def test_db_tool_data_insert_with_versions(config, caplog):
caplog.set_level(logging.DEBUG)
test_db = ToolDatabase(config)
ver1 = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
ver2 = VersionInfo(**FAKE_VERSION_INFO_WITH_CHECKER)
tool_obj = ToolInfo(**FAKE_TOOL_INFO)
tool_obj.versions.append(ver1)
tool_obj.versions.append(ver2)
with test_db.transaction():
test_db.insert_tool_info(tool_obj)
n_tools = test_db.get_tools()
assert len(n_tools) == 1
assert len(n_tools[0].versions) == 2
n_versions = n_tools[0].versions
assert n_versions[0].version == FAKE_VERSION_INFO_NO_CHECKER.get(
"version")
assert n_versions[0].version_type == FAKE_VERSION_INFO_NO_CHECKER.get(
"version_type")
assert n_versions[0].source == FAKE_VERSION_INFO_NO_CHECKER.get(
"source")
assert n_versions[0].tags == FAKE_VERSION_INFO_NO_CHECKER.get("tags")
# DB insert should not update time - should tell information of origin update time
assert n_versions[0].updated == FAKE_VERSION_INFO_NO_CHECKER.get(
"updated")
assert n_versions[0].raw_size() == FAKE_VERSION_INFO_NO_CHECKER.get(
"size")
# Duplicate insert, should be handled gracefully
test_db.insert_tool_info(tool_obj)
n_tools = test_db.get_tools()
# Still two versions
assert len(n_tools[0].versions) == 2
def test_insert_meta_data(caplog, config):
"""Insert metadata of checker"""
caplog.set_level(logging.DEBUG)
test_db = ToolDatabase(config)
ver1 = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
ver2 = VersionInfo(**FAKE_VERSION_INFO_WITH_CHECKER)
assert isinstance(ver2.source, UpstreamChecker)
tool_obj = ToolInfo(**FAKE_TOOL_INFO)
tool_obj.versions.append(ver1)
tool_obj.versions.append(ver2)
with test_db.transaction():
test_db.insert_tool_info(tool_obj)
test_db.insert_meta_info(tool_obj.name, tool_obj.location,
FAKE_CHECKER_CONF)
meta_data = test_db.get_meta_information(
tool_obj.name, FAKE_CHECKER_CONF.get("provider"))[0]
assert meta_data.get("uri") == FAKE_CHECKER_CONF.get("uri")
assert meta_data.get("repository") == FAKE_CHECKER_CONF.get(
"repository")
assert meta_data.get("tool") == FAKE_CHECKER_CONF.get("tool")
assert meta_data.get("provider") == FAKE_CHECKER_CONF.get("provider")
assert meta_data.get("method") == FAKE_CHECKER_CONF.get("method")
assert meta_data.get("suite") == FAKE_CHECKER_CONF.get("suite")
assert meta_data.get("origin") == FAKE_CHECKER_CONF.get("origin")
assert meta_data.get("docker_origin") == FAKE_CHECKER_CONF.get(
"docker_origin")
def test_version_info_from_dict():
obj = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
obj_dict = dict(obj)
obj = VersionInfo.from_dict(obj_dict)
assert obj.version == "0.9"
assert obj.source == "no_checker_case"
assert obj.tags == set(["latest-stable", "latest"])
assert obj.updated == datetime(2020, 3, 3, 13, 37)
assert obj.size == "39.53 MB"
with pytest.raises(TypeError):
VersionInfo.from_dict("not_dict")
def test_version_info_eq():
obj1 = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
obj2 = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
assert obj1 == obj2
obj1.version = 1.2
assert obj1 != obj2
# We can compare for strings.
assert obj1 == "1.2"
assert obj1 != "1.3"
# But not integers
with pytest.raises(ValueError):
assert obj1 != 1
def test_getters_version_info_no_checker():
obj = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
assert obj.version == "0.9"
assert obj.provider == "no_checker_case"
assert not obj.docker_origin
assert obj.extra_info == ""
assert obj.source == "no_checker_case"
assert not obj.origin
assert obj.tags == set(["latest", "latest-stable"])
assert obj.updated == datetime(2020, 3, 3, 13, 37)
assert obj.size == "39.53 MB"
assert obj.raw_size() == 39529754
def test_tool_info_latest_version():
ver1 = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
ver2 = VersionInfo(**FAKE_VERSION_INFO_WITH_CHECKER)
tool_obj = ToolInfo(**FAKE_TOOL_INFO)
tool_obj.versions.append(ver1)
tool_obj.versions.append(ver2)
assert tool_obj.get_latest() == "0.9"
assert tool_obj.get_latest(in_upstream=True) == "1.1"
# No versions at all
tool_obj = ToolInfo(**FAKE_TOOL_INFO)
assert tool_obj.get_latest() == VersionInfo("undefined", VersionType.UNDEFINED, "", set(), datetime.min)
def test_failed_constraints_meta_data(caplog, base_db):
caplog.set_level(logging.DEBUG)
# Null tool data
tmp_conf = deepcopy(FAKE_CHECKER_CONF)
tmp_conf["tool"] = None
tmp_checker = {
"version": "1.9",
"version_type": VersionType.REMOTE,
"source": "no_checker_case",
"tags": {"latest", "latest-stable"},
"updated": datetime(
2021,
3,
3,
13,
37,
),
"size": 89529754,
}
with pytest.raises(sqlite3.IntegrityError):
with base_db.transaction():
base_db.insert_version_info(ToolInfo(**FAKE_TOOL_INFO),
VersionInfo(**tmp_checker))
base_db.insert_meta_info(FAKE_TOOL_INFO.get("name"),
FAKE_TOOL_INFO.get("location"), tmp_conf)
# Rollback should happen, inserted version not found
version = base_db.get_versions_by_tool(
FAKE_TOOL_INFO.get("name"),
provider=tmp_checker.get("source"),
latest=True)
assert version.version != "1.9"
tools = base_db.get_tools()
assert len(tools[0].versions) == 2
def test_db_insert_duplicate_version(caplog, config):
caplog.set_level(logging.DEBUG)
test_db = ToolDatabase(config)
ver1 = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
ver2 = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
cp_FAKE_VERSION_INFO_NO_CHECKER = deepcopy(FAKE_VERSION_INFO_NO_CHECKER)
cp_FAKE_VERSION_INFO_NO_CHECKER["version"] = "1.1"
ver3 = VersionInfo(**cp_FAKE_VERSION_INFO_NO_CHECKER)
tool_obj = ToolInfo(**FAKE_TOOL_INFO)
tool_obj.versions.append(ver1)
tool_obj.versions.append(ver2)
tool_obj.versions.append(ver3)
with test_db.transaction():
test_db.insert_tool_info(tool_obj)
tools_db = test_db.get_tools()
assert len(tools_db[0].versions) == 2
def test_tool_info_from_dict():
t_info = ToolInfo(**FAKE_TOOL_INFO)
t_info.versions.append(VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER))
t_info.versions.append(VersionInfo(**FAKE_VERSION_INFO_WITH_CHECKER))
t_info_dict = dict(t_info)
t_info_from_dict = ToolInfo.from_dict(t_info_dict)
assert t_info.name == t_info_from_dict.name
assert t_info.updated == t_info_from_dict.updated
assert t_info.location == t_info_from_dict.location
assert t_info.versions[1].version == t_info_from_dict.versions[1].version
assert t_info.versions[1].version == "1.1"
with pytest.raises(TypeError):
ToolInfo.from_dict("not_dict")
assert json.dumps(t_info, cls=ToolInfoEncoder)
def base_db(caplog, config):
caplog.set_level(logging.DEBUG)
# Make sample database for other tests
test_db = ToolDatabase(config)
ver1 = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
ver2 = VersionInfo(**FAKE_VERSION_INFO_WITH_CHECKER)
tool_obj = ToolInfo(**FAKE_TOOL_INFO)
tool_obj.versions.append(ver1)
tool_obj.versions.append(ver2)
with test_db.transaction():
test_db.insert_tool_info(tool_obj)
tool_obj2 = ToolInfo(**FAKE_TOOL_INFO2)
tool_obj2.versions.append(ver1)
tool_obj2.versions.append(ver2)
test_db.insert_tool_info(tool_obj2)
yield test_db
def test_create_version_info_no_checker():
"""
Test init method and attribute content
"""
obj = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
assert obj._version == "0.9"
assert obj._source == "no_checker_case"
assert obj._tags == {"latest", "latest-stable"}
assert obj._updated == datetime(2020, 3, 3, 13, 37)
assert obj._origin is False
fake_d = FAKE_VERSION_INFO_NO_CHECKER.copy()
fake_d["updated"] = "invalid_dateformat"
with pytest.raises(ValueError):
VersionInfo(**fake_d)
fake_d["updated"] = FAKE_VERSION_INFO_NO_CHECKER.get("updated")
with pytest.raises(ValueError):
v_obj = VersionInfo(**fake_d)
v_obj.version_type = ""
def test_get_latest_version_by_provider(base_db):
tmp_checker = {
"version": "1.9",
"version_type": VersionType.REMOTE,
"source": "no_checker_case",
"tags": {"latest", "latest-stable"},
"updated": datetime(
2021,
3,
3,
13,
37,
),
"size": 89529754,
}
with base_db.transaction():
base_db.insert_version_info(ToolInfo(**FAKE_TOOL_INFO),
VersionInfo(**tmp_checker))
version = base_db.get_versions_by_tool(
FAKE_TOOL_INFO.get("name"),
provider=tmp_checker.get("source"),
latest=True)
assert version.version == "1.9"
versions = base_db.get_versions_by_tool(
FAKE_TOOL_INFO.get("name"),
provider=tmp_checker.get("source"),
latest=False)
assert len(versions) == 2
# Replace existing record with identical data but different date
tmp_checker["updated"] = datetime(
2018,
3,
3,
13,
37,
)
base_db.insert_version_info(ToolInfo(**FAKE_TOOL_INFO),
VersionInfo(**tmp_checker))
version = base_db.get_versions_by_tool(
FAKE_TOOL_INFO.get("name"),
provider=tmp_checker.get("source"),
latest=True)
assert version.version == "0.9"
def test_tool_info_iter():
t_info = ToolInfo(**FAKE_TOOL_INFO)
t_info.versions.append(VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER))
t_info.versions.append(VersionInfo(**FAKE_VERSION_INFO_WITH_CHECKER))
t_info_dict = dict(t_info)
assert t_info_dict.get("name") == "test_tool"
assert t_info_dict.get("updated") == "2020-03-13T13:37:00"
assert t_info_dict.get("location") == "test_location"
assert t_info_dict.get("description") == "test_description"
assert t_info_dict.get("versions")[0] == {
"version": "0.9",
"version_type": VersionType.REMOTE.value,
"source": "no_checker_case",
"tags": ["latest", "latest-stable"],
"updated": format_time(datetime(2020, 3, 3, 13, 37,)),
"size": "39.53 MB",
"origin": False,
}
def test_create_version_info_with_checker():
"""
Test init and attribute content
"""
obj = VersionInfo(**FAKE_VERSION_INFO_WITH_CHECKER)
obj._source.reset_mock()
assert isinstance(obj._source, UpstreamChecker)
assert obj.provider == "test_provider"
assert obj.docker_origin
assert obj.extra_info == "Test information"
assert obj.version == "1.1"
def test_create_tool_info():
ver1 = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
ver2 = VersionInfo(**FAKE_VERSION_INFO_WITH_CHECKER)
tool_obj = ToolInfo(**FAKE_TOOL_INFO)
tool_obj.versions.append(ver1)
tool_obj.versions.append(ver2)
assert tool_obj.name == "test_tool"
assert tool_obj.updated == datetime(2020, 3, 13, 13, 37)
assert tool_obj.location == "test_location"
assert tool_obj.description == "test_description"
assert tool_obj.versions[0].version == "0.9"
assert tool_obj.versions[1].version == "1.1"
assert len(tool_obj.versions) == 2
with pytest.raises(ValueError):
ToolInfo("", datetime.now(), "test-location")
with pytest.raises(ValueError):
ToolInfo(1234, datetime.now(), "test-location")
def test_tool_info_eq():
ver1 = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
ver2 = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
tool_obj = ToolInfo(**FAKE_TOOL_INFO)
tool_obj.versions.append(ver1)
tool_obj2 = ToolInfo(**FAKE_TOOL_INFO)
tool_obj2.versions.append(ver2)
# Same name and version
assert tool_obj == tool_obj2
# Different version
tool_obj.versions[0].version = "NOT_SAME"
assert tool_obj != tool_obj2
# Test different names
tool_obj = ToolInfo(**FAKE_TOOL_INFO2)
tool_obj.versions.append(ver1)
assert tool_obj != tool_obj2
# Invalid type comparison
with pytest.raises(ValueError):
assert tool_obj == "Heheehe"
def test_tool_info_origin_version():
ver1 = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
ver2 = VersionInfo(**FAKE_VERSION_INFO_WITH_CHECKER)
tool_obj = ToolInfo(**FAKE_TOOL_INFO)
tool_obj.versions.append(ver1)
assert tool_obj.get_origin_version() == VersionInfo(
"Not implemented", VersionType.UNDEFINED, "", set(), datetime.min
)
assert tool_obj.get_docker_origin_version() == VersionInfo(
"Not implemented", VersionType.UNDEFINED, "", set(), datetime.min
)
tool_obj.versions.append(ver2)
assert tool_obj.get_origin_version() == "1.1"
# Above fetch updated timestamp when getting 1.2 version with 'get_version' method, because
# timestamp was older than 1 hour
# However, this is mock object, and does not update original object as real UpstreamCheck
# Object would do - therefore we are getting version 1.1 in next fetch, because VersionInfo
# has timestamp updated
assert tool_obj.get_docker_origin_version() == "1.1"
def update_versions_from_manifest_by_tags(
self, tool_name: str, tag_names: List[str]) -> List[VersionInfo]:
"""
By given tag name list, fetches corresponding manifests and generates version info
"""
available_versions: List[VersionInfo] = []
# Get token only once for one tool because speed
token = self._get_registry_service_token(tool_name)
for t in tag_names:
manifest = self.fetch_manifest(tool_name, t, token)
if not manifest:
continue
container_config = self.fetch_image_config(tool_name,
manifest.config.digest,
token)
if not container_config:
continue
size = sum([layer.size for layer in manifest.layers])
if manifest:
version = self._get_version_from_image_config(container_config)
updated = parse_file_time(container_config.created)
# Get meta data from latest image for upstream checking, skip big files (1MB+). Should be only file
# on final layer
if t == self.config.tag and manifest.layers[
-1].size < self.config.meta_max_size:
meta_blob_resp = self.fetch_blob(
tool_name, manifest.layers[-1].digest, token)
meta_parsed = self._parse_meta_file(
meta_blob_resp, tool_name)
if meta_parsed and isinstance(meta_parsed, Dict):
self.cache_meta_data.put(
(basename(tool_name), self.registry_name,
meta_parsed))
if not version:
version = self.VER_UNDEFINED
match = [v for v in available_versions if version == v.version]
if match:
next(iter(match)).tags.add(t)
else:
ver_info = VersionInfo(version,
VersionType.REMOTE,
self.registry_name, {t},
updated,
size=size)
available_versions.append(ver_info)
return available_versions
def test_version_info_iter():
obj = VersionInfo(**FAKE_VERSION_INFO_WITH_CHECKER)
obj.updated = datetime.now()
test_dict = {
"version": "1.1",
"version_type": "upstream",
"source": {
"uri": "https://test.uri",
"repository": "test_repository",
"tool": "test_tool",
"provider": "test_provider",
"method": "test_release",
"suite": "test_suite",
"origin": True,
"docker_origin": True,
"version": "1.1",
"extra_info": "Test information",
},
"tags": ["latest", "latest-stable"],
"updated": format_time(obj.updated),
"origin": True,
"size": "3.95 MB",
}
assert dict(obj) == test_dict
obj = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
obj.updated = datetime.now()
test_dict2 = {
"version": "0.9",
"version_type": "remote",
"source": "no_checker_case",
"tags": ["latest", "latest-stable"],
"updated": format_time(obj.updated),
"origin": False,
"size": "39.53 MB",
}
assert dict(obj) == test_dict2
assert json.dumps(test_dict)
assert json.dumps(test_dict2)
def test_version_info_str():
obj = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
assert str(obj) == "0.9"
def test_version_info_format():
obj1 = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
assert f"{obj1}" == "0.9"
def test_setters_version_info_no_checker():
obj = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
assert obj.source == "no_checker_case"
obj.source = "new case"
assert obj.source == "new case"
assert obj.version == "0.9"
obj.version = 1.2
assert obj.version == "1.2"
with pytest.raises(ValueError):
obj.version = ""
obj.updated = datetime(2020, 1, 1, 11, 11)
obj.updated == datetime(2020, 1, 1, 11, 11)
# Test invalid time format
with pytest.raises(ValueError):
obj.updated = ""
# size
with pytest.raises(ValueError):
obj.size = "16062006"
obj._size = None
assert obj.size == "NaN"
obj.size = 900
assert obj.size == "900 bytes"
obj.size = 1001
assert obj.size == "1.00 KB"
obj.size = 16062
assert obj.size == "16.06 KB"
obj.size = 16062006
assert obj.size == "16.06 MB"
obj.size = 1606200600
assert obj.size == "1.61 GB"
obj_c = FAKE_VERSION_INFO_NO_CHECKER.copy()
obj_c["size"] = "This is something"
obj = VersionInfo(**obj_c)
assert obj.size == "NaN"
def test_version_info_normalization():
obj1 = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
assert obj1.get_normalized_ver() == [0, 9]
obj1.version = "1.2.3.4.5.6"
assert obj1.get_normalized_ver() == [1, 2, 3, 4, 5, 6]
obj1.version = "1_2_3_4"
assert obj1.get_normalized_ver() == [1, 2, 3, 4]
obj1.version = "ghidra_9.1.2_PUBLIC_20200212"
assert obj1.get_normalized_ver() == [9, 1, 2]
obj1.version = "release-1.2.3"
assert obj1.get_normalized_ver() == [1, 2, 3]
# sha1 test - 40 char
obj1.version = "ee9f16b4b95c28f8f79a39ca6a1840d8a6444c10"
assert obj1.get_normalized_ver(
) == "ee9f16b4b95c28f8f79a39ca6a1840d8a6444c10"
# sha256 test - 64 char
obj1.version = "f8b09fba9fda9ffebae86611261cf628bd71022fb4348d876974f7c48ddcc6d5"
assert (obj1.get_normalized_ver() ==
"f8b09fba9fda9ffebae86611261cf628bd71022fb4348d876974f7c48ddcc6d5")
# missing couple characters from sha256 length
obj1.version = "f809fba9fda9ffebae86611261cf628bd71022fb4348d876974f7c48ddcc65"
assert obj1.get_normalized_ver() == [809998661126162871022434887697474865]
obj1.version = "ABCDEFG"
assert obj1.get_normalized_ver() == "ABCDEFG"