def test_tool_info_set():
tool_obj = ToolInfo(**FAKE_TOOL_INFO)
# Unable to change name
with pytest.raises(AttributeError):
tool_obj.name = "new_name"
with pytest.raises(ValueError):
tool_obj.updated = "16062006"
tool_obj.updated = datetime(2020, 3, 11, 11, 37)
assert tool_obj.updated == datetime(2020, 3, 11, 11, 37)
def test_tool_info_from_dict():
t_info = ToolInfo(**FAKE_TOOL_INFO)
t_info.versions.append(VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER))
t_info.versions.append(VersionInfo(**FAKE_VERSION_INFO_WITH_CHECKER))
t_info_dict = dict(t_info)
t_info_from_dict = ToolInfo.from_dict(t_info_dict)
assert t_info.name == t_info_from_dict.name
assert t_info.updated == t_info_from_dict.updated
assert t_info.location == t_info_from_dict.location
assert t_info.versions[1].version == t_info_from_dict.versions[1].version
assert t_info.versions[1].version == "1.1"
with pytest.raises(TypeError):
ToolInfo.from_dict("not_dict")
assert json.dumps(t_info, cls=ToolInfoEncoder)
def test_failed_constraints_meta_data(caplog, base_db):
caplog.set_level(logging.DEBUG)
# Null tool data
tmp_conf = deepcopy(FAKE_CHECKER_CONF)
tmp_conf["tool"] = None
tmp_checker = {
"version": "1.9",
"version_type": VersionType.REMOTE,
"source": "no_checker_case",
"tags": {"latest", "latest-stable"},
"updated": datetime(
2021,
3,
3,
13,
37,
),
"size": 89529754,
}
with pytest.raises(sqlite3.IntegrityError):
with base_db.transaction():
base_db.insert_version_info(ToolInfo(**FAKE_TOOL_INFO),
VersionInfo(**tmp_checker))
base_db.insert_meta_info(FAKE_TOOL_INFO.get("name"),
FAKE_TOOL_INFO.get("location"), tmp_conf)
# Rollback should happen, inserted version not found
version = base_db.get_versions_by_tool(
FAKE_TOOL_INFO.get("name"),
provider=tmp_checker.get("source"),
latest=True)
assert version.version != "1.9"
tools = base_db.get_tools()
assert len(tools[0].versions) == 2
def test_db_tool_data_insert_with_versions(config, caplog):
caplog.set_level(logging.DEBUG)
test_db = ToolDatabase(config)
ver1 = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
ver2 = VersionInfo(**FAKE_VERSION_INFO_WITH_CHECKER)
tool_obj = ToolInfo(**FAKE_TOOL_INFO)
tool_obj.versions.append(ver1)
tool_obj.versions.append(ver2)
with test_db.transaction():
test_db.insert_tool_info(tool_obj)
n_tools = test_db.get_tools()
assert len(n_tools) == 1
assert len(n_tools[0].versions) == 2
n_versions = n_tools[0].versions
assert n_versions[0].version == FAKE_VERSION_INFO_NO_CHECKER.get(
"version")
assert n_versions[0].version_type == FAKE_VERSION_INFO_NO_CHECKER.get(
"version_type")
assert n_versions[0].source == FAKE_VERSION_INFO_NO_CHECKER.get(
"source")
assert n_versions[0].tags == FAKE_VERSION_INFO_NO_CHECKER.get("tags")
# DB insert should not update time - should tell information of origin update time
assert n_versions[0].updated == FAKE_VERSION_INFO_NO_CHECKER.get(
"updated")
assert n_versions[0].raw_size() == FAKE_VERSION_INFO_NO_CHECKER.get(
"size")
# Duplicate insert, should be handled gracefully
test_db.insert_tool_info(tool_obj)
n_tools = test_db.get_tools()
# Still two versions
assert len(n_tools[0].versions) == 2
def test_insert_meta_data(caplog, config):
"""Insert metadata of checker"""
caplog.set_level(logging.DEBUG)
test_db = ToolDatabase(config)
ver1 = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
ver2 = VersionInfo(**FAKE_VERSION_INFO_WITH_CHECKER)
assert isinstance(ver2.source, UpstreamChecker)
tool_obj = ToolInfo(**FAKE_TOOL_INFO)
tool_obj.versions.append(ver1)
tool_obj.versions.append(ver2)
with test_db.transaction():
test_db.insert_tool_info(tool_obj)
test_db.insert_meta_info(tool_obj.name, tool_obj.location,
FAKE_CHECKER_CONF)
meta_data = test_db.get_meta_information(
tool_obj.name, FAKE_CHECKER_CONF.get("provider"))[0]
assert meta_data.get("uri") == FAKE_CHECKER_CONF.get("uri")
assert meta_data.get("repository") == FAKE_CHECKER_CONF.get(
"repository")
assert meta_data.get("tool") == FAKE_CHECKER_CONF.get("tool")
assert meta_data.get("provider") == FAKE_CHECKER_CONF.get("provider")
assert meta_data.get("method") == FAKE_CHECKER_CONF.get("method")
assert meta_data.get("suite") == FAKE_CHECKER_CONF.get("suite")
assert meta_data.get("origin") == FAKE_CHECKER_CONF.get("origin")
assert meta_data.get("docker_origin") == FAKE_CHECKER_CONF.get(
"docker_origin")
def base_db(caplog, config):
caplog.set_level(logging.DEBUG)
# Make sample database for other tests
test_db = ToolDatabase(config)
ver1 = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
ver2 = VersionInfo(**FAKE_VERSION_INFO_WITH_CHECKER)
tool_obj = ToolInfo(**FAKE_TOOL_INFO)
tool_obj.versions.append(ver1)
tool_obj.versions.append(ver2)
with test_db.transaction():
test_db.insert_tool_info(tool_obj)
tool_obj2 = ToolInfo(**FAKE_TOOL_INFO2)
tool_obj2.versions.append(ver1)
tool_obj2.versions.append(ver2)
test_db.insert_tool_info(tool_obj2)
yield test_db
def test_insert_tool_list(config, caplog):
caplog.set_level(logging.DEBUG)
test_db = ToolDatabase(config)
tools = [ToolInfo(**FAKE_TOOL_INFO), ToolInfo(**FAKE_TOOL_INFO2)]
with test_db.transaction():
test_db.insert_tool_info(tools)
# Read values
with test_db.transaction():
tools_from_db = test_db.get_tools()
assert len(tools_from_db) == 2
assert tools[0].description == FAKE_TOOL_INFO.get("description")
assert tools[0].name == FAKE_TOOL_INFO.get("name")
assert tools[0].updated == FAKE_TOOL_INFO.get("updated")
assert tools[0].location == FAKE_TOOL_INFO.get("location")
assert tools[1].description == FAKE_TOOL_INFO2.get("description")
assert tools[1].name == FAKE_TOOL_INFO2.get("name")
assert tools[1].updated == FAKE_TOOL_INFO2.get("updated")
assert tools[1].location == FAKE_TOOL_INFO2.get("location")
def test_get_latest_version_by_provider(base_db):
tmp_checker = {
"version": "1.9",
"version_type": VersionType.REMOTE,
"source": "no_checker_case",
"tags": {"latest", "latest-stable"},
"updated": datetime(
2021,
3,
3,
13,
37,
),
"size": 89529754,
}
with base_db.transaction():
base_db.insert_version_info(ToolInfo(**FAKE_TOOL_INFO),
VersionInfo(**tmp_checker))
version = base_db.get_versions_by_tool(
FAKE_TOOL_INFO.get("name"),
provider=tmp_checker.get("source"),
latest=True)
assert version.version == "1.9"
versions = base_db.get_versions_by_tool(
FAKE_TOOL_INFO.get("name"),
provider=tmp_checker.get("source"),
latest=False)
assert len(versions) == 2
# Replace existing record with identical data but different date
tmp_checker["updated"] = datetime(
2018,
3,
3,
13,
37,
)
base_db.insert_version_info(ToolInfo(**FAKE_TOOL_INFO),
VersionInfo(**tmp_checker))
version = base_db.get_versions_by_tool(
FAKE_TOOL_INFO.get("name"),
provider=tmp_checker.get("source"),
latest=True)
assert version.version == "0.9"
def fetch_tags(self, tool: ToolInfo, update_cache: bool = False):
"""
Fetches available tags for single tool from quay.io HTTP API
See: https://docs.quay.io/api/swagger/#!/repository/getRepo
"""
if not self.auth_url:
self._set_auth_and_service_location()
# In case name includes tag, separate it
self.logger.info("fetch %s...", tool.name)
tool_name, tool_tag = split_tool_tag(tool.name)
# Use name without registry prefix e.g. quay.io
# name_without_prefix = "/".join(tool_name.split("/")[-2:])
name_without_prefix = f"{self.cincan_namespace}/{tool_name}"
endpoint = f"{self.registry_root}/api/v1/repository/{name_without_prefix}"
params = {
"includeTags": True,
"includeStats": False
}
resp = None
try:
resp = self.session.get(endpoint, params=params)
except requests.exceptions.ConnectionError as e:
self.logger.error(e)
if resp and resp.status_code == 200:
resp_cont = resp.json()
tags = resp_cont.get("tags")
tag_names = tags.keys()
if tag_names:
available_versions = self.update_versions_from_manifest_by_tags(name_without_prefix, tag_names)
else:
self.logger.error(f"No tags found for tool {tool_name}.")
return
tool.versions = available_versions
tool.updated = datetime.datetime.now()
if update_cache:
self.update_cache_by_tool(tool)
else:
self.logger.error(f"Failed to fetch tags for image {tool.name} - not updated")
if resp:
self._quay_api_error(resp)
return
def test_create_tool_info():
ver1 = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
ver2 = VersionInfo(**FAKE_VERSION_INFO_WITH_CHECKER)
tool_obj = ToolInfo(**FAKE_TOOL_INFO)
tool_obj.versions.append(ver1)
tool_obj.versions.append(ver2)
assert tool_obj.name == "test_tool"
assert tool_obj.updated == datetime(2020, 3, 13, 13, 37)
assert tool_obj.location == "test_location"
assert tool_obj.description == "test_description"
assert tool_obj.versions[0].version == "0.9"
assert tool_obj.versions[1].version == "1.1"
assert len(tool_obj.versions) == 2
with pytest.raises(ValueError):
ToolInfo("", datetime.now(), "test-location")
with pytest.raises(ValueError):
ToolInfo(1234, datetime.now(), "test-location")
def test_tool_info_eq():
ver1 = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
ver2 = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
tool_obj = ToolInfo(**FAKE_TOOL_INFO)
tool_obj.versions.append(ver1)
tool_obj2 = ToolInfo(**FAKE_TOOL_INFO)
tool_obj2.versions.append(ver2)
# Same name and version
assert tool_obj == tool_obj2
# Different version
tool_obj.versions[0].version = "NOT_SAME"
assert tool_obj != tool_obj2
# Test different names
tool_obj = ToolInfo(**FAKE_TOOL_INFO2)
tool_obj.versions.append(ver1)
assert tool_obj != tool_obj2
# Invalid type comparison
with pytest.raises(ValueError):
assert tool_obj == "Heheehe"
def test_tool_info_latest_version():
ver1 = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
ver2 = VersionInfo(**FAKE_VERSION_INFO_WITH_CHECKER)
tool_obj = ToolInfo(**FAKE_TOOL_INFO)
tool_obj.versions.append(ver1)
tool_obj.versions.append(ver2)
assert tool_obj.get_latest() == "0.9"
assert tool_obj.get_latest(in_upstream=True) == "1.1"
# No versions at all
tool_obj = ToolInfo(**FAKE_TOOL_INFO)
assert tool_obj.get_latest() == VersionInfo("undefined", VersionType.UNDEFINED, "", set(), datetime.min)
def test_db_tool_data_insert(config, caplog):
caplog.set_level(logging.DEBUG)
test_db = ToolDatabase(config)
tool_obj = ToolInfo(**FAKE_TOOL_INFO)
with test_db.transaction():
test_db.insert_tool_info(tool_obj)
# Duplicate insert, should be handled gracefully
test_db.insert_tool_info(tool_obj)
# Read data
with test_db.transaction():
tools = test_db.get_tools()
assert len(tools) == 1
assert tools[0].description == FAKE_TOOL_INFO.get("description")
assert tools[0].name == FAKE_TOOL_INFO.get("name")
assert tools[0].updated == FAKE_TOOL_INFO.get("updated")
assert tools[0].location == FAKE_TOOL_INFO.get("location")
def test_db_insert_duplicate_version(caplog, config):
caplog.set_level(logging.DEBUG)
test_db = ToolDatabase(config)
ver1 = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
ver2 = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
cp_FAKE_VERSION_INFO_NO_CHECKER = deepcopy(FAKE_VERSION_INFO_NO_CHECKER)
cp_FAKE_VERSION_INFO_NO_CHECKER["version"] = "1.1"
ver3 = VersionInfo(**cp_FAKE_VERSION_INFO_NO_CHECKER)
tool_obj = ToolInfo(**FAKE_TOOL_INFO)
tool_obj.versions.append(ver1)
tool_obj.versions.append(ver2)
tool_obj.versions.append(ver3)
with test_db.transaction():
test_db.insert_tool_info(tool_obj)
tools_db = test_db.get_tools()
assert len(tools_db[0].versions) == 2
def test_tool_info_iter():
t_info = ToolInfo(**FAKE_TOOL_INFO)
t_info.versions.append(VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER))
t_info.versions.append(VersionInfo(**FAKE_VERSION_INFO_WITH_CHECKER))
t_info_dict = dict(t_info)
assert t_info_dict.get("name") == "test_tool"
assert t_info_dict.get("updated") == "2020-03-13T13:37:00"
assert t_info_dict.get("location") == "test_location"
assert t_info_dict.get("description") == "test_description"
assert t_info_dict.get("versions")[0] == {
"version": "0.9",
"version_type": VersionType.REMOTE.value,
"source": "no_checker_case",
"tags": ["latest", "latest-stable"],
"updated": format_time(datetime(2020, 3, 3, 13, 37,)),
"size": "39.53 MB",
"origin": False,
}
async def get_tools(self, defined_tag: str = "", force_update: bool = False) -> Dict[str, ToolInfo]:
"""Get tools from remote registry. Name set without repository prefixes"""
self._set_auth_and_service_location()
available_tools = self.__fetch_available_tools()
tool_list = {}
for t in available_tools:
# name = f"{self.image_prefix}/{t.get('namespace')}/{t.get('name')}"
name = t.get('name')
timestamp = t.get("last_modified")
description = t.get("description")
tool_list[name] = ToolInfo(name, datetime.datetime.fromtimestamp(timestamp),
self.registry_name, description=description)
tools = await self.update_tools_in_parallel(tool_list, self.fetch_tags, force_update)
if defined_tag:
keep = []
for t in tools.keys():
for v in tools[t].versions:
if defined_tag in v.tags:
keep.append(t)
break
tools = {k: tools[k] for k in keep}
return tools
def test_get_tool_by_remote(base_db, caplog):
caplog.set_level(logging.DEBUG)
tmp_tool = {
"name": "test_tool_temp",
"updated": datetime(2021, 3, 13, 13, 37),
"location": "test_location",
"description": "test_description",
}
with base_db.transaction():
base_db.insert_tool_info(ToolInfo(**tmp_tool))
tool = base_db.get_single_tool(tool_name=FAKE_TOOL_INFO.get("name"),
remote_name=FAKE_TOOL_INFO.get("location"))
assert tool.name == FAKE_TOOL_INFO.get("name")
tool = base_db.get_single_tool(tool_name=FAKE_TOOL_INFO.get("name"),
remote_name=FAKE_TOOL_INFO.get("location"),
filter_by=[VersionType.UPSTREAM])
assert len(tool.versions) == 1
assert tool.versions[0].version_type == VersionType.UPSTREAM
tool = base_db.get_single_tool(tool_name=FAKE_TOOL_INFO.get("name"),
remote_name=FAKE_TOOL_INFO.get("location"),
filter_by=[VersionType.REMOTE])
assert len(tool.versions) == 1
assert tool.versions[0].version_type == VersionType.REMOTE
tmp_version = deepcopy(FAKE_VERSION_INFO_NO_CHECKER)
tmp_version["version_type"] = VersionType.LOCAL
with base_db.transaction():
base_db.insert_version_info(tool, tmp_version)
tool = base_db.get_single_tool(
tool_name=FAKE_TOOL_INFO.get("name"),
remote_name=FAKE_TOOL_INFO.get("location"),
filter_by=[VersionType.REMOTE, VersionType.UPSTREAM])
assert len(tool.versions) == 2
assert VersionType.LOCAL not in [i.version_type for i in tool.versions]
tools = base_db.get_tools(remote_name=FAKE_TOOL_INFO.get("location"))
assert len(tools) == 2
def test_tool_info_origin_version():
ver1 = VersionInfo(**FAKE_VERSION_INFO_NO_CHECKER)
ver2 = VersionInfo(**FAKE_VERSION_INFO_WITH_CHECKER)
tool_obj = ToolInfo(**FAKE_TOOL_INFO)
tool_obj.versions.append(ver1)
assert tool_obj.get_origin_version() == VersionInfo(
"Not implemented", VersionType.UNDEFINED, "", set(), datetime.min
)
assert tool_obj.get_docker_origin_version() == VersionInfo(
"Not implemented", VersionType.UNDEFINED, "", set(), datetime.min
)
tool_obj.versions.append(ver2)
assert tool_obj.get_origin_version() == "1.1"
# Above fetch updated timestamp when getting 1.2 version with 'get_version' method, because
# timestamp was older than 1 hour
# However, this is mock object, and does not update original object as real UpstreamCheck
# Object would do - therefore we are getting version 1.1 in next fetch, because VersionInfo
# has timestamp updated
assert tool_obj.get_docker_origin_version() == "1.1"
async def list_versions(
self,
tool: str = "",
to_json: bool = False,
only_updates: bool = False,
force_refresh: bool = False,
):
maintainer = VersionMaintainer(
self.config,
db=self.db,
force_refresh=force_refresh,
)
versions = {}
if tool:
if "/" in tool:
self.logger.error(
"Give only name of the tool, without prefixes or namespaces related to tool image."
f" Tool must be in default registry: {self.default_remote}."
)
sys.exit(1)
tool_name = basename(tool)
# tool_with_namespace = f"{self.remote_registry.full_prefix}/{tool_name}"
l_tool = self.local_registry.create_local_tool_info_by_name(
tool_name)
r_tool = self.remote_registry.read_remote_versions_from_db(
tool_name) if not force_refresh else {}
now = datetime.now()
if not r_tool:
r_tool = ToolInfo(tool_name, datetime.min,
self.remote_registry.registry_name)
if not r_tool.updated or not (
now - timedelta(hours=self.config.cache_lifetime) <=
r_tool.updated <= now):
self.remote_registry.fetch_tags(r_tool, update_cache=True)
if l_tool or (r_tool and not r_tool.updated == datetime.min):
l_tool, r_tool = maintainer.get_versions_single_tool(
tool_name, l_tool, r_tool)
versions = await maintainer.list_versions_single(
l_tool, r_tool, only_updates)
else:
raise FileNotFoundError(
f"Given tool {tool} not found locally or remotely. Please, give only the basename of the tool,"
f"without prefixes.")
else:
remote_tools = await self.remote_registry.get_tools(
force_update=force_refresh)
# Remote tools, with included upstream version information
remote_tools_with_origin_version = await maintainer.check_upstream_versions(
remote_tools)
# Local tools, without checking and corresponding the configured registry
local_tools = await self.local_registry.get_tools(
prefix=self.remote_registry.full_prefix)
for t in remote_tools_with_origin_version:
r_tool = remote_tools_with_origin_version.get(
t) # Contains also upstream version info
l_tool = local_tools.get(t, "")
t_info = await maintainer.list_versions_single(
l_tool, r_tool, only_updates)
if t_info:
versions[t] = t_info
if to_json:
return json.dumps(versions)
else:
return versions
def test_tool_info_to_str():
tool_obj = ToolInfo(**FAKE_TOOL_INFO)
assert str(tool_obj) == "test_tool test_description"