def test_process_npm(default_request, default_toplevel_purl):
pkg = Package.from_json({
"name": "grc-ui",
"type": "npm",
"version": "1.0.0"
})
pkg.id = 1
expected_purl = default_toplevel_purl
dep_commit_id = "7762177aacfb1ddf5ca45cebfe8de1da3b24f0ff"
dep = Package.from_json({
"name":
"security-middleware",
"type":
"npm",
"version":
f"github:open-cluster-management/security-middleware#{dep_commit_id}",
})
dep.id = 2
expected_dep_purl = f"pkg:github/open-cluster-management/security-middleware@{dep_commit_id}"
src = Package.from_json({
"name": "@types/events",
"type": "npm",
"version": "3.0.0"
})
src.id = 3
src.dev = True
expected_src_purl = "pkg:npm/%40types/[email protected]"
cm = ContentManifest(default_request)
# emulate to_json behavior to setup internal packages cache
cm._npm_data.setdefault(pkg.id, {
"purl": expected_purl,
"dependencies": [],
"sources": []
})
cm.process_npm_package(pkg, dep)
cm.process_npm_package(pkg, src)
expected_contents = {
pkg.id: {
"purl": expected_purl,
"dependencies": [{
"purl": expected_dep_purl
}],
"sources": [{
"purl": expected_dep_purl
}, {
"purl": expected_src_purl
}],
}
}
assert cm._npm_data
assert pkg.id in cm._npm_data
assert cm._npm_data == expected_contents
def test_process_pip(default_request, default_toplevel_purl):
pkg = Package.from_json({
"name": "requests",
"type": "pip",
"version": "2.24.0"
})
pkg.id = 1
expected_purl = default_toplevel_purl
dep_commit_id = "58c88e4952e95935c0dd72d4a24b0c44f2249f5b"
dep = Package.from_json({
"name":
"cnr-server",
"type":
"pip",
"version":
f"git+https://github.com/quay/appr@{dep_commit_id}",
})
dep.id = 2
expected_dep_purl = f"pkg:github/quay/appr@{dep_commit_id}"
src = Package.from_json({
"name": "setuptools",
"type": "pip",
"version": "49.1.1"
})
src.id = 3
src.dev = True
expected_src_purl = "pkg:pypi/[email protected]"
cm = ContentManifest(default_request)
# emulate to_json behavior to setup internal packages cache
cm._pip_data.setdefault(pkg.id, {
"purl": expected_purl,
"dependencies": [],
"sources": []
})
cm.process_pip_package(pkg, dep)
cm.process_pip_package(pkg, src)
expected_contents = {
pkg.id: {
"purl": expected_purl,
"dependencies": [{
"purl": expected_dep_purl
}],
"sources": [{
"purl": expected_dep_purl
}, {
"purl": expected_src_purl
}],
}
}
assert cm._pip_data
assert pkg.id in cm._pip_data
assert cm._pip_data == expected_contents
def test_process_go(default_request):
pkg = Package.from_json({
"name": "example.com/org/project",
"type": "go-package",
"version": "1.1.1"
})
pkg.id = 1
expected_purl = "pkg:golang/example.com%2Forg%[email protected]"
dep = Package.from_json({
"name": "example.com/org/project/lib",
"type": "go-package",
"version": "2.2.2"
})
dep.id = 2
expected_dep_purl = "pkg:golang/example.com%2Forg%2Fproject%[email protected]"
src = Package.from_json({
"name": "example.com/anotherorg/project",
"type": "gomod",
"version": "3.3.3"
})
src.id = 3
expected_src_purl = "pkg:golang/example.com%2Fanotherorg%[email protected]"
cm = ContentManifest(default_request)
# emulate to_json behavior to setup internal packages cache
cm._gomod_data.setdefault(pkg.name, {
"purl": "not-important",
"dependencies": []
})
cm._gopkg_data.setdefault(pkg.id, {
"name": pkg.name,
"purl": expected_purl,
"dependencies": [],
"sources": []
})
cm.process_go_package(pkg, dep)
cm.process_gomod(pkg, src)
cm.set_go_package_sources()
expected_contents = {
pkg.id: {
"purl": expected_purl,
"dependencies": [{
"purl": expected_dep_purl
}],
"sources": [{
"purl": expected_src_purl
}],
}
}
assert cm._gopkg_data
assert pkg.id in cm._gopkg_data
assert cm._gopkg_data == expected_contents
def test_to_json(mock_top_level_purl, app, package, subpath):
request = Request()
cm = ContentManifest(request)
image_contents = []
if package:
pkg = Package.from_json(package)
request_package = RequestPackage(package=pkg, subpath=subpath)
request.request_packages.append(request_package)
content = {
"purl": mock_top_level_purl.return_value,
"dependencies": [],
"sources": [],
}
image_contents.append(content)
expected = {
"metadata": {
"icm_version": 1,
"icm_spec": ContentManifest.json_schema_url,
"image_layer_index": -1,
},
"image_contents": image_contents,
}
assert cm.to_json() == expected
if package:
mock_top_level_purl.assert_called_once_with(request, subpath=subpath)
def test_purl_conversion_bogus_forge():
package = {"name": "odd", "type": "npm", "version": "github:something/odd"}
pkg = Package.from_json(package)
msg = f"Could not convert version {pkg.version} to purl"
with pytest.raises(ContentManifestError, match=msg):
pkg.to_purl()
def test_top_level_purl_conversion(pkg_type, purl_method, method_args,
default_request, has_subpath):
pkg = Package(type=pkg_type)
if purl_method is None:
msg = f"{pkg_type!r} is not a valid top level package"
with pytest.raises(ContentManifestError, match=msg):
pkg.to_top_level_purl(default_request)
else:
with mock.patch.object(pkg, purl_method) as mock_purl_method:
mock_purl_method.return_value = "pkg:generic/foo"
purl = pkg.to_top_level_purl(
default_request, subpath="some/path" if has_subpath else None)
assert mock_purl_method.called_once_with(*method_args)
if has_subpath and pkg_type != "git-submodule":
assert purl == "pkg:generic/foo#some/path"
else:
assert purl == "pkg:generic/foo"
def test_purl_conversion(package, expected_purl, defined, known_protocol):
pkg = Package.from_json(package)
if defined and known_protocol:
purl = pkg.to_purl()
assert purl == expected_purl
else:
msg = f"The PURL spec is not defined for {pkg.type} packages"
if defined:
msg = f"Unknown protocol in {pkg.type} package version: {pkg.version}"
with pytest.raises(ContentManifestError, match=msg):
pkg.to_purl()
def test_to_json_with_multiple_packages(mock_generate_icm, app, packages):
request = Request()
cm = ContentManifest(request)
image_contents = []
for package in packages:
pkg = Package.from_json(package)
request_package = RequestPackage(package=pkg)
request.request_packages.append(request_package)
content = {"purl": pkg.to_purl(), "dependencies": [], "sources": []}
image_contents.append(content)
res = cm.to_json()
mock_generate_icm.assert_called_once_with(image_contents)
assert res == mock_generate_icm.return_value
def test_process_gomod_replace_parent_purl(default_request):
module = Package.from_json({
"name": "example.com/org/project",
"type": "gomod",
"version": "1.1.1"
})
module.id = 1
expected_module_purl = "pkg:golang/example.com%2Forg%[email protected]"
module_dep = Package.from_json({
"name":
"example.com/anotherorg/project",
"type":
"gomod",
"version":
"./staging/src/anotherorg/project",
})
module_dep.id = 2
expected_dependency_purl = f"{expected_module_purl}#staging/src/anotherorg/project"
cm = ContentManifest(default_request)
# emulate to_json behavior to setup internal packages cache
cm._gomod_data.setdefault(module.name, {
"purl": expected_module_purl,
"dependencies": []
})
cm.process_gomod(module, module_dep)
assert cm._gomod_data == {
module.name: {
"purl": expected_module_purl,
"dependencies": [{
"purl": expected_dependency_purl
}],
},
}
def test_to_json_properly_sets_internal_data(mock_set_go_sources,
mock_top_level_purl, app, package,
internal_attr, internal_data):
# Half the unit tests "emulate to_json() behaviour" so we should probably test that behaviour
request = Request()
pkg = Package.from_json(package)
pkg.id = 1
request_package = RequestPackage(package=pkg)
request.request_packages.append(request_package)
mock_top_level_purl.return_value = "mock-package-purl"
cm = ContentManifest(request)
cm.to_json()
# Here we are only interested in the setup part of to_json()
# (sidenote: we really need to refactor to_json())
assert getattr(cm, internal_attr) == internal_data
def patch_request(request_id):
"""
Modify the given request.
:param int request_id: the request ID from the URL
:return: a Flask JSON response
:rtype: flask.Response
:raise NotFound: if the request is not found
:raise ValidationError: if the JSON is invalid
"""
payload = flask.request.get_json()
if not isinstance(payload, dict):
raise ValidationError("The input data must be a JSON object")
if not payload:
raise ValidationError(
"At least one key must be specified to update the request")
valid_keys = {
"dependencies",
"environment_variables",
"package",
"package_subpath",
"state",
"state_reason",
}
invalid_keys = set(payload.keys()) - valid_keys
if invalid_keys:
raise ValidationError("The following keys are not allowed: {}".format(
", ".join(invalid_keys)))
for key, value in payload.items():
if key == "dependencies":
if not isinstance(value, list):
raise ValidationError(
'The value for "dependencies" must be an array')
if "package" not in payload:
raise ValidationError(
'The "package" object must also be provided if the "dependencies" array is '
"provided")
for dep in value:
Dependency.validate_json(dep, for_update=True)
elif key == "package":
Package.validate_json(value)
elif key == "environment_variables":
if not isinstance(value, dict):
raise ValidationError(
'The value for "{}" must be an object'.format(key))
for env_var_name, env_var_info in value.items():
EnvironmentVariable.validate_json(env_var_name, env_var_info)
elif not isinstance(value, str):
raise ValidationError(
'The value for "{}" must be a string'.format(key))
if "package_subpath" in payload and "package" not in payload:
raise ValidationError(
'The "package" object must also be provided if "package_subpath" is provided'
)
if "state" in payload and "state_reason" not in payload:
raise ValidationError(
'The "state_reason" key is required when "state" is supplied')
elif "state_reason" in payload and "state" not in payload:
raise ValidationError(
'The "state" key is required when "state_reason" is supplied')
request = Request.query.get_or_404(request_id)
delete_bundle = False
delete_bundle_temp = False
cleanup_nexus = []
delete_logs = False
if "state" in payload and "state_reason" in payload:
new_state = payload["state"]
delete_bundle = new_state == "stale" and request.state.state_name != "failed"
if new_state in ("stale", "failed"):
for pkg_manager in ["npm", "pip", "yarn"]:
if any(p.name == pkg_manager for p in request.pkg_managers):
cleanup_nexus.append(pkg_manager)
delete_bundle_temp = new_state in ("complete", "failed")
delete_logs = new_state == "stale"
new_state_reason = payload["state_reason"]
# This is to protect against a Celery task getting executed twice and setting the
# state each time
if request.state.state_name == new_state and request.state.state_reason == new_state_reason:
flask.current_app.logger.info(
"Not adding a new state since it matches the last state")
else:
request.add_state(new_state, new_state_reason)
package_object = None
if "package" in payload:
package_object = Package.get_or_create(payload["package"])
package_attrs = {}
# The presence of "package_subpath" in payload indicates whether to modify the subpath.
# This is only allowed when creating a new package, so when the PATCH API is used to
# modify an existing package, the user must make sure to use the same subpath (or no
# subpath).
if "package_subpath" in payload:
package_attrs["subpath"] = payload["package_subpath"]
request.add_package(package_object, **package_attrs)
for dep_and_replaces in payload.get("dependencies", []):
dep = copy.deepcopy(dep_and_replaces)
replaces = dep.pop("replaces", None)
dep_object = Dependency.get_or_create(dep)
replaces_object = None
if replaces:
replaces_object = Dependency.get_or_create(replaces)
request.add_dependency(package_object, dep_object, replaces_object)
for env_var_name, env_var_info in payload.get("environment_variables",
{}).items():
env_var_obj = EnvironmentVariable.query.filter_by(
name=env_var_name, **env_var_info).first()
if not env_var_obj:
env_var_obj = EnvironmentVariable.from_json(
env_var_name, env_var_info)
db.session.add(env_var_obj)
if env_var_obj not in request.environment_variables:
request.environment_variables.append(env_var_obj)
db.session.commit()
bundle_dir = RequestBundleDir(
request.id, root=flask.current_app.config["CACHITO_BUNDLES_DIR"])
if delete_bundle and bundle_dir.bundle_archive_file.exists():
flask.current_app.logger.info("Deleting the bundle archive %s",
bundle_dir.bundle_archive_file)
try:
bundle_dir.bundle_archive_file.unlink()
except: # noqa E722
flask.current_app.logger.exception(
"Failed to delete the bundle archive %s",
bundle_dir.bundle_archive_file)
if delete_bundle_temp and bundle_dir.exists():
flask.current_app.logger.debug(
"Deleting the temporary files used to create the bundle at %s",
bundle_dir)
try:
bundle_dir.rmtree()
except: # noqa E722
flask.current_app.logger.exception(
"Failed to delete the temporary files at %s", bundle_dir)
if delete_logs:
request_log_dir = flask.current_app.config[
"CACHITO_REQUEST_FILE_LOGS_DIR"]
path_to_file = os.path.join(request_log_dir, f"{request_id}.log")
try:
os.remove(path_to_file)
except: # noqa E722
flask.current_app.logger.exception(
"Failed to delete the log file %s", path_to_file)
for pkg_mgr in cleanup_nexus:
flask.current_app.logger.info(
"Cleaning up the Nexus %s content for request %d", pkg_mgr,
request_id)
cleanup_task = getattr(tasks, f"cleanup_{pkg_mgr}_request")
try:
cleanup_task.delay(request_id)
except kombu.exceptions.OperationalError:
flask.current_app.logger.exception(
"Failed to schedule the cleanup_%s_request task for request %d. An administrator "
"must clean this up manually.",
pkg_mgr,
request.id,
)
if current_user.is_authenticated:
flask.current_app.logger.info("The user %s patched request %d",
current_user.username, request.id)
else:
flask.current_app.logger.info("An anonymous user patched request %d",
request.id)
return flask.jsonify(request.to_json()), 200
def test_vcs_purl_conversion(repo_url, expected_purl):
pkg = Package(name="foo")
assert pkg.to_vcs_purl(repo_url, GIT_REF) == expected_purl
def patch_request(request_id):
"""
Modify the given request.
:param int request_id: the request ID from the URL
:return: a Flask JSON response
:rtype: flask.Response
:raise NotFound: if the request is not found
:raise ValidationError: if the JSON is invalid
"""
allowed_users = flask.current_app.config['CACHITO_WORKER_USERNAMES']
# current_user.is_authenticated is only ever False when auth is disabled
if current_user.is_authenticated and current_user.username not in allowed_users:
raise Unauthorized(
'This API endpoint is restricted to Cachito workers')
payload = flask.request.get_json()
if not isinstance(payload, dict):
raise ValidationError('The input data must be a JSON object')
if not payload:
raise ValidationError(
'At least one key must be specified to update the request')
valid_keys = {
'dependencies', 'environment_variables', 'packages', 'pkg_managers',
'state', 'state_reason'
}
invalid_keys = set(payload.keys()) - valid_keys
if invalid_keys:
raise ValidationError('The following keys are not allowed: {}'.format(
', '.join(invalid_keys)))
for key, value in payload.items():
if key in ('dependencies', 'packages',
'pkg_managers') and not isinstance(value, list):
raise ValidationError(f'The value for "{key}" must be an array')
if key == 'dependencies':
for dep in value:
Dependency.validate_json(dep, for_update=True)
elif key == 'packages':
for dep in value:
Package.validate_json(dep)
elif key == 'pkg_managers':
for pkg_manager in value:
if not isinstance(pkg_manager, str):
raise ValidationError(
'The value for "pkg_managers" must be an array of strings'
)
elif key == 'environment_variables':
if not isinstance(value, dict):
raise ValidationError(
'The value for "{}" must be an object'.format(key))
for env_var_name, env_var_value in value.items():
EnvironmentVariable.validate_json(env_var_name, env_var_value)
elif not isinstance(value, str):
raise ValidationError(
'The value for "{}" must be a string'.format(key))
if 'state' in payload and 'state_reason' not in payload:
raise ValidationError(
'The "state_reason" key is required when "state" is supplied')
elif 'state_reason' in payload and 'state' not in payload:
raise ValidationError(
'The "state" key is required when "state_reason" is supplied')
request = Request.query.get_or_404(request_id)
delete_bundle = False
delete_bundle_temp = False
if 'state' in payload and 'state_reason' in payload:
new_state = payload['state']
delete_bundle = new_state == 'stale'
delete_bundle_temp = new_state in ('complete', 'failed')
new_state_reason = payload['state_reason']
# This is to protect against a Celery task getting executed twice and setting the
# state each time
if request.state.state_name == new_state and request.state.state_reason == new_state_reason:
flask.current_app.logger.info(
'Not adding a new state since it matches the last state')
else:
request.add_state(new_state, new_state_reason)
if 'dependencies' in payload:
for dep_and_replaces in payload['dependencies']:
dep = copy.deepcopy(dep_and_replaces)
replaces = dep.pop('replaces', None)
dep_object = Dependency.get_or_create(dep)
replaces_object = None
if replaces:
replaces_object = Dependency.get_or_create(replaces)
request.add_dependency(dep_object, replaces_object)
for package in payload.get('packages', []):
package_object = Package.get_or_create(package)
if package_object not in request.packages:
request.packages.append(package_object)
if 'pkg_managers' in payload:
pkg_managers = PackageManager.get_pkg_managers(payload['pkg_managers'])
for pkg_manager in pkg_managers:
if pkg_manager not in request.pkg_managers:
request.pkg_managers.append(pkg_manager)
for name, value in payload.get('environment_variables', {}).items():
env_var_obj = EnvironmentVariable.query.filter_by(name=name,
value=value).first()
if not env_var_obj:
env_var_obj = EnvironmentVariable.from_json(name, value)
db.session.add(env_var_obj)
if env_var_obj not in request.environment_variables:
request.environment_variables.append(env_var_obj)
db.session.commit()
if delete_bundle and os.path.exists(request.bundle_archive):
flask.current_app.logger.info('Deleting the bundle archive %s',
request.bundle_archive)
try:
os.remove(request.bundle_archive)
except: # noqa E722
flask.current_app.logger.exception(
'Failed to delete the bundle archive %s',
request.bundle_archive)
if delete_bundle_temp and os.path.exists(request.bundle_temp_files):
flask.current_app.logger.debug(
'Deleting the temporary files used to create the bundle at %s',
request.bundle_temp_files,
)
try:
shutil.rmtree(request.bundle_temp_files)
except: # noqa E722
flask.current_app.logger.exception(
'Failed to delete the temporary files at %s',
request.bundle_temp_files)
if current_user.is_authenticated:
flask.current_app.logger.info('The user %s patched request %d',
current_user.username, request.id)
else:
flask.current_app.logger.info('An anonymous user patched request %d',
request.id)
return flask.jsonify(request.to_json()), 200