def download_archive(request_id):
"""
Download archive of source code.
:param int request_id: the value of the request ID
:return: a Flask send_file response
:rtype: flask.Response
:raise NotFound: if the request is not found
"""
request = Request.query.get_or_404(request_id)
if request.state.state_name != "complete":
raise ValidationError(
'The request must be in the "complete" state before downloading the archive'
)
bundle_dir = RequestBundleDir(
request.id, root=flask.current_app.config["CACHITO_BUNDLES_DIR"])
if not bundle_dir.bundle_archive_file.exists():
flask.current_app.logger.error(
"The bundle archive at %s for request %d doesn't exist",
bundle_dir.bundle_archive_file,
request_id,
)
raise InternalServerError()
flask.current_app.logger.debug("Sending the bundle at %s for request %d",
bundle_dir.bundle_archive_file, request_id)
return flask.send_file(
str(bundle_dir.bundle_archive_file),
mimetype="application/gzip",
as_attachment=True,
attachment_filename=f"cachito-{request_id}.tar.gz",
)
def test_verify_npm_files(tmpdir):
app_dir = tmpdir.mkdir("temp").mkdir("1").mkdir("app")
app_dir.join("package.json").write(b"{}")
app_dir.join("package-lock.json").write(b"{}")
bundle_dir = RequestBundleDir(1, str(tmpdir))
npm._verify_npm_files(bundle_dir, ["."])
def test_verify_npm_files_no_package_json(tmpdir):
app_dir = tmpdir.mkdir("temp").mkdir("1").mkdir("app").mkdir("client")
app_dir.join("package-lock.json").write(b"{}")
bundle_dir = RequestBundleDir(1, str(tmpdir))
expected = "The client/package.json file must be present for the npm package manager"
with pytest.raises(CachitoError, match=expected):
npm._verify_npm_files(bundle_dir, ["client"])
def mock_bundle_dir(tmp_path):
root_dir = tmp_path / "temp" / "1" / "app"
root_dir.mkdir(parents=True)
sub_dir = root_dir / "sub"
sub_dir.mkdir()
bundle_dir = RequestBundleDir(1, str(tmp_path))
return bundle_dir, root_dir, sub_dir
def test_verify_npm_files_node_modules(tmpdir):
app_dir = tmpdir.mkdir("temp").mkdir("1").mkdir("app").mkdir("client")
app_dir.join("package.json").write(b"{}")
app_dir.join("package-lock.json").write(b"{}")
app_dir.mkdir("node_modules")
bundle_dir = RequestBundleDir(1, str(tmpdir))
expected = "The client/node_modules directory cannot be present in the source repository"
with pytest.raises(CachitoError, match=expected):
npm._verify_npm_files(bundle_dir, ["client"])
def test_verify_yarn_files(tmp_path):
bundle_dir, root, sub = mock_bundle_dir(tmp_path)
(root / "package.json").touch()
(root / "yarn.lock").touch()
(sub / "package.json").touch()
(sub / "yarn.lock").touch()
bundle_dir = RequestBundleDir(1, str(tmp_path))
yarn._verify_yarn_files(bundle_dir, [".", "sub"])
def patch_request(request_id):
"""
Modify the given request.
:param int request_id: the request ID from the URL
:return: a Flask JSON response
:rtype: flask.Response
:raise NotFound: if the request is not found
:raise ValidationError: if the JSON is invalid
"""
payload = flask.request.get_json()
if not isinstance(payload, dict):
raise ValidationError("The input data must be a JSON object")
if not payload:
raise ValidationError(
"At least one key must be specified to update the request")
valid_keys = {
"dependencies",
"environment_variables",
"package",
"package_subpath",
"state",
"state_reason",
}
invalid_keys = set(payload.keys()) - valid_keys
if invalid_keys:
raise ValidationError("The following keys are not allowed: {}".format(
", ".join(invalid_keys)))
for key, value in payload.items():
if key == "dependencies":
if not isinstance(value, list):
raise ValidationError(
'The value for "dependencies" must be an array')
if "package" not in payload:
raise ValidationError(
'The "package" object must also be provided if the "dependencies" array is '
"provided")
for dep in value:
Dependency.validate_json(dep, for_update=True)
elif key == "package":
Package.validate_json(value)
elif key == "environment_variables":
if not isinstance(value, dict):
raise ValidationError(
'The value for "{}" must be an object'.format(key))
for env_var_name, env_var_info in value.items():
EnvironmentVariable.validate_json(env_var_name, env_var_info)
elif not isinstance(value, str):
raise ValidationError(
'The value for "{}" must be a string'.format(key))
if "package_subpath" in payload and "package" not in payload:
raise ValidationError(
'The "package" object must also be provided if "package_subpath" is provided'
)
if "state" in payload and "state_reason" not in payload:
raise ValidationError(
'The "state_reason" key is required when "state" is supplied')
elif "state_reason" in payload and "state" not in payload:
raise ValidationError(
'The "state" key is required when "state_reason" is supplied')
request = Request.query.get_or_404(request_id)
delete_bundle = False
delete_bundle_temp = False
cleanup_nexus = []
delete_logs = False
if "state" in payload and "state_reason" in payload:
new_state = payload["state"]
delete_bundle = new_state == "stale" and request.state.state_name != "failed"
if new_state in ("stale", "failed"):
for pkg_manager in ["npm", "pip", "yarn"]:
if any(p.name == pkg_manager for p in request.pkg_managers):
cleanup_nexus.append(pkg_manager)
delete_bundle_temp = new_state in ("complete", "failed")
delete_logs = new_state == "stale"
new_state_reason = payload["state_reason"]
# This is to protect against a Celery task getting executed twice and setting the
# state each time
if request.state.state_name == new_state and request.state.state_reason == new_state_reason:
flask.current_app.logger.info(
"Not adding a new state since it matches the last state")
else:
request.add_state(new_state, new_state_reason)
package_object = None
if "package" in payload:
package_object = Package.get_or_create(payload["package"])
package_attrs = {}
# The presence of "package_subpath" in payload indicates whether to modify the subpath.
# This is only allowed when creating a new package, so when the PATCH API is used to
# modify an existing package, the user must make sure to use the same subpath (or no
# subpath).
if "package_subpath" in payload:
package_attrs["subpath"] = payload["package_subpath"]
request.add_package(package_object, **package_attrs)
for dep_and_replaces in payload.get("dependencies", []):
dep = copy.deepcopy(dep_and_replaces)
replaces = dep.pop("replaces", None)
dep_object = Dependency.get_or_create(dep)
replaces_object = None
if replaces:
replaces_object = Dependency.get_or_create(replaces)
request.add_dependency(package_object, dep_object, replaces_object)
for env_var_name, env_var_info in payload.get("environment_variables",
{}).items():
env_var_obj = EnvironmentVariable.query.filter_by(
name=env_var_name, **env_var_info).first()
if not env_var_obj:
env_var_obj = EnvironmentVariable.from_json(
env_var_name, env_var_info)
db.session.add(env_var_obj)
if env_var_obj not in request.environment_variables:
request.environment_variables.append(env_var_obj)
db.session.commit()
bundle_dir = RequestBundleDir(
request.id, root=flask.current_app.config["CACHITO_BUNDLES_DIR"])
if delete_bundle and bundle_dir.bundle_archive_file.exists():
flask.current_app.logger.info("Deleting the bundle archive %s",
bundle_dir.bundle_archive_file)
try:
bundle_dir.bundle_archive_file.unlink()
except: # noqa E722
flask.current_app.logger.exception(
"Failed to delete the bundle archive %s",
bundle_dir.bundle_archive_file)
if delete_bundle_temp and bundle_dir.exists():
flask.current_app.logger.debug(
"Deleting the temporary files used to create the bundle at %s",
bundle_dir)
try:
bundle_dir.rmtree()
except: # noqa E722
flask.current_app.logger.exception(
"Failed to delete the temporary files at %s", bundle_dir)
if delete_logs:
request_log_dir = flask.current_app.config[
"CACHITO_REQUEST_FILE_LOGS_DIR"]
path_to_file = os.path.join(request_log_dir, f"{request_id}.log")
try:
os.remove(path_to_file)
except: # noqa E722
flask.current_app.logger.exception(
"Failed to delete the log file %s", path_to_file)
for pkg_mgr in cleanup_nexus:
flask.current_app.logger.info(
"Cleaning up the Nexus %s content for request %d", pkg_mgr,
request_id)
cleanup_task = getattr(tasks, f"cleanup_{pkg_mgr}_request")
try:
cleanup_task.delay(request_id)
except kombu.exceptions.OperationalError:
flask.current_app.logger.exception(
"Failed to schedule the cleanup_%s_request task for request %d. An administrator "
"must clean this up manually.",
pkg_mgr,
request.id,
)
if current_user.is_authenticated:
flask.current_app.logger.info("The user %s patched request %d",
current_user.username, request.id)
else:
flask.current_app.logger.info("An anonymous user patched request %d",
request.id)
return flask.jsonify(request.to_json()), 200