def download_archive(request_id): """ Download archive of source code. :param int request_id: the value of the request ID :return: a Flask send_file response :rtype: flask.Response :raise NotFound: if the request is not found """ request = Request.query.get_or_404(request_id) if request.state.state_name != "complete": raise ValidationError( 'The request must be in the "complete" state before downloading the archive' ) bundle_dir = RequestBundleDir( request.id, root=flask.current_app.config["CACHITO_BUNDLES_DIR"]) if not bundle_dir.bundle_archive_file.exists(): flask.current_app.logger.error( "The bundle archive at %s for request %d doesn't exist", bundle_dir.bundle_archive_file, request_id, ) raise InternalServerError() flask.current_app.logger.debug("Sending the bundle at %s for request %d", bundle_dir.bundle_archive_file, request_id) return flask.send_file( str(bundle_dir.bundle_archive_file), mimetype="application/gzip", as_attachment=True, attachment_filename=f"cachito-{request_id}.tar.gz", )
def test_verify_npm_files(tmpdir): app_dir = tmpdir.mkdir("temp").mkdir("1").mkdir("app") app_dir.join("package.json").write(b"{}") app_dir.join("package-lock.json").write(b"{}") bundle_dir = RequestBundleDir(1, str(tmpdir)) npm._verify_npm_files(bundle_dir, ["."])
def test_verify_npm_files_no_package_json(tmpdir): app_dir = tmpdir.mkdir("temp").mkdir("1").mkdir("app").mkdir("client") app_dir.join("package-lock.json").write(b"{}") bundle_dir = RequestBundleDir(1, str(tmpdir)) expected = "The client/package.json file must be present for the npm package manager" with pytest.raises(CachitoError, match=expected): npm._verify_npm_files(bundle_dir, ["client"])
def mock_bundle_dir(tmp_path): root_dir = tmp_path / "temp" / "1" / "app" root_dir.mkdir(parents=True) sub_dir = root_dir / "sub" sub_dir.mkdir() bundle_dir = RequestBundleDir(1, str(tmp_path)) return bundle_dir, root_dir, sub_dir
def test_verify_npm_files_node_modules(tmpdir): app_dir = tmpdir.mkdir("temp").mkdir("1").mkdir("app").mkdir("client") app_dir.join("package.json").write(b"{}") app_dir.join("package-lock.json").write(b"{}") app_dir.mkdir("node_modules") bundle_dir = RequestBundleDir(1, str(tmpdir)) expected = "The client/node_modules directory cannot be present in the source repository" with pytest.raises(CachitoError, match=expected): npm._verify_npm_files(bundle_dir, ["client"])
def test_verify_yarn_files(tmp_path): bundle_dir, root, sub = mock_bundle_dir(tmp_path) (root / "package.json").touch() (root / "yarn.lock").touch() (sub / "package.json").touch() (sub / "yarn.lock").touch() bundle_dir = RequestBundleDir(1, str(tmp_path)) yarn._verify_yarn_files(bundle_dir, [".", "sub"])
def patch_request(request_id): """ Modify the given request. :param int request_id: the request ID from the URL :return: a Flask JSON response :rtype: flask.Response :raise NotFound: if the request is not found :raise ValidationError: if the JSON is invalid """ payload = flask.request.get_json() if not isinstance(payload, dict): raise ValidationError("The input data must be a JSON object") if not payload: raise ValidationError( "At least one key must be specified to update the request") valid_keys = { "dependencies", "environment_variables", "package", "package_subpath", "state", "state_reason", } invalid_keys = set(payload.keys()) - valid_keys if invalid_keys: raise ValidationError("The following keys are not allowed: {}".format( ", ".join(invalid_keys))) for key, value in payload.items(): if key == "dependencies": if not isinstance(value, list): raise ValidationError( 'The value for "dependencies" must be an array') if "package" not in payload: raise ValidationError( 'The "package" object must also be provided if the "dependencies" array is ' "provided") for dep in value: Dependency.validate_json(dep, for_update=True) elif key == "package": Package.validate_json(value) elif key == "environment_variables": if not isinstance(value, dict): raise ValidationError( 'The value for "{}" must be an object'.format(key)) for env_var_name, env_var_info in value.items(): EnvironmentVariable.validate_json(env_var_name, env_var_info) elif not isinstance(value, str): raise ValidationError( 'The value for "{}" must be a string'.format(key)) if "package_subpath" in payload and "package" not in payload: raise ValidationError( 'The "package" object must also be provided if "package_subpath" is provided' ) if "state" in payload and "state_reason" not in payload: raise ValidationError( 'The "state_reason" key is required when "state" is supplied') elif "state_reason" in payload and "state" not in payload: raise ValidationError( 'The "state" key is required when "state_reason" is supplied') request = Request.query.get_or_404(request_id) delete_bundle = False delete_bundle_temp = False cleanup_nexus = [] delete_logs = False if "state" in payload and "state_reason" in payload: new_state = payload["state"] delete_bundle = new_state == "stale" and request.state.state_name != "failed" if new_state in ("stale", "failed"): for pkg_manager in ["npm", "pip", "yarn"]: if any(p.name == pkg_manager for p in request.pkg_managers): cleanup_nexus.append(pkg_manager) delete_bundle_temp = new_state in ("complete", "failed") delete_logs = new_state == "stale" new_state_reason = payload["state_reason"] # This is to protect against a Celery task getting executed twice and setting the # state each time if request.state.state_name == new_state and request.state.state_reason == new_state_reason: flask.current_app.logger.info( "Not adding a new state since it matches the last state") else: request.add_state(new_state, new_state_reason) package_object = None if "package" in payload: package_object = Package.get_or_create(payload["package"]) package_attrs = {} # The presence of "package_subpath" in payload indicates whether to modify the subpath. # This is only allowed when creating a new package, so when the PATCH API is used to # modify an existing package, the user must make sure to use the same subpath (or no # subpath). if "package_subpath" in payload: package_attrs["subpath"] = payload["package_subpath"] request.add_package(package_object, **package_attrs) for dep_and_replaces in payload.get("dependencies", []): dep = copy.deepcopy(dep_and_replaces) replaces = dep.pop("replaces", None) dep_object = Dependency.get_or_create(dep) replaces_object = None if replaces: replaces_object = Dependency.get_or_create(replaces) request.add_dependency(package_object, dep_object, replaces_object) for env_var_name, env_var_info in payload.get("environment_variables", {}).items(): env_var_obj = EnvironmentVariable.query.filter_by( name=env_var_name, **env_var_info).first() if not env_var_obj: env_var_obj = EnvironmentVariable.from_json( env_var_name, env_var_info) db.session.add(env_var_obj) if env_var_obj not in request.environment_variables: request.environment_variables.append(env_var_obj) db.session.commit() bundle_dir = RequestBundleDir( request.id, root=flask.current_app.config["CACHITO_BUNDLES_DIR"]) if delete_bundle and bundle_dir.bundle_archive_file.exists(): flask.current_app.logger.info("Deleting the bundle archive %s", bundle_dir.bundle_archive_file) try: bundle_dir.bundle_archive_file.unlink() except: # noqa E722 flask.current_app.logger.exception( "Failed to delete the bundle archive %s", bundle_dir.bundle_archive_file) if delete_bundle_temp and bundle_dir.exists(): flask.current_app.logger.debug( "Deleting the temporary files used to create the bundle at %s", bundle_dir) try: bundle_dir.rmtree() except: # noqa E722 flask.current_app.logger.exception( "Failed to delete the temporary files at %s", bundle_dir) if delete_logs: request_log_dir = flask.current_app.config[ "CACHITO_REQUEST_FILE_LOGS_DIR"] path_to_file = os.path.join(request_log_dir, f"{request_id}.log") try: os.remove(path_to_file) except: # noqa E722 flask.current_app.logger.exception( "Failed to delete the log file %s", path_to_file) for pkg_mgr in cleanup_nexus: flask.current_app.logger.info( "Cleaning up the Nexus %s content for request %d", pkg_mgr, request_id) cleanup_task = getattr(tasks, f"cleanup_{pkg_mgr}_request") try: cleanup_task.delay(request_id) except kombu.exceptions.OperationalError: flask.current_app.logger.exception( "Failed to schedule the cleanup_%s_request task for request %d. An administrator " "must clean this up manually.", pkg_mgr, request.id, ) if current_user.is_authenticated: flask.current_app.logger.info("The user %s patched request %d", current_user.username, request.id) else: flask.current_app.logger.info("An anonymous user patched request %d", request.id) return flask.jsonify(request.to_json()), 200