def _instantiate_project( requirements: str, requirements_locked: Optional[str] = None, *, runtime_environment: RuntimeEnvironment = None, constraints: Optional[str] = None, ): """Create Project instance based on arguments passed to CLI.""" try: with open(requirements, "r") as requirements_file: requirements = requirements_file.read() except (OSError, FileNotFoundError): # We we gather values from env vars, un-escape new lines. requirements = requirements.replace("\\n", "\n") if requirements_locked: try: with open(requirements_locked, "r") as requirements_file: requirements_locked = requirements_file.read() except (OSError, FileNotFoundError): # We we gather values from env vars, un-escape new lines. requirements_locked = requirements_locked.replace("\\n", "\n") pipfile = Pipfile.from_string(requirements) pipfile_lock = None if requirements_locked and requirements_locked != "null": pipfile_lock = PipfileLock.from_string(requirements_locked, pipfile) constraints_instance = None if constraints: try: with open(constraints, "r") as constraints_file: constraints_content = constraints_file.read() except (OSError, FileNotFoundError): # We we gather values from env vars, un-escape new lines. constraints_content = constraints.replace("\\n", "\n") try: constraints_instance = Constraints.from_dict( json.loads(constraints_content)) except json.decoder.JSONDecodeError: constraints_instance = Constraints.from_string(constraints_content) runtime_environment = runtime_environment or RuntimeEnvironment.from_dict( {}) if not runtime_environment.platform: runtime_environment.platform = _DEFAULT_PLATFORM project = Project( pipfile=pipfile, pipfile_lock=pipfile_lock, runtime_environment=runtime_environment, constraints=constraints_instance or Constraints(), ) return project
def test_verify_multiple_should_include(self, builder_context: PipelineBuilderContext) -> None: """Verify multiple should_include calls do not loop endlessly.""" builder_context.project.constraints = Constraints.from_file( str(self.data_dir / "constraints" / "constraints_0.txt") ) builder_context.project.runtime_environment.python_version = "3.9" self.verify_multiple_should_include(builder_context)
def test_not_include(self, builder_context: PipelineBuilderContext) -> None: """Test not including this pipeline unit.""" builder_context.project.runtime_environment.python_version = "1.9" builder_context.project.constraints = Constraints.from_file( str(self.data_dir / "constraints" / "constraints_0.txt") ) assert list(self.UNIT_TESTED.should_include(builder_context)) == []
def test_include(self, builder_context: PipelineBuilderContext) -> None: """Test including this pipeline unit.""" builder_context.project.runtime_environment.python_version = "3.9" builder_context.project.constraints = Constraints.from_file( str(self.data_dir / "constraints" / "constraints_0.txt") ) assert list(self.UNIT_TESTED.should_include(builder_context)) == [ {"package_name": "numpy", "specifier": "~=2.0"} ]
def test_include_version(self, builder_context: PipelineBuilderContext, context: "Context") -> None: """Test not including and running this pipeline unit if version is not provided in constraints.""" builder_context.project.runtime_environment.python_version = "3.9" builder_context.project.constraints = Constraints.from_file( str(self.data_dir / "constraints" / "constraints_1.txt") ) config = list(self.UNIT_TESTED.should_include(builder_context)) assert len(config) == 1 assert config == [{"package_name": "numpy", "specifier": "*"}] unit = self.UNIT_TESTED() unit.update_configuration(config[0]) package_version_1 = PackageVersion( name="numpy", version="==0.0.1", index=Source("https://pypi.org/simple"), develop=True ) with unit.assigned_context(context): unit.pre_run() assert list(unit.run((pv for pv in (package_version_1,)))) == [package_version_1]
def post_advise_python( input: dict, recommendation_type: typing.Optional[str] = None, count: typing.Optional[int] = None, limit: typing.Optional[int] = None, source_type: typing.Optional[str] = None, debug: bool = False, force: bool = False, dev: bool = False, origin: typing.Optional[str] = None, token: typing.Optional[str] = None, github_event_type: typing.Optional[str] = None, github_check_run_id: typing.Optional[int] = None, github_installation_id: typing.Optional[int] = None, github_base_repo_url: typing.Optional[str] = None, ): """Compute results for the given package or package stack using adviser.""" parameters = locals() # Translate request body parameters. parameters["application_stack"] = parameters["input"].pop( "application_stack", None) parameters["justification"] = parameters["input"].pop( "justification", None) parameters["stack_info"] = parameters["input"].pop("stack_info", None) parameters["kebechet_metadata"] = parameters["input"].pop( "kebechet_metadata", None) parameters["labels"] = parameters["input"].pop("labels", None) token = parameters.pop("token", None) authenticated = False if token is not None: if Configuration.API_TOKEN != token: return {"error": "Bad token supplied"}, 401 authenticated = True else: for k in _ADVISE_PROTECTED_FIELDS: if parameters[k] is not None: return { "error": f"Parameter {k!r} requires token to be set to perform authenticated request" }, 401 # Always try to parse runtime environment so that we have it available in JSON reports in a unified form. try: parameters["runtime_environment"] = RuntimeEnvironment.from_dict( parameters["input"].pop("runtime_environment", {})).to_dict() except Exception as exc: return { "parameters": parameters, "error": f"Failed to parse runtime environment: {str(exc)}" } try: constraints = Constraints.from_string( parameters["input"].pop("constraints", None) or "") except Exception as exc: return { "parameters": parameters, "error": f"Invalid constraints supplied: {str(exc)}" }, 400 parameters["library_usage"] = parameters["input"].pop( "library_usage", None) parameters.pop("input") force = parameters.pop("force", False) if parameters[ "library_usage"]: # Sort library usage to hit cache properly. for key, value in (parameters["library_usage"].get("report") or {}).items(): parameters["library_usage"]["report"][key] = sorted(value) try: project = Project.from_strings( parameters["application_stack"]["requirements"], parameters["application_stack"].get("requirements_lock"), runtime_environment=RuntimeEnvironment.from_dict( parameters["runtime_environment"]), constraints=constraints, ) except ThothPythonException as exc: return { "parameters": parameters, "error": f"Invalid application stack supplied: {str(exc)}" }, 400 except Exception: return { "parameters": parameters, "error": "Invalid application stack supplied" }, 400 # We could rewrite this to a decorator and make it shared with provenance # checks etc, but there are small glitches why the solution would not be # generic enough to be used for all POST endpoints. adviser_cache = AdvisersCacheStore() adviser_cache.connect() timestamp_now = int(time.mktime(datetime.datetime.utcnow().timetuple())) if authenticated: cached_document_id = _compute_digest_params( dict( **project.to_dict(), count=parameters["count"], limit=parameters["limit"], library_usage=parameters["library_usage"], recommendation_type=recommendation_type, origin=origin, source_type=source_type.upper() if source_type else None, dev=dev, debug=parameters["debug"], github_event_type=parameters["github_event_type"], github_check_run_id=parameters["github_check_run_id"], github_installation_id=parameters["github_installation_id"], github_base_repo_url=parameters["github_base_repo_url"], kebechet_metadata=parameters["kebechet_metadata"], labels=parameters["labels"], )) else: cached_document_id = _compute_digest_params( dict( **project.to_dict(), count=parameters["count"], limit=parameters["limit"], library_usage=parameters["library_usage"], recommendation_type=recommendation_type, dev=dev, debug=parameters["debug"], labels=parameters["labels"], )) if not force: try: cache_record = adviser_cache.retrieve_document_record( cached_document_id) if cache_record[ "timestamp"] + Configuration.THOTH_CACHE_EXPIRATION > timestamp_now: return { "analysis_id": cache_record.pop("analysis_id"), "cached": True, "authenticated": authenticated, "parameters": parameters, }, 202 except CacheMiss: pass # Enum type is checked on thoth-common side to avoid serialization issue in user-api side when providing response parameters["source_type"] = source_type.upper() if source_type else None parameters["constraints"] = constraints.to_dict() parameters["job_id"] = _OPENSHIFT.generate_id("adviser") # Remove data passed via Ceph. message = dict(**parameters, authenticated=authenticated) message.pop("application_stack") message.pop("runtime_environment") message.pop("library_usage") message.pop("labels") message.pop("constraints") response, status = _send_schedule_message(message, adviser_trigger_message, AdviserTriggerContent, with_authentication=True, authenticated=authenticated) if status == 202: adviser_cache.store_document_record( cached_document_id, { "analysis_id": response["analysis_id"], "timestamp": timestamp_now }) # Store the request for traceability. store = AdvisersResultsStore() store.connect() store.store_request(parameters["job_id"], parameters) return response, status