def test_cut_locked(self) -> None: """Test removing a locked package based on direct dependencies.""" tf = PackageVersion( name="tensorflow", version="==2.0.0", index=Source("https://pypi.org/simple"), develop=False, ) context = flexmock(project=Project.from_strings(self._CASE_PIPFILE_LOCKED)) with CutLockedSieve.assigned_context(context): sieve = CutLockedSieve() assert list(sieve.run(p for p in [tf])) == []
def test_os_sieve_no_remove(self): """Test the TensorFlow package is not removed as it has no other candidate.""" package_versions = [ PackageVersion( name="tensorflow", version="==1.9.0", index=Source( "https://tensorflow.pypi.thoth-station.ninja/index/fedora/30/jemalloc/simple/" ), develop=False, ), PackageVersion( name="pytest", version="==3.0.0", index=Source("https://pypi.org/simple"), develop=True, ), ] sieve_context = SieveContext.from_package_versions(package_versions) # Do not assign runtime environment intentionally - it will default to no environment. project = Project.from_strings( pipfile_str=self._PIPFILE_CONTENT_AICOE, runtime_environment=RuntimeEnvironment.from_dict( {"operating_system": { "name": "rhel", "version": "7.5" }})) os_sieve = OperatingSystemSieve(graph=None, project=project) os_sieve.run(sieve_context) expected = { ("pytest", "3.0.0", "https://pypi.org/simple"), ("tensorflow", "1.9.0", "https://tensorflow.pypi.thoth-station.ninja/index/fedora/30/jemalloc/simple/" ), } assert set(sieve_context.iter_direct_dependencies_tuple()) == expected
def test_remove_pre_releases_allowed_noop(self) -> None: """Test removing dependencies not hitting limit causes a noop.""" tf_2_0_0rc = PackageVersion( name="tensorflow", version="==2.0.0rc0", index=Source("https://pypi.org/simple"), develop=False, ) context = flexmock(project=Project.from_strings(self._CASE_ALLOWED_PIPFILE)) with CutPreReleasesSieve.assigned_context(context): sieve = CutPreReleasesSieve() assert list(sieve.run(p for p in [tf_2_0_0rc])) == [tf_2_0_0rc]
def run( self, _: State, package_version: PackageVersion ) -> Optional[Tuple[Optional[float], Optional[List[Dict[str, str]]]]]: """Score the given package regardless of the state.""" # Using seed set to process on the adviser run affects this call - so adviser # with same seed set shared scores generated across runs. score = self._score_history.setdefault( package_version.to_tuple(), random.uniform(self.SCORE_MIN, self.SCORE_MAX) if random.random() <= self.configuration["assign_probability"] else 0.0, ) return score, None
def register_package_version(self, package_version: PackageVersion) -> bool: """Register the given package version to the context.""" package_tuple = package_version.to_tuple() registered = self.package_versions.get(package_tuple) if registered: # If the given package is shared in develop and in the main part, make it main stack part. registered.develop = registered.develop or package_version.develop return True # Direct dependency, no dependency introduced this one. self._note_dependencies(package_tuple=None, dependency_tuple=package_tuple) self.package_versions[package_tuple] = package_version return False
def test_from_pip_compile_files_example_dir3(self) -> None: """Test loading only if only requirements.in is present.""" with cwd(os.path.join(self.data_dir, "requirements", "example_dir3")): project = Project.from_pip_compile_files(allow_without_lock=True) assert project.pipfile_lock is None assert list(project.iter_dependencies()) == [ PackageVersion(name="flask", version="*", develop=False, hashes=[], index=None) ]
def test_db_0_all_versions(self, graph): """Check that resolving can gather all versions available in the graph database.""" resolved = PythonGraphSolver(graph_db=graph).solve( [PackageVersion(name="a", version="*", index=None, develop=False)], graceful=False, all_versions=True, ) assert len(resolved) == 1 assert "a" in resolved assert resolved == { "a": [("1.0.0", "index1"), ("1.1.0", "index1"), ("1.2.0", "index2")] }
def test_noop_dev(self) -> None: """Test no operation if dependencies are not locked.""" pytest = PackageVersion( name="pytest", version="==5.3.1", index=Source("https://pypi.org/simple"), develop=False, ) context = flexmock(project=Project.from_strings(self._CASE_PIPFILE_NOT_LOCKED)) with CutLockedSieve.assigned_context(context): sieve = CutLockedSieve() assert list(sieve.run(p for p in [pytest])) == [pytest]
def test_pre_releases_disallowed_removal(self) -> None: """Test no removals if pre-releases are allowed.""" tf_2_0_0rc0 = PackageVersion( name="tensorflow", version="==2.0.0rc0", index=Source("https://tensorflow.pypi.thoth-station.ninja/index/os/fedora/30/jemalloc/simple/"), develop=False, ) context = flexmock(project=Project.from_strings(self._CASE_DISALLOWED_PIPFILE)) with CutPreReleasesSieve.assigned_context(context): sieve = CutPreReleasesSieve() assert list(sieve.run(p for p in [tf_2_0_0rc0])) == []
def test_remove_all_direct_error(self): """Test raising an error if all the direct deps of a type were removed.""" source = Source("https://pypi.org/simple") direct_dependencies = { ("tensorflow", "2.0.0", source.url): PackageVersion( name="tensorflow", version="==2.0.0", index=source, develop=False, ), ("numpy", "1.0.0rc1", source.url): PackageVersion( name="numpy", version="==1.0.0rc1", index=source, develop=False, ), } paths = { ("tensorflow", "2.0.0", "https://pypi.org/simple"): [ (("tensorflow", "2.0.0", "https://pypi.org/simple"), ("absl-py", "1.0.0", "https://pypi.org/simple")), ] } step_context = StepContext.from_paths(direct_dependencies, paths) project = Project.from_strings(self._CASE_PIPFILE) restrict_indexes = CutPreReleases( graph=None, project=project, library_usage=None, ) with pytest.raises(CannotRemovePackage): restrict_indexes.run(step_context)
def test_sieve_develop_true(self, context: Context, develop: bool) -> None: """Test sieving packages based on develop flag set to true.""" prescription_str = f""" name: SieveUnit type: sieve should_include: times: 1 adviser_pipeline: true match: package_version: name: flask develop: {'true' if develop else 'false'} """ prescription = yaml.safe_load(prescription_str) PRESCRIPTION_SIEVE_SCHEMA(prescription) SievePrescription.set_prescription(prescription) package_versions = [ PackageVersion( name="flask", version="==1.1.2", index=Source("https://pypi.org/simple"), develop=develop, ), PackageVersion( name="flask", version="==1.0.2", index=Source("https://pypi.org/simple"), develop=not develop, ), ] unit = SievePrescription() unit.pre_run() with unit.assigned_context(context): result = list(unit.run((pv for pv in package_versions))) assert len(result) == 1 assert [result[0]] == [pv for pv in package_versions if pv.develop != develop]
def _python_package_name(n: object) -> None: """Validate Python package name.""" if not isinstance(n, str): raise Invalid(f"Value {n!r} is not a valid package name") try: normalized = PackageVersion.normalize_python_package_name(n) except Exception as exc: raise Invalid(f"Failed to parse Python package name {n!r}: {str(exc)}") else: if normalized != n: raise Invalid( f"Python package name {n!r} is not in a normalized form, normalized: {normalized!r}" )
def test_from_pip_compile_files_example_dir1(self) -> None: """Test loading only if requirements.txt is present.""" with cwd(os.path.join(self.data_dir, "requirements", "example_dir1")): with pytest.raises(FileLoadError): Project.from_pip_compile_files() project = Project.from_pip_compile_files(allow_without_lock=True) assert project.pipfile_lock is None assert list(project.iter_dependencies()) == [ PackageVersion( name="click", version="*", develop=False, hashes=[], index=Source("https://pypi.org/simple") ) ]
def _do_try_exclude(sieve_context: SieveContext, package_version: PackageVersion, config: dict) -> None: """Try to exclude the given package, produce warning if exclusion was not successful.""" try: sieve_context.remove_package(package_version) except CannotRemovePackage as exc: # TODO: this should go to sieve context and be reported to the user in final recommendations _LOGGER.warning( "Using package %r which was released for different operating system %r, package" "cannot be removed: %s", package_version.to_tuple(), config["os_version"], str(exc), )
def test_no_tf_avx2(self) -> None: """Test not recommending TensorFlow without AVX2 support.""" package_version = PackageVersion( name="tensorflow", version="==2.2.0", develop=False, index=Source("https://pypi.org/simple"), ) # State and context are unused in the actual pipeline run. state, context = flexmock(), flexmock() with TensorFlowAVX2Step.assigned_context(context): unit = TensorFlowAVX2Step() assert unit.run(state, package_version) is None
def run( self, _: State, package_version: PackageVersion ) -> Optional[Tuple[float, List[Dict[str, str]]]]: """Penalize stacks with a CVE.""" try: cve_records = self.context.graph.get_python_cve_records_all( package_name=package_version.name, package_version=package_version.locked_version, ) except NotFoundError as exc: _LOGGER.warning("Package %r in version %r not found: %r", exc) return None if cve_records: package_version_tuple = package_version.to_tuple() _LOGGER.debug("Found a CVEs for %r: %r", package_version_tuple, cve_records) justification = [] for cve_record in cve_records: message = f"Package {package_version_tuple!r} has a CVE {cve_record['cve_id']!r}" justification.append({ "package_name": package_version.name, "link": cve_record.get("link") or self._JUSTIFICATION_LINK, "advisory": cve_record["details"], "message": message, "type": "WARNING", }) if self.context.recommendation_type not in ( RecommendationType.LATEST, RecommendationType.TESTING): # Penalize only if not latest/testing. penalization = len( cve_records) * self.configuration["cve_penalization"] return max(penalization, -1.0), justification return 0.0, justification else: justification = [{ "package_name": package_version.name, "link": self._JUSTIFICATION_LINK_NO_CVE, "type": "INFO", "message": f"No known CVE known for {package_version.name!r} in " f"version {package_version.locked_version!r}", }] return 0.0, justification
def _get_packages_no_aicoe(): """Get packages which are not specific to AICoE.""" source = Source("https://pypi.org/simple") return [ PackageVersion( name="thoth-python", version="==1.0.0", index=source, develop=False, ), PackageVersion( name="thoth-adviser", version="==1.0.0", index=source, develop=False, ), PackageVersion( name="pytest", version="==3.0.0", index=source, develop=True, ), ]
def run( self, _: State, package_version: PackageVersion ) -> Optional[Tuple[Optional[float], Optional[List[Dict[str, str]]]]]: """Score the given package.""" package_tuple = package_version.to_tuple() score = self._history.get(package_tuple) if score is not None: return score, None idx = self._idx self._idx = (self._idx + 1) % self.configuration["buffer_size"] self._history[package_tuple] = self._buffer[idx] return self._buffer[idx], None
def register_package_tuple( self, package_tuple: Tuple[str, str, str], *, develop: bool, dependent_tuple: Optional[Tuple[str, str, str]] = None, extras: Optional[List[str]] = None, os_name: Optional[str], os_version: Optional[str], python_version: Optional[str], ) -> PackageVersion: """Register the given package tuple to pipeline context and return its package version representative.""" registered = self.package_versions.get(package_tuple) if registered: # If the given package is shared in develop and in the main part, make it main stack part. registered.develop = registered.develop or develop self._note_dependencies( dependent_tuple, package_tuple, os_name=os_name, os_version=os_version, python_version=python_version, ) # This method is called solely on transitive dependencies - for those we do not track # extras as extras are already resolved by solver runs (pre-computed). Keep extras untouched # in this function call. return registered source = self.sources.get(package_tuple[2]) if not source: source = Source(package_tuple[2]) self.sources[package_tuple[2]] = source package_version = PackageVersion( name=package_tuple[0], version="==" + package_tuple[1], index=source, extras=extras, develop=develop, ) self.package_versions[package_tuple] = package_version self._note_dependencies( dependent_tuple, package_tuple, os_name=os_name, os_version=os_version, python_version=python_version, ) return package_version
def test_db_0_package_raises(self, graph: MockedGraphDatabase) -> None: """Check that there is raised an exception if no releases were found.""" with pytest.raises(SolverException): PythonPackageGraphSolver(graph=graph).solve( [ PackageVersion( name="nonexisting-foo", version="==1.0.0", index=None, develop=False, ) ], graceful=False, )
def test_rule(self, context: Context) -> None: """Test if a rule is assigned to a package.""" package_version = PackageVersion( name="flask", version="==1.1.2", index=Source("https://pypi.org/simple"), develop=False) (GraphDatabase.should_receive( "get_python_package_version_solver_rules_all").with_args( "flask", "1.1.2", "https://pypi.org/simple", ).and_return(["foo"])) (GraphDatabase.should_receive( "get_python_package_version_solver_rules_all").with_args( "flask", "1.1.2", ).and_return(["bar"])) context.graph = GraphDatabase() assert not context.stack_info, "No stack info should be provided before test run" sieve = self.UNIT_TESTED() sieve.pre_run() with self.UNIT_TESTED.assigned_context(context): assert list(sieve.run(p for p in [package_version])) == [] assert context.stack_info == [ { "link": "https://thoth-station.ninja/j/rules", "message": "Removing package ('flask', '1.1.2', 'https://pypi.org/simple') based on solver " "rule configured: foo", "type": "WARNING", }, { "link": "https://thoth-station.ninja/j/rules", "message": "Removing package ('flask', '1.1.2', 'https://pypi.org/simple') based on solver " "rule configured: bar", "type": "WARNING", }, ] assert self.verify_justification_schema(context.stack_info) is True
def _get_prepared_step_context() -> StepContext: source = Source("https://pypi.org/simple") direct_dependencies = { ("flask", "0.12.0", "https://pypi.org/simple"): PackageVersion( name="flask", version="==0.12.0", index=source, develop=False, ), ("flask", "0.13.0", "https://pypi.org/simple"): PackageVersion( name="flask", version="==0.13.0", index=source, develop=False, ), } paths = { ("flask", "0.12.0", "https://pypi.org/simple"): [ (("flask", "0.12.0", "https://pypi.org/simple"), ("click", "2.0", "https://pypi.org/simple")), (("click", "2.0", "https://pypi.org/simple"), ("pyyaml", "3.12", "https://pypi.org/simple")), ], ("flask", "0.13.0", "https://pypi.org/simple"): [ (("flask", "0.13.0", "https://pypi.org/simple"), ("click", "2.0", "https://pypi.org/simple")), (("click", "2.0", "https://pypi.org/simple"), ("pyyaml", "3.12", "https://pypi.org/simple")), (("click", "2.0", "https://pypi.org/simple"), ("pyyaml", "4.0", "https://pypi.org/simple")), ], } return StepContext.from_paths(direct_dependencies, paths)
def get_latest_package_version(_, package_name): # noqa: N805 return { "certifi": PackageVersion.parse_semantic_version("2018.10.15"), "chardet": PackageVersion.parse_semantic_version("3.0.4"), "idna": PackageVersion.parse_semantic_version("2.7"), "requests": PackageVersion.parse_semantic_version("3.0.0"), "termcolor": PackageVersion.parse_semantic_version("1.1.0"), "urllib3": PackageVersion.parse_semantic_version("1.23"), }[package_name]
def test_note_dependencies(self, context: Context) -> None: """Test noting dependencies to the context.""" dependency_tuple = ("tensorboard", "2.1.0", "https://pypi.org/simple") package_tuple = ("tensorflow", "2.0.0", "https://pypi.org/simple") context.register_package_version( PackageVersion( name=package_tuple[0], version="==" + package_tuple[1], index=Source(package_tuple[2]), develop=False, )) context.register_package_tuple( dependency_tuple, develop=True, extras=None, dependent_tuple=package_tuple, os_name="fedora", os_version="31", python_version="3.7", ) package_version = context.get_package_version(package_tuple) assert package_version is not None assert package_version.name == package_tuple[0] assert package_version.locked_version == package_tuple[1] assert package_version.index.url == package_tuple[2] package_version = context.get_package_version(dependency_tuple) assert package_version is not None assert package_version.name == dependency_tuple[0] assert package_version.locked_version == dependency_tuple[1] assert package_version.index.url == dependency_tuple[2] assert package_tuple[0] in context.dependencies assert package_tuple in context.dependencies[package_tuple[0]] assert dependency_tuple in context.dependencies[ package_tuple[0]][package_tuple] assert dependency_tuple[0] in context.dependents assert dependency_tuple in context.dependents[dependency_tuple[0]] entry = context.dependents[dependency_tuple[0]][dependency_tuple] assert entry == {(package_tuple, "fedora", "31", "3.7")} # By calling register_package_version we get a notion about direct dependency. assert package_tuple[0] in context.dependents assert package_tuple in context.dependents[package_tuple[0]] assert context.dependents[package_tuple[0]][package_tuple] == set()
def test_no_tf_21(self, context: Context, h5py_version: str, tf_version: str) -> None: """Test no blocking when using h5py<3 or TensorFlow!=2.1.""" h5py_package_version = PackageVersion( name="h5py", version=f"=={h5py_version}", develop=False, index=Source("https://pypi.org/simple"), ) tf_package_version = PackageVersion( name="tensorflow", version=f"=={tf_version}", develop=False, index=Source("https://pypi.org/simple"), ) state = State() state.resolved_dependencies["tensorflow"] = tf_package_version.to_tuple() # Context is not used during the actual pipeline run. context.register_package_version(tf_package_version) with self.UNIT_TESTED.assigned_context(context): unit = self.UNIT_TESTED() assert unit.run(state, h5py_package_version) is None
def test_run_no_yield(self, context: Context, package_name: str, package_version: str) -> None: """Test packages the pipeline unit does not yield respecting Python version compatibility.""" package_version = PackageVersion( name=package_name, version=f"=={package_version}", develop=False, index=Source("https://pypi.org/simple"), ) unit = self.UNIT_TESTED() with unit.assigned_context(context): unit.pre_run() result = list(unit.run((pv for pv in (package_version, )))) assert len(result) == 0
def test_run(self, context: Context, pandas_version: str) -> None: """Test filtering out Pandas that dropped Python 3.6 support.""" package_version = PackageVersion( name="pandas", version=f"=={pandas_version}", develop=False, index=Source("https://pypi.org/simple"), ) unit = PandasPy36Sieve() unit.pre_run() with PandasPy36Sieve.assigned_context(context): assert unit._message_logged is False assert list(unit.run(p for p in [package_version])) == [] assert unit._message_logged is True
def test_run_no_filter(self) -> None: """"Test not filtering a package based on version specifier.""" package_version = PackageVersion( name="tensorboard", version="==2.1.0", index=Source("https://pypi.org/simple"), develop=False, ) unit = VersionConstraintSieve() unit.update_configuration({ "package_name": "tensorboard", "version_specifier": ">2.0", }) unit.pre_run() assert list(unit.run([package_version])) == [package_version]
def test_no_aicoe_release(self) -> None: """Make sure the stack score is untouched if not an AICoE release.""" package_version = PackageVersion( name="tensorflow", version="==2.0.0", index=Source("https://pypi.org/simple"), develop=False, ) context = flexmock() with AICoEReleasesStep.assigned_context(context): step = AICoEReleasesStep() result = step.run(None, package_version) assert result is None
def test_remove_latest_versions_noop(self): """Test removing direct dependencies not hitting limit causes a noop.""" tf_1_1_0 = PackageVersion( name="tensorflow", version="==1.1.0", index=Source( "https://tensorflow.pypi.thoth-station.ninja/index/os/fedora/30/jemalloc/simple/" ), develop=False, ) tf_1_9_0 = PackageVersion( name="tensorflow", version="==1.9.0", index=Source("https://pypi.org/simple"), develop=False, ) tf_2_0_0 = PackageVersion( name="tensorflow", version="==2.0.0", index=Source( "https://tensorflow.pypi.thoth-station.ninja/index/manylinux2010/jemalloc/simple/" ), develop=False, ) sieve = LimitLatestVersionsSieve(graph=None, project=None) sieve.update_parameters(dict([("limit_latest_versions", 3)])) sieve_context = SieveContext.from_package_versions( [tf_1_1_0, tf_1_9_0, tf_2_0_0]) sieve.run(sieve_context) assert list(sieve_context.iter_direct_dependencies()) == [ tf_1_1_0, tf_1_9_0, tf_2_0_0 ]