def main(): deps = testdata.DEPS_MODERATE versions_by_package = depdata.generate_dict_versions_by_package(deps) (edeps, packs_wout_avail_version_info, dists_w_missing_dependencies) = depdata.elaborate_dependencies( deps, versions_by_package ) assert depdata.are_deps_valid(testdata.DEPS_MODERATE) and depdata.are_deps_valid( testdata.DEPS_SIMPLE ), "The test dependencies are coming up as invalid for some reason...." # Clear any pre-existing test database. sqli.initialize(db_fname="data/test_dependencies.db") sqli.delete_all_tables() sqli.populate_sql_with_full_dependency_info( edeps, versions_by_package, packs_wout_avail_version_info, dists_w_missing_dependencies, db_fname="data/test_dependencies.db", ) print("All tests in main() OK.")
def test_depdata(): """ """ assert 41 == len(testdata.DEPS_MODERATE), ( "Set changed: should be len 41 but is len " + str(len(testdata.DEPS_MODERATE)) + " - reconfigure tests" ) json.dump(testdata.DEPS_MODERATE, open("data/test_deps_set.json", "w")) deps = depdata.load_json_db("data/test_deps_set.json") assert testdata.DEPS_MODERATE == deps, "JSON write and load via load_json_db is breaking!" versions_by_package = depdata.generate_dict_versions_by_package(deps) assert 10 == len(versions_by_package) # different package names total_package_versions = sum([len(versions_by_package[p]) for p in versions_by_package]) assert 41 == total_package_versions, ( "Wrong number of versions: " + str(total_package_versions) + "instead" + "of 41." ) (edeps, packs_wout_avail_version_info, dists_w_missing_dependencies) = depdata.elaborate_dependencies( deps, versions_by_package ) # 1 entry in the edeps dict assert 41 == len(edeps), ( "Wrong number of dists in elaborate_dependencies output. " + str(len(edeps)) + "instead of 41." ) # 1 version listed for every possible satisfying dependency n_dependencies_elaborated = 0 for distkey in edeps: for satisfying_package_entry in edeps[distkey]: list_of_satisfying_versions = satisfying_package_entry[1] n_dependencies_elaborated += len(list_of_satisfying_versions) # print(distkey + " -> " + str(list_of_satisfying_versions)) assert 34 == n_dependencies_elaborated, ( "Expecting 34 satisfying versions (1 for every [depending dist]" + ",[satisfying_version] pair. Instead, got " + str(n_dependencies_elaborated) ) assert depdata.are_deps_valid(testdata.DEPS_MODERATE) and depdata.are_deps_valid( testdata.DEPS_SIMPLE ), "The test dependencies are coming up as invalid for some reason...." print("test_depdata(): All tests OK.")
def test_depdata(): """ """ assert 41 == len(testdata.DEPS_MODERATE), \ "Set changed: should be len 41 but is len " + \ str(len(testdata.DEPS_MODERATE)) + " - reconfigure tests" json.dump(testdata.DEPS_MODERATE, open('data/test_deps_set.json', 'w')) deps = depdata.load_json_db('data/test_deps_set.json') assert testdata.DEPS_MODERATE == deps, \ "JSON write and load via load_json_db is breaking!" versions_by_package = depdata.generate_dict_versions_by_package(deps) assert 10 == len(versions_by_package) # different package names total_package_versions = \ sum([len(versions_by_package[p]) for p in versions_by_package]) assert 41 == total_package_versions, \ "Wrong number of versions: " + str(total_package_versions) + "instead" \ + "of 41." (edeps, packs_wout_avail_version_info, dists_w_missing_dependencies) = \ depdata.elaborate_dependencies(deps, versions_by_package) # 1 entry in the edeps dict assert 41 == len(edeps), \ "Wrong number of dists in elaborate_dependencies output. " + \ str(len(edeps)) + "instead of 41." # 1 version listed for every possible satisfying dependency n_dependencies_elaborated = 0 for distkey in edeps: for satisfying_package_entry in edeps[distkey]: list_of_satisfying_versions = satisfying_package_entry[1] n_dependencies_elaborated += len(list_of_satisfying_versions) #print(distkey + " -> " + str(list_of_satisfying_versions)) assert 34 == n_dependencies_elaborated, \ "Expecting 34 satisfying versions (1 for every [depending dist]" + \ ",[satisfying_version] pair. Instead, got " + \ str(n_dependencies_elaborated) assert depdata.are_deps_valid(testdata.DEPS_MODERATE) and \ depdata.are_deps_valid(testdata.DEPS_SIMPLE), \ 'The test dependencies are coming up as invalid for some reason....' print("test_depdata(): All tests OK.")
def test_detect_model_2_conflicts(): """TEST 3: Detection of model 2 conflicts.""" deps = testdata.DEPS_MODEL2 versions_by_package = depdata.generate_dict_versions_by_package(deps) (edeps, packs_wout_avail_version_info, dists_w_missing_dependencies) = \ depdata.elaborate_dependencies(deps, versions_by_package) success = ry.detect_model_2_conflict_from_distkey('motorengine(0.7.4)', edeps, versions_by_package) if not success: logger.error( 'Did not detect model 2 conflict for motorengine(0.7.4). ):') else: logger.info("test_resolver(): Test 3 OK.") return success
def backtracking_satisfy_alpha(distkey_to_satisfy, edeps=None, edeps_alpha=None, edeps_rev=None, versions_by_package=None): """ Small workaround. See https://github.com/awwad/depresolve/issues/12 """ if edeps is None or edeps_alpha is None or edeps_rev is None: depdata.ensure_data_loaded(include_edeps=True, include_sorts=True) edeps = depdata.elaborated_dependencies edeps_alpha = depdata.elaborated_alpha edeps_rev = depdata.elaborated_reverse versions_by_package = depdata.versions_by_package elif versions_by_package is None: versions_by_package = depdata.generate_dict_versions_by_package(edeps) satisfy_output = None # Try three different ways until one works or all fail. for edeps_trying in [edeps_rev, edeps_alpha]: try: satisfy_output = backtracking_satisfy(distkey_to_satisfy, edeps_trying, versions_by_package) except depresolve.UnresolvableConflictError: pass else: assert satisfy_output, 'Programming error. Should not be empty.' break if satisfy_output is None: satisfy_output = backtracking_satisfy(distkey_to_satisfy, edeps, versions_by_package) return satisfy_output
def main(): deps = testdata.DEPS_MODERATE versions_by_package = depdata.generate_dict_versions_by_package(deps) (edeps, packs_wout_avail_version_info, dists_w_missing_dependencies) = \ depdata.elaborate_dependencies(deps, versions_by_package) assert depdata.are_deps_valid(testdata.DEPS_MODERATE) and \ depdata.are_deps_valid(testdata.DEPS_SIMPLE), \ 'The test dependencies are coming up as invalid for some reason....' # Clear any pre-existing test database. sqli.initialize(db_fname='data/test_dependencies.db') sqli.delete_all_tables() sqli.populate_sql_with_full_dependency_info( edeps, versions_by_package, packs_wout_avail_version_info, dists_w_missing_dependencies, db_fname='data/test_dependencies.db') print('All tests in main() OK.')
def backtracking_satisfy(distkey_to_satisfy, edeps=None, versions_by_package=None): """ Provide a list of distributions to install that will fully satisfy a given distribution's dependencies (and its dependencies' dependencies, and so on), without any conflicting or incompatible versions. This is a backtracking dependency resolution algorithm. This recursion is extremely inefficient, and would profit from dynamic programming in general. Note that there must be a level of indirection for the timeout decorator to work as it is currently written. (This function can't call itself directly recursively, but must instead call _backtracking_satisfy, which then can recurse.) Arguments: - distkey_to_satisfy ('django(1.8.3)'), - edeps (dictionary returned by depdata.deps_elaborated; see there.) - versions_by_package (dictionary of all distkeys, keyed by package name) (If not included, it will be generated from edeps.) Returns: - list of distkeys needed as direct or indirect dependencies to install distkey_to_satisfy, including distkey_to_satisfy Throws: - timeout.TimeoutException if the process takes longer than 5 minutes - depresolve.UnresolvableConflictError if not able to generate a solution that satisfies all dependencies of the given package (and their dependencies, etc.). This suggests that there is an unresolvable conflict. - depresolve.ConflictingVersionError (Should not raise, ideally, but might - requires more testing) - depresolve.NoSatisfyingVersionError (Should not raise, ideally, but might - requires more testing) """ if edeps is None: depdata.ensure_data_loaded(include_edeps=True) edeps = depdata.elaborated_dependencies versions_by_package = depdata.versions_by_package elif versions_by_package is None: versions_by_package = depdata.generate_dict_versions_by_package(edeps) try: (satisfying_candidate_set, new_conflicts, child_dotgraph) = \ _backtracking_satisfy(distkey_to_satisfy, edeps, versions_by_package) except depresolve.ConflictingVersionError as e: # Compromise traceback style so as not to give up python2 compatibility. six.reraise( depresolve.UnresolvableConflictError, depresolve.UnresolvableConflictError( 'Unable to find solution' ' to a conflict with one of ' + distkey_to_satisfy + "'s immediate " 'dependencies.'), sys.exc_info()[2]) # Python 3 style (by far the nicest): #raise depresolve.UnresolvableConflictError('Unable to find solution to ' # 'a conflict with one of ' + distkey_to_satisfy + "'s immediate " # 'dependencies.') from e # Original (2 or 3 compatible but not great on either, especially not 2) #raise depresolve.UnresolvableConflictError('Unable to find solution to ' # 'conflict with one of ' + distkey_to_satisfy + "'s immediate " # 'dependencies.' Lower level conflict exception follows: ' + str(e)) else: return satisfying_candidate_set
def naive_satisfy(depender_distkey, edeps, versions_by_package=None, _preexisting_candidates=[], _preexisting_candidate_packs=[]): """ Vaguely pip-like "simple dependency resolution". Recurse and list all dists that together form a simple resolution to a given distribution's dependencies (may have dependency conflicts and not be a true resolution). Where there is ambiguity, select the first result from sort_versions(). If multiple dists depend on the same package, we get both in this result. This has the same level of capability as pip's dependency resolution, though the results are slightly different. Arguments: - depender_distkey ('django(1.8.3)'), - edeps (dictionary returned by depdata.deps_elaborated; see there.) - versions_by_package (dictionary of all distkeys, keyed by package name) Returns: - list of distkeys needed as direct or indirect dependencies to install depender_distkey """ # Avoid circular dependencies and ignore conflicts. satisfying_candidate_set = _preexisting_candidates[:] satisfying_candidate_packs = _preexisting_candidate_packs[:] if depender_distkey in satisfying_candidate_set or \ depdata.get_packname(depender_distkey) in satisfying_candidate_packs: return [] else: satisfying_candidate_set.append(depender_distkey) satisfying_candidate_packs.append( depdata.get_packname(depender_distkey)) if versions_by_package is None: versions_by_package = depdata.generate_dict_versions_by_package(edeps) depdata.assume_dep_data_exists_for(depender_distkey, edeps) my_edeps = edeps[depender_distkey] if not my_edeps: # if no dependencies, return nothing new return satisfying_candidate_set for edep in my_edeps: satisfying_packname = edep[0] satisfying_versions = edep[1] if satisfying_packname in satisfying_candidate_packs: # Avoid circular dependencies and ignore conflicts. continue if not satisfying_versions: raise depresolve.NoSatisfyingVersionError( "Dependency of " + depender_distkey + " on " + satisfying_packname + " with specstring " + edep[2] + " cannot be satisfied: no versions found in elaboration " "attempt.") chosen_version = sort_versions(satisfying_versions)[0] # grab first chosen_distkey = \ depdata.distkey_format(satisfying_packname, chosen_version) #satisfying_candidate_set.append(chosen_distkey) #satisfying_candidate_packs.append(satisfying_packname) # Now recurse. satisfying_candidate_set = naive_satisfy(chosen_distkey, edeps, versions_by_package, satisfying_candidate_set, satisfying_candidate_packs) return satisfying_candidate_set
def are_fully_satisfied(candidates, edeps=None, versions_by_package=None, disregard_setuptools=False, report_issue=False): """ Validates the results of a resolver solution. Given a set of distkeys, determines whether or not all dependencies of all given dists are satisfied by the set (and all dependencies of their dependencies, etc.). Returns True if that is so, else returns False. Note that this depends on the provided dependency information in edeps. If those dependencies were harvested on a system that's different from the one that generated the given candidates (e.g. if even the python versions used are different), there's a chance the dependencies won't actually match since, as we know, PyPI dependencies are not static.............. Arguments: 1. candidates: a list of distkeys indicating which dists have been selected to satisfy each others' dependencies. 2. edeps: elaborated dependencies (see depresolve/depdata.py) If not provided, this will be loaded from the data directory using depdata.ensure_data_loaded. 3. versions_by_package (as generated by depresolve.depdata.generate_dict_versions_by_package(); a dict of all versions for each package name)). If not included, this will be generated from the given (or loaded) edeps. 4. disregard_setuptools: optional. I dislike this hack. Because for the rbtcollins resolver, I'm testing solutions generated by pip installs and harvested by pip freeze, I'm not going to get setuptools listed in the solution set (pip freeze doesn't list it), so ... for that, I pass in disregard_setuptools=True. 5. report_issue: optional. If True, additionally returns a string describing the (first) unsatisfied dependency. Returns: - True or False - (ONLY if report_issue is True), A string description of unsatisfied dependencies. Throws: - depresolve.MissingDependencyInfoError: if the dependencies data lacks info for one of the candidate dists. e.g. if solution employs a version not in the outdated dependency data """ # Lowercase the distkeys for our all-lowercase data, just in case. candidates = [distkey.lower() for distkey in candidates] # Load the dependency library if one wasn't provided. if edeps is None: depdata.ensure_data_loaded(include_edeps=True) edeps = depdata.elaborated_dependencies versions_by_package = depdata.versions_by_package elif versions_by_package is None: versions_by_package = depdata.generate_dict_versions_by_package(edeps) satisfied = True problem = '' for distkey in candidates: depdata.assume_dep_data_exists_for(distkey, edeps) for edep in edeps[distkey]: this_dep_satisfied = is_dep_satisfied( edep, candidates, disregard_setuptools=disregard_setuptools ) # do not use report_issue if not this_dep_satisfied: satisfied = False this_problem = distkey + ' dependency ' + edep[0] + str(edep[2]) + \ ' is not satisfied by candidate set: ' + str(candidates) + \ '. Acceptable versions were: ' + str(edep[1]) logger.info(this_problem) if problem: problem += '. ' problem += this_problem if report_issue: return satisfied, problem else: return True
def test_resolver(resolver_func, expected_result, distkey, deps, versions_by_package=None, edeps=None, expected_exception=None, use_raw_deps=False): """ Returns True if the given resolver function returns the expected result on the given data, else False. More modes described in args notes below. Solutions are compared with intelligent version parsing outsourced partly to pip's pip._vendor.packaging.version classes. For example, 2.0 and 2 and 2.0.0 are all treated as equivalent. Arguments: resolver_func Argument resolver_func should be a function that accepts 3 arguments and returns a solution list: - Arg 1: distkey to generate an install set for, whose installation results in a fully satisfied set of dependencies - Arg 2: dependency data (either deps or edeps, per depdata.py) - Arg 3: versions_by_package, a dict mapping package names to all versions of that package available - Return: a list containing the distkeys for dists to install expected_result This should be a list of distkeys. If the list given matches the solution generated by calling resolver_func with the appropriate arguments, we return True. If this is None, then we don't care what solution is returned by the call to resolver_func, only that no unexpected exceptions were raised and any expected exception was raised. distkey The distkey of the distribution to solve for (find install set that fully satisfies). deps Dependency data to be used in resolution. Optional Arguments: versions_by_package As described elsewhere, the dictionary mapping package names to available versions of those packages. If not provided, this is generated from deps. use_raw_deps If True, we do not try to elaborate dependencies (or use provided elaborated dependencies), instead passing the deps provided on in our call to resolver_func. Some resolvers do not use elaborated dependencies. edeps If provided, we don't elaborate the deps argument, but instead use these. expected_exception The type() of an exception that we expect to receive. If provided, we disregard expected_result and instead expect to catch an exception of the indicated type, returning True if we catch one and False otherwise. Raises: - UnresolvableConflictError (reraise) if unable to resolve and we were not told to expect UnresolvableConflictError. (Same goes for any other exceptions generated by call to resolver_func.) Side Effects: (NO: Used to also write dependency graph to resolver/output/test_resolver_* in graphviz .dot format, but have turned this off for now.) """ if versions_by_package is None: versions_by_package = depdata.generate_dict_versions_by_package(deps) if use_raw_deps: edeps = deps elif edeps is None: (edeps, packs_wout_avail_version_info, dists_w_missing_dependencies) = \ depdata.elaborate_dependencies(deps, versions_by_package) solution = None try: #(solution, _junk_, dotstrings) = \ solution = \ resolver_func(distkey, edeps, versions_by_package) except Exception as e: if expected_exception is None or type(e) is not expected_exception: logger.exception('Test Failure: Unexpectedly unable to resolve ' + distkey) # Compromise traceback style so as not to give up python2 compatibility. six.reraise( UnexpectedException, UnexpectedException('Unexpectedly ' 'unable to resolve ' + distkey), sys.exc_info()[2]) # Python 3 style (by far the nicest): #raise UnexpectedException('Unexpectedly unable to resolve ' + distkey) \ # from e # Original (2 or 3 compatible but not great on either, especially not 2) #raise #return False else: # We expected this error. logger.info('As expected, unable to resolve ' + distkey + ' due to ' + str(type(e)) + '.') logger.info(' Exception caught: ' + e.args[0]) return True else: logger.info('Resolved ' + distkey + '. Solution: ' + str(solution)) #if dotstrings is not None: # If the func provides dotstrings # fobj = open('data/resolver/test_resolver_' + resolver_func.__name__ + # '_' + distkey + '.dot', 'w') # fobj.write('digraph G {\n' + dotstrings + '}\n') # fobj.close() # Were we expecting an exception? (We didn't get one if we're here.) if expected_exception is not None: logger.info('Expecting exception (' + str(expected_exception) + ') but ' 'none were raised. Was solving for ' + distkey + ' using ' + resolver_func.__name__) return False # If expected_result is None, then we didn't care what the result was as # long as there was no unexpected exception / as long as whatever exception # is expected was raised. elif expected_result is None: logger.info( 'No particular solution expected and resolver call did not ' 'raise an exception, therefore result is acceptable. Was solving ' 'for ' + distkey + ' using ' + resolver_func.__name__) return True # Is the solution set as expected? elif ry.dist_lists_are_equal(solution, expected_result): logger.info('Solution is as expected.') return True else: logger.info('Solution does not match while solving for ' + distkey + ' using ' + resolver_func.__name__ + ':') logger.info(' Expected: ' + str(sorted(expected_result))) logger.info(' Produced: ' + str(sorted(solution))) return False