Example #1
0
def main():

    deps = testdata.DEPS_MODERATE
    versions_by_package = depdata.generate_dict_versions_by_package(deps)

    (edeps, packs_wout_avail_version_info, dists_w_missing_dependencies) = depdata.elaborate_dependencies(
        deps, versions_by_package
    )

    assert depdata.are_deps_valid(testdata.DEPS_MODERATE) and depdata.are_deps_valid(
        testdata.DEPS_SIMPLE
    ), "The test dependencies are coming up as invalid for some reason...."

    # Clear any pre-existing test database.
    sqli.initialize(db_fname="data/test_dependencies.db")
    sqli.delete_all_tables()

    sqli.populate_sql_with_full_dependency_info(
        edeps,
        versions_by_package,
        packs_wout_avail_version_info,
        dists_w_missing_dependencies,
        db_fname="data/test_dependencies.db",
    )

    print("All tests in main() OK.")
Example #2
0
def test_depdata():
    """
  """
    assert 41 == len(testdata.DEPS_MODERATE), (
        "Set changed: should be len 41 but is len " + str(len(testdata.DEPS_MODERATE)) + " - reconfigure tests"
    )

    json.dump(testdata.DEPS_MODERATE, open("data/test_deps_set.json", "w"))

    deps = depdata.load_json_db("data/test_deps_set.json")

    assert testdata.DEPS_MODERATE == deps, "JSON write and load via load_json_db is breaking!"

    versions_by_package = depdata.generate_dict_versions_by_package(deps)

    assert 10 == len(versions_by_package)  # different package names

    total_package_versions = sum([len(versions_by_package[p]) for p in versions_by_package])

    assert 41 == total_package_versions, (
        "Wrong number of versions: " + str(total_package_versions) + "instead" + "of 41."
    )

    (edeps, packs_wout_avail_version_info, dists_w_missing_dependencies) = depdata.elaborate_dependencies(
        deps, versions_by_package
    )

    # 1 entry in the edeps dict
    assert 41 == len(edeps), (
        "Wrong number of dists in elaborate_dependencies output. " + str(len(edeps)) + "instead of 41."
    )

    # 1 version listed for every possible satisfying dependency
    n_dependencies_elaborated = 0
    for distkey in edeps:
        for satisfying_package_entry in edeps[distkey]:
            list_of_satisfying_versions = satisfying_package_entry[1]
            n_dependencies_elaborated += len(list_of_satisfying_versions)
            # print(distkey + " -> " + str(list_of_satisfying_versions))

    assert 34 == n_dependencies_elaborated, (
        "Expecting 34 satisfying versions (1 for every [depending dist]"
        + ",[satisfying_version] pair. Instead, got "
        + str(n_dependencies_elaborated)
    )

    assert depdata.are_deps_valid(testdata.DEPS_MODERATE) and depdata.are_deps_valid(
        testdata.DEPS_SIMPLE
    ), "The test dependencies are coming up as invalid for some reason...."

    print("test_depdata(): All tests OK.")
Example #3
0
def test_depdata():
    """
  """
    assert 41 == len(testdata.DEPS_MODERATE), \
        "Set changed: should be len 41 but is len " + \
        str(len(testdata.DEPS_MODERATE)) + " - reconfigure tests"

    json.dump(testdata.DEPS_MODERATE, open('data/test_deps_set.json', 'w'))

    deps = depdata.load_json_db('data/test_deps_set.json')

    assert testdata.DEPS_MODERATE == deps, \
        "JSON write and load via load_json_db is breaking!"

    versions_by_package = depdata.generate_dict_versions_by_package(deps)

    assert 10 == len(versions_by_package)  # different package names

    total_package_versions = \
        sum([len(versions_by_package[p]) for p in versions_by_package])

    assert 41 == total_package_versions, \
        "Wrong number of versions: " + str(total_package_versions) + "instead" \
        + "of 41."

    (edeps, packs_wout_avail_version_info, dists_w_missing_dependencies) = \
        depdata.elaborate_dependencies(deps, versions_by_package)

    # 1 entry in the edeps dict
    assert 41 == len(edeps), \
        "Wrong number of dists in elaborate_dependencies output. " + \
        str(len(edeps)) + "instead of 41."

    # 1 version listed for every possible satisfying dependency
    n_dependencies_elaborated = 0
    for distkey in edeps:
        for satisfying_package_entry in edeps[distkey]:
            list_of_satisfying_versions = satisfying_package_entry[1]
            n_dependencies_elaborated += len(list_of_satisfying_versions)
            #print(distkey + " -> " + str(list_of_satisfying_versions))

    assert 34 == n_dependencies_elaborated, \
        "Expecting 34 satisfying versions (1 for every [depending dist]" + \
        ",[satisfying_version] pair. Instead, got " + \
        str(n_dependencies_elaborated)

    assert depdata.are_deps_valid(testdata.DEPS_MODERATE) and \
        depdata.are_deps_valid(testdata.DEPS_SIMPLE), \
        'The test dependencies are coming up as invalid for some reason....'

    print("test_depdata(): All tests OK.")
Example #4
0
def test_detect_model_2_conflicts():
    """TEST 3: Detection of model 2 conflicts."""
    deps = testdata.DEPS_MODEL2
    versions_by_package = depdata.generate_dict_versions_by_package(deps)
    (edeps, packs_wout_avail_version_info, dists_w_missing_dependencies) = \
        depdata.elaborate_dependencies(deps, versions_by_package)

    success = ry.detect_model_2_conflict_from_distkey('motorengine(0.7.4)',
                                                      edeps,
                                                      versions_by_package)

    if not success:
        logger.error(
            'Did not detect model 2 conflict for motorengine(0.7.4). ):')
    else:
        logger.info("test_resolver(): Test 3 OK.")

    return success
Example #5
0
def backtracking_satisfy_alpha(distkey_to_satisfy,
                               edeps=None,
                               edeps_alpha=None,
                               edeps_rev=None,
                               versions_by_package=None):
    """
  Small workaround.
  See https://github.com/awwad/depresolve/issues/12
  """
    if edeps is None or edeps_alpha is None or edeps_rev is None:
        depdata.ensure_data_loaded(include_edeps=True, include_sorts=True)
        edeps = depdata.elaborated_dependencies
        edeps_alpha = depdata.elaborated_alpha
        edeps_rev = depdata.elaborated_reverse
        versions_by_package = depdata.versions_by_package

    elif versions_by_package is None:
        versions_by_package = depdata.generate_dict_versions_by_package(edeps)

    satisfy_output = None
    # Try three different ways until one works or all fail.
    for edeps_trying in [edeps_rev, edeps_alpha]:
        try:
            satisfy_output = backtracking_satisfy(distkey_to_satisfy,
                                                  edeps_trying,
                                                  versions_by_package)

        except depresolve.UnresolvableConflictError:
            pass

        else:
            assert satisfy_output, 'Programming error. Should not be empty.'
            break

    if satisfy_output is None:
        satisfy_output = backtracking_satisfy(distkey_to_satisfy, edeps,
                                              versions_by_package)

    return satisfy_output
Example #6
0
def main():

    deps = testdata.DEPS_MODERATE
    versions_by_package = depdata.generate_dict_versions_by_package(deps)

    (edeps, packs_wout_avail_version_info, dists_w_missing_dependencies) = \
        depdata.elaborate_dependencies(deps, versions_by_package)

    assert depdata.are_deps_valid(testdata.DEPS_MODERATE) and \
        depdata.are_deps_valid(testdata.DEPS_SIMPLE), \
        'The test dependencies are coming up as invalid for some reason....'

    # Clear any pre-existing test database.
    sqli.initialize(db_fname='data/test_dependencies.db')
    sqli.delete_all_tables()

    sqli.populate_sql_with_full_dependency_info(
        edeps,
        versions_by_package,
        packs_wout_avail_version_info,
        dists_w_missing_dependencies,
        db_fname='data/test_dependencies.db')

    print('All tests in main() OK.')
Example #7
0
def backtracking_satisfy(distkey_to_satisfy,
                         edeps=None,
                         versions_by_package=None):
    """
  Provide a list of distributions to install that will fully satisfy a given
  distribution's dependencies (and its dependencies' dependencies, and so on),
  without any conflicting or incompatible versions.

  This is a backtracking dependency resolution algorithm.
  
  This recursion is extremely inefficient, and would profit from dynamic
  programming in general.

  Note that there must be a level of indirection for the timeout decorator to
  work as it is currently written. (This function can't call itself directly
  recursively, but must instead call _backtracking_satisfy, which then can
  recurse.)


  Arguments:
    - distkey_to_satisfy ('django(1.8.3)'),
    - edeps (dictionary returned by depdata.deps_elaborated; see there.)
    - versions_by_package (dictionary of all distkeys, keyed by package name)
      (If not included, it will be generated from edeps.)

  Returns:
    - list of distkeys needed as direct or indirect dependencies to install
      distkey_to_satisfy, including distkey_to_satisfy

  Throws:
    - timeout.TimeoutException if the process takes longer than 5 minutes
    - depresolve.UnresolvableConflictError if not able to generate a solution
      that satisfies all dependencies of the given package (and their
      dependencies, etc.). This suggests that there is an unresolvable
      conflict.
    - depresolve.ConflictingVersionError
      (Should not raise, ideally, but might - requires more testing)
    - depresolve.NoSatisfyingVersionError
      (Should not raise, ideally, but might - requires more testing)

  """
    if edeps is None:
        depdata.ensure_data_loaded(include_edeps=True)
        edeps = depdata.elaborated_dependencies
        versions_by_package = depdata.versions_by_package

    elif versions_by_package is None:
        versions_by_package = depdata.generate_dict_versions_by_package(edeps)

    try:
        (satisfying_candidate_set, new_conflicts, child_dotgraph) = \
            _backtracking_satisfy(distkey_to_satisfy, edeps, versions_by_package)

    except depresolve.ConflictingVersionError as e:
        # Compromise traceback style so as not to give up python2 compatibility.
        six.reraise(
            depresolve.UnresolvableConflictError,
            depresolve.UnresolvableConflictError(
                'Unable to find solution'
                ' to a conflict with one of ' + distkey_to_satisfy +
                "'s immediate "
                'dependencies.'),
            sys.exc_info()[2])

        # Python 3 style (by far the nicest):
        #raise depresolve.UnresolvableConflictError('Unable to find solution to '
        #    'a conflict with one of ' + distkey_to_satisfy + "'s immediate "
        #    'dependencies.') from e

        # Original (2 or 3 compatible but not great on either, especially not 2)
        #raise depresolve.UnresolvableConflictError('Unable to find solution to '
        #     'conflict with one of ' + distkey_to_satisfy + "'s immediate "
        #     'dependencies.' Lower level conflict exception follows: ' + str(e))

    else:
        return satisfying_candidate_set
Example #8
0
def naive_satisfy(depender_distkey,
                  edeps,
                  versions_by_package=None,
                  _preexisting_candidates=[],
                  _preexisting_candidate_packs=[]):
    """
  Vaguely pip-like "simple dependency resolution". Recurse and list all dists
  that together form a simple resolution to a given distribution's dependencies
  (may have dependency conflicts and not be a true resolution).

  Where there is ambiguity, select the first result from sort_versions().
  If multiple dists depend on the same package, we get both in this result.

  This has the same level of capability as pip's dependency resolution, though
  the results are slightly different.

  Arguments:
    - depender_distkey ('django(1.8.3)'),
    - edeps (dictionary returned by depdata.deps_elaborated; see there.)
    - versions_by_package (dictionary of all distkeys, keyed by package name)

  Returns:
    - list of distkeys needed as direct or indirect dependencies to install
      depender_distkey
  """
    # Avoid circular dependencies and ignore conflicts.
    satisfying_candidate_set = _preexisting_candidates[:]
    satisfying_candidate_packs = _preexisting_candidate_packs[:]
    if depender_distkey in satisfying_candidate_set or \
        depdata.get_packname(depender_distkey) in satisfying_candidate_packs:
        return []
    else:
        satisfying_candidate_set.append(depender_distkey)
        satisfying_candidate_packs.append(
            depdata.get_packname(depender_distkey))

    if versions_by_package is None:
        versions_by_package = depdata.generate_dict_versions_by_package(edeps)

    depdata.assume_dep_data_exists_for(depender_distkey, edeps)

    my_edeps = edeps[depender_distkey]
    if not my_edeps:  # if no dependencies, return nothing new
        return satisfying_candidate_set

    for edep in my_edeps:
        satisfying_packname = edep[0]
        satisfying_versions = edep[1]
        if satisfying_packname in satisfying_candidate_packs:
            # Avoid circular dependencies and ignore conflicts.
            continue
        if not satisfying_versions:
            raise depresolve.NoSatisfyingVersionError(
                "Dependency of " + depender_distkey + " on " +
                satisfying_packname + " with specstring " + edep[2] +
                " cannot be satisfied: no versions found in elaboration "
                "attempt.")
        chosen_version = sort_versions(satisfying_versions)[0]  # grab first
        chosen_distkey = \
            depdata.distkey_format(satisfying_packname, chosen_version)
        #satisfying_candidate_set.append(chosen_distkey)
        #satisfying_candidate_packs.append(satisfying_packname)

        # Now recurse.
        satisfying_candidate_set = naive_satisfy(chosen_distkey, edeps,
                                                 versions_by_package,
                                                 satisfying_candidate_set,
                                                 satisfying_candidate_packs)

    return satisfying_candidate_set
Example #9
0
def are_fully_satisfied(candidates,
                        edeps=None,
                        versions_by_package=None,
                        disregard_setuptools=False,
                        report_issue=False):
    """
  Validates the results of a resolver solution.
  Given a set of distkeys, determines whether or not all dependencies of all
  given dists are satisfied by the set (and all dependencies of their
  dependencies, etc.).
  Returns True if that is so, else returns False.

  Note that this depends on the provided dependency information in edeps.
  If those dependencies were harvested on a system that's different from the
  one that generated the given candidates (e.g. if even the python versions
  used are different), there's a chance the dependencies won't actually match
  since, as we know, PyPI dependencies are not static..............

  Arguments:
    1. candidates: a list of distkeys indicating which dists have been selected
           to satisfy each others' dependencies.
    2. edeps: elaborated dependencies (see depresolve/depdata.py) If not
           provided, this will be loaded from the data directory using
           depdata.ensure_data_loaded.
    3. versions_by_package (as generated by
           depresolve.depdata.generate_dict_versions_by_package(); a dict of
           all versions for each package name)). If not included, this will be
           generated from the given (or loaded) edeps.
    4. disregard_setuptools: optional. I dislike this hack. Because for the
           rbtcollins resolver, I'm testing solutions generated by pip installs
           and harvested by pip freeze, I'm not going to get setuptools listed
           in the solution set (pip freeze doesn't list it), so ... for that, I
           pass in disregard_setuptools=True.
    5. report_issue: optional. If True, additionally returns a string
           describing the (first) unsatisfied dependency.

  Returns:
    - True or False
    - (ONLY if report_issue is True), A string description of unsatisfied
      dependencies.

  Throws:
    - depresolve.MissingDependencyInfoError:
        if the dependencies data lacks info for one of the candidate dists.
        e.g. if solution employs a version not in the outdated dependency data

  """
    # Lowercase the distkeys for our all-lowercase data, just in case.
    candidates = [distkey.lower() for distkey in candidates]

    # Load the dependency library if one wasn't provided.
    if edeps is None:
        depdata.ensure_data_loaded(include_edeps=True)
        edeps = depdata.elaborated_dependencies
        versions_by_package = depdata.versions_by_package

    elif versions_by_package is None:
        versions_by_package = depdata.generate_dict_versions_by_package(edeps)

    satisfied = True
    problem = ''

    for distkey in candidates:

        depdata.assume_dep_data_exists_for(distkey, edeps)

        for edep in edeps[distkey]:
            this_dep_satisfied = is_dep_satisfied(
                edep, candidates, disregard_setuptools=disregard_setuptools
            )  # do not use report_issue

            if not this_dep_satisfied:
                satisfied = False
                this_problem = distkey + ' dependency ' + edep[0] + str(edep[2]) + \
                    ' is not satisfied by candidate set: ' + str(candidates) + \
                    '. Acceptable versions were: ' + str(edep[1])
                logger.info(this_problem)
                if problem:
                    problem += '. '
                problem += this_problem

    if report_issue:
        return satisfied, problem
    else:
        return True
Example #10
0
def test_resolver(resolver_func,
                  expected_result,
                  distkey,
                  deps,
                  versions_by_package=None,
                  edeps=None,
                  expected_exception=None,
                  use_raw_deps=False):
    """
  Returns True if the given resolver function returns the expected result on
  the given data, else False. More modes described in args notes below.

  Solutions are compared with intelligent version parsing outsourced partly to
  pip's pip._vendor.packaging.version classes. For example, 2.0 and 2 and 2.0.0
  are all treated as equivalent.


  Arguments:

    resolver_func
      Argument resolver_func should be a function that accepts 3 arguments and
      returns a solution list:
        - Arg 1:  distkey to generate an install set for, whose installation
                  results in a fully satisfied set of dependencies
        - Arg 2:  dependency data (either deps or edeps, per depdata.py)
        - Arg 3:  versions_by_package, a dict mapping package names to all
                  versions of that package available
        - Return: a list containing the distkeys for dists to install

    expected_result
      This should be a list of distkeys. If the list given matches the solution
      generated by calling resolver_func with the appropriate arguments, we
      return True.
      If this is None, then we don't care what solution is returned by the call
      to resolver_func, only that no unexpected exceptions were raised and
      any expected exception was raised.

    distkey
      The distkey of the distribution to solve for (find install set that
      fully satisfies).
    
    deps
      Dependency data to be used in resolution.


  Optional Arguments:
    
    versions_by_package
      As described elsewhere, the dictionary mapping package names to available
      versions of those packages. If not provided, this is generated from deps.

    use_raw_deps
      If True, we do not try to elaborate dependencies (or use provided
      elaborated dependencies), instead passing the deps provided on in our
      call to resolver_func. Some resolvers do not use elaborated dependencies.

    edeps
      If provided, we don't elaborate the deps argument, but instead use these.

    expected_exception
      The type() of an exception that we expect to receive. If provided, we
      disregard expected_result and instead expect to catch an exception of the
      indicated type, returning True if we catch one and False otherwise.


  Raises:

    - UnresolvableConflictError (reraise) if unable to resolve and we were not
      told to expect UnresolvableConflictError. (Same goes for any other
      exceptions generated by call to resolver_func.)

  Side Effects:
    (NO:
     Used to also write dependency graph to resolver/output/test_resolver_* in
     graphviz .dot format, but have turned this off for now.)

  """

    if versions_by_package is None:
        versions_by_package = depdata.generate_dict_versions_by_package(deps)

    if use_raw_deps:
        edeps = deps

    elif edeps is None:
        (edeps, packs_wout_avail_version_info, dists_w_missing_dependencies) = \
          depdata.elaborate_dependencies(deps, versions_by_package)

    solution = None

    try:
        #(solution, _junk_, dotstrings) = \
        solution = \
            resolver_func(distkey, edeps, versions_by_package)

    except Exception as e:
        if expected_exception is None or type(e) is not expected_exception:
            logger.exception('Test Failure: Unexpectedly unable to resolve ' +
                             distkey)

            # Compromise traceback style so as not to give up python2 compatibility.
            six.reraise(
                UnexpectedException,
                UnexpectedException('Unexpectedly '
                                    'unable to resolve ' + distkey),
                sys.exc_info()[2])

            # Python 3 style (by far the nicest):
            #raise UnexpectedException('Unexpectedly unable to resolve ' + distkey) \
            #    from e

            # Original (2 or 3 compatible but not great on either, especially not 2)
            #raise

            #return False

        else:
            # We expected this error.
            logger.info('As expected, unable to resolve ' + distkey +
                        ' due to ' + str(type(e)) + '.')
            logger.info('  Exception caught: ' + e.args[0])
            return True

    else:
        logger.info('Resolved ' + distkey + '. Solution: ' + str(solution))
        #if dotstrings is not None: # If the func provides dotstrings
        #  fobj = open('data/resolver/test_resolver_' + resolver_func.__name__ +
        #      '_' + distkey + '.dot', 'w')
        #  fobj.write('digraph G {\n' + dotstrings + '}\n')
        #  fobj.close()

        # Were we expecting an exception? (We didn't get one if we're here.)
        if expected_exception is not None:
            logger.info('Expecting exception (' + str(expected_exception) +
                        ') but '
                        'none were raised. Was solving for ' + distkey +
                        ' using ' + resolver_func.__name__)
            return False

        # If expected_result is None, then we didn't care what the result was as
        # long as there was no unexpected exception / as long as whatever exception
        # is expected was raised.
        elif expected_result is None:
            logger.info(
                'No particular solution expected and resolver call did not '
                'raise an exception, therefore result is acceptable. Was solving '
                'for ' + distkey + ' using ' + resolver_func.__name__)
            return True

        # Is the solution set as expected?
        elif ry.dist_lists_are_equal(solution, expected_result):
            logger.info('Solution is as expected.')
            return True

        else:
            logger.info('Solution does not match while solving for ' +
                        distkey + ' using ' + resolver_func.__name__ + ':')
            logger.info('    Expected: ' + str(sorted(expected_result)))
            logger.info('    Produced: ' + str(sorted(solution)))
            return False