def get_missing_packages(requirements_file: str = None, special_reqs: List[str] = []) -> List[Requirement]: needed = [Requirement.parse_line(sr) for sr in special_reqs] if requirements_file is not None: with open(requirements_file, 'r') as fd: needed += [r for r in requirements.parse(fd) if r.specifier] installed = get_installed_packages() needed = [ r for r in needed if pkg_resources.safe_name(r.name) not in installed ] return needed
def sort_requirements(fn=None): """ Prints to stdout the current pip-requirements.txt sorted by dependency. """ ignore_packages = set(['setuptools']) package_names_dep = set() package_names_req = set() package_name_to_version = {} package_name_to_original = {} fn = fn or 'roles/all/pip-requirements.txt' i = 0 for line in open(fn).readlines(): i += 1 try: line = line.strip() parent = Requirement.parse_line(line) print(parent.specs, parent.__dict__) package_name, package_version = line.split('==') if package_name in ignore_packages: continue package_name_to_original[package_name.lower()] = package_name package_names_req.add(package_name.lower()) package_name_to_version[package_name.lower()] = package_version except Exception as e: print('Error on line %i.' % i, file=sys.stderr) raise package_to_deps = defaultdict(set) # {package:set(dependencies)} depends_fn = get_dependencies_fn() reader = csv.DictReader(open(depends_fn)) for line in reader: print(line, file=sys.stderr) package_name = line['package_name'].lower() if package_name in ignore_packages: continue dependency_name = line['dependency_name'].lower() if dependency_name in ignore_packages: continue package_names_dep.add(package_name) package_names_dep.add(dependency_name) package_to_deps[package_name].add(dependency_name) reqs_missing_deps = set(map(str.lower, package_names_req)).difference( set(map(str.lower, package_names_dep))) print('reqs_missing_deps:', reqs_missing_deps, file=sys.stderr) deps_missing_reqs = set(map(str.lower, package_names_dep)).difference( set(map(str.lower, package_names_req))) print('deps_missing_reqs:', deps_missing_reqs, file=sys.stderr) # def sort_by_dep(a_name, b_name): # if a_name in package_to_deps[b_name]: # # b depends on a, so a should come first # return -1 # elif b_name in package_to_deps[a_name]: # # a depends on b, so a should come first # return +1 # #else: # # return cmp(a_name, b_name) # return 0 for package_name in package_names_req: package_to_deps[package_name] all_names = common.topological_sort(package_to_deps) for name in all_names: print('%s==%s' % (package_name_to_original[name], package_name_to_version[name]))
def update_dependency_cache(name=None, output=None): """ Reads all pip package dependencies and saves them to a file for later use with organizing pip-requirements.txt. Outputs CSV to stdout. """ common.set_show(0) try: shutil.rmtree('./.env/build') except OSError: pass env.pip_path_versioned = env.pip_path % env #depends_fn = get_dependencies_fn(output) fout = open(output, 'w') #fout = sys.stdout writer = csv.DictWriter(fout, PIP_DEPENDS_HEADERS) writer.writerow(dict(zip(PIP_DEPENDS_HEADERS, PIP_DEPENDS_HEADERS))) package_to_fqv = {} for dep in pip_to_deps(): #print dep assert dep.name not in package_to_fqv, 'Package %s specified multiple times!' % dep.name package_to_fqv[dep.name] = str(dep) #dep_tree = defaultdict(set) # {package:set([deps])} reqs = list(iter_pip_requirements()) total = len(reqs) i = 0 for line in reqs: i += 1 if name and name not in line: continue print('line %s: %i %i %.02f%%' % (line, i, total, i / float(total) * 100), file=sys.stderr) env.pip_package = line env.pip_download_dir = tempfile.mkdtemp() cmd = env.pip_depend_command % env #with hide('output', 'running', 'warnings'): ret = local_or_dryrun(cmd, capture=True) print('ret:', ret) matches = PIP_DEP_PATTERN.findall(ret) # [(child,parent)] print('matches:', matches) for child, parent in matches: try: child_line = child.strip() #print 'child_line:',child_line #child = Requirement(child_line) child_name = PIP_REQ_NAME_PATTERN.findall(child_line)[0] child_specs = PIP_REQ_SPEC_PATTERN.findall(child_line) #print 'child:',child_name,child_specs parent = Requirement.parse_line(parent.strip().split('->')[0]) #print 'parent:',parent.__dict__ # print('parent.specs:',parent.specs,bool(parent.specs) assert not parent.specs \ or (parent.specs and parent.specs[0][0] in ('==', '>=', '<=', '!=', '<', '>')), \ 'Invalid parent: %s (%s)' % (parent, parent.specs) # if parent.specs and parent.specs[0][0] == '==': # parent.specs[0] = list(parent.specs[0]) # parent.specs[0][0] = '>=' parent_version = '' if parent.specs: parent_version = parent.specs[0][1] writer.writerow( dict( package_name=parent.name, package_version=parent_version, dependency_name=child_name, dependency_specs=';'.join( [''.join(_) for _ in child_specs]), )) fout.flush() except Exception as e: print('Error: %s' % e, file=sys.stderr) print(e, file=sys.stderr) traceback.print_exc(file=sys.stderr) raise