def get_missing_packages(requirements_file: str = None,
special_reqs: List[str] = []) -> List[Requirement]:
needed = [Requirement.parse_line(sr) for sr in special_reqs]
if requirements_file is not None:
with open(requirements_file, 'r') as fd:
needed += [r for r in requirements.parse(fd) if r.specifier]
installed = get_installed_packages()
needed = [
r for r in needed if pkg_resources.safe_name(r.name) not in installed
]
return needed
def sort_requirements(fn=None):
"""
Prints to stdout the current pip-requirements.txt sorted by dependency.
"""
ignore_packages = set(['setuptools'])
package_names_dep = set()
package_names_req = set()
package_name_to_version = {}
package_name_to_original = {}
fn = fn or 'roles/all/pip-requirements.txt'
i = 0
for line in open(fn).readlines():
i += 1
try:
line = line.strip()
parent = Requirement.parse_line(line)
print(parent.specs, parent.__dict__)
package_name, package_version = line.split('==')
if package_name in ignore_packages:
continue
package_name_to_original[package_name.lower()] = package_name
package_names_req.add(package_name.lower())
package_name_to_version[package_name.lower()] = package_version
except Exception as e:
print('Error on line %i.' % i, file=sys.stderr)
raise
package_to_deps = defaultdict(set) # {package:set(dependencies)}
depends_fn = get_dependencies_fn()
reader = csv.DictReader(open(depends_fn))
for line in reader:
print(line, file=sys.stderr)
package_name = line['package_name'].lower()
if package_name in ignore_packages:
continue
dependency_name = line['dependency_name'].lower()
if dependency_name in ignore_packages:
continue
package_names_dep.add(package_name)
package_names_dep.add(dependency_name)
package_to_deps[package_name].add(dependency_name)
reqs_missing_deps = set(map(str.lower, package_names_req)).difference(
set(map(str.lower, package_names_dep)))
print('reqs_missing_deps:', reqs_missing_deps, file=sys.stderr)
deps_missing_reqs = set(map(str.lower, package_names_dep)).difference(
set(map(str.lower, package_names_req)))
print('deps_missing_reqs:', deps_missing_reqs, file=sys.stderr)
# def sort_by_dep(a_name, b_name):
# if a_name in package_to_deps[b_name]:
# # b depends on a, so a should come first
# return -1
# elif b_name in package_to_deps[a_name]:
# # a depends on b, so a should come first
# return +1
# #else:
# # return cmp(a_name, b_name)
# return 0
for package_name in package_names_req:
package_to_deps[package_name]
all_names = common.topological_sort(package_to_deps)
for name in all_names:
print('%s==%s' %
(package_name_to_original[name], package_name_to_version[name]))
def update_dependency_cache(name=None, output=None):
"""
Reads all pip package dependencies and saves them to a file for later use with organizing
pip-requirements.txt.
Outputs CSV to stdout.
"""
common.set_show(0)
try:
shutil.rmtree('./.env/build')
except OSError:
pass
env.pip_path_versioned = env.pip_path % env
#depends_fn = get_dependencies_fn(output)
fout = open(output, 'w')
#fout = sys.stdout
writer = csv.DictWriter(fout, PIP_DEPENDS_HEADERS)
writer.writerow(dict(zip(PIP_DEPENDS_HEADERS, PIP_DEPENDS_HEADERS)))
package_to_fqv = {}
for dep in pip_to_deps():
#print dep
assert dep.name not in package_to_fqv, 'Package %s specified multiple times!' % dep.name
package_to_fqv[dep.name] = str(dep)
#dep_tree = defaultdict(set) # {package:set([deps])}
reqs = list(iter_pip_requirements())
total = len(reqs)
i = 0
for line in reqs:
i += 1
if name and name not in line:
continue
print('line %s: %i %i %.02f%%' %
(line, i, total, i / float(total) * 100),
file=sys.stderr)
env.pip_package = line
env.pip_download_dir = tempfile.mkdtemp()
cmd = env.pip_depend_command % env
#with hide('output', 'running', 'warnings'):
ret = local_or_dryrun(cmd, capture=True)
print('ret:', ret)
matches = PIP_DEP_PATTERN.findall(ret) # [(child,parent)]
print('matches:', matches)
for child, parent in matches:
try:
child_line = child.strip()
#print 'child_line:',child_line
#child = Requirement(child_line)
child_name = PIP_REQ_NAME_PATTERN.findall(child_line)[0]
child_specs = PIP_REQ_SPEC_PATTERN.findall(child_line)
#print 'child:',child_name,child_specs
parent = Requirement.parse_line(parent.strip().split('->')[0])
#print 'parent:',parent.__dict__
# print('parent.specs:',parent.specs,bool(parent.specs)
assert not parent.specs \
or (parent.specs and parent.specs[0][0] in ('==', '>=', '<=', '!=', '<', '>')), \
'Invalid parent: %s (%s)' % (parent, parent.specs)
# if parent.specs and parent.specs[0][0] == '==':
# parent.specs[0] = list(parent.specs[0])
# parent.specs[0][0] = '>='
parent_version = ''
if parent.specs:
parent_version = parent.specs[0][1]
writer.writerow(
dict(
package_name=parent.name,
package_version=parent_version,
dependency_name=child_name,
dependency_specs=';'.join(
[''.join(_) for _ in child_specs]),
))
fout.flush()
except Exception as e:
print('Error: %s' % e, file=sys.stderr)
print(e, file=sys.stderr)
traceback.print_exc(file=sys.stderr)
raise