def find_from_requirements(path):
reqs = find_requirements(path)
names = []
for requirement in reqs:
if requirement.name is not None and requirement.name.lower() in POSSIBLE_LIBRARIES:
names.append(requirement.name.lower())
return names
def find_from_requirements(path):
reqs = find_requirements(path)
names = []
for requirement in reqs:
if (requirement.name is not None
and requirement.name.lower() in POSSIBLE_LIBRARIES):
names.append(requirement.name.lower())
return names
def find_from_requirements(path):
reqs = find_requirements(path)
names = []
for requirement in reqs:
if requirement.name is not None \
and requirement.name.lower() in LIBRARY_ADAPTORS:
names.append(requirement.name.lower())
return names
def find_from_requirements(path):
reqs = find_requirements(path)
names = []
for requirement in reqs:
if requirement.name is not None \
and requirement.name.lower() in LIBRARY_ADAPTORS:
names.append(requirement.name.lower())
return names
def infer_requirements(path):
import requirements_detector
try:
return [
str(requirement)
for requirement in requirements_detector.find_requirements(path)
]
except requirements_detector.detect.RequirementsNotFound:
return []
def get_reqs(path):
"""Extract project requirements."""
_reqs = [(x.name, x.version_specs) for x in find_requirements(path)]
reqs = []
for req in _reqs:
if req[1]:
reqs.append('{}=={}'.format(req[0], req[1][0][1]))
else:
reqs.append('{}'.format(req[0]))
return reqs
def infer_requirements(path: Path) -> Dict[str, str]:
"""Given the directory of a Python project, try to find its
requirements.
"""
import requirements_detector
try:
return {
"requirements":
json.dumps([
str(requirement) for requirement in
requirements_detector.find_requirements(path)
])
}
except requirements_detector.detect.RequirementsNotFound:
return {"requirements": "[]"}
def _get_requirements(path: str) -> list:
"""
Get list of requirements from Python project.
Files supported are setup.py and requierements.txt
:param path: Project path
"""
_reqs = ((x.name, x.version_specs) for x in find_requirements(path))
reqs = []
for req in _reqs:
if req[1]:
reqs.append((req[0], req[1][0][1]))
else:
reqs.append((req[0], None))
return reqs
def run():
if len(sys.argv) > 1:
path = sys.argv[1]
else:
path = os.getcwd()
if not os.path.exists(path):
_die("%s does not exist" % path)
if not os.path.isdir(path):
_die("%s is not a directory" % path)
try:
requirements = find_requirements(path)
except RequirementsNotFound:
_die("Unable to find requirements at %s" % path)
format_name = 'requirements_file' # TODO: other output formats such as JSON
FORMATTERS[format_name](requirements)
sys.exit(0)
def try_setup_analysis():
err_file = 'failed_setup.txt'
if os.path.exists(err_file):
with open(err_file) as f:
failed = set(f.read().split('\n'))
else:
failed = set()
template = r"https://raw.githubusercontent.com/%s/%s/master/setup.py"
with open(err_file, 'a') as f:
for index, package in enumerate(need):
package_tup = full_dict[package]
if package in failed:
continue
setup_path = os.path.join('setups', 'setup.py')
url = template % (package, package)
output_list = []
if index % 100 == 0:
f.flush()
print('committing')
con.commit()
try:
req = requests.get(url, timeout=2)
if req.status_code != 200:
f.write(package)
f.write('\n')
continue
with open(setup_path, 'wb') as output:
print('found setup.py for %s' % package)
output.write(req.content)
try_ours = False
#Run the analysis tool here
try:
tool_output = find_requirements('setups')
except Exception:
print('Could not use req parser. Trying ours')
try_ours = True
tool_output = False
if tool_output:
mini_set = set()
for requ_obj in tool_output:
requirement = requ_obj.requirement.project_name
specifiers = ','.join(''.join(x) for x in requ_obj.requirement.specs)
if (requirement, specifiers) in mini_set:
continue
mini_set.add ( (requirement, specifiers) )
tup = (package, package_tup[1], requirement, specifiers)
print(tup)
output_list.append(tup)
write_dependency(package_tup, output_list)
elif try_ours is True:
try:
deps = getDep.parse_setup(setup_path)
except Exception as e:
print(e)
print('failed %s' % package)
continue
if deps:
dep_list = get_dependencies(package, version, deps)
dep_set(dep_list) #Makes sure each tuple is unique!
write_dependency(package_tup, dep_set)
elif deps is None:
print('failed %s' % package)
else:
print("we think %s is a dependency less package" % package)
write_dependency(package_tup, None)
print('Got something for %s' % package)
except requests.exceptions.Timeout:
print('%s timed out' % package)
except requests.exceptions.ConnectionError:
print('%s got a connenction error. Exiting' % package)
raise
con.commit()
con.close()
