def run(self):
output_requires = {}
output_required_by = {}
known_packages = os.listdir("./datastore")
for idx, package in enumerate(known_packages):
print "working on [%s:%s]: %s" % (idx, len(known_packages),
package)
try:
requirements = _clean_requirements(
open("./datastore/%s/required" %
package).read().split("\n"))
except IOError:
continue
output_requires[package] = requirements
for r in requirements:
if r in output_required_by:
output_required_by[r].append(package)
else:
output_required_by[r] = [package]
counter = 0
for package, req_by in output_required_by.iteritems():
counter = counter + 1
print "writing [%s:%s]: %s" % (
counter, len(output_required_by.keys()), package)
out = open('./output/%s.json' % package, 'w')
try:
out_req = output_requires[package]
except KeyError:
out_req = []
json_out = {
'generated_at': datetime.datetime.now(),
'package': package,
'requires': out_req,
'required_by': req_by
}
out_content = simplejson.dumps(json_out, default=date_handler)
out.write(out_content)
out.close()
#generate a file with the most popular packages
popfile = open('./output/_popular.json', 'w')
popular_output = sorted(output_required_by.items(),
key=lambda k: len(k[1]),
reverse=True)
popfile.write(json.dumps(popular_output[:25]))
popfile.close()
package_list = open('./output/_packagelist.json', 'w')
package_list.write(json.dumps(known_packages))
package_list.close()
def test_garabage(self):
reqs = [
'(%s %s % (d', 'v and v or )).strip()', 'for d',
'v in requires.items()', ''
]
self.assertEqual(_clean_requirements(reqs), [])
reqs = ['line for line in open(requirements.txt)', '']
self.assertEqual(_clean_requirements(reqs), [])
reqs = [
'', 'gocept.httpserverlayer >= 2.0.dev0', 'httpagentparser',
'plone.testing', 'selenium !=2.53.0', '!=2.53.1', '!=2.53.2,<3.0',
'Pillow', 'setuptools', ''
]
clean = [
'gocept.httpserverlayer', 'httpagentparser', 'plone.testing',
'selenium', 'Pillow', 'setuptools', '\\'
]
self.assertEqual(_clean_requirements(reqs), clean)
reqs = ["<3.0.0", '-))', '==dev', 'a.strip()', '/']
self.assertEqual(_clean_requirements(reqs), [])
def test_trim_text(self):
reqs = [' zope.app.intid']
self.assertEqual(_clean_requirements(reqs), ['zope.app.intid'])
def test_strip_comments(self):
reqs = ['#hello world']
self.assertEqual(_clean_requirements(reqs), [])
def test_no_blanks(self):
reqs = ['SQLAlchemy>0.4', '']
self.assertEqual(_clean_requirements(reqs), ['SQLAlchemy'])
def test_remove_reqs_string_part(self):
reqs = ['SQLAlchemy>0.4']
self.assertEqual(_clean_requirements(reqs), ['SQLAlchemy'])