def process(package_name): if fileinfo.exists(package_name): return env_name = '%senv' % (package_name, ) cmds = [ 'virtualenv envs/%s' % env_name, '. envs/%s/bin/activate' % env_name, 'pip install wheel', 'pip install numpy', 'pip install Cython', 'pip install --no-deps %s' % package_name, 'python extractdatapip.py %s' % package_name, 'deactivate' ] cmd = ' && '.join(cmds) subprocess.call(cmd, shell=True) subprocess.call('rm -r envs/%s' % (env_name, ), shell=True)
def process(package_name): if fileinfo.exists(package_name): return env_name = "%senv" % (package_name,) cmds = [ "virtualenv envs/%s" % env_name, ". envs/%s/bin/activate" % env_name, "pip install wheel", "pip install numpy", "pip install Cython", "pip install --no-deps %s" % package_name, "python extractdatapip.py %s" % package_name, "deactivate", ] cmd = " && ".join(cmds) subprocess.call(cmd, shell=True) subprocess.call("rm -r envs/%s" % (env_name,), shell=True)
from redis import Redis from rq import Queue import fileinfo from extractdatapypi import get_package_data ONE_MONTH = 60 * 60 * 24 * 30 if __name__ == '__main__': client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi') filename = 'data/all_packages.bin' if os.path.exists(filename): with open(filename) as f: packages = marshal.load(f) else: packages = client.list_packages() with open(filename, 'wb') as f: marshal.dump(packages, f) packages = [p for p in packages if not fileinfo.exists(p, fileinfo.PYPI_DATA_DIR)] random.shuffle(packages) q = Queue(connection=Redis()) results = [q.enqueue(get_package_data, args=[job], timeout=ONE_MONTH) for job in packages] print "%s tasks queued" % (len(results), )
from extractdatapypi import get_package_data ONE_MONTH = 60 * 60 * 24 * 30 if __name__ == '__main__': client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi') filename = 'data/all_packages.bin' if os.path.exists(filename): with open(filename) as f: packages = marshal.load(f) else: packages = client.list_packages() with open(filename, 'wb') as f: marshal.dump(packages, f) packages = [ p for p in packages if not fileinfo.exists(p, fileinfo.PYPI_DATA_DIR) ] random.shuffle(packages) q = Queue(connection=Redis()) results = [ q.enqueue(get_package_data, args=[job], timeout=ONE_MONTH) for job in packages ] print "%s tasks queued" % (len(results), )
from redis import Redis from rq import Queue import fileinfo from piprun import process ONE_MONTH = 60 * 60 * 24 * 30 if __name__ == '__main__': client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi') filename = 'data/all_packages.bin' if os.path.exists(filename): with open(filename) as f: packages = marshal.load(f) else: packages = client.list_packages() with open(filename, 'wb') as f: marshal.dump(packages, f) packages = [p for p in packages if not fileinfo.exists(p)] random.shuffle(packages) q = Queue(connection=Redis()) results = [q.enqueue(process, args=[job], timeout=ONE_MONTH) for job in packages] print "%s tasks queued" % (len(results), )
from rq import Queue import fileinfo from piprun import process ONE_MONTH = 60 * 60 * 24 * 30 if __name__ == '__main__': client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi') filename = 'data/all_packages.bin' if os.path.exists(filename): with open(filename) as f: packages = marshal.load(f) else: packages = client.list_packages() with open(filename, 'wb') as f: marshal.dump(packages, f) packages = [p for p in packages if not fileinfo.exists(p)] random.shuffle(packages) q = Queue(connection=Redis()) results = [ q.enqueue(process, args=[job], timeout=ONE_MONTH) for job in packages ] print "%s tasks queued" % (len(results), )