def process(package_name):
if fileinfo.exists(package_name):
return
env_name = '%senv' % (package_name, )
cmds = [
'virtualenv envs/%s' % env_name,
'. envs/%s/bin/activate' % env_name, 'pip install wheel',
'pip install numpy', 'pip install Cython',
'pip install --no-deps %s' % package_name,
'python extractdatapip.py %s' % package_name, 'deactivate'
]
cmd = ' && '.join(cmds)
subprocess.call(cmd, shell=True)
subprocess.call('rm -r envs/%s' % (env_name, ), shell=True)
def process(package_name):
if fileinfo.exists(package_name):
return
env_name = "%senv" % (package_name,)
cmds = [
"virtualenv envs/%s" % env_name,
". envs/%s/bin/activate" % env_name,
"pip install wheel",
"pip install numpy",
"pip install Cython",
"pip install --no-deps %s" % package_name,
"python extractdatapip.py %s" % package_name,
"deactivate",
]
cmd = " && ".join(cmds)
subprocess.call(cmd, shell=True)
subprocess.call("rm -r envs/%s" % (env_name,), shell=True)
from redis import Redis
from rq import Queue
import fileinfo
from extractdatapypi import get_package_data
ONE_MONTH = 60 * 60 * 24 * 30
if __name__ == '__main__':
client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
filename = 'data/all_packages.bin'
if os.path.exists(filename):
with open(filename) as f:
packages = marshal.load(f)
else:
packages = client.list_packages()
with open(filename, 'wb') as f:
marshal.dump(packages, f)
packages = [p for p in packages if not fileinfo.exists(p, fileinfo.PYPI_DATA_DIR)]
random.shuffle(packages)
q = Queue(connection=Redis())
results = [q.enqueue(get_package_data, args=[job], timeout=ONE_MONTH) for job in packages]
print "%s tasks queued" % (len(results), )
from extractdatapypi import get_package_data
ONE_MONTH = 60 * 60 * 24 * 30
if __name__ == '__main__':
client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
filename = 'data/all_packages.bin'
if os.path.exists(filename):
with open(filename) as f:
packages = marshal.load(f)
else:
packages = client.list_packages()
with open(filename, 'wb') as f:
marshal.dump(packages, f)
packages = [
p for p in packages if not fileinfo.exists(p, fileinfo.PYPI_DATA_DIR)
]
random.shuffle(packages)
q = Queue(connection=Redis())
results = [
q.enqueue(get_package_data, args=[job], timeout=ONE_MONTH)
for job in packages
]
print "%s tasks queued" % (len(results), )
from redis import Redis
from rq import Queue
import fileinfo
from piprun import process
ONE_MONTH = 60 * 60 * 24 * 30
if __name__ == '__main__':
client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
filename = 'data/all_packages.bin'
if os.path.exists(filename):
with open(filename) as f:
packages = marshal.load(f)
else:
packages = client.list_packages()
with open(filename, 'wb') as f:
marshal.dump(packages, f)
packages = [p for p in packages if not fileinfo.exists(p)]
random.shuffle(packages)
q = Queue(connection=Redis())
results = [q.enqueue(process, args=[job], timeout=ONE_MONTH) for job in packages]
print "%s tasks queued" % (len(results), )
from rq import Queue
import fileinfo
from piprun import process
ONE_MONTH = 60 * 60 * 24 * 30
if __name__ == '__main__':
client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
filename = 'data/all_packages.bin'
if os.path.exists(filename):
with open(filename) as f:
packages = marshal.load(f)
else:
packages = client.list_packages()
with open(filename, 'wb') as f:
marshal.dump(packages, f)
packages = [p for p in packages if not fileinfo.exists(p)]
random.shuffle(packages)
q = Queue(connection=Redis())
results = [
q.enqueue(process, args=[job], timeout=ONE_MONTH) for job in packages
]
print "%s tasks queued" % (len(results), )
