def execute_wrapper(setuppath):
errlist = []
pbs = glob.glob('/usr/bin/python?.?')
if os.path.isfile(setuppath):
pkgpath = os.path.dirname(setuppath)
if os.path.isdir(setuppath):
pkgpath = setuppath
storepath = os.path.join(pkgpath, 'setupargs-pypicontents.json')
for cmd in [(pb, '-m', 'pypicontents.wrapper', setuppath) for pb in pbs]:
try:
with timeout(error='Execution of setup.py took too much time.'):
p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
stdout = u(stdout).strip('\n').strip()
stderr = u(stderr).strip('\n').strip()
if p.poll() is None:
p.kill()
if os.path.isfile(storepath):
with open(storepath) as store:
return json.loads(store.read()), ''
except Exception:
errlist.append(traceback.format_exc())
else:
errlist.append('Execution of {0} failed with the following '
'messages:'.format(' '.join(cmd)))
if stdout:
errlist.append('[stdout] {0}'.format(stdout))
if stderr:
errlist.append('[stderr] {0}'.format(stderr))
if not stdout and not stderr:
errlist.append('Unknown reason.')
return {}, '\n'.join(errlist)
def execute_wrapper(setuppath):
errlist = []
pbs = glob.glob('/usr/bin/python?.?')
if os.path.isfile(setuppath):
pkgpath = os.path.dirname(setuppath)
if os.path.isdir(setuppath):
pkgpath = setuppath
storepath = os.path.join(pkgpath, 'setupargs-pypicontents.json')
for cmd in [(pb, '-m', 'pypicontents.wrapper', setuppath) for pb in pbs]:
try:
with timeout(error='Execution of setup.py took too much time.'):
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
stdout = u(stdout).strip('\n').strip()
stderr = u(stderr).strip('\n').strip()
if p.poll() is None:
p.kill()
if os.path.isfile(storepath):
with open(storepath) as store:
return json.loads(store.read()), ''
except Exception:
errlist.append(traceback.format_exc())
else:
errlist.append('Execution of {0} failed with the following '
'messages:'.format(' '.join(cmd)))
if stdout:
errlist.append('[stdout] {0}'.format(stdout))
if stderr:
errlist.append('[stderr] {0}'.format(stderr))
if not stdout and not stderr:
errlist.append('Unknown reason.')
return {}, '\n'.join(errlist)
def stdlib(**kwargs):
jsondict = {}
outputfile = os.path.abspath(kwargs.get('outputfile'))
pyver = '%s.%s' % (sys.version_info[0], sys.version_info[1])
jsondir = os.path.dirname(outputfile)
if not os.path.isdir(jsondir):
os.makedirs(jsondir)
inv = 'http://docs.python.org/{0}/objects.inv'.format(pyver)
uri = 'http://docs.python.org/{0}'.format(pyver)
inventory = fetch_inventory(uri, inv) or {'py:module': {}}
inventorymodules = list(inventory.get('py:module').keys())
libpackages = get_packages(libdir)
libmodules = get_modules(libpackages)
for f in list_files(libdir, '*.py'):
libmodules.append(os.path.splitext(os.path.basename(f))[0])
modules = sorted(list(set(inventorymodules + libmodules)))
jsondict.update({pyver: [u(m) for m in modules]})
with open(outputfile, 'w') as j:
j.write(json.dumps(jsondict, separators=(',', ': '),
sort_keys=True, indent=4))
def read_inventory_v2(f, uri, bufsize=16 * 1024):
# type: (IO, unicode, Callable, int) -> Inventory
invdata = {} # type: Inventory
line = f.readline()
projname = line.rstrip()[11:].decode('utf-8')
line = f.readline()
version = line.rstrip()[11:].decode('utf-8')
line = f.readline().decode('utf-8')
if 'zlib' not in line:
raise ValueError('This is not a gzipped file.')
def read_chunks():
# type: () -> Iterator[bytes]
decompressor = zlib.decompressobj()
for chunk in iter(lambda: f.read(bufsize), b''):
yield decompressor.decompress(chunk)
yield decompressor.flush()
def split_lines(iter):
# type: (Iterator[bytes]) -> Iterator[unicode]
buf = b''
for chunk in iter:
buf += chunk
lineend = buf.find(b'\n')
while lineend != -1:
yield buf[:lineend].decode('utf-8')
buf = buf[lineend + 1:]
lineend = buf.find(b'\n')
assert not buf
for line in split_lines(read_chunks()):
# be careful to handle names with embedded spaces correctly
m = re.match(r'(?x)(.+?)\s+(\S*:\S*)\s+(-?\d+)\s+(\S+)\s+(.*)',
line.rstrip())
if not m:
continue
name, type, prio, location, dispname = m.groups()
if type == 'py:module' and type in invdata and \
name in invdata[type]: # due to a bug in 1.1 and below,
# two inventory entries are created
# for Python modules, and the first
# one is correct
continue
if location.endswith(u('$')):
location = location[:-1] + name
location = os.path.join(uri, location)
invdata.setdefault(type, {})[name] = (projname, version,
location, dispname)
return invdata
def false_setup(*args, **kwargs):
cmdline = []
setupdir = os.path.dirname(globals['__file__'])
storepath = os.path.join(setupdir, 'setupargs-pypicontents.json')
banned_options = [
'setup_requires', 'test_requires', 'conda_buildnum', 'd2to1',
'distclass', 'email', 'entry_points', 'executables', 'home_page',
'include_package_data', 'install_requires', 'licesne',
'namespace_packages', 'pbr', 'platform', 'use_2to3',
'use_scm_version'
]
from distutils.dist import Distribution
from distutils.command.build_py import build_py
from pkg_resources import EntryPoint
for opt in banned_options:
kwargs.pop(opt, None)
kwargs.update({'script_name': globals['__file__'], 'script_args': []})
bpy = build_py(Distribution(kwargs))
bpy.finalize_options()
_bpy_mods = bpy.find_all_modules()
_join_mods = ['.'.join([p, m]).strip('.') for p, m, f in _bpy_mods]
modules = [
'.'.join(m.split('.')[:-1])
if m.endswith('.__init__') or m.endswith('.__main__') else m
for m in _join_mods
]
if 'scripts' in kwargs:
cmdline.extend([os.path.basename(s) for s in kwargs['scripts']])
if 'entry_points' in kwargs:
entrymap = EntryPoint.parse_map(kwargs['entry_points'])
if 'console_scripts' in entrymap:
cmdline.extend(entrymap['console_scripts'].keys())
with open(storepath, 'w') as store:
store.write(
u(
json.dumps({
'modules': sorted(set(modules)),
'cmdline': sorted(set(cmdline))
})))
def false_setup(*args, **kwargs):
cmdline = []
setupdir = os.path.dirname(globals['__file__'])
storepath = os.path.join(setupdir, 'setupargs-pypicontents.json')
banned_options = ['setup_requires', 'test_requires', 'conda_buildnum',
'd2to1', 'distclass', 'email', 'entry_points',
'executables', 'home_page', 'include_package_data',
'install_requires', 'licesne', 'namespace_packages',
'pbr', 'platform', 'use_2to3', 'use_scm_version']
from distutils.dist import Distribution
from distutils.command.build_py import build_py
from pkg_resources import EntryPoint
for opt in banned_options:
kwargs.pop(opt, None)
kwargs.update({'script_name': globals['__file__'],
'script_args': []})
bpy = build_py(Distribution(kwargs))
bpy.finalize_options()
_bpy_mods = bpy.find_all_modules()
_join_mods = ['.'.join([p, m]).strip('.') for p, m, f in _bpy_mods]
modules = ['.'.join(m.split('.')[:-1])
if m.endswith('.__init__') or m.endswith('.__main__')
else m for m in _join_mods]
if 'scripts' in kwargs:
cmdline.extend([os.path.basename(s) for s in kwargs['scripts']])
if 'entry_points' in kwargs:
entrymap = EntryPoint.parse_map(kwargs['entry_points'])
if 'console_scripts' in entrymap:
cmdline.extend(entrymap['console_scripts'].keys())
with open(storepath, 'w') as store:
store.write(u(json.dumps({'modules': sorted(set(modules)),
'cmdline': sorted(set(cmdline))})))
def pypi(**kwargs):
jsondict = {}
sum_updated = 0
sum_uptodate = 0
sum_nodata = 0
sum_nodown = 0
sum_noapi = 0
sum_nourls = 0
limit_mem_available = 600 * 1024 * 1024
allowed_range = list(string.ascii_lowercase) + list(map(str, range(0, 10)))
start_time = time.time()
logfile = kwargs.get('logfile')
if logfile:
logfile = os.path.abspath(logfile)
outputfile = os.path.abspath(kwargs.get('outputfile'))
extractdir = os.path.abspath(kwargs.get('extractdir'))
cachedir = os.path.abspath(kwargs.get('cachedir'))
limit_mem = kwargs.get('limit_mem')
limit_log_size = kwargs.get('limit_log_size')
letter_range = translate_letter_range(kwargs.get('letter_range'))
limit_time = int(kwargs.get('limit_time'))
clean = kwargs.get('clean')
if limit_mem.isdigit():
limit_mem = '{0}B'.format(limit_mem)
if limit_log_size.isdigit():
limit_log_size = '{0}B'.format(limit_log_size)
limit_mem = human2bytes(limit_mem)
limit_log_size = human2bytes(limit_log_size)
if len(set(letter_range) - set(allowed_range)):
raise RuntimeError('"{0}" not in allowed range.'.format(
', '.join(set(letter_range) - set(allowed_range))))
if not os.path.isdir(extractdir):
os.makedirs(extractdir)
if not os.path.isdir(cachedir):
os.makedirs(cachedir)
if not os.path.isfile(outputfile):
create_file_if_notfound(outputfile)
logger.info('Downloading package list from PyPI ...')
pkglist = filter_package_list(get_pkglist(), letter_range)
if not pkglist:
raise RuntimeError('Couldnt download the PyPI package list.\n'
'There was an error stablishing communication '
'with {0}'.format(pypiurl))
outjsondict = get_outputfile_jsondict(outputfile)
jsondict = prefill_jsondict(pkglist, jsondict, outjsondict)
for pkgname in sorted(jsondict.keys()):
elapsed_time = int(time.time() - start_time)
mem_usage = int(getrusage(RUSAGE_SELF).ru_maxrss * 1024)
mem_available = int(get_free_memory())
for chpid in get_children_processes(os.getpid()):
os.kill(int(chpid), signal.SIGKILL)
if logfile:
logsize = int(os.path.getsize(logfile))
else:
logsize = 0
logger.configpkg(pkgname)
if elapsed_time > limit_time:
logger.configpkg()
logger.warning('')
logger.warning('Processing has taken more than {0} seconds.'
' Interrupting.'.format(limit_time))
logger.warning('Processing will continue in next iteration.')
logger.warning('')
break
if mem_usage > limit_mem:
logger.configpkg()
logger.warning('')
logger.warning('This process is taking more than {0} MB of memory.'
' Interrupting'.format(limit_mem / (1024 * 1024)))
logger.warning('Processing will continue in next iteration.')
logger.warning('')
break
if mem_available < limit_mem_available:
logger.configpkg()
logger.warning('')
logger.warning('This machine is running out of memory.'
' Interrupting.')
logger.warning('Processing will continue in next iteration.')
logger.warning('')
break
if logsize > limit_log_size:
logger.configpkg()
logger.warning('')
logger.warning(
'The log is taking more than {0} MB.'
' Interrupting.'.format(logsize / (1024 * 1024)))
logger.warning('Processing will continue in next iteration.')
logger.warning('')
break
pkgdata, errstring = get_pkgdata(pkgname)
if not pkgdata:
logger.error('Could not get a response from API for this package. '
'See below for details.\n{0}'.format(errstring))
sum_noapi += 1
continue
newversion = pkgdata['info']['version']
currentversion = jsondict[pkgname]['version']
if newversion == currentversion:
logger.info('This package is up to date.')
sum_uptodate += 1
continue
pkgurls, errstring = get_pkgurls(pkgdata, pkgname, newversion)
if not pkgurls:
logger.error('Could not get download URLs for this package. '
'See below for details.\n{0}'.format(errstring))
sum_nourls += 1
continue
if not (pkgurls['sdist'] + pkgurls['bdist_wheel'] +
pkgurls['bdist_egg']):
logger.error('This package does not have downloadable releases.')
sum_nodown += 1
continue
setupargs, pkgpath, tarpath, errstring = get_setupargs(
pkgname, newversion, pkgurls, cachedir,
os.path.join(extractdir, pkgname))
if clean:
if os.path.isfile(tarpath):
os.remove(tarpath)
if os.path.isdir(pkgpath):
shutil.rmtree(pkgpath)
if not setupargs:
logger.error('Could not extract data from this package. '
'See below for details.\n{0}'.format(errstring))
sum_nodata += 1
continue
logger.info('Data has been updated for this package.')
jsondict[pkgname]['version'] = pkgdata['info']['version']
jsondict[pkgname].update(setupargs)
sum_updated += 1
with open(outputfile, 'w') as f:
f.write(u(json.dumps(jsondict, separators=(',', ': '),
sort_keys=True, indent=4)))
sum_proc = (sum_updated + sum_uptodate + sum_noapi + sum_nourls +
sum_nodown + sum_nodata)
sum_not_proc = len(jsondict.keys()) - sum_proc
logger.configpkg('')
logger.info('')
logger.info('')
logger.info('Total packages: {0}'.format(len(jsondict.keys())))
logger.info(' Packages processed: {0}'.format(sum_proc))
logger.info(' Packages updated: {0}'.format(sum_updated))
logger.info(' Packages up-to-date: {0}'.format(sum_uptodate))
logger.info(' Packages without response: {0}'.format(sum_noapi))
logger.info(' Packages without urls: {0}'.format(sum_nourls))
logger.info(' Packages without downloads: {0}'.format(sum_nodown))
logger.info(' Packages with data errors: {0}'.format(sum_nodata))
logger.info(' Packages not processed: {0}'.format(sum_not_proc))
logger.info('')
logger.info('')
basedir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
sys.path.append(basedir)
if __name__ == '__main__':
if os.path.isdir(sys.argv[1]):
setupdir = sys.argv[1]
storepath = os.path.join(setupdir, 'setupargs-pypicontents.json')
os.chdir(setupdir)
try:
with open(storepath, 'w') as store:
store.write(
u(
json.dumps({
'modules':
get_modules(get_packages(setupdir)),
'cmdline': []
})))
except BaseException:
sys.stderr.write(traceback.format_exc())
else:
setuppath = sys.argv[1]
setupdir = os.path.dirname(setuppath)
env = patchedglobals()
env.update({'__file__': setuppath})
os.chdir(setupdir)
sys.path.append(setupdir)
def pypi(**kwargs):
jsondict = {}
sum_updated = 0
sum_uptodate = 0
sum_nodata = 0
sum_nodown = 0
sum_noapi = 0
sum_nourls = 0
limit_mem_available = 600 * 1024 * 1024
allowed_range = list(string.ascii_lowercase) + list(map(str, range(0, 10)))
start_time = time.time()
logfile = kwargs.get('logfile')
if logfile:
logfile = os.path.abspath(logfile)
outputfile = os.path.abspath(kwargs.get('outputfile'))
extractdir = os.path.abspath(kwargs.get('extractdir'))
cachedir = os.path.abspath(kwargs.get('cachedir'))
limit_mem = kwargs.get('limit_mem')
limit_log_size = kwargs.get('limit_log_size')
letter_range = translate_letter_range(kwargs.get('letter_range'))
limit_time = int(kwargs.get('limit_time'))
clean = kwargs.get('clean')
if limit_mem.isdigit():
limit_mem = '{0}B'.format(limit_mem)
if limit_log_size.isdigit():
limit_log_size = '{0}B'.format(limit_log_size)
limit_mem = human2bytes(limit_mem)
limit_log_size = human2bytes(limit_log_size)
if len(set(letter_range) - set(allowed_range)):
raise RuntimeError('"{0}" not in allowed range.'.format(
', '.join(set(letter_range) - set(allowed_range))))
if not os.path.isdir(extractdir):
os.makedirs(extractdir)
if not os.path.isdir(cachedir):
os.makedirs(cachedir)
if not os.path.isfile(outputfile):
create_file_if_notfound(outputfile)
logger.info('Downloading package list from PyPI ...')
pkglist = filter_package_list(get_pkglist(), letter_range)
if not pkglist:
raise RuntimeError('Couldnt download the PyPI package list.\n'
'There was an error stablishing communication '
'with {0}'.format(pypiurl))
outjsondict = get_outputfile_jsondict(outputfile)
jsondict = prefill_jsondict(pkglist, jsondict, outjsondict)
for pkgname in sorted(jsondict.keys()):
elapsed_time = int(time.time() - start_time)
mem_usage = int(getrusage(RUSAGE_SELF).ru_maxrss * 1024)
mem_available = int(get_free_memory())
for chpid in get_children_processes(os.getpid()):
os.kill(int(chpid), signal.SIGKILL)
if logfile:
logsize = int(os.path.getsize(logfile))
else:
logsize = 0
logger.configpkg(pkgname)
if elapsed_time > limit_time:
logger.configpkg()
logger.warning('')
logger.warning('Processing has taken more than {0} seconds.'
' Interrupting.'.format(limit_time))
logger.warning('Processing will continue in next iteration.')
logger.warning('')
break
if mem_usage > limit_mem:
logger.configpkg()
logger.warning('')
logger.warning('This process is taking more than {0} MB of memory.'
' Interrupting'.format(limit_mem / (1024 * 1024)))
logger.warning('Processing will continue in next iteration.')
logger.warning('')
break
if mem_available < limit_mem_available:
logger.configpkg()
logger.warning('')
logger.warning('This machine is running out of memory.'
' Interrupting.')
logger.warning('Processing will continue in next iteration.')
logger.warning('')
break
if logsize > limit_log_size:
logger.configpkg()
logger.warning('')
logger.warning('The log is taking more than {0} MB.'
' Interrupting.'.format(logsize / (1024 * 1024)))
logger.warning('Processing will continue in next iteration.')
logger.warning('')
break
pkgdata, errstring = get_pkgdata(pkgname)
if not pkgdata:
logger.error('Could not get a response from API for this package. '
'See below for details.\n{0}'.format(errstring))
sum_noapi += 1
continue
newversion = pkgdata['info']['version']
currentversion = jsondict[pkgname]['version']
if newversion == currentversion:
logger.info('This package is up to date.')
sum_uptodate += 1
continue
pkgurls, errstring = get_pkgurls(pkgdata, pkgname, newversion)
if not pkgurls:
logger.error('Could not get download URLs for this package. '
'See below for details.\n{0}'.format(errstring))
sum_nourls += 1
continue
if not (pkgurls['sdist'] + pkgurls['bdist_wheel'] +
pkgurls['bdist_egg']):
logger.error('This package does not have downloadable releases.')
sum_nodown += 1
continue
setupargs, pkgpath, tarpath, errstring = get_setupargs(
pkgname, newversion, pkgurls, cachedir,
os.path.join(extractdir, pkgname))
if clean:
if os.path.isfile(tarpath):
os.remove(tarpath)
if os.path.isdir(pkgpath):
shutil.rmtree(pkgpath)
if not setupargs:
logger.error('Could not extract data from this package. '
'See below for details.\n{0}'.format(errstring))
sum_nodata += 1
continue
logger.info('Data has been updated for this package.')
jsondict[pkgname]['version'] = pkgdata['info']['version']
jsondict[pkgname].update(setupargs)
sum_updated += 1
with open(outputfile, 'w') as f:
f.write(
u(
json.dumps(jsondict,
separators=(',', ': '),
sort_keys=True,
indent=4)))
sum_proc = (sum_updated + sum_uptodate + sum_noapi + sum_nourls +
sum_nodown + sum_nodata)
sum_not_proc = len(jsondict.keys()) - sum_proc
logger.configpkg('')
logger.info('')
logger.info('')
logger.info('Total packages: {0}'.format(len(jsondict.keys())))
logger.info(' Packages processed: {0}'.format(sum_proc))
logger.info(' Packages updated: {0}'.format(sum_updated))
logger.info(' Packages up-to-date: {0}'.format(sum_uptodate))
logger.info(' Packages without response: {0}'.format(sum_noapi))
logger.info(' Packages without urls: {0}'.format(sum_nourls))
logger.info(' Packages without downloads: {0}'.format(sum_nodown))
logger.info(' Packages with data errors: {0}'.format(sum_nodata))
logger.info(' Packages not processed: {0}'.format(sum_not_proc))
logger.info('')
logger.info('')
from .core.patches import patchedglobals
except ImportError:
basedir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
sys.path.append(basedir)
if __name__ == '__main__':
if os.path.isdir(sys.argv[1]):
setupdir = sys.argv[1]
storepath = os.path.join(setupdir, 'setupargs-pypicontents.json')
os.chdir(setupdir)
try:
with open(storepath, 'w') as store:
store.write(u(json.dumps({
'modules': get_modules(get_packages(setupdir)),
'cmdline': []})))
except BaseException:
sys.stderr.write(traceback.format_exc())
else:
setuppath = sys.argv[1]
setupdir = os.path.dirname(setuppath)
env = patchedglobals()
env.update({'__file__': setuppath})
os.chdir(setupdir)
sys.path.append(setupdir)
try:
