def run_cleanup_in_subprocess(self):
""" Runs the cleanup job in a subprocess. Necessary as setup.py is often
changing the state of the virtualenv as it goes, and working_set in
memory might not reflect the real state of the world
"""
# Using the entry pount here instead of the module. This is because the module may have
# been moved by the time we go to run it, eg if we just updated pkglib itself.
cmd = [sys.executable, os.path.join(sys.exec_prefix, 'bin', 'pycleanup')]
for arg in ['-v', '-n', '--verbose', '--dry-run']:
if arg in sys.argv:
cmd.append(arg)
cmdline.run(cmd, capture_stdout=False)
def run_graph_easy(entries, renderer, outfile=None):
""" Given the path edge entries, run the graphing tools and produce the
output.
Parameters
----------
entries : `list`
Path edges
renderer : `str`
One of 'ascii', 'boxart' or 'graphviz'
outfile : `str`
File to save to, only for graphviz. If None, it will delete the
generated file.
"""
if not CONFIG.graph_easy:
log.warn("Graph-Easy not configured, please set graph_easy variable in pkglib config")
return
from path import path
if renderer == 'graphviz' and not os.getenv('DISPLAY'):
log.info("No DISPLAY set, using ascii renderer")
renderer = 'ascii'
instream = '\n'.join(entries)
if os.path.isfile(CONFIG.graph_easy / 'bin' / 'graph-easy'):
with chdir(CONFIG.graph_easy / 'lib'):
if renderer == 'graphviz':
delete = False
if not outfile:
tmpdir = path(tempfile.mkdtemp())
outfile = tmpdir / 'depgraph.png'
delete = True
outfile = path(outfile)
outfile = outfile.abspath()
run(['-c', ('../bin/graph-easy --as={0} | '
'/usr/bin/dot -Tpng -o {1}'
.format(renderer, outfile))],
capture_stdout=False, stdin=instream, shell=True)
if not outfile.isfile:
log.error("Failed to create image file.")
return
webbrowser.open('file://%s' % outfile)
if delete:
time.sleep(5)
shutil.rmtree(tmpdir)
else:
log.info("Created graph at %s" % outfile)
else:
run(['../bin/graph-easy', '--as=%s' % renderer],
capture_stdout=False, stdin=instream)
return
log.warn("Can't find graphing tool at %s" % CONFIG.graph_easy)
def mirror_eggs(self, file_root, target_host, target_root,
target_packages=None, subprocesses=10):
""" Mirrors egg files from this PyPi instance to a target host and
path. Used for filling out a cache that can be used by
CONFIG.installer_search_path.
Parameters
----------
file_root : `str`
filesystem path to the root of the file store
target_host : `str`
host to mirror to
target_root : `str`
filesystem path to mirror to on target host
target_packages : `list` or None
list of packages to mirror. Use None for all.
subprocesses : `int`
number of subprocesses to spawn when doing the mirror
"""
from path import path
file_root = path(file_root)
target_root = path(target_root)
pkg_dirs, target_dirs = self.get_mirror_targets(file_root, target_root,
target_packages)
print "Creating target root dirs"
run(['/usr/bin/ssh', target_host, 'mkdir -p ' + ' '.join(target_dirs)])
work = []
for pkg in pkg_dirs:
# Filter non-egg and dev packages out, as this is a site-packages
# mirror which won't work with source packages.
files = [i for i in pkg.files()
if i.basename().endswith('egg')
and not 'dev' in i.basename()]
print "Found %s (%d files)" % (pkg.basename(), len(files))
if files:
cmd = (['/usr/bin/rsync', '-av', '--ignore-existing'] +
[i.abspath().strip() for i in files] +
[target_host + ':' + target_root /
self.get_mirror_dirname(pkg.basename())]
)
work.append(cmd)
# Using multiprocessing here to multiplex the transfers
if subprocesses > 1:
pool = Pool(processes=subprocesses)
pool.map(run, work)
else:
map(run, work)
self.unpack_eggs(files, target_host, target_root)
def run_with_coverage(cmd, pytestconfig, coverage='coverage', cd=None, **kwargs):
"""
Run a given command with coverage enabled. This won't make any sense
if the command isn't a python script.
This must be run within a pytest session that has been setup with
the '--cov=xxx' options, and therefore requires the pytestconfig
argument that can be retrieved from the standard pytest funcarg
of the same name.
Parameters
----------
cmd: `List`
Command to run
pytestconfig: `pytest._config.Config`
Pytest configuration object
coverage: `str`
Path to the coverage executable
cd: `str`
If not None, will change to this directory before running the cmd.
This is the directory that the coverage files will be created in.
kwargs: keyword arguments
Any extra arguments to pass to `pkglib.cmdline.run`
Returns
-------
`str` standard output
Examples
--------
>>> def test_example(pytestconfig):
... cmd = ['python','myscript.py']
... run_with_coverage(cmd, pytestconfig)
"""
if isinstance(cmd, str):
cmd = [cmd]
args = [coverage, 'run', '-p']
if pytestconfig.option.cov_source:
source_dirs = ",".join(pytestconfig.option.cov_source)
args += ['--source=%s' % source_dirs]
args += cmd
if cd:
with manage.chdir(cd):
return run(args, **kwargs)
return run(args, **kwargs)
def get_open_files(self):
""" Returns open files under our site-packages
"""
# It's far cheaper to run lsof for all files and search later than running it with
# the +D option to only return results under a certain directory
cmd = "/usr/sbin/lsof 2>/dev/null | grep %s | awk '{ print $2 \" \" $9 }'" % \
self.site_packages
return [i.split() for i in cmdline.run(cmd, capture_stdout=True, check_rc=False, shell=True).split('\n') if i]
def get_open_files(self):
""" Returns open files under our site-packages
"""
# It's far cheaper to run lsof for all files and search later than
# running it with the +D option to only return results under a certain
# directory
# TODO: this might not be on the path, and be hidden by the >/dev/null
cmd = ("lsof 2>/dev/null | grep {} |"
"awk '{{ print $2 \" \" $9 }}'").format(self.site_packages)
return [i.split() for i in
cmdline.run(cmd, capture_stdout=True, check_rc=False,
shell=True).split('\n') if i]
def unpack_eggs(self, files, target_host, target_root):
""" Unpacks all eggs on the target host and root
"""
print "Unpacking eggs: %r" % files
target_eggs = [target_root / self.get_mirror_dirname(f.parent.basename()) \
/ f.name for f in files]
cmd = """set -x
for EGG in %s; do
if [ -f $EGG ]; then
echo Unzipping $EGG
ZIPFILE=./.tmp.`basename $EGG`
mv $EGG $ZIPFILE && \
mkdir $EGG && \
unzip -q $ZIPFILE -d $EGG && \
rm $ZIPFILE && \
chmod -R 555 $EGG
fi
done""" % ' '.join(target_eggs)
print "Running cmd on %s" % target_host
print cmd
run(['/usr/bin/ssh', target_host, cmd])
