def setup_pkg(virtualenv, pkg_dir, options, test=True, indent_txt='',
deps=True):
"""
Sets up a package in the given virtualenv
Parameters
----------
virtualenv : `str`
virtulenv base dir
pkg_dir : `str`
package checkout dir
options : `optparse.Options`
command-line options
test : `bool`
True/False will install test packages
deps : `bool`
True/False will install dependencies
"""
get_log().info("%s Setting up package in %s" % (indent_txt, pkg_dir))
if getattr(options, 'no_action', False):
return
python = os.path.join(virtualenv, 'bin', 'python')
cmd = [python, 'setup.py']
if options.verbose:
cmd.append('-v')
cmd.append('develop')
if not test:
cmd.append('--no-test')
if getattr(options, 'prefer_final', False):
cmd.append('--prefer-final')
if not deps:
cmd.append('--no-deps')
with cmdline.chdir(pkg_dir):
cmdline.run(cmd, capture_stdout=not options.verbose)
def run_with_coverage(cmd, pytestconfig, coverage=None, cd=None, **kwargs):
"""
Run a given command with coverage enabled. This won't make any sense
if the command isn't a python script.
This must be run within a pytest session that has been setup with
the '--cov=xxx' options, and therefore requires the pytestconfig
argument that can be retrieved from the standard pytest funcarg
of the same name.
Parameters
----------
cmd: `List`
Command to run
pytestconfig: `pytest._config.Config`
Pytest configuration object
coverage: `str`
Path to the coverage executable
cd: `str`
If not None, will change to this directory before running the cmd.
This is the directory that the coverage files will be created in.
kwargs: keyword arguments
Any extra arguments to pass to `pkglib.cmdline.run`
Returns
-------
`str` standard output
Examples
--------
>>> def test_example(pytestconfig):
... cmd = ['python','myscript.py']
... run_with_coverage(cmd, pytestconfig)
"""
if isinstance(cmd, str):
cmd = [cmd]
if coverage is None:
coverage = [sys.executable, '-mcoverage.__main__']
elif isinstance(coverage, str):
coverage = [coverage]
args = coverage + ['run', '-p']
if pytestconfig.option.cov_source:
source_dirs = ",".join(pytestconfig.option.cov_source)
args += ['--source=%s' % source_dirs]
args += cmd
if cd:
with cmdline.chdir(cd):
return cmdline.run(args, **kwargs)
return cmdline.run(args, **kwargs)
def run_graph_easy(entries, renderer, outfile=None):
""" Given the path edge entries, run the graphing tools and produce the
output.
Parameters
----------
entries : `list`
Path edges
renderer : `str`
One of 'ascii', 'boxart' or 'graphviz'
outfile : `str`
File to save to, only for graphviz. If None, it will delete the
generated file.
"""
if not CONFIG.graph_easy:
log.warn("Graph-Easy not configured, please set graph_easy variable in pkglib config")
return
from path import path
if renderer == 'graphviz' and not os.getenv('DISPLAY'):
log.info("No DISPLAY set, using ascii renderer")
renderer = 'ascii'
instream = '\n'.join(entries)
if os.path.isfile(CONFIG.graph_easy / 'bin' / 'graph-easy'):
with cmdline.chdir(CONFIG.graph_easy / 'lib'):
if renderer == 'graphviz':
delete = False
if not outfile:
tmpdir = path(tempfile.mkdtemp())
outfile = tmpdir / 'depgraph.png'
delete = True
outfile = path(outfile)
outfile = outfile.abspath()
cmdline.run(['-c', ('../bin/graph-easy --as={0} | /usr/bin/dot -Tpng -o {1}'
.format(renderer, outfile))],
capture_stdout=False, stdin=instream, shell=True)
if not outfile.isfile:
log.error("Failed to create image file.")
return
webbrowser.open('file://%s' % outfile)
if delete:
time.sleep(5)
shutil.rmtree(tmpdir)
else:
log.info("Created graph at %s" % outfile)
else:
cmdline.run(['../bin/graph-easy', '--as=%s' % renderer],
capture_stdout=False, stdin=instream)
return
log.warn("Can't find graphing tool at %s" % CONFIG.graph_easy)
def get_inhouse_dependencies(pkg_dir, exceptions=[], indent_txt=''):
"""
Yields a list of dependencies to setup.
"""
with cmdline.chdir(pkg_dir):
if os.path.isfile('setup.cfg'):
metadata = parse.parse_pkg_metadata(parse.get_pkg_cfg_parser())
for req in pkg_resources.parse_requirements(
list(r for r in metadata.get('install_requires', [])
if util.is_inhouse_package(r))):
if req.project_name not in exceptions:
yield req.project_name
else:
get_log().warn("{0} Package at {1} has no setup.cfg file, cannot "
"find dependencies.".format(indent_txt, pkg_dir))
def run(self, cmd, capture=False, check_rc=True, cd=None, shell=True, **kwargs):
"""
Run a command relative to a given directory, defaulting to the workspace root
Parameters
----------
cmd : `str`
Command string.
capture : `bool`
Capture and return output
check_rc : `bool`
Assert return code is zero
cd : `str`
Path to chdir to, defaults to workspace root
"""
if isinstance(cmd, str):
cmd = [cmd]
shell = True
if not cd:
cd = self.workspace
with cmdline.chdir(cd):
log.debug("run: %s" % str(cmd))
if capture:
p = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs)
else:
p = subprocess.Popen(cmd, shell=shell, **kwargs)
(out, _) = p.communicate()
if out is not None and not isinstance(out, string_types):
out = out.decode('utf-8')
if self.debug and capture:
log.debug("Stdout/stderr:")
log.debug(out)
if check_rc and p.returncode != 0:
err = subprocess.CalledProcessError(p.returncode, cmd)
err.output = out
if capture and not self.debug:
log.debug("Stdout/stderr:")
log.debug(out)
raise err
return out
def setup():
""" Mirror pkglib's setup() method for each sub-package in this repository.
"""
top_level_parser = parse.get_pkg_cfg_parser()
cfg = parse.parse_section(top_level_parser, 'multipkg', ['pkg_dirs'])
rc = [0]
for dirname in cfg['pkg_dirs']:
with cmdline.chdir(dirname):
# Update sub-package setup.cfg with top-level version if it's specified
if 'version' in cfg:
sub_parser = parse.get_pkg_cfg_parser()
sub_cfg = parse.parse_pkg_metadata(sub_parser)
if sub_cfg['version'] != cfg['version']:
print ("Updating setup.cfg version for {0}: {1} -> {2}"
.format(dirname, sub_cfg['version'], cfg['version']))
sub_parser.set('metadata', 'version', cfg['version'])
with open('setup.cfg', 'w') as sub_cfg_file:
sub_parser.write(sub_cfg_file)
cmd = [sys.executable, "setup.py"] + sys.argv[1:]
print ("In directory {0}: Running '{1}'"
.format(dirname, ' '.join(cmd)))
try:
cmdline.run(cmd, capture_stdout=False, bufsize=0)
except subprocess.CalledProcessError as e:
# Here we exit straight away, unless this was a run as
# 'python setup.py test'. Reason for this is that we want to
# run all the packages' tests through and gather the results.
# Exception: using the -x/--exitfirst option.
# For any other setup.py command, a failure here is likely
# some sort of build or config issue and it's best not to
# plow on.
print "Command failed with exit code {0}".format(e.returncode)
if 'test' in cmd and not '-x' in ' '.join(cmd) \
and not '--exitfirst' in ' '.join(cmd):
rc[0] = e.returncode
else:
sys.exit(e.returncode)
sys.exit(rc[0])
def setup():
""" Mirror pkglib's setup() method for each sub-package in this repository.
"""
top_level_parser = parse.get_pkg_cfg_parser()
cfg = parse.parse_section(top_level_parser, 'multipkg', ['pkg_dirs'])
rc = [0]
for dirname in cfg['pkg_dirs']:
with cmdline.chdir(dirname):
# Update sub-package setup.cfg with top-level version if it's specified
if 'version' in cfg:
sub_parser = parse.get_pkg_cfg_parser()
sub_cfg = parse.parse_pkg_metadata(sub_parser)
if sub_cfg['version'] != cfg['version']:
print("Updating setup.cfg version for {0}: {1} -> {2}".
format(dirname, sub_cfg['version'], cfg['version']))
sub_parser.set('metadata', 'version', cfg['version'])
with open('setup.cfg', 'w') as sub_cfg_file:
sub_parser.write(sub_cfg_file)
cmd = [sys.executable, "setup.py"] + sys.argv[1:]
print("In directory {0}: Running '{1}'".format(
dirname, ' '.join(cmd)))
try:
cmdline.run(cmd, capture_stdout=False, bufsize=0)
except subprocess.CalledProcessError as e:
# Here we exit straight away, unless this was a run as
# 'python setup.py test'. Reason for this is that we want to
# run all the packages' tests through and gather the results.
# Exception: using the -x/--exitfirst option.
# For any other setup.py command, a failure here is likely
# some sort of build or config issue and it's best not to
# plow on.
print "Command failed with exit code {0}".format(e.returncode)
if 'test' in cmd and not '-x' in ' '.join(cmd) \
and not '--exitfirst' in ' '.join(cmd):
rc[0] = e.returncode
else:
sys.exit(e.returncode)
sys.exit(rc[0])
def create_relative_link(self, src, dest):
""" Create relative symlink from src -> dest
"""
with cmdline.chdir(os.path.dirname(src)):
os.symlink(dest, os.path.basename(src))
def test_Shell_func_2():
this_dir = os.path.dirname(__file__)
# start at parent of this directory
with chdir(os.path.dirname(this_dir)):
with cmdline.Shell(['cd %s' % this_dir, 'ls']) as s:
assert os.path.basename(__file__) in s.out.split('\n')
def test_Shell_func_1_as_list():
with chdir(os.path.dirname(__file__)):
with cmdline.Shell(['ls']) as s:
assert os.path.basename(__file__) in s.out.split('\n')
def check_and_remove(adir):
with chdir(adir):
for name in names:
for item in glob.iglob(name):
self.remove_object(adir, item)
def create_relative_link(self, src, dest):
""" Create relative symlink from src -> dest
"""
with cmdline.chdir(os.path.dirname(src)):
os.symlink(dest, os.path.basename(src))
def check_and_remove(adir):
with chdir(adir):
for name in names:
for item in glob.iglob(name):
self.remove_object(adir, item)
def test_chdir():
here = os.getcwd()
with cmdline.chdir('/bin'):
assert os.getcwd() == '/bin'
assert os.getcwd() == here
def test_chdir():
here = os.getcwd()
with cmdline.chdir('/bin'):
assert os.getcwd() == '/bin'
assert os.getcwd() == here
def test_Shell_func_1_as_list():
with chdir(os.path.dirname(__file__)):
with cmdline.Shell(['ls']) as s:
assert os.path.basename(__file__) in s.out.split('\n')
def test_Shell_func_2():
this_dir = os.path.dirname(__file__)
# start at parent of this directory
with chdir(os.path.dirname(this_dir)):
with cmdline.Shell(['cd %s' % this_dir, 'ls']) as s:
assert os.path.basename(__file__) in s.out.split('\n')
def run_graph_easy(entries, renderer, outfile=None):
""" Given the path edge entries, run the graphing tools and produce the
output.
Parameters
----------
entries : `list`
Path edges
renderer : `str`
One of 'ascii', 'boxart' or 'graphviz'
outfile : `str`
File to save to, only for graphviz. If None, it will delete the
generated file.
"""
if not CONFIG.graph_easy:
log.warn(
"Graph-Easy not configured, please set graph_easy variable in pkglib config"
)
return
from path import path
if renderer == 'graphviz' and not os.getenv('DISPLAY'):
log.info("No DISPLAY set, using ascii renderer")
renderer = 'ascii'
instream = '\n'.join(entries)
if os.path.isfile(CONFIG.graph_easy / 'bin' / 'graph-easy'):
with cmdline.chdir(CONFIG.graph_easy / 'lib'):
if renderer == 'graphviz':
delete = False
if not outfile:
tmpdir = path(tempfile.mkdtemp())
outfile = tmpdir / 'depgraph.png'
delete = True
outfile = path(outfile)
outfile = outfile.abspath()
cmdline.run([
'-c',
('../bin/graph-easy --as={0} | /usr/bin/dot -Tpng -o {1}'.
format(renderer, outfile))
],
capture_stdout=False,
stdin=instream,
shell=True)
if not outfile.isfile:
log.error("Failed to create image file.")
return
webbrowser.open('file://%s' % outfile)
if delete:
time.sleep(5)
shutil.rmtree(tmpdir)
else:
log.info("Created graph at %s" % outfile)
else:
cmdline.run(['../bin/graph-easy',
'--as=%s' % renderer],
capture_stdout=False,
stdin=instream)
return
log.warn("Can't find graphing tool at %s" % CONFIG.graph_easy)
