def tox_run(config, opts):
"""
tox test
activate venv and run tox test suite
"""
suite_conf = config.test_suite(opts.suite)
tox_ini = suite_conf.get('tox_ini')
test_opts = suite_conf.get('test_options')
if opts.options:
# command line overrides
test_opts = opts.options
tox_command = "tox"
if tox_ini:
tox_command += " -c {}".format(tox_ini)
if test_opts:
tox_command += " {}".format(test_opts)
if opts.builder is None:
opts.builder = get_builder_plugin()
builder = FACTORY(opts.builder)
activate = builder.activate()
local(
'{0} && {1}'.format(
activate,
tox_command
)
)
def pytest_run(config, opts):
"""
activate venv and run pytest
"""
suite_conf = config.test_suite(opts.suite)
test_opts = suite_conf.get('test_options')
where = suite_conf.get('where', 'tests/unit')
if opts.options:
# command line overrides
test_opts = opts.options
command = "pytest {}".format(where)
if test_opts:
command += " {}".format(test_opts)
if opts.builder is None:
opts.builder = get_builder_plugin()
builder = FACTORY(opts.builder)
activate = builder.activate()
local(
'{0} && {1}'.format(
activate,
command
)
)
def nose_run(config, opts):
"""
_nose_test_
Locally activate vitrualenv and run tests via nose
"""
where = config.test_where(opts.suite)
suite_conf = config.test_suite(opts.suite)
test_opts = suite_conf.get('test_options')
if opts.options:
# command line overrides
test_opts = opts.options
if opts.builder is None:
opts.builder = get_builder_plugin()
builder = FACTORY(opts.builder)
activate = builder.activate()
local(
'{0} && nosetests -w {1} {2}'.format(
activate,
where,
test_opts if test_opts else ""
)
)
def _install_extras(self, extra):
activate = self.activate()
LOGGER.info("Installing extra dependencies: [{}]".format(extra))
local(
'{} && pip install {}[{}]'.format(
activate, self.config.package_name(), extra
)
)
def run_setup_develop(self):
LOGGER.info("Running setup.py develop...")
activate = self.activate()
local(
'{} && python setup.py develop'.format(
activate
)
)
def clean(self, **kwargs):
conda = kwargs.get('conda', self.conda_bin)
if os.path.exists(self.venv_path):
cmd = "{} remove --all -y -p {}".format(
conda,
self.venv_path
)
LOGGER.info("Removing existing conda env: {0}".format(self.venv_path))
local(cmd)
def test_local_ok(self, mock_invoke):
mock_ctx = mock.Mock()
mock_ctx.run = mock.Mock()
mock_invoke.return_value = mock_ctx
comm = "/bin/ls"
local(comm)
self.assertTrue(mock_ctx.run.called)
def test_local_ok(self, mock_invoke):
mock_ctx = mock.Mock()
mock_ctx.run = mock.Mock()
mock_invoke.return_value=mock_ctx
comm = "/bin/ls"
local(comm)
self.assertTrue(mock_ctx.run.called)
def setup_develop(config):
"""
_setup_develop_
run local python setup.py develop via fab
"""
LOGGER.info("running setup.py develop...")
local('git cirrus build --upgrade')
local(' . ./{0}/bin/activate && python setup.py develop'.format(
config.venv_name()))
return
def pip_install(version, update_setuptools=False):
"""pip install the version of cirrus requested"""
pip_req = 'cirrus-cli=={0}'.format(version)
venv_path = virtualenv_home()
venv_name = os.path.basename(venv_path)
LOGGER.info("running pip upgrade...")
if is_anaconda():
if update_setuptools:
local(
'source {0}/bin/activate {1} && pip install --upgrade setuptools'.format(
venv_path, venv_path
)
)
local(
'source {0}/bin/activate {1} && pip install --upgrade {2}'.format(
venv_path, venv_path, pip_req
)
)
else:
if update_setuptools:
local(
' . ./{0}/bin/activate && pip install --upgrade setuptools'.format(
venv_name
)
)
local(
' . ./{0}/bin/activate && pip install --upgrade {1}'.format(
venv_name, pip_req
)
)
def pylint_file(filenames, **kwargs):
"""
apply pylint to the file specified,
return the filename, score
"""
command = "pylint "
if 'rcfile' in kwargs and kwargs['rcfile'] is not None:
command += " --rcfile={0} ".format(kwargs['rcfile'])
command = command + ' '.join(filenames)
# we use fabric to run the pylint command, hiding the normal fab
# output and warnings
result = local(command)
score = 0.0
# parse the output from pylint for the score
for line in result.stdout.split('\n'):
if re.match("E....:.", line):
LOGGER.info(line)
if "Your code has been rated at" in line:
score = re.findall("\d+.\d\d", line)[0]
score = float(score)
return filenames, score
def nose_run(config, opts):
"""
_nose_test_
Locally activate vitrualenv and run tests via nose
"""
where = config.test_where(opts.suite)
suite_conf = config.test_suite(opts.suite)
test_opts = suite_conf.get('test_options')
venv_path = os.path.join(repo_directory(), config.venv_name())
if opts.options:
# command line overrides
test_opts = opts.options
activate = activate_command(venv_path)
local('{0} && nosetests -w {1} {2}'.format(activate, where,
test_opts if test_opts else ""))
def setup_develop(config):
"""
_setup_develop_
run local python setup.py develop via fab
"""
LOGGER.info("running setup.py develop...")
local(
'git cirrus build --upgrade'
)
local(
' . ./{0}/bin/activate && python setup.py develop'.format(
config.venv_name()
)
)
return
def upload(self, opts, build_artifact):
"""
upload to pypi using sdist.
This will use settings from the ~/.pypirc file.
The pypi-url CLI option can be used to give a pypi
server url or an alias specified in the pypirc
"""
pypirc = os.path.expandvars('$HOME/.pypirc')
if not os.path.exists(pypirc):
msg = "$HOME/.pypirc not found, cannot upload to pypi"
LOGGER.error(msg)
raise RuntimeError(msg)
command = 'python setup.py sdist upload'
if opts.pypi_url:
LOGGER.info("using pypi server {}".format(opts.pypi_url))
pypi_url = opts.pypi_url
command += ' -r {}'.format(pypi_url)
LOGGER.info("Executing {} ...".format(command))
local(command)
def tox_run(config, opts):
"""
tox test
activate venv and run tox test suite
"""
suite_conf = config.test_suite(opts.suite)
tox_ini = suite_conf.get('tox_ini')
test_opts = suite_conf.get('test_options')
venv_path = os.path.join(repo_directory(), config.venv_name())
if opts.options:
# command line overrides
test_opts = opts.options
tox_command = "tox"
if tox_ini:
tox_command += " -c {}".format(tox_ini)
if test_opts:
tox_command += " {}".format(test_opts)
activate = activate_command(venv_path)
local('{0} && {1}'.format(activate, tox_command))
def build_docs(make_opts=None):
"""
_build_docs_
Runs 'make' against a Sphinx makefile.
Requires the following cirrus.conf section:
[doc]
sphinx_makefile_dir = /path/to/makefile
:param list make_opts: Options to run the Sphinx 'make' command gathered
from the calling command's --docs option. If None or an empty list,
'make' will be run using 'clean html'
"""
LOGGER.info('Building docs')
config = load_configuration()
build_params = config.get('build', {})
venv_name = build_params.get('virtualenv_name', 'venv')
try:
docs_root = os.path.join(
os.getcwd(),
config['doc']['sphinx_makefile_dir'])
except KeyError:
LOGGER.error(
'Did not find a complete [doc] section in cirrus.conf'
'\nSee below for an example:'
'\n[doc]'
'\nsphinx_makefile_dir = /path/to/sphinx')
sys.exit(1)
cmd = 'cd {} && make clean html'.format(docs_root)
if make_opts:
# additional args were passed after --docs. Pass these to make
cmd = 'cd {} && make {}'.format(docs_root, ' '.join(make_opts))
local('. ./{}/bin/activate && {}'.format(venv_name, cmd))
LOGGER.info('Build command was "{}"'.format(cmd))
def create(self, **kwargs):
python_bin = kwargs.get("python", self.python_bin)
conda = kwargs.get('conda', self.conda_bin)
upgrade = kwargs.get('upgrade', False)
nosetupdevelop = kwargs.get('nosetupdevelop', False)
clean = kwargs.get('clean', False)
if clean:
self.clean(**kwargs)
venv_command = "{} create -y -m -p {} pip virtualenv".format(
conda, self.venv_path)
if python_bin:
# should probably check this is int or int.int format
venv_command += " python={}".format(self.python_bin)
if not os.path.exists(self.venv_path):
LOGGER.info("Bootstrapping conda env: {0}".format(self.venv_path))
local(venv_command)
cmd = build_pip_command(self.config,
self.venv_path,
self.reqs_name,
upgrade=upgrade)
try:
local(cmd)
except OSError as ex:
msg = ("Error running pip install command during build\n"
"Error was {0}\n"
"Running command: {1}\n"
"Working Dir: {2}\n"
"Conda env: {3}\n"
"Requirements: {4}\n").format(ex, cmd, self.working_dir,
self.venv_path,
self.reqs_name)
LOGGER.error(msg)
raise
for req in self.extra_reqs:
cmd = build_pip_command(self.config,
self.venv_path,
req,
upgrade=upgrade)
LOGGER.info("Installing extra requirements... {}".format(cmd))
try:
local(cmd)
except OSError as ex:
msg = ("Error running pip install command extra "
"requirements install: {}\n{}").format(req, ex)
LOGGER.error(msg)
raise
# setup for development
if nosetupdevelop:
msg = "skipping python setup.py develop..."
LOGGER.info(msg)
else:
self.run_setup_develop()
def pip_install(version, update_setuptools=False):
"""pip install the version of cirrus requested"""
pip_req = 'cirrus-cli=={0}'.format(version)
venv_path = virtualenv_home()
venv_name = os.path.basename(venv_path)
LOGGER.info("running pip upgrade...")
if is_anaconda():
if update_setuptools:
local(
'source {0}/bin/activate {1} && pip install --upgrade setuptools'
.format(venv_path, venv_path))
local(
'source {0}/bin/activate {1} && pip install --upgrade {2}'.format(
venv_path, venv_path, pip_req))
else:
if update_setuptools:
local(' . ./{0}/bin/activate && pip install --upgrade setuptools'.
format(venv_name))
local(' . ./{0}/bin/activate && pip install --upgrade {1}'.format(
venv_name, pip_req))
def pyflakes_file(filenames, verbose=False):
"""
_pyflakes_file_
Appyly pyflakes to file specified,
return (filenames, score)
"""
command = 'pyflakes ' + ' '.join(filenames)
# we use fabric to run the pyflakes command, hiding the normal fab
# output and warnings
result = local(command)
flakes = 0
data = [x for x in result.stdout.split('\n') if x.strip()]
if len(data) != 0:
#We have at least one flake, find the rest
flakes = count_flakes(data, verbose) + 1
else:
flakes = 0
return filenames, flakes
def create(self, **kwargs):
python_bin = kwargs.get("python")
if python_bin is not None:
self.python_bin = python_bin
conda = kwargs.get('conda', self.conda_bin)
upgrade = kwargs.get('upgrade', False)
nosetupdevelop = kwargs.get('nosetupdevelop', False)
clean = kwargs.get('clean', False)
channels = self.str_to_list(self.build_config.get('conda_channels', []))
LOGGER.info("channels from conf: {}".format(channels))
if kwargs.get('conda_channels'):
channels = kwargs['conda_channels']
if clean:
self.clean(**kwargs)
channels = ''.join(' -c {}'.format(c) for c in channels)
LOGGER.info("channels: result={}".format(kwargs.get('conda_channels'), self.build_config.get('conda_channels'), channels))
venv_command = "{} create -y -m {} -p {} pip virtualenv".format(
conda,
channels,
self.venv_path
)
if self.python_bin:
# should probably check this is int or int.int format
venv_command += " python={}".format(self.python_bin_for_conda)
if not os.path.exists(self.venv_path):
LOGGER.info("Bootstrapping conda env: {0}".format(self.venv_path))
local(venv_command)
cmd = "conda install {} {} --yes --file {}".format(
'--update-dependencies' if upgrade else '--no-update-dependencies',
channels,
self.reqs_name
)
try:
local(cmd)
except OSError as ex:
msg = (
"Error running conda install command during build\n"
"Error was {0}\n"
"Running command: {1}\n"
"Working Dir: {2}\n"
"Conda env: {3}\n"
"Requirements: {4}\n"
).format(ex, cmd, self.working_dir, self.venv_path, self.reqs_name)
LOGGER.error(msg)
raise
for req in self.extra_reqs:
cmd = "conda install {} {} --yes --file {}".format(
'--update-dependencies' if upgrade else '',
channels,
req
)
LOGGER.info("Installing extra requirements... {}".format(cmd))
try:
local(cmd)
except OSError as ex:
msg = (
"Error running conda install command extra "
"requirements install: {}\n{}"
).format(req, ex)
LOGGER.error(msg)
raise
# setup for development
if nosetupdevelop:
msg = "skipping python setup.py develop..."
LOGGER.info(msg)
else:
self.run_setup_develop()
def clean(self, **kwargs):
if os.path.exists(self.venv_path):
cmd = "rm -rf {0}".format(self.venv_path)
LOGGER.info("Removing virtualenv: {0}".format(self.venv_path))
local(cmd)
def create(self, **kwargs):
"""
build the virtualenv
"""
clean = kwargs.get('clean', False)
if clean:
self.clean(**kwargs)
site_packages = kwargs.get(
'system-site-packages',
self.use_sitepackages
)
upgrade = kwargs.get('upgrade', False)
extras_require = kwargs.get('extras_require', [])
all_extras = kwargs.get('all_extras', False)
if all_extras:
extras_require = self.config.extras_require().keys()
nosetupdevelop = kwargs.get('nosetupdevelop', False)
venv = VirtualEnvironment(
self.venv_path,
python=self.python_bin_for_venv,
system_site_packages=site_packages
)
LOGGER.info("Bootstrapping virtualenv: {}".format(self.venv_path))
venv.open_or_create()
cmd = build_pip_command(
self.config,
self.venv_path,
self.reqs_name,
upgrade=upgrade
)
try:
local(cmd)
except OSError as ex:
msg = (
"Error running pip install command during build\n"
"Error was {0}\n"
"Running command: {1}\n"
"Working Dir: {2}\n"
"Virtualenv: {3}\n"
"Requirements: {4}\n"
).format(ex, cmd, self.working_dir, self.venv_path, self.reqs_name)
LOGGER.error(msg)
raise
for req in self.extra_reqs:
cmd = build_pip_command(
self.config,
self.venv_path,
req,
upgrade=upgrade
)
LOGGER.info("Installing extra requirements... {}".format(cmd))
try:
local(cmd)
except OSError as ex:
msg = (
"Error running pip install command extra "
"requirements install: {}\n{}"
).format(req, ex)
LOGGER.error(msg)
raise
# setup for development
if nosetupdevelop:
msg = "skipping python setup.py develop..."
LOGGER.info(msg)
else:
self.run_setup_develop()
if extras_require:
for extra in extras_require:
self._install_extras(extra)
def create(self, **kwargs):
python_bin = kwargs.get("python")
if python_bin is not None:
self.python_bin = python_bin
conda = kwargs.get('conda', self.conda_bin)
upgrade = kwargs.get('upgrade', False)
extra_pip = self.build_config.get('extra_pip_requirements')
if extra_pip:
extra_pip = self.str_to_list(extra_pip)
if kwargs.get('extra-pip-requirements'):
extra_pip = kwargs['extra-pip-requirements']
extra_conda = self.build_config.get('extra_conda_requirements')
if extra_conda:
extra_conda = self.str_to_list(extra_conda)
if kwargs.get('extra-conda-requirements'):
extra_conda = kwargs['extra-conda-requirements']
# TODO: also accept this from build_config, override from cli
nosetupdevelop = kwargs.get('nosetupdevelop', False)
environment = kwargs.get('environment', None)
if environment is None:
environment = self.build_config.get('conda-environment', None)
if environment is None:
msg = "No conda environment yaml specified in cirrus.conf [build] section or via --environment option"
LOGGER.error(msg)
raise RuntimeError(msg)
clean = kwargs.get('clean', False)
if clean:
self.clean(**kwargs)
venv_command = "{} env create -f {} -p {} ".format(
conda,
environment,
self.venv_path
)
if self.python_bin:
# should probably check this is int or int.int format
venv_command += " python={}".format(self.python_bin_for_conda)
msg = (
"Passing python version {} to conda env setup: "
"Usually it is safer to set this in the environment yaml"
).format(
self.python_bin_for_conda
)
LOGGER.warning(msg)
if not os.path.exists(self.venv_path):
LOGGER.info("Bootstrapping conda env: {0}".format(self.venv_path))
local(venv_command)
if upgrade:
cmd = "{activate} && conda env update {venv} -f {env}".format(
activate=self.activate(),
venv=self.venv_path,
env=environment
)
try:
local(cmd)
except OSError as ex:
msg = (
"Error running conda env update command during build\n"
"Error was {0}\n"
"Running command: {1}\n"
"Working Dir: {2}\n"
"Conda env: {3}\n"
"Requirements: {4}\n"
).format(ex, cmd, self.working_dir, self.venv_path, self.reqs_name)
LOGGER.error(msg)
raise
if extra_conda:
for conda_file in extra_conda:
cmd = "{activate} && conda install {venv} -f {file}".format(
activate=self.activate(),
venv=self.venv_path,
file=conda_file
)
LOGGER.info("Installing extra conda reqs: {}".format(conda_file))
try:
local(cmd)
except OSError as ex:
msg = (
"Error running conda install extras command during build\n"
"Error was {0}\n"
"Running command: {1}\n"
"Working Dir: {2}\n"
"Conda env: {3}\n"
"Extra Conda Requirements: {4}\n"
).format(ex, cmd, self.working_dir, self.venv_path, conda_file)
LOGGER.error(msg)
raise
if extra_pip:
for pip_file in extra_pip:
pip_options = self.config.pip_options()
cmd = "{activate} && pip install -r {reqs}".format(
activate=self.activate(),
venv=self.venv_path,
reqs=pip_file
)
if pip_options:
cmd += " {}".format(pip_options)
LOGGER.info("Installing extra pip reqs {}".format(pip_file))
try:
local(cmd)
except OSError as ex:
msg = (
"Error running pip install extras command during build\n"
"Error was {0}\n"
"Running command: {1}\n"
"Working Dir: {2}\n"
"Conda env: {3}\n"
"Extra pip Requirements: {4}\n"
).format(ex, cmd, self.working_dir, self.venv_path, pip_file)
LOGGER.error(msg)
raise
# setup for development
if nosetupdevelop:
msg = "skipping python setup.py develop..."
LOGGER.info(msg)
else:
self.run_setup_develop()
def __call__(self):
local(self.scp_command)
def create(self, **kwargs):
python_bin = kwargs.get("python")
if python_bin is not None:
self.python_bin = python_bin
LOGGER.info("Overriding python bin from command line: {}".format(python_bin))
conda = kwargs.get('conda', self.conda_bin)
upgrade = kwargs.get('upgrade', False)
nosetupdevelop = kwargs.get('nosetupdevelop', False)
clean = kwargs.get('clean', False)
if clean:
self.clean(**kwargs)
venv_command = "{} create -y -m -p {} pip virtualenv".format(
conda,
self.venv_path
)
if self.python_bin:
LOGGER.info("using python bin: {}".format(self.python_bin_for_conda))
# should probably check this is int or int.int format
venv_command += " python={}".format(self.python_bin_for_conda)
if not os.path.exists(self.venv_path):
LOGGER.info("Bootstrapping conda env: {0}".format(self.venv_path))
local(venv_command)
cmd = build_pip_command(
self.config,
self.venv_path,
self.reqs_name,
upgrade=upgrade
)
try:
local(cmd)
except OSError as ex:
msg = (
"Error running pip install command during build\n"
"Error was {0}\n"
"Running command: {1}\n"
"Working Dir: {2}\n"
"Conda env: {3}\n"
"Requirements: {4}\n"
).format(ex, cmd, self.working_dir, self.venv_path, self.reqs_name)
LOGGER.error(msg)
raise
for req in self.extra_reqs:
cmd = build_pip_command(
self.config,
self.venv_path,
req,
upgrade=upgrade
)
LOGGER.info("Installing extra requirements... {}".format(cmd))
try:
local(cmd)
except OSError as ex:
msg = (
"Error running pip install command extra "
"requirements install: {}\n{}"
).format(req, ex)
LOGGER.error(msg)
raise
# setup for development
if nosetupdevelop:
msg = "skipping python setup.py develop..."
LOGGER.info(msg)
else:
self.run_setup_develop()
