def test_reading(self, mock_shell):
"""test config load """
# test read and accessors
mock_shell.return_value = self.gitconf_str
config = load_configuration(package_dir=self.dir,
gitconfig_file=self.gitconfig)
self.assertEqual(config.package_version(), '1.2.3')
self.assertEqual(config.package_name(), 'cirrus_tests')
self.assertEqual(config.gitflow_branch_name(), 'develop')
self.assertEqual(config.gitflow_release_prefix(), 'release/')
self.assertEqual(config.gitflow_feature_prefix(), 'feature/')
self.assertEqual(config.release_notes(), (None, None))
self.assertEqual(config.version_file(), (None, '__version__'))
self.assertEqual(config.extras_require(), {
'analysis': 'pandas;scipy',
'server': 'Flask==0.0.0'
})
self.failUnless(config.credentials is not None)
self.failUnless(isinstance(config.credentials, Default))
# test updating version
config.update_package_version('1.2.4')
self.assertEqual(config.package_version(), '1.2.4')
config2 = load_configuration(package_dir=self.dir)
self.assertEqual(config2.package_version(), '1.2.4')
def test_reading(self, mock_shell):
"""test config load """
# test read and accessors
mock_shell.return_value = self.gitconf_str
config = load_configuration(
package_dir=self.dir,
gitconfig_file=self.gitconfig
)
self.assertEqual(config.package_version(), '1.2.3')
self.assertEqual(config.package_name(), 'cirrus_tests')
self.assertEqual(config.gitflow_branch_name(), 'develop')
self.assertEqual(config.gitflow_release_prefix(), 'release/')
self.assertEqual(config.gitflow_feature_prefix(), 'feature/')
self.assertEqual(config.release_notes(), (None, None))
self.assertEqual(config.version_file(), (None, '__version__'))
self.assertEqual(
config.extras_require(),
{'analysis': 'pandas;scipy', 'server': 'Flask==0.0.0'}
)
self.failUnless(config.credentials is not None)
self.failUnless(isinstance(config.credentials, Default))
# test updating version
config.update_package_version('1.2.4')
self.assertEqual(config.package_version(), '1.2.4')
config2 = load_configuration(package_dir=self.dir)
self.assertEqual(config2.package_version(), '1.2.4')
def branch_status(branch_name):
"""
_branch_status_
Get the branch status which should include details of CI builds/hooks etc
See:
https://developer.github.com/v3/repos/statuses/#get-the-combined-status-for-a-specific-ref
returns a state which is one of 'failure', 'pending', 'success'
"""
config = load_configuration()
token = get_github_auth()[1]
url = "https://api.github.com/repos/{org}/{repo}/commits/{branch}/status".format(
org=config.organisation_name(),
repo=config.package_name(),
branch=branch_name
)
headers = {
'Authorization': 'token {0}'.format(token),
'Content-Type': 'application/json'
}
resp = requests.get(url, headers=headers)
resp.raise_for_status()
state = resp.json()['state']
return state
def get_builder_plugin():
"""
Get the builder plugin name default.
If not provided by CLI opt, start with
user pref in gitconfig,
Look for hint in cirrus conf or just resort
to a guess based on what python cirrus is using
"""
# TODO look up in git config
config = load_configuration()
builder = None
if config.has_gitconfig_param('builder'):
builder = str(config.get_gitconfig_param('builder'))
if builder:
LOGGER.info("Using Builder Plugin from gitconfig: {}".format(builder))
return builder
build_config = config.get('build', {})
builder = build_config.get('builder')
if builder is not None:
LOGGER.info(
"Using Builder Plugin from cirrus.conf: {}".format(builder))
return builder
# fall back to old defaults
if is_anaconda():
LOGGER.info("Using default CondaPip builder")
builder = "CondaPip"
else:
LOGGER.info("Using default VirtualenvPip builder")
builder = "VirtualenvPip"
return builder
def main():
"""
_main_
Execute test command
"""
opts = build_parser(sys.argv[1:])
config = load_configuration()
mode = config.test_mode(opts.suite)
if opts.mode:
mode = opts.mode
# backwards compat: default to nosetests
if mode is None:
mode = 'nosetests'
if mode == 'nosetests':
nose_run(config, opts)
sys.exit(0)
if mode == 'tox':
tox_run(config, opts)
sys.exit(0)
if mode == 'pytest':
pytest_run(config, opts)
sys.exit(0)
def upload_release(opts):
"""
_upload_release_
"""
LOGGER.info("Uploading release...")
config = load_configuration()
build_artifact = artifact_name(config)
LOGGER.info("Uploading artifact: {0}".format(build_artifact))
if not os.path.exists(build_artifact):
msg = ("Expected build artifact: {0} Not Found, upload aborted\n"
"Did you run git cirrus release build?").format(build_artifact)
LOGGER.error(msg)
raise RuntimeError(msg)
# merge in release branches and tag, push to remote
tag = config.package_version()
LOGGER.info("Loading plugin {}".format(opts.plugin))
plugin = get_plugin(opts.plugin)
if opts.test:
LOGGER.info(
"Uploading {} to pypi disabled by test or option...".format(tag))
return
plugin.upload(opts, build_artifact)
return
def build_doc_artifact():
"""
build sphinx documentation and create an artifact to be uploaded to
a remote server
Requires the following cirrus.conf section:
[doc]
sphinx_makefile_dir = /path/to/makefile
sphinx_doc_dir = /path/to/_build/docdir
artifact_dir = /path/to/archive/dir
"""
LOGGER.info("Building doc archive...")
config = load_configuration()
artifact_name = doc_artifact_name(config)
arcname = os.path.basename(artifact_name).rsplit('.', 2)[0]
doc_dir = config['doc']['sphinx_doc_dir']
if not os.path.exists(doc_dir):
msg = "Documentation path: {0} Not Found".format(doc_dir)
LOGGER.error(msg)
raise RuntimeError(msg)
with tarfile.open(artifact_name, "w:gz") as tar:
tar.add(doc_dir, arcname=arcname)
LOGGER.info("Documentation artifact created at: {}".format(artifact_name))
return artifact_name
def main():
"""
_main_
provide support for some basic docker operations so that
building images can be standardised as part of a workflow
"""
opts = build_parser()
config = load_configuration()
if not config.has_section('docker'):
msg = (
"Unable to find docker section in cirrus.conf"
#TODO: Link to docs here
)
LOGGER.error(msg)
sys.exit(1)
if not is_docker_connected():
LOGGER.error(DOCKER_CONNECTION_HELP)
sys.exit(1)
if opts.command == 'build':
docker_build(opts, config)
if opts.command == 'push':
docker_push(opts, config)
if opts.command == 'test':
# Already called above
pass
def get_builder_plugin():
"""
Get the builder plugin name default.
If not provided by CLI opt, start with
user pref in gitconfig,
Look for hint in cirrus conf or just resort
to a guess based on what python cirrus is using
"""
# TODO look up in git config
config = load_configuration()
builder = None
if config.has_gitconfig_param('builder'):
builder = str(config.get_gitconfig_param('builder'))
if builder:
LOGGER.info("Using Builder Plugin from gitconfig: {}".format(builder))
return builder
build_config = config.get('build', {})
builder = build_config.get('builder')
if builder is not None:
LOGGER.info("Using Builder Plugin from cirrus.conf: {}".format(builder))
return builder
# fall back to old defaults
if is_anaconda():
LOGGER.info("Using default CondaPip builder")
builder = "CondaPip"
else:
LOGGER.info("Using default VirtualenvPip builder")
builder = "VirtualenvPip"
return builder
def branch_status(branch_name):
"""
_branch_status_
Get the branch status which should include details of CI builds/hooks etc
See:
https://developer.github.com/v3/repos/statuses/#get-the-combined-status-for-a-specific-ref
returns a state which is one of 'failure', 'pending', 'success'
"""
config = load_configuration()
token = get_github_auth()[1]
url = "https://api.github.com/repos/{org}/{repo}/commits/{branch}/status".format(
org=config.organisation_name(),
repo=config.package_name(),
branch=branch_name)
headers = {
'Authorization': 'token {0}'.format(token),
'Content-Type': 'application/json'
}
resp = requests.get(url, headers=headers)
resp.raise_for_status()
state = resp.json()['state']
return state
def main():
"""
_main_
provide support for some basic docker operations so that
building images can be standardised as part of a workflow
"""
opts = build_parser()
config = load_configuration()
if not config.has_section('docker'):
msg = (
"Unable to find docker section in cirrus.conf"
#TODO: Link to docs here
)
LOGGER.error(msg)
sys.exit(1)
if not is_docker_connected():
LOGGER.error(DOCKER_CONNECTION_HELP)
sys.exit(1)
if opts.command == 'build':
docker_build(opts, config)
if opts.command == 'push':
docker_push(opts, config)
if opts.command == 'test':
# Already called above
pass
def register_package(
package,
repository,
pypirc='~/.pypirc',
username=None,
password=None,
certfile=None,
client_certfile=None,
comment=None
):
"""
:param package: - sdist/bdist etc artifact to register
:param repository:
"""
cirrus_conf = load_configuration()
if cirrus_conf.has_section('twine'):
username = username or cirrus_conf.get_param('twine', 'username', None)
password = password or cirrus_conf.get_param('twine', 'password', None)
certfile = certfile or cirrus_conf.get_param('twine', 'certfile', None)
client_certfile = client_certfile or cirrus_conf.get_param('twine', 'client_certfile', None)
register(
package,
repository,
username,
password,
comment,
pypirc,
certfile,
client_certfile,
None
)
def build_doc_artifact():
"""
build sphinx documentation and create an artifact to be uploaded to
a remote server
Requires the following cirrus.conf section:
[doc]
sphinx_makefile_dir = /path/to/makefile
sphinx_doc_dir = /path/to/_build/docdir
artifact_dir = /path/to/archive/dir
"""
LOGGER.info("Building doc archive...")
config = load_configuration()
artifact_name = doc_artifact_name(config)
arcname = os.path.basename(artifact_name).rsplit('.', 2)[0]
doc_dir = config['doc']['sphinx_doc_dir']
if not os.path.exists(doc_dir):
msg = "Documentation path: {0} Not Found".format(doc_dir)
LOGGER.error(msg)
raise RuntimeError(msg)
with tarfile.open(artifact_name, "w:gz") as tar:
tar.add(doc_dir, arcname=arcname)
LOGGER.info("Documentation artifact created at: {}".format(artifact_name))
return artifact_name
def __init__(self, repo_dir, package_dir=None):
self.repo_dir = repo_dir
self.repo = git.Repo(repo_dir)
self.config = load_configuration(package_dir)
self.gh_user, self.token = get_github_auth()
self.auth_headers = {
'Authorization': 'token {0}'.format(self.token),
'Content-Type': 'application/json'
}
def __init__(self):
super(Linter, self).__init__()
self.config = load_configuration()
self.linter_config = self.config.get(
'qc/{}'.format(type(self).__name__, {}), {})
self.working_dir = repo_directory()
self.pass_threshold = 0
self.test_mode = False
self.errors = {}
def __init__(self, repo_dir, package_dir=None):
self.repo_dir = repo_dir
self.repo = git.Repo(repo_dir)
self.config = load_configuration(package_dir)
self.gh_user, self.token = get_github_auth()
self.auth_headers = {
'Authorization': 'token {0}'.format(self.token),
'Content-Type': 'application/json'
}
def make_new_version(opts):
LOGGER.info("Updating package version...")
if not highlander([opts.major, opts.minor, opts.micro]):
msg = "Can only specify one of --major, --minor or --micro"
LOGGER.error(msg)
raise RuntimeError(msg)
fields = ['major', 'minor', 'micro']
mask = [opts.major, opts.minor, opts.micro]
field = [x for x in itertools.compress(fields, mask)][0]
config = load_configuration()
current_version = config.package_version()
# need to be on the latest develop
repo_dir = repo_directory()
curr_branch = current_branch(repo_dir)
# make sure repo is clean
if has_unstaged_changes(repo_dir):
msg = ("Error: Unstaged changes are present on the branch {}"
"Please commit them or clean up before proceeding"
).format(curr_branch)
LOGGER.error(msg)
raise RuntimeError(msg)
# update cirrus conf
new_version = bump_version_field(current_version, field)
msg = "Bumping version from {prev} to {new} on branch {branch}".format(
prev=current_version, new=new_version, branch=curr_branch)
LOGGER.info(msg)
config.update_package_version(new_version)
changes = ['cirrus.conf']
if opts.bump:
reqs_file = os.path.join(repo_dir, 'requirements.txt')
for pkg, version in opts.bump:
LOGGER.info("Bumping dependency {} to {}".format(pkg, version))
bump_package(reqs_file, pkg, version)
changes.append(reqs_file)
# update __version__ or equivalent
version_file, version_attr = config.version_file()
if version_file is not None:
LOGGER.info('Updating {0} attribute in {1}'.format(
version_file, version_attr))
update_version(version_file, new_version, version_attr)
changes.append(version_file)
# update files changed
msg = "cirrus release: version bumped for {0}".format(curr_branch)
LOGGER.info('Committing files: {0}'.format(','.join(changes)))
LOGGER.info(msg)
commit_files_optional_push(repo_dir, msg, not opts.no_remote, *changes)
def run(self, opts):
"""
_run_
Run the plugin, calling the overloaded parser
and setup methods
"""
self.opts = opts
project_dir = opts.repo
self.config = load_configuration()
self.setup(project_dir)
def __init__(self):
super(Builder, self).__init__()
self.plugin_parser = argparse.ArgumentParser()
self.config = load_configuration()
self.build_config = self.config.get('build', {})
self.working_dir = repo_directory()
self.venv_name = self.build_config.get('virtualenv_name', 'venv')
self.reqs_name = self.build_config.get('requirements_file', 'requirements.txt')
self.extra_reqs = self.build_config.get('extra_requirements', [])
self.python_bin = self.build_config.get('python', None)
self.extra_reqs = self.str_to_list(self.extra_reqs)
self.venv_path = os.path.join(self.working_dir, self.venv_name)
def __init__(self):
super(Builder, self).__init__()
self.plugin_parser = argparse.ArgumentParser()
self.config = load_configuration()
self.build_config = self.config.get('build', {})
self.working_dir = repo_directory()
self.venv_name = self.build_config.get('virtualenv_name', 'venv')
self.reqs_name = self.build_config.get('requirements_file', 'requirements.txt')
self.extra_reqs = self.build_config.get('extra_requirements', [])
self.python_bin = self.build_config.get('python', None)
self.extra_reqs = self.str_to_list(self.extra_reqs)
self.venv_path = os.path.join(self.working_dir, self.venv_name)
def current_branch_mark_status(repo_dir, state):
"""
_current_branch_mark_status_
Mark the CI status of the current branch.
:param repo_dir: directory of git repository
:param state: state of the last test run, such as "success" or "failure"
"""
LOGGER.info(u"Setting CI status for current branch to {}".format(state))
config = load_configuration()
token = get_github_auth()[1]
sha = git.Repo(repo_dir).head.commit.hexsha
try:
# @HACK: Do a push that we expect will fail -- we just want to
# tell the server about our sha. A more elegant solution would
# probably be to push a detached head.
push(repo_dir)
except RuntimeError as ex:
if "rejected" not in unicode_(ex):
raise
url = "https://api.github.com/repos/{org}/{repo}/statuses/{sha}".format(
org=config.organisation_name(),
repo=config.package_name(),
sha=sha
)
headers = {
'Authorization': 'token {0}'.format(token),
'Content-Type': 'application/json'
}
data = json.dumps(
{
"state": state,
"description": "State after cirrus check.",
# @HACK: use the travis context, which is technically
# true, because we wait for Travis tests to pass before
# cutting a release. In the future, we need to setup a
# "cirrus" context, for clarity.
"context": "continuous-integration/travis-ci"
}
)
resp = requests.post(url, headers=headers, data=data)
resp.raise_for_status()
def create_pull_request(
repo_dir,
pr_info,
token=None):
"""
Creates a pull_request on GitHub and returns the html url of the
pull request created
:param repo_dir: directory of git repository
:param pr_info: dictionary containing title and body of pull request
:param token: auth token
"""
if repo_dir is None:
raise RuntimeError('repo_dir is None')
if 'title' not in pr_info:
raise RuntimeError('title is None')
if 'body' not in pr_info:
raise RuntimeError('body is None')
config = load_configuration()
url = 'https://api.github.com/repos/{0}/{1}/pulls'.format(
config.organisation_name(),
config.package_name())
if token is None:
token = get_github_auth()[1]
headers = {
'Authorization': 'token {0}'.format(token),
'Content-Type': 'application/json'}
data = {
'title': pr_info['title'],
'head': get_active_branch(repo_dir).name,
'base': config.gitflow_branch_name(),
'body': pr_info['body']}
resp = requests.post(url, data=json.dumps(data), headers=headers)
if resp.status_code == 422:
LOGGER.error(
(
"POST to GitHub api returned {0}"
"Have you committed your changes and pushed to remote?"
).format(resp.status_code)
)
resp.raise_for_status()
resp_json = resp.json()
return resp_json['html_url']
def current_branch_mark_status(repo_dir, state):
"""
_current_branch_mark_status_
Mark the CI status of the current branch.
:param repo_dir: directory of git repository
:param state: state of the last test run, such as "success" or "failure"
"""
LOGGER.info(u"Setting CI status for current branch to {}".format(state))
config = load_configuration()
token = get_github_auth()[1]
sha = git.Repo(repo_dir).head.commit.hexsha
try:
# @HACK: Do a push that we expect will fail -- we just want to
# tell the server about our sha. A more elegant solution would
# probably be to push a detached head.
push(repo_dir)
except RuntimeError as ex:
if "rejected" not in unicode_(ex):
raise
url = "https://api.github.com/repos/{org}/{repo}/statuses/{sha}".format(
org=config.organisation_name(),
repo=config.package_name(),
sha=sha
)
headers = {
'Authorization': 'token {0}'.format(token),
'Content-Type': 'application/json'
}
data = json.dumps(
{
"state": state,
"description": "State after cirrus check.",
# @HACK: use the travis context, which is technically
# true, because we wait for Travis tests to pass before
# cutting a release. In the future, we need to setup a
# "cirrus" context, for clarity.
"context": "continuous-integration/travis-ci"
}
)
resp = requests.post(url, headers=headers, data=data)
resp.raise_for_status()
def create_pull_request(
repo_dir,
pr_info,
token=None):
"""
Creates a pull_request on GitHub and returns the html url of the
pull request created
:param repo_dir: directory of git repository
:param pr_info: dictionary containing title and body of pull request
:param token: auth token
"""
if repo_dir is None:
raise RuntimeError('repo_dir is None')
if 'title' not in pr_info:
raise RuntimeError('title is None')
if 'body' not in pr_info:
raise RuntimeError('body is None')
config = load_configuration()
url = 'https://api.github.com/repos/{0}/{1}/pulls'.format(
config.organisation_name(),
config.package_name())
if token is None:
token = get_github_auth()[1]
headers = {
'Authorization': 'token {0}'.format(token),
'Content-Type': 'application/json'}
data = {
'title': pr_info['title'],
'head': get_active_branch(repo_dir).name,
'base': config.gitflow_branch_name(),
'body': pr_info['body']}
resp = requests.post(url, data=json.dumps(data), headers=headers)
if resp.status_code == 422:
LOGGER.error(
(
"POST to GitHub api returned {0}"
"Have you committed your changes and pushed to remote?"
).format(resp.status_code)
)
resp.raise_for_status()
resp_json = resp.json()
return resp_json['html_url']
def publish_documentation(opts):
"""
Publish Sphinx documentation.
If a tarfile exists for the current program version, that file will
be used. Otherwise, the documentation will be built and packaged
before uploading.
Requires the publish_method plugin to be specified in the cirrus.conf [doc]
section, and a section containing the required fields for the publisher
method, e.g:
[doc]
publisher = file_server
:param argparse.Namspace opts: A Namespace of publisher options
"""
LOGGER.info("Preparing to upload documentation...")
config = load_configuration()
doc_params = config.get('doc', {})
doc_artifact = doc_artifact_name(config)
if not os.path.exists(doc_artifact):
msg = 'Documentation tarball not found at {}'.format(doc_artifact)
LOGGER.error(msg)
raise RuntimeError(msg)
try:
publisher = doc_params['publisher']
except KeyError:
LOGGER.error(
'Did not find a publisher in [doc] section of cirrus.conf'
'\nSee below for an example:'
'\n[doc]'
'\npublisher = doc_file_server')
sys.exit(1)
plugin = get_publisher_plugin(publisher)
if opts.test:
LOGGER.info(
"Uploading {}-{}.tar.gz to file server disabled by test or "
"option...".format(
config.package_name(), config.package_version()
)
)
return
plugin.publish(doc_artifact)
return
def publish_documentation(opts):
"""
Publish Sphinx documentation.
If a tarfile exists for the current program version, that file will
be used. Otherwise, the documentation will be built and packaged
before uploading.
Requires the publish_method plugin to be specified in the cirrus.conf [doc]
section, and a section containing the required fields for the publisher
method, e.g:
[doc]
publisher = file_server
:param argparse.Namspace opts: A Namespace of publisher options
"""
LOGGER.info("Preparing to upload documentation...")
config = load_configuration()
doc_params = config.get('doc', {})
doc_artifact = doc_artifact_name(config)
if not os.path.exists(doc_artifact):
msg = 'Documentation tarball not found at {}'.format(doc_artifact)
LOGGER.error(msg)
raise RuntimeError(msg)
try:
publisher = doc_params['publisher']
except KeyError:
LOGGER.error(
'Did not find a publisher in [doc] section of cirrus.conf'
'\nSee below for an example:'
'\n[doc]'
'\npublisher = doc_file_server')
sys.exit(1)
plugin = get_publisher_plugin(publisher)
if opts.test:
LOGGER.info(
"Uploading {}-{}.tar.gz to file server disabled by test or "
"option...".format(
config.package_name(), config.package_version()
)
)
return
plugin.publish(doc_artifact)
return
def test_configuration_map(self):
"""test building config mapping"""
config = load_configuration(package_dir=self.dir, gitconfig_file=self.gitconfig)
mapping = config.configuration_map()
self.failUnless('cirrus' in mapping)
self.failUnless('credentials' in mapping['cirrus'])
self.failUnless('configuration' in mapping['cirrus'])
self.failUnless('github_credentials' in mapping['cirrus']['credentials'])
self.assertEqual(
mapping['cirrus']['credentials']['github_credentials'],
{'github_user': None, 'github_token': None}
)
self.assertEqual(
mapping['cirrus']['configuration']['package']['name'], 'cirrus_tests'
)
def remove_nightly(ghc):
"""
remove the nightly part from the cirrus.conf version
"""
cirrus_conf = load_configuration()
nightly_conf = nightly_config(cirrus_conf)
current = cirrus_conf.package_version()
if is_nightly(current):
new_version = current.split(nightly_conf['nightly_separator'], 1)[0]
cirrus_conf.update_package_version(new_version)
ghc.commit_files_optional_push(
"remove nightly tag from cirrus.conf",
False,
"cirrus.conf"
)
return
def test_configuration_map(self):
"""test building config mapping"""
config = load_configuration(package_dir=self.dir,
gitconfig_file=self.gitconfig)
mapping = config.configuration_map()
self.failUnless('cirrus' in mapping)
self.failUnless('credentials' in mapping['cirrus'])
self.failUnless('configuration' in mapping['cirrus'])
self.failUnless(
'github_credentials' in mapping['cirrus']['credentials'])
self.assertEqual(
mapping['cirrus']['credentials']['github_credentials'], {
'github_user': None,
'github_token': None
})
self.assertEqual(mapping['cirrus']['configuration']['package']['name'],
'cirrus_tests')
def nightly_config(conf=None):
"""
get the nightly config settings from
the release section
"""
result = {
"nightly_format": DEFAULT_FORMAT,
"nightly_separator": "-nightly-"
}
if not conf:
conf = load_configuration()
if not conf.has_section('release'):
return result
result['nightly_format'] = conf.get_param("release", "nightly_format", result['nightly_format'])
result['nightly_separator'] = conf.get_param("release", "nightly_separator", result['nightly_separator'])
return result
def build_release(opts):
"""
_build_release_
run python setup.py sdist to create the release artifact
"""
LOGGER.info("Building release...")
config = load_configuration()
local('python setup.py sdist')
build_artifact = artifact_name(config)
if not os.path.exists(build_artifact):
msg = "Expected build artifact: {0} Not Found".format(build_artifact)
LOGGER.error(msg)
raise RuntimeError(msg)
LOGGER.info("Release artifact created: {0}".format(build_artifact))
return build_artifact
def new_nightly():
"""
generate a new nightly version
"""
cirrus_conf = load_configuration()
nightly_conf = nightly_config(cirrus_conf)
now = datetime.datetime.now()
ts = now.strftime(nightly_conf['nightly_format'])
current = cirrus_conf.package_version()
nightly = "{version}{sep}{ts}".format(
version=current,
sep=nightly_conf['nightly_separator'],
ts=ts
)
return nightly
def test_reading_missing(self, mock_pop, mock_shell):
"""test config load using repo dir"""
mock_result = mock.Mock()
mock_result.communicate = mock.Mock()
mock_result.returncode = 0
mock_result.communicate.return_value = (self.dir, None)
mock_pop.return_value = mock_result
mock_shell.return_value = self.gitconf_str
config = load_configuration(package_dir="womp")
self.failUnless(mock_result.communicate.called)
mock_pop.assert_has_calls(
mock.call(['git', 'rev-parse', '--show-toplevel'], stdout=-1))
self.assertEqual(config.package_version(), '1.2.3')
self.assertEqual(config.package_name(), 'cirrus_tests')
self.failUnless(mock_shell.called)
mock_shell.assert_has_calls(
[mock.call('git config --file {} -l'.format(self.gitconfig))])
def test_reading_missing(self, mock_pop, mock_shell):
"""test config load using repo dir"""
mock_result = mock.Mock()
mock_result.communicate = mock.Mock()
mock_result.returncode = 0
mock_result.communicate.return_value = (self.dir, None)
mock_pop.return_value = mock_result
mock_shell.return_value = self.gitconf_str
config = load_configuration(package_dir="womp")
self.failUnless(mock_result.communicate.called)
mock_pop.assert_has_calls(mock.call(['git', 'rev-parse', '--show-toplevel'], stdout=-1))
self.assertEqual(config.package_version(), '1.2.3')
self.assertEqual(config.package_name(), 'cirrus_tests')
self.failUnless(mock_shell.called)
mock_shell.assert_has_calls([
mock.call('git config --file {} -l'.format(self.gitconfig))
])
def new_feature_branch(opts):
"""
_new_feature_branch_
Checks out branch, creates new branch 'name', optionally
pushes new branch to remote
"""
config = load_configuration()
repo_dir = os.getcwd()
checkout_and_pull(repo_dir, config.gitflow_branch_name())
LOGGER.info("Checked out and pulled {0}".format(
config.gitflow_branch_name()))
branch_name = ''.join((config.gitflow_feature_prefix(), opts.name[0]))
branch(repo_dir, branch_name, config.gitflow_branch_name())
LOGGER.info("Created branch {0}".format(branch_name))
if opts.push:
push(repo_dir)
LOGGER.info("Branch {0} pushed to remote".format(branch_name))
def upload_package(
package,
repository,
pypirc='~/.pypirc',
sign=False,
identity=None,
username=None,
password=None,
comment=None,
sign_with=None,
skip_existing=False,
certfile=None,
client_certfile=None
):
"""
:param package: path to sdist package
:param repository: repo name to upload to
:param pypirc: path to pypirc file
"""
cirrus_conf = load_configuration()
if cirrus_conf.has_section('twine'):
username = username or cirrus_conf.get_param('twine', 'username', None)
password = password or cirrus_conf.get_param('twine', 'password', None)
certfile = certfile or cirrus_conf.get_param('twine', 'certfile', None)
identity = identity or cirrus_conf.get_param('twine', 'identity', None)
sign_with = sign_with or cirrus_conf.get_param('twine', 'sign_with', None)
client_certfile = client_certfile or cirrus_conf.get_param('twine', 'client_certfile', None)
upload(
[package],
repository,
sign,
identity,
username,
password,
comment,
sign_with,
pypirc,
skip_existing,
certfile,
client_certfile,
None
)
def main():
"""
_main_
Execute prestage command:
- look into cirrus conf for requirements to be prestaged
- look for matching version requirement in requirements.txt
- clone each repo locally
- pip install -e the repo into the virtualenv
"""
working_dir = os.getcwd()
repo_cache = os.path.join(working_dir, 'prestage')
if not os.path.exists(repo_cache):
os.makedirs(repo_cache)
config = load_configuration()
prestage_params = config.get('prestage', {})
build_params = config.get('build', {})
venv_name = build_params.get('virtualenv_name', 'venv')
reqs_name = build_params.get('requirements_file', 'requirements.txt')
venv_path = os.path.join(working_dir, venv_name)
venv_command = os.path.join(venv_path, 'bin', 'activate')
reqs = {}
# TODO: this only supports explicit == version reqs right now
# make this more flexible
for req in parse_requirements(reqs_name):
if req.req is not None:
versions = [ x for x in req.absolute_versions]
if len(versions) > 0:
reqs[req.name] = versions[0]
# prestage section contains a map of package name: repo
for req, repo in prestage_params.iteritems():
msg = "Prestaging repo for requirement {req} from {repo}"
tag = reqs.get(req)
if tag is not None:
msg += " with tag {tag}"
msg.format(req=req, repo=repo, tag=tag)
print msg
local_repo = os.path.join(repo_cache, req)
git_clone_repo(repo, local_repo, tag=reqs.get(req))
install_from_repo(venv_command, local_repo)
def main():
"""
_main_
Execute prestage command:
- look into cirrus conf for requirements to be prestaged
- look for matching version requirement in requirements.txt
- clone each repo locally
- pip install -e the repo into the virtualenv
"""
working_dir = os.getcwd()
repo_cache = os.path.join(working_dir, 'prestage')
if not os.path.exists(repo_cache):
os.makedirs(repo_cache)
config = load_configuration()
prestage_params = config.get('prestage', {})
build_params = config.get('build', {})
venv_name = build_params.get('virtualenv_name', 'venv')
reqs_name = build_params.get('requirements_file', 'requirements.txt')
venv_path = os.path.join(working_dir, venv_name)
venv_command = os.path.join(venv_path, 'bin', 'activate')
reqs = {}
# TODO: this only supports explicit == version reqs right now
# make this more flexible
for req in parse_requirements(reqs_name):
if req.req is not None:
versions = [x for x in req.absolute_versions]
if len(versions) > 0:
reqs[req.name] = versions[0]
# prestage section contains a map of package name: repo
for req, repo in prestage_params.iteritems():
msg = "Prestaging repo for requirement {req} from {repo}"
tag = reqs.get(req)
if tag is not None:
msg += " with tag {tag}"
msg.format(req=req, repo=repo, tag=tag)
print msg
local_repo = os.path.join(repo_cache, req)
git_clone_repo(repo, local_repo, tag=reqs.get(req))
install_from_repo(venv_command, local_repo)
def main():
"""
_main_
Execute test command
"""
cirrus_conf = load_configuration()
qc_conf = cirrus_conf.quality_control()
opts = build_parser(sys.argv, qc_conf)
if opts.only_changes:
files = get_diff_files(None)
# we only want python modules
for item in files[:]:
if not item.endswith('.py'):
files.remove(item)
if not files:
LOGGER.info("No modules have been changed.")
sys.exit(0)
opts.include_files = files
run_linters(opts, cirrus_conf, qc_conf)
def main():
"""
_main_
Execute test command
"""
cirrus_conf = load_configuration()
qc_conf = cirrus_conf.quality_control()
opts = build_parser(sys.argv, qc_conf)
if opts.only_changes:
files = get_diff_files(None)
# we only want python modules
for item in files[:]:
if not item.endswith('.py'):
files.remove(item)
if not files:
LOGGER.info("No modules have been changed.")
sys.exit(0)
opts.include_files = files
run_linters(opts, cirrus_conf, qc_conf)
def cleanup_release(opts):
"""
_cleanup_release_
Remove local and remote release branches if they exist
"""
config = load_configuration()
repo_dir = os.getcwd()
pfix = config.gitflow_release_prefix()
branch_name = release_branch_name(config)
if opts.version is not None:
if not opts.version.startswith(pfix):
branch_name = "{0}{1}".format(pfix, opts.version)
else:
branch_name = opts.version
LOGGER.info("Cleaning release branches for {}".format(branch_name))
with GitHubContext(repo_dir) as ghc:
ghc.delete_branch(branch_name, not opts.no_remote)
def legacy_update(opts):
"""update repo installed cirrus"""
install = find_cirrus_install()
with chdir(install):
config = load_configuration()
if opts.branch and opts.version:
msg = "Can specify branch -OR- version, not both"
raise RuntimeError(msg)
if opts.branch is not None:
update_to_branch(opts.branch, config)
setup_develop(config)
return
if opts.version is not None:
tag = opts.version
else:
tag = latest_release(config)
LOGGER.info("Retrieved latest tag: {0}".format(tag))
update_to_tag(tag, config)
setup_develop(config)
def build_docs(make_opts=None):
"""
_build_docs_
Runs 'make' against a Sphinx makefile.
Requires the following cirrus.conf section:
[doc]
sphinx_makefile_dir = /path/to/makefile
:param list make_opts: Options to run the Sphinx 'make' command gathered
from the calling command's --docs option. If None or an empty list,
'make' will be run using 'clean html'
"""
LOGGER.info('Building docs')
config = load_configuration()
build_params = config.get('build', {})
venv_name = build_params.get('virtualenv_name', 'venv')
try:
docs_root = os.path.join(
os.getcwd(),
config['doc']['sphinx_makefile_dir'])
except KeyError:
LOGGER.error(
'Did not find a complete [doc] section in cirrus.conf'
'\nSee below for an example:'
'\n[doc]'
'\nsphinx_makefile_dir = /path/to/sphinx')
sys.exit(1)
cmd = 'cd {} && make clean html'.format(docs_root)
if make_opts:
# additional args were passed after --docs. Pass these to make
cmd = 'cd {} && make {}'.format(docs_root, ' '.join(make_opts))
local('. ./{}/bin/activate && {}'.format(venv_name, cmd))
LOGGER.info('Build command was "{}"'.format(cmd))
def build_docs(make_opts=None):
"""
_build_docs_
Runs 'make' against a Sphinx makefile.
Requires the following cirrus.conf section:
[doc]
sphinx_makefile_dir = /path/to/makefile
:param list make_opts: Options to run the Sphinx 'make' command gathered
from the calling command's --docs option. If None or an empty list,
'make' will be run using 'clean html'
"""
LOGGER.info('Building docs')
config = load_configuration()
build_params = config.get('build', {})
venv_name = build_params.get('virtualenv_name', 'venv')
try:
docs_root = os.path.join(
os.getcwd(),
config['doc']['sphinx_makefile_dir'])
except KeyError:
LOGGER.error(
'Did not find a complete [doc] section in cirrus.conf'
'\nSee below for an example:'
'\n[doc]'
'\nsphinx_makefile_dir = /path/to/sphinx')
sys.exit(1)
cmd = 'cd {} && make clean html'.format(docs_root)
if make_opts:
# additional args were passed after --docs. Pass these to make
cmd = 'cd {} && make {}'.format(docs_root, ' '.join(make_opts))
local('. ./{}/bin/activate && {}'.format(venv_name, cmd))
LOGGER.info('Build command was "{}"'.format(cmd))
def legacy_update(opts):
"""update repo installed cirrus"""
install = find_cirrus_install()
with chdir(install):
config = load_configuration()
if opts.branch and opts.version:
msg = "Can specify branch -OR- version, not both"
raise RuntimeError(msg)
if opts.branch is not None:
update_to_branch(opts.branch, config)
setup_develop(config)
return
if opts.version is not None:
tag = opts.version
else:
tag = latest_release(config)
LOGGER.info("Retrieved latest tag: {0}".format(tag))
update_to_tag(tag, config)
setup_develop(config)
def trigger_release(opts):
"""
_trigger_release_
Alias for "git cirrus release new --micro/minor/major.
- Run the "release new" command
- Capture the new version string
- Pass new version number to external build server
Requires the following sections and values in cirrus.conf:
[build-server]
name = jenkins
[jenkins]
url = http://localhost:8080
job = default
"""
config = load_configuration()
try:
build_server = config['build-server']['name']
build_server_config = config[build_server]
except KeyError:
msg = (
'[build-server] section is incomplete or missing from cirrus.conf. '
'Please see below for an example.\n'
'\n [build-server]'
'\n name = jenkins'
'\n [jenkins]'
'\n url = http://localhost:8080'
'\n job = default')
raise RuntimeError(msg)
new_version, release_level = new_release(opts)
if build_server == 'jenkins':
_trigger_jenkins_release(build_server_config, new_version,
release_level)
def merge_feature_branch(opts):
"""
merge current feature branch into develop
"""
config = load_configuration()
main_branch = config.gitflow_branch_name()
repo_dir = repo_directory()
curr_branch = current_branch(repo_dir)
LOGGER.info("Merging {} into {}".format(curr_branch, main_branch))
# make sure repo is clean
if has_unstaged_changes(repo_dir):
msg = ("Error: Unstaged changes are present on the feature branch {}"
"Please commit them or clean up before proceeding"
).format(curr_branch)
LOGGER.error(msg)
raise RuntimeError(msg)
checkout_and_pull(repo_dir, main_branch, pull=not opts.no_remote)
with GitHubContext(repo_dir) as ghc:
ghc.merge_branch(curr_branch)
if not opts.no_remote:
ghc.push_branch(main_branch)
LOGGER.info("Branch {0} pushed to remote".format(main_branch))
def main():
"""
_main_
Look up the plugin to be invoked for deployment, via the CLI or cirrus config
for this package, and then invoke the deployer plugin
"""
initial_opts = build_parser()
pname = initial_opts.plugin
if initial_opts.plugin is None:
config = load_configuration()
try:
pname = config.get_param('deploy', 'plugin', None)
except KeyError:
pname = None
if pname is None:
msg = "No Plugin specified with --plugin or in cirrus.conf for deploy"
raise RuntimeError(msg)
plugin = get_plugin(pname)
plugin.build_parser()
opts = plugin.parser.parse_args()
plugin.deploy(opts)
def main():
"""
_main_
Look up the plugin to be invoked for deployment, via the CLI or cirrus config
for this package, and then invoke the deployer plugin
"""
initial_opts = build_parser()
pname = initial_opts.plugin
if initial_opts.plugin is None:
config = load_configuration()
try:
pname = config.get_param('deploy', 'plugin', None)
except KeyError:
pname = None
if pname is None:
msg = "No Plugin specified with --plugin or in cirrus.conf for deploy"
raise RuntimeError(msg)
plugin = get_plugin(pname)
plugin.build_parser()
opts = plugin.parser.parse_args()
plugin.deploy(opts)
def merge_feature_branch(opts):
"""
merge current feature branch into develop
"""
config = load_configuration()
main_branch = config.gitflow_branch_name()
repo_dir = repo_directory()
curr_branch = current_branch(repo_dir)
LOGGER.info("Merging {} into {}".format(curr_branch, main_branch))
# make sure repo is clean
if has_unstaged_changes(repo_dir):
msg = (
"Error: Unstaged changes are present on the feature branch {}"
"Please commit them or clean up before proceeding"
).format(curr_branch)
LOGGER.error(msg)
raise RuntimeError(msg)
checkout_and_pull(repo_dir, main_branch, pull=not opts.no_remote)
with GitHubContext(repo_dir) as ghc:
ghc.merge_branch(curr_branch)
if not opts.no_remote:
ghc.push_branch(main_branch)
LOGGER.info("Branch {0} pushed to remote".format(main_branch))
def new_feature_branch(opts):
"""
_new_feature_branch_
Checks out branch, creates new branch 'name', optionally
pushes new branch to remote
"""
config = load_configuration()
repo_dir = repo_directory()
checkout_and_pull(
repo_dir,
config.gitflow_branch_name(),
pull=not opts.no_remote
)
LOGGER.info("Checked out and pulled {0}".format(
config.gitflow_branch_name()))
branch_name = ''.join((config.gitflow_feature_prefix(), opts.name[0]))
branch(repo_dir,
branch_name,
config.gitflow_branch_name())
LOGGER.info("Created branch {0}".format(branch_name))
if not opts.no_remote:
push(repo_dir)
LOGGER.info("Branch {0} pushed to remote".format(branch_name))
def execute_build(opts):
"""
_execute_build_
Execute the build in the current package context.
- reads the config to check for custom build parameters
- defaults to ./venv for virtualenv
- defaults to ./requirements.txt for reqs
- removes existing virtualenv if clean flag is set
- builds the virtualenv
- pip installs the requirements into it
: param argparse.Namspace opts: A Namespace of build options
"""
working_dir = repo_directory()
config = load_configuration()
build_params = config.get('build', {})
# we have custom build controls in the cirrus.conf
venv_name = build_params.get('virtualenv_name', 'venv')
reqs_name = build_params.get('requirements_file', 'requirements.txt')
extra_reqs = build_params.get('extra_requirements', '')
python_bin = build_params.get('python', None)
if opts.python:
python_bin = opts.python
extra_reqs = [x.strip() for x in extra_reqs.split(',') if x.strip()]
if opts.extras:
extra_reqs.extend(opts.extras)
extra_reqs = set(extra_reqs) # dedupe
venv_path = os.path.join(working_dir, venv_name)
if is_anaconda():
conda_venv(venv_path, opts, python_bin)
else:
virtualenv_venv(venv_path, opts, python_bin)
# custom pypi server
pypi_server = config.pypi_url()
pip_options = config.pip_options()
pip_command_base = None
if pypi_server is not None:
pypirc = PypircFile()
if pypi_server in pypirc.index_servers:
pypi_url = pypirc.get_pypi_url(pypi_server)
else:
pypi_conf = get_pypi_auth()
pypi_url = (
"https://{pypi_username}:{pypi_token}@{pypi_server}/simple"
).format(pypi_token=pypi_conf['token'],
pypi_username=pypi_conf['username'],
pypi_server=pypi_server)
pip_command_base = ('{0}/bin/pip install -i {1}').format(
venv_path, pypi_url)
if opts.upgrade:
cmd = ('{0} --upgrade '
'-r {1}').format(pip_command_base, reqs_name)
else:
cmd = ('{0} ' '-r {1}').format(pip_command_base, reqs_name)
else:
pip_command_base = '{0}/bin/pip install'.format(venv_path)
# no pypi server
if opts.upgrade:
cmd = '{0} --upgrade -r {1}'.format(pip_command_base, reqs_name)
else:
cmd = '{0} -r {1}'.format(pip_command_base, reqs_name)
if pip_options:
cmd += " {} ".format(pip_options)
try:
local(cmd)
except OSError as ex:
msg = ("Error running pip install command during build\n"
"Error was {0}\n"
"Running command: {1}\n"
"Working Dir: {2}\n"
"Virtualenv: {3}\n"
"Requirements: {4}\n").format(ex, cmd, working_dir, venv_path,
reqs_name)
LOGGER.error(msg)
sys.exit(1)
if extra_reqs:
if opts.upgrade:
commands = [
"{0} --upgrade -r {1}".format(pip_command_base, reqfile)
for reqfile in extra_reqs
]
else:
commands = [
"{0} -r {1}".format(pip_command_base, reqfile)
for reqfile in extra_reqs
]
for cmd in commands:
LOGGER.info("Installing extra requirements... {}".format(cmd))
try:
local(cmd)
except OSError as ex:
msg = ("Error running pip install command extra "
"requirements install: {}\n{}").format(reqfile, ex)
LOGGER.error(msg)
sys.exit(1)
# setup for development
if opts.nosetupdevelop:
msg = "skipping python setup.py develop..."
LOGGER.info(msg)
else:
LOGGER.info('running python setup.py develop...')
activate = activate_command(venv_path)
local('{} && python setup.py develop'.format(activate))
def __init__(self):
super(Publisher, self).__init__()
self.package_conf = load_configuration()
def __init__(self):
super(Deployer, self).__init__()
self.package_conf = load_configuration()
self.parser = argparse.ArgumentParser()
self.parser.add_argument('--plugin', '-p', dest='plugin', default=None)
def init_container(opts):
"""
Initialise a basic container-template setup
for this package
"""
cirrus_conf = os.path.join(opts.repo, 'cirrus.conf')
if not os.path.exists(cirrus_conf):
msg = "No cirrus.conf found, need to init repo first?"
LOGGER.error(msg)
sys.exit(1)
config = load_configuration(opts.repo)
template_dir = os.path.join(opts.repo, opts.template_dir)
if not os.path.exists(template_dir):
LOGGER.info("Creating Template in {}".format(template_dir))
os.makedirs(template_dir)
docker_file = os.path.join(template_dir, 'Dockerfile.mustache')
dotfile = os.path.join(template_dir, '.dockerstache')
pre_script = os.path.join(template_dir, 'pre_script.sh')
post_script = os.path.join(template_dir, 'post_script.sh')
context = os.path.join(template_dir, 'context.json')
installer_script = os.path.join(template_dir, 'install_script.sh.mustache')
opts.context_file = os.path.join(opts.template_dir, 'context.json')
# make sure repo is clean
if has_unstaged_changes(opts.repo):
msg = (
"Error: Unstaged changes are present on the branch "
"Please commit them or clean up before proceeding"
)
LOGGER.error(msg)
raise RuntimeError(msg)
main_branch = config.gitflow_branch_name()
LOGGER.info("checking out latest {} branch...".format(main_branch))
checkout_and_pull(opts.repo, main_branch, not opts.no_remote)
venv_option = ""
if opts.virtualenv:
venv_option = ". {}/bin/activate".format(opts.virtualenv)
with working_dir(template_dir):
write_basic_dockerfile(opts, config, docker_file)
write_json_file(dotfile, {
"post_script": "post_script.sh",
"pre_script": "pre_script.sh",
"inclusive": True,
"excludes": ["post_script.sh", "post_script.sh", ".dockerstache"]
})
write_json_file(context, {})
# render templates for container scripts
template_context = {
'open_brace': '{{',
'close_brace': '}}',
'package': config.package_name(),
'virtualenv': venv_option,
'pip_options': config.pip_options() if config.pip_options() else ""
}
install_template = find_template('install_script.sh.mustache')
pre_template = find_template('pre_script.sh.mustache')
post_template = find_template('post_script.sh.mustache')
renderer = pystache.Renderer()
install_result = renderer.render_path(install_template, template_context)
with open(installer_script, 'w') as handle:
handle.write(install_result)
post_result = renderer.render_path(post_template, template_context)
with open(post_script, 'w') as handle:
handle.write(post_result)
pre_result = renderer.render_path(pre_template, template_context)
with open(pre_script, 'w') as handle:
handle.write(pre_result)
make_executable(pre_script, opts.repo)
make_executable(post_script, opts.repo)
make_executable(installer_script, opts.repo)
edit_cirrus_conf(opts, config)
modified = [
cirrus_conf,
docker_file,
dotfile,
pre_script,
post_script,
installer_script,
context
]
LOGGER.info("commiting changes...")
commit_files_optional_push(
opts.repo,
"git cirrus package container-init",
not opts.no_remote,
*modified
)
