def test_reading(self, mock_shell): """test config load """ # test read and accessors mock_shell.return_value = self.gitconf_str config = load_configuration(package_dir=self.dir, gitconfig_file=self.gitconfig) self.assertEqual(config.package_version(), '1.2.3') self.assertEqual(config.package_name(), 'cirrus_tests') self.assertEqual(config.gitflow_branch_name(), 'develop') self.assertEqual(config.gitflow_release_prefix(), 'release/') self.assertEqual(config.gitflow_feature_prefix(), 'feature/') self.assertEqual(config.release_notes(), (None, None)) self.assertEqual(config.version_file(), (None, '__version__')) self.assertEqual(config.extras_require(), { 'analysis': 'pandas;scipy', 'server': 'Flask==0.0.0' }) self.failUnless(config.credentials is not None) self.failUnless(isinstance(config.credentials, Default)) # test updating version config.update_package_version('1.2.4') self.assertEqual(config.package_version(), '1.2.4') config2 = load_configuration(package_dir=self.dir) self.assertEqual(config2.package_version(), '1.2.4')
def test_reading(self, mock_shell): """test config load """ # test read and accessors mock_shell.return_value = self.gitconf_str config = load_configuration( package_dir=self.dir, gitconfig_file=self.gitconfig ) self.assertEqual(config.package_version(), '1.2.3') self.assertEqual(config.package_name(), 'cirrus_tests') self.assertEqual(config.gitflow_branch_name(), 'develop') self.assertEqual(config.gitflow_release_prefix(), 'release/') self.assertEqual(config.gitflow_feature_prefix(), 'feature/') self.assertEqual(config.release_notes(), (None, None)) self.assertEqual(config.version_file(), (None, '__version__')) self.assertEqual( config.extras_require(), {'analysis': 'pandas;scipy', 'server': 'Flask==0.0.0'} ) self.failUnless(config.credentials is not None) self.failUnless(isinstance(config.credentials, Default)) # test updating version config.update_package_version('1.2.4') self.assertEqual(config.package_version(), '1.2.4') config2 = load_configuration(package_dir=self.dir) self.assertEqual(config2.package_version(), '1.2.4')
def branch_status(branch_name): """ _branch_status_ Get the branch status which should include details of CI builds/hooks etc See: https://developer.github.com/v3/repos/statuses/#get-the-combined-status-for-a-specific-ref returns a state which is one of 'failure', 'pending', 'success' """ config = load_configuration() token = get_github_auth()[1] url = "https://api.github.com/repos/{org}/{repo}/commits/{branch}/status".format( org=config.organisation_name(), repo=config.package_name(), branch=branch_name ) headers = { 'Authorization': 'token {0}'.format(token), 'Content-Type': 'application/json' } resp = requests.get(url, headers=headers) resp.raise_for_status() state = resp.json()['state'] return state
def get_builder_plugin(): """ Get the builder plugin name default. If not provided by CLI opt, start with user pref in gitconfig, Look for hint in cirrus conf or just resort to a guess based on what python cirrus is using """ # TODO look up in git config config = load_configuration() builder = None if config.has_gitconfig_param('builder'): builder = str(config.get_gitconfig_param('builder')) if builder: LOGGER.info("Using Builder Plugin from gitconfig: {}".format(builder)) return builder build_config = config.get('build', {}) builder = build_config.get('builder') if builder is not None: LOGGER.info( "Using Builder Plugin from cirrus.conf: {}".format(builder)) return builder # fall back to old defaults if is_anaconda(): LOGGER.info("Using default CondaPip builder") builder = "CondaPip" else: LOGGER.info("Using default VirtualenvPip builder") builder = "VirtualenvPip" return builder
def main(): """ _main_ Execute test command """ opts = build_parser(sys.argv[1:]) config = load_configuration() mode = config.test_mode(opts.suite) if opts.mode: mode = opts.mode # backwards compat: default to nosetests if mode is None: mode = 'nosetests' if mode == 'nosetests': nose_run(config, opts) sys.exit(0) if mode == 'tox': tox_run(config, opts) sys.exit(0) if mode == 'pytest': pytest_run(config, opts) sys.exit(0)
def upload_release(opts): """ _upload_release_ """ LOGGER.info("Uploading release...") config = load_configuration() build_artifact = artifact_name(config) LOGGER.info("Uploading artifact: {0}".format(build_artifact)) if not os.path.exists(build_artifact): msg = ("Expected build artifact: {0} Not Found, upload aborted\n" "Did you run git cirrus release build?").format(build_artifact) LOGGER.error(msg) raise RuntimeError(msg) # merge in release branches and tag, push to remote tag = config.package_version() LOGGER.info("Loading plugin {}".format(opts.plugin)) plugin = get_plugin(opts.plugin) if opts.test: LOGGER.info( "Uploading {} to pypi disabled by test or option...".format(tag)) return plugin.upload(opts, build_artifact) return
def build_doc_artifact(): """ build sphinx documentation and create an artifact to be uploaded to a remote server Requires the following cirrus.conf section: [doc] sphinx_makefile_dir = /path/to/makefile sphinx_doc_dir = /path/to/_build/docdir artifact_dir = /path/to/archive/dir """ LOGGER.info("Building doc archive...") config = load_configuration() artifact_name = doc_artifact_name(config) arcname = os.path.basename(artifact_name).rsplit('.', 2)[0] doc_dir = config['doc']['sphinx_doc_dir'] if not os.path.exists(doc_dir): msg = "Documentation path: {0} Not Found".format(doc_dir) LOGGER.error(msg) raise RuntimeError(msg) with tarfile.open(artifact_name, "w:gz") as tar: tar.add(doc_dir, arcname=arcname) LOGGER.info("Documentation artifact created at: {}".format(artifact_name)) return artifact_name
def main(): """ _main_ provide support for some basic docker operations so that building images can be standardised as part of a workflow """ opts = build_parser() config = load_configuration() if not config.has_section('docker'): msg = ( "Unable to find docker section in cirrus.conf" #TODO: Link to docs here ) LOGGER.error(msg) sys.exit(1) if not is_docker_connected(): LOGGER.error(DOCKER_CONNECTION_HELP) sys.exit(1) if opts.command == 'build': docker_build(opts, config) if opts.command == 'push': docker_push(opts, config) if opts.command == 'test': # Already called above pass
def get_builder_plugin(): """ Get the builder plugin name default. If not provided by CLI opt, start with user pref in gitconfig, Look for hint in cirrus conf or just resort to a guess based on what python cirrus is using """ # TODO look up in git config config = load_configuration() builder = None if config.has_gitconfig_param('builder'): builder = str(config.get_gitconfig_param('builder')) if builder: LOGGER.info("Using Builder Plugin from gitconfig: {}".format(builder)) return builder build_config = config.get('build', {}) builder = build_config.get('builder') if builder is not None: LOGGER.info("Using Builder Plugin from cirrus.conf: {}".format(builder)) return builder # fall back to old defaults if is_anaconda(): LOGGER.info("Using default CondaPip builder") builder = "CondaPip" else: LOGGER.info("Using default VirtualenvPip builder") builder = "VirtualenvPip" return builder
def branch_status(branch_name): """ _branch_status_ Get the branch status which should include details of CI builds/hooks etc See: https://developer.github.com/v3/repos/statuses/#get-the-combined-status-for-a-specific-ref returns a state which is one of 'failure', 'pending', 'success' """ config = load_configuration() token = get_github_auth()[1] url = "https://api.github.com/repos/{org}/{repo}/commits/{branch}/status".format( org=config.organisation_name(), repo=config.package_name(), branch=branch_name) headers = { 'Authorization': 'token {0}'.format(token), 'Content-Type': 'application/json' } resp = requests.get(url, headers=headers) resp.raise_for_status() state = resp.json()['state'] return state
def register_package( package, repository, pypirc='~/.pypirc', username=None, password=None, certfile=None, client_certfile=None, comment=None ): """ :param package: - sdist/bdist etc artifact to register :param repository: """ cirrus_conf = load_configuration() if cirrus_conf.has_section('twine'): username = username or cirrus_conf.get_param('twine', 'username', None) password = password or cirrus_conf.get_param('twine', 'password', None) certfile = certfile or cirrus_conf.get_param('twine', 'certfile', None) client_certfile = client_certfile or cirrus_conf.get_param('twine', 'client_certfile', None) register( package, repository, username, password, comment, pypirc, certfile, client_certfile, None )
def __init__(self, repo_dir, package_dir=None): self.repo_dir = repo_dir self.repo = git.Repo(repo_dir) self.config = load_configuration(package_dir) self.gh_user, self.token = get_github_auth() self.auth_headers = { 'Authorization': 'token {0}'.format(self.token), 'Content-Type': 'application/json' }
def __init__(self): super(Linter, self).__init__() self.config = load_configuration() self.linter_config = self.config.get( 'qc/{}'.format(type(self).__name__, {}), {}) self.working_dir = repo_directory() self.pass_threshold = 0 self.test_mode = False self.errors = {}
def make_new_version(opts): LOGGER.info("Updating package version...") if not highlander([opts.major, opts.minor, opts.micro]): msg = "Can only specify one of --major, --minor or --micro" LOGGER.error(msg) raise RuntimeError(msg) fields = ['major', 'minor', 'micro'] mask = [opts.major, opts.minor, opts.micro] field = [x for x in itertools.compress(fields, mask)][0] config = load_configuration() current_version = config.package_version() # need to be on the latest develop repo_dir = repo_directory() curr_branch = current_branch(repo_dir) # make sure repo is clean if has_unstaged_changes(repo_dir): msg = ("Error: Unstaged changes are present on the branch {}" "Please commit them or clean up before proceeding" ).format(curr_branch) LOGGER.error(msg) raise RuntimeError(msg) # update cirrus conf new_version = bump_version_field(current_version, field) msg = "Bumping version from {prev} to {new} on branch {branch}".format( prev=current_version, new=new_version, branch=curr_branch) LOGGER.info(msg) config.update_package_version(new_version) changes = ['cirrus.conf'] if opts.bump: reqs_file = os.path.join(repo_dir, 'requirements.txt') for pkg, version in opts.bump: LOGGER.info("Bumping dependency {} to {}".format(pkg, version)) bump_package(reqs_file, pkg, version) changes.append(reqs_file) # update __version__ or equivalent version_file, version_attr = config.version_file() if version_file is not None: LOGGER.info('Updating {0} attribute in {1}'.format( version_file, version_attr)) update_version(version_file, new_version, version_attr) changes.append(version_file) # update files changed msg = "cirrus release: version bumped for {0}".format(curr_branch) LOGGER.info('Committing files: {0}'.format(','.join(changes))) LOGGER.info(msg) commit_files_optional_push(repo_dir, msg, not opts.no_remote, *changes)
def run(self, opts): """ _run_ Run the plugin, calling the overloaded parser and setup methods """ self.opts = opts project_dir = opts.repo self.config = load_configuration() self.setup(project_dir)
def __init__(self): super(Builder, self).__init__() self.plugin_parser = argparse.ArgumentParser() self.config = load_configuration() self.build_config = self.config.get('build', {}) self.working_dir = repo_directory() self.venv_name = self.build_config.get('virtualenv_name', 'venv') self.reqs_name = self.build_config.get('requirements_file', 'requirements.txt') self.extra_reqs = self.build_config.get('extra_requirements', []) self.python_bin = self.build_config.get('python', None) self.extra_reqs = self.str_to_list(self.extra_reqs) self.venv_path = os.path.join(self.working_dir, self.venv_name)
def current_branch_mark_status(repo_dir, state): """ _current_branch_mark_status_ Mark the CI status of the current branch. :param repo_dir: directory of git repository :param state: state of the last test run, such as "success" or "failure" """ LOGGER.info(u"Setting CI status for current branch to {}".format(state)) config = load_configuration() token = get_github_auth()[1] sha = git.Repo(repo_dir).head.commit.hexsha try: # @HACK: Do a push that we expect will fail -- we just want to # tell the server about our sha. A more elegant solution would # probably be to push a detached head. push(repo_dir) except RuntimeError as ex: if "rejected" not in unicode_(ex): raise url = "https://api.github.com/repos/{org}/{repo}/statuses/{sha}".format( org=config.organisation_name(), repo=config.package_name(), sha=sha ) headers = { 'Authorization': 'token {0}'.format(token), 'Content-Type': 'application/json' } data = json.dumps( { "state": state, "description": "State after cirrus check.", # @HACK: use the travis context, which is technically # true, because we wait for Travis tests to pass before # cutting a release. In the future, we need to setup a # "cirrus" context, for clarity. "context": "continuous-integration/travis-ci" } ) resp = requests.post(url, headers=headers, data=data) resp.raise_for_status()
def create_pull_request( repo_dir, pr_info, token=None): """ Creates a pull_request on GitHub and returns the html url of the pull request created :param repo_dir: directory of git repository :param pr_info: dictionary containing title and body of pull request :param token: auth token """ if repo_dir is None: raise RuntimeError('repo_dir is None') if 'title' not in pr_info: raise RuntimeError('title is None') if 'body' not in pr_info: raise RuntimeError('body is None') config = load_configuration() url = 'https://api.github.com/repos/{0}/{1}/pulls'.format( config.organisation_name(), config.package_name()) if token is None: token = get_github_auth()[1] headers = { 'Authorization': 'token {0}'.format(token), 'Content-Type': 'application/json'} data = { 'title': pr_info['title'], 'head': get_active_branch(repo_dir).name, 'base': config.gitflow_branch_name(), 'body': pr_info['body']} resp = requests.post(url, data=json.dumps(data), headers=headers) if resp.status_code == 422: LOGGER.error( ( "POST to GitHub api returned {0}" "Have you committed your changes and pushed to remote?" ).format(resp.status_code) ) resp.raise_for_status() resp_json = resp.json() return resp_json['html_url']
def publish_documentation(opts): """ Publish Sphinx documentation. If a tarfile exists for the current program version, that file will be used. Otherwise, the documentation will be built and packaged before uploading. Requires the publish_method plugin to be specified in the cirrus.conf [doc] section, and a section containing the required fields for the publisher method, e.g: [doc] publisher = file_server :param argparse.Namspace opts: A Namespace of publisher options """ LOGGER.info("Preparing to upload documentation...") config = load_configuration() doc_params = config.get('doc', {}) doc_artifact = doc_artifact_name(config) if not os.path.exists(doc_artifact): msg = 'Documentation tarball not found at {}'.format(doc_artifact) LOGGER.error(msg) raise RuntimeError(msg) try: publisher = doc_params['publisher'] except KeyError: LOGGER.error( 'Did not find a publisher in [doc] section of cirrus.conf' '\nSee below for an example:' '\n[doc]' '\npublisher = doc_file_server') sys.exit(1) plugin = get_publisher_plugin(publisher) if opts.test: LOGGER.info( "Uploading {}-{}.tar.gz to file server disabled by test or " "option...".format( config.package_name(), config.package_version() ) ) return plugin.publish(doc_artifact) return
def test_configuration_map(self): """test building config mapping""" config = load_configuration(package_dir=self.dir, gitconfig_file=self.gitconfig) mapping = config.configuration_map() self.failUnless('cirrus' in mapping) self.failUnless('credentials' in mapping['cirrus']) self.failUnless('configuration' in mapping['cirrus']) self.failUnless('github_credentials' in mapping['cirrus']['credentials']) self.assertEqual( mapping['cirrus']['credentials']['github_credentials'], {'github_user': None, 'github_token': None} ) self.assertEqual( mapping['cirrus']['configuration']['package']['name'], 'cirrus_tests' )
def remove_nightly(ghc): """ remove the nightly part from the cirrus.conf version """ cirrus_conf = load_configuration() nightly_conf = nightly_config(cirrus_conf) current = cirrus_conf.package_version() if is_nightly(current): new_version = current.split(nightly_conf['nightly_separator'], 1)[0] cirrus_conf.update_package_version(new_version) ghc.commit_files_optional_push( "remove nightly tag from cirrus.conf", False, "cirrus.conf" ) return
def test_configuration_map(self): """test building config mapping""" config = load_configuration(package_dir=self.dir, gitconfig_file=self.gitconfig) mapping = config.configuration_map() self.failUnless('cirrus' in mapping) self.failUnless('credentials' in mapping['cirrus']) self.failUnless('configuration' in mapping['cirrus']) self.failUnless( 'github_credentials' in mapping['cirrus']['credentials']) self.assertEqual( mapping['cirrus']['credentials']['github_credentials'], { 'github_user': None, 'github_token': None }) self.assertEqual(mapping['cirrus']['configuration']['package']['name'], 'cirrus_tests')
def nightly_config(conf=None): """ get the nightly config settings from the release section """ result = { "nightly_format": DEFAULT_FORMAT, "nightly_separator": "-nightly-" } if not conf: conf = load_configuration() if not conf.has_section('release'): return result result['nightly_format'] = conf.get_param("release", "nightly_format", result['nightly_format']) result['nightly_separator'] = conf.get_param("release", "nightly_separator", result['nightly_separator']) return result
def build_release(opts): """ _build_release_ run python setup.py sdist to create the release artifact """ LOGGER.info("Building release...") config = load_configuration() local('python setup.py sdist') build_artifact = artifact_name(config) if not os.path.exists(build_artifact): msg = "Expected build artifact: {0} Not Found".format(build_artifact) LOGGER.error(msg) raise RuntimeError(msg) LOGGER.info("Release artifact created: {0}".format(build_artifact)) return build_artifact
def new_nightly(): """ generate a new nightly version """ cirrus_conf = load_configuration() nightly_conf = nightly_config(cirrus_conf) now = datetime.datetime.now() ts = now.strftime(nightly_conf['nightly_format']) current = cirrus_conf.package_version() nightly = "{version}{sep}{ts}".format( version=current, sep=nightly_conf['nightly_separator'], ts=ts ) return nightly
def test_reading_missing(self, mock_pop, mock_shell): """test config load using repo dir""" mock_result = mock.Mock() mock_result.communicate = mock.Mock() mock_result.returncode = 0 mock_result.communicate.return_value = (self.dir, None) mock_pop.return_value = mock_result mock_shell.return_value = self.gitconf_str config = load_configuration(package_dir="womp") self.failUnless(mock_result.communicate.called) mock_pop.assert_has_calls( mock.call(['git', 'rev-parse', '--show-toplevel'], stdout=-1)) self.assertEqual(config.package_version(), '1.2.3') self.assertEqual(config.package_name(), 'cirrus_tests') self.failUnless(mock_shell.called) mock_shell.assert_has_calls( [mock.call('git config --file {} -l'.format(self.gitconfig))])
def test_reading_missing(self, mock_pop, mock_shell): """test config load using repo dir""" mock_result = mock.Mock() mock_result.communicate = mock.Mock() mock_result.returncode = 0 mock_result.communicate.return_value = (self.dir, None) mock_pop.return_value = mock_result mock_shell.return_value = self.gitconf_str config = load_configuration(package_dir="womp") self.failUnless(mock_result.communicate.called) mock_pop.assert_has_calls(mock.call(['git', 'rev-parse', '--show-toplevel'], stdout=-1)) self.assertEqual(config.package_version(), '1.2.3') self.assertEqual(config.package_name(), 'cirrus_tests') self.failUnless(mock_shell.called) mock_shell.assert_has_calls([ mock.call('git config --file {} -l'.format(self.gitconfig)) ])
def new_feature_branch(opts): """ _new_feature_branch_ Checks out branch, creates new branch 'name', optionally pushes new branch to remote """ config = load_configuration() repo_dir = os.getcwd() checkout_and_pull(repo_dir, config.gitflow_branch_name()) LOGGER.info("Checked out and pulled {0}".format( config.gitflow_branch_name())) branch_name = ''.join((config.gitflow_feature_prefix(), opts.name[0])) branch(repo_dir, branch_name, config.gitflow_branch_name()) LOGGER.info("Created branch {0}".format(branch_name)) if opts.push: push(repo_dir) LOGGER.info("Branch {0} pushed to remote".format(branch_name))
def upload_package( package, repository, pypirc='~/.pypirc', sign=False, identity=None, username=None, password=None, comment=None, sign_with=None, skip_existing=False, certfile=None, client_certfile=None ): """ :param package: path to sdist package :param repository: repo name to upload to :param pypirc: path to pypirc file """ cirrus_conf = load_configuration() if cirrus_conf.has_section('twine'): username = username or cirrus_conf.get_param('twine', 'username', None) password = password or cirrus_conf.get_param('twine', 'password', None) certfile = certfile or cirrus_conf.get_param('twine', 'certfile', None) identity = identity or cirrus_conf.get_param('twine', 'identity', None) sign_with = sign_with or cirrus_conf.get_param('twine', 'sign_with', None) client_certfile = client_certfile or cirrus_conf.get_param('twine', 'client_certfile', None) upload( [package], repository, sign, identity, username, password, comment, sign_with, pypirc, skip_existing, certfile, client_certfile, None )
def main(): """ _main_ Execute prestage command: - look into cirrus conf for requirements to be prestaged - look for matching version requirement in requirements.txt - clone each repo locally - pip install -e the repo into the virtualenv """ working_dir = os.getcwd() repo_cache = os.path.join(working_dir, 'prestage') if not os.path.exists(repo_cache): os.makedirs(repo_cache) config = load_configuration() prestage_params = config.get('prestage', {}) build_params = config.get('build', {}) venv_name = build_params.get('virtualenv_name', 'venv') reqs_name = build_params.get('requirements_file', 'requirements.txt') venv_path = os.path.join(working_dir, venv_name) venv_command = os.path.join(venv_path, 'bin', 'activate') reqs = {} # TODO: this only supports explicit == version reqs right now # make this more flexible for req in parse_requirements(reqs_name): if req.req is not None: versions = [ x for x in req.absolute_versions] if len(versions) > 0: reqs[req.name] = versions[0] # prestage section contains a map of package name: repo for req, repo in prestage_params.iteritems(): msg = "Prestaging repo for requirement {req} from {repo}" tag = reqs.get(req) if tag is not None: msg += " with tag {tag}" msg.format(req=req, repo=repo, tag=tag) print msg local_repo = os.path.join(repo_cache, req) git_clone_repo(repo, local_repo, tag=reqs.get(req)) install_from_repo(venv_command, local_repo)
def main(): """ _main_ Execute prestage command: - look into cirrus conf for requirements to be prestaged - look for matching version requirement in requirements.txt - clone each repo locally - pip install -e the repo into the virtualenv """ working_dir = os.getcwd() repo_cache = os.path.join(working_dir, 'prestage') if not os.path.exists(repo_cache): os.makedirs(repo_cache) config = load_configuration() prestage_params = config.get('prestage', {}) build_params = config.get('build', {}) venv_name = build_params.get('virtualenv_name', 'venv') reqs_name = build_params.get('requirements_file', 'requirements.txt') venv_path = os.path.join(working_dir, venv_name) venv_command = os.path.join(venv_path, 'bin', 'activate') reqs = {} # TODO: this only supports explicit == version reqs right now # make this more flexible for req in parse_requirements(reqs_name): if req.req is not None: versions = [x for x in req.absolute_versions] if len(versions) > 0: reqs[req.name] = versions[0] # prestage section contains a map of package name: repo for req, repo in prestage_params.iteritems(): msg = "Prestaging repo for requirement {req} from {repo}" tag = reqs.get(req) if tag is not None: msg += " with tag {tag}" msg.format(req=req, repo=repo, tag=tag) print msg local_repo = os.path.join(repo_cache, req) git_clone_repo(repo, local_repo, tag=reqs.get(req)) install_from_repo(venv_command, local_repo)
def main(): """ _main_ Execute test command """ cirrus_conf = load_configuration() qc_conf = cirrus_conf.quality_control() opts = build_parser(sys.argv, qc_conf) if opts.only_changes: files = get_diff_files(None) # we only want python modules for item in files[:]: if not item.endswith('.py'): files.remove(item) if not files: LOGGER.info("No modules have been changed.") sys.exit(0) opts.include_files = files run_linters(opts, cirrus_conf, qc_conf)
def cleanup_release(opts): """ _cleanup_release_ Remove local and remote release branches if they exist """ config = load_configuration() repo_dir = os.getcwd() pfix = config.gitflow_release_prefix() branch_name = release_branch_name(config) if opts.version is not None: if not opts.version.startswith(pfix): branch_name = "{0}{1}".format(pfix, opts.version) else: branch_name = opts.version LOGGER.info("Cleaning release branches for {}".format(branch_name)) with GitHubContext(repo_dir) as ghc: ghc.delete_branch(branch_name, not opts.no_remote)
def legacy_update(opts): """update repo installed cirrus""" install = find_cirrus_install() with chdir(install): config = load_configuration() if opts.branch and opts.version: msg = "Can specify branch -OR- version, not both" raise RuntimeError(msg) if opts.branch is not None: update_to_branch(opts.branch, config) setup_develop(config) return if opts.version is not None: tag = opts.version else: tag = latest_release(config) LOGGER.info("Retrieved latest tag: {0}".format(tag)) update_to_tag(tag, config) setup_develop(config)
def build_docs(make_opts=None): """ _build_docs_ Runs 'make' against a Sphinx makefile. Requires the following cirrus.conf section: [doc] sphinx_makefile_dir = /path/to/makefile :param list make_opts: Options to run the Sphinx 'make' command gathered from the calling command's --docs option. If None or an empty list, 'make' will be run using 'clean html' """ LOGGER.info('Building docs') config = load_configuration() build_params = config.get('build', {}) venv_name = build_params.get('virtualenv_name', 'venv') try: docs_root = os.path.join( os.getcwd(), config['doc']['sphinx_makefile_dir']) except KeyError: LOGGER.error( 'Did not find a complete [doc] section in cirrus.conf' '\nSee below for an example:' '\n[doc]' '\nsphinx_makefile_dir = /path/to/sphinx') sys.exit(1) cmd = 'cd {} && make clean html'.format(docs_root) if make_opts: # additional args were passed after --docs. Pass these to make cmd = 'cd {} && make {}'.format(docs_root, ' '.join(make_opts)) local('. ./{}/bin/activate && {}'.format(venv_name, cmd)) LOGGER.info('Build command was "{}"'.format(cmd))
def trigger_release(opts): """ _trigger_release_ Alias for "git cirrus release new --micro/minor/major. - Run the "release new" command - Capture the new version string - Pass new version number to external build server Requires the following sections and values in cirrus.conf: [build-server] name = jenkins [jenkins] url = http://localhost:8080 job = default """ config = load_configuration() try: build_server = config['build-server']['name'] build_server_config = config[build_server] except KeyError: msg = ( '[build-server] section is incomplete or missing from cirrus.conf. ' 'Please see below for an example.\n' '\n [build-server]' '\n name = jenkins' '\n [jenkins]' '\n url = http://localhost:8080' '\n job = default') raise RuntimeError(msg) new_version, release_level = new_release(opts) if build_server == 'jenkins': _trigger_jenkins_release(build_server_config, new_version, release_level)
def merge_feature_branch(opts): """ merge current feature branch into develop """ config = load_configuration() main_branch = config.gitflow_branch_name() repo_dir = repo_directory() curr_branch = current_branch(repo_dir) LOGGER.info("Merging {} into {}".format(curr_branch, main_branch)) # make sure repo is clean if has_unstaged_changes(repo_dir): msg = ("Error: Unstaged changes are present on the feature branch {}" "Please commit them or clean up before proceeding" ).format(curr_branch) LOGGER.error(msg) raise RuntimeError(msg) checkout_and_pull(repo_dir, main_branch, pull=not opts.no_remote) with GitHubContext(repo_dir) as ghc: ghc.merge_branch(curr_branch) if not opts.no_remote: ghc.push_branch(main_branch) LOGGER.info("Branch {0} pushed to remote".format(main_branch))
def main(): """ _main_ Look up the plugin to be invoked for deployment, via the CLI or cirrus config for this package, and then invoke the deployer plugin """ initial_opts = build_parser() pname = initial_opts.plugin if initial_opts.plugin is None: config = load_configuration() try: pname = config.get_param('deploy', 'plugin', None) except KeyError: pname = None if pname is None: msg = "No Plugin specified with --plugin or in cirrus.conf for deploy" raise RuntimeError(msg) plugin = get_plugin(pname) plugin.build_parser() opts = plugin.parser.parse_args() plugin.deploy(opts)
def merge_feature_branch(opts): """ merge current feature branch into develop """ config = load_configuration() main_branch = config.gitflow_branch_name() repo_dir = repo_directory() curr_branch = current_branch(repo_dir) LOGGER.info("Merging {} into {}".format(curr_branch, main_branch)) # make sure repo is clean if has_unstaged_changes(repo_dir): msg = ( "Error: Unstaged changes are present on the feature branch {}" "Please commit them or clean up before proceeding" ).format(curr_branch) LOGGER.error(msg) raise RuntimeError(msg) checkout_and_pull(repo_dir, main_branch, pull=not opts.no_remote) with GitHubContext(repo_dir) as ghc: ghc.merge_branch(curr_branch) if not opts.no_remote: ghc.push_branch(main_branch) LOGGER.info("Branch {0} pushed to remote".format(main_branch))
def new_feature_branch(opts): """ _new_feature_branch_ Checks out branch, creates new branch 'name', optionally pushes new branch to remote """ config = load_configuration() repo_dir = repo_directory() checkout_and_pull( repo_dir, config.gitflow_branch_name(), pull=not opts.no_remote ) LOGGER.info("Checked out and pulled {0}".format( config.gitflow_branch_name())) branch_name = ''.join((config.gitflow_feature_prefix(), opts.name[0])) branch(repo_dir, branch_name, config.gitflow_branch_name()) LOGGER.info("Created branch {0}".format(branch_name)) if not opts.no_remote: push(repo_dir) LOGGER.info("Branch {0} pushed to remote".format(branch_name))
def execute_build(opts): """ _execute_build_ Execute the build in the current package context. - reads the config to check for custom build parameters - defaults to ./venv for virtualenv - defaults to ./requirements.txt for reqs - removes existing virtualenv if clean flag is set - builds the virtualenv - pip installs the requirements into it : param argparse.Namspace opts: A Namespace of build options """ working_dir = repo_directory() config = load_configuration() build_params = config.get('build', {}) # we have custom build controls in the cirrus.conf venv_name = build_params.get('virtualenv_name', 'venv') reqs_name = build_params.get('requirements_file', 'requirements.txt') extra_reqs = build_params.get('extra_requirements', '') python_bin = build_params.get('python', None) if opts.python: python_bin = opts.python extra_reqs = [x.strip() for x in extra_reqs.split(',') if x.strip()] if opts.extras: extra_reqs.extend(opts.extras) extra_reqs = set(extra_reqs) # dedupe venv_path = os.path.join(working_dir, venv_name) if is_anaconda(): conda_venv(venv_path, opts, python_bin) else: virtualenv_venv(venv_path, opts, python_bin) # custom pypi server pypi_server = config.pypi_url() pip_options = config.pip_options() pip_command_base = None if pypi_server is not None: pypirc = PypircFile() if pypi_server in pypirc.index_servers: pypi_url = pypirc.get_pypi_url(pypi_server) else: pypi_conf = get_pypi_auth() pypi_url = ( "https://{pypi_username}:{pypi_token}@{pypi_server}/simple" ).format(pypi_token=pypi_conf['token'], pypi_username=pypi_conf['username'], pypi_server=pypi_server) pip_command_base = ('{0}/bin/pip install -i {1}').format( venv_path, pypi_url) if opts.upgrade: cmd = ('{0} --upgrade ' '-r {1}').format(pip_command_base, reqs_name) else: cmd = ('{0} ' '-r {1}').format(pip_command_base, reqs_name) else: pip_command_base = '{0}/bin/pip install'.format(venv_path) # no pypi server if opts.upgrade: cmd = '{0} --upgrade -r {1}'.format(pip_command_base, reqs_name) else: cmd = '{0} -r {1}'.format(pip_command_base, reqs_name) if pip_options: cmd += " {} ".format(pip_options) try: local(cmd) except OSError as ex: msg = ("Error running pip install command during build\n" "Error was {0}\n" "Running command: {1}\n" "Working Dir: {2}\n" "Virtualenv: {3}\n" "Requirements: {4}\n").format(ex, cmd, working_dir, venv_path, reqs_name) LOGGER.error(msg) sys.exit(1) if extra_reqs: if opts.upgrade: commands = [ "{0} --upgrade -r {1}".format(pip_command_base, reqfile) for reqfile in extra_reqs ] else: commands = [ "{0} -r {1}".format(pip_command_base, reqfile) for reqfile in extra_reqs ] for cmd in commands: LOGGER.info("Installing extra requirements... {}".format(cmd)) try: local(cmd) except OSError as ex: msg = ("Error running pip install command extra " "requirements install: {}\n{}").format(reqfile, ex) LOGGER.error(msg) sys.exit(1) # setup for development if opts.nosetupdevelop: msg = "skipping python setup.py develop..." LOGGER.info(msg) else: LOGGER.info('running python setup.py develop...') activate = activate_command(venv_path) local('{} && python setup.py develop'.format(activate))
def __init__(self): super(Publisher, self).__init__() self.package_conf = load_configuration()
def __init__(self): super(Deployer, self).__init__() self.package_conf = load_configuration() self.parser = argparse.ArgumentParser() self.parser.add_argument('--plugin', '-p', dest='plugin', default=None)
def init_container(opts): """ Initialise a basic container-template setup for this package """ cirrus_conf = os.path.join(opts.repo, 'cirrus.conf') if not os.path.exists(cirrus_conf): msg = "No cirrus.conf found, need to init repo first?" LOGGER.error(msg) sys.exit(1) config = load_configuration(opts.repo) template_dir = os.path.join(opts.repo, opts.template_dir) if not os.path.exists(template_dir): LOGGER.info("Creating Template in {}".format(template_dir)) os.makedirs(template_dir) docker_file = os.path.join(template_dir, 'Dockerfile.mustache') dotfile = os.path.join(template_dir, '.dockerstache') pre_script = os.path.join(template_dir, 'pre_script.sh') post_script = os.path.join(template_dir, 'post_script.sh') context = os.path.join(template_dir, 'context.json') installer_script = os.path.join(template_dir, 'install_script.sh.mustache') opts.context_file = os.path.join(opts.template_dir, 'context.json') # make sure repo is clean if has_unstaged_changes(opts.repo): msg = ( "Error: Unstaged changes are present on the branch " "Please commit them or clean up before proceeding" ) LOGGER.error(msg) raise RuntimeError(msg) main_branch = config.gitflow_branch_name() LOGGER.info("checking out latest {} branch...".format(main_branch)) checkout_and_pull(opts.repo, main_branch, not opts.no_remote) venv_option = "" if opts.virtualenv: venv_option = ". {}/bin/activate".format(opts.virtualenv) with working_dir(template_dir): write_basic_dockerfile(opts, config, docker_file) write_json_file(dotfile, { "post_script": "post_script.sh", "pre_script": "pre_script.sh", "inclusive": True, "excludes": ["post_script.sh", "post_script.sh", ".dockerstache"] }) write_json_file(context, {}) # render templates for container scripts template_context = { 'open_brace': '{{', 'close_brace': '}}', 'package': config.package_name(), 'virtualenv': venv_option, 'pip_options': config.pip_options() if config.pip_options() else "" } install_template = find_template('install_script.sh.mustache') pre_template = find_template('pre_script.sh.mustache') post_template = find_template('post_script.sh.mustache') renderer = pystache.Renderer() install_result = renderer.render_path(install_template, template_context) with open(installer_script, 'w') as handle: handle.write(install_result) post_result = renderer.render_path(post_template, template_context) with open(post_script, 'w') as handle: handle.write(post_result) pre_result = renderer.render_path(pre_template, template_context) with open(pre_script, 'w') as handle: handle.write(pre_result) make_executable(pre_script, opts.repo) make_executable(post_script, opts.repo) make_executable(installer_script, opts.repo) edit_cirrus_conf(opts, config) modified = [ cirrus_conf, docker_file, dotfile, pre_script, post_script, installer_script, context ] LOGGER.info("commiting changes...") commit_files_optional_push( opts.repo, "git cirrus package container-init", not opts.no_remote, *modified )