def _copy_file_to_workspace(self, src: str) -> bool: info = _exec([ 'docker', 'cp', '{}:{}'.format(self.name, src), '{}/{}'.format(self.workspace_dir, self.exec_uuid) ]) if info.returncode == 0: return True logger.error('persist to workspace failed: %s', info)
def copy_workspace_to_container(self, dest: str) -> bool: self.exec('mkdir -p {}'.format(dest)) current_workspace_path = '{}/{}'.format(self.workspace_dir, self.exec_uuid) for file in os.listdir(current_workspace_path): file_path = os.path.join(self.workspace_dir, self.exec_uuid, file) info = _exec( ['docker', 'cp', file_path, '{}:{}'.format(self.name, dest)]) if not info: logger.error('attach workspace failed: %s', info) return False return True
def run(self) -> None: with DockerContainer(self.name, self.spec.get('docker')[0].get('image'), self.exec_uuid, self.clone_url, self.working_directory, self.env_vars, ref=self.ref) as docker: self.steps = [ Step.factory(docker, step) for step in self.spec.get('steps') ] skip = False if self.run_condition.get('branch'): if not re.search(self.run_condition['branch'], self.branch): logger.debug( 'skipping %s because %s doesnt match condition %s', self.name, self.branch, self.run_condition['branch']) self.state = Status.skipped skip = True if self.run_condition.get('tag'): if not re.search(self.run_condition['tag'], self.tag): logger.debug( 'skipping %s because %s doesnt match condition %s', self.name, self.tag, self.run_condition['tag']) self.state = Status.skipped skip = True if not skip: self.state = Status.running try: logger.info('---- Running Job: %s ----', self.name) logger.debug('exec_uuid: %s, env_vars: %s', self.exec_uuid, self.env_vars) for step in self.steps: logger.info('Executing Step: %s', step) output = step.run() if not output: logger.error( f'Job Failed[{self.name}]\n{output.stderr}') self.state = Status.failed return self.state logger.info('Job (%s) Passed in %.2f seconds', self.name, docker.duration) except Exception as e: self.state = Status.failed raise e self.state = Status.passed return self.state
def _setup() -> None: for binary in ('docker', 'git'): try: _exec([binary]) except FileNotFoundError: logger.error('%s not installed', binary) try: os.mkdir(DockerContainer.workspace_dir) except FileExistsError: logger.info( 'workspace directory already exists at %s - this ' 'is harmless providing it\'s what you wanted', DockerContainer.workspace_dir) pass
def _download_repo_build_config(repo_slab, ref) -> dict: """ last commit to master: https://raw.githubusercontent.com/chestm007/Zeus-CI/master/.zeusci/config.yml specific commit: https://raw.githubusercontent.com/chestm007/Zeus-CI/142eb4bdbbc54371cbcc4a0000bd8eeea997d1f2/.zeusci/config.yml specific tag: https://raw.githubusercontent.com/chestm007/Zeus-CI/test-tag/.zeusci/config.yml """ url_format = 'https://raw.githubusercontent.com/{repo_slab}/{ref}/.zeusci/config.yml' try: response = urllib.request.urlopen( url_format.format(repo_slab=repo_slab, ref=ref.split('/')[-1])) except urllib.error.HTTPError: logger.error('couldnt fetch build config') return {} if response.status == 200: _config = yaml.load(response, yaml.Loader) return _config
def main(repo_slab: str = None, env_vars: List[str] = None, threads: int = 1, ref=None) -> bool: if not any([repo_slab, env_vars, threads, ref]): parser = argparse.ArgumentParser( description='Run Zeus-CI jobs locally through docker') parser.add_argument('--env', type=str, nargs='+', help='K=V environment vars to pass to the test') parser.add_argument( '--threads', type=int, help='number of docker containers to run concurrently') parser.add_argument('--ref', type=str, help='git ref(commit/tag) to checkout') args = parser.parse_args() if args.ref: ref = args.ref if args.threads: threads = args.threads if args.env: env_vars = env_vars.extend(args.env) if env_vars else args.env if not repo_slab: repo_slab = repo_slab_of_cwd() if not repo_slab: logger.error( 'not in the root directory of a git repository, exiting') sys.exit(1) _setup() clone_url = 'https://github.com/{}.git'.format(repo_slab) _config = _download_repo_build_config(repo_slab, ref) if not config: return False try: _config['workflows'].pop('version') except KeyError: pass env_vars.append('ZEUS_USERNAME={}'.format(repo_slab.split('/')[0])) workflows = { name: Workflow(name, _config['jobs'], spec, clone_url, threads, env_vars=env_vars, ref=ref) for name, spec in _config['workflows'].items() } results = [] for workflow_name, workflow in workflows.items(): try: results.append(workflow.run()) except Exception as e: logger.error('%s: %s', workflow_name, e) for status in (Status.error, Status.failed): if any(map(lambda r: r == status, results)): return status return Status.passed
def _run_from_queue(self, queue): signal.signal(signal.SIGINT, signal.SIG_IGN) with self.database.get_session() as session: try: for build_id in iter(queue.get, None): build = session.query(Build).filter_by(id=build_id).one() # TODO: this must be called as soon as possible due to a race condition with populating the queue build.status = Status.starting session.commit() github = Github(TokenAuth(build.repo.user.token)) github.update_status(build, GithubStatus.pending) try: ref = None env_vars = build.repo.shell_ready_envvars() if build.ref.startswith('refs/tags/'): ref = build.ref.replace('refs/', '', 1) env_vars.append('ZEUS_TAG={}'.format( build.ref.replace('refs/tags/', ''))) env_vars.append('ZEUS_BRANCH={}'.format( build.json['base_ref'].replace( 'refs/heads/', ''))) elif build.ref.startswith('refs/heads/'): ref = build.json['after'] env_vars.append('ZEUS_TAG=""') env_vars.append('ZEUS_BRANCH={}'.format( build.ref.replace('refs/heads/', ''))) if not ref: logger.error( 'error from worker thread: %s, refn not detected', build.id) else: build.status = Status.running session.commit() logger.debug('executing runner.main process') status = runner.main( build.repo.name, threads=self.config['runner_threads'], ref=ref, env_vars=env_vars) logger.debug("runner main process completed") if status == Status.passed: logger.debug("build passed") github.update_status(build, GithubStatus.success) build.status = Status.passed else: logger.debug("build failed") github.update_status(build, GithubStatus.failure) build.status = Status.failed session.commit() except Exception as e: github.update_status(build, GithubStatus.error) build.status = Status.error raise e except Exception as e: logger.error('error from worker thread: %s', exc_info=True) raise e except KeyboardInterrupt: pass