def _copy_file_to_workspace(self, src: str) -> bool:
info = _exec([
'docker', 'cp', '{}:{}'.format(self.name, src),
'{}/{}'.format(self.workspace_dir, self.exec_uuid)
])
if info.returncode == 0:
return True
logger.error('persist to workspace failed: %s', info)
def copy_workspace_to_container(self, dest: str) -> bool:
self.exec('mkdir -p {}'.format(dest))
current_workspace_path = '{}/{}'.format(self.workspace_dir,
self.exec_uuid)
for file in os.listdir(current_workspace_path):
file_path = os.path.join(self.workspace_dir, self.exec_uuid, file)
info = _exec(
['docker', 'cp', file_path, '{}:{}'.format(self.name, dest)])
if not info:
logger.error('attach workspace failed: %s', info)
return False
return True
def run(self) -> None:
with DockerContainer(self.name,
self.spec.get('docker')[0].get('image'),
self.exec_uuid,
self.clone_url,
self.working_directory,
self.env_vars,
ref=self.ref) as docker:
self.steps = [
Step.factory(docker, step) for step in self.spec.get('steps')
]
skip = False
if self.run_condition.get('branch'):
if not re.search(self.run_condition['branch'], self.branch):
logger.debug(
'skipping %s because %s doesnt match condition %s',
self.name, self.branch, self.run_condition['branch'])
self.state = Status.skipped
skip = True
if self.run_condition.get('tag'):
if not re.search(self.run_condition['tag'], self.tag):
logger.debug(
'skipping %s because %s doesnt match condition %s',
self.name, self.tag, self.run_condition['tag'])
self.state = Status.skipped
skip = True
if not skip:
self.state = Status.running
try:
logger.info('---- Running Job: %s ----', self.name)
logger.debug('exec_uuid: %s, env_vars: %s', self.exec_uuid,
self.env_vars)
for step in self.steps:
logger.info('Executing Step: %s', step)
output = step.run()
if not output:
logger.error(
f'Job Failed[{self.name}]\n{output.stderr}')
self.state = Status.failed
return self.state
logger.info('Job (%s) Passed in %.2f seconds', self.name,
docker.duration)
except Exception as e:
self.state = Status.failed
raise e
self.state = Status.passed
return self.state
def _setup() -> None:
for binary in ('docker', 'git'):
try:
_exec([binary])
except FileNotFoundError:
logger.error('%s not installed', binary)
try:
os.mkdir(DockerContainer.workspace_dir)
except FileExistsError:
logger.info(
'workspace directory already exists at %s - this '
'is harmless providing it\'s what you wanted',
DockerContainer.workspace_dir)
pass
def _download_repo_build_config(repo_slab, ref) -> dict:
"""
last commit to master:
https://raw.githubusercontent.com/chestm007/Zeus-CI/master/.zeusci/config.yml
specific commit:
https://raw.githubusercontent.com/chestm007/Zeus-CI/142eb4bdbbc54371cbcc4a0000bd8eeea997d1f2/.zeusci/config.yml
specific tag:
https://raw.githubusercontent.com/chestm007/Zeus-CI/test-tag/.zeusci/config.yml
"""
url_format = 'https://raw.githubusercontent.com/{repo_slab}/{ref}/.zeusci/config.yml'
try:
response = urllib.request.urlopen(
url_format.format(repo_slab=repo_slab, ref=ref.split('/')[-1]))
except urllib.error.HTTPError:
logger.error('couldnt fetch build config')
return {}
if response.status == 200:
_config = yaml.load(response, yaml.Loader)
return _config
def main(repo_slab: str = None,
env_vars: List[str] = None,
threads: int = 1,
ref=None) -> bool:
if not any([repo_slab, env_vars, threads, ref]):
parser = argparse.ArgumentParser(
description='Run Zeus-CI jobs locally through docker')
parser.add_argument('--env',
type=str,
nargs='+',
help='K=V environment vars to pass to the test')
parser.add_argument(
'--threads',
type=int,
help='number of docker containers to run concurrently')
parser.add_argument('--ref',
type=str,
help='git ref(commit/tag) to checkout')
args = parser.parse_args()
if args.ref:
ref = args.ref
if args.threads:
threads = args.threads
if args.env:
env_vars = env_vars.extend(args.env) if env_vars else args.env
if not repo_slab:
repo_slab = repo_slab_of_cwd()
if not repo_slab:
logger.error(
'not in the root directory of a git repository, exiting')
sys.exit(1)
_setup()
clone_url = 'https://github.com/{}.git'.format(repo_slab)
_config = _download_repo_build_config(repo_slab, ref)
if not config:
return False
try:
_config['workflows'].pop('version')
except KeyError:
pass
env_vars.append('ZEUS_USERNAME={}'.format(repo_slab.split('/')[0]))
workflows = {
name: Workflow(name,
_config['jobs'],
spec,
clone_url,
threads,
env_vars=env_vars,
ref=ref)
for name, spec in _config['workflows'].items()
}
results = []
for workflow_name, workflow in workflows.items():
try:
results.append(workflow.run())
except Exception as e:
logger.error('%s: %s', workflow_name, e)
for status in (Status.error, Status.failed):
if any(map(lambda r: r == status, results)):
return status
return Status.passed
def _run_from_queue(self, queue):
signal.signal(signal.SIGINT, signal.SIG_IGN)
with self.database.get_session() as session:
try:
for build_id in iter(queue.get, None):
build = session.query(Build).filter_by(id=build_id).one()
# TODO: this must be called as soon as possible due to a race condition with populating the queue
build.status = Status.starting
session.commit()
github = Github(TokenAuth(build.repo.user.token))
github.update_status(build, GithubStatus.pending)
try:
ref = None
env_vars = build.repo.shell_ready_envvars()
if build.ref.startswith('refs/tags/'):
ref = build.ref.replace('refs/', '', 1)
env_vars.append('ZEUS_TAG={}'.format(
build.ref.replace('refs/tags/', '')))
env_vars.append('ZEUS_BRANCH={}'.format(
build.json['base_ref'].replace(
'refs/heads/', '')))
elif build.ref.startswith('refs/heads/'):
ref = build.json['after']
env_vars.append('ZEUS_TAG=""')
env_vars.append('ZEUS_BRANCH={}'.format(
build.ref.replace('refs/heads/', '')))
if not ref:
logger.error(
'error from worker thread: %s, refn not detected',
build.id)
else:
build.status = Status.running
session.commit()
logger.debug('executing runner.main process')
status = runner.main(
build.repo.name,
threads=self.config['runner_threads'],
ref=ref,
env_vars=env_vars)
logger.debug("runner main process completed")
if status == Status.passed:
logger.debug("build passed")
github.update_status(build,
GithubStatus.success)
build.status = Status.passed
else:
logger.debug("build failed")
github.update_status(build,
GithubStatus.failure)
build.status = Status.failed
session.commit()
except Exception as e:
github.update_status(build, GithubStatus.error)
build.status = Status.error
raise e
except Exception as e:
logger.error('error from worker thread: %s', exc_info=True)
raise e
except KeyboardInterrupt:
pass