def host_prepare(self):
"""Prepare the commands and environment to start execution.
Returns:
str: The command to execute.
"""
root = scm.get_git_root_folder()
if self.action['uses'] == 'sh':
cmd = self.action.get('runs', [])
if cmd:
cmd[0] = os.path.join(root, cmd[0])
cmd.extend(self.action.get('args', []))
if not self.dry_run:
os.chdir(root)
else:
cmd = self.action.get('runs', ['entrypoint.sh'])
cmd[0] = os.path.join('./', cmd[0])
cmd.extend(self.action.get('args', []))
if not self.dry_run:
if 'repo_dir' in self.action:
os.chdir(self.action['repo_dir'])
cmd[0] = os.path.join(self.action['repo_dir'], cmd[0])
else:
os.chdir(os.path.join(root, self.action['uses']))
cmd[0] = os.path.join(root, self.action['uses'], cmd[0])
return cmd
def instantiate_runners(runtime, wf, workspace, dry_run, skip_pull, wid):
"""Factory of ActionRunner instances, one for each action.
Note:
If the `uses` attribute startswith a './' and does not have
a `Dockerfile` in the referenced directory, we assume that
it is meant to be run on the Host machine and ignore the
runtime argument.
Same is the case when the `uses` attribute is equal to 'sh'.
"""
env = WorkflowRunner.get_workflow_env(wf, workspace)
for _, a in wf.action.items():
if a['uses'] == 'sh':
a['runner'] = HostRunner(
a, workspace, env, dry_run, skip_pull, wid)
continue
if a['uses'].startswith('./'):
if not os.path.isfile(
os.path.join(scm.get_git_root_folder(), a['uses'],
'Dockerfile')):
a['runner'] = HostRunner(
a, workspace, env, dry_run, skip_pull, wid)
continue
if runtime == 'docker':
a['runner'] = DockerRunner(
a, workspace, env, dry_run, skip_pull, wid)
elif runtime == 'singularity':
a['runner'] = SingularityRunner(
a, workspace, env, dry_run, skip_pull, wid)
def get_build_resources(self):
"""Parse the `uses` attribute and get the build
resources from them.
Args:
(bool, str, str): pull/build, image ref, the build source.
"""
build = True
image = None
build_source = None
if 'docker://' in self.action['uses']:
image = self.action['uses']
build = False
elif './' in self.action['uses']:
image = 'action/' + self.action['uses']
build_source = os.path.join(
scm.get_git_root_folder(), self.action['uses'])
else:
image = self.action['uses']
build_source = os.path.join(
self.action['repo_dir'], self.action['action_dir'])
return (build, image, build_source)
def cli(ctx, service):
"""Generates configuration files for distinct CI services. This command
needs to be executed on the root of your Git repository folder.
"""
if service not in ci_files:
log.fail("Unrecognized service " + service)
project_root = scm.get_git_root_folder()
if project_root != os.getcwd():
log.fail('This command needs to be executed on the root of your '
'Git project folder (where the .git/ folder is located).')
for ci_file, ci_file_content in pu.get_items(ci_files[service]):
ci_file_content = ci_file_content
ci_file = os.path.join(project_root, ci_file)
# create parent folder
if not os.path.isdir(os.path.dirname(ci_file)):
os.makedirs(os.path.dirname(ci_file))
# write content
with open(ci_file, 'w') as f:
f.write(ci_file_content)
log.info('Wrote {} configuration successfully.'.format(service))
def cli(ctx, service, install):
"""Generates configuration files for distinct CI services. This command
needs to be executed on the root of your Git repository folder.
"""
project_root = scm.get_git_root_folder()
if project_root != os.getcwd():
log.fail(
'This command needs to be executed on the root of your '
'Git project folder (where the .git/ folder is located).')
for ci_file, ci_file_content in pu.get_items(ci_files[service]):
# Prepare and write the CI config file.
ci_file = os.path.join(project_root, ci_file)
if not os.path.isdir(os.path.dirname(ci_file)):
os.makedirs(os.path.dirname(ci_file))
install_script_cmd = ''
if install:
if service == 'jenkins' or service == 'gitlab':
log.fail(
'Scaffolding of custom install scripts is not '
'supported for Jenkins and Gitlab CI. Include it '
'manually depending upon the CI\'s OS.')
elif service == 'travis':
install_script_cmd = ('before_install: scripts/'
'install_scripts.sh')
elif service == 'circle':
install_script_cmd = 'bash scripts/install_scripts.sh'
elif service == 'brigade':
install_script_cmd = '"bash scripts/install_scripts.sh",'
with open(ci_file, 'w') as f:
f.write(reformat(ci_file_content.safe_substitute(
{'install_scripts': install_script_cmd})))
# Prepare and Write the install scripts.
if install:
install = set(install)
install_script_file = os.path.join(
project_root, 'scripts', 'install_scripts.sh')
script_content = base_script_content
for runtime in install:
script_content += install_scripts_content[runtime]
if not os.path.isdir(os.path.dirname(install_script_file)):
os.makedirs(os.path.dirname(install_script_file))
with open(install_script_file, 'w') as f:
f.write(script_content)
st = os.stat(install_script_file)
os.chmod(install_script_file, st.st_mode | stat.S_IEXEC)
log.info('Wrote {} configuration successfully.'.format(service))
def instantiate_runners(engine, wf, workspace, dry_run, skip_pull, wid):
"""Factory of ActionRunner instances, one for each action.
Note:
If the `uses` attribute startswith a './' and does not have
a `Dockerfile` in the referenced directory, we assume that
it is meant to be run on the Host machine and ignore the
engine argument.
Same is the case when the `uses` attribute is equal to 'sh'.
Args:
engine(str): Identifier of the workflow being executed.
wf(popper.parser.workflow): Instance of the Workflow class.
workspace(str): Location of the workspace.
dry_run(bool): True if workflow flag is being dry-run.
skip_pull(bool): True if pulling action has to be skipped.
wid(str):
Returns:
None
"""
env = WorkflowRunner.get_workflow_env(wf, workspace)
for _, a in wf.action.items():
if a['uses'] == 'sh':
a['runner'] = HostRunner(a, workspace, env, dry_run, skip_pull,
wid)
continue
if a['uses'].startswith('./'):
if not os.path.isfile(
os.path.join(scm.get_git_root_folder(), a['uses'],
'Dockerfile')):
a['runner'] = HostRunner(a, workspace, env, dry_run,
skip_pull, wid)
continue
if engine == 'docker':
a['runner'] = DockerRunner(a, workspace, env, dry_run,
skip_pull, wid)
elif engine == 'singularity':
a['runner'] = SingularityRunner(a, workspace, env, dry_run,
skip_pull, wid)
elif engine == 'vagrant':
a['runner'] = VagrantRunner(a, workspace, env, dry_run,
skip_pull, wid)
def cli(ctx, service, with_singularity):
"""Generates configuration files for distinct CI services. This command
needs to be executed on the root of your Git repository folder.
"""
if service not in ci_files:
log.fail("Unrecognized service " + service)
project_root = scm.get_git_root_folder()
if project_root != os.getcwd():
log.fail('This command needs to be executed on the root of your '
'Git project folder (where the .git/ folder is located).')
for ci_file, ci_file_content in pu.get_items(ci_files[service]):
ci_file_content = ci_file_content
ci_file = os.path.join(project_root, ci_file)
# Create parent folder
if not os.path.isdir(os.path.dirname(ci_file)):
os.makedirs(os.path.dirname(ci_file))
# Customize content
scripts = []
if with_singularity:
if service in ['jenkins', 'gitlab']:
log.fail('Scaffolding of Singularity install script is not '
'supported for Jenkins and Gitlab CI. Include it '
'manually depending upon the CI\'s OS.')
scripts.append(install_scripts['singularity'])
if scripts:
scripts = ' && '.join(scripts)
if service == 'travis':
scripts = '- ' + scripts
if service == 'brigade':
scripts = '"' + scripts + '",'
else:
scripts = ''
# Write content
with open(ci_file, 'w') as f:
f.write(
reformat(
ci_file_content.safe_substitute(
{'install_scripts': scripts})))
log.info('Wrote {} configuration successfully.'.format(service))
def get_build_resources(self):
"""Parse the `uses` attribute and get the build resources from them.
Args:
None
Returns:
bool: pull/build, image ref, the build source
"""
build = True
image = None
build_source = None
if 'docker://' in self.action['uses']:
image = self.action['uses'].replace('docker://', '')
if ':' not in image:
image += ":latest"
build = False
elif './' in self.action['uses']:
action_dir = os.path.basename(self.action['uses'].replace(
'./', ''))
if self.env['GITHUB_REPOSITORY'] == 'unknown':
repo_id = ''
else:
repo_id = self.env['GITHUB_REPOSITORY']
if action_dir:
repo_id += '/'
image = repo_id + action_dir + ':' + self.env['GITHUB_SHA']
build_source = os.path.join(scm.get_git_root_folder(),
self.action['uses'])
else:
_, _, user, repo, _, version = scm.parse(self.action['uses'])
image = '{}/{}:{}'.format(user, repo, version)
build_source = os.path.join(self.action['repo_dir'],
self.action['action_dir'])
image = image.lower()
return (build, image, build_source)
def test_get_git_root_folder(self):
root_folder = scm.get_git_root_folder()
self.assertEqual(root_folder, '/tmp/test_folder/github-actions-demo')
def run(self, reuse=False):
root = scm.get_git_root_folder()
if self.action['uses'] == 'sh':
cmd = self.action.get('runs', [])
if cmd:
cmd[0] = os.path.join(root, cmd[0])
cmd.extend(self.action.get('args', []))
if not self.dry_run:
os.chdir(root)
else:
cmd = self.action.get('runs', ['entrypoint.sh'])
cmd[0] = os.path.join('./', cmd[0])
cmd.extend(self.action.get('args', []))
if not self.dry_run:
if 'repo_dir' in self.action:
os.chdir(self.action['repo_dir'])
cmd[0] = os.path.join(self.action['repo_dir'], cmd[0])
else:
os.chdir(os.path.join(root, self.action['uses']))
cmd[0] = os.path.join(root, self.action['uses'], cmd[0])
os.environ.update(self.action.get('env', {}))
log.info('{}[{}] {}'.format(self.msg_prefix, self.action['name'],
' '.join(cmd)))
if self.dry_run:
return
ecode = 0
try:
log.debug('Executing: {}'.format(' '.join(cmd)))
p = Popen(' '.join(cmd),
stdout=PIPE,
stderr=STDOUT,
shell=True,
universal_newlines=True,
preexec_fn=os.setsid)
popper.cli.process_list.append(p.pid)
log.debug('Reading process output')
for line in iter(p.stdout.readline, ''):
line_decoded = pu.decode(line)
log.info(line_decoded[:-1])
p.wait()
ecode = p.poll()
log.debug('Code returned by process: {}'.format(ecode))
except CalledProcessError as ex:
msg = "Command '{}' failed: {}".format(cmd, ex)
ecode = ex.returncode
log.info(msg)
finally:
log.info()
# remove variables that we added to the environment
for i in self.action.get('env', {}):
os.environ.pop(i)
os.chdir(self.cwd)
if ecode != 0:
log.fail("Action '{}' failed.".format(self.action['name']))