def __skip_steps(wf_data, skip_list=[]):
if not skip_list:
return
filtered_list = []
used = {}
for step in wf_data["steps"]:
if step["id"] in skip_list:
used[step["id"]] = 1
continue
filtered_list.append(step)
wf_data["steps"] = filtered_list
if len(used) != len(skip_list):
log.fail("Not all skipped steps exist in the workflow.")
def _clone_repos(self, wf):
"""Clone steps that reference a repository.
Args:
wf(popper.parser.workflow): Instance of the Workflow class.
config.dry_run(bool): True if workflow flag is being dry-run.
config.skip_clone(bool): True if clonning step has to be skipped.
config.wid(str): id of the workspace
Returns:
None
"""
repo_cache = os.path.join(WorkflowRunner._setup_base_cache(),
self._config.wid)
cloned = set()
infoed = False
for _, a in wf.steps.items():
uses = a['uses']
if ('docker://' in uses or 'shub://' in uses
or 'library://' in uses or './' in uses or uses == 'sh'):
continue
url, service, user, repo, step_dir, version = scm.parse(a['uses'])
repo_dir = os.path.join(repo_cache, service, user, repo)
a['repo_dir'] = repo_dir
a['step_dir'] = step_dir
if self._config.dry_run:
continue
if self._config.skip_clone:
if not os.path.exists(repo_dir):
log.fail(f"Expecting folder '{repo_dir}' not found.")
continue
if not infoed:
log.info('[popper] Cloning step repositories')
infoed = True
if f'{user}/{repo}' in cloned:
continue
log.info(f'[popper] - {url}/{user}/{repo}@{version}')
scm.clone(url, user, repo, repo_dir, version)
cloned.add(f'{user}/{repo}')
def parse(url):
"""Method to parse the git url.
Args:
url (str): The url in string format.
Returns:
tuple(service_url, service, user, repo, action_dir, version)
"""
if url.startswith('ssh://'):
log.fail('The ssh protocol is not supported yet.')
if url.endswith('.git'):
url = url[:-4]
pattern = re.compile(
r'^(http://|https://|git@)?(?:(\w+\.\w+)(?:\/|\:))?'
r'([\w\-]+)(?:\/([^\@^\/]+)\/?([^\@]+)?(?:\@([\w\W]+))?)$')
try:
protocol, service, user, repo, action_dir, version = pattern.search(
url).groups()
except AttributeError:
log.fail('Invalid url. The url should be in any of the 3 forms: \n'
'1) https://github.com/user/repo/path/to/action@version \n'
'2) gitlab.com/user/repo/path/to/action@version \n'
'3) user/repo/path/to/action@version')
if not service:
service = 'github.com'
if not protocol:
protocol = 'https://'
if not action_dir:
action_dir = ''
service_url = protocol + service
log.debug('parse("{}"):'.format(url))
log.debug(' service_url: {}'.format(service_url))
log.debug(' service: {}'.format(service))
log.debug(' user: {}'.format(user))
log.debug(' repo: {}'.format(repo))
log.debug(' action_dir: {}'.format(action_dir))
log.debug(' version: {}'.format(version))
return service_url, service, user, repo, action_dir, version
def check_for_unreachable_steps(self, skip=None):
"""Validates a workflow by checking for unreachable nodes / gaps in the
workflow.
Args:
skip(list, optional): The list of steps to skip if applicable.
(Default value = None)
Returns:
None
"""
if not skip or self.wf_fmt == 'yml':
# noop
return
def _traverse(entrypoint, reachable, steps):
"""
Args:
entrypoint(set): Set containing the entry point of part of the
workflow.
reachable(set): Set containing all the reachable parts of
workflow.
steps(dict): Dictionary containing the identifier of the
workflow and its description.
Returns:
None
"""
for node in entrypoint:
reachable.add(node)
_traverse(steps[node].get('next', []), reachable, steps)
reachable = set()
skipped = set(self.props.get('skip_list', []))
steps = set(map(lambda a: a[0], self.steps.items()))
_traverse(self.root, reachable, self.steps)
unreachable = steps - reachable
if unreachable - skipped:
if skip:
log.fail(
f'Unreachable step(s): {", ".join(unreachable-skipped)}.')
else:
log.warning(f'Unreachable step(s): {", ".join(unreachable)}.')
for a in unreachable:
self.steps.pop(a)
def download_actions(wf, dry_run, skip_clone, wid):
"""Clone actions that reference a repository.
Args:
wf(popper.parser.workflow): Instance of the Workflow class.
dry_run(bool): True if workflow flag is being dry-run.
skip_clone(bool): True if clonning action has to be skipped.
wid(str):
Returns:
None
"""
actions_cache = os.path.join(pu.setup_base_cache(), 'actions', wid)
cloned = set()
infoed = False
for _, a in wf.action.items():
if ('docker://' in a['uses'] or './' in a['uses']
or a['uses'] == 'sh'):
continue
url, service, user, repo, action_dir, version = scm.parse(
a['uses'])
repo_dir = os.path.join(actions_cache, service, user, repo)
a['repo_dir'] = repo_dir
a['action_dir'] = action_dir
if dry_run:
continue
if skip_clone:
if not os.path.exists(repo_dir):
log.fail('The required action folder \'{}\' was not '
'found locally.'.format(repo_dir))
continue
if not infoed:
log.info('[popper] Cloning action repositories')
infoed = True
if '{}/{}'.format(user, repo) in cloned:
continue
log.info('[popper] - {}/{}/{}@{}'.format(url, user, repo, version))
scm.clone(url, user, repo, repo_dir, version)
cloned.add('{}/{}'.format(user, repo))
def handle_exit(self, ecode):
"""Exit handler for the action.
Args:
ecode (int): The exit code of the action's process.
"""
if ecode == 0:
log.info("Action '{}' ran successfully !".format(
self.action['name']))
elif ecode == 78:
log.info("Action '{}' ran successfully !".format(
self.action['name']))
os.kill(os.getpid(), signal.SIGUSR1)
else:
log.fail("Action '{}' failed !".format(self.action['name']))
def cli(ctx, service, wfile):
"""Generates configuration files for distinct CI services. This command
needs to be executed on the root of your Git repository folder.
"""
if not os.path.exists('.git'):
log.fail('This command needs to be executed on the root of your '
'Git project folder (where the .git/ folder is located).')
for ci_file, ci_file_content in ci_files[service].items():
ci_file = os.path.join(os.getcwd(), ci_file)
os.makedirs(os.path.dirname(ci_file), exist_ok=True)
with open(ci_file, 'w') as f:
f.write(ci_file_content.format(wfile))
log.info(f'Wrote {service} configuration successfully.')
def _clone_repos(self, wf):
"""Clone steps that reference a repository.
Args:
wf(popper.parser.workflow): Instance of the Workflow class.
config.dry_run(bool): True if workflow flag is being dry-run.
config.skip_clone(bool): True if clonning step has to be skipped.
config.wid(str): id of the workspace
Returns:
None
"""
# cache directory for this workspace
wf_cache_dir = os.path.join(self._config.cache_dir, self._config.wid)
os.makedirs(wf_cache_dir, exist_ok=True)
cloned = set()
infoed = False
for step in wf.steps:
if ("docker://" in step.uses or "shub://" in step.uses
or "library://" in step.uses or "./" in step.uses
or step.uses == "sh"):
continue
url, service, user, repo, _, version = scm.parse(step.uses)
repo_dir = os.path.join(wf_cache_dir, service, user, repo)
if self._config.dry_run:
continue
if self._config.skip_clone:
if not os.path.exists(repo_dir):
log.fail(f"Expecting folder '{repo_dir}' not found.")
continue
if not infoed:
log.info("Cloning step repositories",
extra={"pretag": "[popper]"})
infoed = True
if f"{user}/{repo}" in cloned:
continue
log.info(f"- {url}/{user}/{repo}@{version}",
extra={"pretag": "[popper]"})
scm.clone(url, user, repo, repo_dir, version)
cloned.add(f"{user}/{repo}")
def docker_pull(self, img):
"""Pull an image from Dockerhub.
Args:
img (str): The image reference to pull.
"""
if not self.skip_pull:
log.info('{}[{}] docker pull {}'.format(self.msg_prefix,
self.action['name'], img))
if self.dry_run:
return
self.d_client.images.pull(repository=img)
else:
if not self.docker_image_exists(img):
log.fail('The required docker image \'{}\' was not found '
'locally.'.format(img))
def __init__(self, init_podman_client=True, **kw):
super(PodmanRunner, self).__init__(**kw)
self._spawned_containers = set()
if not init_podman_client:
return
try:
_, _, self._p_info = HostRunner._exec_cmd(["podman", "info"], logging=False)
self._p_version = HostRunner._exec_cmd(["podman", "version"], logging=False)
except Exception as e:
log.debug(f"Podman error: {e}")
log.fail("Unable to connect to podman, is it installed?")
log.debug(f"Podman info: {pu.prettystr(self._p_info)}")
def make_gh_request(url, err=True, msg=None):
"""Method for making GET requests to GitHub API.
Args:
url (str): URL on which the API request is to be made.
err (bool): Checks if an error message needs to be printed or not.
msg (str): Error message to be printed for a failed request.
Returns:
Response object: Contains a server's response to an HTTP request.
"""
if not msg:
msg = ("Unable to connect. Please check your network connection.")
response = requests.get(url)
if err and response.status_code != 200:
log.fail(msg)
else:
return response
def __init__(self, init_docker_client=True, **kw):
super(DockerRunner, self).__init__(**kw)
self._spawned_containers = set()
self._d = None
if not init_docker_client:
return
try:
self._d = docker.from_env()
self._d.version()
except Exception as e:
log.debug(f'Docker error: {e}')
log.fail(f'Unable to connect to the docker daemon.')
log.debug(f'Docker info: {pu.prettystr(self._d.info())}')
def cli(ctx, wfile):
"""Generates a minimal workflow that can be used as starting point."""
main_workflow_content = """steps:
- uses: "popperized/bin/sh@master"
args: ["ls"]
- uses: "docker://alpine:3.11"
args: ["ls"]
"""
if os.path.exists(wfile):
log.fail(f"File {wfile} already exists")
with open(wfile, "w") as f:
f.write(main_workflow_content)
log.info("Successfully generated a workflow scaffold.")
def run(self, step):
"""Execute a step in a kubernetes cluster."""
self._pod_name = self._base_pod_name + f"-{step.id}"
needs_build, _, img, tag, _ = self._get_build_info(step)
if needs_build:
log.fail(f"Cannot build ")
image = f"{img}:{tag}"
m = f"[{step.id}] kubernetes run {self._namespace}.{self._pod_name}"
log.info(m)
if self._config.dry_run:
return 0
ecode = 1
try:
if not self._vol_claim_created:
if not self._vol_claim_exists():
self._vol_claim_create()
self._vol_claim_created = True
if not self._init_pod_created:
e, self._pod_host_node = self._init_pod_schedule()
if e:
raise Exception("None of the nodes are schedulable.")
self._copy_ctx()
self._init_pod_delete()
self._init_pod_created = True
self._pod_create(step, image, self._pod_host_node)
self._pod_read_log()
ecode = self._pod_exit_code()
except Exception as e:
log.fail(e)
finally:
self._pod_delete()
log.debug(f"returning with {ecode}")
return ecode
def check_secrets(self):
"""Checks whether the secrets defined in the action block is
set in the execution environment or not.
Note:
When the environment variable `CI` is set to `true`,
then the execution fails if secrets are not defined
else it prompts the user to enter the environment vars
during the time of execution itself.
"""
if self.dry_run or self.skip_secrets_prompt:
return
for _, a in self.wf.actions:
for s in a.get('secrets', []):
if s not in os.environ:
if os.environ.get('CI') == "true":
log.fail('Secret {} not defined'.format(s))
else:
val = input("Enter the value for {0}:\n".format(s))
os.environ[s] = val
def run(self, reuse=False):
"""
Args:
reuse: True if existing containers are to be reused.
(Default value = False)
Returns:
None
"""
if reuse:
log.fail('--reuse flag is not supported for actions running '
'on the host.')
cmd = self.host_prepare()
self.prepare_environment(set_env=True)
e = self.host_start(cmd)
self.remove_environment()
self.handle_exit(e)
def make_gh_request(url, err=True, msg=None):
"""Method for making GET requests to GitHub API
Args:
url (str): URL on which the API request is to be made.
err (bool): Checks if an error message needs to be printed or not.
msg (str): Error message to be printed for a failed request.
Returns:
Response object: contains a server's response to an HTTP request.
"""
if not msg:
msg = (
"Unable to connect. If your network is working properly, you might"
" have reached Github's API request limit. Try adding a Github API"
" token to the 'POPPER_GITHUB_API_TOKEN' variable.")
response = requests.get(url, headers=get_gh_headers())
if err and response.status_code != 200:
log.fail(msg)
else:
return response
def get_sha():
"""Runs git rev-parse --short HEAD and returns result.
This function returns 'unknown' if the project folder
is not a git repo. It fails, when the project folder is a
git repo but doesn't have any commit.
Returns:
str: The sha of the head commit or 'unknown'.
"""
repo = init_repo_object()
if repo:
try:
return repo.git.rev_parse(repo.head.object.hexsha, short=True)
except ValueError as e:
log.debug(e)
log.fail(
'Could not obtain revision of repository located at {}'.format(
get_git_root_folder()))
else:
return 'unknown'
def check_secrets(wf, dry_run, skip_secrets_prompt):
"""Checks whether the secrets defined in the action block is
set in the execution environment or not.
Note:
When the environment variable `CI` is set to `true`,
then the execution fails if secrets are not defined
else it prompts the user to enter the environment vars
during the time of execution itself.
"""
if dry_run or skip_secrets_prompt:
return
for _, a in wf.action.items():
for s in a.get('secrets', []):
if s not in os.environ:
if os.environ.get('CI') == 'true':
log.fail('Secret {} not defined'.format(s))
else:
val = getpass.getpass(
'Enter the value for {} : '.format(s))
os.environ[s] = val
def __load_config_file(config_file):
"""Validate and parse the engine configuration file.
Args:
config_file(str): Path to the file to be parsed.
Returns:
dict: Engine configuration.
"""
if isinstance(config_file, dict):
return config_file
if not config_file:
return dict()
if not os.path.exists(config_file):
log.fail(f"File {config_file} was not found.")
if not config_file.endswith(".yml"):
log.fail("Configuration file must be a YAML file.")
with open(config_file, "r") as cf:
data = yaml.load(cf, Loader=yaml.Loader)
if not data:
log.fail("Configuration file is empty.")
return data
def singularity_build_from_image(self, image, container_path):
"""Build a container from Docker image.
Args:
image (str): The docker image to build the container from.
container_path (str): The path of the built container.
"""
container = os.path.basename(container_path)
if not self.skip_pull:
log.info('{}[{}] singularity pull {} {}'.format(
self.msg_prefix, self.action['name'], container, image))
if not self.dry_run:
if not self.singularity_exists(container_path):
s_client.pull(image=image,
name=container,
pull_folder=os.path.dirname(container_path))
else:
if not self.singularity_exists(container_path):
log.fail(
'The required singularity container \'{}\' was not found '
'locally.'.format(container_path))
def __apply_substitution(wf_element, k, v, used_registry):
if isinstance(wf_element, str):
if k in wf_element:
wf_element.replace(k, v)
used_registry[k] = 1
elif isinstance(wf_element, list):
# we assume list of strings
for i, e in enumerate(wf_element):
if k in e:
wf_element[i].replace(k, v)
used_registry[k] = 1
elif isinstance(wf_element, dict):
# we assume list of strings
for ek in wf_element:
if k in ek:
log.fail(
"Substitutions only allowed on keys of dictionaries")
if k in wf_element[ek]:
wf_element[ek].replace(k, v)
used_registry[k] = 1
def run_pipeline(action, wfile, workspace, reuse, dry_run, parallel):
pipeline = WorkflowRunner(wfile, workspace, dry_run, reuse, parallel)
# Saving workflow instance for signal handling
popper.cli.interrupt_params = pipeline
if reuse:
log.warn("Using --reuse ignores any changes made to an action's logic "
"or to an action block in the .workflow file.")
if parallel:
if sys.version_info[0] < 3:
log.fail('--parallel is only supported on Python3')
log.warn("Using --parallel may result in interleaved output. "
"You may use --quiet flag to avoid confusion.")
pipeline.run(action, reuse, parallel)
if action:
log.info('Action "{}" finished successfully.'.format(action))
else:
log.info('Workflow finished successfully.')
def cli(ctx, action, wfile, workspace, reuse, recursive, quiet, debug, dry_run,
parallel, log_file):
"""Executes one or more pipelines and reports on their status.
"""
popper.scm.get_git_root_folder()
level = 'ACTION_INFO'
if quiet:
level = 'INFO'
if debug:
level = 'DEBUG'
log.setLevel(level)
if log_file:
logging.add_log(log, log_file)
if recursive:
wfile_list = pu.find_recursive_wfile()
if not wfile_list:
log.fail("Recursive search couldn't find any .workflow files ")
for wfile in wfile_list:
log.info("Found and running workflow at " + wfile)
run_pipeline(action, wfile, workspace, reuse, dry_run, parallel)
else:
run_pipeline(action, wfile, workspace, reuse, dry_run, parallel)
def run(self, step):
"""Execute the given step in docker."""
cid = pu.sanitized_name(step.id, self._config.wid)
container = self._find_container(cid)
if not container and self._config.reuse:
log.fail(
f"Cannot find an existing container for step '{step.id}' to be reused"
)
if container and not self._config.reuse and not self._config.dry_run:
container.remove(force=True)
container = None
if not container and not self._config.reuse:
container = self._create_container(cid, step)
log.info(f"[{step.id}] docker start")
if self._config.dry_run:
return 0
self._spawned_containers.add(container)
try:
container.start()
if self._config.pty:
dockerpty.start(self._d.api, container.id)
else:
cout = container.logs(stream=True)
for line in cout:
log.step_info(line.decode().rstrip())
e = container.wait()["StatusCode"]
except Exception as exc:
log.fail(exc)
return e
def skip_steps(wf, skip_list=[]):
"""Removes the steps to be skipped from the workflow graph and return
a new `Workflow` object.
Args:
wf(Workflow): The workflow object to operate upon.
skip_list(list): List of steps to be skipped.
(Default value = list())
Returns:
Workflow : The updated workflow object.
"""
if not skip_list:
# noop
return wf
workflow = deepcopy(wf)
for sa_name in skip_list:
if sa_name not in workflow.steps:
log.fail(f"Referenced step '{sa_name} missing.")
sa_block = workflow.steps[sa_name]
# Clear up all connections from sa_block
sa_block.get('next', set()).clear()
del sa_block.get('needs', list())[:]
# Handle skipping of root step's
if sa_name in workflow.root:
workflow.root.remove(sa_name)
# Handle skipping of non-root step's
for a_name, a_block in workflow.steps.items():
if sa_name in a_block.get('next', set()):
a_block['next'].remove(sa_name)
if sa_name in a_block.get('needs', list()):
a_block['needs'].remove(sa_name)
workflow.props['skip_list'] = list(skip_list)
return workflow
def _complete_graph_util(self, entrypoint, nwd):
"""A GHA workflow is defined by specifying edges that point to the
previous nodes they depend on. To make the workflow easier to process,
we add forward edges. This also obtains the root nodes.
Args:
entrypoint (list): List of nodes from where to start
generating the graph.
nwd (set) : Set of nodes without dependencies.
"""
for node in entrypoint:
if self._workflow['action'].get(node, None):
if self._workflow['action'][node].get('needs', None):
for n in self._workflow['action'][node]['needs']:
self._complete_graph_util([n], nwd)
if not self._workflow['action'][n].get('next', None):
self._workflow['action'][n]['next'] = set()
self._workflow['action'][n]['next'].add(node)
else:
nwd.add(node)
else:
log.fail('Action {} doesn\'t exist.'.format(node))
def find_default_wfile(wfile):
"""
Used to find `main.workflow` in $PWD or in `.github`
And returns error if not found
Returns:
path of wfile
"""
if not wfile:
if os.path.isfile("main.workflow"):
wfile = "main.workflow"
elif os.path.isfile(".github/main.workflow"):
wfile = ".github/main.workflow"
if not wfile:
log.fail("Files {} or {} not found.".format("./main.workflow",
".github/main.workflow"))
if not os.path.isfile(wfile):
log.fail("File {} not found.".format(wfile))
exit(1)
return wfile
def cli(ctx, action, wfile, skip_clone, skip_pull, skip, workspace, reuse,
recursive, quiet, debug, dry_run, parallel, log_file,
with_dependencies, on_failure):
"""Executes one or more pipelines and reports on their status.
"""
popper.scm.get_git_root_folder()
level = 'ACTION_INFO'
if quiet:
level = 'INFO'
if debug:
level = 'DEBUG'
log.setLevel(level)
if log_file:
logging.add_log(log, log_file)
if os.environ.get('CI') == 'true':
log.info("Running in CI environment.")
if recursive:
log.warning('When CI variable is set, --recursive is ignored.')
wfile_list = pu.find_recursive_wfile()
wfile_list = workflows_from_commit_message(wfile_list)
else:
if recursive:
if action:
log.fail(
"An 'action' argument and the --recursive flag cannot be "
"both given.")
wfile_list = pu.find_recursive_wfile()
else:
wfile_list = [wfile]
if not wfile_list:
log.fail("No workflow to execute.")
for wfile in wfile_list:
wfile = pu.find_default_wfile(wfile)
log.info("Found and running workflow at " + wfile)
run_pipeline(action, wfile, skip_clone, skip_pull, skip, workspace,
reuse, dry_run, parallel, with_dependencies, on_failure)
def workflows_from_commit_message(workflows):
head_commit = scm.get_head_commit()
if not head_commit:
return workflows
msg = head_commit.message
if 'Merge' in msg:
log.info("Merge detected. Reading message from merged commit.")
if len(head_commit.parents) == 2:
msg = head_commit.parents[1].message
if 'popper:skip[' in msg:
log.info("Found 'popper:skip' keyword.")
re_expr = r'popper:skip\[(.+?)\]'
elif 'popper:whitelist[' in msg:
log.info("Found 'popper:whitelist' keyword.")
re_expr = r'popper:whitelist\[(.+?)\]'
else:
return workflows
try:
workflow_list = re.search(re_expr, msg).group(1).split(',')
except AttributeError:
log.fail("Error parsing commit message keyword.")
if 'skip' in re_expr:
for wf in workflow_list:
if wf in workflows:
workflows.remove(wf)
else:
log.warn('Workflow {} was not found.'.format(wf))
else:
workflows = workflow_list
print('Only running workflows: {}'.format(', '.join(workflows)))
return workflows