def test_get_branch_in_detached_head_state(self):
repo = self.mk_repo()
repo.git.checkout("HEAD~1")
os.environ["TRAVIS_BRANCH"] = "travis"
self.assertEqual("travis", scm.get_branch(repo))
os.environ.pop("TRAVIS_BRANCH")
os.environ["GIT_BRANCH"] = "jenkins"
self.assertEqual("jenkins", scm.get_branch(repo))
os.environ.pop("GIT_BRANCH")
os.environ["CIRCLE_BRANCH"] = "circle"
self.assertEqual("circle", scm.get_branch(repo))
os.environ.pop("CIRCLE_BRANCH")
os.environ["CI_COMMIT_REF_NAME"] = "gitlab"
self.assertEqual("gitlab", scm.get_branch(repo))
os.environ.pop("CI_COMMIT_REF_NAME")
self.assertEqual(scm.get_sha(repo), scm.get_branch(repo))
# None given as arg
self.assertIsNone(scm.get_sha(None))
self.assertIsNone(scm.get_sha(None, short=8))
self.assertIsNone(scm.get_branch(None))
def load(
engine_name=None,
resman_name=None,
config_file=None,
workspace_dir=os.getcwd(),
reuse=False,
dry_run=False,
quiet=False,
skip_pull=False,
skip_clone=False,
pty=False,
allow_undefined_secrets_in_ci=False,
):
"""Loads and creates a configuration, represented by a frozen Box
"""
workspace_dir = os.path.realpath(workspace_dir)
repo = scm.new_repo(workspace_dir)
# path to cache
if os.environ.get("POPPER_CACHE_DIR", None):
cache_dir = os.environ["POPPER_CACHE_DIR"]
else:
cache_dir_default = os.path.join(os.environ["HOME"], ".cache")
cache_dir = os.environ.get("XDG_CACHE_HOME", cache_dir_default)
cache_dir = os.path.join(cache_dir, "popper")
from_file = ConfigLoader.__load_config_from_file(
config_file, engine_name, resman_name
)
pp_config = {
"workspace_dir": workspace_dir,
"reuse": reuse,
"dry_run": dry_run,
"quiet": quiet,
"skip_pull": skip_pull,
"skip_clone": skip_clone,
"pty": pty,
"allow_undefined_secrets_in_ci": allow_undefined_secrets_in_ci,
# if no git repository exists in workspace_dir or its parents, the repo
# variable is None and all git_* variables are assigned to 'na'
"repo": repo,
"git_commit": scm.get_sha(repo),
"git_sha_short": scm.get_sha(repo, short=7),
"git_branch": scm.get_branch(repo),
"git_tag": scm.get_tag(repo),
"git_remote_origin_url": scm.get_remote_url(repo),
# wid is used to associate a unique id to this workspace. This is then
# used by runners to name resources in a way that there is no name
# clash between concurrent workflows being executed
"wid": shake_256(workspace_dir.encode("utf-8")).hexdigest(4),
"cache_dir": cache_dir,
"engine_name": from_file["engine_name"],
"resman_name": from_file["resman_name"],
"engine_opts": from_file["engine_opts"],
"resman_opts": from_file["resman_opts"],
}
return Box(pp_config, default_box=True, frozen_box=True)
def __init__(self,
engine_name=None,
resman_name=None,
config_file=None,
workspace_dir=os.getcwd(),
reuse=False,
dry_run=False,
quiet=False,
skip_pull=False,
skip_clone=False):
self.workspace_dir = os.path.realpath(workspace_dir)
self.reuse = reuse
self.dry_run = dry_run
self.quiet = quiet
self.skip_pull = skip_pull
self.skip_clone = skip_clone
self.repo = scm.new_repo()
self.workspace_sha = scm.get_sha(self.repo)
wid = shake_256(self.workspace_dir.encode('utf-8')).hexdigest(4)
self.wid = wid
from_file = self._load_config_from_file(config_file, engine_name,
resman_name)
self.engine_name = from_file['engine_name']
self.resman_name = from_file['resman_name']
self.engine_opts = from_file['engine_opts']
self.resman_opts = from_file['resman_opts']
def __init__(self, wfile, workspace, quiet, debug, dry_run):
wfile = pu.find_default_wfile(wfile)
with open(wfile, 'r') as fp:
self.wf = hcl.load(fp)
self.workspace = workspace
self.debug = debug
if debug:
self.quiet = False
else:
self.quiet = quiet
self.dry_run = dry_run
self.actions_cache_path = os.path.join('/', 'tmp', 'actions')
self.validate_syntax()
self.check_secrets()
self.normalize()
self.complete_graph()
self.env = {
'GITHUB_WORKSPACE': self.workspace,
'GITHUB_WORKFLOW': self.wf['name'],
'GITHUB_ACTOR': 'popper',
'GITHUB_REPOSITORY': '{}/{}'.format(scm.get_user(),
scm.get_name()),
'GITHUB_EVENT_NAME': self.wf['on'],
'GITHUB_EVENT_PATH': '/{}/{}'.format(self.workspace,
'workflow/event.json'),
'GITHUB_SHA': scm.get_sha(),
'GITHUB_REF': scm.get_ref()
}
for e in dict(self.env):
self.env.update({e.replace('GITHUB_', 'POPPER_'): self.env[e]})
def run(self, action_name=None, reuse=False, parallel=False):
"""Run the pipeline or a specific action"""
os.environ['WORKSPACE'] = self.workspace
if scm.get_user():
repo_id = '{}/{}'.format(scm.get_user(), scm.get_name())
else:
repo_id = 'unknown'
self.env = {
'GITHUB_WORKSPACE': self.workspace,
'GITHUB_WORKFLOW': self.wf.name,
'GITHUB_ACTOR': 'popper',
'GITHUB_REPOSITORY': repo_id,
'GITHUB_EVENT_NAME': self.wf.on,
'GITHUB_EVENT_PATH': '/{}/{}'.format(self.workspace,
'workflow/event.json'),
'GITHUB_SHA': scm.get_sha(),
'GITHUB_REF': scm.get_ref()
}
for e in dict(self.env):
self.env.update({e.replace('GITHUB_', 'POPPER_'): self.env[e]})
self.download_actions()
self.instantiate_runners()
if action_name:
self.wf.get_runner(action_name).run(reuse)
else:
for s in self.wf.get_stages():
self.run_stage(s, reuse, parallel)
def get_workflow_env(wf, workspace):
"""Updates the Popper environment variable with Github environment
variables.
Args:
wf(popper.parser.Workflow): Instance of the Workflow class.
workspace(str): Location of the workspace.
Returns:
dict: dictionary containing Github variables.
"""
if scm.get_user():
repo_id = '{}/{}'.format(scm.get_user(), scm.get_name())
else:
repo_id = 'unknown'
env = {
'HOME': os.environ['HOME'],
'GITHUB_WORKFLOW': wf.name,
'GITHUB_ACTION': '',
'GITHUB_ACTOR': 'popper',
'GITHUB_REPOSITORY': repo_id,
'GITHUB_EVENT_NAME': wf.on,
'GITHUB_EVENT_PATH': '/tmp/github_event.json',
'GITHUB_WORKSPACE': workspace,
'GITHUB_SHA': scm.get_sha(),
'GITHUB_REF': scm.get_ref()
}
for e in dict(env):
env.update({e.replace('GITHUB_', 'POPPER_'): env[e]})
return env
def run(self,
action,
skip_clone,
skip_pull,
skip,
workspace,
reuse,
dry_run,
parallel,
with_dependencies,
skip_secrets_prompt=False):
"""Run the pipeline or a specific action"""
os.environ['WORKSPACE'] = workspace
if scm.get_user():
repo_id = '{}/{}'.format(scm.get_user(), scm.get_name())
else:
repo_id = 'unknown'
if with_dependencies and (not action):
log.fail('`--with-dependencies` can be used only with '
'action argument.')
if skip and action:
log.fail('`--skip` can\'t be used when action argument '
'is passed.')
new_wf = deepcopy(self.wf)
if skip:
new_wf = self.wf.skip_actions(skip)
if action:
new_wf = self.wf.filter_action(action, with_dependencies)
new_wf.check_for_unreachable_actions(skip)
env = {
'GITHUB_WORKSPACE': workspace,
'GITHUB_WORKFLOW': new_wf.name,
'GITHUB_ACTOR': 'popper',
'GITHUB_REPOSITORY': repo_id,
'GITHUB_EVENT_NAME': new_wf.on,
'GITHUB_EVENT_PATH': '/tmp/github_event.json',
'GITHUB_SHA': scm.get_sha(),
'GITHUB_REF': scm.get_ref()
}
for e in dict(env):
env.update({e.replace('GITHUB_', 'POPPER_'): env[e]})
self.check_secrets(new_wf, dry_run, skip_secrets_prompt)
self.download_actions(new_wf, dry_run, skip_clone)
self.instantiate_runners(new_wf, workspace, env, dry_run, skip_pull)
for s in new_wf.get_stages():
self.run_stage(new_wf, s, reuse, parallel)
def test_empty_repo(self):
tempdir = tempfile.mkdtemp()
repo = git.Repo.init(tempdir)
self.assertTrue(scm.is_empty(repo))
self.assertIsNone(scm.get_sha(repo))
self.assertIsNone(scm.get_branch(repo))
self.assertIsNone(scm.get_tag(repo))
repo = self.mk_repo()
self.assertFalse(scm.is_empty(repo))
def test_without_git(self):
shutil.move(self.gitdir, self.gotdor)
# root folder
root_folder = scm.get_project_root_folder(None)
self.assertEqual(os.path.realpath(root_folder),
os.path.realpath(os.path.join(self.tempdir, 'bin')))
# get_remote_url
self.assertEqual(scm.get_remote_url(None), '')
# get sha
sha = scm.get_sha(None)
self.assertEqual(sha, 'na')
def get_workflow_env(wf, workspace):
if scm.get_user():
repo_id = '{}/{}'.format(scm.get_user(), scm.get_name())
else:
repo_id = 'unknown'
env = {
'HOME': os.environ['HOME'],
'GITHUB_WORKFLOW': wf.name,
'GITHUB_ACTION': '',
'GITHUB_ACTOR': 'popper',
'GITHUB_REPOSITORY': repo_id,
'GITHUB_EVENT_NAME': wf.on,
'GITHUB_EVENT_PATH': '/tmp/github_event.json',
'GITHUB_WORKSPACE': workspace,
'GITHUB_SHA': scm.get_sha(),
'GITHUB_REF': scm.get_ref()
}
for e in dict(env):
env.update({e.replace('GITHUB_', 'POPPER_'): env[e]})
return env
def test_with_git(self):
if not os.path.exists(self.gitdir):
shutil.move(self.gotdor, self.gitdir)
# root folder
root_folder = scm.get_project_root_folder(self.repo)
self.assertEqual(os.path.realpath(root_folder),
os.path.realpath(os.path.join(self.tempdir, 'bin')))
# get_remote_url
url = scm.get_remote_url(self.repo)
auth_token = os.getenv('GITHUB_API_TOKEN')
if not auth_token:
self.assertEqual(url, 'https://github.com/popperized/bin')
else:
self.assertTrue('github.com/popperized/bin' in url)
# get sha
sha = scm.get_sha(self.repo)
expected = self.repo.git.rev_parse(self.repo.head.object.hexsha,
short=True)
self.assertEqual(sha, expected)
def test_get_sha(self):
sha = scm.get_sha()
if self.with_git:
self.assertEqual(sha, 'c3c8022')
else:
self.assertEqual(sha, 'unknown')
def test_get_build_info(self):
step = Box(
{"uses": "popperized/bin/sh@master", "args": ["ls"], "id": "one",},
default_box=True,
)
with StepRunner() as r:
build, _, img, tag, build_ctx_path = r._get_build_info(step)
self.assertEqual(build, True)
self.assertEqual(img, "popperized/bin")
self.assertEqual(tag, "master")
self.assertTrue(f"{os.environ['HOME']}/.cache/popper" in build_ctx_path)
self.assertTrue("github.com/popperized/bin/sh" in build_ctx_path)
step = Box(
{
"uses": "docker://alpine:3.9",
"runs": ["sh", "-c", "echo $FOO > hello.txt ; pwd"],
"env": {"FOO": "bar"},
"id": "1",
},
default_box=True,
)
with StepRunner() as r:
build, _, img, tag, build_sources = r._get_build_info(step)
self.assertEqual(build, False)
self.assertEqual(img, "alpine")
self.assertEqual(tag, "3.9")
self.assertEqual(build_sources, None)
step = Box({"uses": "./", "args": ["ls"], "id": "one",}, default_box=True,)
conf = ConfigLoader.load(workspace_dir="/tmp")
with StepRunner(config=conf) as r:
build, _, img, tag, build_ctx_path = r._get_build_info(step)
self.assertEqual(build, True)
self.assertEqual(img, "popper_one_step")
self.assertEqual(tag, "na")
self.assertEqual(build_ctx_path, f"{os.path.realpath('/tmp')}/./")
# test within a git repo
repo = self.mk_repo()
conf = ConfigLoader.load(workspace_dir=repo.working_dir)
with StepRunner(config=conf) as r:
build, _, img, tag, build_ctx_path = r._get_build_info(step)
self.assertEqual(build, True)
self.assertEqual(img, "popper_one_step")
self.assertEqual(tag, scm.get_sha(repo, short=7))
self.assertEqual(build_ctx_path, f"{os.path.realpath(repo.working_dir)}/./")
step = Box(
{
"uses": "docker://alpine:3.9",
"runs": ["sh", "-c", "echo $FOO > hello.txt ; pwd"],
"env": {"FOO": "bar"},
"name": "1",
},
default_box=True,
)
with StepRunner() as r:
build, img_full, _, _, build_ctx_path = r._get_build_info(step)
self.assertEqual(build, False)
self.assertEqual(img_full, "docker://alpine:3.9")
self.assertEqual(build_ctx_path, None)