def test_get_branch_in_detached_head_state(self): repo = self.mk_repo() repo.git.checkout("HEAD~1") os.environ["TRAVIS_BRANCH"] = "travis" self.assertEqual("travis", scm.get_branch(repo)) os.environ.pop("TRAVIS_BRANCH") os.environ["GIT_BRANCH"] = "jenkins" self.assertEqual("jenkins", scm.get_branch(repo)) os.environ.pop("GIT_BRANCH") os.environ["CIRCLE_BRANCH"] = "circle" self.assertEqual("circle", scm.get_branch(repo)) os.environ.pop("CIRCLE_BRANCH") os.environ["CI_COMMIT_REF_NAME"] = "gitlab" self.assertEqual("gitlab", scm.get_branch(repo)) os.environ.pop("CI_COMMIT_REF_NAME") self.assertEqual(scm.get_sha(repo), scm.get_branch(repo)) # None given as arg self.assertIsNone(scm.get_sha(None)) self.assertIsNone(scm.get_sha(None, short=8)) self.assertIsNone(scm.get_branch(None))
def load( engine_name=None, resman_name=None, config_file=None, workspace_dir=os.getcwd(), reuse=False, dry_run=False, quiet=False, skip_pull=False, skip_clone=False, pty=False, allow_undefined_secrets_in_ci=False, ): """Loads and creates a configuration, represented by a frozen Box """ workspace_dir = os.path.realpath(workspace_dir) repo = scm.new_repo(workspace_dir) # path to cache if os.environ.get("POPPER_CACHE_DIR", None): cache_dir = os.environ["POPPER_CACHE_DIR"] else: cache_dir_default = os.path.join(os.environ["HOME"], ".cache") cache_dir = os.environ.get("XDG_CACHE_HOME", cache_dir_default) cache_dir = os.path.join(cache_dir, "popper") from_file = ConfigLoader.__load_config_from_file( config_file, engine_name, resman_name ) pp_config = { "workspace_dir": workspace_dir, "reuse": reuse, "dry_run": dry_run, "quiet": quiet, "skip_pull": skip_pull, "skip_clone": skip_clone, "pty": pty, "allow_undefined_secrets_in_ci": allow_undefined_secrets_in_ci, # if no git repository exists in workspace_dir or its parents, the repo # variable is None and all git_* variables are assigned to 'na' "repo": repo, "git_commit": scm.get_sha(repo), "git_sha_short": scm.get_sha(repo, short=7), "git_branch": scm.get_branch(repo), "git_tag": scm.get_tag(repo), "git_remote_origin_url": scm.get_remote_url(repo), # wid is used to associate a unique id to this workspace. This is then # used by runners to name resources in a way that there is no name # clash between concurrent workflows being executed "wid": shake_256(workspace_dir.encode("utf-8")).hexdigest(4), "cache_dir": cache_dir, "engine_name": from_file["engine_name"], "resman_name": from_file["resman_name"], "engine_opts": from_file["engine_opts"], "resman_opts": from_file["resman_opts"], } return Box(pp_config, default_box=True, frozen_box=True)
def __init__(self, engine_name=None, resman_name=None, config_file=None, workspace_dir=os.getcwd(), reuse=False, dry_run=False, quiet=False, skip_pull=False, skip_clone=False): self.workspace_dir = os.path.realpath(workspace_dir) self.reuse = reuse self.dry_run = dry_run self.quiet = quiet self.skip_pull = skip_pull self.skip_clone = skip_clone self.repo = scm.new_repo() self.workspace_sha = scm.get_sha(self.repo) wid = shake_256(self.workspace_dir.encode('utf-8')).hexdigest(4) self.wid = wid from_file = self._load_config_from_file(config_file, engine_name, resman_name) self.engine_name = from_file['engine_name'] self.resman_name = from_file['resman_name'] self.engine_opts = from_file['engine_opts'] self.resman_opts = from_file['resman_opts']
def __init__(self, wfile, workspace, quiet, debug, dry_run): wfile = pu.find_default_wfile(wfile) with open(wfile, 'r') as fp: self.wf = hcl.load(fp) self.workspace = workspace self.debug = debug if debug: self.quiet = False else: self.quiet = quiet self.dry_run = dry_run self.actions_cache_path = os.path.join('/', 'tmp', 'actions') self.validate_syntax() self.check_secrets() self.normalize() self.complete_graph() self.env = { 'GITHUB_WORKSPACE': self.workspace, 'GITHUB_WORKFLOW': self.wf['name'], 'GITHUB_ACTOR': 'popper', 'GITHUB_REPOSITORY': '{}/{}'.format(scm.get_user(), scm.get_name()), 'GITHUB_EVENT_NAME': self.wf['on'], 'GITHUB_EVENT_PATH': '/{}/{}'.format(self.workspace, 'workflow/event.json'), 'GITHUB_SHA': scm.get_sha(), 'GITHUB_REF': scm.get_ref() } for e in dict(self.env): self.env.update({e.replace('GITHUB_', 'POPPER_'): self.env[e]})
def run(self, action_name=None, reuse=False, parallel=False): """Run the pipeline or a specific action""" os.environ['WORKSPACE'] = self.workspace if scm.get_user(): repo_id = '{}/{}'.format(scm.get_user(), scm.get_name()) else: repo_id = 'unknown' self.env = { 'GITHUB_WORKSPACE': self.workspace, 'GITHUB_WORKFLOW': self.wf.name, 'GITHUB_ACTOR': 'popper', 'GITHUB_REPOSITORY': repo_id, 'GITHUB_EVENT_NAME': self.wf.on, 'GITHUB_EVENT_PATH': '/{}/{}'.format(self.workspace, 'workflow/event.json'), 'GITHUB_SHA': scm.get_sha(), 'GITHUB_REF': scm.get_ref() } for e in dict(self.env): self.env.update({e.replace('GITHUB_', 'POPPER_'): self.env[e]}) self.download_actions() self.instantiate_runners() if action_name: self.wf.get_runner(action_name).run(reuse) else: for s in self.wf.get_stages(): self.run_stage(s, reuse, parallel)
def get_workflow_env(wf, workspace): """Updates the Popper environment variable with Github environment variables. Args: wf(popper.parser.Workflow): Instance of the Workflow class. workspace(str): Location of the workspace. Returns: dict: dictionary containing Github variables. """ if scm.get_user(): repo_id = '{}/{}'.format(scm.get_user(), scm.get_name()) else: repo_id = 'unknown' env = { 'HOME': os.environ['HOME'], 'GITHUB_WORKFLOW': wf.name, 'GITHUB_ACTION': '', 'GITHUB_ACTOR': 'popper', 'GITHUB_REPOSITORY': repo_id, 'GITHUB_EVENT_NAME': wf.on, 'GITHUB_EVENT_PATH': '/tmp/github_event.json', 'GITHUB_WORKSPACE': workspace, 'GITHUB_SHA': scm.get_sha(), 'GITHUB_REF': scm.get_ref() } for e in dict(env): env.update({e.replace('GITHUB_', 'POPPER_'): env[e]}) return env
def run(self, action, skip_clone, skip_pull, skip, workspace, reuse, dry_run, parallel, with_dependencies, skip_secrets_prompt=False): """Run the pipeline or a specific action""" os.environ['WORKSPACE'] = workspace if scm.get_user(): repo_id = '{}/{}'.format(scm.get_user(), scm.get_name()) else: repo_id = 'unknown' if with_dependencies and (not action): log.fail('`--with-dependencies` can be used only with ' 'action argument.') if skip and action: log.fail('`--skip` can\'t be used when action argument ' 'is passed.') new_wf = deepcopy(self.wf) if skip: new_wf = self.wf.skip_actions(skip) if action: new_wf = self.wf.filter_action(action, with_dependencies) new_wf.check_for_unreachable_actions(skip) env = { 'GITHUB_WORKSPACE': workspace, 'GITHUB_WORKFLOW': new_wf.name, 'GITHUB_ACTOR': 'popper', 'GITHUB_REPOSITORY': repo_id, 'GITHUB_EVENT_NAME': new_wf.on, 'GITHUB_EVENT_PATH': '/tmp/github_event.json', 'GITHUB_SHA': scm.get_sha(), 'GITHUB_REF': scm.get_ref() } for e in dict(env): env.update({e.replace('GITHUB_', 'POPPER_'): env[e]}) self.check_secrets(new_wf, dry_run, skip_secrets_prompt) self.download_actions(new_wf, dry_run, skip_clone) self.instantiate_runners(new_wf, workspace, env, dry_run, skip_pull) for s in new_wf.get_stages(): self.run_stage(new_wf, s, reuse, parallel)
def test_empty_repo(self): tempdir = tempfile.mkdtemp() repo = git.Repo.init(tempdir) self.assertTrue(scm.is_empty(repo)) self.assertIsNone(scm.get_sha(repo)) self.assertIsNone(scm.get_branch(repo)) self.assertIsNone(scm.get_tag(repo)) repo = self.mk_repo() self.assertFalse(scm.is_empty(repo))
def test_without_git(self): shutil.move(self.gitdir, self.gotdor) # root folder root_folder = scm.get_project_root_folder(None) self.assertEqual(os.path.realpath(root_folder), os.path.realpath(os.path.join(self.tempdir, 'bin'))) # get_remote_url self.assertEqual(scm.get_remote_url(None), '') # get sha sha = scm.get_sha(None) self.assertEqual(sha, 'na')
def get_workflow_env(wf, workspace): if scm.get_user(): repo_id = '{}/{}'.format(scm.get_user(), scm.get_name()) else: repo_id = 'unknown' env = { 'HOME': os.environ['HOME'], 'GITHUB_WORKFLOW': wf.name, 'GITHUB_ACTION': '', 'GITHUB_ACTOR': 'popper', 'GITHUB_REPOSITORY': repo_id, 'GITHUB_EVENT_NAME': wf.on, 'GITHUB_EVENT_PATH': '/tmp/github_event.json', 'GITHUB_WORKSPACE': workspace, 'GITHUB_SHA': scm.get_sha(), 'GITHUB_REF': scm.get_ref() } for e in dict(env): env.update({e.replace('GITHUB_', 'POPPER_'): env[e]}) return env
def test_with_git(self): if not os.path.exists(self.gitdir): shutil.move(self.gotdor, self.gitdir) # root folder root_folder = scm.get_project_root_folder(self.repo) self.assertEqual(os.path.realpath(root_folder), os.path.realpath(os.path.join(self.tempdir, 'bin'))) # get_remote_url url = scm.get_remote_url(self.repo) auth_token = os.getenv('GITHUB_API_TOKEN') if not auth_token: self.assertEqual(url, 'https://github.com/popperized/bin') else: self.assertTrue('github.com/popperized/bin' in url) # get sha sha = scm.get_sha(self.repo) expected = self.repo.git.rev_parse(self.repo.head.object.hexsha, short=True) self.assertEqual(sha, expected)
def test_get_sha(self): sha = scm.get_sha() if self.with_git: self.assertEqual(sha, 'c3c8022') else: self.assertEqual(sha, 'unknown')
def test_get_build_info(self): step = Box( {"uses": "popperized/bin/sh@master", "args": ["ls"], "id": "one",}, default_box=True, ) with StepRunner() as r: build, _, img, tag, build_ctx_path = r._get_build_info(step) self.assertEqual(build, True) self.assertEqual(img, "popperized/bin") self.assertEqual(tag, "master") self.assertTrue(f"{os.environ['HOME']}/.cache/popper" in build_ctx_path) self.assertTrue("github.com/popperized/bin/sh" in build_ctx_path) step = Box( { "uses": "docker://alpine:3.9", "runs": ["sh", "-c", "echo $FOO > hello.txt ; pwd"], "env": {"FOO": "bar"}, "id": "1", }, default_box=True, ) with StepRunner() as r: build, _, img, tag, build_sources = r._get_build_info(step) self.assertEqual(build, False) self.assertEqual(img, "alpine") self.assertEqual(tag, "3.9") self.assertEqual(build_sources, None) step = Box({"uses": "./", "args": ["ls"], "id": "one",}, default_box=True,) conf = ConfigLoader.load(workspace_dir="/tmp") with StepRunner(config=conf) as r: build, _, img, tag, build_ctx_path = r._get_build_info(step) self.assertEqual(build, True) self.assertEqual(img, "popper_one_step") self.assertEqual(tag, "na") self.assertEqual(build_ctx_path, f"{os.path.realpath('/tmp')}/./") # test within a git repo repo = self.mk_repo() conf = ConfigLoader.load(workspace_dir=repo.working_dir) with StepRunner(config=conf) as r: build, _, img, tag, build_ctx_path = r._get_build_info(step) self.assertEqual(build, True) self.assertEqual(img, "popper_one_step") self.assertEqual(tag, scm.get_sha(repo, short=7)) self.assertEqual(build_ctx_path, f"{os.path.realpath(repo.working_dir)}/./") step = Box( { "uses": "docker://alpine:3.9", "runs": ["sh", "-c", "echo $FOO > hello.txt ; pwd"], "env": {"FOO": "bar"}, "name": "1", }, default_box=True, ) with StepRunner() as r: build, img_full, _, _, build_ctx_path = r._get_build_info(step) self.assertEqual(build, False) self.assertEqual(img_full, "docker://alpine:3.9") self.assertEqual(build_ctx_path, None)