def download_actions(self): """Clone actions that reference a repository.""" infoed = False for _, a in self.wf['action'].items(): if 'docker://' in a['uses'] or './' in a['uses']: continue user = a['uses'].split('/')[0] repo = a['uses'].split('/')[1] if '@' in a['uses']: action_dir = '/'.join(a['uses'].split('@')[0].split('/')[2:]) version = a['uses'].split('@')[1] else: action_dir = '/'.join(a['uses'].split('/')[2:]) version = None action_dir = os.path.join('./', action_dir) repo_parent_dir = os.path.join(self.actions_cache_path, user) if not os.path.exists(repo_parent_dir): os.makedirs(repo_parent_dir) if not infoed: pu.info('[popper] cloning actions from repositories\n') infoed = True scm.clone(user, repo, repo_parent_dir, version) a['repo_dir'] = os.path.join(repo_parent_dir, repo) a['action_dir'] = action_dir
def test_clone(self): tdir = os.path.join(self.tempdir, 'test_clone') os.makedirs(tdir) currdir = os.getcwd() os.chdir(tdir) scm.clone( 'https://github.com', 'popperized', 'github-actions-demo', os.path.join(os.getcwd(), 'gad'), 'develop' ) repo = git.Repo(os.path.join(os.getcwd(), 'gad')) self.assertEqual(repo.active_branch.name, 'develop') scm.clone( 'https://github.com', 'popperized', 'github-actions-demo', os.path.join(os.getcwd(), 'gad'), 'master' ) repo = git.Repo(os.path.join(os.getcwd(), 'gad')) self.assertEqual(repo.active_branch.name, 'master') repo.close() os.chdir(currdir)
def download_actions(self): """Clone actions that reference a repository.""" cloned = set() infoed = False for _, a in self.wf.actions: if ('docker://' in a['uses'] or 'shub://' in a['uses'] or './' in a['uses']): continue url, service, usr, repo, action_dir, version = scm.parse(a['uses']) repo_parent_dir = os.path.join( self.actions_cache_path, service, usr ) a['repo_dir'] = os.path.join(repo_parent_dir, repo) a['action_dir'] = action_dir if self.dry_run: continue if not infoed: log.info('[popper] cloning action repositories') infoed = True if '{}/{}'.format(usr, repo) in cloned: continue if not os.path.exists(repo_parent_dir): os.makedirs(repo_parent_dir) log.info('[popper] - {}/{}/{}@{}'.format(url, usr, repo, version)) scm.clone(url, usr, repo, repo_parent_dir, version) cloned.add('{}/{}'.format(usr, repo))
def test_clone(self): tempdir = tempfile.mkdtemp() tdir = os.path.join(tempdir, "test_clone") os.makedirs(tdir) currdir = os.getcwd() os.chdir(tdir) scm.clone( "https://github.com", "popperized", "github-actions-demo", os.path.join(os.getcwd(), "gad"), "develop", ) repo = git.Repo(os.path.join(os.getcwd(), "gad")) self.assertEqual(repo.active_branch.name, "develop") scm.clone( "https://github.com", "popperized", "github-actions-demo", os.path.join(os.getcwd(), "gad"), "master", ) repo = git.Repo(os.path.join(os.getcwd(), "gad")) self.assertEqual(repo.active_branch.name, "master") repo.close() os.chdir(currdir)
def import_from_repo(action_ref, project_root): url, service, user, repo, action_dir, version = scm.parse(action_ref) cloned_project_dir = os.path.join("/tmp", service, user, repo) scm.clone(url, user, repo, cloned_project_dir, version) if not action_dir: ptw_one = os.path.join(cloned_project_dir, "main.workflow") ptw_two = os.path.join(cloned_project_dir, ".github/main.workflow") if os.path.isfile(ptw_one): path_to_workflow = ptw_one elif os.path.isfile(ptw_two): path_to_workflow = ptw_two else: log.fail("Unable to find main.workflow file") else: path_to_workflow = os.path.join(cloned_project_dir, action_dir) if not os.path.basename(path_to_workflow).endswith('.workflow'): path_to_workflow = os.path.join(path_to_workflow, 'main.workflow') if not os.path.isfile(path_to_workflow): log.fail("Unable to find a main.workflow file") if '.github/' in path_to_workflow: path_to_copy = os.path.dirname(os.path.dirname(path_to_workflow)) else: path_to_copy = os.path.dirname(path_to_workflow) copy_tree(path_to_copy, project_root) log.info("Successfully imported from {}".format(action_ref))
def test_clone(self): os.makedirs('/tmp/test_folder/test_clone') os.chdir('/tmp/test_folder/test_clone') scm.clone('https://github.com', 'JayjeetAtGithub', 'github-actions-demo', os.path.join(os.getcwd(), 'gad'), 'develop') repo = git.Repo(os.path.join(os.getcwd(), 'gad')) self.assertEqual(repo.active_branch.name, 'develop') scm.clone('https://github.com', 'JayjeetAtGithub', 'github-actions-demo', os.path.join(os.getcwd(), 'gad'), 'master') repo = git.Repo(os.path.join(os.getcwd(), 'gad')) self.assertEqual(repo.active_branch.name, 'master')
def _clone_repos(self, wf): """Clone steps that reference a repository. Args: wf(popper.parser.workflow): Instance of the Workflow class. config.dry_run(bool): True if workflow flag is being dry-run. config.skip_clone(bool): True if clonning step has to be skipped. config.wid(str): id of the workspace Returns: None """ # cache directory for this workspace wf_cache_dir = os.path.join(self._config.cache_dir, self._config.wid) os.makedirs(wf_cache_dir, exist_ok=True) cloned = set() infoed = False for step in wf.steps: if ( "docker://" in step.uses or "shub://" in step.uses or "library://" in step.uses or "./" in step.uses or step.uses == "sh" ): continue url, service, user, repo, _, version = scm.parse(step.uses) repo_dir = os.path.join(wf_cache_dir, service, user, repo) if self._config.dry_run: continue if self._config.skip_clone: if not os.path.exists(repo_dir): log.fail(f"Expecting folder '{repo_dir}' not found.") continue if not infoed: log.info("[popper] Cloning step repositories") infoed = True if f"{user}/{repo}" in cloned: continue log.info(f"[popper] - {url}/{user}/{repo}@{version}") scm.clone(url, user, repo, repo_dir, version) cloned.add(f"{user}/{repo}")
def _clone_repos(self, wf): """Clone steps that reference a repository. Args: wf(popper.parser.workflow): Instance of the Workflow class. config.dry_run(bool): True if workflow flag is being dry-run. config.skip_clone(bool): True if clonning step has to be skipped. config.wid(str): id of the workspace Returns: None """ repo_cache = os.path.join(WorkflowRunner._setup_base_cache(), self._config.wid) cloned = set() infoed = False for _, a in wf.steps.items(): uses = a['uses'] if ('docker://' in uses or 'shub://' in uses or 'library://' in uses or './' in uses or uses == 'sh'): continue url, service, user, repo, step_dir, version = scm.parse(a['uses']) repo_dir = os.path.join(repo_cache, service, user, repo) a['repo_dir'] = repo_dir a['step_dir'] = step_dir if self._config.dry_run: continue if self._config.skip_clone: if not os.path.exists(repo_dir): log.fail(f"Expecting folder '{repo_dir}' not found.") continue if not infoed: log.info('[popper] Cloning step repositories') infoed = True if f'{user}/{repo}' in cloned: continue log.info(f'[popper] - {url}/{user}/{repo}@{version}') scm.clone(url, user, repo, repo_dir, version) cloned.add(f'{user}/{repo}')
def import_from_repo(path, project_root): parts = scm.get_parts(path) if len(parts) < 3: log.fail( 'Required url format: \ <url>/<user>/<repo>[/folder[/wf.workflow]]' ) url, service, user, repo, _, version = scm.parse(path) cloned_project_dir = os.path.join("/tmp", service, user, repo) scm.clone(url, user, repo, os.path.dirname( cloned_project_dir), version ) if len(parts) == 3: ptw_one = os.path.join(cloned_project_dir, "main.workflow") ptw_two = os.path.join(cloned_project_dir, ".github/main.workflow") if os.path.isfile(ptw_one): path_to_workflow = ptw_one elif os.path.isfile(ptw_two): path_to_workflow = ptw_two else: log.fail("Unable to find a .workflow file") elif len(parts) >= 4: path_to_workflow = os.path.join( cloned_project_dir, '/'.join(parts[3:])).split("@")[0] if not os.path.basename(path_to_workflow).endswith('.workflow'): path_to_workflow = os.path.join( path_to_workflow, 'main.workflow') if not os.path.isfile(path_to_workflow): log.fail("Unable to find a .workflow file") shutil.copy(path_to_workflow, project_root) log.info("Successfully imported from {}".format(path_to_workflow)) with open(path_to_workflow, 'r') as fp: wf = Workflow(path_to_workflow) action_paths = list() for _, a_block in wf.actions: if a_block['uses'].startswith("./"): action_paths.append(a_block['uses']) action_paths = set([a.split("/")[1] for a in action_paths]) for a in action_paths: copy_tree(os.path.join(cloned_project_dir, a), os.path.join(project_root, a)) log.info("Copied {} to {}...".format(os.path.join( cloned_project_dir, a), project_root))
def download_actions(wf, dry_run, skip_clone, wid): """Clone actions that reference a repository. Args: wf(popper.parser.workflow): Instance of the Workflow class. dry_run(bool): True if workflow flag is being dry-run. skip_clone(bool): True if clonning action has to be skipped. wid(str): Returns: None """ actions_cache = os.path.join(pu.setup_base_cache(), 'actions', wid) cloned = set() infoed = False for _, a in wf.action.items(): if ('docker://' in a['uses'] or './' in a['uses'] or a['uses'] == 'sh'): continue url, service, user, repo, action_dir, version = scm.parse( a['uses']) repo_dir = os.path.join(actions_cache, service, user, repo) a['repo_dir'] = repo_dir a['action_dir'] = action_dir if dry_run: continue if skip_clone: if not os.path.exists(repo_dir): log.fail('The required action folder \'{}\' was not ' 'found locally.'.format(repo_dir)) continue if not infoed: log.info('[popper] Cloning action repositories') infoed = True if '{}/{}'.format(user, repo) in cloned: continue log.info('[popper] - {}/{}/{}@{}'.format(url, user, repo, version)) scm.clone(url, user, repo, repo_dir, version) cloned.add('{}/{}'.format(user, repo))
def setUp(self): if os.environ.get('POPPER_TEST_MODE') == 'with-git': self.with_git = True else: self.with_git = False log.setLevel('CRITICAL') if os.path.exists('/tmp/test_folder'): shutil.rmtree('/tmp/test_folder') os.makedirs('/tmp/test_folder') os.chdir('/tmp/test_folder') scm.clone('https://github.com', 'cplee', 'github-actions-demo', os.path.join(os.getcwd(), 'github-actions-demo')) if not self.with_git: shutil.rmtree('/tmp/test_folder/github-actions-demo/.git') os.chdir('/tmp/test_folder/github-actions-demo')
def download_actions(self, wf, dry_run, skip_clone): """Clone actions that reference a repository.""" cloned = set() infoed = False for _, a in wf.actions.items(): if ('docker://' in a['uses'] or 'shub://' in a['uses'] or './' in a['uses'] or a['uses'] == 'sh'): continue url, service, usr, repo, action_dir, version = scm.parse(a['uses']) repo_dir = os.path.join(self.actions_cache_path, service, usr, repo) a['repo_dir'] = repo_dir a['action_dir'] = action_dir if dry_run: continue if skip_clone: if not os.path.exists(repo_dir): log.fail('Cannot find action folder locally.') continue if not infoed: log.info('[popper] cloning action repositories') infoed = True if '{}/{}'.format(usr, repo) in cloned: continue log.info('[popper] - {}/{}/{}@{}'.format(url, usr, repo, version)) scm.clone(url, usr, repo, repo_dir, version) cloned.add('{}/{}'.format(usr, repo))
def download_actions(self): """Clone actions that reference a repository.""" cloned = set() infoed = False for key, a in self.wf['action'].items(): if 'docker://' in a['uses'] or './' in a['uses']: continue action = None if a['uses'].startswith('https://'): a['uses'] = a['uses'][8:] parts = a['uses'].split('/') url = 'https://' + parts[0] user = parts[1] repo = parts[2] elif a['uses'].startswith('http://'): a['uses'] = a['uses'][7:] parts = a['uses'].split('/') url = 'http://' + parts[0] user = parts[1] repo = parts[2] elif a['uses'].startswith('git@'): url, rest = a['uses'].split(':') user, repo = rest.split('/') elif a['uses'].startswith('ssh://'): pu.fail("The ssh protocol is not supported yet.") else: url = 'https://github.com' parts = a['uses'].split('/') user = a['uses'].split('/')[0] repo = a['uses'].split('/')[1] action = '/'.join(a['uses'].split('/')[1:]) if '@' in repo: action_dir = '/'.join(a['uses'].split('@')[-2].split('/')[-1:]) version = a['uses'].split('@')[-1] elif '@' in action: action_dir = '/'.join(action.split('@')[-2].split('/')[-1:]) version = action.split('@')[-1] else: action_dir = '/'.join(a['uses'].split('/')[2:]) version = None action_dir = os.path.join('./', action_dir) repo_parent_dir = os.path.join(self.actions_cache_path, user) a['repo_dir'] = os.path.join(repo_parent_dir, repo) a['action_dir'] = action_dir if '{}/{}'.format(user, repo) in cloned: continue if not os.path.exists(repo_parent_dir): os.makedirs(repo_parent_dir) if not self.dry_run: if not infoed: pu.info('[popper] cloning actions from repositories\n') infoed = True scm.clone(url, user, repo, repo_parent_dir, version, debug=self.debug) cloned.add('{}/{}'.format(user, repo))