def download_actions(self):
"""Clone actions that reference a repository."""
infoed = False
for _, a in self.wf['action'].items():
if 'docker://' in a['uses'] or './' in a['uses']:
continue
user = a['uses'].split('/')[0]
repo = a['uses'].split('/')[1]
if '@' in a['uses']:
action_dir = '/'.join(a['uses'].split('@')[0].split('/')[2:])
version = a['uses'].split('@')[1]
else:
action_dir = '/'.join(a['uses'].split('/')[2:])
version = None
action_dir = os.path.join('./', action_dir)
repo_parent_dir = os.path.join(self.actions_cache_path, user)
if not os.path.exists(repo_parent_dir):
os.makedirs(repo_parent_dir)
if not infoed:
pu.info('[popper] cloning actions from repositories\n')
infoed = True
scm.clone(user, repo, repo_parent_dir, version)
a['repo_dir'] = os.path.join(repo_parent_dir, repo)
a['action_dir'] = action_dir
def test_clone(self):
tdir = os.path.join(self.tempdir, 'test_clone')
os.makedirs(tdir)
currdir = os.getcwd()
os.chdir(tdir)
scm.clone(
'https://github.com',
'popperized',
'github-actions-demo',
os.path.join(os.getcwd(), 'gad'),
'develop'
)
repo = git.Repo(os.path.join(os.getcwd(), 'gad'))
self.assertEqual(repo.active_branch.name, 'develop')
scm.clone(
'https://github.com',
'popperized',
'github-actions-demo',
os.path.join(os.getcwd(), 'gad'),
'master'
)
repo = git.Repo(os.path.join(os.getcwd(), 'gad'))
self.assertEqual(repo.active_branch.name, 'master')
repo.close()
os.chdir(currdir)
def download_actions(self):
"""Clone actions that reference a repository."""
cloned = set()
infoed = False
for _, a in self.wf.actions:
if ('docker://' in a['uses'] or 'shub://' in a['uses'] or
'./' in a['uses']):
continue
url, service, usr, repo, action_dir, version = scm.parse(a['uses'])
repo_parent_dir = os.path.join(
self.actions_cache_path, service, usr
)
a['repo_dir'] = os.path.join(repo_parent_dir, repo)
a['action_dir'] = action_dir
if self.dry_run:
continue
if not infoed:
log.info('[popper] cloning action repositories')
infoed = True
if '{}/{}'.format(usr, repo) in cloned:
continue
if not os.path.exists(repo_parent_dir):
os.makedirs(repo_parent_dir)
log.info('[popper] - {}/{}/{}@{}'.format(url, usr, repo, version))
scm.clone(url, usr, repo, repo_parent_dir, version)
cloned.add('{}/{}'.format(usr, repo))
def test_clone(self):
tempdir = tempfile.mkdtemp()
tdir = os.path.join(tempdir, "test_clone")
os.makedirs(tdir)
currdir = os.getcwd()
os.chdir(tdir)
scm.clone(
"https://github.com",
"popperized",
"github-actions-demo",
os.path.join(os.getcwd(), "gad"),
"develop",
)
repo = git.Repo(os.path.join(os.getcwd(), "gad"))
self.assertEqual(repo.active_branch.name, "develop")
scm.clone(
"https://github.com",
"popperized",
"github-actions-demo",
os.path.join(os.getcwd(), "gad"),
"master",
)
repo = git.Repo(os.path.join(os.getcwd(), "gad"))
self.assertEqual(repo.active_branch.name, "master")
repo.close()
os.chdir(currdir)
def import_from_repo(action_ref, project_root):
url, service, user, repo, action_dir, version = scm.parse(action_ref)
cloned_project_dir = os.path.join("/tmp", service, user, repo)
scm.clone(url, user, repo, cloned_project_dir, version)
if not action_dir:
ptw_one = os.path.join(cloned_project_dir, "main.workflow")
ptw_two = os.path.join(cloned_project_dir, ".github/main.workflow")
if os.path.isfile(ptw_one):
path_to_workflow = ptw_one
elif os.path.isfile(ptw_two):
path_to_workflow = ptw_two
else:
log.fail("Unable to find main.workflow file")
else:
path_to_workflow = os.path.join(cloned_project_dir, action_dir)
if not os.path.basename(path_to_workflow).endswith('.workflow'):
path_to_workflow = os.path.join(path_to_workflow,
'main.workflow')
if not os.path.isfile(path_to_workflow):
log.fail("Unable to find a main.workflow file")
if '.github/' in path_to_workflow:
path_to_copy = os.path.dirname(os.path.dirname(path_to_workflow))
else:
path_to_copy = os.path.dirname(path_to_workflow)
copy_tree(path_to_copy, project_root)
log.info("Successfully imported from {}".format(action_ref))
def test_clone(self):
os.makedirs('/tmp/test_folder/test_clone')
os.chdir('/tmp/test_folder/test_clone')
scm.clone('https://github.com',
'JayjeetAtGithub', 'github-actions-demo',
os.path.join(os.getcwd(), 'gad'), 'develop')
repo = git.Repo(os.path.join(os.getcwd(), 'gad'))
self.assertEqual(repo.active_branch.name, 'develop')
scm.clone('https://github.com',
'JayjeetAtGithub', 'github-actions-demo',
os.path.join(os.getcwd(), 'gad'), 'master')
repo = git.Repo(os.path.join(os.getcwd(), 'gad'))
self.assertEqual(repo.active_branch.name, 'master')
def _clone_repos(self, wf):
"""Clone steps that reference a repository.
Args:
wf(popper.parser.workflow): Instance of the Workflow class.
config.dry_run(bool): True if workflow flag is being dry-run.
config.skip_clone(bool): True if clonning step has to be skipped.
config.wid(str): id of the workspace
Returns:
None
"""
# cache directory for this workspace
wf_cache_dir = os.path.join(self._config.cache_dir, self._config.wid)
os.makedirs(wf_cache_dir, exist_ok=True)
cloned = set()
infoed = False
for step in wf.steps:
if (
"docker://" in step.uses
or "shub://" in step.uses
or "library://" in step.uses
or "./" in step.uses
or step.uses == "sh"
):
continue
url, service, user, repo, _, version = scm.parse(step.uses)
repo_dir = os.path.join(wf_cache_dir, service, user, repo)
if self._config.dry_run:
continue
if self._config.skip_clone:
if not os.path.exists(repo_dir):
log.fail(f"Expecting folder '{repo_dir}' not found.")
continue
if not infoed:
log.info("[popper] Cloning step repositories")
infoed = True
if f"{user}/{repo}" in cloned:
continue
log.info(f"[popper] - {url}/{user}/{repo}@{version}")
scm.clone(url, user, repo, repo_dir, version)
cloned.add(f"{user}/{repo}")
def _clone_repos(self, wf):
"""Clone steps that reference a repository.
Args:
wf(popper.parser.workflow): Instance of the Workflow class.
config.dry_run(bool): True if workflow flag is being dry-run.
config.skip_clone(bool): True if clonning step has to be skipped.
config.wid(str): id of the workspace
Returns:
None
"""
repo_cache = os.path.join(WorkflowRunner._setup_base_cache(),
self._config.wid)
cloned = set()
infoed = False
for _, a in wf.steps.items():
uses = a['uses']
if ('docker://' in uses or 'shub://' in uses
or 'library://' in uses or './' in uses or uses == 'sh'):
continue
url, service, user, repo, step_dir, version = scm.parse(a['uses'])
repo_dir = os.path.join(repo_cache, service, user, repo)
a['repo_dir'] = repo_dir
a['step_dir'] = step_dir
if self._config.dry_run:
continue
if self._config.skip_clone:
if not os.path.exists(repo_dir):
log.fail(f"Expecting folder '{repo_dir}' not found.")
continue
if not infoed:
log.info('[popper] Cloning step repositories')
infoed = True
if f'{user}/{repo}' in cloned:
continue
log.info(f'[popper] - {url}/{user}/{repo}@{version}')
scm.clone(url, user, repo, repo_dir, version)
cloned.add(f'{user}/{repo}')
def import_from_repo(path, project_root):
parts = scm.get_parts(path)
if len(parts) < 3:
log.fail(
'Required url format: \
<url>/<user>/<repo>[/folder[/wf.workflow]]'
)
url, service, user, repo, _, version = scm.parse(path)
cloned_project_dir = os.path.join("/tmp", service, user, repo)
scm.clone(url, user, repo, os.path.dirname(
cloned_project_dir), version
)
if len(parts) == 3:
ptw_one = os.path.join(cloned_project_dir, "main.workflow")
ptw_two = os.path.join(cloned_project_dir, ".github/main.workflow")
if os.path.isfile(ptw_one):
path_to_workflow = ptw_one
elif os.path.isfile(ptw_two):
path_to_workflow = ptw_two
else:
log.fail("Unable to find a .workflow file")
elif len(parts) >= 4:
path_to_workflow = os.path.join(
cloned_project_dir, '/'.join(parts[3:])).split("@")[0]
if not os.path.basename(path_to_workflow).endswith('.workflow'):
path_to_workflow = os.path.join(
path_to_workflow, 'main.workflow')
if not os.path.isfile(path_to_workflow):
log.fail("Unable to find a .workflow file")
shutil.copy(path_to_workflow, project_root)
log.info("Successfully imported from {}".format(path_to_workflow))
with open(path_to_workflow, 'r') as fp:
wf = Workflow(path_to_workflow)
action_paths = list()
for _, a_block in wf.actions:
if a_block['uses'].startswith("./"):
action_paths.append(a_block['uses'])
action_paths = set([a.split("/")[1] for a in action_paths])
for a in action_paths:
copy_tree(os.path.join(cloned_project_dir, a),
os.path.join(project_root, a))
log.info("Copied {} to {}...".format(os.path.join(
cloned_project_dir, a), project_root))
def download_actions(wf, dry_run, skip_clone, wid):
"""Clone actions that reference a repository.
Args:
wf(popper.parser.workflow): Instance of the Workflow class.
dry_run(bool): True if workflow flag is being dry-run.
skip_clone(bool): True if clonning action has to be skipped.
wid(str):
Returns:
None
"""
actions_cache = os.path.join(pu.setup_base_cache(), 'actions', wid)
cloned = set()
infoed = False
for _, a in wf.action.items():
if ('docker://' in a['uses'] or './' in a['uses']
or a['uses'] == 'sh'):
continue
url, service, user, repo, action_dir, version = scm.parse(
a['uses'])
repo_dir = os.path.join(actions_cache, service, user, repo)
a['repo_dir'] = repo_dir
a['action_dir'] = action_dir
if dry_run:
continue
if skip_clone:
if not os.path.exists(repo_dir):
log.fail('The required action folder \'{}\' was not '
'found locally.'.format(repo_dir))
continue
if not infoed:
log.info('[popper] Cloning action repositories')
infoed = True
if '{}/{}'.format(user, repo) in cloned:
continue
log.info('[popper] - {}/{}/{}@{}'.format(url, user, repo, version))
scm.clone(url, user, repo, repo_dir, version)
cloned.add('{}/{}'.format(user, repo))
def setUp(self):
if os.environ.get('POPPER_TEST_MODE') == 'with-git':
self.with_git = True
else:
self.with_git = False
log.setLevel('CRITICAL')
if os.path.exists('/tmp/test_folder'):
shutil.rmtree('/tmp/test_folder')
os.makedirs('/tmp/test_folder')
os.chdir('/tmp/test_folder')
scm.clone('https://github.com', 'cplee', 'github-actions-demo',
os.path.join(os.getcwd(), 'github-actions-demo'))
if not self.with_git:
shutil.rmtree('/tmp/test_folder/github-actions-demo/.git')
os.chdir('/tmp/test_folder/github-actions-demo')
def download_actions(self, wf, dry_run, skip_clone):
"""Clone actions that reference a repository."""
cloned = set()
infoed = False
for _, a in wf.actions.items():
if ('docker://' in a['uses'] or 'shub://' in a['uses']
or './' in a['uses'] or a['uses'] == 'sh'):
continue
url, service, usr, repo, action_dir, version = scm.parse(a['uses'])
repo_dir = os.path.join(self.actions_cache_path, service, usr,
repo)
a['repo_dir'] = repo_dir
a['action_dir'] = action_dir
if dry_run:
continue
if skip_clone:
if not os.path.exists(repo_dir):
log.fail('Cannot find action folder locally.')
continue
if not infoed:
log.info('[popper] cloning action repositories')
infoed = True
if '{}/{}'.format(usr, repo) in cloned:
continue
log.info('[popper] - {}/{}/{}@{}'.format(url, usr, repo, version))
scm.clone(url, usr, repo, repo_dir, version)
cloned.add('{}/{}'.format(usr, repo))
def download_actions(self):
"""Clone actions that reference a repository."""
cloned = set()
infoed = False
for key, a in self.wf['action'].items():
if 'docker://' in a['uses'] or './' in a['uses']:
continue
action = None
if a['uses'].startswith('https://'):
a['uses'] = a['uses'][8:]
parts = a['uses'].split('/')
url = 'https://' + parts[0]
user = parts[1]
repo = parts[2]
elif a['uses'].startswith('http://'):
a['uses'] = a['uses'][7:]
parts = a['uses'].split('/')
url = 'http://' + parts[0]
user = parts[1]
repo = parts[2]
elif a['uses'].startswith('git@'):
url, rest = a['uses'].split(':')
user, repo = rest.split('/')
elif a['uses'].startswith('ssh://'):
pu.fail("The ssh protocol is not supported yet.")
else:
url = 'https://github.com'
parts = a['uses'].split('/')
user = a['uses'].split('/')[0]
repo = a['uses'].split('/')[1]
action = '/'.join(a['uses'].split('/')[1:])
if '@' in repo:
action_dir = '/'.join(a['uses'].split('@')[-2].split('/')[-1:])
version = a['uses'].split('@')[-1]
elif '@' in action:
action_dir = '/'.join(action.split('@')[-2].split('/')[-1:])
version = action.split('@')[-1]
else:
action_dir = '/'.join(a['uses'].split('/')[2:])
version = None
action_dir = os.path.join('./', action_dir)
repo_parent_dir = os.path.join(self.actions_cache_path, user)
a['repo_dir'] = os.path.join(repo_parent_dir, repo)
a['action_dir'] = action_dir
if '{}/{}'.format(user, repo) in cloned:
continue
if not os.path.exists(repo_parent_dir):
os.makedirs(repo_parent_dir)
if not self.dry_run:
if not infoed:
pu.info('[popper] cloning actions from repositories\n')
infoed = True
scm.clone(url,
user,
repo,
repo_parent_dir,
version,
debug=self.debug)
cloned.add('{}/{}'.format(user, repo))
