def get_info(pipeline):
"""Prints the information about the specified pipeline.
Args:
pipeline (list): [org,repo,pipeline_name]
"""
# Checking if the popperized repositories are present or not
info = {}
org = pipeline[0]
repo = pipeline[1]
pipe = pipeline[2]
commit_url = 'https://api.github.com/repos'
commit_url += '/{}/{}/git/refs/heads/master'.format(org, repo)
r = pu.make_gh_request(commit_url,
msg="Please check if the specified pipeline exists "
"and the internet is connected.")
r = r.json()
info['name'] = pipe
info['url'] = 'https://github.com/{}/{}'.format(org, repo)
info['sha'] = r['object']['sha']
temp = {}
contents = " ".join(pu.read_gh_pipeline(org, repo, pipe)[1:])
if len(contents) != 0:
temp['description'] = contents
pu.print_yaml(info)
if 'description' in temp:
pu.print_yaml(temp)
def test_make_gh_request(self, m):
m.get('http://sample.test', text='response', status_code=200)
response = pu.make_gh_request('http://sample.test')
self.assertEqual(response.text, 'response')
m.get('http://sample.test', status_code=400)
self.assertRaises(SystemExit, pu.make_gh_request, 'http://sample.test',
True)
def search_pipeline(s):
"""Searches for the pipeline inside a github repository.
Word level levenshtein distances are being calculated to find
appropriate results for a given query.
Args:
s (dict): A python dictionary that contains all the conditions
specified by a user during popper search.
Returns:
results (list): List of pipelines
"""
repo_name = s["repo_url"].split('/')[-1]
results = []
pipelines = ""
if not s["skip_update"]:
pipelines_url = s["repo_url"] + "/contents/pipelines"
response = pu.make_gh_request(pipelines_url, err=False)
if response.status_code != 200:
return results
else:
with open(os.path.join(s["cache_dir"], repo_name + '.json'),
'w') as f:
json.dump(response.json(), f)
try:
with open(os.path.join(s["cache_dir"], repo_name + '.json'), 'r') as f:
pipelines = json.load(f)
except FileNotFoundError:
return results
for pipeline in pipelines:
if s["empty_query"]:
temp = "{}/{}/{}".format(s["uname"], repo_name, pipeline['name'])
results.append(temp)
else:
if l_distance(s["keywords"].lower(), pipeline['name'].lower()) < 1:
temp = "{}/{}/{}" \
.format(s["uname"], repo_name, pipeline['name'])
if s["in_readme"]:
contents = pu.read_gh_pipeline(s["uname"], repo_name,
pipeline["name"])
if len(contents) != 0:
contents = " ".join(" ".join(
contents[2:4]).split(".")[0:3])
temp += "\n" + contents + "\n"
results.append(temp)
return results
def cli(ctx, pipeline, folder, branch):
"""Add a pipeline to your repository from the existing popperized
repositories on github. The pipeline argument is provided as owner/repo/
pipeline. For example, 'popper add popperized/quiho-popper/single-node'
adds the 'single-node' pipeline from the 'quiho-popper' repository from the
'popperized' organization.
"""
if len(pipeline.split('/')) != 3:
raise BadArgumentUsage(
"Bad pipeline name. See 'popper add --help' for more info.")
owner, repo, pipe_name = pipeline.split('/')
config = pu.read_config()
if pipe_name in config['pipelines']:
pu.fail("Pipeline {} already in repo.".format(pipe_name))
project_root = pu.get_project_root()
pipelines_dir = os.path.join(project_root, folder)
if not os.path.exists(pipelines_dir):
os.mkdir(pipelines_dir)
gh_url = 'https://github.com/{}/{}/'.format(owner, repo)
gh_url += 'archive/{}.tar.gz'.format(branch)
pu.info("Downloading pipeline {}... ".format(pipe_name))
r = pu.make_gh_request(
gh_url,
msg="Unable to fetch the pipeline. Please check if the name"
" of the pipeline is correct and the internet is connected"
)
# Downloading and extracting the tarfile
with tarfile.open(
mode='r:gz', fileobj=BytesIO(r.content)) as t:
t.extractall()
os.rename('{}-{}/pipelines/{}'.format(
repo, branch, pipe_name), os.path.join(folder, pipe_name))
shutil.rmtree('{}-{}'.format(repo, branch))
pu.info("Updating popper configuration... ")
repo_config = get_config(owner, repo)
config['pipelines'][pipe_name] = repo_config['pipelines'][pipe_name]
config['pipelines'][pipe_name]['path'] = os.path.join(folder, pipe_name)
pu.write_config(config)
pu.info("Pipeline {} has been added successfully.".format(pipe_name),
fg="green")
def get_config(owner, repo):
"""It returns the content of the .popper.yml file of the repository
whose pipeline we are copying.
Args:
owner (str): The username of the owner of the repository
repo (str): The name of the repository from where the
pipeline is being added
Returns:
config (dict): .popper.yml configuarion of the pipeline
"""
yaml_url = 'https://raw.githubusercontent.com/{}/{}/{}/.popper.yml'.format(
owner, repo, 'master')
r = pu.make_gh_request(yaml_url)
config = yaml.load(r.content)
return config
def cli(ctx, keywords, skip_update, add, rm, ls, include_readme):
"""Searches for pipelines on GitHub matching the given keyword(s).
The list of repositories or organizations scraped for Popper pipelines is
specified in the 'popperized' list in the .popper.yml file. By default,
https://github.com/popperized is added to the configuration.
If no keywords are specified, a list of all the pipelines from all
organizations (in the .popper.yml file) and repositories will be returned.
Example:
popper search quiho
would result in:
popperized/quiho-popper
To add or remove orgs/repos to/from the 'popperized' ,
use the --add and --rm flags while searching.
popper search --add org/repo
To remove an organization/person do:
popper search --rm org/repo
To view the list repositories that are available to the search command:
popper search --ls
"""
if (rm or add or ls) and (keywords):
raise BadArgumentUsage(
"'add', 'rm' and 'ls' flags cannot be combined with others.")
project_root = pu.get_project_root()
config = pu.read_config()
popperized_list = config['popperized']
if add:
add = 'github/' + add
if add not in popperized_list:
popperized_list.append(add)
config['popperized'] = popperized_list
pu.write_config(config)
sys.exit(0)
if rm:
rm = 'github/' + rm
if rm in popperized_list:
popperized_list.remove(rm)
config['popperized'] = popperized_list
pu.write_config(config)
sys.exit(0)
result = [] # to store the result of the search query as a list
if ls:
for p in popperized_list:
if p.count('/') == 1:
org_name = p.split('/')[1]
org_url = ('https://api.github.com/users/{}/repos')
org_url = org_url.format(org_name)
response = pu.make_gh_request(org_url)
repos = response.json()
temp = [r["full_name"] for r in repos]
result.extend(temp)
else:
result.extend(p[7:])
if len(result) > 0:
pu.info("The list of available poppperized repositories are:\n")
pu.print_yaml(result)
sys.exit()
else:
fail_msg = "There are no popperized repositores available"
"for search. Use the --add flag to add an org/repo."
pu.fail(fail_msg)
sys.exit(0)
search_params = {}
if not keywords: # checks if the query is empty or not
search_params['empty_query'] = True
else:
search_params['empty_query'] = False
cache_dir = os.path.join(project_root, '.cache')
search_params["keywords"] = keywords
search_params["cache_dir"] = cache_dir
search_params["skip_update"] = True if skip_update else False
search_params["in_readme"] = True if include_readme else False
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
for popperized in popperized_list:
if popperized.count('/') == 1:
# it is an organization
org_name = popperized.split('/')[1]
repos = ""
if not skip_update:
org_url = (
'https://api.github.com/users/{}/repos'.format(org_name))
response = pu.make_gh_request(org_url)
with open(os.path.join(cache_dir, org_name + '_repos.json'),
'w') as f:
json.dump(response.json(), f)
try:
with open(os.path.join(cache_dir, org_name + '_repos.json'),
'r') as f:
repos = json.load(f)
except FileNotFoundError:
pu.fail('No cached metadata has been downloaded')
with click.progressbar(
repos,
show_eta=False,
label='Searching in ' + org_name,
bar_template='[%(bar)s] %(label)s | %(info)s',
show_percent=True) as bar:
for r in bar:
if search_params["empty_query"]:
temp = ' {}/{}'\
.format(org_name, r['name'])
result.append(temp)
elif l_distance(r["name"].lower(), keywords.lower()) < 1:
temp = ' {}/{}' \
.format(org_name, r['name'])
result.append(temp)
else:
search_params["repo_url"] = r["url"]
search_params["uname"] = org_name
result.extend(search_pipeline(search_params))
else:
# it is a repository
user, repo = popperized.split('/')[1:]
repo_url = ('https://api.github.com/repos/{}/{}'.format(
user, repo))
search_params["repo_url"] = repo_url
search_params["uname"] = user
pu.info("Searching in repository : {}".format(repo))
result.extend(search_pipeline(search_params))
if len(result) != 0:
pu.info("\nSearch results:\n", fg="green")
for res in result:
pu.info("> " + res + "\n")
if search_params["in_readme"]:
pu.info("Use popper info command to view the"
" details of a pipeline. See popper info --"
"help")
else:
pu.fail("Unable to find any matching pipelines")
def cli(ctx, pipeline, folder, branch):
"""Add a pipeline to your repository from the existing popperized
repositories on github. The pipeline argument is provided as
<org>/<repo>/<pipeline>. For example:
popper add popperized/quiho-popper/single-node
The above adds the 'single-node' pipeline from the 'quiho-popper'
repository from the 'popperized' organization.
This commands makes use of Github's API, which has a limit on the number of
requests per hour that an unauthenticated user can make. If you reach this
limit, you can provide a Github API token via a POPPER_GITHUB_API_TOKEN
environment variable. If defined, this variable is used to obtain the token
when executing HTTP requests.
"""
if len(pipeline.split('/')) != 3:
raise BadArgumentUsage(
"Bad pipeline name. See 'popper add --help' for more info.")
owner, repo, pipe_name = pipeline.split('/')
new_pipe_name, folder = pu.get_name_and_path_for_new_pipeline(
folder, pipe_name)
config = pu.read_config()
if new_pipe_name in config['pipelines']:
pu.fail("Pipeline {} already in repo.".format(new_pipe_name))
project_root = pu.get_project_root()
pipelines_dir = os.path.join(project_root, folder)
if not os.path.exists(pipelines_dir):
try:
os.makedirs(pipelines_dir)
except (OSError, IOError) as e:
pu.fail("Could not create the necessary path.\n")
elif len(os.listdir(pipelines_dir)) != 0:
pu.fail("The path already exists and is not empty.")
gh_url = 'https://github.com/{}/{}/'.format(owner, repo)
gh_url += 'archive/{}.tar.gz'.format(branch)
pu.info("Downloading pipeline {} as {}...".format(pipe_name,
new_pipe_name))
r = pu.make_gh_request(
gh_url,
msg="Unable to fetch the pipeline. Please check if the name"
" of the pipeline is correct and the internet is connected")
# Downloading and extracting the tarfile
with tarfile.open(mode='r:gz', fileobj=BytesIO(r.content)) as t:
t.extractall()
try:
os.rename('{}-{}/pipelines/{}'.format(repo, branch, pipe_name),
pipelines_dir)
except OSError:
pu.fail("Could not rename {} to {}.".format(
'{}-{}/pipelines/{}'.format(repo, branch, pipe_name),
pipelines_dir))
finally:
shutil.rmtree('{}-{}'.format(repo, branch))
pu.info("Updating popper configuration... ")
repo_config = get_config(owner, repo)
config['pipelines'][new_pipe_name] = repo_config['pipelines'][pipe_name]
config['pipelines'][new_pipe_name]['path'] = folder
pu.write_config(config)
pu.info("Pipeline {} has been added successfully.".format(new_pipe_name),
fg="green")