def __init__(self, args):
super(ProcessingContext, self).__init__(args)
self.config_dir = os.path.realpath(os.path.normpath(args.i))
self.ref = 'devel' if args.devel else 'master'
self.url = GITHUB_CONTENT_API
self.url += GITHUB_API_PARAMETER.format('ref', self.ref)
self.files = None
def get_special_case(f, url, repo, ref, auth):
"""
Get a dictionary of (filename -> file_info) pairs to be used for
named files in place of the file info from the general API call
done for the directory. file_info should contain at least
the elements 'sha' and 'download_url'
"""
url = url.format(repo)
url += '/{}'.format(f)
url += GITHUB_API_PARAMETER.format('ref', ref)
r = gh_request_content(url=url, auth=auth)
Print.debug('Set special GitHub reference -- "{}": "{}"'.format(f, ref))
return {f: r.json()}
def run(args):
"""
Main process that:
* Decide to fetch or not depending on file presence/absence and command-line arguments,
* Gets the GitHub file content from full API URL,
* Backups old file if desired,
* Writes response into table file.
:param ArgumentParser args: Parsed command-line arguments
"""
# Instantiate processing context manager
with ProcessingContext(args) as ctx:
for project in ctx.project:
try:
# Set repository name
repo = REPO_PATTERN.format(project)
# Get the list of available refs for that repository
r = gh_request_content(url=ctx.ref_url.format(repo),
auth=ctx.auth)
refs = [os.path.basename(ref['url']) for ref in r.json()]
# Get refs to fetch
if hasattr(ctx, 'ref'):
if ctx.ref not in refs:
raise GitHubReferenceNotFound(ctx.ref, refs)
fetch_refs = [ctx.ref]
else:
fetch_refs = list(
filter(re.compile(ctx.ref_regex).match, refs))
if not fetch_refs:
raise GitHubReferenceNotFound(ctx.ref_regex.pattern,
refs)
Print.debug('GitHub Available reference(s): {}'.format(
', '.join(sorted(refs))))
Print.info('Selected GitHub reference(s): {}'.format(', '.join(
sorted(fetch_refs))))
# Get special case for CMIP6_CV.json file
special_cases = get_special_case(f='CMIP6_CV.json',
url=ctx.url,
repo=repo,
ref='master',
auth=ctx.auth)
# Fetch each ref
for ref in fetch_refs:
try:
# Set reference url
url = ctx.url.format(repo)
if ref:
url += GITHUB_API_PARAMETER.format('ref', ref)
Print.debug(
'Fetch {} tables from "{}" GitHub reference'.
format(project, ref))
# Build output directory
if ctx.no_subfolder:
outdir = make_outdir(ctx.tables_dir, repo)
else:
outdir = make_outdir(ctx.tables_dir, repo, ref)
# Fetch GitHub reference
fetch_gh_ref(url=url,
outdir=outdir,
auth=ctx.auth,
keep=ctx.keep,
overwrite=ctx.overwrite,
backup_mode=ctx.backup_mode,
filter=ctx.file_filter,
special_cases=special_cases)
except Exception:
exc = traceback.format_exc().splitlines()
msg = TAGS.FAIL
msg += 'Fetching {} tables from {} GitHub reference'.format(
COLORS.HEADER(project), COLORS.HEADER(ref)) + '\n'
msg += '\n'.join(exc)
Print.exception(msg, buffer=True)
ctx.error = True
except Exception:
exc = traceback.format_exc().splitlines()
msg = TAGS.FAIL
msg += 'Fetching {} tables'.format(
COLORS.HEADER(project)) + '\n'
msg += '\n'.join(exc)
Print.exception(msg, buffer=True)
ctx.error = True
# Flush buffer
Print.flush()
# Evaluate errors and exit with appropriated return code
if ctx.error:
sys.exit(1)