def run(args):
"""
Main process that:
* Instantiates processing context,
* Loads previous program instance,
* Parallelizes file processing with threads pools,
* Apply command-line action to the whole DRS tree,
* Evaluate exit status.
:param ArgumentParser args: The command-line arguments parser
"""
# Instantiate processing context
with ProcessingContext(args) as ctx:
logging.info('==> Scan started')
if not ctx.scan:
reader = load(TREE_FILE)
_ = reader.next()
ctx.tree = reader.next()
ctx.scan_err_log = reader.next()
results = reader.next()
# Rollback --commands_file value to command-line argument in any case
ctx.tree.commands_file = ctx.commands_file
msg = 'Skipping incoming files scan (use "--rescan" to force it) -- ' \
'Using cached DRS tree from {}'.format(TREE_FILE)
if ctx.pbar:
print(msg)
logging.warning(msg)
else:
if ctx.use_pool:
processes = ctx.pool.imap(process, ctx.sources)
else:
processes = itertools.imap(process, ctx.sources)
# Process supplied files
results = [x for x in processes]
# Close progress bar
if ctx.pbar:
ctx.pbar.close()
# Get number of files scanned (including skipped files)
ctx.scan_files = len(results)
# Get number of scan errors
ctx.scan_errors = results.count(None)
# Backup tree context for later usage with other command lines
store(TREE_FILE, data=[{key: ctx.__getattribute__(key) for key in CONTROLLED_ARGS},
ctx.tree,
ctx.scan_err_log,
results])
logging.warning('DRS tree recorded for next usage onto {}.'.format(TREE_FILE))
# Evaluates the scan results to trigger the DRS tree action
if evaluate(results):
# Check upgrade uniqueness
ctx.tree.check_uniqueness(ctx.checksum_type)
# Apply tree action
ctx.tree.get_display_lengths()
getattr(ctx.tree, ctx.action)()
def run(args):
"""
Main process that:
* Decide to fetch or not depending on file presence/absence and command-line arguments,
* Gets the GitHub file content from full API URL,
* Backups old file if desired,
* Writes response into INI file.
:param ArgumentParser args: Parsed command-line arguments
"""
# Instantiate processing context manager
with ProcessingContext(args) as ctx:
# Make output directory
outdir = make_outdir(root=ctx.config_dir)
# Counter
progress = 0
for f, info in ctx.files.items():
try:
# Set output file full path
outfile = os.path.join(outdir, f)
# Get checksum
download_url = info['download_url']
sha = info['sha']
# Get GitHub file
fetch(url=download_url,
outfile=outfile,
auth=ctx.auth,
sha=sha,
keep=ctx.keep,
overwrite=ctx.overwrite,
backup_mode=ctx.backup_mode)
except KeyboardInterrupt:
raise
except Exception:
download_url = info['download_url']
exc = traceback.format_exc().splitlines()
msg = TAGS.FAIL + COLORS.HEADER(download_url) + '\n'
msg += '\n'.join(exc)
Print.exception(msg, buffer=True)
ctx.error = True
finally:
progress += 1
percentage = int(progress * 100 / ctx.nfiles)
msg = COLORS.OKBLUE('\rFetching project(s) config: ')
msg += '{}% | {}/{} files'.format(percentage, progress,
ctx.nfiles)
Print.progress(msg)
Print.progress('\n')
# Flush buffer
Print.flush()
# Evaluate errors and exit with appropriated return code
if ctx.error:
sys.exit(1)
def run(args=None):
"""
Main process that:
* Instantiates processing context,
* Defines the referenced time properties,
* Instantiates threads pools,
* Prints or logs the time axis diagnostics.
:param ArgumentParser args: Command-line arguments parser
"""
# Instantiate processing context
with ProcessingContext(args) as ctx:
# Collecting data
Print.progress('\rAnalysing data, please wait...')
ctx.nbfiles = len(ctx.sources)
# Init process context
cctx = {name: getattr(ctx, name) for name in PROCESS_VARS}
if ctx.use_pool:
# Init processes pool
pool = Pool(processes=ctx.processes,
initializer=initializer,
initargs=(cctx.keys(), cctx.values()))
# Process supplied files
processes = pool.imap(process, ctx.sources)
else:
initializer(cctx.keys(), cctx.values())
processes = itertools.imap(process, ctx.sources)
# Process supplied sources
handlers = [x for x in processes if x is not None]
# Close pool of workers if exists
if 'pool' in locals().keys():
locals()['pool'].close()
locals()['pool'].join()
locals()['pool'].terminate()
Print.progress('\n')
# Flush buffer
Print.flush()
# Get number of errors
ctx.nbskip = ctx.nbfiles - len(handlers)
ctx.nberrors = sum(handlers)
# Evaluate errors and exit with appropriate return code
if ctx.nbskip or ctx.nberrors:
sys.exit(ctx.nbskip + ctx.nberrors)
def run(args):
"""
Main process that:
* Instantiates processing context,
* Parallelizes file processing with threads pools,
* Copies mapfile(s) to the output directory,
* Evaluate exit status.
:param ArgumentParser args: Command-line arguments parser
"""
# Instantiate processing context
with ProcessingContext(args) as ctx:
# If dataset ID is submitted for "show" action skip scan
logging.info('==> Scan started')
if ctx.use_pool:
processes = ctx.pool.imap(process, ctx.sources)
else:
processes = itertools.imap(process, ctx.sources)
# Process supplied sources
results = [x for x in processes]
# Close progress bar
if ctx.pbar:
ctx.pbar.close()
# Get number of files scanned (including skipped files)
ctx.scan_files = len(results)
# Get number of scan errors
ctx.scan_errors = results.count(None)
# Get number of generated mapfiles
ctx.nb_map = len(filter(None, set(results)))
# Evaluates the scan results to finalize mapfiles writing
if evaluate(results):
for mapfile in filter(None, set(results)):
# Remove mapfile working extension
if ctx.action == 'show':
# Print mapfiles to be generated
if ctx.pbar or ctx.quiet:
print(remove(WORKING_EXTENSION, mapfile))
logging.info(remove(WORKING_EXTENSION, mapfile))
elif ctx.action == 'make':
# A final mapfile is silently overwritten if already exists
os.rename(mapfile, remove(WORKING_EXTENSION, mapfile))
def run(args):
"""
Main process that:
* Instantiates processing context,
* Loads previous program instance,
* Parallelizes file processing with threads pools,
* Apply command-line action to the whole DRS tree,
* Evaluate exit status.
:param ArgumentParser args: The command-line arguments parser
"""
# Instantiate processing context
with ProcessingContext(args) as ctx:
# Init global variable
global tree
# Init DRS tree
tree = DRSTree(ctx.root, ctx.version, ctx.mode, ctx.commands_file)
# Init process context
cctx = {name: getattr(ctx, name) for name in PROCESS_VARS}
# Disable file scan if a previous DRS tree have generated using same context and no "list" action
if do_scanning(ctx):
if ctx.use_pool:
# Init processes pool
pool = Pool(processes=ctx.processes,
initializer=initializer,
initargs=(cctx.keys(), cctx.values()))
processes = pool.imap(process, ctx.sources)
else:
initializer(cctx.keys(), cctx.values())
processes = itertools.imap(process, ctx.sources)
# Process supplied sources
handlers = [x for x in processes]
# Close pool of workers if exists
if 'pool' in locals().keys():
locals()['pool'].close()
locals()['pool'].join()
Print.progress('\n')
# Build DRS tree
cctx['progress'].value = 0
initializer(cctx.keys(), cctx.values())
results = [x for x in itertools.imap(tree_builder, handlers)]
Print.progress('\n')
else:
reader = load(TREE_FILE)
msg = 'Skip incoming files scan (use "--rescan" to force it) -- '
msg += 'Using cached DRS tree from {}'.format(TREE_FILE)
Print.warning(msg)
_ = reader.next()
tree = reader.next()
handlers = reader.next()
results = reader.next()
# Flush buffer
Print.flush()
# Rollback --commands-file value to command-line argument in any case
tree.commands_file = ctx.commands_file
# Get number of files scanned (including errors/skipped files)
ctx.scan_data = len(results)
# Get number of scan errors
ctx.scan_errors = results.count(None)
# Backup tree context for later usage with other command lines
store(
TREE_FILE,
data=[{key: ctx.__getattribute__(key)
for key in CONTROLLED_ARGS}, tree, handlers, results])
Print.info(TAGS.INFO +
'DRS tree recorded for next usage onto {}.'.format(
COLORS.HEADER(TREE_FILE)))
# Evaluates the scan results to trigger the DRS tree action
if evaluate(results):
# Check upgrade uniqueness
if not ctx.no_checksum:
tree.check_uniqueness(ctx.checksum_type)
# Apply tree action
tree.get_display_lengths()
getattr(tree, ctx.action)()
# Evaluate errors and exit with appropriated return code
if ctx.scan_errors > 0:
sys.exit(ctx.scan_errors)
def run(args):
"""
Main process that:
* Instantiates processing context
* Parses the configuration files options and values,
* Deduces facets and values from directories or dataset lists,
* Compares the values of each facet between both,
* Print or log the checking.
:param ArgumentParser args: The command-line arguments parser
"""
# Instantiate processing context manager
with ProcessingContext(args) as ctx:
# Get facets values used by the source
source_values = dict((facet, set()) for facet in ctx.facets)
for source in ctx.sources:
try:
attributes = re.match(ctx.pattern, source).groupdict()
for facet in ctx.facets:
source_values[facet].add(attributes[facet])
except AttributeError:
logging.error(ExpressionNotMatch(source, ctx.pattern))
ctx.scan_errors += 1
# Get facets values declared in configuration file
config_values = {}
for facet in ctx.facets:
logging.info(
'Collecting values from INI file(s) for "{}" facet...'.format(
facet))
try:
config_values[facet], _ = ctx.cfg.get_options(facet)
if not isinstance(config_values[facet], type(re.compile(""))):
config_values[facet] = set(config_values[facet])
except NoConfigOptions:
for option in ctx.cfg.get_options_from_list('maps'):
maptable = ctx.cfg.get(option)
from_keys, _ = split_map_header(maptable.split('\n')[0])
if facet in from_keys:
config_values[facet] = set(
ctx.cfg.get_options_from_map(option, facet))
finally:
if facet not in config_values.keys():
raise NoConfigOptions(facet)
# Compare values from sources against values from configuration file
print(''.center(WIDTH, '='))
print('{} :: {}'.format('Facet'.ljust(FACET_WIDTH),
'Status'.rjust(STATUS_WIDTH)))
print(''.center(WIDTH, '-'))
for facet in ctx.facets:
if isinstance(config_values[facet], type(re.compile(""))):
config_values[facet] = set([
v for v in source_values[facet]
if config_values[facet].search(v)
])
if not source_values[facet]:
print('{} :: {}'.format(facet.ljust(FACET_WIDTH),
STATUS[2].rjust(STATUS_WIDTH)))
elif not config_values[facet]:
print('{} :: {}'.format(facet.ljust(FACET_WIDTH),
STATUS[3].rjust(STATUS_WIDTH)))
else:
undeclared_values = source_values[facet].difference(
config_values[facet])
updated_values = source_values[facet].union(
config_values[facet])
if undeclared_values:
print('{} :: {}'.format(facet.ljust(FACET_WIDTH),
STATUS[1].rjust(STATUS_WIDTH)))
_values = ', '.join(sorted(undeclared_values))
logging.error('{} :: UNDECLARED VALUES :: {}'.format(
facet, _values))
_values = ', '.join(sorted(updated_values))
logging.error('{} :: UPDATED VALUES :: {}'.format(
facet, _values))
ctx.any_undeclared = True
else:
print('{} :: {}'.format(facet.ljust(FACET_WIDTH),
STATUS[0].rjust(STATUS_WIDTH)))
print(''.center(WIDTH, '='))
def run(args):
"""
Main process that:
* Instantiates processing context,
* Parallelizes file processing with threads pools,
* Copies mapfile(s) to the output directory,
* Evaluate exit status.
:param ArgumentParser args: Command-line arguments parser
"""
# Deal with 'quiet' option separately. If set, turn off all output
# before creating ProcessingContext, and turn it on only when needed
quiet = args.quiet if hasattr(args, 'quiet') else False
if quiet:
output_control = OutputControl()
output_control.stdout_off()
# Instantiate processing context
with ProcessingContext(args) as ctx:
# Init process context
cctx = {name: getattr(ctx, name) for name in PROCESS_VARS}
# Init progress bar
if ctx.use_pool:
# Init processes pool
pool = Pool(processes=ctx.processes,
initializer=initializer,
initargs=(cctx.keys(), cctx.values()))
processes = pool.imap(process, ctx.sources)
else:
initializer(cctx.keys(), cctx.values())
processes = itertools.imap(process, ctx.sources)
# Process supplied sources
results = [x for x in processes]
# Close pool of workers if exists
if 'pool' in locals().keys():
locals()['pool'].close()
locals()['pool'].join()
Print.progress('\n')
# Flush buffer
Print.flush()
# Get number of files scanned (excluding errors/skipped files)
ctx.scan_data = len(filter(None, results))
# Get number of scan errors
ctx.scan_errors = results.count(None)
# Get number of generated mapfiles
ctx.nbmap = len(filter(None, set(results)))
# Evaluates the scan results to finalize mapfiles writing
if evaluate(results):
for mapfile in filter(None, set(results)):
# Remove mapfile working extension
if ctx.action == 'show':
# Print mapfiles to be generated
result = remove(WORKING_EXTENSION, mapfile)
if quiet:
output_control.stdout_on()
print result
output_control.stdout_off()
else:
Print.result(result)
elif ctx.action == 'make':
# A final mapfile is silently overwritten if already exists
os.rename(mapfile, remove(WORKING_EXTENSION, mapfile))
# Evaluate errors and exit with appropriated return code
if ctx.scan_errors > 0:
sys.exit(ctx.scan_errors)
def run(args):
"""
Main process that:
* Instantiates processing context
* Parses the configuration files options and values,
* Deduces facets and values from directories or dataset lists,
* Compares the values of each facet between both,
* Print or log the checking.
:param ArgumentParser args: The command-line arguments parser
"""
# Instantiate processing context manager
with ProcessingContext(args) as ctx:
# Init process context
cctx = {name: getattr(ctx, name) for name in PROCESS_VARS}
cctx['source_values'][0] = dict((facet, set()) for facet in ctx.facets)
if ctx.use_pool:
# Init processes pool
pool = Pool(processes=ctx.processes,
initializer=initializer,
initargs=(cctx.keys(), cctx.values()))
processes = pool.imap(process, ctx.sources)
else:
initializer(cctx.keys(), cctx.values())
processes = itertools.imap(process, ctx.sources)
# Process supplied sources
results = [x for x in processes]
# Close pool of workers if exists
if 'pool' in locals().keys():
locals()['pool'].close()
locals()['pool'].join()
Print.progress('\n')
# Flush buffer
Print.flush()
ctx.scan_data = sum(results)
ctx.scan_errors = results.count(0)
# Get source values
source_values = cctx['source_values'][0]
# Get facets values declared in configuration file
config_values = {}
progress = 0
nfacets = len(ctx.facets)
for facet in ctx.facets:
try:
try:
config_values[facet], _ = ctx.cfg.get_options(facet)
if not isinstance(config_values[facet], type(
re.compile(""))):
config_values[facet] = set(config_values[facet])
except NoConfigOptions:
for option in ctx.cfg.get_options_from_list('maps'):
maptable = ctx.cfg.get(option)
from_keys, _ = split_map_header(
maptable.split('\n')[0])
if facet in from_keys:
config_values[facet] = set(
ctx.cfg.get_options_from_map(option, facet))
finally:
if facet not in config_values.keys():
raise NoConfigOptions(facet)
msg = TAGS.SUCCESS
msg += 'Get values from {} for {}'.format(
COLORS.HEADER(ctx.cfg.file), COLORS.HEADER(facet))
Print.info(msg)
except KeyboardInterrupt:
raise
except Exception:
exc = traceback.format_exc().splitlines()
msg = TAGS.FAIL
msg += 'Get values from {} for {}'.format(
COLORS.HEADER(ctx.cfg.file), COLORS.HEADER(facet)) + '\n'
msg += '\n'.join(exc)
Print.exception(msg, buffer=True)
ctx.scan_errors += 1
finally:
progress += 1
percentage = int(progress * 100 / nfacets)
msg = COLORS.OKBLUE(
'\rCollecting facet values from INI file(s): ')
msg += '{}% | {}/{} facet(s)'.format(percentage, progress,
nfacets)
Print.progress(msg)
Print.progress('\n')
# Flush buffer
Print.flush()
# Compare values from sources against values from configuration file
Print.result(''.center(WIDTH, '='))
Print.result('{} :: {}'.format('Facet'.ljust(FACET_WIDTH),
'Status'.rjust(STATUS_WIDTH)))
Print.result(''.center(WIDTH, '-'))
for facet in ctx.facets:
if isinstance(config_values[facet], type(re.compile(""))):
config_values[facet] = set([
v for v in source_values[facet]
if config_values[facet].search(v)
])
if not source_values[facet]:
line = '{} :: '.format(facet.ljust(FACET_WIDTH))
line += COLORS.WARNING(STATUS[2].rjust(STATUS_WIDTH))
Print.result(line)
elif not config_values[facet]:
line = '{} :: '.format(facet.ljust(FACET_WIDTH))
line += COLORS.WARNING(STATUS[3].rjust(STATUS_WIDTH))
Print.result(line)
else:
undeclared_values = source_values[facet].difference(
config_values[facet])
updated_values = source_values[facet].union(
config_values[facet])
if undeclared_values:
line = '{} :: '.format(facet.ljust(FACET_WIDTH))
line += COLORS.FAIL(STATUS[1].rjust(STATUS_WIDTH))
Print.result(line)
_values = ', '.join(sorted(undeclared_values))
msg = COLORS.FAIL(':: UNDECLARED VALUES :: ')
msg += COLORS.HEADER(facet)
msg += COLOR().bold(' :: {}'.format(_values))
Print.error(msg, buffer=True)
_values = ', '.join(sorted(updated_values))
msg = COLORS.SUCCESS(':: UPDATED VALUES :: ')
msg += COLORS.HEADER(facet)
msg += ' :: {}'.format(_values)
Print.error(msg, buffer=True)
ctx.any_undeclared = True
else:
line = '{} :: '.format(facet.ljust(FACET_WIDTH))
line += COLORS.SUCCESS(STATUS[0].rjust(STATUS_WIDTH))
Print.result(line)
Print.result(''.center(WIDTH, '='))
# Flush buffer
Print.flush()
# Evaluate errors and exit with appropriated return code
if ctx.scan_errors > 0:
sys.exit(ctx.scan_errors)
if ctx.any_undeclared:
sys.exit(-2)
def run(args):
"""
Main process that:
* Decide to fetch or not depending on file presence/absence and command-line arguments,
* Gets the GitHub file content from full API URL,
* Backups old file if desired,
* Writes response into table file.
:param ArgumentParser args: Parsed command-line arguments
"""
# Instantiate processing context manager
with ProcessingContext(args) as ctx:
for project in ctx.project:
try:
# Set repository name
repo = REPO_PATTERN.format(project)
# Get the list of available refs for that repository
r = gh_request_content(url=ctx.ref_url.format(repo), auth=ctx.auth)
refs = [os.path.basename(ref['url']) for ref in r.json()]
# Get refs to fetch
if hasattr(ctx, 'ref'):
if ctx.ref not in refs:
raise GitHubReferenceNotFound(ctx.ref, refs)
fetch_refs = [ctx.ref]
else:
fetch_refs = filter(re.compile(ctx.ref_regex).match, refs)
if not fetch_refs:
raise GitHubReferenceNotFound(ctx.ref_regex.pattern, refs)
Print.debug('GitHub Available reference(s): {}'.format(', '.join(sorted(refs))))
Print.info('Selected GitHub reference(s): {}'.format(', '.join(sorted(fetch_refs))))
# Get special case for CMIP6_CV.json file
special_cases = get_special_case(f='CMIP6_CV.json', url=ctx.url, repo=repo, ref='master', auth=ctx.auth)
# Fetch each ref
for ref in fetch_refs:
try:
# Set reference url
url = ctx.url.format(repo)
if ref:
url += GITHUB_API_PARAMETER.format('ref', ref)
Print.debug('Fetch {} tables from "{}" GitHub reference'.format(project, ref))
# Build output directory
if ctx.no_subfolder:
outdir = make_outdir(ctx.tables_dir, repo)
else:
outdir = make_outdir(ctx.tables_dir, repo, ref)
# Fetch GitHub reference
fetch_gh_ref(url=url,
outdir=outdir,
auth=ctx.auth,
keep=ctx.keep,
overwrite=ctx.overwrite,
backup_mode=ctx.backup_mode,
filter=ctx.file_filter,
special_cases=special_cases)
except Exception:
exc = traceback.format_exc().splitlines()
msg = TAGS.FAIL
msg += 'Fetching {} tables from {} GitHub reference'.format(COLORS.HEADER(project),
COLORS.HEADER(ref)) + '\n'
msg += '\n'.join(exc)
Print.exception(msg, buffer=True)
ctx.error = True
except Exception:
exc = traceback.format_exc().splitlines()
msg = TAGS.FAIL
msg += 'Fetching {} tables'.format(COLORS.HEADER(project)) + '\n'
msg += '\n'.join(exc)
Print.exception(msg, buffer=True)
ctx.error = True
# Flush buffer
Print.flush()
# Evaluate errors and exit with appropriated return code
if ctx.error:
sys.exit(1)
def main():
"""
Run main program
"""
# Get command-line arguments
prog, args = get_args()
# Init print management
Print.init(log=args.log, debug=args.debug, all=args.all, cmd=prog)
# Print command-line
Print.command(COLORS.OKBLUE + 'Command: ' + COLORS.ENDC + ' '.join(sys.argv))
# Instantiate processing context
with ProcessingContext(args) as ctx:
# Collecting data
Print.progress('\rCollecting data, please wait...')
# Get number of XML
ctx.nbxml = len([x for x in yield_filedef(ctx.xml)])
# Get number of files
ctx.nbcdf = len([x for x in yield_files(ctx.directory)])
# Init process context
cctx = {name: getattr(ctx, name) for name in PROCESS_VARS}
# Process XML files
if ctx.use_pool:
# Init processes pool
pool = Pool(processes=ctx.processes, initializer=initializer, initargs=(cctx.keys(),
cctx.values()))
xios_input = [i for x in pool.imap(get_patterns_from_filedef, yield_filedef(ctx.xml)) for i in x]
# Close pool of workers
pool.close()
pool.join()
else:
initializer(cctx.keys(), cctx.values())
xios_input = [i for x in itertools.imap(get_patterns_from_filedef, yield_filedef(ctx.xml)) for i in x]
# Get number of entries
ctx.nbentries = len(xios_input)
Print.progress('\n')
# Reset progress counter
cctx['progress'].value = 0
# Process XIOS files
if ctx.use_pool:
# Init processes pool
pool = Pool(processes=ctx.processes, initializer=initializer, initargs=(cctx.keys(),
cctx.values()))
xios_output = [x for x in pool.imap(get_patterns_from_files, yield_files(ctx.directory))]
# Close pool of workers
pool.close()
pool.join()
else:
initializer(cctx.keys(), cctx.values())
xios_output = [x for x in itertools.imap(get_patterns_from_files, yield_files(ctx.directory))]
# Control
assert ctx.nbcdf == len(xios_output)
# Check sources have been found
if not xios_input:
raise NoPatternFound()
if not xios_output:
raise NoFileFound()
# Remove unchanged facet among all entries for better display
if not args.full_table:
xios_all = [x.get_attrs().values() for x in xios_input]
xios_all.extend([x.get_attrs().values() for x in xios_output])
fn_facets = xios_input[0].get_attrs()
hidden_facets = [fn_facets.keys()[fn_facets.values().index(i[0])] for i in zip(*xios_all) if
len(set(i)) == 1]
for facet in hidden_facets:
del (ctx.facets[facet])
xios_input = tupleized(ctx.facets, [x.mask(hidden_facets) for x in xios_input])
xios_output = tupleized(ctx.facets, [x.mask(hidden_facets) for x in xios_output])
else:
xios_input = tupleized(ctx.facets, [x.get_attrs() for x in xios_input])
xios_output = tupleized(ctx.facets, [x.get_attrs() for x in xios_output])
# Run comparison between input and output
# What is in input AND in output
common = xios_input.intersection(xios_output)
# What is in input BUT NOT in output
missing_output = xios_input.difference(xios_output)
# What is in output BUT NOT in input
missing_input = xios_output.difference(xios_input)
# Build table title
labels = get_labels(ctx.facets)
# Build table width
widths = get_widths(labels, xios_input.union(xios_output))
# Add labels for results
labels.extend([DR2XML_LABEL, XIOS_LABEL])
widths.extend([DR2XML_WIDTH, XIOS_WIDTH])
# Table separators
separators = ['| '] * (len(ctx.facets) - 1)
separators += ['| ', '| ']
# Get total width for display
total_width = sum(widths) + sum([len(sep) for sep in separators]) + 1
# No print if no differences
if missing_input or missing_output:
# Print table header
Print.success('\n\n' + '+' + ''.center(total_width, '-') + '+' + '\n')
Print.success('| ' + align(labels, widths, sep=separators) + '|' + '\n')
Print.success('+' + ''.center(total_width, '-') + '+' + '\n')
# Print results for each entry
lines = 0
input_total = 0
output_total = 0
for tup in sorted(xios_input.union(xios_output)):
line = list(tup)
if tup in common:
if args.all:
line.extend([IS, IS])
input_total += 1
output_total += 1
else:
continue
elif tup in missing_input:
line.extend([ISNOT, IS])
output_total += 1
ctx.notinxml += 1
elif tup in missing_output:
line.extend([IS, ISNOT])
input_total += 1
ctx.notinxios += 1
else:
line.extend([ISNOT, ISNOT])
lines += 1
Print.success('| ' + align(line, widths, sep=separators) + '|' + '\n')
# Print table footer with totals
Print.success('+' + ''.center(total_width, '-') + '+' + '\n')
line = align(fields=['Totals = {}'.format(lines), str(input_total), str(output_total)],
widths=[len(align(labels[:-2], widths[:-2])), widths[-2], widths[-1]],
sep=['| ', '| '])
Print.success('| ' + line + '|' + '\n')
Print.success('+' + ''.center(total_width, '-') + '+')
# Print summary
msg = COLORS.HEADER + '\n\nNumber of DR2XML file(s) scanned: {}'.format(ctx.nbxml) + COLORS.ENDC
msg += COLORS.HEADER + '\nNumber of DR2XML entry(ies) scanned: {}'.format(ctx.nbentries) + COLORS.ENDC
msg += COLORS.HEADER + '\nNumber of netCDF file(s) scanned: {}'.format(ctx.nbcdf) + COLORS.ENDC
if (ctx.notinxml + ctx.notinxios):
msg += COLORS.FAIL
else:
msg += COLORS.OKGREEN
msg += '\nNumber of difference(s) between XML and XIOS outputs: {}'.format(
ctx.notinxml + ctx.notinxios) + COLORS.ENDC
if ctx.notinxml:
msg += COLORS.FAIL
else:
msg += COLORS.OKGREEN
msg += '\nNumber of XIOS output(s) not in XML files: {}'.format(ctx.notinxml) + COLORS.ENDC
if ctx.notinxios:
msg += COLORS.FAIL
else:
msg += COLORS.OKGREEN
msg += '\nNumber of XML entry(ies) not as XIOS output: {}\n'.format(ctx.notinxios) + COLORS.ENDC
# Critical level used to print in any case
Print.summary(msg)
# Print log path if exists
Print.info(COLORS.HEADER + '\nSee log: {}\n'.format(Print.LOGFILE) + COLORS.ENDC)
# Evaluate errors and exit with appropriate return code
if ctx.notinxios or ctx.notinxml:
# Some datasets (at least one) has error(s). Error code = nb datasets with error(s)
sys.exit(1)
else:
# No errors. Error code = 0
sys.exit(0)
def run(args=None):
"""
Main process that:
* Instantiates processing context,
* Deduces start, end and next date from each filenames,
* Builds the DiGraph,
* Detects the shortest path between dates if exists,
* Detects broken path between dates if exists,
* Removes the overlapping files.
:param ArgumentParser args: Command-line arguments parser
"""
# Declare global variables
global graph, patterns, resolve, period_start, period_end
# Instantiate processing context
with ProcessingContext(args) as ctx:
# Collecting data
Print.progress('\rCollecting data, please wait...')
# Get number of files
ctx.nbfiles = len(ctx.sources)
# Init process context
cctx = {name: getattr(ctx, name) for name in PROCESS_VARS}
if ctx.use_pool:
# Init processes pool
pool = Pool(processes=ctx.processes,
initializer=initializer,
initargs=(cctx.keys(), cctx.values()))
# Process supplied files to create nodes in appropriate directed graph
processes = pool.imap(extract_dates, ctx.sources)
else:
initializer(cctx.keys(), cctx.values())
processes = itertools.imap(extract_dates, ctx.sources)
# Process supplied sources
handlers = [x for x in processes if x is not None]
# Close pool of workers if exists
if 'pool' in locals().keys():
locals()['pool'].close()
locals()['pool'].join()
# Get number of files skipped
ctx.skip = ctx.nbfiles - len(handlers)
# Process XML files if card
Print.progress('\n')
patterns = dict()
if ctx.xml:
# Reset progress counter
cctx['progress'].value = 0
# Get number of xml
ctx.nbxml = len([x for x in yield_filedef(ctx.xml)])
cctx['nbxml'] = ctx.nbxml
if ctx.use_pool:
# Init processes pool
pool = Pool(processes=ctx.processes,
initializer=initializer,
initargs=(cctx.keys(), cctx.values()))
for k, v in pool.imap(get_patterns_from_filedef,
yield_filedef(ctx.xml)):
if k not in patterns.keys():
patterns[k] = list()
patterns[k].extend(v)
# Close pool of workers
pool.close()
pool.join()
else:
initializer(cctx.keys(), cctx.values())
for k, v in itertools.imap(get_patterns_from_filedef,
yield_filedef(ctx.xml)):
if k not in patterns.keys():
patterns[k] = list()
patterns[k].extend(v)
Print.progress('\n')
# Initialize Graph()
graph = Graph()
global period_start
period_start = ctx.period_start
period_end = ctx.period_end
# Process filename handler to create nodes
ctx.nbnodes = len([x for x in itertools.imap(create_nodes, handlers)])
# Process each directed graph to create appropriate edges
ctx.nbdsets = len([x for x in itertools.imap(create_edges, graph())])
# Evaluate each graph if a shortest path exist
resolve = ctx.resolve
for path, partial_overlaps, full_overlaps in itertools.imap(
evaluate_graph, graph()):
# Format message about path
msg = format_path(path, partial_overlaps, full_overlaps)
# If broken time series
if 'BREAK' in path:
ctx.broken += 1
Print.error(COLORS.FAIL('Time series broken: ') + msg)
elif 'XML GAP' in path:
Print.success(
COLORS.WARNING('Time series with XML gap(s): ') + msg)
else:
# Print overlaps if exists
if full_overlaps or partial_overlaps:
ctx.overlaps += 1
Print.error(
COLORS.FAIL('Continuous time series with overlaps: ') +
msg)
else:
Print.success(
COLORS.SUCCESS('Continuous time series: ') + msg)
# Resolve overlaps
if resolve:
# Full overlapping files has to be deleted before partial overlapping files are truncated.
for node in full_overlaps:
resolve_overlap(ffp=full_overlaps[node]['path'],
pattern=ctx.pattern,
partial=False)
if not ctx.full_only:
for node in partial_overlaps:
resolve_overlap(
ffp=partial_overlaps[node]['path'],
pattern=ctx.pattern,
from_date=partial_overlaps[node]['cutting_date'],
to_date=partial_overlaps[node]['end'],
cutting_timestep=partial_overlaps[node]
['cutting_timestep'],
partial=True)
# Evaluate errors and exit with appropriate return code
if ctx.overlaps or ctx.broken or ctx.nberrors:
sys.exit(ctx.broken + ctx.overlaps + ctx.nberrors)