def main():
benchmark_dir = os.path.join(os.getcwd(), 'results')
os.makedirs(benchmark_dir)
pull_request_id = os.environ.get('PULL_REQUEST_NUMBER')
commit_hash = get_commit_hash() #[:8]
info = BenchmarksInfo(pull_request_id, commit_hash)
info.datetime.append(datatime_with_format()) #start time
print(f'pull_request_id = {pull_request_id}')
print(f'commit_hash = {commit_hash}')
for arch in benchmark_archs:
#init & run
suites = BenchmarkSuites(arch)
suites.run()
#save result
suites.save(benchmark_dir)
#add benchmark info
info.add_suites_info(arch, suites)
info.datetime.append(datatime_with_format()) #end time
#save commit and benchmark info
info_path = os.path.join(benchmark_dir, '_info.json')
info_str = dump2json(info)
with open(info_path, 'w') as f:
print(info_str, file=f)
def main():
definitions = read_definitions()
for root, dirs, files in os.walk('schema'):
for file_name in files:
schema_path = Path(root) / file_name
schema_path_components = schema_path.with_suffix('').parts
output_path = Path('output') / schema_path
simulation_round = schema_path_components[1]
product = schema_path_components[2]
if product.endswith('InputData'):
category = schema_path_components[3]
sector = None
else:
category = None
sector = schema_path_components[3]
# step 1: read schema template
with open(schema_path, encoding='utf-8') as f:
schema = {
'$schema': 'http://json-schema.org/draft-07/schema#',
'$id': URL + schema_path.as_posix(),
'commit': get_commit_hash()
}
schema.update(json.loads(f.read()))
# step 2: loop over properties/specifiers/properties and add enums from definition files
for identifier, properties in schema['properties']['specifiers'][
'properties'].items():
if identifier in definitions:
rows = definitions[identifier]
enum = []
if product.endswith('InputData'):
for row in filter_rows(rows,
simulation_round,
product,
category=category):
enum.append(
row.get('specifier_file')
or row.get('specifier'))
else:
for row in filter_rows(rows,
simulation_round,
product,
sector=sector):
enum.append(
row.get('specifier_file')
or row.get('specifier'))
properties['enum'] = list(set(enum))
# step 3: write json schema
write_json(output_path, schema)
def main():
definitions = read_definitions()
simulation_rounds = [definition['specifier'] for definition in definitions['simulation_round']]
products = [definition['specifier'] for definition in definitions['product']]
categories = [definition['specifier'] for definition in definitions['category']]
sectors = [definition['specifier'] for definition in definitions['sector']]
for simulation_round in simulation_rounds:
for product in products:
if product.endswith('InputData'):
for category in categories:
output_path = Path('output').joinpath('definitions') \
.joinpath(simulation_round).joinpath(product).joinpath(category) \
.with_suffix('.json')
output_definitions = {
'commit': get_commit_hash()
}
for definition_name, rows in definitions.items():
output_definitions[definition_name] = []
for row in filter_rows(rows, simulation_round, product, category=category):
output_definitions[definition_name].append(filter_row(row, simulation_round, product, category=category))
write_json(output_path, output_definitions)
else:
for sector in sectors:
output_path = Path('output').joinpath('definitions') \
.joinpath(simulation_round).joinpath(product).joinpath(sector) \
.with_suffix('.json')
output_definitions = {
'commit': get_commit_hash()
}
for definition_name, rows in definitions.items():
output_definitions[definition_name] = []
for row in filter_rows(rows, simulation_round, product, sector=sector):
output_definitions[definition_name].append(filter_row(row, simulation_round, product, sector=sector))
write_json(output_path, output_definitions)
def save_as_markdown(self, arch_dir='./'):
current_time = datatime_with_format()
commit_hash = get_commit_hash() #[:8]
file_name = f'{self.suite_name}.md'
file_path = os.path.join(arch_dir, file_name)
with open(file_path, 'w') as f:
lines = [
f'commit_hash: {commit_hash}\n', f'datatime: {current_time}\n'
]
lines += self._get_markdown_lines()
for line in lines:
print(line, file=f)
def main():
glossary = {'commit': get_commit_hash(), 'terms': {}}
for identifier, rows in read_definitions().items():
if identifier not in glossary['terms']:
glossary['terms'][identifier] = {}
for row in rows:
specifier = row.pop('specifier')
glossary['terms'][identifier][specifier] = row
glossary_path = Path('output') / 'glossary.json'
write_json(glossary_path, glossary)
def main():
for root, dirs, files in os.walk('tree'):
for file_name in files:
tree_path = Path(root) / file_name
output_path = Path('output') / tree_path
# step 2: open and read pattern
with open(tree_path, encoding='utf-8') as f:
identifiers = json.loads(f.read())
# create a pattern json from scratch
tree_json = {
'commit': get_commit_hash(),
'identifiers': [identifier.replace(' ', '') for identifier in identifiers]
}
# step 3: write json file
write_json(output_path, tree_json)
def main():
simulation_rounds = json.loads(
open('definitions/simulation_round.json', encoding='utf-8').read())
sectors = json.loads(
open('definitions/sector.json', encoding='utf-8').read())
commit_hash = get_commit_hash()
commit_date = get_commit_date()
commit_url = URL + '/commit/' + commit_hash
# step 1: create hierarchy of simulation_rounds, products, and sectors
pages = []
for simulation_round in simulation_rounds:
simulation_round['sectors'] = []
for sector in sectors:
if 'simulation_rounds' not in sector or simulation_round[
'specifier'] in sector['simulation_rounds']:
simulation_round['sectors'].append(sector)
pages.append(simulation_round)
# step 2: open and read intro
markdown_path = os.path.join('protocol', '10.index.md')
with open(markdown_path, encoding='utf-8') as f:
md = f.read()
html = markdown(md)
# step 3: render content into layout template
template_path = os.path.join('templates', 'index.html')
output_path = os.path.join('output', 'index.html')
enviroment = Environment(loader=FileSystemLoader(['templates']))
os.makedirs(os.path.dirname(output_path), exist_ok=True)
with open(template_path, encoding='utf-8') as f:
template = enviroment.from_string(f.read())
with open(output_path, 'w', encoding='utf-8') as f:
f.write(
template.render(commit_url=commit_url,
commit_hash=commit_hash,
commit_date=commit_date,
pages=pages,
html=html,
base_path='.'))
def main():
for root, dirs, files in os.walk('pattern'):
for file_name in files:
pattern_path = Path(root) / file_name
output_path = Path('output') / pattern_path
# create a pattern json from scratch
pattern_json = {
'commit': get_commit_hash()
}
# step 2: open and read pattern
with open(pattern_path, encoding='utf-8') as f:
pattern = json.loads(f.read())
path = pattern.get('path') or [
"(?P<simulation_round>[A-Za-z0-9]+)",
"(?P<product>[A-Za-z]+)",
"(?P<sector>[a-z0-9-_]+)",
"(?P<model>[A-Za-z0-9-+._]+)",
"(?P<climate_forcing>[a-z0-9-]+)",
"(?P<period>[a-z0-9-_]+)"
]
dataset = pattern.get('dataset') or pattern['file'][:-2]
file = pattern['file']
suffix = pattern.get('suffix') or ['.nc']
pattern_json['path'] = '/'.join(path) + '$'
pattern_json['dataset'] = '^' + '_'.join(dataset)
pattern_json['file'] = '^' + '_'.join(file) + '({})'.format('|'.join(suffix))
pattern_json['suffix'] = suffix
pattern_json['specifiers'] = pattern.get('specifiers', {})
pattern_json['specifiers_map'] = pattern.get('specifiers_map', {})
# step 3: write json file
write_json(output_path, pattern_json)
def main():
simulation_rounds = json.loads(
open('definitions/simulation_round.json', encoding='utf-8').read())
sectors = json.loads(
open('definitions/sector.json', encoding='utf-8').read())
sectors.append({'title': 'all sectors combined', 'specifier': 'index'})
commit_hash = get_commit_hash()
commit_date = get_commit_date()
commit_url = URL + '/commit/' + commit_hash
definitions = read_definitions()
for simulation_round in simulation_rounds:
for sector in sectors:
if 'simulation_rounds' not in sector or simulation_round[
'specifier'] in sector['simulation_rounds']:
protocol_path = os.path.join('protocol', '00.base.md')
pattern_path = os.path.join(
'pattern', simulation_round['specifier'], 'OutputData',
'{}.json'.format(sector['specifier']))
output_path = os.path.join(
'output/protocol', simulation_round['specifier'],
'{}.html'.format(sector['specifier']))
layout_path = os.path.join('templates', 'layout.html')
os.makedirs(os.path.dirname(output_path), exist_ok=True)
# step 1: open and read protocol
with open(protocol_path, encoding='utf-8') as f:
template_string = f.read()
# step 1: open and read protocol
with open(pattern_path, encoding='utf-8') as f:
pattern_list = json.loads(f.read())['file']
pattern = '_'.join(pattern_list) + '.nc'
pattern_simple = get_pattern_simple(pattern_list)
# step 2: render the template using jinja2
enviroment = Environment(loader=FileSystemLoader(
['bibliography', 'protocol', 'templates']))
template = enviroment.from_string(template_string)
md = template.render(simulation_round=simulation_round,
sector=sector,
definitions=definitions,
pattern=pattern,
pattern_simple=pattern_simple,
commit_url=commit_url,
commit_hash=commit_hash,
commit_date=commit_date,
table=Table(simulation_round, sector,
definitions, Counter()))
# step 3: convert markdown to html
html = markdown(md,
extensions=[
'fenced_code', 'attr_list',
TocExtension(toc_depth='2-3')
])
# step 4: replace shortcodes
html = html.replace(
'[mandatory]',
'<span class="badge badge-success">mandatory</span>')
# step 5: render content into layout template
with open(layout_path, encoding='utf-8') as f:
template = Template(f.read(),
trim_blocks=True,
lstrip_blocks=True)
with open(output_path, 'w', encoding='utf-8') as f:
f.write(
template.render(content=html,
simulation_round=simulation_round,
sector=sector,
base_path='../..',
commit_url=commit_url,
commit_hash=commit_hash,
commit_date=commit_date))
def __init__(self):
"""init with commit info"""
self.commit_hash = get_commit_hash()
self.datetime = datatime_with_format()
self.suites = {}
print(f'commit_hash = {self.commit_hash}')