def _verify_repo(i, errs):
"""Verifies that an institutional repo exists.
"""
path = io_mgr.get_folder([i])
if not os.path.exists(path):
errs.append("ERROR: repo not found: {}".format(i.raw_name))
def _verify_models(i, errs):
"""Verifies that a set of institutional RCM model documentation files exists.
"""
# Set identifiers of models to be documented.
identifiers = set(
[j.canonical_name for j in vocabs.get_models_by_institution(i)])
# Set directories.
folder_root = io_mgr.get_folder([i, 'cordex', 'models'])
documented = set([
j for j in os.listdir(folder_root)
if os.path.isdir(os.path.join(folder_root, j))
])
# Set undocuemtned & obsolete models.
undocumented = identifiers.difference(documented)
obsolete = documented.difference(identifiers)
# Verify each model and then model topics.
for model in vocabs.get_models_by_institution(i):
if model.canonical_name in undocumented:
errs.append("model undocumented :: {}".format(
model.canonical_name.upper()))
elif model.canonical_name in obsolete:
errs.append("model obsolete :: {}".format(
model.canonical_name.upper()))
else:
for topic in vocabs.get_topics():
topic_xls = io_mgr.get_model_topic_xls(i, model, topic)
if not os.path.exists(topic_xls):
errs.append("model topic undocumented :: {} :: {}".format(
model.canonical_name, topic.canonical_name))
def _verify_models(institute, errs):
"""Verifies that a set of institutional model documentation files exists.
"""
# Root folder for models.
folder = io_mgr.get_folder([institute, 'cmip6', 'models'])
# Model configurations to be documented.
configurations = vocabs.get_model_configurations(institute)
# Set of model configuration names.
models = set([i.canonical_name for i in configurations])
# Model directories.
folders = set([
i for i in os.listdir(folder) if os.path.isdir(os.path.join(folder, i))
])
# Undocumented models.
undocumented = models.difference(folders)
for model in undocumented:
errs.append("{} --> {} :: model undocumented".format(
institute.raw_name, model))
# Obsolete folders.
obsolete = folders.difference(models)
for model in obsolete:
errs.append("{} --> {} :: model obsolete".format(
institute.raw_name, model))
# Documented models: validate sub-folders / files.
documented = [
i for i in configurations if i.canonical_name not in undocumented
]
for source_id in documented:
# ... model CIM document;
fpath = io_mgr.get_model_cim(institute, source_id)
if not os.path.isfile(fpath):
err = "{} --> {} :: TODO: cmip6-models-generate-cim {}".format(
institute.raw_name, source_id.raw_name,
institute.canonical_name)
errs.append(err)
# ... model topic xls|json|pdf
for topic in vocabs.get_model_topics(source_id):
for fpath_factory, ftype in (
(io_mgr.get_model_topic_xls, 'xls'),
(io_mgr.get_model_topic_json, 'json'),
(io_mgr.get_model_topic_pdf, 'pdf'),
):
fpath = fpath_factory(institute, source_id, topic)
if not os.path.isfile(fpath):
err = "{} --> {} --> {} :: TODO cmip6-models-generate-{} {} {}".format(
institute.raw_name, source_id.raw_name, topic.raw_name,
ftype, institute.canonical_name,
source_id.canonical_name)
errs.append(err)
def _get_submitted_file(institute, source_id):
"""Returns a submission file.
"""
folder = io_mgr.get_folder((institute, 'cmip6', 'submission'))
fname = "cmip6_{}_{}.json".format(institute.canonical_name,
source_id.canonical_name)
return "{}/{}".format(folder, fname)
def _verify_sub_folders(i, errs):
"""Verifies that an institutional repo sub-folder exists.
"""
for sub_folder in SUB_FOLDERS:
path = [i] + sub_folder.split('/')
folder = io_mgr.get_folder(path)
if not os.path.exists(folder):
errs.append("ERROR: sub-folder not found: {} --> {}".format(
i.raw_name, sub_folder))
def _verify_static_files(i, errs):
"""Verifies that a set of institutional static files exists.
"""
for static_file in STATIC_FILES:
folder = io_mgr.get_folder([i])
fname = static_file.format(i.canonical_name)
path = os.path.join(folder, fname)
if not os.path.exists(path):
errs.append("file not found: {}".format(fname))