def _verify_repo(i, errs): """Verifies that an institutional repo exists. """ path = io_mgr.get_folder([i]) if not os.path.exists(path): errs.append("ERROR: repo not found: {}".format(i.raw_name))
def _verify_models(i, errs): """Verifies that a set of institutional RCM model documentation files exists. """ # Set identifiers of models to be documented. identifiers = set( [j.canonical_name for j in vocabs.get_models_by_institution(i)]) # Set directories. folder_root = io_mgr.get_folder([i, 'cordex', 'models']) documented = set([ j for j in os.listdir(folder_root) if os.path.isdir(os.path.join(folder_root, j)) ]) # Set undocuemtned & obsolete models. undocumented = identifiers.difference(documented) obsolete = documented.difference(identifiers) # Verify each model and then model topics. for model in vocabs.get_models_by_institution(i): if model.canonical_name in undocumented: errs.append("model undocumented :: {}".format( model.canonical_name.upper())) elif model.canonical_name in obsolete: errs.append("model obsolete :: {}".format( model.canonical_name.upper())) else: for topic in vocabs.get_topics(): topic_xls = io_mgr.get_model_topic_xls(i, model, topic) if not os.path.exists(topic_xls): errs.append("model topic undocumented :: {} :: {}".format( model.canonical_name, topic.canonical_name))
def _verify_models(institute, errs): """Verifies that a set of institutional model documentation files exists. """ # Root folder for models. folder = io_mgr.get_folder([institute, 'cmip6', 'models']) # Model configurations to be documented. configurations = vocabs.get_model_configurations(institute) # Set of model configuration names. models = set([i.canonical_name for i in configurations]) # Model directories. folders = set([ i for i in os.listdir(folder) if os.path.isdir(os.path.join(folder, i)) ]) # Undocumented models. undocumented = models.difference(folders) for model in undocumented: errs.append("{} --> {} :: model undocumented".format( institute.raw_name, model)) # Obsolete folders. obsolete = folders.difference(models) for model in obsolete: errs.append("{} --> {} :: model obsolete".format( institute.raw_name, model)) # Documented models: validate sub-folders / files. documented = [ i for i in configurations if i.canonical_name not in undocumented ] for source_id in documented: # ... model CIM document; fpath = io_mgr.get_model_cim(institute, source_id) if not os.path.isfile(fpath): err = "{} --> {} :: TODO: cmip6-models-generate-cim {}".format( institute.raw_name, source_id.raw_name, institute.canonical_name) errs.append(err) # ... model topic xls|json|pdf for topic in vocabs.get_model_topics(source_id): for fpath_factory, ftype in ( (io_mgr.get_model_topic_xls, 'xls'), (io_mgr.get_model_topic_json, 'json'), (io_mgr.get_model_topic_pdf, 'pdf'), ): fpath = fpath_factory(institute, source_id, topic) if not os.path.isfile(fpath): err = "{} --> {} --> {} :: TODO cmip6-models-generate-{} {} {}".format( institute.raw_name, source_id.raw_name, topic.raw_name, ftype, institute.canonical_name, source_id.canonical_name) errs.append(err)
def _get_submitted_file(institute, source_id): """Returns a submission file. """ folder = io_mgr.get_folder((institute, 'cmip6', 'submission')) fname = "cmip6_{}_{}.json".format(institute.canonical_name, source_id.canonical_name) return "{}/{}".format(folder, fname)
def _verify_sub_folders(i, errs): """Verifies that an institutional repo sub-folder exists. """ for sub_folder in SUB_FOLDERS: path = [i] + sub_folder.split('/') folder = io_mgr.get_folder(path) if not os.path.exists(folder): errs.append("ERROR: sub-folder not found: {} --> {}".format( i.raw_name, sub_folder))
def _verify_static_files(i, errs): """Verifies that a set of institutional static files exists. """ for static_file in STATIC_FILES: folder = io_mgr.get_folder([i]) fname = static_file.format(i.canonical_name) path = os.path.join(folder, fname) if not os.path.exists(path): errs.append("file not found: {}".format(fname))