def validateFile(self, fileobj):
"""
for CMIP6, this will first verify if the data is written by CMOR at the correct version set in the ini file.
If so, the file is declared valid. If not, file will go through PrePARE (CV) check. PrePARE runs CFChecker
Raises ESGPublishError if settings are missing or file fails the checks.
Raise ESGInvalidMetadataFormat if the file cannot be processed by this handler.
"""
validator = PrePARE.PrePARE
f = fileobj.path
config = getConfig()
projectSection = 'project:' + self.name
min_cmor_version = config.get(projectSection,
"min_cmor_version",
default="0.0.0")
file_cmor_version = "0.0.0"
try:
file_cmor_version = fileobj.getAttribute('cmor_version', None)
except:
debug(
'File %s missing cmor_version attribute; will proceed with PrePARE check'
% f)
if compareLibVersions(min_cmor_version, file_cmor_version):
debug('File %s cmor-ized at version %s, passed!"' %
(f, file_cmor_version))
return
# PrePARE is going to handle the CF check now
# min_cf_version = config.get(projectSection, "min_cf_version", defaut="")
# if len(min_cf_version) == 0:
# raise ESGPublishError("Minimum CF version not set in esg.ini")
# fakeversion = ["cfchecker.py", "-v", min_cf_version
# , "foo"]
# (badc,coards,uploader,useFileName,standardName,areaTypes,udunitsDat,version,files)=getargs(fakeversion)
# CF_Chk_obj = CFChecker(uploader=uploader, useFileName=useFileName, badc=badc, coards=coards, cfStandardNamesXML=standardName, cfAreaTypesXML=areaTypes, udunitsDat=udunitsDat, version=version)
# rc = CF_Chk_obj.checker(f)
# if (rc > 0):
# raise ESGPublishError("File %s fails CF check"%f)
file_data_specs_version = None
try:
file_data_specs_version = fileobj.getAttribute(
'data_specs_version', None)
except Exception as e:
raise ESGPublishError(
"File %s missing required data_specs_version global attribute"
% f)
table = None
try:
table = fileobj.getAttribute('table_id', None)
except:
raise ESGPublishError(
"File %s missing required table_id global attribute" % f)
try:
variable_id = fileobj.getAttribute('variable_id', None)
except:
raise ESGPublishError(
"File %s missing required variable_id global attribute" % f)
project_section = 'config:cmip6'
cmor_table_path = ""
try:
cmor_table_path = config.get(projectSection,
"cmor_table_path",
defaut="")
except:
debug("Missing cmor_table_path setting. Using default location")
if cmor_table_path == "":
cmor_table_path = DEFAULT_CMOR_TABLE_PATH
checkAndUpdateRepo(cmor_table_path, self, file_data_specs_version)
table_file = cmor_table_path + '/CMIP6_' + table + '.json'
fakeargs = ['--variable', variable_id, table_file, f]
parser = argparse.ArgumentParser(prog='esgpublisher')
parser.add_argument('--variable')
parser.add_argument('cmip6_table', action=validator.JSONAction)
parser.add_argument('infile', action=validator.CDMSAction)
parser.add_argument('outfile',
nargs='?',
help='Output file (default stdout)',
type=argparse.FileType('w'),
default=sys.stdout)
args = parser.parse_args(fakeargs)
# print "About to CV check:", f
try:
process = validator.checkCMIP6(args)
if process is None:
raise ESGPublishError(
"File %s failed the CV check - object create failure" % f)
process.ControlVocab()
except:
raise ESGPublishError("File %s failed the CV check" % f)
def validateFile(self, fileobj):
"""
for CMIP6, this will first verify if the data is written by CMOR at the correct version set in the ini file.
If so, the file is declared valid. If not, file will go through PrePARE (CV) check. PrePARE runs CFChecker
Raises ESGPublishError if settings are missing or file fails the checks.
Raise ESGInvalidMetadataFormat if the file cannot be processed by this handler.
"""
validator = PrePARE.PrePARE
f = fileobj.path
if self.replica:
debug("skipping PrePARE for replica (file %s)" % f)
return
# todo refactoring these could loaded upfront in the constructor
config = getConfig()
project_section = 'project:' + self.name
project_config_section = 'config:' + self.name
min_cmor_version = config.get(project_section, "min_cmor_version", default="0.0.0")
min_ds_version = config.get(project_section, "min_data_specs_version", default="0.0.0")
data_specs_version = config.get(project_config_section, "data_specs_version", default="master")
cmor_table_path = config.get(project_config_section, "cmor_table_path", default=DEFAULT_CMOR_TABLE_PATH)
try:
file_cmor_version = fileobj.getAttribute('cmor_version', None)
except:
file_cmor_version = None
debug('File %s missing cmor_version attribute; will proceed with PrePARE check' % f)
passed_cmor = False
if compareLibVersions(min_cmor_version, file_cmor_version):
debug('File %s cmor-ized at version %s, passed!"'%(f, file_cmor_version))
passed_cmor = True
try:
table = fileobj.getAttribute('table_id', None)
except:
raise ESGPublishError("File %s missing required table_id global attribute" % f)
try:
variable_id = fileobj.getAttribute('variable_id', None)
except:
raise ESGPublishError("File %s missing required variable_id global attribute" % f)
# data_specs_version drives CMOR table fetching
# Behavior A (default): fetches "master" branch" (if not "data_specs_version" in esg.ini")
# Behavior A: fetches branch specified by "data_specs_version=my_branch" into esg.ini
# Behavior B: fetches branch specified by file global attributes using "data_specs_version=file" into esg.ini
try:
file_data_specs_version = fileobj.getAttribute('data_specs_version', None)
except Exception as e:
raise ESGPublishError("File %s missing required data_specs_version global attribute"%f)
if not compareLibVersions(min_ds_version, file_data_specs_version):
raise ESGPublishError("File %s data_specs_version is %s, which is less than the required minimum version of %s"%(f,file_data_specs_version,min_ds_version))
# at this point the file has the correct data specs version.
# if also was CMORized and has the correct version tag, we can exit
if passed_cmor:
return
if data_specs_version == "file":
data_specs_version = file_data_specs_version
checkAndUpdateRepo(cmor_table_path, data_specs_version)
try:
process = validator.checkCMIP6(cmor_table_path)
if process is None:
raise ESGPublishError("File %s failed the CV check - object create failure"%f)
process.ControlVocab(f)
except:
raise ESGPublishError("File %s failed the CV check"%f)
def validateFile(self, fileobj):
"""
for CMIP6, this will first verify if the data is written by CMOR at the correct version set in the ini file.
If so, the file is declared valid. If not, file will go through PrePARE (CV) check. PrePARE runs CFChecker
Raises ESGPublishError if settings are missing or file fails the checks.
Raise ESGInvalidMetadataFormat if the file cannot be processed by this handler.
"""
validator = PrePARE.PrePARE
f = fileobj.path
# todo refactoring these could loaded upfront in the constructor
config = getConfig()
project_section = 'project:' + self.name
project_config_section = 'config:' + self.name
min_cmor_version = config.get(project_section, "min_cmor_version", default="0.0.0")
min_ds_version = config.get(project_section, "min_data_specs_version", default="0.0.0")
data_specs_version = config.get(project_config_section, "data_specs_version", default="master")
cmor_table_path = config.get(project_config_section, "cmor_table_path", default=DEFAULT_CMOR_TABLE_PATH)
force_validation = config.getboolean(project_config_section, "force_validation", default=False)
cmor_table_subdirs = config.getboolean(project_config_section, "cmor_table_subdirs", default=False)
if not force_validation:
if self.replica:
info("skipping PrePARE for replica (file %s)" % f)
return
try:
file_cmor_version = fileobj.getAttribute('cmor_version', None)
except:
file_cmor_version = None
debug('File %s missing cmor_version attribute; will proceed with PrePARE check' % f)
passed_cmor = False
if compareLibVersions(min_cmor_version, file_cmor_version):
debug('File %s cmor-ized at version %s, passed!'%(f, file_cmor_version))
passed_cmor = True
try:
table = fileobj.getAttribute('table_id', None)
except:
raise ESGPublishError("File %s missing required table_id global attribute" % f)
try:
variable_id = fileobj.getAttribute('variable_id', None)
except:
raise ESGPublishError("File %s missing required variable_id global attribute" % f)
# data_specs_version drives CMOR table fetching
# Behavior A (default): fetches "master" branch" (if not "data_specs_version" in esg.ini")
# Behavior A: fetches branch specified by "data_specs_version=my_branch" into esg.ini
# Behavior B: fetches branch specified by file global attributes using "data_specs_version=file" into esg.ini
try:
file_data_specs_version = fileobj.getAttribute('data_specs_version', None)
except Exception as e:
raise ESGPublishError("File %s missing required data_specs_version global attribute"%f)
if not compareLibVersions(min_ds_version, file_data_specs_version):
raise ESGPublishError("File %s data_specs_version is %s, which is less than the required minimum version of %s"%(f,file_data_specs_version,min_ds_version))
# at this point the file has the correct data specs version.
# if also was CMORized and has the correct version tag, we can exit
if (not force_validation) and passed_cmor:
return
if data_specs_version == "file":
data_specs_version = file_data_specs_version
table_dir = getTableDir(cmor_table_path, data_specs_version, cmor_table_subdirs)
debug("Validating {} using tables dir: {}".format(f, table_dir))
try:
process = validator.checkCMIP6(table_dir)
if process is None:
raise ESGPublishError("File %s failed the CV check - object create failure"%f)
process.ControlVocab(f)
except:
raise ESGPublishError("File %s failed the CV check"%f)