def test_cmip5_institute(self):
del self.cube.attributes['institution_id']
self.cube.attributes['institute_id'] = 'EFGH'
self.expected['institute'] = 'EFGH'
actual = identify_contents_metadata(self.cube, 'abc.nc')
self.assertEqual(actual, self.expected)
def _identify_and_validate_file(filename, project, file_format, output,
error_event):
"""
Do the validation of a file.
:param str filename: The name of the file
:param str project: The name of the project
:param str file_format: The format of the file (CMIP5 or CMIP6)
:param multiprocessing.Manager.list output: A list containing the output
metadata dictionaries for each file
:param multiprocessing.Manager.Event error_event: If set then a catastrophic
error has occurred in another process and processing should end
"""
try:
basename = os.path.basename(filename)
if DataFile.objects.filter(name=basename).count() > 0:
msg = 'File {} already exists in the database.'.format(basename)
raise FileValidationError(msg)
metadata = identify_filename_metadata(filename, file_format)
if metadata['table'].startswith('Prim'):
metadata['project'] = 'PRIMAVERA'
else:
metadata['project'] = project
if 'fx' in metadata['table']:
cf = iris.fileformats.cf.CFReader(filename)
metadata.update(identify_cell_measures_metadata(cf, filename))
validate_cell_measures_contents(cf, metadata)
else:
cube = load_cube(filename)
metadata.update(identify_contents_metadata(cube, filename))
validate_file_contents(cube, metadata)
_contents_hdf_check(cube, metadata, cmd_args.data_limit)
verify_fk_relationships(metadata)
calculate_checksum(metadata)
except SubmissionError:
msg = ('A serious file error means the submission cannot continue: '
'{}'.format(filename))
logger.error(msg)
error_event.set()
except FileValidationError as fve:
msg = 'File failed validation. {}'.format(fve.__str__())
logger.warning(msg)
else:
output.append(metadata)
def main(args):
"""
Run the checks
"""
# the metadata found by the checks is used in the online PRIMAVERA
# validation but can be ignored in this simple check
_output = []
num_errors_found = 0
if args.single_file:
data_files = [args.directory]
else:
data_files = list_files(
os.path.expandvars(os.path.expanduser(args.directory)))
if not data_files:
msg = 'No data files found in directory: {}'.format(args.directory)
logger.error(msg)
sys.exit(1)
logger.debug('%s files found.', len(data_files))
for filename in data_files:
try:
# run the four checks that the validate script runs
metadata = identify_filename_metadata(filename, args.file_format)
if not args.cell_measure:
cube = load_cube(filename)
metadata.update(identify_contents_metadata(cube, filename))
validate_file_contents(cube, metadata)
else:
cf = iris.fileformats.cf.CFReader(filename)
metadata.update(identify_cell_measures_metadata(cf, filename))
validate_cell_measures_contents(cf, metadata)
except FileValidationError as exc:
logger.warning('File failed validation:\n%s', exc.__str__())
num_errors_found += 1
else:
_output.append(metadata)
if num_errors_found:
logger.error('%s files failed validation', num_errors_found)
sys.exit(1)
else:
logger.debug('All files successfully validated.')
sys.exit(0)
def test_custom_activity(self):
self.cube.attributes['activity_id'] = 'ZYXW'
self.expected['activity_id'] = 'ZYXW'
actual = identify_contents_metadata(self.cube, 'abc.nc')
self.assertEqual(actual, self.expected)
def test_default_activity(self):
actual = identify_contents_metadata(self.cube, 'abc.nc')
self.assertEqual(actual, self.expected)