def test_broken_yaml(self):
"""test against broken YAML"""
iso_os = ISO19139OutputSchema()
with self.assertRaises(MCFReadError):
render_j2_template(read_mcf(get_abspath('broken-yaml.yml')),
iso_os.template_dir)
def test_pre1900_dates(self):
"""test datestrings that are pre-1900"""
iso_os = ISO19139OutputSchema()
xml = render_j2_template(read_mcf(get_abspath('dates-pre-1900.yml')),
iso_os.template_dir)
self.assertIsInstance(xml, str, 'Expected unicode string')
def load_collection_level_metadata(self):
logger.debug('Loading collection level metadata')
for clm in os.listdir(COLLECTION_LEVEL_METADATA):
logger.debug(f'collection metadata file: {clm}')
clm_ = os.path.join(COLLECTION_LEVEL_METADATA, clm)
clm_mcf = read_mcf(clm_)
clm_iso = ISO19139OutputSchema().write(clm_mcf)
logger.debug(f'Upserting metadata: {clm_}')
self._parse_and_upsert_metadata(clm_iso)
def from_cwl(self, cwl_item: str, public_s3_url: str) -> str:
mcf = deepcopy(self.mcf)
now = datetime.now().isoformat()
cwl = yaml.load(cwl_item, Loader=yaml.SafeLoader)
wf = list(filter(lambda x: x['class'] == 'Workflow', cwl['$graph']))[0]
mcf['metadata']['identifier'] = wf['id']
mcf['metadata']['hierarchylevel'] = 'application'
mcf['metadata']['datestamp'] = now
mcf['identification']['title'] = wf['label']
mcf['identification']['abstract'] = wf['doc']
mcf['identification']['keywords']['default'] = {
'keywords': [
f'softwareVersion:{cwl["s:softwareVersion"]}', 'application',
'CWL'
],
'keywords_type':
'theme'
}
mcf['identification']['dates'] = {'creation': now}
mcf['distribution']['cwl'] = {
'url': self.base_url.rstrip('/'),
'type': 'application/x-yaml',
'name': wf['label'],
'description': wf['doc'],
'function': 'information'
}
mcf['distribution']['http'] = {
'url': public_s3_url,
'type': 'application/x-yaml',
'name': wf['label'],
'description': wf['doc'],
'function': 'information'
}
mcf['identification']['extents'] = {
'spatial': [{
'bbox': [-180, -90, 180, 90],
'crs': 4326
}],
}
logger.debug(f'MCF: {mcf}')
iso_os = ISO19139OutputSchema()
return iso_os.write(mcf)
def test_pretty_print(self):
"""Test pretty-printing"""
iso_os = ISO19139OutputSchema()
xml = render_j2_template(read_mcf(get_abspath('../sample.yml')),
iso_os.template_dir)
xml2 = pretty_print(xml)
self.assertIsInstance(xml2, str, 'Expected unicode string')
self.assertEqual(xml2[-1], '>', 'Expected closing bracket')
self.assertTrue(xml2.startswith('<?xml'), 'Expected XML declaration')
def test_render_j2_template(self):
"""test template rendering"""
test_mcf_paths = [
'../sample.yml', 'unilingual.yml',
'nil-identification-language.yml'
]
for mcf_path in test_mcf_paths:
iso_os = ISO19139OutputSchema()
# working template directory
xml = render_j2_template(read_mcf(get_abspath(mcf_path)),
iso_os.template_dir)
self.assertIsInstance(xml, str, 'Expected unicode string')
# no template directory or local schema provided
with self.assertRaises(RuntimeError):
render_j2_template(read_mcf(get_abspath(mcf_path)))
# bad template directory provided
with self.assertRaises(RuntimeError):
xml = render_j2_template(read_mcf(get_abspath(mcf_path)),
'bad_dir')
# bad j2 template_dir provided
with self.assertRaises(RuntimeError):
xml = render_j2_template(read_mcf(get_abspath(mcf_path)),
template_dir='/bad_schema/path')
# good j2 template_dir provided
xml = render_j2_template(
read_mcf(get_abspath(mcf_path)),
template_dir=get_abspath('sample_schema_j2')) # noqa
# good sample output schema
s_os = SampleOutputSchema()
_ = s_os.write(read_mcf(get_abspath(mcf_path)))
def test_import(self):
"""test metadata import"""
schema = ISO19139OutputSchema()
with open(get_abspath('md-SMJP01RJTD-gmd.xml')) as fh:
mcf = schema.import_(fh.read())
self.assertEqual(mcf['identification']['title'],
'WIS/GTS bulletin SMJP01 RJTD in FM12 SYNOP',
'Expected specific title')
self.assertEqual(len(mcf['distribution']), 1,
'Expected specific number of links')
result_bbox = mcf['identification']['extents']['spatial'][0][
'bbox'] # noqa
expected_bbox = [124.167, 24.333, 145.583, 45.4]
self.assertEqual(expected_bbox, result_bbox,
'Expected specific BBOX')
with open(get_abspath(
'urn:x-wmo:md:int.wmo.wis::ISMD01EDZW.xml')) as fh: # noqa
mcf = schema.import_(fh.read())
self.assertEqual(
mcf['identification']['title'],
'GTS Bulletin: ISMD01 EDZW - Observational data (Binary coded) - BUFR (details are described in the abstract)', # noqa
'Expected specific title')
self.assertEqual(len(mcf['distribution']), 1,
'Expected specific number of links')
result_bbox = mcf['identification']['extents']['spatial'][0][
'bbox'] # noqa
expected_bbox = [6.3467, 47.7244, 14.1203, 55.0111]
self.assertEqual(expected_bbox, result_bbox,
'Expected specific BBOX')
def indexDir(dir, dir_out, dir_out_mode, mode, dbtype, profile, db):
if not dir:
dir = "."
if not dir_out_mode or dir_out_mode not in ["flat", "nested"]:
dir_out_mode = "flat"
if not mode or mode not in ["init", "update", "export"]:
mode = "init"
if not dbtype or dbtype not in ["path", "sqlite", "postgres"]:
dbtype = "path"
if not db:
db = dir
if not profile or profile not in ["iso19139", "dcat"]:
profile = "iso19139"
print(mode + ' metadata in ' + dir + ' as ' + profile + ' in ' + db)
if mode == "export":
if dbtype == 'sqlite':
dir_out = os.path.join(dir_out, db)
createIndexIfDoesntExist(dir_out)
elif dbtype == "path":
if not os.path.exists(dir_out):
print('creating out folder ' + dir_out)
os.makedirs(dir_out)
else:
print("postgis not supported")
# core metadata gets populated by merging the index.yaml content from parent folders
coreMetadata = {}
# identify if there is a path change
prvPath = "dummy"
for path, dirs, files in os.walk(dir):
if mode == 'export':
# if dir has index.yaml merge it to parent
f = os.path.join(path, 'index.yaml')
if os.path.exists(f):
if prvPath != path:
print('Indexing path ' + path)
prvPath = path
with open(os.path.join(f), mode="r",
encoding="utf-8") as yf:
pathMetadata = yaml.load(yf, Loader=SafeLoader)
pathMetadata.pop('index')
pathMetadata.pop('mode')
dict_merge(pathMetadata, coreMetadata)
coreMetadata = pathMetadata
else:
print(f + ' does not exist') # create it?
for file in files:
fname = os.path.join(path, file)
if '.' in file:
base, extension = file.rsplit('.', 1)
if extension.lower() in SPATIAL_FILE_TYPES:
print('Indexing file ' + fname)
yf = os.path.join(path, base + '.yaml')
if (mode == 'update'
or (not os.path.exists(yf) and mode == 'init')):
# mode init for spatial files without metadata or update
cnt = indexSpatialFile(fname, extension)
if (mode == 'update'
): # keep manual changes on the original
try:
with open(os.path.join(yf),
mode="r",
encoding="utf-8") as f:
orig = yaml.load(f, Loader=SafeLoader)
dict_merge(
orig, cnt
) # or should we overwrite some values from cnt explicitely?
cnt = orig
except Exception as e:
print('Failed to merge original:', f, e)
md = asPGM(cnt)
# write yf
try:
with open(os.path.join(yf), 'w') as f:
yaml.dump(md, f, sort_keys=False)
except Exception as e:
print('Failed to dump yaml:', e)
elif mode == 'export':
try:
with open(os.path.join(yf),
mode="r",
encoding="utf-8") as f:
cnf = yaml.load(f, Loader=SafeLoader)
dict_merge(cnf, coreMetadata)
if dbtype == 'sqlite' or dbtype == 'postgres':
insert_or_update(cnt, dir_out)
elif dbtype == "path":
#load yml as mcf
md = read_mcf(cnf)
#yaml to iso/dcat
if schemaPath and os.path.exists(
schemaPath):
print('Using schema', schemaPath)
xml_string = render_j2_template(
md,
template_dir="{}/iso19139".format(
schemaPath))
else:
print('Using default iso19139 schema')
iso_os = ISO19139OutputSchema()
xml_string = iso_os.write(md)
if dir_out_mode == "flat":
pth = os.path.join(
dir_out,
cnf['metadata']['identifier'] +
'.xml')
else:
pth = os.path.join(path, base + '.xml')
print("write to file: " + pth)
with open(pth, 'w+') as ff:
ff.write(xml_string)
print('iso19139 xml generated at ' +
pth)
except Exception as e:
print('Failed to create xml:', e)
else:
None
# print('Skipping {}, not approved file type: {}'.format(fname, extension))
else:
None