def dump(self, dest=None, **kwargs):
if dest is None:
# raise IOError("no destination metadata file defined")
try:
cat = self.category.get('code')
except:
cat = self.category
dest = osp.join("%s.json" % cat)
elif not isinstance(dest, string_types):
raise TypeError("Wrong destination metadata file '%s'" % dest)
try:
assert osp.exists(dest)
except AssertionError:
warnings.warn('\n! Destination metadata file will be created !')
else:
warnings.warn(
'\n! Destination metadata file will be overwritten !')
meta = {
k: v
for (k, v) in dict(self.copy()).items() if k in self.keys()
}
if meta == {}:
raise IOError("No metadata variable available")
with open(dest, 'w') as fp:
try:
Json.dump(meta, fp, **kwargs)
except:
raise IOError("Error saving metadata file")
def __init__(self, *args, **kwargs):
if not args in ((), (None, )):
try:
meta = deepcopy(
args[0]
) # deepcopy to avoid updating the default variables!!!
except:
meta = args[0]
else:
meta = {}
if isinstance(meta, MetaDat):
meta = dict(meta).copy()
elif isinstance(meta, string_types):
if not osp.exists(meta):
raise IOError("Input metadata filename '%s' not recognised" %
meta)
with open(meta, 'rt') as fp:
try:
meta = Json.load(fp,
serialize=kwargs.pop('serialize', False))
except:
raise IOError(
"Input metadata file '%s' must be in JSON format" %
meta)
elif not isinstance(meta, Mapping):
raise TypeError(
"Input metadata format '%s' not recognised - must be a mapping dictionary or a string"
% type(meta))
meta.update(kwargs)
super(MetaDat, self).__init__(**meta)
self.__dict__ = self
def dumps(self, **kwargs):
meta = {
k: v
for (k, v) in dict(self.copy()).items() if k in self.keys()
}
if meta == {}:
raise IOError("No metadata variable available")
try:
return Json.dumps(meta, **kwargs)
except:
raise IOError("Error dumping metadata file")
def load(self, src=None, **kwargs):
if src is None:
# raise IOError("no source metadata file defined")
try:
cat = self.category.get('code')
except:
cat = self.category
src = osp.join("%s.json" % cat)
elif not isinstance(src, string_types):
raise TypeError("Wrong source metadata file '%s'" % src)
try:
assert osp.exists(src)
except AssertionError:
raise IOError("Metadata file '%s' do not exist" % src)
with open(src, 'r') as fp:
try:
meta = Json.load(fp, **kwargs)
except:
raise IOError("Error saving metadata file")
if 'category' not in meta.keys() and self.category != self.CATEGORY:
meta.update({'category': self.category})
return meta
def harmoniseService(facility, country=None, gc=None, **kwargs):
"""Generic harmonisation function.
>>> harmonise.harmoniseService(facility, country = None, gc = None, **kwargs)
"""
#if facility is None:
# facility = list(FACILITIES.keys())
if not isinstance(facility, string_types):
raise TypeError(
"Wrong type for input service - must be the facility type")
elif not facility in FACILITIES.keys():
raise IOError(
"Service type not recognised - must be a string in the list '%s'" %
list(FACILITIES.keys()))
if country is None:
country = list(COUNTRIES.keys())
if not isinstance(country, string_types) and isinstance(country, Sequence):
for ctry in country:
try:
harmoniseService(facility, country=ctry, gc=gc, **kwargs)
except:
continue
return
elif not isinstance(country, string_types):
raise TypeError(
"Wrong type for input country code - must be the ISO 2-letter string"
)
elif not country in COUNTRIES.keys():
raise IOError(
"Country code not recognised - must be a code of the '%s' area(s)"
% list(COUNTRIES.keys()))
if not (gc is None or isinstance(gc, (string_types, Mapping))):
raise TypeError(
"Coder type not recognised - must be a dictionary or a single string"
)
CC, METADATNAT = None, {}
metadir = FACILITIES[facility].get('code')
# generic name
ccname = "%s%s" % (country, BASENAME.get(facility, ''))
# load country-dedicated module wmmhen available
modname = ccname
fname = '%s.py' % ccname
try:
assert osp.exists(osp.join(__THISDIR, metadir, fname))
# import_module('%s.%s' % (PACKNAME,modname) )
imp = import_module('.%s' % modname, '%s.%s' % (PACKNAME, metadir))
except AssertionError:
logging.warning(
"\n! No country py-file '%s' found - will proceed without !" %
fname)
except ImportError:
logging.warning(
"\n! No country py-module '%s' found - will proceed without !" %
modname)
except:
raise ImportError("No country py-module '%s' loaded" % modname)
else:
logging.warning("\n! Country py-module '%s' found !" % imp.__name__)
try:
assert imp in sysmod.values()
except:
raise ImportError("Country py-module '%s' not loaded correctly" %
imp.__name__)
try:
# assert 'CC' in dir(imp)
CC = getattr(imp, 'CC', None)
except:
logging.warning("\n! Global variable 'CC' not set - use default !")
# try:
# # assert 'META' in dir(imp)
# METADATNAT = getattr(imp, 'META', None)
# assert METADATNAT is not None
# except:
# logging.warning("\n! No default metadata dictionary 'META' available !")
# else:
# logging.warning("\n! Default hard-coded metadata dictionary 'META' found !")
try:
# assert 'harmonise' in dir(imp)
harmonise = getattr(imp, HARMONISE, None)
assert harmonise is not None
except:
logging.warning('\n! Generic formatting/harmonisation methods used !')
harmonise = harmoniseFacilityData
else:
logging.warning(
'\n! Country-specific formatting/harmonisation methods used !')
if 'methods' not in kwargs:
kwargs.update({'methods': {p: None for p in PROCESSES}})
for proc in PROCESSES:
try:
# assert proc in dir(imp)
proc_data = getattr(imp, '%s_data' % proc, None)
assert proc_data is not None
except:
# logging.warning("! no data '%s' method used !" % proc)
pass # proc_data = None
else:
logging.warning("\n! country-specific '%s_data' method loaded !" %
proc)
finally:
kwargs['methods'].update({proc: proc_data})
# load country-dedicated metadata when available
metadata = None
metafname = '%s.json' % ccname
try:
metafname = osp.join(__THISDIR, metadir, metafname)
assert osp.exists(metafname)
with open(metafname, 'r') as fp:
metadata = Json.load(fp)
except (AssertionError, FileNotFoundError):
logging.warning(
"\n! No metadata JSON-file '%s' found - will proceed without !" %
metafname)
else:
logging.warning("! Ad-hoc metadata found - JSON-file '%s' loaded !" %
metafname)
# define the actual metadata: the one loaded, or the default
# metadata = metadata or METADATNAT
if metadata in (None, {}):
raise IOError('No metadata parsed - this cannot end up well')
try:
kwargs.update({'country': {'code': CC or country}})
if 'options' in kwargs.keys():
if 'locate' in kwargs['options'].keys():
kwargs['options']['locate'].update({'gc': gc})
else:
kwargs['options'].update({'locate': {'gc': gc}})
else:
kwargs.update({'options': {'locate': {'gc': gc}}})
res = harmonise(facility, metadata, **kwargs)
except:
raise IOError("Harmonisation process for country '%s' failed..." %
country)
else:
logging.warning("\n! Harmonised data for country '%s' generated !" %
country)
return res