class Writer(object):
"""Writer class"""
def __init__(self, output_name: object, dataset_path: object, vars_to_write: object, endpoint_url: object, tiled=False, tile_shape=[]) -> object:
"""
Writer class constructor
:parameter output_name: Name of output file
:parameter dataset_path: Path to dataset
:parameter vars_to_write: List of variables to write
:parameter tiled: Boolean value (default False)
:parameter tile_shape: List containing shape of tiles
:parameter endpoint_url: MongoDB endpoint for CovJSON persistence
"""
self.output_name = output_name
self.tile_shape = tile_shape
self.vars_to_write = vars_to_write
self.url_template = 'localhost:8080/{t}.covjson'
self.tiled = tiled
if tiled:
self.range_type = 'TiledNdArray'
else:
self.range_type = 'NdArray'
self.dataset_path = dataset_path
self.reader = Reader(dataset_path)
self.axis_dict = self.reader.get_axes()
self.axis_list = list(self.axis_dict.keys())
self.ref_list = []
if 't' in self.axis_list and 'z' in self.axis_list:
self.ref_list.append(TemporalReferenceSystem())
self.ref_list.append(SpatialReferenceSystem3d())
if 't' in self.axis_list and 'z' not in self.axis_list:
self.ref_list.append(TemporalReferenceSystem())
self.ref_list.append(SpatialReferenceSystem2d())
elif 't' not in self.axis_list and 'z' not in self.axis_list:
self.ref_list.append(SpatialReferenceSystem2d())
if endpoint_url is not None:
self.endpoint_url = endpoint_url
else:
self.endpoint_url = None
def write(self):
"""
Writes Coverage object to local disk or MongoDB
"""
coverage = self._construct_coverage()
if self.endpoint_url is not None:
if self.tiled:
self.save_covjson_tiled(coverage, self.endpoint_url)
else:
self._save_covjson(coverage, self.endpoint_url)
else:
if self.tiled:
self.save_covjson_tiled(coverage, self.output_name)
else:
self._save_covjson(coverage, self.output_name)
def _construct_coverage(self):
"""
Constructs Coverage object from constituent parts
:return: coverage object
"""
coverage = Coverage(self._construct_domain(), self._construct_range(
), self._construct_params(), self._construct_refs()).to_dict()
return coverage
def _construct_domain(self):
"""
Constructs Domain object, populates with values
:return: domain object
"""
domain_type = 'Grid'
x_values = self.reader.get_x().flatten().tolist()
y_values = self.reader.get_y().flatten().tolist()
t_values = []
z_values = []
if 't' in self.axis_list:
t_values = self.reader.get_t()
if 'z' in self.axis_list:
z_values = self.reader.get_z().flatten().tolist()
domain = Domain(domain_type, x_values, y_values, z_values, t_values)
return domain
def _construct_params(self):
"""
Construct parameter object from constituent parts
:return: Parameter object
"""
for variable in self.vars_to_write:
description = self.reader.get_std_name(variable)
unit = self.reader.get_units(variable)
symbol = self.reader.dataset[variable].units
label = self.reader.dataset[variable].long_name
params = Parameter(description=description, variable_name=variable,
symbol=symbol, unit=unit, observed_property=label)
return params
def _construct_refs(self):
"""
Construct reference object
:return: refs
"""
refs = Reference(self.ref_list)
return refs
def _construct_range(self):
"""
Construct range object
:return: range
"""
for variable in self.vars_to_write:
print("Constructing Range from variable:", variable)
axis_names = list(map(str.lower, list(self.reader.get_axis(variable))))
if self.tiled:
tile_set_obj = TileSet(self.tile_shape, self.urlTemplate)
variable_type = self.reader.get_type(variable)
variable_shape = self.reader.get_shape(variable)
print('Variable shape:', variable_shape)
count = 0
for tile in tile_set_obj.get_tiles(self.tile_shape, self.reader.dataset[variable].values):
count += 1
covrange = {'ranges': Range('NdArray', data_type=variable_type, axes=tile[
1], shape=variable_shape, values=tile[0].flatten().tolist()).to_dict()}
self.save_covjson_range(covrange, str(count) + '.covjson')
url_template = tile_set_obj.generate_url_template(base_url='localhost:8080',
axis_names=['t'])
tileset = TileSet(variable_shape, url_template).create_tileset(self.tile_shape)
covrange = Range('TiledNdArray', data_type=variable_type, variable_name=variable,
axes=axis_names, tile_sets=tileset, shape=variable_shape)
return covrange
else:
shape = self.reader.get_shape(variable)
values = self.reader.get_values(variable).flatten().tolist()
data_type = self.reader.get_type(variable)
axes = self.reader.get_axis(variable)
covrange = Range(range_type='NdArray', data_type=data_type, values=values, shape=shape,
variable_name=variable, axes=axis_names)
return covrange
# Adapted from
# https://github.com/the-iea/ecem/blob/master/preprocess/ecem/util.py -
# letmaik
def _save_json(self, obj, path, **kw):
"""Save json object to disk"""
with open(path, 'w') as fp:
print("Converting....")
start = time.clock()
jsonstr = json.dumps(obj, fp, cls=CustomEncoder, **kw)
fp.write(jsonstr)
stop = time.clock()
print("Completed in: '%s' seconds." % (stop - start))
def _save_covjson(self, obj, resource):
"""
Skip indentation of certain fields to make JSON more compact but still human readable
:param obj: the CovJSON object to write
:param resource: either a local file path or a MongoDB endpoint
"""
for axis in obj['domain']['axes'].values():
self.compact(axis, 'values')
for ref in obj['domain']['referencing']:
self.no_indent(ref, 'coordinates')
for covrange in obj['ranges'].values():
self.no_indent(covrange, 'axisNames', 'shape')
self.compact(covrange, 'values')
self.save_json(obj, resource, indent=2)
def save_covjson_tiled(self, obj, resource):
"""
Skip indentation of certain fields to make JSON more compact but still human readable
:param obj: the CovJSON object to write
:param resource: either a local file path or a MongoDB endpoint
"""
for axis in obj['domain']['axes'].values():
self.compact(axis, 'values')
for ref in obj['domain']['referencing']:
self.no_indent(ref, 'coordinates')
self.save_json(obj, resource, indent=2)
def save_json(self, obj, resource, **kw):
print("Attempting to write CovJSON manifestation to '%s'" % (resource))
start = time.clock()
if resource[0].startswith('mongo'):
mongo_client = MongoDBClient(obj, resource).write()
else:
with open(resource, 'w') as fp:
jsonstr = json.dumps(obj, cls=CustomEncoder, **kw)
fp.write(jsonstr)
stop = time.clock()
print("Completed in: '%s' seconds." % (stop - start))
def save_covjson_range(self, obj, path):
for covrange in obj['ranges'].values():
self.no_indent(covrange, 'axisNames', 'shape')
self.compact(covrange, 'values')
self.save_json(obj, path, indent=2)
def compact(self, obj, *names):
for name in names:
obj[name] = Custom(obj[name], separators=(',', ':'))
def no_indent(self, obj, *names):
for name in names:
obj[name] = Custom(obj[name])
