def read():
dir_name = os.path.dirname(__file__)
testfile = 'test_xy.nc'
dataset_path = os.path.join(dir_name, 'testdata', testfile)
# 1.
reader = NetCDFReader(dataset_path)
coverage = reader.read()
def main():
"""
Command line interface for pycovjson - Converts Scientific Data Formats into CovJSON and saves to disk.
:argument -i: Input file path.
:argument -o: Output file name.
:argument -t: Use Tiling.
:argument -v: Which variable to populate coverage with.
:argument -s: [tile shape]: Tile shape.
:argument -n: Use interactive mode.
:argument -u: MongoDB URL
"""
parser = argparse.ArgumentParser(
description='Convert Scientific Data Formats into CovJSON.')
parser.add_argument('-i', '--input', dest='inputfile',
help='Name of input file', required=True)
parser.add_argument('-o', '--output', dest='outputfile',
help='Name and location of output file', default='coverage.covjson')
parser.add_argument('-t', '--tiled', action='store_true', help='Apply tiling')
parser.add_argument('-s', '--shape', nargs='+',
help='Tile shape, list', type=int)
parser.add_argument('-v', dest='variable',
help='Variable to populate coverage with', required=True)
parser.add_argument('-n', '--interactive', action='store_true', help='Enter interactive mode')
parser.add_argument('-u', '--endpoint_url', dest='endpoint_url', nargs=1,
help='MongoDB endpoint for CovJSON persistence')
args = parser.parse_args()
inputfile = args.inputfile
outputfile = args.outputfile
variable = args.variable
tiled = args.tiled
tile_shape = args.shape
interactive = args.interactive
endpoint_url = args.endpoint_url
if interactive:
axis = input('Which Axis?', Reader.get_axis(variable))
if tiled and len(tile_shape) == 0:
reader = Reader(inputfile)
shape_list = reader.get_shape(variable)
dims = reader.get_dimensions(variable)
print(list(zip(dims, shape_list)))
tile_shape = input(
'Enter the shape tile shape as a list of comma separated integers')
tile_shape = tile_shape.split(',')
tile_shape = list(map(int, tile_shape))
print(tile_shape)
if outputfile == None:
outputfile = outputfile.default
Writer(outputfile, inputfile, [variable],
tiled=tiled, tile_shape=tile_shape, endpoint_url=endpoint_url).write()
def test():
dir_name = os.path.dirname(__file__)
json_template = os.path.join(dir_name, '..', 'data', 'jsont_template.json')
testfile = 'test_xy.nc'
dataset_path = os.path.join(dir_name, 'testdata', testfile)
dataset = np.arange(60)
dataset.reshape(10, 6)
urlTemplate = 'localhost:8080/'
tile_shape = [2, 1]
# tileSet = TileSet(tile_shape, urlTemplate, dataset)
try:
variable_names = NetCDFReader(testfile).get_var_names()
except OSError:
print('Error: ', OSError)
# for tile in tileSet.get_tiles(tile_shape, 'data'):
# print(tile, '\n')
# variable_names = pycovjson.readNetCDFOOP.get_var_names(netcdf_dataset)
assert len(variable_names) > 0
def main():
parser = argparse.ArgumentParser(description="View Scientific Data files.")
parser.add_argument("inputfile", action="store", help="Name of input file")
parser.add_argument("-v", "--variables,", dest="variables", help="Display variables", action="store_true")
args = parser.parse_args()
inputfile = args.inputfile
variables = args.variables
reader = Reader(inputfile)
ds = reader.get_xarray()
# TODO
# if variables:
# reader.get_vars_with_long_name(inputfile)
print(ds)
def __init__(self, output_name: object, dataset_path: object, vars_to_write: object, tiled=False, tile_shape=[]) -> object:
"""
Writer class constructor
:parameter output_name: Name of output file
:parameter dataset_path: Path to dataset
:parameter vars_to_write: List of variables to write
:parameter tiled: Boolean value (default False)
:parameter tile_shape: List containing shape of tiles
"""
self.output_name = output_name
self.dataset_path = dataset_path
self.tile_shape = tile_shape
self.vars_to_write = vars_to_write
self.urlTemplate = 'localhost:8080/{t}.covjson'
self.tiled = tiled
if tiled:
self.range_type = 'TiledNdArray'
else:
self.range_type = 'NdArray'
self.dataset_path = dataset_path
self.Reader = Reader(dataset_path)
self.axis_dict = self.Reader.get_axes()
self.axis_list = list(self.axis_dict.keys())
self.ref_list = []
if 't' in self.axis_list and 'z' in self.axis_list:
self.ref_list.append(TemporalReferenceSystem())
self.ref_list.append(SpatialReferenceSystem3d())
if 't' in self.axis_list and 'z' not in self.axis_list:
self.ref_list.append(TemporalReferenceSystem())
self.ref_list.append(SpatialReferenceSystem2d())
elif 't' not in self.axis_list and 'z' not in self.axis_list:
self.ref_list.append(SpatialReferenceSystem2d())
def main():
parser = argparse.ArgumentParser(description='View Scientific Data files.')
parser.add_argument('inputfile', action ='store',
help='Name of input file')
parser.add_argument('-v', '--variables,', dest='variables',
help='Display variables', action='store_true')
args = parser.parse_args()
inputfile = args.inputfile
variables = args.variables
reader = Reader(inputfile)
ds = reader.get_xarray()
# TODO
# if variables:
# reader.get_vars_with_long_name(inputfile)
print(ds)
def tile_by_axis(variable, axis):
"""
:param variable:
:param axis:
:return: tile_shape
"""
# Get shape of variable
shape = Reader.get_shape(variable)
# Set axis to slice by to 1
TileSet.create_tileset()
# Generate new tile shape
return tile_shape
def tile_by_axis(variable, axis):
"""
:param variable:
:param axis:
:return: tile_shape
"""
# Get shape of variable
shape = Reader.get_shape(variable)
# Set axis to slice by to 1
TileSet.create_tileset()
# Generate new tile shape
return tile_shape
def __init__(self, output_name: object, dataset_path: object, vars_to_write: object, endpoint_url: object, tiled=False, tile_shape=[]) -> object:
"""
Writer class constructor
:parameter output_name: Name of output file
:parameter dataset_path: Path to dataset
:parameter vars_to_write: List of variables to write
:parameter tiled: Boolean value (default False)
:parameter tile_shape: List containing shape of tiles
:parameter endpoint_url: MongoDB endpoint for CovJSON persistence
"""
self.output_name = output_name
self.tile_shape = tile_shape
self.vars_to_write = vars_to_write
self.url_template = 'localhost:8080/{t}.covjson'
self.tiled = tiled
if tiled:
self.range_type = 'TiledNdArray'
else:
self.range_type = 'NdArray'
self.dataset_path = dataset_path
self.reader = Reader(dataset_path)
self.axis_dict = self.reader.get_axes()
self.axis_list = list(self.axis_dict.keys())
self.ref_list = []
if 't' in self.axis_list and 'z' in self.axis_list:
self.ref_list.append(TemporalReferenceSystem())
self.ref_list.append(SpatialReferenceSystem3d())
if 't' in self.axis_list and 'z' not in self.axis_list:
self.ref_list.append(TemporalReferenceSystem())
self.ref_list.append(SpatialReferenceSystem2d())
elif 't' not in self.axis_list and 'z' not in self.axis_list:
self.ref_list.append(SpatialReferenceSystem2d())
if endpoint_url is not None:
self.endpoint_url = endpoint_url
else:
self.endpoint_url = None
class Writer(object):
"""Writer class"""
def __init__(self, output_name: object, dataset_path: object, vars_to_write: object, endpoint_url: object, tiled=False, tile_shape=[]) -> object:
"""
Writer class constructor
:parameter output_name: Name of output file
:parameter dataset_path: Path to dataset
:parameter vars_to_write: List of variables to write
:parameter tiled: Boolean value (default False)
:parameter tile_shape: List containing shape of tiles
:parameter endpoint_url: MongoDB endpoint for CovJSON persistence
"""
self.output_name = output_name
self.tile_shape = tile_shape
self.vars_to_write = vars_to_write
self.url_template = 'localhost:8080/{t}.covjson'
self.tiled = tiled
if tiled:
self.range_type = 'TiledNdArray'
else:
self.range_type = 'NdArray'
self.dataset_path = dataset_path
self.reader = Reader(dataset_path)
self.axis_dict = self.reader.get_axes()
self.axis_list = list(self.axis_dict.keys())
self.ref_list = []
if 't' in self.axis_list and 'z' in self.axis_list:
self.ref_list.append(TemporalReferenceSystem())
self.ref_list.append(SpatialReferenceSystem3d())
if 't' in self.axis_list and 'z' not in self.axis_list:
self.ref_list.append(TemporalReferenceSystem())
self.ref_list.append(SpatialReferenceSystem2d())
elif 't' not in self.axis_list and 'z' not in self.axis_list:
self.ref_list.append(SpatialReferenceSystem2d())
if endpoint_url is not None:
self.endpoint_url = endpoint_url
else:
self.endpoint_url = None
def write(self):
"""
Writes Coverage object to local disk or MongoDB
"""
coverage = self._construct_coverage()
if self.endpoint_url is not None:
if self.tiled:
self.save_covjson_tiled(coverage, self.endpoint_url)
else:
self._save_covjson(coverage, self.endpoint_url)
else:
if self.tiled:
self.save_covjson_tiled(coverage, self.output_name)
else:
self._save_covjson(coverage, self.output_name)
def _construct_coverage(self):
"""
Constructs Coverage object from constituent parts
:return: coverage object
"""
coverage = Coverage(self._construct_domain(), self._construct_range(
), self._construct_params(), self._construct_refs()).to_dict()
return coverage
def _construct_domain(self):
"""
Constructs Domain object, populates with values
:return: domain object
"""
domain_type = 'Grid'
x_values = self.reader.get_x().flatten().tolist()
y_values = self.reader.get_y().flatten().tolist()
t_values = []
z_values = []
if 't' in self.axis_list:
t_values = self.reader.get_t()
if 'z' in self.axis_list:
z_values = self.reader.get_z().flatten().tolist()
domain = Domain(domain_type, x_values, y_values, z_values, t_values)
return domain
def _construct_params(self):
"""
Construct parameter object from constituent parts
:return: Parameter object
"""
for variable in self.vars_to_write:
description = self.reader.get_std_name(variable)
unit = self.reader.get_units(variable)
symbol = self.reader.dataset[variable].units
label = self.reader.dataset[variable].long_name
params = Parameter(description=description, variable_name=variable,
symbol=symbol, unit=unit, observed_property=label)
return params
def _construct_refs(self):
"""
Construct reference object
:return: refs
"""
refs = Reference(self.ref_list)
return refs
def _construct_range(self):
"""
Construct range object
:return: range
"""
for variable in self.vars_to_write:
print("Constructing Range from variable:", variable)
axis_names = list(map(str.lower, list(self.reader.get_axis(variable))))
if self.tiled:
tile_set_obj = TileSet(self.tile_shape, self.urlTemplate)
variable_type = self.reader.get_type(variable)
variable_shape = self.reader.get_shape(variable)
print('Variable shape:', variable_shape)
count = 0
for tile in tile_set_obj.get_tiles(self.tile_shape, self.reader.dataset[variable].values):
count += 1
covrange = {'ranges': Range('NdArray', data_type=variable_type, axes=tile[
1], shape=variable_shape, values=tile[0].flatten().tolist()).to_dict()}
self.save_covjson_range(covrange, str(count) + '.covjson')
url_template = tile_set_obj.generate_url_template(base_url='localhost:8080',
axis_names=['t'])
tileset = TileSet(variable_shape, url_template).create_tileset(self.tile_shape)
covrange = Range('TiledNdArray', data_type=variable_type, variable_name=variable,
axes=axis_names, tile_sets=tileset, shape=variable_shape)
return covrange
else:
shape = self.reader.get_shape(variable)
values = self.reader.get_values(variable).flatten().tolist()
data_type = self.reader.get_type(variable)
axes = self.reader.get_axis(variable)
covrange = Range(range_type='NdArray', data_type=data_type, values=values, shape=shape,
variable_name=variable, axes=axis_names)
return covrange
# Adapted from
# https://github.com/the-iea/ecem/blob/master/preprocess/ecem/util.py -
# letmaik
def _save_json(self, obj, path, **kw):
"""Save json object to disk"""
with open(path, 'w') as fp:
print("Converting....")
start = time.clock()
jsonstr = json.dumps(obj, fp, cls=CustomEncoder, **kw)
fp.write(jsonstr)
stop = time.clock()
print("Completed in: '%s' seconds." % (stop - start))
def _save_covjson(self, obj, resource):
"""
Skip indentation of certain fields to make JSON more compact but still human readable
:param obj: the CovJSON object to write
:param resource: either a local file path or a MongoDB endpoint
"""
for axis in obj['domain']['axes'].values():
self.compact(axis, 'values')
for ref in obj['domain']['referencing']:
self.no_indent(ref, 'coordinates')
for covrange in obj['ranges'].values():
self.no_indent(covrange, 'axisNames', 'shape')
self.compact(covrange, 'values')
self.save_json(obj, resource, indent=2)
def save_covjson_tiled(self, obj, resource):
"""
Skip indentation of certain fields to make JSON more compact but still human readable
:param obj: the CovJSON object to write
:param resource: either a local file path or a MongoDB endpoint
"""
for axis in obj['domain']['axes'].values():
self.compact(axis, 'values')
for ref in obj['domain']['referencing']:
self.no_indent(ref, 'coordinates')
self.save_json(obj, resource, indent=2)
def save_json(self, obj, resource, **kw):
print("Attempting to write CovJSON manifestation to '%s'" % (resource))
start = time.clock()
if resource[0].startswith('mongo'):
mongo_client = MongoDBClient(obj, resource).write()
else:
with open(resource, 'w') as fp:
jsonstr = json.dumps(obj, cls=CustomEncoder, **kw)
fp.write(jsonstr)
stop = time.clock()
print("Completed in: '%s' seconds." % (stop - start))
def save_covjson_range(self, obj, path):
for covrange in obj['ranges'].values():
self.no_indent(covrange, 'axisNames', 'shape')
self.compact(covrange, 'values')
self.save_json(obj, path, indent=2)
def compact(self, obj, *names):
for name in names:
obj[name] = Custom(obj[name], separators=(',', ':'))
def no_indent(self, obj, *names):
for name in names:
obj[name] = Custom(obj[name])
def main():
"""
Command line interface for pycovjson - Converts Scientific Data Formats into CovJSON and saves to disk.
:argument -i: Input file path.
:argument -o: Output file name.
:argument -t: Use Tiling.
:argument -v: Which variable to populate coverage with.
:argument -s: [tile shape]: Tile shape.
:argument -n: Use interactive mode.
:argument -u: MongoDB URL
"""
parser = argparse.ArgumentParser(
description='Convert Scientific Data Formats into CovJSON.')
parser.add_argument('-i',
'--input',
dest='inputfile',
help='Name of input file',
required=True)
parser.add_argument('-o',
'--output',
dest='outputfile',
help='Name and location of output file',
default='coverage.covjson')
parser.add_argument('-t',
'--tiled',
action='store_true',
help='Apply tiling')
parser.add_argument('-s',
'--shape',
nargs='+',
help='Tile shape, list',
type=int)
parser.add_argument('-v',
dest='variable',
help='Variable to populate coverage with',
required=True)
parser.add_argument('-n',
'--interactive',
action='store_true',
help='Enter interactive mode')
parser.add_argument('-u',
'--endpoint_url',
dest='endpoint_url',
nargs=1,
help='MongoDB endpoint for CovJSON persistence')
args = parser.parse_args()
inputfile = args.inputfile
outputfile = args.outputfile
variable = args.variable
tiled = args.tiled
tile_shape = args.shape
interactive = args.interactive
endpoint_url = args.endpoint_url
if interactive:
axis = input('Which Axis?', Reader.get_axis(variable))
if tiled and len(tile_shape) == 0:
reader = Reader(inputfile)
shape_list = reader.get_shape(variable)
dims = reader.get_dimensions(variable)
print(list(zip(dims, shape_list)))
tile_shape = input(
'Enter the shape tile shape as a list of comma separated integers'
)
tile_shape = tile_shape.split(',')
tile_shape = list(map(int, tile_shape))
print(tile_shape)
if outputfile == None:
outputfile = outputfile.default
Writer(outputfile,
inputfile, [variable],
tiled=tiled,
tile_shape=tile_shape,
endpoint_url=endpoint_url).write()
