def create_coverage(cls):
pdict = cls.create_parameters()
sdom, tdom = cls.create_domains()
scov = SimplexCoverage('sample grid coverage_model', pdict, tdom, sdom)
return scov
def parse_constraints(self, environ):
base = os.path.split(self.filepath)
coverage = SimplexCoverage.load(base[0], base[1],mode='r')
last_modified = formatdate(time.mktime(time.localtime(os.stat(self.filepath)[ST_MTIME])))
environ['pydap.headers'].append(('Last-modified', last_modified))
atts = {}
atts['title'] = coverage.name
dataset = DatasetType(coverage.name) #, attributes=atts)
fields, queries = environ['pydap.ce']
response = environ['pydap.response']
queries = filter(bool, queries) # fix for older version of pydap
all_vars = coverage.list_parameters()
fill_index = -1
if response == "dods":
time_context = coverage.get_parameter_context(coverage.temporal_parameter_name)
time_fill_value = time_context.fill_value
time_data = coverage.get_parameter_values(coverage.temporal_parameter_name)
try:
fill_index = np.where(time_data == time_fill_value)[0][0]
except IndexError:
pass
# If no fields have been explicitly requested, of if the sequence
# has been requested directly, return all variables.
if not fields:
fields = [[(name, ())] for name in all_vars]
dataset = self.get_dataset(coverage, fields, fill_index, dataset, response)
return dataset
def create_coverage(self, mapping_file, param_dict):
if param_dict:
self._param_mapper = ParameterMapper(pmap_file=mapping_file, parameter_dictionary=param_dict)
# Construct temporal Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape("temporal", [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline)
self._coverage = SimplexCoverage(
"test_data",
self.create_guid(),
os.path.splitext(os.path.basename("test_data/test.csv"))[0],
parameter_dictionary=param_dict,
temporal_domain=tdom,
)
shp = self._parser.get_var_shape("time")
self._coverage.insert_timesteps(shp[0])
mapping = self._param_mapper.get_mapping()
for var in self._parser.get_col_names():
# parameter mapping goes here
print var
print mapping[var]
vals = self._parser.get_values(var_name=var)
print vals
self._coverage.set_parameter_values(mapping[var], value=vals)
print self._coverage.get_parameter_values(mapping[var])
class coverage_creator:
def __init__(self, mod_name="", class_name="", file_path=""):
module = __import__(mod_name, fromlist=[class_name])
classobj = getattr(module, class_name)
self._parser = classobj(file_path)
self._coverage = None
def create_coverage(self, mapping_file, param_dict):
if param_dict:
self._param_mapper = ParameterMapper(pmap_file=mapping_file, parameter_dictionary=param_dict)
# Construct temporal Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape("temporal", [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline)
self._coverage = SimplexCoverage(
"test_data",
self.create_guid(),
os.path.splitext(os.path.basename("test_data/test.csv"))[0],
parameter_dictionary=param_dict,
temporal_domain=tdom,
)
shp = self._parser.get_var_shape("time")
self._coverage.insert_timesteps(shp[0])
mapping = self._param_mapper.get_mapping()
for var in self._parser.get_col_names():
# parameter mapping goes here
print var
print mapping[var]
vals = self._parser.get_values(var_name=var)
print vals
self._coverage.set_parameter_values(mapping[var], value=vals)
print self._coverage.get_parameter_values(mapping[var])
def create_guid(self):
"""
@retval Return global unique id string
"""
# guids seem to be more readable if they are UPPERCASE
return str(uuid.uuid4()).upper()
def get_dataset_xml(self, coverage_path):
result = ''
paths = os.path.split(coverage_path)
cov = SimplexCoverage.load(coverage_path)
#ds = open_url(url)
doc = xml.dom.minidom.Document()
#Get lists of variables with unique sets of dimensions.
#Datasets can only have variables with the same sets of dimensions
datasets = {}
for key in cov.list_parameters():
pc = cov.get_parameter_context(key)
if np.dtype(pc.param_type.value_encoding).char == 'O':
continue
param = cov.get_parameter(key)
dims = (cov.temporal_parameter_name,)
if len(param.shape) == 2:
dims = (cov.temporal_parameter_name, cov.spatial_domain.shape.name)
if not dims in datasets.keys():
datasets[dims] = []
datasets[dims].append(key)
index = 0
for dims, vars in datasets.iteritems():
if not (len(dims) == 1 and dims[0] == vars[0]):
dataset_element = doc.createElement('dataset')
dataset_element.setAttribute('type', 'EDDGridFromDap')
dataset_element.setAttribute('datasetID', '{0}_{1}'.format(paths[1], index))
dataset_element.setAttribute('active', 'True')
source_element = doc.createElement('sourceUrl')
text_node = doc.createTextNode(self.pydap_url + paths[1])
source_element.appendChild(text_node)
dataset_element.appendChild(source_element)
reload_element = doc.createElement('reloadEveryNMinutes')
text_node = doc.createTextNode('1440')
reload_element.appendChild(text_node)
dataset_element.appendChild(reload_element)
add_attributes_element = doc.createElement('addAttributes')
atts = {}
atts['title'] = urllib.unquote(cov.name)
atts['infoUrl'] = self.pydap_url + paths[1]
atts['summary'] = cov.name
atts['institution'] = 'ASA'
for key, val in atts.iteritems():
att_element = doc.createElement('att')
att_element.setAttribute('name', key)
text_node = doc.createTextNode(val)
att_element.appendChild(text_node)
add_attributes_element.appendChild(att_element)
if len(add_attributes_element.childNodes) > 0:
dataset_element.appendChild(add_attributes_element)
for var_name in vars:
param = cov.get_parameter(var_name)
var = param.context
units = var.uom
if len(param.shape) >=1 and not param.is_coordinate: #dataVariable
data_element = doc.createElement('dataVariable')
source_name_element = doc.createElement('sourceName')
text_node = doc.createTextNode(var.name)
source_name_element.appendChild(text_node)
data_element.appendChild(source_name_element)
destination_name_element = doc.createElement('destinationName')
text_node = doc.createTextNode(var.name)
destination_name_element.appendChild(text_node)
data_element.appendChild(destination_name_element)
add_attributes_element = doc.createElement('addAttributes')
if not var.attributes is None:
for key, val in var.attributes.iteritems():
att_element = doc.createElement('att')
att_element.setAttribute('name', key)
text_node = doc.createTextNode(val)
att_element.appendChild(text_node)
add_attributes_element.appendChild(att_element)
att_element = doc.createElement('att')
att_element.setAttribute('name', 'ioos_category')
text_node = doc.createTextNode(self.get_ioos_category(var.name, units))
att_element.appendChild(text_node)
add_attributes_element.appendChild(att_element)
data_element.appendChild(add_attributes_element)
dataset_element.appendChild(data_element)
#add category
#add long_name
#add standard_name
#add units
for dim_name in dims: #axisVariable
param = cov.get_parameter(dim_name)
dim = param.context
units = var.uom
axis_element = doc.createElement('axisVariable')
source_name_element = doc.createElement('sourceName')
text_node = doc.createTextNode(dim.name)
source_name_element.appendChild(text_node)
axis_element.appendChild(source_name_element)
destination_name_element = doc.createElement('destinationName')
text_node = doc.createTextNode(dim.name)
destination_name_element.appendChild(text_node)
axis_element.appendChild(destination_name_element)
add_attributes_element = doc.createElement('addAttributes')
if not dim.attributes is None:
for key, val in dim.attributes.iteritems():
att_element = doc.createElement('att')
att_element.setAttribute('name', key)
text_node = doc.createTextNode(val)
att_element.appendChild(text_node)
add_attributes_element.appendChild(att_element)
att_element = doc.createElement('att')
att_element.setAttribute('name', 'ioos_category')
text_node = doc.createTextNode(self.get_ioos_category(dim.name, units))
att_element.appendChild(text_node)
add_attributes_element.appendChild(att_element)
axis_element.appendChild(add_attributes_element)
dataset_element.appendChild(axis_element)
index += 1
result += dataset_element.toprettyxml() + '\n'
cov.close()
return result
def parse_constraints(self, environ):
seq_name = 'data'
base = os.path.split(self.filepath)
cov = SimplexCoverage.load(base[0], base[1],mode='r')
last_modified = formatdate(time.mktime(time.localtime(os.stat(self.filepath)[ST_MTIME])))
environ['pydap.headers'].append(('Last-modified', last_modified))
dataset = DatasetType(cov.name)
#find time fill index
time_context = cov.get_parameter_context(cov.temporal_parameter_name)
time_fill_value = time_context.fill_value
time_data = cov.get_parameter_values(cov.temporal_parameter_name)
fill_index = -1
try:
fill_index = np.where(time_data == time_fill_value)[0][0]
except IndexError:
pass
fields, queries = environ['pydap.ce']
queries = filter(bool, queries) # fix for older version of pydap
all_vars = []
for name in cov.list_parameters():
pc = cov.get_parameter_context(name)
#if isinstance(pc.param_type, QuantityType):
#if name in ["time", "lon", "lat", "density"]:
all_vars.append(name)
if not fields:
fields = [[(name, ())] for name in all_vars]
dataset[seq_name] = SequenceType(name=seq_name)
for var in fields:
while var:
name, slice_ = var.pop(0)
name = urllib.unquote(name)
if seq_name == name:
continue
slice_ = self.update_slice_object(slice_, fill_index)
if slice_ is None:
continue
pc = cov.get_parameter_context(name)
try:
attrs = {}
try:
attrs['units'] = pc.uom
except:
pass
attrs['long_name'] = pc.long_name
data = cov.get_parameter_values(name, tdoa=slice_)
#fix data for scalars
if isinstance(pc.param_type, ConstantRangeType):
if isinstance(data, tuple):
data = [data]
data = np.asanyarray(data)
if not data.shape:
data.shape = (1,)
if isinstance(pc.param_type, QuantityType):
dataset[seq_name][name] = BaseType(name=name, data=data, type=data.dtype.char, attributes=attrs)
if isinstance(pc.param_type, ConstantType):
dataset[seq_name][name] = BaseType(name=name, data=data, type=data.dtype.char, attributes=attrs)
if isinstance(pc.param_type, ConstantRangeType):
try:
#convert to string
result = []
for d in data:
f = [str(d[0]),str(d[1])]
result.append('_'.join(f))
data = np.asanyarray(result)
except Exception, e:
data = np.asanyarray(['None' for d in data])
dataset[seq_name][name] = BaseType(name=name, data=data, type=data.dtype.char, attributes=attrs)
if isinstance(pc.param_type,BooleanType):
boolean_values = np.asanyarray(data, dtype='int32')
dataset[seq_name][name] = BaseType(name=name, data=boolean_values, type=self.get_numpy_type(boolean_values), attributes=attrs)
if isinstance(pc.param_type,CategoryType):
dataset[seq_name][name] = BaseType(name=name, data=data, type=self.get_numpy_type(data), attributes=attrs)
if isinstance(pc.param_type,ArrayType):
dataset[seq_name][name] = BaseType(name=name, data=data, type=self.get_numpy_type(data), attributes=attrs)
if isinstance(pc.param_type,RecordType):
try:
result = []
for ddict in data:
d = ['_'.join([k,v]) for k,v in ddict.iteritems()]
result = result + d
result = np.asanyarray(result)
except Exception, e:
result = np.asanyarray(['None' for d in data])
ttype = self.get_numpy_type(result)
dataset[seq_name][name] = BaseType(name=name, data=result, type=ttype, attributes=attrs)
except Exception, e:
log.exception('Problem reading cov %s %s', cov.name, e)
continue
def parse_constraints(self, environ):
base = os.path.split(self.filepath)
coverage = SimplexCoverage.load(base[0], base[1])
atts = {}
atts['title'] = coverage.name
dataset = DatasetType(coverage.name) #, attributes=atts)
fields, queries = environ['pydap.ce']
queries = filter(bool, queries) # fix for older version of pydap
all_vars = coverage.list_parameters()
t = []
for param in all_vars:
if numpy.dtype(coverage.get_parameter_context(param).param_type.value_encoding).char == 'O':
t.append(param)
[all_vars.remove(i) for i in t]
time_context = coverage.get_parameter_context(coverage.temporal_parameter_name)
time_fill_value = time_context.fill_value
time_data = coverage.get_parameter_values(coverage.temporal_parameter_name)
fill_index = -1
try:
fill_index = numpy.where(time_data == time_fill_value)[0][0]
except IndexError:
pass
# If no fields have been explicitly requested, of if the sequence
# has been requested directly, return all variables.
if not fields:
fields = [[(name, ())] for name in all_vars]
for var in fields:
target = dataset
while var:
name, slice_ = var.pop(0)
covname = urllib.unquote(name)
param = coverage.get_parameter(covname)
#need to truncate slice here in case time has fill values
if len(slice_) == 0 and fill_index >= 0:
slice_ = slice(0, fill_index, 1)
if len(slice_) == 1:
slice_ = slice_[0]
if fill_index > slice_.start:
continue
if fill_index > slice_.stop:
slice_.stop = fill_index
if param.is_coordinate or target is not dataset:
target[name] = get_var(coverage,name,slice_)
elif var:
target.setdefault(name, StructureType(name=name, attributes={'units':coverage.get_parameter_context(name).uom}))
target = target[name]
else: # return grid
grid = target[name] = GridType(name=name)
grid[name] = get_var(coverage,name, slice_)
dim = coverage.temporal_parameter_name
grid[dim] = get_var(coverage,dim,slice_)
dataset._set_id()
dataset.close = coverage.close
return dataset
