def create_coverage(cls): pdict = cls.create_parameters() sdom, tdom = cls.create_domains() scov = SimplexCoverage('sample grid coverage_model', pdict, tdom, sdom) return scov
def parse_constraints(self, environ): base = os.path.split(self.filepath) coverage = SimplexCoverage.load(base[0], base[1],mode='r') last_modified = formatdate(time.mktime(time.localtime(os.stat(self.filepath)[ST_MTIME]))) environ['pydap.headers'].append(('Last-modified', last_modified)) atts = {} atts['title'] = coverage.name dataset = DatasetType(coverage.name) #, attributes=atts) fields, queries = environ['pydap.ce'] response = environ['pydap.response'] queries = filter(bool, queries) # fix for older version of pydap all_vars = coverage.list_parameters() fill_index = -1 if response == "dods": time_context = coverage.get_parameter_context(coverage.temporal_parameter_name) time_fill_value = time_context.fill_value time_data = coverage.get_parameter_values(coverage.temporal_parameter_name) try: fill_index = np.where(time_data == time_fill_value)[0][0] except IndexError: pass # If no fields have been explicitly requested, of if the sequence # has been requested directly, return all variables. if not fields: fields = [[(name, ())] for name in all_vars] dataset = self.get_dataset(coverage, fields, fill_index, dataset, response) return dataset
def create_coverage(self, mapping_file, param_dict): if param_dict: self._param_mapper = ParameterMapper(pmap_file=mapping_file, parameter_dictionary=param_dict) # Construct temporal Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape("temporal", [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) self._coverage = SimplexCoverage( "test_data", self.create_guid(), os.path.splitext(os.path.basename("test_data/test.csv"))[0], parameter_dictionary=param_dict, temporal_domain=tdom, ) shp = self._parser.get_var_shape("time") self._coverage.insert_timesteps(shp[0]) mapping = self._param_mapper.get_mapping() for var in self._parser.get_col_names(): # parameter mapping goes here print var print mapping[var] vals = self._parser.get_values(var_name=var) print vals self._coverage.set_parameter_values(mapping[var], value=vals) print self._coverage.get_parameter_values(mapping[var])
class coverage_creator: def __init__(self, mod_name="", class_name="", file_path=""): module = __import__(mod_name, fromlist=[class_name]) classobj = getattr(module, class_name) self._parser = classobj(file_path) self._coverage = None def create_coverage(self, mapping_file, param_dict): if param_dict: self._param_mapper = ParameterMapper(pmap_file=mapping_file, parameter_dictionary=param_dict) # Construct temporal Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape("temporal", [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) self._coverage = SimplexCoverage( "test_data", self.create_guid(), os.path.splitext(os.path.basename("test_data/test.csv"))[0], parameter_dictionary=param_dict, temporal_domain=tdom, ) shp = self._parser.get_var_shape("time") self._coverage.insert_timesteps(shp[0]) mapping = self._param_mapper.get_mapping() for var in self._parser.get_col_names(): # parameter mapping goes here print var print mapping[var] vals = self._parser.get_values(var_name=var) print vals self._coverage.set_parameter_values(mapping[var], value=vals) print self._coverage.get_parameter_values(mapping[var]) def create_guid(self): """ @retval Return global unique id string """ # guids seem to be more readable if they are UPPERCASE return str(uuid.uuid4()).upper()
def get_dataset_xml(self, coverage_path): result = '' paths = os.path.split(coverage_path) cov = SimplexCoverage.load(coverage_path) #ds = open_url(url) doc = xml.dom.minidom.Document() #Get lists of variables with unique sets of dimensions. #Datasets can only have variables with the same sets of dimensions datasets = {} for key in cov.list_parameters(): pc = cov.get_parameter_context(key) if np.dtype(pc.param_type.value_encoding).char == 'O': continue param = cov.get_parameter(key) dims = (cov.temporal_parameter_name,) if len(param.shape) == 2: dims = (cov.temporal_parameter_name, cov.spatial_domain.shape.name) if not dims in datasets.keys(): datasets[dims] = [] datasets[dims].append(key) index = 0 for dims, vars in datasets.iteritems(): if not (len(dims) == 1 and dims[0] == vars[0]): dataset_element = doc.createElement('dataset') dataset_element.setAttribute('type', 'EDDGridFromDap') dataset_element.setAttribute('datasetID', '{0}_{1}'.format(paths[1], index)) dataset_element.setAttribute('active', 'True') source_element = doc.createElement('sourceUrl') text_node = doc.createTextNode(self.pydap_url + paths[1]) source_element.appendChild(text_node) dataset_element.appendChild(source_element) reload_element = doc.createElement('reloadEveryNMinutes') text_node = doc.createTextNode('1440') reload_element.appendChild(text_node) dataset_element.appendChild(reload_element) add_attributes_element = doc.createElement('addAttributes') atts = {} atts['title'] = urllib.unquote(cov.name) atts['infoUrl'] = self.pydap_url + paths[1] atts['summary'] = cov.name atts['institution'] = 'ASA' for key, val in atts.iteritems(): att_element = doc.createElement('att') att_element.setAttribute('name', key) text_node = doc.createTextNode(val) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) if len(add_attributes_element.childNodes) > 0: dataset_element.appendChild(add_attributes_element) for var_name in vars: param = cov.get_parameter(var_name) var = param.context units = var.uom if len(param.shape) >=1 and not param.is_coordinate: #dataVariable data_element = doc.createElement('dataVariable') source_name_element = doc.createElement('sourceName') text_node = doc.createTextNode(var.name) source_name_element.appendChild(text_node) data_element.appendChild(source_name_element) destination_name_element = doc.createElement('destinationName') text_node = doc.createTextNode(var.name) destination_name_element.appendChild(text_node) data_element.appendChild(destination_name_element) add_attributes_element = doc.createElement('addAttributes') if not var.attributes is None: for key, val in var.attributes.iteritems(): att_element = doc.createElement('att') att_element.setAttribute('name', key) text_node = doc.createTextNode(val) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) att_element = doc.createElement('att') att_element.setAttribute('name', 'ioos_category') text_node = doc.createTextNode(self.get_ioos_category(var.name, units)) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) data_element.appendChild(add_attributes_element) dataset_element.appendChild(data_element) #add category #add long_name #add standard_name #add units for dim_name in dims: #axisVariable param = cov.get_parameter(dim_name) dim = param.context units = var.uom axis_element = doc.createElement('axisVariable') source_name_element = doc.createElement('sourceName') text_node = doc.createTextNode(dim.name) source_name_element.appendChild(text_node) axis_element.appendChild(source_name_element) destination_name_element = doc.createElement('destinationName') text_node = doc.createTextNode(dim.name) destination_name_element.appendChild(text_node) axis_element.appendChild(destination_name_element) add_attributes_element = doc.createElement('addAttributes') if not dim.attributes is None: for key, val in dim.attributes.iteritems(): att_element = doc.createElement('att') att_element.setAttribute('name', key) text_node = doc.createTextNode(val) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) att_element = doc.createElement('att') att_element.setAttribute('name', 'ioos_category') text_node = doc.createTextNode(self.get_ioos_category(dim.name, units)) att_element.appendChild(text_node) add_attributes_element.appendChild(att_element) axis_element.appendChild(add_attributes_element) dataset_element.appendChild(axis_element) index += 1 result += dataset_element.toprettyxml() + '\n' cov.close() return result
def parse_constraints(self, environ): seq_name = 'data' base = os.path.split(self.filepath) cov = SimplexCoverage.load(base[0], base[1],mode='r') last_modified = formatdate(time.mktime(time.localtime(os.stat(self.filepath)[ST_MTIME]))) environ['pydap.headers'].append(('Last-modified', last_modified)) dataset = DatasetType(cov.name) #find time fill index time_context = cov.get_parameter_context(cov.temporal_parameter_name) time_fill_value = time_context.fill_value time_data = cov.get_parameter_values(cov.temporal_parameter_name) fill_index = -1 try: fill_index = np.where(time_data == time_fill_value)[0][0] except IndexError: pass fields, queries = environ['pydap.ce'] queries = filter(bool, queries) # fix for older version of pydap all_vars = [] for name in cov.list_parameters(): pc = cov.get_parameter_context(name) #if isinstance(pc.param_type, QuantityType): #if name in ["time", "lon", "lat", "density"]: all_vars.append(name) if not fields: fields = [[(name, ())] for name in all_vars] dataset[seq_name] = SequenceType(name=seq_name) for var in fields: while var: name, slice_ = var.pop(0) name = urllib.unquote(name) if seq_name == name: continue slice_ = self.update_slice_object(slice_, fill_index) if slice_ is None: continue pc = cov.get_parameter_context(name) try: attrs = {} try: attrs['units'] = pc.uom except: pass attrs['long_name'] = pc.long_name data = cov.get_parameter_values(name, tdoa=slice_) #fix data for scalars if isinstance(pc.param_type, ConstantRangeType): if isinstance(data, tuple): data = [data] data = np.asanyarray(data) if not data.shape: data.shape = (1,) if isinstance(pc.param_type, QuantityType): dataset[seq_name][name] = BaseType(name=name, data=data, type=data.dtype.char, attributes=attrs) if isinstance(pc.param_type, ConstantType): dataset[seq_name][name] = BaseType(name=name, data=data, type=data.dtype.char, attributes=attrs) if isinstance(pc.param_type, ConstantRangeType): try: #convert to string result = [] for d in data: f = [str(d[0]),str(d[1])] result.append('_'.join(f)) data = np.asanyarray(result) except Exception, e: data = np.asanyarray(['None' for d in data]) dataset[seq_name][name] = BaseType(name=name, data=data, type=data.dtype.char, attributes=attrs) if isinstance(pc.param_type,BooleanType): boolean_values = np.asanyarray(data, dtype='int32') dataset[seq_name][name] = BaseType(name=name, data=boolean_values, type=self.get_numpy_type(boolean_values), attributes=attrs) if isinstance(pc.param_type,CategoryType): dataset[seq_name][name] = BaseType(name=name, data=data, type=self.get_numpy_type(data), attributes=attrs) if isinstance(pc.param_type,ArrayType): dataset[seq_name][name] = BaseType(name=name, data=data, type=self.get_numpy_type(data), attributes=attrs) if isinstance(pc.param_type,RecordType): try: result = [] for ddict in data: d = ['_'.join([k,v]) for k,v in ddict.iteritems()] result = result + d result = np.asanyarray(result) except Exception, e: result = np.asanyarray(['None' for d in data]) ttype = self.get_numpy_type(result) dataset[seq_name][name] = BaseType(name=name, data=result, type=ttype, attributes=attrs) except Exception, e: log.exception('Problem reading cov %s %s', cov.name, e) continue
def parse_constraints(self, environ): base = os.path.split(self.filepath) coverage = SimplexCoverage.load(base[0], base[1]) atts = {} atts['title'] = coverage.name dataset = DatasetType(coverage.name) #, attributes=atts) fields, queries = environ['pydap.ce'] queries = filter(bool, queries) # fix for older version of pydap all_vars = coverage.list_parameters() t = [] for param in all_vars: if numpy.dtype(coverage.get_parameter_context(param).param_type.value_encoding).char == 'O': t.append(param) [all_vars.remove(i) for i in t] time_context = coverage.get_parameter_context(coverage.temporal_parameter_name) time_fill_value = time_context.fill_value time_data = coverage.get_parameter_values(coverage.temporal_parameter_name) fill_index = -1 try: fill_index = numpy.where(time_data == time_fill_value)[0][0] except IndexError: pass # If no fields have been explicitly requested, of if the sequence # has been requested directly, return all variables. if not fields: fields = [[(name, ())] for name in all_vars] for var in fields: target = dataset while var: name, slice_ = var.pop(0) covname = urllib.unquote(name) param = coverage.get_parameter(covname) #need to truncate slice here in case time has fill values if len(slice_) == 0 and fill_index >= 0: slice_ = slice(0, fill_index, 1) if len(slice_) == 1: slice_ = slice_[0] if fill_index > slice_.start: continue if fill_index > slice_.stop: slice_.stop = fill_index if param.is_coordinate or target is not dataset: target[name] = get_var(coverage,name,slice_) elif var: target.setdefault(name, StructureType(name=name, attributes={'units':coverage.get_parameter_context(name).uom})) target = target[name] else: # return grid grid = target[name] = GridType(name=name) grid[name] = get_var(coverage,name, slice_) dim = coverage.temporal_parameter_name grid[dim] = get_var(coverage,dim,slice_) dataset._set_id() dataset.close = coverage.close return dataset