def create_dataset(self, name='', datastore_name='', view_name='', stream_id='', parameter_dict=None, spatial_domain=None, temporal_domain=None, parameter_dictionary_id='', description=''): validate_true(parameter_dict or parameter_dictionary_id, 'A parameter dictionary must be supplied to register a new dataset.') validate_is_not_none(spatial_domain, 'A spatial domain must be supplied to register a new dataset.') validate_is_not_none(temporal_domain, 'A temporal domain must be supplied to register a new dataset.') if parameter_dictionary_id: pd = self.read_parameter_dictionary(parameter_dictionary_id) pcs = self.read_parameter_contexts(parameter_dictionary_id, id_only=False) parameter_dict = self._merge_contexts([ParameterContext.load(i.parameter_context) for i in pcs], pd.temporal_context) parameter_dict = parameter_dict.dump() dataset = Dataset() dataset.description = description dataset.name = name dataset.primary_view_key = stream_id or None dataset.datastore_name = datastore_name or self.DEFAULT_DATASTORE dataset.view_name = view_name or self.DEFAULT_VIEW dataset.parameter_dictionary = parameter_dict dataset.temporal_domain = temporal_domain dataset.spatial_domain = spatial_domain dataset.registered = False dataset_id, _ = self.clients.resource_registry.create(dataset) if stream_id: self.add_stream(dataset_id,stream_id) log.debug('creating dataset: %s', dataset_id) cov = self._create_coverage(dataset_id, description or dataset_id, parameter_dict, spatial_domain, temporal_domain) self._save_coverage(cov) cov.close() return dataset_id
def get_cov(self): # Instantiate a ParameterDictionary pdict = ParameterDictionary() # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary t_ctxt = ParameterContext( 'time', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(t_ctxt, is_temporal=True) lat_ctxt = ParameterContext( 'lat', param_type=QuantityType(value_encoding=np.dtype('float32'))) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext( 'lon', param_type=QuantityType(value_encoding=np.dtype('float32'))) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' pdict.add_context(lon_ctxt) # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE ) # 0d spatial topology (station/trajectory) # Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain scov = SimplexCoverage(self.working_dir, create_guid(), 'sample coverage_model', parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom, inline_data_writes=True, in_memory_storage=False) # Insert some timesteps (automatically expands other arrays) nt = 200000 scov.insert_timesteps(nt) # Add data for each parameter scov.set_parameter_values('time', value=np.arange(nt)) scov.set_parameter_values('lat', value=45) scov.set_parameter_values('lon', value=-71) dset = Dataset() dset.parameter_dictionary = pdict.dump() dset.spatial_domain = sdom.dump() dset.temporal_domain = tdom.dump() return scov, dset
def create_dataset(self, name='', datastore_name='', view_name='', stream_id='', parameter_dict=None, spatial_domain=None, temporal_domain=None, parameter_dictionary_id='', description=''): validate_true( parameter_dict or parameter_dictionary_id, 'A parameter dictionary must be supplied to register a new dataset.' ) validate_is_not_none( spatial_domain, 'A spatial domain must be supplied to register a new dataset.') validate_is_not_none( temporal_domain, 'A temporal domain must be supplied to register a new dataset.') if parameter_dictionary_id: pd = self.read_parameter_dictionary(parameter_dictionary_id) pcs = self.read_parameter_contexts(parameter_dictionary_id, id_only=False) parameter_dict = self._merge_contexts( [ParameterContext.load(i.parameter_context) for i in pcs], pd.temporal_context) parameter_dict = parameter_dict.dump() dataset = Dataset() dataset.description = description dataset.name = name dataset.primary_view_key = stream_id or None dataset.datastore_name = datastore_name or self.DEFAULT_DATASTORE dataset.view_name = view_name or self.DEFAULT_VIEW dataset.parameter_dictionary = parameter_dict dataset.temporal_domain = temporal_domain dataset.spatial_domain = spatial_domain dataset.registered = False dataset_id, _ = self.clients.resource_registry.create(dataset) if stream_id: self.add_stream(dataset_id, stream_id) log.debug('creating dataset: %s', dataset_id) cov = self._create_coverage(dataset_id, description or dataset_id, parameter_dict, spatial_domain, temporal_domain) self._save_coverage(cov) cov.close() return dataset_id
def get_cov(self): # Instantiate a ParameterDictionary pdict = ParameterDictionary() # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(t_ctxt, is_temporal=True) lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.dtype('float32'))) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.dtype('float32'))) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' pdict.add_context(lon_ctxt) # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 0d spatial topology (station/trajectory) # Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain scov = SimplexCoverage(self.working_dir, create_guid(), 'sample coverage_model', parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom, inline_data_writes=True, in_memory_storage=False) # Insert some timesteps (automatically expands other arrays) nt = 200000 # Add data for each parameter data = { scov.temporal_parameter_name: np.arange(nt), 'lat': np.empty(nt), 'lon': np.empty(nt) } data['lat'][:] = 45 data['lon'][:] = -71 scov.set_parameter_values(make_parameter_data_dict(data)) dset = Dataset() dset.parameter_dictionary = pdict.dump() dset.spatial_domain = sdom.dump() dset.temporal_domain = tdom.dump() return scov, dset
def create_dataset(self, name='', datastore_name='', view_name='', stream_id='', parameter_dict=None, parameter_dictionary_id='', description='', parent_dataset_id=''): validate_true(parameter_dict or parameter_dictionary_id, 'A parameter dictionary must be supplied to register a new dataset.') if parameter_dictionary_id: pd = self.read_parameter_dictionary(parameter_dictionary_id) pcs = self.read_parameter_contexts(parameter_dictionary_id, id_only=False) parameter_dict = self._merge_contexts([ParameterContext.load(i.parameter_context) for i in pcs], pd.temporal_context) parameter_dict = parameter_dict.dump() parameter_dict = self.numpy_walk(parameter_dict) dataset = Dataset() dataset.description = description dataset.name = name dataset.primary_view_key = stream_id or None dataset.datastore_name = datastore_name or self.DEFAULT_DATASTORE dataset.view_name = view_name or self.DEFAULT_VIEW dataset.parameter_dictionary = parameter_dict dataset.registered = False dataset_id, _ = self.clients.resource_registry.create(dataset) if stream_id: self.add_stream(dataset_id, stream_id) log.debug('creating dataset: %s', dataset_id) if parent_dataset_id: vcov = self._create_view_coverage(dataset_id, description or dataset_id, parent_dataset_id) vcov.close() return dataset_id cov = self._create_coverage(dataset_id, description or dataset_id, parameter_dict) self._save_coverage(cov) cov.close() #table loader create resource if self._get_eoi_service_available(): log.debug('DM:create dataset: %s -- dataset_id: %s', name, dataset_id) self._create_single_resource(dataset_id, parameter_dict) return dataset_id