def create_dataset(self, name='', datastore_name='', view_name='', stream_id='', parameter_dict=None, spatial_domain=None, temporal_domain=None, parameter_dictionary_id='', description=''):
validate_true(parameter_dict or parameter_dictionary_id, 'A parameter dictionary must be supplied to register a new dataset.')
validate_is_not_none(spatial_domain, 'A spatial domain must be supplied to register a new dataset.')
validate_is_not_none(temporal_domain, 'A temporal domain must be supplied to register a new dataset.')
if parameter_dictionary_id:
pd = self.read_parameter_dictionary(parameter_dictionary_id)
pcs = self.read_parameter_contexts(parameter_dictionary_id, id_only=False)
parameter_dict = self._merge_contexts([ParameterContext.load(i.parameter_context) for i in pcs], pd.temporal_context)
parameter_dict = parameter_dict.dump()
dataset = Dataset()
dataset.description = description
dataset.name = name
dataset.primary_view_key = stream_id or None
dataset.datastore_name = datastore_name or self.DEFAULT_DATASTORE
dataset.view_name = view_name or self.DEFAULT_VIEW
dataset.parameter_dictionary = parameter_dict
dataset.temporal_domain = temporal_domain
dataset.spatial_domain = spatial_domain
dataset.registered = False
dataset_id, _ = self.clients.resource_registry.create(dataset)
if stream_id:
self.add_stream(dataset_id,stream_id)
log.debug('creating dataset: %s', dataset_id)
cov = self._create_coverage(dataset_id, description or dataset_id, parameter_dict, spatial_domain, temporal_domain)
self._save_coverage(cov)
cov.close()
return dataset_id
def get_cov(self):
# Instantiate a ParameterDictionary
pdict = ParameterDictionary()
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
t_ctxt = ParameterContext(
'time', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt, is_temporal=True)
lat_ctxt = ParameterContext(
'lat', param_type=QuantityType(value_encoding=np.dtype('float32')))
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext(
'lon', param_type=QuantityType(value_encoding=np.dtype('float32')))
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
pdict.add_context(lon_ctxt)
# Construct temporal and spatial Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT])
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape('temporal', [0]), tcrs,
MutabilityEnum.EXTENSIBLE) # 1d (timeline)
sdom = GridDomain(GridShape('spatial',
[0]), scrs, MutabilityEnum.IMMUTABLE
) # 0d spatial topology (station/trajectory)
# Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain
scov = SimplexCoverage(self.working_dir,
create_guid(),
'sample coverage_model',
parameter_dictionary=pdict,
temporal_domain=tdom,
spatial_domain=sdom,
inline_data_writes=True,
in_memory_storage=False)
# Insert some timesteps (automatically expands other arrays)
nt = 200000
scov.insert_timesteps(nt)
# Add data for each parameter
scov.set_parameter_values('time', value=np.arange(nt))
scov.set_parameter_values('lat', value=45)
scov.set_parameter_values('lon', value=-71)
dset = Dataset()
dset.parameter_dictionary = pdict.dump()
dset.spatial_domain = sdom.dump()
dset.temporal_domain = tdom.dump()
return scov, dset
def create_dataset(self,
name='',
datastore_name='',
view_name='',
stream_id='',
parameter_dict=None,
spatial_domain=None,
temporal_domain=None,
parameter_dictionary_id='',
description=''):
validate_true(
parameter_dict or parameter_dictionary_id,
'A parameter dictionary must be supplied to register a new dataset.'
)
validate_is_not_none(
spatial_domain,
'A spatial domain must be supplied to register a new dataset.')
validate_is_not_none(
temporal_domain,
'A temporal domain must be supplied to register a new dataset.')
if parameter_dictionary_id:
pd = self.read_parameter_dictionary(parameter_dictionary_id)
pcs = self.read_parameter_contexts(parameter_dictionary_id,
id_only=False)
parameter_dict = self._merge_contexts(
[ParameterContext.load(i.parameter_context) for i in pcs],
pd.temporal_context)
parameter_dict = parameter_dict.dump()
dataset = Dataset()
dataset.description = description
dataset.name = name
dataset.primary_view_key = stream_id or None
dataset.datastore_name = datastore_name or self.DEFAULT_DATASTORE
dataset.view_name = view_name or self.DEFAULT_VIEW
dataset.parameter_dictionary = parameter_dict
dataset.temporal_domain = temporal_domain
dataset.spatial_domain = spatial_domain
dataset.registered = False
dataset_id, _ = self.clients.resource_registry.create(dataset)
if stream_id:
self.add_stream(dataset_id, stream_id)
log.debug('creating dataset: %s', dataset_id)
cov = self._create_coverage(dataset_id, description or dataset_id,
parameter_dict, spatial_domain,
temporal_domain)
self._save_coverage(cov)
cov.close()
return dataset_id
def get_cov(self):
# Instantiate a ParameterDictionary
pdict = ParameterDictionary()
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt, is_temporal=True)
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.dtype('float32')))
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.dtype('float32')))
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
pdict.add_context(lon_ctxt)
# Construct temporal and spatial Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT])
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline)
sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 0d spatial topology (station/trajectory)
# Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain
scov = SimplexCoverage(self.working_dir, create_guid(), 'sample coverage_model', parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom, inline_data_writes=True, in_memory_storage=False)
# Insert some timesteps (automatically expands other arrays)
nt = 200000
# Add data for each parameter
data = { scov.temporal_parameter_name: np.arange(nt),
'lat': np.empty(nt),
'lon': np.empty(nt)
}
data['lat'][:] = 45
data['lon'][:] = -71
scov.set_parameter_values(make_parameter_data_dict(data))
dset = Dataset()
dset.parameter_dictionary = pdict.dump()
dset.spatial_domain = sdom.dump()
dset.temporal_domain = tdom.dump()
return scov, dset
def create_dataset(self, name='', datastore_name='', view_name='', stream_id='', parameter_dict=None, parameter_dictionary_id='', description='', parent_dataset_id=''):
validate_true(parameter_dict or parameter_dictionary_id, 'A parameter dictionary must be supplied to register a new dataset.')
if parameter_dictionary_id:
pd = self.read_parameter_dictionary(parameter_dictionary_id)
pcs = self.read_parameter_contexts(parameter_dictionary_id, id_only=False)
parameter_dict = self._merge_contexts([ParameterContext.load(i.parameter_context) for i in pcs], pd.temporal_context)
parameter_dict = parameter_dict.dump()
parameter_dict = self.numpy_walk(parameter_dict)
dataset = Dataset()
dataset.description = description
dataset.name = name
dataset.primary_view_key = stream_id or None
dataset.datastore_name = datastore_name or self.DEFAULT_DATASTORE
dataset.view_name = view_name or self.DEFAULT_VIEW
dataset.parameter_dictionary = parameter_dict
dataset.registered = False
dataset_id, _ = self.clients.resource_registry.create(dataset)
if stream_id:
self.add_stream(dataset_id, stream_id)
log.debug('creating dataset: %s', dataset_id)
if parent_dataset_id:
vcov = self._create_view_coverage(dataset_id, description or dataset_id, parent_dataset_id)
vcov.close()
return dataset_id
cov = self._create_coverage(dataset_id, description or dataset_id, parameter_dict)
self._save_coverage(cov)
cov.close()
#table loader create resource
if self._get_eoi_service_available():
log.debug('DM:create dataset: %s -- dataset_id: %s', name, dataset_id)
self._create_single_resource(dataset_id, parameter_dict)
return dataset_id