def validate_stream_defs(self, in_stream_definition_id, out_stream_definition_id):
stream_def_in = self.read_stream_definition(in_stream_definition_id)
stream_def_out = self.read_stream_definition(out_stream_definition_id)
if not stream_def_in.available_fields:
log.info("no available fields on input stream definition")
return False
incoming_pdict_dump = stream_def_in.parameter_dictionary
outgoing_pdict_dump = stream_def_out.parameter_dictionary
incoming_pdict = ParameterDictionary.load(incoming_pdict_dump)
outgoing_pdict = ParameterDictionary.load(outgoing_pdict_dump)
incoming_ctxt = [pc for name,(n,pc) in incoming_pdict.iteritems()]
outgoing_ctxt = [pc for name,(n,pc) in outgoing_pdict.iteritems()]
incoming_ctxt_af = [pc for name,(n,pc) in incoming_pdict.iteritems() if name in stream_def_in.available_fields]
pfv = ParameterFunctionValidator(incoming_ctxt_af, incoming_ctxt, outgoing_ctxt)
try:
for name in stream_def_out.available_fields:
pfv.validate(name)
except ParameterFunctionException, e:
log.info('validation failed %s to %s is not reachable' % (stream_def_in.available_fields,stream_def_out.available_fields))
log.info(e)
return False
def validate_stream_defs(self, in_stream_definition_id,
out_stream_definition_id):
stream_def_in = self.read_stream_definition(in_stream_definition_id)
stream_def_out = self.read_stream_definition(out_stream_definition_id)
if not stream_def_in.available_fields:
log.info("no available fields on input stream definition")
return False
incoming_pdict_dump = stream_def_in.parameter_dictionary
outgoing_pdict_dump = stream_def_out.parameter_dictionary
incoming_pdict = ParameterDictionary.load(incoming_pdict_dump)
outgoing_pdict = ParameterDictionary.load(outgoing_pdict_dump)
incoming_ctxt = [pc for name, (n, pc) in incoming_pdict.iteritems()]
outgoing_ctxt = [pc for name, (n, pc) in outgoing_pdict.iteritems()]
incoming_ctxt_af = [
pc for name, (n, pc) in incoming_pdict.iteritems()
if name in stream_def_in.available_fields
]
pfv = ParameterFunctionValidator(incoming_ctxt_af, incoming_ctxt,
outgoing_ctxt)
try:
for name in stream_def_out.available_fields:
pfv.validate(name)
except ParameterFunctionException, e:
log.info('validation failed %s to %s is not reachable' %
(stream_def_in.available_fields,
stream_def_out.available_fields))
log.info(e)
return False
def _merge_contexts(cls, contexts, temporal):
pdict = ParameterDictionary()
for context in contexts:
if context.name == temporal:
context.axis = AxisTypeEnum.TIME
pdict.add_context(context, is_temporal=True)
else:
pdict.add_context(context)
return pdict
def _splice_coverage(cls, dataset_id, scov):
file_root = FileSystem.get_url(FS.CACHE, 'datasets')
vcov = cls._get_coverage(dataset_id, mode='a')
scov_pth = scov.persistence_dir
if isinstance(vcov.reference_coverage, SimplexCoverage):
ccov = ComplexCoverage(
file_root,
uuid4().hex,
'Complex coverage for %s' % dataset_id,
reference_coverage_locs=[
vcov.head_coverage_path,
],
parameter_dictionary=ParameterDictionary(),
complex_type=ComplexCoverageType.TEMPORAL_AGGREGATION)
log.info('Creating Complex Coverage: %s', ccov.persistence_dir)
ccov.append_reference_coverage(scov_pth)
ccov_pth = ccov.persistence_dir
ccov.close()
vcov.replace_reference_coverage(ccov_pth)
elif isinstance(vcov.reference_coverage, ComplexCoverage):
log.info('Appending simplex coverage to complex coverage')
#vcov.reference_coverage.append_reference_coverage(scov_pth)
dir_path = vcov.reference_coverage.persistence_dir
vcov.close()
ccov = AbstractCoverage.load(dir_path, mode='a')
ccov.append_reference_coverage(scov_pth)
ccov.refresh()
ccov.close()
vcov.refresh()
vcov.close()
def _create_coverage(self, dataset_id, description, parameter_dict, spatial_domain,temporal_domain):
pdict = ParameterDictionary.load(parameter_dict)
sdom = GridDomain.load(spatial_domain)
tdom = GridDomain.load(temporal_domain)
file_root = FileSystem.get_url(FS.CACHE,'datasets')
scov = SimplexCoverage(file_root,dataset_id,description or dataset_id,parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom, inline_data_writes=self.inline_data_writes)
return scov
def _create_coverage(self, dataset_id, description, parameter_dict):
#file_root = FileSystem.get_url(FS.CACHE,'datasets')
temporal_domain, spatial_domain = time_series_domain()
pdict = ParameterDictionary.load(parameter_dict)
scov = self._create_simplex_coverage(dataset_id, pdict, spatial_domain, temporal_domain, self.inline_data_writes)
#vcov = ViewCoverage(file_root, dataset_id, description or dataset_id, reference_coverage_location=scov.persistence_dir)
scov.close()
return scov
def _create_coverage(self, dataset_id, parameter_dict_id, time_dom, spatial_dom):
pd = self.dataset_management_client.read_parameter_dictionary(parameter_dict_id)
pdict = ParameterDictionary.load(pd)
sdom = GridDomain.load(spatial_dom.dump())
tdom = GridDomain.load(time_dom.dump())
file_root = FileSystem.get_url(FS.CACHE,'datasets')
scov = SimplexCoverage(file_root, dataset_id, dataset_id, parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom)
return scov
def _create_coverage(self, dataset_id, description, parameter_dict):
#file_root = FileSystem.get_url(FS.CACHE,'datasets')
temporal_domain, spatial_domain = time_series_domain()
pdict = ParameterDictionary.load(parameter_dict)
scov = self._create_simplex_coverage(dataset_id, pdict, spatial_domain,
temporal_domain)
#vcov = ViewCoverage(file_root, dataset_id, description or dataset_id, reference_coverage_location=scov.persistence_dir)
scov.close()
return scov
def _create_coverage(self, dataset_id, description, parameter_dict, spatial_domain,temporal_domain):
file_root = FileSystem.get_url(FS.CACHE,'datasets')
pdict = ParameterDictionary.load(parameter_dict)
sdom = GridDomain.load(spatial_domain)
tdom = GridDomain.load(temporal_domain)
scov = self._create_simplex_coverage(dataset_id, pdict, sdom, tdom, self.inline_data_writes)
vcov = ViewCoverage(file_root, dataset_id, description or dataset_id, reference_coverage_location=scov.persistence_dir)
scov.close()
return vcov
def _create_complex_coverage(cls, dataset_id, description, parameter_dict):
pdict = ParameterDictionary.load(parameter_dict)
file_root = FileSystem.get_url(FS.CACHE, 'datasets')
ccov = ComplexCoverage(
file_root,
dataset_id,
'Complex Coverage for %s' % dataset_id,
parameter_dictionary=pdict,
complex_type=ComplexCoverageType.TEMPORAL_AGGREGATION)
return ccov
def _create_coverage(self, description, parameter_dict, spatial_domain,temporal_domain):
#--------------------------------------------------------------------------------
# Create coversions and deserialization then craft the coverage
#--------------------------------------------------------------------------------
pdict = ParameterDictionary.load(parameter_dict)
sdom = GridDomain.load(spatial_domain)
tdom = GridDomain.load(temporal_domain)
scov = SimplexCoverage(description, parameter_dictionary=pdict, spatial_domain=sdom, temporal_domain=tdom)
return scov
def dataset_temporal_bounds(self, dataset_id):
dataset = self.read_dataset(dataset_id)
if not dataset:
return {}
pdict = ParameterDictionary.load(dataset.parameter_dictionary)
temporal_parameter = pdict.temporal_parameter_name
units = pdict.get_temporal_context().uom
bounds = self.dataset_bounds(dataset_id)
if not bounds:
return {}
bounds = bounds[temporal_parameter or 'time']
bounds = [TimeUtils.units_to_ts(units, i) for i in bounds]
return bounds
def has_lookup_values(self, stream_definition_id=''):
stream_definition = self.read_stream_definition(stream_definition_id)
pdict = ParameterDictionary.load(stream_definition.parameter_dictionary)
ret = []
for key in pdict.keys():
p_context = pdict.get_context(key)
if hasattr(p_context, 'lookup_value'):
if stream_definition.available_fields and key in stream_definition.available_fields:
ret.append(p_context.name)
elif not stream_definition.available_fields:
ret.append(p_context.name)
return ret
def _create_coverage(self, dataset_id, description, parameter_dict,
spatial_domain, temporal_domain):
file_root = FileSystem.get_url(FS.CACHE, 'datasets')
pdict = ParameterDictionary.load(parameter_dict)
sdom = GridDomain.load(spatial_domain)
tdom = GridDomain.load(temporal_domain)
scov = self._create_simplex_coverage(dataset_id, pdict, sdom, tdom,
self.inline_data_writes)
vcov = ViewCoverage(file_root,
dataset_id,
description or dataset_id,
reference_coverage_location=scov.persistence_dir)
scov.close()
return vcov
def _create_coverage(self, dataset_id, description, parameter_dict,
spatial_domain, temporal_domain):
pdict = ParameterDictionary.load(parameter_dict)
sdom = GridDomain.load(spatial_domain)
tdom = GridDomain.load(temporal_domain)
file_root = FileSystem.get_url(FS.CACHE, 'datasets')
scov = SimplexCoverage(file_root,
dataset_id,
description or dataset_id,
parameter_dictionary=pdict,
temporal_domain=tdom,
spatial_domain=sdom,
inline_data_writes=self.inline_data_writes)
return scov
def _merge_contexts(cls, contexts, temporal):
pdict = ParameterDictionary()
for context in contexts:
if context.name == temporal:
context.axis = AxisTypeEnum.TIME
pdict.add_context(context, is_temporal=True)
else:
pdict.add_context(context)
return pdict
def _create_coverage(self, dataset_id, parameter_dict_id, time_dom,
spatial_dom):
pd = self.dataset_management_client.read_parameter_dictionary(
parameter_dict_id)
pdict = ParameterDictionary.load(pd)
sdom = GridDomain.load(spatial_dom.dump())
tdom = GridDomain.load(time_dom.dump())
file_root = FileSystem.get_url(FS.CACHE, 'datasets')
scov = SimplexCoverage(file_root,
dataset_id,
dataset_id,
parameter_dictionary=pdict,
temporal_domain=tdom,
spatial_domain=sdom)
return scov
def has_lookup_values(self, stream_definition_id=''):
stream_definition = self.read_stream_definition(stream_definition_id)
pdict = ParameterDictionary.load(
stream_definition.parameter_dictionary)
ret = []
for key in pdict.keys():
p_context = pdict.get_context(key)
if hasattr(p_context, 'lookup_value'):
if stream_definition.available_fields and key in stream_definition.available_fields:
ret.append(p_context.name)
elif not stream_definition.available_fields:
ret.append(p_context.name)
return ret
def _create_complex_coverage(cls, dataset_id, description, parameter_dict):
pdict = ParameterDictionary.load(parameter_dict)
file_root = FileSystem.get_url(FS.CACHE, 'datasets')
ccov = ComplexCoverage(file_root, dataset_id, 'Complex Coverage for %s' % dataset_id, parameter_dictionary=pdict, complex_type=ComplexCoverageType.TEMPORAL_AGGREGATION)
return ccov
def _compare_pdicts(cls, pdict1, pdict2):
if pdict1:
pdict1 = ParameterDictionary.load(pdict1) or {}
if pdict2:
pdict2 = ParameterDictionary.load(pdict2) or {}
return bool(pdict1 == pdict2)
def _create_parameter(self):
pdict = ParameterDictionary()
pdict = self._add_location_time_ctxt(pdict)
pres_ctxt = ParameterContext('pressure', param_type=QuantityType(value_encoding=numpy.float32))
pres_ctxt.uom = 'Pascal'
pres_ctxt.fill_value = 0x0
pdict.add_context(pres_ctxt)
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=numpy.float32))
temp_ctxt.uom = 'degree_Celsius'
temp_ctxt.fill_value = 0e0
pdict.add_context(temp_ctxt)
cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=numpy.float32))
cond_ctxt.uom = 'unknown'
cond_ctxt.fill_value = 0e0
pdict.add_context(cond_ctxt)
raw_fixed_ctxt = ParameterContext('raw_fixed', param_type=QuantityType(value_encoding=numpy.float32))
raw_fixed_ctxt.uom = 'unknown'
raw_fixed_ctxt.fill_value = 0e0
pdict.add_context(raw_fixed_ctxt)
raw_blob_ctxt = ParameterContext('raw_blob', param_type=QuantityType(value_encoding=numpy.float32))
raw_blob_ctxt.uom = 'unknown'
raw_blob_ctxt.fill_value = 0e0
pdict.add_context(raw_blob_ctxt)
return pdict
def _compare_pdicts(cls, pdict1, pdict2):
if pdict1:
pdict1 = ParameterDictionary.load(pdict1) or {}
if pdict2:
pdict2 = ParameterDictionary.load(pdict2) or {}
return bool(pdict1 == pdict2)