def on_start(self): super(DatasetManagementService, self).on_start() using_eoi_services = self.CFG.get_safe('eoi.meta.use_eoi_services', False) if using_eoi_services: self.resource_parser = ResourceParser() else: self.resource_parser = None
def on_initial_bootstrap(self, process, config, **kwargs): """ EOI BootstrapPlugin Resets the geoserver datastore... """ using_eoi_services = config.get_safe('eoi.meta.use_eoi_services', False) if using_eoi_services: try: r = ResourceParser() r.reset() except Exception: log.error("Error resetting EOI Services") else: log.info("EOI Services are not enabled")
def on_initial_bootstrap(self, process, config, **kwargs): """ EOI BootstrapPlugin Resets the geoserver datastore... Performs inital parsing of the eoi datasources """ using_eoi_services = config.get_safe('eoi.meta.use_eoi_services', False) if using_eoi_services: try: r = ResourceParser() r.reset() except Exception: log.error("Error resetting EOI Services") else: log.info("EOI Services are not enabled")
def on_start(self): super(DatasetManagementService,self).on_start() using_eoi_services = self.CFG.get_safe('eoi.meta.use_eoi_services', False) if using_eoi_services: self.resource_parser = ResourceParser() else: self.resource_parser = None
def on_start(self): super(DatasetManagementService,self).on_start() self.datastore_name = self.CFG.get_safe('process.datastore_name', self.DEFAULT_DATASTORE) self.inline_data_writes = self.CFG.get_safe('service.ingestion_management.inline_data_writes', True) #self.db = self.container.datastore_manager.get_datastore(self.datastore_name,DataStore.DS_PROFILE.SCIDATA) using_eoi_services = self.CFG.get_safe('eoi.meta.use_eoi_services', False) if using_eoi_services: self.resource_parser = ResourceParser() else: self.resource_parser = None
def on_start(self): super(DatasetManagementService,self).on_start() self.datastore_name = self.CFG.get_safe('process.datastore_name', self.DEFAULT_DATASTORE) self.inline_data_writes = self.CFG.get_safe('service.ingestion_management.inline_data_writes', True) #self.db = self.container.datastore_manager.get_datastore(self.datastore_name,DataStore.DS_PROFILE.SCIDATA) self.rr_table_loader = ResourceParser() self.geos_available = False #check that the services are available if self.rr_table_loader.use_geo_services: self.geos_available = True # if they are proceed with the reset try: self.rr_table_loader.reset() self.geos_available = True pass except Exception as e: #check the eoi geoserver importer service is started raise e
class DatasetManagementService(BaseDatasetManagementService): DEFAULT_DATASTORE = 'datasets' DEFAULT_VIEW = 'manifest/by_dataset' def __init__(self, *args, **kwargs): super(DatasetManagementService, self).__init__(*args,**kwargs) self.logging_name = '(DatasetManagementService %s)' % (self.name or self.id) def on_start(self): super(DatasetManagementService,self).on_start() self.datastore_name = self.CFG.get_safe('process.datastore_name', self.DEFAULT_DATASTORE) self.inline_data_writes = self.CFG.get_safe('service.ingestion_management.inline_data_writes', True) #self.db = self.container.datastore_manager.get_datastore(self.datastore_name,DataStore.DS_PROFILE.SCIDATA) using_eoi_services = self.CFG.get_safe('eoi.meta.use_eoi_services', False) if using_eoi_services: self.resource_parser = ResourceParser() else: self.resource_parser = None #-------- def create_dataset(self, name='', datastore_name='', view_name='', stream_id='', parameter_dict=None, spatial_domain=None, temporal_domain=None, parameter_dictionary_id='', description='', parent_dataset_id=''): validate_true(parameter_dict or parameter_dictionary_id, 'A parameter dictionary must be supplied to register a new dataset.') validate_is_not_none(spatial_domain, 'A spatial domain must be supplied to register a new dataset.') validate_is_not_none(temporal_domain, 'A temporal domain must be supplied to register a new dataset.') if parameter_dictionary_id: pd = self.read_parameter_dictionary(parameter_dictionary_id) pcs = self.read_parameter_contexts(parameter_dictionary_id, id_only=False) parameter_dict = self._merge_contexts([ParameterContext.load(i.parameter_context) for i in pcs], pd.temporal_context) parameter_dict = parameter_dict.dump() parameter_dict = self.numpy_walk(parameter_dict) dataset = Dataset() dataset.description = description dataset.name = name dataset.primary_view_key = stream_id or None dataset.datastore_name = datastore_name or self.DEFAULT_DATASTORE dataset.view_name = view_name or self.DEFAULT_VIEW dataset.parameter_dictionary = parameter_dict dataset.temporal_domain = temporal_domain dataset.spatial_domain = spatial_domain dataset.registered = False dataset_id, _ = self.clients.resource_registry.create(dataset) if stream_id: self.add_stream(dataset_id, stream_id) log.debug('creating dataset: %s', dataset_id) if parent_dataset_id: vcov = self._create_view_coverage(dataset_id, description or dataset_id, parent_dataset_id) vcov.close() return dataset_id cov = self._create_coverage(dataset_id, description or dataset_id, parameter_dict, spatial_domain, temporal_domain) self._save_coverage(cov) cov.close() #table loader create resource if self._get_eoi_service_available(): log.debug('DM:create dataset: %s -- dataset_id: %s', name, dataset_id) self._create_single_resource(dataset_id, parameter_dict) return dataset_id def read_dataset(self, dataset_id=''): retval = self.clients.resource_registry.read(dataset_id) validate_is_instance(retval,Dataset) return retval def update_dataset(self, dataset=None): if not (dataset and dataset._id): raise BadRequest('%s: Dataset either not provided or malformed.' % self.logging_name) self.clients.resource_registry.update(dataset) #@todo: Check to make sure retval is boolean log.debug('DM:update dataset: dataset_id: %s', dataset._id) if self._get_eoi_service_available(): self._remove_single_resource(dataset._id) self._create_single_resource(dataset._id, dataset.parameter_dictionary) return True def delete_dataset(self, dataset_id=''): assocs = self.clients.resource_registry.find_associations(subject=dataset_id, predicate=PRED.hasStream) for assoc in assocs: self.clients.resource_registry.delete_association(assoc) self.clients.resource_registry.delete(dataset_id) log.debug('DM:delete dataset: dataset_id: %s', dataset_id) if self._get_eoi_service_available(): self._remove_single_resource(dataset_id) def register_dataset(self, data_product_id=''): raise BadRequest("register_dataset is no longer supported, please use create_catalog_entry in data product management") #-------- def add_parameter_to_dataset(self, parameter_context_id='', dataset_id=''): cov = self._get_simplex_coverage(dataset_id, mode='r+') parameter_ctx_res = self.read_parameter_context(parameter_context_id) pc = ParameterContext.load(parameter_ctx_res.parameter_context) cov.append_parameter(pc) cov.close() dataset = self.read_dataset(dataset_id) pdict = cov.parameter_dictionary dataset.parameter_dictionary = pdict.dump() self.update_dataset(dataset) return True #-------- def add_stream(self,dataset_id='', stream_id=''): log.info('Adding stream %s to dataset %s', stream_id, dataset_id) validate_true(dataset_id and stream_id, 'Clients must provide both the dataset_id and stream_id') self.clients.resource_registry.create_association(subject=dataset_id, predicate=PRED.hasStream,object=stream_id) def remove_stream(self,dataset_id='', stream_id=''): log.info('Removing stream %s from dataset %s', stream_id, dataset_id) validate_true(dataset_id and stream_id, 'Clients must provide both the dataset_id and stream_id') assocs = self.clients.resource_registry.find_associations(subject=dataset_id, predicate=PRED.hasStream,object=stream_id) for assoc in assocs: self.clients.resource_registry.delete_association(assoc) #-------- def get_dataset_info(self,dataset_id=''): coverage = self._get_coverage(dataset_id, mode='r') return coverage.info def get_dataset_parameters(self, dataset_id=''): coverage = self._get_coverage(dataset_id, mode='r') return coverage.parameter_dictionary.dump() def get_dataset_length(self, dataset_id=''): coverage = self._get_coverage(dataset_id, mode='r') return coverage.num_timesteps #-------- @classmethod def numpy_walk(cls,obj): try: if np.isnan(obj): return {'__nan__':0} except TypeError: pass except NotImplementedError: pass except ValueError: pass if isinstance(obj, np.number): return np.asscalar(obj) if isinstance(obj, np.dtype): return {'__np__':obj.str} if isinstance(obj,dict): if '__nan__' in obj and len(obj)==1: return np.nan if '__np__' in obj and len(obj)==1: return np.dtype(obj['__np__']) return {k:cls.numpy_walk(v) for k,v in obj.iteritems()} if isinstance(obj,list): return map(cls.numpy_walk, obj) if isinstance(obj, tuple): return tuple(map(cls.numpy_walk, obj)) return obj def create_parameter(self, parameter_context=None): """ Creates a parameter context using the IonObject """ context = self.get_coverage_parameter(parameter_context) parameter_context.parameter_context = context.dump() parameter_context = self.numpy_walk(parameter_context) parameter_context_id, _ = self.clients.resource_registry.create(parameter_context) if parameter_context.parameter_function_id: self.read_parameter_function(parameter_context.parameter_function_id) self.clients.resource_registry.create_association( subject=parameter_context_id, predicate=PRED.hasParameterFunction, object=parameter_context.parameter_function_id) return parameter_context_id @classmethod def get_coverage_parameter(cls, parameter_context): """ Creates a Coverage Model based Parameter Context given the ParameterContext IonObject. Note: If the parameter is a parameter function and depends on dynamically created calibrations, this will fail. """ # Only CF and netCDF compliant variable names parameter_context.name = re.sub(r'[^a-zA-Z0-9_]', '_', parameter_context.name) from ion.services.dm.utility.types import TypesManager # The TypesManager does all the parsing and converting to the coverage model instances tm = TypesManager(None, {}, {}) # First thing to do is create the parameter type param_type = tm.get_parameter_type( parameter_context.parameter_type, parameter_context.value_encoding, parameter_context.code_report, parameter_context.parameter_function_id, parameter_context.parameter_function_map) # Ugh, I hate it but I did copy this section from # ion/processes/bootstrap/ion_loader.py context = ParameterContext(name=parameter_context.name, param_type=param_type) # Now copy over all the attrs context.uom = parameter_context.units try: if isinstance(context.uom, basestring): tm.get_unit(context.uom) except UdunitsError: log.warning('Parameter %s has invalid units: %s', parameter_context.name, context.uom) # Fill values can be a bit tricky... context.fill_value = tm.get_fill_value(parameter_context.fill_value, parameter_context.value_encoding, param_type) context.reference_urls = parameter_context.reference_urls context.internal_name = parameter_context.name context.display_name = parameter_context.display_name context.standard_name = parameter_context.standard_name context.ooi_short_name = parameter_context.ooi_short_name context.description = parameter_context.description context.precision = parameter_context.precision context.visible = parameter_context.visible return context def create_parameter_context(self, name='', parameter_context=None, description='', reference_urls=None, parameter_type='', internal_name='', value_encoding='', code_report='', units='', fill_value='', display_name='', parameter_function_id='', parameter_function_map='', standard_name='', ooi_short_name='', precision='', visible=True): validate_true(name, 'Name field may not be empty') validate_is_instance(parameter_context, dict, 'parameter_context field is not dictable.') name = re.sub(r'[^a-zA-Z0-9_]', '_', name) parameter_context = self.numpy_walk(parameter_context) parameter_context['name'] = name pc_res = ParameterContextResource(name=name, parameter_context=parameter_context, description=description) pc_res.reference_urls = reference_urls or [] pc_res.parameter_type = parameter_type pc_res.internal_name = internal_name or name pc_res.value_encoding = value_encoding pc_res.code_report = code_report or '' pc_res.units = units pc_res.fill_value = fill_value pc_res.display_name = display_name pc_res.parameter_function_id = parameter_function_id pc_res.parameter_function_map = parameter_function_map pc_res.standard_name = standard_name pc_res.ooi_short_name = ooi_short_name pc_res.precision = precision or '5' pc_res.visible = visible pc_id, ver = self.clients.resource_registry.create(pc_res) if parameter_function_id: self.read_parameter_function(parameter_function_id) self.clients.resource_registry.create_association(subject=pc_id, predicate=PRED.hasParameterFunction, object=parameter_function_id) return pc_id def read_parameter_context(self, parameter_context_id=''): res = self.clients.resource_registry.read(parameter_context_id) validate_is_instance(res,ParameterContextResource) res.parameter_context = self.numpy_walk(res.parameter_context) return res def delete_parameter_context(self, parameter_context_id=''): self.read_parameter_context(parameter_context_id) self.clients.resource_registry.delete(parameter_context_id) return True #-------- def read_parameter_context_by_name(self, name='', id_only=False): res, _ = self.clients.resource_registry.find_resources(restype=RT.ParameterContext, name=name, id_only=id_only) if not len(res): raise NotFound('Unable to locate context with name: %s' % name) retval = res[0] if not id_only: retval.parameter_context = self.numpy_walk(retval.parameter_context) return retval #-------- def create_parameter_function(self, parameter_function=None): validate_is_instance(parameter_function, ParameterFunctionResource) pf_id, ver = self.clients.resource_registry.create(parameter_function) return pf_id def read_parameter_function(self, parameter_function_id=''): res = self.clients.resource_registry.read(parameter_function_id) validate_is_instance(res, ParameterFunctionResource) return res def delete_parameter_function(self, parameter_function_id=''): self.read_parameter_function(parameter_function_id) self.clients.resource_registry.retire(parameter_function_id) return True @classmethod def get_coverage_function(self, parameter_function): func = None if parameter_function.function_type == PFT.PYTHON: func = PythonFunction(name=parameter_function.name, owner=parameter_function.owner, func_name=parameter_function.function, arg_list=parameter_function.args, kwarg_map=None, param_map=None, egg_uri=parameter_function.egg_uri) elif parameter_function.function_type == PFT.NUMEXPR: func = NumexprFunction(name=parameter_function.name, expression=parameter_function.function, arg_list=parameter_function.args) if not isinstance(func, AbstractFunction): raise Conflict("Incompatible parameter function loaded: %s" % parameter_function._id) return func #-------- def load_parameter_function(self, row): name = row['Name'] ftype = row['Function Type'] func_expr = row['Function'] owner = row['Owner'] args = ast.literal_eval(row['Args']) #kwargs = row['Kwargs'] descr = row['Description'] data_process_management = DataProcessManagementServiceProcessClient(self) function_type=None if ftype == 'PythonFunction': function_type = PFT.PYTHON elif ftype == 'NumexprFunction': function_type = PFT.NUMEXPR else: raise Conflict('Unsupported Function Type: %s' % ftype) parameter_function = ParameterFunctionResource( name=name, function=func_expr, function_type=function_type, owner=owner, args=args, description=descr) parameter_function.alt_ids = ['PRE:' + row['ID']] parameter_function_id = self.create_parameter_function(parameter_function) dpd = DataProcessDefinition() dpd.name = name dpd.description = 'Parameter Function Definition for %s' % name dpd.data_process_type = DataProcessTypeEnum.PARAMETER_FUNCTION dpd.parameters = args data_process_management.create_data_process_definition(dpd, parameter_function_id) return parameter_function_id #-------- def read_parameter_function_by_name(self, name='', id_only=False): res, _ = self.clients.resource_registry.find_resources(restype=RT.ParameterFunction,name=name, id_only=id_only) if not len(res): raise NotFound('Unable to locate parameter function with name: %s' % name) retval = res[0] retval.parameter_function = self.numpy_walk(retval.parameter_function) return retval #-------- def create_parameter_dictionary(self, name='', parameter_context_ids=None, temporal_context='', description=''): res, _ = self.clients.resource_registry.find_resources(restype=RT.ParameterDictionary, name=name, id_only=True) if len(res): context_ids,_ = self.clients.resource_registry.find_objects(subject=res[0], predicate=PRED.hasParameterContext, id_only=True) context_ids.sort() parameter_context_ids.sort() if context_ids == parameter_context_ids: return res[0] else: raise Conflict('A parameter dictionary with name %s already exists and has a different definition' % name) validate_true(name, 'Name field may not be empty.') parameter_context_ids = parameter_context_ids or [] pd_res = ParameterDictionaryResource(name=name, temporal_context=temporal_context, description=description) pd_res_id, ver = self.clients.resource_registry.create(pd_res) for pc_id in parameter_context_ids: self._link_pcr_to_pdr(pc_id, pd_res_id) return pd_res_id def read_parameter_dictionary(self, parameter_dictionary_id=''): res = self.clients.resource_registry.read(parameter_dictionary_id) validate_is_instance(res, ParameterDictionaryResource, 'Resource is not a valid ParameterDictionaryResource') return res def delete_parameter_dictionary(self, parameter_dictionary_id=''): self.read_parameter_dictionary(parameter_dictionary_id) self._unlink_pdr(parameter_dictionary_id) self.clients.resource_registry.delete(parameter_dictionary_id) return True #-------- def read_parameter_contexts(self, parameter_dictionary_id='', id_only=False): pcs, assocs = self.clients.resource_registry.find_objects(subject=parameter_dictionary_id, predicate=PRED.hasParameterContext, id_only=id_only) if not id_only: for pc in pcs: pc.parameter_context = self.numpy_walk(pc.parameter_context) return pcs def read_parameter_dictionary_by_name(self, name='', id_only=False): res, _ = self.clients.resource_registry.find_resources(restype=RT.ParameterDictionary, name=name, id_only=id_only) if not len(res): raise NotFound('Unable to locate dictionary with name: %s' % name) return res[0] #-------- def dataset_bounds(self, dataset_id='', parameters=None): self.read_dataset(dataset_id) # Validates proper dataset parameters = parameters or None try: doc = self.container.object_store.read_doc(dataset_id) except NotFound: return {} if parameters is not None: retval = {} for p in parameters: if p in doc['bounds']: retval[p] = doc['bounds'][p] return retval return doc['bounds'] def dataset_bounds_by_axis(self, dataset_id='', axis=None): bounds = self.dataset_bounds(dataset_id) if bounds and axis and axis in bounds: return bounds[axis] return {} def dataset_temporal_bounds(self, dataset_id): dataset = self.read_dataset(dataset_id) if not dataset: return {} pdict = ParameterDictionary.load(dataset.parameter_dictionary) temporal_parameter = pdict.temporal_parameter_name units = pdict.get_temporal_context().uom bounds = self.dataset_bounds(dataset_id) if not bounds: return {} bounds = bounds[temporal_parameter or 'time'] bounds = [TimeUtils.units_to_ts(units, i) for i in bounds] return bounds def dataset_extents(self, dataset_id='', parameters=None): self.read_dataset(dataset_id) # Validates proper dataset parameters = parameters or None try: doc = self.container.object_store.read_doc(dataset_id) except NotFound: return {} if parameters is not None: if isinstance(parameters, list): retval = {} for p in parameters: retval[p] = doc['extents'][p] return retval elif isinstance(parameters, basestring): return doc['extents'][parameters] return doc['extents'] def dataset_extents_by_axis(self, dataset_id='', axis=None): extents = self.dataset_extents(dataset_id) if extents and axis and axis in extents: return extents[axis] return {} def dataset_size(self,dataset_id='', in_bytes=False): self.read_dataset(dataset_id) # Validates proper dataset try: doc = self.container.object_store.read_doc(dataset_id) except NotFound: return 0. size = doc['size'] if not in_bytes: size = size / 1024. return size def dataset_latest(self, dataset_id=''): self.read_dataset(dataset_id) # Validates proper dataset try: doc = self.container.object_store.read_doc(dataset_id) except NotFound: return {} return doc['last_values'] #-------- @classmethod def get_parameter_context(cls, parameter_context_id=''): """ Preferred client-side class method for constructing a parameter context from a service call. """ dms_cli = DatasetManagementServiceClient() pc_res = dms_cli.read_parameter_context(parameter_context_id=parameter_context_id) pc = ParameterContext.load(pc_res.parameter_context) pc._identifier = pc_res._id return pc @classmethod def get_parameter_function(cls, parameter_function_id=''): """ Preferred client-side class method for constructing a parameter function """ dms_cli = DatasetManagementServiceClient() pf_res = dms_cli.read_parameter_function(parameter_function_id=parameter_function_id) pf = AbstractFunction.load(pf_res.parameter_function) pf._identifier = pf._id return pf @classmethod def get_parameter_context_by_name(cls, name=''): dms_cli = DatasetManagementServiceClient() pc_res = dms_cli.read_parameter_context_by_name(name=name, id_only=False) pc = ParameterContext.load(pc_res.parameter_context) pc._identifier = pc_res._id return pc @classmethod def get_parameter_dictionary(cls, parameter_dictionary_id=''): """ Preferred client-side class method for constructing a parameter dictionary from a service call. """ dms_cli = DatasetManagementServiceClient() pd = dms_cli.read_parameter_dictionary(parameter_dictionary_id) pcs = dms_cli.read_parameter_contexts(parameter_dictionary_id=parameter_dictionary_id, id_only=False) return cls.build_parameter_dictionary(pd, pcs) @classmethod def build_parameter_dictionary(cls, parameter_dictionary_obj, parameter_contexts): pdict = cls._merge_contexts([ParameterContext.load(i.parameter_context) for i in parameter_contexts], parameter_dictionary_obj.temporal_context) pdict._identifier = parameter_dictionary_obj._id return pdict @classmethod def get_parameter_dictionary_by_name(cls, name=''): dms_cli = DatasetManagementServiceClient() pd_res = dms_cli.read_parameter_dictionary_by_name(name=name, id_only=True) return cls.get_parameter_dictionary(pd_res) #-------- def create_parameters_mult(self, parameter_function_list=None, parameter_context_list=None, parameter_dictionary_list=None, parameter_dictionary_assocs=None): pass #-------- def _link_pcr_to_pdr(self, pcr_id, pdr_id): self.clients.resource_registry.create_association(subject=pdr_id, predicate=PRED.hasParameterContext,object=pcr_id) def _unlink_pcr_to_pdr(self, pcr_id, pdr_id): assocs = self.clients.resource_registry.find_associations(subject=pdr_id, predicate=PRED.hasParameterContext, object=pcr_id, id_only=True) for assoc in assocs: self.clients.resource_registry.delete_association(assoc) def _unlink_pdr(self, pdr_id): objects, assocs = self.clients.resource_registry.find_objects(subject=pdr_id, predicate=PRED.hasParameterContext, id_only=True) for assoc in assocs: self.clients.resource_registry.delete_association(assoc) def _create_coverage(self, dataset_id, description, parameter_dict, spatial_domain,temporal_domain): #file_root = FileSystem.get_url(FS.CACHE,'datasets') pdict = ParameterDictionary.load(parameter_dict) sdom = GridDomain.load(spatial_domain) tdom = GridDomain.load(temporal_domain) scov = self._create_simplex_coverage(dataset_id, pdict, sdom, tdom, self.inline_data_writes) #vcov = ViewCoverage(file_root, dataset_id, description or dataset_id, reference_coverage_location=scov.persistence_dir) scov.close() return scov def _create_view_coverage(self, dataset_id, description, parent_dataset_id): # As annoying as it is we need to load the view coverage belonging to parent dataset id and use the information # inside to build the new one... file_root = FileSystem.get_url(FS.CACHE,'datasets') pscov = self._get_simplex_coverage(parent_dataset_id, mode='r') scov_location = pscov.persistence_dir pscov.close() vcov = ViewCoverage(file_root, dataset_id, description or dataset_id, reference_coverage_location=scov_location) return vcov @classmethod def _create_simplex_coverage(cls, dataset_id, parameter_dictionary, spatial_domain, temporal_domain, inline_data_writes=True): file_root = FileSystem.get_url(FS.CACHE,'datasets') #scov = SimplexCoverage(file_root,uuid4().hex,'Simplex Coverage for %s' % dataset_id, parameter_dictionary=parameter_dictionary, temporal_domain=temporal_domain, spatial_domain=spatial_domain, inline_data_writes=inline_data_writes) scov = SimplexCoverage(file_root,dataset_id,'Simplex Coverage for %s' % dataset_id, parameter_dictionary=parameter_dictionary, temporal_domain=temporal_domain, spatial_domain=spatial_domain, inline_data_writes=inline_data_writes) return scov @classmethod def _splice_coverage(cls, dataset_id, scov): file_root = FileSystem.get_url(FS.CACHE,'datasets') vcov = cls._get_coverage(dataset_id,mode='a') scov_pth = scov.persistence_dir if isinstance(vcov.reference_coverage, SimplexCoverage): ccov = ComplexCoverage(file_root, uuid4().hex, 'Complex coverage for %s' % dataset_id, reference_coverage_locs=[vcov.head_coverage_path,], parameter_dictionary=ParameterDictionary(), complex_type=ComplexCoverageType.TEMPORAL_AGGREGATION) log.info('Creating Complex Coverage: %s', ccov.persistence_dir) ccov.append_reference_coverage(scov_pth) ccov_pth = ccov.persistence_dir ccov.close() vcov.replace_reference_coverage(ccov_pth) elif isinstance(vcov.reference_coverage, ComplexCoverage): log.info('Appending simplex coverage to complex coverage') #vcov.reference_coverage.append_reference_coverage(scov_pth) dir_path = vcov.reference_coverage.persistence_dir vcov.close() ccov = AbstractCoverage.load(dir_path, mode='a') ccov.append_reference_coverage(scov_pth) ccov.refresh() ccov.close() vcov.refresh() vcov.close() @classmethod def _save_coverage(cls, coverage): coverage.flush() @classmethod def _get_coverage(cls,dataset_id,mode='r'): file_root = FileSystem.get_url(FS.CACHE,'datasets') coverage = AbstractCoverage.load(file_root, dataset_id, mode=mode) return coverage @classmethod def _get_nonview_coverage(cls, dataset_id, mode='r'): cov = cls._get_coverage(dataset_id, mode) if isinstance(cov, ViewCoverage): rcov = cov.reference_coverage pdir = rcov.persistence_dir rcov = None cov.close() cov = AbstractCoverage.load(pdir, mode=mode) return cov @classmethod def _get_simplex_coverage(cls, dataset_id, mode='r'): cov = cls._get_coverage(dataset_id, mode=mode) if isinstance(cov, SimplexCoverage): return cov if isinstance(cov, ViewCoverage): path = cov.head_coverage_path guid = os.path.basename(path) cov.close() return cls._get_simplex_coverage(guid, mode=mode) raise BadRequest('Unsupported coverage type found: %s' % type(cov)) @classmethod def _get_coverage_path(cls, dataset_id): file_root = FileSystem.get_url(FS.CACHE,'datasets') return os.path.join(file_root, '%s' % dataset_id) @classmethod def _compare_pc(cls, pc1, pc2): if pc1: pc1 = ParameterContext.load(pc1) or {} if pc2: pc2 = ParameterContext.load(pc2) or {} if hasattr(pc1,'lookup_value') or hasattr(pc2,'lookup_value'): if hasattr(pc1,'lookup_value') and hasattr(pc2,'lookup_value'): return bool(pc1 == pc2) and pc1.document_key == pc2.document_key return False return bool(pc1 == pc2) @classmethod def _merge_contexts(cls, contexts, temporal): pdict = ParameterDictionary() for context in contexts: if context.name == temporal: context.axis = AxisTypeEnum.TIME pdict.add_context(context, is_temporal=True) else: pdict.add_context(context) return pdict def read_qc_table(self, obj_id): obj = self.container.object_store.read(obj_id) if '_type' in obj and obj['_type'] == 'QC': return obj else: raise BadRequest('obj_id %s not QC' % obj_id) def _create_single_resource(self,dataset_id, param_dict): ''' EOI Creates a foreign data table and a geoserver layer for the given dataset and parameter dictionary ''' self.resource_parser.create_single_resource(dataset_id,param_dict) def _remove_single_resource(self,dataset_id): ''' EOI Removes foreign data table and geoserver layer for the given dataset ''' self.resource_parser.remove_single_resource(dataset_id) def _get_eoi_service_available(self): ''' EOI Returns true if geoserver endpoint is running and verified by table loader process. Once a true is returned, the result is cached and the process is no longer queried ''' return self.resource_parser and self.resource_parser.get_eoi_service_available()
class DatasetManagementService(BaseDatasetManagementService): def __init__(self, *args, **kwargs): super(DatasetManagementService, self).__init__(*args, **kwargs) self.logging_name = '(DatasetManagementService %s)' % (self.name or self.id) def on_start(self): super(DatasetManagementService, self).on_start() using_eoi_services = self.CFG.get_safe('eoi.meta.use_eoi_services', False) if using_eoi_services: self.resource_parser = ResourceParser() else: self.resource_parser = None #-------- def create_dataset(self, dataset=None, parameter_dict=None, parameter_dictionary_id=''): if parameter_dict is not None: log.warning( "Creating a parameter dictionary raw with coverage objects will soon be deprecated" ) if parameter_dictionary_id: parameter_dict = self._coverage_parameter_dictionary( parameter_dictionary_id) parameter_dict = parameter_dict.dump() # Serialize it dataset.coverage_version = 'UNSET' dataset_id, rev = self.clients.resource_registry.create(dataset) try: if dataset.coverage_type == CoverageTypeEnum.SIMPLEX: cov = self._create_coverage(dataset_id, dataset.description or dataset_id, parameter_dict) self._save_coverage(cov) cov.close() elif dataset.coverage_type == CoverageTypeEnum.COMPLEX: cov = self._create_complex_coverage( dataset_id, dataset.description or dataset_id, parameter_dict) cov.close() else: raise BadRequest("Unknown Coverage Type") except Exception: # Clean up dangling resource if there's no coverage self.delete_dataset(dataset_id) raise dataset.coverage_version = "TODO" dataset._id = dataset_id dataset._rev = rev self.update_dataset(dataset) log.debug('creating dataset: %s', dataset_id) #table loader create resource if dataset.visibility == ResourceVisibilityEnum.PUBLIC: log.debug('dataset visible: %s', dataset_id) if self._get_eoi_service_available() and parameter_dictionary_id: params = self.read_parameter_contexts(parameter_dictionary_id) param_defs = {} for p in params: param_defs[p.name] = { "value_encoding": p.value_encoding, "parameter_type": p.parameter_type, "units": p.units, "standard_name": p.name, "display_name": p.display_name, "description": p.description, "fill_value": p.fill_value } self._create_single_resource(dataset_id, param_defs) self.clients.resource_registry.create_association( dataset_id, PRED.hasParameterDictionary, parameter_dictionary_id) return dataset_id def read_dataset(self, dataset_id=''): retval = self.clients.resource_registry.read(dataset_id) validate_is_instance(retval, Dataset) return retval def update_dataset(self, dataset=None): if not (dataset and dataset._id): raise BadRequest('%s: Dataset either not provided or malformed.' % self.logging_name) self.clients.resource_registry.update(dataset) #@todo: Check to make sure retval is boolean log.debug('DM:update dataset: dataset_id: %s', dataset._id) return True def delete_dataset(self, dataset_id=''): assocs = self.clients.resource_registry.find_associations( subject=dataset_id, predicate=PRED.hasStream) for assoc in assocs: self.clients.resource_registry.delete_association(assoc) self.clients.resource_registry.delete(dataset_id) log.debug('DM:delete dataset: dataset_id: %s', dataset_id) if self._get_eoi_service_available(): self._remove_single_resource(dataset_id) def register_dataset(self, data_product_id=''): raise BadRequest( "register_dataset is no longer supported, please use create_catalog_entry in data product management" ) def add_dataset_window_to_complex(self, device_dataset_id='', window=None, site_dataset_id=''): ''' Adds target dataset to the complex coverage for the window specified ''' if window is None: raise BadRequest("Window must be specified") device_path = self._get_coverage_path(device_dataset_id) site_path = self._get_coverage_path(site_dataset_id) ccov = ComplexCoverage.load(site_path) ccov.append_reference_coverage( device_path, ReferenceCoverageExtents('', device_dataset_id, time_extents=window)) ccov.close() #-------- def add_parameter_to_dataset(self, parameter_context_id='', dataset_id=''): cov = self._get_simplex_coverage(dataset_id, mode='r+') parameter_ctx_res = self.read_parameter_context(parameter_context_id) pc = ParameterContext.load(parameter_ctx_res.parameter_context) cov.append_parameter(pc) cov.close() return True #-------- def add_stream(self, dataset_id='', stream_id=''): log.info('Adding stream %s to dataset %s', stream_id, dataset_id) validate_true( dataset_id and stream_id, 'Clients must provide both the dataset_id and stream_id') self.clients.resource_registry.create_association( subject=dataset_id, predicate=PRED.hasStream, object=stream_id) def remove_stream(self, dataset_id='', stream_id=''): log.info('Removing stream %s from dataset %s', stream_id, dataset_id) validate_true( dataset_id and stream_id, 'Clients must provide both the dataset_id and stream_id') assocs = self.clients.resource_registry.find_associations( subject=dataset_id, predicate=PRED.hasStream, object=stream_id) for assoc in assocs: self.clients.resource_registry.delete_association(assoc) #-------- def get_dataset_info(self, dataset_id=''): coverage = self._get_coverage(dataset_id, mode='r') return coverage.info def get_dataset_parameters(self, dataset_id=''): coverage = self._get_coverage(dataset_id, mode='r') return coverage.parameter_dictionary.dump() def get_dataset_length(self, dataset_id=''): coverage = self._get_coverage(dataset_id, mode='r') return coverage.num_timesteps #-------- @classmethod def numpy_walk(cls, obj): try: if np.isnan(obj): return {'__nan__': 0} except TypeError: pass except NotImplementedError: pass except ValueError: pass if isinstance(obj, np.number): return np.asscalar(obj) if isinstance(obj, np.dtype): return {'__np__': obj.str} if isinstance(obj, dict): if '__nan__' in obj and len(obj) == 1: return np.nan if '__np__' in obj and len(obj) == 1: return np.dtype(obj['__np__']) return {k: cls.numpy_walk(v) for k, v in obj.iteritems()} if isinstance(obj, list): return map(cls.numpy_walk, obj) if isinstance(obj, tuple): return tuple(map(cls.numpy_walk, obj)) return obj def create_parameter(self, parameter_context=None): """ Creates a parameter context using the IonObject """ context = self.get_coverage_parameter(parameter_context) parameter_context.parameter_context = self.numpy_walk(context.dump()) parameter_context_id, _ = self.clients.resource_registry.create( parameter_context) if parameter_context.parameter_function_id: self.read_parameter_function( parameter_context.parameter_function_id) self.clients.resource_registry.create_association( subject=parameter_context_id, predicate=PRED.hasParameterFunction, object=parameter_context.parameter_function_id) return parameter_context_id @classmethod def get_coverage_parameter(cls, parameter_context): """ Creates a Coverage Model based Parameter Context given the ParameterContext IonObject. Note: If the parameter is a parameter function and depends on dynamically created calibrations, this will fail. """ # Only CF and netCDF compliant variable names parameter_context.name = re.sub(r'[^a-zA-Z0-9_]', '_', parameter_context.name) from ion.services.dm.utility.types import TypesManager # The TypesManager does all the parsing and converting to the coverage model instances tm = TypesManager(None, {}, {}) # First thing to do is create the parameter type param_type = tm.get_parameter_type( parameter_context.parameter_type, parameter_context.value_encoding, parameter_context.code_report, parameter_context.parameter_function_id, parameter_context.parameter_function_map) # Ugh, I hate it but I did copy this section from # ion/processes/bootstrap/ion_loader.py context = ParameterContext(name=parameter_context.name, param_type=param_type) # Now copy over all the attrs context.uom = parameter_context.units try: if isinstance(context.uom, basestring): tm.get_unit(context.uom) except UdunitsError: log.warning('Parameter %s has invalid units: %s', parameter_context.name, context.uom) # Fill values can be a bit tricky... context.fill_value = tm.get_fill_value( parameter_context.fill_value, parameter_context.value_encoding, param_type) context.reference_urls = parameter_context.reference_urls context.internal_name = parameter_context.name context.display_name = parameter_context.display_name context.standard_name = parameter_context.standard_name context.ooi_short_name = parameter_context.ooi_short_name context.description = parameter_context.description context.precision = parameter_context.precision context.visible = parameter_context.visible return context def create_parameter_context(self, name='', parameter_context=None, description='', reference_urls=None, parameter_type='', internal_name='', value_encoding='', code_report='', units='', fill_value='', display_name='', parameter_function_id='', parameter_function_map='', standard_name='', ooi_short_name='', precision='', visible=True): validate_true(name, 'Name field may not be empty') validate_is_instance(parameter_context, dict, 'parameter_context field is not dictable.') name = re.sub(r'[^a-zA-Z0-9_]', '_', name) parameter_context = self.numpy_walk(parameter_context) parameter_context['name'] = name pc_res = ParameterContextResource(name=name, parameter_context=parameter_context, description=description) pc_res.reference_urls = reference_urls or [] pc_res.parameter_type = parameter_type pc_res.internal_name = internal_name or name pc_res.value_encoding = value_encoding pc_res.code_report = code_report or '' pc_res.units = units pc_res.fill_value = fill_value pc_res.display_name = display_name pc_res.parameter_function_id = parameter_function_id pc_res.parameter_function_map = parameter_function_map pc_res.standard_name = standard_name pc_res.ooi_short_name = ooi_short_name pc_res.precision = precision or '5' pc_res.visible = visible pc_id, ver = self.clients.resource_registry.create(pc_res) if parameter_function_id: self.read_parameter_function(parameter_function_id) self.clients.resource_registry.create_association( subject=pc_id, predicate=PRED.hasParameterFunction, object=parameter_function_id) return pc_id def read_parameter_context(self, parameter_context_id=''): res = self.clients.resource_registry.read(parameter_context_id) validate_is_instance(res, ParameterContextResource) res.parameter_context = self.numpy_walk(res.parameter_context) return res def delete_parameter_context(self, parameter_context_id=''): self.read_parameter_context(parameter_context_id) self.clients.resource_registry.delete(parameter_context_id) return True #-------- def read_parameter_context_by_name(self, name='', id_only=False): res, _ = self.clients.resource_registry.find_resources( restype=RT.ParameterContext, name=name, id_only=id_only) if not len(res): raise NotFound('Unable to locate context with name: %s' % name) retval = res[0] if not id_only: retval.parameter_context = self.numpy_walk( retval.parameter_context) return retval #-------- def create_parameter_function(self, parameter_function=None): validate_is_instance(parameter_function, ParameterFunctionResource) pf_id, ver = self.clients.resource_registry.create(parameter_function) return pf_id def read_parameter_function(self, parameter_function_id=''): res = self.clients.resource_registry.read(parameter_function_id) validate_is_instance(res, ParameterFunctionResource) return res def delete_parameter_function(self, parameter_function_id=''): self.read_parameter_function(parameter_function_id) self.clients.resource_registry.retire(parameter_function_id) return True @classmethod def get_coverage_function(self, parameter_function): func = None if parameter_function.function_type == PFT.PYTHON: func = PythonFunction(name=parameter_function.name, owner=parameter_function.owner, func_name=parameter_function.function, arg_list=parameter_function.args, kwarg_map=None, param_map=None, egg_uri=parameter_function.egg_uri) elif parameter_function.function_type == PFT.NUMEXPR: func = NumexprFunction(name=parameter_function.name, expression=parameter_function.function, arg_list=parameter_function.args) if not isinstance(func, AbstractFunction): raise Conflict("Incompatible parameter function loaded: %s" % parameter_function._id) return func #-------- def load_parameter_function(self, row): name = row['Name'] ftype = row['Function Type'] func_expr = row['Function'] owner = row['Owner'] args = ast.literal_eval(row['Args']) #kwargs = row['Kwargs'] descr = row['Description'] data_process_management = DataProcessManagementServiceProcessClient( self) function_type = None if ftype == 'PythonFunction': function_type = PFT.PYTHON elif ftype == 'NumexprFunction': function_type = PFT.NUMEXPR else: raise Conflict('Unsupported Function Type: %s' % ftype) parameter_function = ParameterFunctionResource( name=name, function=func_expr, function_type=function_type, owner=owner, args=args, description=descr) parameter_function.alt_ids = ['PRE:' + row['ID']] parameter_function_id = self.create_parameter_function( parameter_function) dpd = DataProcessDefinition() dpd.name = name dpd.description = 'Parameter Function Definition for %s' % name dpd.data_process_type = DataProcessTypeEnum.PARAMETER_FUNCTION dpd.parameters = args data_process_management.create_data_process_definition( dpd, parameter_function_id) return parameter_function_id #-------- def read_parameter_function_by_name(self, name='', id_only=False): res, _ = self.clients.resource_registry.find_resources( restype=RT.ParameterFunction, name=name, id_only=id_only) if not len(res): raise NotFound( 'Unable to locate parameter function with name: %s' % name) retval = res[0] retval.parameter_function = self.numpy_walk(retval.parameter_function) return retval #-------- def create_parameter_dictionary(self, name='', parameter_context_ids=None, temporal_context='', description=''): validate_true(name, 'Name field may not be empty.') parameter_context_ids = parameter_context_ids or [] pd_res = ParameterDictionaryResource(name=name, temporal_context=temporal_context, description=description) pd_res_id, ver = self.clients.resource_registry.create(pd_res) for pc_id in parameter_context_ids: self._link_pcr_to_pdr(pc_id, pd_res_id) return pd_res_id def read_parameter_dictionary(self, parameter_dictionary_id=''): res = self.clients.resource_registry.read(parameter_dictionary_id) validate_is_instance( res, ParameterDictionaryResource, 'Resource is not a valid ParameterDictionaryResource') return res def delete_parameter_dictionary(self, parameter_dictionary_id=''): self.read_parameter_dictionary(parameter_dictionary_id) self._unlink_pdr(parameter_dictionary_id) self.clients.resource_registry.delete(parameter_dictionary_id) return True #-------- def read_parameter_contexts(self, parameter_dictionary_id='', id_only=False): pcs, assocs = self.clients.resource_registry.find_objects( subject=parameter_dictionary_id, predicate=PRED.hasParameterContext, id_only=id_only) if not id_only: for pc in pcs: pc.parameter_context = self.numpy_walk(pc.parameter_context) return pcs def read_parameter_dictionary_by_name(self, name='', id_only=False): res, _ = self.clients.resource_registry.find_resources( restype=RT.ParameterDictionary, name=name, id_only=id_only) if not len(res): raise NotFound('Unable to locate dictionary with name: %s' % name) return res[0] #-------- def dataset_bounds(self, dataset_id='', parameters=None): self.read_dataset(dataset_id) # Validates proper dataset parameters = parameters or None try: doc = self.container.object_store.read_doc(dataset_id) except NotFound: return {} if parameters is not None: retval = {} for p in parameters: if p in doc['bounds']: retval[p] = doc['bounds'][p] return retval return doc['bounds'] def dataset_bounds_by_axis(self, dataset_id='', axis=None): bounds = self.dataset_bounds(dataset_id) if bounds and axis and axis in bounds: return bounds[axis] return {} def dataset_temporal_bounds(self, dataset_id): dataset = self.read_dataset(dataset_id) parameter_dictionary_id = self.clients.resource_registry.find_objects( dataset_id, PRED.hasParameterDictionary, id_only=True)[0][0] pdict = self._coverage_parameter_dictionary(parameter_dictionary_id) if not dataset: return {} temporal_parameter = pdict.temporal_parameter_name units = pdict.get_temporal_context().uom bounds = self.dataset_bounds(dataset_id) if not bounds: return {} bounds = bounds[temporal_parameter or 'time'] bounds = [TimeUtils.units_to_ts(units, i) for i in bounds] return bounds def dataset_extents(self, dataset_id='', parameters=None): self.read_dataset(dataset_id) # Validates proper dataset parameters = parameters or None try: doc = self.container.object_store.read_doc(dataset_id) except NotFound: return {} if parameters is not None: if isinstance(parameters, list): retval = {} for p in parameters: retval[p] = doc['extents'][p] return retval elif isinstance(parameters, basestring): return doc['extents'][parameters] return doc['extents'] def dataset_extents_by_axis(self, dataset_id='', axis=None): extents = self.dataset_extents(dataset_id) if extents and axis and axis in extents: return extents[axis] return {} def dataset_size(self, dataset_id='', in_bytes=False): self.read_dataset(dataset_id) # Validates proper dataset try: doc = self.container.object_store.read_doc(dataset_id) except NotFound: return 0. size = doc['size'] if not in_bytes: size = size / 1024. return size def dataset_latest(self, dataset_id=''): self.read_dataset(dataset_id) # Validates proper dataset try: doc = self.container.object_store.read_doc(dataset_id) except NotFound: return {} return doc['last_values'] #-------- @classmethod def get_parameter_context(cls, parameter_context_id=''): """ Preferred client-side class method for constructing a parameter context from a service call. """ dms_cli = DatasetManagementServiceClient() pc_res = dms_cli.read_parameter_context( parameter_context_id=parameter_context_id) pc = ParameterContext.load(pc_res.parameter_context) pc._identifier = pc_res._id return pc @classmethod def get_parameter_function(cls, parameter_function_id=''): """ Preferred client-side class method for constructing a parameter function """ dms_cli = DatasetManagementServiceClient() pf_res = dms_cli.read_parameter_function( parameter_function_id=parameter_function_id) pf = AbstractFunction.load(pf_res.parameter_function) pf._identifier = pf._id return pf @classmethod def get_parameter_context_by_name(cls, name=''): dms_cli = DatasetManagementServiceClient() pc_res = dms_cli.read_parameter_context_by_name(name=name, id_only=False) pc = ParameterContext.load(pc_res.parameter_context) pc._identifier = pc_res._id return pc @classmethod def get_parameter_dictionary(cls, parameter_dictionary_id=''): """ Class method to return a CoverageModel ParameterDictionary object from the ION Resources. The object is built from the associated parameter contexts. """ dms_cli = DatasetManagementServiceClient() pd = dms_cli.read_parameter_dictionary(parameter_dictionary_id) pcs = dms_cli.read_parameter_contexts( parameter_dictionary_id=parameter_dictionary_id, id_only=False) return cls.build_parameter_dictionary(pd, pcs) @classmethod def build_parameter_dictionary(cls, parameter_dictionary_obj, parameter_contexts): pdict = cls._merge_contexts([ ParameterContext.load(i.parameter_context) for i in parameter_contexts ], parameter_dictionary_obj.temporal_context) pdict._identifier = parameter_dictionary_obj._id return pdict def _coverage_parameter_dictionary(self, parameter_dictionary_id): pd = self.read_parameter_dictionary(parameter_dictionary_id) pcs = self.read_parameter_contexts(parameter_dictionary_id, id_only=False) parameter_dict = self._merge_contexts( [ParameterContext.load(i.parameter_context) for i in pcs], pd.temporal_context) return parameter_dict @classmethod def get_parameter_dictionary_by_name(cls, name=''): dms_cli = DatasetManagementServiceClient() pd_res = dms_cli.read_parameter_dictionary_by_name(name=name, id_only=True) return cls.get_parameter_dictionary(pd_res) #-------- def create_parameters_mult(self, parameter_function_list=None, parameter_context_list=None, parameter_dictionary_list=None, parameter_dictionary_assocs=None): pass #-------- def _link_pcr_to_pdr(self, pcr_id, pdr_id): self.clients.resource_registry.create_association( subject=pdr_id, predicate=PRED.hasParameterContext, object=pcr_id) def _unlink_pcr_to_pdr(self, pcr_id, pdr_id): assocs = self.clients.resource_registry.find_associations( subject=pdr_id, predicate=PRED.hasParameterContext, object=pcr_id, id_only=True) for assoc in assocs: self.clients.resource_registry.delete_association(assoc) def _unlink_pdr(self, pdr_id): objects, assocs = self.clients.resource_registry.find_objects( subject=pdr_id, predicate=PRED.hasParameterContext, id_only=True) for assoc in assocs: self.clients.resource_registry.delete_association(assoc) def _create_coverage(self, dataset_id, description, parameter_dict): #file_root = FileSystem.get_url(FS.CACHE,'datasets') temporal_domain, spatial_domain = time_series_domain() pdict = ParameterDictionary.load(parameter_dict) scov = self._create_simplex_coverage(dataset_id, pdict, spatial_domain, temporal_domain) #vcov = ViewCoverage(file_root, dataset_id, description or dataset_id, reference_coverage_location=scov.persistence_dir) scov.close() return scov def _create_view_coverage(self, dataset_id, description, parent_dataset_id): # As annoying as it is we need to load the view coverage belonging to parent dataset id and use the information # inside to build the new one... file_root = FileSystem.get_url(FS.CACHE, 'datasets') pscov = self._get_simplex_coverage(parent_dataset_id, mode='r') scov_location = pscov.persistence_dir pscov.close() vcov = ViewCoverage(file_root, dataset_id, description or dataset_id, reference_coverage_location=scov_location) return vcov @classmethod def _create_complex_coverage(cls, dataset_id, description, parameter_dict): pdict = ParameterDictionary.load(parameter_dict) file_root = FileSystem.get_url(FS.CACHE, 'datasets') ccov = ComplexCoverage( file_root, dataset_id, 'Complex Coverage for %s' % dataset_id, parameter_dictionary=pdict, complex_type=ComplexCoverageType.TEMPORAL_AGGREGATION) return ccov @classmethod def _create_simplex_coverage(cls, dataset_id, parameter_dictionary, spatial_domain, temporal_domain): file_root = FileSystem.get_url(FS.CACHE, 'datasets') scov = SimplexCoverage(file_root, dataset_id, 'Simplex Coverage for %s' % dataset_id, parameter_dictionary=parameter_dictionary, temporal_domain=temporal_domain, spatial_domain=spatial_domain) return scov @classmethod def _splice_coverage(cls, dataset_id, scov): file_root = FileSystem.get_url(FS.CACHE, 'datasets') vcov = cls._get_coverage(dataset_id, mode='a') scov_pth = scov.persistence_dir if isinstance(vcov.reference_coverage, SimplexCoverage): ccov = ComplexCoverage( file_root, uuid4().hex, 'Complex coverage for %s' % dataset_id, reference_coverage_locs=[ vcov.head_coverage_path, ], parameter_dictionary=ParameterDictionary(), complex_type=ComplexCoverageType.TEMPORAL_AGGREGATION) log.info('Creating Complex Coverage: %s', ccov.persistence_dir) ccov.append_reference_coverage(scov_pth) ccov_pth = ccov.persistence_dir ccov.close() vcov.replace_reference_coverage(ccov_pth) elif isinstance(vcov.reference_coverage, ComplexCoverage): log.info('Appending simplex coverage to complex coverage') #vcov.reference_coverage.append_reference_coverage(scov_pth) dir_path = vcov.reference_coverage.persistence_dir vcov.close() ccov = AbstractCoverage.load(dir_path, mode='a') ccov.append_reference_coverage(scov_pth) ccov.refresh() ccov.close() vcov.refresh() vcov.close() @classmethod def _save_coverage(cls, coverage): coverage.flush() @classmethod def _get_coverage(cls, dataset_id, mode='r'): file_root = FileSystem.get_url(FS.CACHE, 'datasets') coverage = AbstractCoverage.load(file_root, dataset_id, mode=mode) return coverage @classmethod def _get_nonview_coverage(cls, dataset_id, mode='r'): cov = cls._get_coverage(dataset_id, mode) if isinstance(cov, ViewCoverage): rcov = cov.reference_coverage pdir = rcov.persistence_dir rcov = None cov.close() cov = AbstractCoverage.load(pdir, mode=mode) return cov @classmethod def _get_simplex_coverage(cls, dataset_id, mode='r'): cov = cls._get_coverage(dataset_id, mode=mode) if isinstance(cov, SimplexCoverage): return cov if isinstance(cov, ViewCoverage): path = cov.head_coverage_path guid = os.path.basename(path) cov.close() return cls._get_simplex_coverage(guid, mode=mode) raise BadRequest('Unsupported coverage type found: %s' % type(cov)) @classmethod def _get_coverage_path(cls, dataset_id): file_root = FileSystem.get_url(FS.CACHE, 'datasets') return os.path.join(file_root, '%s' % dataset_id) @classmethod def _compare_pc(cls, pc1, pc2): if pc1: pc1 = ParameterContext.load(pc1) or {} if pc2: pc2 = ParameterContext.load(pc2) or {} if hasattr(pc1, 'lookup_value') or hasattr(pc2, 'lookup_value'): if hasattr(pc1, 'lookup_value') and hasattr(pc2, 'lookup_value'): return bool( pc1 == pc2) and pc1.document_key == pc2.document_key return False return bool(pc1 == pc2) @classmethod def _merge_contexts(cls, contexts, temporal): pdict = ParameterDictionary() for context in contexts: if context.name == temporal: context.axis = AxisTypeEnum.TIME pdict.add_context(context, is_temporal=True) else: pdict.add_context(context) return pdict def read_qc_table(self, obj_id): obj = self.container.object_store.read(obj_id) if '_type' in obj and obj['_type'] == 'QC': return obj else: raise BadRequest('obj_id %s not QC' % obj_id) def _create_single_resource(self, dataset_id, param_dict): ''' EOI Creates a foreign data table and a geoserver layer for the given dataset and parameter dictionary ''' self.resource_parser.create_single_resource(dataset_id, param_dict) def _remove_single_resource(self, dataset_id): ''' EOI Removes foreign data table and geoserver layer for the given dataset ''' self.resource_parser.remove_single_resource(dataset_id) def _get_eoi_service_available(self): ''' EOI Returns true if geoserver endpoint is running and verified by table loader process. Once a true is returned, the result is cached and the process is no longer queried ''' return self.resource_parser and self.resource_parser.get_eoi_service_available( )