def delete_dataset(self, agent_instance_id, resource_id): """Deletes dataset and coverage files for all of a device's data products""" res_obj = self.rr.read(resource_id) dpms = DataProductManagementServiceProcessClient(process=self) # Find data products from device id count_ds = 0 dp_objs, _ = self.rr.find_objects(resource_id, PRED.hasOutputProduct, RT.DataProduct, id_only=False) for dp_obj in dp_objs: if dpms.is_persisted(dp_obj._id, headers=self._get_system_actor_headers()): if self.force: log.warn("DataProduct %s '%s' is currently persisted - continuing", dp_obj._id, dp_obj.name) else: raise BadRequest("DataProduct %s '%s' is currently persisted. Use force=True to ignore", dp_obj._id, dp_obj.name) ds_objs, _ = self.rr.find_objects(dp_obj._id, PRED.hasDataset, RT.Dataset, id_only=False) for ds_obj in ds_objs: # Delete coverage cov_path = DatasetManagementService._get_coverage_path(ds_obj._id) if os.path.exists(cov_path): log.info("Removing coverage tree at %s", cov_path) shutil.rmtree(cov_path) else: log.warn("Coverage path does not exist %s" % cov_path) # Delete Dataset and associations self.rr.delete(ds_obj._id) count_ds += 1 log.info("Datasets and coverages deleted for device %s '%s': %s", resource_id, res_obj.name, count_ds)
def delete_dataset(self, agent_instance_id, resource_id): res_obj = self.rr.read(resource_id) dpms = DataProductManagementServiceProcessClient(process=self) # Find data products from device id count_ds = 0 dp_objs, _ = self.rr.find_objects(resource_id, PRED.hasOutputProduct, RT.DataProduct, id_only=False) for dp_obj in dp_objs: if dpms.is_persisted(dp_obj._id): raise BadRequest("DataProduct %s '%s' is currently persisted", dp_obj._id, dp_obj.name) ds_objs, _ = self.rr.find_objects(dp_obj._id, PRED.hasDataset, RT.Dataset, id_only=False) for ds_obj in ds_objs: # Delete coverage cov_path = DatasetManagementService._get_coverage_path(ds_obj._id) if os.path.exists(cov_path): log.info("Removing coverage tree at %s", cov_path) shutil.rmtree(cov_path) else: raise OSError("Coverage path does not exist %s" % cov_path) # Delete Dataset and associations self.rr.delete(ds_obj._id) count_ds += 1 log.info("Datasets and coverages deleted for device %s '%s': %s", resource_id, res_obj.name, count_ds)
def _set_calibration_for_data_product(self, dp_obj, dev_cfg): from ion.util.direct_coverage_utils import DirectCoverageAccess from coverage_model import SparseConstantType log.debug("Setting calibration for data product '%s'", dp_obj.name) dataset_ids, _ = self.rr.find_objects(dp_obj, PRED.hasDataset, id_only=True) publisher = EventPublisher(OT.InformationContentModifiedEvent) if not dataset_ids: data_product_management = DataProductManagementServiceProcessClient(process=self) log.debug(" Creating dataset for data product %s", dp_obj.name) data_product_management.create_dataset_for_data_product(dp_obj._id) dataset_ids, _ = self.rr.find_objects(dp_obj, PRED.hasDataset, id_only=True) if not dataset_ids: raise NotFound('No datasets were found for this data product, ensure that it was created') for dataset_id in dataset_ids: # Synchronize with ingestion with DirectCoverageAccess() as dca: cov = dca.get_editable_coverage(dataset_id) # Iterate over the calibrations for cal_name, contents in dev_cfg.iteritems(): if cal_name in cov.list_parameters() and isinstance(cov.get_parameter_context(cal_name).param_type, SparseConstantType): value = float(contents['value']) log.info(' Updating Calibrations for %s in %s', cal_name, dataset_id) cov.set_parameter_values(cal_name, value) else: log.warn(" Calibration %s not found in dataset", cal_name) publisher.publish_event(origin=dataset_id, description="Calibrations Updated") publisher.close() log.info("Calibration set for data product '%s' in %s coverages", dp_obj.name, len(dataset_ids))
def delete_all_data(self, agent_instance_id, resource_id): # Delete Dataset and coverage for all original DataProducts self.delete_dataset(agent_instance_id, resource_id) res_obj = self.rr.read(resource_id) # Find parsed data product from device id dpms = DataProductManagementServiceProcessClient(process=self) count_dp, count_sd, count_st = 0, 0, 0 dp_objs, _ = self.rr.find_subjects(RT.DataProduct, PRED.hasSource, resource_id, id_only=False) for dp_obj in dp_objs: if dpms.is_persisted(dp_obj._id, headers=self._get_system_actor_headers()): if self.force: log.warn("DataProduct %s '%s' is currently persisted - continuing", dp_obj._id, dp_obj.name) else: raise BadRequest("DataProduct %s '%s' is currently persisted. Use force=True to ignore", dp_obj._id, dp_obj.name) # Find and delete Stream st_objs, _ = self.rr.find_objects(dp_obj._id, PRED.hasStream, RT.Stream, id_only=False) for st_obj in st_objs: self.rr.delete(st_obj._id) count_st += 1 # Find and delete StreamDefinition sd_objs, _ = self.rr.find_objects(dp_obj._id, PRED.hasStreamDefinition, RT.StreamDefinition, id_only=False) for sd_obj in sd_objs: self.rr.delete(sd_obj._id) count_sd += 1 # Delete DataProduct self.rr.delete(dp_obj._id) count_dp += 1 log.info("Data resources deleted for device %s '%s': %s DataProduct, %s StreamDefinition, %s Stream", resource_id, res_obj.name, count_dp, count_sd, count_st)
def create_dataset(self, agent_instance_id, resource_id): # Find hasOutputProduct DataProducts # Execute call to create Dataset and coverage res_obj = self.rr.read(resource_id) dpms = DataProductManagementServiceProcessClient(process=self) # Find data products from device id dp_objs, _ = self.rr.find_objects(resource_id, PRED.hasOutputProduct, RT.DataProduct, id_only=False) for dp_obj in dp_objs: dpms.create_dataset_for_data_product(dp_obj._id) log.info("Checked datasets for device %s '%s': %s", resource_id, res_obj.name, len(dp_objs))
def suspend_persistence(self, agent_instance_id, resource_id): dpms = DataProductManagementServiceProcessClient(process=self) dp_objs, _ = self.rr.find_objects(resource_id, PRED.hasOutputProduct, id_only=False) for dp in dp_objs: try: log.info("Suspending persistence for '%s'", dp.name) dpms.suspend_data_product_persistence(dp._id, headers=self._get_system_actor_headers(), timeout=self.timeout) except Exception: self._log_error(agent_instance_id, resource_id, logexc=True, msg="Could not suspend persistence for dp_id=%s" % (dp._id)) if self.autoclean: self._cleanup_persistence(dp)
def suspend_persistence(self, agent_instance_id, resource_id): if not agent_instance_id or not resource_id: log.warn("Could not %s agent %s for device %s", self.op, agent_instance_id, resource_id) return dpms = DataProductManagementServiceProcessClient(process=self) dp_objs, _ = self.rr.find_objects(resource_id, PRED.hasOutputProduct, id_only=False) for dp in dp_objs: try: log.info("Suspending persistence for '%s'", dp.name) dpms.suspend_data_product_persistence(dp._id) except Exception: log.exception("Could not suspend persistence")
def activate_persistence(self, agent_instance_id, resource_id): if not resource_id: log.warn("Could not %s for device %s", self.op, resource_id) return dpms = DataProductManagementServiceProcessClient(process=self) dp_objs, _ = self.rr.find_objects(resource_id, PRED.hasOutputProduct, id_only=False) for dp in dp_objs: try: log.info("Activating persistence for '%s'", dp.name) dpms.activate_data_product_persistence(dp._id, headers=self._get_system_actor_headers(), timeout=self.timeout) except Exception: self._log_error(agent_instance_id, resource_id, logexc=True, msg="Could not activate persistence for dp_id=%s" % (dp._id))
def seed_gov(container, process=FakeProcess()): dp_client = DataProductManagementServiceProcessClient(node=container.node, process=process) dp_obj = IonObject(RT.DataProduct, name='DataProd1', description='some new dp') dp_client.create_data_product(dp_obj) dp_obj = IonObject(RT.DataProduct, name='DataProd2', description='and of course another new dp') dp_client.create_data_product(dp_obj,) dp_obj = IonObject(RT.DataProduct, name='DataProd3', description='yet another new dp') dp_client.create_data_product(dp_obj,) log.debug('Data Products') dp_list = dp_client.find_data_products() for dp_obj in dp_list: log.debug( str(dp_obj)) ims_client = InstrumentManagementServiceProcessClient(node=container.node, process=process) ia_obj = IonObject(RT.InstrumentAgent, name='Instrument Agent1', description='The first Instrument Agent') ims_client.create_instrument_agent(ia_obj) ia_obj = IonObject(RT.InstrumentAgent, name='Instrument Agent2', description='The second Instrument Agent') ims_client.create_instrument_agent(ia_obj) log.debug( 'Instrument Agents') ia_list = ims_client.find_instrument_agents() for ia_obj in ia_list: log.debug( str(ia_obj)) org_client = OrgManagementServiceProcessClient(node=container.node, process=process) ion_org = org_client.find_org() policy_client = PolicyManagementServiceProcessClient(node=container.node, process=process) role_obj = IonObject(RT.UserRole, name='Instrument Operator', description='Users assigned to this role are instrument operators') role_id = policy_client.create_role(role_obj) org_client.add_user_role(ion_org._id, role_id) try: role_id = policy_client.create_role(role_obj) org_client.add_user_role(ion_org._id, role_id) except Exception, e: log.info("This should fail") log.info(e.message)
def setUp(self): # Start container logging.disable(logging.ERROR) self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # simulate preloading preload_ion_params(self.container) logging.disable(logging.NOTSET) #Instantiate a process to represent the test process=VisualizationServiceTestProcess() # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceProcessClient(node=self.container.node, process=process) self.damsclient = DataAcquisitionManagementServiceProcessClient(node=self.container.node, process=process) self.pubsubclient = PubsubManagementServiceProcessClient(node=self.container.node, process=process) self.ingestclient = IngestionManagementServiceProcessClient(node=self.container.node, process=process) self.imsclient = InstrumentManagementServiceProcessClient(node=self.container.node, process=process) self.dataproductclient = DataProductManagementServiceProcessClient(node=self.container.node, process=process) self.dataprocessclient = DataProcessManagementServiceProcessClient(node=self.container.node, process=process) self.datasetclient = DatasetManagementServiceProcessClient(node=self.container.node, process=process) self.workflowclient = WorkflowManagementServiceProcessClient(node=self.container.node, process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient(node=self.container.node, process=process) self.data_retriever = DataRetrieverServiceProcessClient(node=self.container.node, process=process) self.vis_client = VisualizationServiceProcessClient(node=self.container.node, process=process) self.ctd_stream_def = SBE37_CDM_stream_definition()
def activate_persistence(self, agent_instance_id, resource_id): dpms = DataProductManagementServiceProcessClient(process=self) dp_objs, _ = self.rr.find_objects(resource_id, PRED.hasOutputProduct, id_only=False) for dp in dp_objs: try: if dpms.is_persisted(dp._id, headers=self._get_system_actor_headers()): if self.force: log.warn("DataProduct %s '%s' is currently persisted - continuing", dp._id, dp.name) else: log.warn("DataProduct %s '%s' is currently persisted", dp._id, dp.name) continue log.info("Activating persistence for '%s'", dp.name) dpms.activate_data_product_persistence(dp._id, headers=self._get_system_actor_headers(), timeout=self.timeout) except Exception: self._log_error(agent_instance_id, resource_id, logexc=True, msg="Could not activate persistence for dp_id=%s" % (dp._id))
def on_start(self): ImmediateProcess.on_start(self) # necessary arguments, passed in via configuration kwarg to schedule_process. process namespace to avoid collisions fuc_id = self.CFG.get_safe('process.fuc_id', None) # FileUploadContext ID # Clients self.object_store = self.container.object_store self.resource_registry = self.container.resource_registry self.event_publisher = EventPublisher(OT.ResetQCEvent) self.data_product_management = DataProductManagementServiceProcessClient( process=self) self.create_map() # run process if fuc_id: self.process(fuc_id) # cleanup self.event_publisher.close()
def on_start(self): ImmediateProcess.on_start(self) # necessary arguments, passed in via configuration kwarg to schedule_process. process namespace to avoid collisions fuc_id = self.CFG.get_safe('process.fuc_id',None) # FileUploadContext ID # Clients self.object_store = self.container.object_store self.resource_registry = self.container.resource_registry self.event_publisher = EventPublisher(OT.ResetQCEvent) self.data_product_management = DataProductManagementServiceProcessClient(process=self) self.create_map() # run process if fuc_id: self.process(fuc_id) # cleanup self.event_publisher.close()
class UploadCalibrationProcessing(ImmediateProcess): """ Upload Calibration Processing Process This process provides the capability to ION clients and operators to process uploaded calibration coefficients to calibrate data products. This parameters that this process accepts as configurations are: - fuc_id: The FileUploadContext identifier, required, stores where the file was written """ def on_start(self): ImmediateProcess.on_start(self) # necessary arguments, passed in via configuration kwarg to schedule_process. process namespace to avoid collisions fuc_id = self.CFG.get_safe('process.fuc_id',None) # FileUploadContext ID # Clients self.object_store = self.container.object_store self.resource_registry = self.container.resource_registry self.event_publisher = EventPublisher(OT.ResetQCEvent) self.data_product_management = DataProductManagementServiceProcessClient(process=self) self.create_map() # run process if fuc_id: self.process(fuc_id) # cleanup self.event_publisher.close() def process(self,fuc_id): # get the Object (dict) containing details of the uploaded file fuc = self.object_store.read(fuc_id) if fuc['filetype'] == 'ZIP': raise BadRequest("ZIP format not determined by project scientists yet (2014-04-21)") #self.process_zip(fuc) else: self.process_csv(fuc) def create_map(self): ''' Creates a map from property numbers to datasets ''' self.property_map = {} for instrument_device in self.resource_registry.find_resources(restype=RT.InstrumentDevice)[0]: if instrument_device.ooi_property_number: self.property_map[instrument_device.ooi_property_number] = self.data_products_for_device(instrument_device) def data_products_for_device(self, device): data_products, _ = self.resource_registry.find_objects(device, PRED.hasOutputProduct, id_only=True) return data_products def dataset_for_data_product(self, data_product): datasets, _ = self.resource_registry.find_objects(data_product, PRED.hasDataset, id_only=True) return datasets[0] def do_something_with_the_update(self, updates): for property_no, calibration_update in updates.iteritems(): # Check to see if we even have an instrument with this property number if property_no not in self.property_map: continue # Get the data product listings for this instrument data_products = self.property_map[property_no] # Go through each data product and update the data IF # - There is a set of parameters that match those in the calibration for data_product in data_products: self.update_data_product(data_product, calibration_update) def update_data_product(self, data_product, calibration_update): parameters = [p.name for p in self.data_product_management.get_data_product_parameters(data_product)] dataset_updates = [] for cal_name in calibration_update.iterkeys(): if cal_name in parameters: dataset_id = self.dataset_for_data_product(data_product) dataset_updates.append(dataset_id) for dataset in dataset_updates: self.apply_to_dataset(dataset, calibration_update) def apply_to_dataset(self, dataset, calibration_update): cov = DatasetManagementService._get_coverage(dataset, mode='r+') try: self.set_sparse_values(cov, calibration_update) self.publish_calibration_event(dataset, calibration_update.keys()) finally: cov.close() def set_sparse_values(self, cov, calibration_update): for calibration_name, updates in calibration_update.iteritems(): if calibration_name not in cov.list_parameters(): continue for update in updates: np_dict = {} self.check_units(cov, calibration_name, update['units']) start_date = self.ntp_from_iso(update['start_date']) np_dict[calibration_name] = ConstantOverTime(calibration_name, update['value'], time_start=start_date) cov.set_parameter_values(np_dict) def check_units(self, cov, calibration_name, units): pass def publish_calibration_event(self, dataset, calibrations): publisher = EventPublisher(OT.DatasetCalibrationEvent) publisher.publish_event(origin=dataset, calibrations=calibrations) def ntp_from_iso(self, iso): return TimeUtils.ntp_from_iso(iso) def process_csv(self, fuc): # CSV file open here csv_filename = fuc.get('path', None) if csv_filename is None: raise BadRequest("uploaded file has no path") # keep track of the number of calibrations we actually process nupdates = 0 updates = {} # keys are reference_designators, use to update object store after parsing CSV with open(csv_filename, 'rb') as csvfile: # eliminate blank lines csvfile = (row for row in csvfile if len(row.strip()) > 0) # eliminate commented lines csvfile = (row for row in csvfile if not row.startswith('#')) # open CSV reader csv_reader = csv.reader(csvfile, delimiter=',') # skip commented lines # iterate the rows returned by csv.reader for row in csv_reader: if len(row) != 6: log.warn("invalid calibration line %s" % ','.join(row)) continue try: ipn = row[0] # instrument_property_number name = row[1] # calibration_name value = float(row[2]) # calibration_value units = row[3] description = row[4] # description start_date = row[5] # start_date TODO date object? d = { 'value':value, 'units':units, 'description':description, 'start_date':start_date } except ValueError as e: continue #TODO error message? or just skip? # get ipn key if ipn not in updates: updates[ipn] = {} # initialize empty array if name not in updates[ipn]: updates[ipn][name] = [] # will be array of dicts updates[ipn][name].append(d) nupdates = nupdates + 1 self.do_something_with_the_update(updates) # insert the updates into object store self.update_object_store(updates) # update FileUploadContext object (change status to complete) fuc['status'] = 'UploadCalibrationProcessing process complete - %d updates added to object store' % nupdates self.object_store.update_doc(fuc) # remove uploaded file try: os.remove(csv_filename) except OSError: pass # TODO take action to get this removed def process_zip(self,fuc): pass def update_object_store(self, updates): '''inserts the updates into object store''' for i in updates: # loops the instrument_property_number(ipn) in the updates object try: # if i exists in object_store, read it ipn = self.object_store.read(i) #TODO: what will this be? except: # if does not yet exist in object_store, create it (can't use update_doc because need to set id) ipn = self.object_store.create_doc({'_type':'CALIBRATION'},i) # CAUTION: this returns a tuple, not a dict like read() returns ipn = self.object_store.read(i) # read so we have a dict like we expect # merge all from updates[i] into dict destined for the object_store (ipn) for name in updates[i]: # loops the calibration_names under each IPN in updates #TODO: if name not initialized, will append work? if so, can use same op for both if name not in ipn: # if name doesn't exist, we can just add the entire object (dict of lists) ipn[name] = updates[i][name] else: # if it does, we need to append to each of the lists ipn[name].append(updates[i][name]) # append the list from updates # store updated ipn keyed object in object_store (should overwrite full object, contains all previous too) self.object_store.update_doc(ipn) # publish ResetQCEvent event (one for each instrument_property_number [AKA ipn]) self.event_publisher.publish_event(origin=i)
try: org_client.revoke_role(ion_org._id, user._id, 'INSTRUMENT_OPERATOR', headers={'ion-actor-id': user._id, 'ion-actor-roles': header_roles }) except Exception, e: log.info('This revoke role should be denied:' + e.message) header_roles = get_role_message_headers(org_client.find_all_roles_by_user(system_actor._id)) org_client.revoke_role(ion_org._id, user._id, 'INSTRUMENT_OPERATOR', headers={'ion-actor-id': system_actor._id, 'ion-actor-roles': header_roles }) roles = org_client.find_roles_by_user(ion_org._id, user._id) for r in roles: log.info('User UserRole: ' +str(r)) dp_client = DataProductManagementServiceProcessClient(node=container.node, process=process) dp_obj = IonObject(RT.DataProduct, name='DataProd1', description='some new dp') dp_client.create_data_product(dp_obj, headers={'ion-actor-id': system_actor._id}) dp_obj = IonObject(RT.DataProduct, name='DataProd2', description='and of course another new dp') dp_client.create_data_product(dp_obj, headers={'ion-actor-id': system_actor._id}) dp_obj = IonObject(RT.DataProduct, name='DataProd3', description='yet another new dp')
class UploadCalibrationProcessing(ImmediateProcess): """ Upload Calibration Processing Process This process provides the capability to ION clients and operators to process uploaded calibration coefficients to calibrate data products. This parameters that this process accepts as configurations are: - fuc_id: The FileUploadContext identifier, required, stores where the file was written """ def on_start(self): ImmediateProcess.on_start(self) # necessary arguments, passed in via configuration kwarg to schedule_process. process namespace to avoid collisions fuc_id = self.CFG.get_safe('process.fuc_id', None) # FileUploadContext ID # Clients self.object_store = self.container.object_store self.resource_registry = self.container.resource_registry self.event_publisher = EventPublisher(OT.ResetQCEvent) self.data_product_management = DataProductManagementServiceProcessClient( process=self) self.create_map() # run process if fuc_id: self.process(fuc_id) # cleanup self.event_publisher.close() def process(self, fuc_id): # get the Object (dict) containing details of the uploaded file fuc = self.object_store.read(fuc_id) if fuc['filetype'] == 'ZIP': raise BadRequest( "ZIP format not determined by project scientists yet (2014-04-21)" ) #self.process_zip(fuc) else: self.process_csv(fuc) def create_map(self): ''' Creates a map from property numbers to datasets ''' self.property_map = {} for instrument_device in self.resource_registry.find_resources( restype=RT.InstrumentDevice)[0]: if instrument_device.ooi_property_number: self.property_map[ instrument_device. ooi_property_number] = self.data_products_for_device( instrument_device) def data_products_for_device(self, device): data_products, _ = self.resource_registry.find_objects( device, PRED.hasOutputProduct, id_only=True) return data_products def dataset_for_data_product(self, data_product): datasets, _ = self.resource_registry.find_objects(data_product, PRED.hasDataset, id_only=True) return datasets[0] def do_something_with_the_update(self, updates): for property_no, calibration_update in updates.iteritems(): # Check to see if we even have an instrument with this property number if property_no not in self.property_map: continue # Get the data product listings for this instrument data_products = self.property_map[property_no] # Go through each data product and update the data IF # - There is a set of parameters that match those in the calibration for data_product in data_products: self.update_data_product(data_product, calibration_update) def update_data_product(self, data_product, calibration_update): parameters = [ p.name for p in self.data_product_management.get_data_product_parameters( data_product) ] dataset_updates = [] for cal_name in calibration_update.iterkeys(): if cal_name in parameters: dataset_id = self.dataset_for_data_product(data_product) dataset_updates.append(dataset_id) for dataset in dataset_updates: self.apply_to_dataset(dataset, calibration_update) def apply_to_dataset(self, dataset, calibration_update): cov = DatasetManagementService._get_coverage(dataset, mode='r+') try: self.set_sparse_values(cov, calibration_update) self.publish_calibration_event(dataset, calibration_update.keys()) finally: cov.close() def set_sparse_values(self, cov, calibration_update): for calibration_name, updates in calibration_update.iteritems(): if calibration_name not in cov.list_parameters(): continue for update in updates: np_dict = {} self.check_units(cov, calibration_name, update['units']) start_date = self.ntp_from_iso(update['start_date']) np_dict[calibration_name] = ConstantOverTime( calibration_name, update['value'], time_start=start_date) cov.set_parameter_values(np_dict) def check_units(self, cov, calibration_name, units): pass def publish_calibration_event(self, dataset, calibrations): publisher = EventPublisher(OT.DatasetCalibrationEvent) publisher.publish_event(origin=dataset, calibrations=calibrations) def ntp_from_iso(self, iso): return TimeUtils.ntp_from_iso(iso) def process_csv(self, fuc): # CSV file open here csv_filename = fuc.get('path', None) if csv_filename is None: raise BadRequest("uploaded file has no path") # keep track of the number of calibrations we actually process nupdates = 0 updates = { } # keys are reference_designators, use to update object store after parsing CSV with open(csv_filename, 'rb') as csvfile: # eliminate blank lines csvfile = (row for row in csvfile if len(row.strip()) > 0) # eliminate commented lines csvfile = (row for row in csvfile if not row.startswith('#')) # open CSV reader csv_reader = csv.reader(csvfile, delimiter=',') # skip commented lines # iterate the rows returned by csv.reader for row in csv_reader: if len(row) != 6: log.warn("invalid calibration line %s" % ','.join(row)) continue try: ipn = row[0] # instrument_property_number name = row[1] # calibration_name value = float(row[2]) # calibration_value units = row[3] description = row[4] # description start_date = row[5] # start_date TODO date object? d = { 'value': value, 'units': units, 'description': description, 'start_date': start_date } except ValueError as e: continue #TODO error message? or just skip? # get ipn key if ipn not in updates: updates[ipn] = {} # initialize empty array if name not in updates[ipn]: updates[ipn][name] = [] # will be array of dicts updates[ipn][name].append(d) nupdates = nupdates + 1 self.do_something_with_the_update(updates) # insert the updates into object store self.update_object_store(updates) # update FileUploadContext object (change status to complete) fuc['status'] = 'UploadCalibrationProcessing process complete - %d updates added to object store' % nupdates self.object_store.update_doc(fuc) # remove uploaded file try: os.remove(csv_filename) except OSError: pass # TODO take action to get this removed def process_zip(self, fuc): pass def update_object_store(self, updates): '''inserts the updates into object store''' for i in updates: # loops the instrument_property_number(ipn) in the updates object try: # if i exists in object_store, read it ipn = self.object_store.read(i) #TODO: what will this be? except: # if does not yet exist in object_store, create it (can't use update_doc because need to set id) ipn = self.object_store.create_doc( {'_type': 'CALIBRATION'}, i ) # CAUTION: this returns a tuple, not a dict like read() returns ipn = self.object_store.read( i) # read so we have a dict like we expect # merge all from updates[i] into dict destined for the object_store (ipn) for name in updates[ i]: # loops the calibration_names under each IPN in updates #TODO: if name not initialized, will append work? if so, can use same op for both if name not in ipn: # if name doesn't exist, we can just add the entire object (dict of lists) ipn[name] = updates[i][name] else: # if it does, we need to append to each of the lists ipn[name].append( updates[i][name]) # append the list from updates # store updated ipn keyed object in object_store (should overwrite full object, contains all previous too) self.object_store.update_doc(ipn) # publish ResetQCEvent event (one for each instrument_property_number [AKA ipn]) self.event_publisher.publish_event(origin=i)
header_roles = get_role_message_headers( org_client.find_all_roles_by_user(system_actor._id)) org_client.revoke_role(ion_org._id, user._id, 'INSTRUMENT_OPERATOR', headers={ 'ion-actor-id': system_actor._id, 'ion-actor-roles': header_roles }) roles = org_client.find_roles_by_user(ion_org._id, user._id) for r in roles: log.info('User UserRole: ' + str(r)) dp_client = DataProductManagementServiceProcessClient(node=container.node, process=process) dp_obj = IonObject(RT.DataProduct, name='DataProd1', description='some new dp') dp_client.create_data_product(dp_obj, headers={'ion-actor-id': system_actor._id}) dp_obj = IonObject(RT.DataProduct, name='DataProd2', description='and of course another new dp') dp_client.create_data_product(dp_obj, headers={'ion-actor-id': system_actor._id})