def delete_dataset(self, agent_instance_id, resource_id):
"""Deletes dataset and coverage files for all of a device's data products"""
res_obj = self.rr.read(resource_id)
dpms = DataProductManagementServiceProcessClient(process=self)
# Find data products from device id
count_ds = 0
dp_objs, _ = self.rr.find_objects(resource_id, PRED.hasOutputProduct, RT.DataProduct, id_only=False)
for dp_obj in dp_objs:
if dpms.is_persisted(dp_obj._id, headers=self._get_system_actor_headers()):
if self.force:
log.warn("DataProduct %s '%s' is currently persisted - continuing", dp_obj._id, dp_obj.name)
else:
raise BadRequest("DataProduct %s '%s' is currently persisted. Use force=True to ignore", dp_obj._id, dp_obj.name)
ds_objs, _ = self.rr.find_objects(dp_obj._id, PRED.hasDataset, RT.Dataset, id_only=False)
for ds_obj in ds_objs:
# Delete coverage
cov_path = DatasetManagementService._get_coverage_path(ds_obj._id)
if os.path.exists(cov_path):
log.info("Removing coverage tree at %s", cov_path)
shutil.rmtree(cov_path)
else:
log.warn("Coverage path does not exist %s" % cov_path)
# Delete Dataset and associations
self.rr.delete(ds_obj._id)
count_ds += 1
log.info("Datasets and coverages deleted for device %s '%s': %s", resource_id, res_obj.name, count_ds)
def delete_dataset(self, agent_instance_id, resource_id):
res_obj = self.rr.read(resource_id)
dpms = DataProductManagementServiceProcessClient(process=self)
# Find data products from device id
count_ds = 0
dp_objs, _ = self.rr.find_objects(resource_id, PRED.hasOutputProduct, RT.DataProduct, id_only=False)
for dp_obj in dp_objs:
if dpms.is_persisted(dp_obj._id):
raise BadRequest("DataProduct %s '%s' is currently persisted", dp_obj._id, dp_obj.name)
ds_objs, _ = self.rr.find_objects(dp_obj._id, PRED.hasDataset, RT.Dataset, id_only=False)
for ds_obj in ds_objs:
# Delete coverage
cov_path = DatasetManagementService._get_coverage_path(ds_obj._id)
if os.path.exists(cov_path):
log.info("Removing coverage tree at %s", cov_path)
shutil.rmtree(cov_path)
else:
raise OSError("Coverage path does not exist %s" % cov_path)
# Delete Dataset and associations
self.rr.delete(ds_obj._id)
count_ds += 1
log.info("Datasets and coverages deleted for device %s '%s': %s", resource_id, res_obj.name, count_ds)
def _set_calibration_for_data_product(self, dp_obj, dev_cfg):
from ion.util.direct_coverage_utils import DirectCoverageAccess
from coverage_model import SparseConstantType
log.debug("Setting calibration for data product '%s'", dp_obj.name)
dataset_ids, _ = self.rr.find_objects(dp_obj, PRED.hasDataset, id_only=True)
publisher = EventPublisher(OT.InformationContentModifiedEvent)
if not dataset_ids:
data_product_management = DataProductManagementServiceProcessClient(process=self)
log.debug(" Creating dataset for data product %s", dp_obj.name)
data_product_management.create_dataset_for_data_product(dp_obj._id)
dataset_ids, _ = self.rr.find_objects(dp_obj, PRED.hasDataset, id_only=True)
if not dataset_ids:
raise NotFound('No datasets were found for this data product, ensure that it was created')
for dataset_id in dataset_ids:
# Synchronize with ingestion
with DirectCoverageAccess() as dca:
cov = dca.get_editable_coverage(dataset_id)
# Iterate over the calibrations
for cal_name, contents in dev_cfg.iteritems():
if cal_name in cov.list_parameters() and isinstance(cov.get_parameter_context(cal_name).param_type, SparseConstantType):
value = float(contents['value'])
log.info(' Updating Calibrations for %s in %s', cal_name, dataset_id)
cov.set_parameter_values(cal_name, value)
else:
log.warn(" Calibration %s not found in dataset", cal_name)
publisher.publish_event(origin=dataset_id, description="Calibrations Updated")
publisher.close()
log.info("Calibration set for data product '%s' in %s coverages", dp_obj.name, len(dataset_ids))
def delete_all_data(self, agent_instance_id, resource_id):
# Delete Dataset and coverage for all original DataProducts
self.delete_dataset(agent_instance_id, resource_id)
res_obj = self.rr.read(resource_id)
# Find parsed data product from device id
dpms = DataProductManagementServiceProcessClient(process=self)
count_dp, count_sd, count_st = 0, 0, 0
dp_objs, _ = self.rr.find_subjects(RT.DataProduct, PRED.hasSource, resource_id, id_only=False)
for dp_obj in dp_objs:
if dpms.is_persisted(dp_obj._id, headers=self._get_system_actor_headers()):
if self.force:
log.warn("DataProduct %s '%s' is currently persisted - continuing", dp_obj._id, dp_obj.name)
else:
raise BadRequest("DataProduct %s '%s' is currently persisted. Use force=True to ignore", dp_obj._id, dp_obj.name)
# Find and delete Stream
st_objs, _ = self.rr.find_objects(dp_obj._id, PRED.hasStream, RT.Stream, id_only=False)
for st_obj in st_objs:
self.rr.delete(st_obj._id)
count_st += 1
# Find and delete StreamDefinition
sd_objs, _ = self.rr.find_objects(dp_obj._id, PRED.hasStreamDefinition, RT.StreamDefinition, id_only=False)
for sd_obj in sd_objs:
self.rr.delete(sd_obj._id)
count_sd += 1
# Delete DataProduct
self.rr.delete(dp_obj._id)
count_dp += 1
log.info("Data resources deleted for device %s '%s': %s DataProduct, %s StreamDefinition, %s Stream",
resource_id, res_obj.name, count_dp, count_sd, count_st)
def create_dataset(self, agent_instance_id, resource_id):
# Find hasOutputProduct DataProducts
# Execute call to create Dataset and coverage
res_obj = self.rr.read(resource_id)
dpms = DataProductManagementServiceProcessClient(process=self)
# Find data products from device id
dp_objs, _ = self.rr.find_objects(resource_id, PRED.hasOutputProduct, RT.DataProduct, id_only=False)
for dp_obj in dp_objs:
dpms.create_dataset_for_data_product(dp_obj._id)
log.info("Checked datasets for device %s '%s': %s", resource_id, res_obj.name, len(dp_objs))
def suspend_persistence(self, agent_instance_id, resource_id):
dpms = DataProductManagementServiceProcessClient(process=self)
dp_objs, _ = self.rr.find_objects(resource_id, PRED.hasOutputProduct, id_only=False)
for dp in dp_objs:
try:
log.info("Suspending persistence for '%s'", dp.name)
dpms.suspend_data_product_persistence(dp._id, headers=self._get_system_actor_headers(), timeout=self.timeout)
except Exception:
self._log_error(agent_instance_id, resource_id, logexc=True,
msg="Could not suspend persistence for dp_id=%s" % (dp._id))
if self.autoclean:
self._cleanup_persistence(dp)
def suspend_persistence(self, agent_instance_id, resource_id):
if not agent_instance_id or not resource_id:
log.warn("Could not %s agent %s for device %s", self.op, agent_instance_id, resource_id)
return
dpms = DataProductManagementServiceProcessClient(process=self)
dp_objs, _ = self.rr.find_objects(resource_id, PRED.hasOutputProduct, id_only=False)
for dp in dp_objs:
try:
log.info("Suspending persistence for '%s'", dp.name)
dpms.suspend_data_product_persistence(dp._id)
except Exception:
log.exception("Could not suspend persistence")
def activate_persistence(self, agent_instance_id, resource_id):
if not resource_id:
log.warn("Could not %s for device %s", self.op, resource_id)
return
dpms = DataProductManagementServiceProcessClient(process=self)
dp_objs, _ = self.rr.find_objects(resource_id, PRED.hasOutputProduct, id_only=False)
for dp in dp_objs:
try:
log.info("Activating persistence for '%s'", dp.name)
dpms.activate_data_product_persistence(dp._id, headers=self._get_system_actor_headers(), timeout=self.timeout)
except Exception:
self._log_error(agent_instance_id, resource_id, logexc=True,
msg="Could not activate persistence for dp_id=%s" % (dp._id))
def seed_gov(container, process=FakeProcess()):
dp_client = DataProductManagementServiceProcessClient(node=container.node, process=process)
dp_obj = IonObject(RT.DataProduct, name='DataProd1', description='some new dp')
dp_client.create_data_product(dp_obj)
dp_obj = IonObject(RT.DataProduct,
name='DataProd2',
description='and of course another new dp')
dp_client.create_data_product(dp_obj,)
dp_obj = IonObject(RT.DataProduct,
name='DataProd3',
description='yet another new dp')
dp_client.create_data_product(dp_obj,)
log.debug('Data Products')
dp_list = dp_client.find_data_products()
for dp_obj in dp_list:
log.debug( str(dp_obj))
ims_client = InstrumentManagementServiceProcessClient(node=container.node, process=process)
ia_obj = IonObject(RT.InstrumentAgent, name='Instrument Agent1', description='The first Instrument Agent')
ims_client.create_instrument_agent(ia_obj)
ia_obj = IonObject(RT.InstrumentAgent, name='Instrument Agent2', description='The second Instrument Agent')
ims_client.create_instrument_agent(ia_obj)
log.debug( 'Instrument Agents')
ia_list = ims_client.find_instrument_agents()
for ia_obj in ia_list:
log.debug( str(ia_obj))
org_client = OrgManagementServiceProcessClient(node=container.node, process=process)
ion_org = org_client.find_org()
policy_client = PolicyManagementServiceProcessClient(node=container.node, process=process)
role_obj = IonObject(RT.UserRole, name='Instrument Operator', description='Users assigned to this role are instrument operators')
role_id = policy_client.create_role(role_obj)
org_client.add_user_role(ion_org._id, role_id)
try:
role_id = policy_client.create_role(role_obj)
org_client.add_user_role(ion_org._id, role_id)
except Exception, e:
log.info("This should fail")
log.info(e.message)
def setUp(self):
# Start container
logging.disable(logging.ERROR)
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# simulate preloading
preload_ion_params(self.container)
logging.disable(logging.NOTSET)
#Instantiate a process to represent the test
process=VisualizationServiceTestProcess()
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceProcessClient(node=self.container.node, process=process)
self.damsclient = DataAcquisitionManagementServiceProcessClient(node=self.container.node, process=process)
self.pubsubclient = PubsubManagementServiceProcessClient(node=self.container.node, process=process)
self.ingestclient = IngestionManagementServiceProcessClient(node=self.container.node, process=process)
self.imsclient = InstrumentManagementServiceProcessClient(node=self.container.node, process=process)
self.dataproductclient = DataProductManagementServiceProcessClient(node=self.container.node, process=process)
self.dataprocessclient = DataProcessManagementServiceProcessClient(node=self.container.node, process=process)
self.datasetclient = DatasetManagementServiceProcessClient(node=self.container.node, process=process)
self.workflowclient = WorkflowManagementServiceProcessClient(node=self.container.node, process=process)
self.process_dispatcher = ProcessDispatcherServiceProcessClient(node=self.container.node, process=process)
self.data_retriever = DataRetrieverServiceProcessClient(node=self.container.node, process=process)
self.vis_client = VisualizationServiceProcessClient(node=self.container.node, process=process)
self.ctd_stream_def = SBE37_CDM_stream_definition()
def activate_persistence(self, agent_instance_id, resource_id):
dpms = DataProductManagementServiceProcessClient(process=self)
dp_objs, _ = self.rr.find_objects(resource_id, PRED.hasOutputProduct, id_only=False)
for dp in dp_objs:
try:
if dpms.is_persisted(dp._id, headers=self._get_system_actor_headers()):
if self.force:
log.warn("DataProduct %s '%s' is currently persisted - continuing", dp._id, dp.name)
else:
log.warn("DataProduct %s '%s' is currently persisted", dp._id, dp.name)
continue
log.info("Activating persistence for '%s'", dp.name)
dpms.activate_data_product_persistence(dp._id, headers=self._get_system_actor_headers(), timeout=self.timeout)
except Exception:
self._log_error(agent_instance_id, resource_id, logexc=True,
msg="Could not activate persistence for dp_id=%s" % (dp._id))
def on_start(self):
ImmediateProcess.on_start(self)
# necessary arguments, passed in via configuration kwarg to schedule_process. process namespace to avoid collisions
fuc_id = self.CFG.get_safe('process.fuc_id',
None) # FileUploadContext ID
# Clients
self.object_store = self.container.object_store
self.resource_registry = self.container.resource_registry
self.event_publisher = EventPublisher(OT.ResetQCEvent)
self.data_product_management = DataProductManagementServiceProcessClient(
process=self)
self.create_map()
# run process
if fuc_id:
self.process(fuc_id)
# cleanup
self.event_publisher.close()
def on_start(self):
ImmediateProcess.on_start(self)
# necessary arguments, passed in via configuration kwarg to schedule_process. process namespace to avoid collisions
fuc_id = self.CFG.get_safe('process.fuc_id',None) # FileUploadContext ID
# Clients
self.object_store = self.container.object_store
self.resource_registry = self.container.resource_registry
self.event_publisher = EventPublisher(OT.ResetQCEvent)
self.data_product_management = DataProductManagementServiceProcessClient(process=self)
self.create_map()
# run process
if fuc_id:
self.process(fuc_id)
# cleanup
self.event_publisher.close()
class UploadCalibrationProcessing(ImmediateProcess):
"""
Upload Calibration Processing Process
This process provides the capability to ION clients and operators to process uploaded calibration
coefficients to calibrate data products.
This parameters that this process accepts as configurations are:
- fuc_id: The FileUploadContext identifier, required, stores where the file was written
"""
def on_start(self):
ImmediateProcess.on_start(self)
# necessary arguments, passed in via configuration kwarg to schedule_process. process namespace to avoid collisions
fuc_id = self.CFG.get_safe('process.fuc_id',None) # FileUploadContext ID
# Clients
self.object_store = self.container.object_store
self.resource_registry = self.container.resource_registry
self.event_publisher = EventPublisher(OT.ResetQCEvent)
self.data_product_management = DataProductManagementServiceProcessClient(process=self)
self.create_map()
# run process
if fuc_id:
self.process(fuc_id)
# cleanup
self.event_publisher.close()
def process(self,fuc_id):
# get the Object (dict) containing details of the uploaded file
fuc = self.object_store.read(fuc_id)
if fuc['filetype'] == 'ZIP':
raise BadRequest("ZIP format not determined by project scientists yet (2014-04-21)")
#self.process_zip(fuc)
else:
self.process_csv(fuc)
def create_map(self):
'''
Creates a map from property numbers to datasets
'''
self.property_map = {}
for instrument_device in self.resource_registry.find_resources(restype=RT.InstrumentDevice)[0]:
if instrument_device.ooi_property_number:
self.property_map[instrument_device.ooi_property_number] = self.data_products_for_device(instrument_device)
def data_products_for_device(self, device):
data_products, _ = self.resource_registry.find_objects(device, PRED.hasOutputProduct, id_only=True)
return data_products
def dataset_for_data_product(self, data_product):
datasets, _ = self.resource_registry.find_objects(data_product, PRED.hasDataset, id_only=True)
return datasets[0]
def do_something_with_the_update(self, updates):
for property_no, calibration_update in updates.iteritems():
# Check to see if we even have an instrument with this property number
if property_no not in self.property_map:
continue
# Get the data product listings for this instrument
data_products = self.property_map[property_no]
# Go through each data product and update the data IF
# - There is a set of parameters that match those in the calibration
for data_product in data_products:
self.update_data_product(data_product, calibration_update)
def update_data_product(self, data_product, calibration_update):
parameters = [p.name for p in self.data_product_management.get_data_product_parameters(data_product)]
dataset_updates = []
for cal_name in calibration_update.iterkeys():
if cal_name in parameters:
dataset_id = self.dataset_for_data_product(data_product)
dataset_updates.append(dataset_id)
for dataset in dataset_updates:
self.apply_to_dataset(dataset, calibration_update)
def apply_to_dataset(self, dataset, calibration_update):
cov = DatasetManagementService._get_coverage(dataset, mode='r+')
try:
self.set_sparse_values(cov, calibration_update)
self.publish_calibration_event(dataset, calibration_update.keys())
finally:
cov.close()
def set_sparse_values(self, cov, calibration_update):
for calibration_name, updates in calibration_update.iteritems():
if calibration_name not in cov.list_parameters():
continue
for update in updates:
np_dict = {}
self.check_units(cov, calibration_name, update['units'])
start_date = self.ntp_from_iso(update['start_date'])
np_dict[calibration_name] = ConstantOverTime(calibration_name, update['value'], time_start=start_date)
cov.set_parameter_values(np_dict)
def check_units(self, cov, calibration_name, units):
pass
def publish_calibration_event(self, dataset, calibrations):
publisher = EventPublisher(OT.DatasetCalibrationEvent)
publisher.publish_event(origin=dataset, calibrations=calibrations)
def ntp_from_iso(self, iso):
return TimeUtils.ntp_from_iso(iso)
def process_csv(self, fuc):
# CSV file open here
csv_filename = fuc.get('path', None)
if csv_filename is None:
raise BadRequest("uploaded file has no path")
# keep track of the number of calibrations we actually process
nupdates = 0
updates = {} # keys are reference_designators, use to update object store after parsing CSV
with open(csv_filename, 'rb') as csvfile:
# eliminate blank lines
csvfile = (row for row in csvfile if len(row.strip()) > 0)
# eliminate commented lines
csvfile = (row for row in csvfile if not row.startswith('#'))
# open CSV reader
csv_reader = csv.reader(csvfile, delimiter=',') # skip commented lines
# iterate the rows returned by csv.reader
for row in csv_reader:
if len(row) != 6:
log.warn("invalid calibration line %s" % ','.join(row))
continue
try:
ipn = row[0] # instrument_property_number
name = row[1] # calibration_name
value = float(row[2]) # calibration_value
units = row[3]
description = row[4] # description
start_date = row[5] # start_date TODO date object?
d = {
'value':value,
'units':units,
'description':description,
'start_date':start_date
}
except ValueError as e:
continue #TODO error message? or just skip?
# get ipn key
if ipn not in updates:
updates[ipn] = {} # initialize empty array
if name not in updates[ipn]:
updates[ipn][name] = [] # will be array of dicts
updates[ipn][name].append(d)
nupdates = nupdates + 1
self.do_something_with_the_update(updates)
# insert the updates into object store
self.update_object_store(updates)
# update FileUploadContext object (change status to complete)
fuc['status'] = 'UploadCalibrationProcessing process complete - %d updates added to object store' % nupdates
self.object_store.update_doc(fuc)
# remove uploaded file
try:
os.remove(csv_filename)
except OSError:
pass # TODO take action to get this removed
def process_zip(self,fuc):
pass
def update_object_store(self, updates):
'''inserts the updates into object store'''
for i in updates: # loops the instrument_property_number(ipn) in the updates object
try: # if i exists in object_store, read it
ipn = self.object_store.read(i) #TODO: what will this be?
except: # if does not yet exist in object_store, create it (can't use update_doc because need to set id)
ipn = self.object_store.create_doc({'_type':'CALIBRATION'},i) # CAUTION: this returns a tuple, not a dict like read() returns
ipn = self.object_store.read(i) # read so we have a dict like we expect
# merge all from updates[i] into dict destined for the object_store (ipn)
for name in updates[i]: # loops the calibration_names under each IPN in updates
#TODO: if name not initialized, will append work? if so, can use same op for both
if name not in ipn: # if name doesn't exist, we can just add the entire object (dict of lists)
ipn[name] = updates[i][name]
else: # if it does, we need to append to each of the lists
ipn[name].append(updates[i][name]) # append the list from updates
# store updated ipn keyed object in object_store (should overwrite full object, contains all previous too)
self.object_store.update_doc(ipn)
# publish ResetQCEvent event (one for each instrument_property_number [AKA ipn])
self.event_publisher.publish_event(origin=i)
try:
org_client.revoke_role(ion_org._id, user._id, 'INSTRUMENT_OPERATOR', headers={'ion-actor-id': user._id, 'ion-actor-roles': header_roles })
except Exception, e:
log.info('This revoke role should be denied:' + e.message)
header_roles = get_role_message_headers(org_client.find_all_roles_by_user(system_actor._id))
org_client.revoke_role(ion_org._id, user._id, 'INSTRUMENT_OPERATOR', headers={'ion-actor-id': system_actor._id, 'ion-actor-roles': header_roles })
roles = org_client.find_roles_by_user(ion_org._id, user._id)
for r in roles:
log.info('User UserRole: ' +str(r))
dp_client = DataProductManagementServiceProcessClient(node=container.node, process=process)
dp_obj = IonObject(RT.DataProduct, name='DataProd1', description='some new dp')
dp_client.create_data_product(dp_obj, headers={'ion-actor-id': system_actor._id})
dp_obj = IonObject(RT.DataProduct,
name='DataProd2',
description='and of course another new dp')
dp_client.create_data_product(dp_obj, headers={'ion-actor-id': system_actor._id})
dp_obj = IonObject(RT.DataProduct,
name='DataProd3',
description='yet another new dp')
class UploadCalibrationProcessing(ImmediateProcess):
"""
Upload Calibration Processing Process
This process provides the capability to ION clients and operators to process uploaded calibration
coefficients to calibrate data products.
This parameters that this process accepts as configurations are:
- fuc_id: The FileUploadContext identifier, required, stores where the file was written
"""
def on_start(self):
ImmediateProcess.on_start(self)
# necessary arguments, passed in via configuration kwarg to schedule_process. process namespace to avoid collisions
fuc_id = self.CFG.get_safe('process.fuc_id',
None) # FileUploadContext ID
# Clients
self.object_store = self.container.object_store
self.resource_registry = self.container.resource_registry
self.event_publisher = EventPublisher(OT.ResetQCEvent)
self.data_product_management = DataProductManagementServiceProcessClient(
process=self)
self.create_map()
# run process
if fuc_id:
self.process(fuc_id)
# cleanup
self.event_publisher.close()
def process(self, fuc_id):
# get the Object (dict) containing details of the uploaded file
fuc = self.object_store.read(fuc_id)
if fuc['filetype'] == 'ZIP':
raise BadRequest(
"ZIP format not determined by project scientists yet (2014-04-21)"
)
#self.process_zip(fuc)
else:
self.process_csv(fuc)
def create_map(self):
'''
Creates a map from property numbers to datasets
'''
self.property_map = {}
for instrument_device in self.resource_registry.find_resources(
restype=RT.InstrumentDevice)[0]:
if instrument_device.ooi_property_number:
self.property_map[
instrument_device.
ooi_property_number] = self.data_products_for_device(
instrument_device)
def data_products_for_device(self, device):
data_products, _ = self.resource_registry.find_objects(
device, PRED.hasOutputProduct, id_only=True)
return data_products
def dataset_for_data_product(self, data_product):
datasets, _ = self.resource_registry.find_objects(data_product,
PRED.hasDataset,
id_only=True)
return datasets[0]
def do_something_with_the_update(self, updates):
for property_no, calibration_update in updates.iteritems():
# Check to see if we even have an instrument with this property number
if property_no not in self.property_map:
continue
# Get the data product listings for this instrument
data_products = self.property_map[property_no]
# Go through each data product and update the data IF
# - There is a set of parameters that match those in the calibration
for data_product in data_products:
self.update_data_product(data_product, calibration_update)
def update_data_product(self, data_product, calibration_update):
parameters = [
p.name
for p in self.data_product_management.get_data_product_parameters(
data_product)
]
dataset_updates = []
for cal_name in calibration_update.iterkeys():
if cal_name in parameters:
dataset_id = self.dataset_for_data_product(data_product)
dataset_updates.append(dataset_id)
for dataset in dataset_updates:
self.apply_to_dataset(dataset, calibration_update)
def apply_to_dataset(self, dataset, calibration_update):
cov = DatasetManagementService._get_coverage(dataset, mode='r+')
try:
self.set_sparse_values(cov, calibration_update)
self.publish_calibration_event(dataset, calibration_update.keys())
finally:
cov.close()
def set_sparse_values(self, cov, calibration_update):
for calibration_name, updates in calibration_update.iteritems():
if calibration_name not in cov.list_parameters():
continue
for update in updates:
np_dict = {}
self.check_units(cov, calibration_name, update['units'])
start_date = self.ntp_from_iso(update['start_date'])
np_dict[calibration_name] = ConstantOverTime(
calibration_name, update['value'], time_start=start_date)
cov.set_parameter_values(np_dict)
def check_units(self, cov, calibration_name, units):
pass
def publish_calibration_event(self, dataset, calibrations):
publisher = EventPublisher(OT.DatasetCalibrationEvent)
publisher.publish_event(origin=dataset, calibrations=calibrations)
def ntp_from_iso(self, iso):
return TimeUtils.ntp_from_iso(iso)
def process_csv(self, fuc):
# CSV file open here
csv_filename = fuc.get('path', None)
if csv_filename is None:
raise BadRequest("uploaded file has no path")
# keep track of the number of calibrations we actually process
nupdates = 0
updates = {
} # keys are reference_designators, use to update object store after parsing CSV
with open(csv_filename, 'rb') as csvfile:
# eliminate blank lines
csvfile = (row for row in csvfile if len(row.strip()) > 0)
# eliminate commented lines
csvfile = (row for row in csvfile if not row.startswith('#'))
# open CSV reader
csv_reader = csv.reader(csvfile,
delimiter=',') # skip commented lines
# iterate the rows returned by csv.reader
for row in csv_reader:
if len(row) != 6:
log.warn("invalid calibration line %s" % ','.join(row))
continue
try:
ipn = row[0] # instrument_property_number
name = row[1] # calibration_name
value = float(row[2]) # calibration_value
units = row[3]
description = row[4] # description
start_date = row[5] # start_date TODO date object?
d = {
'value': value,
'units': units,
'description': description,
'start_date': start_date
}
except ValueError as e:
continue #TODO error message? or just skip?
# get ipn key
if ipn not in updates:
updates[ipn] = {} # initialize empty array
if name not in updates[ipn]:
updates[ipn][name] = [] # will be array of dicts
updates[ipn][name].append(d)
nupdates = nupdates + 1
self.do_something_with_the_update(updates)
# insert the updates into object store
self.update_object_store(updates)
# update FileUploadContext object (change status to complete)
fuc['status'] = 'UploadCalibrationProcessing process complete - %d updates added to object store' % nupdates
self.object_store.update_doc(fuc)
# remove uploaded file
try:
os.remove(csv_filename)
except OSError:
pass # TODO take action to get this removed
def process_zip(self, fuc):
pass
def update_object_store(self, updates):
'''inserts the updates into object store'''
for i in updates: # loops the instrument_property_number(ipn) in the updates object
try: # if i exists in object_store, read it
ipn = self.object_store.read(i) #TODO: what will this be?
except: # if does not yet exist in object_store, create it (can't use update_doc because need to set id)
ipn = self.object_store.create_doc(
{'_type': 'CALIBRATION'}, i
) # CAUTION: this returns a tuple, not a dict like read() returns
ipn = self.object_store.read(
i) # read so we have a dict like we expect
# merge all from updates[i] into dict destined for the object_store (ipn)
for name in updates[
i]: # loops the calibration_names under each IPN in updates
#TODO: if name not initialized, will append work? if so, can use same op for both
if name not in ipn: # if name doesn't exist, we can just add the entire object (dict of lists)
ipn[name] = updates[i][name]
else: # if it does, we need to append to each of the lists
ipn[name].append(
updates[i][name]) # append the list from updates
# store updated ipn keyed object in object_store (should overwrite full object, contains all previous too)
self.object_store.update_doc(ipn)
# publish ResetQCEvent event (one for each instrument_property_number [AKA ipn])
self.event_publisher.publish_event(origin=i)
header_roles = get_role_message_headers(
org_client.find_all_roles_by_user(system_actor._id))
org_client.revoke_role(ion_org._id,
user._id,
'INSTRUMENT_OPERATOR',
headers={
'ion-actor-id': system_actor._id,
'ion-actor-roles': header_roles
})
roles = org_client.find_roles_by_user(ion_org._id, user._id)
for r in roles:
log.info('User UserRole: ' + str(r))
dp_client = DataProductManagementServiceProcessClient(node=container.node,
process=process)
dp_obj = IonObject(RT.DataProduct,
name='DataProd1',
description='some new dp')
dp_client.create_data_product(dp_obj,
headers={'ion-actor-id': system_actor._id})
dp_obj = IonObject(RT.DataProduct,
name='DataProd2',
description='and of course another new dp')
dp_client.create_data_product(dp_obj,
headers={'ion-actor-id': system_actor._id})