def override_clients(self, new_clients):
"""
Replaces the service clients with a new set of them... and makes sure they go to the right places
"""
#shortcut names for the import sub-services
# we hide these behind checks even though we expect them so that
# the resource_impl_metatests will work
if hasattr(self.clients, "resource_registry"):
self.RR = self.clients.resource_registry
if hasattr(self.clients, "data_acquisition_management"):
self.DAMS = self.clients.data_acquisition_management
if hasattr(self.clients, "data_product_management"):
self.DPMS = self.clients.data_product_management
#farm everything out to the impls
self.instrument_agent = InstrumentAgentImpl(self.clients)
self.instrument_agent_instance = InstrumentAgentInstanceImpl(self.clients)
self.instrument_model = InstrumentModelImpl(self.clients)
self.instrument_device = InstrumentDeviceImpl(self.clients)
self.platform_agent = PlatformAgentImpl(self.clients)
self.platform_agent_instance = PlatformAgentInstanceImpl(self.clients)
self.platform_model = PlatformModelImpl(self.clients)
self.platform_device = PlatformDeviceImpl(self.clients)
self.sensor_model = SensorModelImpl(self.clients)
self.sensor_device = SensorDeviceImpl(self.clients)
#TODO: may not belong in this service
self.data_product = DataProductImpl(self.clients)
self.data_producer = DataProducerImpl(self.clients)
self.logical_instrument = LogicalInstrumentImpl(self.clients)
class InstrumentManagementService(BaseInstrumentManagementService):
"""
@brief Service to manage instrument, platform, and sensor resources, their relationships, and direct access
"""
def on_init(self):
#suppress a few "variable declared but not used" annoying pyflakes errors
IonObject("Resource")
log
self.override_clients(self.clients)
def override_clients(self, new_clients):
"""
Replaces the service clients with a new set of them... and makes sure they go to the right places
"""
#shortcut names for the import sub-services
# we hide these behind checks even though we expect them so that
# the resource_impl_metatests will work
if hasattr(self.clients, "resource_registry"):
self.RR = self.clients.resource_registry
if hasattr(self.clients, "data_acquisition_management"):
self.DAMS = self.clients.data_acquisition_management
if hasattr(self.clients, "data_product_management"):
self.DPMS = self.clients.data_product_management
#farm everything out to the impls
self.instrument_agent = InstrumentAgentImpl(self.clients)
self.instrument_agent_instance = InstrumentAgentInstanceImpl(self.clients)
self.instrument_model = InstrumentModelImpl(self.clients)
self.instrument_device = InstrumentDeviceImpl(self.clients)
self.platform_agent = PlatformAgentImpl(self.clients)
self.platform_agent_instance = PlatformAgentInstanceImpl(self.clients)
self.platform_model = PlatformModelImpl(self.clients)
self.platform_device = PlatformDeviceImpl(self.clients)
self.sensor_model = SensorModelImpl(self.clients)
self.sensor_device = SensorDeviceImpl(self.clients)
#TODO: may not belong in this service
self.data_product = DataProductImpl(self.clients)
self.data_producer = DataProducerImpl(self.clients)
self.logical_instrument = LogicalInstrumentImpl(self.clients)
##########################################################################
#
# INSTRUMENT AGENT INSTANCE
#
##########################################################################
def create_instrument_agent_instance(self, instrument_agent_instance=None):
"""
create a new instance
@param instrument_agent_instance the object to be created as a resource
@retval instrument_agent_instance_id the id of the new object
@throws BadRequest if the incoming _id field is set
@throws BadReqeust if the incoming name already exists
"""
return self.instrument_agent_instance.create_one(instrument_agent_instance)
def update_instrument_agent_instance(self, instrument_agent_instance=None):
"""
update an existing instance
@param instrument_agent_instance the object to be created as a resource
@retval success whether we succeeded
@throws BadRequest if the incoming _id field is not set
@throws BadReqeust if the incoming name already exists
"""
return self.instrument_agent_instance.update_one(instrument_agent_instance)
def read_instrument_agent_instance(self, instrument_agent_instance_id=''):
"""
fetch a resource by ID
@param instrument_agent_instance_id the id of the object to be fetched
@retval InstrumentAgentInstance resource
"""
return self.instrument_agent_instance.read_one(instrument_agent_instance_id)
def delete_instrument_agent_instance(self, instrument_agent_instance_id=''):
"""
delete a resource, including its history (for less ominous deletion, use retire)
@param instrument_agent_instance_id the id of the object to be deleted
@retval success whether it succeeded
"""
return self.instrument_agent_instance.delete_one(instrument_agent_instance_id)
def find_instrument_agent_instances(self, filters=None):
"""
"""
return self.instrument_agent_instance.find_some(filters)
##########################################################################
#
# INSTRUMENT AGENT
#
##########################################################################
def create_instrument_agent(self, instrument_agent=None):
"""
create a new instance
@param instrument_agent the object to be created as a resource
@retval instrument_agent_id the id of the new object
@throws BadRequest if the incoming _id field is set
@throws BadReqeust if the incoming name already exists
"""
return self.instrument_agent.create_one(instrument_agent)
def update_instrument_agent(self, instrument_agent=None):
"""
update an existing instance
@param instrument_agent the object to be created as a resource
@retval success whether we succeeded
@throws BadRequest if the incoming _id field is not set
@throws BadReqeust if the incoming name already exists
"""
return self.instrument_agent.update_one(instrument_agent)
def read_instrument_agent(self, instrument_agent_id=''):
"""
fetch a resource by ID
@param instrument_agent_id the id of the object to be fetched
@retval InstrumentAgent resource
"""
return self.instrument_agent.read_one(instrument_agent_id)
def delete_instrument_agent(self, instrument_agent_id=''):
"""
delete a resource, including its history (for less ominous deletion, use retire)
@param instrument_agent_id the id of the object to be deleted
@retval success whether it succeeded
"""
return self.instrument_agent.delete_one(instrument_agent_id)
def find_instrument_agents(self, filters=None):
"""
"""
return self.instrument_agent.find_some(filters)
##########################################################################
#
# INSTRUMENT MODEL
#
##########################################################################
def create_instrument_model(self, instrument_model=None):
"""
create a new instance
@param instrument_model the object to be created as a resource
@retval instrument_model_id the id of the new object
@throws BadRequest if the incoming _id field is set
@throws BadReqeust if the incoming name already exists
"""
return self.instrument_model.create_one(instrument_model)
def update_instrument_model(self, instrument_model=None):
"""
update an existing instance
@param instrument_model the object to be created as a resource
@retval success whether we succeeded
@throws BadRequest if the incoming _id field is not set
@throws BadReqeust if the incoming name already exists
"""
return self.instrument_model.update_one(instrument_model)
def read_instrument_model(self, instrument_model_id=''):
"""
fetch a resource by ID
@param instrument_model_id the id of the object to be fetched
@retval InstrumentModel resource
"""
return self.instrument_model.read_one(instrument_model_id)
def delete_instrument_model(self, instrument_model_id=''):
"""
delete a resource, including its history (for less ominous deletion, use retire)
@param instrument_model_id the id of the object to be deleted
@retval success whether it succeeded
"""
return self.instrument_model.delete_one(instrument_model_id)
def find_instrument_models(self, filters=None):
"""
"""
return self.instrument_model.find_some(filters)
##########################################################################
#
# PHYSICAL INSTRUMENT
#
##########################################################################
def setup_data_production_chain(self, instrument_device_id=''):
"""
create a data product (L0) for the instrument, and establish provenance
between the corresponding data producers
"""
#get instrument object and instrument's data producer
inst_obj = self.instrument_device.read_one(instrument_device_id)
inst_pducers = self.instrument_device.find_stemming_data_producer(instrument_device_id)
inst_pducer_id = inst_pducers[0]
log.debug("instrument data producer id='%s'" % inst_pducer_id)
#create a new data product
dpms_pduct_obj = IonObject(RT.DataProduct,
name=str(inst_obj.name + " L0 Product"),
description=str("L0 DataProduct for " + inst_obj.name))
pduct_id = self.DPMS.create_data_product(dpms_pduct_obj)
#TODO: DPMS isn't creating a data producer for new data products. not sure why.
#
prod_pducer_id = self.data_producer.create_one(IonObject(RT.DataProducer,
name=str(inst_obj.name + " L0 Producer"),
description=str("L0 DataProducer for " + inst_obj.name)))
self.data_product.link_data_producer(pduct_id, prod_pducer_id)
# get data product's data producer (via association)
#TODO: this belongs in DPMS
prod_pducers = self.data_product.find_stemming_data_producer(pduct_id)
# (TODO: there should only be one assoc_id. what error to raise?)
# TODO: what error to raise if there are no assoc ids?
prod_pducer_id = prod_pducers[0]
# instrument data producer is the parent of the data product producer
#TODO: this belongs in DAMS
self.data_producer.link_input_data_producer(prod_pducer_id, inst_pducer_id)
#TODO: error checking
def create_instrument_device(self, instrument_device=None):
"""
create a new instance
@param instrument_device the object to be created as a resource
@retval instrument_device_id the id of the new object
@throws BadRequest if the incoming _id field is set
@throws BadReqeust if the incoming name already exists
"""
instrument_device_id = self.instrument_device.create_one(instrument_device)
self.DAMS.register_instrument(instrument_device_id)
#TODO: create data producer and product
self.setup_data_production_chain(instrument_device_id)
return instrument_device_id
def update_instrument_device(self, instrument_device=None):
"""
update an existing instance
@param instrument_device the object to be created as a resource
@retval success whether we succeeded
@throws BadRequest if the incoming _id field is not set
@throws BadReqeust if the incoming name already exists
"""
return self.instrument_device.update_one(instrument_device)
def read_instrument_device(self, instrument_device_id=''):
"""
fetch a resource by ID
@param instrument_device_id the id of the object to be fetched
@retval InstrumentDevice resource
"""
return self.instrument_device.read_one(instrument_device_id)
def delete_instrument_device(self, instrument_device_id=''):
"""
delete a resource, including its history (for less ominous deletion, use retire)
@param instrument_device_id the id of the object to be deleted
@retval success whether it succeeded
"""
return self.instrument_device.delete_one(instrument_device_id)
def find_instrument_devices(self, filters=None):
"""
"""
return self.instrument_device.find_some(filters)
##
##
## DIRECT ACCESS
##
##
def request_direct_access(self, instrument_device_id=''):
"""
"""
# determine whether id is for physical or logical instrument
# look up instrument if not
# Validate request; current instrument state, policy, and other
# Retrieve and save current instrument settings
# Request DA channel, save reference
# Return direct access channel
raise NotImplementedError()
pass
def stop_direct_access(self, instrument_device_id=''):
"""
"""
# Return Value
# ------------
# {success: true}
#
raise NotImplementedError()
pass
##########################################################################
#
# PLATFORM AGENT INSTANCE
#
##########################################################################
def create_platform_agent_instance(self, platform_agent_instance=None):
"""
create a new instance
@param platform_agent_instance the object to be created as a resource
@retval platform_agent_instance_id the id of the new object
@throws BadRequest if the incoming _id field is set
@throws BadReqeust if the incoming name already exists
"""
return self.platform_agent_instance.create_one(platform_agent_instance)
def update_platform_agent_instance(self, platform_agent_instance=None):
"""
update an existing instance
@param platform_agent_instance the object to be created as a resource
@retval success whether we succeeded
@throws BadRequest if the incoming _id field is not set
@throws BadReqeust if the incoming name already exists
"""
return self.platform_agent_instance.update_one(platform_agent_instance)
def read_platform_agent_instance(self, platform_agent_instance_id=''):
"""
fetch a resource by ID
@param platform_agent_instance_id the id of the object to be fetched
@retval PlatformAgentInstance resource
"""
return self.platform_agent_instance.read_one(platform_agent_instance_id)
def delete_platform_agent_instance(self, platform_agent_instance_id=''):
"""
delete a resource, including its history (for less ominous deletion, use retire)
@param platform_agent_instance_id the id of the object to be deleted
@retval success whether it succeeded
"""
return self.platform_agent_instance.delete_one(platform_agent_instance_id)
def find_platform_agent_instances(self, filters=None):
"""
"""
return self.platform_agent_instance.find_some(filters)
##########################################################################
#
# PLATFORM AGENT
#
##########################################################################
def create_platform_agent(self, platform_agent=None):
"""
create a new instance
@param platform_agent the object to be created as a resource
@retval platform_agent_id the id of the new object
@throws BadRequest if the incoming _id field is set
@throws BadReqeust if the incoming name already exists
"""
return self.platform_agent.create_one(platform_agent)
def update_platform_agent(self, platform_agent=None):
"""
update an existing instance
@param platform_agent the object to be created as a resource
@retval success whether we succeeded
@throws BadRequest if the incoming _id field is not set
@throws BadReqeust if the incoming name already exists
"""
return self.platform_agent.update_one(platform_agent)
def read_platform_agent(self, platform_agent_id=''):
"""
fetch a resource by ID
@param platform_agent_id the id of the object to be fetched
@retval PlatformAgent resource
"""
return self.platform_agent.read_one(platform_agent_id)
def delete_platform_agent(self, platform_agent_id=''):
"""
delete a resource, including its history (for less ominous deletion, use retire)
@param platform_agent_id the id of the object to be deleted
@retval success whether it succeeded
"""
return self.platform_agent.delete_one(platform_agent_id)
def find_platform_agents(self, filters=None):
"""
"""
return self.platform_agent.find_some(filters)
##########################################################################
#
# PLATFORM MODEL
#
##########################################################################
def create_platform_model(self, platform_model=None):
"""
create a new instance
@param platform_model the object to be created as a resource
@retval platform_model_id the id of the new object
@throws BadRequest if the incoming _id field is set
@throws BadReqeust if the incoming name already exists
"""
return self.platform_model.create_one(platform_model)
def update_platform_model(self, platform_model=None):
"""
update an existing instance
@param platform_model the object to be created as a resource
@retval success whether we succeeded
@throws BadRequest if the incoming _id field is not set
@throws BadReqeust if the incoming name already exists
"""
return self.platform_model.update_one(platform_model)
def read_platform_model(self, platform_model_id=''):
"""
fetch a resource by ID
@param platform_model_id the id of the object to be fetched
@retval PlatformModel resource
"""
return self.platform_model.read_one(platform_model_id)
def delete_platform_model(self, platform_model_id=''):
"""
delete a resource, including its history (for less ominous deletion, use retire)
@param platform_model_id the id of the object to be deleted
@retval success whether it succeeded
"""
return self.platform_model.delete_one(platform_model_id)
def find_platform_models(self, filters=None):
"""
"""
return self.platform_model.find_some(filters)
##########################################################################
#
# PHYSICAL PLATFORM
#
##########################################################################
def create_platform_device(self, platform_device=None):
"""
create a new instance
@param platform_device the object to be created as a resource
@retval platform_device_id the id of the new object
@throws BadRequest if the incoming _id field is set
@throws BadReqeust if the incoming name already exists
"""
return self.platform_device.create_one(platform_device)
def update_platform_device(self, platform_device=None):
"""
update an existing instance
@param platform_device the object to be created as a resource
@retval success whether we succeeded
@throws BadRequest if the incoming _id field is not set
@throws BadReqeust if the incoming name already exists
"""
return self.platform_device.update_one(platform_device)
def read_platform_device(self, platform_device_id=''):
"""
fetch a resource by ID
@param platform_device_id the id of the object to be fetched
@retval PlatformDevice resource
"""
return self.platform_device.read_one(platform_device_id)
def delete_platform_device(self, platform_device_id=''):
"""
delete a resource, including its history (for less ominous deletion, use retire)
@param platform_device_id the id of the object to be deleted
@retval success whether it succeeded
"""
return self.platform_device.delete_one(platform_device_id)
def find_platform_devices(self, filters=None):
"""
"""
return self.platform_device.find_some(filters)
##########################################################################
#
# SENSOR MODEL
#
##########################################################################
def create_sensor_model(self, sensor_model=None):
"""
create a new instance
@param sensor_model the object to be created as a resource
@retval sensor_model_id the id of the new object
@throws BadRequest if the incoming _id field is set
@throws BadReqeust if the incoming name already exists
"""
return self.sensor_model.create_one(sensor_model)
def update_sensor_model(self, sensor_model=None):
"""
update an existing instance
@param sensor_model the object to be created as a resource
@retval success whether we succeeded
@throws BadRequest if the incoming _id field is not set
@throws BadReqeust if the incoming name already exists
"""
return self.sensor_model.update_one(sensor_model)
def read_sensor_model(self, sensor_model_id=''):
"""
fetch a resource by ID
@param sensor_model_id the id of the object to be fetched
@retval SensorModel resource
"""
return self.sensor_model.read_one(sensor_model_id)
def delete_sensor_model(self, sensor_model_id=''):
"""
delete a resource, including its history (for less ominous deletion, use retire)
@param sensor_model_id the id of the object to be deleted
@retval success whether it succeeded
"""
return self.sensor_model.delete_one(sensor_model_id)
def find_sensor_models(self, filters=None):
"""
"""
return self.sensor_model.find_some(filters)
##########################################################################
#
# PHYSICAL SENSOR
#
##########################################################################
def create_sensor_device(self, sensor_device=None):
"""
create a new instance
@param sensor_device the object to be created as a resource
@retval sensor_device_id the id of the new object
@throws BadRequest if the incoming _id field is set
@throws BadReqeust if the incoming name already exists
"""
return self.sensor_device.create_one(sensor_device)
def update_sensor_device(self, sensor_device=None):
"""
update an existing instance
@param sensor_device the object to be created as a resource
@retval success whether we succeeded
@throws BadRequest if the incoming _id field is not set
@throws BadReqeust if the incoming name already exists
"""
return self.sensor_device.update_one(sensor_device)
def read_sensor_device(self, sensor_device_id=''):
"""
fetch a resource by ID
@param sensor_device_id the id of the object to be fetched
@retval SensorDevice resource
"""
return self.sensor_device.read_one(sensor_device_id)
def delete_sensor_device(self, sensor_device_id=''):
"""
delete a resource, including its history (for less ominous deletion, use retire)
@param sensor_device_id the id of the object to be deleted
@retval success whether it succeeded
"""
return self.sensor_device.delete_one(sensor_device_id)
def find_sensor_devices(self, filters=None):
"""
"""
return self.sensor_device.find_some(filters)
##########################################################################
#
# ASSOCIATIONS
#
##########################################################################
def assign_instrument_model_to_instrument_device(self, instrument_model_id='', instrument_device_id=''):
self.instrument_device.link_model(instrument_device_id, instrument_model_id)
def unassign_instrument_model_from_instrument_device(self, instrument_model_id='', instrument_device_id=''):
self.instrument_device.unlink_model(instrument_device_id, instrument_model_id)
def assign_instrument_model_to_instrument_agent(self, instrument_model_id='', instrument_agent_id=''):
self.instrument_agent.link_model(instrument_agent_id, instrument_model_id)
def unassign_instrument_model_from_instrument_agent(self, instrument_model_id='', instrument_agent_id=''):
self.instrument_agent.unlink_model(instrument_agent_id, instrument_model_id)
def assign_sensor_model_to_sensor_device(self, sensor_model_id='', sensor_device_id=''):
self.sensor_device.link_model(sensor_device_id, sensor_model_id)
def unassign_sensor_model_from_sensor_device(self, sensor_model_id='', sensor_device_id=''):
self.sensor_device.unlink_model(sensor_device_id, sensor_model_id)
def assign_platform_model_to_platform_device(self, platform_model_id='', platform_device_id=''):
self.platform_device.link_model(platform_device_id, platform_model_id)
def unassign_platform_model_from_platform_device(self, platform_model_id='', platform_device_id=''):
self.platform_device.unlink_model(platform_device_id, platform_model_id)
def assign_instrument_device_to_platform_device(self, instrument_device_id='', platform_device_id=''):
self.platform_device.link_instrument(platform_device_id, instrument_device_id)
def unassign_instrument_device_from_platform_device(self, instrument_device_id='', platform_device_id=''):
self.platform_device.unlink_instrument(platform_device_id, instrument_device_id)
def assign_logical_instrument_to_instrument_device(self, logical_instrument_id='', instrument_device_id=''):
self.instrument_device.link_assignment(instrument_device_id, logical_instrument_id)
def unassign_logical_instrument_from_instrument_device(self, logical_instrument_id='', instrument_device_id=''):
self.instrument_device.unlink_assignment(instrument_device_id, logical_instrument_id)
def assign_logical_platform_to_platform_device(self, logical_platform_id='', platform_device_id=''):
self.platform_device.link_assignment(platform_device_id, logical_platform_id)
def unassign_logical_platform_from_platform_device(self, logical_platform_id='', platform_device_id=''):
self.platform_device.unlink_assignment(platform_device_id, logical_platform_id)
def assign_platform_agent_instance_to_platform_agent(self, platform_agent_instance_id='', platform_agent_id=''):
self.platform_agent.link_instance(platform_agent_id, platform_agent_instance_id)
def unassign_platform_agent_instance_from_platform_agent(self, platform_agent_instance_id='', platform_agent_id=''):
self.platform_agent.unlink_instance(platform_agent_id, platform_agent_instance_id)
def assign_instrument_agent_instance_to_instrument_agent(self, instrument_agent_instance_id='', instrument_agent_id=''):
self.instrument_agent.link_instance(instrument_agent_id, instrument_agent_instance_id)
def unassign_instrument_agent_instance_from_instrument_agent(self, instrument_agent_instance_id='', instrument_agent_id=''):
self.instrument_agent.unlink_instance(instrument_agent_id, instrument_agent_instance_id)
# reassigning a logical instrument to an instrument device is a little bit special
# TODO: someday we may be able to dig up the correct data products automatically,
# but once we have them this is the function that does all the work.
def reassign_logical_instrument_to_instrument_device(self, logical_instrument_id='',
old_instrument_device_id='',
new_instrument_device_id='',
logical_data_product_ids=[],
old_instrument_data_product_ids=[],
new_instrument_data_product_ids=[]):
"""
associate a logical instrument with a physical one. this involves linking the
physical instrument's data product(s) to the logical one(s).
the 2 lists of data products must be of equal length, and will map 1-1
@param logical_instrument_id
@param instrument_device_id
@param logical_data_product_ids a list of data products associated to a logical instrument
@param instrument_data_product_ids a list of data products coming from an instrument device
"""
def verify_dp_origin(supplied_dps, assigned_dps, instrument_id, instrument_label):
"""
check that the supplied dps (data products) are in the set of what's actually assigned
@param supplied_dps list of data product ids
@param assigned_dps list of data product ids
@param instrument_id a logical or instrument device id
"""
badones = []
for p in supplied_dps:
if not p in assigned_dps:
badones.append(p)
if 0 < len(badones):
raise BadRequest("want to assign %s's data products, but the following were supplied " +
"that don't seem to come from %s '%s': [%s]" %
(instrument_label, instrument_label, instrument_id, ", ".join(badones)))
log.info("Checking consistency of existing logical/instrument assignments")
existing_assignments = self.instrument_device.find_having_assignment(logical_instrument_id)
if 1 < len(existing_assignments):
raise Inconsistent("There is more than 1 instrument device associated with logical instrument '%s'" %
logical_instrument_id)
log.info("Checking whether supplied logical/instrument arguments are proper")
if 0 < len(existing_assignments):
if not old_instrument_device_id:
raise BadRequest(("Tried to assign logical instrument '%s' for the first time, but it is already " +
"assigned to instrument device '%s'") % (logical_instrument_id, existing_assignments[0]))
elif old_instrument_device_id != existing_assignments[0]:
raise BadRequest(("Tried to reassign logical instrument '%s' from instrument device '%s' but it is " +
"actually associated to instrument device '%s'") %
(logical_instrument_id, old_instrument_device_id, existing_assignments[0]))
# log.info("Checking whether supplied data products are proper")
# existing_logical_data_products = self.logical_instrument.find_stemming_data_product(logical_instrument_id)
#
#TODO: need a check that all the logical data products are being provided for
#
# log.info("Checking whether all logical data products are provided")
# if len(logical_data_product_ids) != len(existing_logical_data_products):
# raise BadRequest("tried to assign logical instrument but only provided %d of %d " +
# "data products" % (len(logical_data_product_ids), len(existing_logical_data_products)))
#
# log.info("Checking that supplied logical data products are properly rooted")
# verify_dp_origin(logical_data_product_ids,
# existing_logical_data_products,
# logical_instrument_id,
# "logical_instrument")
if old_instrument_device_id:
log.info("Checking that the data product to be dissociated are properly rooted")
verify_dp_origin(old_instrument_data_product_ids,
self.find_data_product_by_instrument_device(old_instrument_device_id),
old_instrument_device_id,
"instrument_device")
log.info("Checking that all data products to be dissociated have been supplied")
if len(logical_data_product_ids) != len(old_instrument_data_product_ids):
raise BadRequest("Can't unmap %d instrument data products from %d logical products" %
(len(old_instrument_data_product_ids), len(logical_data_product_ids)))
log.info("Checking that supplied instrument data products are properly rooted")
verify_dp_origin(new_instrument_data_product_ids,
self.find_data_product_by_instrument_device(new_instrument_device_id),
new_instrument_device_id,
"instrument_device")
log.info("Checking that all data products to be associated have been supplied")
if len(logical_data_product_ids) != len(new_instrument_data_product_ids):
raise BadRequest("Can't map %d instrument data products to %d logical products" %
(len(new_instrument_data_product_ids), len(logical_data_product_ids)))
log.info("Assigning the instruments themselves")
if "" != old_instrument_device_id:
self.instrument_device.unlink_assignment(old_instrument_device_id, logical_instrument_id)
self.instrument_device.link_assignment(new_instrument_device_id, logical_instrument_id)
# functions to link and unlink data products as appropriate
def link_logical_dp_to_instrument_dp(logical_dp_id, inst_dp_id):
# TODO: this should be a function call, probably to DPMS,
# which sets up inst_dp to copy its data stream
# directly into the logical_dp
pass
def unlink_logical_dp_from_instrument_dp(logical_dp_id, inst_dp_id):
#TODO: undo the above
pass
if old_instrument_device_id:
log.info("Unlinking existing instrument data product(s) from logical instrument's product(s)")
map(unlink_logical_dp_from_instrument_dp, logical_data_product_ids, old_instrument_data_product_ids)
log.info("Linking new instrument data products with logical instrument's product(s)")
map(link_logical_dp_to_instrument_dp, logical_data_product_ids, new_instrument_data_product_ids)
############################
#
# ASSOCIATION FIND METHODS
#
############################
def find_instrument_model_by_instrument_device(self, instrument_device_id=''):
return self.instrument_device.find_stemming_model(instrument_device_id)
def find_instrument_device_by_instrument_model(self, instrument_model_id=''):
return self.instrument_device.find_having_model(instrument_model_id)
def find_platform_model_by_platform_device(self, platform_device_id=''):
return self.platform_device.find_stemming_model(platform_device_id)
def find_platform_device_by_platform_model(self, platform_model_id=''):
return self.platform_device.find_having_model(platform_model_id)
def find_instrument_model_by_instrument_agent(self, instrument_agent_id=''):
return self.instrument_agent.find_stemming_model(instrument_agent_id)
def find_instrument_agent_by_instrument_model(self, instrument_model_id=''):
return self.instrument_agent.find_having_model(instrument_model_id)
def find_instrument_device_by_platform_device(self, platform_device_id=''):
return self.platform_device.find_stemming_instrument(platform_device_id)
def find_platform_device_by_instrument_device(self, instrument_device_id=''):
return self.platform_device.find_having_instrument(instrument_device_id)
def find_instrument_device_by_logical_instrument(self, logical_instrument_id=''):
return self.instrument_device.find_having_assignment(logical_instrument_id)
def find_logical_instrument_by_instrument_device(self, instrument_device_id=''):
return self.instrument_device.find_stemming_assignment(instrument_device_id)
def find_platform_device_by_logical_platform(self, logical_platform_id=''):
return self.platform_device.find_having_assignment(logical_platform_id)
def find_logical_platform_by_platform_device(self, platform_device_id=''):
return self.platform_device.find_stemming_assignment(platform_device_id)
############################
#
# SPECIALIZED FIND METHODS
#
############################
def find_data_product_by_instrument_device(self, instrument_device_id=''):
log.debug("FIND DATA PRODUCT BY INSTRUMENT DEVICE")
#init return value, a list of data products
data_products = []
seen_data_producers = []
#init working set of data producers to walk
data_producers = []
pducers = self.instrument_device.find_stemming_data_producer(instrument_device_id)
data_producers += pducers
#iterate through all un-processed data producers (could also do recursively)
while 0 < len(data_producers):
producer_id = data_producers.pop()
if producer_id in seen_data_producers:
raise Inconsistent("There is a cycle in data producers that includes '%s'" % producer_id)
seen_data_producers.append(producer_id)
log.debug("Analyzing data producer '%s'" % producer_id)
#get any products that are associated with this data producer and return them
#TODO: this belongs in DPMS
new_data_products = self.data_product.find_having_data_producer(producer_id)
#get any producers that receive input from this data producer
#TODO: this belongs in DAMS
new_data_producers = self.data_producer.find_having_input_data_producer(producer_id)
log.debug("Got %d new products, %d new producers" % (len(new_data_products),
len(new_data_producers)))
data_products += new_data_products
data_producers += new_data_producers
return data_products
def find_instrument_device_by_data_product(self, data_product_id=''):
log.debug("FIND INSTRUMENT DEVICE BY DATA PRODUCT")
#init return value, a list of instrument devices
instrument_devices = []
seen_data_producers = []
#init working set of data producers to walk
data_producers = []
#TODO: this belongs in DPMS
pducers = self.data_product.find_stemming_data_producer(data_product_id)
data_producers += pducers
#iterate through all un-processed data producers (could also do recursively)
while 0 < len(data_producers):
producer_id = data_producers.pop()
if producer_id in seen_data_producers:
raise Inconsistent("There is a cycle in data producers that includes '%s'" % producer_id)
seen_data_producers.append(producer_id)
log.debug("Analyzing data producer '%s'" % producer_id)
#get any devices that are associated with this data producer and return them
new_instrument_devices = self.instrument_device.find_having_data_producer(producer_id)
#get any producers that give input to this data producer
#TODO: this belongs in DPMS
new_data_producers = self.data_producer.find_stemming_input_data_producer(producer_id)
log.debug("Got %d new devices, %d new producers" % (len(new_instrument_devices),
len(new_data_producers)))
instrument_devices += new_instrument_devices
data_producers += new_data_producers
return instrument_devices
def find_data_product_by_platform_device(self, platform_device_id=''):
ret = []
for i in self.find_instrument_device_by_platform_device(platform_device_id):
data_products = self.find_data_product_by_instrument_device(i)
for d in data_products:
if not d in ret:
ret.append(d)
return ret
############################
#
# LIFECYCLE TRANSITIONS
#
############################
def set_instrument_agent_lifecycle(self, instrument_agent_id="", lifecycle_state=""):
"""
declare a instrument_agent to be in a given state
@param instrument_agent_id the resource id
"""
return self.instrument_agent.advance_lcs(instrument_agent_id, lifecycle_state)
def set_instrument_agent_instance_lifecycle(self, instrument_agent_instance_id="", lifecycle_state=""):
"""
declare a instrument_agent_instance to be in a given state
@param instrument_agent_instance_id the resource id
"""
return self.instrument_agent_instance.advance_lcs(instrument_agent_instance_id, lifecycle_state)
def set_instrument_model_lifecycle(self, instrument_model_id="", lifecycle_state=""):
"""
declare a instrument_model to be in a given state
@param instrument_model_id the resource id
"""
return self.instrument_model.advance_lcs(instrument_model_id, lifecycle_state)
def set_instrument_device_lifecycle(self, instrument_device_id="", lifecycle_state=""):
"""
declare an instrument_device to be in a given state
@param instrument_device_id the resource id
"""
return self.instrument_device.advance_lcs(instrument_device_id, lifecycle_state)
def set_platform_agent_lifecycle(self, platform_agent_id="", lifecycle_state=""):
"""
declare a platform_agent to be in a given state
@param platform_agent_id the resource id
"""
return self.platform_agent.advance_lcs(platform_agent_id, lifecycle_state)
def set_platform_agent_instance_lifecycle(self, platform_agent_instance_id="", lifecycle_state=""):
"""
declare a platform_agent_instance to be in a given state
@param platform_agent_instance_id the resource id
"""
return self.platform_agent_instance.advance_lcs(platform_agent_instance_id, lifecycle_state)
def set_platform_model_lifecycle(self, platform_model_id="", lifecycle_state=""):
"""
declare a platform_model to be in a given state
@param platform_model_id the resource id
"""
return self.platform_model.advance_lcs(platform_model_id, lifecycle_state)
def set_platform_device_lifecycle(self, platform_device_id="", lifecycle_state=""):
"""
declare a platform_device to be in a given state
@param platform_device_id the resource id
"""
return self.platform_device.advance_lcs(platform_device_id, lifecycle_state)
def set_sensor_model_lifecycle(self, sensor_model_id="", lifecycle_state=""):
"""
declare a sensor_model to be in a given state
@param sensor_model_id the resource id
"""
return self.sensor_model.advance_lcs(sensor_model_id, lifecycle_state)
def set_sensor_device_lifecycle(self, sensor_device_id="", lifecycle_state=""):
"""
declare a sensor_device to be in a given state
@param sensor_device_id the resource id
"""
return self.sensor_device.advance_lcs(sensor_device_id, lifecycle_state)
class DataProductManagementService(BaseDataProductManagementService):
""" @author Bill Bollenbacher
@file ion/services/sa/product/data_product_management_service.py
@brief Implementation of the data product management service
"""
def on_init(self):
self.override_clients(self.clients)
def override_clients(self, new_clients):
"""
Replaces the service clients with a new set of them... and makes sure they go to the right places
"""
self.data_product = DataProductImpl(self.clients)
def create_data_product(self, data_product=None, stream_definition_id=""):
"""
@param data_product IonObject which defines the general data product resource
@param source_resource_id IonObject id which defines the source for the data
@retval data_product_id
"""
# 1. Verify that a data product with same name does not already exist
# 2. Validate that the data product IonObject does not contain an id_ element
# 3. Create a new data product
# - User must supply the name in the data product
# Create will validate and register a new data product within the system
# Validate - TBD by the work that Karen Stocks is driving with John Graybeal
# Register - create and store a new DataProduct resource using provided metadata
log.debug("DataProductManagementService:create_data_product: %s" % str(data_product))
data_product_id = self.data_product.create_one(data_product)
# Create the stream if a stream definition is provided
log.debug("DataProductManagementService:create_data_product: stream definition id = %s" % stream_definition_id)
if stream_definition_id:
stream_id = self.clients.pubsub_management.create_stream(
name=data_product.name, description=data_product.description, stream_definition_id=stream_definition_id
)
log.debug("create_data_product: create stream stream_id %s" % stream_id)
# Associate the Stream with the main Data Product
self.clients.resource_registry.create_association(data_product_id, PRED.hasStream, stream_id)
# Return a resource ref to the new data product
return data_product_id
def read_data_product(self, data_product_id=""):
"""
method docstring
"""
# Retrieve all metadata for a specific data product
# Return data product resource
log.debug("DataProductManagementService:read_data_product: %s" % str(data_product_id))
result = self.data_product.read_one(data_product_id)
return result
def update_data_product(self, data_product=None):
"""
@todo document this interface!!!
@param data_product DataProduct
@throws NotFound object with specified id does not exist
"""
log.debug("DataProductManagementService:update_data_product: %s" % str(data_product))
self.clients.resource_registry.update(data_product)
# TODO: any changes to producer? Call DataAcquisitionMgmtSvc?
return
def delete_data_product(self, data_product_id=""):
# Check if this data product is associated to a producer
producer_ids, _ = self.clients.resource_registry.find_objects(
data_product_id, PRED.hasDataProducer, RT.DataProducer, id_only=True
)
if producer_ids:
log.debug("DataProductManagementService:delete_data_product: %s" % str(producer_ids))
self.clients.data_acquisition_management.unassign_data_product(data_product_id)
# Delete the data product
self.clients.resource_registry.delete(data_product_id)
return
def find_data_products(self, filters=None):
"""
method docstring
"""
# Validate the input filter and augment context as required
# Define set of resource attributes to filter on, change parameter from "filter" to include attributes and filter values.
# potentially: title, keywords, date_created, creator_name, project, geospatial coords, time range
# Call DM DiscoveryService to query the catalog for matches
# Organize and return the list of matches with summary metadata (title, summary, keywords)
# find the items in the store
if filters is None:
objects, _ = self.clients.resource_registry.find_resources(RT.DataProduct, None, None, False)
else: # TODO: code for all the filter types
objects = []
return objects
def activate_data_product_persistence(self, data_product_id="", persist_data=True, persist_metadata=True):
"""Persist data product data into a data set
@param data_product_id str
@throws NotFound object with specified id does not exist
"""
# retrieve the data_process object
data_product_obj = self.clients.resource_registry.read(data_product_id)
# get the Stream associated with this data set; if no stream then create one, if multiple streams then Throw
streams, _ = self.clients.resource_registry.find_objects(data_product_id, PRED.hasStream, RT.Stream, True)
if not streams:
raise BadRequest("Data Product %s must have one stream associated" % str(data_product_id))
stream = streams[0]
log.debug("activate_data_product_persistence: stream = %s" % str(stream))
# Find THE ingestion configuration in the RR to create a ingestion configuration
# todo: how are multiple ingest configs for a site managed?
ingest_config_objs, _ = self.clients.resource_registry.find_resources(
restype=RT.IngestionConfiguration, id_only=False
)
if len(ingest_config_objs) != 1:
log.debug("activate_data_product_persistence: ERROR ingest_config_objs = %s" % str(ingest_config_objs))
raise BadRequest("Data Product must have one ingestion configuration %s" % str(data_product_id))
ingestion_configuration_obj = ingest_config_objs[0]
log.debug(
"activate_data_product_persistence: ingestion_configuration_obj = %s" % str(ingestion_configuration_obj)
)
if data_product_obj.dataset_id:
objs, _ = self.clients.resource_registry.find_objects(
data_product_obj.dataset_id,
PRED.hasIngestionConfiguration,
RT.DatasetIngestionConfiguration,
id_only=False,
)
if not objs:
log.debug(
"activate_data_product_persistence: Calling create_dataset_configuration for EXISTING Dataset"
)
dataset_configuration_id = self.clients.ingestion_management.create_dataset_configuration(
dataset_id=data_product_obj.dataset_id,
archive_data=persist_data,
archive_metadata=persist_metadata,
ingestion_configuration_id=ingestion_configuration_obj._id,
)
log.debug(
"activate_data_product_persistence: create_dataset_configuration = %s"
% str(dataset_configuration_id)
)
else:
dataset_configuration_obj = objs[0]
dataset_configuration_obj.configuration.archive_data = persist_data
dataset_configuration_obj.configuration.archive_metadata = persist_metadata
# call ingestion management to update a dataset configuration
log.debug("activate_data_product_persistence: Calling update_dataset_config")
dataset_configuration_id = self.clients.ingestion_management.update_dataset_config(
dataset_configuration_obj
)
log.debug(
"activate_data_product_persistence: update_dataset_config = %s" % str(dataset_configuration_id)
)
else:
# create the dataset for the data
# !!!!!!!! (Currently) The Datastore name MUST MATCH the ingestion configuration name!!!
data_product_obj.dataset_id = self.clients.dataset_management.create_dataset(
stream_id=stream,
datastore_name=ingestion_configuration_obj.couch_storage.datastore_name,
description=data_product_obj.description,
)
log.debug("activate_data_product_persistence: create_dataset = %s" % str(data_product_obj.dataset_id))
self.update_data_product(data_product_obj)
# Need to read again, because the _rev has changed. Otherwise error on update later.
data_product_obj = self.clients.resource_registry.read(data_product_id)
# call ingestion management to create a dataset configuration
log.debug("activate_data_product_persistence: Calling create_dataset_configuration")
dataset_configuration_id = self.clients.ingestion_management.create_dataset_configuration(
dataset_id=data_product_obj.dataset_id,
archive_data=persist_data,
archive_metadata=persist_metadata,
ingestion_configuration_id=ingestion_configuration_obj._id,
)
log.debug(
"activate_data_product_persistence: create_dataset_configuration = %s" % str(dataset_configuration_id)
)
# save the dataset_configuration_id in the product resource? Can this be found via the stream id?
data_product_obj.dataset_configuration_id = dataset_configuration_id
self.update_data_product(data_product_obj)
def suspend_data_product_persistence(self, data_product_id=""):
"""Suspend data product data persistnce into a data set, multiple options
@param data_product_id str
@param type str
@throws NotFound object with specified id does not exist
"""
# retrieve the data_process object
data_product_obj = self.clients.resource_registry.read(data_product_id)
if data_product_obj is None:
raise NotFound("Data Product %s does not exist" % data_product_id)
if data_product_obj.dataset_configuration_id is None:
raise NotFound("Data Product %s dataset configuration does not exist" % data_product_id)
# retrieve the dataset configuation object so that attrs can be changed
dataset_configuration_obj = self.clients.resource_registry.read(data_product_obj.dataset_configuration_id)
if dataset_configuration_obj is None:
raise NotFound("Dataset Configuration %s does not exist" % data_product_obj.dataset_configuration_id)
# Set the dataset config archive data/metadata attrs to false
dataset_configuration_obj.configuration.archive_data = False
dataset_configuration_obj.configuration.archive_metadata = False
ret = self.clients.ingestion_management.update_dataset_config(dataset_configuration_obj)
log.debug("suspend_data_product_persistence: deactivate = %s" % str(ret))
def set_data_product_lifecycle(self, data_product_id="", lifecycle_state=""):
"""
declare a data_product to be in a given state
@param data_product_id the resource id
"""
return self.data_product.advance_lcs(data_product_id, lifecycle_state)
def get_last_update(self, data_product_id=""):
"""@todo document this interface!!!
@param data_product_id str
@retval last_update LastUpdate
@throws NotFound Data product not found or cache for data product not found.
"""
from ion.processes.data.last_update_cache import CACHE_DATASTORE_NAME
datastore_name = CACHE_DATASTORE_NAME
db = self.container.datastore_manager.get_datastore(datastore_name)
stream_ids, other = self.clients.resource_registry.find_objects(
subject=data_product_id, predicate=PRED.hasStream, id_only=True
)
retval = {}
for stream_id in stream_ids:
try:
lu = db.read(stream_id)
retval[stream_id] = lu
except NotFound:
continue
return retval
def override_clients(self, new_clients):
"""
Replaces the service clients with a new set of them... and makes sure they go to the right places
"""
self.data_product = DataProductImpl(self.clients)
class DataProductManagementService(BaseDataProductManagementService):
""" @author Bill Bollenbacher
@file ion/services/sa/product/data_product_management_service.py
@brief Implementation of the data product management service
"""
def on_init(self):
self.override_clients(self.clients)
def override_clients(self, new_clients):
"""
Replaces the service clients with a new set of them... and makes sure they go to the right places
"""
self.data_product = DataProductImpl(self.clients)
def create_data_product(self, data_product=None, source_resource_id=''):
"""
@param data_product IonObject which defines the general data product resource
@param source_resource_id IonObject id which defines the source for the data
@retval data_product_id
"""
# 1. Verify that a data product with same name does not already exist
# 2. Validate that the data product IonObject does not contain an id_ element
# 3. Create a new data product
# - User must supply the name in the data product
# Create will validate and register a new data product within the system
# Validate - TBD by the work that Karen Stocks is driving with John Graybeal
# Register - create and store a new DataProduct resource using provided metadata
log.debug("DataProductManagementService:create_data_product: %s" % str(data_product))
data_product_id = self.data_product.create_one(data_product)
if source_resource_id:
log.debug("DataProductManagementService:create_data_product: source resource id = %s" % source_resource_id)
# TODO: currently create stream for the product is ALWAYS on, this should be surfaced
# Call Data Aquisition Mgmt Svc:assign_data_product to coordinate connection of the data product to data producer and to the source resource
self.clients.data_acquisition_management.assign_data_product(source_resource_id, data_product_id, True) # TODO: what errors can occur here?
# todo: should this method create the hasOutputProduct association?
self.clients.resource_registry.create_association(source_resource_id, PRED.hasOutputProduct, data_product_id)
# Return a resource ref to the new data product
return data_product_id
def read_data_product(self, data_product_id=''):
"""
method docstring
"""
# Retrieve all metadata for a specific data product
# Return data product resource
log.debug("DataProductManagementService:read_data_product: %s" % str(data_product_id))
result = self.data_product.read_one(data_product_id)
return result
def update_data_product(self, data_product=None):
"""
@todo document this interface!!!
@param data_product DataProduct
@throws NotFound object with specified id does not exist
"""
log.debug("DataProductManagementService:update_data_product: %s" % str(data_product))
self.data_product.update_one(data_product)
#TODO: any changes to producer? Call DataAcquisitionMgmtSvc?
return
def delete_data_product(self, data_product_id=''):
#Check if this data product is associated to a producer
producer_ids, _ = self.clients.resource_registry.find_objects(data_product_id, PRED.hasDataProducer, RT.DataProducer, id_only=True)
if producer_ids:
log.debug("DataProductManagementService:delete_data_product: %s" % str(producer_ids))
self.clients.data_acquisition_management.unassign_data_product(data_product_id)
# Delete the data process
self.clients.resource_registry.delete(data_product_id)
return
def find_data_products(self, filters=None):
"""
method docstring
"""
# Validate the input filter and augment context as required
# Define set of resource attributes to filter on, change parameter from "filter" to include attributes and filter values.
# potentially: title, keywords, date_created, creator_name, project, geospatial coords, time range
# Call DM DiscoveryService to query the catalog for matches
# Organize and return the list of matches with summary metadata (title, summary, keywords)
#find the items in the store
if filters is None:
objects, _ = self.clients.resource_registry.find_resources(RT.DataProduct, None, None, False)
else: # TODO: code for all the filter types
objects = []
return objects
def activate_data_product_persistence(self, data_product_id='', persist_data=True, persist_metadata=True):
"""Persist data product data into a data set
@param data_product_id str
@throws NotFound object with specified id does not exist
"""
# retrieve the data_process object
data_product_obj = self.clients.resource_registry.read(data_product_id)
if data_product_obj is None:
raise NotFound("Data Product %s does not exist" % data_product_id)
# get the Stream associated with this data set; if no stream then create one, if multiple streams then Throw
streams, _ = self.clients.resource_registry.find_objects(data_product_id, PRED.hasStream, RT.Stream, True)
if len(streams) > 1 or len(streams) == 0:
raise BadRequest('Data Product must have one stream associated%s' % str(data_product_id))
stream = streams[0]
# Call ingestion management to create a ingestion configuration
# Configure ingestion using eight workers, ingesting to test_dm_integration datastore with the SCIDATA profile
log.debug('activate_data_product_persistence: Calling create_ingestion_configuration')
data_product_obj.ingestion_configuration_id = self.clients.ingestion_management.create_ingestion_configuration(
exchange_point_id='science_data',
couch_storage=CouchStorage(datastore_name=self.datastore_name,datastore_profile='SCIDATA'),
number_of_workers=8
)
#todo: does DPMS need to save the ingest _config_id in the product resource? Can this be found via the stream id?
# activate an ingestion configuration
#todo: Does DPMS call activate?
ret = self.clients.ingestion_management.activate_ingestion_configuration(data_product_obj.ingestion_configuration_id)
# create the dataset for the data
data_product_obj.dataset_id = self.clients.dataset_management.create_dataset(self, stream, data_product_obj.name, data_product_obj.description)
self.clients.resource_registry.update(data_product_obj)
# Call ingestion management to create a dataset configuration
log.debug('activate_data_product_persistence: Calling create_dataset_configuration')
dataset_configuration_id = self.clients.ingestion_management.create_dataset_configuration( dataset_id, persist_data, persist_metadata, ingestion_configuration_id)
#todo: does DPMS need to save the dataset_configuration_id in the product resource? Can this be found via the stream id?
return
def suspend_data_product_persistence(self, data_product_id=''):
"""Suspend data product data persistnce into a data set, multiple options
@param data_product_id str
@param type str
@throws NotFound object with specified id does not exist
"""
# retrieve the data_process object
data_product_obj = self.clients.resource_registry.read(data_product_id)
if data_product_obj is None:
raise NotFound("Data Product %s does not exist" % data_product_id)
if data_product_obj.ingestion_configuration_id is None:
raise NotFound("Data Product %s ingestion configuration does not exist" % data_product_id)
# Change the stream policy to stop ingestion
self.clients.ingestion_management.deactivate_ingestion_configuration(data_product_obj.ingestion_configuration_id)
return
def set_data_product_lifecycle(self, data_product_id="", lifecycle_state=""):
"""
declare a data_product to be in a given state
@param data_product_id the resource id
"""
return self.data_product.advance_lcs(data_product_id, lifecycle_state)