def override_clients(self, new_clients):
        """
        Replaces the service clients with a new set of them... and makes sure they go to the right places
        """

        #shortcut names for the import sub-services
        # we hide these behind checks even though we expect them so that
        # the resource_impl_metatests will work
        if hasattr(self.clients, "resource_registry"):
            self.RR    = self.clients.resource_registry
            
        if hasattr(self.clients, "data_acquisition_management"):
            self.DAMS  = self.clients.data_acquisition_management

        if hasattr(self.clients, "data_product_management"):
            self.DPMS  = self.clients.data_product_management

        #farm everything out to the impls

        self.instrument_agent           = InstrumentAgentImpl(self.clients)
        self.instrument_agent_instance  = InstrumentAgentInstanceImpl(self.clients)
        self.instrument_model           = InstrumentModelImpl(self.clients)
        self.instrument_device          = InstrumentDeviceImpl(self.clients)

        self.platform_agent           = PlatformAgentImpl(self.clients)
        self.platform_agent_instance  = PlatformAgentInstanceImpl(self.clients)
        self.platform_model           = PlatformModelImpl(self.clients)
        self.platform_device          = PlatformDeviceImpl(self.clients)

        self.sensor_model    = SensorModelImpl(self.clients)
        self.sensor_device   = SensorDeviceImpl(self.clients)

        #TODO: may not belong in this service
        self.data_product        = DataProductImpl(self.clients)
        self.data_producer       = DataProducerImpl(self.clients)
        self.logical_instrument  = LogicalInstrumentImpl(self.clients)
class InstrumentManagementService(BaseInstrumentManagementService):
    """
    @brief Service to manage instrument, platform, and sensor resources, their relationships, and direct access

    """
    def on_init(self):
        #suppress a few "variable declared but not used" annoying pyflakes errors
        IonObject("Resource") 
        log 

        self.override_clients(self.clients)

    def override_clients(self, new_clients):
        """
        Replaces the service clients with a new set of them... and makes sure they go to the right places
        """

        #shortcut names for the import sub-services
        # we hide these behind checks even though we expect them so that
        # the resource_impl_metatests will work
        if hasattr(self.clients, "resource_registry"):
            self.RR    = self.clients.resource_registry
            
        if hasattr(self.clients, "data_acquisition_management"):
            self.DAMS  = self.clients.data_acquisition_management

        if hasattr(self.clients, "data_product_management"):
            self.DPMS  = self.clients.data_product_management

        #farm everything out to the impls

        self.instrument_agent           = InstrumentAgentImpl(self.clients)
        self.instrument_agent_instance  = InstrumentAgentInstanceImpl(self.clients)
        self.instrument_model           = InstrumentModelImpl(self.clients)
        self.instrument_device          = InstrumentDeviceImpl(self.clients)

        self.platform_agent           = PlatformAgentImpl(self.clients)
        self.platform_agent_instance  = PlatformAgentInstanceImpl(self.clients)
        self.platform_model           = PlatformModelImpl(self.clients)
        self.platform_device          = PlatformDeviceImpl(self.clients)

        self.sensor_model    = SensorModelImpl(self.clients)
        self.sensor_device   = SensorDeviceImpl(self.clients)

        #TODO: may not belong in this service
        self.data_product        = DataProductImpl(self.clients)
        self.data_producer       = DataProducerImpl(self.clients)
        self.logical_instrument  = LogicalInstrumentImpl(self.clients)


    ##########################################################################
    #
    # INSTRUMENT AGENT INSTANCE
    #
    ##########################################################################

    def create_instrument_agent_instance(self, instrument_agent_instance=None):
        """
        create a new instance
        @param instrument_agent_instance the object to be created as a resource
        @retval instrument_agent_instance_id the id of the new object
        @throws BadRequest if the incoming _id field is set
        @throws BadReqeust if the incoming name already exists
        """
        return self.instrument_agent_instance.create_one(instrument_agent_instance)

    def update_instrument_agent_instance(self, instrument_agent_instance=None):
        """
        update an existing instance
        @param instrument_agent_instance the object to be created as a resource
        @retval success whether we succeeded
        @throws BadRequest if the incoming _id field is not set
        @throws BadReqeust if the incoming name already exists
        """
        return self.instrument_agent_instance.update_one(instrument_agent_instance)


    def read_instrument_agent_instance(self, instrument_agent_instance_id=''):
        """
        fetch a resource by ID
        @param instrument_agent_instance_id the id of the object to be fetched
        @retval InstrumentAgentInstance resource
        """
        return self.instrument_agent_instance.read_one(instrument_agent_instance_id)

    def delete_instrument_agent_instance(self, instrument_agent_instance_id=''):
        """
        delete a resource, including its history (for less ominous deletion, use retire)
        @param instrument_agent_instance_id the id of the object to be deleted
        @retval success whether it succeeded

        """
        return self.instrument_agent_instance.delete_one(instrument_agent_instance_id)

    def find_instrument_agent_instances(self, filters=None):
        """

        """
        return self.instrument_agent_instance.find_some(filters)




    ##########################################################################
    #
    # INSTRUMENT AGENT
    #
    ##########################################################################

    def create_instrument_agent(self, instrument_agent=None):
        """
        create a new instance
        @param instrument_agent the object to be created as a resource
        @retval instrument_agent_id the id of the new object
        @throws BadRequest if the incoming _id field is set
        @throws BadReqeust if the incoming name already exists
        """
        return self.instrument_agent.create_one(instrument_agent)

    def update_instrument_agent(self, instrument_agent=None):
        """
        update an existing instance
        @param instrument_agent the object to be created as a resource
        @retval success whether we succeeded
        @throws BadRequest if the incoming _id field is not set
        @throws BadReqeust if the incoming name already exists
        """
        return self.instrument_agent.update_one(instrument_agent)


    def read_instrument_agent(self, instrument_agent_id=''):
        """
        fetch a resource by ID
        @param instrument_agent_id the id of the object to be fetched
        @retval InstrumentAgent resource
        """
        return self.instrument_agent.read_one(instrument_agent_id)

    def delete_instrument_agent(self, instrument_agent_id=''):
        """
        delete a resource, including its history (for less ominous deletion, use retire)
        @param instrument_agent_id the id of the object to be deleted
        @retval success whether it succeeded

        """
        return self.instrument_agent.delete_one(instrument_agent_id)

    def find_instrument_agents(self, filters=None):
        """

        """
        return self.instrument_agent.find_some(filters)



    ##########################################################################
    #
    # INSTRUMENT MODEL
    #
    ##########################################################################

    def create_instrument_model(self, instrument_model=None):
        """
        create a new instance
        @param instrument_model the object to be created as a resource
        @retval instrument_model_id the id of the new object
        @throws BadRequest if the incoming _id field is set
        @throws BadReqeust if the incoming name already exists
        """
        return self.instrument_model.create_one(instrument_model)

    def update_instrument_model(self, instrument_model=None):
        """
        update an existing instance
        @param instrument_model the object to be created as a resource
        @retval success whether we succeeded
        @throws BadRequest if the incoming _id field is not set
        @throws BadReqeust if the incoming name already exists
        """
        return self.instrument_model.update_one(instrument_model)


    def read_instrument_model(self, instrument_model_id=''):
        """
        fetch a resource by ID
        @param instrument_model_id the id of the object to be fetched
        @retval InstrumentModel resource
        """
        return self.instrument_model.read_one(instrument_model_id)

    def delete_instrument_model(self, instrument_model_id=''):
        """
        delete a resource, including its history (for less ominous deletion, use retire)
        @param instrument_model_id the id of the object to be deleted
        @retval success whether it succeeded

        """
        return self.instrument_model.delete_one(instrument_model_id)

    def find_instrument_models(self, filters=None):
        """

        """
        return self.instrument_model.find_some(filters)





    ##########################################################################
    #
    # PHYSICAL INSTRUMENT
    #
    ##########################################################################


    def setup_data_production_chain(self, instrument_device_id=''):
        """
        create a data product (L0) for the instrument, and establish provenance
        between the corresponding data producers
        """

        #get instrument object and instrument's data producer
        inst_obj = self.instrument_device.read_one(instrument_device_id)
        inst_pducers = self.instrument_device.find_stemming_data_producer(instrument_device_id)
        inst_pducer_id = inst_pducers[0]
        log.debug("instrument data producer id='%s'" % inst_pducer_id)

        #create a new data product
        dpms_pduct_obj = IonObject(RT.DataProduct,
                                   name=str(inst_obj.name + " L0 Product"),
                                   description=str("L0 DataProduct for " + inst_obj.name))

        pduct_id = self.DPMS.create_data_product(dpms_pduct_obj)

        #TODO: DPMS isn't creating a data producer for new data products. not sure why.
        #
        prod_pducer_id = self.data_producer.create_one(IonObject(RT.DataProducer,
                                                                 name=str(inst_obj.name + " L0 Producer"),
                                                                 description=str("L0 DataProducer for " + inst_obj.name)))
        self.data_product.link_data_producer(pduct_id, prod_pducer_id)

        # get data product's data producer (via association)
        #TODO: this belongs in DPMS
        prod_pducers = self.data_product.find_stemming_data_producer(pduct_id)
        
        # (TODO: there should only be one assoc_id.  what error to raise?)
        # TODO: what error to raise if there are no assoc ids?
        prod_pducer_id = prod_pducers[0]


        # instrument data producer is the parent of the data product producer
        #TODO: this belongs in DAMS
        self.data_producer.link_input_data_producer(prod_pducer_id, inst_pducer_id)

        #TODO: error checking



    def create_instrument_device(self, instrument_device=None):
        """
        create a new instance
        @param instrument_device the object to be created as a resource
        @retval instrument_device_id the id of the new object
        @throws BadRequest if the incoming _id field is set
        @throws BadReqeust if the incoming name already exists
        """
        instrument_device_id = self.instrument_device.create_one(instrument_device)
        self.DAMS.register_instrument(instrument_device_id)
        
        #TODO: create data producer and product
        self.setup_data_production_chain(instrument_device_id)

        return instrument_device_id

    def update_instrument_device(self, instrument_device=None):
        """
        update an existing instance
        @param instrument_device the object to be created as a resource
        @retval success whether we succeeded
        @throws BadRequest if the incoming _id field is not set
        @throws BadReqeust if the incoming name already exists
        """
        return self.instrument_device.update_one(instrument_device)


    def read_instrument_device(self, instrument_device_id=''):
        """
        fetch a resource by ID
        @param instrument_device_id the id of the object to be fetched
        @retval InstrumentDevice resource

        """
        return self.instrument_device.read_one(instrument_device_id)

    def delete_instrument_device(self, instrument_device_id=''):
        """
        delete a resource, including its history (for less ominous deletion, use retire)
        @param instrument_device_id the id of the object to be deleted
        @retval success whether it succeeded

        """
        return self.instrument_device.delete_one(instrument_device_id)

    def find_instrument_devices(self, filters=None):
        """

        """
        return self.instrument_device.find_some(filters)



    ##
    ##
    ##  DIRECT ACCESS
    ##
    ##

    def request_direct_access(self, instrument_device_id=''):
        """

        """

        # determine whether id is for physical or logical instrument
        # look up instrument if not

        # Validate request; current instrument state, policy, and other

        # Retrieve and save current instrument settings

        # Request DA channel, save reference

        # Return direct access channel
        raise NotImplementedError()
        pass

    def stop_direct_access(self, instrument_device_id=''):
        """

        """
        # Return Value
        # ------------
        # {success: true}
        #
        raise NotImplementedError()
        pass







    ##########################################################################
    #
    # PLATFORM AGENT INSTANCE
    #
    ##########################################################################

    def create_platform_agent_instance(self, platform_agent_instance=None):
        """
        create a new instance
        @param platform_agent_instance the object to be created as a resource
        @retval platform_agent_instance_id the id of the new object
        @throws BadRequest if the incoming _id field is set
        @throws BadReqeust if the incoming name already exists
        """
        return self.platform_agent_instance.create_one(platform_agent_instance)

    def update_platform_agent_instance(self, platform_agent_instance=None):
        """
        update an existing instance
        @param platform_agent_instance the object to be created as a resource
        @retval success whether we succeeded
        @throws BadRequest if the incoming _id field is not set
        @throws BadReqeust if the incoming name already exists
        """
        return self.platform_agent_instance.update_one(platform_agent_instance)


    def read_platform_agent_instance(self, platform_agent_instance_id=''):
        """
        fetch a resource by ID
        @param platform_agent_instance_id the id of the object to be fetched
        @retval PlatformAgentInstance resource
        """
        return self.platform_agent_instance.read_one(platform_agent_instance_id)

    def delete_platform_agent_instance(self, platform_agent_instance_id=''):
        """
        delete a resource, including its history (for less ominous deletion, use retire)
        @param platform_agent_instance_id the id of the object to be deleted
        @retval success whether it succeeded

        """
        return self.platform_agent_instance.delete_one(platform_agent_instance_id)

    def find_platform_agent_instances(self, filters=None):
        """

        """
        return self.platform_agent_instance.find_some(filters)






    ##########################################################################
    #
    # PLATFORM AGENT
    #
    ##########################################################################


    def create_platform_agent(self, platform_agent=None):
        """
        create a new instance
        @param platform_agent the object to be created as a resource
        @retval platform_agent_id the id of the new object
        @throws BadRequest if the incoming _id field is set
        @throws BadReqeust if the incoming name already exists
        """
        return self.platform_agent.create_one(platform_agent)

    def update_platform_agent(self, platform_agent=None):
        """
        update an existing instance
        @param platform_agent the object to be created as a resource
        @retval success whether we succeeded
        @throws BadRequest if the incoming _id field is not set
        @throws BadReqeust if the incoming name already exists

        """
        return self.platform_agent.update_one(platform_agent)


    def read_platform_agent(self, platform_agent_id=''):
        """
        fetch a resource by ID
        @param platform_agent_id the id of the object to be fetched
        @retval PlatformAgent resource

        """
        return self.platform_agent.read_one(platform_agent_id)

    def delete_platform_agent(self, platform_agent_id=''):
        """
        delete a resource, including its history (for less ominous deletion, use retire)
        @param platform_agent_id the id of the object to be deleted
        @retval success whether it succeeded

        """
        return self.platform_agent.delete_one(platform_agent_id)

    def find_platform_agents(self, filters=None):
        """

        """
        return self.platform_agent.find_some(filters)



    ##########################################################################
    #
    # PLATFORM MODEL
    #
    ##########################################################################


    def create_platform_model(self, platform_model=None):
        """
        create a new instance
        @param platform_model the object to be created as a resource
        @retval platform_model_id the id of the new object
        @throws BadRequest if the incoming _id field is set
        @throws BadReqeust if the incoming name already exists
        """
        return self.platform_model.create_one(platform_model)

    def update_platform_model(self, platform_model=None):
        """
        update an existing instance
        @param platform_model the object to be created as a resource
        @retval success whether we succeeded
        @throws BadRequest if the incoming _id field is not set
        @throws BadReqeust if the incoming name already exists
        """
        return self.platform_model.update_one(platform_model)


    def read_platform_model(self, platform_model_id=''):
        """
        fetch a resource by ID
        @param platform_model_id the id of the object to be fetched
        @retval PlatformModel resource

        """
        return self.platform_model.read_one(platform_model_id)

    def delete_platform_model(self, platform_model_id=''):
        """
        delete a resource, including its history (for less ominous deletion, use retire)
        @param platform_model_id the id of the object to be deleted
        @retval success whether it succeeded

        """
        return self.platform_model.delete_one(platform_model_id)

    def find_platform_models(self, filters=None):
        """

        """
        return self.platform_model.find_some(filters)




    ##########################################################################
    #
    # PHYSICAL PLATFORM
    #
    ##########################################################################



    def create_platform_device(self, platform_device=None):
        """
        create a new instance
        @param platform_device the object to be created as a resource
        @retval platform_device_id the id of the new object
        @throws BadRequest if the incoming _id field is set
        @throws BadReqeust if the incoming name already exists
        """
        return self.platform_device.create_one(platform_device)

    def update_platform_device(self, platform_device=None):
        """
        update an existing instance
        @param platform_device the object to be created as a resource
        @retval success whether we succeeded
        @throws BadRequest if the incoming _id field is not set
        @throws BadReqeust if the incoming name already exists

        """
        return self.platform_device.update_one(platform_device)


    def read_platform_device(self, platform_device_id=''):
        """
        fetch a resource by ID
        @param platform_device_id the id of the object to be fetched
        @retval PlatformDevice resource

        """
        return self.platform_device.read_one(platform_device_id)

    def delete_platform_device(self, platform_device_id=''):
        """
        delete a resource, including its history (for less ominous deletion, use retire)
        @param platform_device_id the id of the object to be deleted
        @retval success whether it succeeded

        """
        return self.platform_device.delete_one(platform_device_id)

    def find_platform_devices(self, filters=None):
        """

        """
        return self.platform_device.find_some(filters)






    ##########################################################################
    #
    # SENSOR MODEL
    #
    ##########################################################################


    def create_sensor_model(self, sensor_model=None):
        """
        create a new instance
        @param sensor_model the object to be created as a resource
        @retval sensor_model_id the id of the new object
        @throws BadRequest if the incoming _id field is set
        @throws BadReqeust if the incoming name already exists
        """
        return self.sensor_model.create_one(sensor_model)

    def update_sensor_model(self, sensor_model=None):
        """
        update an existing instance
        @param sensor_model the object to be created as a resource
        @retval success whether we succeeded
        @throws BadRequest if the incoming _id field is not set
        @throws BadReqeust if the incoming name already exists

        """
        return self.sensor_model.update_one(sensor_model)


    def read_sensor_model(self, sensor_model_id=''):
        """
        fetch a resource by ID
        @param sensor_model_id the id of the object to be fetched
        @retval SensorModel resource

        """
        return self.sensor_model.read_one(sensor_model_id)

    def delete_sensor_model(self, sensor_model_id=''):
        """
        delete a resource, including its history (for less ominous deletion, use retire)
        @param sensor_model_id the id of the object to be deleted
        @retval success whether it succeeded

        """
        return self.sensor_model.delete_one(sensor_model_id)

    def find_sensor_models(self, filters=None):
        """

        """
        return self.sensor_model.find_some(filters)



    ##########################################################################
    #
    # PHYSICAL SENSOR
    #
    ##########################################################################



    def create_sensor_device(self, sensor_device=None):
        """
        create a new instance
        @param sensor_device the object to be created as a resource
        @retval sensor_device_id the id of the new object
        @throws BadRequest if the incoming _id field is set
        @throws BadReqeust if the incoming name already exists
        """
        return self.sensor_device.create_one(sensor_device)

    def update_sensor_device(self, sensor_device=None):
        """
        update an existing instance
        @param sensor_device the object to be created as a resource
        @retval success whether we succeeded
        @throws BadRequest if the incoming _id field is not set
        @throws BadReqeust if the incoming name already exists

        """
        return self.sensor_device.update_one(sensor_device)


    def read_sensor_device(self, sensor_device_id=''):
        """
        fetch a resource by ID
        @param sensor_device_id the id of the object to be fetched
        @retval SensorDevice resource

        """
        return self.sensor_device.read_one(sensor_device_id)

    def delete_sensor_device(self, sensor_device_id=''):
        """
        delete a resource, including its history (for less ominous deletion, use retire)
        @param sensor_device_id the id of the object to be deleted
        @retval success whether it succeeded

        """
        return self.sensor_device.delete_one(sensor_device_id)

    def find_sensor_devices(self, filters=None):
        """

        """
        return self.sensor_device.find_some(filters)



    ##########################################################################
    #
    # ASSOCIATIONS
    #
    ##########################################################################


    def assign_instrument_model_to_instrument_device(self, instrument_model_id='', instrument_device_id=''):
        self.instrument_device.link_model(instrument_device_id, instrument_model_id)

    def unassign_instrument_model_from_instrument_device(self, instrument_model_id='', instrument_device_id=''):
        self.instrument_device.unlink_model(instrument_device_id, instrument_model_id)

    def assign_instrument_model_to_instrument_agent(self, instrument_model_id='', instrument_agent_id=''):
        self.instrument_agent.link_model(instrument_agent_id, instrument_model_id)

    def unassign_instrument_model_from_instrument_agent(self, instrument_model_id='', instrument_agent_id=''):
        self.instrument_agent.unlink_model(instrument_agent_id, instrument_model_id)


    def assign_sensor_model_to_sensor_device(self, sensor_model_id='', sensor_device_id=''):
        self.sensor_device.link_model(sensor_device_id, sensor_model_id)

    def unassign_sensor_model_from_sensor_device(self, sensor_model_id='', sensor_device_id=''):
        self.sensor_device.unlink_model(sensor_device_id, sensor_model_id)

    def assign_platform_model_to_platform_device(self, platform_model_id='', platform_device_id=''):
        self.platform_device.link_model(platform_device_id, platform_model_id)

    def unassign_platform_model_from_platform_device(self, platform_model_id='', platform_device_id=''):
        self.platform_device.unlink_model(platform_device_id, platform_model_id)

    def assign_instrument_device_to_platform_device(self, instrument_device_id='', platform_device_id=''):
        self.platform_device.link_instrument(platform_device_id, instrument_device_id)

    def unassign_instrument_device_from_platform_device(self, instrument_device_id='', platform_device_id=''):
        self.platform_device.unlink_instrument(platform_device_id, instrument_device_id)

    def assign_logical_instrument_to_instrument_device(self, logical_instrument_id='', instrument_device_id=''):
        self.instrument_device.link_assignment(instrument_device_id, logical_instrument_id)

    def unassign_logical_instrument_from_instrument_device(self, logical_instrument_id='', instrument_device_id=''):
        self.instrument_device.unlink_assignment(instrument_device_id, logical_instrument_id)

    def assign_logical_platform_to_platform_device(self, logical_platform_id='', platform_device_id=''):
        self.platform_device.link_assignment(platform_device_id, logical_platform_id)

    def unassign_logical_platform_from_platform_device(self, logical_platform_id='', platform_device_id=''):
        self.platform_device.unlink_assignment(platform_device_id, logical_platform_id)

    def assign_platform_agent_instance_to_platform_agent(self, platform_agent_instance_id='', platform_agent_id=''):
        self.platform_agent.link_instance(platform_agent_id, platform_agent_instance_id)

    def unassign_platform_agent_instance_from_platform_agent(self, platform_agent_instance_id='', platform_agent_id=''):
        self.platform_agent.unlink_instance(platform_agent_id, platform_agent_instance_id)

    def assign_instrument_agent_instance_to_instrument_agent(self, instrument_agent_instance_id='', instrument_agent_id=''):
        self.instrument_agent.link_instance(instrument_agent_id, instrument_agent_instance_id)

    def unassign_instrument_agent_instance_from_instrument_agent(self, instrument_agent_instance_id='', instrument_agent_id=''):
        self.instrument_agent.unlink_instance(instrument_agent_id, instrument_agent_instance_id)


    # reassigning a logical instrument to an instrument device is a little bit special
    # TODO: someday we may be able to dig up the correct data products automatically,
    #       but once we have them this is the function that does all the work.
    def reassign_logical_instrument_to_instrument_device(self, logical_instrument_id='', 
                                                         old_instrument_device_id='', 
                                                         new_instrument_device_id='',
                                                         logical_data_product_ids=[],
                                                         old_instrument_data_product_ids=[],
                                                         new_instrument_data_product_ids=[]):
        """
        associate a logical instrument with a physical one.  this involves linking the
        physical instrument's data product(s) to the logical one(s).
        
        the 2 lists of data products must be of equal length, and will map 1-1

        @param logical_instrument_id
        @param instrument_device_id
        @param logical_data_product_ids a list of data products associated to a logical instrument
        @param instrument_data_product_ids a list of data products coming from an instrument device
        """
 
        
        def verify_dp_origin(supplied_dps, assigned_dps, instrument_id, instrument_label):
            """
            check that the supplied dps (data products) are in the set of what's actually assigned
            @param supplied_dps list of data product ids
            @param assigned_dps list of data product ids
            @param instrument_id a logical or instrument device id
            """
            badones = []
            for p in supplied_dps:
                if not p in assigned_dps:
                    badones.append(p)
                    if 0 < len(badones):
                        raise BadRequest("want to assign %s's data products, but the following were supplied " +
                                         "that don't seem to come from %s '%s': [%s]" %
                                         (instrument_label, instrument_label, instrument_id, ", ".join(badones)))


        log.info("Checking consistency of existing logical/instrument assignments")
        existing_assignments = self.instrument_device.find_having_assignment(logical_instrument_id)
        if 1 < len(existing_assignments):
            raise Inconsistent("There is more than 1 instrument device associated with logical instrument '%s'" %
                               logical_instrument_id)

        log.info("Checking whether supplied logical/instrument arguments are proper")
        if 0 < len(existing_assignments):
            if not old_instrument_device_id:
                raise BadRequest(("Tried to assign logical instrument '%s' for the first time, but it is already " + 
                                  "assigned to instrument device '%s'") % (logical_instrument_id, existing_assignments[0]))
            elif old_instrument_device_id != existing_assignments[0]:
                raise BadRequest(("Tried to reassign logical instrument '%s' from instrument device '%s' but it is " +
                                  "actually associated to instrument device '%s'") % 
                                 (logical_instrument_id, old_instrument_device_id, existing_assignments[0]))


        # log.info("Checking whether supplied data products are proper")
        #  existing_logical_data_products = self.logical_instrument.find_stemming_data_product(logical_instrument_id)
        #
        #TODO: need a check that all the logical data products are being provided for
        #
        # log.info("Checking whether all logical data products are provided")
        # if len(logical_data_product_ids) != len(existing_logical_data_products):
        #     raise BadRequest("tried to assign logical instrument but only provided %d of %d " +
        #                      "data products" % (len(logical_data_product_ids), len(existing_logical_data_products)))
        #
        # log.info("Checking that supplied logical data products are properly rooted")
        # verify_dp_origin(logical_data_product_ids,
        #                  existing_logical_data_products,
        #                  logical_instrument_id,
        #                  "logical_instrument")


        
        if old_instrument_device_id:
            log.info("Checking that the data product to be dissociated are properly rooted")
            verify_dp_origin(old_instrument_data_product_ids,
                             self.find_data_product_by_instrument_device(old_instrument_device_id),
                             old_instrument_device_id,
                             "instrument_device")

            log.info("Checking that all data products to be dissociated have been supplied")
            if len(logical_data_product_ids) != len(old_instrument_data_product_ids):
                raise BadRequest("Can't unmap %d instrument data products from %d logical products" %
                                 (len(old_instrument_data_product_ids), len(logical_data_product_ids)))


        log.info("Checking that supplied instrument data products are properly rooted")
        verify_dp_origin(new_instrument_data_product_ids,
                         self.find_data_product_by_instrument_device(new_instrument_device_id),
                         new_instrument_device_id,
                         "instrument_device")

        log.info("Checking that all data products to be associated have been supplied")
        if len(logical_data_product_ids) != len(new_instrument_data_product_ids):
            raise BadRequest("Can't map %d instrument data products to %d logical products" %
                             (len(new_instrument_data_product_ids), len(logical_data_product_ids)))

        log.info("Assigning the instruments themselves")
        if "" != old_instrument_device_id:
            self.instrument_device.unlink_assignment(old_instrument_device_id, logical_instrument_id)
        self.instrument_device.link_assignment(new_instrument_device_id, logical_instrument_id)


        # functions to link and unlink data products as appropriate

        def link_logical_dp_to_instrument_dp(logical_dp_id, inst_dp_id):
            # TODO: this should be a function call, probably to DPMS,
            #       which sets up inst_dp to copy its data stream
            #       directly into the logical_dp
            pass

        def unlink_logical_dp_from_instrument_dp(logical_dp_id, inst_dp_id):
            #TODO: undo the above
            pass


        if old_instrument_device_id:
            log.info("Unlinking existing instrument data product(s) from logical instrument's product(s)")
            map(unlink_logical_dp_from_instrument_dp, logical_data_product_ids, old_instrument_data_product_ids)

        log.info("Linking new instrument data products with logical instrument's product(s)")
        map(link_logical_dp_to_instrument_dp, logical_data_product_ids, new_instrument_data_product_ids)





    ############################
    #
    #  ASSOCIATION FIND METHODS
    #
    ############################


    def find_instrument_model_by_instrument_device(self, instrument_device_id=''):
        return self.instrument_device.find_stemming_model(instrument_device_id)

    def find_instrument_device_by_instrument_model(self, instrument_model_id=''):
        return self.instrument_device.find_having_model(instrument_model_id)

    def find_platform_model_by_platform_device(self, platform_device_id=''):
        return self.platform_device.find_stemming_model(platform_device_id)

    def find_platform_device_by_platform_model(self, platform_model_id=''):
        return self.platform_device.find_having_model(platform_model_id)

    def find_instrument_model_by_instrument_agent(self, instrument_agent_id=''):
        return self.instrument_agent.find_stemming_model(instrument_agent_id)

    def find_instrument_agent_by_instrument_model(self, instrument_model_id=''):
        return self.instrument_agent.find_having_model(instrument_model_id)

    def find_instrument_device_by_platform_device(self, platform_device_id=''):
        return self.platform_device.find_stemming_instrument(platform_device_id)

    def find_platform_device_by_instrument_device(self, instrument_device_id=''):
        return self.platform_device.find_having_instrument(instrument_device_id)

    def find_instrument_device_by_logical_instrument(self, logical_instrument_id=''):
        return self.instrument_device.find_having_assignment(logical_instrument_id)

    def find_logical_instrument_by_instrument_device(self, instrument_device_id=''):
        return self.instrument_device.find_stemming_assignment(instrument_device_id)

    def find_platform_device_by_logical_platform(self, logical_platform_id=''):
        return self.platform_device.find_having_assignment(logical_platform_id)

    def find_logical_platform_by_platform_device(self, platform_device_id=''):
        return self.platform_device.find_stemming_assignment(platform_device_id)



    ############################
    #
    #  SPECIALIZED FIND METHODS
    #
    ############################

    def find_data_product_by_instrument_device(self, instrument_device_id=''):
        log.debug("FIND DATA PRODUCT BY INSTRUMENT DEVICE")
        #init return value, a list of data products
        data_products = []
        seen_data_producers = []

        #init working set of data producers to walk
        data_producers = []
        pducers = self.instrument_device.find_stemming_data_producer(instrument_device_id)
        data_producers += pducers


        #iterate through all un-processed data producers (could also do recursively)
        while 0 < len(data_producers):
           producer_id = data_producers.pop()
           if producer_id in seen_data_producers:
               raise Inconsistent("There is a cycle in data producers that includes '%s'" % producer_id)
           seen_data_producers.append(producer_id)

           log.debug("Analyzing data producer '%s'" % producer_id)
           #get any products that are associated with this data producer and return them
           #TODO: this belongs in DPMS
           new_data_products = self.data_product.find_having_data_producer(producer_id)
           #get any producers that receive input from this data producer
           #TODO: this belongs in DAMS
           new_data_producers = self.data_producer.find_having_input_data_producer(producer_id)

           log.debug("Got %d new products, %d new producers" % (len(new_data_products), 
                                                                len(new_data_producers)))

           data_products  += new_data_products
           data_producers += new_data_producers

        return data_products

    def find_instrument_device_by_data_product(self, data_product_id=''):
        log.debug("FIND INSTRUMENT DEVICE BY DATA PRODUCT")
        #init return value, a list of instrument devices
        instrument_devices = []
        seen_data_producers = []

        #init working set of data producers to walk
        data_producers = []
        #TODO: this belongs in DPMS
        pducers = self.data_product.find_stemming_data_producer(data_product_id)
        data_producers += pducers

        #iterate through all un-processed data producers (could also do recursively)
        while 0 < len(data_producers):
           producer_id = data_producers.pop()
           if producer_id in seen_data_producers:
               raise Inconsistent("There is a cycle in data producers that includes '%s'" % producer_id)
           seen_data_producers.append(producer_id)

           log.debug("Analyzing data producer '%s'" % producer_id)
           #get any devices that are associated with this data producer and return them
           new_instrument_devices = self.instrument_device.find_having_data_producer(producer_id)
           #get any producers that give input to this data producer
           #TODO: this belongs in DPMS
           new_data_producers = self.data_producer.find_stemming_input_data_producer(producer_id)

           log.debug("Got %d new devices, %d new producers" % (len(new_instrument_devices), 
                                                                len(new_data_producers)))

           instrument_devices  += new_instrument_devices
           data_producers      += new_data_producers

        return instrument_devices


    def find_data_product_by_platform_device(self, platform_device_id=''):
        ret = []
        for i in self.find_instrument_device_by_platform_device(platform_device_id):
            data_products = self.find_data_product_by_instrument_device(i)
            for d in data_products:
                if not d in ret:
                    ret.append(d)

        return ret
        


    ############################
    #
    #  LIFECYCLE TRANSITIONS
    #
    ############################


    def set_instrument_agent_lifecycle(self, instrument_agent_id="", lifecycle_state=""):
       """
       declare a instrument_agent to be in a given state
       @param instrument_agent_id the resource id
       """
       return self.instrument_agent.advance_lcs(instrument_agent_id, lifecycle_state)

    def set_instrument_agent_instance_lifecycle(self, instrument_agent_instance_id="", lifecycle_state=""):
       """
       declare a instrument_agent_instance to be in a given state
       @param instrument_agent_instance_id the resource id
       """
       return self.instrument_agent_instance.advance_lcs(instrument_agent_instance_id, lifecycle_state)

    def set_instrument_model_lifecycle(self, instrument_model_id="", lifecycle_state=""):
       """
       declare a instrument_model to be in a given state
       @param instrument_model_id the resource id
       """
       return self.instrument_model.advance_lcs(instrument_model_id, lifecycle_state)

    def set_instrument_device_lifecycle(self, instrument_device_id="", lifecycle_state=""):
       """
       declare an instrument_device to be in a given state
       @param instrument_device_id the resource id
       """
       return self.instrument_device.advance_lcs(instrument_device_id, lifecycle_state)

    def set_platform_agent_lifecycle(self, platform_agent_id="", lifecycle_state=""):
       """
       declare a platform_agent to be in a given state
       @param platform_agent_id the resource id
       """
       return self.platform_agent.advance_lcs(platform_agent_id, lifecycle_state)

    def set_platform_agent_instance_lifecycle(self, platform_agent_instance_id="", lifecycle_state=""):
       """
       declare a platform_agent_instance to be in a given state
       @param platform_agent_instance_id the resource id
       """
       return self.platform_agent_instance.advance_lcs(platform_agent_instance_id, lifecycle_state)

    def set_platform_model_lifecycle(self, platform_model_id="", lifecycle_state=""):
       """
       declare a platform_model to be in a given state
       @param platform_model_id the resource id
       """
       return self.platform_model.advance_lcs(platform_model_id, lifecycle_state)

    def set_platform_device_lifecycle(self, platform_device_id="", lifecycle_state=""):
       """
       declare a platform_device to be in a given state
       @param platform_device_id the resource id
       """
       return self.platform_device.advance_lcs(platform_device_id, lifecycle_state)

    def set_sensor_model_lifecycle(self, sensor_model_id="", lifecycle_state=""):
       """
       declare a sensor_model to be in a given state
       @param sensor_model_id the resource id
       """
       return self.sensor_model.advance_lcs(sensor_model_id, lifecycle_state)

    def set_sensor_device_lifecycle(self, sensor_device_id="", lifecycle_state=""):
       """
       declare a sensor_device to be in a given state
       @param sensor_device_id the resource id
       """
       return self.sensor_device.advance_lcs(sensor_device_id, lifecycle_state)
class DataProductManagementService(BaseDataProductManagementService):
    """ @author     Bill Bollenbacher
        @file       ion/services/sa/product/data_product_management_service.py
        @brief      Implementation of the data product management service
    """

    def on_init(self):
        self.override_clients(self.clients)

    def override_clients(self, new_clients):
        """
        Replaces the service clients with a new set of them... and makes sure they go to the right places
        """
        self.data_product = DataProductImpl(self.clients)

    def create_data_product(self, data_product=None, stream_definition_id=""):
        """
        @param      data_product IonObject which defines the general data product resource
        @param      source_resource_id IonObject id which defines the source for the data
        @retval     data_product_id
        """
        #   1. Verify that a data product with same name does not already exist
        #   2. Validate that the data product IonObject does not contain an id_ element
        #   3. Create a new data product
        #       - User must supply the name in the data product

        # Create will validate and register a new data product within the system

        # Validate - TBD by the work that Karen Stocks is driving with John Graybeal

        # Register - create and store a new DataProduct resource using provided metadata
        log.debug("DataProductManagementService:create_data_product: %s" % str(data_product))
        data_product_id = self.data_product.create_one(data_product)

        # Create the stream if a stream definition is provided
        log.debug("DataProductManagementService:create_data_product: stream definition id = %s" % stream_definition_id)

        if stream_definition_id:
            stream_id = self.clients.pubsub_management.create_stream(
                name=data_product.name, description=data_product.description, stream_definition_id=stream_definition_id
            )
            log.debug("create_data_product: create stream stream_id %s" % stream_id)
            # Associate the Stream with the main Data Product
            self.clients.resource_registry.create_association(data_product_id, PRED.hasStream, stream_id)

        # Return a resource ref to the new data product
        return data_product_id

    def read_data_product(self, data_product_id=""):
        """
        method docstring
        """
        # Retrieve all metadata for a specific data product
        # Return data product resource

        log.debug("DataProductManagementService:read_data_product: %s" % str(data_product_id))

        result = self.data_product.read_one(data_product_id)

        return result

    def update_data_product(self, data_product=None):
        """
        @todo document this interface!!!

        @param data_product    DataProduct
        @throws NotFound    object with specified id does not exist
        """

        log.debug("DataProductManagementService:update_data_product: %s" % str(data_product))

        self.clients.resource_registry.update(data_product)

        # TODO: any changes to producer? Call DataAcquisitionMgmtSvc?

        return

    def delete_data_product(self, data_product_id=""):

        # Check if this data product is associated to a producer
        producer_ids, _ = self.clients.resource_registry.find_objects(
            data_product_id, PRED.hasDataProducer, RT.DataProducer, id_only=True
        )
        if producer_ids:
            log.debug("DataProductManagementService:delete_data_product: %s" % str(producer_ids))
            self.clients.data_acquisition_management.unassign_data_product(data_product_id)

        # Delete the data product
        self.clients.resource_registry.delete(data_product_id)
        return

    def find_data_products(self, filters=None):
        """
        method docstring
        """
        # Validate the input filter and augment context as required

        # Define set of resource attributes to filter on, change parameter from "filter" to include attributes and filter values.
        #     potentially: title, keywords, date_created, creator_name, project, geospatial coords, time range

        # Call DM DiscoveryService to query the catalog for matches

        # Organize and return the list of matches with summary metadata (title, summary, keywords)

        # find the items in the store
        if filters is None:
            objects, _ = self.clients.resource_registry.find_resources(RT.DataProduct, None, None, False)
        else:  # TODO: code for all the filter types
            objects = []
        return objects

    def activate_data_product_persistence(self, data_product_id="", persist_data=True, persist_metadata=True):
        """Persist data product data into a data set

        @param data_product_id    str
        @throws NotFound    object with specified id does not exist
        """
        # retrieve the data_process object
        data_product_obj = self.clients.resource_registry.read(data_product_id)

        # get the Stream associated with this data set; if no stream then create one, if multiple streams then Throw
        streams, _ = self.clients.resource_registry.find_objects(data_product_id, PRED.hasStream, RT.Stream, True)
        if not streams:
            raise BadRequest("Data Product %s must have one stream associated" % str(data_product_id))

        stream = streams[0]
        log.debug("activate_data_product_persistence: stream = %s" % str(stream))

        # Find THE ingestion configuration in the RR to create a ingestion configuration
        # todo: how are multiple ingest configs for a site managed?
        ingest_config_objs, _ = self.clients.resource_registry.find_resources(
            restype=RT.IngestionConfiguration, id_only=False
        )
        if len(ingest_config_objs) != 1:
            log.debug("activate_data_product_persistence: ERROR ingest_config_objs = %s" % str(ingest_config_objs))
            raise BadRequest("Data Product must have one ingestion configuration %s" % str(data_product_id))

        ingestion_configuration_obj = ingest_config_objs[0]
        log.debug(
            "activate_data_product_persistence: ingestion_configuration_obj = %s" % str(ingestion_configuration_obj)
        )

        if data_product_obj.dataset_id:
            objs, _ = self.clients.resource_registry.find_objects(
                data_product_obj.dataset_id,
                PRED.hasIngestionConfiguration,
                RT.DatasetIngestionConfiguration,
                id_only=False,
            )
            if not objs:
                log.debug(
                    "activate_data_product_persistence: Calling create_dataset_configuration for EXISTING Dataset"
                )
                dataset_configuration_id = self.clients.ingestion_management.create_dataset_configuration(
                    dataset_id=data_product_obj.dataset_id,
                    archive_data=persist_data,
                    archive_metadata=persist_metadata,
                    ingestion_configuration_id=ingestion_configuration_obj._id,
                )
                log.debug(
                    "activate_data_product_persistence: create_dataset_configuration = %s"
                    % str(dataset_configuration_id)
                )
            else:
                dataset_configuration_obj = objs[0]

                dataset_configuration_obj.configuration.archive_data = persist_data
                dataset_configuration_obj.configuration.archive_metadata = persist_metadata

                # call ingestion management to update a dataset configuration
                log.debug("activate_data_product_persistence: Calling update_dataset_config")
                dataset_configuration_id = self.clients.ingestion_management.update_dataset_config(
                    dataset_configuration_obj
                )
                log.debug(
                    "activate_data_product_persistence: update_dataset_config = %s" % str(dataset_configuration_id)
                )
        else:
            # create the dataset for the data
            # !!!!!!!! (Currently) The Datastore name MUST MATCH the ingestion configuration name!!!
            data_product_obj.dataset_id = self.clients.dataset_management.create_dataset(
                stream_id=stream,
                datastore_name=ingestion_configuration_obj.couch_storage.datastore_name,
                description=data_product_obj.description,
            )
            log.debug("activate_data_product_persistence: create_dataset = %s" % str(data_product_obj.dataset_id))

            self.update_data_product(data_product_obj)

            # Need to read again, because the _rev has changed. Otherwise error on update later.
            data_product_obj = self.clients.resource_registry.read(data_product_id)

            # call ingestion management to create a dataset configuration
            log.debug("activate_data_product_persistence: Calling create_dataset_configuration")
            dataset_configuration_id = self.clients.ingestion_management.create_dataset_configuration(
                dataset_id=data_product_obj.dataset_id,
                archive_data=persist_data,
                archive_metadata=persist_metadata,
                ingestion_configuration_id=ingestion_configuration_obj._id,
            )
            log.debug(
                "activate_data_product_persistence: create_dataset_configuration = %s" % str(dataset_configuration_id)
            )

        # save the dataset_configuration_id in the product resource? Can this be found via the stream id?

        data_product_obj.dataset_configuration_id = dataset_configuration_id
        self.update_data_product(data_product_obj)

    def suspend_data_product_persistence(self, data_product_id=""):
        """Suspend data product data persistnce into a data set, multiple options

        @param data_product_id    str
        @param type    str
        @throws NotFound    object with specified id does not exist
        """

        # retrieve the data_process object
        data_product_obj = self.clients.resource_registry.read(data_product_id)
        if data_product_obj is None:
            raise NotFound("Data Product %s does not exist" % data_product_id)
        if data_product_obj.dataset_configuration_id is None:
            raise NotFound("Data Product %s dataset configuration does not exist" % data_product_id)

        # retrieve the dataset configuation object so that attrs can be changed
        dataset_configuration_obj = self.clients.resource_registry.read(data_product_obj.dataset_configuration_id)
        if dataset_configuration_obj is None:
            raise NotFound("Dataset Configuration %s does not exist" % data_product_obj.dataset_configuration_id)

        # Set the dataset config archive data/metadata attrs to false
        dataset_configuration_obj.configuration.archive_data = False
        dataset_configuration_obj.configuration.archive_metadata = False

        ret = self.clients.ingestion_management.update_dataset_config(dataset_configuration_obj)

        log.debug("suspend_data_product_persistence: deactivate = %s" % str(ret))

    def set_data_product_lifecycle(self, data_product_id="", lifecycle_state=""):
        """
       declare a data_product to be in a given state
       @param data_product_id the resource id
       """
        return self.data_product.advance_lcs(data_product_id, lifecycle_state)

    def get_last_update(self, data_product_id=""):
        """@todo document this interface!!!

        @param data_product_id    str
        @retval last_update    LastUpdate
        @throws NotFound    Data product not found or cache for data product not found.
        """
        from ion.processes.data.last_update_cache import CACHE_DATASTORE_NAME

        datastore_name = CACHE_DATASTORE_NAME
        db = self.container.datastore_manager.get_datastore(datastore_name)
        stream_ids, other = self.clients.resource_registry.find_objects(
            subject=data_product_id, predicate=PRED.hasStream, id_only=True
        )
        retval = {}
        for stream_id in stream_ids:
            try:
                lu = db.read(stream_id)
                retval[stream_id] = lu
            except NotFound:
                continue
        return retval
 def override_clients(self, new_clients):
     """
     Replaces the service clients with a new set of them... and makes sure they go to the right places
     """
     self.data_product = DataProductImpl(self.clients)
class DataProductManagementService(BaseDataProductManagementService):
    """ @author     Bill Bollenbacher
        @file       ion/services/sa/product/data_product_management_service.py
        @brief      Implementation of the data product management service
    """
    
    def on_init(self):
        self.override_clients(self.clients)

    def override_clients(self, new_clients):
        """
        Replaces the service clients with a new set of them... and makes sure they go to the right places
        """
        self.data_product   = DataProductImpl(self.clients)

    

    def create_data_product(self, data_product=None, source_resource_id=''):
        """
        @param      data_product IonObject which defines the general data product resource
        @param      source_resource_id IonObject id which defines the source for the data
        @retval     data_product_id
        """ 
        #   1. Verify that a data product with same name does not already exist 
        #   2. Validate that the data product IonObject does not contain an id_ element     
        #   3. Create a new data product
        #       - User must supply the name in the data product
        
        # Create will validate and register a new data product within the system

        # Validate - TBD by the work that Karen Stocks is driving with John Graybeal

        # Register - create and store a new DataProduct resource using provided metadata
        log.debug("DataProductManagementService:create_data_product: %s" % str(data_product))
        data_product_id = self.data_product.create_one(data_product)


        if source_resource_id:
            log.debug("DataProductManagementService:create_data_product: source resource id = %s" % source_resource_id)
            # TODO: currently create stream for the product is ALWAYS on, this should be surfaced
            # Call Data Aquisition Mgmt Svc:assign_data_product to coordinate connection of the data product to data producer and to the source resource
            self.clients.data_acquisition_management.assign_data_product(source_resource_id, data_product_id, True)  # TODO: what errors can occur here?

            # todo: should this method create the hasOutputProduct association?
            self.clients.resource_registry.create_association(source_resource_id,  PRED.hasOutputProduct,  data_product_id)

        # Return a resource ref to the new data product
        return data_product_id


    def read_data_product(self, data_product_id=''):
        """
        method docstring
        """
        # Retrieve all metadata for a specific data product
        # Return data product resource

        log.debug("DataProductManagementService:read_data_product: %s" % str(data_product_id))
        
        result = self.data_product.read_one(data_product_id)
        
        return result


    def update_data_product(self, data_product=None):
        """
        @todo document this interface!!!

        @param data_product    DataProduct
        @throws NotFound    object with specified id does not exist
        """
 
        log.debug("DataProductManagementService:update_data_product: %s" % str(data_product))
               
        self.data_product.update_one(data_product)

        #TODO: any changes to producer? Call DataAcquisitionMgmtSvc?

        return


    def delete_data_product(self, data_product_id=''):

        #Check if this data product is associated to a producer
        producer_ids, _ = self.clients.resource_registry.find_objects(data_product_id, PRED.hasDataProducer, RT.DataProducer, id_only=True)
        if producer_ids:
            log.debug("DataProductManagementService:delete_data_product: %s" % str(producer_ids))
            self.clients.data_acquisition_management.unassign_data_product(data_product_id)
        
        # Delete the data process
        self.clients.resource_registry.delete(data_product_id)
        return

    def find_data_products(self, filters=None):
        """
        method docstring
        """
        # Validate the input filter and augment context as required

        # Define set of resource attributes to filter on, change parameter from "filter" to include attributes and filter values.
        #     potentially: title, keywords, date_created, creator_name, project, geospatial coords, time range

        # Call DM DiscoveryService to query the catalog for matches

        # Organize and return the list of matches with summary metadata (title, summary, keywords)

        #find the items in the store
        if filters is None:
            objects, _ = self.clients.resource_registry.find_resources(RT.DataProduct, None, None, False)
        else:  # TODO: code for all the filter types
            objects = []
        return objects


    def activate_data_product_persistence(self, data_product_id='', persist_data=True, persist_metadata=True):
        """Persist data product data into a data set

        @param data_product_id    str
        @throws NotFound    object with specified id does not exist
        """
        # retrieve the data_process object
        data_product_obj = self.clients.resource_registry.read(data_product_id)
        if data_product_obj is None:
            raise NotFound("Data Product %s does not exist" % data_product_id)

        # get the Stream associated with this data set; if no stream then create one, if multiple streams then Throw
        streams, _ = self.clients.resource_registry.find_objects(data_product_id, PRED.hasStream, RT.Stream, True)
        if len(streams) > 1 or len(streams) == 0:
            raise BadRequest('Data Product must have one stream associated%s' % str(data_product_id))

        stream = streams[0]

        # Call ingestion management to create a ingestion configuration
        # Configure ingestion using eight workers, ingesting to test_dm_integration datastore with the SCIDATA profile
        log.debug('activate_data_product_persistence: Calling create_ingestion_configuration')
        data_product_obj.ingestion_configuration_id = self.clients.ingestion_management.create_ingestion_configuration(
            exchange_point_id='science_data',
            couch_storage=CouchStorage(datastore_name=self.datastore_name,datastore_profile='SCIDATA'),
            number_of_workers=8
        )
        #todo: does DPMS need to save the ingest _config_id in the product resource? Can this be found via the stream id?

        # activate an ingestion configuration
        #todo: Does DPMS call activate?
        ret = self.clients.ingestion_management.activate_ingestion_configuration(data_product_obj.ingestion_configuration_id)

        # create the dataset for the data
        data_product_obj.dataset_id = self.clients.dataset_management.create_dataset(self, stream, data_product_obj.name, data_product_obj.description)
        self.clients.resource_registry.update(data_product_obj)

        # Call ingestion management to create a dataset configuration
        log.debug('activate_data_product_persistence: Calling create_dataset_configuration')
        dataset_configuration_id = self.clients.ingestion_management.create_dataset_configuration( dataset_id, persist_data, persist_metadata, ingestion_configuration_id)
        #todo: does DPMS need to save the dataset_configuration_id in the product resource? Can this be found via the stream id?

        return

    def suspend_data_product_persistence(self, data_product_id=''):
        """Suspend data product data persistnce into a data set, multiple options

        @param data_product_id    str
        @param type    str
        @throws NotFound    object with specified id does not exist
        """

        # retrieve the data_process object
        data_product_obj = self.clients.resource_registry.read(data_product_id)
        if data_product_obj is None:
            raise NotFound("Data Product %s does not exist" % data_product_id)
        if data_product_obj.ingestion_configuration_id is None:
            raise NotFound("Data Product %s ingestion configuration does not exist" % data_product_id)

        # Change the stream policy to stop ingestion
        self.clients.ingestion_management.deactivate_ingestion_configuration(data_product_obj.ingestion_configuration_id)

        return

    def set_data_product_lifecycle(self, data_product_id="", lifecycle_state=""):
       """
       declare a data_product to be in a given state
       @param data_product_id the resource id
       """
       return self.data_product.advance_lcs(data_product_id, lifecycle_state)