コード例 #1
0
    def _do_test_find_objects_mult(self):
        dp = DataProcess()
        transform = Transform()
        pd = ProcessDefinition()

        dp_id, _ = self.resource_registry_service.create(dp)
        transform_id, _ = self.resource_registry_service.create(transform)
        pd_id, _ = self.resource_registry_service.create(pd)

        self.resource_registry_service.create_association(
            subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
        self.resource_registry_service.create_association(
            subject=transform_id,
            object=pd_id,
            predicate=PRED.hasProcessDefinition)

        results, _ = self.resource_registry_service.find_objects_mult(
            subjects=[dp_id], id_only=True)
        self.assertTrue(results == [transform_id])

        results, _ = self.resource_registry_service.find_objects_mult(
            subjects=[dp_id, transform_id], id_only=True)
        results.sort()
        correct = [transform_id, pd_id]
        correct.sort()
        self.assertTrue(results == correct)
コード例 #2
0
    def test_iterative_associative_searching(self):
        #--------------------------------------------------------------------------------
        # Tests the ability to limit the iterations
        #--------------------------------------------------------------------------------
        dp        = DataProcess()
        transform = Transform()
        pd        = ProcessDefinition()

        dp_id, _        = self.rr.create(dp)
        transform_id, _ = self.rr.create(transform)
        pd_id, _        = self.rr.create(pd)

        self.rr.create_association(subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
        self.rr.create_association(subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition)

        search_string = "belongs to '%s' depth 1" % dp_id
        results = self.poll(5, self.discovery.parse,search_string)
        results = list([i._id for i in results])
        correct = [transform_id]
        self.assertTrue(results == correct, '%s' % results)

        search_string = "belongs to '%s' depth 2" % dp_id
        results = self.poll(5, self.discovery.parse,search_string)
        results = list([i._id for i in results])
        results.sort()
        correct = [transform_id, pd_id]
        correct.sort()
        self.assertTrue(results == correct)
コード例 #3
0
    def create_data_process2(self, data_process_definition_id='', in_data_product_ids=None, out_data_product_ids=None, configuration=None):
        '''
        Creates a DataProcess resource and launches the process.
        A DataProcess is a process that receives one (or more) data products and produces one (or more) data products.

        @param data_process_definition_id : The Data Process Definition to use, if none is specified the standard TransformDataProcess is used
        @param in_data_product_ids : A list of input data product identifiers
        @param out_data_product_ids : A list of output data product identifiers
        @param configuration : The configuration dictionary for the process, and the routing table:

        The routing table is defined as such:
            { (in_data_product_id, out_data_product_id) : actor }

        Routes are specified in the configuration dictionary under the item "routes"
        actor is either None (for ParameterFunctions) or a valid TransformFunction identifier
        '''
        configuration = configuration or DotDict()
        routes = configuration.get_safe('process.routes', {})
        self.validate_compatibility(in_data_product_ids, out_data_product_ids)
        configuration.process.routes = self._manage_routes(routes)

        dproc = DataProcess()
        dproc.name = 'data_process_%s' % self.get_unique_id()
        dproc.configuration = configuration
        dproc_id, rev = self.clients.resource_registry.create(dproc)
        dproc._id = dproc_id
        dproc._rev = rev

        self._manage_producers(dproc_id, out_data_product_ids)

        self._manage_attachments()

        queue_name = self._create_subscription(dproc, in_data_product_ids)

        pid = self._launch_data_process(
                queue_name=queue_name,
                data_process_definition_id=data_process_definition_id,
                out_data_product_ids=out_data_product_ids,
                configuration=configuration)

        self.clients.resource_registry.create_association(subject=dproc_id, predicate=PRED.hasProcess, object=pid)


        return dproc_id
コード例 #4
0
    def test_events_search(self):
        # Create a resource to force a new event

        dp = DataProcess()
        dp_id, rev = self.rr.create(dp)

        search_string = "SEARCH 'origin' IS '%s' FROM 'events_index'" % dp_id

        results = self.poll(9, self.discovery.parse,search_string)
        origin_type = results[0]['_source'].origin_type
        origin_id = results[0]['_source'].origin

        self.assertTrue(origin_type == RT.DataProcess)
        self.assertTrue(origin_id == dp_id)
コード例 #5
0
    def test_traversal(self):
        dp        = DataProcess()
        transform = Transform()
        pd        = ProcessDefinition()

        dp_id, _        = self.rr.create(dp)
        transform_id, _ = self.rr.create(transform)
        pd_id, _        = self.rr.create(pd)

        self.rr.create_association(subject=dp_id, object=transform_id, predicate=PRED.hasTransform)
        self.rr.create_association(subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition)

        results = self.discovery.traverse(dp_id)
        results.sort()
        correct = [pd_id, transform_id]
        correct.sort()
        self.assertTrue(results == correct, '%s' % results)
コード例 #6
0
    def create_data_process(self, data_process_definition_id='', in_data_product_ids=None, out_data_product_ids=None, configuration=None):
        '''
        Creates a DataProcess resource and launches the process.
        A DataProcess is a process that receives one (or more) data products and produces one (or more) data products.

        @param data_process_definition_id : The Data Process Definition to use, if none is specified the standard TransformDataProcess is used
        @param in_data_product_ids : A list of input data product identifiers
        @param out_data_product_ids : A list of output data product identifiers
        @param configuration : The configuration dictionary for the process, and the routing table:

        The routing table is defined as such:
            { in_data_product_id: {out_data_product_id : actor }}

        Routes are specified in the configuration dictionary under the item "routes"
        actor is either None (for ParameterFunctions) or a valid TransformFunction identifier
        '''
        configuration = DotDict(configuration or {}) 
        in_data_product_ids = in_data_product_ids or []
        out_data_product_ids = out_data_product_ids or []
        routes = configuration.get_safe('process.routes', {})
        if not routes and (1==len(in_data_product_ids)==len(out_data_product_ids)):
            routes = {in_data_product_ids[0]: {out_data_product_ids[0]:None}}
        # Routes are not supported for processes with discrete data process definitions
        elif not routes and not data_process_definition_id:
            raise BadRequest('No valid route defined for this data process.')

        self.validate_compatibility(data_process_definition_id, in_data_product_ids, out_data_product_ids, routes)
        routes = self._manage_routes(routes)
        configuration.process.input_products = in_data_product_ids
        configuration.process.output_products = out_data_product_ids
        configuration.process.routes = routes
        if 'lookup_docs' in configuration.process:
            configuration.process.lookup_docs.extend(self._get_lookup_docs(in_data_product_ids, out_data_product_ids))
        else:
            configuration.process.lookup_docs = self._get_lookup_docs(in_data_product_ids, out_data_product_ids)
        dproc = DataProcess()
        dproc.name = 'data_process_%s' % self.get_unique_id()
        dproc.configuration = configuration
        dproc_id, rev = self.clients.resource_registry.create(dproc)
        dproc._id = dproc_id
        dproc._rev = rev

        for data_product_id in in_data_product_ids:
            self.clients.resource_registry.create_association(subject=dproc_id, predicate=PRED.hasInputProduct, object=data_product_id)

        if data_process_definition_id:
            self.clients.resource_registry.create_association(data_process_definition_id, PRED.hasDataProcess ,dproc_id)

        self._manage_producers(dproc_id, out_data_product_ids)

        self._manage_attachments()

        queue_name = self._create_subscription(dproc, in_data_product_ids)

        pid = self._launch_data_process(
                queue_name=queue_name,
                data_process_definition_id=data_process_definition_id,
                out_data_product_ids=out_data_product_ids,
                configuration=configuration)

        self.clients.resource_registry.create_association(subject=dproc_id, predicate=PRED.hasProcess, object=pid)


        return dproc_id
コード例 #7
0
    def test_get_data_product_provenance_report(self):
        #Create a test device
        device_obj = Device(name='Device1', description='test instrument site')
        device_id, _ = self.rrclient.create(device_obj)
        self.addCleanup(self.rrclient.delete, device_id)

        #Create a test DataProduct
        data_product1_obj = DataProduct(name='DataProduct1',
                                        description='test data product 1')
        data_product1_id, _ = self.rrclient.create(data_product1_obj)
        self.addCleanup(self.rrclient.delete, data_product1_id)

        #Create a test DataProcess
        data_process_obj = DataProcess(name='DataProcess',
                                       description='test data process')
        data_process_id, _ = self.rrclient.create(data_process_obj)
        self.addCleanup(self.rrclient.delete, data_process_id)

        #Create a second test DataProduct
        data_product2_obj = DataProduct(name='DataProduct2',
                                        description='test data product 2')
        data_product2_id, _ = self.rrclient.create(data_product2_obj)
        self.addCleanup(self.rrclient.delete, data_product2_id)

        #Create a test DataProducer
        data_producer_obj = DataProducer(name='DataProducer',
                                         description='test data producer')
        data_producer_id, rev = self.rrclient.create(data_producer_obj)

        #Link the DataProcess to the second DataProduct manually
        assoc_id, _ = self.rrclient.create_association(
            subject=data_process_id,
            predicate=PRED.hasInputProduct,
            object=data_product2_id)
        self.addCleanup(self.rrclient.delete_association, assoc_id)

        # Register the instrument and process. This links the device and the data process
        # with their own producers
        self.damsclient.register_instrument(device_id)
        self.addCleanup(self.damsclient.unregister_instrument, device_id)
        self.damsclient.register_process(data_process_id)
        self.addCleanup(self.damsclient.unregister_process, data_process_id)

        #Manually link the first DataProduct with the test DataProducer
        assoc_id, _ = self.rrclient.create_association(
            subject=data_product1_id,
            predicate=PRED.hasDataProducer,
            object=data_producer_id)

        #Get the DataProducer linked to the DataProcess (created in register_process above)
        #Associate that with with DataProduct1's DataProducer
        data_process_producer_ids, _ = self.rrclient.find_objects(
            subject=data_process_id,
            predicate=PRED.hasDataProducer,
            object_type=RT.DataProducer,
            id_only=True)
        assoc_id, _ = self.rrclient.create_association(
            subject=data_process_producer_ids[0],
            predicate=PRED.hasParent,
            object=data_producer_id)
        self.addCleanup(self.rrclient.delete_association, assoc_id)

        #Get the DataProducer linked to the Device (created in register_instrument
        #Associate that with the DataProcess's DataProducer
        device_producer_ids, _ = self.rrclient.find_objects(
            subject=device_id,
            predicate=PRED.hasDataProducer,
            object_type=RT.DataProducer,
            id_only=True)
        assoc_id, _ = self.rrclient.create_association(
            subject=data_producer_id,
            predicate=PRED.hasParent,
            object=device_producer_ids[0])

        #Create the links between the Device, DataProducts, DataProcess, and all DataProducers
        self.damsclient.assign_data_product(input_resource_id=device_id,
                                            data_product_id=data_product1_id)
        self.addCleanup(self.damsclient.unassign_data_product, device_id,
                        data_product1_id)
        self.damsclient.assign_data_product(input_resource_id=data_process_id,
                                            data_product_id=data_product2_id)
        self.addCleanup(self.damsclient.unassign_data_product, data_process_id,
                        data_product2_id)

        #Traverse through the relationships to get the links between objects
        res = self.dpmsclient.get_data_product_provenance_report(
            data_product2_id)

        #Make sure there are four keys
        self.assertEqual(len(res.keys()), 4)

        parent_count = 0
        config_count = 0
        for v in res.itervalues():
            if 'parent' in v:
                parent_count += 1
            if 'config' in v:
                config_count += 1

        #Make sure there are three parents and four configs
        self.assertEqual(parent_count, 3)
        self.assertEqual(config_count, 4)
コード例 #8
0
    def create_data_process(self,
                            data_process_definition_id='',
                            in_data_product_ids=None,
                            out_data_product_ids=None,
                            configuration=None):
        '''
        Creates a DataProcess resource and launches the process.
        A DataProcess is a process that receives one (or more) data products and produces one (or more) data products.

        @param data_process_definition_id : The Data Process Definition to use, if none is specified the standard TransformDataProcess is used
        @param in_data_product_ids : A list of input data product identifiers
        @param out_data_product_ids : A list of output data product identifiers
        @param configuration : The configuration dictionary for the process, and the routing table:

        The routing table is defined as such:
            { in_data_product_id: {out_data_product_id : actor }}

        Routes are specified in the configuration dictionary under the item "routes"
        actor is either None (for ParameterFunctions) or a valid TransformFunction identifier
        '''
        configuration = DotDict(configuration or {})
        in_data_product_ids = in_data_product_ids or []
        out_data_product_ids = out_data_product_ids or []
        routes = configuration.get_safe('process.routes', {})
        if not routes and (1 == len(in_data_product_ids) ==
                           len(out_data_product_ids)):
            routes = {in_data_product_ids[0]: {out_data_product_ids[0]: None}}
        # Routes are not supported for processes with discrete data process definitions
        elif not routes and not data_process_definition_id:
            raise BadRequest('No valid route defined for this data process.')

        self.validate_compatibility(data_process_definition_id,
                                    in_data_product_ids, out_data_product_ids,
                                    routes)
        routes = self._manage_routes(routes)
        configuration.process.input_products = in_data_product_ids
        configuration.process.output_products = out_data_product_ids
        configuration.process.routes = routes
        if 'lookup_docs' in configuration.process:
            configuration.process.lookup_docs.extend(
                self._get_lookup_docs(in_data_product_ids,
                                      out_data_product_ids))
        else:
            configuration.process.lookup_docs = self._get_lookup_docs(
                in_data_product_ids, out_data_product_ids)
        dproc = DataProcess()
        dproc.name = 'data_process_%s' % self.get_unique_id()
        dproc.configuration = configuration
        dproc_id, rev = self.clients.resource_registry.create(dproc)
        dproc._id = dproc_id
        dproc._rev = rev

        for data_product_id in in_data_product_ids:
            self.clients.resource_registry.create_association(
                subject=dproc_id,
                predicate=PRED.hasInputProduct,
                object=data_product_id)

        if data_process_definition_id:
            self.clients.resource_registry.create_association(
                data_process_definition_id, PRED.hasDataProcess, dproc_id)

        self._manage_producers(dproc_id, out_data_product_ids)

        self._manage_attachments()

        queue_name = self._create_subscription(dproc, in_data_product_ids)

        pid = self._launch_data_process(
            queue_name=queue_name,
            data_process_definition_id=data_process_definition_id,
            out_data_product_ids=out_data_product_ids,
            configuration=configuration)

        self.clients.resource_registry.create_association(
            subject=dproc_id, predicate=PRED.hasProcess, object=pid)

        return dproc_id