def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2dm.yml') self.unsc = UserNotificationServiceClient(node=self.container.node) self.rrc = ResourceRegistryServiceClient(node=self.container.node) self.imc = IdentityManagementServiceClient(node=self.container.node)
def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.dpsc_cli = DataProductManagementServiceClient() self.rrclient = ResourceRegistryServiceClient() self.damsclient = DataAcquisitionManagementServiceClient() self.pubsubcli = PubsubManagementServiceClient() self.ingestclient = IngestionManagementServiceClient() self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() self.unsc = UserNotificationServiceClient() self.data_retriever = DataRetrieverServiceClient() self.identcli = IdentityManagementServiceClient() #------------------------------------------ # Create the environment #------------------------------------------ self.stream_def_id = self.pubsubcli.create_stream_definition( name='SBE37_CDM') self.process_definitions = {} ingestion_worker_definition = ProcessDefinition( name='ingestion worker') ingestion_worker_definition.executable = { 'module': 'ion.processes.data.ingestion.science_granule_ingestion_worker', 'class': 'ScienceGranuleIngestionWorker' } process_definition_id = self.process_dispatcher.create_process_definition( process_definition=ingestion_worker_definition) self.process_definitions['ingestion_worker'] = process_definition_id self.pids = [] self.exchange_points = [] self.exchange_names = [] #------------------------------------------------------------------------------------------------ # First launch the ingestors #------------------------------------------------------------------------------------------------ self.exchange_space = 'science_granule_ingestion' self.exchange_point = 'science_data' config = DotDict() config.process.datastore_name = 'datasets' config.process.queue_name = self.exchange_space self.exchange_names.append(self.exchange_space) self.exchange_points.append(self.exchange_point) pid = self.process_dispatcher.schedule_process( self.process_definitions['ingestion_worker'], configuration=config) log.debug("the ingestion worker process id: %s", pid) self.pids.append(pid) self.addCleanup(self.cleaning_up)
def setUp(self): self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254" # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2coi.yml') self.identity_management_service = IdentityManagementServiceClient( node=self.container.node)
def setUp(self): self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254" # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.resource_registry = ResourceRegistryServiceClient() self.identity_management_service = IdentityManagementServiceClient() self.org_client = OrgManagementServiceClient()
def test_non_anonymous_resource_registry_operations_through_gateway(self): id_client = IdentityManagementServiceClient(node=self.container.node) actor_id, valid_until, registered = id_client.signon(USER1_CERTIFICATE, True) response = self.test_app.get('/ion-service/resource_registry/find_resources?name=TestDataProduct&id_only=True&requester=' + actor_id) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json['data']) response_data = response.json['data'][GATEWAY_RESPONSE] self.assertEqual(len(response_data),2 ) self.assertEqual(len(response_data[0]), 0 ) id_client.delete_actor_identity(actor_id)
def setUp(self): """ Test setup. """ # Resources used in the tests. # General resources. self.actor_id = None self.user_info_id = None self.org_id = None self.obs_id = None # Cabled infrastructure. self.cabled_platform_model_id = None self.cabled_platform_site_id = None self.cabled_platform_device_id = None self.cabled_platform_agent_id = None self.cabled_platform_agent_instance_id = None self.cabled_platform_deployment_id = None self.cabled_instrument_deployment_id = None self.cabled_instrument_model_id = None self.cabled_instrument_site_id = None self.cabled_instrument_device_id = None self.cabled_instrument_agent_id = None self.cabled_instrument_agent_instance_id = None self.cabled_instrument_deployment_id = None # Uncabled infrastructure. self.uncabled_platform_model_id = None self.uncabled_platform_site_id = None self.uncabled_platform_device_id = None self.uncabled_platform_agent_id = None self.uncabled_platform_agent_instance_id = None self.uncabled_instrument_model_id = None self.uncabled_instrument_site_id = None self.uncabled_instrument_device_id = None self.uncabled_instrument_agent_id = None self.uncabled_instrument_agent_instance_id = None self.uncabled_site_deployment_id = None # Start container. log.info('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message) log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Setup service clients. self.idms = IdentityManagementServiceClient(node=self.container.node) self.oms = ObservatoryManagementServiceClient(node=self.container.node) # Add generic resources. self._load_system_actors() self._create_user() self._create_org() self._create_observatory() # Add cleanup routine. self.addCleanup(self._cleanup_resources)
def setUp(self): self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254" # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2coi.yml') self.identity_management_service = IdentityManagementServiceClient(node=self.container.node)
def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.dpsc_cli = DataProductManagementServiceClient() self.rrclient = ResourceRegistryServiceClient() self.damsclient = DataAcquisitionManagementServiceClient() self.pubsubcli = PubsubManagementServiceClient() self.ingestclient = IngestionManagementServiceClient() self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() self.unsc = UserNotificationServiceClient() self.data_retriever = DataRetrieverServiceClient() self.identcli = IdentityManagementServiceClient() #------------------------------------------ # Create the environment #------------------------------------------ self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM') self.process_definitions = {} ingestion_worker_definition = ProcessDefinition(name='ingestion worker') ingestion_worker_definition.executable = { 'module':'ion.processes.data.ingestion.science_granule_ingestion_worker', 'class' :'ScienceGranuleIngestionWorker' } process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition) self.process_definitions['ingestion_worker'] = process_definition_id self.pids = [] self.exchange_points = [] self.exchange_names = [] #------------------------------------------------------------------------------------------------ # First launch the ingestors #------------------------------------------------------------------------------------------------ self.exchange_space = 'science_granule_ingestion' self.exchange_point = 'science_data' config = DotDict() config.process.datastore_name = 'datasets' config.process.queue_name = self.exchange_space self.exchange_names.append(self.exchange_space) self.exchange_points.append(self.exchange_point) pid = self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config) log.debug("the ingestion worker process id: %s", pid) self.pids.append(pid) self.addCleanup(self.cleaning_up)
def setUp(self): self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254" # Start container self._start_container() # Establish endpoint with container container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) container_client.start_rel_from_url('res/deploy/r2coi.yml') self.identity_management_service = IdentityManagementServiceClient(node=self.container.node)
def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' unittest # suppress an pycharm inspector error if all unittest.skip references are commented out self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.OMS = ObservatoryManagementServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR)
def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) print 'started services'
def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) print 'started services'
def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR)
def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient( node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.DSC = DatasetManagementServiceClient() self.IDS = IdentityManagementServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) # Use the network definition provided by RSN OMS directly. rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri']) self._network_definition = RsnOmsUtil.build_network_definition(rsn_oms) # get serialized version for the configuration: self._network_definition_ser = NetworkUtil.serialize_network_definition( self._network_definition) if log.isEnabledFor(logging.TRACE): log.trace("NetworkDefinition serialization:\n%s", self._network_definition_ser) self._async_data_result = AsyncResult() self._data_subscribers = [] self._samples_received = [] self.addCleanup(self._stop_data_subscribers) self._async_event_result = AsyncResult() self._event_subscribers = [] self._events_received = [] self.addCleanup(self._stop_event_subscribers) self._start_event_subscriber()
def test_non_anonymous_resource_registry_operations_with_token(self): rr = self.container.resource_registry id_client = IdentityManagementServiceClient() create_request = { "serviceRequest": { "serviceName": "resource_registry", "serviceOp": "create", "params": { "object": { "name": "Instrument1", "type_": "InstrumentDevice" } } } } # Create without actor response = self.test_app.post( '/ion-service/resource_registry/create', {'payload': simplejson.dumps(create_request)}) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json['data']) response_data = response.json['data'][GATEWAY_RESPONSE] self.assertEqual(len(response_data), 2) self.assertGreaterEqual(len(response_data[0]), 20) # This is a resource_id inst_id = str(response_data[0]) inst_obj = rr.read(inst_id) self.assertEquals(inst_obj.type_, RT.InstrumentDevice) self.assertEquals(inst_obj.name, "Instrument1") self.assertEquals(inst_obj.visibility, ResourceVisibilityEnum.PUBLIC) act_objs, assocs = rr.find_objects(inst_id, PRED.hasOwner, RT.ActorIdentity, id_only=False) self.assertEquals(len(act_objs), 0) # Anonymous query shows PUBLIC visibility resource anymore response = self.test_app.get( '/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True' ) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json['data']) response_data = response.json['data'][GATEWAY_RESPONSE] self.assertEqual(len(response_data), 2) self.assertEqual(len(response_data[0]), 1) self.assertEqual(len(response_data[1]), 1) self.assertEqual(response_data[0][0], inst_id) inst_obj.visibility = ResourceVisibilityEnum.OWNER rr.update(inst_obj) # Now the anonymous query should not show the resource anymore response = self.test_app.get( '/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True' ) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json['data']) response_data = response.json['data'][GATEWAY_RESPONSE] self.assertEqual(len(response_data), 2) self.assertEqual(len(response_data[0]), 0) rr.delete(inst_id) # Create with actor actor_id, valid_until, registered = id_client.signon( USER1_CERTIFICATE, True) create_request = { "serviceRequest": { "serviceName": "resource_registry", "serviceOp": "create", "requester": actor_id, "params": { "object": { "name": "Instrument1", "type_": "InstrumentDevice", "visibility": ResourceVisibilityEnum.OWNER } } } } response = self.test_app.post( '/ion-service/resource_registry/create', {'payload': simplejson.dumps(create_request)}) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json['data']) response_data = response.json['data'][GATEWAY_RESPONSE] self.assertEqual(len(response_data), 2) self.assertGreaterEqual(len(response_data[0]), 20) # This is a resource_id inst_id = str(response_data[0]) inst_obj = rr.read(inst_id) self.assertEquals(inst_obj.type_, RT.InstrumentDevice) self.assertEquals(inst_obj.name, "Instrument1") self.assertEquals(inst_obj.visibility, ResourceVisibilityEnum.OWNER) act_objs, assocs = rr.find_objects(inst_id, PRED.hasOwner, RT.ActorIdentity, id_only=False) self.assertEquals(len(act_objs), 1) self.assertEquals(act_objs[0]._id, actor_id) # Anonymous query should not show the resource anymore response = self.test_app.get( '/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True' ) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json['data']) response_data = response.json['data'][GATEWAY_RESPONSE] self.assertEqual(len(response_data), 2) self.assertEqual(len(response_data[0]), 0) # Authenticated request with owner shows resource response = self.test_app.get( '/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&requester=' + actor_id) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json['data']) response_data = response.json['data'][GATEWAY_RESPONSE] self.assertEqual(len(response_data), 2) self.assertEqual(len(response_data[0]), 1) self.assertEqual(len(response_data[1]), 1) self.assertEqual(response_data[0][0], inst_id) token_str = id_client.create_authentication_token(actor_id, validity=2) # Request with authentication token for owner shows resource response = self.test_app.get( '/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&authtoken=' + token_str) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json['data']) response_data = response.json['data'][GATEWAY_RESPONSE] self.assertEqual(len(response_data), 2) self.assertEqual(len(response_data[0]), 1) self.assertEqual(len(response_data[1]), 1) self.assertEqual(response_data[0][0], inst_id) gevent.sleep(2.1) # Request with expired authentication token for owner does not show resource (no error though) response = self.test_app.get( '/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&authtoken=' + token_str) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json['data']) response_data = response.json['data'][GATEWAY_RESPONSE] self.assertEqual(len(response_data), 2) self.assertEqual(len(response_data[0]), 0) # Request with valid authentication token for different user does not show resource actor_id2, _ = rr.create(IonObject(RT.ActorIdentity, name="Actor2")) token_str2 = id_client.create_authentication_token(actor_id2, validity=2) response = self.test_app.get( '/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&authtoken=' + token_str2) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json['data']) response_data = response.json['data'][GATEWAY_RESPONSE] self.assertEqual(len(response_data), 2) self.assertEqual(len(response_data[0]), 0) token_str3 = id_client.create_authentication_token(actor_id, validity=2) # Request with new authentication token for owner shows resource response = self.test_app.get( '/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&authtoken=' + token_str3) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json['data']) response_data = response.json['data'][GATEWAY_RESPONSE] self.assertEqual(len(response_data), 2) self.assertEqual(len(response_data[0]), 1) self.assertEqual(len(response_data[1]), 1) self.assertEqual(response_data[0][0], inst_id) id_client.invalidate_authentication_token(token_str3) response = self.test_app.get( '/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&authtoken=' + token_str3) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json['data']) response_data = response.json['data'][GATEWAY_RESPONSE] self.assertEqual(len(response_data), 2) self.assertEqual(len(response_data[0]), 0) # Cleanup rr.delete(inst_id) rr.delete(actor_id2) id_client.delete_actor_identity(actor_id)
class UserNotificationIntTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2dm.yml') self.unsc = UserNotificationServiceClient(node=self.container.node) self.rrc = ResourceRegistryServiceClient(node=self.container.node) self.imc = IdentityManagementServiceClient(node=self.container.node) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_email(self): proc1 = self.container.proc_manager.procs_by_name['user_notification'] # Create a user and get the user_id user = UserInfo(name='new_user') user_id, _ = self.rrc.create(user) # set up.... notification_id = self.unsc.create_email( event_type='ResourceLifecycleEvent', event_subtype=None, origin='Some_Resource_Agent_ID1', origin_type=None, user_id=user_id, email='*****@*****.**', mode=DeliveryMode.DIGEST, message_header='message_header', parser='parser', period=1) #------------------------------------------------------------------------------------------------------ # Setup so as to be able to get the message and headers going into the # subscription callback method of the EmailEventProcessor #------------------------------------------------------------------------------------------------------ # publish an event for each notification to generate the emails rle_publisher = EventPublisher("ResourceLifecycleEvent") rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event") msg_tuple = proc1.event_processors[ notification_id].smtp_client.sentmail.get(timeout=4) self.assertTrue(proc1.event_processors[notification_id].smtp_client. sentmail.empty()) message = msg_tuple[2] list_lines = message.split("\n") #------------------------------------------------------- # parse the message body #------------------------------------------------------- message_dict = {} for line in list_lines: key_item = line.split(": ") if key_item[0] == 'Subject': message_dict['Subject'] = key_item[1] + key_item[2] else: try: message_dict[key_item[0]] = key_item[1] except IndexError as exc: # these IndexError exceptions happen only because the message sometimes # has successive /r/n (i.e. new lines) and therefore, # the indexing goes out of range. These new lines # can just be ignored. So we ignore the exceptions here. pass #------------------------------------------------------- # make assertions #------------------------------------------------------- self.assertEquals(msg_tuple[1], '*****@*****.**') #self.assertEquals(msg_tuple[0], ION_NOTIFICATION_EMAIL_ADDRESS) #self.assertEquals(message_dict['From'], ION_NOTIFICATION_EMAIL_ADDRESS) self.assertEquals(message_dict['To'], '*****@*****.**') self.assertEquals(message_dict['Event'].rstrip('\r'), 'ResourceLifecycleEvent') self.assertEquals(message_dict['Originator'].rstrip('\r'), 'Some_Resource_Agent_ID1') self.assertEquals(message_dict['Description'].rstrip('\r'), 'RLE test event') @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_sms(self): proc1 = self.container.proc_manager.procs_by_name['user_notification'] # Create a user and get the user_id user = UserInfo(name='new_user') user_id, _ = self.rrc.create(user) # set up.... notification_id = self.unsc.create_sms( event_type='ResourceLifecycleEvent', event_subtype=None, origin='Some_Resource_Agent_ID1', origin_type=None, user_id=user_id, phone='401-XXX-XXXX', provider='T-Mobile', message_header='message_header', parser='parser', ) #------------------------------------------------------------------------------------------------------ # Setup so as to be able to get the message and headers going into the # subscription callback method of the EmailEventProcessor #------------------------------------------------------------------------------------------------------ # publish an event for each notification to generate the emails rle_publisher = EventPublisher("ResourceLifecycleEvent") rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event") msg_tuple = proc1.event_processors[ notification_id].smtp_client.sentmail.get(timeout=4) self.assertTrue(proc1.event_processors[notification_id].smtp_client. sentmail.empty()) message = msg_tuple[2] list_lines = message.split("\n") #------------------------------------------------------- # parse the message body #------------------------------------------------------- message_dict = {} for line in list_lines: key_item = line.split(": ") if key_item[0] == 'Subject': message_dict['Subject'] = key_item[1] + key_item[2] else: try: message_dict[key_item[0]] = key_item[1] except IndexError as exc: # these IndexError exceptions happen only because the message sometimes # has successive /r/n (i.e. new lines) and therefore, # the indexing goes out of range. These new lines # can just be ignored. So we ignore the exceptions here. pass #------------------------------------------------------- # make assertions #------------------------------------------------------- self.assertEquals(msg_tuple[1], '*****@*****.**') #self.assertEquals(msg_tuple[0], ION_NOTIFICATION_EMAIL_ADDRESS) self.assertEquals(message_dict['Description'].rstrip('\r'), 'RLE test event') @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_event_detection(self): proc1 = self.container.proc_manager.procs_by_name['user_notification'] # Create a user and get the user_id user = UserInfo(name='new_user') user_id, _ = self.rrc.create(user) # Create detection notification dfilt = DetectionFilterConfig() dfilt.processing['condition'] = 5 dfilt.processing['comparator'] = '>' dfilt.processing['filter_field'] = 'voltage' dfilt.delivery['message'] = 'I got my detection event!' notification_id = self.unsc.create_detection_filter( event_type='ExampleDetectableEvent', event_subtype=None, origin='Some_Resource_Agent_ID1', origin_type=None, user_id=user_id, filter_config=dfilt) #--------------------------------------------------------------------------------- # Create event subscription for resulting detection event #--------------------------------------------------------------------------------- # Create an email notification so that when the DetectionEventProcessor # detects an event and fires its own output event, this will caught by an # EmailEventProcessor and an email will be sent to the user notification_id_2 = self.unsc.create_email( event_type='DetectionEvent', event_subtype=None, origin='DetectionEventProcessor', origin_type=None, user_id=user_id, email='*****@*****.**', mode=DeliveryMode.UNFILTERED, message_header='Detection event', parser='parser', period=1) # Send event that is not detected # publish an event for each notification to generate the emails rle_publisher = EventPublisher("ExampleDetectableEvent") # since the voltage field in this event is less than 5, it will not be detected rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event", voltage=3) # Check at the end of the test to make sure this event never triggered a Detectable! # Send Event that is detected # publish an event for each notification to generate the emails # since the voltage field in this event is greater than 5, it WILL be detected rle_publisher = EventPublisher("ExampleDetectableEvent") rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event", voltage=10) #------------------------------------------------------- # make assertions #------------------------------------------------------- msg_tuple = proc1.event_processors[ notification_id_2].smtp_client.sentmail.get(timeout=4) # The first event never triggered an email because the voltage was less than 5, the queue is now empty self.assertTrue(proc1.event_processors[notification_id_2].smtp_client. sentmail.empty()) self.assertEquals(msg_tuple[1], '*****@*****.**') #self.assertEquals(msg_tuple[0], ION_NOTIFICATION_EMAIL_ADDRESS) # parse the message body message = msg_tuple[2] list_lines = message.split("\n") message_dict = {} for line in list_lines: key_item = line.split(": ") if key_item[0] == 'Subject': message_dict['Subject'] = key_item[1] + key_item[2] else: try: message_dict[key_item[0]] = key_item[1] except IndexError as exc: # these IndexError exceptions happen only because the message sometimes # has successive /r/n (i.e. new lines) and therefore, # the indexing goes out of range. These new lines # can just be ignored. So we ignore the exceptions here. pass #self.assertEquals(message_dict['From'], ION_NOTIFICATION_EMAIL_ADDRESS) self.assertEquals(message_dict['To'], '*****@*****.**') self.assertEquals(message_dict['Event'].rstrip('\r'), 'DetectionEvent') self.assertEquals(message_dict['Originator'].rstrip('\r'), 'DetectionEventProcessor') self.assertEquals(message_dict['Description'].rstrip('\r'), 'Event was detected by DetectionEventProcessor') @unittest.skip('interface has changed!') def test_find_event_types_for_resource(self): # create a dataset object in the RR to pass into the UNS method dataset_object = IonObject(RT.DataSet, name="dataset1") dataset_id, version = self.rrc.create(dataset_object) # get the list of event types for the dataset events = self.unsc.find_event_types_for_resource(dataset_id) log.debug("dataset events = " + str(events)) if not events == ['dataset_supplement_added', 'dataset_change']: self.fail("failed to return correct list of event types") # try to pass in an id of a resource that doesn't exist (should fail) try: events = self.unsc.find_event_types_for_resource("bogus_id") self.fail("failed to detect non-existant resource") except: pass @unittest.skip('interface has changed!') def test_create_two_user_notifications(self): # create user with email address in RR user_identty_object = IonObject(RT.ActorIdentity, name="user1") user_id = self.imc.create_actor_identity(user_identty_object) user_info_object = IonObject( RT.UserInfo, { "name": "user1_info", "contact": { "email": '*****@*****.**' } }) self.imc.create_user_info(user_id, user_info_object) # create first notification notification_object1 = IonObject( RT.NotificationRequest, { "name": "notification1", "origin_list": ['Some_Resource_Agent_ID1'], "events_list": ['ResourceLifecycleEvent'] }) notification_id1 = self.unsc.create_notification( notification_object1, user_id) # create second notification notification_object2 = IonObject( RT.NotificationRequest, { "name": "notification2", "origin_list": ['Some_Resource_Agent_ID2'], "events_list": ['DataEvent'] }) notification_id2 = self.unsc.create_notification( notification_object2, user_id) # read the notifications back and check that they are correct n1 = self.unsc.read_notification(notification_id1) if n1.name != notification_object1.name or \ n1.origin_list != notification_object1.origin_list or \ n1.events_list != notification_object1.events_list: self.fail("notification was not correct") n2 = self.unsc.read_notification(notification_id2) if n2.name != notification_object2.name or \ n2.origin_list != notification_object2.origin_list or \ n2.events_list != notification_object2.events_list: self.fail("notification was not correct") @unittest.skip('interface has changed!') def test_delete_user_notifications(self): # create user with email address in RR user_identty_object = IonObject(RT.ActorIdentity, name="user1") user_id = self.imc.create_actor_identity(user_identty_object) user_info_object = IonObject( RT.UserInfo, { "name": "user1_info", "contact": { "email": '*****@*****.**' } }) self.imc.create_user_info(user_id, user_info_object) # create first notification notification_object1 = IonObject( RT.NotificationRequest, { "name": "notification1", "origin_list": ['Some_Resource_Agent_ID1'], "events_list": ['ResourceLifecycleEvent'] }) notification1_id = self.unsc.create_notification( notification_object1, user_id) # create second notification notification_object2 = IonObject( RT.NotificationRequest, { "name": "notification2", "origin_list": ['Some_Resource_Agent_ID2'], "events_list": ['DataEvent'] }) notification2_id = self.unsc.create_notification( notification_object2, user_id) # delete both notifications self.unsc.delete_notification(notification1_id) self.unsc.delete_notification(notification2_id) # check that the notifications are not there try: n1 = self.unsc.read_notification(notification1_id) except: try: n2 = self.unsc.read_notification(notification2_id) except: return self.fail("failed to delete notifications") @unittest.skip('interface has changed!') def test_find_user_notifications(self): # create user with email address in RR user_identty_object = IonObject(RT.ActorIdentity, name="user1") user_id = self.imc.create_actor_identity(user_identty_object) user_info_object = IonObject( RT.UserInfo, { "name": "user1_info", "contact": { "email": '*****@*****.**' } }) self.imc.create_user_info(user_id, user_info_object) # create first notification notification_object = IonObject( RT.NotificationRequest, { "name": "notification1", "origin_list": ['Some_Resource_Agent_ID1'], "events_list": ['ResourceLifecycleEvent'] }) self.unsc.create_notification(notification_object, user_id) # create second notification notification_object = IonObject( RT.NotificationRequest, { "name": "notification2", "origin_list": ['Some_Resource_Agent_ID2'], "events_list": ['DataEvent'] }) self.unsc.create_notification(notification_object, user_id) # try to find all notifications for user notifications = self.unsc.find_notifications_by_user(user_id) if len(notifications) != 2: self.fail("failed to find all notifications") @unittest.skip('interface has changed!') def test_update_user_notification(self): # create user with email address in RR user_identty_object = IonObject(RT.ActorIdentity, name="user1") user_id = self.imc.create_actor_identity(user_identty_object) user_info_object = IonObject( RT.UserInfo, { "name": "user1_info", "contact": { "email": '*****@*****.**' } }) self.imc.create_user_info(user_id, user_info_object) # create a notification notification_object = IonObject( RT.NotificationRequest, { "name": "notification1", "origin_list": ['Some_Resource_Agent_ID1'], "events_list": ['ResourceLifecycleEvent'] }) notification_id = self.unsc.create_notification( notification_object, user_id) # read back the notification and change it notification = self.unsc.read_notification(notification_id) notification.origin_list = ['Some_Resource_Agent_ID5'] self.unsc.update_notification(notification) # read back the notification and check that it got changed notification = self.unsc.read_notification(notification_id) if notification.origin_list != ['Some_Resource_Agent_ID5']: self.fail("failed to change notification") @unittest.skip('interface has changed!') def test_send_notification_emails(self): # create user with email address in RR user_identty_object = IonObject(RT.ActorIdentity, name="user1") user_id = self.imc.create_actor_identity(user_identty_object) user_info_object = IonObject(RT.UserInfo, { "name": "user1_info", "contact": { "email": '*****@*****.**' } }) self.imc.create_user_info(user_id, user_info_object) # create first notification notification_object = IonObject( RT.NotificationRequest, { "name": "notification1", "origin_list": ['Some_Resource_Agent_ID1'], "events_list": ['ResourceLifecycleEvent'] }) self.unsc.create_notification(notification_object, user_id) # create second notification notification_object = IonObject( RT.NotificationRequest, { "name": "notification2", "origin_list": ['Some_Resource_Agent_ID2'], "events_list": ['DataEvent'] }) self.unsc.create_notification(notification_object, user_id) # publish an event for each notification to generate the emails # this can't be easily check in SW so need to check for these at the [email protected] account rle_publisher = EventPublisher("ResourceLifecycleEvent") rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event") de_publisher = EventPublisher("DataEvent") de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event") gevent.sleep(1) @unittest.skip('interface has changed!') def test_find_events(self): # publish some events for the event repository rle_publisher = EventPublisher("ResourceLifecycleEvent") de_publisher = EventPublisher("DataEvent") rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event1") rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event2") rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event3") de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event1") de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event2") de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event3") # find all events for the originator 'Some_Resource_Agent_ID1' events = self.unsc.find_events(origin='Some_Resource_Agent_ID1') if len(events) != 3: self.fail("failed to find all events") for event in events: log.debug("event=" + str(event)) if event[1][0] != 'Some_Resource_Agent_ID1': self.fail("failed to find correct events") # find all events for the originator 'DataEvent' events = self.unsc.find_events(type='DataEvent') if len(events) != 3: self.fail("failed to find all events") for event in events: log.debug("event=" + str(event)) if event[1][0] != 'DataEvent': self.fail("failed to find correct events") # find 2 events for the originator 'Some_Resource_Agent_ID1' events = self.unsc.find_events(origin='Some_Resource_Agent_ID2', limit=2) if len(events) != 2: self.fail("failed to find all events") for event in events: log.debug("event=" + str(event)) if event[1][0] != 'Some_Resource_Agent_ID2': self.fail("failed to find correct events") # find all events for the originator 'Some_Resource_Agent_ID1' in reverse time order events = self.unsc.find_events(origin='Some_Resource_Agent_ID1', descending=True) if len(events) != 3: self.fail("failed to find all events") for event in events: log.debug("event=" + str(event)) if event[1][0] != 'Some_Resource_Agent_ID1': self.fail("failed to find correct events")
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.dpsc_cli = DataProductManagementServiceClient() self.rrclient = ResourceRegistryServiceClient() self.damsclient = DataAcquisitionManagementServiceClient() self.pubsubcli = PubsubManagementServiceClient() self.ingestclient = IngestionManagementServiceClient() self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() self.unsc = UserNotificationServiceClient() self.data_retriever = DataRetrieverServiceClient() self.identcli = IdentityManagementServiceClient() #------------------------------------------ # Create the environment #------------------------------------------ self.stream_def_id = self.pubsubcli.create_stream_definition( name='SBE37_CDM') self.process_definitions = {} ingestion_worker_definition = ProcessDefinition( name='ingestion worker') ingestion_worker_definition.executable = { 'module': 'ion.processes.data.ingestion.science_granule_ingestion_worker', 'class': 'ScienceGranuleIngestionWorker' } process_definition_id = self.process_dispatcher.create_process_definition( process_definition=ingestion_worker_definition) self.process_definitions['ingestion_worker'] = process_definition_id self.pids = [] self.exchange_points = [] self.exchange_names = [] #------------------------------------------------------------------------------------------------ # First launch the ingestors #------------------------------------------------------------------------------------------------ self.exchange_space = 'science_granule_ingestion' self.exchange_point = 'science_data' config = DotDict() config.process.datastore_name = 'datasets' config.process.queue_name = self.exchange_space self.exchange_names.append(self.exchange_space) self.exchange_points.append(self.exchange_point) pid = self.process_dispatcher.schedule_process( self.process_definitions['ingestion_worker'], configuration=config) log.debug("the ingestion worker process id: %s", pid) self.pids.append(pid) self.addCleanup(self.cleaning_up) def cleaning_up(self): for pid in self.pids: log.debug("number of pids to be terminated: %s", len(self.pids)) try: self.process_dispatcher.cancel_process(pid) log.debug("Terminated the process: %s", pid) except: log.debug("could not terminate the process id: %s" % pid) IngestionManagementIntTest.clean_subscriptions() for xn in self.exchange_names: xni = self.container.ex_manager.create_xn_queue(xn) xni.delete() for xp in self.exchange_points: xpi = self.container.ex_manager.create_xp(xp) xpi.delete() def get_datastore(self, dataset_id): dataset = self.dataset_management.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore( datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore @attr('EXT') @attr('PREP') def test_create_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ parameter_dictionary = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict') ctd_stream_def_id = self.pubsubcli.create_stream_definition( name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary._id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp') dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0 dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0 dp_obj.ooi_product_name = "PRODNAME" #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product( data_product=dp_obj, stream_definition_id=ctd_stream_def_id) # Assert that the data product has an associated stream at this stage stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True) self.assertNotEquals(len(stream_ids), 0) # Assert that the data product has an associated stream def at this stage stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStreamDefinition, RT.StreamDefinition, True) self.assertNotEquals(len(stream_ids), 0) self.dpsc_cli.activate_data_product_persistence(dp_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertIsNotNone(dp_obj) self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0) log.debug('Created data product %s', dp_obj) #------------------------------------------------------------------------------------------------ # test creating a new data product with a stream definition #------------------------------------------------------------------------------------------------ log.debug('Creating new data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name='DP2', description='some new dp') dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id) self.dpsc_cli.activate_data_product_persistence(dp_id2) log.debug('new dp_id = %s' % dp_id2) #------------------------------------------------------------------------------------------------ #make sure data product is associated with stream def #------------------------------------------------------------------------------------------------ streamdefs = [] streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True) for s in streams: log.debug("Checking stream %s" % s) sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True) for sd in sdefs: log.debug("Checking streamdef %s" % sd) streamdefs.append(sd) self.assertIn(ctd_stream_def_id, streamdefs) group_names = self.dpsc_cli.get_data_product_group_list() self.assertIn("PRODNAME", group_names) #---------------------------------------------------------------------------------------- # Create users then notifications to this data product for each user #---------------------------------------------------------------------------------------- # user_1 user_1 = UserInfo() user_1.name = 'user_1' user_1.contact.email = '*****@*****.**' # user_2 user_2 = UserInfo() user_2.name = 'user_2' user_2.contact.email = '*****@*****.**' #user1 is a complete user self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254" actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) actor_id = self.identcli.create_actor_identity(actor_identity_obj) user_credentials_obj = IonObject("UserCredentials", {"name": self.subject}) self.identcli.register_user_credentials(actor_id, user_credentials_obj) user_id_1 = self.identcli.create_user_info(actor_id, user_1) user_id_2, _ = self.rrclient.create(user_2) delivery_config1a = IonObject( OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH) delivery_config1b = IonObject( OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH) notification_request_1 = NotificationRequest( name="notification_1", origin=dp_id, origin_type="type_1", event_type=OT.ResourceLifecycleEvent, disabled_by_system=False, delivery_configurations=[delivery_config1a, delivery_config1b]) delivery_config2a = IonObject( OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH) delivery_config2b = IonObject( OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH) notification_request_2 = NotificationRequest( name="notification_2", origin=dp_id, origin_type="type_2", disabled_by_system=False, event_type=OT.DetectionEvent, delivery_configurations=[delivery_config2a, delivery_config2b]) notification_request_1_id = self.unsc.create_notification( notification=notification_request_1, user_id=user_id_1) notification_request_2_id = self.unsc.create_notification( notification=notification_request_2, user_id=user_id_2) self.unsc.delete_notification(notification_request_1_id) # test reading a non-existent data product log.debug('reading non-existent data product') with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product('some_fake_id') # update a data product (tests read also) log.debug('Updating data product') # first get the existing dp object dp_obj = self.dpsc_cli.read_data_product(dp_id) # now tweak the object dp_obj.description = 'the very first dp' dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 20.0 dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -20.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 20.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -20.0 # now write the dp back to the registry update_result = self.dpsc_cli.update_data_product(dp_obj) # now get the dp back to see if it was updated dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertEquals(dp_obj.description, 'the very first dp') self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0) log.debug('Updated data product %s', dp_obj) #test extension extended_product = self.dpsc_cli.get_data_product_extension(dp_id) #validate that there is one active and one retired user notification for this data product self.assertEqual( 1, len(extended_product.computed.active_user_subscriptions.value)) self.assertEqual( 1, len(extended_product.computed.past_user_subscriptions.value)) self.assertEqual(dp_id, extended_product._id) self.assertEqual( ComputedValueAvailability.PROVIDED, extended_product.computed.product_download_size_estimated.status) self.assertEqual( 0, extended_product.computed.product_download_size_estimated.value) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.computed.parameters.status) #log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value) def ion_object_encoder(obj): return obj.__dict__ #test prepare for create data_product_data = self.dpsc_cli.prepare_data_product_support() #print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2) self.assertEqual(data_product_data._id, "") self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport) self.assertEqual( len(data_product_data.associations['StreamDefinition'].resources), 2) self.assertEqual( len(data_product_data.associations['Dataset'].resources), 0) self.assertEqual( len(data_product_data.associations['StreamDefinition']. associated_resources), 0) self.assertEqual( len(data_product_data.associations['Dataset'].associated_resources ), 0) #test prepare for update data_product_data = self.dpsc_cli.prepare_data_product_support(dp_id) #print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2) self.assertEqual(data_product_data._id, dp_id) self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport) self.assertEqual( len(data_product_data.associations['StreamDefinition'].resources), 2) self.assertEqual( len(data_product_data.associations['Dataset'].resources), 1) self.assertEqual( len(data_product_data.associations['StreamDefinition']. associated_resources), 1) self.assertEqual( data_product_data.associations['StreamDefinition']. associated_resources[0].s, dp_id) self.assertEqual( len(data_product_data.associations['Dataset'].associated_resources ), 1) self.assertEqual( data_product_data.associations['Dataset'].associated_resources[0]. s, dp_id) # now 'delete' the data product log.debug("deleting data product: %s" % dp_id) self.dpsc_cli.delete_data_product(dp_id) # Assert that there are no associated streams leftover after deleting the data product stream_ids, assoc_ids = self.rrclient.find_objects( dp_id, PRED.hasStream, RT.Stream, True) self.assertEquals(len(stream_ids), 0) self.assertEquals(len(assoc_ids), 0) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product(dp_id) # Get the events corresponding to the data product ret = self.unsc.get_recent_events(resource_id=dp_id) events = ret.value for event in events: log.debug("event time: %s" % event.ts_created) self.assertTrue(len(events) > 0) def test_data_product_stream_def(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition( name='Simulated CTD data', parameter_dictionary_id=pdict_id) dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp') dp_id = self.dpsc_cli.create_data_product( data_product=dp_obj, stream_definition_id=ctd_stream_def_id) stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id) self.assertEquals(ctd_stream_def_id, stream_def_id) def test_derived_data_product(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition( name='ctd parsed', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsubcli.delete_stream_definition, ctd_stream_def_id) dp = DataProduct(name='Instrument DP') dp_id = self.dpsc_cli.create_data_product( dp, stream_definition_id=ctd_stream_def_id) self.addCleanup(self.dpsc_cli.force_delete_data_product, dp_id) self.dpsc_cli.activate_data_product_persistence(dp_id) self.addCleanup(self.dpsc_cli.suspend_data_product_persistence, dp_id) dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True) if not dataset_ids: raise NotFound("Data Product %s dataset does not exist" % str(dp_id)) dataset_id = dataset_ids[0] # Make the derived data product simple_stream_def_id = self.pubsubcli.create_stream_definition( name='TEMPWAT stream def', parameter_dictionary_id=pdict_id, available_fields=['time', 'temp']) tempwat_dp = DataProduct(name='TEMPWAT', category=DataProductTypeEnum.DERIVED) tempwat_dp_id = self.dpsc_cli.create_data_product( tempwat_dp, stream_definition_id=simple_stream_def_id, parent_data_product_id=dp_id) self.addCleanup(self.dpsc_cli.delete_data_product, tempwat_dp_id) # Check that the streams associated with the data product are persisted with stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True) for stream_id in stream_ids: self.assertTrue(self.ingestclient.is_persisted(stream_id)) stream_id = stream_ids[0] route = self.pubsubcli.read_stream_route(stream_id=stream_id) rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id) rdt['time'] = np.arange(20) rdt['temp'] = np.arange(20) rdt['pressure'] = np.arange(20) publisher = StandaloneStreamPublisher(stream_id, route) dataset_modified = Event() def cb(*args, **kwargs): dataset_modified.set() es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True) es.start() self.addCleanup(es.stop) publisher.publish(rdt.to_granule()) self.assertTrue(dataset_modified.wait(30)) tempwat_dataset_ids, _ = self.rrclient.find_objects(tempwat_dp_id, PRED.hasDataset, id_only=True) tempwat_dataset_id = tempwat_dataset_ids[0] granule = self.data_retriever.retrieve( tempwat_dataset_id, delivery_format=simple_stream_def_id) rdt = RecordDictionaryTool.load_from_granule(granule) np.testing.assert_array_equal(rdt['time'], np.arange(20)) self.assertEquals(set(rdt.fields), set(['time', 'temp'])) def test_activate_suspend_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition( name='Simulated CTD data', parameter_dictionary_id=pdict_id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ # Construct temporal and spatial Coordinate Reference System objects dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp') log.debug("Created an IonObject for a data product: %s" % dp_obj) #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product( data_product=dp_obj, stream_definition_id=ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # Subscribe to persist events #------------------------------------------------------------------------------------------------ queue = gevent.queue.Queue() def info_event_received(message, headers): queue.put(message) es = EventSubscriber(event_type=OT.InformationContentStatusEvent, callback=info_event_received, origin=dp_id, auto_delete=True) es.start() self.addCleanup(es.stop) #------------------------------------------------------------------------------------------------ # test activate and suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.activate_data_product_persistence(dp_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertIsNotNone(dp_obj) dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True) if not dataset_ids: raise NotFound("Data Product %s dataset does not exist" % str(dp_id)) dataset_id = dataset_ids[0] # Check that the streams associated with the data product are persisted with stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True) for stream_id in stream_ids: self.assertTrue(self.ingestclient.is_persisted(stream_id)) stream_id = stream_ids[0] route = self.pubsubcli.read_stream_route(stream_id=stream_id) rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id) rdt['time'] = np.arange(20) rdt['temp'] = np.arange(20) publisher = StandaloneStreamPublisher(stream_id, route) dataset_modified = Event() def cb(*args, **kwargs): dataset_modified.set() es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True) es.start() self.addCleanup(es.stop) publisher.publish(rdt.to_granule()) self.assertTrue(dataset_modified.wait(30)) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.data_retriever.retrieve(dataset_ids[0]) self.assertIsInstance(replay_data, Granule) log.debug( "The data retriever was able to replay the dataset that was attached to the data product " "we wanted to be persisted. Therefore the data product was indeed persisted with " "otherwise we could not have retrieved its dataset using the data retriever. Therefore " "this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'" ) data_product_object = self.rrclient.read(dp_id) self.assertEquals(data_product_object.name, 'DP1') self.assertEquals(data_product_object.description, 'some new dp') log.debug( "Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. " " Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the " "resource registry, name='%s', desc='%s'" % (dp_obj.name, dp_obj.description, data_product_object.name, data_product_object.description)) #------------------------------------------------------------------------------------------------ # test suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.suspend_data_product_persistence(dp_id) dataset_modified.clear() rdt['time'] = np.arange(20, 40) publisher.publish(rdt.to_granule()) self.assertFalse(dataset_modified.wait(2)) self.dpsc_cli.activate_data_product_persistence(dp_id) dataset_modified.clear() publisher.publish(rdt.to_granule()) self.assertTrue(dataset_modified.wait(30)) granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(granule) np.testing.assert_array_almost_equal(rdt['time'], np.arange(40)) dataset_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasDataset, id_only=True) self.assertEquals(len(dataset_ids), 1) self.dpsc_cli.suspend_data_product_persistence(dp_id) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object with self.assertRaises(NotFound): dp_obj = self.rrclient.read(dp_id) info_event_counter = 0 runtime = 0 starttime = time.time() caught_events = [] #check that the four InfoStatusEvents were received while info_event_counter < 4 and runtime < 60: a = queue.get(timeout=60) caught_events.append(a) info_event_counter += 1 runtime = time.time() - starttime self.assertEquals(info_event_counter, 4)
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' unittest # suppress an pycharm inspector error if all unittest.skip references are commented out self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient( node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.OMS = ObservatoryManagementServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) # @unittest.skip('this test just for debugging setup') # def test_just_the_setup(self): # return @attr('EXT') def test_resources_associations_extensions(self): """ create one of each resource and association used by IMS to guard against problems in ion-definitions """ #stuff we control instrument_agent_instance_id, _ = self.RR.create( any_old(RT.InstrumentAgentInstance)) instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent)) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) instrument_site_id, _ = self.RR.create(any_old(RT.InstrumentSite)) platform_agent_instance_id, _ = self.RR.create( any_old(RT.PlatformAgentInstance)) platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent)) platform_site_id, _ = self.RR.create(any_old(RT.PlatformSite)) platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice)) sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel)) #stuff we associate to data_producer_id, _ = self.RR.create(any_old(RT.DataProducer)) org_id, _ = self.RR.create(any_old(RT.Org)) #instrument_agent_instance_id #is only a target #instrument_agent self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id) self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id) self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device_id) instrument_model_id #is only a target platform_agent_instance_id #is only a target #platform_agent self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id) self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(instrument_site_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(platform_site_id, PRED.hasDevice, platform_device_id) self.RR.create_association(platform_site_id, PRED.hasSite, instrument_site_id) platform_model_id #is only a target #sensor_device self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id) sensor_model_id #is only a target #create a parsed product for this instrument output tdom, sdom = time_series_domain() tdom = tdom.dump() sdom = sdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', processing_level_code='Parsed_Canonical', temporal_domain=tdom, spatial_domain=sdom) pdict_id = self.DSC.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition( name='parsed', parameter_dictionary_id=pdict_id) data_product_id1 = self.DP.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug('new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1) def addInstOwner(inst_id, subject): actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject}) user_id = self.IDS.create_actor_identity(actor_identity_obj) user_info_obj = any_old(RT.UserInfo) user_info_id = self.IDS.create_user_info(user_id, user_info_obj) self.RR.create_association(inst_id, PRED.hasOwner, user_id) #Testing multiple instrument owners addInstOwner( instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254") addInstOwner( instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256") extended_instrument = self.IMS.get_instrument_device_extension( instrument_device_id) self.assertEqual(instrument_device_id, extended_instrument._id) self.assertEqual(len(extended_instrument.owners), 2) self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id) # Lifecycle self.assertEquals(len(extended_instrument.lcstate_transitions), 5) self.assertEquals( set(extended_instrument.lcstate_transitions.keys()), set(['develop', 'deploy', 'retire', 'plan', 'integrate'])) self.assertEquals(len(extended_instrument.availability_transitions), 2) self.assertEquals( set(extended_instrument.availability_transitions.keys()), set(['enable', 'announce'])) # Verify that computed attributes exist for the extended instrument self.assertIsInstance( extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) self.assertIsInstance( extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance( extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance( extended_instrument.computed.location_status_roll_up, ComputedIntValue) log.debug("extended_instrument.computed: %s", extended_instrument.computed) #check model inst_model_obj = self.RR.read(instrument_model_id) self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name) #check agent instance inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id) self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name) #check agent inst_agent_obj = self.RR.read(instrument_agent_id) #compound assoc return list of lists so check the first element self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent.name) #check platform device plat_device_obj = self.RR.read(platform_device_id) self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name) extended_platform = self.IMS.get_platform_device_extension( platform_device_id) self.assertEqual(1, len(extended_platform.portals)) self.assertEqual(1, len(extended_platform.portal_instruments)) #self.assertEqual(1, len(extended_platform.computed.portal_status.value)) # no agent started so NO statuses reported self.assertEqual(1, len(extended_platform.instrument_devices)) self.assertEqual(instrument_device_id, extended_platform.instrument_devices[0]._id) self.assertEqual(1, len(extended_platform.instrument_models)) self.assertEqual(instrument_model_id, extended_platform.instrument_models[0]._id) self.assertEquals(extended_platform.platform_agent._id, platform_agent_id) self.assertEquals(len(extended_platform.lcstate_transitions), 5) self.assertEquals( set(extended_platform.lcstate_transitions.keys()), set(['develop', 'deploy', 'retire', 'plan', 'integrate'])) self.assertEquals(len(extended_platform.availability_transitions), 2) self.assertEquals( set(extended_platform.availability_transitions.keys()), set(['enable', 'announce'])) #check sensor devices self.assertEqual(1, len(extended_instrument.sensor_devices)) #check data_product_parameters_set self.assertEqual( ComputedValueAvailability.PROVIDED, extended_instrument.computed.data_product_parameters_set.status) self.assertTrue('Parsed_Canonical' in extended_instrument.computed. data_product_parameters_set.value) # the ctd parameters should include 'temp' self.assertTrue('temp' in extended_instrument.computed. data_product_parameters_set.value['Parsed_Canonical']) #none of these will work because there is no agent # self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, # extended_instrument.computed.firmware_version.status) # self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, # extended_instrument.computed.operational_state.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.power_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.communications_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.data_status_roll_up.status) # self.assertEqual(DeviceStatusType.STATUS_OK, # extended_instrument.computed.data_status_roll_up.value) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.location_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.recent_events.status) # self.assertEqual([], extended_instrument.computed.recent_events.value) # cleanup c = DotDict() c.resource_registry = self.RR self.RR2.pluck(instrument_agent_id) self.RR2.pluck(instrument_model_id) self.RR2.pluck(instrument_device_id) self.RR2.pluck(platform_agent_id) self.RR2.pluck(sensor_device_id) self.IMS.force_delete_instrument_agent(instrument_agent_id) self.IMS.force_delete_instrument_model(instrument_model_id) self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_platform_agent_instance( platform_agent_instance_id) self.IMS.force_delete_platform_agent(platform_agent_id) self.OMS.force_delete_instrument_site(instrument_site_id) self.OMS.force_delete_platform_site(platform_site_id) self.IMS.force_delete_platform_device(platform_device_id) self.IMS.force_delete_platform_model(platform_model_id) self.IMS.force_delete_sensor_device(sensor_device_id) self.IMS.force_delete_sensor_model(sensor_model_id) #stuff we associate to self.RR.delete(data_producer_id) self.RR.delete(org_id) def test_custom_attributes(self): """ Test assignment of custom attributes """ instModel_obj = IonObject(OT.CustomAttribute, name='SBE37IMModelAttr', description="model custom attr") instrument_model_id, _ = self.RR.create( any_old(RT.InstrumentModel, {"custom_attributes": [instModel_obj]})) instrument_device_id, _ = self.RR.create( any_old( RT.InstrumentDevice, { "custom_attributes": { "favorite_color": "red", "bogus_attr": "should raise warning" } })) self.IMS.assign_instrument_model_to_instrument_device( instrument_model_id, instrument_device_id) # cleanup self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_instrument_model(instrument_model_id) def _get_datastore(self, dataset_id): dataset = self.DSC.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore( datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def test_data_producer(self): idevice_id = self.IMS.create_instrument_device( any_old(RT.InstrumentDevice)) self.assertEqual( 1, len( self.RR2. find_data_producer_ids_of_instrument_device_using_has_data_producer( idevice_id))) pdevice_id = self.IMS.create_platform_device(any_old( RT.PlatformDevice)) self.assertEqual( 1, len( self.RR2. find_data_producer_ids_of_platform_device_using_has_data_producer( pdevice_id))) @attr('PREP') def test_prepare_resource_support(self): """ create one of each resource and association used by IMS to guard against problems in ion-definitions """ #stuff we control instrument_agent_instance_id, _ = self.RR.create( any_old(RT.InstrumentAgentInstance)) instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent)) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) platform_agent_instance_id, _ = self.RR.create( any_old(RT.PlatformAgentInstance)) platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent)) platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice)) sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel)) instrument_device2_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) instrument_device3_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) platform_device2_id, _ = self.RR.create(any_old(RT.PlatformDevice)) sensor_device2_id, _ = self.RR.create(any_old(RT.SensorDevice)) #stuff we associate to data_producer_id, _ = self.RR.create(any_old(RT.DataProducer)) org_id, _ = self.RR.create(any_old(RT.Org)) #instrument_agent_instance_id #is only a target #instrument_agent self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id) self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id) self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device_id) self.RR.create_association(instrument_device2_id, PRED.hasModel, instrument_model_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device2_id) instrument_model_id #is only a target platform_agent_instance_id #is only a target #platform_agent self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id) self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(platform_device2_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device2_id, PRED.hasDevice, instrument_device2_id) platform_model_id #is only a target #sensor_device self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(sensor_device2_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device2_id, PRED.hasDevice, instrument_device2_id) sensor_model_id #is only a target #set lcstate - used for testing prepare - not setting all to DEVELOP, only some self.RR.execute_lifecycle_transition(instrument_agent_id, LCE.DEVELOP) self.RR.execute_lifecycle_transition(instrument_device_id, LCE.DEVELOP) self.RR.execute_lifecycle_transition(instrument_device2_id, LCE.DEVELOP) self.RR.execute_lifecycle_transition(platform_device_id, LCE.DEVELOP) self.RR.execute_lifecycle_transition(platform_device2_id, LCE.DEVELOP) self.RR.execute_lifecycle_transition(platform_agent_id, LCE.DEVELOP) #create a parsed product for this instrument output tdom, sdom = time_series_domain() tdom = tdom.dump() sdom = sdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', processing_level_code='Parsed_Canonical', temporal_domain=tdom, spatial_domain=sdom) pdict_id = self.DSC.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition( name='parsed', parameter_dictionary_id=pdict_id) data_product_id1 = self.DP.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug('new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1) def addInstOwner(inst_id, subject): actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject}) user_id = self.IDS.create_actor_identity(actor_identity_obj) user_info_obj = any_old(RT.UserInfo) user_info_id = self.IDS.create_user_info(user_id, user_info_obj) self.RR.create_association(inst_id, PRED.hasOwner, user_id) #Testing multiple instrument owners addInstOwner( instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254") addInstOwner( instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256") def ion_object_encoder(obj): return obj.__dict__ #First call to create instrument_data = self.IMS.prepare_instrument_device_support() #print simplejson.dumps(instrument_data, default=ion_object_encoder, indent=2) self.assertEqual(instrument_data._id, '') self.assertEqual(instrument_data.type_, OT.InstrumentDevicePrepareSupport) self.assertEqual( len(instrument_data.associations['InstrumentModel'].resources), 1) self.assertEqual( instrument_data.associations['InstrumentModel'].resources[0]._id, instrument_model_id) self.assertEqual( len(instrument_data.associations['InstrumentAgentInstance']. resources), 1) self.assertEqual( instrument_data.associations['InstrumentAgentInstance']. resources[0]._id, instrument_agent_instance_id) self.assertEqual( len(instrument_data.associations['InstrumentModel']. associated_resources), 0) self.assertEqual( len(instrument_data.associations['InstrumentAgentInstance']. associated_resources), 0) self.assertEqual( len(instrument_data.associations['SensorDevice'].resources), 0) #Next call to update instrument_data = self.IMS.prepare_instrument_device_support( instrument_device_id) #print 'Update results' #print simplejson.dumps(instrument_data, default=ion_object_encoder, indent=2) self.assertEqual(instrument_data._id, instrument_device_id) self.assertEqual(instrument_data.type_, OT.InstrumentDevicePrepareSupport) self.assertEqual( len(instrument_data.associations['InstrumentModel'].resources), 1) self.assertEqual( instrument_data.associations['InstrumentModel'].resources[0]._id, instrument_model_id) self.assertEqual( len(instrument_data.associations['InstrumentAgentInstance']. resources), 1) self.assertEqual( instrument_data.associations['InstrumentAgentInstance']. resources[0]._id, instrument_agent_instance_id) self.assertEqual( len(instrument_data.associations['InstrumentModel']. associated_resources), 1) self.assertEqual( instrument_data.associations['InstrumentModel']. associated_resources[0].s, instrument_device_id) self.assertEqual( instrument_data.associations['InstrumentModel']. associated_resources[0].o, instrument_model_id) self.assertEqual( len(instrument_data.associations['InstrumentAgentInstance']. associated_resources), 1) self.assertEqual( instrument_data.associations['InstrumentAgentInstance']. associated_resources[0].o, instrument_agent_instance_id) self.assertEqual( instrument_data.associations['InstrumentAgentInstance']. associated_resources[0].s, instrument_device_id) self.assertEqual( len(instrument_data.associations['SensorDevice'].resources), 1) self.assertEqual( instrument_data.associations['SensorDevice'].resources[0]._id, sensor_device_id) self.assertEqual( len(instrument_data.associations['SensorDevice']. associated_resources), 1) self.assertEqual( instrument_data.associations['SensorDevice']. associated_resources[0].o, instrument_device_id) self.assertEqual( instrument_data.associations['SensorDevice']. associated_resources[0].s, sensor_device_id) self.assertEqual( instrument_data.associations['InstrumentModel'].assign_request. request_parameters['instrument_device_id'], instrument_device_id) #test prepare for create of instrument agent instance instrument_agent_data = self.IMS.prepare_instrument_agent_instance_support( ) #print 'Update results' #print simplejson.dumps(instrument_agent_data, default=ion_object_encoder, indent=2) self.assertEqual(instrument_agent_data._id, '') self.assertEqual(instrument_agent_data.type_, OT.InstrumentAgentInstancePrepareSupport) self.assertEqual( len(instrument_agent_data.associations['InstrumentDevice']. resources), 2) self.assertEqual( len(instrument_agent_data.associations['InstrumentAgent'].resources ), 1) self.assertEqual( instrument_agent_data.associations['InstrumentAgent'].resources[0]. _id, instrument_agent_id) self.assertEqual( len(instrument_agent_data.associations['InstrumentDevice']. associated_resources), 0) self.assertEqual( len(instrument_agent_data.associations['InstrumentAgent']. associated_resources), 0) #test prepare for update of instrument agent instance to see if it is associated with the instrument that was created instrument_agent_data = self.IMS.prepare_instrument_agent_instance_support( instrument_agent_instance_id=instrument_agent_instance_id) #print 'Update results' #print simplejson.dumps(instrument_agent_data, default=ion_object_encoder, indent=2) self.assertEqual(instrument_agent_data._id, instrument_agent_instance_id) self.assertEqual(instrument_agent_data.type_, OT.InstrumentAgentInstancePrepareSupport) self.assertEqual( len(instrument_agent_data.associations['InstrumentDevice']. resources), 3) self.assertEqual( len(instrument_agent_data.associations['InstrumentAgent'].resources ), 1) self.assertEqual( instrument_agent_data.associations['InstrumentAgent'].resources[0]. _id, instrument_agent_id) self.assertEqual( len(instrument_agent_data.associations['InstrumentDevice']. associated_resources), 1) self.assertEqual( instrument_agent_data.associations['InstrumentDevice']. associated_resources[0].s, instrument_device_id) self.assertEqual( instrument_agent_data.associations['InstrumentDevice']. associated_resources[0].o, instrument_agent_instance_id) self.assertEqual( len(instrument_agent_data.associations['InstrumentAgent']. associated_resources), 1) self.assertEqual( instrument_agent_data.associations['InstrumentAgent']. associated_resources[0].o, instrument_agent_id) self.assertEqual( instrument_agent_data.associations['InstrumentAgent']. associated_resources[0].s, instrument_agent_instance_id) self.assertEqual( instrument_agent_data.associations['InstrumentAgent']. assign_request.request_parameters['instrument_agent_instance_id'], instrument_agent_instance_id) #test prepare for update of data product to see if it is associated with the instrument that was created data_product_data = self.DP.prepare_data_product_support( data_product_id1) #print simplejson.dumps(data_product_data, default=ion_object_encoder, indent=2) self.assertEqual(data_product_data._id, data_product_id1) self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport) self.assertEqual( len(data_product_data.associations['StreamDefinition'].resources), 1) self.assertEqual( len(data_product_data.associations['Dataset'].resources), 0) self.assertEqual( len(data_product_data.associations['StreamDefinition']. associated_resources), 1) self.assertEqual( data_product_data.associations['StreamDefinition']. associated_resources[0].s, data_product_id1) self.assertEqual( len(data_product_data.associations['Dataset'].associated_resources ), 0) self.assertEqual( len(data_product_data. associations['InstrumentDeviceHasOutputProduct'].resources), 3) self.assertEqual( len(data_product_data.associations[ 'InstrumentDeviceHasOutputProduct'].associated_resources), 1) self.assertEqual( data_product_data.associations['InstrumentDeviceHasOutputProduct']. associated_resources[0].s, instrument_device_id) self.assertEqual( data_product_data.associations['InstrumentDeviceHasOutputProduct']. associated_resources[0].o, data_product_id1) self.assertEqual( len(data_product_data.associations['PlatformDevice'].resources), 2) platform_data = self.IMS.prepare_platform_device_support() #print simplejson.dumps(platform_data, default=ion_object_encoder, indent=2) self.assertEqual(platform_data._id, '') self.assertEqual(platform_data.type_, OT.PlatformDevicePrepareSupport) self.assertEqual( len(platform_data.associations['PlatformModel'].resources), 1) self.assertEqual( platform_data.associations['PlatformModel'].resources[0]._id, platform_model_id) self.assertEqual( len(platform_data.associations['PlatformAgentInstance'].resources), 1) self.assertEqual( platform_data.associations['PlatformAgentInstance'].resources[0]. _id, platform_agent_instance_id) self.assertEqual( len(platform_data.associations['PlatformModel']. associated_resources), 0) self.assertEqual( len(platform_data.associations['PlatformAgentInstance']. associated_resources), 0) self.assertEqual( len(platform_data.associations['InstrumentDevice'].resources), 1) platform_data = self.IMS.prepare_platform_device_support( platform_device_id) #print simplejson.dumps(platform_data, default=ion_object_encoder, indent=2) self.assertEqual(platform_data._id, platform_device_id) self.assertEqual(platform_data.type_, OT.PlatformDevicePrepareSupport) self.assertEqual( len(platform_data.associations['PlatformModel'].resources), 1) self.assertEqual( platform_data.associations['PlatformModel'].resources[0]._id, platform_model_id) self.assertEqual( len(platform_data.associations['PlatformAgentInstance'].resources), 1) self.assertEqual( platform_data.associations['PlatformAgentInstance'].resources[0]. _id, platform_agent_instance_id) self.assertEqual( len(platform_data.associations['PlatformModel']. associated_resources), 1) self.assertEqual( platform_data.associations['PlatformModel']. associated_resources[0].s, platform_device_id) self.assertEqual( platform_data.associations['PlatformModel']. associated_resources[0].o, platform_model_id) self.assertEqual( len(platform_data.associations['PlatformAgentInstance']. associated_resources), 1) self.assertEqual( platform_data.associations['PlatformAgentInstance']. associated_resources[0].o, platform_agent_instance_id) self.assertEqual( platform_data.associations['PlatformAgentInstance']. associated_resources[0].s, platform_device_id) self.assertEqual( len(platform_data.associations['InstrumentDevice'].resources), 2) #self.assertEqual(len(platform_data.associations['InstrumentDevice'].associated_resources), 1) #self.assertEqual(platform_data.associations['InstrumentDevice'].associated_resources[0].s, platform_device_id) #self.assertEqual(platform_data.associations['InstrumentDevice'].associated_resources[0].o, instrument_device_id) self.assertEqual( platform_data.associations['PlatformModel'].assign_request. request_parameters['platform_device_id'], platform_device_id) # cleanup c = DotDict() c.resource_registry = self.RR self.RR2.pluck(instrument_agent_id) self.RR2.pluck(instrument_model_id) self.RR2.pluck(instrument_device_id) self.RR2.pluck(platform_agent_id) self.RR2.pluck(sensor_device_id) self.RR2.pluck(sensor_device2_id) self.IMS.force_delete_instrument_agent(instrument_agent_id) self.IMS.force_delete_instrument_model(instrument_model_id) self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_instrument_device(instrument_device2_id) self.IMS.force_delete_platform_agent_instance( platform_agent_instance_id) self.IMS.force_delete_platform_agent(platform_agent_id) self.IMS.force_delete_platform_device(platform_device_id) self.IMS.force_delete_platform_device(platform_device2_id) self.IMS.force_delete_platform_model(platform_model_id) self.IMS.force_delete_sensor_device(sensor_device_id) self.IMS.force_delete_sensor_device(sensor_device2_id) self.IMS.force_delete_sensor_model(sensor_model_id) #stuff we associate to self.RR.delete(data_producer_id) self.RR.delete(org_id)
class TestIdentityManagementServiceInt(IonIntegrationTestCase): def setUp(self): self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254" # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.resource_registry = ResourceRegistryServiceClient( node=self.container.node) self.identity_management_service = IdentityManagementServiceClient( node=self.container.node) self.org_client = OrgManagementServiceClient(node=self.container.node) def test_actor_identity(self): actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) user_id = self.identity_management_service.create_actor_identity( actor_identity_obj) actor_identity = self.identity_management_service.read_actor_identity( user_id) actor_identity.name = 'Updated subject' self.identity_management_service.update_actor_identity(actor_identity) ai = self.identity_management_service.find_actor_identity_by_name( actor_identity.name) self._baseAssertEqual(ai.name, actor_identity.name) with self.assertRaises(NotFound): ai = self.identity_management_service.find_actor_identity_by_name( "Yeah, well, you know, that's just, like, your opinion, man.") self._baseAssertEqual(ai.name, actor_identity.name) self.identity_management_service.delete_actor_identity(user_id) with self.assertRaises(NotFound) as cm: self.identity_management_service.read_actor_identity(user_id) self.assertTrue("does not exist" in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.delete_actor_identity(user_id) self.assertTrue("does not exist" in cm.exception.message) def test_user_credentials(self): actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) user_id = self.identity_management_service.create_actor_identity( actor_identity_obj) user_credentials_obj = IonObject("UserCredentials", {"name": self.subject}) self.identity_management_service.register_user_credentials( user_id, user_credentials_obj) with self.assertRaises(NotFound) as cm: self.identity_management_service.unregister_user_credentials( "bad", self.subject) self.assertTrue("does not exist" in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.unregister_user_credentials( user_id, "bad") self.assertTrue("does not exist" in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.unregister_user_credentials( 'bad', 'bad') self.assertTrue("does not exist" in cm.exception.message) self.identity_management_service.unregister_user_credentials( user_id, self.subject) self.identity_management_service.delete_actor_identity(user_id) def test_user_info(self): actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) user_id = self.identity_management_service.create_actor_identity( actor_identity_obj) user_credentials_obj = IonObject("UserCredentials", {"name": self.subject}) self.identity_management_service.register_user_credentials( user_id, user_credentials_obj) user_info_obj = IonObject("UserInfo", {"name": "Foo"}) user_info = self.identity_management_service.create_user_info( user_id, user_info_obj) with self.assertRaises(Conflict) as cm: self.identity_management_service.create_user_info( user_id, user_info_obj) self.assertTrue( "UserInfo already exists for user id" in cm.exception.message) user_info_obj = self.identity_management_service.find_user_info_by_id( user_id) user_info_obj = self.identity_management_service.find_user_info_by_name( "Foo") user_info_obj = self.identity_management_service.find_user_info_by_subject( self.subject) user_info_obj = self.identity_management_service.read_user_info( user_info) user_info_obj.name = 'Jane Doe' self.identity_management_service.update_user_info(user_info_obj) self.identity_management_service.delete_user_info(user_info) with self.assertRaises(NotFound) as cm: self.identity_management_service.read_user_info(user_info) self.assertTrue('does not exist' in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.delete_user_info(user_info) self.assertTrue('does not exist' in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.find_user_info_by_name("John Doe") self.assertEqual(cm.exception.message, 'UserInfo with name John Doe does not exist') with self.assertRaises(NotFound) as cm: self.identity_management_service.find_user_info_by_subject( "Bogus subject") self.assertEqual( cm.exception.message, "UserCredentials with subject Bogus subject does not exist") self.identity_management_service.unregister_user_credentials( user_id, self.subject) self.identity_management_service.delete_actor_identity(user_id) def test_signon(self): certificate = """-----BEGIN CERTIFICATE----- MIIEMzCCAxugAwIBAgICBQAwDQYJKoZIhvcNAQEFBQAwajETMBEGCgmSJomT8ixkARkWA29yZzEX MBUGCgmSJomT8ixkARkWB2NpbG9nb24xCzAJBgNVBAYTAlVTMRAwDgYDVQQKEwdDSUxvZ29uMRsw GQYDVQQDExJDSUxvZ29uIEJhc2ljIENBIDEwHhcNMTAxMTE4MjIyNTA2WhcNMTAxMTE5MTAzMDA2 WjBvMRMwEQYKCZImiZPyLGQBGRMDb3JnMRcwFQYKCZImiZPyLGQBGRMHY2lsb2dvbjELMAkGA1UE BhMCVVMxFzAVBgNVBAoTDlByb3RlY3ROZXR3b3JrMRkwFwYDVQQDExBSb2dlciBVbndpbiBBMjU0 MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6QhsWxhUXbIxg+1ZyEc7d+hIGvchVmtb g0kKLmivgoVsA4U7swNDRH6svW242THta0oTf6crkRx7kOKg6jma2lcAC1sjOSddqX7/92ChoUPq 7LWt2T6GVVA10ex5WAeB/o7br/Z4U8/75uCBis+ru7xEDl09PToK20mrkcz9M4HqIv1eSoPkrs3b 2lUtQc6cjuHRDU4NknXaVMXTBHKPM40UxEDHJueFyCiZJFg3lvQuSsAl4JL5Z8pC02T8/bODBuf4 dszsqn2SC8YDw1xrujvW2Bd7Q7BwMQ/gO+dZKM1mLJFpfEsR9WrjMeg6vkD2TMWLMr0/WIkGC8u+ 6M6SMQIDAQABo4HdMIHaMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgSwMBMGA1UdJQQMMAoG CCsGAQUFBwMCMBgGA1UdIAQRMA8wDQYLKwYBBAGCkTYBAgEwagYDVR0fBGMwYTAuoCygKoYoaHR0 cDovL2NybC5jaWxvZ29uLm9yZy9jaWxvZ29uLWJhc2ljLmNybDAvoC2gK4YpaHR0cDovL2NybC5k b2Vncmlkcy5vcmcvY2lsb2dvbi1iYXNpYy5jcmwwHwYDVR0RBBgwFoEUaXRzYWdyZWVuMUB5YWhv by5jb20wDQYJKoZIhvcNAQEFBQADggEBAEYHQPMY9Grs19MHxUzMwXp1GzCKhGpgyVKJKW86PJlr HGruoWvx+DLNX75Oj5FC4t8bOUQVQusZGeGSEGegzzfIeOI/jWP1UtIjzvTFDq3tQMNvsgROSCx5 CkpK4nS0kbwLux+zI7BWON97UpMIzEeE05pd7SmNAETuWRsHMP+x6i7hoUp/uad4DwbzNUGIotdK f8b270icOVgkOKRdLP/Q4r/x8skKSCRz1ZsRdR+7+B/EgksAJj7Ut3yiWoUekEMxCaTdAHPTMD/g Mh9xL90hfMJyoGemjJswG5g3fAdTP/Lv0I6/nWeH/cLjwwpQgIEjEAVXl7KHuzX5vPD/wqQ= -----END CERTIFICATE-----""" id, valid_until, registered = self.identity_management_service.signon( certificate, True) self.assertFalse(registered) id2, valid_until2, registered2 = self.identity_management_service.signon( certificate, True) self.assertFalse(registered2) self.assertTrue(id == id2) self.assertTrue(valid_until == valid_until2) user_info_obj = IonObject("UserInfo", {"name": "Foo"}) self.identity_management_service.create_user_info(id, user_info_obj) id3, valid_until3, registered3 = self.identity_management_service.signon( certificate, True) self.assertTrue(registered3) self.assertTrue(id == id3) self.assertTrue(valid_until == valid_until3) @attr('EXT') def test_get_extended_user_identity(self): actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) actor_id = self.identity_management_service.create_actor_identity( actor_identity_obj) user_credentials_obj = IonObject("UserCredentials", {"name": self.subject}) self.identity_management_service.register_user_credentials( actor_id, user_credentials_obj) user_info_obj = IonObject("UserInfo", {"name": "Foo"}) user_info_id = self.identity_management_service.create_user_info( actor_id, user_info_obj) ion_org = self.org_client.find_org() #Build the Service Agreement Proposal to to request a role but never close it sap = IonObject(OT.RequestRoleProposal, consumer=actor_id, provider=ion_org._id, role_name=ORG_MANAGER_ROLE) sap_response = self.org_client.negotiate(sap) #Just grant the role anyway #self.org_client.grant_role(ion_org._id, actor_id, ORG_MANAGER_ROLE) with self.assertRaises(NotFound): self.identity_management_service.get_user_info_extension( 'That rug really tied the room together.') with self.assertRaises(BadRequest): self.identity_management_service.get_user_info_extension() #Check the user without the negotiation role request extended_user = self.identity_management_service.get_user_info_extension( user_info_id, org_id=ion_org._id) self.assertEqual(user_info_obj.type_, extended_user.resource.type_) self.assertEqual(len(extended_user.roles), 1) self.assertEqual(len(extended_user.open_requests), 1) self.assertEqual(extended_user.open_requests[0].org_id, ion_org._id) self.assertEqual(extended_user.open_requests[0].user_id, user_info_id) self.assertEqual(extended_user.open_requests[0].request_type, OT.RequestRoleProposal) self.assertEqual(len(extended_user.closed_requests), 0) self.assertEqual(extended_user.open_requests[0]._id, extended_user.open_requests[0].negotiation_id) neg = self.resource_registry.read( object_id=extended_user.open_requests[0].negotiation_id) sap_response = Negotiation.create_counter_proposal( neg, ProposalStatusEnum.ACCEPTED, ProposalOriginatorEnum.PROVIDER) sap_response2 = self.org_client.negotiate(sap_response) #Now check the user after the negotiation has been accepted and the role granted extended_user = self.identity_management_service.get_user_info_extension( user_info_id, org_id=ion_org._id) self.assertEqual(user_info_obj.type_, extended_user.resource.type_) self.assertEqual(len(extended_user.roles), 2) self.assertEqual(len(extended_user.open_requests), 0) self.assertEqual(len(extended_user.closed_requests), 1) self.assertEqual(extended_user.closed_requests[0].org_id, ion_org._id) self.assertEqual(extended_user.closed_requests[0].user_id, user_info_id) self.assertEqual(extended_user.closed_requests[0].request_type, OT.RequestRoleProposal) self.identity_management_service.delete_user_info(user_info_id) self.org_client.revoke_role(org_id=ion_org._id, actor_id=actor_id, role_name=ORG_MANAGER_ROLE) self.identity_management_service.unregister_user_credentials( actor_id, self.subject) self.identity_management_service.delete_actor_identity(actor_id) def test_account_merge(self): certificate = """-----BEGIN CERTIFICATE----- MIIEMzCCAxugAwIBAgICBQAwDQYJKoZIhvcNAQEFBQAwajETMBEGCgmSJomT8ixkARkWA29yZzEX MBUGCgmSJomT8ixkARkWB2NpbG9nb24xCzAJBgNVBAYTAlVTMRAwDgYDVQQKEwdDSUxvZ29uMRsw GQYDVQQDExJDSUxvZ29uIEJhc2ljIENBIDEwHhcNMTAxMTE4MjIyNTA2WhcNMTAxMTE5MTAzMDA2 WjBvMRMwEQYKCZImiZPyLGQBGRMDb3JnMRcwFQYKCZImiZPyLGQBGRMHY2lsb2dvbjELMAkGA1UE BhMCVVMxFzAVBgNVBAoTDlByb3RlY3ROZXR3b3JrMRkwFwYDVQQDExBSb2dlciBVbndpbiBBMjU0 MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6QhsWxhUXbIxg+1ZyEc7d+hIGvchVmtb g0kKLmivgoVsA4U7swNDRH6svW242THta0oTf6crkRx7kOKg6jma2lcAC1sjOSddqX7/92ChoUPq 7LWt2T6GVVA10ex5WAeB/o7br/Z4U8/75uCBis+ru7xEDl09PToK20mrkcz9M4HqIv1eSoPkrs3b 2lUtQc6cjuHRDU4NknXaVMXTBHKPM40UxEDHJueFyCiZJFg3lvQuSsAl4JL5Z8pC02T8/bODBuf4 dszsqn2SC8YDw1xrujvW2Bd7Q7BwMQ/gO+dZKM1mLJFpfEsR9WrjMeg6vkD2TMWLMr0/WIkGC8u+ 6M6SMQIDAQABo4HdMIHaMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgSwMBMGA1UdJQQMMAoG CCsGAQUFBwMCMBgGA1UdIAQRMA8wDQYLKwYBBAGCkTYBAgEwagYDVR0fBGMwYTAuoCygKoYoaHR0 cDovL2NybC5jaWxvZ29uLm9yZy9jaWxvZ29uLWJhc2ljLmNybDAvoC2gK4YpaHR0cDovL2NybC5k b2Vncmlkcy5vcmcvY2lsb2dvbi1iYXNpYy5jcmwwHwYDVR0RBBgwFoEUaXRzYWdyZWVuMUB5YWhv by5jb20wDQYJKoZIhvcNAQEFBQADggEBAEYHQPMY9Grs19MHxUzMwXp1GzCKhGpgyVKJKW86PJlr HGruoWvx+DLNX75Oj5FC4t8bOUQVQusZGeGSEGegzzfIeOI/jWP1UtIjzvTFDq3tQMNvsgROSCx5 CkpK4nS0kbwLux+zI7BWON97UpMIzEeE05pd7SmNAETuWRsHMP+x6i7hoUp/uad4DwbzNUGIotdK f8b270icOVgkOKRdLP/Q4r/x8skKSCRz1ZsRdR+7+B/EgksAJj7Ut3yiWoUekEMxCaTdAHPTMD/g Mh9xL90hfMJyoGemjJswG5g3fAdTP/Lv0I6/nWeH/cLjwwpQgIEjEAVXl7KHuzX5vPD/wqQ= -----END CERTIFICATE-----""" subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254" certificate_2 = """-----BEGIN CERTIFICATE----- MIIEMzCCAxugAwIBAgIDAJ/lMA0GCSqGSIb3DQEBCwUAMGsxEzARBgoJkiaJk/IsZAEZFgNvcmcx FzAVBgoJkiaJk/IsZAEZFgdjaWxvZ29uMQswCQYDVQQGEwJVUzEQMA4GA1UEChMHQ0lMb2dvbjEc MBoGA1UEAxMTQ0lMb2dvbiBPcGVuSUQgQ0EgMTAeFw0xMjEwMTcwMDE2NDlaFw0xMjEwMTcxMjIx NDlaMGkxEzARBgoJkiaJk/IsZAEZEwNvcmcxFzAVBgoJkiaJk/IsZAEZEwdjaWxvZ29uMQswCQYD VQQGEwJVUzEPMA0GA1UEChMGR29vZ2xlMRswGQYDVQQDExJPd2VuIE93bmVycmVwIEE4OTMwggEi MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDYLdpgg88sntivH+af4oamlp7blsUQcCQ5Yc/b VDP/dwEKfxTcW36tMV3asLO7GcL7z4FESG761LAe86siT9rcwg2ttLkRjI9KeA3sFjC28N8XjKZ1 estCqG3odqw2pjo3VEFaU57219vIYMJhjmHKEgSnlMQeChMYun/sYIO5uNFba9BfiB6/PRS+bgee cXRsIAm1vkB89AHdEjqdvH0uSN+jGjF6aAPXsESh70DUAHzs14lbFAomig7AZafT+weh0G5pnayC lutVnhb9SyS3s1+A6kx8z9mkDUwY/NKXisuDeXa+WbRVq51D+Lc7ffOI+Ph+ynyfFGMcCBzbMADX AgMBAAGjgeEwgd4wDAYDVR0TAQH/BAIwADAOBgNVHQ8BAf8EBAMCBLAwEwYDVR0lBAwwCgYIKwYB BQUHAwIwGAYDVR0gBBEwDzANBgsrBgEEAYKRNgEDAzBsBgNVHR8EZTBjMC+gLaArhilodHRwOi8v Y3JsLmNpbG9nb24ub3JnL2NpbG9nb24tb3BlbmlkLmNybDAwoC6gLIYqaHR0cDovL2NybC5kb2Vn cmlkcy5vcmcvY2lsb2dvbi1vcGVuaWQuY3JsMCEGA1UdEQQaMBiBFm93ZW5vd25lcnJlcEBnbWFp bC5jb20wDQYJKoZIhvcNAQELBQADggEBAHWd6ZOjSmJyOUyyLgZAPJpkSuk7DT5mFRhszJhfTGnu gANHRIJZMs5e/LCMypE+ftxb8mnhAE+kURA2DmeucazHUDP5oYofU+8KMYqcNKnPpLnuiw+bCJPa 3BDxrYoi+vVislHb0U+QDjVYtUtQ2b1/Xhv8ShH89O9i65bbOq+sqez6z2AD9RWOEwRwpQLc9D65 9lkrsKGmJtuG8q3NTpZ1DSuaLOtn0QqttdmCg3pu5edRtgdpGadaSGR4s222JasV439bSTL8Z0Ug HtjSclGqi8IBmvRkTZI61zTVbGdOKMP90LV1p8noJVLRkZpWRjLxI5xy9El8daAWMdjfrSc= -----END CERTIFICATE-----""" subject_2 = "/DC=org/DC=cilogon/C=US/O=Google/CN=Owen Ownerrep A893" # Try to merge with nonexistent email account with self.assertRaises(NotFound): self.identity_management_service.initiate_account_merge( "*****@*****.**") with self.assertRaises(BadRequest): self.identity_management_service.initiate_account_merge() # Create two users id, valid_until, registered = self.identity_management_service.signon( certificate, True) self.assertFalse(registered) id_2, valid_until_2, registered_2 = self.identity_management_service.signon( certificate_2, True) self.assertFalse(registered_2) # Validate the two accounts are different self.assertNotEqual( id, id_2, "The two accounts should have two different user id") # Create UserInfo contact_info_obj = IonObject("ContactInformation", {"email": "*****@*****.**"}) user_info_obj = IonObject("UserInfo", { "name": "Dude", "contact": contact_info_obj }) user_info_id = self.identity_management_service.create_user_info( id, user_info_obj) contact_info_obj_2 = IonObject("ContactInformation", {"email": "*****@*****.**"}) user_info_obj_2 = IonObject("UserInfo", { "name": "theDude", "contact": contact_info_obj_2 }) user_info_id_2 = self.identity_management_service.create_user_info( id_2, user_info_obj_2) # Make sure the two users are registered id, valid_until, registered = self.identity_management_service.signon( certificate, True) self.assertTrue(registered) id_2, valid_until_2, registered_2 = self.identity_management_service.signon( certificate_2, True) self.assertTrue(registered_2) token = self.identity_management_service.initiate_account_merge( "*****@*****.**", headers={'ion-actor-id': id}) # Try merging accounts with invalid token string with self.assertRaises(NotFound): self.identity_management_service.complete_account_merge( token_string="0xBeeF", headers={'ion-actor-id': id}) with self.assertRaises(BadRequest): self.identity_management_service.complete_account_merge() # Try merging accounts with a different user # Since this user hasn't initiated account merge, the token doesn't exist in his/her UserInfo with self.assertRaises(NotFound): self.identity_management_service.complete_account_merge( token, headers={'ion-actor-id': id_2}) self.identity_management_service.complete_account_merge( token, headers={'ion-actor-id': id}) # Try merging the account again with self.assertRaises(BadRequest): self.identity_management_service.complete_account_merge( token, headers={'ion-actor-id': id}) # Signon again and verify the two accounts have been merged id, valid_until, registered = self.identity_management_service.signon( certificate, True) self.assertTrue(registered) id_2, valid_until_2, registered_2 = self.identity_management_service.signon( certificate_2, True) self.assertTrue(registered_2) # Validate the two accounts are the same self.assertEqual(id, id_2, "The two accounts should have the same id") # Try to merge to your own account with self.assertRaises(BadRequest): token = self.identity_management_service.initiate_account_merge( "*****@*****.**", headers={'ion-actor-id': id}) # Done testing. Delete user self.identity_management_service.delete_user_info(user_info_id) self.identity_management_service.unregister_user_credentials( id, subject) self.identity_management_service.delete_actor_identity(id)
class TestIdentityManagementServiceInt(IonIntegrationTestCase): def setUp(self): self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254" # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2coi.yml') self.identity_management_service = IdentityManagementServiceClient(node=self.container.node) def test_actor_identity(self): actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) user_id = self.identity_management_service.create_actor_identity(actor_identity_obj) actor_identity = self.identity_management_service.read_actor_identity(user_id) actor_identity.name = 'Updated subject' self.identity_management_service.update_actor_identity(actor_identity) self.identity_management_service.delete_actor_identity(user_id) with self.assertRaises(NotFound) as cm: self.identity_management_service.read_actor_identity(user_id) self.assertTrue("does not exist" in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.delete_actor_identity(user_id) self.assertTrue("does not exist" in cm.exception.message) def test_user_credentials(self): actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) user_id = self.identity_management_service.create_actor_identity(actor_identity_obj) user_credentials_obj = IonObject("UserCredentials", {"name": self.subject}) self.identity_management_service.register_user_credentials(user_id, user_credentials_obj) with self.assertRaises(NotFound) as cm: self.identity_management_service.unregister_user_credentials("bad", self.subject) self.assertTrue("does not exist" in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.unregister_user_credentials(user_id, "bad") self.assertTrue("does not exist" in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.unregister_user_credentials('bad', 'bad') self.assertTrue("does not exist" in cm.exception.message) self.identity_management_service.unregister_user_credentials(user_id, self.subject) self.identity_management_service.delete_actor_identity(user_id) def test_user_info(self): actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) user_id = self.identity_management_service.create_actor_identity(actor_identity_obj) user_credentials_obj = IonObject("UserCredentials", {"name": self.subject}) self.identity_management_service.register_user_credentials(user_id, user_credentials_obj) user_info_obj = IonObject("UserInfo", {"name": "Foo"}) user_info = self.identity_management_service.create_user_info(user_id, user_info_obj) with self.assertRaises(Conflict) as cm: self.identity_management_service.create_user_info(user_id, user_info_obj) self.assertTrue("UserInfo already exists for user id" in cm.exception.message) user_info_obj = self.identity_management_service.find_user_info_by_id(user_id) user_info_obj = self.identity_management_service.find_user_info_by_name("Foo") user_info_obj = self.identity_management_service.find_user_info_by_subject(self.subject) user_info_obj = self.identity_management_service.read_user_info(user_info) user_info_obj.name = 'Jane Doe' self.identity_management_service.update_user_info(user_info_obj) self.identity_management_service.delete_user_info(user_info) with self.assertRaises(NotFound) as cm: self.identity_management_service.read_user_info(user_info) self.assertTrue('does not exist' in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.delete_user_info(user_info) self.assertTrue('does not exist' in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.find_user_info_by_name("John Doe") self.assertEqual(cm.exception.message, 'UserInfo with name John Doe does not exist') with self.assertRaises(NotFound) as cm: self.identity_management_service.find_user_info_by_subject("Bogus subject") self.assertEqual(cm.exception.message, "UserCredentials with subject Bogus subject does not exist") self.identity_management_service.unregister_user_credentials(user_id, self.subject) self.identity_management_service.delete_actor_identity(user_id) def test_signon(self): certificate = """-----BEGIN CERTIFICATE----- MIIEMzCCAxugAwIBAgICBQAwDQYJKoZIhvcNAQEFBQAwajETMBEGCgmSJomT8ixkARkWA29yZzEX MBUGCgmSJomT8ixkARkWB2NpbG9nb24xCzAJBgNVBAYTAlVTMRAwDgYDVQQKEwdDSUxvZ29uMRsw GQYDVQQDExJDSUxvZ29uIEJhc2ljIENBIDEwHhcNMTAxMTE4MjIyNTA2WhcNMTAxMTE5MTAzMDA2 WjBvMRMwEQYKCZImiZPyLGQBGRMDb3JnMRcwFQYKCZImiZPyLGQBGRMHY2lsb2dvbjELMAkGA1UE BhMCVVMxFzAVBgNVBAoTDlByb3RlY3ROZXR3b3JrMRkwFwYDVQQDExBSb2dlciBVbndpbiBBMjU0 MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6QhsWxhUXbIxg+1ZyEc7d+hIGvchVmtb g0kKLmivgoVsA4U7swNDRH6svW242THta0oTf6crkRx7kOKg6jma2lcAC1sjOSddqX7/92ChoUPq 7LWt2T6GVVA10ex5WAeB/o7br/Z4U8/75uCBis+ru7xEDl09PToK20mrkcz9M4HqIv1eSoPkrs3b 2lUtQc6cjuHRDU4NknXaVMXTBHKPM40UxEDHJueFyCiZJFg3lvQuSsAl4JL5Z8pC02T8/bODBuf4 dszsqn2SC8YDw1xrujvW2Bd7Q7BwMQ/gO+dZKM1mLJFpfEsR9WrjMeg6vkD2TMWLMr0/WIkGC8u+ 6M6SMQIDAQABo4HdMIHaMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgSwMBMGA1UdJQQMMAoG CCsGAQUFBwMCMBgGA1UdIAQRMA8wDQYLKwYBBAGCkTYBAgEwagYDVR0fBGMwYTAuoCygKoYoaHR0 cDovL2NybC5jaWxvZ29uLm9yZy9jaWxvZ29uLWJhc2ljLmNybDAvoC2gK4YpaHR0cDovL2NybC5k b2Vncmlkcy5vcmcvY2lsb2dvbi1iYXNpYy5jcmwwHwYDVR0RBBgwFoEUaXRzYWdyZWVuMUB5YWhv by5jb20wDQYJKoZIhvcNAQEFBQADggEBAEYHQPMY9Grs19MHxUzMwXp1GzCKhGpgyVKJKW86PJlr HGruoWvx+DLNX75Oj5FC4t8bOUQVQusZGeGSEGegzzfIeOI/jWP1UtIjzvTFDq3tQMNvsgROSCx5 CkpK4nS0kbwLux+zI7BWON97UpMIzEeE05pd7SmNAETuWRsHMP+x6i7hoUp/uad4DwbzNUGIotdK f8b270icOVgkOKRdLP/Q4r/x8skKSCRz1ZsRdR+7+B/EgksAJj7Ut3yiWoUekEMxCaTdAHPTMD/g Mh9xL90hfMJyoGemjJswG5g3fAdTP/Lv0I6/nWeH/cLjwwpQgIEjEAVXl7KHuzX5vPD/wqQ= -----END CERTIFICATE-----""" id, valid_until, registered = self.identity_management_service.signon(certificate, True) self.assertFalse(registered) id2, valid_until2, registered2 = self.identity_management_service.signon(certificate, True) self.assertFalse(registered2) self.assertTrue(id == id2) self.assertTrue(valid_until == valid_until2) user_info_obj = IonObject("UserInfo", {"name": "Foo"}) self.identity_management_service.create_user_info(id, user_info_obj) id3, valid_until3, registered3 = self.identity_management_service.signon(certificate, True) self.assertTrue(registered3) self.assertTrue(id == id3) self.assertTrue(valid_until == valid_until3)
class TestIdentityManagementServiceInt(IonIntegrationTestCase): def setUp(self): self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254" # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.resource_registry = ResourceRegistryServiceClient(node=self.container.node) self.identity_management_service = IdentityManagementServiceClient(node=self.container.node) self.org_client = OrgManagementServiceClient(node=self.container.node) def test_actor_identity(self): actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) user_id = self.identity_management_service.create_actor_identity(actor_identity_obj) actor_identity = self.identity_management_service.read_actor_identity(user_id) actor_identity.name = 'Updated subject' self.identity_management_service.update_actor_identity(actor_identity) ai = self.identity_management_service.find_actor_identity_by_name(actor_identity.name) self._baseAssertEqual(ai.name, actor_identity.name) with self.assertRaises(NotFound): ai = self.identity_management_service.find_actor_identity_by_name("Yeah, well, you know, that's just, like, your opinion, man.") self._baseAssertEqual(ai.name, actor_identity.name) self.identity_management_service.delete_actor_identity(user_id) with self.assertRaises(NotFound) as cm: self.identity_management_service.read_actor_identity(user_id) self.assertTrue("does not exist" in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.delete_actor_identity(user_id) self.assertTrue("does not exist" in cm.exception.message) def test_user_credentials(self): actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) user_id = self.identity_management_service.create_actor_identity(actor_identity_obj) user_credentials_obj = IonObject("UserCredentials", {"name": self.subject}) self.identity_management_service.register_user_credentials(user_id, user_credentials_obj) with self.assertRaises(NotFound) as cm: self.identity_management_service.unregister_user_credentials("bad", self.subject) self.assertTrue("does not exist" in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.unregister_user_credentials(user_id, "bad") self.assertTrue("does not exist" in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.unregister_user_credentials('bad', 'bad') self.assertTrue("does not exist" in cm.exception.message) self.identity_management_service.unregister_user_credentials(user_id, self.subject) self.identity_management_service.delete_actor_identity(user_id) def test_user_info(self): actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) user_id = self.identity_management_service.create_actor_identity(actor_identity_obj) user_credentials_obj = IonObject("UserCredentials", {"name": self.subject}) self.identity_management_service.register_user_credentials(user_id, user_credentials_obj) user_info_obj = IonObject("UserInfo", {"name": "Foo"}) user_info = self.identity_management_service.create_user_info(user_id, user_info_obj) with self.assertRaises(Conflict) as cm: self.identity_management_service.create_user_info(user_id, user_info_obj) self.assertTrue("UserInfo already exists for user id" in cm.exception.message) user_info_obj = self.identity_management_service.find_user_info_by_id(user_id) user_info_obj = self.identity_management_service.find_user_info_by_name("Foo") user_info_obj = self.identity_management_service.find_user_info_by_subject(self.subject) user_info_obj = self.identity_management_service.read_user_info(user_info) user_info_obj.name = 'Jane Doe' self.identity_management_service.update_user_info(user_info_obj) self.identity_management_service.delete_user_info(user_info) with self.assertRaises(NotFound) as cm: self.identity_management_service.read_user_info(user_info) self.assertTrue('does not exist' in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.delete_user_info(user_info) self.assertTrue('does not exist' in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.find_user_info_by_name("John Doe") self.assertEqual(cm.exception.message, 'UserInfo with name John Doe does not exist') with self.assertRaises(NotFound) as cm: self.identity_management_service.find_user_info_by_subject("Bogus subject") self.assertEqual(cm.exception.message, "UserCredentials with subject Bogus subject does not exist") self.identity_management_service.unregister_user_credentials(user_id, self.subject) self.identity_management_service.delete_actor_identity(user_id) def test_signon(self): certificate = """-----BEGIN CERTIFICATE----- MIIEMzCCAxugAwIBAgICBQAwDQYJKoZIhvcNAQEFBQAwajETMBEGCgmSJomT8ixkARkWA29yZzEX MBUGCgmSJomT8ixkARkWB2NpbG9nb24xCzAJBgNVBAYTAlVTMRAwDgYDVQQKEwdDSUxvZ29uMRsw GQYDVQQDExJDSUxvZ29uIEJhc2ljIENBIDEwHhcNMTAxMTE4MjIyNTA2WhcNMTAxMTE5MTAzMDA2 WjBvMRMwEQYKCZImiZPyLGQBGRMDb3JnMRcwFQYKCZImiZPyLGQBGRMHY2lsb2dvbjELMAkGA1UE BhMCVVMxFzAVBgNVBAoTDlByb3RlY3ROZXR3b3JrMRkwFwYDVQQDExBSb2dlciBVbndpbiBBMjU0 MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6QhsWxhUXbIxg+1ZyEc7d+hIGvchVmtb g0kKLmivgoVsA4U7swNDRH6svW242THta0oTf6crkRx7kOKg6jma2lcAC1sjOSddqX7/92ChoUPq 7LWt2T6GVVA10ex5WAeB/o7br/Z4U8/75uCBis+ru7xEDl09PToK20mrkcz9M4HqIv1eSoPkrs3b 2lUtQc6cjuHRDU4NknXaVMXTBHKPM40UxEDHJueFyCiZJFg3lvQuSsAl4JL5Z8pC02T8/bODBuf4 dszsqn2SC8YDw1xrujvW2Bd7Q7BwMQ/gO+dZKM1mLJFpfEsR9WrjMeg6vkD2TMWLMr0/WIkGC8u+ 6M6SMQIDAQABo4HdMIHaMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgSwMBMGA1UdJQQMMAoG CCsGAQUFBwMCMBgGA1UdIAQRMA8wDQYLKwYBBAGCkTYBAgEwagYDVR0fBGMwYTAuoCygKoYoaHR0 cDovL2NybC5jaWxvZ29uLm9yZy9jaWxvZ29uLWJhc2ljLmNybDAvoC2gK4YpaHR0cDovL2NybC5k b2Vncmlkcy5vcmcvY2lsb2dvbi1iYXNpYy5jcmwwHwYDVR0RBBgwFoEUaXRzYWdyZWVuMUB5YWhv by5jb20wDQYJKoZIhvcNAQEFBQADggEBAEYHQPMY9Grs19MHxUzMwXp1GzCKhGpgyVKJKW86PJlr HGruoWvx+DLNX75Oj5FC4t8bOUQVQusZGeGSEGegzzfIeOI/jWP1UtIjzvTFDq3tQMNvsgROSCx5 CkpK4nS0kbwLux+zI7BWON97UpMIzEeE05pd7SmNAETuWRsHMP+x6i7hoUp/uad4DwbzNUGIotdK f8b270icOVgkOKRdLP/Q4r/x8skKSCRz1ZsRdR+7+B/EgksAJj7Ut3yiWoUekEMxCaTdAHPTMD/g Mh9xL90hfMJyoGemjJswG5g3fAdTP/Lv0I6/nWeH/cLjwwpQgIEjEAVXl7KHuzX5vPD/wqQ= -----END CERTIFICATE-----""" id, valid_until, registered = self.identity_management_service.signon(certificate, True) self.assertFalse(registered) id2, valid_until2, registered2 = self.identity_management_service.signon(certificate, True) self.assertFalse(registered2) self.assertTrue(id == id2) self.assertTrue(valid_until == valid_until2) user_info_obj = IonObject("UserInfo", {"name": "Foo"}) self.identity_management_service.create_user_info(id, user_info_obj) id3, valid_until3, registered3 = self.identity_management_service.signon(certificate, True) self.assertTrue(registered3) self.assertTrue(id == id3) self.assertTrue(valid_until == valid_until3) @attr('EXT') def test_get_extended_user_identity(self): actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) actor_id = self.identity_management_service.create_actor_identity(actor_identity_obj) user_credentials_obj = IonObject("UserCredentials", {"name": self.subject}) self.identity_management_service.register_user_credentials(actor_id, user_credentials_obj) user_info_obj = IonObject("UserInfo", {"name": "Foo"}) user_info_id = self.identity_management_service.create_user_info(actor_id, user_info_obj) ion_org = self.org_client.find_org() #Build the Service Agreement Proposal to to request a role but never close it sap = IonObject(OT.RequestRoleProposal,consumer=actor_id, provider=ion_org._id, role_name=ORG_MANAGER_ROLE ) sap_response = self.org_client.negotiate(sap) #Just grant the role anyway #self.org_client.grant_role(ion_org._id, actor_id, ORG_MANAGER_ROLE) with self.assertRaises(NotFound): self.identity_management_service.get_user_info_extension('That rug really tied the room together.') with self.assertRaises(BadRequest): self.identity_management_service.get_user_info_extension() #Check the user without the negotiation role request extended_user = self.identity_management_service.get_user_info_extension(user_info_id, org_id=ion_org._id) self.assertEqual(user_info_obj.type_,extended_user.resource.type_) self.assertEqual(len(extended_user.roles),1) self.assertEqual(len(extended_user.open_requests),1) self.assertEqual(extended_user.open_requests[0].org_id, ion_org._id) self.assertEqual(extended_user.open_requests[0].user_id, user_info_id) self.assertEqual(extended_user.open_requests[0].request_type, OT.RequestRoleProposal) self.assertEqual(len(extended_user.closed_requests),0) self.assertEqual(extended_user.open_requests[0]._id, extended_user.open_requests[0].negotiation_id) neg = self.resource_registry.read(object_id=extended_user.open_requests[0].negotiation_id) sap_response = Negotiation.create_counter_proposal(neg, ProposalStatusEnum.ACCEPTED, ProposalOriginatorEnum.PROVIDER) sap_response2 = self.org_client.negotiate(sap_response) #Now check the user after the negotiation has been accepted and the role granted extended_user = self.identity_management_service.get_user_info_extension(user_info_id, org_id=ion_org._id) self.assertEqual(user_info_obj.type_,extended_user.resource.type_) self.assertEqual(len(extended_user.roles),2) self.assertEqual(len(extended_user.open_requests),0) self.assertEqual(len(extended_user.closed_requests),1) self.assertEqual(extended_user.closed_requests[0].org_id, ion_org._id) self.assertEqual(extended_user.closed_requests[0].user_id, user_info_id) self.assertEqual(extended_user.closed_requests[0].request_type, OT.RequestRoleProposal) self.identity_management_service.delete_user_info(user_info_id) self.org_client.revoke_role(org_id=ion_org._id, actor_id=actor_id, role_name=ORG_MANAGER_ROLE) self.identity_management_service.unregister_user_credentials(actor_id, self.subject) self.identity_management_service.delete_actor_identity(actor_id) def test_account_merge(self): certificate = """-----BEGIN CERTIFICATE----- MIIEMzCCAxugAwIBAgICBQAwDQYJKoZIhvcNAQEFBQAwajETMBEGCgmSJomT8ixkARkWA29yZzEX MBUGCgmSJomT8ixkARkWB2NpbG9nb24xCzAJBgNVBAYTAlVTMRAwDgYDVQQKEwdDSUxvZ29uMRsw GQYDVQQDExJDSUxvZ29uIEJhc2ljIENBIDEwHhcNMTAxMTE4MjIyNTA2WhcNMTAxMTE5MTAzMDA2 WjBvMRMwEQYKCZImiZPyLGQBGRMDb3JnMRcwFQYKCZImiZPyLGQBGRMHY2lsb2dvbjELMAkGA1UE BhMCVVMxFzAVBgNVBAoTDlByb3RlY3ROZXR3b3JrMRkwFwYDVQQDExBSb2dlciBVbndpbiBBMjU0 MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6QhsWxhUXbIxg+1ZyEc7d+hIGvchVmtb g0kKLmivgoVsA4U7swNDRH6svW242THta0oTf6crkRx7kOKg6jma2lcAC1sjOSddqX7/92ChoUPq 7LWt2T6GVVA10ex5WAeB/o7br/Z4U8/75uCBis+ru7xEDl09PToK20mrkcz9M4HqIv1eSoPkrs3b 2lUtQc6cjuHRDU4NknXaVMXTBHKPM40UxEDHJueFyCiZJFg3lvQuSsAl4JL5Z8pC02T8/bODBuf4 dszsqn2SC8YDw1xrujvW2Bd7Q7BwMQ/gO+dZKM1mLJFpfEsR9WrjMeg6vkD2TMWLMr0/WIkGC8u+ 6M6SMQIDAQABo4HdMIHaMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgSwMBMGA1UdJQQMMAoG CCsGAQUFBwMCMBgGA1UdIAQRMA8wDQYLKwYBBAGCkTYBAgEwagYDVR0fBGMwYTAuoCygKoYoaHR0 cDovL2NybC5jaWxvZ29uLm9yZy9jaWxvZ29uLWJhc2ljLmNybDAvoC2gK4YpaHR0cDovL2NybC5k b2Vncmlkcy5vcmcvY2lsb2dvbi1iYXNpYy5jcmwwHwYDVR0RBBgwFoEUaXRzYWdyZWVuMUB5YWhv by5jb20wDQYJKoZIhvcNAQEFBQADggEBAEYHQPMY9Grs19MHxUzMwXp1GzCKhGpgyVKJKW86PJlr HGruoWvx+DLNX75Oj5FC4t8bOUQVQusZGeGSEGegzzfIeOI/jWP1UtIjzvTFDq3tQMNvsgROSCx5 CkpK4nS0kbwLux+zI7BWON97UpMIzEeE05pd7SmNAETuWRsHMP+x6i7hoUp/uad4DwbzNUGIotdK f8b270icOVgkOKRdLP/Q4r/x8skKSCRz1ZsRdR+7+B/EgksAJj7Ut3yiWoUekEMxCaTdAHPTMD/g Mh9xL90hfMJyoGemjJswG5g3fAdTP/Lv0I6/nWeH/cLjwwpQgIEjEAVXl7KHuzX5vPD/wqQ= -----END CERTIFICATE-----""" subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254" certificate_2 = """-----BEGIN CERTIFICATE----- MIIEMzCCAxugAwIBAgIDAJ/lMA0GCSqGSIb3DQEBCwUAMGsxEzARBgoJkiaJk/IsZAEZFgNvcmcx FzAVBgoJkiaJk/IsZAEZFgdjaWxvZ29uMQswCQYDVQQGEwJVUzEQMA4GA1UEChMHQ0lMb2dvbjEc MBoGA1UEAxMTQ0lMb2dvbiBPcGVuSUQgQ0EgMTAeFw0xMjEwMTcwMDE2NDlaFw0xMjEwMTcxMjIx NDlaMGkxEzARBgoJkiaJk/IsZAEZEwNvcmcxFzAVBgoJkiaJk/IsZAEZEwdjaWxvZ29uMQswCQYD VQQGEwJVUzEPMA0GA1UEChMGR29vZ2xlMRswGQYDVQQDExJPd2VuIE93bmVycmVwIEE4OTMwggEi MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDYLdpgg88sntivH+af4oamlp7blsUQcCQ5Yc/b VDP/dwEKfxTcW36tMV3asLO7GcL7z4FESG761LAe86siT9rcwg2ttLkRjI9KeA3sFjC28N8XjKZ1 estCqG3odqw2pjo3VEFaU57219vIYMJhjmHKEgSnlMQeChMYun/sYIO5uNFba9BfiB6/PRS+bgee cXRsIAm1vkB89AHdEjqdvH0uSN+jGjF6aAPXsESh70DUAHzs14lbFAomig7AZafT+weh0G5pnayC lutVnhb9SyS3s1+A6kx8z9mkDUwY/NKXisuDeXa+WbRVq51D+Lc7ffOI+Ph+ynyfFGMcCBzbMADX AgMBAAGjgeEwgd4wDAYDVR0TAQH/BAIwADAOBgNVHQ8BAf8EBAMCBLAwEwYDVR0lBAwwCgYIKwYB BQUHAwIwGAYDVR0gBBEwDzANBgsrBgEEAYKRNgEDAzBsBgNVHR8EZTBjMC+gLaArhilodHRwOi8v Y3JsLmNpbG9nb24ub3JnL2NpbG9nb24tb3BlbmlkLmNybDAwoC6gLIYqaHR0cDovL2NybC5kb2Vn cmlkcy5vcmcvY2lsb2dvbi1vcGVuaWQuY3JsMCEGA1UdEQQaMBiBFm93ZW5vd25lcnJlcEBnbWFp bC5jb20wDQYJKoZIhvcNAQELBQADggEBAHWd6ZOjSmJyOUyyLgZAPJpkSuk7DT5mFRhszJhfTGnu gANHRIJZMs5e/LCMypE+ftxb8mnhAE+kURA2DmeucazHUDP5oYofU+8KMYqcNKnPpLnuiw+bCJPa 3BDxrYoi+vVislHb0U+QDjVYtUtQ2b1/Xhv8ShH89O9i65bbOq+sqez6z2AD9RWOEwRwpQLc9D65 9lkrsKGmJtuG8q3NTpZ1DSuaLOtn0QqttdmCg3pu5edRtgdpGadaSGR4s222JasV439bSTL8Z0Ug HtjSclGqi8IBmvRkTZI61zTVbGdOKMP90LV1p8noJVLRkZpWRjLxI5xy9El8daAWMdjfrSc= -----END CERTIFICATE-----""" subject_2 = "/DC=org/DC=cilogon/C=US/O=Google/CN=Owen Ownerrep A893" # Try to merge with nonexistent email account with self.assertRaises(NotFound): self.identity_management_service.initiate_account_merge("*****@*****.**") with self.assertRaises(BadRequest): self.identity_management_service.initiate_account_merge() # Create two users id, valid_until, registered = self.identity_management_service.signon(certificate, True) self.assertFalse(registered) id_2, valid_until_2, registered_2 = self.identity_management_service.signon(certificate_2, True) self.assertFalse(registered_2) # Validate the two accounts are different self.assertNotEqual(id, id_2, "The two accounts should have two different user id") # Create UserInfo contact_info_obj = IonObject("ContactInformation",{"email": "*****@*****.**"}) user_info_obj = IonObject("UserInfo", {"name": "Dude", "contact": contact_info_obj}) user_info_id = self.identity_management_service.create_user_info(id, user_info_obj) contact_info_obj_2 = IonObject("ContactInformation",{"email": "*****@*****.**"}) user_info_obj_2 = IonObject("UserInfo", {"name": "theDude", "contact": contact_info_obj_2}) user_info_id_2 = self.identity_management_service.create_user_info(id_2, user_info_obj_2) # Make sure the two users are registered id, valid_until, registered = self.identity_management_service.signon(certificate, True) self.assertTrue(registered) id_2, valid_until_2, registered_2 = self.identity_management_service.signon(certificate_2, True) self.assertTrue(registered_2) token = self.identity_management_service.initiate_account_merge("*****@*****.**", headers={'ion-actor-id':id}) # Try merging accounts with invalid token string with self.assertRaises(NotFound): self.identity_management_service.complete_account_merge(token_string="0xBeeF", headers={'ion-actor-id':id}) with self.assertRaises(BadRequest): self.identity_management_service.complete_account_merge() # Try merging accounts with a different user # Since this user hasn't initiated account merge, the token doesn't exist in his/her UserInfo with self.assertRaises(NotFound): self.identity_management_service.complete_account_merge(token, headers={'ion-actor-id':id_2}) self.identity_management_service.complete_account_merge(token, headers={'ion-actor-id':id}) # Try merging the account again with self.assertRaises(BadRequest): self.identity_management_service.complete_account_merge(token, headers={'ion-actor-id':id}) # Signon again and verify the two accounts have been merged id, valid_until, registered = self.identity_management_service.signon(certificate, True) self.assertTrue(registered) id_2, valid_until_2, registered_2 = self.identity_management_service.signon(certificate_2, True) self.assertTrue(registered_2) # Validate the two accounts are the same self.assertEqual(id, id_2, "The two accounts should have the same id") # Try to merge to your own account with self.assertRaises(BadRequest): token = self.identity_management_service.initiate_account_merge("*****@*****.**", headers={'ion-actor-id':id}) # Done testing. Delete user self.identity_management_service.delete_user_info(user_info_id) self.identity_management_service.unregister_user_credentials(id, subject) self.identity_management_service.delete_actor_identity(id)
def test_non_anonymous_resource_registry_operations_with_token(self): rr = self.container.resource_registry id_client = IdentityManagementServiceClient() create_request = { "serviceRequest": { "serviceName": "resource_registry", "serviceOp": "create", "params": {"object": {"name": "Instrument1", "type_": "InstrumentDevice"}}, } } # Create without actor response = self.test_app.post( "/ion-service/resource_registry/create", {"payload": simplejson.dumps(create_request)} ) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json["data"]) response_data = response.json["data"][GATEWAY_RESPONSE] self.assertEqual(len(response_data), 2) self.assertGreaterEqual(len(response_data[0]), 20) # This is a resource_id inst_id = str(response_data[0]) inst_obj = rr.read(inst_id) self.assertEquals(inst_obj.type_, RT.InstrumentDevice) self.assertEquals(inst_obj.name, "Instrument1") self.assertEquals(inst_obj.visibility, ResourceVisibilityEnum.PUBLIC) act_objs, assocs = rr.find_objects(inst_id, PRED.hasOwner, RT.ActorIdentity, id_only=False) self.assertEquals(len(act_objs), 0) # Anonymous query shows PUBLIC visibility resource anymore response = self.test_app.get("/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True") self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json["data"]) response_data = response.json["data"][GATEWAY_RESPONSE] self.assertEqual(len(response_data), 2) self.assertEqual(len(response_data[0]), 1) self.assertEqual(len(response_data[1]), 1) self.assertEqual(response_data[0][0], inst_id) inst_obj.visibility = ResourceVisibilityEnum.OWNER rr.update(inst_obj) # Now the anonymous query should not show the resource anymore response = self.test_app.get("/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True") self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json["data"]) response_data = response.json["data"][GATEWAY_RESPONSE] self.assertEqual(len(response_data), 2) self.assertEqual(len(response_data[0]), 0) rr.delete(inst_id) # Create with actor actor_id, valid_until, registered = id_client.signon(USER1_CERTIFICATE, True) create_request = { "serviceRequest": { "serviceName": "resource_registry", "serviceOp": "create", "requester": actor_id, "params": { "object": { "name": "Instrument1", "type_": "InstrumentDevice", "visibility": ResourceVisibilityEnum.OWNER, } }, } } response = self.test_app.post( "/ion-service/resource_registry/create", {"payload": simplejson.dumps(create_request)} ) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json["data"]) response_data = response.json["data"][GATEWAY_RESPONSE] self.assertEqual(len(response_data), 2) self.assertGreaterEqual(len(response_data[0]), 20) # This is a resource_id inst_id = str(response_data[0]) inst_obj = rr.read(inst_id) self.assertEquals(inst_obj.type_, RT.InstrumentDevice) self.assertEquals(inst_obj.name, "Instrument1") self.assertEquals(inst_obj.visibility, ResourceVisibilityEnum.OWNER) act_objs, assocs = rr.find_objects(inst_id, PRED.hasOwner, RT.ActorIdentity, id_only=False) self.assertEquals(len(act_objs), 1) self.assertEquals(act_objs[0]._id, actor_id) # Anonymous query should not show the resource anymore response = self.test_app.get("/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True") self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json["data"]) response_data = response.json["data"][GATEWAY_RESPONSE] self.assertEqual(len(response_data), 2) self.assertEqual(len(response_data[0]), 0) # Authenticated request with owner shows resource response = self.test_app.get( "/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&requester=" + actor_id ) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json["data"]) response_data = response.json["data"][GATEWAY_RESPONSE] self.assertEqual(len(response_data), 2) self.assertEqual(len(response_data[0]), 1) self.assertEqual(len(response_data[1]), 1) self.assertEqual(response_data[0][0], inst_id) token_str = id_client.create_authentication_token(actor_id, validity=2) # Request with authentication token for owner shows resource response = self.test_app.get( "/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&authtoken=" + token_str ) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json["data"]) response_data = response.json["data"][GATEWAY_RESPONSE] self.assertEqual(len(response_data), 2) self.assertEqual(len(response_data[0]), 1) self.assertEqual(len(response_data[1]), 1) self.assertEqual(response_data[0][0], inst_id) gevent.sleep(2.1) # Request with expired authentication token for owner does not show resource (no error though) response = self.test_app.get( "/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&authtoken=" + token_str ) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json["data"]) response_data = response.json["data"][GATEWAY_RESPONSE] self.assertEqual(len(response_data), 2) self.assertEqual(len(response_data[0]), 0) # Request with valid authentication token for different user does not show resource actor_id2, _ = rr.create(IonObject(RT.ActorIdentity, name="Actor2")) token_str2 = id_client.create_authentication_token(actor_id2, validity=2) response = self.test_app.get( "/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&authtoken=" + token_str2 ) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json["data"]) response_data = response.json["data"][GATEWAY_RESPONSE] self.assertEqual(len(response_data), 2) self.assertEqual(len(response_data[0]), 0) token_str3 = id_client.create_authentication_token(actor_id, validity=2) # Request with new authentication token for owner shows resource response = self.test_app.get( "/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&authtoken=" + token_str3 ) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json["data"]) response_data = response.json["data"][GATEWAY_RESPONSE] self.assertEqual(len(response_data), 2) self.assertEqual(len(response_data[0]), 1) self.assertEqual(len(response_data[1]), 1) self.assertEqual(response_data[0][0], inst_id) id_client.invalidate_authentication_token(token_str3) response = self.test_app.get( "/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&authtoken=" + token_str3 ) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json["data"]) response_data = response.json["data"][GATEWAY_RESPONSE] self.assertEqual(len(response_data), 2) self.assertEqual(len(response_data[0]), 0) # Cleanup rr.delete(inst_id) rr.delete(actor_id2) id_client.delete_actor_identity(actor_id)
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) print 'started services' # @unittest.skip('this test just for debugging setup') # def test_just_the_setup(self): # return @attr('EXT') def test_resources_associations_extensions(self): """ create one of each resource and association used by IMS to guard against problems in ion-definitions """ #stuff we control instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance)) instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent)) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance)) platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent)) platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice)) sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel)) #stuff we associate to data_producer_id, _ = self.RR.create(any_old(RT.DataProducer)) org_id, _ = self.RR.create(any_old(RT.Org)) #instrument_agent_instance_id #is only a target #instrument_agent self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id) self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id) self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device_id) instrument_model_id #is only a target platform_agent_instance_id #is only a target #platform_agent self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id) self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) platform_model_id #is only a target #sensor_device self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id) sensor_model_id #is only a target #create a parsed product for this instrument output tdom, sdom = time_series_domain() tdom = tdom.dump() sdom = sdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', processing_level_code='Parsed_Canonical', temporal_domain = tdom, spatial_domain = sdom) pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1) def addInstOwner(inst_id, subject): actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject}) user_id = self.IDS.create_actor_identity(actor_identity_obj) user_info_obj = any_old(RT.UserInfo) user_info_id = self.IDS.create_user_info(user_id, user_info_obj) self.RR.create_association(inst_id, PRED.hasOwner, user_id) #Testing multiple instrument owners addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254") addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256") extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id) self.assertEqual(instrument_device_id, extended_instrument._id) self.assertEqual(len(extended_instrument.owners), 2) self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id) # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_instrument.computed.firmware_version, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.last_calibration_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue) log.debug("extended_instrument.computed: %s", extended_instrument.computed) #check model inst_model_obj = self.RR.read(instrument_model_id) self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name) #check agent instance inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id) self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name) #check agent inst_agent_obj = self.RR.read(instrument_agent_id) #compound assoc return list of lists so check the first element self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent[0].name) #check platform device plat_device_obj = self.RR.read(platform_device_id) self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name) extended_platform = self.IMS.get_platform_device_extension(platform_device_id) self.assertEqual(1, len(extended_platform.instrument_devices)) self.assertEqual(instrument_device_id, extended_platform.instrument_devices[0]._id) self.assertEqual(1, len(extended_platform.instrument_models)) self.assertEqual(instrument_model_id, extended_platform.instrument_models[0]._id) #check sensor devices self.assertEqual(1, len(extended_instrument.sensor_devices)) #check data_product_parameters_set self.assertEqual(ComputedValueAvailability.PROVIDED, extended_instrument.computed.data_product_parameters_set.status) self.assertTrue( 'Parsed_Canonical' in extended_instrument.computed.data_product_parameters_set.value) # the ctd parameters should include 'temp' self.assertTrue( 'temp' in extended_instrument.computed.data_product_parameters_set.value['Parsed_Canonical']) #none of these will work because there is no agent self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, extended_instrument.computed.firmware_version.status) # self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, # extended_instrument.computed.operational_state.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.power_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.communications_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.data_status_roll_up.status) # self.assertEqual(StatusType.STATUS_OK, # extended_instrument.computed.data_status_roll_up.value) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.location_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.recent_events.status) # self.assertEqual([], extended_instrument.computed.recent_events.value) # cleanup c = DotDict() c.resource_registry = self.RR self.RR2.pluck(instrument_agent_id) self.RR2.pluck(instrument_model_id) self.RR2.pluck(instrument_device_id) self.RR2.pluck(platform_agent_id) self.IMS.force_delete_instrument_agent(instrument_agent_id) self.IMS.force_delete_instrument_model(instrument_model_id) self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id) self.IMS.force_delete_platform_agent(platform_agent_id) self.IMS.force_delete_platform_device(platform_device_id) self.IMS.force_delete_platform_model(platform_model_id) self.IMS.force_delete_sensor_device(sensor_device_id) self.IMS.force_delete_sensor_model(sensor_model_id) #stuff we associate to self.RR.delete(data_producer_id) self.RR.delete(org_id) def test_custom_attributes(self): """ Test assignment of custom attributes """ instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel, {"custom_attributes": {"favorite_color": "attr desc goes here"} })) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice, {"custom_attributes": {"favorite_color": "red", "bogus_attr": "should raise warning" } })) self.IMS.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id) # cleanup self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_instrument_model(instrument_model_id) def _get_datastore(self, dataset_id): dataset = self.DSC.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def test_resource_state_save_restore(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.IMS.create_instrument_model(instModel_obj) log.debug( 'new InstrumentModel id = %s ', instModel_id) # Create InstrumentAgent raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 ) instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg", stream_configurations = [raw_config, parsed_config] ) instAgent_id = self.IMS.create_instrument_agent(instAgent_obj) log.debug( 'new InstrumentAgent id = %s', instAgent_id) self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 ' + '(SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj) self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=spdict_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) self.DP.activate_data_product_persistence(data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug( 'Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] #create the datastore at the beginning of each int test that persists data dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug( 'new dp_id = %s', str(data_product_id2)) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.DP.activate_data_product_persistence(data_product_id=data_product_id2) # spin up agent self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.IMS.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id) gate = ProcessStateGate(self.PDC.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id) # take snapshot of config snap_id = self.IMS.save_resource_state(instDevice_id, "xyzzy snapshot") snap_obj = self.RR.read_attachment(snap_id, include_content=True) print "Saved config:" print snap_obj.content #modify config instance_obj.driver_config["comms_config"] = "BAD_DATA" self.RR.update(instance_obj) #restore config self.IMS.restore_resource_state(instDevice_id, snap_id) instance_obj = self.RR.read(instAgentInstance_id) self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"]) self.DP.delete_data_product(data_product_id1) self.DP.delete_data_product(data_product_id2) def test_agent_instance_config(self): """ Verify that agent configurations are being built properly """ clients = DotDict() clients.resource_registry = self.RR clients.pubsub_management = self.PSC clients.dataset_management = self.DSC pconfig_builder = PlatformAgentConfigurationBuilder(clients) iconfig_builder = InstrumentAgentConfigurationBuilder(clients) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() org_id = self.RR2.create(any_old(RT.Org)) inst_startup_config = {'startup': 'config'} required_config_keys = [ 'org_name', 'device_type', 'agent', 'driver_config', 'stream_config', 'startup_config', 'alarm_defs', 'children'] def verify_instrument_config(config, device_id): for key in required_config_keys: self.assertIn(key, config) self.assertEqual('Org_1', config['org_name']) self.assertEqual(RT.InstrumentDevice, config['device_type']) self.assertIn('driver_config', config) driver_config = config['driver_config'] expected_driver_fields = {'process_type': ('ZMQPyClassDriverLauncher',), } for k, v in expected_driver_fields.iteritems(): self.assertIn(k, driver_config) self.assertEqual(v, driver_config[k]) self.assertEqual self.assertEqual({'resource_id': device_id}, config['agent']) self.assertEqual(inst_startup_config, config['startup_config']) self.assertIn('stream_config', config) for key in ['alarm_defs', 'children']: self.assertEqual({}, config[key]) def verify_child_config(config, device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual('Org_1', config['org_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher',)}, config['driver_config']) self.assertEqual({'resource_id': device_id}, config['agent']) self.assertIn('stream_config', config) if None is inst_device_id: for key in ['alarm_defs', 'children', 'startup_config']: self.assertEqual({}, config[key]) else: for key in ['alarm_defs', 'startup_config']: self.assertEqual({}, config[key]) self.assertIn(inst_device_id, config['children']) verify_instrument_config(config['children'][inst_device_id], inst_device_id) def verify_parent_config(config, parent_device_id, child_device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual('Org_1', config['org_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher',)}, config['driver_config']) self.assertEqual({'resource_id': parent_device_id}, config['agent']) self.assertIn('stream_config', config) for key in ['alarm_defs', 'startup_config']: self.assertEqual({}, config[key]) self.assertIn(child_device_id, config['children']) verify_child_config(config['children'][child_device_id], child_device_id, inst_device_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id) #todo: create org and figure out which agent resource needs to get assigned to it def _make_platform_agent_structure(agent_config=None): if None is agent_config: agent_config = {} # instance creation platform_agent_instance_obj = any_old(RT.PlatformAgentInstance) platform_agent_instance_obj.agent_config = agent_config platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj) # agent creation raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) platform_agent_obj = any_old(RT.PlatformAgent, {"stream_configurations":[raw_config]}) platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj) # device creation platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) # assignments self.RR2.assign_platform_agent_instance_to_platform_device(platform_agent_instance_id, platform_device_id) self.RR2.assign_platform_agent_to_platform_agent_instance(platform_agent_id, platform_agent_instance_id) self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, org_id) return platform_agent_instance_id, platform_agent_id, platform_device_id def _make_instrument_agent_structure(agent_config=None): if None is agent_config: agent_config = {} # instance creation instrument_agent_instance_obj = any_old(RT.InstrumentAgentInstance, {"startup_config": inst_startup_config}) instrument_agent_instance_obj.agent_config = agent_config instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj) # agent creation raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) instrument_agent_obj = any_old(RT.InstrumentAgent, {"stream_configurations":[raw_config]}) instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj) # device creation instrument_device_id = self.IMS.create_instrument_device(any_old(RT.InstrumentDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) # assignments self.RR2.assign_instrument_agent_instance_to_instrument_device(instrument_agent_instance_id, instrument_device_id) self.RR2.assign_instrument_agent_to_instrument_agent_instance(instrument_agent_id, instrument_agent_instance_id) self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id) return instrument_agent_instance_id, instrument_agent_id, instrument_device_id # can't do anything without an agent instance obj log.debug("Testing that preparing a launcher without agent instance raises an error") self.assertRaises(AssertionError, pconfig_builder.prepare, will_launch=False) log.debug("Making the structure for a platform agent, which will be the child") platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure() platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id) log.debug("Preparing a valid agent instance launch, for config only") pconfig_builder.set_agent_instance_object(platform_agent_instance_child_obj) child_config = pconfig_builder.prepare(will_launch=False) verify_child_config(child_config, platform_device_child_id) log.debug("Making the structure for a platform agent, which will be the parent") platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure() platform_agent_instance_parent_obj = self.RR2.read(platform_agent_instance_parent_id) log.debug("Testing child-less parent as a child config") pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj) parent_config = pconfig_builder.prepare(will_launch=False) verify_child_config(parent_config, platform_device_parent_id) log.debug("assigning child platform to parent") self.RR2.assign_platform_device_to_platform_device(platform_device_child_id, platform_device_parent_id) child_device_ids = self.RR2.find_platform_device_ids_of_device(platform_device_parent_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing parent + child as parent config") pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj) parent_config = pconfig_builder.prepare(will_launch=False) verify_parent_config(parent_config, platform_device_parent_id, platform_device_child_id) log.debug("making the structure for an instrument agent") instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure() instrument_agent_instance_obj = self.RR2.read(instrument_agent_instance_id) log.debug("Testing instrument config") iconfig_builder.set_agent_instance_object(instrument_agent_instance_obj) instrument_config = iconfig_builder.prepare(will_launch=False) verify_instrument_config(instrument_config, instrument_device_id) log.debug("assigning instrument to platform") self.RR2.assign_instrument_device_to_platform_device(instrument_device_id, platform_device_child_id) child_device_ids = self.RR2.find_instrument_device_ids_of_device(platform_device_child_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing entire config") pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj) full_config = pconfig_builder.prepare(will_launch=False) verify_parent_config(full_config, platform_device_parent_id, platform_device_child_id, instrument_device_id) #self.fail(parent_config) #plauncher.prepare(will_launch=False) def sample_nested_platform_agent_instance_config(self): """ for informational purposes """ ret = {'org_name': 'Org_1', 'alarm_defs': {}, 'driver_config': {'process_type': ('ZMQPyClassDriverLauncher',)}, 'stream_config': {'parameter_dictionary': 'lots of stuff'}, 'agent': {'resource_id': '33e54106c4444444862da082098bc123'}, 'startup_config': {}, 'device_type': 'PlatformDevice', 'children': {'76a39596eeff4fd5b409c4cb93f0e581': {'org_name': 'Org_1', 'alarm_defs': {}, 'driver_config': {'process_type': ('ZMQPyClassDriverLauncher',)}, 'stream_config': {'parameter_dictionary': 'lots of stuff'}, 'agent': {'resource_id': '76a39596eeff4fd5b409c4cb93f0e581'}, 'startup_config': {}, 'device_type': 'PlatformDevice', 'children': {}}}} return ret
class TestRegisterAndActivate(IonIntegrationTestCase): """ Integration test cases to confirm registration and activation services for marine device resources. """ def setUp(self): """ Test setup. """ # Resources used in the tests. # General resources. self.actor_id = None self.user_info_id = None self.org_id = None self.obs_id = None # Cabled infrastructure. self.cabled_platform_model_id = None self.cabled_platform_site_id = None self.cabled_platform_device_id = None self.cabled_platform_agent_id = None self.cabled_platform_agent_instance_id = None self.cabled_platform_deployment_id = None self.cabled_instrument_deployment_id = None self.cabled_instrument_model_id = None self.cabled_instrument_site_id = None self.cabled_instrument_device_id = None self.cabled_instrument_agent_id = None self.cabled_instrument_agent_instance_id = None self.cabled_instrument_deployment_id = None # Uncabled infrastructure. self.uncabled_platform_model_id = None self.uncabled_platform_site_id = None self.uncabled_platform_device_id = None self.uncabled_platform_agent_id = None self.uncabled_platform_agent_instance_id = None self.uncabled_instrument_model_id = None self.uncabled_instrument_site_id = None self.uncabled_instrument_device_id = None self.uncabled_instrument_agent_id = None self.uncabled_instrument_agent_instance_id = None self.uncabled_site_deployment_id = None # Start container. log.info('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message) log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Setup service clients. self.idms = IdentityManagementServiceClient(node=self.container.node) self.oms = ObservatoryManagementServiceClient(node=self.container.node) # Add generic resources. self._load_system_actors() self._create_user() self._create_org() self._create_observatory() # Add cleanup routine. self.addCleanup(self._cleanup_resources) def _cleanup_resources(self): """ Delete resources created by the tests. """ # Check and clean up cabled resources. if self.cabled_instrument_model_id: self.oms.unassign_instrument_model_from_instrument_site( self.cabled_instrument_model_id, self.cabled_instrument_site_id) self.ims.delete_instrument_model(self.cabled_instrument_model_id) self.cabled_instrument_model_id = None if self.cabled_platform_model_id: self.oms.unassign_platform_model_from_platform_site( self.cabled_platform_model_id, self.cabled_platform_site_id) self.ims.delete_platform_model(self.cabled_platform_model_id) self.cabled_platform_model_id = None if self.cabled_instrument_site_id: self.oms.unassign_site_from_site(self.cabled_instrument_site_id, self.cabled_platform_site_id) self.oms.delete_instrument_site(self.cabled_instrument_site_id) self.cabled_instrument_site_id = None if self.cabled_platform_site_id: self.oms.unassign_site_from_site(self.cabled_platform_site_id, self.obs_id) self.oms.delete_platform_site(self.cabled_platform_site_id) self.cabled_platform_site_id = None # Check and clean up unclabled resources. # TODO # Clean up generic resources. if self.user_info_id: self.idms.delete_user_info(self.user_info_id) self.user_info_id = None if self.actor_id: self.idms.delete_actor_identity(self.actor_id) self.actor_id = None if self.obs_id: self.oms.delete_observatory(self.obs_id) self.obs_id = None if self.org_id: self.container.resource_registry.delete(self.org_id) self.org_id = None def _load_system_actors(self): """ Retrieve system and webauth actors and headers for later use. """ # Retrieve and store system actor and headers. system_actor, _ = self.container.resource_registry.find_resources( RT.ActorIdentity, name=CFG.system.system_actor, id_only=False) self.system_actor = system_actor[0] if system_actor else None self.system_actor_id = system_actor[0]._id if system_actor \ else 'anonymous' self.system_actor_headers = { 'ion-actor-id': self.system_actor_id, 'ion-actor-roles': {'ION': ['ION_MANAGER', 'ORG_MANAGER']}, 'expiry':'0' } # Retrieve and store webauth actor and headers. webauth_actor, _ = self.container.resource_registry.find_resources( RT.ActorIdentity, name=CFG.get_safe("system.web_authentication_actor", "web_authentication"), id_only=False) self.webauth_actor = webauth_actor[0] if webauth_actor else None self.webauth_actor_id = webauth_actor[0]._id if webauth_actor \ else 'anonymous' self.webauth_actor_headers = { 'ion-actor-id': self.webauth_actor_id, 'ion-actor-roles': {'ION': ['ION_MANAGER', 'ORG_MANAGER']}, 'expiry':'0' } def _create_user(self): """ Create user resources that serve as device owners. This test user does not have contact information, user credentials or notification preferences. Results in these objects: ActorIdentity({'_rev': '1', '_id': '07f92986b34e426bba0fca00b73cf4a5', 'lcstate': 'DEPLOYED', 'alt_ids': [], 'description': '', 'ts_updated': '1391542388312', 'actor_type': 1, 'addl': {}, 'ts_created': '1391542388312', 'availability': 'AVAILABLE', 'name': 'Identity for Adam Activationtest'}) UserInfo({'_rev': '1', '_id': 'ac8d368e6ea247d996fd60dd0f9c7f89', 'lcstate': 'DEPLOYED', 'alt_ids': [], 'description': 'Activation Test User', 'tokens': [], 'ts_updated': '1391542388345', 'contact': ContactInformation({'individual_names_given': '', 'city': '', 'roles': [], 'administrative_area': '', 'url': '', 'country': '', 'variables': [{'name': '', 'value': ''}], 'organization_name': '', 'postal_code': '', 'individual_name_family': '', 'phones': [], 'position_name': '', 'email': '', 'street_address': ''}), 'variables': [{'name': '', 'value': ''}], 'addl': {}, 'ts_created': '1391542388345', 'availability': 'AVAILABLE', 'name': 'Adam Activationtest'}) """ # Basic user attributes for test user. user_attrs = { 'name' : 'Adam Activationtest', 'description' : 'Activation Test User' } # Create ActorIdentity. actor_name = "Identity for %s" % user_attrs['name'] actor_identity_obj = IonObject("ActorIdentity", name=actor_name) log.trace("creating user %s with headers: %r", user_attrs['name'], self.webauth_actor_headers) self.actor_id = self.idms.create_actor_identity(actor_identity_obj, headers=self.webauth_actor_headers) # Create UserInfo. user_info_obj = IonObject("UserInfo", **user_attrs) self.user_info_id = self.idms.create_user_info(self.actor_id, user_info_obj,headers=self.webauth_actor_headers) def _create_org(self): """ Create an org that contains all test infrastructure. Results in this object: Org({'message_controllable': True, '_rev': '1', '_id': '9ff82d9f6c7b41f886c6137f54a3086c', 'lcstate': 'DEPLOYED', 'alt_ids': [], 'url': '', 'description': 'An Org for Activation Tests', 'contacts': [], 'org_governance_name': 'ActiveOrg', 'institution': Institution({'website': '', 'phone': '', 'name': '', 'email': ''}), 'ts_updated': '1391542388395', 'monitorable': True, 'org_type': 2, 'addl': {}, 'ts_created': '1391542388395', 'availability': 'AVAILABLE', 'name': 'ActiveOrg'}) """ org_attrs = { 'name' : 'ActiveOrg', 'description' : 'An Org for Activation Tests', 'org_type' : OrgTypeEnum.MARINE_FACILITY } org_obj = IonObject('Org', **org_attrs) self.org_id = self.oms.create_marine_facility(org_obj, headers=self.system_actor_headers) def _create_observatory(self): """ Create a top level obsevaotry site for the tests. Results in this object: Observatory({'reference_designator': '', 'spatial_area_name': '', '_id': 'fdcda51901464575913858f98aaf0f41', '_rev': '1', 'lcstate': 'DEPLOYED', 'alt_ids': [], 'url': '', 'description': 'An Observatory for Activation Tests', 'coordinate_reference_system': GeospatialCoordinateReferenceSystem( {'geospatial_latitude_units': '', 'geospatial_vertical_crs': '', 'geospatial_geodetic_crs': '', 'geospatial_vertical_positive': '', 'geospatial_vertical_units': '', 'geospatial_longitude_units': ''}), 'constraint_list': [], 'environment': 1, 'ts_updated': '1391544601340', 'local_name': '', 'geospatial_point_center': GeospatialIndex({'lat': 0.0, 'lon': 0.0}), 'addl': {}, 'ts_created': '1391544601340', 'availability': 'AVAILABLE', 'name': 'ActiveObservatory'}) """ obs_attrs = { 'name': 'ActiveObservatory', 'description' : 'An Observatory for Activation Tests' } obs_obj = IonObject('Observatory', **obs_attrs) self.obs_id = self.oms.create_observatory(obs_obj, self.org_id) def _create_cabled_resources(self): """ Create preexisting infrastructure for the cabled test environment: sites, deployments, models. These are resources that already exist in the system due to preload or incremental preload updates. PlatformModel InstrumentModel PlatformSite InstrumentSite """ platform_model_attrs = { 'name' : 'LP Jbox', 'description' : 'Node Type: LJ', 'manufacturer' : 'University of Washington', 'platform_type' : 'Cable Node', 'platform_family' : 'Low Power JBox', 'ci_onboard' : False, 'shore_networked' : True } instrument_model_attrs = { 'name': 'Diffuse Vent Fluid 3-D Temperature Array (TMPSF-A)', 'description': 'Measures temperatures of diffuse flow across the seafloor', 'reference_designator': 'TMPSFA', 'class_name': 'Temperature seafloor', 'mixed_sampling_mode': True, 'integrated_inductive_modem_available': True, 'internal_battery': True, 'addl': {'comments': '', 'connector': '', 'makemodel_description': 'XR-420', 'input_voltage_range': '', 'interface': '', 'output_description': '', 'class_long_name': 'Temperature_seafloor'}, 'ooi_make_model': 'XR-420', 'series_name': 'TMPSF Series A', 'inline_management': True, 'series_id': 'TMPSFA', 'subseries_name': 'TMPSF Series A Sub 01', 'primary_interface': 1, 'manufacturer': 'RBR Global', 'family_name': 'Seafloor Properties', 'class_description': 'Measures temperatures of diffuse flow across the seafloor', 'class_alternate_name': 'Diffuse Vent Fluid 3-D Temperature Array', 'subseries_id': 'TMPSFA01', 'class_id': 'TMPSF', 'family_id': 'SFL', 'has_clock': True } platform_site_attrs = { 'name' : 'Cabled LP JBOX Platform Site', 'description' : 'Test Site for a Cabled LP JBOX Platform' } instrument_site_attrs = { 'name' : 'Cabled TMPSF Instrument Site', 'description' : 'Test Site for a Cabled TMPSF Instrument' } # Create the cabled model preloaded resources. platform_model = IonObject('PlatformModel', **platform_model_attrs) self.cabled_platform_model_id = self.ims.create_platform_model(platform_model) instrument_model = IonObject('InstrumentModel', **instrument_model_attrs) self.cabled_instrument_model_id = self.ims.create_instrument_model( instrument_model) # Create the cabled sites and link them appropriately. platform_site = IonObject('', **platform_site_attrs) self.cabled_platform_site_id = self.oms.create_platform_site(platform_site) self.oms.assign_site_to_site(self.cabled_platform_site_id, self.obs_id) instrument_site = IonObject('', **instrument_site_attrs) self.cabled_instrument_site_id = self.oms.create_instrument_site(instrument_site) self.oms.assign_site_to_site(self.cabled_instrument_site_id, self.cabled_platform_site_id) # Assign models to available sites. self.oms.assign_platform_model_to_platform_site( self.cabled_platform_model_id, self.cabled_platform_site_id) self.oms.assign_instrument_model_to_instrument_site( self.cabled_instrument_model_id, self.cabled_instrument_site_id) def _create_uncabled_resources(self): """ Create preexisting infrastructure for the uncabled test environment: sites, deployments, models. """ pass def test_cabled_device_activation(self): """ Test registration and activation of cabled device infrastructure. """ pass def test_uncabled_device_activation(self): """ Test registration and activation of uncabled device infrastructure. """ pass
def instrument_test_driver(container): org_client = OrgManagementServiceClient(node=container.node) id_client = IdentityManagementServiceClient(node=container.node) system_actor = id_client.find_actor_identity_by_name(name=CFG.system.system_actor) log.info("system actor:" + system_actor._id) sa_header_roles = get_role_message_headers(org_client.find_all_roles_by_user(system_actor._id)) # Names of agent data streams to be configured. parsed_stream_name = "ctd_parsed" raw_stream_name = "ctd_raw" # Driver configuration. # Simulator driver_config = { "svr_addr": "localhost", "cmd_port": 5556, "evt_port": 5557, "dvr_mod": "ion.services.mi.drivers.sbe37_driver", "dvr_cls": "SBE37Driver", "comms_config": { SBE37Channel.CTD: { "method": "ethernet", "device_addr": CFG.device.sbe37.host, "device_port": CFG.device.sbe37.port, "server_addr": "localhost", "server_port": 8888, } }, } # Hardware _container_client = ContainerAgentClient(node=container.node, name=container.name) # Create a pubsub client to create streams. _pubsub_client = PubsubManagementServiceClient(node=container.node) # A callback for processing subscribed-to data. def consume(message, headers): log.info("Subscriber received message: %s", str(message)) # Create a stream subscriber registrar to create subscribers. subscriber_registrar = StreamSubscriberRegistrar(process=container, node=container.node) subs = [] # Create streams for each stream named in driver. stream_config = {} for (stream_name, val) in PACKET_CONFIG.iteritems(): stream_def = ctd_stream_definition(stream_id=None) stream_def_id = _pubsub_client.create_stream_definition(container=stream_def) stream_id = _pubsub_client.create_stream( name=stream_name, stream_definition_id=stream_def_id, original=True, encoding="ION R2", headers={"ion-actor-id": system_actor._id, "ion-actor-roles": sa_header_roles}, ) stream_config[stream_name] = stream_id # Create subscriptions for each stream. exchange_name = "%s_queue" % stream_name sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name, callback=consume) sub.start() query = StreamQuery(stream_ids=[stream_id]) sub_id = _pubsub_client.create_subscription(query=query, exchange_name=exchange_name) _pubsub_client.activate_subscription(sub_id) subs.append(sub) # Create agent config. agent_resource_id = "123xyz" agent_config = { "driver_config": driver_config, "stream_config": stream_config, "agent": {"resource_id": agent_resource_id}, } # Launch an instrument agent process. _ia_name = "agent007" _ia_mod = "ion.services.mi.instrument_agent" _ia_class = "InstrumentAgent" _ia_pid = _container_client.spawn_process(name=_ia_name, module=_ia_mod, cls=_ia_class, config=agent_config) log.info("got pid=%s for resource_id=%s" % (str(_ia_pid), str(agent_resource_id)))
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' unittest # suppress an pycharm inspector error if all unittest.skip references are commented out self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.OMS = ObservatoryManagementServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) # @unittest.skip('this test just for debugging setup') # def test_just_the_setup(self): # return @attr('EXT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode as it depends on modifying CFG on service side') def test_resources_associations_extensions(self): """ create one of each resource and association used by IMS to guard against problems in ion-definitions """ self.patch_cfg(CFG["container"], {"extended_resources": {"strip_results": False}}) #stuff we control instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance)) instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent)) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) instrument_site_id, _ = self.RR.create(any_old(RT.InstrumentSite)) platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance)) platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent)) platform_site_id, _ = self.RR.create(any_old(RT.PlatformSite)) platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice)) sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel)) #stuff we associate to data_producer_id, _ = self.RR.create(any_old(RT.DataProducer)) org_id, _ = self.RR.create(any_old(RT.Org)) #instrument_agent_instance_id #is only a target #instrument_agent self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id) self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id) self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device_id) instrument_model_id #is only a target platform_agent_instance_id #is only a target #platform_agent self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id) self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(instrument_site_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(platform_site_id, PRED.hasDevice, platform_device_id) self.RR.create_association(platform_site_id, PRED.hasSite, instrument_site_id) platform_model_id #is only a target #sensor_device self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id) sensor_model_id #is only a target #create a parsed product for this instrument output dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', processing_level_code='Parsed_Canonical') pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1) def addInstOwner(inst_id, subject): actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject}) user_id = self.IDS.create_actor_identity(actor_identity_obj) user_info_obj = any_old(RT.UserInfo) user_info_id = self.IDS.create_user_info(user_id, user_info_obj) self.RR.create_association(inst_id, PRED.hasOwner, user_id) #Testing multiple instrument owners addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254") addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256") extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id) self.assertEqual(instrument_device_id, extended_instrument._id) self.assertEqual(len(extended_instrument.owners), 2) self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id) # Lifecycle self.assertEquals(len(extended_instrument.lcstate_transitions), 6) self.assertEquals(set(extended_instrument.lcstate_transitions.keys()), set(['develop', 'deploy', 'retire', 'plan', 'integrate', 'delete'])) self.assertEquals(len(extended_instrument.availability_transitions), 2) self.assertEquals(set(extended_instrument.availability_transitions.keys()), set(['enable', 'announce'])) # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue) log.debug("extended_instrument.computed: %s", extended_instrument.computed) #check model inst_model_obj = self.RR.read(instrument_model_id) self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name) #check agent instance inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id) self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name) #check agent inst_agent_obj = self.RR.read(instrument_agent_id) #compound assoc return list of lists so check the first element self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent.name) #check platform device plat_device_obj = self.RR.read(platform_device_id) self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name) extended_platform = self.IMS.get_platform_device_extension(platform_device_id) self.assertEqual(1, len(extended_platform.portals)) self.assertEqual(1, len(extended_platform.portal_instruments)) #self.assertEqual(1, len(extended_platform.computed.portal_status.value)) # no agent started so NO statuses reported self.assertEqual(1, len(extended_platform.instrument_devices)) self.assertEqual(instrument_device_id, extended_platform.instrument_devices[0]._id) self.assertEqual(1, len(extended_platform.instrument_models)) self.assertEqual(instrument_model_id, extended_platform.instrument_models[0]._id) self.assertEquals(extended_platform.platform_agent._id, platform_agent_id) self.assertEquals(len(extended_platform.lcstate_transitions), 6) self.assertEquals(set(extended_platform.lcstate_transitions.keys()), set(['develop', 'deploy', 'retire', 'plan', 'integrate', 'delete'])) self.assertEquals(len(extended_platform.availability_transitions), 2) self.assertEquals(set(extended_platform.availability_transitions.keys()), set(['enable', 'announce'])) #check sensor devices self.assertEqual(1, len(extended_instrument.sensor_devices)) ##check data_product_parameters_set # !!! OOIION-1342 The UI does not use data_product_parameters_set and it is an expensive calc so the attribute calc was disabled # !!! Remove check in this test #self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.data_product_parameters_set.status) #self.assertTrue( 'Parsed_Canonical' in extended_instrument.computed.data_product_parameters_set.value) ## the ctd parameters should include 'temp' #self.assertTrue( 'temp' in extended_instrument.computed.data_product_parameters_set.value['Parsed_Canonical']) #none of these will work because there is no agent # self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, # extended_instrument.computed.firmware_version.status) # self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, # extended_instrument.computed.operational_state.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.power_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.communications_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.data_status_roll_up.status) # self.assertEqual(DeviceStatusType.STATUS_OK, # extended_instrument.computed.data_status_roll_up.value) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.location_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.recent_events.status) # self.assertEqual([], extended_instrument.computed.recent_events.value) # cleanup c = DotDict() c.resource_registry = self.RR self.RR2.pluck(instrument_agent_id) self.RR2.pluck(instrument_model_id) self.RR2.pluck(instrument_device_id) self.RR2.pluck(platform_agent_id) self.RR2.pluck(sensor_device_id) self.IMS.force_delete_instrument_agent(instrument_agent_id) self.IMS.force_delete_instrument_model(instrument_model_id) self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id) self.IMS.force_delete_platform_agent(platform_agent_id) self.OMS.force_delete_instrument_site(instrument_site_id) self.OMS.force_delete_platform_site(platform_site_id) self.IMS.force_delete_platform_device(platform_device_id) self.IMS.force_delete_platform_model(platform_model_id) self.IMS.force_delete_sensor_device(sensor_device_id) self.IMS.force_delete_sensor_model(sensor_model_id) #stuff we associate to self.RR.delete(data_producer_id) self.RR.delete(org_id) def test_custom_attributes(self): """ Test assignment of custom attributes """ instModel_obj = IonObject(OT.CustomAttribute, name='SBE37IMModelAttr', description="model custom attr") instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel, {"custom_attributes": [instModel_obj] })) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice, {"custom_attributes": {"favorite_color": "red", "bogus_attr": "should raise warning" } })) self.IMS.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id) # cleanup self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_instrument_model(instrument_model_id) def _get_datastore(self, dataset_id): dataset = self.DSC.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def test_data_producer(self): idevice_id = self.IMS.create_instrument_device(any_old(RT.InstrumentDevice)) self.assertEqual(1, len(self.RR2.find_data_producer_ids_of_instrument_device_using_has_data_producer(idevice_id))) pdevice_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice)) self.assertEqual(1, len(self.RR2.find_data_producer_ids_of_platform_device_using_has_data_producer(pdevice_id))) @attr('PREP') def test_prepare_resource_support(self): """ create one of each resource and association used by IMS to guard against problems in ion-definitions """ #stuff we control instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance)) instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent)) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance)) platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent)) platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice)) sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel)) instrument_device2_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) instrument_device3_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) platform_device2_id, _ = self.RR.create(any_old(RT.PlatformDevice)) sensor_device2_id, _ = self.RR.create(any_old(RT.SensorDevice)) #stuff we associate to data_producer_id, _ = self.RR.create(any_old(RT.DataProducer)) org_id, _ = self.RR.create(any_old(RT.Org)) #instrument_agent_instance_id #is only a target #instrument_agent self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id) self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id) self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device_id) self.RR.create_association(instrument_device2_id, PRED.hasModel, instrument_model_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device2_id) instrument_model_id #is only a target platform_agent_instance_id #is only a target #platform_agent self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id) self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(platform_device2_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device2_id, PRED.hasDevice, instrument_device2_id) platform_model_id #is only a target #sensor_device self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(sensor_device2_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device2_id, PRED.hasDevice, instrument_device2_id) sensor_model_id #is only a target #set lcstate - used for testing prepare - not setting all to DEVELOP, only some self.RR.execute_lifecycle_transition(instrument_agent_id, LCE.DEVELOP) self.RR.execute_lifecycle_transition(instrument_device_id, LCE.DEVELOP) self.RR.execute_lifecycle_transition(instrument_device2_id, LCE.DEVELOP) self.RR.execute_lifecycle_transition(platform_device_id, LCE.DEVELOP) self.RR.execute_lifecycle_transition(platform_device2_id, LCE.DEVELOP) self.RR.execute_lifecycle_transition(platform_agent_id, LCE.DEVELOP) #create a parsed product for this instrument output dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', processing_level_code='Parsed_Canonical') pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1) def addInstOwner(inst_id, subject): actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject}) user_id = self.IDS.create_actor_identity(actor_identity_obj) user_info_obj = any_old(RT.UserInfo) user_info_id = self.IDS.create_user_info(user_id, user_info_obj) self.RR.create_association(inst_id, PRED.hasOwner, user_id) #Testing multiple instrument owners addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254") addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256") def ion_object_encoder(obj): return obj.__dict__ #First call to create instrument_data = self.IMS.prepare_instrument_device_support() #print simplejson.dumps(instrument_data, default=ion_object_encoder, indent=2) self.assertEqual(instrument_data._id, '') self.assertEqual(instrument_data.type_, OT.InstrumentDevicePrepareSupport) self.assertEqual(len(instrument_data.associations['InstrumentModel'].resources), 1) self.assertEqual(instrument_data.associations['InstrumentModel'].resources[0]._id, instrument_model_id) self.assertEqual(len(instrument_data.associations['InstrumentAgentInstance'].resources), 1) self.assertEqual(instrument_data.associations['InstrumentAgentInstance'].resources[0]._id, instrument_agent_instance_id) self.assertEqual(len(instrument_data.associations['InstrumentModel'].associated_resources), 0) self.assertEqual(len(instrument_data.associations['InstrumentAgentInstance'].associated_resources), 0) self.assertEqual(len(instrument_data.associations['SensorDevice'].resources), 0) #Next call to update instrument_data = self.IMS.prepare_instrument_device_support(instrument_device_id) #print 'Update results' #print simplejson.dumps(instrument_data, default=ion_object_encoder, indent=2) self.assertEqual(instrument_data._id, instrument_device_id) self.assertEqual(instrument_data.type_, OT.InstrumentDevicePrepareSupport) self.assertEqual(len(instrument_data.associations['InstrumentModel'].resources), 1) self.assertEqual(instrument_data.associations['InstrumentModel'].resources[0]._id, instrument_model_id) self.assertEqual(len(instrument_data.associations['InstrumentAgentInstance'].resources), 1) self.assertEqual(instrument_data.associations['InstrumentAgentInstance'].resources[0]._id, instrument_agent_instance_id) self.assertEqual(len(instrument_data.associations['InstrumentModel'].associated_resources), 1) self.assertEqual(instrument_data.associations['InstrumentModel'].associated_resources[0].s, instrument_device_id) self.assertEqual(instrument_data.associations['InstrumentModel'].associated_resources[0].o, instrument_model_id) self.assertEqual(len(instrument_data.associations['InstrumentAgentInstance'].associated_resources), 1) self.assertEqual(instrument_data.associations['InstrumentAgentInstance'].associated_resources[0].o, instrument_agent_instance_id) self.assertEqual(instrument_data.associations['InstrumentAgentInstance'].associated_resources[0].s, instrument_device_id) self.assertEqual(len(instrument_data.associations['SensorDevice'].resources), 1) self.assertEqual(instrument_data.associations['SensorDevice'].resources[0]._id, sensor_device_id) self.assertEqual(len(instrument_data.associations['SensorDevice'].associated_resources), 1) self.assertEqual(instrument_data.associations['SensorDevice'].associated_resources[0].o, instrument_device_id) self.assertEqual(instrument_data.associations['SensorDevice'].associated_resources[0].s, sensor_device_id) self.assertEqual(instrument_data.associations['InstrumentModel'].assign_request.request_parameters['instrument_device_id'], instrument_device_id) #test prepare for create of instrument agent instance instrument_agent_data = self.IMS.prepare_instrument_agent_instance_support() #print 'Update results' #print simplejson.dumps(instrument_agent_data, default=ion_object_encoder, indent=2) self.assertEqual(instrument_agent_data._id, '') self.assertEqual(instrument_agent_data.type_, OT.InstrumentAgentInstancePrepareSupport) self.assertEqual(len(instrument_agent_data.associations['InstrumentDevice'].resources), 2) self.assertEqual(len(instrument_agent_data.associations['InstrumentAgent'].resources), 1) self.assertEqual(instrument_agent_data.associations['InstrumentAgent'].resources[0]._id, instrument_agent_id) self.assertEqual(len(instrument_agent_data.associations['InstrumentDevice'].associated_resources), 0) self.assertEqual(len(instrument_agent_data.associations['InstrumentAgent'].associated_resources), 0) #test prepare for update of instrument agent instance to see if it is associated with the instrument that was created instrument_agent_data = self.IMS.prepare_instrument_agent_instance_support(instrument_agent_instance_id=instrument_agent_instance_id) #print 'Update results' #print simplejson.dumps(instrument_agent_data, default=ion_object_encoder, indent=2) self.assertEqual(instrument_agent_data._id, instrument_agent_instance_id) self.assertEqual(instrument_agent_data.type_, OT.InstrumentAgentInstancePrepareSupport) self.assertEqual(len(instrument_agent_data.associations['InstrumentDevice'].resources), 3) self.assertEqual(len(instrument_agent_data.associations['InstrumentAgent'].resources), 1) self.assertEqual(instrument_agent_data.associations['InstrumentAgent'].resources[0]._id, instrument_agent_id) self.assertEqual(len(instrument_agent_data.associations['InstrumentDevice'].associated_resources), 1) self.assertEqual(instrument_agent_data.associations['InstrumentDevice'].associated_resources[0].s, instrument_device_id) self.assertEqual(instrument_agent_data.associations['InstrumentDevice'].associated_resources[0].o, instrument_agent_instance_id) self.assertEqual(len(instrument_agent_data.associations['InstrumentAgent'].associated_resources), 1) self.assertEqual(instrument_agent_data.associations['InstrumentAgent'].associated_resources[0].o, instrument_agent_id) self.assertEqual(instrument_agent_data.associations['InstrumentAgent'].associated_resources[0].s, instrument_agent_instance_id) self.assertEqual(instrument_agent_data.associations['InstrumentAgent'].assign_request.request_parameters['instrument_agent_instance_id'], instrument_agent_instance_id) #test prepare for update of data product to see if it is associated with the instrument that was created data_product_data = self.DP.prepare_data_product_support(data_product_id1) #print simplejson.dumps(data_product_data, default=ion_object_encoder, indent=2) self.assertEqual(data_product_data._id, data_product_id1) self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport) self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 1) self.assertEqual(len(data_product_data.associations['Dataset'].resources), 0) self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 1) self.assertEqual(data_product_data.associations['StreamDefinition'].associated_resources[0].s, data_product_id1) self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 0) self.assertEqual(len(data_product_data.associations['InstrumentDeviceHasOutputProduct'].resources), 3) self.assertEqual(len(data_product_data.associations['InstrumentDeviceHasOutputProduct'].associated_resources), 1) self.assertEqual(data_product_data.associations['InstrumentDeviceHasOutputProduct'].associated_resources[0].s, instrument_device_id) self.assertEqual(data_product_data.associations['InstrumentDeviceHasOutputProduct'].associated_resources[0].o, data_product_id1) self.assertEqual(len(data_product_data.associations['PlatformDevice'].resources), 2) platform_data = self.IMS.prepare_platform_device_support() #print simplejson.dumps(platform_data, default=ion_object_encoder, indent=2) self.assertEqual(platform_data._id, '') self.assertEqual(platform_data.type_, OT.PlatformDevicePrepareSupport) self.assertEqual(len(platform_data.associations['PlatformModel'].resources), 1) self.assertEqual(platform_data.associations['PlatformModel'].resources[0]._id, platform_model_id) self.assertEqual(len(platform_data.associations['PlatformAgentInstance'].resources), 1) self.assertEqual(platform_data.associations['PlatformAgentInstance'].resources[0]._id, platform_agent_instance_id) self.assertEqual(len(platform_data.associations['PlatformModel'].associated_resources), 0) self.assertEqual(len(platform_data.associations['PlatformAgentInstance'].associated_resources), 0) self.assertEqual(len(platform_data.associations['InstrumentDevice'].resources), 1) platform_data = self.IMS.prepare_platform_device_support(platform_device_id) #print simplejson.dumps(platform_data, default=ion_object_encoder, indent=2) self.assertEqual(platform_data._id, platform_device_id) self.assertEqual(platform_data.type_, OT.PlatformDevicePrepareSupport) self.assertEqual(len(platform_data.associations['PlatformModel'].resources), 1) self.assertEqual(platform_data.associations['PlatformModel'].resources[0]._id, platform_model_id) self.assertEqual(len(platform_data.associations['PlatformAgentInstance'].resources), 1) self.assertEqual(platform_data.associations['PlatformAgentInstance'].resources[0]._id, platform_agent_instance_id) self.assertEqual(len(platform_data.associations['PlatformModel'].associated_resources), 1) self.assertEqual(platform_data.associations['PlatformModel'].associated_resources[0].s, platform_device_id) self.assertEqual(platform_data.associations['PlatformModel'].associated_resources[0].o, platform_model_id) self.assertEqual(len(platform_data.associations['PlatformAgentInstance'].associated_resources), 1) self.assertEqual(platform_data.associations['PlatformAgentInstance'].associated_resources[0].o, platform_agent_instance_id) self.assertEqual(platform_data.associations['PlatformAgentInstance'].associated_resources[0].s, platform_device_id) self.assertEqual(len(platform_data.associations['InstrumentDevice'].resources), 2) #self.assertEqual(len(platform_data.associations['InstrumentDevice'].associated_resources), 1) #self.assertEqual(platform_data.associations['InstrumentDevice'].associated_resources[0].s, platform_device_id) #self.assertEqual(platform_data.associations['InstrumentDevice'].associated_resources[0].o, instrument_device_id) self.assertEqual(platform_data.associations['PlatformModel'].assign_request.request_parameters['platform_device_id'], platform_device_id) # cleanup c = DotDict() c.resource_registry = self.RR self.RR2.pluck(instrument_agent_id) self.RR2.pluck(instrument_model_id) self.RR2.pluck(instrument_device_id) self.RR2.pluck(platform_agent_id) self.RR2.pluck(sensor_device_id) self.RR2.pluck(sensor_device2_id) self.IMS.force_delete_instrument_agent(instrument_agent_id) self.IMS.force_delete_instrument_model(instrument_model_id) self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_instrument_device(instrument_device2_id) self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id) self.IMS.force_delete_platform_agent(platform_agent_id) self.IMS.force_delete_platform_device(platform_device_id) self.IMS.force_delete_platform_device(platform_device2_id) self.IMS.force_delete_platform_model(platform_model_id) self.IMS.force_delete_sensor_device(sensor_device_id) self.IMS.force_delete_sensor_device(sensor_device2_id) self.IMS.force_delete_sensor_model(sensor_model_id) #stuff we associate to self.RR.delete(data_producer_id) self.RR.delete(org_id)
class UserNotificationIntTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2dm.yml') self.unsc = UserNotificationServiceClient(node=self.container.node) self.rrc = ResourceRegistryServiceClient(node=self.container.node) self.imc = IdentityManagementServiceClient(node=self.container.node) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_email(self): proc1 = self.container.proc_manager.procs_by_name['user_notification'] # Create a user and get the user_id user = UserInfo(name = 'new_user') user_id, _ = self.rrc.create(user) # set up.... notification_id = self.unsc.create_email(event_type='ResourceLifecycleEvent', event_subtype=None, origin='Some_Resource_Agent_ID1', origin_type=None, user_id=user_id, email='*****@*****.**', mode = DeliveryMode.DIGEST, message_header='message_header', parser='parser', period=1) #------------------------------------------------------------------------------------------------------ # Setup so as to be able to get the message and headers going into the # subscription callback method of the EmailEventProcessor #------------------------------------------------------------------------------------------------------ # publish an event for each notification to generate the emails rle_publisher = EventPublisher("ResourceLifecycleEvent") rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event") msg_tuple = proc1.event_processors[notification_id].smtp_client.sentmail.get(timeout=4) self.assertTrue(proc1.event_processors[notification_id].smtp_client.sentmail.empty()) message = msg_tuple[2] list_lines = message.split("\n") #------------------------------------------------------- # parse the message body #------------------------------------------------------- message_dict = {} for line in list_lines: key_item = line.split(": ") if key_item[0] == 'Subject': message_dict['Subject'] = key_item[1] + key_item[2] else: try: message_dict[key_item[0]] = key_item[1] except IndexError as exc: # these IndexError exceptions happen only because the message sometimes # has successive /r/n (i.e. new lines) and therefore, # the indexing goes out of range. These new lines # can just be ignored. So we ignore the exceptions here. pass #------------------------------------------------------- # make assertions #------------------------------------------------------- self.assertEquals(msg_tuple[1], '*****@*****.**' ) #self.assertEquals(msg_tuple[0], ION_NOTIFICATION_EMAIL_ADDRESS) #self.assertEquals(message_dict['From'], ION_NOTIFICATION_EMAIL_ADDRESS) self.assertEquals(message_dict['To'], '*****@*****.**') self.assertEquals(message_dict['Event'].rstrip('\r'), 'ResourceLifecycleEvent') self.assertEquals(message_dict['Originator'].rstrip('\r'), 'Some_Resource_Agent_ID1') self.assertEquals(message_dict['Description'].rstrip('\r'), 'RLE test event') @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_sms(self): proc1 = self.container.proc_manager.procs_by_name['user_notification'] # Create a user and get the user_id user = UserInfo(name = 'new_user') user_id, _ = self.rrc.create(user) # set up.... notification_id = self.unsc.create_sms(event_type='ResourceLifecycleEvent', event_subtype=None, origin='Some_Resource_Agent_ID1', origin_type=None, user_id=user_id, phone = '401-XXX-XXXX', provider='T-Mobile', message_header='message_header', parser='parser', ) #------------------------------------------------------------------------------------------------------ # Setup so as to be able to get the message and headers going into the # subscription callback method of the EmailEventProcessor #------------------------------------------------------------------------------------------------------ # publish an event for each notification to generate the emails rle_publisher = EventPublisher("ResourceLifecycleEvent") rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event") msg_tuple = proc1.event_processors[notification_id].smtp_client.sentmail.get(timeout=4) self.assertTrue(proc1.event_processors[notification_id].smtp_client.sentmail.empty()) message = msg_tuple[2] list_lines = message.split("\n") #------------------------------------------------------- # parse the message body #------------------------------------------------------- message_dict = {} for line in list_lines: key_item = line.split(": ") if key_item[0] == 'Subject': message_dict['Subject'] = key_item[1] + key_item[2] else: try: message_dict[key_item[0]] = key_item[1] except IndexError as exc: # these IndexError exceptions happen only because the message sometimes # has successive /r/n (i.e. new lines) and therefore, # the indexing goes out of range. These new lines # can just be ignored. So we ignore the exceptions here. pass #------------------------------------------------------- # make assertions #------------------------------------------------------- self.assertEquals(msg_tuple[1], '*****@*****.**' ) #self.assertEquals(msg_tuple[0], ION_NOTIFICATION_EMAIL_ADDRESS) self.assertEquals(message_dict['Description'].rstrip('\r'), 'RLE test event') @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_event_detection(self): proc1 = self.container.proc_manager.procs_by_name['user_notification'] # Create a user and get the user_id user = UserInfo(name = 'new_user') user_id, _ = self.rrc.create(user) # Create detection notification dfilt = DetectionFilterConfig() dfilt.processing['condition'] = 5 dfilt.processing['comparator'] = '>' dfilt.processing['filter_field'] = 'voltage' dfilt.delivery['message'] = 'I got my detection event!' notification_id = self.unsc.create_detection_filter(event_type='ExampleDetectableEvent', event_subtype=None, origin='Some_Resource_Agent_ID1', origin_type=None, user_id=user_id, filter_config=dfilt ) #--------------------------------------------------------------------------------- # Create event subscription for resulting detection event #--------------------------------------------------------------------------------- # Create an email notification so that when the DetectionEventProcessor # detects an event and fires its own output event, this will caught by an # EmailEventProcessor and an email will be sent to the user notification_id_2 = self.unsc.create_email(event_type='DetectionEvent', event_subtype=None, origin='DetectionEventProcessor', origin_type=None, user_id=user_id, email='*****@*****.**', mode = DeliveryMode.UNFILTERED, message_header='Detection event', parser='parser', period=1) # Send event that is not detected # publish an event for each notification to generate the emails rle_publisher = EventPublisher("ExampleDetectableEvent") # since the voltage field in this event is less than 5, it will not be detected rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event", voltage = 3) # Check at the end of the test to make sure this event never triggered a Detectable! # Send Event that is detected # publish an event for each notification to generate the emails # since the voltage field in this event is greater than 5, it WILL be detected rle_publisher = EventPublisher("ExampleDetectableEvent") rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event", voltage = 10) #------------------------------------------------------- # make assertions #------------------------------------------------------- msg_tuple = proc1.event_processors[notification_id_2].smtp_client.sentmail.get(timeout=4) # The first event never triggered an email because the voltage was less than 5, the queue is now empty self.assertTrue(proc1.event_processors[notification_id_2].smtp_client.sentmail.empty()) self.assertEquals(msg_tuple[1], '*****@*****.**' ) #self.assertEquals(msg_tuple[0], ION_NOTIFICATION_EMAIL_ADDRESS) # parse the message body message = msg_tuple[2] list_lines = message.split("\n") message_dict = {} for line in list_lines: key_item = line.split(": ") if key_item[0] == 'Subject': message_dict['Subject'] = key_item[1] + key_item[2] else: try: message_dict[key_item[0]] = key_item[1] except IndexError as exc: # these IndexError exceptions happen only because the message sometimes # has successive /r/n (i.e. new lines) and therefore, # the indexing goes out of range. These new lines # can just be ignored. So we ignore the exceptions here. pass #self.assertEquals(message_dict['From'], ION_NOTIFICATION_EMAIL_ADDRESS) self.assertEquals(message_dict['To'], '*****@*****.**') self.assertEquals(message_dict['Event'].rstrip('\r'), 'DetectionEvent') self.assertEquals(message_dict['Originator'].rstrip('\r'), 'DetectionEventProcessor') self.assertEquals(message_dict['Description'].rstrip('\r'), 'Event was detected by DetectionEventProcessor') @unittest.skip('interface has changed!') def test_find_event_types_for_resource(self): # create a dataset object in the RR to pass into the UNS method dataset_object = IonObject(RT.DataSet, name="dataset1") dataset_id, version = self.rrc.create(dataset_object) # get the list of event types for the dataset events = self.unsc.find_event_types_for_resource(dataset_id) log.debug("dataset events = " + str(events)) if not events == ['dataset_supplement_added', 'dataset_change']: self.fail("failed to return correct list of event types") # try to pass in an id of a resource that doesn't exist (should fail) try: events = self.unsc.find_event_types_for_resource("bogus_id") self.fail("failed to detect non-existant resource") except: pass @unittest.skip('interface has changed!') def test_create_two_user_notifications(self): # create user with email address in RR user_identty_object = IonObject(RT.ActorIdentity, name="user1") user_id = self.imc.create_actor_identity(user_identty_object) user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}}) self.imc.create_user_info(user_id, user_info_object) # create first notification notification_object1 = IonObject(RT.NotificationRequest, {"name":"notification1", "origin_list":['Some_Resource_Agent_ID1'], "events_list":['ResourceLifecycleEvent']}) notification_id1 = self.unsc.create_notification(notification_object1, user_id) # create second notification notification_object2 = IonObject(RT.NotificationRequest, {"name":"notification2", "origin_list":['Some_Resource_Agent_ID2'], "events_list":['DataEvent']}) notification_id2 = self.unsc.create_notification(notification_object2, user_id) # read the notifications back and check that they are correct n1 = self.unsc.read_notification(notification_id1) if n1.name != notification_object1.name or \ n1.origin_list != notification_object1.origin_list or \ n1.events_list != notification_object1.events_list: self.fail("notification was not correct") n2 = self.unsc.read_notification(notification_id2) if n2.name != notification_object2.name or \ n2.origin_list != notification_object2.origin_list or \ n2.events_list != notification_object2.events_list: self.fail("notification was not correct") @unittest.skip('interface has changed!') def test_delete_user_notifications(self): # create user with email address in RR user_identty_object = IonObject(RT.ActorIdentity, name="user1") user_id = self.imc.create_actor_identity(user_identty_object) user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}}) self.imc.create_user_info(user_id, user_info_object) # create first notification notification_object1 = IonObject(RT.NotificationRequest, {"name":"notification1", "origin_list":['Some_Resource_Agent_ID1'], "events_list":['ResourceLifecycleEvent']}) notification1_id = self.unsc.create_notification(notification_object1, user_id) # create second notification notification_object2 = IonObject(RT.NotificationRequest, {"name":"notification2", "origin_list":['Some_Resource_Agent_ID2'], "events_list":['DataEvent']}) notification2_id = self.unsc.create_notification(notification_object2, user_id) # delete both notifications self.unsc.delete_notification(notification1_id) self.unsc.delete_notification(notification2_id) # check that the notifications are not there try: n1 = self.unsc.read_notification(notification1_id) except: try: n2 = self.unsc.read_notification(notification2_id) except: return self.fail("failed to delete notifications") @unittest.skip('interface has changed!') def test_find_user_notifications(self): # create user with email address in RR user_identty_object = IonObject(RT.ActorIdentity, name="user1") user_id = self.imc.create_actor_identity(user_identty_object) user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}}) self.imc.create_user_info(user_id, user_info_object) # create first notification notification_object = IonObject(RT.NotificationRequest, {"name":"notification1", "origin_list":['Some_Resource_Agent_ID1'], "events_list":['ResourceLifecycleEvent']}) self.unsc.create_notification(notification_object, user_id) # create second notification notification_object = IonObject(RT.NotificationRequest, {"name":"notification2", "origin_list":['Some_Resource_Agent_ID2'], "events_list":['DataEvent']}) self.unsc.create_notification(notification_object, user_id) # try to find all notifications for user notifications = self.unsc.find_notifications_by_user(user_id) if len(notifications) != 2: self.fail("failed to find all notifications") @unittest.skip('interface has changed!') def test_update_user_notification(self): # create user with email address in RR user_identty_object = IonObject(RT.ActorIdentity, name="user1") user_id = self.imc.create_actor_identity(user_identty_object) user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}}) self.imc.create_user_info(user_id, user_info_object) # create a notification notification_object = IonObject(RT.NotificationRequest, {"name":"notification1", "origin_list":['Some_Resource_Agent_ID1'], "events_list":['ResourceLifecycleEvent']}) notification_id = self.unsc.create_notification(notification_object, user_id) # read back the notification and change it notification = self.unsc.read_notification(notification_id) notification.origin_list = ['Some_Resource_Agent_ID5'] self.unsc.update_notification(notification) # read back the notification and check that it got changed notification = self.unsc.read_notification(notification_id) if notification.origin_list != ['Some_Resource_Agent_ID5']: self.fail("failed to change notification") @unittest.skip('interface has changed!') def test_send_notification_emails(self): # create user with email address in RR user_identty_object = IonObject(RT.ActorIdentity, name="user1") user_id = self.imc.create_actor_identity(user_identty_object) user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}}) self.imc.create_user_info(user_id, user_info_object) # create first notification notification_object = IonObject(RT.NotificationRequest, {"name":"notification1", "origin_list":['Some_Resource_Agent_ID1'], "events_list":['ResourceLifecycleEvent']}) self.unsc.create_notification(notification_object, user_id) # create second notification notification_object = IonObject(RT.NotificationRequest, {"name":"notification2", "origin_list":['Some_Resource_Agent_ID2'], "events_list":['DataEvent']}) self.unsc.create_notification(notification_object, user_id) # publish an event for each notification to generate the emails # this can't be easily check in SW so need to check for these at the [email protected] account rle_publisher = EventPublisher("ResourceLifecycleEvent") rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event") de_publisher = EventPublisher("DataEvent") de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event") gevent.sleep(1) @unittest.skip('interface has changed!') def test_find_events(self): # publish some events for the event repository rle_publisher = EventPublisher("ResourceLifecycleEvent") de_publisher = EventPublisher("DataEvent") rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event1") rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event2") rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event3") de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event1") de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event2") de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event3") # find all events for the originator 'Some_Resource_Agent_ID1' events = self.unsc.find_events(origin='Some_Resource_Agent_ID1') if len(events) != 3: self.fail("failed to find all events") for event in events: log.debug("event=" + str(event)) if event[1][0] != 'Some_Resource_Agent_ID1': self.fail("failed to find correct events") # find all events for the originator 'DataEvent' events = self.unsc.find_events(type='DataEvent') if len(events) != 3: self.fail("failed to find all events") for event in events: log.debug("event=" + str(event)) if event[1][0] != 'DataEvent': self.fail("failed to find correct events") # find 2 events for the originator 'Some_Resource_Agent_ID1' events = self.unsc.find_events(origin='Some_Resource_Agent_ID2', limit=2) if len(events) != 2: self.fail("failed to find all events") for event in events: log.debug("event=" + str(event)) if event[1][0] != 'Some_Resource_Agent_ID2': self.fail("failed to find correct events") # find all events for the originator 'Some_Resource_Agent_ID1' in reverse time order events = self.unsc.find_events(origin='Some_Resource_Agent_ID1', descending=True) if len(events) != 3: self.fail("failed to find all events") for event in events: log.debug("event=" + str(event)) if event[1][0] != 'Some_Resource_Agent_ID1': self.fail("failed to find correct events")
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) print 'started services' # @unittest.skip('this test just for debugging setup') # def test_just_the_setup(self): # return @attr('EXT') def test_resources_associations_extensions(self): """ create one of each resource and association used by IMS to guard against problems in ion-definitions """ #stuff we control # instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance)) instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent)) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance)) platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent)) platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice)) sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel)) #stuff we associate to data_producer_id, _ = self.RR.create(any_old(RT.DataProducer)) #instrument_agent_instance_id #is only a target #instrument_agent self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id) # self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) # self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id) self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id) self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id) instrument_model_id #is only a target platform_agent_instance_id #is only a target #platform_agent self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id) self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) platform_model_id #is only a target #sensor_device self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id) sensor_model_id #is only a target def addInstOwner(inst_id, subject): actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject}) user_id = self.IDS.create_actor_identity(actor_identity_obj) user_info_obj = any_old(RT.UserInfo) user_info_id = self.IDS.create_user_info(user_id, user_info_obj) self.RR.create_association(inst_id, PRED.hasOwner, user_id) #Testing multiple instrument owners addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254") addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256") #testing data products dp_id, _ = self.RR.create(any_old(RT.DataProduct)) self.RR.create_association(instrument_device_id, PRED.hasOutputProduct, dp_id) extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id) self.assertEqual(instrument_device_id, extended_instrument._id) self.assertEqual(len(extended_instrument.owners), 2) self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id) #check data products self.assertEqual(1, len(extended_instrument.data_products)) #check model inst_model_obj = self.RR.read(instrument_model_id) self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name) #check agent inst_agent_obj = self.RR.read(instrument_agent_id) self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent.name) #check platform device plat_device_obj = self.RR.read(platform_device_id) self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name) #check sensor devices self.assertEqual(1, len(extended_instrument.sensor_devices)) self.assertEqual("1.1", extended_instrument.computed.firmware_version) self.assertEqual("42", extended_instrument.computed.last_data_received_time) self.assertEqual("23", extended_instrument.computed.operational_state) self.assertEqual("34", extended_instrument.computed.last_command_status) self.assertEqual("45", extended_instrument.computed.last_command_date) self.assertEqual("56", extended_instrument.computed.last_command) self.assertEqual("67", extended_instrument.computed.last_commanded_by) self.assertEqual("78", extended_instrument.computed.power_status_roll_up) self.assertEqual("89", extended_instrument.computed.communications_status_roll_up) self.assertEqual("98", extended_instrument.computed.data_status_roll_up) self.assertEqual("87", extended_instrument.computed.location_status_roll_up) self.assertEqual(['mon', 'tue', 'wed'], extended_instrument.computed.recent_events)
class TestIdentityManagementServiceInt(IonIntegrationTestCase): def setUp(self): self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254" # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2coi.yml') self.identity_management_service = IdentityManagementServiceClient( node=self.container.node) def test_actor_identity(self): actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) user_id = self.identity_management_service.create_actor_identity( actor_identity_obj) actor_identity = self.identity_management_service.read_actor_identity( user_id) actor_identity.name = 'Updated subject' self.identity_management_service.update_actor_identity(actor_identity) self.identity_management_service.delete_actor_identity(user_id) with self.assertRaises(NotFound) as cm: self.identity_management_service.read_actor_identity(user_id) self.assertTrue("does not exist" in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.delete_actor_identity(user_id) self.assertTrue("does not exist" in cm.exception.message) def test_user_credentials(self): actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) user_id = self.identity_management_service.create_actor_identity( actor_identity_obj) user_credentials_obj = IonObject("UserCredentials", {"name": self.subject}) self.identity_management_service.register_user_credentials( user_id, user_credentials_obj) with self.assertRaises(NotFound) as cm: self.identity_management_service.unregister_user_credentials( "bad", self.subject) self.assertTrue("does not exist" in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.unregister_user_credentials( user_id, "bad") self.assertTrue("does not exist" in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.unregister_user_credentials( 'bad', 'bad') self.assertTrue("does not exist" in cm.exception.message) self.identity_management_service.unregister_user_credentials( user_id, self.subject) self.identity_management_service.delete_actor_identity(user_id) def test_user_info(self): actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) user_id = self.identity_management_service.create_actor_identity( actor_identity_obj) user_credentials_obj = IonObject("UserCredentials", {"name": self.subject}) self.identity_management_service.register_user_credentials( user_id, user_credentials_obj) user_info_obj = IonObject("UserInfo", {"name": "Foo"}) user_info = self.identity_management_service.create_user_info( user_id, user_info_obj) with self.assertRaises(Conflict) as cm: self.identity_management_service.create_user_info( user_id, user_info_obj) self.assertTrue( "UserInfo already exists for user id" in cm.exception.message) user_info_obj = self.identity_management_service.find_user_info_by_id( user_id) user_info_obj = self.identity_management_service.find_user_info_by_name( "Foo") user_info_obj = self.identity_management_service.find_user_info_by_subject( self.subject) user_info_obj = self.identity_management_service.read_user_info( user_info) user_info_obj.name = 'Jane Doe' self.identity_management_service.update_user_info(user_info_obj) self.identity_management_service.delete_user_info(user_info) with self.assertRaises(NotFound) as cm: self.identity_management_service.read_user_info(user_info) self.assertTrue('does not exist' in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.delete_user_info(user_info) self.assertTrue('does not exist' in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.find_user_info_by_name("John Doe") self.assertEqual(cm.exception.message, 'UserInfo with name John Doe does not exist') with self.assertRaises(NotFound) as cm: self.identity_management_service.find_user_info_by_subject( "Bogus subject") self.assertEqual( cm.exception.message, "UserCredentials with subject Bogus subject does not exist") self.identity_management_service.unregister_user_credentials( user_id, self.subject) self.identity_management_service.delete_actor_identity(user_id) def test_signon(self): certificate = """-----BEGIN CERTIFICATE----- MIIEMzCCAxugAwIBAgICBQAwDQYJKoZIhvcNAQEFBQAwajETMBEGCgmSJomT8ixkARkWA29yZzEX MBUGCgmSJomT8ixkARkWB2NpbG9nb24xCzAJBgNVBAYTAlVTMRAwDgYDVQQKEwdDSUxvZ29uMRsw GQYDVQQDExJDSUxvZ29uIEJhc2ljIENBIDEwHhcNMTAxMTE4MjIyNTA2WhcNMTAxMTE5MTAzMDA2 WjBvMRMwEQYKCZImiZPyLGQBGRMDb3JnMRcwFQYKCZImiZPyLGQBGRMHY2lsb2dvbjELMAkGA1UE BhMCVVMxFzAVBgNVBAoTDlByb3RlY3ROZXR3b3JrMRkwFwYDVQQDExBSb2dlciBVbndpbiBBMjU0 MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6QhsWxhUXbIxg+1ZyEc7d+hIGvchVmtb g0kKLmivgoVsA4U7swNDRH6svW242THta0oTf6crkRx7kOKg6jma2lcAC1sjOSddqX7/92ChoUPq 7LWt2T6GVVA10ex5WAeB/o7br/Z4U8/75uCBis+ru7xEDl09PToK20mrkcz9M4HqIv1eSoPkrs3b 2lUtQc6cjuHRDU4NknXaVMXTBHKPM40UxEDHJueFyCiZJFg3lvQuSsAl4JL5Z8pC02T8/bODBuf4 dszsqn2SC8YDw1xrujvW2Bd7Q7BwMQ/gO+dZKM1mLJFpfEsR9WrjMeg6vkD2TMWLMr0/WIkGC8u+ 6M6SMQIDAQABo4HdMIHaMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgSwMBMGA1UdJQQMMAoG CCsGAQUFBwMCMBgGA1UdIAQRMA8wDQYLKwYBBAGCkTYBAgEwagYDVR0fBGMwYTAuoCygKoYoaHR0 cDovL2NybC5jaWxvZ29uLm9yZy9jaWxvZ29uLWJhc2ljLmNybDAvoC2gK4YpaHR0cDovL2NybC5k b2Vncmlkcy5vcmcvY2lsb2dvbi1iYXNpYy5jcmwwHwYDVR0RBBgwFoEUaXRzYWdyZWVuMUB5YWhv by5jb20wDQYJKoZIhvcNAQEFBQADggEBAEYHQPMY9Grs19MHxUzMwXp1GzCKhGpgyVKJKW86PJlr HGruoWvx+DLNX75Oj5FC4t8bOUQVQusZGeGSEGegzzfIeOI/jWP1UtIjzvTFDq3tQMNvsgROSCx5 CkpK4nS0kbwLux+zI7BWON97UpMIzEeE05pd7SmNAETuWRsHMP+x6i7hoUp/uad4DwbzNUGIotdK f8b270icOVgkOKRdLP/Q4r/x8skKSCRz1ZsRdR+7+B/EgksAJj7Ut3yiWoUekEMxCaTdAHPTMD/g Mh9xL90hfMJyoGemjJswG5g3fAdTP/Lv0I6/nWeH/cLjwwpQgIEjEAVXl7KHuzX5vPD/wqQ= -----END CERTIFICATE-----""" id, valid_until, registered = self.identity_management_service.signon( certificate, True) self.assertFalse(registered) id2, valid_until2, registered2 = self.identity_management_service.signon( certificate, True) self.assertFalse(registered2) self.assertTrue(id == id2) self.assertTrue(valid_until == valid_until2) user_info_obj = IonObject("UserInfo", {"name": "Foo"}) self.identity_management_service.create_user_info(id, user_info_obj) id3, valid_until3, registered3 = self.identity_management_service.signon( certificate, True) self.assertTrue(registered3) self.assertTrue(id == id3) self.assertTrue(valid_until == valid_until3)
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) print 'started services' # @unittest.skip('this test just for debugging setup') # def test_just_the_setup(self): # return @attr('EXT') def test_resources_associations_extensions(self): """ create one of each resource and association used by IMS to guard against problems in ion-definitions """ #stuff we control instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance)) instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent)) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance)) platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent)) platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice)) sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel)) #stuff we associate to data_producer_id, _ = self.RR.create(any_old(RT.DataProducer)) org_id, _ = self.RR.create(any_old(RT.Org)) #instrument_agent_instance_id #is only a target #instrument_agent self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id) self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id) self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device_id) instrument_model_id #is only a target platform_agent_instance_id #is only a target #platform_agent self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id) self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) platform_model_id #is only a target #sensor_device self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id) sensor_model_id #is only a target #create a parsed product for this instrument output tdom, sdom = time_series_domain() tdom = tdom.dump() sdom = sdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', processing_level_code='Parsed_Canonical', temporal_domain = tdom, spatial_domain = sdom) pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1) def addInstOwner(inst_id, subject): actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject}) user_id = self.IDS.create_actor_identity(actor_identity_obj) user_info_obj = any_old(RT.UserInfo) user_info_id = self.IDS.create_user_info(user_id, user_info_obj) self.RR.create_association(inst_id, PRED.hasOwner, user_id) #Testing multiple instrument owners addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254") addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256") extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id) self.assertEqual(instrument_device_id, extended_instrument._id) self.assertEqual(len(extended_instrument.owners), 2) self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id) #check model inst_model_obj = self.RR.read(instrument_model_id) self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name) #check agent instance inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id) self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name) #check agent inst_agent_obj = self.RR.read(instrument_agent_id) #compound assoc return list of lists so check the first element self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent[0].name) #check platform device plat_device_obj = self.RR.read(platform_device_id) self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name) #check sensor devices self.assertEqual(1, len(extended_instrument.sensor_devices)) #check data_product_parameters_set self.assertEqual(ComputedValueAvailability.PROVIDED, extended_instrument.computed.data_product_parameters_set.status) self.assertTrue( 'Parsed_Canonical' in extended_instrument.computed.data_product_parameters_set.value) # the ctd parameters should include 'temp' self.assertTrue( 'temp' in extended_instrument.computed.data_product_parameters_set.value['Parsed_Canonical']) #none of these will work because there is no agent self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, extended_instrument.computed.firmware_version.status) self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, extended_instrument.computed.operational_state.status) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_instrument.computed.power_status_roll_up.status) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_instrument.computed.communications_status_roll_up.status) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_instrument.computed.data_status_roll_up.status) self.assertEqual(StatusType.STATUS_OK, extended_instrument.computed.data_status_roll_up.value) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_instrument.computed.location_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.recent_events.status) # self.assertEqual([], extended_instrument.computed.recent_events.value) # cleanup c = DotDict() c.resource_registry = self.RR resource_impl = ResourceImpl(c) resource_impl.pluck(instrument_agent_id) resource_impl.pluck(instrument_model_id) resource_impl.pluck(instrument_device_id) resource_impl.pluck(platform_agent_id) self.IMS.force_delete_instrument_agent(instrument_agent_id) self.IMS.force_delete_instrument_model(instrument_model_id) self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id) self.IMS.force_delete_platform_agent(platform_agent_id) self.IMS.force_delete_platform_device(platform_device_id) self.IMS.force_delete_platform_model(platform_model_id) self.IMS.force_delete_sensor_device(sensor_device_id) self.IMS.force_delete_sensor_model(sensor_model_id) #stuff we associate to self.RR.delete(data_producer_id) self.RR.delete(org_id) def test_custom_attributes(self): """ Test assignment of custom attributes """ instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel, {"custom_attributes": {"favorite_color": "attr desc goes here"} })) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice, {"custom_attributes": {"favorite_color": "red", "bogus_attr": "should raise warning" } })) self.IMS.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id) # cleanup self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_instrument_model(instrument_model_id) def _get_datastore(self, dataset_id): dataset = self.DSC.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def test_checkpoint_restore(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", stream_configuration= {'raw': 'ctd_raw_param_dict' , 'parsed': 'ctd_parsed_param_dict' }) instModel_id = self.IMS.create_instrument_model(instModel_obj) log.debug( 'new InstrumentModel id = %s ', instModel_id) # Create InstrumentAgent instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver" ) instAgent_id = self.IMS.create_instrument_agent(instAgent_obj) log.debug( 'new InstrumentAgent id = %s', instAgent_id) self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 ' + '(SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj) self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': 'sbe37-simulator.oceanobservatories.org', 'device_port': 4001, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'command_port': 4002, 'data_port': 4003, 'log_level': 5, } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_config = port_agent_config) instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict') parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary=spdict_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict') raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary=rpdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id, parameter_dictionary=spdict_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug( 'Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] #create the datastore at the beginning of each int test that persists data self._get_datastore(self.parsed_dataset) self.DP.activate_data_product_persistence(data_product_id=data_product_id1) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id, parameter_dictionary=rpdict_id) log.debug( 'new dp_id = %s', str(data_product_id2)) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.DP.activate_data_product_persistence(data_product_id=data_product_id2) # spin up agent self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.IMS.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id) gate = ProcessStateGate(self.PDC.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id) # take snapshot of config snap_id = self.IMS.agent_state_checkpoint(instDevice_id, "xyzzy snapshot") snap_obj = self.RR.read_attachment(snap_id, include_content=True) print "Saved config:" print snap_obj.content #modify config instance_obj.driver_config["comms_config"] = "BAD_DATA" self.RR.update(instance_obj) #restore config self.IMS.agent_state_restore(instDevice_id, snap_id) instance_obj = self.RR.read(instAgentInstance_id) self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"])
def test_user_role_cache(self): #Create a user id_client = IdentityManagementServiceClient(node=self.container.node) actor_id, valid_until, registered = id_client.signon( USER1_CERTIFICATE, True) #Make a request with this new user to get it into the cache response = self.test_app.get( '/ion-service/resource_registry/find_resources?name=TestDataProduct&id_only=True&requester=' + actor_id) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json['data']) #Check the contents of the user role cache for this user service_gateway_user_cache = self.container.proc_manager.procs_by_name[ 'service_gateway'].user_data_cache self.assertEqual(service_gateway_user_cache.has_key(actor_id), True) role_header = service_gateway_user_cache.get(actor_id) self.assertIn('ION', role_header) self.assertEqual(len(role_header['ION']), 1) self.assertIn('ORG_MEMBER', role_header['ION']) org_client = OrgManagementServiceClient(node=self.container.node) ion_org = org_client.find_org() manager_role = org_client.find_org_role_by_name( org_id=ion_org._id, role_name='ORG_MANAGER') org_client.grant_role(org_id=ion_org._id, user_id=actor_id, role_name='ORG_MANAGER') #Just allow some time for event processing on slower platforms gevent.sleep(2) #The user should be evicted from the cache due to a change in roles self.assertEqual(service_gateway_user_cache.has_key(actor_id), False) #Do it again to check for new roles response = self.test_app.get( '/ion-service/resource_registry/find_resources?name=TestDataProduct&id_only=True&requester=' + actor_id) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json['data']) #Check the contents of the user role cache for this user self.assertEqual(service_gateway_user_cache.has_key(actor_id), True) role_header = service_gateway_user_cache.get(actor_id) self.assertIn('ION', role_header) self.assertEqual(len(role_header['ION']), 2) self.assertIn('ORG_MEMBER', role_header['ION']) self.assertIn('ORG_MANAGER', role_header['ION']) #Now flush the user_role_cache and make sure it was flushed event_publisher = EventPublisher() event_publisher.publish_event(event_type=OT.UserRoleCacheResetEvent) #Just allow some time for event processing on slower platforms gevent.sleep(2) self.assertEqual(service_gateway_user_cache.has_key(actor_id), False) self.assertEqual(service_gateway_user_cache.size(), 0) #Change the role once again and see if it is there again org_client.revoke_role(org_id=ion_org._id, user_id=actor_id, role_name='ORG_MANAGER') #Just allow some time for event processing on slower platforms gevent.sleep(2) #The user should still not be there self.assertEqual(service_gateway_user_cache.has_key(actor_id), False) #Do it again to check for new roles response = self.test_app.get( '/ion-service/resource_registry/find_resources?name=TestDataProduct&id_only=True&requester=' + actor_id) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json['data']) #Check the contents of the user role cache for this user self.assertEqual(service_gateway_user_cache.has_key(actor_id), True) role_header = service_gateway_user_cache.get(actor_id) self.assertIn('ION', role_header) self.assertEqual(len(role_header['ION']), 1) self.assertIn('ORG_MEMBER', role_header['ION']) id_client.delete_actor_identity(actor_id)
def test_user_role_cache(self): # Create a user id_client = IdentityManagementServiceClient() actor_id, valid_until, registered = id_client.signon(USER1_CERTIFICATE, True) # Make a request with this new user to get it into the cache response = self.test_app.get( "/ion-service/resource_registry/find_resources?name=TestDataProduct&id_only=True&requester=" + actor_id ) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json["data"]) # Check the contents of the user role cache for this user service_gateway_user_role_cache = self.container.proc_manager.procs_by_name["service_gateway"].user_role_cache self.assertEqual(service_gateway_user_role_cache.has_key(actor_id), True) role_header = service_gateway_user_role_cache.get(actor_id) self.assertIn("ION", role_header) self.assertEqual(len(role_header["ION"]), 1) self.assertIn("ORG_MEMBER", role_header["ION"]) org_client = OrgManagementServiceClient() ion_org = org_client.find_org() manager_role = org_client.find_org_role_by_name(org_id=ion_org._id, role_name="ORG_MANAGER") org_client.grant_role(org_id=ion_org._id, actor_id=actor_id, role_name="ORG_MANAGER") # Just allow some time for event processing on slower platforms gevent.sleep(2) # The user should be evicted from the cache due to a change in roles self.assertEqual(service_gateway_user_role_cache.has_key(actor_id), False) # Do it again to check for new roles response = self.test_app.get( "/ion-service/resource_registry/find_resources?name=TestDataProduct&id_only=True&requester=" + actor_id ) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json["data"]) # Check the contents of the user role cache for this user self.assertEqual(service_gateway_user_role_cache.has_key(actor_id), True) role_header = service_gateway_user_role_cache.get(actor_id) self.assertIn("ION", role_header) self.assertEqual(len(role_header["ION"]), 2) self.assertIn("ORG_MEMBER", role_header["ION"]) self.assertIn("ORG_MANAGER", role_header["ION"]) # Now flush the user_role_cache and make sure it was flushed event_publisher = EventPublisher() event_publisher.publish_event(event_type=OT.UserRoleCacheResetEvent) # Just allow some time for event processing on slower platforms gevent.sleep(2) self.assertEqual(service_gateway_user_role_cache.has_key(actor_id), False) self.assertEqual(service_gateway_user_role_cache.size(), 0) # Change the role once again and see if it is there again org_client.revoke_role(org_id=ion_org._id, actor_id=actor_id, role_name="ORG_MANAGER") # Just allow some time for event processing on slower platforms gevent.sleep(2) # The user should still not be there self.assertEqual(service_gateway_user_role_cache.has_key(actor_id), False) # Do it again to check for new roles response = self.test_app.get( "/ion-service/resource_registry/find_resources?name=TestDataProduct&id_only=True&requester=" + actor_id ) self.check_response_headers(response) self.assertIn(GATEWAY_RESPONSE, response.json["data"]) # Check the contents of the user role cache for this user self.assertEqual(service_gateway_user_role_cache.has_key(actor_id), True) role_header = service_gateway_user_role_cache.get(actor_id) self.assertIn("ION", role_header) self.assertEqual(len(role_header["ION"]), 1) self.assertIn("ORG_MEMBER", role_header["ION"]) id_client.delete_actor_identity(actor_id)
class UserNotificationIntTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2dm.yml') self.unsc = UserNotificationServiceClient(node=self.container.node) self.rrc = ResourceRegistryServiceClient(node=self.container.node) self.imc = IdentityManagementServiceClient(node=self.container.node) def test_find_event_types_for_resource(self): # create a dataset object in the RR to pass into the UNS method dataset_object = IonObject(RT.DataSet, name="dataset1") dataset_id, version = self.rrc.create(dataset_object) # get the list of event types for the dataset events = self.unsc.find_event_types_for_resource(dataset_id) log.debug("dataset events = " + str(events)) if not events == ['dataset_supplement_added', 'dataset_change']: self.fail("failed to return correct list of event types") # try to pass in an id of a resource that doesn't exist (should fail) try: events = self.unsc.find_event_types_for_resource("bogus_id") self.fail("failed to detect non-existant resource") except: pass def test_create_two_user_notifications(self): # create user with email address in RR user_identty_object = IonObject(RT.ActorIdentity, name="user1") user_id = self.imc.create_actor_identity(user_identty_object) user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}}) self.imc.create_user_info(user_id, user_info_object) # create first notification notification_object1 = IonObject(RT.NotificationRequest, {"name":"notification1", "origin_list":['Some_Resource_Agent_ID1'], "events_list":['ResourceLifecycleEvent']}) notification_id1 = self.unsc.create_notification(notification_object1, user_id) # create second notification notification_object2 = IonObject(RT.NotificationRequest, {"name":"notification2", "origin_list":['Some_Resource_Agent_ID2'], "events_list":['DataEvent']}) notification_id2 = self.unsc.create_notification(notification_object2, user_id) # read the notifications back and check that they are correct n1 = self.unsc.read_notification(notification_id1) if n1.name != notification_object1.name or \ n1.origin_list != notification_object1.origin_list or \ n1.events_list != notification_object1.events_list: self.fail("notification was not correct") n2 = self.unsc.read_notification(notification_id2) if n2.name != notification_object2.name or \ n2.origin_list != notification_object2.origin_list or \ n2.events_list != notification_object2.events_list: self.fail("notification was not correct") def test_delete_user_notifications(self): # create user with email address in RR user_identty_object = IonObject(RT.ActorIdentity, name="user1") user_id = self.imc.create_actor_identity(user_identty_object) user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}}) self.imc.create_user_info(user_id, user_info_object) # create first notification notification_object1 = IonObject(RT.NotificationRequest, {"name":"notification1", "origin_list":['Some_Resource_Agent_ID1'], "events_list":['ResourceLifecycleEvent']}) notification1_id = self.unsc.create_notification(notification_object1, user_id) # create second notification notification_object2 = IonObject(RT.NotificationRequest, {"name":"notification2", "origin_list":['Some_Resource_Agent_ID2'], "events_list":['DataEvent']}) notification2_id = self.unsc.create_notification(notification_object2, user_id) # delete both notifications self.unsc.delete_notification(notification1_id) self.unsc.delete_notification(notification2_id) # check that the notifications are not there try: n1 = self.unsc.read_notification(notification1_id) except: try: n2 = self.unsc.read_notification(notification2_id) except: return self.fail("failed to delete notifications") def test_find_user_notifications(self): # create user with email address in RR user_identty_object = IonObject(RT.ActorIdentity, name="user1") user_id = self.imc.create_actor_identity(user_identty_object) user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}}) self.imc.create_user_info(user_id, user_info_object) # create first notification notification_object = IonObject(RT.NotificationRequest, {"name":"notification1", "origin_list":['Some_Resource_Agent_ID1'], "events_list":['ResourceLifecycleEvent']}) self.unsc.create_notification(notification_object, user_id) # create second notification notification_object = IonObject(RT.NotificationRequest, {"name":"notification2", "origin_list":['Some_Resource_Agent_ID2'], "events_list":['DataEvent']}) self.unsc.create_notification(notification_object, user_id) # try to find all notifications for user notifications = self.unsc.find_notifications_by_user(user_id) if len(notifications) != 2: self.fail("failed to find all notifications") def test_update_user_notification(self): # create user with email address in RR user_identty_object = IonObject(RT.ActorIdentity, name="user1") user_id = self.imc.create_actor_identity(user_identty_object) user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}}) self.imc.create_user_info(user_id, user_info_object) # create a notification notification_object = IonObject(RT.NotificationRequest, {"name":"notification1", "origin_list":['Some_Resource_Agent_ID1'], "events_list":['ResourceLifecycleEvent']}) notification_id = self.unsc.create_notification(notification_object, user_id) # read back the notification and change it notification = self.unsc.read_notification(notification_id) notification.origin_list = ['Some_Resource_Agent_ID5'] self.unsc.update_notification(notification) # read back the notification and check that it got changed notification = self.unsc.read_notification(notification_id) if notification.origin_list != ['Some_Resource_Agent_ID5']: self.fail("failed to change notification") def test_send_notification_emails(self): # create user with email address in RR user_identty_object = IonObject(RT.ActorIdentity, name="user1") user_id = self.imc.create_actor_identity(user_identty_object) user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}}) self.imc.create_user_info(user_id, user_info_object) # create first notification notification_object = IonObject(RT.NotificationRequest, {"name":"notification1", "origin_list":['Some_Resource_Agent_ID1'], "events_list":['ResourceLifecycleEvent']}) self.unsc.create_notification(notification_object, user_id) # create second notification notification_object = IonObject(RT.NotificationRequest, {"name":"notification2", "origin_list":['Some_Resource_Agent_ID2'], "events_list":['DataEvent']}) self.unsc.create_notification(notification_object, user_id) # publish an event for each notification to generate the emails # this can't be easily check in SW so need to check for these at the [email protected] account rle_publisher = EventPublisher("ResourceLifecycleEvent") rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event") de_publisher = EventPublisher("DataEvent") de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event") gevent.sleep(1) def test_find_events(self): # publish some events for the event repository rle_publisher = EventPublisher("ResourceLifecycleEvent") de_publisher = EventPublisher("DataEvent") rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event1") rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event2") rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event3") de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event1") de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event2") de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event3") # find all events for the originator 'Some_Resource_Agent_ID1' events = self.unsc.find_events(origin='Some_Resource_Agent_ID1') if len(events) != 3: self.fail("failed to find all events") for event in events: log.debug("event=" + str(event)) if event[1][0] != 'Some_Resource_Agent_ID1': self.fail("failed to find correct events") # find all events for the originator 'DataEvent' events = self.unsc.find_events(type='DataEvent') if len(events) != 3: self.fail("failed to find all events") for event in events: log.debug("event=" + str(event)) if event[1][0] != 'DataEvent': self.fail("failed to find correct events") # find 2 events for the originator 'Some_Resource_Agent_ID1' events = self.unsc.find_events(origin='Some_Resource_Agent_ID2', limit=2) if len(events) != 2: self.fail("failed to find all events") for event in events: log.debug("event=" + str(event)) if event[1][0] != 'Some_Resource_Agent_ID2': self.fail("failed to find correct events") # find all events for the originator 'Some_Resource_Agent_ID1' in reverse time order events = self.unsc.find_events(origin='Some_Resource_Agent_ID1', descending=True) if len(events) != 3: self.fail("failed to find all events") for event in events: log.debug("event=" + str(event)) if event[1][0] != 'Some_Resource_Agent_ID1': self.fail("failed to find correct events")
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.dpsc_cli = DataProductManagementServiceClient() self.rrclient = ResourceRegistryServiceClient() self.damsclient = DataAcquisitionManagementServiceClient() self.pubsubcli = PubsubManagementServiceClient() self.ingestclient = IngestionManagementServiceClient() self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() self.unsc = UserNotificationServiceClient() self.data_retriever = DataRetrieverServiceClient() self.identcli = IdentityManagementServiceClient() #------------------------------------------ # Create the environment #------------------------------------------ self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM') self.process_definitions = {} ingestion_worker_definition = ProcessDefinition(name='ingestion worker') ingestion_worker_definition.executable = { 'module':'ion.processes.data.ingestion.science_granule_ingestion_worker', 'class' :'ScienceGranuleIngestionWorker' } process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition) self.process_definitions['ingestion_worker'] = process_definition_id self.pids = [] self.exchange_points = [] self.exchange_names = [] #------------------------------------------------------------------------------------------------ # First launch the ingestors #------------------------------------------------------------------------------------------------ self.exchange_space = 'science_granule_ingestion' self.exchange_point = 'science_data' config = DotDict() config.process.datastore_name = 'datasets' config.process.queue_name = self.exchange_space self.exchange_names.append(self.exchange_space) self.exchange_points.append(self.exchange_point) pid = self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config) log.debug("the ingestion worker process id: %s", pid) self.pids.append(pid) self.addCleanup(self.cleaning_up) def cleaning_up(self): for pid in self.pids: log.debug("number of pids to be terminated: %s", len(self.pids)) try: self.process_dispatcher.cancel_process(pid) log.debug("Terminated the process: %s", pid) except: log.debug("could not terminate the process id: %s" % pid) IngestionManagementIntTest.clean_subscriptions() for xn in self.exchange_names: xni = self.container.ex_manager.create_xn_queue(xn) xni.delete() for xp in self.exchange_points: xpi = self.container.ex_manager.create_xp(xp) xpi.delete() def get_datastore(self, dataset_id): dataset = self.dataset_management.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore @attr('EXT') @attr('PREP') def test_create_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ parameter_dictionary = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict') ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary._id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp') dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0 dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0 dp_obj.ooi_product_name = "PRODNAME" #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product( data_product= dp_obj, stream_definition_id=ctd_stream_def_id) # Assert that the data product has an associated stream at this stage stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True) self.assertNotEquals(len(stream_ids), 0) # Assert that the data product has an associated stream def at this stage stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStreamDefinition, RT.StreamDefinition, True) self.assertNotEquals(len(stream_ids), 0) self.dpsc_cli.activate_data_product_persistence(dp_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertIsNotNone(dp_obj) self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0) log.debug('Created data product %s', dp_obj) #------------------------------------------------------------------------------------------------ # test creating a new data product with a stream definition #------------------------------------------------------------------------------------------------ log.debug('Creating new data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name='DP2', description='some new dp') dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id) self.dpsc_cli.activate_data_product_persistence(dp_id2) log.debug('new dp_id = %s' % dp_id2) #------------------------------------------------------------------------------------------------ #make sure data product is associated with stream def #------------------------------------------------------------------------------------------------ streamdefs = [] streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True) for s in streams: log.debug("Checking stream %s" % s) sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True) for sd in sdefs: log.debug("Checking streamdef %s" % sd) streamdefs.append(sd) self.assertIn(ctd_stream_def_id, streamdefs) group_names = self.dpsc_cli.get_data_product_group_list() self.assertIn("PRODNAME", group_names) #---------------------------------------------------------------------------------------- # Create users then notifications to this data product for each user #---------------------------------------------------------------------------------------- # user_1 user_1 = UserInfo() user_1.name = 'user_1' user_1.contact.email = '*****@*****.**' # user_2 user_2 = UserInfo() user_2.name = 'user_2' user_2.contact.email = '*****@*****.**' #user1 is a complete user self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254" actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) actor_id = self.identcli.create_actor_identity(actor_identity_obj) user_credentials_obj = IonObject("UserCredentials", {"name": self.subject}) self.identcli.register_user_credentials(actor_id, user_credentials_obj) user_id_1 = self.identcli.create_user_info(actor_id, user_1) user_id_2, _ = self.rrclient.create(user_2) delivery_config1a = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH) delivery_config1b = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH) notification_request_1 = NotificationRequest( name = "notification_1", origin=dp_id, origin_type="type_1", event_type=OT.ResourceLifecycleEvent, disabled_by_system = False, delivery_configurations=[delivery_config1a, delivery_config1b]) delivery_config2a = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH) delivery_config2b = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH) notification_request_2 = NotificationRequest( name = "notification_2", origin=dp_id, origin_type="type_2", disabled_by_system = False, event_type=OT.DetectionEvent, delivery_configurations=[delivery_config2a, delivery_config2b]) notification_request_1_id = self.unsc.create_notification(notification=notification_request_1, user_id=user_id_1) notification_request_2_id = self.unsc.create_notification(notification=notification_request_2, user_id=user_id_2) self.unsc.delete_notification(notification_request_1_id) # test reading a non-existent data product log.debug('reading non-existent data product') with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product('some_fake_id') # update a data product (tests read also) log.debug('Updating data product') # first get the existing dp object dp_obj = self.dpsc_cli.read_data_product(dp_id) # now tweak the object dp_obj.description = 'the very first dp' dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 20.0 dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -20.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 20.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -20.0 # now write the dp back to the registry update_result = self.dpsc_cli.update_data_product(dp_obj) # now get the dp back to see if it was updated dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertEquals(dp_obj.description,'the very first dp') self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0) log.debug('Updated data product %s', dp_obj) #test extension extended_product = self.dpsc_cli.get_data_product_extension(dp_id) #validate that there is one active and one retired user notification for this data product self.assertEqual(1, len(extended_product.computed.active_user_subscriptions.value)) self.assertEqual(1, len(extended_product.computed.past_user_subscriptions.value)) self.assertEqual(dp_id, extended_product._id) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.computed.product_download_size_estimated.status) self.assertEqual(0, extended_product.computed.product_download_size_estimated.value) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.computed.parameters.status) #log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value) def ion_object_encoder(obj): return obj.__dict__ #test prepare for create data_product_data = self.dpsc_cli.prepare_data_product_support() #print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2) self.assertEqual(data_product_data._id, "") self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport) self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 2) self.assertEqual(len(data_product_data.associations['Dataset'].resources), 0) self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 0) self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 0) #test prepare for update data_product_data = self.dpsc_cli.prepare_data_product_support(dp_id) #print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2) self.assertEqual(data_product_data._id, dp_id) self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport) self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 2) self.assertEqual(len(data_product_data.associations['Dataset'].resources), 1) self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 1) self.assertEqual(data_product_data.associations['StreamDefinition'].associated_resources[0].s, dp_id) self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 1) self.assertEqual(data_product_data.associations['Dataset'].associated_resources[0].s, dp_id) # now 'delete' the data product log.debug("deleting data product: %s" % dp_id) self.dpsc_cli.delete_data_product(dp_id) # Assert that there are no associated streams leftover after deleting the data product stream_ids, assoc_ids = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True) self.assertEquals(len(stream_ids), 0) self.assertEquals(len(assoc_ids), 0) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product(dp_id) # Get the events corresponding to the data product ret = self.unsc.get_recent_events(resource_id=dp_id) events = ret.value for event in events: log.debug("event time: %s" % event.ts_created) self.assertTrue(len(events) > 0) def test_data_product_stream_def(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp') dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj, stream_definition_id=ctd_stream_def_id) stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id) self.assertEquals(ctd_stream_def_id, stream_def_id) def test_derived_data_product(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='ctd parsed', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsubcli.delete_stream_definition, ctd_stream_def_id) dp = DataProduct(name='Instrument DP') dp_id = self.dpsc_cli.create_data_product(dp, stream_definition_id=ctd_stream_def_id) self.addCleanup(self.dpsc_cli.force_delete_data_product, dp_id) self.dpsc_cli.activate_data_product_persistence(dp_id) self.addCleanup(self.dpsc_cli.suspend_data_product_persistence, dp_id) dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True) if not dataset_ids: raise NotFound("Data Product %s dataset does not exist" % str(dp_id)) dataset_id = dataset_ids[0] # Make the derived data product simple_stream_def_id = self.pubsubcli.create_stream_definition(name='TEMPWAT stream def', parameter_dictionary_id=pdict_id, available_fields=['time','temp']) tempwat_dp = DataProduct(name='TEMPWAT', category=DataProductTypeEnum.DERIVED) tempwat_dp_id = self.dpsc_cli.create_data_product(tempwat_dp, stream_definition_id=simple_stream_def_id, parent_data_product_id=dp_id) self.addCleanup(self.dpsc_cli.delete_data_product, tempwat_dp_id) # Check that the streams associated with the data product are persisted with stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True) for stream_id in stream_ids: self.assertTrue(self.ingestclient.is_persisted(stream_id)) stream_id = stream_ids[0] route = self.pubsubcli.read_stream_route(stream_id=stream_id) rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id) rdt['time'] = np.arange(20) rdt['temp'] = np.arange(20) rdt['pressure'] = np.arange(20) publisher = StandaloneStreamPublisher(stream_id,route) dataset_modified = Event() def cb(*args, **kwargs): dataset_modified.set() es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True) es.start() self.addCleanup(es.stop) publisher.publish(rdt.to_granule()) self.assertTrue(dataset_modified.wait(30)) tempwat_dataset_ids, _ = self.rrclient.find_objects(tempwat_dp_id, PRED.hasDataset, id_only=True) tempwat_dataset_id = tempwat_dataset_ids[0] granule = self.data_retriever.retrieve(tempwat_dataset_id, delivery_format=simple_stream_def_id) rdt = RecordDictionaryTool.load_from_granule(granule) np.testing.assert_array_equal(rdt['time'], np.arange(20)) self.assertEquals(set(rdt.fields), set(['time','temp'])) def test_activate_suspend_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ # Construct temporal and spatial Coordinate Reference System objects dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp') log.debug("Created an IonObject for a data product: %s" % dp_obj) #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj, stream_definition_id=ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # Subscribe to persist events #------------------------------------------------------------------------------------------------ queue = gevent.queue.Queue() def info_event_received(message, headers): queue.put(message) es = EventSubscriber(event_type=OT.InformationContentStatusEvent, callback=info_event_received, origin=dp_id, auto_delete=True) es.start() self.addCleanup(es.stop) #------------------------------------------------------------------------------------------------ # test activate and suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.activate_data_product_persistence(dp_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertIsNotNone(dp_obj) dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True) if not dataset_ids: raise NotFound("Data Product %s dataset does not exist" % str(dp_id)) dataset_id = dataset_ids[0] # Check that the streams associated with the data product are persisted with stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True) for stream_id in stream_ids: self.assertTrue(self.ingestclient.is_persisted(stream_id)) stream_id = stream_ids[0] route = self.pubsubcli.read_stream_route(stream_id=stream_id) rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id) rdt['time'] = np.arange(20) rdt['temp'] = np.arange(20) publisher = StandaloneStreamPublisher(stream_id,route) dataset_modified = Event() def cb(*args, **kwargs): dataset_modified.set() es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True) es.start() self.addCleanup(es.stop) publisher.publish(rdt.to_granule()) self.assertTrue(dataset_modified.wait(30)) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.data_retriever.retrieve(dataset_ids[0]) self.assertIsInstance(replay_data, Granule) log.debug("The data retriever was able to replay the dataset that was attached to the data product " "we wanted to be persisted. Therefore the data product was indeed persisted with " "otherwise we could not have retrieved its dataset using the data retriever. Therefore " "this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'") data_product_object = self.rrclient.read(dp_id) self.assertEquals(data_product_object.name,'DP1') self.assertEquals(data_product_object.description,'some new dp') log.debug("Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. " " Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the " "resource registry, name='%s', desc='%s'" % (dp_obj.name, dp_obj.description,data_product_object.name, data_product_object.description)) #------------------------------------------------------------------------------------------------ # test suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.suspend_data_product_persistence(dp_id) dataset_modified.clear() rdt['time'] = np.arange(20,40) publisher.publish(rdt.to_granule()) self.assertFalse(dataset_modified.wait(2)) self.dpsc_cli.activate_data_product_persistence(dp_id) dataset_modified.clear() publisher.publish(rdt.to_granule()) self.assertTrue(dataset_modified.wait(30)) granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(granule) np.testing.assert_array_almost_equal(rdt['time'], np.arange(40)) dataset_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasDataset, id_only=True) self.assertEquals(len(dataset_ids), 1) self.dpsc_cli.suspend_data_product_persistence(dp_id) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object with self.assertRaises(NotFound): dp_obj = self.rrclient.read(dp_id) info_event_counter = 0 runtime = 0 starttime = time.time() caught_events = [] #check that the four InfoStatusEvents were received while info_event_counter < 4 and runtime < 60 : a = queue.get(timeout=60) caught_events.append(a) info_event_counter += 1 runtime = time.time() - starttime self.assertEquals(info_event_counter, 4)
class UserNotificationIntTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2dm.yml') self.unsc = UserNotificationServiceClient(node=self.container.node) self.rrc = ResourceRegistryServiceClient(node=self.container.node) self.imc = IdentityManagementServiceClient(node=self.container.node) def xtest_find_event_types_for_resource(self): dataset_object = IonObject(RT.DataSet, name="dataset1") dataset_id, version = self.rrc.create(dataset_object) events = self.unsc.find_event_types_for_resource(dataset_id) log.debug("dataset events = " + str(events)) try: events = self.unsc.find_event_types_for_resource("bogus_id") self.fail("failed to detect non-existant resource") except: pass def test_create_two_user_notifications(self): user_identty_object = IonObject(RT.UserIdentity, name="user1") user_id = self.imc.create_user_identity(user_identty_object) user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}}) self.imc.create_user_info(user_id, user_info_object) notification_object = IonObject(RT.NotificationRequest, {"name":"notification1", "origin_list":['Some_Resource_Agent_ID1'], "events_list":['resource_lifecycle']}) self.unsc.create_notification(notification_object, user_id) notification_object = IonObject(RT.NotificationRequest, {"name":"notification2", "origin_list":['Some_Resource_Agent_ID2'], "events_list":['data']}) self.unsc.create_notification(notification_object, user_id) def test_delete_user_notifications(self): user_identty_object = IonObject(RT.UserIdentity, name="user1") user_id = self.imc.create_user_identity(user_identty_object) user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}}) self.imc.create_user_info(user_id, user_info_object) notification_object1 = IonObject(RT.NotificationRequest, {"name":"notification1", "origin_list":['Some_Resource_Agent_ID1'], "events_list":['resource_lifecycle']}) notification1_id = self.unsc.create_notification(notification_object1, user_id) notification_object2 = IonObject(RT.NotificationRequest, {"name":"notification2", "origin_list":['Some_Resource_Agent_ID2'], "events_list":['data']}) notification2_id = self.unsc.create_notification(notification_object2, user_id) self.unsc.delete_notification(notification1_id) self.unsc.delete_notification(notification2_id) def test_find_user_notifications(self): user_identty_object = IonObject(RT.UserIdentity, name="user1") user_id = self.imc.create_user_identity(user_identty_object) user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}}) self.imc.create_user_info(user_id, user_info_object) notification_object = IonObject(RT.NotificationRequest, {"name":"notification1", "origin_list":['Some_Resource_Agent_ID1'], "events_list":['resource_lifecycle']}) self.unsc.create_notification(notification_object, user_id) notification_object = IonObject(RT.NotificationRequest, {"name":"notification2", "origin_list":['Some_Resource_Agent_ID2'], "events_list":['data']}) self.unsc.create_notification(notification_object, user_id) notifications = self.unsc.find_notifications_by_user(user_id) for n in notifications: log.debug("n = " +str(n)) def test_update_user_notification(self): user_identty_object = IonObject(RT.UserIdentity, name="user1") user_id = self.imc.create_user_identity(user_identty_object) user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}}) self.imc.create_user_info(user_id, user_info_object) notification_object = IonObject(RT.NotificationRequest, {"name":"notification1", "origin_list":['Some_Resource_Agent_ID1'], "events_list":['resource_lifecycle']}) notification_id = self.unsc.create_notification(notification_object, user_id) notification = self.rrc.read(notification_id) notification.origin_list = ['Some_Resource_Agent_ID5'] self.unsc.update_notification(notification) def test_send_notification_emails(self): user_identty_object = IonObject(RT.UserIdentity, name="user1") user_id = self.imc.create_user_identity(user_identty_object) user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}}) self.imc.create_user_info(user_id, user_info_object) notification_object = IonObject(RT.NotificationRequest, {"name":"notification1", "origin_list":['Some_Resource_Agent_ID1'], "events_list":['resource_lifecycle']}) self.unsc.create_notification(notification_object, user_id) notification_object = IonObject(RT.NotificationRequest, {"name":"notification2", "origin_list":['Some_Resource_Agent_ID2'], "events_list":['data']}) self.unsc.create_notification(notification_object, user_id) rle_publisher = ResourceLifecycleEventPublisher() rle_publisher.create_and_publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event") de_publisher = DataEventPublisher() de_publisher.create_and_publish_event(origin='Some_Resource_Agent_ID2', description="DE test event") gevent.sleep(1) def test_find_events(self): rle_publisher = ResourceLifecycleEventPublisher(event_repo=self.container.event_repository) rle_publisher.create_and_publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event1") rle_publisher.create_and_publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event2") de_publisher = DataEventPublisher(event_repo=self.container.event_repository) de_publisher.create_and_publish_event(origin='Some_Resource_Agent_ID2', description="DE test event1") de_publisher.create_and_publish_event(origin='Some_Resource_Agent_ID2', description="DE test event2") events = self.unsc.find_events(origin='Some_Resource_Agent_ID1') for event in events: log.debug("event=" + str(event)) events = self.unsc.find_events(type='DataEvent') for event in events: log.debug("event=" + str(event))
def instrument_test_driver(container): org_client = OrgManagementServiceClient(node=container.node) id_client = IdentityManagementServiceClient(node=container.node) system_actor = id_client.find_actor_identity_by_name(name=CFG.system.system_actor) log.info('system actor:' + system_actor._id) sa_header_roles = get_role_message_headers(org_client.find_all_roles_by_user(system_actor._id)) # Names of agent data streams to be configured. parsed_stream_name = 'ctd_parsed' raw_stream_name = 'ctd_raw' # Driver configuration. #Simulator driver_config = { 'svr_addr': 'localhost', 'cmd_port': 5556, 'evt_port': 5557, 'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver', 'dvr_cls': 'SBE37Driver', 'comms_config': { SBE37Channel.CTD: { 'method':'ethernet', 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'server_addr': 'localhost', 'server_port': 8888 } } } #Hardware _container_client = ContainerAgentClient(node=container.node, name=container.name) # Create a pubsub client to create streams. _pubsub_client = PubsubManagementServiceClient(node=container.node) # A callback for processing subscribed-to data. def consume(message, headers): log.info('Subscriber received message: %s', str(message)) # Create a stream subscriber registrar to create subscribers. subscriber_registrar = StreamSubscriberRegistrar(process=container, node=container.node) subs = [] # Create streams for each stream named in driver. stream_config = {} for (stream_name, val) in PACKET_CONFIG.iteritems(): stream_def = ctd_stream_definition(stream_id=None) stream_def_id = _pubsub_client.create_stream_definition( container=stream_def) stream_id = _pubsub_client.create_stream( name=stream_name, stream_definition_id=stream_def_id, original=True, encoding='ION R2', headers={'ion-actor-id': system_actor._id, 'ion-actor-roles': sa_header_roles }) stream_config[stream_name] = stream_id # Create subscriptions for each stream. exchange_name = '%s_queue' % stream_name sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name, callback=consume) sub.start() query = StreamQuery(stream_ids=[stream_id]) sub_id = _pubsub_client.create_subscription(\ query=query, exchange_name=exchange_name ) _pubsub_client.activate_subscription(sub_id) subs.append(sub) # Create agent config. agent_resource_id = '123xyz' agent_config = { 'driver_config' : driver_config, 'stream_config' : stream_config, 'agent' : {'resource_id': agent_resource_id} } # Launch an instrument agent process. _ia_name = 'agent007' _ia_mod = 'ion.agents.instrument.instrument_agent' _ia_class = 'InstrumentAgent' _ia_pid = _container_client.spawn_process(name=_ia_name, module=_ia_mod, cls=_ia_class, config=agent_config) log.info('got pid=%s for resource_id=%s' % (str(_ia_pid), str(agent_resource_id)))
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) # @unittest.skip('this test just for debugging setup') # def test_just_the_setup(self): # return @attr('EXT') def test_resources_associations_extensions(self): """ create one of each resource and association used by IMS to guard against problems in ion-definitions """ #stuff we control instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance)) instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent)) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance)) platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent)) platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice)) sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel)) #stuff we associate to data_producer_id, _ = self.RR.create(any_old(RT.DataProducer)) org_id, _ = self.RR.create(any_old(RT.Org)) #instrument_agent_instance_id #is only a target #instrument_agent self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id) self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id) self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device_id) instrument_model_id #is only a target platform_agent_instance_id #is only a target #platform_agent self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id) self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) platform_model_id #is only a target #sensor_device self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id) sensor_model_id #is only a target #create a parsed product for this instrument output tdom, sdom = time_series_domain() tdom = tdom.dump() sdom = sdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', processing_level_code='Parsed_Canonical', temporal_domain = tdom, spatial_domain = sdom) pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1) def addInstOwner(inst_id, subject): actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject}) user_id = self.IDS.create_actor_identity(actor_identity_obj) user_info_obj = any_old(RT.UserInfo) user_info_id = self.IDS.create_user_info(user_id, user_info_obj) self.RR.create_association(inst_id, PRED.hasOwner, user_id) #Testing multiple instrument owners addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254") addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256") extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id) self.assertEqual(instrument_device_id, extended_instrument._id) self.assertEqual(len(extended_instrument.owners), 2) self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id) # Lifecycle self.assertEquals(len(extended_instrument.lcstate_transitions), 7) self.assertEquals(set(extended_instrument.lcstate_transitions.keys()), set(['enable', 'develop', 'deploy', 'retire', 'plan', 'integrate', 'announce'])) # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_instrument.computed.firmware_version, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.last_calibration_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue) log.debug("extended_instrument.computed: %s", extended_instrument.computed) #check model inst_model_obj = self.RR.read(instrument_model_id) self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name) #check agent instance inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id) self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name) #check agent inst_agent_obj = self.RR.read(instrument_agent_id) #compound assoc return list of lists so check the first element self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent.name) #check platform device plat_device_obj = self.RR.read(platform_device_id) self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name) extended_platform = self.IMS.get_platform_device_extension(platform_device_id) self.assertEqual(1, len(extended_platform.instrument_devices)) self.assertEqual(instrument_device_id, extended_platform.instrument_devices[0]._id) self.assertEqual(1, len(extended_platform.instrument_models)) self.assertEqual(instrument_model_id, extended_platform.instrument_models[0]._id) self.assertEquals(extended_platform.platform_agent._id, platform_agent_id) self.assertEquals(len(extended_platform.lcstate_transitions), 7) self.assertEquals(set(extended_platform.lcstate_transitions.keys()), set(['enable', 'develop', 'deploy', 'retire', 'plan', 'integrate', 'announce'])) #check sensor devices self.assertEqual(1, len(extended_instrument.sensor_devices)) #check data_product_parameters_set self.assertEqual(ComputedValueAvailability.PROVIDED, extended_instrument.computed.data_product_parameters_set.status) self.assertTrue( 'Parsed_Canonical' in extended_instrument.computed.data_product_parameters_set.value) # the ctd parameters should include 'temp' self.assertTrue( 'temp' in extended_instrument.computed.data_product_parameters_set.value['Parsed_Canonical']) #none of these will work because there is no agent self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, extended_instrument.computed.firmware_version.status) # self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, # extended_instrument.computed.operational_state.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.power_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.communications_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.data_status_roll_up.status) # self.assertEqual(StatusType.STATUS_OK, # extended_instrument.computed.data_status_roll_up.value) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.location_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.recent_events.status) # self.assertEqual([], extended_instrument.computed.recent_events.value) # cleanup c = DotDict() c.resource_registry = self.RR self.RR2.pluck(instrument_agent_id) self.RR2.pluck(instrument_model_id) self.RR2.pluck(instrument_device_id) self.RR2.pluck(platform_agent_id) self.RR2.pluck(sensor_device_id) self.IMS.force_delete_instrument_agent(instrument_agent_id) self.IMS.force_delete_instrument_model(instrument_model_id) self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id) self.IMS.force_delete_platform_agent(platform_agent_id) self.IMS.force_delete_platform_device(platform_device_id) self.IMS.force_delete_platform_model(platform_model_id) self.IMS.force_delete_sensor_device(sensor_device_id) self.IMS.force_delete_sensor_model(sensor_model_id) #stuff we associate to self.RR.delete(data_producer_id) self.RR.delete(org_id) def test_custom_attributes(self): """ Test assignment of custom attributes """ instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel, {"custom_attributes": {"favorite_color": "attr desc goes here"} })) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice, {"custom_attributes": {"favorite_color": "red", "bogus_attr": "should raise warning" } })) self.IMS.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id) # cleanup self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_instrument_model(instrument_model_id) def _get_datastore(self, dataset_id): dataset = self.DSC.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def test_resource_state_save_restore(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.IMS.create_instrument_model(instModel_obj) log.debug( 'new InstrumentModel id = %s ', instModel_id) # Create InstrumentAgent raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 ) instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg", stream_configurations = [raw_config, parsed_config] ) instAgent_id = self.IMS.create_instrument_agent(instAgent_obj) log.debug( 'new InstrumentAgent id = %s', instAgent_id) self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 ' + '(SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj) self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=spdict_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) self.DP.activate_data_product_persistence(data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug( 'Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] #create the datastore at the beginning of each int test that persists data dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug( 'new dp_id = %s', str(data_product_id2)) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.DP.activate_data_product_persistence(data_product_id=data_product_id2) # spin up agent self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.IMS.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id) gate = ProcessStateGate(self.PDC.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id) # take snapshot of config snap_id = self.IMS.save_resource_state(instDevice_id, "xyzzy snapshot") snap_obj = self.RR.read_attachment(snap_id, include_content=True) print "Saved config:" print snap_obj.content #modify config instance_obj.driver_config["comms_config"] = "BAD_DATA" self.RR.update(instance_obj) #restore config self.IMS.restore_resource_state(instDevice_id, snap_id) instance_obj = self.RR.read(instAgentInstance_id) self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"]) self.DP.delete_data_product(data_product_id1) self.DP.delete_data_product(data_product_id2) def test_agent_instance_config(self): """ Verify that agent configurations are being built properly """ clients = DotDict() clients.resource_registry = self.RR clients.pubsub_management = self.PSC clients.dataset_management = self.DSC pconfig_builder = PlatformAgentConfigurationBuilder(clients) iconfig_builder = InstrumentAgentConfigurationBuilder(clients) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() org_obj = any_old(RT.Org) org_id = self.RR2.create(org_obj) inst_startup_config = {'startup': 'config'} generic_alerts_config = {'lvl1': {'lvl2': 'lvl3val'}} required_config_keys = [ 'org_name', 'device_type', 'agent', 'driver_config', 'stream_config', 'startup_config', 'aparam_alert_config', 'children'] def verify_instrument_config(config, device_id): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.name, config['org_name']) self.assertEqual(RT.InstrumentDevice, config['device_type']) self.assertIn('driver_config', config) driver_config = config['driver_config'] expected_driver_fields = {'process_type': ('ZMQPyClassDriverLauncher',), } for k, v in expected_driver_fields.iteritems(): self.assertIn(k, driver_config) self.assertEqual(v, driver_config[k]) self.assertEqual self.assertEqual({'resource_id': device_id}, config['agent']) self.assertEqual(inst_startup_config, config['startup_config']) self.assertIn('aparam_alert_config', config) self.assertEqual(generic_alerts_config, config['aparam_alert_config']) self.assertIn('stream_config', config) for key in ['children']: self.assertEqual({}, config[key]) def verify_child_config(config, device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.name, config['org_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertEqual({'resource_id': device_id}, config['agent']) self.assertIn('aparam_alert_config', config) self.assertEqual(generic_alerts_config, config['aparam_alert_config']) self.assertIn('stream_config', config) self.assertIn('driver_config', config) self.assertIn('foo', config['driver_config']) self.assertEqual('bar', config['driver_config']['foo']) self.assertIn('process_type', config['driver_config']) self.assertEqual(('ZMQPyClassDriverLauncher',), config['driver_config']['process_type']) if None is inst_device_id: for key in ['children', 'startup_config']: self.assertEqual({}, config[key]) else: for key in ['startup_config']: self.assertEqual({}, config[key]) self.assertIn(inst_device_id, config['children']) verify_instrument_config(config['children'][inst_device_id], inst_device_id) def verify_parent_config(config, parent_device_id, child_device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.name, config['org_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertIn('process_type', config['driver_config']) self.assertEqual(('ZMQPyClassDriverLauncher',), config['driver_config']['process_type']) self.assertEqual({'resource_id': parent_device_id}, config['agent']) self.assertIn('aparam_alert_config', config) self.assertEqual(generic_alerts_config, config['aparam_alert_config']) self.assertIn('stream_config', config) for key in ['startup_config']: self.assertEqual({}, config[key]) self.assertIn(child_device_id, config['children']) verify_child_config(config['children'][child_device_id], child_device_id, inst_device_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id) #todo: create org and figure out which agent resource needs to get assigned to it def _make_platform_agent_structure(agent_config=None): if None is agent_config: agent_config = {} # instance creation platform_agent_instance_obj = any_old(RT.PlatformAgentInstance, {'driver_config': {'foo': 'bar'}, 'alerts': generic_alerts_config}) platform_agent_instance_obj.agent_config = agent_config platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj) # agent creation raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) platform_agent_obj = any_old(RT.PlatformAgent, {"stream_configurations":[raw_config]}) platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj) # device creation platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) # assignments self.RR2.assign_platform_agent_instance_to_platform_device(platform_agent_instance_id, platform_device_id) self.RR2.assign_platform_agent_to_platform_agent_instance(platform_agent_id, platform_agent_instance_id) self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, org_id) return platform_agent_instance_id, platform_agent_id, platform_device_id def _make_instrument_agent_structure(agent_config=None): if None is agent_config: agent_config = {} # instance creation instrument_agent_instance_obj = any_old(RT.InstrumentAgentInstance, {"startup_config": inst_startup_config, 'alerts': generic_alerts_config}) instrument_agent_instance_obj.agent_config = agent_config instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj) # agent creation raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) instrument_agent_obj = any_old(RT.InstrumentAgent, {"stream_configurations":[raw_config]}) instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj) # device creation instrument_device_id = self.IMS.create_instrument_device(any_old(RT.InstrumentDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) # assignments self.RR2.assign_instrument_agent_instance_to_instrument_device(instrument_agent_instance_id, instrument_device_id) self.RR2.assign_instrument_agent_to_instrument_agent_instance(instrument_agent_id, instrument_agent_instance_id) self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id) return instrument_agent_instance_id, instrument_agent_id, instrument_device_id # can't do anything without an agent instance obj log.debug("Testing that preparing a launcher without agent instance raises an error") self.assertRaises(AssertionError, pconfig_builder.prepare, will_launch=False) log.debug("Making the structure for a platform agent, which will be the child") platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure() platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id) log.debug("Preparing a valid agent instance launch, for config only") pconfig_builder.set_agent_instance_object(platform_agent_instance_child_obj) child_config = pconfig_builder.prepare(will_launch=False) verify_child_config(child_config, platform_device_child_id) log.debug("Making the structure for a platform agent, which will be the parent") platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure() platform_agent_instance_parent_obj = self.RR2.read(platform_agent_instance_parent_id) log.debug("Testing child-less parent as a child config") pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj) parent_config = pconfig_builder.prepare(will_launch=False) verify_child_config(parent_config, platform_device_parent_id) log.debug("assigning child platform to parent") self.RR2.assign_platform_device_to_platform_device(platform_device_child_id, platform_device_parent_id) child_device_ids = self.RR2.find_platform_device_ids_of_device(platform_device_parent_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing parent + child as parent config") pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj) parent_config = pconfig_builder.prepare(will_launch=False) verify_parent_config(parent_config, platform_device_parent_id, platform_device_child_id) log.debug("making the structure for an instrument agent") instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure() instrument_agent_instance_obj = self.RR2.read(instrument_agent_instance_id) log.debug("Testing instrument config") iconfig_builder.set_agent_instance_object(instrument_agent_instance_obj) instrument_config = iconfig_builder.prepare(will_launch=False) verify_instrument_config(instrument_config, instrument_device_id) log.debug("assigning instrument to platform") self.RR2.assign_instrument_device_to_platform_device(instrument_device_id, platform_device_child_id) child_device_ids = self.RR2.find_instrument_device_ids_of_device(platform_device_child_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing entire config") pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj) full_config = pconfig_builder.prepare(will_launch=False) verify_parent_config(full_config, platform_device_parent_id, platform_device_child_id, instrument_device_id)