def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.unsc = UserNotificationServiceClient(node=self.container.node)
self.rrc = ResourceRegistryServiceClient(node=self.container.node)
self.imc = IdentityManagementServiceClient(node=self.container.node)
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dpsc_cli = DataProductManagementServiceClient()
self.rrclient = ResourceRegistryServiceClient()
self.damsclient = DataAcquisitionManagementServiceClient()
self.pubsubcli = PubsubManagementServiceClient()
self.ingestclient = IngestionManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.unsc = UserNotificationServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.identcli = IdentityManagementServiceClient()
#------------------------------------------
# Create the environment
#------------------------------------------
self.stream_def_id = self.pubsubcli.create_stream_definition(
name='SBE37_CDM')
self.process_definitions = {}
ingestion_worker_definition = ProcessDefinition(
name='ingestion worker')
ingestion_worker_definition.executable = {
'module':
'ion.processes.data.ingestion.science_granule_ingestion_worker',
'class': 'ScienceGranuleIngestionWorker'
}
process_definition_id = self.process_dispatcher.create_process_definition(
process_definition=ingestion_worker_definition)
self.process_definitions['ingestion_worker'] = process_definition_id
self.pids = []
self.exchange_points = []
self.exchange_names = []
#------------------------------------------------------------------------------------------------
# First launch the ingestors
#------------------------------------------------------------------------------------------------
self.exchange_space = 'science_granule_ingestion'
self.exchange_point = 'science_data'
config = DotDict()
config.process.datastore_name = 'datasets'
config.process.queue_name = self.exchange_space
self.exchange_names.append(self.exchange_space)
self.exchange_points.append(self.exchange_point)
pid = self.process_dispatcher.schedule_process(
self.process_definitions['ingestion_worker'], configuration=config)
log.debug("the ingestion worker process id: %s", pid)
self.pids.append(pid)
self.addCleanup(self.cleaning_up)
def setUp(self):
self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254"
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2coi.yml')
self.identity_management_service = IdentityManagementServiceClient(
node=self.container.node)
def setUp(self):
self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254"
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.resource_registry = ResourceRegistryServiceClient()
self.identity_management_service = IdentityManagementServiceClient()
self.org_client = OrgManagementServiceClient()
def test_non_anonymous_resource_registry_operations_through_gateway(self):
id_client = IdentityManagementServiceClient(node=self.container.node)
actor_id, valid_until, registered = id_client.signon(USER1_CERTIFICATE, True)
response = self.test_app.get('/ion-service/resource_registry/find_resources?name=TestDataProduct&id_only=True&requester=' + actor_id)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
self.assertEqual(len(response_data),2 )
self.assertEqual(len(response_data[0]), 0 )
id_client.delete_actor_identity(actor_id)
def test_non_anonymous_resource_registry_operations_through_gateway(self):
id_client = IdentityManagementServiceClient(node=self.container.node)
actor_id, valid_until, registered = id_client.signon(USER1_CERTIFICATE, True)
response = self.test_app.get('/ion-service/resource_registry/find_resources?name=TestDataProduct&id_only=True&requester=' + actor_id)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
self.assertEqual(len(response_data),2 )
self.assertEqual(len(response_data[0]), 0 )
id_client.delete_actor_identity(actor_id)
def setUp(self):
"""
Test setup.
"""
# Resources used in the tests.
# General resources.
self.actor_id = None
self.user_info_id = None
self.org_id = None
self.obs_id = None
# Cabled infrastructure.
self.cabled_platform_model_id = None
self.cabled_platform_site_id = None
self.cabled_platform_device_id = None
self.cabled_platform_agent_id = None
self.cabled_platform_agent_instance_id = None
self.cabled_platform_deployment_id = None
self.cabled_instrument_deployment_id = None
self.cabled_instrument_model_id = None
self.cabled_instrument_site_id = None
self.cabled_instrument_device_id = None
self.cabled_instrument_agent_id = None
self.cabled_instrument_agent_instance_id = None
self.cabled_instrument_deployment_id = None
# Uncabled infrastructure.
self.uncabled_platform_model_id = None
self.uncabled_platform_site_id = None
self.uncabled_platform_device_id = None
self.uncabled_platform_agent_id = None
self.uncabled_platform_agent_instance_id = None
self.uncabled_instrument_model_id = None
self.uncabled_instrument_site_id = None
self.uncabled_instrument_device_id = None
self.uncabled_instrument_agent_id = None
self.uncabled_instrument_agent_instance_id = None
self.uncabled_site_deployment_id = None
# Start container.
log.info('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message)
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Setup service clients.
self.idms = IdentityManagementServiceClient(node=self.container.node)
self.oms = ObservatoryManagementServiceClient(node=self.container.node)
# Add generic resources.
self._load_system_actors()
self._create_user()
self._create_org()
self._create_observatory()
# Add cleanup routine.
self.addCleanup(self._cleanup_resources)
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.unsc = UserNotificationServiceClient(node=self.container.node)
self.rrc = ResourceRegistryServiceClient(node=self.container.node)
self.imc = IdentityManagementServiceClient(node=self.container.node)
def setUp(self):
self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254"
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2coi.yml')
self.identity_management_service = IdentityManagementServiceClient(node=self.container.node)
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dpsc_cli = DataProductManagementServiceClient()
self.rrclient = ResourceRegistryServiceClient()
self.damsclient = DataAcquisitionManagementServiceClient()
self.pubsubcli = PubsubManagementServiceClient()
self.ingestclient = IngestionManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.unsc = UserNotificationServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.identcli = IdentityManagementServiceClient()
#------------------------------------------
# Create the environment
#------------------------------------------
self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM')
self.process_definitions = {}
ingestion_worker_definition = ProcessDefinition(name='ingestion worker')
ingestion_worker_definition.executable = {
'module':'ion.processes.data.ingestion.science_granule_ingestion_worker',
'class' :'ScienceGranuleIngestionWorker'
}
process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition)
self.process_definitions['ingestion_worker'] = process_definition_id
self.pids = []
self.exchange_points = []
self.exchange_names = []
#------------------------------------------------------------------------------------------------
# First launch the ingestors
#------------------------------------------------------------------------------------------------
self.exchange_space = 'science_granule_ingestion'
self.exchange_point = 'science_data'
config = DotDict()
config.process.datastore_name = 'datasets'
config.process.queue_name = self.exchange_space
self.exchange_names.append(self.exchange_space)
self.exchange_points.append(self.exchange_point)
pid = self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config)
log.debug("the ingestion worker process id: %s", pid)
self.pids.append(pid)
self.addCleanup(self.cleaning_up)
def setUp(self):
self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254"
# Start container
self._start_container()
# Establish endpoint with container
container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
container_client.start_rel_from_url('res/deploy/r2coi.yml')
self.identity_management_service = IdentityManagementServiceClient(node=self.container.node)
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
unittest # suppress an pycharm inspector error if all unittest.skip references are commented out
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.OMS = ObservatoryManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
print 'started services'
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
print 'started services'
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient()
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
# Use the network definition provided by RSN OMS directly.
rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri'])
self._network_definition = RsnOmsUtil.build_network_definition(rsn_oms)
# get serialized version for the configuration:
self._network_definition_ser = NetworkUtil.serialize_network_definition(
self._network_definition)
if log.isEnabledFor(logging.TRACE):
log.trace("NetworkDefinition serialization:\n%s",
self._network_definition_ser)
self._async_data_result = AsyncResult()
self._data_subscribers = []
self._samples_received = []
self.addCleanup(self._stop_data_subscribers)
self._async_event_result = AsyncResult()
self._event_subscribers = []
self._events_received = []
self.addCleanup(self._stop_event_subscribers)
self._start_event_subscriber()
def test_non_anonymous_resource_registry_operations_with_token(self):
rr = self.container.resource_registry
id_client = IdentityManagementServiceClient()
create_request = {
"serviceRequest": {
"serviceName": "resource_registry",
"serviceOp": "create",
"params": {
"object": {
"name": "Instrument1",
"type_": "InstrumentDevice"
}
}
}
}
# Create without actor
response = self.test_app.post(
'/ion-service/resource_registry/create',
{'payload': simplejson.dumps(create_request)})
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertGreaterEqual(len(response_data[0]),
20) # This is a resource_id
inst_id = str(response_data[0])
inst_obj = rr.read(inst_id)
self.assertEquals(inst_obj.type_, RT.InstrumentDevice)
self.assertEquals(inst_obj.name, "Instrument1")
self.assertEquals(inst_obj.visibility, ResourceVisibilityEnum.PUBLIC)
act_objs, assocs = rr.find_objects(inst_id,
PRED.hasOwner,
RT.ActorIdentity,
id_only=False)
self.assertEquals(len(act_objs), 0)
# Anonymous query shows PUBLIC visibility resource anymore
response = self.test_app.get(
'/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True'
)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 1)
self.assertEqual(len(response_data[1]), 1)
self.assertEqual(response_data[0][0], inst_id)
inst_obj.visibility = ResourceVisibilityEnum.OWNER
rr.update(inst_obj)
# Now the anonymous query should not show the resource anymore
response = self.test_app.get(
'/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True'
)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 0)
rr.delete(inst_id)
# Create with actor
actor_id, valid_until, registered = id_client.signon(
USER1_CERTIFICATE, True)
create_request = {
"serviceRequest": {
"serviceName": "resource_registry",
"serviceOp": "create",
"requester": actor_id,
"params": {
"object": {
"name": "Instrument1",
"type_": "InstrumentDevice",
"visibility": ResourceVisibilityEnum.OWNER
}
}
}
}
response = self.test_app.post(
'/ion-service/resource_registry/create',
{'payload': simplejson.dumps(create_request)})
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertGreaterEqual(len(response_data[0]),
20) # This is a resource_id
inst_id = str(response_data[0])
inst_obj = rr.read(inst_id)
self.assertEquals(inst_obj.type_, RT.InstrumentDevice)
self.assertEquals(inst_obj.name, "Instrument1")
self.assertEquals(inst_obj.visibility, ResourceVisibilityEnum.OWNER)
act_objs, assocs = rr.find_objects(inst_id,
PRED.hasOwner,
RT.ActorIdentity,
id_only=False)
self.assertEquals(len(act_objs), 1)
self.assertEquals(act_objs[0]._id, actor_id)
# Anonymous query should not show the resource anymore
response = self.test_app.get(
'/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True'
)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 0)
# Authenticated request with owner shows resource
response = self.test_app.get(
'/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&requester='
+ actor_id)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 1)
self.assertEqual(len(response_data[1]), 1)
self.assertEqual(response_data[0][0], inst_id)
token_str = id_client.create_authentication_token(actor_id, validity=2)
# Request with authentication token for owner shows resource
response = self.test_app.get(
'/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&authtoken='
+ token_str)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 1)
self.assertEqual(len(response_data[1]), 1)
self.assertEqual(response_data[0][0], inst_id)
gevent.sleep(2.1)
# Request with expired authentication token for owner does not show resource (no error though)
response = self.test_app.get(
'/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&authtoken='
+ token_str)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 0)
# Request with valid authentication token for different user does not show resource
actor_id2, _ = rr.create(IonObject(RT.ActorIdentity, name="Actor2"))
token_str2 = id_client.create_authentication_token(actor_id2,
validity=2)
response = self.test_app.get(
'/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&authtoken='
+ token_str2)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 0)
token_str3 = id_client.create_authentication_token(actor_id,
validity=2)
# Request with new authentication token for owner shows resource
response = self.test_app.get(
'/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&authtoken='
+ token_str3)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 1)
self.assertEqual(len(response_data[1]), 1)
self.assertEqual(response_data[0][0], inst_id)
id_client.invalidate_authentication_token(token_str3)
response = self.test_app.get(
'/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&authtoken='
+ token_str3)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 0)
# Cleanup
rr.delete(inst_id)
rr.delete(actor_id2)
id_client.delete_actor_identity(actor_id)
class UserNotificationIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.unsc = UserNotificationServiceClient(node=self.container.node)
self.rrc = ResourceRegistryServiceClient(node=self.container.node)
self.imc = IdentityManagementServiceClient(node=self.container.node)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_email(self):
proc1 = self.container.proc_manager.procs_by_name['user_notification']
# Create a user and get the user_id
user = UserInfo(name='new_user')
user_id, _ = self.rrc.create(user)
# set up....
notification_id = self.unsc.create_email(
event_type='ResourceLifecycleEvent',
event_subtype=None,
origin='Some_Resource_Agent_ID1',
origin_type=None,
user_id=user_id,
email='*****@*****.**',
mode=DeliveryMode.DIGEST,
message_header='message_header',
parser='parser',
period=1)
#------------------------------------------------------------------------------------------------------
# Setup so as to be able to get the message and headers going into the
# subscription callback method of the EmailEventProcessor
#------------------------------------------------------------------------------------------------------
# publish an event for each notification to generate the emails
rle_publisher = EventPublisher("ResourceLifecycleEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1',
description="RLE test event")
msg_tuple = proc1.event_processors[
notification_id].smtp_client.sentmail.get(timeout=4)
self.assertTrue(proc1.event_processors[notification_id].smtp_client.
sentmail.empty())
message = msg_tuple[2]
list_lines = message.split("\n")
#-------------------------------------------------------
# parse the message body
#-------------------------------------------------------
message_dict = {}
for line in list_lines:
key_item = line.split(": ")
if key_item[0] == 'Subject':
message_dict['Subject'] = key_item[1] + key_item[2]
else:
try:
message_dict[key_item[0]] = key_item[1]
except IndexError as exc:
# these IndexError exceptions happen only because the message sometimes
# has successive /r/n (i.e. new lines) and therefore,
# the indexing goes out of range. These new lines
# can just be ignored. So we ignore the exceptions here.
pass
#-------------------------------------------------------
# make assertions
#-------------------------------------------------------
self.assertEquals(msg_tuple[1], '*****@*****.**')
#self.assertEquals(msg_tuple[0], ION_NOTIFICATION_EMAIL_ADDRESS)
#self.assertEquals(message_dict['From'], ION_NOTIFICATION_EMAIL_ADDRESS)
self.assertEquals(message_dict['To'], '*****@*****.**')
self.assertEquals(message_dict['Event'].rstrip('\r'),
'ResourceLifecycleEvent')
self.assertEquals(message_dict['Originator'].rstrip('\r'),
'Some_Resource_Agent_ID1')
self.assertEquals(message_dict['Description'].rstrip('\r'),
'RLE test event')
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_sms(self):
proc1 = self.container.proc_manager.procs_by_name['user_notification']
# Create a user and get the user_id
user = UserInfo(name='new_user')
user_id, _ = self.rrc.create(user)
# set up....
notification_id = self.unsc.create_sms(
event_type='ResourceLifecycleEvent',
event_subtype=None,
origin='Some_Resource_Agent_ID1',
origin_type=None,
user_id=user_id,
phone='401-XXX-XXXX',
provider='T-Mobile',
message_header='message_header',
parser='parser',
)
#------------------------------------------------------------------------------------------------------
# Setup so as to be able to get the message and headers going into the
# subscription callback method of the EmailEventProcessor
#------------------------------------------------------------------------------------------------------
# publish an event for each notification to generate the emails
rle_publisher = EventPublisher("ResourceLifecycleEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1',
description="RLE test event")
msg_tuple = proc1.event_processors[
notification_id].smtp_client.sentmail.get(timeout=4)
self.assertTrue(proc1.event_processors[notification_id].smtp_client.
sentmail.empty())
message = msg_tuple[2]
list_lines = message.split("\n")
#-------------------------------------------------------
# parse the message body
#-------------------------------------------------------
message_dict = {}
for line in list_lines:
key_item = line.split(": ")
if key_item[0] == 'Subject':
message_dict['Subject'] = key_item[1] + key_item[2]
else:
try:
message_dict[key_item[0]] = key_item[1]
except IndexError as exc:
# these IndexError exceptions happen only because the message sometimes
# has successive /r/n (i.e. new lines) and therefore,
# the indexing goes out of range. These new lines
# can just be ignored. So we ignore the exceptions here.
pass
#-------------------------------------------------------
# make assertions
#-------------------------------------------------------
self.assertEquals(msg_tuple[1], '*****@*****.**')
#self.assertEquals(msg_tuple[0], ION_NOTIFICATION_EMAIL_ADDRESS)
self.assertEquals(message_dict['Description'].rstrip('\r'),
'RLE test event')
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_event_detection(self):
proc1 = self.container.proc_manager.procs_by_name['user_notification']
# Create a user and get the user_id
user = UserInfo(name='new_user')
user_id, _ = self.rrc.create(user)
# Create detection notification
dfilt = DetectionFilterConfig()
dfilt.processing['condition'] = 5
dfilt.processing['comparator'] = '>'
dfilt.processing['filter_field'] = 'voltage'
dfilt.delivery['message'] = 'I got my detection event!'
notification_id = self.unsc.create_detection_filter(
event_type='ExampleDetectableEvent',
event_subtype=None,
origin='Some_Resource_Agent_ID1',
origin_type=None,
user_id=user_id,
filter_config=dfilt)
#---------------------------------------------------------------------------------
# Create event subscription for resulting detection event
#---------------------------------------------------------------------------------
# Create an email notification so that when the DetectionEventProcessor
# detects an event and fires its own output event, this will caught by an
# EmailEventProcessor and an email will be sent to the user
notification_id_2 = self.unsc.create_email(
event_type='DetectionEvent',
event_subtype=None,
origin='DetectionEventProcessor',
origin_type=None,
user_id=user_id,
email='*****@*****.**',
mode=DeliveryMode.UNFILTERED,
message_header='Detection event',
parser='parser',
period=1)
# Send event that is not detected
# publish an event for each notification to generate the emails
rle_publisher = EventPublisher("ExampleDetectableEvent")
# since the voltage field in this event is less than 5, it will not be detected
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1',
description="RLE test event",
voltage=3)
# Check at the end of the test to make sure this event never triggered a Detectable!
# Send Event that is detected
# publish an event for each notification to generate the emails
# since the voltage field in this event is greater than 5, it WILL be detected
rle_publisher = EventPublisher("ExampleDetectableEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1',
description="RLE test event",
voltage=10)
#-------------------------------------------------------
# make assertions
#-------------------------------------------------------
msg_tuple = proc1.event_processors[
notification_id_2].smtp_client.sentmail.get(timeout=4)
# The first event never triggered an email because the voltage was less than 5, the queue is now empty
self.assertTrue(proc1.event_processors[notification_id_2].smtp_client.
sentmail.empty())
self.assertEquals(msg_tuple[1], '*****@*****.**')
#self.assertEquals(msg_tuple[0], ION_NOTIFICATION_EMAIL_ADDRESS)
# parse the message body
message = msg_tuple[2]
list_lines = message.split("\n")
message_dict = {}
for line in list_lines:
key_item = line.split(": ")
if key_item[0] == 'Subject':
message_dict['Subject'] = key_item[1] + key_item[2]
else:
try:
message_dict[key_item[0]] = key_item[1]
except IndexError as exc:
# these IndexError exceptions happen only because the message sometimes
# has successive /r/n (i.e. new lines) and therefore,
# the indexing goes out of range. These new lines
# can just be ignored. So we ignore the exceptions here.
pass
#self.assertEquals(message_dict['From'], ION_NOTIFICATION_EMAIL_ADDRESS)
self.assertEquals(message_dict['To'], '*****@*****.**')
self.assertEquals(message_dict['Event'].rstrip('\r'), 'DetectionEvent')
self.assertEquals(message_dict['Originator'].rstrip('\r'),
'DetectionEventProcessor')
self.assertEquals(message_dict['Description'].rstrip('\r'),
'Event was detected by DetectionEventProcessor')
@unittest.skip('interface has changed!')
def test_find_event_types_for_resource(self):
# create a dataset object in the RR to pass into the UNS method
dataset_object = IonObject(RT.DataSet, name="dataset1")
dataset_id, version = self.rrc.create(dataset_object)
# get the list of event types for the dataset
events = self.unsc.find_event_types_for_resource(dataset_id)
log.debug("dataset events = " + str(events))
if not events == ['dataset_supplement_added', 'dataset_change']:
self.fail("failed to return correct list of event types")
# try to pass in an id of a resource that doesn't exist (should fail)
try:
events = self.unsc.find_event_types_for_resource("bogus_id")
self.fail("failed to detect non-existant resource")
except:
pass
@unittest.skip('interface has changed!')
def test_create_two_user_notifications(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(
RT.UserInfo, {
"name": "user1_info",
"contact": {
"email": '*****@*****.**'
}
})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object1 = IonObject(
RT.NotificationRequest, {
"name": "notification1",
"origin_list": ['Some_Resource_Agent_ID1'],
"events_list": ['ResourceLifecycleEvent']
})
notification_id1 = self.unsc.create_notification(
notification_object1, user_id)
# create second notification
notification_object2 = IonObject(
RT.NotificationRequest, {
"name": "notification2",
"origin_list": ['Some_Resource_Agent_ID2'],
"events_list": ['DataEvent']
})
notification_id2 = self.unsc.create_notification(
notification_object2, user_id)
# read the notifications back and check that they are correct
n1 = self.unsc.read_notification(notification_id1)
if n1.name != notification_object1.name or \
n1.origin_list != notification_object1.origin_list or \
n1.events_list != notification_object1.events_list:
self.fail("notification was not correct")
n2 = self.unsc.read_notification(notification_id2)
if n2.name != notification_object2.name or \
n2.origin_list != notification_object2.origin_list or \
n2.events_list != notification_object2.events_list:
self.fail("notification was not correct")
@unittest.skip('interface has changed!')
def test_delete_user_notifications(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(
RT.UserInfo, {
"name": "user1_info",
"contact": {
"email": '*****@*****.**'
}
})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object1 = IonObject(
RT.NotificationRequest, {
"name": "notification1",
"origin_list": ['Some_Resource_Agent_ID1'],
"events_list": ['ResourceLifecycleEvent']
})
notification1_id = self.unsc.create_notification(
notification_object1, user_id)
# create second notification
notification_object2 = IonObject(
RT.NotificationRequest, {
"name": "notification2",
"origin_list": ['Some_Resource_Agent_ID2'],
"events_list": ['DataEvent']
})
notification2_id = self.unsc.create_notification(
notification_object2, user_id)
# delete both notifications
self.unsc.delete_notification(notification1_id)
self.unsc.delete_notification(notification2_id)
# check that the notifications are not there
try:
n1 = self.unsc.read_notification(notification1_id)
except:
try:
n2 = self.unsc.read_notification(notification2_id)
except:
return
self.fail("failed to delete notifications")
@unittest.skip('interface has changed!')
def test_find_user_notifications(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(
RT.UserInfo, {
"name": "user1_info",
"contact": {
"email": '*****@*****.**'
}
})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object = IonObject(
RT.NotificationRequest, {
"name": "notification1",
"origin_list": ['Some_Resource_Agent_ID1'],
"events_list": ['ResourceLifecycleEvent']
})
self.unsc.create_notification(notification_object, user_id)
# create second notification
notification_object = IonObject(
RT.NotificationRequest, {
"name": "notification2",
"origin_list": ['Some_Resource_Agent_ID2'],
"events_list": ['DataEvent']
})
self.unsc.create_notification(notification_object, user_id)
# try to find all notifications for user
notifications = self.unsc.find_notifications_by_user(user_id)
if len(notifications) != 2:
self.fail("failed to find all notifications")
@unittest.skip('interface has changed!')
def test_update_user_notification(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(
RT.UserInfo, {
"name": "user1_info",
"contact": {
"email": '*****@*****.**'
}
})
self.imc.create_user_info(user_id, user_info_object)
# create a notification
notification_object = IonObject(
RT.NotificationRequest, {
"name": "notification1",
"origin_list": ['Some_Resource_Agent_ID1'],
"events_list": ['ResourceLifecycleEvent']
})
notification_id = self.unsc.create_notification(
notification_object, user_id)
# read back the notification and change it
notification = self.unsc.read_notification(notification_id)
notification.origin_list = ['Some_Resource_Agent_ID5']
self.unsc.update_notification(notification)
# read back the notification and check that it got changed
notification = self.unsc.read_notification(notification_id)
if notification.origin_list != ['Some_Resource_Agent_ID5']:
self.fail("failed to change notification")
@unittest.skip('interface has changed!')
def test_send_notification_emails(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {
"name": "user1_info",
"contact": {
"email": '*****@*****.**'
}
})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object = IonObject(
RT.NotificationRequest, {
"name": "notification1",
"origin_list": ['Some_Resource_Agent_ID1'],
"events_list": ['ResourceLifecycleEvent']
})
self.unsc.create_notification(notification_object, user_id)
# create second notification
notification_object = IonObject(
RT.NotificationRequest, {
"name": "notification2",
"origin_list": ['Some_Resource_Agent_ID2'],
"events_list": ['DataEvent']
})
self.unsc.create_notification(notification_object, user_id)
# publish an event for each notification to generate the emails
# this can't be easily check in SW so need to check for these at the [email protected] account
rle_publisher = EventPublisher("ResourceLifecycleEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1',
description="RLE test event")
de_publisher = EventPublisher("DataEvent")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2',
description="DE test event")
gevent.sleep(1)
@unittest.skip('interface has changed!')
def test_find_events(self):
# publish some events for the event repository
rle_publisher = EventPublisher("ResourceLifecycleEvent")
de_publisher = EventPublisher("DataEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1',
description="RLE test event1")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1',
description="RLE test event2")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1',
description="RLE test event3")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2',
description="DE test event1")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2',
description="DE test event2")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2',
description="DE test event3")
# find all events for the originator 'Some_Resource_Agent_ID1'
events = self.unsc.find_events(origin='Some_Resource_Agent_ID1')
if len(events) != 3:
self.fail("failed to find all events")
for event in events:
log.debug("event=" + str(event))
if event[1][0] != 'Some_Resource_Agent_ID1':
self.fail("failed to find correct events")
# find all events for the originator 'DataEvent'
events = self.unsc.find_events(type='DataEvent')
if len(events) != 3:
self.fail("failed to find all events")
for event in events:
log.debug("event=" + str(event))
if event[1][0] != 'DataEvent':
self.fail("failed to find correct events")
# find 2 events for the originator 'Some_Resource_Agent_ID1'
events = self.unsc.find_events(origin='Some_Resource_Agent_ID2',
limit=2)
if len(events) != 2:
self.fail("failed to find all events")
for event in events:
log.debug("event=" + str(event))
if event[1][0] != 'Some_Resource_Agent_ID2':
self.fail("failed to find correct events")
# find all events for the originator 'Some_Resource_Agent_ID1' in reverse time order
events = self.unsc.find_events(origin='Some_Resource_Agent_ID1',
descending=True)
if len(events) != 3:
self.fail("failed to find all events")
for event in events:
log.debug("event=" + str(event))
if event[1][0] != 'Some_Resource_Agent_ID1':
self.fail("failed to find correct events")
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dpsc_cli = DataProductManagementServiceClient()
self.rrclient = ResourceRegistryServiceClient()
self.damsclient = DataAcquisitionManagementServiceClient()
self.pubsubcli = PubsubManagementServiceClient()
self.ingestclient = IngestionManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.unsc = UserNotificationServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.identcli = IdentityManagementServiceClient()
#------------------------------------------
# Create the environment
#------------------------------------------
self.stream_def_id = self.pubsubcli.create_stream_definition(
name='SBE37_CDM')
self.process_definitions = {}
ingestion_worker_definition = ProcessDefinition(
name='ingestion worker')
ingestion_worker_definition.executable = {
'module':
'ion.processes.data.ingestion.science_granule_ingestion_worker',
'class': 'ScienceGranuleIngestionWorker'
}
process_definition_id = self.process_dispatcher.create_process_definition(
process_definition=ingestion_worker_definition)
self.process_definitions['ingestion_worker'] = process_definition_id
self.pids = []
self.exchange_points = []
self.exchange_names = []
#------------------------------------------------------------------------------------------------
# First launch the ingestors
#------------------------------------------------------------------------------------------------
self.exchange_space = 'science_granule_ingestion'
self.exchange_point = 'science_data'
config = DotDict()
config.process.datastore_name = 'datasets'
config.process.queue_name = self.exchange_space
self.exchange_names.append(self.exchange_space)
self.exchange_points.append(self.exchange_point)
pid = self.process_dispatcher.schedule_process(
self.process_definitions['ingestion_worker'], configuration=config)
log.debug("the ingestion worker process id: %s", pid)
self.pids.append(pid)
self.addCleanup(self.cleaning_up)
def cleaning_up(self):
for pid in self.pids:
log.debug("number of pids to be terminated: %s", len(self.pids))
try:
self.process_dispatcher.cancel_process(pid)
log.debug("Terminated the process: %s", pid)
except:
log.debug("could not terminate the process id: %s" % pid)
IngestionManagementIntTest.clean_subscriptions()
for xn in self.exchange_names:
xni = self.container.ex_manager.create_xn_queue(xn)
xni.delete()
for xp in self.exchange_points:
xpi = self.container.ex_manager.create_xp(xp)
xpi.delete()
def get_datastore(self, dataset_id):
dataset = self.dataset_management.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(
datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
@attr('EXT')
@attr('PREP')
def test_create_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
parameter_dictionary = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict')
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
name='Simulated CTD data',
parameter_dictionary_id=parameter_dictionary._id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0
dp_obj.ooi_product_name = "PRODNAME"
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product(
data_product=dp_obj, stream_definition_id=ctd_stream_def_id)
# Assert that the data product has an associated stream at this stage
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream,
RT.Stream, True)
self.assertNotEquals(len(stream_ids), 0)
# Assert that the data product has an associated stream def at this stage
stream_ids, _ = self.rrclient.find_objects(dp_id,
PRED.hasStreamDefinition,
RT.StreamDefinition, True)
self.assertNotEquals(len(stream_ids), 0)
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Created data product %s', dp_obj)
#------------------------------------------------------------------------------------------------
# test creating a new data product with a stream definition
#------------------------------------------------------------------------------------------------
log.debug('Creating new data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='DP2',
description='some new dp')
dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id)
self.dpsc_cli.activate_data_product_persistence(dp_id2)
log.debug('new dp_id = %s' % dp_id2)
#------------------------------------------------------------------------------------------------
#make sure data product is associated with stream def
#------------------------------------------------------------------------------------------------
streamdefs = []
streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream,
RT.Stream, True)
for s in streams:
log.debug("Checking stream %s" % s)
sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition,
RT.StreamDefinition, True)
for sd in sdefs:
log.debug("Checking streamdef %s" % sd)
streamdefs.append(sd)
self.assertIn(ctd_stream_def_id, streamdefs)
group_names = self.dpsc_cli.get_data_product_group_list()
self.assertIn("PRODNAME", group_names)
#----------------------------------------------------------------------------------------
# Create users then notifications to this data product for each user
#----------------------------------------------------------------------------------------
# user_1
user_1 = UserInfo()
user_1.name = 'user_1'
user_1.contact.email = '*****@*****.**'
# user_2
user_2 = UserInfo()
user_2.name = 'user_2'
user_2.contact.email = '*****@*****.**'
#user1 is a complete user
self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254"
actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject})
actor_id = self.identcli.create_actor_identity(actor_identity_obj)
user_credentials_obj = IonObject("UserCredentials",
{"name": self.subject})
self.identcli.register_user_credentials(actor_id, user_credentials_obj)
user_id_1 = self.identcli.create_user_info(actor_id, user_1)
user_id_2, _ = self.rrclient.create(user_2)
delivery_config1a = IonObject(
OT.DeliveryConfiguration,
email='*****@*****.**',
mode=DeliveryModeEnum.EMAIL,
frequency=NotificationFrequencyEnum.BATCH)
delivery_config1b = IonObject(
OT.DeliveryConfiguration,
email='*****@*****.**',
mode=DeliveryModeEnum.EMAIL,
frequency=NotificationFrequencyEnum.BATCH)
notification_request_1 = NotificationRequest(
name="notification_1",
origin=dp_id,
origin_type="type_1",
event_type=OT.ResourceLifecycleEvent,
disabled_by_system=False,
delivery_configurations=[delivery_config1a, delivery_config1b])
delivery_config2a = IonObject(
OT.DeliveryConfiguration,
email='*****@*****.**',
mode=DeliveryModeEnum.EMAIL,
frequency=NotificationFrequencyEnum.BATCH)
delivery_config2b = IonObject(
OT.DeliveryConfiguration,
email='*****@*****.**',
mode=DeliveryModeEnum.EMAIL,
frequency=NotificationFrequencyEnum.BATCH)
notification_request_2 = NotificationRequest(
name="notification_2",
origin=dp_id,
origin_type="type_2",
disabled_by_system=False,
event_type=OT.DetectionEvent,
delivery_configurations=[delivery_config2a, delivery_config2b])
notification_request_1_id = self.unsc.create_notification(
notification=notification_request_1, user_id=user_id_1)
notification_request_2_id = self.unsc.create_notification(
notification=notification_request_2, user_id=user_id_2)
self.unsc.delete_notification(notification_request_1_id)
# test reading a non-existent data product
log.debug('reading non-existent data product')
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product('some_fake_id')
# update a data product (tests read also)
log.debug('Updating data product')
# first get the existing dp object
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# now tweak the object
dp_obj.description = 'the very first dp'
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 20.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -20.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 20.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -20.0
# now write the dp back to the registry
update_result = self.dpsc_cli.update_data_product(dp_obj)
# now get the dp back to see if it was updated
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertEquals(dp_obj.description, 'the very first dp')
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Updated data product %s', dp_obj)
#test extension
extended_product = self.dpsc_cli.get_data_product_extension(dp_id)
#validate that there is one active and one retired user notification for this data product
self.assertEqual(
1, len(extended_product.computed.active_user_subscriptions.value))
self.assertEqual(
1, len(extended_product.computed.past_user_subscriptions.value))
self.assertEqual(dp_id, extended_product._id)
self.assertEqual(
ComputedValueAvailability.PROVIDED,
extended_product.computed.product_download_size_estimated.status)
self.assertEqual(
0, extended_product.computed.product_download_size_estimated.value)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.parameters.status)
#log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value)
def ion_object_encoder(obj):
return obj.__dict__
#test prepare for create
data_product_data = self.dpsc_cli.prepare_data_product_support()
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2)
self.assertEqual(data_product_data._id, "")
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(
len(data_product_data.associations['StreamDefinition'].resources),
2)
self.assertEqual(
len(data_product_data.associations['Dataset'].resources), 0)
self.assertEqual(
len(data_product_data.associations['StreamDefinition'].
associated_resources), 0)
self.assertEqual(
len(data_product_data.associations['Dataset'].associated_resources
), 0)
#test prepare for update
data_product_data = self.dpsc_cli.prepare_data_product_support(dp_id)
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2)
self.assertEqual(data_product_data._id, dp_id)
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(
len(data_product_data.associations['StreamDefinition'].resources),
2)
self.assertEqual(
len(data_product_data.associations['Dataset'].resources), 1)
self.assertEqual(
len(data_product_data.associations['StreamDefinition'].
associated_resources), 1)
self.assertEqual(
data_product_data.associations['StreamDefinition'].
associated_resources[0].s, dp_id)
self.assertEqual(
len(data_product_data.associations['Dataset'].associated_resources
), 1)
self.assertEqual(
data_product_data.associations['Dataset'].associated_resources[0].
s, dp_id)
# now 'delete' the data product
log.debug("deleting data product: %s" % dp_id)
self.dpsc_cli.delete_data_product(dp_id)
# Assert that there are no associated streams leftover after deleting the data product
stream_ids, assoc_ids = self.rrclient.find_objects(
dp_id, PRED.hasStream, RT.Stream, True)
self.assertEquals(len(stream_ids), 0)
self.assertEquals(len(assoc_ids), 0)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# Get the events corresponding to the data product
ret = self.unsc.get_recent_events(resource_id=dp_id)
events = ret.value
for event in events:
log.debug("event time: %s" % event.ts_created)
self.assertTrue(len(events) > 0)
def test_data_product_stream_def(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
name='Simulated CTD data', parameter_dictionary_id=pdict_id)
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
dp_id = self.dpsc_cli.create_data_product(
data_product=dp_obj, stream_definition_id=ctd_stream_def_id)
stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id)
self.assertEquals(ctd_stream_def_id, stream_def_id)
def test_derived_data_product(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
name='ctd parsed', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsubcli.delete_stream_definition,
ctd_stream_def_id)
dp = DataProduct(name='Instrument DP')
dp_id = self.dpsc_cli.create_data_product(
dp, stream_definition_id=ctd_stream_def_id)
self.addCleanup(self.dpsc_cli.force_delete_data_product, dp_id)
self.dpsc_cli.activate_data_product_persistence(dp_id)
self.addCleanup(self.dpsc_cli.suspend_data_product_persistence, dp_id)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id,
predicate=PRED.hasDataset,
id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" %
str(dp_id))
dataset_id = dataset_ids[0]
# Make the derived data product
simple_stream_def_id = self.pubsubcli.create_stream_definition(
name='TEMPWAT stream def',
parameter_dictionary_id=pdict_id,
available_fields=['time', 'temp'])
tempwat_dp = DataProduct(name='TEMPWAT',
category=DataProductTypeEnum.DERIVED)
tempwat_dp_id = self.dpsc_cli.create_data_product(
tempwat_dp,
stream_definition_id=simple_stream_def_id,
parent_data_product_id=dp_id)
self.addCleanup(self.dpsc_cli.delete_data_product, tempwat_dp_id)
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream,
RT.Stream, True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
rdt['pressure'] = np.arange(20)
publisher = StandaloneStreamPublisher(stream_id, route)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified,
callback=cb,
origin=dataset_id,
auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
tempwat_dataset_ids, _ = self.rrclient.find_objects(tempwat_dp_id,
PRED.hasDataset,
id_only=True)
tempwat_dataset_id = tempwat_dataset_ids[0]
granule = self.data_retriever.retrieve(
tempwat_dataset_id, delivery_format=simple_stream_def_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['time'], np.arange(20))
self.assertEquals(set(rdt.fields), set(['time', 'temp']))
def test_activate_suspend_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
name='Simulated CTD data', parameter_dictionary_id=pdict_id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
# Construct temporal and spatial Coordinate Reference System objects
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
log.debug("Created an IonObject for a data product: %s" % dp_obj)
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product(
data_product=dp_obj, stream_definition_id=ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# Subscribe to persist events
#------------------------------------------------------------------------------------------------
queue = gevent.queue.Queue()
def info_event_received(message, headers):
queue.put(message)
es = EventSubscriber(event_type=OT.InformationContentStatusEvent,
callback=info_event_received,
origin=dp_id,
auto_delete=True)
es.start()
self.addCleanup(es.stop)
#------------------------------------------------------------------------------------------------
# test activate and suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id,
predicate=PRED.hasDataset,
id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" %
str(dp_id))
dataset_id = dataset_ids[0]
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream,
RT.Stream, True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
publisher = StandaloneStreamPublisher(stream_id, route)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified,
callback=cb,
origin=dataset_id,
auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_ids[0])
self.assertIsInstance(replay_data, Granule)
log.debug(
"The data retriever was able to replay the dataset that was attached to the data product "
"we wanted to be persisted. Therefore the data product was indeed persisted with "
"otherwise we could not have retrieved its dataset using the data retriever. Therefore "
"this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'"
)
data_product_object = self.rrclient.read(dp_id)
self.assertEquals(data_product_object.name, 'DP1')
self.assertEquals(data_product_object.description, 'some new dp')
log.debug(
"Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. "
" Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the "
"resource registry, name='%s', desc='%s'" %
(dp_obj.name, dp_obj.description, data_product_object.name,
data_product_object.description))
#------------------------------------------------------------------------------------------------
# test suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.suspend_data_product_persistence(dp_id)
dataset_modified.clear()
rdt['time'] = np.arange(20, 40)
publisher.publish(rdt.to_granule())
self.assertFalse(dataset_modified.wait(2))
self.dpsc_cli.activate_data_product_persistence(dp_id)
dataset_modified.clear()
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_almost_equal(rdt['time'], np.arange(40))
dataset_ids, _ = self.rrclient.find_objects(dp_id,
PRED.hasDataset,
id_only=True)
self.assertEquals(len(dataset_ids), 1)
self.dpsc_cli.suspend_data_product_persistence(dp_id)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.rrclient.read(dp_id)
info_event_counter = 0
runtime = 0
starttime = time.time()
caught_events = []
#check that the four InfoStatusEvents were received
while info_event_counter < 4 and runtime < 60:
a = queue.get(timeout=60)
caught_events.append(a)
info_event_counter += 1
runtime = time.time() - starttime
self.assertEquals(info_event_counter, 4)
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
unittest # suppress an pycharm inspector error if all unittest.skip references are commented out
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.DSC = DatasetManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.OMS = ObservatoryManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
# @unittest.skip('this test just for debugging setup')
# def test_just_the_setup(self):
# return
@attr('EXT')
def test_resources_associations_extensions(self):
"""
create one of each resource and association used by IMS
to guard against problems in ion-definitions
"""
#stuff we control
instrument_agent_instance_id, _ = self.RR.create(
any_old(RT.InstrumentAgentInstance))
instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent))
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
instrument_site_id, _ = self.RR.create(any_old(RT.InstrumentSite))
platform_agent_instance_id, _ = self.RR.create(
any_old(RT.PlatformAgentInstance))
platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent))
platform_site_id, _ = self.RR.create(any_old(RT.PlatformSite))
platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice))
platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel))
sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice))
sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel))
#stuff we associate to
data_producer_id, _ = self.RR.create(any_old(RT.DataProducer))
org_id, _ = self.RR.create(any_old(RT.Org))
#instrument_agent_instance_id #is only a target
#instrument_agent
self.RR.create_association(instrument_agent_id, PRED.hasModel,
instrument_model_id)
self.RR.create_association(instrument_agent_instance_id,
PRED.hasAgentDefinition,
instrument_agent_id)
#instrument_device
self.RR.create_association(instrument_device_id, PRED.hasModel,
instrument_model_id)
self.RR.create_association(instrument_device_id, PRED.hasAgentInstance,
instrument_agent_instance_id)
self.RR.create_association(instrument_device_id, PRED.hasDataProducer,
data_producer_id)
self.RR.create_association(instrument_device_id, PRED.hasDevice,
sensor_device_id)
self.RR.create_association(org_id, PRED.hasResource,
instrument_device_id)
instrument_model_id #is only a target
platform_agent_instance_id #is only a target
#platform_agent
self.RR.create_association(platform_agent_id, PRED.hasModel,
platform_model_id)
self.RR.create_association(platform_agent_instance_id,
PRED.hasAgentDefinition, platform_agent_id)
#platform_device
self.RR.create_association(platform_device_id, PRED.hasModel,
platform_model_id)
self.RR.create_association(platform_device_id, PRED.hasAgentInstance,
platform_agent_instance_id)
self.RR.create_association(platform_device_id, PRED.hasDevice,
instrument_device_id)
self.RR.create_association(instrument_site_id, PRED.hasDevice,
instrument_device_id)
self.RR.create_association(platform_site_id, PRED.hasDevice,
platform_device_id)
self.RR.create_association(platform_site_id, PRED.hasSite,
instrument_site_id)
platform_model_id #is only a target
#sensor_device
self.RR.create_association(sensor_device_id, PRED.hasModel,
sensor_model_id)
self.RR.create_association(sensor_device_id, PRED.hasDevice,
instrument_device_id)
sensor_model_id #is only a target
#create a parsed product for this instrument output
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
processing_level_code='Parsed_Canonical',
temporal_domain=tdom,
spatial_domain=sdom)
pdict_id = self.DSC.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(
name='parsed', parameter_dictionary_id=pdict_id)
data_product_id1 = self.DP.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug('new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id,
data_product_id=data_product_id1)
def addInstOwner(inst_id, subject):
actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject})
user_id = self.IDS.create_actor_identity(actor_identity_obj)
user_info_obj = any_old(RT.UserInfo)
user_info_id = self.IDS.create_user_info(user_id, user_info_obj)
self.RR.create_association(inst_id, PRED.hasOwner, user_id)
#Testing multiple instrument owners
addInstOwner(
instrument_device_id,
"/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254")
addInstOwner(
instrument_device_id,
"/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256")
extended_instrument = self.IMS.get_instrument_device_extension(
instrument_device_id)
self.assertEqual(instrument_device_id, extended_instrument._id)
self.assertEqual(len(extended_instrument.owners), 2)
self.assertEqual(extended_instrument.instrument_model._id,
instrument_model_id)
# Lifecycle
self.assertEquals(len(extended_instrument.lcstate_transitions), 5)
self.assertEquals(
set(extended_instrument.lcstate_transitions.keys()),
set(['develop', 'deploy', 'retire', 'plan', 'integrate']))
self.assertEquals(len(extended_instrument.availability_transitions), 2)
self.assertEquals(
set(extended_instrument.availability_transitions.keys()),
set(['enable', 'announce']))
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(
extended_instrument.computed.last_data_received_datetime,
ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.uptime,
ComputedStringValue)
self.assertIsInstance(
extended_instrument.computed.power_status_roll_up,
ComputedIntValue)
self.assertIsInstance(
extended_instrument.computed.communications_status_roll_up,
ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.data_status_roll_up,
ComputedIntValue)
self.assertIsInstance(
extended_instrument.computed.location_status_roll_up,
ComputedIntValue)
log.debug("extended_instrument.computed: %s",
extended_instrument.computed)
#check model
inst_model_obj = self.RR.read(instrument_model_id)
self.assertEqual(inst_model_obj.name,
extended_instrument.instrument_model.name)
#check agent instance
inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id)
self.assertEqual(inst_agent_instance_obj.name,
extended_instrument.agent_instance.name)
#check agent
inst_agent_obj = self.RR.read(instrument_agent_id)
#compound assoc return list of lists so check the first element
self.assertEqual(inst_agent_obj.name,
extended_instrument.instrument_agent.name)
#check platform device
plat_device_obj = self.RR.read(platform_device_id)
self.assertEqual(plat_device_obj.name,
extended_instrument.platform_device.name)
extended_platform = self.IMS.get_platform_device_extension(
platform_device_id)
self.assertEqual(1, len(extended_platform.portals))
self.assertEqual(1, len(extended_platform.portal_instruments))
#self.assertEqual(1, len(extended_platform.computed.portal_status.value)) # no agent started so NO statuses reported
self.assertEqual(1, len(extended_platform.instrument_devices))
self.assertEqual(instrument_device_id,
extended_platform.instrument_devices[0]._id)
self.assertEqual(1, len(extended_platform.instrument_models))
self.assertEqual(instrument_model_id,
extended_platform.instrument_models[0]._id)
self.assertEquals(extended_platform.platform_agent._id,
platform_agent_id)
self.assertEquals(len(extended_platform.lcstate_transitions), 5)
self.assertEquals(
set(extended_platform.lcstate_transitions.keys()),
set(['develop', 'deploy', 'retire', 'plan', 'integrate']))
self.assertEquals(len(extended_platform.availability_transitions), 2)
self.assertEquals(
set(extended_platform.availability_transitions.keys()),
set(['enable', 'announce']))
#check sensor devices
self.assertEqual(1, len(extended_instrument.sensor_devices))
#check data_product_parameters_set
self.assertEqual(
ComputedValueAvailability.PROVIDED,
extended_instrument.computed.data_product_parameters_set.status)
self.assertTrue('Parsed_Canonical' in extended_instrument.computed.
data_product_parameters_set.value)
# the ctd parameters should include 'temp'
self.assertTrue('temp' in extended_instrument.computed.
data_product_parameters_set.value['Parsed_Canonical'])
#none of these will work because there is no agent
# self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
# extended_instrument.computed.firmware_version.status)
# self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
# extended_instrument.computed.operational_state.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.power_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.communications_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.data_status_roll_up.status)
# self.assertEqual(DeviceStatusType.STATUS_OK,
# extended_instrument.computed.data_status_roll_up.value)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.location_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.recent_events.status)
# self.assertEqual([], extended_instrument.computed.recent_events.value)
# cleanup
c = DotDict()
c.resource_registry = self.RR
self.RR2.pluck(instrument_agent_id)
self.RR2.pluck(instrument_model_id)
self.RR2.pluck(instrument_device_id)
self.RR2.pluck(platform_agent_id)
self.RR2.pluck(sensor_device_id)
self.IMS.force_delete_instrument_agent(instrument_agent_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_platform_agent_instance(
platform_agent_instance_id)
self.IMS.force_delete_platform_agent(platform_agent_id)
self.OMS.force_delete_instrument_site(instrument_site_id)
self.OMS.force_delete_platform_site(platform_site_id)
self.IMS.force_delete_platform_device(platform_device_id)
self.IMS.force_delete_platform_model(platform_model_id)
self.IMS.force_delete_sensor_device(sensor_device_id)
self.IMS.force_delete_sensor_model(sensor_model_id)
#stuff we associate to
self.RR.delete(data_producer_id)
self.RR.delete(org_id)
def test_custom_attributes(self):
"""
Test assignment of custom attributes
"""
instModel_obj = IonObject(OT.CustomAttribute,
name='SBE37IMModelAttr',
description="model custom attr")
instrument_model_id, _ = self.RR.create(
any_old(RT.InstrumentModel,
{"custom_attributes": [instModel_obj]}))
instrument_device_id, _ = self.RR.create(
any_old(
RT.InstrumentDevice, {
"custom_attributes": {
"favorite_color": "red",
"bogus_attr": "should raise warning"
}
}))
self.IMS.assign_instrument_model_to_instrument_device(
instrument_model_id, instrument_device_id)
# cleanup
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
def _get_datastore(self, dataset_id):
dataset = self.DSC.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(
datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def test_data_producer(self):
idevice_id = self.IMS.create_instrument_device(
any_old(RT.InstrumentDevice))
self.assertEqual(
1,
len(
self.RR2.
find_data_producer_ids_of_instrument_device_using_has_data_producer(
idevice_id)))
pdevice_id = self.IMS.create_platform_device(any_old(
RT.PlatformDevice))
self.assertEqual(
1,
len(
self.RR2.
find_data_producer_ids_of_platform_device_using_has_data_producer(
pdevice_id)))
@attr('PREP')
def test_prepare_resource_support(self):
"""
create one of each resource and association used by IMS
to guard against problems in ion-definitions
"""
#stuff we control
instrument_agent_instance_id, _ = self.RR.create(
any_old(RT.InstrumentAgentInstance))
instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent))
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
platform_agent_instance_id, _ = self.RR.create(
any_old(RT.PlatformAgentInstance))
platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent))
platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice))
platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel))
sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice))
sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel))
instrument_device2_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
instrument_device3_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
platform_device2_id, _ = self.RR.create(any_old(RT.PlatformDevice))
sensor_device2_id, _ = self.RR.create(any_old(RT.SensorDevice))
#stuff we associate to
data_producer_id, _ = self.RR.create(any_old(RT.DataProducer))
org_id, _ = self.RR.create(any_old(RT.Org))
#instrument_agent_instance_id #is only a target
#instrument_agent
self.RR.create_association(instrument_agent_id, PRED.hasModel,
instrument_model_id)
self.RR.create_association(instrument_agent_instance_id,
PRED.hasAgentDefinition,
instrument_agent_id)
#instrument_device
self.RR.create_association(instrument_device_id, PRED.hasModel,
instrument_model_id)
self.RR.create_association(instrument_device_id, PRED.hasAgentInstance,
instrument_agent_instance_id)
self.RR.create_association(instrument_device_id, PRED.hasDataProducer,
data_producer_id)
self.RR.create_association(instrument_device_id, PRED.hasDevice,
sensor_device_id)
self.RR.create_association(org_id, PRED.hasResource,
instrument_device_id)
self.RR.create_association(instrument_device2_id, PRED.hasModel,
instrument_model_id)
self.RR.create_association(org_id, PRED.hasResource,
instrument_device2_id)
instrument_model_id #is only a target
platform_agent_instance_id #is only a target
#platform_agent
self.RR.create_association(platform_agent_id, PRED.hasModel,
platform_model_id)
self.RR.create_association(platform_agent_instance_id,
PRED.hasAgentDefinition, platform_agent_id)
#platform_device
self.RR.create_association(platform_device_id, PRED.hasModel,
platform_model_id)
self.RR.create_association(platform_device_id, PRED.hasAgentInstance,
platform_agent_instance_id)
self.RR.create_association(platform_device_id, PRED.hasDevice,
instrument_device_id)
self.RR.create_association(platform_device2_id, PRED.hasModel,
platform_model_id)
self.RR.create_association(platform_device2_id, PRED.hasDevice,
instrument_device2_id)
platform_model_id #is only a target
#sensor_device
self.RR.create_association(sensor_device_id, PRED.hasModel,
sensor_model_id)
self.RR.create_association(sensor_device_id, PRED.hasDevice,
instrument_device_id)
self.RR.create_association(sensor_device2_id, PRED.hasModel,
sensor_model_id)
self.RR.create_association(sensor_device2_id, PRED.hasDevice,
instrument_device2_id)
sensor_model_id #is only a target
#set lcstate - used for testing prepare - not setting all to DEVELOP, only some
self.RR.execute_lifecycle_transition(instrument_agent_id, LCE.DEVELOP)
self.RR.execute_lifecycle_transition(instrument_device_id, LCE.DEVELOP)
self.RR.execute_lifecycle_transition(instrument_device2_id,
LCE.DEVELOP)
self.RR.execute_lifecycle_transition(platform_device_id, LCE.DEVELOP)
self.RR.execute_lifecycle_transition(platform_device2_id, LCE.DEVELOP)
self.RR.execute_lifecycle_transition(platform_agent_id, LCE.DEVELOP)
#create a parsed product for this instrument output
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
processing_level_code='Parsed_Canonical',
temporal_domain=tdom,
spatial_domain=sdom)
pdict_id = self.DSC.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(
name='parsed', parameter_dictionary_id=pdict_id)
data_product_id1 = self.DP.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug('new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id,
data_product_id=data_product_id1)
def addInstOwner(inst_id, subject):
actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject})
user_id = self.IDS.create_actor_identity(actor_identity_obj)
user_info_obj = any_old(RT.UserInfo)
user_info_id = self.IDS.create_user_info(user_id, user_info_obj)
self.RR.create_association(inst_id, PRED.hasOwner, user_id)
#Testing multiple instrument owners
addInstOwner(
instrument_device_id,
"/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254")
addInstOwner(
instrument_device_id,
"/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256")
def ion_object_encoder(obj):
return obj.__dict__
#First call to create
instrument_data = self.IMS.prepare_instrument_device_support()
#print simplejson.dumps(instrument_data, default=ion_object_encoder, indent=2)
self.assertEqual(instrument_data._id, '')
self.assertEqual(instrument_data.type_,
OT.InstrumentDevicePrepareSupport)
self.assertEqual(
len(instrument_data.associations['InstrumentModel'].resources), 1)
self.assertEqual(
instrument_data.associations['InstrumentModel'].resources[0]._id,
instrument_model_id)
self.assertEqual(
len(instrument_data.associations['InstrumentAgentInstance'].
resources), 1)
self.assertEqual(
instrument_data.associations['InstrumentAgentInstance'].
resources[0]._id, instrument_agent_instance_id)
self.assertEqual(
len(instrument_data.associations['InstrumentModel'].
associated_resources), 0)
self.assertEqual(
len(instrument_data.associations['InstrumentAgentInstance'].
associated_resources), 0)
self.assertEqual(
len(instrument_data.associations['SensorDevice'].resources), 0)
#Next call to update
instrument_data = self.IMS.prepare_instrument_device_support(
instrument_device_id)
#print 'Update results'
#print simplejson.dumps(instrument_data, default=ion_object_encoder, indent=2)
self.assertEqual(instrument_data._id, instrument_device_id)
self.assertEqual(instrument_data.type_,
OT.InstrumentDevicePrepareSupport)
self.assertEqual(
len(instrument_data.associations['InstrumentModel'].resources), 1)
self.assertEqual(
instrument_data.associations['InstrumentModel'].resources[0]._id,
instrument_model_id)
self.assertEqual(
len(instrument_data.associations['InstrumentAgentInstance'].
resources), 1)
self.assertEqual(
instrument_data.associations['InstrumentAgentInstance'].
resources[0]._id, instrument_agent_instance_id)
self.assertEqual(
len(instrument_data.associations['InstrumentModel'].
associated_resources), 1)
self.assertEqual(
instrument_data.associations['InstrumentModel'].
associated_resources[0].s, instrument_device_id)
self.assertEqual(
instrument_data.associations['InstrumentModel'].
associated_resources[0].o, instrument_model_id)
self.assertEqual(
len(instrument_data.associations['InstrumentAgentInstance'].
associated_resources), 1)
self.assertEqual(
instrument_data.associations['InstrumentAgentInstance'].
associated_resources[0].o, instrument_agent_instance_id)
self.assertEqual(
instrument_data.associations['InstrumentAgentInstance'].
associated_resources[0].s, instrument_device_id)
self.assertEqual(
len(instrument_data.associations['SensorDevice'].resources), 1)
self.assertEqual(
instrument_data.associations['SensorDevice'].resources[0]._id,
sensor_device_id)
self.assertEqual(
len(instrument_data.associations['SensorDevice'].
associated_resources), 1)
self.assertEqual(
instrument_data.associations['SensorDevice'].
associated_resources[0].o, instrument_device_id)
self.assertEqual(
instrument_data.associations['SensorDevice'].
associated_resources[0].s, sensor_device_id)
self.assertEqual(
instrument_data.associations['InstrumentModel'].assign_request.
request_parameters['instrument_device_id'], instrument_device_id)
#test prepare for create of instrument agent instance
instrument_agent_data = self.IMS.prepare_instrument_agent_instance_support(
)
#print 'Update results'
#print simplejson.dumps(instrument_agent_data, default=ion_object_encoder, indent=2)
self.assertEqual(instrument_agent_data._id, '')
self.assertEqual(instrument_agent_data.type_,
OT.InstrumentAgentInstancePrepareSupport)
self.assertEqual(
len(instrument_agent_data.associations['InstrumentDevice'].
resources), 2)
self.assertEqual(
len(instrument_agent_data.associations['InstrumentAgent'].resources
), 1)
self.assertEqual(
instrument_agent_data.associations['InstrumentAgent'].resources[0].
_id, instrument_agent_id)
self.assertEqual(
len(instrument_agent_data.associations['InstrumentDevice'].
associated_resources), 0)
self.assertEqual(
len(instrument_agent_data.associations['InstrumentAgent'].
associated_resources), 0)
#test prepare for update of instrument agent instance to see if it is associated with the instrument that was created
instrument_agent_data = self.IMS.prepare_instrument_agent_instance_support(
instrument_agent_instance_id=instrument_agent_instance_id)
#print 'Update results'
#print simplejson.dumps(instrument_agent_data, default=ion_object_encoder, indent=2)
self.assertEqual(instrument_agent_data._id,
instrument_agent_instance_id)
self.assertEqual(instrument_agent_data.type_,
OT.InstrumentAgentInstancePrepareSupport)
self.assertEqual(
len(instrument_agent_data.associations['InstrumentDevice'].
resources), 3)
self.assertEqual(
len(instrument_agent_data.associations['InstrumentAgent'].resources
), 1)
self.assertEqual(
instrument_agent_data.associations['InstrumentAgent'].resources[0].
_id, instrument_agent_id)
self.assertEqual(
len(instrument_agent_data.associations['InstrumentDevice'].
associated_resources), 1)
self.assertEqual(
instrument_agent_data.associations['InstrumentDevice'].
associated_resources[0].s, instrument_device_id)
self.assertEqual(
instrument_agent_data.associations['InstrumentDevice'].
associated_resources[0].o, instrument_agent_instance_id)
self.assertEqual(
len(instrument_agent_data.associations['InstrumentAgent'].
associated_resources), 1)
self.assertEqual(
instrument_agent_data.associations['InstrumentAgent'].
associated_resources[0].o, instrument_agent_id)
self.assertEqual(
instrument_agent_data.associations['InstrumentAgent'].
associated_resources[0].s, instrument_agent_instance_id)
self.assertEqual(
instrument_agent_data.associations['InstrumentAgent'].
assign_request.request_parameters['instrument_agent_instance_id'],
instrument_agent_instance_id)
#test prepare for update of data product to see if it is associated with the instrument that was created
data_product_data = self.DP.prepare_data_product_support(
data_product_id1)
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent=2)
self.assertEqual(data_product_data._id, data_product_id1)
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(
len(data_product_data.associations['StreamDefinition'].resources),
1)
self.assertEqual(
len(data_product_data.associations['Dataset'].resources), 0)
self.assertEqual(
len(data_product_data.associations['StreamDefinition'].
associated_resources), 1)
self.assertEqual(
data_product_data.associations['StreamDefinition'].
associated_resources[0].s, data_product_id1)
self.assertEqual(
len(data_product_data.associations['Dataset'].associated_resources
), 0)
self.assertEqual(
len(data_product_data.
associations['InstrumentDeviceHasOutputProduct'].resources), 3)
self.assertEqual(
len(data_product_data.associations[
'InstrumentDeviceHasOutputProduct'].associated_resources), 1)
self.assertEqual(
data_product_data.associations['InstrumentDeviceHasOutputProduct'].
associated_resources[0].s, instrument_device_id)
self.assertEqual(
data_product_data.associations['InstrumentDeviceHasOutputProduct'].
associated_resources[0].o, data_product_id1)
self.assertEqual(
len(data_product_data.associations['PlatformDevice'].resources), 2)
platform_data = self.IMS.prepare_platform_device_support()
#print simplejson.dumps(platform_data, default=ion_object_encoder, indent=2)
self.assertEqual(platform_data._id, '')
self.assertEqual(platform_data.type_, OT.PlatformDevicePrepareSupport)
self.assertEqual(
len(platform_data.associations['PlatformModel'].resources), 1)
self.assertEqual(
platform_data.associations['PlatformModel'].resources[0]._id,
platform_model_id)
self.assertEqual(
len(platform_data.associations['PlatformAgentInstance'].resources),
1)
self.assertEqual(
platform_data.associations['PlatformAgentInstance'].resources[0].
_id, platform_agent_instance_id)
self.assertEqual(
len(platform_data.associations['PlatformModel'].
associated_resources), 0)
self.assertEqual(
len(platform_data.associations['PlatformAgentInstance'].
associated_resources), 0)
self.assertEqual(
len(platform_data.associations['InstrumentDevice'].resources), 1)
platform_data = self.IMS.prepare_platform_device_support(
platform_device_id)
#print simplejson.dumps(platform_data, default=ion_object_encoder, indent=2)
self.assertEqual(platform_data._id, platform_device_id)
self.assertEqual(platform_data.type_, OT.PlatformDevicePrepareSupport)
self.assertEqual(
len(platform_data.associations['PlatformModel'].resources), 1)
self.assertEqual(
platform_data.associations['PlatformModel'].resources[0]._id,
platform_model_id)
self.assertEqual(
len(platform_data.associations['PlatformAgentInstance'].resources),
1)
self.assertEqual(
platform_data.associations['PlatformAgentInstance'].resources[0].
_id, platform_agent_instance_id)
self.assertEqual(
len(platform_data.associations['PlatformModel'].
associated_resources), 1)
self.assertEqual(
platform_data.associations['PlatformModel'].
associated_resources[0].s, platform_device_id)
self.assertEqual(
platform_data.associations['PlatformModel'].
associated_resources[0].o, platform_model_id)
self.assertEqual(
len(platform_data.associations['PlatformAgentInstance'].
associated_resources), 1)
self.assertEqual(
platform_data.associations['PlatformAgentInstance'].
associated_resources[0].o, platform_agent_instance_id)
self.assertEqual(
platform_data.associations['PlatformAgentInstance'].
associated_resources[0].s, platform_device_id)
self.assertEqual(
len(platform_data.associations['InstrumentDevice'].resources), 2)
#self.assertEqual(len(platform_data.associations['InstrumentDevice'].associated_resources), 1)
#self.assertEqual(platform_data.associations['InstrumentDevice'].associated_resources[0].s, platform_device_id)
#self.assertEqual(platform_data.associations['InstrumentDevice'].associated_resources[0].o, instrument_device_id)
self.assertEqual(
platform_data.associations['PlatformModel'].assign_request.
request_parameters['platform_device_id'], platform_device_id)
# cleanup
c = DotDict()
c.resource_registry = self.RR
self.RR2.pluck(instrument_agent_id)
self.RR2.pluck(instrument_model_id)
self.RR2.pluck(instrument_device_id)
self.RR2.pluck(platform_agent_id)
self.RR2.pluck(sensor_device_id)
self.RR2.pluck(sensor_device2_id)
self.IMS.force_delete_instrument_agent(instrument_agent_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_instrument_device(instrument_device2_id)
self.IMS.force_delete_platform_agent_instance(
platform_agent_instance_id)
self.IMS.force_delete_platform_agent(platform_agent_id)
self.IMS.force_delete_platform_device(platform_device_id)
self.IMS.force_delete_platform_device(platform_device2_id)
self.IMS.force_delete_platform_model(platform_model_id)
self.IMS.force_delete_sensor_device(sensor_device_id)
self.IMS.force_delete_sensor_device(sensor_device2_id)
self.IMS.force_delete_sensor_model(sensor_model_id)
#stuff we associate to
self.RR.delete(data_producer_id)
self.RR.delete(org_id)
class TestIdentityManagementServiceInt(IonIntegrationTestCase):
def setUp(self):
self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254"
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.resource_registry = ResourceRegistryServiceClient(
node=self.container.node)
self.identity_management_service = IdentityManagementServiceClient(
node=self.container.node)
self.org_client = OrgManagementServiceClient(node=self.container.node)
def test_actor_identity(self):
actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject})
user_id = self.identity_management_service.create_actor_identity(
actor_identity_obj)
actor_identity = self.identity_management_service.read_actor_identity(
user_id)
actor_identity.name = 'Updated subject'
self.identity_management_service.update_actor_identity(actor_identity)
ai = self.identity_management_service.find_actor_identity_by_name(
actor_identity.name)
self._baseAssertEqual(ai.name, actor_identity.name)
with self.assertRaises(NotFound):
ai = self.identity_management_service.find_actor_identity_by_name(
"Yeah, well, you know, that's just, like, your opinion, man.")
self._baseAssertEqual(ai.name, actor_identity.name)
self.identity_management_service.delete_actor_identity(user_id)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.read_actor_identity(user_id)
self.assertTrue("does not exist" in cm.exception.message)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.delete_actor_identity(user_id)
self.assertTrue("does not exist" in cm.exception.message)
def test_user_credentials(self):
actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject})
user_id = self.identity_management_service.create_actor_identity(
actor_identity_obj)
user_credentials_obj = IonObject("UserCredentials",
{"name": self.subject})
self.identity_management_service.register_user_credentials(
user_id, user_credentials_obj)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.unregister_user_credentials(
"bad", self.subject)
self.assertTrue("does not exist" in cm.exception.message)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.unregister_user_credentials(
user_id, "bad")
self.assertTrue("does not exist" in cm.exception.message)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.unregister_user_credentials(
'bad', 'bad')
self.assertTrue("does not exist" in cm.exception.message)
self.identity_management_service.unregister_user_credentials(
user_id, self.subject)
self.identity_management_service.delete_actor_identity(user_id)
def test_user_info(self):
actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject})
user_id = self.identity_management_service.create_actor_identity(
actor_identity_obj)
user_credentials_obj = IonObject("UserCredentials",
{"name": self.subject})
self.identity_management_service.register_user_credentials(
user_id, user_credentials_obj)
user_info_obj = IonObject("UserInfo", {"name": "Foo"})
user_info = self.identity_management_service.create_user_info(
user_id, user_info_obj)
with self.assertRaises(Conflict) as cm:
self.identity_management_service.create_user_info(
user_id, user_info_obj)
self.assertTrue(
"UserInfo already exists for user id" in cm.exception.message)
user_info_obj = self.identity_management_service.find_user_info_by_id(
user_id)
user_info_obj = self.identity_management_service.find_user_info_by_name(
"Foo")
user_info_obj = self.identity_management_service.find_user_info_by_subject(
self.subject)
user_info_obj = self.identity_management_service.read_user_info(
user_info)
user_info_obj.name = 'Jane Doe'
self.identity_management_service.update_user_info(user_info_obj)
self.identity_management_service.delete_user_info(user_info)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.read_user_info(user_info)
self.assertTrue('does not exist' in cm.exception.message)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.delete_user_info(user_info)
self.assertTrue('does not exist' in cm.exception.message)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.find_user_info_by_name("John Doe")
self.assertEqual(cm.exception.message,
'UserInfo with name John Doe does not exist')
with self.assertRaises(NotFound) as cm:
self.identity_management_service.find_user_info_by_subject(
"Bogus subject")
self.assertEqual(
cm.exception.message,
"UserCredentials with subject Bogus subject does not exist")
self.identity_management_service.unregister_user_credentials(
user_id, self.subject)
self.identity_management_service.delete_actor_identity(user_id)
def test_signon(self):
certificate = """-----BEGIN CERTIFICATE-----
MIIEMzCCAxugAwIBAgICBQAwDQYJKoZIhvcNAQEFBQAwajETMBEGCgmSJomT8ixkARkWA29yZzEX
MBUGCgmSJomT8ixkARkWB2NpbG9nb24xCzAJBgNVBAYTAlVTMRAwDgYDVQQKEwdDSUxvZ29uMRsw
GQYDVQQDExJDSUxvZ29uIEJhc2ljIENBIDEwHhcNMTAxMTE4MjIyNTA2WhcNMTAxMTE5MTAzMDA2
WjBvMRMwEQYKCZImiZPyLGQBGRMDb3JnMRcwFQYKCZImiZPyLGQBGRMHY2lsb2dvbjELMAkGA1UE
BhMCVVMxFzAVBgNVBAoTDlByb3RlY3ROZXR3b3JrMRkwFwYDVQQDExBSb2dlciBVbndpbiBBMjU0
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6QhsWxhUXbIxg+1ZyEc7d+hIGvchVmtb
g0kKLmivgoVsA4U7swNDRH6svW242THta0oTf6crkRx7kOKg6jma2lcAC1sjOSddqX7/92ChoUPq
7LWt2T6GVVA10ex5WAeB/o7br/Z4U8/75uCBis+ru7xEDl09PToK20mrkcz9M4HqIv1eSoPkrs3b
2lUtQc6cjuHRDU4NknXaVMXTBHKPM40UxEDHJueFyCiZJFg3lvQuSsAl4JL5Z8pC02T8/bODBuf4
dszsqn2SC8YDw1xrujvW2Bd7Q7BwMQ/gO+dZKM1mLJFpfEsR9WrjMeg6vkD2TMWLMr0/WIkGC8u+
6M6SMQIDAQABo4HdMIHaMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgSwMBMGA1UdJQQMMAoG
CCsGAQUFBwMCMBgGA1UdIAQRMA8wDQYLKwYBBAGCkTYBAgEwagYDVR0fBGMwYTAuoCygKoYoaHR0
cDovL2NybC5jaWxvZ29uLm9yZy9jaWxvZ29uLWJhc2ljLmNybDAvoC2gK4YpaHR0cDovL2NybC5k
b2Vncmlkcy5vcmcvY2lsb2dvbi1iYXNpYy5jcmwwHwYDVR0RBBgwFoEUaXRzYWdyZWVuMUB5YWhv
by5jb20wDQYJKoZIhvcNAQEFBQADggEBAEYHQPMY9Grs19MHxUzMwXp1GzCKhGpgyVKJKW86PJlr
HGruoWvx+DLNX75Oj5FC4t8bOUQVQusZGeGSEGegzzfIeOI/jWP1UtIjzvTFDq3tQMNvsgROSCx5
CkpK4nS0kbwLux+zI7BWON97UpMIzEeE05pd7SmNAETuWRsHMP+x6i7hoUp/uad4DwbzNUGIotdK
f8b270icOVgkOKRdLP/Q4r/x8skKSCRz1ZsRdR+7+B/EgksAJj7Ut3yiWoUekEMxCaTdAHPTMD/g
Mh9xL90hfMJyoGemjJswG5g3fAdTP/Lv0I6/nWeH/cLjwwpQgIEjEAVXl7KHuzX5vPD/wqQ=
-----END CERTIFICATE-----"""
id, valid_until, registered = self.identity_management_service.signon(
certificate, True)
self.assertFalse(registered)
id2, valid_until2, registered2 = self.identity_management_service.signon(
certificate, True)
self.assertFalse(registered2)
self.assertTrue(id == id2)
self.assertTrue(valid_until == valid_until2)
user_info_obj = IonObject("UserInfo", {"name": "Foo"})
self.identity_management_service.create_user_info(id, user_info_obj)
id3, valid_until3, registered3 = self.identity_management_service.signon(
certificate, True)
self.assertTrue(registered3)
self.assertTrue(id == id3)
self.assertTrue(valid_until == valid_until3)
@attr('EXT')
def test_get_extended_user_identity(self):
actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject})
actor_id = self.identity_management_service.create_actor_identity(
actor_identity_obj)
user_credentials_obj = IonObject("UserCredentials",
{"name": self.subject})
self.identity_management_service.register_user_credentials(
actor_id, user_credentials_obj)
user_info_obj = IonObject("UserInfo", {"name": "Foo"})
user_info_id = self.identity_management_service.create_user_info(
actor_id, user_info_obj)
ion_org = self.org_client.find_org()
#Build the Service Agreement Proposal to to request a role but never close it
sap = IonObject(OT.RequestRoleProposal,
consumer=actor_id,
provider=ion_org._id,
role_name=ORG_MANAGER_ROLE)
sap_response = self.org_client.negotiate(sap)
#Just grant the role anyway
#self.org_client.grant_role(ion_org._id, actor_id, ORG_MANAGER_ROLE)
with self.assertRaises(NotFound):
self.identity_management_service.get_user_info_extension(
'That rug really tied the room together.')
with self.assertRaises(BadRequest):
self.identity_management_service.get_user_info_extension()
#Check the user without the negotiation role request
extended_user = self.identity_management_service.get_user_info_extension(
user_info_id, org_id=ion_org._id)
self.assertEqual(user_info_obj.type_, extended_user.resource.type_)
self.assertEqual(len(extended_user.roles), 1)
self.assertEqual(len(extended_user.open_requests), 1)
self.assertEqual(extended_user.open_requests[0].org_id, ion_org._id)
self.assertEqual(extended_user.open_requests[0].user_id, user_info_id)
self.assertEqual(extended_user.open_requests[0].request_type,
OT.RequestRoleProposal)
self.assertEqual(len(extended_user.closed_requests), 0)
self.assertEqual(extended_user.open_requests[0]._id,
extended_user.open_requests[0].negotiation_id)
neg = self.resource_registry.read(
object_id=extended_user.open_requests[0].negotiation_id)
sap_response = Negotiation.create_counter_proposal(
neg, ProposalStatusEnum.ACCEPTED, ProposalOriginatorEnum.PROVIDER)
sap_response2 = self.org_client.negotiate(sap_response)
#Now check the user after the negotiation has been accepted and the role granted
extended_user = self.identity_management_service.get_user_info_extension(
user_info_id, org_id=ion_org._id)
self.assertEqual(user_info_obj.type_, extended_user.resource.type_)
self.assertEqual(len(extended_user.roles), 2)
self.assertEqual(len(extended_user.open_requests), 0)
self.assertEqual(len(extended_user.closed_requests), 1)
self.assertEqual(extended_user.closed_requests[0].org_id, ion_org._id)
self.assertEqual(extended_user.closed_requests[0].user_id,
user_info_id)
self.assertEqual(extended_user.closed_requests[0].request_type,
OT.RequestRoleProposal)
self.identity_management_service.delete_user_info(user_info_id)
self.org_client.revoke_role(org_id=ion_org._id,
actor_id=actor_id,
role_name=ORG_MANAGER_ROLE)
self.identity_management_service.unregister_user_credentials(
actor_id, self.subject)
self.identity_management_service.delete_actor_identity(actor_id)
def test_account_merge(self):
certificate = """-----BEGIN CERTIFICATE-----
MIIEMzCCAxugAwIBAgICBQAwDQYJKoZIhvcNAQEFBQAwajETMBEGCgmSJomT8ixkARkWA29yZzEX
MBUGCgmSJomT8ixkARkWB2NpbG9nb24xCzAJBgNVBAYTAlVTMRAwDgYDVQQKEwdDSUxvZ29uMRsw
GQYDVQQDExJDSUxvZ29uIEJhc2ljIENBIDEwHhcNMTAxMTE4MjIyNTA2WhcNMTAxMTE5MTAzMDA2
WjBvMRMwEQYKCZImiZPyLGQBGRMDb3JnMRcwFQYKCZImiZPyLGQBGRMHY2lsb2dvbjELMAkGA1UE
BhMCVVMxFzAVBgNVBAoTDlByb3RlY3ROZXR3b3JrMRkwFwYDVQQDExBSb2dlciBVbndpbiBBMjU0
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6QhsWxhUXbIxg+1ZyEc7d+hIGvchVmtb
g0kKLmivgoVsA4U7swNDRH6svW242THta0oTf6crkRx7kOKg6jma2lcAC1sjOSddqX7/92ChoUPq
7LWt2T6GVVA10ex5WAeB/o7br/Z4U8/75uCBis+ru7xEDl09PToK20mrkcz9M4HqIv1eSoPkrs3b
2lUtQc6cjuHRDU4NknXaVMXTBHKPM40UxEDHJueFyCiZJFg3lvQuSsAl4JL5Z8pC02T8/bODBuf4
dszsqn2SC8YDw1xrujvW2Bd7Q7BwMQ/gO+dZKM1mLJFpfEsR9WrjMeg6vkD2TMWLMr0/WIkGC8u+
6M6SMQIDAQABo4HdMIHaMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgSwMBMGA1UdJQQMMAoG
CCsGAQUFBwMCMBgGA1UdIAQRMA8wDQYLKwYBBAGCkTYBAgEwagYDVR0fBGMwYTAuoCygKoYoaHR0
cDovL2NybC5jaWxvZ29uLm9yZy9jaWxvZ29uLWJhc2ljLmNybDAvoC2gK4YpaHR0cDovL2NybC5k
b2Vncmlkcy5vcmcvY2lsb2dvbi1iYXNpYy5jcmwwHwYDVR0RBBgwFoEUaXRzYWdyZWVuMUB5YWhv
by5jb20wDQYJKoZIhvcNAQEFBQADggEBAEYHQPMY9Grs19MHxUzMwXp1GzCKhGpgyVKJKW86PJlr
HGruoWvx+DLNX75Oj5FC4t8bOUQVQusZGeGSEGegzzfIeOI/jWP1UtIjzvTFDq3tQMNvsgROSCx5
CkpK4nS0kbwLux+zI7BWON97UpMIzEeE05pd7SmNAETuWRsHMP+x6i7hoUp/uad4DwbzNUGIotdK
f8b270icOVgkOKRdLP/Q4r/x8skKSCRz1ZsRdR+7+B/EgksAJj7Ut3yiWoUekEMxCaTdAHPTMD/g
Mh9xL90hfMJyoGemjJswG5g3fAdTP/Lv0I6/nWeH/cLjwwpQgIEjEAVXl7KHuzX5vPD/wqQ=
-----END CERTIFICATE-----"""
subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254"
certificate_2 = """-----BEGIN CERTIFICATE-----
MIIEMzCCAxugAwIBAgIDAJ/lMA0GCSqGSIb3DQEBCwUAMGsxEzARBgoJkiaJk/IsZAEZFgNvcmcx
FzAVBgoJkiaJk/IsZAEZFgdjaWxvZ29uMQswCQYDVQQGEwJVUzEQMA4GA1UEChMHQ0lMb2dvbjEc
MBoGA1UEAxMTQ0lMb2dvbiBPcGVuSUQgQ0EgMTAeFw0xMjEwMTcwMDE2NDlaFw0xMjEwMTcxMjIx
NDlaMGkxEzARBgoJkiaJk/IsZAEZEwNvcmcxFzAVBgoJkiaJk/IsZAEZEwdjaWxvZ29uMQswCQYD
VQQGEwJVUzEPMA0GA1UEChMGR29vZ2xlMRswGQYDVQQDExJPd2VuIE93bmVycmVwIEE4OTMwggEi
MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDYLdpgg88sntivH+af4oamlp7blsUQcCQ5Yc/b
VDP/dwEKfxTcW36tMV3asLO7GcL7z4FESG761LAe86siT9rcwg2ttLkRjI9KeA3sFjC28N8XjKZ1
estCqG3odqw2pjo3VEFaU57219vIYMJhjmHKEgSnlMQeChMYun/sYIO5uNFba9BfiB6/PRS+bgee
cXRsIAm1vkB89AHdEjqdvH0uSN+jGjF6aAPXsESh70DUAHzs14lbFAomig7AZafT+weh0G5pnayC
lutVnhb9SyS3s1+A6kx8z9mkDUwY/NKXisuDeXa+WbRVq51D+Lc7ffOI+Ph+ynyfFGMcCBzbMADX
AgMBAAGjgeEwgd4wDAYDVR0TAQH/BAIwADAOBgNVHQ8BAf8EBAMCBLAwEwYDVR0lBAwwCgYIKwYB
BQUHAwIwGAYDVR0gBBEwDzANBgsrBgEEAYKRNgEDAzBsBgNVHR8EZTBjMC+gLaArhilodHRwOi8v
Y3JsLmNpbG9nb24ub3JnL2NpbG9nb24tb3BlbmlkLmNybDAwoC6gLIYqaHR0cDovL2NybC5kb2Vn
cmlkcy5vcmcvY2lsb2dvbi1vcGVuaWQuY3JsMCEGA1UdEQQaMBiBFm93ZW5vd25lcnJlcEBnbWFp
bC5jb20wDQYJKoZIhvcNAQELBQADggEBAHWd6ZOjSmJyOUyyLgZAPJpkSuk7DT5mFRhszJhfTGnu
gANHRIJZMs5e/LCMypE+ftxb8mnhAE+kURA2DmeucazHUDP5oYofU+8KMYqcNKnPpLnuiw+bCJPa
3BDxrYoi+vVislHb0U+QDjVYtUtQ2b1/Xhv8ShH89O9i65bbOq+sqez6z2AD9RWOEwRwpQLc9D65
9lkrsKGmJtuG8q3NTpZ1DSuaLOtn0QqttdmCg3pu5edRtgdpGadaSGR4s222JasV439bSTL8Z0Ug
HtjSclGqi8IBmvRkTZI61zTVbGdOKMP90LV1p8noJVLRkZpWRjLxI5xy9El8daAWMdjfrSc=
-----END CERTIFICATE-----"""
subject_2 = "/DC=org/DC=cilogon/C=US/O=Google/CN=Owen Ownerrep A893"
# Try to merge with nonexistent email account
with self.assertRaises(NotFound):
self.identity_management_service.initiate_account_merge(
"*****@*****.**")
with self.assertRaises(BadRequest):
self.identity_management_service.initiate_account_merge()
# Create two users
id, valid_until, registered = self.identity_management_service.signon(
certificate, True)
self.assertFalse(registered)
id_2, valid_until_2, registered_2 = self.identity_management_service.signon(
certificate_2, True)
self.assertFalse(registered_2)
# Validate the two accounts are different
self.assertNotEqual(
id, id_2, "The two accounts should have two different user id")
# Create UserInfo
contact_info_obj = IonObject("ContactInformation",
{"email": "*****@*****.**"})
user_info_obj = IonObject("UserInfo", {
"name": "Dude",
"contact": contact_info_obj
})
user_info_id = self.identity_management_service.create_user_info(
id, user_info_obj)
contact_info_obj_2 = IonObject("ContactInformation",
{"email": "*****@*****.**"})
user_info_obj_2 = IonObject("UserInfo", {
"name": "theDude",
"contact": contact_info_obj_2
})
user_info_id_2 = self.identity_management_service.create_user_info(
id_2, user_info_obj_2)
# Make sure the two users are registered
id, valid_until, registered = self.identity_management_service.signon(
certificate, True)
self.assertTrue(registered)
id_2, valid_until_2, registered_2 = self.identity_management_service.signon(
certificate_2, True)
self.assertTrue(registered_2)
token = self.identity_management_service.initiate_account_merge(
"*****@*****.**", headers={'ion-actor-id': id})
# Try merging accounts with invalid token string
with self.assertRaises(NotFound):
self.identity_management_service.complete_account_merge(
token_string="0xBeeF", headers={'ion-actor-id': id})
with self.assertRaises(BadRequest):
self.identity_management_service.complete_account_merge()
# Try merging accounts with a different user
# Since this user hasn't initiated account merge, the token doesn't exist in his/her UserInfo
with self.assertRaises(NotFound):
self.identity_management_service.complete_account_merge(
token, headers={'ion-actor-id': id_2})
self.identity_management_service.complete_account_merge(
token, headers={'ion-actor-id': id})
# Try merging the account again
with self.assertRaises(BadRequest):
self.identity_management_service.complete_account_merge(
token, headers={'ion-actor-id': id})
# Signon again and verify the two accounts have been merged
id, valid_until, registered = self.identity_management_service.signon(
certificate, True)
self.assertTrue(registered)
id_2, valid_until_2, registered_2 = self.identity_management_service.signon(
certificate_2, True)
self.assertTrue(registered_2)
# Validate the two accounts are the same
self.assertEqual(id, id_2, "The two accounts should have the same id")
# Try to merge to your own account
with self.assertRaises(BadRequest):
token = self.identity_management_service.initiate_account_merge(
"*****@*****.**", headers={'ion-actor-id': id})
# Done testing. Delete user
self.identity_management_service.delete_user_info(user_info_id)
self.identity_management_service.unregister_user_credentials(
id, subject)
self.identity_management_service.delete_actor_identity(id)
class TestIdentityManagementServiceInt(IonIntegrationTestCase):
def setUp(self):
self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254"
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2coi.yml')
self.identity_management_service = IdentityManagementServiceClient(node=self.container.node)
def test_actor_identity(self):
actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject})
user_id = self.identity_management_service.create_actor_identity(actor_identity_obj)
actor_identity = self.identity_management_service.read_actor_identity(user_id)
actor_identity.name = 'Updated subject'
self.identity_management_service.update_actor_identity(actor_identity)
self.identity_management_service.delete_actor_identity(user_id)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.read_actor_identity(user_id)
self.assertTrue("does not exist" in cm.exception.message)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.delete_actor_identity(user_id)
self.assertTrue("does not exist" in cm.exception.message)
def test_user_credentials(self):
actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject})
user_id = self.identity_management_service.create_actor_identity(actor_identity_obj)
user_credentials_obj = IonObject("UserCredentials", {"name": self.subject})
self.identity_management_service.register_user_credentials(user_id, user_credentials_obj)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.unregister_user_credentials("bad", self.subject)
self.assertTrue("does not exist" in cm.exception.message)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.unregister_user_credentials(user_id, "bad")
self.assertTrue("does not exist" in cm.exception.message)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.unregister_user_credentials('bad', 'bad')
self.assertTrue("does not exist" in cm.exception.message)
self.identity_management_service.unregister_user_credentials(user_id, self.subject)
self.identity_management_service.delete_actor_identity(user_id)
def test_user_info(self):
actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject})
user_id = self.identity_management_service.create_actor_identity(actor_identity_obj)
user_credentials_obj = IonObject("UserCredentials", {"name": self.subject})
self.identity_management_service.register_user_credentials(user_id, user_credentials_obj)
user_info_obj = IonObject("UserInfo", {"name": "Foo"})
user_info = self.identity_management_service.create_user_info(user_id, user_info_obj)
with self.assertRaises(Conflict) as cm:
self.identity_management_service.create_user_info(user_id, user_info_obj)
self.assertTrue("UserInfo already exists for user id" in cm.exception.message)
user_info_obj = self.identity_management_service.find_user_info_by_id(user_id)
user_info_obj = self.identity_management_service.find_user_info_by_name("Foo")
user_info_obj = self.identity_management_service.find_user_info_by_subject(self.subject)
user_info_obj = self.identity_management_service.read_user_info(user_info)
user_info_obj.name = 'Jane Doe'
self.identity_management_service.update_user_info(user_info_obj)
self.identity_management_service.delete_user_info(user_info)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.read_user_info(user_info)
self.assertTrue('does not exist' in cm.exception.message)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.delete_user_info(user_info)
self.assertTrue('does not exist' in cm.exception.message)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.find_user_info_by_name("John Doe")
self.assertEqual(cm.exception.message, 'UserInfo with name John Doe does not exist')
with self.assertRaises(NotFound) as cm:
self.identity_management_service.find_user_info_by_subject("Bogus subject")
self.assertEqual(cm.exception.message, "UserCredentials with subject Bogus subject does not exist")
self.identity_management_service.unregister_user_credentials(user_id, self.subject)
self.identity_management_service.delete_actor_identity(user_id)
def test_signon(self):
certificate = """-----BEGIN CERTIFICATE-----
MIIEMzCCAxugAwIBAgICBQAwDQYJKoZIhvcNAQEFBQAwajETMBEGCgmSJomT8ixkARkWA29yZzEX
MBUGCgmSJomT8ixkARkWB2NpbG9nb24xCzAJBgNVBAYTAlVTMRAwDgYDVQQKEwdDSUxvZ29uMRsw
GQYDVQQDExJDSUxvZ29uIEJhc2ljIENBIDEwHhcNMTAxMTE4MjIyNTA2WhcNMTAxMTE5MTAzMDA2
WjBvMRMwEQYKCZImiZPyLGQBGRMDb3JnMRcwFQYKCZImiZPyLGQBGRMHY2lsb2dvbjELMAkGA1UE
BhMCVVMxFzAVBgNVBAoTDlByb3RlY3ROZXR3b3JrMRkwFwYDVQQDExBSb2dlciBVbndpbiBBMjU0
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6QhsWxhUXbIxg+1ZyEc7d+hIGvchVmtb
g0kKLmivgoVsA4U7swNDRH6svW242THta0oTf6crkRx7kOKg6jma2lcAC1sjOSddqX7/92ChoUPq
7LWt2T6GVVA10ex5WAeB/o7br/Z4U8/75uCBis+ru7xEDl09PToK20mrkcz9M4HqIv1eSoPkrs3b
2lUtQc6cjuHRDU4NknXaVMXTBHKPM40UxEDHJueFyCiZJFg3lvQuSsAl4JL5Z8pC02T8/bODBuf4
dszsqn2SC8YDw1xrujvW2Bd7Q7BwMQ/gO+dZKM1mLJFpfEsR9WrjMeg6vkD2TMWLMr0/WIkGC8u+
6M6SMQIDAQABo4HdMIHaMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgSwMBMGA1UdJQQMMAoG
CCsGAQUFBwMCMBgGA1UdIAQRMA8wDQYLKwYBBAGCkTYBAgEwagYDVR0fBGMwYTAuoCygKoYoaHR0
cDovL2NybC5jaWxvZ29uLm9yZy9jaWxvZ29uLWJhc2ljLmNybDAvoC2gK4YpaHR0cDovL2NybC5k
b2Vncmlkcy5vcmcvY2lsb2dvbi1iYXNpYy5jcmwwHwYDVR0RBBgwFoEUaXRzYWdyZWVuMUB5YWhv
by5jb20wDQYJKoZIhvcNAQEFBQADggEBAEYHQPMY9Grs19MHxUzMwXp1GzCKhGpgyVKJKW86PJlr
HGruoWvx+DLNX75Oj5FC4t8bOUQVQusZGeGSEGegzzfIeOI/jWP1UtIjzvTFDq3tQMNvsgROSCx5
CkpK4nS0kbwLux+zI7BWON97UpMIzEeE05pd7SmNAETuWRsHMP+x6i7hoUp/uad4DwbzNUGIotdK
f8b270icOVgkOKRdLP/Q4r/x8skKSCRz1ZsRdR+7+B/EgksAJj7Ut3yiWoUekEMxCaTdAHPTMD/g
Mh9xL90hfMJyoGemjJswG5g3fAdTP/Lv0I6/nWeH/cLjwwpQgIEjEAVXl7KHuzX5vPD/wqQ=
-----END CERTIFICATE-----"""
id, valid_until, registered = self.identity_management_service.signon(certificate, True)
self.assertFalse(registered)
id2, valid_until2, registered2 = self.identity_management_service.signon(certificate, True)
self.assertFalse(registered2)
self.assertTrue(id == id2)
self.assertTrue(valid_until == valid_until2)
user_info_obj = IonObject("UserInfo", {"name": "Foo"})
self.identity_management_service.create_user_info(id, user_info_obj)
id3, valid_until3, registered3 = self.identity_management_service.signon(certificate, True)
self.assertTrue(registered3)
self.assertTrue(id == id3)
self.assertTrue(valid_until == valid_until3)
class TestIdentityManagementServiceInt(IonIntegrationTestCase):
def setUp(self):
self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254"
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.resource_registry = ResourceRegistryServiceClient(node=self.container.node)
self.identity_management_service = IdentityManagementServiceClient(node=self.container.node)
self.org_client = OrgManagementServiceClient(node=self.container.node)
def test_actor_identity(self):
actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject})
user_id = self.identity_management_service.create_actor_identity(actor_identity_obj)
actor_identity = self.identity_management_service.read_actor_identity(user_id)
actor_identity.name = 'Updated subject'
self.identity_management_service.update_actor_identity(actor_identity)
ai = self.identity_management_service.find_actor_identity_by_name(actor_identity.name)
self._baseAssertEqual(ai.name, actor_identity.name)
with self.assertRaises(NotFound):
ai = self.identity_management_service.find_actor_identity_by_name("Yeah, well, you know, that's just, like, your opinion, man.")
self._baseAssertEqual(ai.name, actor_identity.name)
self.identity_management_service.delete_actor_identity(user_id)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.read_actor_identity(user_id)
self.assertTrue("does not exist" in cm.exception.message)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.delete_actor_identity(user_id)
self.assertTrue("does not exist" in cm.exception.message)
def test_user_credentials(self):
actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject})
user_id = self.identity_management_service.create_actor_identity(actor_identity_obj)
user_credentials_obj = IonObject("UserCredentials", {"name": self.subject})
self.identity_management_service.register_user_credentials(user_id, user_credentials_obj)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.unregister_user_credentials("bad", self.subject)
self.assertTrue("does not exist" in cm.exception.message)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.unregister_user_credentials(user_id, "bad")
self.assertTrue("does not exist" in cm.exception.message)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.unregister_user_credentials('bad', 'bad')
self.assertTrue("does not exist" in cm.exception.message)
self.identity_management_service.unregister_user_credentials(user_id, self.subject)
self.identity_management_service.delete_actor_identity(user_id)
def test_user_info(self):
actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject})
user_id = self.identity_management_service.create_actor_identity(actor_identity_obj)
user_credentials_obj = IonObject("UserCredentials", {"name": self.subject})
self.identity_management_service.register_user_credentials(user_id, user_credentials_obj)
user_info_obj = IonObject("UserInfo", {"name": "Foo"})
user_info = self.identity_management_service.create_user_info(user_id, user_info_obj)
with self.assertRaises(Conflict) as cm:
self.identity_management_service.create_user_info(user_id, user_info_obj)
self.assertTrue("UserInfo already exists for user id" in cm.exception.message)
user_info_obj = self.identity_management_service.find_user_info_by_id(user_id)
user_info_obj = self.identity_management_service.find_user_info_by_name("Foo")
user_info_obj = self.identity_management_service.find_user_info_by_subject(self.subject)
user_info_obj = self.identity_management_service.read_user_info(user_info)
user_info_obj.name = 'Jane Doe'
self.identity_management_service.update_user_info(user_info_obj)
self.identity_management_service.delete_user_info(user_info)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.read_user_info(user_info)
self.assertTrue('does not exist' in cm.exception.message)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.delete_user_info(user_info)
self.assertTrue('does not exist' in cm.exception.message)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.find_user_info_by_name("John Doe")
self.assertEqual(cm.exception.message, 'UserInfo with name John Doe does not exist')
with self.assertRaises(NotFound) as cm:
self.identity_management_service.find_user_info_by_subject("Bogus subject")
self.assertEqual(cm.exception.message, "UserCredentials with subject Bogus subject does not exist")
self.identity_management_service.unregister_user_credentials(user_id, self.subject)
self.identity_management_service.delete_actor_identity(user_id)
def test_signon(self):
certificate = """-----BEGIN CERTIFICATE-----
MIIEMzCCAxugAwIBAgICBQAwDQYJKoZIhvcNAQEFBQAwajETMBEGCgmSJomT8ixkARkWA29yZzEX
MBUGCgmSJomT8ixkARkWB2NpbG9nb24xCzAJBgNVBAYTAlVTMRAwDgYDVQQKEwdDSUxvZ29uMRsw
GQYDVQQDExJDSUxvZ29uIEJhc2ljIENBIDEwHhcNMTAxMTE4MjIyNTA2WhcNMTAxMTE5MTAzMDA2
WjBvMRMwEQYKCZImiZPyLGQBGRMDb3JnMRcwFQYKCZImiZPyLGQBGRMHY2lsb2dvbjELMAkGA1UE
BhMCVVMxFzAVBgNVBAoTDlByb3RlY3ROZXR3b3JrMRkwFwYDVQQDExBSb2dlciBVbndpbiBBMjU0
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6QhsWxhUXbIxg+1ZyEc7d+hIGvchVmtb
g0kKLmivgoVsA4U7swNDRH6svW242THta0oTf6crkRx7kOKg6jma2lcAC1sjOSddqX7/92ChoUPq
7LWt2T6GVVA10ex5WAeB/o7br/Z4U8/75uCBis+ru7xEDl09PToK20mrkcz9M4HqIv1eSoPkrs3b
2lUtQc6cjuHRDU4NknXaVMXTBHKPM40UxEDHJueFyCiZJFg3lvQuSsAl4JL5Z8pC02T8/bODBuf4
dszsqn2SC8YDw1xrujvW2Bd7Q7BwMQ/gO+dZKM1mLJFpfEsR9WrjMeg6vkD2TMWLMr0/WIkGC8u+
6M6SMQIDAQABo4HdMIHaMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgSwMBMGA1UdJQQMMAoG
CCsGAQUFBwMCMBgGA1UdIAQRMA8wDQYLKwYBBAGCkTYBAgEwagYDVR0fBGMwYTAuoCygKoYoaHR0
cDovL2NybC5jaWxvZ29uLm9yZy9jaWxvZ29uLWJhc2ljLmNybDAvoC2gK4YpaHR0cDovL2NybC5k
b2Vncmlkcy5vcmcvY2lsb2dvbi1iYXNpYy5jcmwwHwYDVR0RBBgwFoEUaXRzYWdyZWVuMUB5YWhv
by5jb20wDQYJKoZIhvcNAQEFBQADggEBAEYHQPMY9Grs19MHxUzMwXp1GzCKhGpgyVKJKW86PJlr
HGruoWvx+DLNX75Oj5FC4t8bOUQVQusZGeGSEGegzzfIeOI/jWP1UtIjzvTFDq3tQMNvsgROSCx5
CkpK4nS0kbwLux+zI7BWON97UpMIzEeE05pd7SmNAETuWRsHMP+x6i7hoUp/uad4DwbzNUGIotdK
f8b270icOVgkOKRdLP/Q4r/x8skKSCRz1ZsRdR+7+B/EgksAJj7Ut3yiWoUekEMxCaTdAHPTMD/g
Mh9xL90hfMJyoGemjJswG5g3fAdTP/Lv0I6/nWeH/cLjwwpQgIEjEAVXl7KHuzX5vPD/wqQ=
-----END CERTIFICATE-----"""
id, valid_until, registered = self.identity_management_service.signon(certificate, True)
self.assertFalse(registered)
id2, valid_until2, registered2 = self.identity_management_service.signon(certificate, True)
self.assertFalse(registered2)
self.assertTrue(id == id2)
self.assertTrue(valid_until == valid_until2)
user_info_obj = IonObject("UserInfo", {"name": "Foo"})
self.identity_management_service.create_user_info(id, user_info_obj)
id3, valid_until3, registered3 = self.identity_management_service.signon(certificate, True)
self.assertTrue(registered3)
self.assertTrue(id == id3)
self.assertTrue(valid_until == valid_until3)
@attr('EXT')
def test_get_extended_user_identity(self):
actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject})
actor_id = self.identity_management_service.create_actor_identity(actor_identity_obj)
user_credentials_obj = IonObject("UserCredentials", {"name": self.subject})
self.identity_management_service.register_user_credentials(actor_id, user_credentials_obj)
user_info_obj = IonObject("UserInfo", {"name": "Foo"})
user_info_id = self.identity_management_service.create_user_info(actor_id, user_info_obj)
ion_org = self.org_client.find_org()
#Build the Service Agreement Proposal to to request a role but never close it
sap = IonObject(OT.RequestRoleProposal,consumer=actor_id, provider=ion_org._id, role_name=ORG_MANAGER_ROLE )
sap_response = self.org_client.negotiate(sap)
#Just grant the role anyway
#self.org_client.grant_role(ion_org._id, actor_id, ORG_MANAGER_ROLE)
with self.assertRaises(NotFound):
self.identity_management_service.get_user_info_extension('That rug really tied the room together.')
with self.assertRaises(BadRequest):
self.identity_management_service.get_user_info_extension()
#Check the user without the negotiation role request
extended_user = self.identity_management_service.get_user_info_extension(user_info_id, org_id=ion_org._id)
self.assertEqual(user_info_obj.type_,extended_user.resource.type_)
self.assertEqual(len(extended_user.roles),1)
self.assertEqual(len(extended_user.open_requests),1)
self.assertEqual(extended_user.open_requests[0].org_id, ion_org._id)
self.assertEqual(extended_user.open_requests[0].user_id, user_info_id)
self.assertEqual(extended_user.open_requests[0].request_type, OT.RequestRoleProposal)
self.assertEqual(len(extended_user.closed_requests),0)
self.assertEqual(extended_user.open_requests[0]._id, extended_user.open_requests[0].negotiation_id)
neg = self.resource_registry.read(object_id=extended_user.open_requests[0].negotiation_id)
sap_response = Negotiation.create_counter_proposal(neg, ProposalStatusEnum.ACCEPTED, ProposalOriginatorEnum.PROVIDER)
sap_response2 = self.org_client.negotiate(sap_response)
#Now check the user after the negotiation has been accepted and the role granted
extended_user = self.identity_management_service.get_user_info_extension(user_info_id, org_id=ion_org._id)
self.assertEqual(user_info_obj.type_,extended_user.resource.type_)
self.assertEqual(len(extended_user.roles),2)
self.assertEqual(len(extended_user.open_requests),0)
self.assertEqual(len(extended_user.closed_requests),1)
self.assertEqual(extended_user.closed_requests[0].org_id, ion_org._id)
self.assertEqual(extended_user.closed_requests[0].user_id, user_info_id)
self.assertEqual(extended_user.closed_requests[0].request_type, OT.RequestRoleProposal)
self.identity_management_service.delete_user_info(user_info_id)
self.org_client.revoke_role(org_id=ion_org._id, actor_id=actor_id, role_name=ORG_MANAGER_ROLE)
self.identity_management_service.unregister_user_credentials(actor_id, self.subject)
self.identity_management_service.delete_actor_identity(actor_id)
def test_account_merge(self):
certificate = """-----BEGIN CERTIFICATE-----
MIIEMzCCAxugAwIBAgICBQAwDQYJKoZIhvcNAQEFBQAwajETMBEGCgmSJomT8ixkARkWA29yZzEX
MBUGCgmSJomT8ixkARkWB2NpbG9nb24xCzAJBgNVBAYTAlVTMRAwDgYDVQQKEwdDSUxvZ29uMRsw
GQYDVQQDExJDSUxvZ29uIEJhc2ljIENBIDEwHhcNMTAxMTE4MjIyNTA2WhcNMTAxMTE5MTAzMDA2
WjBvMRMwEQYKCZImiZPyLGQBGRMDb3JnMRcwFQYKCZImiZPyLGQBGRMHY2lsb2dvbjELMAkGA1UE
BhMCVVMxFzAVBgNVBAoTDlByb3RlY3ROZXR3b3JrMRkwFwYDVQQDExBSb2dlciBVbndpbiBBMjU0
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6QhsWxhUXbIxg+1ZyEc7d+hIGvchVmtb
g0kKLmivgoVsA4U7swNDRH6svW242THta0oTf6crkRx7kOKg6jma2lcAC1sjOSddqX7/92ChoUPq
7LWt2T6GVVA10ex5WAeB/o7br/Z4U8/75uCBis+ru7xEDl09PToK20mrkcz9M4HqIv1eSoPkrs3b
2lUtQc6cjuHRDU4NknXaVMXTBHKPM40UxEDHJueFyCiZJFg3lvQuSsAl4JL5Z8pC02T8/bODBuf4
dszsqn2SC8YDw1xrujvW2Bd7Q7BwMQ/gO+dZKM1mLJFpfEsR9WrjMeg6vkD2TMWLMr0/WIkGC8u+
6M6SMQIDAQABo4HdMIHaMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgSwMBMGA1UdJQQMMAoG
CCsGAQUFBwMCMBgGA1UdIAQRMA8wDQYLKwYBBAGCkTYBAgEwagYDVR0fBGMwYTAuoCygKoYoaHR0
cDovL2NybC5jaWxvZ29uLm9yZy9jaWxvZ29uLWJhc2ljLmNybDAvoC2gK4YpaHR0cDovL2NybC5k
b2Vncmlkcy5vcmcvY2lsb2dvbi1iYXNpYy5jcmwwHwYDVR0RBBgwFoEUaXRzYWdyZWVuMUB5YWhv
by5jb20wDQYJKoZIhvcNAQEFBQADggEBAEYHQPMY9Grs19MHxUzMwXp1GzCKhGpgyVKJKW86PJlr
HGruoWvx+DLNX75Oj5FC4t8bOUQVQusZGeGSEGegzzfIeOI/jWP1UtIjzvTFDq3tQMNvsgROSCx5
CkpK4nS0kbwLux+zI7BWON97UpMIzEeE05pd7SmNAETuWRsHMP+x6i7hoUp/uad4DwbzNUGIotdK
f8b270icOVgkOKRdLP/Q4r/x8skKSCRz1ZsRdR+7+B/EgksAJj7Ut3yiWoUekEMxCaTdAHPTMD/g
Mh9xL90hfMJyoGemjJswG5g3fAdTP/Lv0I6/nWeH/cLjwwpQgIEjEAVXl7KHuzX5vPD/wqQ=
-----END CERTIFICATE-----"""
subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254"
certificate_2 = """-----BEGIN CERTIFICATE-----
MIIEMzCCAxugAwIBAgIDAJ/lMA0GCSqGSIb3DQEBCwUAMGsxEzARBgoJkiaJk/IsZAEZFgNvcmcx
FzAVBgoJkiaJk/IsZAEZFgdjaWxvZ29uMQswCQYDVQQGEwJVUzEQMA4GA1UEChMHQ0lMb2dvbjEc
MBoGA1UEAxMTQ0lMb2dvbiBPcGVuSUQgQ0EgMTAeFw0xMjEwMTcwMDE2NDlaFw0xMjEwMTcxMjIx
NDlaMGkxEzARBgoJkiaJk/IsZAEZEwNvcmcxFzAVBgoJkiaJk/IsZAEZEwdjaWxvZ29uMQswCQYD
VQQGEwJVUzEPMA0GA1UEChMGR29vZ2xlMRswGQYDVQQDExJPd2VuIE93bmVycmVwIEE4OTMwggEi
MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDYLdpgg88sntivH+af4oamlp7blsUQcCQ5Yc/b
VDP/dwEKfxTcW36tMV3asLO7GcL7z4FESG761LAe86siT9rcwg2ttLkRjI9KeA3sFjC28N8XjKZ1
estCqG3odqw2pjo3VEFaU57219vIYMJhjmHKEgSnlMQeChMYun/sYIO5uNFba9BfiB6/PRS+bgee
cXRsIAm1vkB89AHdEjqdvH0uSN+jGjF6aAPXsESh70DUAHzs14lbFAomig7AZafT+weh0G5pnayC
lutVnhb9SyS3s1+A6kx8z9mkDUwY/NKXisuDeXa+WbRVq51D+Lc7ffOI+Ph+ynyfFGMcCBzbMADX
AgMBAAGjgeEwgd4wDAYDVR0TAQH/BAIwADAOBgNVHQ8BAf8EBAMCBLAwEwYDVR0lBAwwCgYIKwYB
BQUHAwIwGAYDVR0gBBEwDzANBgsrBgEEAYKRNgEDAzBsBgNVHR8EZTBjMC+gLaArhilodHRwOi8v
Y3JsLmNpbG9nb24ub3JnL2NpbG9nb24tb3BlbmlkLmNybDAwoC6gLIYqaHR0cDovL2NybC5kb2Vn
cmlkcy5vcmcvY2lsb2dvbi1vcGVuaWQuY3JsMCEGA1UdEQQaMBiBFm93ZW5vd25lcnJlcEBnbWFp
bC5jb20wDQYJKoZIhvcNAQELBQADggEBAHWd6ZOjSmJyOUyyLgZAPJpkSuk7DT5mFRhszJhfTGnu
gANHRIJZMs5e/LCMypE+ftxb8mnhAE+kURA2DmeucazHUDP5oYofU+8KMYqcNKnPpLnuiw+bCJPa
3BDxrYoi+vVislHb0U+QDjVYtUtQ2b1/Xhv8ShH89O9i65bbOq+sqez6z2AD9RWOEwRwpQLc9D65
9lkrsKGmJtuG8q3NTpZ1DSuaLOtn0QqttdmCg3pu5edRtgdpGadaSGR4s222JasV439bSTL8Z0Ug
HtjSclGqi8IBmvRkTZI61zTVbGdOKMP90LV1p8noJVLRkZpWRjLxI5xy9El8daAWMdjfrSc=
-----END CERTIFICATE-----"""
subject_2 = "/DC=org/DC=cilogon/C=US/O=Google/CN=Owen Ownerrep A893"
# Try to merge with nonexistent email account
with self.assertRaises(NotFound):
self.identity_management_service.initiate_account_merge("*****@*****.**")
with self.assertRaises(BadRequest):
self.identity_management_service.initiate_account_merge()
# Create two users
id, valid_until, registered = self.identity_management_service.signon(certificate, True)
self.assertFalse(registered)
id_2, valid_until_2, registered_2 = self.identity_management_service.signon(certificate_2, True)
self.assertFalse(registered_2)
# Validate the two accounts are different
self.assertNotEqual(id, id_2, "The two accounts should have two different user id")
# Create UserInfo
contact_info_obj = IonObject("ContactInformation",{"email": "*****@*****.**"})
user_info_obj = IonObject("UserInfo", {"name": "Dude", "contact": contact_info_obj})
user_info_id = self.identity_management_service.create_user_info(id, user_info_obj)
contact_info_obj_2 = IonObject("ContactInformation",{"email": "*****@*****.**"})
user_info_obj_2 = IonObject("UserInfo", {"name": "theDude", "contact": contact_info_obj_2})
user_info_id_2 = self.identity_management_service.create_user_info(id_2, user_info_obj_2)
# Make sure the two users are registered
id, valid_until, registered = self.identity_management_service.signon(certificate, True)
self.assertTrue(registered)
id_2, valid_until_2, registered_2 = self.identity_management_service.signon(certificate_2, True)
self.assertTrue(registered_2)
token = self.identity_management_service.initiate_account_merge("*****@*****.**", headers={'ion-actor-id':id})
# Try merging accounts with invalid token string
with self.assertRaises(NotFound):
self.identity_management_service.complete_account_merge(token_string="0xBeeF", headers={'ion-actor-id':id})
with self.assertRaises(BadRequest):
self.identity_management_service.complete_account_merge()
# Try merging accounts with a different user
# Since this user hasn't initiated account merge, the token doesn't exist in his/her UserInfo
with self.assertRaises(NotFound):
self.identity_management_service.complete_account_merge(token, headers={'ion-actor-id':id_2})
self.identity_management_service.complete_account_merge(token, headers={'ion-actor-id':id})
# Try merging the account again
with self.assertRaises(BadRequest):
self.identity_management_service.complete_account_merge(token, headers={'ion-actor-id':id})
# Signon again and verify the two accounts have been merged
id, valid_until, registered = self.identity_management_service.signon(certificate, True)
self.assertTrue(registered)
id_2, valid_until_2, registered_2 = self.identity_management_service.signon(certificate_2, True)
self.assertTrue(registered_2)
# Validate the two accounts are the same
self.assertEqual(id, id_2, "The two accounts should have the same id")
# Try to merge to your own account
with self.assertRaises(BadRequest):
token = self.identity_management_service.initiate_account_merge("*****@*****.**", headers={'ion-actor-id':id})
# Done testing. Delete user
self.identity_management_service.delete_user_info(user_info_id)
self.identity_management_service.unregister_user_credentials(id, subject)
self.identity_management_service.delete_actor_identity(id)
def test_non_anonymous_resource_registry_operations_with_token(self):
rr = self.container.resource_registry
id_client = IdentityManagementServiceClient()
create_request = {
"serviceRequest": {
"serviceName": "resource_registry",
"serviceOp": "create",
"params": {"object": {"name": "Instrument1", "type_": "InstrumentDevice"}},
}
}
# Create without actor
response = self.test_app.post(
"/ion-service/resource_registry/create", {"payload": simplejson.dumps(create_request)}
)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json["data"])
response_data = response.json["data"][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertGreaterEqual(len(response_data[0]), 20) # This is a resource_id
inst_id = str(response_data[0])
inst_obj = rr.read(inst_id)
self.assertEquals(inst_obj.type_, RT.InstrumentDevice)
self.assertEquals(inst_obj.name, "Instrument1")
self.assertEquals(inst_obj.visibility, ResourceVisibilityEnum.PUBLIC)
act_objs, assocs = rr.find_objects(inst_id, PRED.hasOwner, RT.ActorIdentity, id_only=False)
self.assertEquals(len(act_objs), 0)
# Anonymous query shows PUBLIC visibility resource anymore
response = self.test_app.get("/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True")
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json["data"])
response_data = response.json["data"][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 1)
self.assertEqual(len(response_data[1]), 1)
self.assertEqual(response_data[0][0], inst_id)
inst_obj.visibility = ResourceVisibilityEnum.OWNER
rr.update(inst_obj)
# Now the anonymous query should not show the resource anymore
response = self.test_app.get("/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True")
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json["data"])
response_data = response.json["data"][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 0)
rr.delete(inst_id)
# Create with actor
actor_id, valid_until, registered = id_client.signon(USER1_CERTIFICATE, True)
create_request = {
"serviceRequest": {
"serviceName": "resource_registry",
"serviceOp": "create",
"requester": actor_id,
"params": {
"object": {
"name": "Instrument1",
"type_": "InstrumentDevice",
"visibility": ResourceVisibilityEnum.OWNER,
}
},
}
}
response = self.test_app.post(
"/ion-service/resource_registry/create", {"payload": simplejson.dumps(create_request)}
)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json["data"])
response_data = response.json["data"][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertGreaterEqual(len(response_data[0]), 20) # This is a resource_id
inst_id = str(response_data[0])
inst_obj = rr.read(inst_id)
self.assertEquals(inst_obj.type_, RT.InstrumentDevice)
self.assertEquals(inst_obj.name, "Instrument1")
self.assertEquals(inst_obj.visibility, ResourceVisibilityEnum.OWNER)
act_objs, assocs = rr.find_objects(inst_id, PRED.hasOwner, RT.ActorIdentity, id_only=False)
self.assertEquals(len(act_objs), 1)
self.assertEquals(act_objs[0]._id, actor_id)
# Anonymous query should not show the resource anymore
response = self.test_app.get("/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True")
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json["data"])
response_data = response.json["data"][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 0)
# Authenticated request with owner shows resource
response = self.test_app.get(
"/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&requester=" + actor_id
)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json["data"])
response_data = response.json["data"][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 1)
self.assertEqual(len(response_data[1]), 1)
self.assertEqual(response_data[0][0], inst_id)
token_str = id_client.create_authentication_token(actor_id, validity=2)
# Request with authentication token for owner shows resource
response = self.test_app.get(
"/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&authtoken=" + token_str
)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json["data"])
response_data = response.json["data"][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 1)
self.assertEqual(len(response_data[1]), 1)
self.assertEqual(response_data[0][0], inst_id)
gevent.sleep(2.1)
# Request with expired authentication token for owner does not show resource (no error though)
response = self.test_app.get(
"/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&authtoken=" + token_str
)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json["data"])
response_data = response.json["data"][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 0)
# Request with valid authentication token for different user does not show resource
actor_id2, _ = rr.create(IonObject(RT.ActorIdentity, name="Actor2"))
token_str2 = id_client.create_authentication_token(actor_id2, validity=2)
response = self.test_app.get(
"/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&authtoken=" + token_str2
)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json["data"])
response_data = response.json["data"][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 0)
token_str3 = id_client.create_authentication_token(actor_id, validity=2)
# Request with new authentication token for owner shows resource
response = self.test_app.get(
"/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&authtoken=" + token_str3
)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json["data"])
response_data = response.json["data"][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 1)
self.assertEqual(len(response_data[1]), 1)
self.assertEqual(response_data[0][0], inst_id)
id_client.invalidate_authentication_token(token_str3)
response = self.test_app.get(
"/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&authtoken=" + token_str3
)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json["data"])
response_data = response.json["data"][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 0)
# Cleanup
rr.delete(inst_id)
rr.delete(actor_id2)
id_client.delete_actor_identity(actor_id)
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
print 'started services'
# @unittest.skip('this test just for debugging setup')
# def test_just_the_setup(self):
# return
@attr('EXT')
def test_resources_associations_extensions(self):
"""
create one of each resource and association used by IMS
to guard against problems in ion-definitions
"""
#stuff we control
instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance))
instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent))
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance))
platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent))
platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice))
platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel))
sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice))
sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel))
#stuff we associate to
data_producer_id, _ = self.RR.create(any_old(RT.DataProducer))
org_id, _ = self.RR.create(any_old(RT.Org))
#instrument_agent_instance_id #is only a target
#instrument_agent
self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id)
#instrument_device
self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id)
self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id)
self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id)
self.RR.create_association(org_id, PRED.hasResource, instrument_device_id)
instrument_model_id #is only a target
platform_agent_instance_id #is only a target
#platform_agent
self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id)
#platform_device
self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id)
self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id)
platform_model_id #is only a target
#sensor_device
self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id)
self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id)
sensor_model_id #is only a target
#create a parsed product for this instrument output
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
processing_level_code='Parsed_Canonical',
temporal_domain = tdom,
spatial_domain = sdom)
pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1)
def addInstOwner(inst_id, subject):
actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject})
user_id = self.IDS.create_actor_identity(actor_identity_obj)
user_info_obj = any_old(RT.UserInfo)
user_info_id = self.IDS.create_user_info(user_id, user_info_obj)
self.RR.create_association(inst_id, PRED.hasOwner, user_id)
#Testing multiple instrument owners
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254")
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256")
extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id)
self.assertEqual(instrument_device_id, extended_instrument._id)
self.assertEqual(len(extended_instrument.owners), 2)
self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id)
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(extended_instrument.computed.firmware_version, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.last_calibration_datetime, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue)
self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue)
log.debug("extended_instrument.computed: %s", extended_instrument.computed)
#check model
inst_model_obj = self.RR.read(instrument_model_id)
self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name)
#check agent instance
inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id)
self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name)
#check agent
inst_agent_obj = self.RR.read(instrument_agent_id)
#compound assoc return list of lists so check the first element
self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent[0].name)
#check platform device
plat_device_obj = self.RR.read(platform_device_id)
self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name)
extended_platform = self.IMS.get_platform_device_extension(platform_device_id)
self.assertEqual(1, len(extended_platform.instrument_devices))
self.assertEqual(instrument_device_id, extended_platform.instrument_devices[0]._id)
self.assertEqual(1, len(extended_platform.instrument_models))
self.assertEqual(instrument_model_id, extended_platform.instrument_models[0]._id)
#check sensor devices
self.assertEqual(1, len(extended_instrument.sensor_devices))
#check data_product_parameters_set
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_instrument.computed.data_product_parameters_set.status)
self.assertTrue( 'Parsed_Canonical' in extended_instrument.computed.data_product_parameters_set.value)
# the ctd parameters should include 'temp'
self.assertTrue( 'temp' in extended_instrument.computed.data_product_parameters_set.value['Parsed_Canonical'])
#none of these will work because there is no agent
self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
extended_instrument.computed.firmware_version.status)
# self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
# extended_instrument.computed.operational_state.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.power_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.communications_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.data_status_roll_up.status)
# self.assertEqual(StatusType.STATUS_OK,
# extended_instrument.computed.data_status_roll_up.value)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.location_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.recent_events.status)
# self.assertEqual([], extended_instrument.computed.recent_events.value)
# cleanup
c = DotDict()
c.resource_registry = self.RR
self.RR2.pluck(instrument_agent_id)
self.RR2.pluck(instrument_model_id)
self.RR2.pluck(instrument_device_id)
self.RR2.pluck(platform_agent_id)
self.IMS.force_delete_instrument_agent(instrument_agent_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id)
self.IMS.force_delete_platform_agent(platform_agent_id)
self.IMS.force_delete_platform_device(platform_device_id)
self.IMS.force_delete_platform_model(platform_model_id)
self.IMS.force_delete_sensor_device(sensor_device_id)
self.IMS.force_delete_sensor_model(sensor_model_id)
#stuff we associate to
self.RR.delete(data_producer_id)
self.RR.delete(org_id)
def test_custom_attributes(self):
"""
Test assignment of custom attributes
"""
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel,
{"custom_attributes":
{"favorite_color": "attr desc goes here"}
}))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice,
{"custom_attributes":
{"favorite_color": "red",
"bogus_attr": "should raise warning"
}
}))
self.IMS.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id)
# cleanup
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
def _get_datastore(self, dataset_id):
dataset = self.DSC.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def test_resource_state_save_restore(self):
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.IMS.create_instrument_model(instModel_obj)
log.debug( 'new InstrumentModel id = %s ', instModel_id)
# Create InstrumentAgent
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 )
parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 )
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg",
stream_configurations = [raw_config, parsed_config] )
instAgent_id = self.IMS.create_instrument_agent(instAgent_obj)
log.debug( 'new InstrumentAgent id = %s', instAgent_id)
self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 '
+ '(SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345" )
instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj)
self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=spdict_id)
rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
self.DP.activate_data_product_persistence(data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug( 'Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True)
log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
#create the datastore at the beginning of each int test that persists data
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id2 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=raw_stream_def_id)
log.debug( 'new dp_id = %s', str(data_product_id2))
self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.DP.activate_data_product_persistence(data_product_id=data_product_id2)
# spin up agent
self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.addCleanup(self.IMS.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id)
gate = ProcessStateGate(self.PDC.read_process,
instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" %
instance_obj.agent_process_id)
# take snapshot of config
snap_id = self.IMS.save_resource_state(instDevice_id, "xyzzy snapshot")
snap_obj = self.RR.read_attachment(snap_id, include_content=True)
print "Saved config:"
print snap_obj.content
#modify config
instance_obj.driver_config["comms_config"] = "BAD_DATA"
self.RR.update(instance_obj)
#restore config
self.IMS.restore_resource_state(instDevice_id, snap_id)
instance_obj = self.RR.read(instAgentInstance_id)
self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"])
self.DP.delete_data_product(data_product_id1)
self.DP.delete_data_product(data_product_id2)
def test_agent_instance_config(self):
"""
Verify that agent configurations are being built properly
"""
clients = DotDict()
clients.resource_registry = self.RR
clients.pubsub_management = self.PSC
clients.dataset_management = self.DSC
pconfig_builder = PlatformAgentConfigurationBuilder(clients)
iconfig_builder = InstrumentAgentConfigurationBuilder(clients)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
org_id = self.RR2.create(any_old(RT.Org))
inst_startup_config = {'startup': 'config'}
required_config_keys = [
'org_name',
'device_type',
'agent',
'driver_config',
'stream_config',
'startup_config',
'alarm_defs',
'children']
def verify_instrument_config(config, device_id):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual('Org_1', config['org_name'])
self.assertEqual(RT.InstrumentDevice, config['device_type'])
self.assertIn('driver_config', config)
driver_config = config['driver_config']
expected_driver_fields = {'process_type': ('ZMQPyClassDriverLauncher',),
}
for k, v in expected_driver_fields.iteritems():
self.assertIn(k, driver_config)
self.assertEqual(v, driver_config[k])
self.assertEqual
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertEqual(inst_startup_config, config['startup_config'])
self.assertIn('stream_config', config)
for key in ['alarm_defs', 'children']:
self.assertEqual({}, config[key])
def verify_child_config(config, device_id, inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual('Org_1', config['org_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher',)}, config['driver_config'])
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertIn('stream_config', config)
if None is inst_device_id:
for key in ['alarm_defs', 'children', 'startup_config']:
self.assertEqual({}, config[key])
else:
for key in ['alarm_defs', 'startup_config']:
self.assertEqual({}, config[key])
self.assertIn(inst_device_id, config['children'])
verify_instrument_config(config['children'][inst_device_id], inst_device_id)
def verify_parent_config(config, parent_device_id, child_device_id, inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual('Org_1', config['org_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher',)}, config['driver_config'])
self.assertEqual({'resource_id': parent_device_id}, config['agent'])
self.assertIn('stream_config', config)
for key in ['alarm_defs', 'startup_config']:
self.assertEqual({}, config[key])
self.assertIn(child_device_id, config['children'])
verify_child_config(config['children'][child_device_id], child_device_id, inst_device_id)
rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id)
#todo: create org and figure out which agent resource needs to get assigned to it
def _make_platform_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
platform_agent_instance_obj = any_old(RT.PlatformAgentInstance)
platform_agent_instance_obj.agent_config = agent_config
platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 )
platform_agent_obj = any_old(RT.PlatformAgent, {"stream_configurations":[raw_config]})
platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj)
# device creation
platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_platform_agent_instance_to_platform_device(platform_agent_instance_id, platform_device_id)
self.RR2.assign_platform_agent_to_platform_agent_instance(platform_agent_id, platform_agent_instance_id)
self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, org_id)
return platform_agent_instance_id, platform_agent_id, platform_device_id
def _make_instrument_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
instrument_agent_instance_obj = any_old(RT.InstrumentAgentInstance, {"startup_config": inst_startup_config})
instrument_agent_instance_obj.agent_config = agent_config
instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 )
instrument_agent_obj = any_old(RT.InstrumentAgent, {"stream_configurations":[raw_config]})
instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj)
# device creation
instrument_device_id = self.IMS.create_instrument_device(any_old(RT.InstrumentDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_instrument_agent_instance_to_instrument_device(instrument_agent_instance_id, instrument_device_id)
self.RR2.assign_instrument_agent_to_instrument_agent_instance(instrument_agent_id, instrument_agent_instance_id)
self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id)
return instrument_agent_instance_id, instrument_agent_id, instrument_device_id
# can't do anything without an agent instance obj
log.debug("Testing that preparing a launcher without agent instance raises an error")
self.assertRaises(AssertionError, pconfig_builder.prepare, will_launch=False)
log.debug("Making the structure for a platform agent, which will be the child")
platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure()
platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id)
log.debug("Preparing a valid agent instance launch, for config only")
pconfig_builder.set_agent_instance_object(platform_agent_instance_child_obj)
child_config = pconfig_builder.prepare(will_launch=False)
verify_child_config(child_config, platform_device_child_id)
log.debug("Making the structure for a platform agent, which will be the parent")
platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure()
platform_agent_instance_parent_obj = self.RR2.read(platform_agent_instance_parent_id)
log.debug("Testing child-less parent as a child config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
verify_child_config(parent_config, platform_device_parent_id)
log.debug("assigning child platform to parent")
self.RR2.assign_platform_device_to_platform_device(platform_device_child_id, platform_device_parent_id)
child_device_ids = self.RR2.find_platform_device_ids_of_device(platform_device_parent_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing parent + child as parent config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
verify_parent_config(parent_config, platform_device_parent_id, platform_device_child_id)
log.debug("making the structure for an instrument agent")
instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure()
instrument_agent_instance_obj = self.RR2.read(instrument_agent_instance_id)
log.debug("Testing instrument config")
iconfig_builder.set_agent_instance_object(instrument_agent_instance_obj)
instrument_config = iconfig_builder.prepare(will_launch=False)
verify_instrument_config(instrument_config, instrument_device_id)
log.debug("assigning instrument to platform")
self.RR2.assign_instrument_device_to_platform_device(instrument_device_id, platform_device_child_id)
child_device_ids = self.RR2.find_instrument_device_ids_of_device(platform_device_child_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing entire config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
full_config = pconfig_builder.prepare(will_launch=False)
verify_parent_config(full_config, platform_device_parent_id, platform_device_child_id, instrument_device_id)
#self.fail(parent_config)
#plauncher.prepare(will_launch=False)
def sample_nested_platform_agent_instance_config(self):
"""
for informational purposes
"""
ret = {'org_name': 'Org_1',
'alarm_defs': {},
'driver_config': {'process_type': ('ZMQPyClassDriverLauncher',)},
'stream_config': {'parameter_dictionary': 'lots of stuff'},
'agent': {'resource_id': '33e54106c4444444862da082098bc123'},
'startup_config': {},
'device_type': 'PlatformDevice',
'children': {'76a39596eeff4fd5b409c4cb93f0e581':
{'org_name': 'Org_1',
'alarm_defs': {},
'driver_config': {'process_type': ('ZMQPyClassDriverLauncher',)},
'stream_config': {'parameter_dictionary': 'lots of stuff'},
'agent': {'resource_id': '76a39596eeff4fd5b409c4cb93f0e581'},
'startup_config': {},
'device_type': 'PlatformDevice',
'children': {}}}}
return ret
class TestRegisterAndActivate(IonIntegrationTestCase):
"""
Integration test cases to confirm registration and activation services
for marine device resources.
"""
def setUp(self):
"""
Test setup.
"""
# Resources used in the tests.
# General resources.
self.actor_id = None
self.user_info_id = None
self.org_id = None
self.obs_id = None
# Cabled infrastructure.
self.cabled_platform_model_id = None
self.cabled_platform_site_id = None
self.cabled_platform_device_id = None
self.cabled_platform_agent_id = None
self.cabled_platform_agent_instance_id = None
self.cabled_platform_deployment_id = None
self.cabled_instrument_deployment_id = None
self.cabled_instrument_model_id = None
self.cabled_instrument_site_id = None
self.cabled_instrument_device_id = None
self.cabled_instrument_agent_id = None
self.cabled_instrument_agent_instance_id = None
self.cabled_instrument_deployment_id = None
# Uncabled infrastructure.
self.uncabled_platform_model_id = None
self.uncabled_platform_site_id = None
self.uncabled_platform_device_id = None
self.uncabled_platform_agent_id = None
self.uncabled_platform_agent_instance_id = None
self.uncabled_instrument_model_id = None
self.uncabled_instrument_site_id = None
self.uncabled_instrument_device_id = None
self.uncabled_instrument_agent_id = None
self.uncabled_instrument_agent_instance_id = None
self.uncabled_site_deployment_id = None
# Start container.
log.info('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message)
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Setup service clients.
self.idms = IdentityManagementServiceClient(node=self.container.node)
self.oms = ObservatoryManagementServiceClient(node=self.container.node)
# Add generic resources.
self._load_system_actors()
self._create_user()
self._create_org()
self._create_observatory()
# Add cleanup routine.
self.addCleanup(self._cleanup_resources)
def _cleanup_resources(self):
"""
Delete resources created by the tests.
"""
# Check and clean up cabled resources.
if self.cabled_instrument_model_id:
self.oms.unassign_instrument_model_from_instrument_site(
self.cabled_instrument_model_id, self.cabled_instrument_site_id)
self.ims.delete_instrument_model(self.cabled_instrument_model_id)
self.cabled_instrument_model_id = None
if self.cabled_platform_model_id:
self.oms.unassign_platform_model_from_platform_site(
self.cabled_platform_model_id, self.cabled_platform_site_id)
self.ims.delete_platform_model(self.cabled_platform_model_id)
self.cabled_platform_model_id = None
if self.cabled_instrument_site_id:
self.oms.unassign_site_from_site(self.cabled_instrument_site_id,
self.cabled_platform_site_id)
self.oms.delete_instrument_site(self.cabled_instrument_site_id)
self.cabled_instrument_site_id = None
if self.cabled_platform_site_id:
self.oms.unassign_site_from_site(self.cabled_platform_site_id,
self.obs_id)
self.oms.delete_platform_site(self.cabled_platform_site_id)
self.cabled_platform_site_id = None
# Check and clean up unclabled resources.
# TODO
# Clean up generic resources.
if self.user_info_id:
self.idms.delete_user_info(self.user_info_id)
self.user_info_id = None
if self.actor_id:
self.idms.delete_actor_identity(self.actor_id)
self.actor_id = None
if self.obs_id:
self.oms.delete_observatory(self.obs_id)
self.obs_id = None
if self.org_id:
self.container.resource_registry.delete(self.org_id)
self.org_id = None
def _load_system_actors(self):
"""
Retrieve system and webauth actors and headers for later use.
"""
# Retrieve and store system actor and headers.
system_actor, _ = self.container.resource_registry.find_resources(
RT.ActorIdentity,
name=CFG.system.system_actor,
id_only=False)
self.system_actor = system_actor[0] if system_actor else None
self.system_actor_id = system_actor[0]._id if system_actor \
else 'anonymous'
self.system_actor_headers = {
'ion-actor-id': self.system_actor_id,
'ion-actor-roles': {'ION': ['ION_MANAGER', 'ORG_MANAGER']},
'expiry':'0'
}
# Retrieve and store webauth actor and headers.
webauth_actor, _ = self.container.resource_registry.find_resources(
RT.ActorIdentity,
name=CFG.get_safe("system.web_authentication_actor",
"web_authentication"), id_only=False)
self.webauth_actor = webauth_actor[0] if webauth_actor else None
self.webauth_actor_id = webauth_actor[0]._id if webauth_actor \
else 'anonymous'
self.webauth_actor_headers = {
'ion-actor-id': self.webauth_actor_id,
'ion-actor-roles': {'ION': ['ION_MANAGER', 'ORG_MANAGER']},
'expiry':'0'
}
def _create_user(self):
"""
Create user resources that serve as device owners.
This test user does not have contact information,
user credentials or notification preferences.
Results in these objects:
ActorIdentity({'_rev': '1',
'_id': '07f92986b34e426bba0fca00b73cf4a5',
'lcstate': 'DEPLOYED',
'alt_ids': [],
'description': '',
'ts_updated': '1391542388312',
'actor_type': 1,
'addl': {},
'ts_created': '1391542388312',
'availability': 'AVAILABLE',
'name': 'Identity for Adam Activationtest'})
UserInfo({'_rev': '1',
'_id': 'ac8d368e6ea247d996fd60dd0f9c7f89',
'lcstate': 'DEPLOYED',
'alt_ids': [],
'description': 'Activation Test User',
'tokens': [],
'ts_updated': '1391542388345',
'contact': ContactInformation({'individual_names_given': '',
'city': '', 'roles': [], 'administrative_area': '', 'url': '',
'country': '', 'variables': [{'name': '', 'value': ''}],
'organization_name': '', 'postal_code': '',
'individual_name_family': '', 'phones': [], 'position_name': '',
'email': '', 'street_address': ''}),
'variables': [{'name': '', 'value': ''}],
'addl': {},
'ts_created': '1391542388345',
'availability': 'AVAILABLE',
'name': 'Adam Activationtest'})
"""
# Basic user attributes for test user.
user_attrs = {
'name' : 'Adam Activationtest',
'description' : 'Activation Test User'
}
# Create ActorIdentity.
actor_name = "Identity for %s" % user_attrs['name']
actor_identity_obj = IonObject("ActorIdentity", name=actor_name)
log.trace("creating user %s with headers: %r", user_attrs['name'],
self.webauth_actor_headers)
self.actor_id = self.idms.create_actor_identity(actor_identity_obj,
headers=self.webauth_actor_headers)
# Create UserInfo.
user_info_obj = IonObject("UserInfo", **user_attrs)
self.user_info_id = self.idms.create_user_info(self.actor_id,
user_info_obj,headers=self.webauth_actor_headers)
def _create_org(self):
"""
Create an org that contains all test infrastructure.
Results in this object:
Org({'message_controllable': True,
'_rev': '1',
'_id': '9ff82d9f6c7b41f886c6137f54a3086c',
'lcstate': 'DEPLOYED',
'alt_ids': [],
'url': '',
'description':
'An Org for Activation Tests',
'contacts': [],
'org_governance_name': 'ActiveOrg',
'institution': Institution({'website': '', 'phone': '',
'name': '', 'email': ''}),
'ts_updated': '1391542388395',
'monitorable': True,
'org_type': 2,
'addl': {},
'ts_created': '1391542388395',
'availability': 'AVAILABLE',
'name': 'ActiveOrg'})
"""
org_attrs = {
'name' : 'ActiveOrg',
'description' : 'An Org for Activation Tests',
'org_type' : OrgTypeEnum.MARINE_FACILITY
}
org_obj = IonObject('Org', **org_attrs)
self.org_id = self.oms.create_marine_facility(org_obj,
headers=self.system_actor_headers)
def _create_observatory(self):
"""
Create a top level obsevaotry site for the tests.
Results in this object:
Observatory({'reference_designator': '',
'spatial_area_name': '',
'_id': 'fdcda51901464575913858f98aaf0f41',
'_rev': '1',
'lcstate': 'DEPLOYED',
'alt_ids': [],
'url': '',
'description': 'An Observatory for Activation Tests',
'coordinate_reference_system': GeospatialCoordinateReferenceSystem(
{'geospatial_latitude_units': '',
'geospatial_vertical_crs': '',
'geospatial_geodetic_crs': '',
'geospatial_vertical_positive': '',
'geospatial_vertical_units': '',
'geospatial_longitude_units': ''}),
'constraint_list': [],
'environment': 1,
'ts_updated': '1391544601340',
'local_name': '',
'geospatial_point_center': GeospatialIndex({'lat': 0.0, 'lon': 0.0}),
'addl': {},
'ts_created': '1391544601340',
'availability': 'AVAILABLE',
'name': 'ActiveObservatory'})
"""
obs_attrs = {
'name': 'ActiveObservatory',
'description' : 'An Observatory for Activation Tests'
}
obs_obj = IonObject('Observatory', **obs_attrs)
self.obs_id = self.oms.create_observatory(obs_obj, self.org_id)
def _create_cabled_resources(self):
"""
Create preexisting infrastructure for the cabled test environment:
sites, deployments, models. These are resources that already
exist in the system due to preload or incremental preload updates.
PlatformModel
InstrumentModel
PlatformSite
InstrumentSite
"""
platform_model_attrs = {
'name' : 'LP Jbox',
'description' : 'Node Type: LJ',
'manufacturer' : 'University of Washington',
'platform_type' : 'Cable Node',
'platform_family' : 'Low Power JBox',
'ci_onboard' : False,
'shore_networked' : True
}
instrument_model_attrs = {
'name': 'Diffuse Vent Fluid 3-D Temperature Array (TMPSF-A)',
'description': 'Measures temperatures of diffuse flow across the seafloor',
'reference_designator': 'TMPSFA',
'class_name': 'Temperature seafloor',
'mixed_sampling_mode': True,
'integrated_inductive_modem_available': True,
'internal_battery': True,
'addl': {'comments': '', 'connector': '',
'makemodel_description': 'XR-420',
'input_voltage_range': '',
'interface': '',
'output_description': '',
'class_long_name': 'Temperature_seafloor'},
'ooi_make_model': 'XR-420',
'series_name': 'TMPSF Series A',
'inline_management': True,
'series_id': 'TMPSFA',
'subseries_name': 'TMPSF Series A Sub 01',
'primary_interface': 1,
'manufacturer': 'RBR Global',
'family_name': 'Seafloor Properties',
'class_description': 'Measures temperatures of diffuse flow across the seafloor',
'class_alternate_name': 'Diffuse Vent Fluid 3-D Temperature Array',
'subseries_id': 'TMPSFA01',
'class_id': 'TMPSF',
'family_id': 'SFL',
'has_clock': True
}
platform_site_attrs = {
'name' : 'Cabled LP JBOX Platform Site',
'description' : 'Test Site for a Cabled LP JBOX Platform'
}
instrument_site_attrs = {
'name' : 'Cabled TMPSF Instrument Site',
'description' : 'Test Site for a Cabled TMPSF Instrument'
}
# Create the cabled model preloaded resources.
platform_model = IonObject('PlatformModel', **platform_model_attrs)
self.cabled_platform_model_id = self.ims.create_platform_model(platform_model)
instrument_model = IonObject('InstrumentModel', **instrument_model_attrs)
self.cabled_instrument_model_id = self.ims.create_instrument_model(
instrument_model)
# Create the cabled sites and link them appropriately.
platform_site = IonObject('', **platform_site_attrs)
self.cabled_platform_site_id = self.oms.create_platform_site(platform_site)
self.oms.assign_site_to_site(self.cabled_platform_site_id, self.obs_id)
instrument_site = IonObject('', **instrument_site_attrs)
self.cabled_instrument_site_id = self.oms.create_instrument_site(instrument_site)
self.oms.assign_site_to_site(self.cabled_instrument_site_id, self.cabled_platform_site_id)
# Assign models to available sites.
self.oms.assign_platform_model_to_platform_site(
self.cabled_platform_model_id, self.cabled_platform_site_id)
self.oms.assign_instrument_model_to_instrument_site(
self.cabled_instrument_model_id, self.cabled_instrument_site_id)
def _create_uncabled_resources(self):
"""
Create preexisting infrastructure for the uncabled test environment:
sites, deployments, models.
"""
pass
def test_cabled_device_activation(self):
"""
Test registration and activation of cabled device infrastructure.
"""
pass
def test_uncabled_device_activation(self):
"""
Test registration and activation of uncabled device infrastructure.
"""
pass
def instrument_test_driver(container):
org_client = OrgManagementServiceClient(node=container.node)
id_client = IdentityManagementServiceClient(node=container.node)
system_actor = id_client.find_actor_identity_by_name(name=CFG.system.system_actor)
log.info("system actor:" + system_actor._id)
sa_header_roles = get_role_message_headers(org_client.find_all_roles_by_user(system_actor._id))
# Names of agent data streams to be configured.
parsed_stream_name = "ctd_parsed"
raw_stream_name = "ctd_raw"
# Driver configuration.
# Simulator
driver_config = {
"svr_addr": "localhost",
"cmd_port": 5556,
"evt_port": 5557,
"dvr_mod": "ion.services.mi.drivers.sbe37_driver",
"dvr_cls": "SBE37Driver",
"comms_config": {
SBE37Channel.CTD: {
"method": "ethernet",
"device_addr": CFG.device.sbe37.host,
"device_port": CFG.device.sbe37.port,
"server_addr": "localhost",
"server_port": 8888,
}
},
}
# Hardware
_container_client = ContainerAgentClient(node=container.node, name=container.name)
# Create a pubsub client to create streams.
_pubsub_client = PubsubManagementServiceClient(node=container.node)
# A callback for processing subscribed-to data.
def consume(message, headers):
log.info("Subscriber received message: %s", str(message))
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(process=container, node=container.node)
subs = []
# Create streams for each stream named in driver.
stream_config = {}
for (stream_name, val) in PACKET_CONFIG.iteritems():
stream_def = ctd_stream_definition(stream_id=None)
stream_def_id = _pubsub_client.create_stream_definition(container=stream_def)
stream_id = _pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
original=True,
encoding="ION R2",
headers={"ion-actor-id": system_actor._id, "ion-actor-roles": sa_header_roles},
)
stream_config[stream_name] = stream_id
# Create subscriptions for each stream.
exchange_name = "%s_queue" % stream_name
sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name, callback=consume)
sub.start()
query = StreamQuery(stream_ids=[stream_id])
sub_id = _pubsub_client.create_subscription(query=query, exchange_name=exchange_name)
_pubsub_client.activate_subscription(sub_id)
subs.append(sub)
# Create agent config.
agent_resource_id = "123xyz"
agent_config = {
"driver_config": driver_config,
"stream_config": stream_config,
"agent": {"resource_id": agent_resource_id},
}
# Launch an instrument agent process.
_ia_name = "agent007"
_ia_mod = "ion.services.mi.instrument_agent"
_ia_class = "InstrumentAgent"
_ia_pid = _container_client.spawn_process(name=_ia_name, module=_ia_mod, cls=_ia_class, config=agent_config)
log.info("got pid=%s for resource_id=%s" % (str(_ia_pid), str(agent_resource_id)))
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
unittest # suppress an pycharm inspector error if all unittest.skip references are commented out
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.OMS = ObservatoryManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
# @unittest.skip('this test just for debugging setup')
# def test_just_the_setup(self):
# return
@attr('EXT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode as it depends on modifying CFG on service side')
def test_resources_associations_extensions(self):
"""
create one of each resource and association used by IMS
to guard against problems in ion-definitions
"""
self.patch_cfg(CFG["container"], {"extended_resources": {"strip_results": False}})
#stuff we control
instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance))
instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent))
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
instrument_site_id, _ = self.RR.create(any_old(RT.InstrumentSite))
platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance))
platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent))
platform_site_id, _ = self.RR.create(any_old(RT.PlatformSite))
platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice))
platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel))
sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice))
sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel))
#stuff we associate to
data_producer_id, _ = self.RR.create(any_old(RT.DataProducer))
org_id, _ = self.RR.create(any_old(RT.Org))
#instrument_agent_instance_id #is only a target
#instrument_agent
self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id)
#instrument_device
self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id)
self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id)
self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id)
self.RR.create_association(org_id, PRED.hasResource, instrument_device_id)
instrument_model_id #is only a target
platform_agent_instance_id #is only a target
#platform_agent
self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id)
#platform_device
self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id)
self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id)
self.RR.create_association(instrument_site_id, PRED.hasDevice, instrument_device_id)
self.RR.create_association(platform_site_id, PRED.hasDevice, platform_device_id)
self.RR.create_association(platform_site_id, PRED.hasSite, instrument_site_id)
platform_model_id #is only a target
#sensor_device
self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id)
self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id)
sensor_model_id #is only a target
#create a parsed product for this instrument output
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
processing_level_code='Parsed_Canonical')
pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1)
def addInstOwner(inst_id, subject):
actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject})
user_id = self.IDS.create_actor_identity(actor_identity_obj)
user_info_obj = any_old(RT.UserInfo)
user_info_id = self.IDS.create_user_info(user_id, user_info_obj)
self.RR.create_association(inst_id, PRED.hasOwner, user_id)
#Testing multiple instrument owners
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254")
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256")
extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id)
self.assertEqual(instrument_device_id, extended_instrument._id)
self.assertEqual(len(extended_instrument.owners), 2)
self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id)
# Lifecycle
self.assertEquals(len(extended_instrument.lcstate_transitions), 6)
self.assertEquals(set(extended_instrument.lcstate_transitions.keys()), set(['develop', 'deploy', 'retire', 'plan', 'integrate', 'delete']))
self.assertEquals(len(extended_instrument.availability_transitions), 2)
self.assertEquals(set(extended_instrument.availability_transitions.keys()), set(['enable', 'announce']))
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue)
self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue)
log.debug("extended_instrument.computed: %s", extended_instrument.computed)
#check model
inst_model_obj = self.RR.read(instrument_model_id)
self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name)
#check agent instance
inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id)
self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name)
#check agent
inst_agent_obj = self.RR.read(instrument_agent_id)
#compound assoc return list of lists so check the first element
self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent.name)
#check platform device
plat_device_obj = self.RR.read(platform_device_id)
self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name)
extended_platform = self.IMS.get_platform_device_extension(platform_device_id)
self.assertEqual(1, len(extended_platform.portals))
self.assertEqual(1, len(extended_platform.portal_instruments))
#self.assertEqual(1, len(extended_platform.computed.portal_status.value)) # no agent started so NO statuses reported
self.assertEqual(1, len(extended_platform.instrument_devices))
self.assertEqual(instrument_device_id, extended_platform.instrument_devices[0]._id)
self.assertEqual(1, len(extended_platform.instrument_models))
self.assertEqual(instrument_model_id, extended_platform.instrument_models[0]._id)
self.assertEquals(extended_platform.platform_agent._id, platform_agent_id)
self.assertEquals(len(extended_platform.lcstate_transitions), 6)
self.assertEquals(set(extended_platform.lcstate_transitions.keys()), set(['develop', 'deploy', 'retire', 'plan', 'integrate', 'delete']))
self.assertEquals(len(extended_platform.availability_transitions), 2)
self.assertEquals(set(extended_platform.availability_transitions.keys()), set(['enable', 'announce']))
#check sensor devices
self.assertEqual(1, len(extended_instrument.sensor_devices))
##check data_product_parameters_set
# !!! OOIION-1342 The UI does not use data_product_parameters_set and it is an expensive calc so the attribute calc was disabled
# !!! Remove check in this test
#self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.data_product_parameters_set.status)
#self.assertTrue( 'Parsed_Canonical' in extended_instrument.computed.data_product_parameters_set.value)
## the ctd parameters should include 'temp'
#self.assertTrue( 'temp' in extended_instrument.computed.data_product_parameters_set.value['Parsed_Canonical'])
#none of these will work because there is no agent
# self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
# extended_instrument.computed.firmware_version.status)
# self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
# extended_instrument.computed.operational_state.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.power_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.communications_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.data_status_roll_up.status)
# self.assertEqual(DeviceStatusType.STATUS_OK,
# extended_instrument.computed.data_status_roll_up.value)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.location_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.recent_events.status)
# self.assertEqual([], extended_instrument.computed.recent_events.value)
# cleanup
c = DotDict()
c.resource_registry = self.RR
self.RR2.pluck(instrument_agent_id)
self.RR2.pluck(instrument_model_id)
self.RR2.pluck(instrument_device_id)
self.RR2.pluck(platform_agent_id)
self.RR2.pluck(sensor_device_id)
self.IMS.force_delete_instrument_agent(instrument_agent_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id)
self.IMS.force_delete_platform_agent(platform_agent_id)
self.OMS.force_delete_instrument_site(instrument_site_id)
self.OMS.force_delete_platform_site(platform_site_id)
self.IMS.force_delete_platform_device(platform_device_id)
self.IMS.force_delete_platform_model(platform_model_id)
self.IMS.force_delete_sensor_device(sensor_device_id)
self.IMS.force_delete_sensor_model(sensor_model_id)
#stuff we associate to
self.RR.delete(data_producer_id)
self.RR.delete(org_id)
def test_custom_attributes(self):
"""
Test assignment of custom attributes
"""
instModel_obj = IonObject(OT.CustomAttribute,
name='SBE37IMModelAttr',
description="model custom attr")
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel,
{"custom_attributes":
[instModel_obj]
}))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice,
{"custom_attributes":
{"favorite_color": "red",
"bogus_attr": "should raise warning"
}
}))
self.IMS.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id)
# cleanup
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
def _get_datastore(self, dataset_id):
dataset = self.DSC.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def test_data_producer(self):
idevice_id = self.IMS.create_instrument_device(any_old(RT.InstrumentDevice))
self.assertEqual(1, len(self.RR2.find_data_producer_ids_of_instrument_device_using_has_data_producer(idevice_id)))
pdevice_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice))
self.assertEqual(1, len(self.RR2.find_data_producer_ids_of_platform_device_using_has_data_producer(pdevice_id)))
@attr('PREP')
def test_prepare_resource_support(self):
"""
create one of each resource and association used by IMS
to guard against problems in ion-definitions
"""
#stuff we control
instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance))
instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent))
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance))
platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent))
platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice))
platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel))
sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice))
sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel))
instrument_device2_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
instrument_device3_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
platform_device2_id, _ = self.RR.create(any_old(RT.PlatformDevice))
sensor_device2_id, _ = self.RR.create(any_old(RT.SensorDevice))
#stuff we associate to
data_producer_id, _ = self.RR.create(any_old(RT.DataProducer))
org_id, _ = self.RR.create(any_old(RT.Org))
#instrument_agent_instance_id #is only a target
#instrument_agent
self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id)
#instrument_device
self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id)
self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id)
self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id)
self.RR.create_association(org_id, PRED.hasResource, instrument_device_id)
self.RR.create_association(instrument_device2_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(org_id, PRED.hasResource, instrument_device2_id)
instrument_model_id #is only a target
platform_agent_instance_id #is only a target
#platform_agent
self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id)
#platform_device
self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id)
self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id)
self.RR.create_association(platform_device2_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_device2_id, PRED.hasDevice, instrument_device2_id)
platform_model_id #is only a target
#sensor_device
self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id)
self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id)
self.RR.create_association(sensor_device2_id, PRED.hasModel, sensor_model_id)
self.RR.create_association(sensor_device2_id, PRED.hasDevice, instrument_device2_id)
sensor_model_id #is only a target
#set lcstate - used for testing prepare - not setting all to DEVELOP, only some
self.RR.execute_lifecycle_transition(instrument_agent_id, LCE.DEVELOP)
self.RR.execute_lifecycle_transition(instrument_device_id, LCE.DEVELOP)
self.RR.execute_lifecycle_transition(instrument_device2_id, LCE.DEVELOP)
self.RR.execute_lifecycle_transition(platform_device_id, LCE.DEVELOP)
self.RR.execute_lifecycle_transition(platform_device2_id, LCE.DEVELOP)
self.RR.execute_lifecycle_transition(platform_agent_id, LCE.DEVELOP)
#create a parsed product for this instrument output
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
processing_level_code='Parsed_Canonical')
pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1)
def addInstOwner(inst_id, subject):
actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject})
user_id = self.IDS.create_actor_identity(actor_identity_obj)
user_info_obj = any_old(RT.UserInfo)
user_info_id = self.IDS.create_user_info(user_id, user_info_obj)
self.RR.create_association(inst_id, PRED.hasOwner, user_id)
#Testing multiple instrument owners
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254")
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256")
def ion_object_encoder(obj):
return obj.__dict__
#First call to create
instrument_data = self.IMS.prepare_instrument_device_support()
#print simplejson.dumps(instrument_data, default=ion_object_encoder, indent=2)
self.assertEqual(instrument_data._id, '')
self.assertEqual(instrument_data.type_, OT.InstrumentDevicePrepareSupport)
self.assertEqual(len(instrument_data.associations['InstrumentModel'].resources), 1)
self.assertEqual(instrument_data.associations['InstrumentModel'].resources[0]._id, instrument_model_id)
self.assertEqual(len(instrument_data.associations['InstrumentAgentInstance'].resources), 1)
self.assertEqual(instrument_data.associations['InstrumentAgentInstance'].resources[0]._id, instrument_agent_instance_id)
self.assertEqual(len(instrument_data.associations['InstrumentModel'].associated_resources), 0)
self.assertEqual(len(instrument_data.associations['InstrumentAgentInstance'].associated_resources), 0)
self.assertEqual(len(instrument_data.associations['SensorDevice'].resources), 0)
#Next call to update
instrument_data = self.IMS.prepare_instrument_device_support(instrument_device_id)
#print 'Update results'
#print simplejson.dumps(instrument_data, default=ion_object_encoder, indent=2)
self.assertEqual(instrument_data._id, instrument_device_id)
self.assertEqual(instrument_data.type_, OT.InstrumentDevicePrepareSupport)
self.assertEqual(len(instrument_data.associations['InstrumentModel'].resources), 1)
self.assertEqual(instrument_data.associations['InstrumentModel'].resources[0]._id, instrument_model_id)
self.assertEqual(len(instrument_data.associations['InstrumentAgentInstance'].resources), 1)
self.assertEqual(instrument_data.associations['InstrumentAgentInstance'].resources[0]._id, instrument_agent_instance_id)
self.assertEqual(len(instrument_data.associations['InstrumentModel'].associated_resources), 1)
self.assertEqual(instrument_data.associations['InstrumentModel'].associated_resources[0].s, instrument_device_id)
self.assertEqual(instrument_data.associations['InstrumentModel'].associated_resources[0].o, instrument_model_id)
self.assertEqual(len(instrument_data.associations['InstrumentAgentInstance'].associated_resources), 1)
self.assertEqual(instrument_data.associations['InstrumentAgentInstance'].associated_resources[0].o, instrument_agent_instance_id)
self.assertEqual(instrument_data.associations['InstrumentAgentInstance'].associated_resources[0].s, instrument_device_id)
self.assertEqual(len(instrument_data.associations['SensorDevice'].resources), 1)
self.assertEqual(instrument_data.associations['SensorDevice'].resources[0]._id, sensor_device_id)
self.assertEqual(len(instrument_data.associations['SensorDevice'].associated_resources), 1)
self.assertEqual(instrument_data.associations['SensorDevice'].associated_resources[0].o, instrument_device_id)
self.assertEqual(instrument_data.associations['SensorDevice'].associated_resources[0].s, sensor_device_id)
self.assertEqual(instrument_data.associations['InstrumentModel'].assign_request.request_parameters['instrument_device_id'], instrument_device_id)
#test prepare for create of instrument agent instance
instrument_agent_data = self.IMS.prepare_instrument_agent_instance_support()
#print 'Update results'
#print simplejson.dumps(instrument_agent_data, default=ion_object_encoder, indent=2)
self.assertEqual(instrument_agent_data._id, '')
self.assertEqual(instrument_agent_data.type_, OT.InstrumentAgentInstancePrepareSupport)
self.assertEqual(len(instrument_agent_data.associations['InstrumentDevice'].resources), 2)
self.assertEqual(len(instrument_agent_data.associations['InstrumentAgent'].resources), 1)
self.assertEqual(instrument_agent_data.associations['InstrumentAgent'].resources[0]._id, instrument_agent_id)
self.assertEqual(len(instrument_agent_data.associations['InstrumentDevice'].associated_resources), 0)
self.assertEqual(len(instrument_agent_data.associations['InstrumentAgent'].associated_resources), 0)
#test prepare for update of instrument agent instance to see if it is associated with the instrument that was created
instrument_agent_data = self.IMS.prepare_instrument_agent_instance_support(instrument_agent_instance_id=instrument_agent_instance_id)
#print 'Update results'
#print simplejson.dumps(instrument_agent_data, default=ion_object_encoder, indent=2)
self.assertEqual(instrument_agent_data._id, instrument_agent_instance_id)
self.assertEqual(instrument_agent_data.type_, OT.InstrumentAgentInstancePrepareSupport)
self.assertEqual(len(instrument_agent_data.associations['InstrumentDevice'].resources), 3)
self.assertEqual(len(instrument_agent_data.associations['InstrumentAgent'].resources), 1)
self.assertEqual(instrument_agent_data.associations['InstrumentAgent'].resources[0]._id, instrument_agent_id)
self.assertEqual(len(instrument_agent_data.associations['InstrumentDevice'].associated_resources), 1)
self.assertEqual(instrument_agent_data.associations['InstrumentDevice'].associated_resources[0].s, instrument_device_id)
self.assertEqual(instrument_agent_data.associations['InstrumentDevice'].associated_resources[0].o, instrument_agent_instance_id)
self.assertEqual(len(instrument_agent_data.associations['InstrumentAgent'].associated_resources), 1)
self.assertEqual(instrument_agent_data.associations['InstrumentAgent'].associated_resources[0].o, instrument_agent_id)
self.assertEqual(instrument_agent_data.associations['InstrumentAgent'].associated_resources[0].s, instrument_agent_instance_id)
self.assertEqual(instrument_agent_data.associations['InstrumentAgent'].assign_request.request_parameters['instrument_agent_instance_id'], instrument_agent_instance_id)
#test prepare for update of data product to see if it is associated with the instrument that was created
data_product_data = self.DP.prepare_data_product_support(data_product_id1)
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent=2)
self.assertEqual(data_product_data._id, data_product_id1)
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 1)
self.assertEqual(len(data_product_data.associations['Dataset'].resources), 0)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 1)
self.assertEqual(data_product_data.associations['StreamDefinition'].associated_resources[0].s, data_product_id1)
self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 0)
self.assertEqual(len(data_product_data.associations['InstrumentDeviceHasOutputProduct'].resources), 3)
self.assertEqual(len(data_product_data.associations['InstrumentDeviceHasOutputProduct'].associated_resources), 1)
self.assertEqual(data_product_data.associations['InstrumentDeviceHasOutputProduct'].associated_resources[0].s, instrument_device_id)
self.assertEqual(data_product_data.associations['InstrumentDeviceHasOutputProduct'].associated_resources[0].o, data_product_id1)
self.assertEqual(len(data_product_data.associations['PlatformDevice'].resources), 2)
platform_data = self.IMS.prepare_platform_device_support()
#print simplejson.dumps(platform_data, default=ion_object_encoder, indent=2)
self.assertEqual(platform_data._id, '')
self.assertEqual(platform_data.type_, OT.PlatformDevicePrepareSupport)
self.assertEqual(len(platform_data.associations['PlatformModel'].resources), 1)
self.assertEqual(platform_data.associations['PlatformModel'].resources[0]._id, platform_model_id)
self.assertEqual(len(platform_data.associations['PlatformAgentInstance'].resources), 1)
self.assertEqual(platform_data.associations['PlatformAgentInstance'].resources[0]._id, platform_agent_instance_id)
self.assertEqual(len(platform_data.associations['PlatformModel'].associated_resources), 0)
self.assertEqual(len(platform_data.associations['PlatformAgentInstance'].associated_resources), 0)
self.assertEqual(len(platform_data.associations['InstrumentDevice'].resources), 1)
platform_data = self.IMS.prepare_platform_device_support(platform_device_id)
#print simplejson.dumps(platform_data, default=ion_object_encoder, indent=2)
self.assertEqual(platform_data._id, platform_device_id)
self.assertEqual(platform_data.type_, OT.PlatformDevicePrepareSupport)
self.assertEqual(len(platform_data.associations['PlatformModel'].resources), 1)
self.assertEqual(platform_data.associations['PlatformModel'].resources[0]._id, platform_model_id)
self.assertEqual(len(platform_data.associations['PlatformAgentInstance'].resources), 1)
self.assertEqual(platform_data.associations['PlatformAgentInstance'].resources[0]._id, platform_agent_instance_id)
self.assertEqual(len(platform_data.associations['PlatformModel'].associated_resources), 1)
self.assertEqual(platform_data.associations['PlatformModel'].associated_resources[0].s, platform_device_id)
self.assertEqual(platform_data.associations['PlatformModel'].associated_resources[0].o, platform_model_id)
self.assertEqual(len(platform_data.associations['PlatformAgentInstance'].associated_resources), 1)
self.assertEqual(platform_data.associations['PlatformAgentInstance'].associated_resources[0].o, platform_agent_instance_id)
self.assertEqual(platform_data.associations['PlatformAgentInstance'].associated_resources[0].s, platform_device_id)
self.assertEqual(len(platform_data.associations['InstrumentDevice'].resources), 2)
#self.assertEqual(len(platform_data.associations['InstrumentDevice'].associated_resources), 1)
#self.assertEqual(platform_data.associations['InstrumentDevice'].associated_resources[0].s, platform_device_id)
#self.assertEqual(platform_data.associations['InstrumentDevice'].associated_resources[0].o, instrument_device_id)
self.assertEqual(platform_data.associations['PlatformModel'].assign_request.request_parameters['platform_device_id'], platform_device_id)
# cleanup
c = DotDict()
c.resource_registry = self.RR
self.RR2.pluck(instrument_agent_id)
self.RR2.pluck(instrument_model_id)
self.RR2.pluck(instrument_device_id)
self.RR2.pluck(platform_agent_id)
self.RR2.pluck(sensor_device_id)
self.RR2.pluck(sensor_device2_id)
self.IMS.force_delete_instrument_agent(instrument_agent_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_instrument_device(instrument_device2_id)
self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id)
self.IMS.force_delete_platform_agent(platform_agent_id)
self.IMS.force_delete_platform_device(platform_device_id)
self.IMS.force_delete_platform_device(platform_device2_id)
self.IMS.force_delete_platform_model(platform_model_id)
self.IMS.force_delete_sensor_device(sensor_device_id)
self.IMS.force_delete_sensor_device(sensor_device2_id)
self.IMS.force_delete_sensor_model(sensor_model_id)
#stuff we associate to
self.RR.delete(data_producer_id)
self.RR.delete(org_id)
class UserNotificationIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.unsc = UserNotificationServiceClient(node=self.container.node)
self.rrc = ResourceRegistryServiceClient(node=self.container.node)
self.imc = IdentityManagementServiceClient(node=self.container.node)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_email(self):
proc1 = self.container.proc_manager.procs_by_name['user_notification']
# Create a user and get the user_id
user = UserInfo(name = 'new_user')
user_id, _ = self.rrc.create(user)
# set up....
notification_id = self.unsc.create_email(event_type='ResourceLifecycleEvent',
event_subtype=None,
origin='Some_Resource_Agent_ID1',
origin_type=None,
user_id=user_id,
email='*****@*****.**',
mode = DeliveryMode.DIGEST,
message_header='message_header',
parser='parser',
period=1)
#------------------------------------------------------------------------------------------------------
# Setup so as to be able to get the message and headers going into the
# subscription callback method of the EmailEventProcessor
#------------------------------------------------------------------------------------------------------
# publish an event for each notification to generate the emails
rle_publisher = EventPublisher("ResourceLifecycleEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event")
msg_tuple = proc1.event_processors[notification_id].smtp_client.sentmail.get(timeout=4)
self.assertTrue(proc1.event_processors[notification_id].smtp_client.sentmail.empty())
message = msg_tuple[2]
list_lines = message.split("\n")
#-------------------------------------------------------
# parse the message body
#-------------------------------------------------------
message_dict = {}
for line in list_lines:
key_item = line.split(": ")
if key_item[0] == 'Subject':
message_dict['Subject'] = key_item[1] + key_item[2]
else:
try:
message_dict[key_item[0]] = key_item[1]
except IndexError as exc:
# these IndexError exceptions happen only because the message sometimes
# has successive /r/n (i.e. new lines) and therefore,
# the indexing goes out of range. These new lines
# can just be ignored. So we ignore the exceptions here.
pass
#-------------------------------------------------------
# make assertions
#-------------------------------------------------------
self.assertEquals(msg_tuple[1], '*****@*****.**' )
#self.assertEquals(msg_tuple[0], ION_NOTIFICATION_EMAIL_ADDRESS)
#self.assertEquals(message_dict['From'], ION_NOTIFICATION_EMAIL_ADDRESS)
self.assertEquals(message_dict['To'], '*****@*****.**')
self.assertEquals(message_dict['Event'].rstrip('\r'), 'ResourceLifecycleEvent')
self.assertEquals(message_dict['Originator'].rstrip('\r'), 'Some_Resource_Agent_ID1')
self.assertEquals(message_dict['Description'].rstrip('\r'), 'RLE test event')
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_sms(self):
proc1 = self.container.proc_manager.procs_by_name['user_notification']
# Create a user and get the user_id
user = UserInfo(name = 'new_user')
user_id, _ = self.rrc.create(user)
# set up....
notification_id = self.unsc.create_sms(event_type='ResourceLifecycleEvent',
event_subtype=None,
origin='Some_Resource_Agent_ID1',
origin_type=None,
user_id=user_id,
phone = '401-XXX-XXXX',
provider='T-Mobile',
message_header='message_header',
parser='parser',
)
#------------------------------------------------------------------------------------------------------
# Setup so as to be able to get the message and headers going into the
# subscription callback method of the EmailEventProcessor
#------------------------------------------------------------------------------------------------------
# publish an event for each notification to generate the emails
rle_publisher = EventPublisher("ResourceLifecycleEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event")
msg_tuple = proc1.event_processors[notification_id].smtp_client.sentmail.get(timeout=4)
self.assertTrue(proc1.event_processors[notification_id].smtp_client.sentmail.empty())
message = msg_tuple[2]
list_lines = message.split("\n")
#-------------------------------------------------------
# parse the message body
#-------------------------------------------------------
message_dict = {}
for line in list_lines:
key_item = line.split(": ")
if key_item[0] == 'Subject':
message_dict['Subject'] = key_item[1] + key_item[2]
else:
try:
message_dict[key_item[0]] = key_item[1]
except IndexError as exc:
# these IndexError exceptions happen only because the message sometimes
# has successive /r/n (i.e. new lines) and therefore,
# the indexing goes out of range. These new lines
# can just be ignored. So we ignore the exceptions here.
pass
#-------------------------------------------------------
# make assertions
#-------------------------------------------------------
self.assertEquals(msg_tuple[1], '*****@*****.**' )
#self.assertEquals(msg_tuple[0], ION_NOTIFICATION_EMAIL_ADDRESS)
self.assertEquals(message_dict['Description'].rstrip('\r'), 'RLE test event')
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_event_detection(self):
proc1 = self.container.proc_manager.procs_by_name['user_notification']
# Create a user and get the user_id
user = UserInfo(name = 'new_user')
user_id, _ = self.rrc.create(user)
# Create detection notification
dfilt = DetectionFilterConfig()
dfilt.processing['condition'] = 5
dfilt.processing['comparator'] = '>'
dfilt.processing['filter_field'] = 'voltage'
dfilt.delivery['message'] = 'I got my detection event!'
notification_id = self.unsc.create_detection_filter(event_type='ExampleDetectableEvent',
event_subtype=None,
origin='Some_Resource_Agent_ID1',
origin_type=None,
user_id=user_id,
filter_config=dfilt
)
#---------------------------------------------------------------------------------
# Create event subscription for resulting detection event
#---------------------------------------------------------------------------------
# Create an email notification so that when the DetectionEventProcessor
# detects an event and fires its own output event, this will caught by an
# EmailEventProcessor and an email will be sent to the user
notification_id_2 = self.unsc.create_email(event_type='DetectionEvent',
event_subtype=None,
origin='DetectionEventProcessor',
origin_type=None,
user_id=user_id,
email='*****@*****.**',
mode = DeliveryMode.UNFILTERED,
message_header='Detection event',
parser='parser',
period=1)
# Send event that is not detected
# publish an event for each notification to generate the emails
rle_publisher = EventPublisher("ExampleDetectableEvent")
# since the voltage field in this event is less than 5, it will not be detected
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1',
description="RLE test event",
voltage = 3)
# Check at the end of the test to make sure this event never triggered a Detectable!
# Send Event that is detected
# publish an event for each notification to generate the emails
# since the voltage field in this event is greater than 5, it WILL be detected
rle_publisher = EventPublisher("ExampleDetectableEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1',
description="RLE test event",
voltage = 10)
#-------------------------------------------------------
# make assertions
#-------------------------------------------------------
msg_tuple = proc1.event_processors[notification_id_2].smtp_client.sentmail.get(timeout=4)
# The first event never triggered an email because the voltage was less than 5, the queue is now empty
self.assertTrue(proc1.event_processors[notification_id_2].smtp_client.sentmail.empty())
self.assertEquals(msg_tuple[1], '*****@*****.**' )
#self.assertEquals(msg_tuple[0], ION_NOTIFICATION_EMAIL_ADDRESS)
# parse the message body
message = msg_tuple[2]
list_lines = message.split("\n")
message_dict = {}
for line in list_lines:
key_item = line.split(": ")
if key_item[0] == 'Subject':
message_dict['Subject'] = key_item[1] + key_item[2]
else:
try:
message_dict[key_item[0]] = key_item[1]
except IndexError as exc:
# these IndexError exceptions happen only because the message sometimes
# has successive /r/n (i.e. new lines) and therefore,
# the indexing goes out of range. These new lines
# can just be ignored. So we ignore the exceptions here.
pass
#self.assertEquals(message_dict['From'], ION_NOTIFICATION_EMAIL_ADDRESS)
self.assertEquals(message_dict['To'], '*****@*****.**')
self.assertEquals(message_dict['Event'].rstrip('\r'), 'DetectionEvent')
self.assertEquals(message_dict['Originator'].rstrip('\r'), 'DetectionEventProcessor')
self.assertEquals(message_dict['Description'].rstrip('\r'), 'Event was detected by DetectionEventProcessor')
@unittest.skip('interface has changed!')
def test_find_event_types_for_resource(self):
# create a dataset object in the RR to pass into the UNS method
dataset_object = IonObject(RT.DataSet, name="dataset1")
dataset_id, version = self.rrc.create(dataset_object)
# get the list of event types for the dataset
events = self.unsc.find_event_types_for_resource(dataset_id)
log.debug("dataset events = " + str(events))
if not events == ['dataset_supplement_added', 'dataset_change']:
self.fail("failed to return correct list of event types")
# try to pass in an id of a resource that doesn't exist (should fail)
try:
events = self.unsc.find_event_types_for_resource("bogus_id")
self.fail("failed to detect non-existant resource")
except:
pass
@unittest.skip('interface has changed!')
def test_create_two_user_notifications(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object1 = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['ResourceLifecycleEvent']})
notification_id1 = self.unsc.create_notification(notification_object1, user_id)
# create second notification
notification_object2 = IonObject(RT.NotificationRequest, {"name":"notification2",
"origin_list":['Some_Resource_Agent_ID2'],
"events_list":['DataEvent']})
notification_id2 = self.unsc.create_notification(notification_object2, user_id)
# read the notifications back and check that they are correct
n1 = self.unsc.read_notification(notification_id1)
if n1.name != notification_object1.name or \
n1.origin_list != notification_object1.origin_list or \
n1.events_list != notification_object1.events_list:
self.fail("notification was not correct")
n2 = self.unsc.read_notification(notification_id2)
if n2.name != notification_object2.name or \
n2.origin_list != notification_object2.origin_list or \
n2.events_list != notification_object2.events_list:
self.fail("notification was not correct")
@unittest.skip('interface has changed!')
def test_delete_user_notifications(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object1 = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['ResourceLifecycleEvent']})
notification1_id = self.unsc.create_notification(notification_object1, user_id)
# create second notification
notification_object2 = IonObject(RT.NotificationRequest, {"name":"notification2",
"origin_list":['Some_Resource_Agent_ID2'],
"events_list":['DataEvent']})
notification2_id = self.unsc.create_notification(notification_object2, user_id)
# delete both notifications
self.unsc.delete_notification(notification1_id)
self.unsc.delete_notification(notification2_id)
# check that the notifications are not there
try:
n1 = self.unsc.read_notification(notification1_id)
except:
try:
n2 = self.unsc.read_notification(notification2_id)
except:
return
self.fail("failed to delete notifications")
@unittest.skip('interface has changed!')
def test_find_user_notifications(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['ResourceLifecycleEvent']})
self.unsc.create_notification(notification_object, user_id)
# create second notification
notification_object = IonObject(RT.NotificationRequest, {"name":"notification2",
"origin_list":['Some_Resource_Agent_ID2'],
"events_list":['DataEvent']})
self.unsc.create_notification(notification_object, user_id)
# try to find all notifications for user
notifications = self.unsc.find_notifications_by_user(user_id)
if len(notifications) != 2:
self.fail("failed to find all notifications")
@unittest.skip('interface has changed!')
def test_update_user_notification(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
# create a notification
notification_object = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['ResourceLifecycleEvent']})
notification_id = self.unsc.create_notification(notification_object, user_id)
# read back the notification and change it
notification = self.unsc.read_notification(notification_id)
notification.origin_list = ['Some_Resource_Agent_ID5']
self.unsc.update_notification(notification)
# read back the notification and check that it got changed
notification = self.unsc.read_notification(notification_id)
if notification.origin_list != ['Some_Resource_Agent_ID5']:
self.fail("failed to change notification")
@unittest.skip('interface has changed!')
def test_send_notification_emails(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['ResourceLifecycleEvent']})
self.unsc.create_notification(notification_object, user_id)
# create second notification
notification_object = IonObject(RT.NotificationRequest, {"name":"notification2",
"origin_list":['Some_Resource_Agent_ID2'],
"events_list":['DataEvent']})
self.unsc.create_notification(notification_object, user_id)
# publish an event for each notification to generate the emails
# this can't be easily check in SW so need to check for these at the [email protected] account
rle_publisher = EventPublisher("ResourceLifecycleEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event")
de_publisher = EventPublisher("DataEvent")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event")
gevent.sleep(1)
@unittest.skip('interface has changed!')
def test_find_events(self):
# publish some events for the event repository
rle_publisher = EventPublisher("ResourceLifecycleEvent")
de_publisher = EventPublisher("DataEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event1")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event2")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event3")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event1")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event2")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event3")
# find all events for the originator 'Some_Resource_Agent_ID1'
events = self.unsc.find_events(origin='Some_Resource_Agent_ID1')
if len(events) != 3:
self.fail("failed to find all events")
for event in events:
log.debug("event=" + str(event))
if event[1][0] != 'Some_Resource_Agent_ID1':
self.fail("failed to find correct events")
# find all events for the originator 'DataEvent'
events = self.unsc.find_events(type='DataEvent')
if len(events) != 3:
self.fail("failed to find all events")
for event in events:
log.debug("event=" + str(event))
if event[1][0] != 'DataEvent':
self.fail("failed to find correct events")
# find 2 events for the originator 'Some_Resource_Agent_ID1'
events = self.unsc.find_events(origin='Some_Resource_Agent_ID2', limit=2)
if len(events) != 2:
self.fail("failed to find all events")
for event in events:
log.debug("event=" + str(event))
if event[1][0] != 'Some_Resource_Agent_ID2':
self.fail("failed to find correct events")
# find all events for the originator 'Some_Resource_Agent_ID1' in reverse time order
events = self.unsc.find_events(origin='Some_Resource_Agent_ID1', descending=True)
if len(events) != 3:
self.fail("failed to find all events")
for event in events:
log.debug("event=" + str(event))
if event[1][0] != 'Some_Resource_Agent_ID1':
self.fail("failed to find correct events")
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
print 'started services'
# @unittest.skip('this test just for debugging setup')
# def test_just_the_setup(self):
# return
@attr('EXT')
def test_resources_associations_extensions(self):
"""
create one of each resource and association used by IMS
to guard against problems in ion-definitions
"""
#stuff we control
# instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance))
instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel))
platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance))
platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent))
platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice))
platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel))
sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice))
sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel))
#stuff we associate to
data_producer_id, _ = self.RR.create(any_old(RT.DataProducer))
#instrument_agent_instance_id #is only a target
#instrument_agent
self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id)
# self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id)
#instrument_device
self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id)
# self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id)
self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id)
self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id)
instrument_model_id #is only a target
platform_agent_instance_id #is only a target
#platform_agent
self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id)
#platform_device
self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id)
self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id)
platform_model_id #is only a target
#sensor_device
self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id)
self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id)
sensor_model_id #is only a target
def addInstOwner(inst_id, subject):
actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject})
user_id = self.IDS.create_actor_identity(actor_identity_obj)
user_info_obj = any_old(RT.UserInfo)
user_info_id = self.IDS.create_user_info(user_id, user_info_obj)
self.RR.create_association(inst_id, PRED.hasOwner, user_id)
#Testing multiple instrument owners
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254")
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256")
#testing data products
dp_id, _ = self.RR.create(any_old(RT.DataProduct))
self.RR.create_association(instrument_device_id, PRED.hasOutputProduct, dp_id)
extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id)
self.assertEqual(instrument_device_id, extended_instrument._id)
self.assertEqual(len(extended_instrument.owners), 2)
self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id)
#check data products
self.assertEqual(1, len(extended_instrument.data_products))
#check model
inst_model_obj = self.RR.read(instrument_model_id)
self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name)
#check agent
inst_agent_obj = self.RR.read(instrument_agent_id)
self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent.name)
#check platform device
plat_device_obj = self.RR.read(platform_device_id)
self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name)
#check sensor devices
self.assertEqual(1, len(extended_instrument.sensor_devices))
self.assertEqual("1.1", extended_instrument.computed.firmware_version)
self.assertEqual("42", extended_instrument.computed.last_data_received_time)
self.assertEqual("23", extended_instrument.computed.operational_state)
self.assertEqual("34", extended_instrument.computed.last_command_status)
self.assertEqual("45", extended_instrument.computed.last_command_date)
self.assertEqual("56", extended_instrument.computed.last_command)
self.assertEqual("67", extended_instrument.computed.last_commanded_by)
self.assertEqual("78", extended_instrument.computed.power_status_roll_up)
self.assertEqual("89", extended_instrument.computed.communications_status_roll_up)
self.assertEqual("98", extended_instrument.computed.data_status_roll_up)
self.assertEqual("87", extended_instrument.computed.location_status_roll_up)
self.assertEqual(['mon', 'tue', 'wed'], extended_instrument.computed.recent_events)
class TestIdentityManagementServiceInt(IonIntegrationTestCase):
def setUp(self):
self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254"
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2coi.yml')
self.identity_management_service = IdentityManagementServiceClient(
node=self.container.node)
def test_actor_identity(self):
actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject})
user_id = self.identity_management_service.create_actor_identity(
actor_identity_obj)
actor_identity = self.identity_management_service.read_actor_identity(
user_id)
actor_identity.name = 'Updated subject'
self.identity_management_service.update_actor_identity(actor_identity)
self.identity_management_service.delete_actor_identity(user_id)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.read_actor_identity(user_id)
self.assertTrue("does not exist" in cm.exception.message)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.delete_actor_identity(user_id)
self.assertTrue("does not exist" in cm.exception.message)
def test_user_credentials(self):
actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject})
user_id = self.identity_management_service.create_actor_identity(
actor_identity_obj)
user_credentials_obj = IonObject("UserCredentials",
{"name": self.subject})
self.identity_management_service.register_user_credentials(
user_id, user_credentials_obj)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.unregister_user_credentials(
"bad", self.subject)
self.assertTrue("does not exist" in cm.exception.message)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.unregister_user_credentials(
user_id, "bad")
self.assertTrue("does not exist" in cm.exception.message)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.unregister_user_credentials(
'bad', 'bad')
self.assertTrue("does not exist" in cm.exception.message)
self.identity_management_service.unregister_user_credentials(
user_id, self.subject)
self.identity_management_service.delete_actor_identity(user_id)
def test_user_info(self):
actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject})
user_id = self.identity_management_service.create_actor_identity(
actor_identity_obj)
user_credentials_obj = IonObject("UserCredentials",
{"name": self.subject})
self.identity_management_service.register_user_credentials(
user_id, user_credentials_obj)
user_info_obj = IonObject("UserInfo", {"name": "Foo"})
user_info = self.identity_management_service.create_user_info(
user_id, user_info_obj)
with self.assertRaises(Conflict) as cm:
self.identity_management_service.create_user_info(
user_id, user_info_obj)
self.assertTrue(
"UserInfo already exists for user id" in cm.exception.message)
user_info_obj = self.identity_management_service.find_user_info_by_id(
user_id)
user_info_obj = self.identity_management_service.find_user_info_by_name(
"Foo")
user_info_obj = self.identity_management_service.find_user_info_by_subject(
self.subject)
user_info_obj = self.identity_management_service.read_user_info(
user_info)
user_info_obj.name = 'Jane Doe'
self.identity_management_service.update_user_info(user_info_obj)
self.identity_management_service.delete_user_info(user_info)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.read_user_info(user_info)
self.assertTrue('does not exist' in cm.exception.message)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.delete_user_info(user_info)
self.assertTrue('does not exist' in cm.exception.message)
with self.assertRaises(NotFound) as cm:
self.identity_management_service.find_user_info_by_name("John Doe")
self.assertEqual(cm.exception.message,
'UserInfo with name John Doe does not exist')
with self.assertRaises(NotFound) as cm:
self.identity_management_service.find_user_info_by_subject(
"Bogus subject")
self.assertEqual(
cm.exception.message,
"UserCredentials with subject Bogus subject does not exist")
self.identity_management_service.unregister_user_credentials(
user_id, self.subject)
self.identity_management_service.delete_actor_identity(user_id)
def test_signon(self):
certificate = """-----BEGIN CERTIFICATE-----
MIIEMzCCAxugAwIBAgICBQAwDQYJKoZIhvcNAQEFBQAwajETMBEGCgmSJomT8ixkARkWA29yZzEX
MBUGCgmSJomT8ixkARkWB2NpbG9nb24xCzAJBgNVBAYTAlVTMRAwDgYDVQQKEwdDSUxvZ29uMRsw
GQYDVQQDExJDSUxvZ29uIEJhc2ljIENBIDEwHhcNMTAxMTE4MjIyNTA2WhcNMTAxMTE5MTAzMDA2
WjBvMRMwEQYKCZImiZPyLGQBGRMDb3JnMRcwFQYKCZImiZPyLGQBGRMHY2lsb2dvbjELMAkGA1UE
BhMCVVMxFzAVBgNVBAoTDlByb3RlY3ROZXR3b3JrMRkwFwYDVQQDExBSb2dlciBVbndpbiBBMjU0
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6QhsWxhUXbIxg+1ZyEc7d+hIGvchVmtb
g0kKLmivgoVsA4U7swNDRH6svW242THta0oTf6crkRx7kOKg6jma2lcAC1sjOSddqX7/92ChoUPq
7LWt2T6GVVA10ex5WAeB/o7br/Z4U8/75uCBis+ru7xEDl09PToK20mrkcz9M4HqIv1eSoPkrs3b
2lUtQc6cjuHRDU4NknXaVMXTBHKPM40UxEDHJueFyCiZJFg3lvQuSsAl4JL5Z8pC02T8/bODBuf4
dszsqn2SC8YDw1xrujvW2Bd7Q7BwMQ/gO+dZKM1mLJFpfEsR9WrjMeg6vkD2TMWLMr0/WIkGC8u+
6M6SMQIDAQABo4HdMIHaMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgSwMBMGA1UdJQQMMAoG
CCsGAQUFBwMCMBgGA1UdIAQRMA8wDQYLKwYBBAGCkTYBAgEwagYDVR0fBGMwYTAuoCygKoYoaHR0
cDovL2NybC5jaWxvZ29uLm9yZy9jaWxvZ29uLWJhc2ljLmNybDAvoC2gK4YpaHR0cDovL2NybC5k
b2Vncmlkcy5vcmcvY2lsb2dvbi1iYXNpYy5jcmwwHwYDVR0RBBgwFoEUaXRzYWdyZWVuMUB5YWhv
by5jb20wDQYJKoZIhvcNAQEFBQADggEBAEYHQPMY9Grs19MHxUzMwXp1GzCKhGpgyVKJKW86PJlr
HGruoWvx+DLNX75Oj5FC4t8bOUQVQusZGeGSEGegzzfIeOI/jWP1UtIjzvTFDq3tQMNvsgROSCx5
CkpK4nS0kbwLux+zI7BWON97UpMIzEeE05pd7SmNAETuWRsHMP+x6i7hoUp/uad4DwbzNUGIotdK
f8b270icOVgkOKRdLP/Q4r/x8skKSCRz1ZsRdR+7+B/EgksAJj7Ut3yiWoUekEMxCaTdAHPTMD/g
Mh9xL90hfMJyoGemjJswG5g3fAdTP/Lv0I6/nWeH/cLjwwpQgIEjEAVXl7KHuzX5vPD/wqQ=
-----END CERTIFICATE-----"""
id, valid_until, registered = self.identity_management_service.signon(
certificate, True)
self.assertFalse(registered)
id2, valid_until2, registered2 = self.identity_management_service.signon(
certificate, True)
self.assertFalse(registered2)
self.assertTrue(id == id2)
self.assertTrue(valid_until == valid_until2)
user_info_obj = IonObject("UserInfo", {"name": "Foo"})
self.identity_management_service.create_user_info(id, user_info_obj)
id3, valid_until3, registered3 = self.identity_management_service.signon(
certificate, True)
self.assertTrue(registered3)
self.assertTrue(id == id3)
self.assertTrue(valid_until == valid_until3)
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
print 'started services'
# @unittest.skip('this test just for debugging setup')
# def test_just_the_setup(self):
# return
@attr('EXT')
def test_resources_associations_extensions(self):
"""
create one of each resource and association used by IMS
to guard against problems in ion-definitions
"""
#stuff we control
instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance))
instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent))
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance))
platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent))
platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice))
platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel))
sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice))
sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel))
#stuff we associate to
data_producer_id, _ = self.RR.create(any_old(RT.DataProducer))
org_id, _ = self.RR.create(any_old(RT.Org))
#instrument_agent_instance_id #is only a target
#instrument_agent
self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id)
#instrument_device
self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id)
self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id)
self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id)
self.RR.create_association(org_id, PRED.hasResource, instrument_device_id)
instrument_model_id #is only a target
platform_agent_instance_id #is only a target
#platform_agent
self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id)
#platform_device
self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id)
self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id)
platform_model_id #is only a target
#sensor_device
self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id)
self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id)
sensor_model_id #is only a target
#create a parsed product for this instrument output
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
processing_level_code='Parsed_Canonical',
temporal_domain = tdom,
spatial_domain = sdom)
pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1)
def addInstOwner(inst_id, subject):
actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject})
user_id = self.IDS.create_actor_identity(actor_identity_obj)
user_info_obj = any_old(RT.UserInfo)
user_info_id = self.IDS.create_user_info(user_id, user_info_obj)
self.RR.create_association(inst_id, PRED.hasOwner, user_id)
#Testing multiple instrument owners
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254")
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256")
extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id)
self.assertEqual(instrument_device_id, extended_instrument._id)
self.assertEqual(len(extended_instrument.owners), 2)
self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id)
#check model
inst_model_obj = self.RR.read(instrument_model_id)
self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name)
#check agent instance
inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id)
self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name)
#check agent
inst_agent_obj = self.RR.read(instrument_agent_id)
#compound assoc return list of lists so check the first element
self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent[0].name)
#check platform device
plat_device_obj = self.RR.read(platform_device_id)
self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name)
#check sensor devices
self.assertEqual(1, len(extended_instrument.sensor_devices))
#check data_product_parameters_set
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_instrument.computed.data_product_parameters_set.status)
self.assertTrue( 'Parsed_Canonical' in extended_instrument.computed.data_product_parameters_set.value)
# the ctd parameters should include 'temp'
self.assertTrue( 'temp' in extended_instrument.computed.data_product_parameters_set.value['Parsed_Canonical'])
#none of these will work because there is no agent
self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
extended_instrument.computed.firmware_version.status)
self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
extended_instrument.computed.operational_state.status)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_instrument.computed.power_status_roll_up.status)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_instrument.computed.communications_status_roll_up.status)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_instrument.computed.data_status_roll_up.status)
self.assertEqual(StatusType.STATUS_OK,
extended_instrument.computed.data_status_roll_up.value)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_instrument.computed.location_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.recent_events.status)
# self.assertEqual([], extended_instrument.computed.recent_events.value)
# cleanup
c = DotDict()
c.resource_registry = self.RR
resource_impl = ResourceImpl(c)
resource_impl.pluck(instrument_agent_id)
resource_impl.pluck(instrument_model_id)
resource_impl.pluck(instrument_device_id)
resource_impl.pluck(platform_agent_id)
self.IMS.force_delete_instrument_agent(instrument_agent_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id)
self.IMS.force_delete_platform_agent(platform_agent_id)
self.IMS.force_delete_platform_device(platform_device_id)
self.IMS.force_delete_platform_model(platform_model_id)
self.IMS.force_delete_sensor_device(sensor_device_id)
self.IMS.force_delete_sensor_model(sensor_model_id)
#stuff we associate to
self.RR.delete(data_producer_id)
self.RR.delete(org_id)
def test_custom_attributes(self):
"""
Test assignment of custom attributes
"""
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel,
{"custom_attributes":
{"favorite_color": "attr desc goes here"}
}))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice,
{"custom_attributes":
{"favorite_color": "red",
"bogus_attr": "should raise warning"
}
}))
self.IMS.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id)
# cleanup
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
def _get_datastore(self, dataset_id):
dataset = self.DSC.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def test_checkpoint_restore(self):
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel",
stream_configuration= {'raw': 'ctd_raw_param_dict' , 'parsed': 'ctd_parsed_param_dict' })
instModel_id = self.IMS.create_instrument_model(instModel_obj)
log.debug( 'new InstrumentModel id = %s ', instModel_id)
# Create InstrumentAgent
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver" )
instAgent_id = self.IMS.create_instrument_agent(instAgent_obj)
log.debug( 'new InstrumentAgent id = %s', instAgent_id)
self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 '
+ '(SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345" )
instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj)
self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
port_agent_config = {
'device_addr': 'sbe37-simulator.oceanobservatories.org',
'device_port': 4001,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'command_port': 4002,
'data_port': 4003,
'log_level': 5,
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
comms_device_address='sbe37-simulator.oceanobservatories.org',
comms_device_port=4001,
port_agent_config = port_agent_config)
instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict')
parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary=spdict_id)
rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict')
raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary=rpdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=parsed_stream_def_id,
parameter_dictionary=spdict_id)
log.debug( 'new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug( 'Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True)
log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
#create the datastore at the beginning of each int test that persists data
self._get_datastore(self.parsed_dataset)
self.DP.activate_data_product_persistence(data_product_id=data_product_id1)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id2 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=raw_stream_def_id,
parameter_dictionary=rpdict_id)
log.debug( 'new dp_id = %s', str(data_product_id2))
self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.DP.activate_data_product_persistence(data_product_id=data_product_id2)
# spin up agent
self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.addCleanup(self.IMS.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id)
gate = ProcessStateGate(self.PDC.read_process,
instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" %
instance_obj.agent_process_id)
# take snapshot of config
snap_id = self.IMS.agent_state_checkpoint(instDevice_id, "xyzzy snapshot")
snap_obj = self.RR.read_attachment(snap_id, include_content=True)
print "Saved config:"
print snap_obj.content
#modify config
instance_obj.driver_config["comms_config"] = "BAD_DATA"
self.RR.update(instance_obj)
#restore config
self.IMS.agent_state_restore(instDevice_id, snap_id)
instance_obj = self.RR.read(instAgentInstance_id)
self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"])
def test_user_role_cache(self):
#Create a user
id_client = IdentityManagementServiceClient(node=self.container.node)
actor_id, valid_until, registered = id_client.signon(
USER1_CERTIFICATE, True)
#Make a request with this new user to get it into the cache
response = self.test_app.get(
'/ion-service/resource_registry/find_resources?name=TestDataProduct&id_only=True&requester='
+ actor_id)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
#Check the contents of the user role cache for this user
service_gateway_user_cache = self.container.proc_manager.procs_by_name[
'service_gateway'].user_data_cache
self.assertEqual(service_gateway_user_cache.has_key(actor_id), True)
role_header = service_gateway_user_cache.get(actor_id)
self.assertIn('ION', role_header)
self.assertEqual(len(role_header['ION']), 1)
self.assertIn('ORG_MEMBER', role_header['ION'])
org_client = OrgManagementServiceClient(node=self.container.node)
ion_org = org_client.find_org()
manager_role = org_client.find_org_role_by_name(
org_id=ion_org._id, role_name='ORG_MANAGER')
org_client.grant_role(org_id=ion_org._id,
user_id=actor_id,
role_name='ORG_MANAGER')
#Just allow some time for event processing on slower platforms
gevent.sleep(2)
#The user should be evicted from the cache due to a change in roles
self.assertEqual(service_gateway_user_cache.has_key(actor_id), False)
#Do it again to check for new roles
response = self.test_app.get(
'/ion-service/resource_registry/find_resources?name=TestDataProduct&id_only=True&requester='
+ actor_id)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
#Check the contents of the user role cache for this user
self.assertEqual(service_gateway_user_cache.has_key(actor_id), True)
role_header = service_gateway_user_cache.get(actor_id)
self.assertIn('ION', role_header)
self.assertEqual(len(role_header['ION']), 2)
self.assertIn('ORG_MEMBER', role_header['ION'])
self.assertIn('ORG_MANAGER', role_header['ION'])
#Now flush the user_role_cache and make sure it was flushed
event_publisher = EventPublisher()
event_publisher.publish_event(event_type=OT.UserRoleCacheResetEvent)
#Just allow some time for event processing on slower platforms
gevent.sleep(2)
self.assertEqual(service_gateway_user_cache.has_key(actor_id), False)
self.assertEqual(service_gateway_user_cache.size(), 0)
#Change the role once again and see if it is there again
org_client.revoke_role(org_id=ion_org._id,
user_id=actor_id,
role_name='ORG_MANAGER')
#Just allow some time for event processing on slower platforms
gevent.sleep(2)
#The user should still not be there
self.assertEqual(service_gateway_user_cache.has_key(actor_id), False)
#Do it again to check for new roles
response = self.test_app.get(
'/ion-service/resource_registry/find_resources?name=TestDataProduct&id_only=True&requester='
+ actor_id)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
#Check the contents of the user role cache for this user
self.assertEqual(service_gateway_user_cache.has_key(actor_id), True)
role_header = service_gateway_user_cache.get(actor_id)
self.assertIn('ION', role_header)
self.assertEqual(len(role_header['ION']), 1)
self.assertIn('ORG_MEMBER', role_header['ION'])
id_client.delete_actor_identity(actor_id)
def test_user_role_cache(self):
# Create a user
id_client = IdentityManagementServiceClient()
actor_id, valid_until, registered = id_client.signon(USER1_CERTIFICATE, True)
# Make a request with this new user to get it into the cache
response = self.test_app.get(
"/ion-service/resource_registry/find_resources?name=TestDataProduct&id_only=True&requester=" + actor_id
)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json["data"])
# Check the contents of the user role cache for this user
service_gateway_user_role_cache = self.container.proc_manager.procs_by_name["service_gateway"].user_role_cache
self.assertEqual(service_gateway_user_role_cache.has_key(actor_id), True)
role_header = service_gateway_user_role_cache.get(actor_id)
self.assertIn("ION", role_header)
self.assertEqual(len(role_header["ION"]), 1)
self.assertIn("ORG_MEMBER", role_header["ION"])
org_client = OrgManagementServiceClient()
ion_org = org_client.find_org()
manager_role = org_client.find_org_role_by_name(org_id=ion_org._id, role_name="ORG_MANAGER")
org_client.grant_role(org_id=ion_org._id, actor_id=actor_id, role_name="ORG_MANAGER")
# Just allow some time for event processing on slower platforms
gevent.sleep(2)
# The user should be evicted from the cache due to a change in roles
self.assertEqual(service_gateway_user_role_cache.has_key(actor_id), False)
# Do it again to check for new roles
response = self.test_app.get(
"/ion-service/resource_registry/find_resources?name=TestDataProduct&id_only=True&requester=" + actor_id
)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json["data"])
# Check the contents of the user role cache for this user
self.assertEqual(service_gateway_user_role_cache.has_key(actor_id), True)
role_header = service_gateway_user_role_cache.get(actor_id)
self.assertIn("ION", role_header)
self.assertEqual(len(role_header["ION"]), 2)
self.assertIn("ORG_MEMBER", role_header["ION"])
self.assertIn("ORG_MANAGER", role_header["ION"])
# Now flush the user_role_cache and make sure it was flushed
event_publisher = EventPublisher()
event_publisher.publish_event(event_type=OT.UserRoleCacheResetEvent)
# Just allow some time for event processing on slower platforms
gevent.sleep(2)
self.assertEqual(service_gateway_user_role_cache.has_key(actor_id), False)
self.assertEqual(service_gateway_user_role_cache.size(), 0)
# Change the role once again and see if it is there again
org_client.revoke_role(org_id=ion_org._id, actor_id=actor_id, role_name="ORG_MANAGER")
# Just allow some time for event processing on slower platforms
gevent.sleep(2)
# The user should still not be there
self.assertEqual(service_gateway_user_role_cache.has_key(actor_id), False)
# Do it again to check for new roles
response = self.test_app.get(
"/ion-service/resource_registry/find_resources?name=TestDataProduct&id_only=True&requester=" + actor_id
)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json["data"])
# Check the contents of the user role cache for this user
self.assertEqual(service_gateway_user_role_cache.has_key(actor_id), True)
role_header = service_gateway_user_role_cache.get(actor_id)
self.assertIn("ION", role_header)
self.assertEqual(len(role_header["ION"]), 1)
self.assertIn("ORG_MEMBER", role_header["ION"])
id_client.delete_actor_identity(actor_id)
class UserNotificationIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.unsc = UserNotificationServiceClient(node=self.container.node)
self.rrc = ResourceRegistryServiceClient(node=self.container.node)
self.imc = IdentityManagementServiceClient(node=self.container.node)
def test_find_event_types_for_resource(self):
# create a dataset object in the RR to pass into the UNS method
dataset_object = IonObject(RT.DataSet, name="dataset1")
dataset_id, version = self.rrc.create(dataset_object)
# get the list of event types for the dataset
events = self.unsc.find_event_types_for_resource(dataset_id)
log.debug("dataset events = " + str(events))
if not events == ['dataset_supplement_added', 'dataset_change']:
self.fail("failed to return correct list of event types")
# try to pass in an id of a resource that doesn't exist (should fail)
try:
events = self.unsc.find_event_types_for_resource("bogus_id")
self.fail("failed to detect non-existant resource")
except:
pass
def test_create_two_user_notifications(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object1 = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['ResourceLifecycleEvent']})
notification_id1 = self.unsc.create_notification(notification_object1, user_id)
# create second notification
notification_object2 = IonObject(RT.NotificationRequest, {"name":"notification2",
"origin_list":['Some_Resource_Agent_ID2'],
"events_list":['DataEvent']})
notification_id2 = self.unsc.create_notification(notification_object2, user_id)
# read the notifications back and check that they are correct
n1 = self.unsc.read_notification(notification_id1)
if n1.name != notification_object1.name or \
n1.origin_list != notification_object1.origin_list or \
n1.events_list != notification_object1.events_list:
self.fail("notification was not correct")
n2 = self.unsc.read_notification(notification_id2)
if n2.name != notification_object2.name or \
n2.origin_list != notification_object2.origin_list or \
n2.events_list != notification_object2.events_list:
self.fail("notification was not correct")
def test_delete_user_notifications(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object1 = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['ResourceLifecycleEvent']})
notification1_id = self.unsc.create_notification(notification_object1, user_id)
# create second notification
notification_object2 = IonObject(RT.NotificationRequest, {"name":"notification2",
"origin_list":['Some_Resource_Agent_ID2'],
"events_list":['DataEvent']})
notification2_id = self.unsc.create_notification(notification_object2, user_id)
# delete both notifications
self.unsc.delete_notification(notification1_id)
self.unsc.delete_notification(notification2_id)
# check that the notifications are not there
try:
n1 = self.unsc.read_notification(notification1_id)
except:
try:
n2 = self.unsc.read_notification(notification2_id)
except:
return
self.fail("failed to delete notifications")
def test_find_user_notifications(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['ResourceLifecycleEvent']})
self.unsc.create_notification(notification_object, user_id)
# create second notification
notification_object = IonObject(RT.NotificationRequest, {"name":"notification2",
"origin_list":['Some_Resource_Agent_ID2'],
"events_list":['DataEvent']})
self.unsc.create_notification(notification_object, user_id)
# try to find all notifications for user
notifications = self.unsc.find_notifications_by_user(user_id)
if len(notifications) != 2:
self.fail("failed to find all notifications")
def test_update_user_notification(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
# create a notification
notification_object = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['ResourceLifecycleEvent']})
notification_id = self.unsc.create_notification(notification_object, user_id)
# read back the notification and change it
notification = self.unsc.read_notification(notification_id)
notification.origin_list = ['Some_Resource_Agent_ID5']
self.unsc.update_notification(notification)
# read back the notification and check that it got changed
notification = self.unsc.read_notification(notification_id)
if notification.origin_list != ['Some_Resource_Agent_ID5']:
self.fail("failed to change notification")
def test_send_notification_emails(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['ResourceLifecycleEvent']})
self.unsc.create_notification(notification_object, user_id)
# create second notification
notification_object = IonObject(RT.NotificationRequest, {"name":"notification2",
"origin_list":['Some_Resource_Agent_ID2'],
"events_list":['DataEvent']})
self.unsc.create_notification(notification_object, user_id)
# publish an event for each notification to generate the emails
# this can't be easily check in SW so need to check for these at the [email protected] account
rle_publisher = EventPublisher("ResourceLifecycleEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event")
de_publisher = EventPublisher("DataEvent")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event")
gevent.sleep(1)
def test_find_events(self):
# publish some events for the event repository
rle_publisher = EventPublisher("ResourceLifecycleEvent")
de_publisher = EventPublisher("DataEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event1")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event2")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event3")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event1")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event2")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event3")
# find all events for the originator 'Some_Resource_Agent_ID1'
events = self.unsc.find_events(origin='Some_Resource_Agent_ID1')
if len(events) != 3:
self.fail("failed to find all events")
for event in events:
log.debug("event=" + str(event))
if event[1][0] != 'Some_Resource_Agent_ID1':
self.fail("failed to find correct events")
# find all events for the originator 'DataEvent'
events = self.unsc.find_events(type='DataEvent')
if len(events) != 3:
self.fail("failed to find all events")
for event in events:
log.debug("event=" + str(event))
if event[1][0] != 'DataEvent':
self.fail("failed to find correct events")
# find 2 events for the originator 'Some_Resource_Agent_ID1'
events = self.unsc.find_events(origin='Some_Resource_Agent_ID2', limit=2)
if len(events) != 2:
self.fail("failed to find all events")
for event in events:
log.debug("event=" + str(event))
if event[1][0] != 'Some_Resource_Agent_ID2':
self.fail("failed to find correct events")
# find all events for the originator 'Some_Resource_Agent_ID1' in reverse time order
events = self.unsc.find_events(origin='Some_Resource_Agent_ID1', descending=True)
if len(events) != 3:
self.fail("failed to find all events")
for event in events:
log.debug("event=" + str(event))
if event[1][0] != 'Some_Resource_Agent_ID1':
self.fail("failed to find correct events")
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dpsc_cli = DataProductManagementServiceClient()
self.rrclient = ResourceRegistryServiceClient()
self.damsclient = DataAcquisitionManagementServiceClient()
self.pubsubcli = PubsubManagementServiceClient()
self.ingestclient = IngestionManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.unsc = UserNotificationServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.identcli = IdentityManagementServiceClient()
#------------------------------------------
# Create the environment
#------------------------------------------
self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM')
self.process_definitions = {}
ingestion_worker_definition = ProcessDefinition(name='ingestion worker')
ingestion_worker_definition.executable = {
'module':'ion.processes.data.ingestion.science_granule_ingestion_worker',
'class' :'ScienceGranuleIngestionWorker'
}
process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition)
self.process_definitions['ingestion_worker'] = process_definition_id
self.pids = []
self.exchange_points = []
self.exchange_names = []
#------------------------------------------------------------------------------------------------
# First launch the ingestors
#------------------------------------------------------------------------------------------------
self.exchange_space = 'science_granule_ingestion'
self.exchange_point = 'science_data'
config = DotDict()
config.process.datastore_name = 'datasets'
config.process.queue_name = self.exchange_space
self.exchange_names.append(self.exchange_space)
self.exchange_points.append(self.exchange_point)
pid = self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config)
log.debug("the ingestion worker process id: %s", pid)
self.pids.append(pid)
self.addCleanup(self.cleaning_up)
def cleaning_up(self):
for pid in self.pids:
log.debug("number of pids to be terminated: %s", len(self.pids))
try:
self.process_dispatcher.cancel_process(pid)
log.debug("Terminated the process: %s", pid)
except:
log.debug("could not terminate the process id: %s" % pid)
IngestionManagementIntTest.clean_subscriptions()
for xn in self.exchange_names:
xni = self.container.ex_manager.create_xn_queue(xn)
xni.delete()
for xp in self.exchange_points:
xpi = self.container.ex_manager.create_xp(xp)
xpi.delete()
def get_datastore(self, dataset_id):
dataset = self.dataset_management.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
@attr('EXT')
@attr('PREP')
def test_create_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
parameter_dictionary = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict')
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary._id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0
dp_obj.ooi_product_name = "PRODNAME"
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product( data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
# Assert that the data product has an associated stream at this stage
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True)
self.assertNotEquals(len(stream_ids), 0)
# Assert that the data product has an associated stream def at this stage
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStreamDefinition, RT.StreamDefinition, True)
self.assertNotEquals(len(stream_ids), 0)
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Created data product %s', dp_obj)
#------------------------------------------------------------------------------------------------
# test creating a new data product with a stream definition
#------------------------------------------------------------------------------------------------
log.debug('Creating new data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='DP2',
description='some new dp')
dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id)
self.dpsc_cli.activate_data_product_persistence(dp_id2)
log.debug('new dp_id = %s' % dp_id2)
#------------------------------------------------------------------------------------------------
#make sure data product is associated with stream def
#------------------------------------------------------------------------------------------------
streamdefs = []
streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True)
for s in streams:
log.debug("Checking stream %s" % s)
sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True)
for sd in sdefs:
log.debug("Checking streamdef %s" % sd)
streamdefs.append(sd)
self.assertIn(ctd_stream_def_id, streamdefs)
group_names = self.dpsc_cli.get_data_product_group_list()
self.assertIn("PRODNAME", group_names)
#----------------------------------------------------------------------------------------
# Create users then notifications to this data product for each user
#----------------------------------------------------------------------------------------
# user_1
user_1 = UserInfo()
user_1.name = 'user_1'
user_1.contact.email = '*****@*****.**'
# user_2
user_2 = UserInfo()
user_2.name = 'user_2'
user_2.contact.email = '*****@*****.**'
#user1 is a complete user
self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254"
actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject})
actor_id = self.identcli.create_actor_identity(actor_identity_obj)
user_credentials_obj = IonObject("UserCredentials", {"name": self.subject})
self.identcli.register_user_credentials(actor_id, user_credentials_obj)
user_id_1 = self.identcli.create_user_info(actor_id, user_1)
user_id_2, _ = self.rrclient.create(user_2)
delivery_config1a = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH)
delivery_config1b = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH)
notification_request_1 = NotificationRequest( name = "notification_1",
origin=dp_id,
origin_type="type_1",
event_type=OT.ResourceLifecycleEvent,
disabled_by_system = False,
delivery_configurations=[delivery_config1a, delivery_config1b])
delivery_config2a = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH)
delivery_config2b = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH)
notification_request_2 = NotificationRequest( name = "notification_2",
origin=dp_id,
origin_type="type_2",
disabled_by_system = False,
event_type=OT.DetectionEvent,
delivery_configurations=[delivery_config2a, delivery_config2b])
notification_request_1_id = self.unsc.create_notification(notification=notification_request_1, user_id=user_id_1)
notification_request_2_id = self.unsc.create_notification(notification=notification_request_2, user_id=user_id_2)
self.unsc.delete_notification(notification_request_1_id)
# test reading a non-existent data product
log.debug('reading non-existent data product')
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product('some_fake_id')
# update a data product (tests read also)
log.debug('Updating data product')
# first get the existing dp object
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# now tweak the object
dp_obj.description = 'the very first dp'
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 20.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -20.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 20.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -20.0
# now write the dp back to the registry
update_result = self.dpsc_cli.update_data_product(dp_obj)
# now get the dp back to see if it was updated
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertEquals(dp_obj.description,'the very first dp')
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Updated data product %s', dp_obj)
#test extension
extended_product = self.dpsc_cli.get_data_product_extension(dp_id)
#validate that there is one active and one retired user notification for this data product
self.assertEqual(1, len(extended_product.computed.active_user_subscriptions.value))
self.assertEqual(1, len(extended_product.computed.past_user_subscriptions.value))
self.assertEqual(dp_id, extended_product._id)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.product_download_size_estimated.status)
self.assertEqual(0, extended_product.computed.product_download_size_estimated.value)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.parameters.status)
#log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value)
def ion_object_encoder(obj):
return obj.__dict__
#test prepare for create
data_product_data = self.dpsc_cli.prepare_data_product_support()
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2)
self.assertEqual(data_product_data._id, "")
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 2)
self.assertEqual(len(data_product_data.associations['Dataset'].resources), 0)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 0)
self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 0)
#test prepare for update
data_product_data = self.dpsc_cli.prepare_data_product_support(dp_id)
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2)
self.assertEqual(data_product_data._id, dp_id)
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 2)
self.assertEqual(len(data_product_data.associations['Dataset'].resources), 1)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 1)
self.assertEqual(data_product_data.associations['StreamDefinition'].associated_resources[0].s, dp_id)
self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 1)
self.assertEqual(data_product_data.associations['Dataset'].associated_resources[0].s, dp_id)
# now 'delete' the data product
log.debug("deleting data product: %s" % dp_id)
self.dpsc_cli.delete_data_product(dp_id)
# Assert that there are no associated streams leftover after deleting the data product
stream_ids, assoc_ids = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True)
self.assertEquals(len(stream_ids), 0)
self.assertEquals(len(assoc_ids), 0)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# Get the events corresponding to the data product
ret = self.unsc.get_recent_events(resource_id=dp_id)
events = ret.value
for event in events:
log.debug("event time: %s" % event.ts_created)
self.assertTrue(len(events) > 0)
def test_data_product_stream_def(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id)
self.assertEquals(ctd_stream_def_id, stream_def_id)
def test_derived_data_product(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='ctd parsed', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsubcli.delete_stream_definition, ctd_stream_def_id)
dp = DataProduct(name='Instrument DP')
dp_id = self.dpsc_cli.create_data_product(dp, stream_definition_id=ctd_stream_def_id)
self.addCleanup(self.dpsc_cli.force_delete_data_product, dp_id)
self.dpsc_cli.activate_data_product_persistence(dp_id)
self.addCleanup(self.dpsc_cli.suspend_data_product_persistence, dp_id)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" % str(dp_id))
dataset_id = dataset_ids[0]
# Make the derived data product
simple_stream_def_id = self.pubsubcli.create_stream_definition(name='TEMPWAT stream def', parameter_dictionary_id=pdict_id, available_fields=['time','temp'])
tempwat_dp = DataProduct(name='TEMPWAT', category=DataProductTypeEnum.DERIVED)
tempwat_dp_id = self.dpsc_cli.create_data_product(tempwat_dp, stream_definition_id=simple_stream_def_id, parent_data_product_id=dp_id)
self.addCleanup(self.dpsc_cli.delete_data_product, tempwat_dp_id)
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
rdt['pressure'] = np.arange(20)
publisher = StandaloneStreamPublisher(stream_id,route)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
tempwat_dataset_ids, _ = self.rrclient.find_objects(tempwat_dp_id, PRED.hasDataset, id_only=True)
tempwat_dataset_id = tempwat_dataset_ids[0]
granule = self.data_retriever.retrieve(tempwat_dataset_id, delivery_format=simple_stream_def_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['time'], np.arange(20))
self.assertEquals(set(rdt.fields), set(['time','temp']))
def test_activate_suspend_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
# Construct temporal and spatial Coordinate Reference System objects
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
log.debug("Created an IonObject for a data product: %s" % dp_obj)
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# Subscribe to persist events
#------------------------------------------------------------------------------------------------
queue = gevent.queue.Queue()
def info_event_received(message, headers):
queue.put(message)
es = EventSubscriber(event_type=OT.InformationContentStatusEvent, callback=info_event_received, origin=dp_id, auto_delete=True)
es.start()
self.addCleanup(es.stop)
#------------------------------------------------------------------------------------------------
# test activate and suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" % str(dp_id))
dataset_id = dataset_ids[0]
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
publisher = StandaloneStreamPublisher(stream_id,route)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_ids[0])
self.assertIsInstance(replay_data, Granule)
log.debug("The data retriever was able to replay the dataset that was attached to the data product "
"we wanted to be persisted. Therefore the data product was indeed persisted with "
"otherwise we could not have retrieved its dataset using the data retriever. Therefore "
"this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'")
data_product_object = self.rrclient.read(dp_id)
self.assertEquals(data_product_object.name,'DP1')
self.assertEquals(data_product_object.description,'some new dp')
log.debug("Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. "
" Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the "
"resource registry, name='%s', desc='%s'" % (dp_obj.name, dp_obj.description,data_product_object.name,
data_product_object.description))
#------------------------------------------------------------------------------------------------
# test suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.suspend_data_product_persistence(dp_id)
dataset_modified.clear()
rdt['time'] = np.arange(20,40)
publisher.publish(rdt.to_granule())
self.assertFalse(dataset_modified.wait(2))
self.dpsc_cli.activate_data_product_persistence(dp_id)
dataset_modified.clear()
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_almost_equal(rdt['time'], np.arange(40))
dataset_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasDataset, id_only=True)
self.assertEquals(len(dataset_ids), 1)
self.dpsc_cli.suspend_data_product_persistence(dp_id)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.rrclient.read(dp_id)
info_event_counter = 0
runtime = 0
starttime = time.time()
caught_events = []
#check that the four InfoStatusEvents were received
while info_event_counter < 4 and runtime < 60 :
a = queue.get(timeout=60)
caught_events.append(a)
info_event_counter += 1
runtime = time.time() - starttime
self.assertEquals(info_event_counter, 4)
class UserNotificationIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.unsc = UserNotificationServiceClient(node=self.container.node)
self.rrc = ResourceRegistryServiceClient(node=self.container.node)
self.imc = IdentityManagementServiceClient(node=self.container.node)
def xtest_find_event_types_for_resource(self):
dataset_object = IonObject(RT.DataSet, name="dataset1")
dataset_id, version = self.rrc.create(dataset_object)
events = self.unsc.find_event_types_for_resource(dataset_id)
log.debug("dataset events = " + str(events))
try:
events = self.unsc.find_event_types_for_resource("bogus_id")
self.fail("failed to detect non-existant resource")
except:
pass
def test_create_two_user_notifications(self):
user_identty_object = IonObject(RT.UserIdentity, name="user1")
user_id = self.imc.create_user_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
notification_object = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['resource_lifecycle']})
self.unsc.create_notification(notification_object, user_id)
notification_object = IonObject(RT.NotificationRequest, {"name":"notification2",
"origin_list":['Some_Resource_Agent_ID2'],
"events_list":['data']})
self.unsc.create_notification(notification_object, user_id)
def test_delete_user_notifications(self):
user_identty_object = IonObject(RT.UserIdentity, name="user1")
user_id = self.imc.create_user_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
notification_object1 = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['resource_lifecycle']})
notification1_id = self.unsc.create_notification(notification_object1, user_id)
notification_object2 = IonObject(RT.NotificationRequest, {"name":"notification2",
"origin_list":['Some_Resource_Agent_ID2'],
"events_list":['data']})
notification2_id = self.unsc.create_notification(notification_object2, user_id)
self.unsc.delete_notification(notification1_id)
self.unsc.delete_notification(notification2_id)
def test_find_user_notifications(self):
user_identty_object = IonObject(RT.UserIdentity, name="user1")
user_id = self.imc.create_user_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
notification_object = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['resource_lifecycle']})
self.unsc.create_notification(notification_object, user_id)
notification_object = IonObject(RT.NotificationRequest, {"name":"notification2",
"origin_list":['Some_Resource_Agent_ID2'],
"events_list":['data']})
self.unsc.create_notification(notification_object, user_id)
notifications = self.unsc.find_notifications_by_user(user_id)
for n in notifications:
log.debug("n = " +str(n))
def test_update_user_notification(self):
user_identty_object = IonObject(RT.UserIdentity, name="user1")
user_id = self.imc.create_user_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
notification_object = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['resource_lifecycle']})
notification_id = self.unsc.create_notification(notification_object, user_id)
notification = self.rrc.read(notification_id)
notification.origin_list = ['Some_Resource_Agent_ID5']
self.unsc.update_notification(notification)
def test_send_notification_emails(self):
user_identty_object = IonObject(RT.UserIdentity, name="user1")
user_id = self.imc.create_user_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
notification_object = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['resource_lifecycle']})
self.unsc.create_notification(notification_object, user_id)
notification_object = IonObject(RT.NotificationRequest, {"name":"notification2",
"origin_list":['Some_Resource_Agent_ID2'],
"events_list":['data']})
self.unsc.create_notification(notification_object, user_id)
rle_publisher = ResourceLifecycleEventPublisher()
rle_publisher.create_and_publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event")
de_publisher = DataEventPublisher()
de_publisher.create_and_publish_event(origin='Some_Resource_Agent_ID2', description="DE test event")
gevent.sleep(1)
def test_find_events(self):
rle_publisher = ResourceLifecycleEventPublisher(event_repo=self.container.event_repository)
rle_publisher.create_and_publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event1")
rle_publisher.create_and_publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event2")
de_publisher = DataEventPublisher(event_repo=self.container.event_repository)
de_publisher.create_and_publish_event(origin='Some_Resource_Agent_ID2', description="DE test event1")
de_publisher.create_and_publish_event(origin='Some_Resource_Agent_ID2', description="DE test event2")
events = self.unsc.find_events(origin='Some_Resource_Agent_ID1')
for event in events:
log.debug("event=" + str(event))
events = self.unsc.find_events(type='DataEvent')
for event in events:
log.debug("event=" + str(event))
def instrument_test_driver(container):
org_client = OrgManagementServiceClient(node=container.node)
id_client = IdentityManagementServiceClient(node=container.node)
system_actor = id_client.find_actor_identity_by_name(name=CFG.system.system_actor)
log.info('system actor:' + system_actor._id)
sa_header_roles = get_role_message_headers(org_client.find_all_roles_by_user(system_actor._id))
# Names of agent data streams to be configured.
parsed_stream_name = 'ctd_parsed'
raw_stream_name = 'ctd_raw'
# Driver configuration.
#Simulator
driver_config = {
'svr_addr': 'localhost',
'cmd_port': 5556,
'evt_port': 5557,
'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'comms_config': {
SBE37Channel.CTD: {
'method':'ethernet',
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'server_addr': 'localhost',
'server_port': 8888
}
}
}
#Hardware
_container_client = ContainerAgentClient(node=container.node,
name=container.name)
# Create a pubsub client to create streams.
_pubsub_client = PubsubManagementServiceClient(node=container.node)
# A callback for processing subscribed-to data.
def consume(message, headers):
log.info('Subscriber received message: %s', str(message))
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(process=container,
node=container.node)
subs = []
# Create streams for each stream named in driver.
stream_config = {}
for (stream_name, val) in PACKET_CONFIG.iteritems():
stream_def = ctd_stream_definition(stream_id=None)
stream_def_id = _pubsub_client.create_stream_definition(
container=stream_def)
stream_id = _pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
original=True,
encoding='ION R2', headers={'ion-actor-id': system_actor._id, 'ion-actor-roles': sa_header_roles })
stream_config[stream_name] = stream_id
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name, callback=consume)
sub.start()
query = StreamQuery(stream_ids=[stream_id])
sub_id = _pubsub_client.create_subscription(\
query=query, exchange_name=exchange_name )
_pubsub_client.activate_subscription(sub_id)
subs.append(sub)
# Create agent config.
agent_resource_id = '123xyz'
agent_config = {
'driver_config' : driver_config,
'stream_config' : stream_config,
'agent' : {'resource_id': agent_resource_id}
}
# Launch an instrument agent process.
_ia_name = 'agent007'
_ia_mod = 'ion.agents.instrument.instrument_agent'
_ia_class = 'InstrumentAgent'
_ia_pid = _container_client.spawn_process(name=_ia_name,
module=_ia_mod, cls=_ia_class,
config=agent_config)
log.info('got pid=%s for resource_id=%s' % (str(_ia_pid), str(agent_resource_id)))
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
# @unittest.skip('this test just for debugging setup')
# def test_just_the_setup(self):
# return
@attr('EXT')
def test_resources_associations_extensions(self):
"""
create one of each resource and association used by IMS
to guard against problems in ion-definitions
"""
#stuff we control
instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance))
instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent))
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance))
platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent))
platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice))
platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel))
sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice))
sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel))
#stuff we associate to
data_producer_id, _ = self.RR.create(any_old(RT.DataProducer))
org_id, _ = self.RR.create(any_old(RT.Org))
#instrument_agent_instance_id #is only a target
#instrument_agent
self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id)
#instrument_device
self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id)
self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id)
self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id)
self.RR.create_association(org_id, PRED.hasResource, instrument_device_id)
instrument_model_id #is only a target
platform_agent_instance_id #is only a target
#platform_agent
self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id)
#platform_device
self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id)
self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id)
platform_model_id #is only a target
#sensor_device
self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id)
self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id)
sensor_model_id #is only a target
#create a parsed product for this instrument output
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
processing_level_code='Parsed_Canonical',
temporal_domain = tdom,
spatial_domain = sdom)
pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1)
def addInstOwner(inst_id, subject):
actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject})
user_id = self.IDS.create_actor_identity(actor_identity_obj)
user_info_obj = any_old(RT.UserInfo)
user_info_id = self.IDS.create_user_info(user_id, user_info_obj)
self.RR.create_association(inst_id, PRED.hasOwner, user_id)
#Testing multiple instrument owners
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254")
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256")
extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id)
self.assertEqual(instrument_device_id, extended_instrument._id)
self.assertEqual(len(extended_instrument.owners), 2)
self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id)
# Lifecycle
self.assertEquals(len(extended_instrument.lcstate_transitions), 7)
self.assertEquals(set(extended_instrument.lcstate_transitions.keys()), set(['enable', 'develop', 'deploy', 'retire', 'plan', 'integrate', 'announce']))
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(extended_instrument.computed.firmware_version, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.last_calibration_datetime, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue)
self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue)
log.debug("extended_instrument.computed: %s", extended_instrument.computed)
#check model
inst_model_obj = self.RR.read(instrument_model_id)
self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name)
#check agent instance
inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id)
self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name)
#check agent
inst_agent_obj = self.RR.read(instrument_agent_id)
#compound assoc return list of lists so check the first element
self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent.name)
#check platform device
plat_device_obj = self.RR.read(platform_device_id)
self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name)
extended_platform = self.IMS.get_platform_device_extension(platform_device_id)
self.assertEqual(1, len(extended_platform.instrument_devices))
self.assertEqual(instrument_device_id, extended_platform.instrument_devices[0]._id)
self.assertEqual(1, len(extended_platform.instrument_models))
self.assertEqual(instrument_model_id, extended_platform.instrument_models[0]._id)
self.assertEquals(extended_platform.platform_agent._id, platform_agent_id)
self.assertEquals(len(extended_platform.lcstate_transitions), 7)
self.assertEquals(set(extended_platform.lcstate_transitions.keys()), set(['enable', 'develop', 'deploy', 'retire', 'plan', 'integrate', 'announce']))
#check sensor devices
self.assertEqual(1, len(extended_instrument.sensor_devices))
#check data_product_parameters_set
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_instrument.computed.data_product_parameters_set.status)
self.assertTrue( 'Parsed_Canonical' in extended_instrument.computed.data_product_parameters_set.value)
# the ctd parameters should include 'temp'
self.assertTrue( 'temp' in extended_instrument.computed.data_product_parameters_set.value['Parsed_Canonical'])
#none of these will work because there is no agent
self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
extended_instrument.computed.firmware_version.status)
# self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
# extended_instrument.computed.operational_state.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.power_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.communications_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.data_status_roll_up.status)
# self.assertEqual(StatusType.STATUS_OK,
# extended_instrument.computed.data_status_roll_up.value)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.location_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.recent_events.status)
# self.assertEqual([], extended_instrument.computed.recent_events.value)
# cleanup
c = DotDict()
c.resource_registry = self.RR
self.RR2.pluck(instrument_agent_id)
self.RR2.pluck(instrument_model_id)
self.RR2.pluck(instrument_device_id)
self.RR2.pluck(platform_agent_id)
self.RR2.pluck(sensor_device_id)
self.IMS.force_delete_instrument_agent(instrument_agent_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id)
self.IMS.force_delete_platform_agent(platform_agent_id)
self.IMS.force_delete_platform_device(platform_device_id)
self.IMS.force_delete_platform_model(platform_model_id)
self.IMS.force_delete_sensor_device(sensor_device_id)
self.IMS.force_delete_sensor_model(sensor_model_id)
#stuff we associate to
self.RR.delete(data_producer_id)
self.RR.delete(org_id)
def test_custom_attributes(self):
"""
Test assignment of custom attributes
"""
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel,
{"custom_attributes":
{"favorite_color": "attr desc goes here"}
}))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice,
{"custom_attributes":
{"favorite_color": "red",
"bogus_attr": "should raise warning"
}
}))
self.IMS.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id)
# cleanup
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
def _get_datastore(self, dataset_id):
dataset = self.DSC.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def test_resource_state_save_restore(self):
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.IMS.create_instrument_model(instModel_obj)
log.debug( 'new InstrumentModel id = %s ', instModel_id)
# Create InstrumentAgent
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 )
parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 )
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg",
stream_configurations = [raw_config, parsed_config] )
instAgent_id = self.IMS.create_instrument_agent(instAgent_obj)
log.debug( 'new InstrumentAgent id = %s', instAgent_id)
self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 '
+ '(SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345" )
instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj)
self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=spdict_id)
rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
self.DP.activate_data_product_persistence(data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug( 'Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True)
log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
#create the datastore at the beginning of each int test that persists data
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id2 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=raw_stream_def_id)
log.debug( 'new dp_id = %s', str(data_product_id2))
self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.DP.activate_data_product_persistence(data_product_id=data_product_id2)
# spin up agent
self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.addCleanup(self.IMS.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id)
gate = ProcessStateGate(self.PDC.read_process,
instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" %
instance_obj.agent_process_id)
# take snapshot of config
snap_id = self.IMS.save_resource_state(instDevice_id, "xyzzy snapshot")
snap_obj = self.RR.read_attachment(snap_id, include_content=True)
print "Saved config:"
print snap_obj.content
#modify config
instance_obj.driver_config["comms_config"] = "BAD_DATA"
self.RR.update(instance_obj)
#restore config
self.IMS.restore_resource_state(instDevice_id, snap_id)
instance_obj = self.RR.read(instAgentInstance_id)
self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"])
self.DP.delete_data_product(data_product_id1)
self.DP.delete_data_product(data_product_id2)
def test_agent_instance_config(self):
"""
Verify that agent configurations are being built properly
"""
clients = DotDict()
clients.resource_registry = self.RR
clients.pubsub_management = self.PSC
clients.dataset_management = self.DSC
pconfig_builder = PlatformAgentConfigurationBuilder(clients)
iconfig_builder = InstrumentAgentConfigurationBuilder(clients)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
org_obj = any_old(RT.Org)
org_id = self.RR2.create(org_obj)
inst_startup_config = {'startup': 'config'}
generic_alerts_config = {'lvl1': {'lvl2': 'lvl3val'}}
required_config_keys = [
'org_name',
'device_type',
'agent',
'driver_config',
'stream_config',
'startup_config',
'aparam_alert_config',
'children']
def verify_instrument_config(config, device_id):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(org_obj.name, config['org_name'])
self.assertEqual(RT.InstrumentDevice, config['device_type'])
self.assertIn('driver_config', config)
driver_config = config['driver_config']
expected_driver_fields = {'process_type': ('ZMQPyClassDriverLauncher',),
}
for k, v in expected_driver_fields.iteritems():
self.assertIn(k, driver_config)
self.assertEqual(v, driver_config[k])
self.assertEqual
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertEqual(inst_startup_config, config['startup_config'])
self.assertIn('aparam_alert_config', config)
self.assertEqual(generic_alerts_config, config['aparam_alert_config'])
self.assertIn('stream_config', config)
for key in ['children']:
self.assertEqual({}, config[key])
def verify_child_config(config, device_id, inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(org_obj.name, config['org_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertIn('aparam_alert_config', config)
self.assertEqual(generic_alerts_config, config['aparam_alert_config'])
self.assertIn('stream_config', config)
self.assertIn('driver_config', config)
self.assertIn('foo', config['driver_config'])
self.assertEqual('bar', config['driver_config']['foo'])
self.assertIn('process_type', config['driver_config'])
self.assertEqual(('ZMQPyClassDriverLauncher',), config['driver_config']['process_type'])
if None is inst_device_id:
for key in ['children', 'startup_config']:
self.assertEqual({}, config[key])
else:
for key in ['startup_config']:
self.assertEqual({}, config[key])
self.assertIn(inst_device_id, config['children'])
verify_instrument_config(config['children'][inst_device_id], inst_device_id)
def verify_parent_config(config, parent_device_id, child_device_id, inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(org_obj.name, config['org_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertIn('process_type', config['driver_config'])
self.assertEqual(('ZMQPyClassDriverLauncher',), config['driver_config']['process_type'])
self.assertEqual({'resource_id': parent_device_id}, config['agent'])
self.assertIn('aparam_alert_config', config)
self.assertEqual(generic_alerts_config, config['aparam_alert_config'])
self.assertIn('stream_config', config)
for key in ['startup_config']:
self.assertEqual({}, config[key])
self.assertIn(child_device_id, config['children'])
verify_child_config(config['children'][child_device_id], child_device_id, inst_device_id)
rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id)
#todo: create org and figure out which agent resource needs to get assigned to it
def _make_platform_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
platform_agent_instance_obj = any_old(RT.PlatformAgentInstance, {'driver_config': {'foo': 'bar'},
'alerts': generic_alerts_config})
platform_agent_instance_obj.agent_config = agent_config
platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 )
platform_agent_obj = any_old(RT.PlatformAgent, {"stream_configurations":[raw_config]})
platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj)
# device creation
platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_platform_agent_instance_to_platform_device(platform_agent_instance_id, platform_device_id)
self.RR2.assign_platform_agent_to_platform_agent_instance(platform_agent_id, platform_agent_instance_id)
self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, org_id)
return platform_agent_instance_id, platform_agent_id, platform_device_id
def _make_instrument_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
instrument_agent_instance_obj = any_old(RT.InstrumentAgentInstance, {"startup_config": inst_startup_config,
'alerts': generic_alerts_config})
instrument_agent_instance_obj.agent_config = agent_config
instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(stream_name='raw',
parameter_dictionary_name='ctd_raw_param_dict',
records_per_granule=2,
granule_publish_rate=5 )
instrument_agent_obj = any_old(RT.InstrumentAgent, {"stream_configurations":[raw_config]})
instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj)
# device creation
instrument_device_id = self.IMS.create_instrument_device(any_old(RT.InstrumentDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_instrument_agent_instance_to_instrument_device(instrument_agent_instance_id, instrument_device_id)
self.RR2.assign_instrument_agent_to_instrument_agent_instance(instrument_agent_id, instrument_agent_instance_id)
self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id)
return instrument_agent_instance_id, instrument_agent_id, instrument_device_id
# can't do anything without an agent instance obj
log.debug("Testing that preparing a launcher without agent instance raises an error")
self.assertRaises(AssertionError, pconfig_builder.prepare, will_launch=False)
log.debug("Making the structure for a platform agent, which will be the child")
platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure()
platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id)
log.debug("Preparing a valid agent instance launch, for config only")
pconfig_builder.set_agent_instance_object(platform_agent_instance_child_obj)
child_config = pconfig_builder.prepare(will_launch=False)
verify_child_config(child_config, platform_device_child_id)
log.debug("Making the structure for a platform agent, which will be the parent")
platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure()
platform_agent_instance_parent_obj = self.RR2.read(platform_agent_instance_parent_id)
log.debug("Testing child-less parent as a child config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
verify_child_config(parent_config, platform_device_parent_id)
log.debug("assigning child platform to parent")
self.RR2.assign_platform_device_to_platform_device(platform_device_child_id, platform_device_parent_id)
child_device_ids = self.RR2.find_platform_device_ids_of_device(platform_device_parent_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing parent + child as parent config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
verify_parent_config(parent_config, platform_device_parent_id, platform_device_child_id)
log.debug("making the structure for an instrument agent")
instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure()
instrument_agent_instance_obj = self.RR2.read(instrument_agent_instance_id)
log.debug("Testing instrument config")
iconfig_builder.set_agent_instance_object(instrument_agent_instance_obj)
instrument_config = iconfig_builder.prepare(will_launch=False)
verify_instrument_config(instrument_config, instrument_device_id)
log.debug("assigning instrument to platform")
self.RR2.assign_instrument_device_to_platform_device(instrument_device_id, platform_device_child_id)
child_device_ids = self.RR2.find_instrument_device_ids_of_device(platform_device_child_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing entire config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
full_config = pconfig_builder.prepare(will_launch=False)
verify_parent_config(full_config, platform_device_parent_id, platform_device_child_id, instrument_device_id)
