def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.unsc = UserNotificationServiceClient(node=self.container.node)
self.rrc = ResourceRegistryServiceClient(node=self.container.node)
self.imc = IdentityManagementServiceClient(node=self.container.node)
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dpsc_cli = DataProductManagementServiceClient()
self.rrclient = ResourceRegistryServiceClient()
self.damsclient = DataAcquisitionManagementServiceClient()
self.pubsubcli = PubsubManagementServiceClient()
self.ingestclient = IngestionManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.unsc = UserNotificationServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.identcli = IdentityManagementServiceClient()
#------------------------------------------
# Create the environment
#------------------------------------------
self.stream_def_id = self.pubsubcli.create_stream_definition(
name='SBE37_CDM')
self.process_definitions = {}
ingestion_worker_definition = ProcessDefinition(
name='ingestion worker')
ingestion_worker_definition.executable = {
'module':
'ion.processes.data.ingestion.science_granule_ingestion_worker',
'class': 'ScienceGranuleIngestionWorker'
}
process_definition_id = self.process_dispatcher.create_process_definition(
process_definition=ingestion_worker_definition)
self.process_definitions['ingestion_worker'] = process_definition_id
self.pids = []
self.exchange_points = []
self.exchange_names = []
#------------------------------------------------------------------------------------------------
# First launch the ingestors
#------------------------------------------------------------------------------------------------
self.exchange_space = 'science_granule_ingestion'
self.exchange_point = 'science_data'
config = DotDict()
config.process.datastore_name = 'datasets'
config.process.queue_name = self.exchange_space
self.exchange_names.append(self.exchange_space)
self.exchange_points.append(self.exchange_point)
pid = self.process_dispatcher.schedule_process(
self.process_definitions['ingestion_worker'], configuration=config)
log.debug("the ingestion worker process id: %s", pid)
self.pids.append(pid)
self.addCleanup(self.cleaning_up)
def setUp(self):
self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254"
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2coi.yml')
self.identity_management_service = IdentityManagementServiceClient(
node=self.container.node)
def setUp(self):
self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254"
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.resource_registry = ResourceRegistryServiceClient()
self.identity_management_service = IdentityManagementServiceClient()
self.org_client = OrgManagementServiceClient()
def test_non_anonymous_resource_registry_operations_through_gateway(self):
id_client = IdentityManagementServiceClient(node=self.container.node)
actor_id, valid_until, registered = id_client.signon(USER1_CERTIFICATE, True)
response = self.test_app.get('/ion-service/resource_registry/find_resources?name=TestDataProduct&id_only=True&requester=' + actor_id)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
self.assertEqual(len(response_data),2 )
self.assertEqual(len(response_data[0]), 0 )
id_client.delete_actor_identity(actor_id)
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
unittest # suppress an pycharm inspector error if all unittest.skip references are commented out
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.OMS = ObservatoryManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
print 'started services'
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient()
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
# Use the network definition provided by RSN OMS directly.
rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri'])
self._network_definition = RsnOmsUtil.build_network_definition(rsn_oms)
# get serialized version for the configuration:
self._network_definition_ser = NetworkUtil.serialize_network_definition(
self._network_definition)
if log.isEnabledFor(logging.TRACE):
log.trace("NetworkDefinition serialization:\n%s",
self._network_definition_ser)
self._async_data_result = AsyncResult()
self._data_subscribers = []
self._samples_received = []
self.addCleanup(self._stop_data_subscribers)
self._async_event_result = AsyncResult()
self._event_subscribers = []
self._events_received = []
self.addCleanup(self._stop_event_subscribers)
self._start_event_subscriber()
def instrument_test_driver(container):
org_client = OrgManagementServiceClient(node=container.node)
id_client = IdentityManagementServiceClient(node=container.node)
system_actor = id_client.find_actor_identity_by_name(name=CFG.system.system_actor)
log.info('system actor:' + system_actor._id)
sa_header_roles = get_role_message_headers(org_client.find_all_roles_by_user(system_actor._id))
# Names of agent data streams to be configured.
parsed_stream_name = 'ctd_parsed'
raw_stream_name = 'ctd_raw'
# Driver configuration.
#Simulator
driver_config = {
'svr_addr': 'localhost',
'cmd_port': 5556,
'evt_port': 5557,
'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'comms_config': {
SBE37Channel.CTD: {
'method':'ethernet',
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'server_addr': 'localhost',
'server_port': 8888
}
}
}
#Hardware
_container_client = ContainerAgentClient(node=container.node,
name=container.name)
# Create a pubsub client to create streams.
_pubsub_client = PubsubManagementServiceClient(node=container.node)
# A callback for processing subscribed-to data.
def consume(message, headers):
log.info('Subscriber received message: %s', str(message))
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(process=container,
node=container.node)
subs = []
# Create streams for each stream named in driver.
stream_config = {}
for (stream_name, val) in PACKET_CONFIG.iteritems():
stream_def = ctd_stream_definition(stream_id=None)
stream_def_id = _pubsub_client.create_stream_definition(
container=stream_def)
stream_id = _pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
original=True,
encoding='ION R2', headers={'ion-actor-id': system_actor._id, 'ion-actor-roles': sa_header_roles })
stream_config[stream_name] = stream_id
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name, callback=consume)
sub.start()
query = StreamQuery(stream_ids=[stream_id])
sub_id = _pubsub_client.create_subscription(\
query=query, exchange_name=exchange_name )
_pubsub_client.activate_subscription(sub_id)
subs.append(sub)
# Create agent config.
agent_resource_id = '123xyz'
agent_config = {
'driver_config' : driver_config,
'stream_config' : stream_config,
'agent' : {'resource_id': agent_resource_id}
}
# Launch an instrument agent process.
_ia_name = 'agent007'
_ia_mod = 'ion.agents.instrument.instrument_agent'
_ia_class = 'InstrumentAgent'
_ia_pid = _container_client.spawn_process(name=_ia_name,
module=_ia_mod, cls=_ia_class,
config=agent_config)
log.info('got pid=%s for resource_id=%s' % (str(_ia_pid), str(agent_resource_id)))
def test_user_role_cache(self):
#Create a user
id_client = IdentityManagementServiceClient(node=self.container.node)
actor_id, valid_until, registered = id_client.signon(
USER1_CERTIFICATE, True)
#Make a request with this new user to get it into the cache
response = self.test_app.get(
'/ion-service/resource_registry/find_resources?name=TestDataProduct&id_only=True&requester='
+ actor_id)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
#Check the contents of the user role cache for this user
service_gateway_user_cache = self.container.proc_manager.procs_by_name[
'service_gateway'].user_data_cache
self.assertEqual(service_gateway_user_cache.has_key(actor_id), True)
role_header = service_gateway_user_cache.get(actor_id)
self.assertIn('ION', role_header)
self.assertEqual(len(role_header['ION']), 1)
self.assertIn('ORG_MEMBER', role_header['ION'])
org_client = OrgManagementServiceClient(node=self.container.node)
ion_org = org_client.find_org()
manager_role = org_client.find_org_role_by_name(
org_id=ion_org._id, role_name='ORG_MANAGER')
org_client.grant_role(org_id=ion_org._id,
user_id=actor_id,
role_name='ORG_MANAGER')
#Just allow some time for event processing on slower platforms
gevent.sleep(2)
#The user should be evicted from the cache due to a change in roles
self.assertEqual(service_gateway_user_cache.has_key(actor_id), False)
#Do it again to check for new roles
response = self.test_app.get(
'/ion-service/resource_registry/find_resources?name=TestDataProduct&id_only=True&requester='
+ actor_id)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
#Check the contents of the user role cache for this user
self.assertEqual(service_gateway_user_cache.has_key(actor_id), True)
role_header = service_gateway_user_cache.get(actor_id)
self.assertIn('ION', role_header)
self.assertEqual(len(role_header['ION']), 2)
self.assertIn('ORG_MEMBER', role_header['ION'])
self.assertIn('ORG_MANAGER', role_header['ION'])
#Now flush the user_role_cache and make sure it was flushed
event_publisher = EventPublisher()
event_publisher.publish_event(event_type=OT.UserRoleCacheResetEvent)
#Just allow some time for event processing on slower platforms
gevent.sleep(2)
self.assertEqual(service_gateway_user_cache.has_key(actor_id), False)
self.assertEqual(service_gateway_user_cache.size(), 0)
#Change the role once again and see if it is there again
org_client.revoke_role(org_id=ion_org._id,
user_id=actor_id,
role_name='ORG_MANAGER')
#Just allow some time for event processing on slower platforms
gevent.sleep(2)
#The user should still not be there
self.assertEqual(service_gateway_user_cache.has_key(actor_id), False)
#Do it again to check for new roles
response = self.test_app.get(
'/ion-service/resource_registry/find_resources?name=TestDataProduct&id_only=True&requester='
+ actor_id)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
#Check the contents of the user role cache for this user
self.assertEqual(service_gateway_user_cache.has_key(actor_id), True)
role_header = service_gateway_user_cache.get(actor_id)
self.assertIn('ION', role_header)
self.assertEqual(len(role_header['ION']), 1)
self.assertIn('ORG_MEMBER', role_header['ION'])
id_client.delete_actor_identity(actor_id)
def test_non_anonymous_resource_registry_operations_with_token(self):
rr = self.container.resource_registry
id_client = IdentityManagementServiceClient()
create_request = {
"serviceRequest": {
"serviceName": "resource_registry",
"serviceOp": "create",
"params": {
"object": {
"name": "Instrument1",
"type_": "InstrumentDevice"
}
}
}
}
# Create without actor
response = self.test_app.post(
'/ion-service/resource_registry/create',
{'payload': simplejson.dumps(create_request)})
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertGreaterEqual(len(response_data[0]),
20) # This is a resource_id
inst_id = str(response_data[0])
inst_obj = rr.read(inst_id)
self.assertEquals(inst_obj.type_, RT.InstrumentDevice)
self.assertEquals(inst_obj.name, "Instrument1")
self.assertEquals(inst_obj.visibility, ResourceVisibilityEnum.PUBLIC)
act_objs, assocs = rr.find_objects(inst_id,
PRED.hasOwner,
RT.ActorIdentity,
id_only=False)
self.assertEquals(len(act_objs), 0)
# Anonymous query shows PUBLIC visibility resource anymore
response = self.test_app.get(
'/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True'
)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 1)
self.assertEqual(len(response_data[1]), 1)
self.assertEqual(response_data[0][0], inst_id)
inst_obj.visibility = ResourceVisibilityEnum.OWNER
rr.update(inst_obj)
# Now the anonymous query should not show the resource anymore
response = self.test_app.get(
'/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True'
)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 0)
rr.delete(inst_id)
# Create with actor
actor_id, valid_until, registered = id_client.signon(
USER1_CERTIFICATE, True)
create_request = {
"serviceRequest": {
"serviceName": "resource_registry",
"serviceOp": "create",
"requester": actor_id,
"params": {
"object": {
"name": "Instrument1",
"type_": "InstrumentDevice",
"visibility": ResourceVisibilityEnum.OWNER
}
}
}
}
response = self.test_app.post(
'/ion-service/resource_registry/create',
{'payload': simplejson.dumps(create_request)})
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertGreaterEqual(len(response_data[0]),
20) # This is a resource_id
inst_id = str(response_data[0])
inst_obj = rr.read(inst_id)
self.assertEquals(inst_obj.type_, RT.InstrumentDevice)
self.assertEquals(inst_obj.name, "Instrument1")
self.assertEquals(inst_obj.visibility, ResourceVisibilityEnum.OWNER)
act_objs, assocs = rr.find_objects(inst_id,
PRED.hasOwner,
RT.ActorIdentity,
id_only=False)
self.assertEquals(len(act_objs), 1)
self.assertEquals(act_objs[0]._id, actor_id)
# Anonymous query should not show the resource anymore
response = self.test_app.get(
'/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True'
)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 0)
# Authenticated request with owner shows resource
response = self.test_app.get(
'/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&requester='
+ actor_id)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 1)
self.assertEqual(len(response_data[1]), 1)
self.assertEqual(response_data[0][0], inst_id)
token_str = id_client.create_authentication_token(actor_id, validity=2)
# Request with authentication token for owner shows resource
response = self.test_app.get(
'/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&authtoken='
+ token_str)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 1)
self.assertEqual(len(response_data[1]), 1)
self.assertEqual(response_data[0][0], inst_id)
gevent.sleep(2.1)
# Request with expired authentication token for owner does not show resource (no error though)
response = self.test_app.get(
'/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&authtoken='
+ token_str)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 0)
# Request with valid authentication token for different user does not show resource
actor_id2, _ = rr.create(IonObject(RT.ActorIdentity, name="Actor2"))
token_str2 = id_client.create_authentication_token(actor_id2,
validity=2)
response = self.test_app.get(
'/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&authtoken='
+ token_str2)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 0)
token_str3 = id_client.create_authentication_token(actor_id,
validity=2)
# Request with new authentication token for owner shows resource
response = self.test_app.get(
'/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&authtoken='
+ token_str3)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 1)
self.assertEqual(len(response_data[1]), 1)
self.assertEqual(response_data[0][0], inst_id)
id_client.invalidate_authentication_token(token_str3)
response = self.test_app.get(
'/ion-service/resource_registry/find_resources?name=Instrument1&id_only=True&authtoken='
+ token_str3)
self.check_response_headers(response)
self.assertIn(GATEWAY_RESPONSE, response.json['data'])
response_data = response.json['data'][GATEWAY_RESPONSE]
self.assertEqual(len(response_data), 2)
self.assertEqual(len(response_data[0]), 0)
# Cleanup
rr.delete(inst_id)
rr.delete(actor_id2)
id_client.delete_actor_identity(actor_id)
