def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
print 'started services'
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
def setUp(self):
# Start container
super(TestActivateRSNVel3DInstrument, self).setUp()
config = DotDict()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml', config)
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
self.OMS = ObservatoryManagementServiceClient(node=self.container.node)
self.org_management_service = OrgManagementServiceClient(
node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
#print 'TestObservatoryManagementServiceIntegration: started services'
self.event_publisher = EventPublisher()
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
print 'started services'
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(
node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dataset_management = self.datasetclient
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(
node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.workflowclient = WorkflowManagementServiceClient(
node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient(
node=self.container.node)
self.ctd_stream_def = SBE37_CDM_stream_definition()
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(
node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(
node=self.container.node)
self.dpmsclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(
node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(
RT.DataProcess, None, None, True)[0]:
self.dataprocessclient.deactivate_data_process(proc_id)
self.dataprocessclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
def is_service_available(self, service_name, local_rr_only=False):
try:
service_resource = None
#from pyon.core.bootstrap import container_instance
from mi.core.bootstrap import container_instance
from interface.objects import ServiceStateEnum
# Use container direct RR connection if available, otherwise use messaging to the RR service
if hasattr(container_instance, 'has_capability') and container_instance.has_capability('RESOURCE_REGISTRY'):
service_resource, _ = container_instance.resource_registry.find_resources(restype='Service', name=service_name)
elif not local_rr_only:
from interface.services.coi.iresource_registry_service import ResourceRegistryServiceClient
rr_client = ResourceRegistryServiceClient(container_instance.node)
service_resource, _ = rr_client.find_resources(restype='Service', name=service_name)
else:
log.warn("is_service_available(%s) - No RR connection" % service_name)
# The service is available only of there is a single RR object for it and it is in one of these states:
if service_resource and len(service_resource) > 1:
log.warn("is_service_available(%s) - Found multiple service instances: %s", service_name, service_resource)
# MM 2013-08-17: Added PENDING, because this means service will be there shortly
if service_resource and service_resource[0].state in (ServiceStateEnum.READY, ServiceStateEnum.STEADY, ServiceStateEnum.PENDING):
return True
elif service_resource:
log.warn("is_service_available(%s) - Service resource in invalid state", service_resource)
return False
except Exception as ex:
return False
def _update_new_data_check_attachment(cls, res_id, new_content):
"""
Update the list of data files for the external dataset resource
@param res_id the ID of the external dataset resource
@param new_content list of new files found on data host
@throws InstrumentException if external dataset resource can't be found
@retval the attachment ID
"""
rr_cli = ResourceRegistryServiceClient()
try:
# Delete any attachments with the "NewDataCheck" keyword
attachment_objs = rr_cli.find_attachments(resource_id=res_id, include_content=False, id_only=False)
for attachment_obj in attachment_objs:
kwds = set(attachment_obj.keywords)
if 'NewDataCheck' in kwds:
log.debug('Delete NewDataCheck attachment: {0}'.format(attachment_obj._id))
rr_cli.delete_attachment(attachment_obj._id)
else:
log.debug('Found attachment: {0}'.format(attachment_obj))
# Create the new attachment
att = Attachment(name='new_data_check', attachment_type=AttachmentType.ASCII, keywords=['NewDataCheck', ], content_type='text/plain', content=msgpack.packb(new_content))
att_id = rr_cli.create_attachment(resource_id=res_id, attachment=att)
except NotFound:
raise InstrumentException('ExternalDatasetResource \'{0}\' not found'.format(res_id))
return att_id
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.dpmsclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
# create missing data process definition
dpd_obj = IonObject(RT.DataProcessDefinition,
name=LOGICAL_TRANSFORM_DEFINITION_NAME,
description="normally in preload",
module='ion.processes.data.transforms.logical_transform',
class_name='logical_transform')
self.dataprocessclient.create_data_process_definition(dpd_obj)
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]:
self.dataprocessclient.deactivate_data_process(proc_id)
self.dataprocessclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
def helper_create_highcharts_workflow_def(container):
from interface.services.coi.iresource_registry_service import ResourceRegistryServiceClient
rrclient = ResourceRegistryServiceClient(node=container.node)
# Check to see if the workflow defnition already exist
workflow_def_ids,_ = rrclient.find_resources(restype=RT.WorkflowDefinition, name='Realtime_HighCharts', id_only=True)
if len(workflow_def_ids) > 0:
workflow_def_id = workflow_def_ids[0]
else:
# Build the workflow definition
workflow_def_obj = IonObject(RT.WorkflowDefinition, name='Realtime_HighCharts',description='Convert stream data to HighCharts data')
#Add a transformation process definition
procdef_id = helper_create_highcharts_data_process_definition(container)
workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=procdef_id)
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#Create it in the resource registry
from interface.services.ans.iworkflow_management_service import WorkflowManagementServiceClient
workflowclient = WorkflowManagementServiceClient(node=container.node)
workflow_def_id = workflowclient.create_workflow_definition(workflow_def_obj)
return workflow_def_id
def setUp(self):
super(DiscoveryIntTest, self).setUp()
self._start_container()
self.addCleanup(DiscoveryIntTest.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.discovery = DiscoveryServiceClient()
self.catalog = CatalogManagementServiceClient()
self.ims = IndexManagementServiceClient()
self.rr = ResourceRegistryServiceClient()
if use_es:
self.es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
self.es_port = CFG.get_safe('server.elasticsearch.port', '9200')
CFG.server.elasticsearch.shards = 1
CFG.server.elasticsearch.replicas = 0
CFG.server.elasticsearch.river_shards = 1
CFG.server.elasticsearch.river_replicas = 0
self.es = ep.ElasticSearch(
host=self.es_host,
port=self.es_port,
timeout=10,
verbose=True
)
op = DotDict(CFG)
op.op = 'clean_bootstrap'
self.container.spawn_process('index_bootstrap','ion.processes.bootstrap.index_bootstrap','IndexBootStrap', op)
def is_service_available(self, service_name, local_rr_only=False):
try:
service_resource = None
from pyon.core.bootstrap import container_instance
from interface.objects import ServiceStateEnum
#Use container direct RR connection if available, otherwise use messaging to the RR service
if hasattr(container_instance,
'has_capability') and container_instance.has_capability(
'RESOURCE_REGISTRY'):
service_resource, _ = container_instance.resource_registry.find_resources(
restype='Service', name=service_name)
else:
if not local_rr_only:
from interface.services.coi.iresource_registry_service import ResourceRegistryServiceClient
rr_client = ResourceRegistryServiceClient(
container_instance.node)
service_resource, _ = rr_client.find_resources(
restype='Service', name=service_name)
#The service is available only of there is a single RR object for it and it is in one of these states:
if service_resource and (
service_resource[0].state == ServiceStateEnum.READY
or service_resource[0].state == ServiceStateEnum.STEADY):
return True
return False
except Exception, e:
return False
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
self.event_publisher = EventPublisher()
def is_service_available(self, service_name, local_rr_only=False):
try:
service_resource = None
from pyon.core.bootstrap import container_instance
from interface.objects import ServiceStateEnum
#Use container direct RR connection if available, otherwise use messaging to the RR service
if hasattr(container_instance, 'has_capability') and container_instance.has_capability('RESOURCE_REGISTRY'):
service_resource, _ = container_instance.resource_registry.find_resources(restype='Service', name=service_name)
else:
if not local_rr_only:
from interface.services.coi.iresource_registry_service import ResourceRegistryServiceClient
rr_client = ResourceRegistryServiceClient(container_instance.node)
service_resource, _ = rr_client.find_resources(restype='Service', name=service_name)
#The service is available only of there is a single RR object for it and it is in one of these states:
if service_resource and len(service_resource) > 1:
log.warn("Found multiple service instances registered under name %s: %s", service_name, service_resource)
if service_resource and ( service_resource[0].state == ServiceStateEnum.READY or service_resource[0].state == ServiceStateEnum.STEADY ):
return True
elif service_resource:
log.warn("Call to is_service_available() failed although a Service resource exists: %s", service_resource)
return False
except Exception, e:
return False
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.unsc = UserNotificationServiceClient(node=self.container.node)
self.rrc = ResourceRegistryServiceClient(node=self.container.node)
self.imc = IdentityManagementServiceClient(node=self.container.node)
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dmpsclient = DataProductManagementServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.psmsclient = PubsubManagementServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.c = DotDict()
self.c.resource_registry = self.rrclient
self.RR2 = EnhancedResourceRegistryClient(self.rrclient)
self.dsmsclient = DataProcessManagementServiceClient(node=self.container.node)
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]:
self.dsmsclient.deactivate_data_process(proc_id)
self.dsmsclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to bank service
self.resource_registry_service = ResourceRegistryServiceClient()
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Instantiate a process to represent the test
process = TransformWorkerTestProcess()
self.dataset_management_client = DatasetManagementServiceClient(
node=self.container.node)
self.pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceProcessClient(
node=self.container.node, process=process)
self.time_dom, self.spatial_dom = time_series_domain()
self.ph = ParameterHelper(self.dataset_management_client,
self.addCleanup)
self.wait_time = CFG.get_safe('endpoint.receive.timeout', 10)
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.ims_cli = IndexManagementServiceClient()
self.rr_cli = ResourceRegistryServiceClient()
self.index_name = 'test_index'
def test_create_event_process(self):
"""
Test creating an event process
"""
process_definition = ProcessDefinition(name='test')
process_definition.definition = ''
rrc = ResourceRegistryServiceClient(node = self.container.node)
process_definition_id = rrc.create(process_definition)
self.mock_rr_client.find_objects = Mock()
self.mock_rr_client.find_objects.return_value = ['stream_id_1'], 'obj_assoc_1'
# self.mock_pd_client.schedule_process = Mock()
# self.mock_pd_client.schedule_process.return_value = 'process_id'
self.mock_rr_client.create_association = mocksignature(self.mock_rr_client.create_association)
pid = self.event_management.create_event_process(process_definition_id=process_definition_id,
event_types=['type_1', 'type_2'],
sub_types=['subtype_1', 'subtype_2'],
origins=['or_1', 'or_2'],
origin_types=['t1', 't2'],
out_data_products={'conductivity': 'id1'}
)
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dpsc_cli = DataProductManagementServiceClient()
self.rrclient = ResourceRegistryServiceClient()
self.damsclient = DataAcquisitionManagementServiceClient()
self.pubsubcli = PubsubManagementServiceClient()
self.ingestclient = IngestionManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.unsc = UserNotificationServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.identcli = IdentityManagementServiceClient()
#------------------------------------------
# Create the environment
#------------------------------------------
self.stream_def_id = self.pubsubcli.create_stream_definition(
name='SBE37_CDM')
self.process_definitions = {}
ingestion_worker_definition = ProcessDefinition(
name='ingestion worker')
ingestion_worker_definition.executable = {
'module':
'ion.processes.data.ingestion.science_granule_ingestion_worker',
'class': 'ScienceGranuleIngestionWorker'
}
process_definition_id = self.process_dispatcher.create_process_definition(
process_definition=ingestion_worker_definition)
self.process_definitions['ingestion_worker'] = process_definition_id
self.pids = []
self.exchange_points = []
self.exchange_names = []
#------------------------------------------------------------------------------------------------
# First launch the ingestors
#------------------------------------------------------------------------------------------------
self.exchange_space = 'science_granule_ingestion'
self.exchange_point = 'science_data'
config = DotDict()
config.process.datastore_name = 'datasets'
config.process.queue_name = self.exchange_space
self.exchange_names.append(self.exchange_space)
self.exchange_points.append(self.exchange_point)
pid = self.process_dispatcher.schedule_process(
self.process_definitions['ingestion_worker'], configuration=config)
log.debug("the ingestion worker process id: %s", pid)
self.pids.append(pid)
self.addCleanup(self.cleaning_up)
class TestContainerExchangeToEms(IonIntegrationTestCase):
# these tests should auto contact the EMS to do the work
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2coi.yml')
self.ems = ExchangeManagementServiceClient()
self.rr = ResourceRegistryServiceClient()
# we want the ex manager to do its thing, but without actual calls to broker
# just mock out the transport
self.container.ex_manager._priviledged_transport = Mock(BaseTransport)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Test reaches into container, doesn\'t work with CEI')
def test_create_xs_talks_to_ems(self):
self.patch_cfg('pyon.ion.exchange.CFG',
container=DotDict(CFG.container,
exchange=DotDict(auto_register=True)))
xs = self.container.ex_manager.create_xs('house')
self.addCleanup(xs.delete)
# should have called EMS and set RR items
res, _ = self.rr.find_resources(RT.ExchangeSpace, name='house')
self.assertEquals(res[0].name, 'house')
# should have tried to call broker as well
self.assertEquals(
self.container.ex_manager._priviledged_transport.
declare_exchange_impl.call_count, 1)
self.assertIn(
'house', self.container.ex_manager._priviledged_transport.
declare_exchange_impl.call_args[0][0])
@patch.dict('pyon.ion.exchange.CFG',
container=DotDict(CFG.container,
exchange=DotDict(auto_register=False)))
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Test reaches into container, doesn\'t work with CEI')
def test_create_xs_with_no_flag_only_uses_ex_manager(self):
self.patch_cfg('pyon.ion.exchange.CFG',
container=DotDict(
CFG.container,
exchange=DotDict(auto_register=False)))
xs = self.container.ex_manager.create_xs('house')
self.addCleanup(xs.delete)
e1, e2 = self.rr.find_resources(RT.ExchangeSpace, name='house')
self.assertEquals(e1, [])
self.assertEquals(e2, [])
self.assertEquals(
self.container.ex_manager._priviledged_transport.
declare_exchange_impl.call_count, 1)
self.assertIn(
'house', self.container.ex_manager._priviledged_transport.
declare_exchange_impl.call_args[0][0])
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2coi.yml')
self.resource_registry = ResourceRegistryServiceClient(node=self.container.node)
self.org_management_service = OrgManagementServiceClient(node=self.container.node)
def setUp(self):
super(CatalogManagementIntTest,self).setUp()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.cms_cli = CatalogManagementServiceClient()
self.rr_cli = ResourceRegistryServiceClient()
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataAcquisitionManagementService
self.client = DataAcquisitionManagementServiceClient(node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dams_client = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsub_client = PubsubManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.data_retriever = DataRetrieverServiceClient(node=self.container.node)
self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
# Data async and subscription TODO: Replace with new subscriber
self._finished_count = None
#TODO: Switch to gevent.queue.Queue
self._async_finished_result = AsyncResult()
self._finished_events_received = []
self._finished_event_subscriber = None
self._start_finished_event_subscriber()
self.addCleanup(self._stop_finished_event_subscriber)
self.DVR_CONFIG = {}
self.DVR_CONFIG = {
'dvr_mod' : 'ion.agents.data.handlers.slocum_data_handler',
'dvr_cls' : 'SlocumDataHandler',
}
self._setup_resources()
self.agent_config = {
'driver_config' : self.DVR_CONFIG,
'stream_config' : {},
'agent' : {'resource_id': self.EDA_RESOURCE_ID},
'test_mode' : True
}
datasetagent_instance_obj = IonObject(RT.ExternalDatasetAgentInstance, name='ExternalDatasetAgentInstance1', description='external data agent instance',
handler_module=self.EDA_MOD, handler_class=self.EDA_CLS,
dataset_driver_config=self.DVR_CONFIG, dataset_agent_config=self.agent_config )
self.dataset_agent_instance_id = self.dams_client.create_external_dataset_agent_instance(external_dataset_agent_instance=datasetagent_instance_obj,
external_dataset_agent_id=self.datasetagent_id, external_dataset_id=self.EDA_RESOURCE_ID)
#TG: Setup/configure the granule logger to log granules as they're published
pid = self.dams_client.start_external_dataset_agent_instance(self.dataset_agent_instance_id)
dataset_agent_instance_obj= self.dams_client.read_external_dataset_agent_instance(self.dataset_agent_instance_id)
print 'TestBulkIngest: Dataset agent instance obj: = ', dataset_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient('datasetagentclient', name=pid, process=FakeProcess())
log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client))
def setUp(self):
self._start_container()
self._pp = pprint.PrettyPrinter()
log.debug("oms_uri = %s", OMS_URI)
self.oms = CIOMSClientFactory.create_instance(OMS_URI)
self._get_platform_attributes()
url = OmsTestMixin.start_http_server()
log.info("TestPlatformInstrument:setup http url %s", url)
result = self.oms.event.register_event_listener(url)
log.info(
"TestPlatformInstrument:setup register_event_listener result %s",
result)
# response = self.oms.port.get_platform_ports('LPJBox_CI_Ben_Hall')
# log.info("TestPlatformInstrument:setup get_platform_ports %s", response)
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.RR2 = EnhancedResourceRegistryClient(self.rrclient)
self.org_id = self.RR2.create(any_old(RT.Org))
log.debug("Org created: %s", self.org_id)
# see _set_receive_timeout
self._receive_timeout = 177
self.instrument_device = ''
self.platform_device = ''
self.platform_agent_instance_id = ''
self._pa_client = ''
def done():
CIOMSClientFactory.destroy_instance(self.oms)
event_notifications = OmsTestMixin.stop_http_server()
log.info("event_notifications = %s" % str(event_notifications))
self.addCleanup(done)
def _new_dataset(self, stream_id):
'''
Adds a new dataset to the internal cache of the ingestion worker
'''
rr_client = ResourceRegistryServiceClient()
datasets, _ = rr_client.find_subjects(subject_type=RT.Dataset,predicate=PRED.hasStream,object=stream_id,id_only=True)
if datasets:
return datasets[0]
return None
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.generic_int_helper_rr = ResourceRegistryServiceClient(
node=self.container.node)
self.generic_int_helper_rr2 = EnhancedResourceRegistryClient(
self.generic_int_helper_rr)
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.ingestion_management = IngestionManagementServiceClient()
self.resource_registry = ResourceRegistryServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.ingest_name = 'basic'
self.exchange = 'testdata'
class TestIntDataProcessManagementService(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
#print 'started container'
# Establish endpoint with container
container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
#print 'got CC client'
container_client.start_rel_from_url('res/deploy/r2sa.yml')
print 'started services'
# Now create client to DataProcessManagementService
self.DPMSclient = DataProcessManagementServiceClient(node=self.container.node)
self.RRclient = ResourceRegistryServiceClient(node=self.container.node)
def test_createDataProcess(self):
# test creating a new data process
print 'Creating new data process definition'
dpd_obj = IonObject(RT.DataProcessDefinition,
name='DPD1',
description='some new dpd',
process_source='some_source_reference')
try:
dprocd_id = self.DPMSclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new data process definition: %s" %ex)
print 'new dpd_id = ', dprocd_id
print 'Creating new subscription'
s_obj = IonObject(RT.Subscription,
name='S1',
description='some new subscription')
try:
s_id, version = self.RRclient.create(s_obj)
except BadRequest as ex:
self.fail("failed to create new subscription: %s" %ex)
print 'new s_id = ', s_id
print 'Creating new data product'
dprod_obj = IonObject(RT.Subscription,
name='DProd1',
description='some new data product')
try:
dprod_id, version = self.RRclient.create(dprod_obj)
except BadRequest as ex:
self.fail("failed to create new data product: %s" %ex)
print 'new dprod_id = ', dprod_id
print 'Creating new data process'
try:
dproc_id = self.DPMSclient.create_data_process(dprocd_id, s_id, dprod_id)
except (BadRequest, NotFound, Conflict) as ex:
self.fail("failed to create new data process definition: %s" %ex)
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.pubsub_management = PubsubManagementServiceClient()
self.resource_registry = ResourceRegistryServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.pdicts = {}
self.queue_cleanup = list()
self.exchange_cleanup = list()
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
def setUp(self):
self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254"
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.resource_registry = ResourceRegistryServiceClient()
self.identity_management_service = IdentityManagementServiceClient()
self.org_client = OrgManagementServiceClient()
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataAcquisitionManagementService
self.client = DataAcquisitionManagementServiceClient(node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
def __init__(self):
self.ingestion_management = IngestionManagementServiceClient()
self.resource_registry = ResourceRegistryServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self._paused_streams = []
self._w_covs = {}
self._ro_covs = {}
self._context_managed = False
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2coi.yml')
self.ems = ExchangeManagementServiceClient()
self.rr = ResourceRegistryServiceClient()
# we want the ex manager to do its thing, but without actual calls to broker
# just mock out the transport
self.container.ex_manager._priviledged_transport = Mock(BaseTransport)
def _find_new_data_check_attachment(self, res_id):
rr_cli = ResourceRegistryServiceClient()
try:
attachment_objs = rr_cli.find_attachments(resource_id=res_id, include_content=False, id_only=False)
for attachment_obj in attachment_objs:
kwds = set(attachment_obj.keywords)
if 'NewDataCheck' in kwds:
log.debug('Found NewDataCheck attachment: {0}'.format(attachment_obj._id))
return attachment_obj.content
else:
log.debug('Found attachment: {0}'.format(attachment_obj))
except NotFound:
raise InstrumentException('ExternalDatasetResource \'{0}\' not found'.format(res_id))
def _start_raw_ingestion(self):
dpsc_cli = DataProductManagementServiceClient()
rrclient = ResourceRegistryServiceClient()
RR2 = EnhancedResourceRegistryClient(rrclient)
# Generic time-series data domain creation
tdom, sdom = time_series_domain()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom.dump(),
spatial_domain = sdom.dump())
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0
dp_obj.ooi_product_name = "PRODNAME"
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
log.info("Create data product... raw stream id: %s", self._raw_stream_id)
dp_id = dpsc_cli.create_data_product_(data_product= dp_obj)
dataset_id = self.create_dataset(self._raw_stream_pdict_id)
RR2.assign_stream_definition_to_data_product_with_has_stream_definition(self._raw_stream_def_id, dp_id)
RR2.assign_stream_to_data_product_with_has_stream(self._raw_stream_id, dp_id)
RR2.assign_dataset_to_data_product_with_has_dataset(dataset_id, dp_id)
self._raw_dataset_id = dataset_id
log.info("Create data product...Complete")
# Assert that the data product has an associated stream at this stage
stream_ids, _ = rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True)
self.assertNotEquals(len(stream_ids), 0)
# Assert that the data product has an associated stream def at this stage
stream_ids, _ = rrclient.find_objects(dp_id, PRED.hasStreamDefinition, RT.StreamDefinition, True)
self.assertNotEquals(len(stream_ids), 0)
log.info("Activate data product persistence")
dpsc_cli.activate_data_product_persistence(dp_id)
log.info("Read data product")
dp_obj = dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Created data product %s', dp_obj)
def clean_subscriptions():
ingestion_management = IngestionManagementServiceClient()
pubsub = PubsubManagementServiceClient()
rr = ResourceRegistryServiceClient()
ingestion_config_ids = ingestion_management.list_ingestion_configurations(id_only=True)
for ic in ingestion_config_ids:
subscription_ids, assocs = rr.find_objects(subject=ic, predicate=PRED.hasSubscription, id_only=True)
for subscription_id, assoc in zip(subscription_ids, assocs):
rr.delete_association(assoc)
try:
pubsub.deactivate_subscription(subscription_id)
except:
log.exception("Unable to decativate subscription: %s", subscription_id)
pubsub.delete_subscription(subscription_id)
def clean_subscriptions():
ingestion_management = IngestionManagementServiceClient()
pubsub = PubsubManagementServiceClient()
rr = ResourceRegistryServiceClient()
ingestion_config_ids = ingestion_management.list_ingestion_configurations(id_only=True)
for ic in ingestion_config_ids:
assocs = rr.find_associations(subject=ic, predicate=PRED.hasSubscription, id_only=False)
for assoc in assocs:
rr.delete_association(assoc)
try:
pubsub.deactivate_subscription(assoc.o)
except:
pass
pubsub.delete_subscription(assoc.o)
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.unsc = UserNotificationServiceClient(node=self.container.node)
self.rrc = ResourceRegistryServiceClient(node=self.container.node)
self.imc = IdentityManagementServiceClient(node=self.container.node)
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
self.event_publisher = EventPublisher()
self.egg_url_good = "http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1a-py2.7.egg"
self.egg_url_bad = "http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.1a-py2.7.egg"
self.egg_url_404 = "http://sddevrepo.oceanobservatories.org/releases/completely_made_up_404.egg"
def setUp(self):
super(DataRetrieverIntTestAlpha,self).setUp()
self._start_container()
config = DotDict()
config.bootstrap.processes.ingestion.module = 'ion.processes.data.ingestion.ingestion_worker_a'
config.bootstrap.processes.replay.module = 'ion.processes.data.replay.replay_process_a'
self.container.start_rel_from_url('res/deploy/r2dm.yml', config)
self.datastore_name = 'test_datasets'
self.datastore = self.container.datastore_manager.get_datastore(self.datastore_name, profile=DataStore.DS_PROFILE.SCIDATA)
self.data_retriever = DataRetrieverServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.resource_registry = ResourceRegistryServiceClient()
xs_dot_xp = CFG.core_xps.science_data
try:
self.XS, xp_base = xs_dot_xp.split('.')
self.XP = '.'.join([get_sys_name(), xp_base])
except ValueError:
raise StandardError('Invalid CFG for core_xps.science_data: "%s"; must have "xs.xp" structure' % xs_dot_xp)
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.generic_int_helper_rr = ResourceRegistryServiceClient(node=self.container.node)
self.generic_int_helper_rr2 = EnhancedResourceRegistryClient(self.generic_int_helper_rr)
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
print 'started services'
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient()
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
# Use the network definition provided by RSN OMS directly.
rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri'])
self._network_definition = RsnOmsUtil.build_network_definition(rsn_oms)
# get serialized version for the configuration:
self._network_definition_ser = NetworkUtil.serialize_network_definition(self._network_definition)
if log.isEnabledFor(logging.TRACE):
log.trace("NetworkDefinition serialization:\n%s", self._network_definition_ser)
self._async_data_result = AsyncResult()
self._data_subscribers = []
self._samples_received = []
self.addCleanup(self._stop_data_subscribers)
self._async_event_result = AsyncResult()
self._event_subscribers = []
self._events_received = []
self.addCleanup(self._stop_event_subscribers)
self._start_event_subscriber()
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dmpsclient = DataProductManagementServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.psmsclient = PubsubManagementServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.c = DotDict()
self.c.resource_registry = self.rrclient
self.RR2 = EnhancedResourceRegistryClient(self.rrclient)
self.dsmsclient = DataProcessManagementServiceClient(node=self.container.node)
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]:
self.dsmsclient.deactivate_data_process(proc_id)
self.dsmsclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dmpsclient = DataProductManagementServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.psmsclient = PubsubManagementServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.c = DotDict()
self.c.resource_registry = self.rrclient
self.RR2 = EnhancedResourceRegistryClient(self.rrclient)
# create missing data process definition
self.dsmsclient = DataProcessManagementServiceClient(node=self.container.node)
dpd_obj = IonObject(RT.DataProcessDefinition,
name=LOGICAL_TRANSFORM_DEFINITION_NAME,
description="normally in preload",
module='ion.processes.data.transforms.logical_transform',
class_name='logical_transform')
self.dsmsclient.create_data_process_definition(dpd_obj)
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]:
self.dsmsclient.deactivate_data_process(proc_id)
self.dsmsclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.dpmsclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]:
self.dataprocessclient.deactivate_data_process(proc_id)
self.dataprocessclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
def setUp(self):
super(DiscoveryIntTest, self).setUp()
self._start_container()
self.addCleanup(DiscoveryIntTest.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.discovery = DiscoveryServiceClient()
self.catalog = CatalogManagementServiceClient()
self.ims = IndexManagementServiceClient()
self.rr = ResourceRegistryServiceClient()
if use_es:
self.es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
self.es_port = CFG.get_safe('server.elasticsearch.port', '9200')
CFG.server.elasticsearch.shards = 1
CFG.server.elasticsearch.replicas = 0
CFG.server.elasticsearch.river_shards = 1
CFG.server.elasticsearch.river_replicas = 0
self.es = ep.ElasticSearch(
host=self.es_host,
port=self.es_port,
timeout=10,
verbose=True
)
op = DotDict(CFG)
op.op = 'clean_bootstrap'
self.container.spawn_process('index_bootstrap','ion.processes.bootstrap.index_bootstrap','IndexBootStrap', op)
def setUp(self):
# set up the container
self._start_container()
self.cc = ContainerAgentClient(node=self.container.node,name=self.container.name)
self.cc.start_rel_from_url('res/deploy/r2deploy.yml')
self.pubsub_cli = PubsubManagementServiceClient(node=self.cc.node)
self.tms_cli = TransformManagementServiceClient(node=self.cc.node)
self.rr_cli = ResourceRegistryServiceClient(node=self.cc.node)
self.input_stream = IonObject(RT.Stream,name='ctd1 output', description='output from a ctd')
self.input_stream.original = True
self.input_stream.mimetype = 'hdf'
self.input_stream_id = self.pubsub_cli.create_stream(self.input_stream)
self.input_subscription = IonObject(RT.Subscription,name='ctd1 subscription', description='subscribe to this if you want ctd1 data')
self.input_subscription.query['stream_id'] = self.input_stream_id
self.input_subscription.exchange_name = 'a queue'
self.input_subscription_id = self.pubsub_cli.create_subscription(self.input_subscription)
self.output_stream = IonObject(RT.Stream,name='transform output', description='output from the transform process')
self.output_stream.original = True
self.output_stream.mimetype='raw'
self.output_stream_id = self.pubsub_cli.create_stream(self.output_stream)
self.process_definition = IonObject(RT.ProcessDefinition,name='transform_process')
self.process_definition.executable = {'module': 'ion.services.dm.transformation.example.transform_example',
'class':'TransformExample'}
self.process_definition_id, _= self.rr_cli.create(self.process_definition)
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url("res/deploy/r2coi.yml")
# Now create client to bank service
self.resource_registry_service = ResourceRegistryServiceClient(node=self.container.node)
class TestIntDataProcessManagementServiceMultiOut(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient()
self.damsclient = DataAcquisitionManagementServiceClient()
self.pubsubclient = PubsubManagementServiceClient()
self.ingestclient = IngestionManagementServiceClient()
self.imsclient = InstrumentManagementServiceClient()
self.dataproductclient = DataProductManagementServiceClient()
self.dataprocessclient = DataProcessManagementServiceClient()
self.datasetclient = DatasetManagementServiceClient()
self.dataset_management = self.datasetclient
self.process_dispatcher = ProcessDispatcherServiceClient()
def test_create_data_process_definition(self):
pfunc = ParameterFunction(name='test_func')
func_id = self.dataset_management.create_parameter_function(pfunc)
data_process_definition = DataProcessDefinition()
data_process_definition.name = 'Simple'
dpd_id = self.dataprocessclient.create_data_process_definition(data_process_definition, func_id)
self.addCleanup(self.dataprocessclient.delete_data_process_definition, dpd_id)
self.addCleanup(self.dataset_management.delete_parameter_function, func_id)
objs, _ = self.rrclient.find_objects(dpd_id, PRED.hasParameterFunction, id_only=False)
self.assertEquals(len(objs), 1)
self.assertIsInstance(objs[0], ParameterFunction)
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
self.event_publisher = EventPublisher()
def setUp(self):
self._start_container()
self.container.start_rel_from_url("res/deploy/r2dm.yml")
self.ims_cli = IndexManagementServiceClient()
self.rr_cli = ResourceRegistryServiceClient()
self.index_name = "test_index"
def setUp(self):
# set up the container
self._start_container()
self.container.start_rel_from_url("res/deploy/r2dm.yml")
self.pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
self.tms_cli = TransformManagementServiceClient(node=self.container.node)
self.rr_cli = ResourceRegistryServiceClient(node=self.container.node)
self.procd_cli = ProcessDispatcherServiceClient(node=self.container.node)
self.input_stream_id = self.pubsub_cli.create_stream(name="input_stream", original=True)
self.input_subscription_id = self.pubsub_cli.create_subscription(
query=StreamQuery(stream_ids=[self.input_stream_id]),
exchange_name="transform_input",
name="input_subscription",
)
self.output_stream_id = self.pubsub_cli.create_stream(name="output_stream", original=True)
self.process_definition = ProcessDefinition(name="basic_transform_definition")
self.process_definition.executable = {
"module": "ion.processes.data.transforms.transform_example",
"class": "TransformExample",
}
self.process_definition_id = self.procd_cli.create_process_definition(
process_definition=self.process_definition
)
def setUp(self):
# Start container
super(TestActivateInstrumentIntegration, self).setUp()
config = DotDict()
config.bootstrap.use_es = True
self._start_container()
self.addCleanup(TestActivateInstrumentIntegration.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2deploy.yml', config)
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.usernotificationclient = UserNotificationServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
self.event_publisher = EventPublisher()
def _on_init(self):
ResourceAgent._on_init(self)
# log.debug(">>funcs:\n%s" % dir(self))
# log.debug(">>type:\n%s" % type(self))
# log.debug(">>CFG:\n%s" % self.CFG)
# log.debug(">>_proc_type:\n%s" % self._proc_type)
ext_dataset_id = self.CFG.get('process',{}).get('eoa',{}).get('dataset_id','')
log.debug(">>ds_id:\n\t%s" % ext_dataset_id)
self.resreg_cli = ResourceRegistryServiceClient()
self.pubsub_cli = PubsubManagementServiceClient()
# retrieve all the ExternalDataset resource associated resources
ext_ds_res = self.resreg_cli.read(object_id=ext_dataset_id)
log.debug("Retrieved ExternalDataset: %s" % ext_ds_res)
dsrc_res, dsrc_acc = self.resreg_cli.find_objects(subject=ext_dataset_id, predicate=PRED.hasSource, object_type=RT.DataSource)
dsrc_res = dsrc_res[0]
dsrc_id = dsrc_acc[0].o
log.debug("Found associated DataSource: %s" % dsrc_id)
edp_res, edp_acc = self.resreg_cli.find_objects(subject=dsrc_id, predicate=PRED.hasProvider, object_type=RT.ExternalDataProvider)
edp_res = edp_res[0]
edp_id = edp_acc[0].o
log.debug("Found associated ExternalDataProvider: %s" % edp_id)
mdl_res, mdl_acc = self.resreg_cli.find_objects(subject=dsrc_id, predicate=PRED.hasModel, object_type=RT.ExternalDataSourceModel)
mdl_res = mdl_res[0]
mdl_id = mdl_acc[0].o
log.debug("Found associated ExternalDataSourceModel: %s" % mdl_id)
dprod_id, _ = self.resreg_cli.find_objects(subject=ext_dataset_id, predicate=PRED.hasDataProducer, object_type=RT.DataProducer, id_only=True)
dprod_id = dprod_id[0]
log.debug("Found associated DataProducer: %s" % dprod_id)
stream_id, _ = self.resreg_cli.find_objects(subject=dprod_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True)
stream_id = stream_id[0]
log.debug("Found associated Stream: %s" % stream_id)
# configure the stream publisher
log.debug("Configure StreamPublisher")
stream_route = self.pubsub_cli.read_stream(stream_id=stream_id)
log.debug("StreamRoute: %s" % stream_route)
self._stream_publisher = StreamPublisher(stream_route)
# instantiate the data_handler instance
modpath = mdl_res.data_handler_module
log.debug("ExternalDataHandler module: %s" % modpath)
classname = mdl_res.data_handler_class
log.debug("ExternalDataHandler class: %s" % classname)
module = __import__(modpath, fromlist=[classname])
classobj = getattr(module, classname)
self._data_handler = classobj(data_provider=edp_res, data_source=dsrc_res, ext_dataset=ext_ds_res)
assert isinstance(self._data_handler, BaseExternalDataHandler), "Instantiated service not a BaseExternalDataHandler %r" % self._data_handler
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url("res/deploy/r2coi.yml")
self.resource_registry = ResourceRegistryServiceClient(node=self.container.node)
self.org_management_service = OrgManagementServiceClient(node=self.container.node)
