def test_create_deployment(self): #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) start = str(int(time.mktime(datetime.datetime(2013, 1, 1).timetuple()))) end = str(int(time.mktime(datetime.datetime(2014, 1, 1).timetuple()))) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start, end_datetime=end) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.assign_site_to_deployment(site_id, deployment_id) self.omsclient.assign_device_to_deployment(device_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id)) #retrieve the deployment objects and check that the assoc site and device are attached read_deployment_obj = self.omsclient.read_deployment(deployment_id) log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj)) site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(site_ids), 1) device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(device_ids), 1) #delete the deployment self.omsclient.force_delete_deployment(deployment_id) # now try to get the deleted dp object try: self.omsclient.read_deployment(deployment_id) except NotFound: pass else: self.fail("deleted deployment was found during read")
def test_get_resource_extension(self): #Testing multiple instrument owners subject1 = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254" actor_identity_obj1 = IonObject(RT.ActorIdentity, {"name": subject1}) actor_id1, _ = self.resource_registry_service.create( actor_identity_obj1) user_info_obj1 = IonObject(RT.UserInfo, {"name": "Foo"}) user_info_id1, _ = self.resource_registry_service.create( user_info_obj1) self.resource_registry_service.create_association( actor_id1, PRED.hasInfo, user_info_id1) subject2 = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256" actor_identity_obj2 = IonObject(RT.ActorIdentity, {"name": subject2}) actor_id2, _ = self.resource_registry_service.create( actor_identity_obj2) user_info_obj2 = IonObject(RT.UserInfo, {"name": "Foo2"}) user_info_id2, _ = self.resource_registry_service.create( user_info_obj2) self.resource_registry_service.create_association( actor_id2, PRED.hasInfo, user_info_id2) test_obj = IonObject(RT.InformationResource, {"name": "TestResource"}) test_obj_id, _ = self.resource_registry_service.create(test_obj) self.resource_registry_service.create_association( test_obj_id, PRED.hasOwner, actor_id1) self.resource_registry_service.create_association( test_obj_id, PRED.hasOwner, actor_id2) extended_resource = self.resource_registry_service.get_resource_extension( test_obj_id, OT.ExtendedInformationResource) self.assertEqual(test_obj_id, extended_resource._id) self.assertEqual(len(extended_resource.owners), 2) extended_resource_list = self.resource_registry_service.get_resource_extension( str([user_info_id1, user_info_id2]), OT.ExtendedInformationResource) self.assertEqual(len(extended_resource_list), 2) optional_args = {'user_id': user_info_id1} extended_resource = self.resource_registry_service.get_resource_extension( test_obj_id, OT.TestExtendedInformationResource, optional_args=optional_args) self.assertEqual(test_obj_id, extended_resource._id) self.assertEqual(len(extended_resource.owners), 2) self.assertEqual(extended_resource.user_id, user_info_id1)
def test_org_crud(self): with self.assertRaises(BadRequest) as br: self.org_management_service.create_org(IonObject("Org", {"name": "Test Facility", "org_governance_name": "Test Facility" })) self.assertTrue("can only contain alphanumeric and underscore characters" in br.exception.message) with self.assertRaises(BadRequest): self.org_management_service.create_org() org_obj = IonObject("Org", {"name": "Test Facility"}) org_id = self.org_management_service.create_org(org_obj) self.assertNotEqual(org_id, None) org = None org = self.org_management_service.read_org(org_id) self.assertNotEqual(org, None) self.assertEqual(org.org_governance_name, 'Test_Facility') #Check that the roles got associated to them role_list = self.org_management_service.find_org_roles(org_id) self.assertEqual(len(role_list),2 ) with self.assertRaises(BadRequest): self.org_management_service.update_org() org.name = 'Updated Test Facility' self.org_management_service.update_org(org) org = None org = self.org_management_service.read_org(org_id) self.assertNotEqual(org, None) self.assertEqual(org.name, 'Updated Test Facility') self.assertEqual(org.org_governance_name, 'Test_Facility') user_role = self.org_management_service.find_org_role_by_name(org_id, ORG_MANAGER_ROLE) self.assertNotEqual(user_role, None) self.org_management_service.remove_user_role(org_id, ORG_MANAGER_ROLE) with self.assertRaises(BadRequest) as cm: user_role = self.org_management_service.find_org_role_by_name(org_id, ORG_MANAGER_ROLE) self.assertIn("The User Role 'ORG_MANAGER' does not exist for this Org", cm.exception.message) with self.assertRaises(BadRequest): self.org_management_service.delete_org() self.org_management_service.delete_org(org_id) with self.assertRaises(NotFound) as cm: self.org_management_service.read_org(org_id) self.assertIn("does not exist", cm.exception.message) with self.assertRaises(NotFound) as cm: self.org_management_service.delete_org(org_id) self.assertIn("does not exist", cm.exception.message)
def _create_l1_out_data_products(self): ctd_l1_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L1_Conductivity', description='transform output L1 conductivity', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product( ctd_l1_conductivity_output_dp_obj, self.outgoing_stream_l1_conductivity_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l1_conductivity_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects( self.ctd_l1_conductivity_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_conductivity', stream_ids[0]) self.loggerpids.append(pid) ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L1_Pressure', description='transform output L1 pressure', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product( ctd_l1_pressure_output_dp_obj, self.outgoing_stream_l1_pressure_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l1_pressure_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects( self.ctd_l1_pressure_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_pressure', stream_ids[0]) self.loggerpids.append(pid) ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L1_Temperature', description='transform output L1 temperature', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product( ctd_l1_temperature_output_dp_obj, self.outgoing_stream_l1_temperature_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l1_temperature_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects( self.ctd_l1_temperature_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_temperature', stream_ids[0]) self.loggerpids.append(pid)
def parse_phones(text): if ':' in text: out = [] for type, number in parse_dict(text).iteritems(): out.append(IonObject("Phone", phone_number=number, phone_type=type)) return out elif text: return [ IonObject("Phone", phone_number=text.strip(), phone_type='office') ] else: return []
def test_register_instrument(self): # Register an instrument as a data producer in coordination with DM PubSub: create stream, register and create producer object # set up initial instrument to register instrument_obj = IonObject( RT.InstrumentDevice, name='Inst1', description='an instrument that is creating the data product') instrument_id, rev = self.rrclient.create(instrument_obj) dataproduct_obj = IonObject(RT.DataProduct, name='DataProduct1', description='sample data product') dataproduct_id, rev = self.rrclient.create(dataproduct_obj) # test registering a new data producer try: ds_id = self.client.register_instrument(instrument_id) except BadRequest as ex: self.fail("failed to create new data producer: %s" % ex) print 'new data producer id = ', ds_id # test assigning a data product to an instrument, creating the stream for the product try: self.client.assign_data_product(instrument_id, dataproduct_id, True) except BadRequest as ex: self.fail("failed to assign data product to data producer: %s" % ex) except NotFound as ex: self.fail("failed to assign data product to data producer: %s" % ex) # test UNassigning a data product from instrument, deleting the stream for the product try: self.client.unassign_data_product(instrument_id, dataproduct_id, True) except BadRequest as ex: self.fail( "failed to failed to UNassign data product to data producer data producer: %s" % ex) except NotFound as ex: self.fail( "failed to failed to UNassign data product to data producer data producer: %s" % ex) # test UNregistering a new data producer try: ds_id = self.client.unregister_instrument(instrument_id) except NotFound as ex: self.fail("failed to unregister instrument producer: %s" % ex)
def create_basic_deployment(self, name='', description=''): start = IonTime(datetime.datetime(2013, 1, 1)) end = IonTime(datetime.datetime(2014, 1, 1)) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string()) deployment_obj = IonObject(RT.Deployment, name=name, description=description, context=IonObject( OT.CabledNodeDeploymentContext), constraint_list=[temporal_bounds]) return self.OMS.create_deployment(deployment_obj)
def _create_l0_output_data_products(self, outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id): out_data_prods = [] ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) out_data_prods.append(self.ctd_l0_conductivity_output_dp_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l0_conductivity_output_dp_id) ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) out_data_prods.append(self.ctd_l0_pressure_output_dp_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l0_pressure_output_dp_id) ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) out_data_prods.append(self.ctd_l0_temperature_output_dp_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l0_temperature_output_dp_id) return out_data_prods
def setUp(self): mock_clients = self._create_service_mock('workflow_management') self.workflow_management_service = WorkflowManagementService() self.workflow_management_service.clients = mock_clients # Rename to save some typing self.mock_create = mock_clients.resource_registry.create self.mock_read = mock_clients.resource_registry.read self.mock_update = mock_clients.resource_registry.update self.mock_delete = mock_clients.resource_registry.delete self.mock_create_association = mock_clients.resource_registry.create_association self.mock_delete_association = mock_clients.resource_registry.delete_association self.mock_find_objects = mock_clients.resource_registry.find_objects self.mock_find_resources = mock_clients.resource_registry.find_resources self.mock_find_subjects = mock_clients.resource_registry.find_subjects self.mock_find_associations = mock_clients.resource_registry.find_associations # workflow definition self.workflow_definition = Mock() self.workflow_definition.name = "Foo" self.workflow_definition.description = "This is a test workflow definition" self.workflow_definition.workflow_steps = [] workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id='123') self.workflow_definition.workflow_steps.append(workflow_step_obj) workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id='456') self.workflow_definition.workflow_steps.append(workflow_step_obj) # WorkflowDefinition to DataProcessDefinition associations self.workflow_definition_to_dataprocess_definition_association = Mock() self.workflow_definition_to_dataprocess_definition_association._id = 'abc' self.workflow_definition_to_dataprocess_definition_association.s = "111" self.workflow_definition_to_dataprocess_definition_association.st = RT.WorkflowDefinition self.workflow_definition_to_dataprocess_definition_association.p = PRED.hasDataProcessDefinition self.workflow_definition_to_dataprocess_definition_association.o = "123" self.workflow_definition_to_dataprocess_definition_association.ot = RT.DataProcessDefinition self.workflow_definition_to_dataprocess_definition_association2 = Mock( ) self.workflow_definition_to_dataprocess_definition_association2._id = 'def' self.workflow_definition_to_dataprocess_definition_association2.s = "111" self.workflow_definition_to_dataprocess_definition_association2.st = RT.WorkflowDefinition self.workflow_definition_to_dataprocess_definition_association2.p = PRED.hasDataProcessDefinition self.workflow_definition_to_dataprocess_definition_association2.o = "456" self.workflow_definition_to_dataprocess_definition_association2.ot = RT.DataProcessDefinition
def _create_subsequent_deployment(self, prior_dep_info): platform_device_obj = IonObject(RT.PlatformDevice, name='PlatformDevice2', description='test platform device') platform_device_id = self.imsclient.create_platform_device( platform_device_obj) instrument_device_obj = IonObject(RT.InstrumentDevice, name='InstrumentDevice2', description='test instrument device') instrument_device_id = self.imsclient.create_instrument_device( instrument_device_obj) self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) self.imsclient.assign_platform_model_to_platform_device( prior_dep_info.platform_model_id, platform_device_id) self.imsclient.assign_instrument_model_to_instrument_device( prior_dep_info.instrument_model_id, instrument_device_id) start = str(int(time.mktime(datetime.datetime(2013, 6, 1).timetuple()))) end = str(int(time.mktime(datetime.datetime(2020, 6, 1).timetuple()))) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start, end_datetime=end) deployment_obj = IonObject(RT.Deployment, name='TestDeployment2', description='some new deployment', context=IonObject( OT.CabledNodeDeploymentContext), constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.assign_site_to_deployment( prior_dep_info.platform_site_id, deployment_id) self.omsclient.assign_device_to_deployment( prior_dep_info.platform_device_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id)) ret = DotDict(instrument_device_id=instrument_device_id, platform_device_id=platform_device_id, deployment_id=deployment_id) return ret
def test_register_external_data_set(self): # Register an external data set as a data producer in coordination with DM PubSub: create stream, register and create producer object # set up initial instrument to register ext_dataset_obj = IonObject(RT.ExternalDataset, name='DataSet1', description='an external data feed') ext_dataset_id, rev = self.rrclient.create(ext_dataset_obj) dataproduct_obj = IonObject(RT.DataProduct, name='DataProduct1', description='sample data product') dataproduct_id, rev = self.rrclient.create(dataproduct_obj) # test registering a new external data set try: ds_id = self.client.register_external_data_set(ext_dataset_id) except BadRequest as ex: self.fail("failed to create new data producer: %s" % ex) print 'new data producer id = ', ds_id # test assigning a data product to an ext_dataset_id, creating the stream for the product try: self.client.assign_data_product(ext_dataset_id, dataproduct_id) except BadRequest as ex: self.fail("failed to assign data product to data producer: %s" % ex) except NotFound as ex: self.fail("failed to assign data product to data producer: %s" % ex) # test UNassigning a data product from ext_dataset_id, deleting the stream for the product try: self.client.unassign_data_product(ext_dataset_id, dataproduct_id) except BadRequest as ex: self.fail( "failed to failed to UNassign data product to data producer data producer: %s" % ex) except NotFound as ex: self.fail( "failed to failed to UNassign data product to data producer data producer: %s" % ex) # test UNregistering a external data set try: ds_id = self.client.unregister_external_data_set(ext_dataset_id) except NotFound as ex: self.fail("failed to unregister instrument producer: %s" % ex)
def trigger_container_snapshot(self, snapshot_id='', include_snapshots=None, exclude_snapshots=None, take_at_time='', clear_all=False, persist_snapshot=True, snapshot_kwargs=None): if not snapshot_id: snapshot_id = get_ion_ts() if not snapshot_kwargs: snapshot_kwargs = {} self.perform_action( ALL_CONTAINERS_INSTANCE, IonObject(OT.TriggerContainerSnapshot, snapshot_id=snapshot_id, include_snapshots=include_snapshots, exclude_snapshots=exclude_snapshots, take_at_time=take_at_time, clear_all=clear_all, persist_snapshot=persist_snapshot, snapshot_kwargs=snapshot_kwargs)) log.info("Event to trigger container snapshots sent. snapshot_id=%s" % snapshot_id)
def trigger_garbage_collection(self): """Triggers a garbage collection in all containers @throws BadRequest None """ self.perform_action(ALL_CONTAINERS_INSTANCE, IonObject(OT.TriggerGarbageCollection))
def reset_policy_cache(self, headers=None, timeout=None): """Clears and reloads the policy caches in all of the containers. @throws BadRequest None """ self.perform_action(ALL_CONTAINERS_INSTANCE, IonObject(OT.ResetPolicyCache))
def set_log_level(self, logger='', level='', recursive=False): self.perform_action( ALL_CONTAINERS_INSTANCE, IonObject(OT.ChangeLogLevel, logger=logger, level=level, recursive=recursive))
def _do_test_lifecycle(self): # Lifecycle tests att = IonObject("InstrumentDevice", name='mine', description='desc') rid,rev = self.resource_registry_service.create(att) att1 = self.resource_registry_service.read(rid) self.assertEquals(att1.name, att.name) self.assertEquals(att1.lcstate, LCS.DRAFT) self.assertEquals(att1.availability, AS.PRIVATE) new_state = self.resource_registry_service.execute_lifecycle_transition(rid, LCE.PLAN) self.assertEquals(new_state, lcstate(LCS.PLANNED, AS.PRIVATE)) att2 = self.resource_registry_service.read(rid) self.assertEquals(att2.lcstate, LCS.PLANNED) self.assertEquals(att2.availability, AS.PRIVATE) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.execute_lifecycle_transition(rid, LCE.UNANNOUNCE) self.assertTrue("type=InstrumentDevice, lcstate=PLANNED_PRIVATE has no transition for event unannounce" in cm.exception.message) new_state = self.resource_registry_service.execute_lifecycle_transition(rid, LCE.DEVELOP) self.assertEquals(new_state, lcstate(LCS.DEVELOPED, AS.PRIVATE)) with self.assertRaises(BadRequest): self.resource_registry_service.execute_lifecycle_transition( resource_id=rid, transition_event='NONE##') self.resource_registry_service.set_lifecycle_state(rid, lcstate(LCS.INTEGRATED, AS.PRIVATE)) att1 = self.resource_registry_service.read(rid) self.assertEquals(att1.lcstate, LCS.INTEGRATED) self.assertEquals(att1.availability, AS.PRIVATE)
def set_object_field(self, obj, field, field_val): """Recursively set sub object field values. TODO: This may be an expensive operation. May also be redundant with object code """ if isinstance(field_val, dict) and field != "kwargs": sub_obj = getattr(obj, field) if isinstance(sub_obj, IonObjectBase): if "type_" in field_val and field_val["type_"] != sub_obj.type_: if issubtype(field_val["type_"], sub_obj.type_): sub_obj = IonObject(field_val["type_"]) setattr(obj, field, sub_obj) else: raise Inconsistent( "Unable to walk the field %s - types don't match: %s %s" % (field, sub_obj.type_, field_val["type_"])) for sub_field in field_val: self.set_object_field(sub_obj, sub_field, field_val.get(sub_field)) elif isinstance(sub_obj, dict): setattr(obj, field, field_val) else: for sub_field in field_val: self.set_object_field(sub_obj, sub_field, field_val.get(sub_field)) else: # type_ already exists in the class. if field != "type_": setattr(obj, field, field_val)
def run_reverse_transform(self): ''' Runs a reverse transform example and displays the results of performing the transform ''' tms_cli = TransformManagementServiceClient(node=self.container.node) procd_cli = ProcessDispatcherServiceClient(node=self.container.node) #------------------------------- # Process Definition #------------------------------- process_definition = IonObject(RT.ProcessDefinition, name='transform_process_definition') process_definition.executable = { 'module': 'ion.processes.data.transforms.transform_example', 'class': 'ReverseTransform' } process_definition_id = procd_cli.create_process_definition( process_definition) #------------------------------- # Execute Transform #------------------------------- input = [1, 2, 3, 4] retval = tms_cli.execute_transform( process_definition_id=process_definition_id, data=[1, 2, 3, 4], configuration={}) log.debug('Transform Input: %s', input) log.debug('Transform Output: %s', retval)
def get_recent_events(self, resource_id='', limit=100): """ Get recent events for use in extended resource computed attribute @param resource_id str @param limit int @retval ComputedListValue with value list of 4-tuple with Event objects """ now = get_ion_ts() events = self.find_events(origin=resource_id, limit=limit, max_datetime=now, descending=True) ret = IonObject(OT.ComputedEventListValue) if events: ret.value = events ret.computed_list = [ get_event_computed_attributes(event) for event in events ] ret.status = ComputedValueAvailability.PROVIDED else: ret.status = ComputedValueAvailability.NOTAVAILABLE return ret
def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.dpmsclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() # create missing data process definition dpd_obj = IonObject(RT.DataProcessDefinition, name=LOGICAL_TRANSFORM_DEFINITION_NAME, description="normally in preload", module='ion.processes.data.transforms.logical_transform', class_name='logical_transform') self.dataprocessclient.create_data_process_definition(dpd_obj) # deactivate all data processes when tests are complete def killAllDataProcesses(): for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]: self.dataprocessclient.deactivate_data_process(proc_id) self.dataprocessclient.delete_data_process(proc_id) self.addCleanup(killAllDataProcesses)
def create_salinity_doubler_data_process_definition(self): #First look to see if it exists and if not, then create it dpd, _ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='salinity_doubler') if len(dpd) > 0: return dpd[0] # Salinity Doubler: Data Process Definition log.debug("Create data process definition SalinityDoublerTransform") dpd_obj = IonObject( RT.DataProcessDefinition, name='salinity_doubler', description='create a salinity doubler data product', module='ion.processes.data.transforms.example_double_salinity', class_name='SalinityDoubler') try: salinity_doubler_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) except Exception as ex: self.fail( "failed to create new SalinityDoubler data process definition: %s" % ex) # create a stream definition for the data from the salinity Transform ctd_pdict_id = self.datasetclient.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) salinity_double_stream_def_id = self.pubsubclient.create_stream_definition( name='SalinityDoubler', parameter_dictionary_id=ctd_pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( salinity_double_stream_def_id, salinity_doubler_dprocdef_id, binding='salinity') return salinity_doubler_dprocdef_id
def any_old(resource_type, extra_fields=None): """ Create any old resource... a generic and unique object of a given type @param resource_type the resource type @param extra_fields dict of any extra fields to set """ if not extra_fields: extra_fields = {} if resource_type not in _sa_test_helpers_ionobj_count: _sa_test_helpers_ionobj_count[resource_type] = 0 _sa_test_helpers_ionobj_count[ resource_type] = _sa_test_helpers_ionobj_count[resource_type] + 1 name = "%s_%d" % (resource_type, _sa_test_helpers_ionobj_count[resource_type]) desc = "My %s #%d" % (resource_type, _sa_test_helpers_ionobj_count[resource_type]) log.debug("Creating any old %s IonObject (#%d)" % (resource_type, _sa_test_helpers_ionobj_count[resource_type])) ret = IonObject(resource_type, name=name, description=desc) #add any extra fields for k, v in extra_fields.iteritems(): setattr(ret, k, v) return ret
def acquire_resource(self, org_id='', user_id='', resource_id=''): """Acquire the specified resource for a specified user withing the specified Org. Once shared, the resource is committed to the user. Throws a NotFound exception if none of the ids are found. @param org_id str @param user_id str @param resource_id str @retval success bool @throws NotFound object with specified id does not exist """ param_objects = self._validate_parameters(org_id=org_id, user_id=user_id, resource_id=resource_id) commitment = IonObject(RT.ResourceCommitment, name='', org_id=org_id, user_id=user_id, resource_id=resource_id, description='Resource Commitment') commitment_id, commitment_rev = self.clients.resource_registry.create( commitment) commitment._id = commitment_id commitment._rev = commitment_rev self.clients.resource_registry.create_association( user_id, PRED.hasCommitment, commitment) self.clients.resource_registry.create_association( resource_id, PRED.hasCommitment, commitment) return True
def create_mpl_graphs_data_process_definition(self): #First look to see if it exists and if not, then create it dpd, _ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='mpl_graphs_transform') if len(dpd) > 0: return dpd[0] #Data Process Definition log.debug("Create data process definition MatplotlibGraphsTransform") dpd_obj = IonObject( RT.DataProcessDefinition, name='mpl_graphs_transform', description='Convert data streams to Matplotlib graphs', module='ion.processes.data.transforms.viz.matplotlib_graphs', class_name='VizTransformMatplotlibGraphs') try: procdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) except Exception as ex: self.fail( "failed to create new VizTransformMatplotlibGraphs data process definition: %s" % ex) pdict_id = self.datasetclient.read_parameter_dictionary_by_name( 'graph_image_param_dict', id_only=True) # create a stream definition for the data stream_def_id = self.pubsubclient.create_stream_definition( name='VizTransformMatplotlibGraphs', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( stream_def_id, procdef_id, binding='graph_image_param_dict') return procdef_id
def test_custom_attributes(self): """ Test assignment of custom attributes """ instModel_obj = IonObject(OT.CustomAttribute, name='SBE37IMModelAttr', description="model custom attr") instrument_model_id, _ = self.RR.create( any_old(RT.InstrumentModel, {"custom_attributes": [instModel_obj]})) instrument_device_id, _ = self.RR.create( any_old( RT.InstrumentDevice, { "custom_attributes": { "favorite_color": "red", "bogus_attr": "should raise warning" } })) self.IMS.assign_instrument_model_to_instrument_device( instrument_model_id, instrument_device_id) # cleanup self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_instrument_model(instrument_model_id)
def buy_bonds(self, account_id='', cash_amount=0.0): """ Purchase the specified amount of bonds. Check is first made that the cash account has sufficient funds. """ account_obj = self.clients.resource_registry.read(account_id) if not account_obj: raise NotFound("Account %s does not exist" % account_id) if account_obj.cash_balance < cash_amount: raise BadRequest("Insufficient funds") owner_obj = self.clients.resource_registry.find_subjects( "BankCustomer", PRED.hasAccount, account_obj, False)[0][0] # Create order object and call trade service order_obj = IonObject("Order", type="buy", on_behalf=owner_obj.name, cash_amount=cash_amount) # Make the call to trade service confirmation_obj = self.clients.trade.exercise(order_obj) if confirmation_obj.status == "complete": account_obj.cash_balance -= cash_amount account_obj.bond_balance += confirmation_obj.proceeds self.clients.resource_registry.update(account_obj) return "Balances after bond purchase: cash %f, bonds: %s" % ( account_obj.cash_balance, account_obj.bond_balance) return "Bond purchase status is: %s" % confirmation_obj.status
def test_createDataProduct_and_DataProducer_success(self): # setup self.clients.resource_registry.find_resources.return_value = ( [], 'do not care') self.clients.resource_registry.find_associations.return_value = [] self.clients.resource_registry.create.return_value = ('SOME_RR_ID1', 'Version_1') self.clients.data_acquisition_management.assign_data_product.return_value = None self.clients.pubsub_management.create_stream.return_value = "stream_id", "route_id" # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE ) # 1d spatial topology (station/trajectory) sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain=tdom, spatial_domain=sdom) # test call dp_id = self.data_product_management_service.create_data_product( data_product=dp_obj, stream_definition_id='a stream def id')
def create_inst_agent_instance(self, agent_id, device_id): port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.client.IMS.create_instrument_agent_instance(instAgentInstance_obj, agent_id, device_id) return instAgentInstance_id
def _get_type_interface(self, res_type): """ Creates a merge of params and commands up the type inheritance chain. Note: Entire param and command entries if subtypes replace their super types definition. """ res_interface = dict(params={}, commands={}) base_types = IonObject(res_type)._get_extends() base_types.insert(0, res_type) for rt in reversed(base_types): type_interface = self.resource_interface.get(rt, None) if not type_interface: continue for tpar, tval in type_interface.iteritems(): if tpar in res_interface: rval = res_interface[tpar] if isinstance(rval, dict): rval.update(tval) else: res_interface[tpar] = tval else: res_interface[tpar] = dict(tval) if isinstance( tval, dict) else tval return res_interface
def _do_test_find_resources(self): with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_resources( RT.UserInfo, LCS.DRAFT, "name", False) self.assertTrue( cm.exception.message == "find by name does not support lcstate") ret = self.resource_registry_service.find_resources( RT.UserInfo, None, "name", False) self.assertEquals(len(ret[0]), 0) # Instantiate an object obj = IonObject("InstrumentAgentInstance", name="name") # Persist object and read it back obj_id, obj_rev = self.resource_registry_service.create(obj) read_obj = self.resource_registry_service.read(obj_id) ret = self.resource_registry_service.find_resources( RT.InstrumentAgentInstance, None, "name", False) self.assertEquals(len(ret[0]), 1) self.assertEquals(ret[0][0]._id, read_obj._id) ret = self.resource_registry_service.find_resources( RT.InstrumentAgentInstance, LCS.DEPLOYED, None, False) self.assertEquals(len(ret[0]), 1) self.assertEquals(ret[0][0]._id, read_obj._id)