def activate_deployment(self, agent_instance_id, resource_id):
# For current device, find all deployments. Activate the one that
dep_objs, _ = self.rr.find_objects(resource_id, PRED.hasDeployment, RT.Deployment, id_only=False)
if dep_objs:
current_dep = self._get_current_deployment(dep_objs, for_activate=True)
if current_dep:
obs_ms = ObservatoryManagementServiceClient()
obs_ms.activate_deployment(current_dep._id)
class TestDeployment(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dmpsclient = DataProductManagementServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.psmsclient = PubsubManagementServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.c = DotDict()
self.c.resource_registry = self.rrclient
self.RR2 = EnhancedResourceRegistryClient(self.rrclient)
self.dsmsclient = DataProcessManagementServiceClient(node=self.container.node)
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]:
self.dsmsclient.deactivate_data_process(proc_id)
self.dsmsclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
#@unittest.skip("targeting")
def test_create_deployment(self):
#create a deployment with metadata and an initial site and device
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site')
site_id = self.omsclient.create_platform_site(platform_site__obj)
platform_device__obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
device_id = self.imsclient.create_platform_device(platform_device__obj)
start = IonTime(datetime.datetime(2013,1,1))
end = IonTime(datetime.datetime(2014,1,1))
temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string())
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment',
constraint_list=[temporal_bounds])
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.deploy_platform_site(site_id, deployment_id)
self.imsclient.deploy_platform_device(device_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
#retrieve the deployment objects and check that the assoc site and device are attached
read_deployment_obj = self.omsclient.read_deployment(deployment_id)
log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj) )
site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True)
self.assertEqual(len(site_ids), 1)
device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True)
self.assertEqual(len(device_ids), 1)
#delete the deployment
self.RR2.pluck(deployment_id)
self.omsclient.force_delete_deployment(deployment_id)
# now try to get the deleted dp object
try:
self.omsclient.read_deployment(deployment_id)
except NotFound:
pass
else:
self.fail("deleted deployment was found during read")
#@unittest.skip("targeting")
def test_prepare_deployment_support(self):
deploy_sup = self.omsclient.prepare_deployment_support()
self.assertTrue(deploy_sup)
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].type_, "AssocDeploymentInstDevice")
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].associated_resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasPlatformDevice'].type_, "AssocDeploymentPlatDevice")
self.assertEquals(deploy_sup.associations['DeploymentHasPlatformDevice'].resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasPlatformDevice'].associated_resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].type_, "AssocDeploymentInstSite")
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].associated_resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasPlatformSite'].type_, "AssocDeploymentPlatSite")
self.assertEquals(deploy_sup.associations['DeploymentHasPlatformSite'].resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasPlatformSite'].associated_resources, [])
#create a deployment with metadata and an initial site and device
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site')
site_id = self.omsclient.create_platform_site(platform_site__obj)
platform_device__obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
device_id = self.imsclient.create_platform_device(platform_device__obj)
start = IonTime(datetime.datetime(2013,1,1))
end = IonTime(datetime.datetime(2014,1,1))
temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string())
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment',
constraint_list=[temporal_bounds])
deployment_id = self.omsclient.create_deployment(deployment_obj)
deploy_sup = self.omsclient.prepare_deployment_support(deployment_id)
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].associated_resources, [])
self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformDevice'].resources), 1)
self.assertEquals(deploy_sup.associations['DeploymentHasPlatformDevice'].associated_resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].associated_resources, [])
self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformSite'].resources), 1)
self.assertEquals(deploy_sup.associations['DeploymentHasPlatformSite'].associated_resources, [])
self.omsclient.assign_site_to_deployment(site_id, deployment_id)
self.omsclient.assign_device_to_deployment(device_id, deployment_id)
deploy_sup = self.omsclient.prepare_deployment_support(deployment_id)
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].associated_resources, [])
self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformDevice'].resources), 1)
self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformDevice'].associated_resources), 1)
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].associated_resources, [])
self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformSite'].resources), 1)
self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformSite'].associated_resources), 1)
#delete the deployment
self.RR2.pluck(deployment_id)
self.omsclient.force_delete_deployment(deployment_id)
# now try to get the deleted dp object
try:
self.omsclient.read_deployment(deployment_id)
except NotFound:
pass
else:
self.fail("deleted deployment was found during read")
#@unittest.skip("targeting")
def base_activate_deployment(self):
#-------------------------------------------------------------------------------------
# Create platform site, platform device, platform model
#-------------------------------------------------------------------------------------
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site')
platform_site_id = self.omsclient.create_platform_site(platform_site__obj)
platform_device_obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
platform_device_id = self.imsclient.create_platform_device(platform_device_obj)
platform_model__obj = IonObject(RT.PlatformModel,
name='PlatformModel1',
description='test platform model')
platform_model_id = self.imsclient.create_platform_model(platform_model__obj)
#-------------------------------------------------------------------------------------
# Create instrument site
#-------------------------------------------------------------------------------------
instrument_site_obj = IonObject(RT.InstrumentSite,
name='InstrumentSite1',
description='test instrument site')
instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, platform_site_id)
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.psmsclient.create_stream_definition(name='SBE37_CDM', parameter_dictionary_id=pdict_id)
#----------------------------------------------------------------------------------------------------
# Create an instrument device
#----------------------------------------------------------------------------------------------------
instrument_device_obj = IonObject(RT.InstrumentDevice,
name='InstrumentDevice1',
description='test instrument device')
instrument_device_id = self.imsclient.create_instrument_device(instrument_device_obj)
self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id)
#----------------------------------------------------------------------------------------------------
# Create an instrument model
#----------------------------------------------------------------------------------------------------
instrument_model_obj = IonObject(RT.InstrumentModel,
name='InstrumentModel1',
description='test instrument model')
instrument_model_id = self.imsclient.create_instrument_model(instrument_model_obj)
#----------------------------------------------------------------------------------------------------
# Create a deployment object
#----------------------------------------------------------------------------------------------------
start = IonTime(datetime.datetime(2013,1,1))
end = IonTime(datetime.datetime(2014,1,1))
temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string())
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment',
context=IonObject(OT.CabledNodeDeploymentContext),
constraint_list=[temporal_bounds])
deployment_id = self.omsclient.create_deployment(deployment_obj)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
ret = DotDict(instrument_site_id=instrument_site_id,
instrument_device_id=instrument_device_id,
instrument_model_id=instrument_model_id,
platform_site_id=platform_site_id,
platform_device_id=platform_device_id,
platform_model_id=platform_model_id,
deployment_id=deployment_id)
return ret
#@unittest.skip("targeting")
def test_activate_deployment_normal(self):
res = self.base_activate_deployment()
log.debug("assigning platform and instrument models")
self.imsclient.assign_platform_model_to_platform_device(res.platform_model_id, res.platform_device_id)
self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id)
self.omsclient.assign_platform_model_to_platform_site(res.platform_model_id, res.platform_site_id)
self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id)
log.debug("adding instrument site and device to deployment")
self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id)
self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id)
log.debug("adding platform site and device to deployment")
self.omsclient.deploy_platform_site(res.platform_site_id, res.deployment_id)
self.imsclient.deploy_platform_device(res.platform_device_id, res.deployment_id)
log.debug("activating deployment, expecting success")
self.omsclient.activate_deployment(res.deployment_id)
log.debug("deactivatin deployment, expecting success")
self.omsclient.deactivate_deployment(res.deployment_id)
#@unittest.skip("targeting")
def test_activate_deployment_nomodels(self):
res = self.base_activate_deployment()
self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id)
self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id)
log.debug("activating deployment without site+device models, expecting fail")
self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1")
log.debug("assigning instrument site model")
self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id)
log.debug("activating deployment without device models, expecting fail")
self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1")
#@unittest.skip("targeting")
def test_activate_deployment_nosite(self):
res = self.base_activate_deployment()
log.debug("assigning instrument models")
self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id)
self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id)
log.debug("deploying instrument device only")
self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id)
log.debug("activating deployment without instrument site, expecting fail")
self.assert_deploy_fail(res.deployment_id, BadRequest, "Devices in this deployment outnumber sites")
#@unittest.skip("targeting")
def test_activate_deployment_nodevice(self):
res = self.base_activate_deployment()
log.debug("assigning platform and instrument models")
self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id)
self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id)
log.debug("deploying instrument site only")
self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id)
log.debug("activating deployment without device, expecting fail")
self.assert_deploy_fail(res.deployment_id, BadRequest, "No devices were found in the deployment")
def test_activate_deployment_asymmetric_children(self):
"""
P0
| \
P1 P2
|
I1
Complex deployment using CSP
P1, P2, and P3 share the same platform model. The CSP solver should be able to work this out
based on relationships to parents
"""
log.debug("create models")
imodel_id = self.RR2.create(any_old(RT.InstrumentModel))
pmodel_id = self.RR2.create(any_old(RT.PlatformModel))
log.debug("create devices")
idevice_id = self.RR2.create(any_old(RT.InstrumentDevice))
pdevice_id = [self.RR2.create(any_old(RT.PlatformDevice)) for _ in range(3)]
log.debug("create sites")
isite_id = self.RR2.create(any_old(RT.InstrumentSite))
psite_id = [self.RR2.create(any_old(RT.PlatformSite)) for _ in range(3)]
log.debug("assign models")
self.RR2.assign_instrument_model_to_instrument_device_with_has_model(imodel_id, idevice_id)
self.RR2.assign_instrument_model_to_instrument_site_with_has_model(imodel_id, isite_id)
for x in range(3):
self.RR2.assign_platform_model_to_platform_device_with_has_model(pmodel_id, pdevice_id[x])
self.RR2.assign_platform_model_to_platform_site_with_has_model(pmodel_id, psite_id[x])
log.debug("assign hierarchy")
self.RR2.assign_instrument_device_to_platform_device_with_has_device(idevice_id, pdevice_id[1])
self.RR2.assign_instrument_site_to_platform_site_with_has_site(isite_id, psite_id[1])
for x in range(1,3):
self.RR2.assign_platform_device_to_platform_device_with_has_device(pdevice_id[x], pdevice_id[0])
self.RR2.assign_platform_site_to_platform_site_with_has_site(psite_id[x], psite_id[0])
log.debug("create and activate deployment")
dep_id = self.RR2.create(any_old(RT.Deployment, {"context": IonObject(OT.RemotePlatformDeploymentContext)}))
self.RR2.assign_deployment_to_platform_device_with_has_deployment(dep_id, pdevice_id[0])
self.RR2.assign_deployment_to_platform_site_with_has_deployment(dep_id, psite_id[0])
self.omsclient.activate_deployment(dep_id)
log.debug("verifying deployment")
self.assertEqual(idevice_id, self.RR2.find_instrument_device_id_of_instrument_site_using_has_device(isite_id),
"The instrument device was not assigned to the instrument site")
for x in range(3):
self.assertEqual(pdevice_id[x], self.RR2.find_platform_device_id_of_platform_site_using_has_device(psite_id[x]),
"Platform device %d was not assigned to platform site %d" % (x, x))
def assert_deploy_fail(self, deployment_id, err_type=BadRequest, fail_message="did not specify fail_message"):
with self.assertRaises(err_type) as cm:
self.omsclient.activate_deployment(deployment_id)
self.assertIn(fail_message, cm.exception.message)
def test_3x3_matchups_remoteplatform(self):
self.base_3x3_matchups(IonObject(OT.RemotePlatformDeploymentContext))
def test_3x3_matchups_cabledinstrument(self):
self.base_3x3_matchups(IonObject(OT.CabledInstrumentDeploymentContext))
def test_3x3_matchups_cablednode(self):
self.base_3x3_matchups(IonObject(OT.CabledNodeDeploymentContext))
def base_3x3_matchups(self, deployment_context):
"""
This will be 1 root platform, 3 sub platforms (2 of one model, 1 of another) and 3 sub instruments each (2-to-1)
"""
deployment_context_type = type(deployment_context).__name__
instrument_model_id = [self.RR2.create(any_old(RT.InstrumentModel)) for _ in range(6)]
platform_model_id = [self.RR2.create(any_old(RT.PlatformModel)) for _ in range(3)]
instrument_device_id = [self.RR2.create(any_old(RT.InstrumentDevice)) for _ in range(9)]
platform_device_id = [self.RR2.create(any_old(RT.PlatformDevice)) for _ in range(4)]
instrument_site_id = [self.RR2.create(any_old(RT.InstrumentSite,
{"planned_uplink_port":
IonObject(OT.PlatformPort,
reference_designator="instport_%d" % (i+1))}))
for i in range(9)]
platform_site_id = [self.RR2.create(any_old(RT.PlatformSite,
{"planned_uplink_port":
IonObject(OT.PlatformPort,
reference_designator="platport_%d" % (i+1))}))
for i in range(4)]
def instrument_model_at(platform_idx, instrument_idx):
m = platform_idx * 2
if instrument_idx > 0:
m += 1
return m
def platform_model_at(platform_idx):
if platform_idx > 0:
return 1
return 0
def instrument_at(platform_idx, instrument_idx):
return platform_idx * 3 + instrument_idx
# set up the structure
for p in range(3):
m = platform_model_at(p)
self.RR2.assign_platform_model_to_platform_site_with_has_model(platform_model_id[m], platform_site_id[p])
self.RR2.assign_platform_model_to_platform_device_with_has_model(platform_model_id[m], platform_device_id[p])
self.RR2.assign_platform_device_to_platform_device_with_has_device(platform_device_id[p], platform_device_id[3])
self.RR2.assign_platform_site_to_platform_site_with_has_site(platform_site_id[p], platform_site_id[3])
for i in range(3):
m = instrument_model_at(p, i)
idx = instrument_at(p, i)
self.RR2.assign_instrument_model_to_instrument_site_with_has_model(instrument_model_id[m], instrument_site_id[idx])
self.RR2.assign_instrument_model_to_instrument_device_with_has_model(instrument_model_id[m], instrument_device_id[idx])
self.RR2.assign_instrument_device_to_platform_device_with_has_device(instrument_device_id[idx], platform_device_id[p])
self.RR2.assign_instrument_site_to_platform_site_with_has_site(instrument_site_id[idx], platform_site_id[p])
# top level models
self.RR2.assign_platform_model_to_platform_device_with_has_model(platform_model_id[2], platform_device_id[3])
self.RR2.assign_platform_model_to_platform_site_with_has_model(platform_model_id[2], platform_site_id[3])
# verify structure
for p in range(3):
parent_id = self.RR2.find_platform_device_id_by_platform_device_using_has_device(platform_device_id[p])
self.assertEqual(platform_device_id[3], parent_id)
parent_id = self.RR2.find_platform_site_id_by_platform_site_using_has_site(platform_site_id[p])
self.assertEqual(platform_site_id[3], parent_id)
for i in range(len(platform_site_id)):
self.assertEqual(self.RR2.find_platform_model_of_platform_device_using_has_model(platform_device_id[i]),
self.RR2.find_platform_model_of_platform_site_using_has_model(platform_site_id[i]))
for i in range(len(instrument_site_id)):
self.assertEqual(self.RR2.find_instrument_model_of_instrument_device_using_has_model(instrument_device_id[i]),
self.RR2.find_instrument_model_of_instrument_site_using_has_model(instrument_site_id[i]))
port_assignments = {}
for p in range(3):
port_assignments[platform_device_id[p]] = "platport_%d" % (p+1)
for i in range(3):
idx = instrument_at(p, i)
port_assignments[instrument_device_id[idx]] = "instport_%d" % (idx+1)
deployment_id = self.RR2.create(any_old(RT.Deployment,
{"context": deployment_context,
"port_assignments": port_assignments}))
log.debug("assigning device/site to %s deployment", deployment_context_type)
if OT.RemotePlatformDeploymentContext == deployment_context_type:
self.RR2.assign_deployment_to_platform_device_with_has_deployment(deployment_id, platform_device_id[3])
self.RR2.assign_deployment_to_platform_site_with_has_deployment(deployment_id, platform_site_id[3])
elif OT.CabledInstrumentDeploymentContext == deployment_context_type:
self.RR2.assign_deployment_to_instrument_device_with_has_deployment(deployment_id, instrument_device_id[1])
self.RR2.assign_deployment_to_instrument_site_with_has_deployment(deployment_id, instrument_site_id[1])
elif OT.CabledNodeDeploymentContext == deployment_context_type:
self.RR2.assign_deployment_to_platform_device_with_has_deployment(deployment_id, platform_device_id[1])
self.RR2.assign_deployment_to_platform_site_with_has_deployment(deployment_id, platform_site_id[1])
log.debug("activation of %s deployment", deployment_context_type)
self.omsclient.activate_deployment(deployment_id)
log.debug("validation of %s deployment", deployment_context_type)
if OT.RemotePlatformDeploymentContext == deployment_context_type:
# verify proper associations
for i, d in enumerate(platform_device_id):
self.assertEqual(d, self.RR2.find_platform_device_id_of_platform_site_using_has_device(platform_site_id[i]))
for i, d in enumerate(instrument_device_id):
self.assertEqual(d, self.RR2.find_instrument_device_id_of_instrument_site_using_has_device(instrument_site_id[i]))
elif OT.CabledInstrumentDeploymentContext == deployment_context_type:
self.assertEqual(instrument_device_id[1],
self.RR2.find_instrument_device_id_of_instrument_site_using_has_device(instrument_site_id[1]))
elif OT.CabledNodeDeploymentContext == deployment_context_type:
expected_platforms = [1]
expected_instruments = [3, 4, 5]
# verify proper associations
for i, d in enumerate(platform_device_id):
self.assertEqual(i in expected_platforms,
d in self.RR2.find_platform_device_ids_of_platform_site_using_has_device(platform_site_id[i]))
for i, d in enumerate(instrument_device_id):
self.assertEqual(i in expected_instruments,
d in self.RR2.find_instrument_device_ids_of_instrument_site_using_has_device(instrument_site_id[i]))
class TestDeployment(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dmpsclient = DataProductManagementServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.psmsclient = PubsubManagementServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.c = DotDict()
self.c.resource_registry = self.rrclient
self.RR2 = EnhancedResourceRegistryClient(self.rrclient)
self.dsmsclient = DataProcessManagementServiceClient(node=self.container.node)
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]:
self.dsmsclient.deactivate_data_process(proc_id)
self.dsmsclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
#@unittest.skip("targeting")
def test_create_deployment(self):
#create a deployment with metadata and an initial site and device
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site')
site_id = self.omsclient.create_platform_site(platform_site__obj)
platform_device__obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
device_id = self.imsclient.create_platform_device(platform_device__obj)
start = IonTime(datetime.datetime(2013,1,1))
end = IonTime(datetime.datetime(2014,1,1))
temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string())
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment',
constraint_list=[temporal_bounds])
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.deploy_platform_site(site_id, deployment_id)
self.imsclient.deploy_platform_device(device_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
#retrieve the deployment objects and check that the assoc site and device are attached
read_deployment_obj = self.omsclient.read_deployment(deployment_id)
log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj) )
site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True)
self.assertEqual(len(site_ids), 1)
device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True)
self.assertEqual(len(device_ids), 1)
#delete the deployment
self.RR2.pluck(deployment_id)
self.omsclient.force_delete_deployment(deployment_id)
# now try to get the deleted dp object
try:
self.omsclient.read_deployment(deployment_id)
except NotFound:
pass
else:
self.fail("deleted deployment was found during read")
#@unittest.skip("targeting")
def base_activate_deployment(self):
#-------------------------------------------------------------------------------------
# Create platform site, platform device, platform model
#-------------------------------------------------------------------------------------
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site')
platform_site_id = self.omsclient.create_platform_site(platform_site__obj)
platform_device_obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
platform_device_id = self.imsclient.create_platform_device(platform_device_obj)
platform_model__obj = IonObject(RT.PlatformModel,
name='PlatformModel1',
description='test platform model')
platform_model_id = self.imsclient.create_platform_model(platform_model__obj)
#-------------------------------------------------------------------------------------
# Create instrument site
#-------------------------------------------------------------------------------------
instrument_site_obj = IonObject(RT.InstrumentSite,
name='InstrumentSite1',
description='test instrument site')
instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, platform_site_id)
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.psmsclient.create_stream_definition(name='SBE37_CDM', parameter_dictionary_id=pdict_id)
#----------------------------------------------------------------------------------------------------
# Create an instrument device
#----------------------------------------------------------------------------------------------------
instrument_device_obj = IonObject(RT.InstrumentDevice,
name='InstrumentDevice1',
description='test instrument device')
instrument_device_id = self.imsclient.create_instrument_device(instrument_device_obj)
self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id)
#----------------------------------------------------------------------------------------------------
# Create an instrument model
#----------------------------------------------------------------------------------------------------
instrument_model_obj = IonObject(RT.InstrumentModel,
name='InstrumentModel1',
description='test instrument model')
instrument_model_id = self.imsclient.create_instrument_model(instrument_model_obj)
#----------------------------------------------------------------------------------------------------
# Create a deployment object
#----------------------------------------------------------------------------------------------------
start = IonTime(datetime.datetime(2013,1,1))
end = IonTime(datetime.datetime(2014,1,1))
temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string())
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment',
context=IonObject(OT.CabledNodeDeploymentContext),
constraint_list=[temporal_bounds])
deployment_id = self.omsclient.create_deployment(deployment_obj)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
ret = DotDict(instrument_site_id=instrument_site_id,
instrument_device_id=instrument_device_id,
instrument_model_id=instrument_model_id,
platform_site_id=platform_site_id,
platform_device_id=platform_device_id,
platform_model_id=platform_model_id,
deployment_id=deployment_id)
return ret
#@unittest.skip("targeting")
def test_activate_deployment_normal(self):
res = self.base_activate_deployment()
log.debug("assigning platform and instrument models")
self.imsclient.assign_platform_model_to_platform_device(res.platform_model_id, res.platform_device_id)
self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id)
self.omsclient.assign_platform_model_to_platform_site(res.platform_model_id, res.platform_site_id)
self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id)
log.debug("adding instrument site and device to deployment")
self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id)
self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id)
log.debug("adding platform site and device to deployment")
self.omsclient.deploy_platform_site(res.platform_site_id, res.deployment_id)
self.imsclient.deploy_platform_device(res.platform_device_id, res.deployment_id)
log.debug("activating deployment, expecting success")
self.omsclient.activate_deployment(res.deployment_id)
log.debug("deactivatin deployment, expecting success")
self.omsclient.deactivate_deployment(res.deployment_id)
#@unittest.skip("targeting")
def test_activate_deployment_nomodels(self):
res = self.base_activate_deployment()
self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id)
self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id)
log.debug("activating deployment without site+device models, expecting fail")
self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1")
log.debug("assigning instrument site model")
self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id)
log.debug("activating deployment without device models, expecting fail")
self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1")
#@unittest.skip("targeting")
def test_activate_deployment_nosite(self):
res = self.base_activate_deployment()
log.debug("assigning instrument models")
self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id)
self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id)
log.debug("deploying instrument device only")
self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id)
log.debug("activating deployment without instrument site, expecting fail")
self.assert_deploy_fail(res.deployment_id, BadRequest, "Devices in this deployment outnumber sites")
#@unittest.skip("targeting")
def test_activate_deployment_nodevice(self):
res = self.base_activate_deployment()
log.debug("assigning platform and instrument models")
self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id)
self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id)
log.debug("deploying instrument site only")
self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id)
log.debug("activating deployment without device, expecting fail")
self.assert_deploy_fail(res.deployment_id, BadRequest, "No devices were found in the deployment")
def test_activate_deployment_asymmetric_children(self):
"""
P0
| \
P1 P2
|
I1
Complex deployment using CSP
P1, P2, and P3 share the same platform model. The CSP solver should be able to work this out
based on relationships to parents
"""
log.debug("create models")
imodel_id = self.RR2.create(any_old(RT.InstrumentModel))
pmodel_id = self.RR2.create(any_old(RT.PlatformModel))
log.debug("create devices")
idevice_id = self.RR2.create(any_old(RT.InstrumentDevice))
pdevice_id = [self.RR2.create(any_old(RT.PlatformDevice)) for _ in range(3)]
log.debug("create sites")
isite_id = self.RR2.create(any_old(RT.InstrumentSite))
psite_id = [self.RR2.create(any_old(RT.PlatformSite)) for _ in range(3)]
log.debug("assign models")
self.RR2.assign_instrument_model_to_instrument_device_with_has_model(imodel_id, idevice_id)
self.RR2.assign_instrument_model_to_instrument_site_with_has_model(imodel_id, isite_id)
for x in range(3):
self.RR2.assign_platform_model_to_platform_device_with_has_model(pmodel_id, pdevice_id[x])
self.RR2.assign_platform_model_to_platform_site_with_has_model(pmodel_id, psite_id[x])
log.debug("assign hierarchy")
self.RR2.assign_instrument_device_to_platform_device_with_has_device(idevice_id, pdevice_id[1])
self.RR2.assign_instrument_site_to_platform_site_with_has_site(isite_id, psite_id[1])
for x in range(1,3):
self.RR2.assign_platform_device_to_platform_device_with_has_device(pdevice_id[x], pdevice_id[0])
self.RR2.assign_platform_site_to_platform_site_with_has_site(psite_id[x], psite_id[0])
log.debug("create and activate deployment")
dep_id = self.RR2.create(any_old(RT.Deployment, {"context": IonObject(OT.RemotePlatformDeploymentContext)}))
self.RR2.assign_deployment_to_platform_device_with_has_deployment(dep_id, pdevice_id[0])
self.RR2.assign_deployment_to_platform_site_with_has_deployment(dep_id, psite_id[0])
self.omsclient.activate_deployment(dep_id)
log.debug("verifying deployment")
self.assertEqual(idevice_id, self.RR2.find_instrument_device_id_of_instrument_site_using_has_device(isite_id),
"The instrument device was not assigned to the instrument site")
for x in range(3):
self.assertEqual(pdevice_id[x], self.RR2.find_platform_device_id_of_platform_site_using_has_device(psite_id[x]),
"Platform device %d was not assigned to platform site %d" % (x, x))
def assert_deploy_fail(self, deployment_id, err_type=BadRequest, fail_message="did not specify fail_message"):
with self.assertRaises(err_type) as cm:
self.omsclient.activate_deployment(deployment_id)
self.assertIn(fail_message, cm.exception.message)
def test_3x3_matchups_remoteplatform(self):
self.base_3x3_matchups(IonObject(OT.RemotePlatformDeploymentContext))
def test_3x3_matchups_cabledinstrument(self):
self.base_3x3_matchups(IonObject(OT.CabledInstrumentDeploymentContext))
def test_3x3_matchups_cablednode(self):
self.base_3x3_matchups(IonObject(OT.CabledNodeDeploymentContext))
def base_3x3_matchups(self, deployment_context):
"""
This will be 1 root platform, 3 sub platforms (2 of one model, 1 of another) and 3 sub instruments each (2-to-1)
"""
deployment_context_type = type(deployment_context).__name__
instrument_model_id = [self.RR2.create(any_old(RT.InstrumentModel)) for _ in range(6)]
platform_model_id = [self.RR2.create(any_old(RT.PlatformModel)) for _ in range(3)]
instrument_device_id = [self.RR2.create(any_old(RT.InstrumentDevice)) for _ in range(9)]
platform_device_id = [self.RR2.create(any_old(RT.PlatformDevice)) for _ in range(4)]
instrument_site_id = [self.RR2.create(any_old(RT.InstrumentSite,
{"planned_uplink_port":
IonObject(OT.PlatformPort,
reference_designator="instport_%d" % (i+1))}))
for i in range(9)]
platform_site_id = [self.RR2.create(any_old(RT.PlatformSite,
{"planned_uplink_port":
IonObject(OT.PlatformPort,
reference_designator="platport_%d" % (i+1))}))
for i in range(4)]
def instrument_model_at(platform_idx, instrument_idx):
m = platform_idx * 2
if instrument_idx > 0:
m += 1
return m
def platform_model_at(platform_idx):
if platform_idx > 0:
return 1
return 0
def instrument_at(platform_idx, instrument_idx):
return platform_idx * 3 + instrument_idx
# set up the structure
for p in range(3):
m = platform_model_at(p)
self.RR2.assign_platform_model_to_platform_site_with_has_model(platform_model_id[m], platform_site_id[p])
self.RR2.assign_platform_model_to_platform_device_with_has_model(platform_model_id[m], platform_device_id[p])
self.RR2.assign_platform_device_to_platform_device_with_has_device(platform_device_id[p], platform_device_id[3])
self.RR2.assign_platform_site_to_platform_site_with_has_site(platform_site_id[p], platform_site_id[3])
for i in range(3):
m = instrument_model_at(p, i)
idx = instrument_at(p, i)
self.RR2.assign_instrument_model_to_instrument_site_with_has_model(instrument_model_id[m], instrument_site_id[idx])
self.RR2.assign_instrument_model_to_instrument_device_with_has_model(instrument_model_id[m], instrument_device_id[idx])
self.RR2.assign_instrument_device_to_platform_device_with_has_device(instrument_device_id[idx], platform_device_id[p])
self.RR2.assign_instrument_site_to_platform_site_with_has_site(instrument_site_id[idx], platform_site_id[p])
# top level models
self.RR2.assign_platform_model_to_platform_device_with_has_model(platform_model_id[2], platform_device_id[3])
self.RR2.assign_platform_model_to_platform_site_with_has_model(platform_model_id[2], platform_site_id[3])
# verify structure
for p in range(3):
parent_id = self.RR2.find_platform_device_id_by_platform_device_using_has_device(platform_device_id[p])
self.assertEqual(platform_device_id[3], parent_id)
parent_id = self.RR2.find_platform_site_id_by_platform_site_using_has_site(platform_site_id[p])
self.assertEqual(platform_site_id[3], parent_id)
for i in range(len(platform_site_id)):
self.assertEqual(self.RR2.find_platform_model_of_platform_device_using_has_model(platform_device_id[i]),
self.RR2.find_platform_model_of_platform_site_using_has_model(platform_site_id[i]))
for i in range(len(instrument_site_id)):
self.assertEqual(self.RR2.find_instrument_model_of_instrument_device_using_has_model(instrument_device_id[i]),
self.RR2.find_instrument_model_of_instrument_site_using_has_model(instrument_site_id[i]))
port_assignments = {}
for p in range(3):
port_assignments[platform_device_id[p]] = "platport_%d" % (p+1)
for i in range(3):
idx = instrument_at(p, i)
port_assignments[instrument_device_id[idx]] = "instport_%d" % (idx+1)
deployment_id = self.RR2.create(any_old(RT.Deployment,
{"context": deployment_context,
"port_assignments": port_assignments}))
log.debug("assigning device/site to %s deployment", deployment_context_type)
if OT.RemotePlatformDeploymentContext == deployment_context_type:
self.RR2.assign_deployment_to_platform_device_with_has_deployment(deployment_id, platform_device_id[3])
self.RR2.assign_deployment_to_platform_site_with_has_deployment(deployment_id, platform_site_id[3])
elif OT.CabledInstrumentDeploymentContext == deployment_context_type:
self.RR2.assign_deployment_to_instrument_device_with_has_deployment(deployment_id, instrument_device_id[1])
self.RR2.assign_deployment_to_instrument_site_with_has_deployment(deployment_id, instrument_site_id[1])
elif OT.CabledNodeDeploymentContext == deployment_context_type:
self.RR2.assign_deployment_to_platform_device_with_has_deployment(deployment_id, platform_device_id[1])
self.RR2.assign_deployment_to_platform_site_with_has_deployment(deployment_id, platform_site_id[1])
log.debug("activation of %s deployment", deployment_context_type)
self.omsclient.activate_deployment(deployment_id)
log.debug("validation of %s deployment", deployment_context_type)
if OT.RemotePlatformDeploymentContext == deployment_context_type:
# verify proper associations
for i, d in enumerate(platform_device_id):
self.assertEqual(d, self.RR2.find_platform_device_id_of_platform_site_using_has_device(platform_site_id[i]))
for i, d in enumerate(instrument_device_id):
self.assertEqual(d, self.RR2.find_instrument_device_id_of_instrument_site_using_has_device(instrument_site_id[i]))
elif OT.CabledInstrumentDeploymentContext == deployment_context_type:
self.assertEqual(instrument_device_id[1],
self.RR2.find_instrument_device_id_of_instrument_site_using_has_device(instrument_site_id[1]))
elif OT.CabledNodeDeploymentContext == deployment_context_type:
expected_platforms = [1]
expected_instruments = [3, 4, 5]
# verify proper associations
for i, d in enumerate(platform_device_id):
self.assertEqual(i in expected_platforms,
d in self.RR2.find_platform_device_ids_of_platform_site_using_has_device(platform_site_id[i]))
for i, d in enumerate(instrument_device_id):
self.assertEqual(i in expected_instruments,
d in self.RR2.find_instrument_device_ids_of_instrument_site_using_has_device(instrument_site_id[i]))
class TestObservatoryManagementFullIntegration(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient()
self.RR2 = EnhancedResourceRegistryClient(self.RR)
self.OMS = ObservatoryManagementServiceClient()
self.org_management_service = OrgManagementServiceClient()
self.IMS = InstrumentManagementServiceClient()
self.dpclient = DataProductManagementServiceClient()
self.pubsubcli = PubsubManagementServiceClient()
self.damsclient = DataAcquisitionManagementServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self._load_stage = 0
self._resources = {}
def preload_ooi(self, stage=STAGE_LOAD_ASSETS):
# Preloads OOI up to a given stage
if self._load_stage >= stage:
return
if self._load_stage < STAGE_LOAD_ORGS:
log.info("--------------------------------------------------------------------------------------------------------")
log.info("Preloading stage: %s (OOIR2 Orgs, users, roles)", STAGE_LOAD_ORGS)
# load_OOIR2_scenario
self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=dict(
op="load",
scenario="OOIR2",
path="master",
))
self._load_stage = STAGE_LOAD_ORGS
if self._load_stage < STAGE_LOAD_PARAMS:
log.info("--------------------------------------------------------------------------------------------------------")
log.info("Preloading stage: %s (BASE params, streamdefs)", STAGE_LOAD_PARAMS)
# load_parameter_scenarios
self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=dict(
op="load",
scenario="BETA",
path="master",
categories="ParameterFunctions,ParameterDefs,ParameterDictionary,StreamDefinition",
clearcols="owner_id,org_ids",
assets="res/preload/r2_ioc/ooi_assets",
parseooi="True",
))
self._load_stage = STAGE_LOAD_PARAMS
if self._load_stage < STAGE_LOAD_AGENTS:
log.info("--------------------------------------------------------------------------------------------------------")
log.info("Preloading stage: %s (OOIR2_I agents, model links)", STAGE_LOAD_AGENTS)
# load_OOIR2_agents
self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=dict(
op="load",
scenario="OOIR2_I",
path="master",
))
self._load_stage = STAGE_LOAD_AGENTS
if self._load_stage < STAGE_LOAD_ASSETS:
log.info("--------------------------------------------------------------------------------------------------------")
log.info("Preloading stage: %s (OOI assets linked to params, agents)", STAGE_LOAD_ASSETS)
# load_ooi_assets
self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=dict(
op="load",
loadooi="True",
path="master",
assets="res/preload/r2_ioc/ooi_assets",
bulk="True",
debug="True",
ooiuntil="9/1/2013",
ooiparams="True",
#excludecategories: DataProduct,DataProductLink,Deployment,Workflow,WorkflowDefinition
))
self._load_stage = STAGE_LOAD_ASSETS
# 'DataProduct,DataProductLink,WorkflowDefinition,ExternalDataProvider,ExternalDatasetModel,ExternalDataset,ExternalDatasetAgent,ExternalDatasetAgentInstance',
@unittest.skip('Work in progress')
def test_observatory(self):
self._load_stage = 0
self._resources = {}
passing = True
self.assertTrue(True)
# LOAD STEP 1
self.preload_ooi(stage=STAGE_LOAD_ORGS)
passing &= self.orguserrole_assertions()
# LOAD STEP 2
self.preload_ooi(stage=STAGE_LOAD_PARAMS)
passing &= self.parameter_assertions()
# LOAD STEP 3
self.preload_ooi(stage=STAGE_LOAD_AGENTS)
passing &= self.agent_assertions()
# LOAD STEP 4
self.preload_ooi(stage=STAGE_LOAD_ASSETS)
# Check OOI preloaded resources to see if they match needs for this test and for correctness
passing &= self.sites_assertions()
passing &= self.device_assertions()
passing &= self.deployment_assertions()
# Extensive tests on select RSN nodes
passing &= self.rsn_node_checks()
# Extensive tests on select RSN instruments
passing &= self.check_rsn_instrument()
passing &= self.check_rsn_instrument_data_product()
# Extensive tests on a glider
#passing &= self.check_glider()
# Extensive tests on a CG assembly
#passing &= self.check_cg_assembly()
# Add a new instrument agent
# Add a new instrument agent instance
# Check DataProducts
# Check Provenance
IonIntegrationTestCase.assertTrue(self, passing)
# -------------------------------------------------------------------------
def orguserrole_assertions(self):
passing = True
passing &= self._check_marine_facility("MF_CGSN")
passing &= self._check_marine_facility("MF_RSN")
passing &= self._check_marine_facility("MF_EA")
return passing
def _check_marine_facility(self, preload_id):
passing = True
log.debug("Checking marine facility %s and associations", preload_id)
mf_obj = self.retrieve_ooi_asset(preload_id)
mf_id = mf_obj._id
self._resources[preload_id] = mf_id
passing &= self.assertEquals(mf_obj.lcstate, LCS.DEPLOYED)
res_list, _ = self.RR.find_objects(subject=mf_id, predicate=PRED.hasMembership, id_only=True)
passing &= self.assertTrue(len(res_list) >= 3)
res_list, _ = self.RR.find_objects(subject=mf_id, predicate=PRED.hasRole, id_only=False)
passing &= self.assertTrue(len(res_list) >= 5)
passing &= self._check_role_assignments(res_list, "ORG_MANAGER")
passing &= self._check_role_assignments(res_list, "OBSERVATORY_OPERATOR")
passing &= self._check_role_assignments(res_list, "INSTRUMENT_OPERATOR")
return passing
def _check_role_assignments(self, role_list, role_name):
passing = True
role_obj = self._find_resource_in_list(role_list, "governance_name", role_name)
if role_obj:
res_list = self.RR.find_subjects(predicate=PRED.hasRole, object=role_obj._id, id_only=True)
passing &= self.assertTrue(len(res_list) >= 1)
return passing
def parameter_assertions(self):
passing = True
pctx_list, _ = self.RR.find_resources_ext(restype=RT.ParameterContext)
passing &= self.assertTrue(len(pctx_list) >= 10)
pdict_list, _ = self.RR.find_resources_ext(restype=RT.ParameterDictionary)
passing &= self.assertTrue(len(pdict_list) >= 10)
sdef_list, _ = self.RR.find_resources_ext(restype=RT.StreamDefinition)
passing &= self.assertTrue(len(sdef_list) >= 10)
# Verify that a PDict has the appropriate QC parameters defined
pdicts, _ = self.RR.find_resources_ext(restype=RT.ParameterDictionary, alt_id_ns='PRE', alt_id='DICT110')
passing &= self.assertTrue(len(pdicts)==1)
if not pdicts:
return passing
pdict = pdicts[0]
# According to the latest SAF, density should NOT have trend
parameters, _ = self.RR.find_objects(pdict, PRED.hasParameterContext)
names = [i.name for i in parameters if i.name.startswith('density')]
passing &= self.assertTrue('density_trndtst_qc' not in names)
return passing
def agent_assertions(self):
passing = True
# TODO: More tests?
return passing
def sites_assertions(self):
passing = True
observatory_list, _ = self.RR.find_resources_ext(restype=RT.Observatory)
passing &= self.assertTrue(len(observatory_list) >= 40)
for obs in observatory_list:
passing &= self.assertEquals(obs.lcstate, LCS.DEPLOYED)
platform_site_list, _ = self.RR.find_resources(RT.PlatformSite, id_only=False)
log.debug('platform sites: %s', [ps.name for ps in platform_site_list])
passing &= self.assertTrue(len(platform_site_list) >= 30)
return passing
def device_assertions(self):
passing = True
platform_device_list, _ = self.RR.find_resources(RT.PlatformDevice, id_only=False)
passing &= self.assertTrue(len(platform_device_list) >= 30)
for pdev in platform_device_list:
log.debug('platform device: %s', pdev.name)
passing &= self.assertEquals(pdev.lcstate, LCS.PLANNED)
platform_agent_list, _ = self.RR.find_resources(RT.PlatformAgent, id_only=False)
passing &= self.assertTrue(len(platform_agent_list) >= 2)
for pagent in platform_agent_list:
log.debug('platform agent: %s', pagent.name)
passing &= self.assertEquals(pagent.lcstate, LCS.DEPLOYED)
instrument_agent_list, _ = self.RR.find_resources(RT.InstrumentAgent, id_only=False)
passing &= self.assertTrue(len(instrument_agent_list) >= 3)
for iagent in instrument_agent_list:
log.debug('instrument agent: %s', iagent.name)
passing &= self.assertEquals(iagent.lcstate, LCS.DEPLOYED)
model_list, _ = self.RR.find_objects(subject=iagent._id, predicate=PRED.hasModel, id_only=True)
passing &= self.assertTrue(len(model_list) >= 1, "IA %s" % iagent.name)
return passing
def deployment_assertions(self):
passing = True
deployment_list, _ = self.RR.find_resources(RT.Deployment, id_only=False)
passing &= self.assertTrue(len(deployment_list) >= 30)
for deploy in deployment_list:
log.debug('deployment: %s', deploy.name)
passing &= self.assertEquals(deploy.lcstate, LCS.DEPLOYED)
return passing
def rsn_node_checks(self):
"""
Current preload creates:
- PlatformDevice in PLANNED
- PlatformSite in DEPLOYED
- Deployment in DEPLOYED
- Deployment is NOT activated
"""
passing = True
dp_obj = self.retrieve_ooi_asset("CE04OSHY-PN01C_DEP")
passing &= self.assertEquals(dp_obj.lcstate, LCS.DEPLOYED)
passing &= self.assertEquals(dp_obj.availability, AS.AVAILABLE)
log.debug('test_observatory retrieve CE04OSHY-PN01C_DEP deployment: %s', dp_obj)
# Check existing RSN node CE04OSHY-LV01C Deployment (PLANNED lcstate)
CE04OSHY_LV01C_deployment = self.retrieve_ooi_asset('CE04OSHY-LV01C_DEP')
passing &= self.assertEquals(CE04OSHY_LV01C_deployment.lcstate, LCS.DEPLOYED)
passing &= self.assertEquals(CE04OSHY_LV01C_deployment.availability, AS.AVAILABLE)
#self.dump_deployment(CE04OSHY_LV01C_deployment._id)
log.debug('test_observatory retrieve RSN node CE04OSHY-LV01C Deployment: %s', CE04OSHY_LV01C_deployment)
CE04OSHY_LV01C_device = self.retrieve_ooi_asset('CE04OSHY-LV01C_PD')
# Set CE04OSHY-LV01C device to DEVELOPED state
passing &= self.transition_lcs_then_verify(resource_id=CE04OSHY_LV01C_device._id, new_lcs_state=LCE.DEVELOP, verify=LCS.DEVELOPED)
# Set CE04OSHY-LV01C device to INTEGRATED state
passing &= self.transition_lcs_then_verify(resource_id=CE04OSHY_LV01C_device._id, new_lcs_state=LCE.INTEGRATE, verify=LCS.INTEGRATED)
# Set CE04OSHY-LV01C device to DEPLOYED state
passing &= self.transition_lcs_then_verify(resource_id=CE04OSHY_LV01C_device._id, new_lcs_state=LCE.DEPLOY, verify=LCS.DEPLOYED)
# Set CE04OSHY-LV01C Deployment to DEPLOYED state
# NOTE: Deployments are created in DEPLOYED state, currently
#self.transition_lcs_then_verify(resource_id=CE04OSHY_LV01C_deployment._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED')
# Activate Deployment for CE04OSHY-LV01C
self.OMS.activate_deployment(CE04OSHY_LV01C_deployment._id)
log.debug('--------- activate_deployment CE04OSHY_LV01C_deployment -------------- ')
self.dump_deployment(CE04OSHY_LV01C_deployment._id)
passing &= self.validate_deployment_activated(CE04OSHY_LV01C_deployment._id)
# (optional) Start CE04OSHY-LV01C platform agent with simulator
# NOTE: DataProduct is generated in DEPLOYED state
# # Set DataProduct for CE04OSHY-LV01C platform to DEPLOYED state
# output_data_product_ids, assns =self.RR.find_objects(subject=CE04OSHY_LV01C_device._id, predicate=PRED.hasOutputProduct, id_only=True)
# if output_data_product_ids:
# #self.assertEquals(len(child_devs), 3)
# for output_data_product_id in output_data_product_ids:
# log.debug('DataProduct for CE04OSHY-LV01C platform: %s', output_data_product_id)
# self.transition_lcs_then_verify(resource_id=output_data_product_id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED')
# Check events for CE04OSHY-LV01C platform
# Check existing CE04OSBP-LJ01C Deployment (PLANNED lcstate)
# dp_list, _ = self.RR.find_resources_ext(alt_id_ns="PRE", alt_id="CE04OSBP-LJ01C_DEP")
# self.assertEquals(len(dp_list), 1)
# CE04OSHY_LV01C_deployment = dp_list[0]
# self.assertEquals(CE04OSHY_LV01C_deployment.lcstate, 'PLANNED')
# log.debug('test_observatory retrieve RSN node CE04OSBP-LJ01C Deployment: %s', CE04OSHY_LV01C_deployment)
# Set CE04OSBP-LJ01C Deployment to DEPLOYED state
# Update description and other attributes for CE04OSBP-LJ01C device resource
# Create attachment (JPG image) for CE04OSBP-LJ01C device resource
# Activate Deployment for CE04OSBP-LJ01C
# (optional) Add/register CE04OSBP-LJ01C platform agent to parent agent
# (optional) Start CE04OSBP-LJ01C platform agent
return passing
def check_rsn_instrument(self):
"""
Check existing RSN instrument CE04OSBP-LJ01C-06-CTDBPO108 Deployment (PLANNED lcstate)
Current preload creates:
- InstrumentDevice in PLANNED
- InstrumentSite in DEPLOYED
- Deployment in DEPLOYED
- Deployment is activated
"""
passing = True
CE04OSBP_LJ01C_06_CTDBPO108_deploy = self.retrieve_ooi_asset('CE04OSBP-LJ01C-06-CTDBPO108_DEP')
self.dump_deployment(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id)
#passing &= self.assertEquals(CE04OSBP_LJ01C_06_CTDBPO108_deploy.lcstate, 'PLANNED')
# Set CE04OSBP-LJ01C-06-CTDBPO108 device to DEVELOPED state
CE04OSBP_LJ01C_06_CTDBPO108_device = self.retrieve_ooi_asset('CE04OSBP-LJ01C-06-CTDBPO108_ID')
passing &= self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id, new_lcs_state=LCE.DEVELOP, verify='DEVELOPED')
# Set CE04OSBP-LJ01C-06-CTDBPO108 device to INTEGRATED state
passing &= self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id, new_lcs_state=LCE.INTEGRATE, verify='INTEGRATED')
# Set CE04OSBP-LJ01C-06-CTDBPO108 device to DEPLOYED state
passing &= self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED')
# Set CE04OSBP-LJ01C-06-CTDBPO108 Deployment to DEPLOYED state
#self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_deploy._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED')
# Activate Deployment for CE04OSBP-LJ01C-06-CTDBPO108 instrument
log.debug('--------- activate_deployment CE04OSBP-LJ01C-06-CTDBPO108 deployment -------------- ')
self.OMS.activate_deployment(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id)
passing &= self.validate_deployment_activated(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id)
# (optional) Add/register CE04OSBP-LJ01C-06-CTDBPO108 instrument agent to parent agent
# (optional) Start CE04OSBP-LJ01C-06-CTDBPO108 instrument agent with simulator
# Set all DataProducts for CE04OSBP-LJ01C-06-CTDBPO108 to DEPLOYED state
# (optional) Create a substitute Deployment for site CE04OSBP-LJ01C-06-CTDBPO108 with a comparable device
CE04OSBP_LJ01C_06_CTDBPO108_isite = self.retrieve_ooi_asset('CE04OSBP-LJ01C-06-CTDBPO108')
## create device here: retrieve CTD Mooring on Mooring Riser 001 - similiar?
GP03FLMB_RI001_10_CTDMOG999_ID_idevice = self.retrieve_ooi_asset('GP03FLMB-RI001-10-CTDMOG999_ID')
deploy_id_2 = self.create_basic_deployment(name='CE04OSBP-LJ01C-06-CTDBPO108_DEP2', description='substitute Deployment for site CE04OSBP-LJ01C-06-CTDBPO108 with a comparable device')
self.IMS.deploy_instrument_device(instrument_device_id=GP03FLMB_RI001_10_CTDMOG999_ID_idevice._id, deployment_id=deploy_id_2)
self.OMS.deploy_instrument_site(instrument_site_id=CE04OSBP_LJ01C_06_CTDBPO108_isite._id, deployment_id=deploy_id_2)
self.dump_deployment(deploy_id_2)
# (optional) Activate this second deployment - check first deployment is deactivated
self.OMS.deactivate_deployment(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id)
passing &= self.validate_deployment_deactivated(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id)
# log.debug('Activate deployment deploy_id_2')
# self.get_deployment_ids(deploy_id_2)
# self.dump_deployment(deploy_id_2, "deploy_id_2")
# self.OMS.activate_deployment(deploy_id_2)
# passing &= self.validate_deployment_deactivated(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id)
#
# # (optional) Set first CE04OSBP-LJ01C-06-CTDBPO108 Deployment to INTEGRATED state
# passing &= self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_deploy._id, new_lcs_state=LCE.INTEGRATE, verify='INTEGRATED')
#
# # Set first CE04OSBP-LJ01C-06-CTDBPO108 device to INTEGRATED state
# passing &= self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id, new_lcs_state=LCE.INTEGRATE, verify='INTEGRATED')
#
#
# # (optional) Create a third Deployment for site CE04OSBP-LJ01C-06-CTDBPO108 with a same device from first deployment
# deploy_id_3 = self.create_basic_deployment(name='CE04OSBP-LJ01C-06-CTDBPO108_DEP3', description='substitute Deployment for site CE04OSBP-LJ01C-06-CTDBPO108 with same device as first')
# self.IMS.deploy_instrument_device(instrument_device_id=GP03FLMB_RI001_10_CTDMOG999_ID_idevice._id, deployment_id=deploy_id_3)
# self.OMS.deploy_instrument_site(instrument_site_id=CE04OSBP_LJ01C_06_CTDBPO108_isite._id, deployment_id=deploy_id_3)
# self.dump_deployment(deploy_id_3)
#
#
# # Set first CE04OSBP-LJ01C-06-CTDBPO108 device to DEPLOYED state
# passing &= self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED')
#
# # (optional) Activate this third deployment - check second deployment is deactivated
# log.debug('Activate deployment deploy_id_3')
# self.dump_deployment(deploy_id_3)
# self.OMS.activate_deployment(deploy_id_3)
# #todo: check second deployment is deactivated
return passing
def check_data_product_reference(self, reference_designator, output=[]):
passing = True
data_product_ids, _ = self.RR.find_resources_ext(alt_id_ns='PRE', alt_id='%s_DPI1' % reference_designator, id_only=True) # Assuming DPI1 is parsed
passing &= self.assertEquals(len(data_product_ids), 1)
if not data_product_ids:
return passing
# Let's go ahead and activate it
data_product_id = data_product_ids[0]
self.dpclient.activate_data_product_persistence(data_product_id)
self.addCleanup(self.dpclient.suspend_data_product_persistence, data_product_id)
dataset_ids, _ = self.RR.find_objects(data_product_id, PRED.hasDataset, id_only=True)
passing &= self.assertEquals(len(dataset_ids), 1)
if not dataset_ids:
return passing
dataset_id = dataset_ids[0]
stream_def_ids, _ = self.RR.find_objects(data_product_id, PRED.hasStreamDefinition, id_only=True)
passing &= self.assertEquals(len(dataset_ids), 1)
if not stream_def_ids:
return passing
stream_def_id = stream_def_ids[0]
output.append((data_product_id, stream_def_id, dataset_id))
return passing
def check_tempsf_instrument_data_product(self, reference_designator):
passing = True
info_list = []
passing &= self.check_data_product_reference(reference_designator, info_list)
if not passing: return passing
data_product_id, stream_def_id, dataset_id = info_list.pop()
now = time.time()
ntp_now = now + 2208988800
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [ntp_now]
rdt['temperature'] = [[ 25.3884, 26.9384, 24.3394, 23.3401, 22.9832,
29.4434, 26.9873, 15.2883, 16.3374, 14.5883, 15.7253, 18.4383,
15.3488, 17.2993, 10.2111, 11.5993, 10.9345, 9.4444, 9.9876,
10.9834, 11.0098, 5.3456, 4.2994, 4.3009]]
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
passing &= self.assertTrue(dataset_monitor.event.wait(20))
if not passing: return passing
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
passing &= self.assert_array_almost_equal(rdt['time'], [ntp_now])
passing &= self.assert_array_almost_equal(rdt['temperature'], [[
25.3884, 26.9384, 24.3394, 23.3401, 22.9832, 29.4434, 26.9873,
15.2883, 16.3374, 14.5883, 15.7253, 18.4383, 15.3488, 17.2993,
10.2111, 11.5993, 10.9345, 9.4444, 9.9876, 10.9834, 11.0098,
5.3456, 4.2994, 4.3009]])
return passing
def check_trhph_instrument_data_products(self, reference_designator):
passing = True
info_list = []
passing &= self.check_data_product_reference(reference_designator, info_list)
if not passing:
return passing
data_product_id, stream_def_id, dataset_id = info_list.pop()
pdict = self.RR2.find_parameter_dictionary_of_stream_definition_using_has_parameter_dictionary(stream_def_id)
passing &= self.assertEquals(pdict.name, 'trhph_sample')
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
# calibration constants
a = 1.98e-9
b = -2.45e-6
c = 9.28e-4
d = -0.0888
e = 0.731
V_s = 1.506
V_c = 0.
T = 11.8
r1 = 0.906
r2 = 4.095
r3 = 4.095
ORP_V = 1.806
Cl = np.nan
offset = 2008
gain = 4.0
# Normally this would be 50 per the DPS but the precision is %4.0f which truncates the values to the nearest 1...
ORP = ((ORP_V * 1000.) - offset) / gain
ntp_now = time.time() + 2208988800
rdt['cc_a'] = [a]
rdt['cc_b'] = [b]
rdt['cc_c'] = [c]
rdt['cc_d'] = [d]
rdt['cc_e'] = [e]
rdt['ref_temp_volts'] = [V_s]
rdt['resistivity_temp_volts'] = [V_c]
rdt['eh_sensor'] = [ORP_V]
rdt['resistivity_5'] = [r1]
rdt['resistivity_x1'] = [r2]
rdt['resistivity_x5'] = [r3]
rdt['cc_offset'] = [offset]
rdt['cc_gain'] = [gain]
rdt['time'] = [ntp_now]
passing &= self.assert_array_almost_equal(rdt['vent_fluid_temperaure'], [T], 2)
passing &= self.assert_array_almost_equal(rdt['vent_fluid_chloride_conc'], [Cl], 4)
passing &= self.assert_array_almost_equal(rdt['vent_fluid_orp'], [ORP], 4)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
passing &= self.assertTrue(dataset_monitor.event.wait(60))
if not passing: return passing
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
passing &= self.assert_array_almost_equal(rdt['vent_fluid_temperaure'], [T], 2)
passing &= self.assert_array_almost_equal(rdt['vent_fluid_chloride_conc'], [Cl], 4)
passing &= self.assert_array_almost_equal(rdt['vent_fluid_orp'], [ORP], 4)
return passing
def check_vel3d_instrument_data_products(self, reference_designator):
passing = True
info_list = []
passing &= self.check_data_product_reference(reference_designator, info_list)
if not passing:
return passing
data_product_id, stream_def_id, dataset_id = info_list.pop()
pdict = self.RR2.find_parameter_dictionary_of_stream_definition_using_has_parameter_dictionary(stream_def_id)
self.assertEquals(pdict.name, 'vel3d_b_sample')
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
lat = 14.6846
lon = -51.044
ts = np.array([3319563600, 3319567200, 3319570800, 3319574400,
3319578000, 3319581600, 3319585200, 3319588800, 3319592400,
3319596000], dtype=np.float)
ve = np.array([ -3.2, 0.1, 0. , 2.3, -0.1, 5.6, 5.1, 5.8,
8.8, 10.3])
vn = np.array([ 18.2, 9.9, 12. , 6.6, 7.4, 3.4, -2.6, 0.2,
-1.5, 4.1])
vu = np.array([-1.1, -0.6, -1.4, -2, -1.7, -2, 1.3, -1.6, -1.1, -4.5])
ve_expected = np.array([-0.085136, -0.028752, -0.036007, 0.002136,
-0.023158, 0.043218, 0.056451, 0.054727, 0.088446, 0.085952])
vn_expected = np.array([ 0.164012, 0.094738, 0.114471, 0.06986, 0.07029,
0.049237, -0.009499, 0.019311, 0.012096, 0.070017])
vu_expected = np.array([-0.011, -0.006, -0.014, -0.02, -0.017, -0.02,
0.013, -0.016, -0.011, -0.045])
rdt['time'] = ts
rdt['lat'] = [lat] * 10
rdt['lon'] = [lon] * 10
rdt['turbulent_velocity_east'] = ve
rdt['turbulent_velocity_north'] = vn
rdt['turbulent_velocity_up'] = vu
passing &= self.assert_array_almost_equal(rdt['eastward_turbulent_velocity'],
ve_expected)
passing &= self.assert_array_almost_equal(rdt['northward_turbulent_velocity'],
vn_expected)
passing &= self.assert_array_almost_equal(rdt['upward_turbulent_velocity'],
vu_expected)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
passing &= self.assertTrue(dataset_monitor.event.wait(20))
if not passing: return passing
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
passing &= self.assert_array_almost_equal(rdt['eastward_turbulent_velocity'],
ve_expected)
passing &= self.assert_array_almost_equal(rdt['northward_turbulent_velocity'],
vn_expected)
passing &= self.assert_array_almost_equal(rdt['upward_turbulent_velocity'],
vu_expected)
return passing
def check_presta_instrument_data_products(self, reference_designator):
# Check the parsed data product make sure it's got everything it needs and can be published persisted etc.
# Absolute Pressure (SFLPRES_L0) is what comes off the instrumnet, SFLPRES_L1 is a pfunc
# Let's go ahead and publish some fake data!!!
# According to https://alfresco.oceanobservatories.org/alfresco/d/d/workspace/SpacesStore/63e16865-9d9e-4b11-b0b3-d5658faa5080/1341-00230_Data_Product_Spec_SFLPRES_OOI.pdf
# Appendix A. Example 1.
# p_psia_tide = 14.8670
# the tide should be 10.2504
passing = True
info_list = []
passing &= self.check_data_product_reference(reference_designator, info_list)
if not passing:
return passing
data_product_id, stream_def_id, dataset_id = info_list.pop()
now = time.time()
ntp_now = now + 2208988800.
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [ntp_now]
rdt['absolute_pressure'] = [14.8670]
passing &= self.assert_array_almost_equal(rdt['seafloor_pressure'], [10.2504], 4)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(dataset_monitor.event.wait(20)) # Bumped to 20 to keep buildbot happy
if not passing: return passing
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
passing &= self.assert_array_almost_equal(rdt['time'], [ntp_now])
passing &= self.assert_array_almost_equal(rdt['seafloor_pressure'], [10.2504], 4)
passing &= self.assert_array_almost_equal(rdt['absolute_pressure'], [14.8670], 4)
return passing
def check_rsn_instrument_data_product(self):
passing = True
# for RS03AXBS-MJ03A-06-PRESTA301 (PREST-A) there are a few listed data products
# Parsed, Engineering
# SFLPRES-0 SFLPRES-1
# Check for the two data products and make sure they have the proper parameters
# SFLPRES-0 should
data_products, _ = self.RR.find_resources_ext(alt_id_ns='PRE', alt_id='RS03AXBS-MJ03A-06-PRESTA301_SFLPRES_L0_DPID', id_only=True)
passing &=self.assertTrue(len(data_products)==1)
if not data_products:
return passing
data_product_id = data_products[0]
stream_defs, _ = self.RR.find_objects(data_product_id,PRED.hasStreamDefinition,id_only=False)
passing &= self.assertTrue(len(stream_defs)==1)
if not stream_defs:
return passing
# Assert that the stream definition has the correct reference designator
stream_def = stream_defs[0]
passing &= self.assertEquals(stream_def.stream_configuration['reference_designator'], 'RS03AXBS-MJ03A-06-PRESTA301')
# Get the pdict and make sure that the parameters corresponding to the available fields
# begin with the appropriate data product identifier
pdict_ids, _ = self.RR.find_objects(stream_def, PRED.hasParameterDictionary, id_only=True)
passing &= self.assertEquals(len(pdict_ids), 1)
if not pdict_ids:
return passing
pdict_id = pdict_ids[0]
pdict = DatasetManagementService.get_parameter_dictionary(pdict_id)
available_params = [pdict.get_context(i) for i in pdict.keys() if i in stream_def.available_fields]
for p in available_params:
if p.name=='time': # Ignore the domain parameter
continue
passing &= self.assertTrue(p.ooi_short_name.startswith('SFLPRES'))
passing &= self.check_presta_instrument_data_products('RS01SLBS-MJ01A-06-PRESTA101')
passing &= self.check_vel3d_instrument_data_products( 'RS01SLBS-MJ01A-12-VEL3DB101')
passing &= self.check_presta_instrument_data_products('RS03AXBS-MJ03A-06-PRESTA301')
passing &= self.check_vel3d_instrument_data_products( 'RS03AXBS-MJ03A-12-VEL3DB301')
passing &= self.check_tempsf_instrument_data_product( 'RS03ASHS-MJ03B-07-TMPSFA301')
passing &= self.check_vel3d_instrument_data_products( 'RS03INT2-MJ03D-12-VEL3DB304')
passing &= self.check_trhph_instrument_data_products( 'RS03INT1-MJ03C-10-TRHPHA301')
self.data_product_management.activate_data_product_persistence(data_product_id)
dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(data_product_id)
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
self.assert_array_almost_equal(rdt['seafloor_pressure'], [10.2504], 4)
self.assert_array_almost_equal(rdt['absolute_pressure'], [14.8670], 4)
self.data_product_management.suspend_data_product_persistence(data_product_id) # Should do nothing and not raise anything
return passing
def check_glider(self):
'''
# Check that glider GP05MOAS-GL001 assembly is defined by OOI preload (3 instruments)
'''
passing = True
GP05MOAS_GL001_device = self.retrieve_ooi_asset('GP05MOAS-GL001_PD')
child_devs, assns =self.RR.find_objects(subject=GP05MOAS_GL001_device._id, predicate=PRED.hasDevice, id_only=True)
passing &= self.assertEquals(len(child_devs), 3)
# Set GP05MOAS-GL001 Deployment to DEPLOYED
GP05MOAS_GL001_deploy = self.retrieve_ooi_asset('GP05MOAS-GL001_DEP')
passing &= self.transition_lcs_then_verify(resource_id=GP05MOAS_GL001_deploy._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED')
# Activate Deployment for GP05MOAS-GL001
#self.OMS.activate_deployment(GP05MOAS_GL001_deploy._id)
# Deactivate Deployment for GP05MOAS-GL001
#self.OMS.deactivate_deployment(GP05MOAS_GL001_deploy._id)
# Create a new Deployment resource X without any assignment
x_deploy_id = self.create_basic_deployment(name='X_Deployment', description='new Deployment resource X without any assignment')
# Assign Deployment X to site GP05MOAS-GL001
GP05MOAS_GL001_psite = self.retrieve_ooi_asset('GP05MOAS-GL001')
self.OMS.deploy_platform_site(GP05MOAS_GL001_psite._id, x_deploy_id)
# Assign Deployment X to first device for GP05MOAS-GL001
GP05MOAS_GL001_device = self.retrieve_ooi_asset('GP05MOAS-GL001_PD')
self.IMS.deploy_platform_device(GP05MOAS_GL001_device._id, x_deploy_id)
# Set GP05MOAS-GL001 Deployment to PLANNED state
#self.transition_lcs_then_verify(resource_id=x_deploy_id, new_lcs_state=LCE.PLAN, verify='PLANNED')
# ??? already in planned
# Set second GP05MOAS-GL001 Deployment to DEPLOYED
passing &= self.transition_lcs_then_verify(resource_id=x_deploy_id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED')
self.dump_deployment(x_deploy_id)
# Activate second Deployment for GP05MOAS-GL001
#self.OMS.activate_deployment(x_deploy_id)
# Deactivate second Deployment for GP05MOAS-GL001
#self.OMS.deactivate_deployment(x_deploy_id)
return passing
def check_cg_assembly(self):
passing = True
# Set several CE01ISSM-RI002-* instrument devices to DEVELOPED state
# Assemble several CE01ISSM-RI002-* instruments to a CG CE01ISSM-RI002 component platform
# Set several CE01ISSM-RI002-* instrument devices to INTEGRATED state
# Assemble CE01ISSM-RI002 platform to CG CE01ISSM-LM001 station platform
# Set CE01ISSM-RI002 component device to INTEGRATED state
# Set CE01ISSM-LM001 station device to INTEGRATED state
# Set CE01ISSM-LM001 station device to DEPLOYED state (children maybe too?)
# Set CE01ISSM-LM001 Deployment to DEPLOYED
# Activate CE01ISSM-LM001 platform assembly deployment
# Dectivate CE01ISSM-LM001 platform assembly deployment
# Set CE01ISSM-LM001 Deployment to INTEGRATED state
# Set CE01ISSM-LM001 station device to INTEGRATED state
# Set CE01ISSM-RI002 component device to INTEGRATED state
# Set CE01ISSM-RI002 component device to INTEGRATED state
# Disassemble CE01ISSM-RI002 platform from CG CE01ISSM-LM001 station platform
# Disassemble all CE01ISSM-RI002-* instruments from a CG CE01ISSM-RI002 component platform
# Retire instrument one for CE01ISSM-RI002-*
# Retire device one for CE01ISSM-RI002
# Retire device one for CE01ISSM-LM001
return passing
# -------------------------------------------------------------------------
def retrieve_ooi_asset(self, alt_id='', namespace='PRE'):
dp_list, _ = self.RR.find_resources_ext(alt_id_ns=namespace, alt_id=alt_id)
self.assertEquals(len(dp_list), 1)
return dp_list[0]
def transition_lcs_then_verify(self, resource_id, new_lcs_state, verify):
ret = self.RR2.advance_lcs(resource_id, new_lcs_state)
resource_obj = self.RR.read(resource_id)
return self.assertEquals(resource_obj.lcstate, verify)
def create_basic_deployment(self, name='', description=''):
start = IonTime(datetime.datetime(2013,1,1))
end = IonTime(datetime.datetime(2014,1,1))
temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string())
deployment_obj = IonObject(RT.Deployment,
name=name,
description=description,
context=IonObject(OT.CabledNodeDeploymentContext),
constraint_list=[temporal_bounds])
return self.OMS.create_deployment(deployment_obj)
def validate_deployment_activated(self, deployment_id=''):
site_id, device_id = self.get_deployment_ids(deployment_id)
assocs = self.RR.find_associations(subject=site_id, predicate=PRED.hasDevice, object=device_id)
return self.assertEquals(len(assocs), 1)
def validate_deployment_deactivated(self, deployment_id=''):
site_id, device_id = self.get_deployment_ids(deployment_id)
assocs = self.RR.find_associations(subject=site_id, predicate=PRED.hasDevice, object=device_id)
return self.assertEquals(len(assocs), 0)
def dump_deployment(self, deployment_id='', name=""):
#site_id, device_id = self.get_deployment_ids(deployment_id)
resource_list,_ = self.RR.find_subjects(predicate=PRED.hasDeployment, object=deployment_id, id_only=True)
resource_list.append(deployment_id)
resources = self.RR.read_mult(resource_list )
log.debug('--------- dump_deployment %s summary---------------', name)
for resource in resources:
log.debug('%s: %s (%s)', resource._get_type(), resource.name, resource._id)
log.debug('--------- dump_deployment %s full dump ---------------', name)
for resource in resources:
log.debug('resource: %s ', resource)
log.debug('--------- dump_deployment %s end ---------------', name)
#assocs = self.container.resource_registry.find_assoctiations(anyside=deployment_id)
# assocs = Container.instance.resource_registry.find_assoctiations(anyside=deployment_id)
# log.debug('--------- dump_deployment ---------------')
# for assoc in assocs:
# log.debug('SUBJECT: %s PREDICATE: %s OBJET: %s', assoc.s, assoc.p, assoc.o)
# log.debug('--------- dump_deployment end ---------------')
def get_deployment_ids(self, deployment_id=''):
devices = []
sites = []
idevice_list,_ = self.RR.find_subjects(RT.InstrumentDevice, PRED.hasDeployment, deployment_id, id_only=True)
pdevice_list,_ = self.RR.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, id_only=True)
devices = idevice_list + pdevice_list
self.assertEquals(1, len(devices))
isite_list,_ = self.RR.find_subjects(RT.InstrumentSite, PRED.hasDeployment, deployment_id, id_only=True)
psite_list,_ = self.RR.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, id_only=True)
sites = isite_list + psite_list
self.assertEquals(1, len(sites))
return sites[0], devices[0]
def _find_resource_in_list(self, res_list, attr, attr_val, assert_found=True):
for res in res_list:
v = getattr(res, attr, None)
if v == attr_val:
return res
if assert_found:
self.assertTrue(False, "Attribute %s value %s not found in list" % (attr, attr_val))
return None
# -------------------------------------------------------------------------
def _get_caller(self):
s = inspect.stack()
return "%s:%s" % (s[2][1], s[2][2])
@assertion_wrapper
def assert_array_almost_equal(self, *args, **kwargs):
np.testing.assert_array_almost_equal(*args, **kwargs)
@assertion_wrapper
def assertEquals(self, *args, **kwargs):
IonIntegrationTestCase.assertEquals(self, *args, **kwargs)
@assertion_wrapper
def assertTrue(self, *args, **kwargs):
IonIntegrationTestCase.assertTrue(self, *args, **kwargs)
class TestObservatoryManagementFullIntegration(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient()
self.RR2 = EnhancedResourceRegistryClient(self.RR)
self.OMS = ObservatoryManagementServiceClient()
self.org_management_service = OrgManagementServiceClient()
self.IMS = InstrumentManagementServiceClient()
self.dpclient = DataProductManagementServiceClient()
self.pubsubcli = PubsubManagementServiceClient()
self.damsclient = DataAcquisitionManagementServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self._load_stage = 0
self._resources = {}
def preload_ooi(self, stage=STAGE_LOAD_ASSETS):
# Preloads OOI up to a given stage
if self._load_stage >= stage:
return
if self._load_stage < STAGE_LOAD_ORGS:
log.info(
"--------------------------------------------------------------------------------------------------------"
)
log.info("Preloading stage: %s (OOIR2 Orgs, users, roles)",
STAGE_LOAD_ORGS)
# load_OOIR2_scenario
self.container.spawn_process("Loader",
"ion.processes.bootstrap.ion_loader",
"IONLoader",
config=dict(
op="load",
scenario="OOIR2",
path="master",
))
self._load_stage = STAGE_LOAD_ORGS
if self._load_stage < STAGE_LOAD_PARAMS:
log.info(
"--------------------------------------------------------------------------------------------------------"
)
log.info("Preloading stage: %s (BASE params, streamdefs)",
STAGE_LOAD_PARAMS)
# load_parameter_scenarios
self.container.spawn_process(
"Loader",
"ion.processes.bootstrap.ion_loader",
"IONLoader",
config=dict(
op="load",
scenario="BETA",
path="master",
categories=
"ParameterFunctions,ParameterDefs,ParameterDictionary,StreamDefinition",
clearcols="owner_id,org_ids",
assets="res/preload/r2_ioc/ooi_assets",
parseooi="True",
))
self._load_stage = STAGE_LOAD_PARAMS
if self._load_stage < STAGE_LOAD_AGENTS:
log.info(
"--------------------------------------------------------------------------------------------------------"
)
log.info("Preloading stage: %s (OOIR2_I agents, model links)",
STAGE_LOAD_AGENTS)
# load_OOIR2_agents
self.container.spawn_process("Loader",
"ion.processes.bootstrap.ion_loader",
"IONLoader",
config=dict(
op="load",
scenario="OOIR2_I",
path="master",
))
self._load_stage = STAGE_LOAD_AGENTS
if self._load_stage < STAGE_LOAD_ASSETS:
log.info(
"--------------------------------------------------------------------------------------------------------"
)
log.info(
"Preloading stage: %s (OOI assets linked to params, agents)",
STAGE_LOAD_ASSETS)
# load_ooi_assets
self.container.spawn_process(
"Loader",
"ion.processes.bootstrap.ion_loader",
"IONLoader",
config=dict(
op="load",
loadooi="True",
path="master",
assets="res/preload/r2_ioc/ooi_assets",
bulk="True",
debug="True",
ooiuntil="9/1/2013",
ooiparams="True",
#excludecategories: DataProduct,DataProductLink,Deployment,Workflow,WorkflowDefinition
))
self._load_stage = STAGE_LOAD_ASSETS
# 'DataProduct,DataProductLink,WorkflowDefinition,ExternalDataProvider,ExternalDatasetModel,ExternalDataset,ExternalDatasetAgent,ExternalDatasetAgentInstance',
@unittest.skip('Work in progress')
def test_observatory(self):
self._load_stage = 0
self._resources = {}
passing = True
self.assertTrue(True)
# LOAD STEP 1
self.preload_ooi(stage=STAGE_LOAD_ORGS)
passing &= self.orguserrole_assertions()
# LOAD STEP 2
self.preload_ooi(stage=STAGE_LOAD_PARAMS)
passing &= self.parameter_assertions()
# LOAD STEP 3
self.preload_ooi(stage=STAGE_LOAD_AGENTS)
passing &= self.agent_assertions()
# LOAD STEP 4
self.preload_ooi(stage=STAGE_LOAD_ASSETS)
# Check OOI preloaded resources to see if they match needs for this test and for correctness
passing &= self.sites_assertions()
passing &= self.device_assertions()
passing &= self.deployment_assertions()
# Extensive tests on select RSN nodes
passing &= self.rsn_node_checks()
# Extensive tests on select RSN instruments
passing &= self.check_rsn_instrument()
passing &= self.check_rsn_instrument_data_product()
# Extensive tests on a glider
#passing &= self.check_glider()
# Extensive tests on a CG assembly
#passing &= self.check_cg_assembly()
# Add a new instrument agent
# Add a new instrument agent instance
# Check DataProducts
# Check Provenance
IonIntegrationTestCase.assertTrue(self, passing)
# -------------------------------------------------------------------------
def orguserrole_assertions(self):
passing = True
passing &= self._check_marine_facility("MF_CGSN")
passing &= self._check_marine_facility("MF_RSN")
passing &= self._check_marine_facility("MF_EA")
return passing
def _check_marine_facility(self, preload_id):
passing = True
log.debug("Checking marine facility %s and associations", preload_id)
mf_obj = self.retrieve_ooi_asset(preload_id)
mf_id = mf_obj._id
self._resources[preload_id] = mf_id
passing &= self.assertEquals(mf_obj.lcstate, LCS.DEPLOYED)
res_list, _ = self.RR.find_objects(subject=mf_id,
predicate=PRED.hasMembership,
id_only=True)
passing &= self.assertTrue(len(res_list) >= 3)
res_list, _ = self.RR.find_objects(subject=mf_id,
predicate=PRED.hasRole,
id_only=False)
passing &= self.assertTrue(len(res_list) >= 5)
passing &= self._check_role_assignments(res_list, "ORG_MANAGER")
passing &= self._check_role_assignments(res_list,
"OBSERVATORY_OPERATOR")
passing &= self._check_role_assignments(res_list,
"INSTRUMENT_OPERATOR")
return passing
def _check_role_assignments(self, role_list, role_name):
passing = True
role_obj = self._find_resource_in_list(role_list, "governance_name",
role_name)
if role_obj:
res_list = self.RR.find_subjects(predicate=PRED.hasRole,
object=role_obj._id,
id_only=True)
passing &= self.assertTrue(len(res_list) >= 1)
return passing
def parameter_assertions(self):
passing = True
pctx_list, _ = self.RR.find_resources_ext(restype=RT.ParameterContext)
passing &= self.assertTrue(len(pctx_list) >= 10)
pdict_list, _ = self.RR.find_resources_ext(
restype=RT.ParameterDictionary)
passing &= self.assertTrue(len(pdict_list) >= 10)
sdef_list, _ = self.RR.find_resources_ext(restype=RT.StreamDefinition)
passing &= self.assertTrue(len(sdef_list) >= 10)
# Verify that a PDict has the appropriate QC parameters defined
pdicts, _ = self.RR.find_resources_ext(restype=RT.ParameterDictionary,
alt_id_ns='PRE',
alt_id='DICT110')
passing &= self.assertTrue(len(pdicts) == 1)
if not pdicts:
return passing
pdict = pdicts[0]
# According to the latest SAF, density should NOT have trend
parameters, _ = self.RR.find_objects(pdict, PRED.hasParameterContext)
names = [i.name for i in parameters if i.name.startswith('density')]
passing &= self.assertTrue('density_trndtst_qc' not in names)
return passing
def agent_assertions(self):
passing = True
# TODO: More tests?
return passing
def sites_assertions(self):
passing = True
observatory_list, _ = self.RR.find_resources_ext(
restype=RT.Observatory)
passing &= self.assertTrue(len(observatory_list) >= 40)
for obs in observatory_list:
passing &= self.assertEquals(obs.lcstate, LCS.DEPLOYED)
platform_site_list, _ = self.RR.find_resources(RT.PlatformSite,
id_only=False)
log.debug('platform sites: %s', [ps.name for ps in platform_site_list])
passing &= self.assertTrue(len(platform_site_list) >= 30)
return passing
def device_assertions(self):
passing = True
platform_device_list, _ = self.RR.find_resources(RT.PlatformDevice,
id_only=False)
passing &= self.assertTrue(len(platform_device_list) >= 30)
for pdev in platform_device_list:
log.debug('platform device: %s', pdev.name)
passing &= self.assertEquals(pdev.lcstate, LCS.PLANNED)
platform_agent_list, _ = self.RR.find_resources(RT.PlatformAgent,
id_only=False)
passing &= self.assertTrue(len(platform_agent_list) >= 2)
for pagent in platform_agent_list:
log.debug('platform agent: %s', pagent.name)
passing &= self.assertEquals(pagent.lcstate, LCS.DEPLOYED)
instrument_agent_list, _ = self.RR.find_resources(RT.InstrumentAgent,
id_only=False)
passing &= self.assertTrue(len(instrument_agent_list) >= 3)
for iagent in instrument_agent_list:
log.debug('instrument agent: %s', iagent.name)
passing &= self.assertEquals(iagent.lcstate, LCS.DEPLOYED)
model_list, _ = self.RR.find_objects(subject=iagent._id,
predicate=PRED.hasModel,
id_only=True)
passing &= self.assertTrue(
len(model_list) >= 1, "IA %s" % iagent.name)
return passing
def deployment_assertions(self):
passing = True
deployment_list, _ = self.RR.find_resources(RT.Deployment,
id_only=False)
passing &= self.assertTrue(len(deployment_list) >= 30)
for deploy in deployment_list:
log.debug('deployment: %s', deploy.name)
passing &= self.assertEquals(deploy.lcstate, LCS.DEPLOYED)
return passing
def rsn_node_checks(self):
"""
Current preload creates:
- PlatformDevice in PLANNED
- PlatformSite in DEPLOYED
- Deployment in DEPLOYED
- Deployment is NOT activated
"""
passing = True
dp_obj = self.retrieve_ooi_asset("CE04OSHY-PN01C_DEP")
passing &= self.assertEquals(dp_obj.lcstate, LCS.DEPLOYED)
passing &= self.assertEquals(dp_obj.availability, AS.AVAILABLE)
log.debug(
'test_observatory retrieve CE04OSHY-PN01C_DEP deployment: %s',
dp_obj)
# Check existing RSN node CE04OSHY-LV01C Deployment (PLANNED lcstate)
CE04OSHY_LV01C_deployment = self.retrieve_ooi_asset(
'CE04OSHY-LV01C_DEP')
passing &= self.assertEquals(CE04OSHY_LV01C_deployment.lcstate,
LCS.DEPLOYED)
passing &= self.assertEquals(CE04OSHY_LV01C_deployment.availability,
AS.AVAILABLE)
#self.dump_deployment(CE04OSHY_LV01C_deployment._id)
log.debug(
'test_observatory retrieve RSN node CE04OSHY-LV01C Deployment: %s',
CE04OSHY_LV01C_deployment)
CE04OSHY_LV01C_device = self.retrieve_ooi_asset('CE04OSHY-LV01C_PD')
# Set CE04OSHY-LV01C device to DEVELOPED state
passing &= self.transition_lcs_then_verify(
resource_id=CE04OSHY_LV01C_device._id,
new_lcs_state=LCE.DEVELOP,
verify=LCS.DEVELOPED)
# Set CE04OSHY-LV01C device to INTEGRATED state
passing &= self.transition_lcs_then_verify(
resource_id=CE04OSHY_LV01C_device._id,
new_lcs_state=LCE.INTEGRATE,
verify=LCS.INTEGRATED)
# Set CE04OSHY-LV01C device to DEPLOYED state
passing &= self.transition_lcs_then_verify(
resource_id=CE04OSHY_LV01C_device._id,
new_lcs_state=LCE.DEPLOY,
verify=LCS.DEPLOYED)
# Set CE04OSHY-LV01C Deployment to DEPLOYED state
# NOTE: Deployments are created in DEPLOYED state, currently
#self.transition_lcs_then_verify(resource_id=CE04OSHY_LV01C_deployment._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED')
# Activate Deployment for CE04OSHY-LV01C
self.OMS.activate_deployment(CE04OSHY_LV01C_deployment._id)
log.debug(
'--------- activate_deployment CE04OSHY_LV01C_deployment -------------- '
)
self.dump_deployment(CE04OSHY_LV01C_deployment._id)
passing &= self.validate_deployment_activated(
CE04OSHY_LV01C_deployment._id)
# (optional) Start CE04OSHY-LV01C platform agent with simulator
# NOTE: DataProduct is generated in DEPLOYED state
# # Set DataProduct for CE04OSHY-LV01C platform to DEPLOYED state
# output_data_product_ids, assns =self.RR.find_objects(subject=CE04OSHY_LV01C_device._id, predicate=PRED.hasOutputProduct, id_only=True)
# if output_data_product_ids:
# #self.assertEquals(len(child_devs), 3)
# for output_data_product_id in output_data_product_ids:
# log.debug('DataProduct for CE04OSHY-LV01C platform: %s', output_data_product_id)
# self.transition_lcs_then_verify(resource_id=output_data_product_id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED')
# Check events for CE04OSHY-LV01C platform
# Check existing CE04OSBP-LJ01C Deployment (PLANNED lcstate)
# dp_list, _ = self.RR.find_resources_ext(alt_id_ns="PRE", alt_id="CE04OSBP-LJ01C_DEP")
# self.assertEquals(len(dp_list), 1)
# CE04OSHY_LV01C_deployment = dp_list[0]
# self.assertEquals(CE04OSHY_LV01C_deployment.lcstate, 'PLANNED')
# log.debug('test_observatory retrieve RSN node CE04OSBP-LJ01C Deployment: %s', CE04OSHY_LV01C_deployment)
# Set CE04OSBP-LJ01C Deployment to DEPLOYED state
# Update description and other attributes for CE04OSBP-LJ01C device resource
# Create attachment (JPG image) for CE04OSBP-LJ01C device resource
# Activate Deployment for CE04OSBP-LJ01C
# (optional) Add/register CE04OSBP-LJ01C platform agent to parent agent
# (optional) Start CE04OSBP-LJ01C platform agent
return passing
def check_rsn_instrument(self):
"""
Check existing RSN instrument CE04OSBP-LJ01C-06-CTDBPO108 Deployment (PLANNED lcstate)
Current preload creates:
- InstrumentDevice in PLANNED
- InstrumentSite in DEPLOYED
- Deployment in DEPLOYED
- Deployment is activated
"""
passing = True
CE04OSBP_LJ01C_06_CTDBPO108_deploy = self.retrieve_ooi_asset(
'CE04OSBP-LJ01C-06-CTDBPO108_DEP')
self.dump_deployment(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id)
#passing &= self.assertEquals(CE04OSBP_LJ01C_06_CTDBPO108_deploy.lcstate, 'PLANNED')
# Set CE04OSBP-LJ01C-06-CTDBPO108 device to DEVELOPED state
CE04OSBP_LJ01C_06_CTDBPO108_device = self.retrieve_ooi_asset(
'CE04OSBP-LJ01C-06-CTDBPO108_ID')
passing &= self.transition_lcs_then_verify(
resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id,
new_lcs_state=LCE.DEVELOP,
verify='DEVELOPED')
# Set CE04OSBP-LJ01C-06-CTDBPO108 device to INTEGRATED state
passing &= self.transition_lcs_then_verify(
resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id,
new_lcs_state=LCE.INTEGRATE,
verify='INTEGRATED')
# Set CE04OSBP-LJ01C-06-CTDBPO108 device to DEPLOYED state
passing &= self.transition_lcs_then_verify(
resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id,
new_lcs_state=LCE.DEPLOY,
verify='DEPLOYED')
# Set CE04OSBP-LJ01C-06-CTDBPO108 Deployment to DEPLOYED state
#self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_deploy._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED')
# Activate Deployment for CE04OSBP-LJ01C-06-CTDBPO108 instrument
log.debug(
'--------- activate_deployment CE04OSBP-LJ01C-06-CTDBPO108 deployment -------------- '
)
self.OMS.activate_deployment(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id)
passing &= self.validate_deployment_activated(
CE04OSBP_LJ01C_06_CTDBPO108_deploy._id)
# (optional) Add/register CE04OSBP-LJ01C-06-CTDBPO108 instrument agent to parent agent
# (optional) Start CE04OSBP-LJ01C-06-CTDBPO108 instrument agent with simulator
# Set all DataProducts for CE04OSBP-LJ01C-06-CTDBPO108 to DEPLOYED state
# (optional) Create a substitute Deployment for site CE04OSBP-LJ01C-06-CTDBPO108 with a comparable device
CE04OSBP_LJ01C_06_CTDBPO108_isite = self.retrieve_ooi_asset(
'CE04OSBP-LJ01C-06-CTDBPO108')
## create device here: retrieve CTD Mooring on Mooring Riser 001 - similiar?
GP03FLMB_RI001_10_CTDMOG999_ID_idevice = self.retrieve_ooi_asset(
'GP03FLMB-RI001-10-CTDMOG999_ID')
deploy_id_2 = self.create_basic_deployment(
name='CE04OSBP-LJ01C-06-CTDBPO108_DEP2',
description=
'substitute Deployment for site CE04OSBP-LJ01C-06-CTDBPO108 with a comparable device'
)
self.OMS.assign_device_to_deployment(
instrument_device_id=GP03FLMB_RI001_10_CTDMOG999_ID_idevice._id,
deployment_id=deploy_id_2)
self.OMS.assign_site_to_deployment(
instrument_site_id=CE04OSBP_LJ01C_06_CTDBPO108_isite._id,
deployment_id=deploy_id_2)
self.dump_deployment(deploy_id_2)
# (optional) Activate this second deployment - check first deployment is deactivated
self.OMS.deactivate_deployment(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id)
passing &= self.validate_deployment_deactivated(
CE04OSBP_LJ01C_06_CTDBPO108_deploy._id)
# log.debug('Activate deployment deploy_id_2')
# self.get_deployment_ids(deploy_id_2)
# self.dump_deployment(deploy_id_2, "deploy_id_2")
# self.OMS.activate_deployment(deploy_id_2)
# passing &= self.validate_deployment_deactivated(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id)
#
# # (optional) Set first CE04OSBP-LJ01C-06-CTDBPO108 Deployment to INTEGRATED state
# passing &= self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_deploy._id, new_lcs_state=LCE.INTEGRATE, verify='INTEGRATED')
#
# # Set first CE04OSBP-LJ01C-06-CTDBPO108 device to INTEGRATED state
# passing &= self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id, new_lcs_state=LCE.INTEGRATE, verify='INTEGRATED')
#
#
# # (optional) Create a third Deployment for site CE04OSBP-LJ01C-06-CTDBPO108 with a same device from first deployment
# deploy_id_3 = self.create_basic_deployment(name='CE04OSBP-LJ01C-06-CTDBPO108_DEP3', description='substitute Deployment for site CE04OSBP-LJ01C-06-CTDBPO108 with same device as first')
# self.IMS.deploy_instrument_device(instrument_device_id=GP03FLMB_RI001_10_CTDMOG999_ID_idevice._id, deployment_id=deploy_id_3)
# self.OMS.deploy_instrument_site(instrument_site_id=CE04OSBP_LJ01C_06_CTDBPO108_isite._id, deployment_id=deploy_id_3)
# self.dump_deployment(deploy_id_3)
#
#
# # Set first CE04OSBP-LJ01C-06-CTDBPO108 device to DEPLOYED state
# passing &= self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED')
#
# # (optional) Activate this third deployment - check second deployment is deactivated
# log.debug('Activate deployment deploy_id_3')
# self.dump_deployment(deploy_id_3)
# self.OMS.activate_deployment(deploy_id_3)
# #todo: check second deployment is deactivated
return passing
def check_data_product_reference(self, reference_designator, output=[]):
passing = True
data_product_ids, _ = self.RR.find_resources_ext(
alt_id_ns='PRE',
alt_id='%s_DPI1' % reference_designator,
id_only=True) # Assuming DPI1 is parsed
passing &= self.assertEquals(len(data_product_ids), 1)
if not data_product_ids:
return passing
# Let's go ahead and activate it
data_product_id = data_product_ids[0]
self.dpclient.activate_data_product_persistence(data_product_id)
self.addCleanup(self.dpclient.suspend_data_product_persistence,
data_product_id)
dataset_ids, _ = self.RR.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)
passing &= self.assertEquals(len(dataset_ids), 1)
if not dataset_ids:
return passing
dataset_id = dataset_ids[0]
stream_def_ids, _ = self.RR.find_objects(data_product_id,
PRED.hasStreamDefinition,
id_only=True)
passing &= self.assertEquals(len(dataset_ids), 1)
if not stream_def_ids:
return passing
stream_def_id = stream_def_ids[0]
output.append((data_product_id, stream_def_id, dataset_id))
return passing
def check_tempsf_instrument_data_product(self, reference_designator):
passing = True
info_list = []
passing &= self.check_data_product_reference(reference_designator,
info_list)
if not passing: return passing
data_product_id, stream_def_id, dataset_id = info_list.pop()
now = time.time()
ntp_now = now + 2208988800
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [ntp_now]
rdt['temperature'] = [[
25.3884, 26.9384, 24.3394, 23.3401, 22.9832, 29.4434, 26.9873,
15.2883, 16.3374, 14.5883, 15.7253, 18.4383, 15.3488, 17.2993,
10.2111, 11.5993, 10.9345, 9.4444, 9.9876, 10.9834, 11.0098,
5.3456, 4.2994, 4.3009
]]
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
passing &= self.assertTrue(dataset_monitor.wait())
if not passing: return passing
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
passing &= self.assert_array_almost_equal(rdt['time'], [ntp_now])
passing &= self.assert_array_almost_equal(rdt['temperature'], [[
25.3884, 26.9384, 24.3394, 23.3401, 22.9832, 29.4434, 26.9873,
15.2883, 16.3374, 14.5883, 15.7253, 18.4383, 15.3488, 17.2993,
10.2111, 11.5993, 10.9345, 9.4444, 9.9876, 10.9834, 11.0098,
5.3456, 4.2994, 4.3009
]])
return passing
def check_trhph_instrument_data_products(self, reference_designator):
passing = True
info_list = []
passing &= self.check_data_product_reference(reference_designator,
info_list)
if not passing:
return passing
data_product_id, stream_def_id, dataset_id = info_list.pop()
pdict = self.RR2.find_parameter_dictionary_of_stream_definition_using_has_parameter_dictionary(
stream_def_id)
passing &= self.assertEquals(pdict.name, 'trhph_sample')
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
# calibration constants
a = 1.98e-9
b = -2.45e-6
c = 9.28e-4
d = -0.0888
e = 0.731
V_s = 1.506
V_c = 0.
T = 11.8
r1 = 0.906
r2 = 4.095
r3 = 4.095
ORP_V = 1.806
Cl = np.nan
offset = 2008
gain = 4.0
# Normally this would be 50 per the DPS but the precision is %4.0f which truncates the values to the nearest 1...
ORP = ((ORP_V * 1000.) - offset) / gain
ntp_now = time.time() + 2208988800
rdt['cc_a'] = [a]
rdt['cc_b'] = [b]
rdt['cc_c'] = [c]
rdt['cc_d'] = [d]
rdt['cc_e'] = [e]
rdt['ref_temp_volts'] = [V_s]
rdt['resistivity_temp_volts'] = [V_c]
rdt['eh_sensor'] = [ORP_V]
rdt['resistivity_5'] = [r1]
rdt['resistivity_x1'] = [r2]
rdt['resistivity_x5'] = [r3]
rdt['cc_offset'] = [offset]
rdt['cc_gain'] = [gain]
rdt['time'] = [ntp_now]
passing &= self.assert_array_almost_equal(rdt['vent_fluid_temperaure'],
[T], 2)
passing &= self.assert_array_almost_equal(
rdt['vent_fluid_chloride_conc'], [Cl], 4)
passing &= self.assert_array_almost_equal(rdt['vent_fluid_orp'], [ORP],
4)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
passing &= self.assertTrue(dataset_monitor.wait())
if not passing: return passing
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
passing &= self.assert_array_almost_equal(rdt['vent_fluid_temperaure'],
[T], 2)
passing &= self.assert_array_almost_equal(
rdt['vent_fluid_chloride_conc'], [Cl], 4)
passing &= self.assert_array_almost_equal(rdt['vent_fluid_orp'], [ORP],
4)
return passing
def check_vel3d_instrument_data_products(self, reference_designator):
passing = True
info_list = []
passing &= self.check_data_product_reference(reference_designator,
info_list)
if not passing:
return passing
data_product_id, stream_def_id, dataset_id = info_list.pop()
pdict = self.RR2.find_parameter_dictionary_of_stream_definition_using_has_parameter_dictionary(
stream_def_id)
self.assertEquals(pdict.name, 'vel3d_b_sample')
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
lat = 14.6846
lon = -51.044
ts = np.array([
3319563600, 3319567200, 3319570800, 3319574400, 3319578000,
3319581600, 3319585200, 3319588800, 3319592400, 3319596000
],
dtype=np.float)
ve = np.array([-3.2, 0.1, 0., 2.3, -0.1, 5.6, 5.1, 5.8, 8.8, 10.3])
vn = np.array([18.2, 9.9, 12., 6.6, 7.4, 3.4, -2.6, 0.2, -1.5, 4.1])
vu = np.array([-1.1, -0.6, -1.4, -2, -1.7, -2, 1.3, -1.6, -1.1, -4.5])
ve_expected = np.array([
-0.085136, -0.028752, -0.036007, 0.002136, -0.023158, 0.043218,
0.056451, 0.054727, 0.088446, 0.085952
])
vn_expected = np.array([
0.164012, 0.094738, 0.114471, 0.06986, 0.07029, 0.049237,
-0.009499, 0.019311, 0.012096, 0.070017
])
vu_expected = np.array([
-0.011, -0.006, -0.014, -0.02, -0.017, -0.02, 0.013, -0.016,
-0.011, -0.045
])
rdt['time'] = ts
rdt['lat'] = [lat] * 10
rdt['lon'] = [lon] * 10
rdt['turbulent_velocity_east'] = ve
rdt['turbulent_velocity_north'] = vn
rdt['turbulent_velocity_up'] = vu
passing &= self.assert_array_almost_equal(
rdt['eastward_turbulent_velocity'], ve_expected)
passing &= self.assert_array_almost_equal(
rdt['northward_turbulent_velocity'], vn_expected)
passing &= self.assert_array_almost_equal(
rdt['upward_turbulent_velocity'], vu_expected)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
passing &= self.assertTrue(dataset_monitor.wait())
if not passing: return passing
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
passing &= self.assert_array_almost_equal(
rdt['eastward_turbulent_velocity'], ve_expected)
passing &= self.assert_array_almost_equal(
rdt['northward_turbulent_velocity'], vn_expected)
passing &= self.assert_array_almost_equal(
rdt['upward_turbulent_velocity'], vu_expected)
return passing
def check_presta_instrument_data_products(self, reference_designator):
# Check the parsed data product make sure it's got everything it needs and can be published persisted etc.
# Absolute Pressure (SFLPRES_L0) is what comes off the instrumnet, SFLPRES_L1 is a pfunc
# Let's go ahead and publish some fake data!!!
# According to https://alfresco.oceanobservatories.org/alfresco/d/d/workspace/SpacesStore/63e16865-9d9e-4b11-b0b3-d5658faa5080/1341-00230_Data_Product_Spec_SFLPRES_OOI.pdf
# Appendix A. Example 1.
# p_psia_tide = 14.8670
# the tide should be 10.2504
passing = True
info_list = []
passing &= self.check_data_product_reference(reference_designator,
info_list)
if not passing:
return passing
data_product_id, stream_def_id, dataset_id = info_list.pop()
now = time.time()
ntp_now = now + 2208988800.
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [ntp_now]
rdt['absolute_pressure'] = [14.8670]
passing &= self.assert_array_almost_equal(rdt['seafloor_pressure'],
[10.2504], 4)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(
dataset_monitor.wait()) # Bumped to 20 to keep buildbot happy
if not passing: return passing
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
passing &= self.assert_array_almost_equal(rdt['time'], [ntp_now])
passing &= self.assert_array_almost_equal(rdt['seafloor_pressure'],
[10.2504], 4)
passing &= self.assert_array_almost_equal(rdt['absolute_pressure'],
[14.8670], 4)
return passing
def check_rsn_instrument_data_product(self):
passing = True
# for RS03AXBS-MJ03A-06-PRESTA301 (PREST-A) there are a few listed data products
# Parsed, Engineering
# SFLPRES-0 SFLPRES-1
# Check for the two data products and make sure they have the proper parameters
# SFLPRES-0 should
data_products, _ = self.RR.find_resources_ext(
alt_id_ns='PRE',
alt_id='RS03AXBS-MJ03A-06-PRESTA301_SFLPRES_L0_DPID',
id_only=True)
passing &= self.assertTrue(len(data_products) == 1)
if not data_products:
return passing
data_product_id = data_products[0]
stream_defs, _ = self.RR.find_objects(data_product_id,
PRED.hasStreamDefinition,
id_only=False)
passing &= self.assertTrue(len(stream_defs) == 1)
if not stream_defs:
return passing
# Assert that the stream definition has the correct reference designator
stream_def = stream_defs[0]
passing &= self.assertEquals(
stream_def.stream_configuration['reference_designator'],
'RS03AXBS-MJ03A-06-PRESTA301')
# Get the pdict and make sure that the parameters corresponding to the available fields
# begin with the appropriate data product identifier
pdict_ids, _ = self.RR.find_objects(stream_def,
PRED.hasParameterDictionary,
id_only=True)
passing &= self.assertEquals(len(pdict_ids), 1)
if not pdict_ids:
return passing
pdict_id = pdict_ids[0]
pdict = DatasetManagementService.get_parameter_dictionary(pdict_id)
available_params = [
pdict.get_context(i) for i in pdict.keys()
if i in stream_def.available_fields
]
for p in available_params:
if p.name == 'time': # Ignore the domain parameter
continue
passing &= self.assertTrue(p.ooi_short_name.startswith('SFLPRES'))
passing &= self.check_presta_instrument_data_products(
'RS01SLBS-MJ01A-06-PRESTA101')
passing &= self.check_vel3d_instrument_data_products(
'RS01SLBS-MJ01A-12-VEL3DB101')
passing &= self.check_presta_instrument_data_products(
'RS03AXBS-MJ03A-06-PRESTA301')
passing &= self.check_vel3d_instrument_data_products(
'RS03AXBS-MJ03A-12-VEL3DB301')
passing &= self.check_tempsf_instrument_data_product(
'RS03ASHS-MJ03B-07-TMPSFA301')
passing &= self.check_vel3d_instrument_data_products(
'RS03INT2-MJ03D-12-VEL3DB304')
passing &= self.check_trhph_instrument_data_products(
'RS03INT1-MJ03C-10-TRHPHA301')
self.data_product_management.activate_data_product_persistence(
data_product_id)
dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(
data_product_id)
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
self.assert_array_almost_equal(rdt['seafloor_pressure'], [10.2504], 4)
self.assert_array_almost_equal(rdt['absolute_pressure'], [14.8670], 4)
self.data_product_management.suspend_data_product_persistence(
data_product_id) # Should do nothing and not raise anything
return passing
def check_glider(self):
'''
# Check that glider GP05MOAS-GL001 assembly is defined by OOI preload (3 instruments)
'''
passing = True
GP05MOAS_GL001_device = self.retrieve_ooi_asset('GP05MOAS-GL001_PD')
child_devs, assns = self.RR.find_objects(
subject=GP05MOAS_GL001_device._id,
predicate=PRED.hasDevice,
id_only=True)
passing &= self.assertEquals(len(child_devs), 3)
# Set GP05MOAS-GL001 Deployment to DEPLOYED
GP05MOAS_GL001_deploy = self.retrieve_ooi_asset('GP05MOAS-GL001_DEP')
passing &= self.transition_lcs_then_verify(
resource_id=GP05MOAS_GL001_deploy._id,
new_lcs_state=LCE.DEPLOY,
verify='DEPLOYED')
# Activate Deployment for GP05MOAS-GL001
#self.OMS.activate_deployment(GP05MOAS_GL001_deploy._id)
# Deactivate Deployment for GP05MOAS-GL001
#self.OMS.deactivate_deployment(GP05MOAS_GL001_deploy._id)
# Create a new Deployment resource X without any assignment
x_deploy_id = self.create_basic_deployment(
name='X_Deployment',
description='new Deployment resource X without any assignment')
# Assign Deployment X to site GP05MOAS-GL001
GP05MOAS_GL001_psite = self.retrieve_ooi_asset('GP05MOAS-GL001')
self.OMS.assign_site_to_deployment(GP05MOAS_GL001_psite._id,
x_deploy_id)
# Assign Deployment X to first device for GP05MOAS-GL001
GP05MOAS_GL001_device = self.retrieve_ooi_asset('GP05MOAS-GL001_PD')
self.OMS.assign_device_to_deployment(GP05MOAS_GL001_device._id,
x_deploy_id)
# Set GP05MOAS-GL001 Deployment to PLANNED state
#self.transition_lcs_then_verify(resource_id=x_deploy_id, new_lcs_state=LCE.PLAN, verify='PLANNED')
# ??? already in planned
# Set second GP05MOAS-GL001 Deployment to DEPLOYED
passing &= self.transition_lcs_then_verify(resource_id=x_deploy_id,
new_lcs_state=LCE.DEPLOY,
verify='DEPLOYED')
self.dump_deployment(x_deploy_id)
# Activate second Deployment for GP05MOAS-GL001
#self.OMS.activate_deployment(x_deploy_id)
# Deactivate second Deployment for GP05MOAS-GL001
#self.OMS.deactivate_deployment(x_deploy_id)
return passing
def check_cg_assembly(self):
passing = True
# Set several CE01ISSM-RI002-* instrument devices to DEVELOPED state
# Assemble several CE01ISSM-RI002-* instruments to a CG CE01ISSM-RI002 component platform
# Set several CE01ISSM-RI002-* instrument devices to INTEGRATED state
# Assemble CE01ISSM-RI002 platform to CG CE01ISSM-LM001 station platform
# Set CE01ISSM-RI002 component device to INTEGRATED state
# Set CE01ISSM-LM001 station device to INTEGRATED state
# Set CE01ISSM-LM001 station device to DEPLOYED state (children maybe too?)
# Set CE01ISSM-LM001 Deployment to DEPLOYED
# Activate CE01ISSM-LM001 platform assembly deployment
# Dectivate CE01ISSM-LM001 platform assembly deployment
# Set CE01ISSM-LM001 Deployment to INTEGRATED state
# Set CE01ISSM-LM001 station device to INTEGRATED state
# Set CE01ISSM-RI002 component device to INTEGRATED state
# Set CE01ISSM-RI002 component device to INTEGRATED state
# Disassemble CE01ISSM-RI002 platform from CG CE01ISSM-LM001 station platform
# Disassemble all CE01ISSM-RI002-* instruments from a CG CE01ISSM-RI002 component platform
# Retire instrument one for CE01ISSM-RI002-*
# Retire device one for CE01ISSM-RI002
# Retire device one for CE01ISSM-LM001
return passing
# -------------------------------------------------------------------------
def retrieve_ooi_asset(self, alt_id='', namespace='PRE'):
dp_list, _ = self.RR.find_resources_ext(alt_id_ns=namespace,
alt_id=alt_id)
self.assertEquals(len(dp_list), 1)
return dp_list[0]
def transition_lcs_then_verify(self, resource_id, new_lcs_state, verify):
ret = self.RR2.advance_lcs(resource_id, new_lcs_state)
resource_obj = self.RR.read(resource_id)
return self.assertEquals(resource_obj.lcstate, verify)
def create_basic_deployment(self, name='', description=''):
start = IonTime(datetime.datetime(2013, 1, 1))
end = IonTime(datetime.datetime(2014, 1, 1))
temporal_bounds = IonObject(OT.TemporalBounds,
name='planned',
start_datetime=start.to_string(),
end_datetime=end.to_string())
deployment_obj = IonObject(RT.Deployment,
name=name,
description=description,
context=IonObject(
OT.CabledNodeDeploymentContext),
constraint_list=[temporal_bounds])
return self.OMS.create_deployment(deployment_obj)
def validate_deployment_activated(self, deployment_id=''):
site_id, device_id = self.get_deployment_ids(deployment_id)
assocs = self.RR.find_associations(subject=site_id,
predicate=PRED.hasDevice,
object=device_id)
return self.assertEquals(len(assocs), 1)
def validate_deployment_deactivated(self, deployment_id=''):
site_id, device_id = self.get_deployment_ids(deployment_id)
assocs = self.RR.find_associations(subject=site_id,
predicate=PRED.hasDevice,
object=device_id)
return self.assertEquals(len(assocs), 0)
def dump_deployment(self, deployment_id='', name=""):
#site_id, device_id = self.get_deployment_ids(deployment_id)
resource_list, _ = self.RR.find_subjects(predicate=PRED.hasDeployment,
object=deployment_id,
id_only=True)
resource_list.append(deployment_id)
resources = self.RR.read_mult(resource_list)
log.debug('--------- dump_deployment %s summary---------------',
name)
for resource in resources:
log.debug('%s: %s (%s)', resource._get_type(), resource.name,
resource._id)
log.debug('--------- dump_deployment %s full dump ---------------',
name)
for resource in resources:
log.debug('resource: %s ', resource)
log.debug('--------- dump_deployment %s end ---------------', name)
#assocs = self.container.resource_registry.find_assoctiations(anyside=deployment_id)
# assocs = Container.instance.resource_registry.find_assoctiations(anyside=deployment_id)
# log.debug('--------- dump_deployment ---------------')
# for assoc in assocs:
# log.debug('SUBJECT: %s PREDICATE: %s OBJET: %s', assoc.s, assoc.p, assoc.o)
# log.debug('--------- dump_deployment end ---------------')
def get_deployment_ids(self, deployment_id=''):
devices = []
sites = []
idevice_list, _ = self.RR.find_subjects(RT.InstrumentDevice,
PRED.hasDeployment,
deployment_id,
id_only=True)
pdevice_list, _ = self.RR.find_subjects(RT.PlatformDevice,
PRED.hasDeployment,
deployment_id,
id_only=True)
devices = idevice_list + pdevice_list
self.assertEquals(1, len(devices))
isite_list, _ = self.RR.find_subjects(RT.InstrumentSite,
PRED.hasDeployment,
deployment_id,
id_only=True)
psite_list, _ = self.RR.find_subjects(RT.PlatformSite,
PRED.hasDeployment,
deployment_id,
id_only=True)
sites = isite_list + psite_list
self.assertEquals(1, len(sites))
return sites[0], devices[0]
def _find_resource_in_list(self,
res_list,
attr,
attr_val,
assert_found=True):
for res in res_list:
v = getattr(res, attr, None)
if v == attr_val:
return res
if assert_found:
self.assertTrue(
False,
"Attribute %s value %s not found in list" % (attr, attr_val))
return None
# -------------------------------------------------------------------------
def _get_caller(self):
s = inspect.stack()
return "%s:%s" % (s[2][1], s[2][2])
@assertion_wrapper
def assert_array_almost_equal(self, *args, **kwargs):
np.testing.assert_array_almost_equal(*args, **kwargs)
@assertion_wrapper
def assertEquals(self, *args, **kwargs):
IonIntegrationTestCase.assertEquals(self, *args, **kwargs)
@assertion_wrapper
def assertTrue(self, *args, **kwargs):
IonIntegrationTestCase.assertTrue(self, *args, **kwargs)
class TestDeployment(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dmpsclient = DataProductManagementServiceClient(
node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.psmsclient = PubsubManagementServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.c = DotDict()
self.c.resource_registry = self.rrclient
self.RR2 = EnhancedResourceRegistryClient(self.rrclient)
# create missing data process definition
self.dsmsclient = DataProcessManagementServiceClient(
node=self.container.node)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name=LOGICAL_TRANSFORM_DEFINITION_NAME,
description="normally in preload",
module='ion.processes.data.transforms.logical_transform',
class_name='logical_transform')
self.dsmsclient.create_data_process_definition(dpd_obj)
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(
RT.DataProcess, None, None, True)[0]:
self.dsmsclient.deactivate_data_process(proc_id)
self.dsmsclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
#@unittest.skip("targeting")
def test_create_deployment(self):
#create a deployment with metadata and an initial site and device
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site')
site_id = self.omsclient.create_platform_site(platform_site__obj)
platform_device__obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
device_id = self.imsclient.create_platform_device(platform_device__obj)
start = IonTime(datetime.datetime(2013, 1, 1))
end = IonTime(datetime.datetime(2014, 1, 1))
temporal_bounds = IonObject(OT.TemporalBounds,
name='planned',
start_datetime=start.to_string(),
end_datetime=end.to_string())
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment',
constraint_list=[temporal_bounds])
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.deploy_platform_site(site_id, deployment_id)
self.imsclient.deploy_platform_device(device_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ",
str(deployment_id))
#retrieve the deployment objects and check that the assoc site and device are attached
read_deployment_obj = self.omsclient.read_deployment(deployment_id)
log.debug("test_create_deployment: created deployment obj: %s ",
str(read_deployment_obj))
site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite,
PRED.hasDeployment,
deployment_id, True)
self.assertEqual(len(site_ids), 1)
device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice,
PRED.hasDeployment,
deployment_id, True)
self.assertEqual(len(device_ids), 1)
#delete the deployment
self.RR2.pluck(deployment_id)
self.omsclient.force_delete_deployment(deployment_id)
# now try to get the deleted dp object
try:
self.omsclient.read_deployment(deployment_id)
except NotFound:
pass
else:
self.fail("deleted deployment was found during read")
#@unittest.skip("targeting")
def base_activate_deployment(self):
#-------------------------------------------------------------------------------------
# Create platform site, platform device, platform model
#-------------------------------------------------------------------------------------
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site')
platform_site_id = self.omsclient.create_platform_site(
platform_site__obj)
platform_device_obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
platform_device_id = self.imsclient.create_platform_device(
platform_device_obj)
platform_model__obj = IonObject(RT.PlatformModel,
name='PlatformModel1',
description='test platform model')
platform_model_id = self.imsclient.create_platform_model(
platform_model__obj)
#-------------------------------------------------------------------------------------
# Create instrument site
#-------------------------------------------------------------------------------------
instrument_site_obj = IonObject(RT.InstrumentSite,
name='InstrumentSite1',
description='test instrument site')
instrument_site_id = self.omsclient.create_instrument_site(
instrument_site_obj, platform_site_id)
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.psmsclient.create_stream_definition(
name='SBE37_CDM', parameter_dictionary_id=pdict_id)
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='Log Data Product',
description='some new dp',
temporal_domain=tdom,
spatial_domain=sdom)
out_log_data_product_id = self.dmpsclient.create_data_product(
dp_obj, ctd_stream_def_id)
#----------------------------------------------------------------------------------------------------
# Start the transform (a logical transform) that acts as an instrument site
#----------------------------------------------------------------------------------------------------
self.omsclient.create_site_data_product(
site_id=instrument_site_id,
data_product_id=out_log_data_product_id)
#----------------------------------------------------------------------------------------------------
# Create an instrument device
#----------------------------------------------------------------------------------------------------
instrument_device_obj = IonObject(RT.InstrumentDevice,
name='InstrumentDevice1',
description='test instrument device')
instrument_device_id = self.imsclient.create_instrument_device(
instrument_device_obj)
self.rrclient.create_association(platform_device_id, PRED.hasDevice,
instrument_device_id)
dp_obj = IonObject(RT.DataProduct,
name='Instrument Data Product',
description='some new dp',
temporal_domain=tdom,
spatial_domain=sdom)
inst_data_product_id = self.dmpsclient.create_data_product(
dp_obj, ctd_stream_def_id)
#assign data products appropriately
self.damsclient.assign_data_product(
input_resource_id=instrument_device_id,
data_product_id=inst_data_product_id)
#----------------------------------------------------------------------------------------------------
# Create an instrument model
#----------------------------------------------------------------------------------------------------
instrument_model_obj = IonObject(RT.InstrumentModel,
name='InstrumentModel1',
description='test instrument model')
instrument_model_id = self.imsclient.create_instrument_model(
instrument_model_obj)
#----------------------------------------------------------------------------------------------------
# Create a deployment object
#----------------------------------------------------------------------------------------------------
start = IonTime(datetime.datetime(2013, 1, 1))
end = IonTime(datetime.datetime(2014, 1, 1))
temporal_bounds = IonObject(OT.TemporalBounds,
name='planned',
start_datetime=start.to_string(),
end_datetime=end.to_string())
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment',
constraint_list=[temporal_bounds])
deployment_id = self.omsclient.create_deployment(deployment_obj)
log.debug("test_create_deployment: created deployment id: %s ",
str(deployment_id))
ret = DotDict(instrument_site_id=instrument_site_id,
instrument_device_id=instrument_device_id,
instrument_model_id=instrument_model_id,
platform_site_id=platform_site_id,
platform_device_id=platform_device_id,
platform_model_id=platform_model_id,
deployment_id=deployment_id)
return ret
#@unittest.skip("targeting")
def test_activate_deployment_normal(self):
res = self.base_activate_deployment()
log.debug("assigning platform and instrument models")
self.imsclient.assign_platform_model_to_platform_device(
res.platform_model_id, res.platform_device_id)
self.imsclient.assign_instrument_model_to_instrument_device(
res.instrument_model_id, res.instrument_device_id)
self.omsclient.assign_platform_model_to_platform_site(
res.platform_model_id, res.platform_site_id)
self.omsclient.assign_instrument_model_to_instrument_site(
res.instrument_model_id, res.instrument_site_id)
log.debug("adding instrument site and device to deployment")
self.omsclient.deploy_instrument_site(res.instrument_site_id,
res.deployment_id)
self.imsclient.deploy_instrument_device(res.instrument_device_id,
res.deployment_id)
log.debug("adding platform site and device to deployment")
self.omsclient.deploy_platform_site(res.platform_site_id,
res.deployment_id)
self.imsclient.deploy_platform_device(res.platform_device_id,
res.deployment_id)
log.debug("activating deployment, expecting success")
self.omsclient.activate_deployment(res.deployment_id)
#@unittest.skip("targeting")
def test_activate_deployment_nomodels(self):
res = self.base_activate_deployment()
self.omsclient.deploy_instrument_site(res.instrument_site_id,
res.deployment_id)
self.imsclient.deploy_instrument_device(res.instrument_device_id,
res.deployment_id)
log.debug(
"activating deployment without site+device models, expecting fail")
self.assert_deploy_fail(
res.deployment_id, "Expected at least 1 model for InstrumentSite")
log.debug("assigning instrument site model")
self.omsclient.assign_instrument_model_to_instrument_site(
res.instrument_model_id, res.instrument_site_id)
log.debug(
"activating deployment without device models, expecting fail")
self.assert_deploy_fail(res.deployment_id,
"Expected 1 model for InstrumentDevice")
#@unittest.skip("targeting")
def test_activate_deployment_nosite(self):
res = self.base_activate_deployment()
log.debug("assigning instrument models")
self.imsclient.assign_instrument_model_to_instrument_device(
res.instrument_model_id, res.instrument_device_id)
self.omsclient.assign_instrument_model_to_instrument_site(
res.instrument_model_id, res.instrument_site_id)
log.debug("deploying instrument device only")
self.imsclient.deploy_instrument_device(res.instrument_device_id,
res.deployment_id)
log.debug(
"activating deployment without device models, expecting fail")
self.assert_deploy_fail(res.deployment_id,
"No sites were found in the deployment")
#@unittest.skip("targeting")
def test_activate_deployment_nodevice(self):
res = self.base_activate_deployment()
log.debug("assigning platform and instrument models")
self.imsclient.assign_instrument_model_to_instrument_device(
res.instrument_model_id, res.instrument_device_id)
self.omsclient.assign_instrument_model_to_instrument_site(
res.instrument_model_id, res.instrument_site_id)
log.debug("deploying instrument site only")
self.omsclient.deploy_instrument_site(res.instrument_site_id,
res.deployment_id)
log.debug(
"activating deployment without device models, expecting fail")
self.assert_deploy_fail(
res.deployment_id,
"The set of devices could not be mapped to the set of sites")
def assert_deploy_fail(self,
deployment_id,
fail_message="did not specify fail_message"):
with self.assertRaises(BadRequest) as cm:
self.omsclient.activate_deployment(deployment_id)
self.assertIn(fail_message, cm.exception.message)
class TestDataProductProvenance(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.dpmsclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
#@unittest.skip('not ready')
def test_get_provenance(self):
#create a deployment with metadata and an initial site and device
instrument_site_obj = IonObject(RT.InstrumentSite,
name='InstrumentSite1',
description='test instrument site')
instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, "")
log.debug( 'test_get_provenance: new instrument_site_id id = %s ', str(instrument_site_id))
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel",
stream_configuration= {'parsed': 'ctd_parsed_param_dict' } )
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" %ex)
log.debug( 'test_get_provenance: new InstrumentModel id = %s ', str(instModel_id))
self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrument_site_id)
# Create InstrumentAgent
instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver" )
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" %ex)
log.debug( 'test_get_provenance:new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_get_provenance: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
log.debug("test_get_provenance: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id)
#-------------------------------
# Create CTD Parsed data product
#-------------------------------
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
log.debug( 'test_get_provenance:Creating new CDM data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product = self.dpmsclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', ctd_parsed_data_product)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product)
#-------------------------------
# create a data product for the site to pass the OMS check.... we need to remove this check
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
log_data_product_id = self.dpmsclient.create_data_product(dp_obj, parsed_stream_def_id)
self.omsclient.create_site_data_product(instrument_site_id, log_data_product_id)
#-------------------------------
# Deploy instrument device to instrument site
#-------------------------------
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment')
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id)
self.imsclient.deploy_instrument_device(instDevice_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
self.omsclient.activate_deployment(deployment_id)
inst_device_objs, _ = self.rrclient.find_objects(subject=instrument_site_id, predicate=PRED.hasDevice, object_type=RT.InstrumetDevice, id_only=False)
log.debug("test_create_deployment: deployed device: %s ", str(inst_device_objs[0]) )
#-------------------------------
# Create the agent instance
#-------------------------------
port_agent_config = {
'device_addr': 'sbe37-simulator.oceanobservatories.org',
'device_port': 4001,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'command_port': 4003,
'data_port': 4000,
'log_level': 5,
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
#-------------------------------
# L1 Conductivity: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1ConductivityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_conductivity',
description='create the L1 conductivity data product',
module='ion.processes.data.transforms.ctd.ctd_L1_conductivity',
class_name='CTDL1ConductivityTransform')
try:
ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex)
#-------------------------------
# L1 Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1PressureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_pressure',
description='create the L1 pressure data product',
module='ion.processes.data.transforms.ctd.ctd_L1_pressure',
class_name='CTDL1PressureTransform')
try:
ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex)
#-------------------------------
# L1 Temperature: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1TemperatureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_temperature',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L1_temperature',
class_name='CTDL1TemperatureTransform')
try:
ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex)
#-------------------------------
# L2 Salinity: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition SalinityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_salinity',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new SalinityTransform data process definition: %s" %ex)
#-------------------------------
# L2 Density: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition DensityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_density',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_density',
class_name='DensityTransform')
try:
ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new DensityTransform data process definition: %s" %ex)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' )
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' )
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' )
self.output_products={}
log.debug("TestDataProductProvenance: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
log.debug("TestDataProductProvenance: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_pressure_output_dp_obj,
outgoing_stream_l0_pressure_id)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
log.debug("TestDataProductProvenance: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
#-------------------------------
# L1 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(name='L1_conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id, binding='conductivity' )
outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(name='L1_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id, binding='pressure' )
outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(name='L1_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id, binding='temperature' )
log.debug("TestDataProductProvenance: create output data product L1 conductivity")
ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct,
name='L1_Conductivity',
description='transform output L1 conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_conductivity_output_dp_obj,
outgoing_stream_l1_conductivity_id)
log.debug("TestDataProductProvenance: create output data product L1 pressure")
ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Pressure',
description='transform output L1 pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_pressure_output_dp_obj,
outgoing_stream_l1_pressure_id)
log.debug("TestDataProductProvenance: create output data product L1 temperature")
ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Temperature',
description='transform output L1 temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_temperature_output_dp_obj,
outgoing_stream_l1_temperature_id)
#-------------------------------
# L2 Salinity - Density: Output Data Products
#-------------------------------
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(name='L2_salinity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id, binding='salinity' )
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(name='L2_Density', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id, binding='density' )
log.debug("TestDataProductProvenance: create output data product L2 Salinity")
ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Salinity',
description='transform output L2 salinity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_salinity_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_salinity_output_dp_obj,
outgoing_stream_l2_salinity_id)
log.debug("TestDataProductProvenance: create output data product L2 Density")
# ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
# name='L2_Density',
# description='transform output pressure',
# temporal_domain = tdom,
# spatial_domain = sdom)
#
# ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj,
# outgoing_stream_l2_density_id,
# parameter_dictionary)
contactInfo = ContactInformation()
contactInfo.individual_names_given = "Bill"
contactInfo.individual_name_family = "Smith"
contactInfo.street_address = "111 First St"
contactInfo.city = "San Diego"
contactInfo.email = "*****@*****.**"
contactInfo.phones = ["858-555-6666"]
contactInfo.country = "USA"
contactInfo.postal_code = "92123"
ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Density',
description='transform output pressure',
contacts = [contactInfo],
iso_topic_category = "my_iso_topic_category_here",
quality_control_level = "1",
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj,
outgoing_stream_l2_density_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L0 all data_process start")
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products)
#activate only this data process just for coverage
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
contents = "this is the lookup table contents, replace with a file..."
att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
deviceAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att)
log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment)
log.debug("TestDataProductProvenance: create L0 all data_process return")
#-------------------------------
# L1 Conductivity: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1 Conductivity data_process start")
try:
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_conductivity_dprocdef_id, [ctd_l0_conductivity_output_dp_id], {'conductivity':ctd_l1_conductivity_output_dp_id})
self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L1 Pressure: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1_Pressure data_process start")
try:
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_pressure_dprocdef_id, [ctd_l0_pressure_output_dp_id], {'pressure':ctd_l1_pressure_output_dp_id})
self.dataprocessclient.activate_data_process(l1_pressure_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L1 Temperature: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1_Pressure data_process start")
try:
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_temperature_dprocdef_id, [ctd_l0_temperature_output_dp_id], {'temperature':ctd_l1_temperature_output_dp_id})
self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L2 Salinity: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L2_salinity data_process start")
try:
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_salinity_dprocdef_id, [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], {'salinity':ctd_l2_salinity_output_dp_id})
self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L2 Density: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L2_Density data_process start")
try:
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_density_dprocdef_id, [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], {'density':ctd_l2_density_output_dp_id})
self.dataprocessclient.activate_data_process(l2_density_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
print 'TestDataProductProvenance: Instrument agent instance obj: = ', inst_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
# self._ia_client = ResourceAgentClient('iaclient', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess())
# print 'activate_instrument: got ia client %s', self._ia_client
# log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client))
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.dataprocessclient.deactivate_data_process(l2_density_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l2_salinity_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_temperature_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_pressure_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_conductivity_data_process_id)
self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id)
#-------------------------------
# Retrieve the provenance info for the ctd density data product
#-------------------------------
provenance_dict = self.dpmsclient.get_data_product_provenance(ctd_l2_density_output_dp_id)
log.debug("TestDataProductProvenance: provenance_dict %s", str(provenance_dict))
#validate that products are represented
self.assertTrue (provenance_dict[str(ctd_l1_conductivity_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l0_conductivity_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l2_density_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l1_temperature_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l0_temperature_output_dp_id)])
density_dict = (provenance_dict[str(ctd_l2_density_output_dp_id)])
self.assertEquals(density_dict['producer'], [l2_density_all_data_process_id])
#-------------------------------
# Retrieve the extended resource for this data product
#-------------------------------
extended_product = self.dpmsclient.get_data_product_extension(ctd_l2_density_output_dp_id)
#self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.provenance_product_list.status)
log.debug("TestDataProductProvenance: DataProduct provenance_product_list %s", str(extended_product.provenance_product_list))
# log.debug("TestDataProductProvenance: DataProduct data_processes %s", str(extended_product.data_processes))
# log.debug("TestDataProductProvenance: DataProduct process_input_data_products %s", str(extended_product.process_input_data_products))
#log.debug("TestDataProductProvenance: provenance %s", str(extended_product.computed.provenance.value))
#-------------------------------
# Retrieve the extended resource for this data process
#-------------------------------
extended_process_def = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id)
# log.debug("TestDataProductProvenance: DataProcess extended_process_def %s", str(extended_process_def))
# log.debug("TestDataProductProvenance: DataProcess data_processes %s", str(extended_process_def.data_processes))
# log.debug("TestDataProductProvenance: DataProcess data_products %s", str(extended_process_def.data_products))
self.assertEqual(1, len(extended_process_def.data_processes) )
self.assertEqual(3, len(extended_process_def.output_stream_definitions) )
self.assertEqual(1, len(extended_process_def.data_products) ) #one list because of one data process
self.assertEqual(3, len(extended_process_def.data_products[0]) ) #inside that inner list are the three output data products
#-------------------------------
# Request the xml report
#-------------------------------
results = self.dpmsclient.get_data_product_provenance_report(ctd_l2_density_output_dp_id)
class TestDeployment(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dmpsclient = DataProductManagementServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.psmsclient = PubsubManagementServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.c = DotDict()
self.c.resource_registry = self.rrclient
self.RR2 = EnhancedResourceRegistryClient(self.rrclient)
# create missing data process definition
self.dsmsclient = DataProcessManagementServiceClient(node=self.container.node)
dpd_obj = IonObject(RT.DataProcessDefinition,
name=LOGICAL_TRANSFORM_DEFINITION_NAME,
description="normally in preload",
module='ion.processes.data.transforms.logical_transform',
class_name='logical_transform')
self.dsmsclient.create_data_process_definition(dpd_obj)
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]:
self.dsmsclient.deactivate_data_process(proc_id)
self.dsmsclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
#@unittest.skip("targeting")
def test_create_deployment(self):
#create a deployment with metadata and an initial site and device
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site')
site_id = self.omsclient.create_platform_site(platform_site__obj)
platform_device__obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
device_id = self.imsclient.create_platform_device(platform_device__obj)
start = IonTime(datetime.datetime(2013,1,1))
end = IonTime(datetime.datetime(2014,1,1))
temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string())
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment',
constraint_list=[temporal_bounds])
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.deploy_platform_site(site_id, deployment_id)
self.imsclient.deploy_platform_device(device_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
#retrieve the deployment objects and check that the assoc site and device are attached
read_deployment_obj = self.omsclient.read_deployment(deployment_id)
log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj) )
site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True)
self.assertEqual(len(site_ids), 1)
device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True)
self.assertEqual(len(device_ids), 1)
#delete the deployment
self.RR2.pluck(deployment_id)
self.omsclient.force_delete_deployment(deployment_id)
# now try to get the deleted dp object
try:
self.omsclient.read_deployment(deployment_id)
except NotFound:
pass
else:
self.fail("deleted deployment was found during read")
#@unittest.skip("targeting")
def base_activate_deployment(self):
#-------------------------------------------------------------------------------------
# Create platform site, platform device, platform model
#-------------------------------------------------------------------------------------
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site')
platform_site_id = self.omsclient.create_platform_site(platform_site__obj)
platform_device_obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
platform_device_id = self.imsclient.create_platform_device(platform_device_obj)
platform_model__obj = IonObject(RT.PlatformModel,
name='PlatformModel1',
description='test platform model')
platform_model_id = self.imsclient.create_platform_model(platform_model__obj)
#-------------------------------------------------------------------------------------
# Create instrument site
#-------------------------------------------------------------------------------------
instrument_site_obj = IonObject(RT.InstrumentSite,
name='InstrumentSite1',
description='test instrument site')
instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, platform_site_id)
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.psmsclient.create_stream_definition(name='SBE37_CDM', parameter_dictionary_id=pdict_id)
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='Log Data Product',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
out_log_data_product_id = self.dmpsclient.create_data_product(dp_obj, ctd_stream_def_id)
#----------------------------------------------------------------------------------------------------
# Start the transform (a logical transform) that acts as an instrument site
#----------------------------------------------------------------------------------------------------
self.omsclient.create_site_data_product( site_id= instrument_site_id,
data_product_id = out_log_data_product_id)
#----------------------------------------------------------------------------------------------------
# Create an instrument device
#----------------------------------------------------------------------------------------------------
instrument_device_obj = IonObject(RT.InstrumentDevice,
name='InstrumentDevice1',
description='test instrument device')
instrument_device_id = self.imsclient.create_instrument_device(instrument_device_obj)
self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id)
dp_obj = IonObject(RT.DataProduct,
name='Instrument Data Product',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
inst_data_product_id = self.dmpsclient.create_data_product(dp_obj, ctd_stream_def_id)
#assign data products appropriately
self.damsclient.assign_data_product(input_resource_id=instrument_device_id,
data_product_id=inst_data_product_id)
#----------------------------------------------------------------------------------------------------
# Create an instrument model
#----------------------------------------------------------------------------------------------------
instrument_model_obj = IonObject(RT.InstrumentModel,
name='InstrumentModel1',
description='test instrument model')
instrument_model_id = self.imsclient.create_instrument_model(instrument_model_obj)
#----------------------------------------------------------------------------------------------------
# Create a deployment object
#----------------------------------------------------------------------------------------------------
start = IonTime(datetime.datetime(2013,1,1))
end = IonTime(datetime.datetime(2014,1,1))
temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string())
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment',
constraint_list=[temporal_bounds])
deployment_id = self.omsclient.create_deployment(deployment_obj)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
ret = DotDict(instrument_site_id=instrument_site_id,
instrument_device_id=instrument_device_id,
instrument_model_id=instrument_model_id,
platform_site_id=platform_site_id,
platform_device_id=platform_device_id,
platform_model_id=platform_model_id,
deployment_id=deployment_id)
return ret
#@unittest.skip("targeting")
def test_activate_deployment_normal(self):
res = self.base_activate_deployment()
log.debug("assigning platform and instrument models")
self.imsclient.assign_platform_model_to_platform_device(res.platform_model_id, res.platform_device_id)
self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id)
self.omsclient.assign_platform_model_to_platform_site(res.platform_model_id, res.platform_site_id)
self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id)
log.debug("adding instrument site and device to deployment")
self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id)
self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id)
log.debug("adding platform site and device to deployment")
self.omsclient.deploy_platform_site(res.platform_site_id, res.deployment_id)
self.imsclient.deploy_platform_device(res.platform_device_id, res.deployment_id)
log.debug("activating deployment, expecting success")
self.omsclient.activate_deployment(res.deployment_id)
#@unittest.skip("targeting")
def test_activate_deployment_nomodels(self):
res = self.base_activate_deployment()
self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id)
self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id)
log.debug("activating deployment without site+device models, expecting fail")
self.assert_deploy_fail(res.deployment_id, "Expected at least 1 model for InstrumentSite")
log.debug("assigning instrument site model")
self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id)
log.debug("activating deployment without device models, expecting fail")
self.assert_deploy_fail(res.deployment_id, "Expected 1 model for InstrumentDevice")
#@unittest.skip("targeting")
def test_activate_deployment_nosite(self):
res = self.base_activate_deployment()
log.debug("assigning instrument models")
self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id)
self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id)
log.debug("deploying instrument device only")
self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id)
log.debug("activating deployment without device models, expecting fail")
self.assert_deploy_fail(res.deployment_id, "No sites were found in the deployment")
#@unittest.skip("targeting")
def test_activate_deployment_nodevice(self):
res = self.base_activate_deployment()
log.debug("assigning platform and instrument models")
self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id)
self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id)
log.debug("deploying instrument site only")
self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id)
log.debug("activating deployment without device models, expecting fail")
self.assert_deploy_fail(res.deployment_id, "The set of devices could not be mapped to the set of sites")
def assert_deploy_fail(self, deployment_id, fail_message="did not specify fail_message"):
with self.assertRaises(BadRequest) as cm:
self.omsclient.activate_deployment(deployment_id)
self.assertIn(fail_message, cm.exception.message)
class TestDeployment(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dmpsclient = DataProductManagementServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.psmsclient = PubsubManagementServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.c = DotDict()
self.c.resource_registry = self.rrclient
self.RR2 = EnhancedResourceRegistryClient(self.rrclient)
self.dsmsclient = DataProcessManagementServiceClient(node=self.container.node)
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]:
self.dsmsclient.deactivate_data_process(proc_id)
self.dsmsclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
#@unittest.skip("targeting")
def test_create_deployment(self):
#create a deployment with metadata and an initial site and device
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site')
site_id = self.omsclient.create_platform_site(platform_site__obj)
platform_device__obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
device_id = self.imsclient.create_platform_device(platform_device__obj)
start = str(int(time.mktime(datetime.datetime(2013, 1, 1).timetuple())))
end = str(int(time.mktime(datetime.datetime(2014, 1, 1).timetuple())))
temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start, end_datetime=end)
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment',
constraint_list=[temporal_bounds])
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.assign_site_to_deployment(site_id, deployment_id)
self.omsclient.assign_device_to_deployment(device_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
#retrieve the deployment objects and check that the assoc site and device are attached
read_deployment_obj = self.omsclient.read_deployment(deployment_id)
log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj) )
site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True)
self.assertEqual(len(site_ids), 1)
device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True)
self.assertEqual(len(device_ids), 1)
#delete the deployment
self.omsclient.force_delete_deployment(deployment_id)
# now try to get the deleted dp object
try:
self.omsclient.read_deployment(deployment_id)
except NotFound:
pass
else:
self.fail("deleted deployment was found during read")
#@unittest.skip("targeting")
def test_prepare_deployment_support(self):
deploy_sup = self.omsclient.prepare_deployment_support()
self.assertTrue(deploy_sup)
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].type_, "AssocDeploymentInstDevice")
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].associated_resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasPlatformDevice'].type_, "AssocDeploymentPlatDevice")
self.assertEquals(deploy_sup.associations['DeploymentHasPlatformDevice'].resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasPlatformDevice'].associated_resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].type_, "AssocDeploymentInstSite")
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].associated_resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasPlatformSite'].type_, "AssocDeploymentPlatSite")
self.assertEquals(deploy_sup.associations['DeploymentHasPlatformSite'].resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasPlatformSite'].associated_resources, [])
#create a deployment with metadata and an initial site and device
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site')
site_id = self.omsclient.create_platform_site(platform_site__obj)
platform_device__obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
device_id = self.imsclient.create_platform_device(platform_device__obj)
start = str(int(time.mktime(datetime.datetime(2013, 1, 1).timetuple())))
end = str(int(time.mktime(datetime.datetime(2014, 1, 1).timetuple())))
temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start, end_datetime=end)
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment',
constraint_list=[temporal_bounds])
deployment_id = self.omsclient.create_deployment(deployment_obj)
deploy_sup = self.omsclient.prepare_deployment_support(deployment_id)
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].associated_resources, [])
self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformDevice'].resources), 1)
self.assertEquals(deploy_sup.associations['DeploymentHasPlatformDevice'].associated_resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].associated_resources, [])
self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformSite'].resources), 1)
self.assertEquals(deploy_sup.associations['DeploymentHasPlatformSite'].associated_resources, [])
self.omsclient.assign_site_to_deployment(site_id, deployment_id)
self.omsclient.assign_device_to_deployment(device_id, deployment_id)
deploy_sup = self.omsclient.prepare_deployment_support(deployment_id)
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].associated_resources, [])
self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformDevice'].resources), 1)
self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformDevice'].associated_resources), 1)
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].associated_resources, [])
self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformSite'].resources), 1)
self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformSite'].associated_resources), 1)
#delete the deployment
self.omsclient.force_delete_deployment(deployment_id)
# now try to get the deleted dp object
try:
self.omsclient.read_deployment(deployment_id)
except NotFound:
pass
else:
self.fail("deleted deployment was found during read")
#@unittest.skip("targeting")
def base_activate_deployment(self, make_assigns=False):
# Create platform site, platform device, platform model
bounds = GeospatialBounds(geospatial_latitude_limit_north=float(5),
geospatial_latitude_limit_south=float(5),
geospatial_longitude_limit_west=float(15),
geospatial_longitude_limit_east=float(15),
geospatial_vertical_min=float(0),
geospatial_vertical_max=float(1000))
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site',
constraint_list=[bounds])
platform_site_id = self.omsclient.create_platform_site(platform_site__obj)
platform_device_obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
platform_device_id = self.imsclient.create_platform_device(platform_device_obj)
platform_model__obj = IonObject(RT.PlatformModel,
name='PlatformModel1',
description='test platform model')
platform_model_id = self.imsclient.create_platform_model(platform_model__obj)
# Create instrument site
#-------------------------------------------------------------------------------------
bounds = GeospatialBounds(geospatial_latitude_limit_north=float(45),
geospatial_latitude_limit_south=float(40),
geospatial_longitude_limit_west=float(-75),
geospatial_longitude_limit_east=float(-70),
geospatial_vertical_min=float(0),
geospatial_vertical_max=float(500))
instrument_site_obj = IonObject(RT.InstrumentSite,
name='InstrumentSite1',
description='test instrument site',
reference_designator='GA01SUMO-FI003-01-CTDMO0999',
constraint_list=[bounds])
instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, platform_site_id)
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.psmsclient.create_stream_definition(name='SBE37_CDM', parameter_dictionary_id=pdict_id)
# Create an instrument device
instrument_device_obj = IonObject(RT.InstrumentDevice,
name='InstrumentDevice1',
description='test instrument device')
instrument_device_id = self.imsclient.create_instrument_device(instrument_device_obj)
self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id)
pp_obj = IonObject(OT.PlatformPort, reference_designator='GA01SUMO-FI003-01-CTDMO0999', port_type= PortTypeEnum.PAYLOAD, ip_address='1' )
port_assignments = {instrument_device_id : pp_obj}
#----------------------------------------------------------------------------------------------------
# Create an instrument model
instrument_model_obj = IonObject(RT.InstrumentModel,
name='InstrumentModel1',
description='test instrument model')
instrument_model_id = self.imsclient.create_instrument_model(instrument_model_obj)
# Create a deployment object
#----------------------------------------------------------------------------------------------------
start = str(int(time.mktime(datetime.datetime(2013, 1, 1).timetuple())))
end = str(int(time.mktime(datetime.datetime(2020, 1, 1).timetuple())))
temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start, end_datetime=end)
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment',
context=IonObject(OT.CabledNodeDeploymentContext),
port_assignments=port_assignments,
constraint_list=[temporal_bounds])
deployment_id = self.omsclient.create_deployment(deployment_obj)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
if make_assigns:
self.imsclient.assign_platform_model_to_platform_device(platform_model_id, platform_device_id)
self.imsclient.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id)
self.omsclient.assign_platform_model_to_platform_site(platform_model_id, platform_site_id)
self.omsclient.assign_instrument_model_to_instrument_site(instrument_model_id, instrument_site_id)
self.omsclient.assign_site_to_deployment(platform_site_id, deployment_id)
self.omsclient.assign_device_to_deployment(platform_device_id, deployment_id)
ret = DotDict(instrument_site_id=instrument_site_id,
instrument_device_id=instrument_device_id,
instrument_model_id=instrument_model_id,
platform_site_id=platform_site_id,
platform_device_id=platform_device_id,
platform_model_id=platform_model_id,
deployment_id=deployment_id)
return ret
def _create_subsequent_deployment(self, prior_dep_info):
platform_device_obj = IonObject(RT.PlatformDevice,
name='PlatformDevice2',
description='test platform device')
platform_device_id = self.imsclient.create_platform_device(platform_device_obj)
instrument_device_obj = IonObject(RT.InstrumentDevice,
name='InstrumentDevice2',
description='test instrument device')
instrument_device_id = self.imsclient.create_instrument_device(instrument_device_obj)
self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id)
self.imsclient.assign_platform_model_to_platform_device(prior_dep_info.platform_model_id, platform_device_id)
self.imsclient.assign_instrument_model_to_instrument_device(prior_dep_info.instrument_model_id, instrument_device_id)
start = str(int(time.mktime(datetime.datetime(2013, 6, 1).timetuple())))
end = str(int(time.mktime(datetime.datetime(2020, 6, 1).timetuple())))
temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start, end_datetime=end)
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment2',
description='some new deployment',
context=IonObject(OT.CabledNodeDeploymentContext),
constraint_list=[temporal_bounds])
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.assign_site_to_deployment(prior_dep_info.platform_site_id, deployment_id)
self.omsclient.assign_device_to_deployment(prior_dep_info.platform_device_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
ret = DotDict(instrument_device_id=instrument_device_id,
platform_device_id=platform_device_id,
deployment_id=deployment_id)
return ret
#@unittest.skip("targeting")
def test_activate_deployment_normal(self):
res = self.base_activate_deployment(make_assigns=True)
before_activate_instrument_device_obj = self.rrclient.read(res.instrument_device_id)
self.assertNotEquals(before_activate_instrument_device_obj.lcstate, LCS.DEPLOYED)
log.debug("activating deployment, expecting success")
self.omsclient.activate_deployment(res.deployment_id)
# OOIION-1239: retrieve the extended resource and validate that only two sites are in the list of portals
extended_deployment = self.omsclient.get_deployment_extension(res.deployment_id)
self.assertEquals( len(extended_deployment.computed.portals.value), 2)
def assertGeospatialBoundsEquals(a, b):
self.assertEquals(a['geospatial_latitude_limit_north'],b['geospatial_latitude_limit_north'])
self.assertEquals(a['geospatial_latitude_limit_south'],b['geospatial_latitude_limit_south'])
self.assertEquals(a['geospatial_longitude_limit_west'],b['geospatial_longitude_limit_west'])
self.assertEquals(a['geospatial_longitude_limit_east'],b['geospatial_longitude_limit_east'])
def assertGeospatialBoundsNotEquals(a, b):
self.assertNotEquals(a['geospatial_latitude_limit_north'],b['geospatial_latitude_limit_north'])
self.assertNotEquals(a['geospatial_latitude_limit_south'],b['geospatial_latitude_limit_south'])
self.assertNotEquals(a['geospatial_longitude_limit_west'],b['geospatial_longitude_limit_west'])
self.assertNotEquals(a['geospatial_longitude_limit_east'],b['geospatial_longitude_limit_east'])
after_activate_instrument_device_obj = self.rrclient.read(res.instrument_device_id)
assertGeospatialBoundsNotEquals(before_activate_instrument_device_obj.geospatial_bounds,after_activate_instrument_device_obj.geospatial_bounds)
deployment_obj = self.RR2.read(res.deployment_id)
self.assertEquals(deployment_obj.lcstate, LCS.DEPLOYED)
extended_deployment = self.omsclient.get_deployment_extension(res.deployment_id)
# two sites in this test
self.assertEquals(len(extended_deployment.computed.portals.value), 2)
# only one portal instrument
self.assertEquals(len(extended_deployment.portal_instruments), 1)
log.debug("deactivatin deployment, expecting success")
self.omsclient.deactivate_deployment(res.deployment_id)
after_deactivate_instrument_device_obj = self.rrclient.read(res.instrument_device_id)
assertGeospatialBoundsNotEquals(after_activate_instrument_device_obj.geospatial_bounds, after_deactivate_instrument_device_obj.geospatial_bounds)
deployment_obj = self.RR2.read(res.deployment_id)
self.assertEquals(deployment_obj.lcstate, LCS.INTEGRATED)
def test_activate_deployment_redeploy(self):
dep_util = DeploymentUtil(self.container)
res = self.base_activate_deployment(make_assigns=True)
log.debug("activating first deployment, expecting success")
self.omsclient.activate_deployment(res.deployment_id)
deployment_obj1 = self.RR2.read(res.deployment_id)
self.assertEquals(deployment_obj1.lcstate, LCS.DEPLOYED)
next_dep_info = self._create_subsequent_deployment(res)
deployment_obj2 = self.RR2.read(next_dep_info.deployment_id)
self.assertNotEquals(deployment_obj2.lcstate, LCS.DEPLOYED)
log.debug("activating subsequent deployment, expecting success")
self.omsclient.activate_deployment(next_dep_info.deployment_id)
deployment_obj1 = self.RR2.read(res.deployment_id)
self.assertEquals(deployment_obj1.lcstate, LCS.INTEGRATED)
deployment_obj2 = self.RR2.read(next_dep_info.deployment_id)
self.assertEquals(deployment_obj2.lcstate, LCS.DEPLOYED)
dep1_tc = dep_util.get_temporal_constraint(deployment_obj1)
dep2_tc = dep_util.get_temporal_constraint(deployment_obj2)
self.assertLessEqual(float(dep1_tc.end_datetime), float(dep2_tc.end_datetime))
log.debug("deactivating second deployment, expecting success")
self.omsclient.deactivate_deployment(next_dep_info.deployment_id)
deployment_obj2 = self.RR2.read(next_dep_info.deployment_id)
self.assertEquals(deployment_obj2.lcstate, LCS.INTEGRATED)
#@unittest.skip("targeting")
def test_activate_deployment_nomodels(self):
res = self.base_activate_deployment()
self.omsclient.assign_site_to_deployment(res.platform_site_id, res.deployment_id)
self.omsclient.assign_device_to_deployment(res.platform_device_id, res.deployment_id)
log.debug("activating deployment without site+device models, expecting fail")
self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1")
log.debug("assigning instrument site model")
self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id)
log.debug("activating deployment without device models, expecting fail")
self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1")
#@unittest.skip("targeting")
def test_activate_deployment_nosite(self):
res = self.base_activate_deployment()
log.debug("assigning instrument models")
self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id)
self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id)
log.debug("deploying instrument device only")
self.omsclient.assign_device_to_deployment(res.instrument_device_id, res.deployment_id)
log.debug("activating deployment without instrument site, expecting fail")
self.assert_deploy_fail(res.deployment_id, BadRequest)
#@unittest.skip("targeting")
def test_activate_deployment_nodevice(self):
res = self.base_activate_deployment()
log.debug("assigning platform and instrument models")
self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id)
self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id)
log.debug("deploying instrument site only")
self.omsclient.assign_site_to_deployment(res.instrument_site_id, res.deployment_id)
log.debug("activating deployment without device, expecting fail")
self.assert_deploy_fail(res.deployment_id, BadRequest, "No devices were found in the deployment")
def assert_deploy_fail(self, deployment_id, err_type=BadRequest, fail_message=""):
with self.assertRaises(err_type) as cm:
self.omsclient.activate_deployment(deployment_id)
log.debug("assert_deploy_fail cm: %s", str(cm) )
if fail_message:
self.assertIn(fail_message, cm.exception.message)
def test_3x3_matchups_remoteplatform(self):
self.base_3x3_matchups(IonObject(OT.RemotePlatformDeploymentContext))
def test_3x3_matchups_cabledinstrument(self):
self.base_3x3_matchups(IonObject(OT.CabledInstrumentDeploymentContext))
def test_3x3_matchups_cablednode(self):
self.base_3x3_matchups(IonObject(OT.CabledNodeDeploymentContext))
def base_3x3_matchups(self, deployment_context):
"""
This will be 1 root platform, 3 sub platforms (2 of one model, 1 of another) and 3 sub instruments each (2-to-1)
"""
deployment_context_type = type(deployment_context).__name__
instrument_model_id = [self.RR2.create(any_old(RT.InstrumentModel)) for _ in range(6)]
platform_model_id = [self.RR2.create(any_old(RT.PlatformModel)) for _ in range(3)]
instrument_device_id = [self.RR2.create(any_old(RT.InstrumentDevice)) for _ in range(9)]
platform_device_id = [self.RR2.create(any_old(RT.PlatformDevice)) for _ in range(4)]
instrument_site_id = [self.RR2.create(any_old(RT.InstrumentSite,
{ "reference_designator" : "GA01SUMO-FI003-0%s-CTDMO0999" % (i+1),
"planned_uplink_port":
IonObject(OT.PlatformPort,
reference_designator="GA01SUMO-FI003-0%s-CTDMO0999" % (i+1) )}))
for i in range(9)]
platform_site_id = [self.RR2.create(any_old(RT.PlatformSite,
{ "reference_designator" : "GA01SUMO-FI003-0%s-CTDMO0888" % (i+1) ,
"planned_uplink_port":
IonObject(OT.PlatformPort,
reference_designator="GA01SUMO-FI003-0%s-CTDMO0888" % (i+1))}))
for i in range(4)]
def instrument_model_at(platform_idx, instrument_idx):
m = platform_idx * 2
if instrument_idx > 0:
m += 1
return m
def platform_model_at(platform_idx):
if platform_idx > 0:
return 1
return 0
def instrument_at(platform_idx, instrument_idx):
return platform_idx * 3 + instrument_idx
# set up the structure
for p in range(3):
m = platform_model_at(p)
self.RR2.assign_platform_model_to_platform_site_with_has_model(platform_model_id[m], platform_site_id[p])
self.RR2.assign_platform_model_to_platform_device_with_has_model(platform_model_id[m], platform_device_id[p])
self.RR2.assign_platform_device_to_platform_device_with_has_device(platform_device_id[p], platform_device_id[3])
self.RR2.assign_platform_site_to_platform_site_with_has_site(platform_site_id[p], platform_site_id[3])
for i in range(3):
m = instrument_model_at(p, i)
idx = instrument_at(p, i)
self.RR2.assign_instrument_model_to_instrument_site_with_has_model(instrument_model_id[m], instrument_site_id[idx])
self.RR2.assign_instrument_model_to_instrument_device_with_has_model(instrument_model_id[m], instrument_device_id[idx])
self.RR2.assign_instrument_device_to_platform_device_with_has_device(instrument_device_id[idx], platform_device_id[p])
self.RR2.assign_instrument_site_to_platform_site_with_has_site(instrument_site_id[idx], platform_site_id[p])
# top level models
self.RR2.assign_platform_model_to_platform_device_with_has_model(platform_model_id[2], platform_device_id[3])
self.RR2.assign_platform_model_to_platform_site_with_has_model(platform_model_id[2], platform_site_id[3])
# verify structure
for p in range(3):
parent_id = self.RR2.find_platform_device_id_by_platform_device_using_has_device(platform_device_id[p])
self.assertEqual(platform_device_id[3], parent_id)
parent_id = self.RR2.find_platform_site_id_by_platform_site_using_has_site(platform_site_id[p])
self.assertEqual(platform_site_id[3], parent_id)
for i in range(len(platform_site_id)):
self.assertEqual(self.RR2.find_platform_model_of_platform_device_using_has_model(platform_device_id[i]),
self.RR2.find_platform_model_of_platform_site_using_has_model(platform_site_id[i]))
for i in range(len(instrument_site_id)):
self.assertEqual(self.RR2.find_instrument_model_of_instrument_device_using_has_model(instrument_device_id[i]),
self.RR2.find_instrument_model_of_instrument_site_using_has_model(instrument_site_id[i]))
# OOIReferenceDesignator format: GA01SUMO-FI003-03-CTDMO0999 (site-platform_id-port-device_id)
port_assignments = {}
for p in range(3):
ref_desig = "GA01SUMO-FI003-0%s-CTDMO0888" % (p+1)
pp_obj = IonObject(OT.PlatformPort, reference_designator=ref_desig, port_type= PortTypeEnum.PAYLOAD, ip_address=str(p) )
port_assignments[platform_device_id[p]] = pp_obj
for i in range(3):
ref_desig = "GA01SUMO-FI003-0%s-CTDMO0999" % ((p*3)+i+1)
pp_obj = IonObject(OT.PlatformPort, reference_designator=ref_desig, port_type= PortTypeEnum.PAYLOAD, ip_address=str(p) )
idx = instrument_at(p, i)
port_assignments[instrument_device_id[idx]] = pp_obj
deployment_id = self.RR2.create(any_old(RT.Deployment,
{"context": deployment_context,
"port_assignments": port_assignments}))
log.debug("assigning device/site to %s deployment", deployment_context_type)
if OT.RemotePlatformDeploymentContext == deployment_context_type:
self.RR2.assign_deployment_to_platform_device_with_has_deployment(deployment_id, platform_device_id[3])
self.RR2.assign_deployment_to_platform_site_with_has_deployment(deployment_id, platform_site_id[3])
elif OT.CabledInstrumentDeploymentContext == deployment_context_type:
self.RR2.assign_deployment_to_instrument_device_with_has_deployment(deployment_id, instrument_device_id[1])
self.RR2.assign_deployment_to_instrument_site_with_has_deployment(deployment_id, instrument_site_id[1])
elif OT.CabledNodeDeploymentContext == deployment_context_type:
self.RR2.assign_deployment_to_platform_device_with_has_deployment(deployment_id, platform_device_id[1])
self.RR2.assign_deployment_to_platform_site_with_has_deployment(deployment_id, platform_site_id[1])
log.debug("activation of %s deployment", deployment_context_type)
self.omsclient.activate_deployment(deployment_id)
log.debug("validation of %s deployment", deployment_context_type)
if OT.RemotePlatformDeploymentContext == deployment_context_type:
# verify proper associations
for i, d in enumerate(platform_device_id):
self.assertEqual(d, self.RR2.find_platform_device_id_of_platform_site_using_has_device(platform_site_id[i]))
for i, d in enumerate(instrument_device_id):
self.assertEqual(d, self.RR2.find_instrument_device_id_of_instrument_site_using_has_device(instrument_site_id[i]))
elif OT.CabledInstrumentDeploymentContext == deployment_context_type:
self.assertEqual(instrument_device_id[1],
self.RR2.find_instrument_device_id_of_instrument_site_using_has_device(instrument_site_id[1]))
elif OT.CabledNodeDeploymentContext == deployment_context_type:
expected_platforms = [1]
# verify proper associations
for i, d in enumerate(platform_device_id):
self.assertEqual(i in expected_platforms,
d in self.RR2.find_platform_device_ids_of_platform_site_using_has_device(platform_site_id[i]))
class TestDeployment(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
#@unittest.skip("targeting")
def test_create_deployment(self):
#create a deployment with metadata and an initial site and device
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site')
site_id = self.omsclient.create_platform_site(platform_site__obj)
platform_device__obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
device_id = self.imsclient.create_platform_device(platform_device__obj)
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment')
deployment_id = self.omsclient.create_deployment(
deployment_obj, site_id, device_id)
log.debug("test_create_deployment: created deployment id: %s ",
str(deployment_id))
#retrieve the deployment objects and check that the assoc site and device are attached
read_deployment_obj = self.omsclient.read_deployment(deployment_id)
log.debug("test_create_deployment: created deployment obj: %s ",
str(read_deployment_obj))
site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite,
PRED.hasDeployment,
deployment_id, True)
self.assertEqual(len(site_ids), 1)
device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice,
PRED.hasDeployment,
deployment_id, True)
self.assertEqual(len(device_ids), 1)
#delete the deployment
self.omsclient.delete_deployment(deployment_id)
# now try to get the deleted dp object
try:
deployment_obj = self.omsclient.read_deployment(deployment_id)
except NotFound as ex:
pass
else:
self.fail("deleted deployment was found during read")
#@unittest.skip("targeting")
def test_activate_deployment(self):
#create a deployment with metadata and an initial site and device
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site')
site_id = self.omsclient.create_platform_site(platform_site__obj)
platform_device__obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
device_id = self.imsclient.create_platform_device(platform_device__obj)
platform_model__obj = IonObject(RT.PlatformModel,
name='PlatformModel1',
description='test platform model')
model_id = self.imsclient.create_platform_model(platform_model__obj)
self.imsclient.assign_platform_model_to_platform_device(
model_id, device_id)
self.omsclient.assign_platform_model_to_platform_site(
model_id, site_id)
#create a deployment with metadata and an initial site and device
instrument_site__obj = IonObject(RT.InstrumentSite,
name='InstrumentSite1',
description='test instrument site')
instrument_site_id = self.omsclient.create_instrument_site(
instrument_site__obj, site_id)
instrument_device__obj = IonObject(
RT.InstrumentDevice,
name='InstrumentDevice1',
description='test instrument device')
instrument_device_id = self.imsclient.create_instrument_device(
instrument_device__obj)
self.rrclient.create_association(device_id, PRED.hasDevice,
instrument_device_id)
instrument_model__obj = IonObject(RT.InstrumentModel,
name='InstrumentModel1',
description='test instrument model')
instrument_model_id = self.imsclient.create_instrument_model(
instrument_model__obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instrument_model_id, instrument_device_id)
self.omsclient.assign_instrument_model_to_instrument_site(
instrument_model_id, instrument_site_id)
#self.rrclient.create_association(instrument_site_id, PRED.hasModel, instrument_model_id)
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment')
deployment_id = self.omsclient.create_deployment(
deployment_obj, site_id, device_id)
log.debug("test_create_deployment: created deployment id: %s ",
str(deployment_id))
self.omsclient.activate_deployment(deployment_id)
class TestObservatoryManagementFullIntegration(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
self.OMS = ObservatoryManagementServiceClient(node=self.container.node)
self.org_management_service = OrgManagementServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
def assert_can_load(self, scenarios, loadui=False, loadooi=False, path=TESTED_DOC, ui_path='default'):
""" perform preload for given scenarios and raise exception if there is a problem with the data """
config = dict(op="load",
scenario=scenarios,
attachments="res/preload/r2_ioc/attachments",
loadui=loadui,
loadooi=loadooi,
path=path, ui_path=ui_path,
assets='res/preload/r2_ioc/ooi_assets',
bulk=loadooi,
debug=True,
ooiexclude='DataProduct,DataProductLink',
)
self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=config)
# 'DataProduct,DataProductLink,WorkflowDefinition,ExternalDataProvider,ExternalDatasetModel,ExternalDataset,ExternalDatasetAgent,ExternalDatasetAgentInstance',
def load_summer_deploy_assets(self):
""" make sure UI assets are valid using DEFAULT_UI_ASSETS = 'https://userexperience.oceanobservatories.org/database-exports/' """
ui_path = 'https://userexperience.oceanobservatories.org/database-exports/Candidates'
self.assert_can_load(scenarios="X", loadui=True, loadooi=True, ui_path=ui_path)
@unittest.skip('under construction.')
def test_observatory(self):
# Perform OOI preload for summer deployments (production mode, no debug, no bulk)
self.load_summer_deploy_assets()
#test an asset
res_list, _ = self.RR.find_resources_ext(alt_id_ns="OOI", alt_id="CE04OSBP-LJ01C-06-CTDBPO108")
log.debug('test_observatory retrieve test: %s', res_list)
# Check OOI preloaded resources to see if they match needs for this test and for correctness
observatory_list, _ = self.RR.find_resources_ext(restype=RT.Observatory)
self.assertEquals(42, len(observatory_list))
platform_site_list, _ = self.RR.find_resources(RT.PlatformSite, None, None, False)
for ps in platform_site_list:
log.debug('platform site: %s', ps.name)
self.assertEquals(38, len(platform_site_list))
platform_device_list, _ = self.RR.find_resources(RT.PlatformDevice, None, None, False)
for pd in platform_device_list:
log.debug('platform device: %s', pd.name)
self.assertEquals(38, len(platform_site_list))
platform_agent_list, _ = self.RR.find_resources(RT.PlatformAgent, None, None, False)
self.assertEquals(2, len(platform_agent_list))
for pa in platform_agent_list:
log.debug('platform agent: %s', pa.name)
deployment_list, _ = self.RR.find_resources(RT.Deployment, None, None, False)
self.assertEquals(62, len(deployment_list))
for d in deployment_list:
log.debug('deployment: %s', d.name)
# Check lcstates for select OOI resources: Some PLANNED, some INTEGRATED, some DEPLOYED
for obs in observatory_list:
self.assertEquals(obs.lcstate, 'DRAFT')
for pdev in platform_device_list:
self.assertEquals(pdev.lcstate, 'PLANNED')
for pagent in platform_agent_list:
self.assertEquals(pagent.lcstate, 'DEPLOYED')
# See if Deployment for primary nodes is already active and in DEPLOYED lcstate, in particular CE04OSHY-PN01C
for deploy in deployment_list:
self.assertEquals(deploy.lcstate, 'PLANNED')
dp_list, _ = self.RR.find_resources_ext(alt_id_ns="PRE", alt_id="CE04OSHY-PN01C_DEP")
self.assertEquals(len(dp_list), 1)
self.assertEquals(dp_list[0].availability, 'AVAILABLE')
log.debug('test_observatory retrieve CE04OSHY-PN01C_DEP deployment: %s', res_list[0])
# Check existing RSN node CE04OSHY-LV01C Deployment (PLANNED lcstate)
CE04OSHY_LV01C_deployment = self.retrieve_ooi_asset(namespace='PRE', alt_id='CE04OSHY-LV01C_DEP')
#self.dump_deployment(CE04OSHY_LV01C_deployment._id)
self.assertEquals(CE04OSHY_LV01C_deployment.lcstate, 'PLANNED')
log.debug('test_observatory retrieve RSN node CE04OSHY-LV01C Deployment: %s', CE04OSHY_LV01C_deployment)
# Set CE04OSHY-LV01C device to DEVELOPED state
# MATURITY = ['DRAFT', 'PLANNED', 'DEVELOPED', 'INTEGRATED', 'DEPLOYED', 'RETIRED']
CE04OSHY_LV01C_device = self.retrieve_ooi_asset(namespace='PRE', alt_id='CE04OSHY-LV01C_PD')
#ret = self.RR.execute_lifecycle_transition(resource_id=CE04OSHY_LV01C_device._id, transition_event=LCE.PLANNED)
self.transition_lcs_then_verify(resource_id=CE04OSHY_LV01C_device._id, new_lcs_state=LCE.DEVELOP, verify='DEVELOPED')
# Set CE04OSHY-LV01C device to INTEGRATED state
self.transition_lcs_then_verify(resource_id=CE04OSHY_LV01C_device._id, new_lcs_state=LCE.INTEGRATE, verify='INTEGRATED')
# Set CE04OSHY-LV01C device to DEPLOYED state
self.transition_lcs_then_verify(resource_id=CE04OSHY_LV01C_device._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED')
# Set CE04OSHY-LV01C Deployment to DEPLOYED state
self.transition_lcs_then_verify(resource_id=CE04OSHY_LV01C_deployment._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED')
# Activate Deployment for CE04OSHY-LV01C
self.OMS.activate_deployment(CE04OSHY_LV01C_deployment._id)
log.debug('--------- activate_deployment CE04OSHY_LV01C_deployment -------------- ')
self.dump_deployment(CE04OSHY_LV01C_deployment._id)
self.validate_deployment_activated(CE04OSHY_LV01C_deployment._id)
# (optional) Start CE04OSHY-LV01C platform agent with simulator
# Set DataProduct for CE04OSHY-LV01C platform to DEPLOYED state
output_data_product_ids, assns =self.RR.find_objects(subject=CE04OSHY_LV01C_device._id, predicate=PRED.hasOutputProduct, id_only=True)
if output_data_product_ids:
#self.assertEquals(len(child_devs), 3)
for output_data_product_id in output_data_product_ids:
log.debug('DataProduct for CE04OSHY-LV01C platform: %s', output_data_product_id)
self.transition_lcs_then_verify(resource_id=output_data_product_id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED')
# Check events for CE04OSHY-LV01C platform
# Check existing CE04OSBP-LJ01C Deployment (PLANNED lcstate)
# dp_list, _ = self.RR.find_resources_ext(alt_id_ns="PRE", alt_id="CE04OSBP-LJ01C_DEP")
# self.assertEquals(len(dp_list), 1)
# CE04OSHY_LV01C_deployment = dp_list[0]
# self.assertEquals(CE04OSHY_LV01C_deployment.lcstate, 'PLANNED')
# log.debug('test_observatory retrieve RSN node CE04OSBP-LJ01C Deployment: %s', CE04OSHY_LV01C_deployment)
# Set CE04OSBP-LJ01C Deployment to DEPLOYED state
# Update description and other attributes for CE04OSBP-LJ01C device resource
# Create attachment (JPG image) for CE04OSBP-LJ01C device resource
# Activate Deployment for CE04OSBP-LJ01C
# (optional) Add/register CE04OSBP-LJ01C platform agent to parent agent
# (optional) Start CE04OSBP-LJ01C platform agent
log.debug('--------- ------------------------------------------------------------------------------------------------------------ -------------- ')
# Check existing RSN instrument CE04OSBP-LJ01C-06-CTDBPO108 Deployment (PLANNED lcstate)
CE04OSBP_LJ01C_06_CTDBPO108_deploy = self.retrieve_ooi_asset(namespace='PRE', alt_id='CE04OSBP-LJ01C-06-CTDBPO108_DEP')
self.dump_deployment(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id)
self.assertEquals(CE04OSBP_LJ01C_06_CTDBPO108_deploy.lcstate, 'PLANNED')
# Set CE04OSBP-LJ01C-06-CTDBPO108 device to DEVELOPED state
CE04OSBP_LJ01C_06_CTDBPO108_device = self.retrieve_ooi_asset(namespace='PRE', alt_id='CE04OSBP-LJ01C-06-CTDBPO108_ID')
self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id, new_lcs_state=LCE.DEVELOP, verify='DEVELOPED')
# Set CE04OSBP-LJ01C-06-CTDBPO108 device to INTEGRATED state
self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id, new_lcs_state=LCE.INTEGRATE, verify='INTEGRATED')
# Set CE04OSBP-LJ01C-06-CTDBPO108 device to DEPLOYED state
self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED')
# Set CE04OSBP-LJ01C-06-CTDBPO108 Deployment to DEPLOYED state
self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_deploy._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED')
# Activate Deployment for CE04OSBP-LJ01C-06-CTDBPO108 instrument
log.debug('--------- activate_deployment CE04OSBP-LJ01C-06-CTDBPO108 deployment -------------- ')
self.OMS.activate_deployment(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id)
self.validate_deployment_activated(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id)
# (optional) Add/register CE04OSBP-LJ01C-06-CTDBPO108 instrument agent to parent agent
# (optional) Start CE04OSBP-LJ01C-06-CTDBPO108 instrument agent with simulator
# Set all DataProducts for CE04OSBP-LJ01C-06-CTDBPO108 to DEPLOYED state
# (optional) Create a substitute Deployment for site CE04OSBP-LJ01C-06-CTDBPO108 with a comparable device
CE04OSBP_LJ01C_06_CTDBPO108_isite = self.retrieve_ooi_asset(namespace='PRE', alt_id='CE04OSBP-LJ01C-06-CTDBPO108')
## create device here: retrieve CTD Mooring on Mooring Riser 001 - similiar?
GP03FLMB_RI001_10_CTDMOG999_ID_idevice = self.retrieve_ooi_asset(namespace='PRE', alt_id='GP03FLMB-RI001-10-CTDMOG999_ID')
deploy_id_2 = self.create_basic_deployment(name='CE04OSBP-LJ01C-06-CTDBPO108_DEP2', description='substitute Deployment for site CE04OSBP-LJ01C-06-CTDBPO108 with a comparable device')
self.IMS.deploy_instrument_device(instrument_device_id=GP03FLMB_RI001_10_CTDMOG999_ID_idevice._id, deployment_id=deploy_id_2)
self.OMS.deploy_instrument_site(instrument_site_id=CE04OSBP_LJ01C_06_CTDBPO108_isite._id, deployment_id=deploy_id_2)
self.dump_deployment(deploy_id_2)
# (optional) Activate this second deployment - check first deployment is deactivated
self.OMS.deactivate_deployment(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id)
self.validate_deployment_deactivated(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id)
log.debug('Activate deployment deploy_id_2')
self.get_deployment_ids(deploy_id_2)
self.dump_deployment(deploy_id_2, "deploy_id_2")
self.OMS.activate_deployment(deploy_id_2)
self.validate_deployment_deactivated(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id)
# (optional) Set first CE04OSBP-LJ01C-06-CTDBPO108 Deployment to INTEGRATED state
self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_deploy._id, new_lcs_state=LCE.INTEGRATE, verify='INTEGRATED')
# Set first CE04OSBP-LJ01C-06-CTDBPO108 device to INTEGRATED state
self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id, new_lcs_state=LCE.INTEGRATE, verify='INTEGRATED')
# (optional) Create a third Deployment for site CE04OSBP-LJ01C-06-CTDBPO108 with a same device from first deployment
deploy_id_3 = self.create_basic_deployment(name='CE04OSBP-LJ01C-06-CTDBPO108_DEP3', description='substitute Deployment for site CE04OSBP-LJ01C-06-CTDBPO108 with same device as first')
self.IMS.deploy_instrument_device(instrument_device_id=GP03FLMB_RI001_10_CTDMOG999_ID_idevice._id, deployment_id=deploy_id_3)
self.OMS.deploy_instrument_site(instrument_site_id=CE04OSBP_LJ01C_06_CTDBPO108_isite._id, deployment_id=deploy_id_3)
self.dump_deployment(deploy_id_3)
# Set first CE04OSBP-LJ01C-06-CTDBPO108 device to DEPLOYED state
self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED')
# (optional) Activate this third deployment - check second deployment is deactivated
log.debug('Activate deployment deploy_id_3')
self.dump_deployment(deploy_id_3)
self.OMS.activate_deployment(deploy_id_3)
#todo: check second deployment is deactivated
# Check that glider GP05MOAS-GL001 assembly is defined by OOI preload (3 instruments)
GP05MOAS_GL001_device = self.retrieve_ooi_asset(namespace='PRE', alt_id='GP05MOAS-GL001_PD')
child_devs, assns =self.RR.find_objects(subject=GP05MOAS_GL001_device._id, predicate=PRED.hasDevice, id_only=True)
self.assertEquals(len(child_devs), 3)
# Set GP05MOAS-GL001 Deployment to DEPLOYED
GP05MOAS_GL001_deploy = self.retrieve_ooi_asset(namespace='PRE', alt_id='GP05MOAS-GL001_DEP')
self.transition_lcs_then_verify(resource_id=GP05MOAS_GL001_deploy._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED')
# Activate Deployment for GP05MOAS-GL001
#self.OMS.activate_deployment(GP05MOAS_GL001_deploy._id)
# Deactivate Deployment for GP05MOAS-GL001
#self.OMS.deactivate_deployment(GP05MOAS_GL001_deploy._id)
# Create a new Deployment resource X without any assignment
x_deploy_id = self.create_basic_deployment(name='X_Deployment', description='new Deployment resource X without any assignment')
# Assign Deployment X to site GP05MOAS-GL001
GP05MOAS_GL001_psite = self.retrieve_ooi_asset(namespace='PRE', alt_id='GP05MOAS-GL001')
self.OMS.deploy_platform_site(GP05MOAS_GL001_psite._id, x_deploy_id)
# Assign Deployment X to first device for GP05MOAS-GL001
GP05MOAS_GL001_device = self.retrieve_ooi_asset(namespace='PRE', alt_id='GP05MOAS-GL001_PD')
self.IMS.deploy_platform_device(GP05MOAS_GL001_device._id, x_deploy_id)
# Set GP05MOAS-GL001 Deployment to PLANNED state
#self.transition_lcs_then_verify(resource_id=x_deploy_id, new_lcs_state=LCE.PLAN, verify='PLANNED')
# ??? already in planned
# Set second GP05MOAS-GL001 Deployment to DEPLOYED
self.transition_lcs_then_verify(resource_id=x_deploy_id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED')
self.dump_deployment(x_deploy_id)
# Activate second Deployment for GP05MOAS-GL001
#self.OMS.activate_deployment(x_deploy_id)
# Deactivate second Deployment for GP05MOAS-GL001
#self.OMS.deactivate_deployment(x_deploy_id)
# Set several CE01ISSM-RI002-* instrument devices to DEVELOPED state
# Assemble several CE01ISSM-RI002-* instruments to a CG CE01ISSM-RI002 component platform
# Set several CE01ISSM-RI002-* instrument devices to INTEGRATED state
# Assemble CE01ISSM-RI002 platform to CG CE01ISSM-LM001 station platform
# Set CE01ISSM-RI002 component device to INTEGRATED state
# Set CE01ISSM-LM001 station device to INTEGRATED state
# Set CE01ISSM-LM001 station device to DEPLOYED state (children maybe too?)
# Set CE01ISSM-LM001 Deployment to DEPLOYED
# Activate CE01ISSM-LM001 platform assembly deployment
# Dectivate CE01ISSM-LM001 platform assembly deployment
# Set CE01ISSM-LM001 Deployment to INTEGRATED state
# Set CE01ISSM-LM001 station device to INTEGRATED state
# Set CE01ISSM-RI002 component device to INTEGRATED state
# Set CE01ISSM-RI002 component device to INTEGRATED state
# Disassemble CE01ISSM-RI002 platform from CG CE01ISSM-LM001 station platform
# Disassemble all CE01ISSM-RI002-* instruments from a CG CE01ISSM-RI002 component platform
# Retire instrument one for CE01ISSM-RI002-*
# Retire device one for CE01ISSM-RI002
# Retire device one for CE01ISSM-LM001
# Add a new instrument agent
# Add a new instrument agent instance
# Check DataProducts
# Check provenance
pass
def retrieve_ooi_asset(self, namespace='', alt_id=''):
dp_list, _ = self.RR.find_resources_ext(alt_id_ns=namespace, alt_id=alt_id)
self.assertEquals(len(dp_list), 1)
return dp_list[0]
def transition_lcs_then_verify(self, resource_id, new_lcs_state, verify):
ret = self.RR2.advance_lcs(resource_id, new_lcs_state)
resource_obj = self.RR.read(resource_id)
self.assertEquals(resource_obj.lcstate, verify)
def create_basic_deployment(self, name='', description=''):
start = IonTime(datetime.datetime(2013,1,1))
end = IonTime(datetime.datetime(2014,1,1))
temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string())
deployment_obj = IonObject(RT.Deployment,
name=name,
description=description,
context=IonObject(OT.CabledNodeDeploymentContext),
constraint_list=[temporal_bounds])
return self.OMS.create_deployment(deployment_obj)
def validate_deployment_activated(self, deployment_id=''):
site_id, device_id = self.get_deployment_ids(deployment_id)
assocs = self.RR.find_associations(subject=site_id, predicate=PRED.hasDevice, object=device_id)
self.assertEquals(len(assocs), 1)
def validate_deployment_deactivated(self, deployment_id=''):
site_id, device_id = self.get_deployment_ids(deployment_id)
assocs = self.RR.find_associations(subject=site_id, predicate=PRED.hasDevice, object=device_id)
self.assertEquals(len(assocs), 0)
def dump_deployment(self, deployment_id='', name=""):
#site_id, device_id = self.get_deployment_ids(deployment_id)
resource_list,_ = self.RR.find_subjects(predicate=PRED.hasDeployment, object=deployment_id, id_only=True)
resource_list.append(deployment_id)
resources = self.RR.read_mult(resource_list )
log.debug('--------- dump_deployment %s summary---------------', name)
for resource in resources:
log.debug('%s: %s (%s)', resource._get_type(), resource.name, resource._id)
log.debug('--------- dump_deployment %s full dump ---------------', name)
for resource in resources:
log.debug('resource: %s ', resource)
log.debug('--------- dump_deployment %s end ---------------', name)
#assocs = self.container.resource_registry.find_assoctiations(anyside=deployment_id)
# assocs = Container.instance.resource_registry.find_assoctiations(anyside=deployment_id)
# log.debug('--------- dump_deployment ---------------')
# for assoc in assocs:
# log.debug('SUBJECT: %s PREDICATE: %s OBJET: %s', assoc.s, assoc.p, assoc.o)
# log.debug('--------- dump_deployment end ---------------')
def get_deployment_ids(self, deployment_id=''):
devices = []
sites = []
idevice_list,_ = self.RR.find_subjects(RT.InstrumentDevice, PRED.hasDeployment, deployment_id, id_only=True)
pdevice_list,_ = self.RR.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, id_only=True)
devices = idevice_list + pdevice_list
self.assertEquals(1, len(devices))
isite_list,_ = self.RR.find_subjects(RT.InstrumentSite, PRED.hasDeployment, deployment_id, id_only=True)
psite_list,_ = self.RR.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, id_only=True)
sites = isite_list + psite_list
self.assertEquals(1, len(sites))
return sites[0], devices[0]
class TestIMSDeployAsPrimaryDevice(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
#self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
print 'started services'
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
def cleanupprocs(self):
stm = os.popen('ps -e | grep ion.agents.port.logger_process')
procs = stm.read()
if len(procs) > 0:
procs = procs.split()
if procs[0].isdigit():
pid = int(procs[0])
os.kill(pid,signal.SIGKILL)
stm = os.popen('ps -e | grep ion.agents.instrument.zmq_driver_process')
procs = stm.read()
if len(procs) > 0:
procs = procs.split()
if procs[0].isdigit():
pid = int(procs[0])
os.kill(pid,signal.SIGKILL)
# stm = os.popen('rm /tmp/*.pid.txt')
@unittest.skip ("Deprecated by IngestionManagement refactor, timeout on start inst agent?")
def test_deploy_activate_full(self):
# ensure no processes or pids are left around by agents or Sims
self.cleanupprocs()
# Set up the preconditions
# Set up the preconditions
# ingestion configuration parameters
exchange_point_id = 'science_data'
exchange_name = 'ingestion_queue'
#-------------------------------
# Create ingestion configuration and activate it
#-------------------------------
q = IngestionQueue()
ingestion_configuration_id = self.ingestclient.create_ingestion_configuration(
name=exchange_name,
exchange_point_id=exchange_point_id,
queues=[q])
print 'test_deployAsPrimaryDevice: ingestion_configuration_id', ingestion_configuration_id
# activate an ingestion configuration
# = self.ingestclient.activate_ingestion_configuration(ingestion_configuration_id)
#-------------------------------
# Create InstrumentModel
#-------------------------------
instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model="SBE37IMModel" )
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" %ex)
print 'test_deployAsPrimaryDevice: new InstrumentModel id = ', instModel_id
#-------------------------------
# Create InstrumentAgent
#-------------------------------
instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent" )
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" %ex)
print 'test_deployAsPrimaryDevice: new InstrumentAgent id = ', instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
#-------------------------------
# Create Instrument Site
#-------------------------------
instrumentSite_obj = IonObject(RT.InstrumentSite, name='instrumentSite1', description="SBE37IMInstrumentSite" )
try:
instrumentSite_id = self.omsclient.create_instrument_site(instrument_site=instrumentSite_obj, parent_id='')
except BadRequest as ex:
self.fail("failed to create new InstrumentSite: %s" %ex)
print 'test_deployAsPrimaryDevice: new instrumentSite id = ', instrumentSite_id
self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrumentSite_id)
#-------------------------------
# Logical Transform: Output Data Products
#-------------------------------
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def)
log.debug("test_deployAsPrimaryDevice: create output parsed data product for Logical Instrument")
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters()
parameter_dictionary = parameter_dictionary.dump()
ctd_logical_output_dp_obj = IonObject(RT.DataProduct,
name='ctd_parsed_logical',
description='ctd parsed from the logical instrument',
temporal_domain = tdom,
spatial_domain = sdom)
instrument_site_output_dp_id = self.dataproductclient.create_data_product(ctd_logical_output_dp_obj,
ctd_stream_def_id,
parameter_dictionary)
self.dataproductclient.activate_data_product_persistence(data_product_id=instrument_site_output_dp_id)
self.omsclient.create_site_data_product(instrumentSite_id, instrument_site_output_dp_id)
#-------------------------------
# Create Old InstrumentDevice
#-------------------------------
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDeviceYear1',
description="SBE37IMDevice for the FIRST year of deployment",
serial_number="12345" )
try:
oldInstDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, oldInstDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
print 'test_deployAsPrimaryDevice: new Year 1 InstrumentDevice id = ', oldInstDevice_id
self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.DEPLOY)
self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.ENABLE)
#-------------------------------
# Create Old Deployment
#-------------------------------
deployment_obj = IonObject(RT.Deployment, name='first deployment')
oldDeployment_id = self.omsclient.create_deployment(deployment_obj)
# deploy this device to the logical slot
self.imsclient.deploy_instrument_device(oldInstDevice_id, oldDeployment_id)
self.omsclient.deploy_instrument_site(instrumentSite_id, oldDeployment_id)
#-------------------------------
# Create InstrumentAgentInstance for OldInstrumentDevice to hold configuration information
# cmd_port=5556, evt_port=5557, comms_method="ethernet", comms_device_address=CFG.device.sbe37.host, comms_device_port=CFG.device.sbe37.port,
#-------------------------------
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance,
name='SBE37IMAgentInstanceYear1',
description="SBE37IMAgentInstance Year 1",
svr_addr="localhost",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver",
cmd_port=5556,
evt_port=5557,
comms_method="ethernet",
comms_device_address="localhost",
comms_device_port=4001,
comms_server_address="localhost",
comms_server_port=8888)
oldInstAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
oldInstDevice_id)
#-------------------------------
# Create CTD Parsed as the Year 1 data product and attach to instrument
#-------------------------------
print 'test_deployAsPrimaryDevice: new Stream Definition id = ', ctd_stream_def_id
print 'Creating new CDM data product with a stream definition'
dp_obj = IonObject(RT.DataProduct,
name='ctd_parsed_year1',
description='ctd stream test year 1',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product_year1 = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary)
print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product_year1
self.damsclient.assign_data_product(input_resource_id=oldInstDevice_id,
data_product_id=ctd_parsed_data_product_year1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_year1, PRED.hasStream, None, True)
print 'test_deployAsPrimaryDevice: Data product streams1 = ', stream_ids
#-------------------------------
# Create New InstrumentDevice
#-------------------------------
instDevice_obj_2 = IonObject(RT.InstrumentDevice,
name='SBE37IMDeviceYear2',
description="SBE37IMDevice for the SECOND year of deployment",
serial_number="67890" )
try:
newInstDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj_2)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, newInstDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
print 'test_deployAsPrimaryDevice: new Year 2 InstrumentDevice id = ', newInstDevice_id
#set the LCSTATE
self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.DEPLOY)
self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.ENABLE)
instDevice_obj_2 = self.rrclient.read(newInstDevice_id)
log.debug("test_deployAsPrimaryDevice: Create New InstrumentDevice LCSTATE: %s ",
str(instDevice_obj_2.lcstate))
#-------------------------------
# Create Old Deployment
#-------------------------------
deployment_obj = IonObject(RT.Deployment, name='second deployment')
newDeployment_id = self.omsclient.create_deployment(deployment_obj)
# deploy this device to the logical slot
self.imsclient.deploy_instrument_device(newInstDevice_id, newDeployment_id)
self.omsclient.deploy_instrument_site(instrumentSite_id, newDeployment_id)
#-------------------------------
# Create InstrumentAgentInstance for NewInstrumentDevice to hold configuration information
#-------------------------------
instAgentInstance_new__obj = IonObject(RT.InstrumentAgentInstance,
name='SBE37IMAgentInstanceYear2',
description="SBE37IMAgentInstance Year 2",
svr_addr="localhost",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver",
cmd_port=5556, evt_port=5557,
comms_method="ethernet",
comms_device_address="localhost",
comms_device_port=4002,
comms_server_address="localhost",
comms_server_port=8888)
newInstAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_new__obj,
instAgent_id,
newInstDevice_id)
#-------------------------------
# Create CTD Parsed as the Year 2 data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
# ctd_stream_def = SBE37_CDM_stream_definition()
# ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def)
print 'test_deployAsPrimaryDevice: new Stream Definition id = ', ctd_stream_def_id
print 'Creating new CDM data product with a stream definition'
dp_obj = IonObject(RT.DataProduct,
name='ctd_parsed_year2',
description='ctd stream test year 2',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product_year2 = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary)
print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product_year2
self.damsclient.assign_data_product(input_resource_id=newInstDevice_id,
data_product_id=ctd_parsed_data_product_year2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_year2, PRED.hasStream, None, True)
print 'test_deployAsPrimaryDevice: Data product streams2 = ', stream_ids
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("test_deployAsPrimaryDevice: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all',
process_source='some_source_reference')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity = L0_conductivity_stream_definition()
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_conductivity, name='L0_Conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_pressure = L0_pressure_stream_definition()
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_pressure, name='L0_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_temperature = L0_temperature_stream_definition()
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_temperature, name='L0_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id )
self.output_products={}
log.debug("test_deployAsPrimaryDevice: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject(RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id,
parameter_dictionary)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
#self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id)
log.debug("test_deployAsPrimaryDevice: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_pressure_output_dp_obj,
outgoing_stream_l0_pressure_id,
parameter_dictionary )
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
#self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id)
log.debug("test_deployAsPrimaryDevice: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id,
parameter_dictionary)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
#self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process, listening to Sim1 (later: logical instrument output product)
#-------------------------------
log.debug("test_deployAsPrimaryDevice: create L0 all data_process start")
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product_year1], self.output_products)
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_deployAsPrimaryDevice: create L0 all data_process return")
#--------------------------------
# Activate the deployment
#--------------------------------
self.omsclient.activate_deployment(oldDeployment_id)
#-------------------------------
# Launch InstrumentAgentInstance Sim1, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=oldInstAgentInstance_id)
inst_agent1_instance_obj= self.imsclient.read_instrument_agent_instance(oldInstAgentInstance_id)
print 'test_deployAsPrimaryDevice: Instrument agent instance obj: = ', inst_agent1_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client_sim1 = ResourceAgentClient('iaclient Sim1', name=inst_agent1_instance_obj.agent_process_id, process=FakeProcess())
print 'activate_instrument: got _ia_client_sim1 %s', self._ia_client_sim1
log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim1 %s", str(self._ia_client_sim1))
#-------------------------------
# Launch InstrumentAgentInstance Sim2, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=newInstAgentInstance_id)
inst_agent2_instance_obj= self.imsclient.read_instrument_agent_instance(newInstAgentInstance_id)
print 'test_deployAsPrimaryDevice: Instrument agent instance obj: = ', inst_agent2_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client_sim2 = ResourceAgentClient('iaclient Sim2', name=inst_agent2_instance_obj.agent_process_id, process=FakeProcess())
print 'activate_instrument: got _ia_client_sim2 %s', self._ia_client_sim2
log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim2 %s", str(self._ia_client_sim2))
#-------------------------------
# Streaming Sim1 (old instrument)
#-------------------------------
cmd = AgentCommand(command='initialize')
retval = self._ia_client_sim1.execute_agent(cmd)
print retval
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 initialize %s", str(retval))
time.sleep(2)
cmd = AgentCommand(command='go_active')
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 go_active %s", str(reply))
time.sleep(2)
cmd = AgentCommand(command='run')
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 run %s", str(reply))
time.sleep(2)
log.debug("test_activateInstrument: calling go_streaming ")
cmd = AgentCommand(command='go_streaming')
reply = self._ia_client_sim1.execute(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 go_streaming %s", str(reply))
#-------------------------------
# Streaming Sim 2 (new instrument)
#-------------------------------
cmd = AgentCommand(command='initialize')
retval = self._ia_client_sim2.execute_agent(cmd)
print retval
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 initialize %s", str(retval))
time.sleep(2)
cmd = AgentCommand(command='go_active')
reply = self._ia_client_sim2.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 go_active %s", str(reply))
time.sleep(2)
cmd = AgentCommand(command='run')
reply = self._ia_client_sim2.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 run %s", str(reply))
time.sleep(2)
log.debug("test_activateInstrument: calling go_streaming ")
cmd = AgentCommand(command='go_streaming')
reply = self._ia_client_sim2.execute(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 go_streaming %s", str(reply))
time.sleep(20)
#-------------------------------
# Shutdown Sim1 (old instrument)
#-------------------------------
log.debug("test_activateInstrument: calling go_observatory")
cmd = AgentCommand(command='go_observatory')
reply = self._ia_client_sim1.execute(cmd)
log.debug("test_activateInstrument: _ia_client_sim1 return from go_observatory %s", str(reply))
time.sleep(5)
log.debug("test_deployAsPrimaryDevice:: calling go_inactive ")
cmd = AgentCommand(command='go_inactive')
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 return from go_inactive %s", str(reply))
time.sleep(2)
log.debug("test_deployAsPrimaryDevice:: calling reset ")
cmd = AgentCommand(command='reset')
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 return from reset %s", str(reply))
time.sleep(2)
#-------------------------------
# Shutdown Sim2 (old instrument)
#-------------------------------
log.debug("test_activateInstrument: calling go_observatory")
cmd = AgentCommand(command='go_observatory')
reply = self._ia_client_sim2.execute(cmd)
log.debug("test_activateInstrument: _ia_client_sim2 return from go_observatory %s", str(reply))
time.sleep(8)
log.debug("test_deployAsPrimaryDevice:: calling go_inactive ")
cmd = AgentCommand(command='go_inactive')
reply = self._ia_client_sim2.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 return from go_inactive %s", str(reply))
time.sleep(2)
log.debug("test_deployAsPrimaryDevice:: calling reset ")
cmd = AgentCommand(command='reset')
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 return from reset %s", str(reply))
time.sleep(2)
log.debug("test_deployAsPrimaryDevice:: calling reset ")
cmd = AgentCommand(command='reset')
reply = self._ia_client_sim2.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 return from reset %s", str(reply))
time.sleep(2)
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=oldInstAgentInstance_id)
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=newInstAgentInstance_id)
class TestDeployment(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url("res/deploy/r2deploy.yml")
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dmpsclient = DataProductManagementServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.psmsclient = PubsubManagementServiceClient(node=self.container.node)
# @unittest.skip("targeting")
def test_create_deployment(self):
# create a deployment with metadata and an initial site and device
platform_site__obj = IonObject(RT.PlatformSite, name="PlatformSite1", description="test platform site")
site_id = self.omsclient.create_platform_site(platform_site__obj)
platform_device__obj = IonObject(RT.PlatformDevice, name="PlatformDevice1", description="test platform device")
device_id = self.imsclient.create_platform_device(platform_device__obj)
deployment_obj = IonObject(RT.Deployment, name="TestDeployment", description="some new deployment")
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.deploy_platform_site(site_id, deployment_id)
self.imsclient.deploy_platform_device(device_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id))
# retrieve the deployment objects and check that the assoc site and device are attached
read_deployment_obj = self.omsclient.read_deployment(deployment_id)
log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj))
site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True)
self.assertEqual(len(site_ids), 1)
device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True)
self.assertEqual(len(device_ids), 1)
# delete the deployment
self.omsclient.delete_deployment(deployment_id)
# now try to get the deleted dp object
try:
deployment_obj = self.omsclient.read_deployment(deployment_id)
except NotFound as ex:
pass
else:
self.fail("deleted deployment was found during read")
# @unittest.skip("targeting")
def test_activate_deployment(self):
# create a deployment with metadata and an initial site and device
platform_site__obj = IonObject(RT.PlatformSite, name="PlatformSite1", description="test platform site")
site_id = self.omsclient.create_platform_site(platform_site__obj)
platform_device_obj = IonObject(RT.PlatformDevice, name="PlatformDevice1", description="test platform device")
platform_device_id = self.imsclient.create_platform_device(platform_device_obj)
platform_model__obj = IonObject(RT.PlatformModel, name="PlatformModel1", description="test platform model")
model_id = self.imsclient.create_platform_model(platform_model__obj)
self.imsclient.assign_platform_model_to_platform_device(model_id, platform_device_id)
self.omsclient.assign_platform_model_to_platform_site(model_id, site_id)
# create a deployment with metadata and an initial site and device
instrument_site_obj = IonObject(RT.InstrumentSite, name="InstrumentSite1", description="test instrument site")
instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, site_id)
# assign data products appropriately
# set up stream (this would be preload)
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.psmsclient.create_stream_definition(container=ctd_stream_def)
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters()
parameter_dictionary = parameter_dictionary.dump()
dp_obj = IonObject(
RT.DataProduct, name="DP1", description="some new dp", temporal_domain=tdom, spatial_domain=sdom
)
log_data_product_id = self.dmpsclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary)
self.omsclient.create_site_data_product(instrument_site_id, log_data_product_id)
instrument_device_obj = IonObject(
RT.InstrumentDevice, name="InstrumentDevice1", description="test instrument device"
)
instrument_device_id = self.imsclient.create_instrument_device(instrument_device_obj)
self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id)
dp_obj = IonObject(
RT.DataProduct, name="DP1", description="some new dp", temporal_domain=tdom, spatial_domain=sdom
)
inst_data_product_id = self.dmpsclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary)
# assign data products appropriately
self.damsclient.assign_data_product(
input_resource_id=instrument_device_id, data_product_id=inst_data_product_id
)
instrument_model_obj = IonObject(
RT.InstrumentModel, name="InstrumentModel1", description="test instrument model"
)
instrument_model_id = self.imsclient.create_instrument_model(instrument_model_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id)
self.omsclient.assign_instrument_model_to_instrument_site(instrument_model_id, instrument_site_id)
# self.rrclient.create_association(instrument_site_id, PRED.hasModel, instrument_model_id)
deployment_obj = IonObject(RT.Deployment, name="TestDeployment", description="some new deployment")
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id)
self.imsclient.deploy_instrument_device(instrument_device_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id))
self.omsclient.activate_deployment(deployment_id)
class TestDeployment(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
#@unittest.skip("targeting")
def test_create_deployment(self):
#create a deployment with metadata and an initial site and device
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site')
site_id = self.omsclient.create_platform_site(platform_site__obj)
platform_device__obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
device_id = self.imsclient.create_platform_device(platform_device__obj)
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment')
deployment_id = self.omsclient.create_deployment(deployment_obj, site_id, device_id)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
#retrieve the deployment objects and check that the assoc site and device are attached
read_deployment_obj = self.omsclient.read_deployment(deployment_id)
log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj) )
site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True)
self.assertEqual(len(site_ids), 1)
device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True)
self.assertEqual(len(device_ids), 1)
#delete the deployment
self.omsclient.delete_deployment(deployment_id)
# now try to get the deleted dp object
try:
deployment_obj = self.omsclient.read_deployment(deployment_id)
except NotFound as ex:
pass
else:
self.fail("deleted deployment was found during read")
#@unittest.skip("targeting")
def test_activate_deployment(self):
#create a deployment with metadata and an initial site and device
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site')
site_id = self.omsclient.create_platform_site(platform_site__obj)
platform_device__obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
device_id = self.imsclient.create_platform_device(platform_device__obj)
platform_model__obj = IonObject(RT.PlatformModel,
name='PlatformModel1',
description='test platform model')
model_id = self.imsclient.create_platform_model(platform_model__obj)
self.imsclient.assign_platform_model_to_platform_device(model_id, device_id)
self.omsclient.assign_platform_model_to_platform_site(model_id, site_id)
#create a deployment with metadata and an initial site and device
instrument_site__obj = IonObject(RT.InstrumentSite,
name='InstrumentSite1',
description='test instrument site')
instrument_site_id = self.omsclient.create_instrument_site(instrument_site__obj, site_id)
instrument_device__obj = IonObject(RT.InstrumentDevice,
name='InstrumentDevice1',
description='test instrument device')
instrument_device_id = self.imsclient.create_instrument_device(instrument_device__obj)
self.rrclient.create_association(device_id, PRED.hasDevice, instrument_device_id)
instrument_model__obj = IonObject(RT.InstrumentModel,
name='InstrumentModel1',
description='test instrument model')
instrument_model_id = self.imsclient.create_instrument_model(instrument_model__obj)
self.imsclient.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id)
self.omsclient.assign_instrument_model_to_instrument_site(instrument_model_id, instrument_site_id)
#self.rrclient.create_association(instrument_site_id, PRED.hasModel, instrument_model_id)
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment')
deployment_id = self.omsclient.create_deployment(deployment_obj, site_id, device_id)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
self.omsclient.activate_deployment(deployment_id)
class TestIMSDeployAsPrimaryDevice(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
# self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.container.start_rel_from_url("res/deploy/r2deploy.yml")
print "started services"
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
def create_logger(self, name, stream_id=""):
# logger process
producer_definition = ProcessDefinition(name=name + "_logger")
producer_definition.executable = {
"module": "ion.processes.data.stream_granule_logger",
"class": "StreamGranuleLogger",
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {"process": {"stream_id": stream_id}}
pid = self.processdispatchclient.schedule_process(
process_definition_id=logger_procdef_id, configuration=configuration
)
return pid
def cleanupprocs(self):
stm = os.popen("ps -e | grep ion.agents.port.logger_process")
procs = stm.read()
if len(procs) > 0:
procs = procs.split()
if procs[0].isdigit():
pid = int(procs[0])
os.kill(pid, signal.SIGKILL)
stm = os.popen("ps -e | grep ion.agents.instrument.zmq_driver_process")
procs = stm.read()
if len(procs) > 0:
procs = procs.split()
if procs[0].isdigit():
pid = int(procs[0])
os.kill(pid, signal.SIGKILL)
# stm = os.popen('rm /tmp/*.pid.txt')
@unittest.skip("Deprecated by IngestionManagement refactor, timeout on start inst agent?")
def test_deploy_activate_full(self):
# ensure no processes or pids are left around by agents or Sims
# self.cleanupprocs()
self.loggerpids = []
# -------------------------------
# Create InstrumentModel
# -------------------------------
instModel_obj = IonObject(RT.InstrumentModel, name="SBE37IMModel", description="SBE37IMModel")
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" % ex)
# -------------------------------
# Create InstrumentAgent
# -------------------------------
instAgent_obj = IonObject(
RT.InstrumentAgent,
name="agent007",
description="SBE37IMAgent",
driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg",
)
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" % ex)
log.debug("new InstrumentAgent id = %s", instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# -------------------------------
# Create Instrument Site
# -------------------------------
instrumentSite_obj = IonObject(RT.InstrumentSite, name="instrumentSite1", description="SBE37IMInstrumentSite")
try:
instrumentSite_id = self.omsclient.create_instrument_site(instrument_site=instrumentSite_obj, parent_id="")
except BadRequest as ex:
self.fail("failed to create new InstrumentSite: %s" % ex)
print "test_deployAsPrimaryDevice: new instrumentSite id = ", instrumentSite_id
self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrumentSite_id)
# -------------------------------
# Logical Transform: Output Data Products
# -------------------------------
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
"ctd_parsed_param_dict", id_only=True
)
parsed_stream_def_id = self.pubsubclient.create_stream_definition(
name="parsed", parameter_dictionary_id=parsed_pdict_id
)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_raw_param_dict", id_only=True)
raw_stream_def_id = self.pubsubclient.create_stream_definition(name="raw", parameter_dictionary_id=raw_pdict_id)
# -------------------------------
# Create Old InstrumentDevice
# -------------------------------
instDevice_obj = IonObject(
RT.InstrumentDevice,
name="SBE37IMDeviceYear1",
description="SBE37IMDevice for the FIRST year of deployment",
serial_number="12345",
)
try:
oldInstDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, oldInstDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
print "test_deployAsPrimaryDevice: new Year 1 InstrumentDevice id = ", oldInstDevice_id
self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.DEPLOY)
self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.ENABLE)
# -------------------------------
# Create Raw and Parsed Data Products for the device
# -------------------------------
dp_obj = IonObject(
RT.DataProduct,
name="SiteDataProduct",
description="SiteDataProduct",
temporal_domain=tdom,
spatial_domain=sdom,
)
instrument_site_output_dp_id = self.dataproductclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id
)
self.damsclient.assign_data_product(
input_resource_id=oldInstDevice_id, data_product_id=instrument_site_output_dp_id
)
# self.dataproductclient.activate_data_product_persistence(data_product_id=instrument_site_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(instrument_site_output_dp_id, PRED.hasStream, None, True)
log.debug("Data product streams1 = %s", stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(instrument_site_output_dp_id, PRED.hasDataset, RT.Dataset, True)
log.debug("Data set for data_product_id1 = %s", dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
pid = self.create_logger("ctd_parsed", stream_ids[0])
self.loggerpids.append(pid)
self.omsclient.create_site_data_product(instrumentSite_id, instrument_site_output_dp_id)
# -------------------------------
# Create Old Deployment
# -------------------------------
deployment_obj = IonObject(RT.Deployment, name="first deployment")
oldDeployment_id = self.omsclient.create_deployment(deployment_obj)
# deploy this device to the logical slot
self.imsclient.deploy_instrument_device(oldInstDevice_id, oldDeployment_id)
self.omsclient.deploy_instrument_site(instrumentSite_id, oldDeployment_id)
# -------------------------------
# Create InstrumentAgentInstance for OldInstrumentDevice to hold configuration information
# cmd_port=5556, evt_port=5557, comms_method="ethernet", comms_device_address=CFG.device.sbe37.host, comms_device_port=CFG.device.sbe37.port,
# -------------------------------
port_agent_config = {
"device_addr": CFG.device.sbe37.host,
"device_port": CFG.device.sbe37.port,
"process_type": PortAgentProcessType.UNIX,
"binary_path": "port_agent",
"port_agent_addr": "localhost",
"command_port": CFG.device.sbe37.port_agent_cmd_port,
"data_port": CFG.device.sbe37.port_agent_data_port,
"log_level": 5,
"type": PortAgentType.ETHERNET,
}
raw_config = StreamConfiguration(
stream_name="raw",
parameter_dictionary_name="ctd_raw_param_dict",
records_per_granule=2,
granule_publish_rate=5,
)
parsed_config = StreamConfiguration(
stream_name="parsed",
parameter_dictionary_name="ctd_parsed_param_dict",
records_per_granule=2,
granule_publish_rate=5,
)
instAgentInstance_obj = IonObject(
RT.InstrumentAgentInstance,
name="SBE37IMAgentInstanceYear1",
description="SBE37IMAgentInstanceYear1",
comms_device_address="sbe37-simulator.oceanobservatories.org",
comms_device_port=4001,
port_agent_config=port_agent_config,
stream_configurations=[raw_config, parsed_config],
)
oldInstAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, oldInstDevice_id
)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
# -------------------------------
# Create CTD Parsed as the Year 1 data product and attach to instrument
# -------------------------------
print "Creating new CDM data product with a stream definition"
dp_obj = IonObject(
RT.DataProduct,
name="ctd_parsed_year1",
description="ctd stream test year 1",
temporal_domain=tdom,
spatial_domain=sdom,
)
ctd_parsed_data_product_year1 = self.dataproductclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id
)
print "new ctd_parsed_data_product_id = ", ctd_parsed_data_product_year1
self.damsclient.assign_data_product(
input_resource_id=oldInstDevice_id, data_product_id=ctd_parsed_data_product_year1
)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_year1, PRED.hasStream, None, True)
print "test_deployAsPrimaryDevice: Data product streams1 = ", stream_ids
# -------------------------------
# Create New InstrumentDevice
# -------------------------------
instDevice_obj_2 = IonObject(
RT.InstrumentDevice,
name="SBE37IMDeviceYear2",
description="SBE37IMDevice for the SECOND year of deployment",
serial_number="67890",
)
try:
newInstDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj_2)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, newInstDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
print "test_deployAsPrimaryDevice: new Year 2 InstrumentDevice id = ", newInstDevice_id
# set the LCSTATE
self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.DEPLOY)
self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.ENABLE)
instDevice_obj_2 = self.rrclient.read(newInstDevice_id)
log.debug("test_deployAsPrimaryDevice: Create New InstrumentDevice LCSTATE: %s ", str(instDevice_obj_2.lcstate))
# -------------------------------
# Create Old Deployment
# -------------------------------
deployment_obj = IonObject(RT.Deployment, name="second deployment")
newDeployment_id = self.omsclient.create_deployment(deployment_obj)
# deploy this device to the logical slot
self.imsclient.deploy_instrument_device(newInstDevice_id, newDeployment_id)
self.omsclient.deploy_instrument_site(instrumentSite_id, newDeployment_id)
# -------------------------------
# Create InstrumentAgentInstance for NewInstrumentDevice to hold configuration information
# -------------------------------
port_agent_config = {
"device_addr": "sbe37-simulator.oceanobservatories.org",
"device_port": 4004,
"process_type": PortAgentProcessType.UNIX,
"binary_path": "port_agent",
"port_agent_addr": "localhost",
"command_port": 4005,
"data_port": 4006,
"log_level": 5,
"type": PortAgentType.ETHERNET,
}
instAgentInstance_obj = IonObject(
RT.InstrumentAgentInstance,
name="SBE37IMAgentInstanceYear2",
description="SBE37IMAgentInstanceYear2",
comms_device_address="sbe37-simulator.oceanobservatories.org",
comms_device_port=4004,
port_agent_config=port_agent_config,
)
newInstAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, newInstDevice_id
)
# -------------------------------
# Create CTD Parsed as the Year 2 data product
# -------------------------------
dp_obj = IonObject(
RT.DataProduct,
name="ctd_parsed_year2",
description="ctd stream test year 2",
temporal_domain=tdom,
spatial_domain=sdom,
)
ctd_parsed_data_product_year2 = self.dataproductclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id
)
print "new ctd_parsed_data_product_id = ", ctd_parsed_data_product_year2
self.damsclient.assign_data_product(
input_resource_id=newInstDevice_id, data_product_id=ctd_parsed_data_product_year2
)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_year2, PRED.hasStream, None, True)
print "test_deployAsPrimaryDevice: Data product streams2 = ", stream_ids
# -------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
# -------------------------------
log.debug("test_deployAsPrimaryDevice: create data process definition ctd_L0_all")
dpd_obj = IonObject(
RT.DataProcessDefinition,
name="ctd_L0_all",
description="transform ctd package into three separate L0 streams",
module="ion.processes.data.transforms.ctd.ctd_L0_all",
class_name="ctd_L0_all",
)
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" % ex)
# -------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
# -------------------------------
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(
name="L0_Conductivity", parameter_dictionary_id=parsed_pdict_id
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding="conductivity"
)
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(
name="L0_Pressure", parameter_dictionary_id=parsed_pdict_id
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding="pressure"
)
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(
name="L0_Temperature", parameter_dictionary_id=parsed_pdict_id
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding="temperature"
)
self.output_products = {}
log.debug("test_deployAsPrimaryDevice: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject(
RT.DataProduct,
name="L0_Conductivity",
description="transform output conductivity",
temporal_domain=tdom,
spatial_domain=sdom,
)
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(
data_product=ctd_l0_conductivity_output_dp_obj, stream_definition_id=parsed_stream_def_id
)
self.output_products["conductivity"] = ctd_l0_conductivity_output_dp_id
# self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id)
log.debug("test_deployAsPrimaryDevice: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject(
RT.DataProduct,
name="L0_Pressure",
description="transform output pressure",
temporal_domain=tdom,
spatial_domain=sdom,
)
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(
data_product=ctd_l0_pressure_output_dp_obj, stream_definition_id=parsed_stream_def_id
)
self.output_products["pressure"] = ctd_l0_pressure_output_dp_id
# self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id)
log.debug("test_deployAsPrimaryDevice: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject(
RT.DataProduct,
name="L0_Temperature",
description="transform output temperature",
temporal_domain=tdom,
spatial_domain=sdom,
)
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(
data_product=ctd_l0_temperature_output_dp_obj, stream_definition_id=parsed_stream_def_id
)
self.output_products["temperature"] = ctd_l0_temperature_output_dp_id
# self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id)
# -------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process, listening to Sim1 (later: logical instrument output product)
# -------------------------------
log.debug("test_deployAsPrimaryDevice: create L0 all data_process start")
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(
ctd_L0_all_dprocdef_id, [ctd_parsed_data_product_year1], self.output_products
)
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug("test_deployAsPrimaryDevice: create L0 all data_process return")
# --------------------------------
# Activate the deployment
# --------------------------------
self.omsclient.activate_deployment(oldDeployment_id)
# -------------------------------
# Launch InstrumentAgentInstance Sim1, connect to the resource agent client
# -------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=oldInstAgentInstance_id)
self.addCleanup(
self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=oldInstAgentInstance_id
)
# wait for start
instance_obj = self.imsclient.read_instrument_agent_instance(oldInstAgentInstance_id)
gate = ProcessStateGate(
self.processdispatchclient.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING
)
self.assertTrue(
gate.await(30),
"The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id,
)
inst_agent1_instance_obj = self.imsclient.read_instrument_agent_instance(oldInstAgentInstance_id)
print "test_deployAsPrimaryDevice: Instrument agent instance obj: = ", inst_agent1_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client_sim1 = ResourceAgentClient(
"iaclient Sim1", name=inst_agent1_instance_obj.agent_process_id, process=FakeProcess()
)
print "activate_instrument: got _ia_client_sim1 %s", self._ia_client_sim1
log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim1 %s", str(self._ia_client_sim1))
# -------------------------------
# Launch InstrumentAgentInstance Sim2, connect to the resource agent client
# -------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=newInstAgentInstance_id)
self.addCleanup(
self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=newInstAgentInstance_id
)
# wait for start
instance_obj = self.imsclient.read_instrument_agent_instance(newInstAgentInstance_id)
gate = ProcessStateGate(
self.processdispatchclient.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING
)
self.assertTrue(
gate.await(30),
"The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id,
)
inst_agent2_instance_obj = self.imsclient.read_instrument_agent_instance(newInstAgentInstance_id)
print "test_deployAsPrimaryDevice: Instrument agent instance obj: = ", inst_agent2_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client_sim2 = ResourceAgentClient(
"iaclient Sim2", name=inst_agent2_instance_obj.agent_process_id, process=FakeProcess()
)
print "activate_instrument: got _ia_client_sim2 %s", self._ia_client_sim2
log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim2 %s", str(self._ia_client_sim2))
# -------------------------------
# Streaming Sim1 (old instrument)
# -------------------------------
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice: initialize %s", str(retval))
log.debug("(L4-CI-SA-RQ-334): Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice: return value from go_active %s", str(reply))
self.assertTrue(reply)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client_sim1.execute_agent(cmd)
state = retval.result
log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice: run %s", str(reply))
gevent.sleep(2)
cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
retval = self._ia_client_sim1.execute_resource(cmd)
log.debug("test_activateInstrumentSample: return from START_AUTOSAMPLE: %s", str(retval))
# -------------------------------
# Streaming Sim 2 (new instrument)
# -------------------------------
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client_sim2.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice: initialize_sim2 %s", str(retval))
log.debug("(L4-CI-SA-RQ-334): Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client_sim2.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice: return value from go_active_sim2 %s", str(reply))
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client_sim2.execute_agent(cmd)
state = retval.result
log.debug("(L4-CI-SA-RQ-334): current state after sending go_active_sim2 command %s", str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client_sim2.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice: run %s", str(reply))
gevent.sleep(2)
cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
retval = self._ia_client_sim2.execute_resource(cmd)
log.debug("test_activateInstrumentSample: return from START_AUTOSAMPLE_sim2: %s", str(retval))
gevent.sleep(10)
# -------------------------------
# Shutdown Sim1 (old instrument)
# -------------------------------
cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
retval = self._ia_client_sim1.execute_resource(cmd)
log.debug("test_activateInstrumentSample: return from STOP_AUTOSAMPLE: %s", str(retval))
log.debug("test_activateInstrumentSample: calling reset ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset %s", str(reply))
time.sleep(5)
# -------------------------------
# Shutdown Sim2 (old instrument)
# -------------------------------
cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
retval = self._ia_client_sim2.execute_resource(cmd)
log.debug("test_activateInstrumentSample: return from STOP_AUTOSAMPLE_sim2: %s", str(retval))
log.debug("test_activateInstrumentSample: calling reset_sim2 ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset_sim2 %s", str(reply))
time.sleep(5)
class TestDataProductProvenance(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.dpmsclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
# create missing data process definition
dpd_obj = IonObject(RT.DataProcessDefinition,
name=LOGICAL_TRANSFORM_DEFINITION_NAME,
description="normally in preload",
module='ion.processes.data.transforms.logical_transform',
class_name='logical_transform')
self.dataprocessclient.create_data_process_definition(dpd_obj)
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]:
self.dataprocessclient.deactivate_data_process(proc_id)
self.dataprocessclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
#@unittest.skip('not ready')
def test_get_provenance(self):
#create a deployment with metadata and an initial site and device
instrument_site_obj = IonObject(RT.InstrumentSite,
name='InstrumentSite1',
description='test instrument site')
instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, "")
log.debug( 'test_get_provenance: new instrument_site_id id = %s ', str(instrument_site_id))
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel" )
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" %ex)
log.debug( 'test_get_provenance: new InstrumentModel id = %s ', str(instModel_id))
self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrument_site_id)
# Create InstrumentAgent
parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 )
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg",
stream_configurations = [parsed_config] )
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" %ex)
log.debug( 'test_get_provenance:new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_get_provenance: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
log.debug("test_get_provenance: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id)
#-------------------------------
# Create CTD Parsed data product
#-------------------------------
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
log.debug( 'test_get_provenance:Creating new CDM data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product = self.dpmsclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', ctd_parsed_data_product)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product)
#-------------------------------
# create a data product for the site to pass the OMS check.... we need to remove this check
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
log_data_product_id = self.dpmsclient.create_data_product(dp_obj, parsed_stream_def_id)
self.omsclient.create_site_data_product(instrument_site_id, log_data_product_id)
#-------------------------------
# Deploy instrument device to instrument site
#-------------------------------
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment')
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id)
self.imsclient.deploy_instrument_device(instDevice_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
self.omsclient.activate_deployment(deployment_id)
inst_device_objs, _ = self.rrclient.find_objects(subject=instrument_site_id, predicate=PRED.hasDevice, object_type=RT.InstrumetDevice, id_only=False)
log.debug("test_create_deployment: deployed device: %s ", str(inst_device_objs[0]) )
#-------------------------------
# Create the agent instance
#-------------------------------
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
#-------------------------------
# L1 Conductivity: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1ConductivityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_conductivity',
description='create the L1 conductivity data product',
module='ion.processes.data.transforms.ctd.ctd_L1_conductivity',
class_name='CTDL1ConductivityTransform')
try:
ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex)
#-------------------------------
# L1 Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1PressureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_pressure',
description='create the L1 pressure data product',
module='ion.processes.data.transforms.ctd.ctd_L1_pressure',
class_name='CTDL1PressureTransform')
try:
ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex)
#-------------------------------
# L1 Temperature: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1TemperatureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_temperature',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L1_temperature',
class_name='CTDL1TemperatureTransform')
try:
ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex)
#-------------------------------
# L2 Salinity: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition SalinityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_salinity',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new SalinityTransform data process definition: %s" %ex)
#-------------------------------
# L2 Density: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition DensityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_density',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_density',
class_name='DensityTransform')
try:
ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new DensityTransform data process definition: %s" %ex)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' )
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' )
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' )
self.output_products={}
log.debug("TestDataProductProvenance: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
log.debug("TestDataProductProvenance: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_pressure_output_dp_obj,
outgoing_stream_l0_pressure_id)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
log.debug("TestDataProductProvenance: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
#-------------------------------
# L1 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(name='L1_conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id, binding='conductivity' )
outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(name='L1_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id, binding='pressure' )
outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(name='L1_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id, binding='temperature' )
log.debug("TestDataProductProvenance: create output data product L1 conductivity")
ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct,
name='L1_Conductivity',
description='transform output L1 conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_conductivity_output_dp_obj,
outgoing_stream_l1_conductivity_id)
log.debug("TestDataProductProvenance: create output data product L1 pressure")
ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Pressure',
description='transform output L1 pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_pressure_output_dp_obj,
outgoing_stream_l1_pressure_id)
log.debug("TestDataProductProvenance: create output data product L1 temperature")
ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Temperature',
description='transform output L1 temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_temperature_output_dp_obj,
outgoing_stream_l1_temperature_id)
#-------------------------------
# L2 Salinity - Density: Output Data Products
#-------------------------------
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(name='L2_salinity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id, binding='salinity' )
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(name='L2_Density', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id, binding='density' )
log.debug("TestDataProductProvenance: create output data product L2 Salinity")
ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Salinity',
description='transform output L2 salinity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_salinity_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_salinity_output_dp_obj,
outgoing_stream_l2_salinity_id)
log.debug("TestDataProductProvenance: create output data product L2 Density")
# ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
# name='L2_Density',
# description='transform output pressure',
# temporal_domain = tdom,
# spatial_domain = sdom)
#
# ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj,
# outgoing_stream_l2_density_id,
# parameter_dictionary)
contactInfo = ContactInformation()
contactInfo.individual_names_given = "Bill"
contactInfo.individual_name_family = "Smith"
contactInfo.street_address = "111 First St"
contactInfo.city = "San Diego"
contactInfo.email = "*****@*****.**"
contactInfo.phones = ["858-555-6666"]
contactInfo.country = "USA"
contactInfo.postal_code = "92123"
ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Density',
description='transform output pressure',
contacts = [contactInfo],
iso_topic_category = "my_iso_topic_category_here",
quality_control_level = "1",
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj,
outgoing_stream_l2_density_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L0 all data_process start")
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products)
#activate only this data process just for coverage
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
contents = "this is the lookup table contents, replace with a file..."
att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
deviceAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att)
log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment)
log.debug("TestDataProductProvenance: create L0 all data_process return")
#-------------------------------
# L1 Conductivity: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1 Conductivity data_process start")
try:
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_conductivity_dprocdef_id, [ctd_l0_conductivity_output_dp_id], {'conductivity':ctd_l1_conductivity_output_dp_id})
self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L1 Pressure: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1_Pressure data_process start")
try:
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_pressure_dprocdef_id, [ctd_l0_pressure_output_dp_id], {'pressure':ctd_l1_pressure_output_dp_id})
self.dataprocessclient.activate_data_process(l1_pressure_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L1 Temperature: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1_Pressure data_process start")
try:
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_temperature_dprocdef_id, [ctd_l0_temperature_output_dp_id], {'temperature':ctd_l1_temperature_output_dp_id})
self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L2 Salinity: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L2_salinity data_process start")
try:
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_salinity_dprocdef_id, [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], {'salinity':ctd_l2_salinity_output_dp_id})
self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L2 Density: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L2_Density data_process start")
try:
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_density_dprocdef_id, [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], {'density':ctd_l2_density_output_dp_id})
self.dataprocessclient.activate_data_process(l2_density_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
print 'TestDataProductProvenance: Instrument agent instance obj: = ', inst_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
# self._ia_client = ResourceAgentClient('iaclient', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess())
# print 'activate_instrument: got ia client %s', self._ia_client
# log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client))
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.dataprocessclient.deactivate_data_process(l2_density_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l2_salinity_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_temperature_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_pressure_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_conductivity_data_process_id)
self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id)
#-------------------------------
# Retrieve the provenance info for the ctd density data product
#-------------------------------
provenance_dict = self.dpmsclient.get_data_product_provenance(ctd_l2_density_output_dp_id)
log.debug("TestDataProductProvenance: provenance_dict %s", str(provenance_dict))
#validate that products are represented
self.assertTrue (provenance_dict[str(ctd_l1_conductivity_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l0_conductivity_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l2_density_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l1_temperature_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l0_temperature_output_dp_id)])
density_dict = (provenance_dict[str(ctd_l2_density_output_dp_id)])
self.assertEquals(density_dict['producer'], [l2_density_all_data_process_id])
#-------------------------------
# Retrieve the extended resource for this data product
#-------------------------------
extended_product = self.dpmsclient.get_data_product_extension(ctd_l2_density_output_dp_id)
self.assertEqual(1, len(extended_product.data_processes) )
self.assertEqual(3, len(extended_product.process_input_data_products) )
# log.debug("TestDataProductProvenance: DataProduct provenance_product_list %s", str(extended_product.provenance_product_list))
# log.debug("TestDataProductProvenance: DataProduct data_processes %s", str(extended_product.data_processes))
# log.debug("TestDataProductProvenance: DataProduct process_input_data_products %s", str(extended_product.process_input_data_products))
# log.debug("TestDataProductProvenance: provenance %s", str(extended_product.computed.provenance.value))
#-------------------------------
# Retrieve the extended resource for this data process
#-------------------------------
extended_process_def = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id)
# log.debug("TestDataProductProvenance: DataProcess extended_process_def %s", str(extended_process_def))
# log.debug("TestDataProductProvenance: DataProcess data_processes %s", str(extended_process_def.data_processes))
# log.debug("TestDataProductProvenance: DataProcess data_products %s", str(extended_process_def.data_products))
self.assertEqual(1, len(extended_process_def.data_processes) )
self.assertEqual(3, len(extended_process_def.output_stream_definitions) )
self.assertEqual(3, len(extended_process_def.data_products) ) #one list because of one data process
#-------------------------------
# Request the xml report
#-------------------------------
results = self.dpmsclient.get_data_product_provenance_report(ctd_l2_density_output_dp_id)
#-------------------------------
# Cleanup
#-------------------------------
self.dpmsclient.delete_data_product(ctd_parsed_data_product)
self.dpmsclient.delete_data_product(log_data_product_id)
self.dpmsclient.delete_data_product(ctd_l0_conductivity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l0_pressure_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l0_temperature_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_conductivity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_pressure_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_temperature_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l2_salinity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l2_density_output_dp_id)
class TestDeployment(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dmpsclient = DataProductManagementServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.psmsclient = PubsubManagementServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.c = DotDict()
self.c.resource_registry = self.rrclient
self.resource_impl = ResourceImpl(self.c)
#@unittest.skip("targeting")
def test_create_deployment(self):
#create a deployment with metadata and an initial site and device
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site')
site_id = self.omsclient.create_platform_site(platform_site__obj)
platform_device__obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
device_id = self.imsclient.create_platform_device(platform_device__obj)
start = IonTime(datetime.datetime(2013,1,1))
end = IonTime(datetime.datetime(2014,1,1))
temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string())
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment',
constraint_list=[temporal_bounds])
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.deploy_platform_site(site_id, deployment_id)
self.imsclient.deploy_platform_device(device_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
#retrieve the deployment objects and check that the assoc site and device are attached
read_deployment_obj = self.omsclient.read_deployment(deployment_id)
log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj) )
site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True)
self.assertEqual(len(site_ids), 1)
device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True)
self.assertEqual(len(device_ids), 1)
#delete the deployment
self.resource_impl.pluck(deployment_id)
self.omsclient.force_delete_deployment(deployment_id)
# now try to get the deleted dp object
try:
deployment_obj = self.omsclient.read_deployment(deployment_id)
except NotFound:
pass
else:
self.fail("deleted deployment was found during read")
#@unittest.skip("targeting")
def test_activate_deployment(self):
#-------------------------------------------------------------------------------------
# Create platform site, platform device, platform model
#-------------------------------------------------------------------------------------
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site')
site_id = self.omsclient.create_platform_site(platform_site__obj)
platform_device_obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
platform_device_id = self.imsclient.create_platform_device(platform_device_obj)
platform_model__obj = IonObject(RT.PlatformModel,
name='PlatformModel1',
description='test platform model')
model_id = self.imsclient.create_platform_model(platform_model__obj)
#-------------------------------------------------------------------------------------
# Assign platform model to platform device and site
#-------------------------------------------------------------------------------------
self.imsclient.assign_platform_model_to_platform_device(model_id, platform_device_id)
self.omsclient.assign_platform_model_to_platform_site(model_id, site_id)
#-------------------------------------------------------------------------------------
# Create instrument site
#-------------------------------------------------------------------------------------
instrument_site_obj = IonObject(RT.InstrumentSite,
name='InstrumentSite1',
description='test instrument site')
instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, site_id)
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.psmsclient.create_stream_definition(name='SBE37_CDM', parameter_dictionary_id=pdict_id)
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='Log Data Product',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
out_log_data_product_id = self.dmpsclient.create_data_product(dp_obj, ctd_stream_def_id)
#----------------------------------------------------------------------------------------------------
# Start the transform (a logical transform) that acts as an instrument site
#----------------------------------------------------------------------------------------------------
self.omsclient.create_site_data_product( site_id= instrument_site_id,
data_product_id = out_log_data_product_id)
#----------------------------------------------------------------------------------------------------
# Create an instrument device
#----------------------------------------------------------------------------------------------------
instrument_device_obj = IonObject(RT.InstrumentDevice,
name='InstrumentDevice1',
description='test instrument device')
instrument_device_id = self.imsclient.create_instrument_device(instrument_device_obj)
self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id)
dp_obj = IonObject(RT.DataProduct,
name='Instrument Data Product',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
inst_data_product_id = self.dmpsclient.create_data_product(dp_obj, ctd_stream_def_id)
#assign data products appropriately
self.damsclient.assign_data_product(input_resource_id=instrument_device_id,
data_product_id=inst_data_product_id)
#----------------------------------------------------------------------------------------------------
# Create an instrument model
#----------------------------------------------------------------------------------------------------
instrument_model_obj = IonObject(RT.InstrumentModel,
name='InstrumentModel1',
description='test instrument model')
instrument_model_id = self.imsclient.create_instrument_model(instrument_model_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id)
self.omsclient.assign_instrument_model_to_instrument_site(instrument_model_id, instrument_site_id)
#----------------------------------------------------------------------------------------------------
# Create a deployment object
#----------------------------------------------------------------------------------------------------
start = IonTime(datetime.datetime(2013,1,1))
end = IonTime(datetime.datetime(2014,1,1))
temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string())
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment',
constraint_list=[temporal_bounds])
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id)
self.imsclient.deploy_instrument_device(instrument_device_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
self.omsclient.activate_deployment(deployment_id)
class TestDeployment(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dmpsclient = DataProductManagementServiceClient(
node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.psmsclient = PubsubManagementServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.c = DotDict()
self.c.resource_registry = self.rrclient
self.RR2 = EnhancedResourceRegistryClient(self.rrclient)
self.dsmsclient = DataProcessManagementServiceClient(
node=self.container.node)
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(
RT.DataProcess, None, None, True)[0]:
self.dsmsclient.deactivate_data_process(proc_id)
self.dsmsclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
#@unittest.skip("targeting")
def test_create_deployment(self):
#create a deployment with metadata and an initial site and device
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site')
site_id = self.omsclient.create_platform_site(platform_site__obj)
platform_device__obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
device_id = self.imsclient.create_platform_device(platform_device__obj)
start = str(int(time.mktime(datetime.datetime(2013, 1,
1).timetuple())))
end = str(int(time.mktime(datetime.datetime(2014, 1, 1).timetuple())))
temporal_bounds = IonObject(OT.TemporalBounds,
name='planned',
start_datetime=start,
end_datetime=end)
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment',
constraint_list=[temporal_bounds])
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.assign_site_to_deployment(site_id, deployment_id)
self.omsclient.assign_device_to_deployment(device_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ",
str(deployment_id))
#retrieve the deployment objects and check that the assoc site and device are attached
read_deployment_obj = self.omsclient.read_deployment(deployment_id)
log.debug("test_create_deployment: created deployment obj: %s ",
str(read_deployment_obj))
site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite,
PRED.hasDeployment,
deployment_id, True)
self.assertEqual(len(site_ids), 1)
device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice,
PRED.hasDeployment,
deployment_id, True)
self.assertEqual(len(device_ids), 1)
#delete the deployment
self.omsclient.force_delete_deployment(deployment_id)
# now try to get the deleted dp object
try:
self.omsclient.read_deployment(deployment_id)
except NotFound:
pass
else:
self.fail("deleted deployment was found during read")
#@unittest.skip("targeting")
def test_prepare_deployment_support(self):
deploy_sup = self.omsclient.prepare_deployment_support()
self.assertTrue(deploy_sup)
self.assertEquals(
deploy_sup.associations['DeploymentHasInstrumentDevice'].type_,
"AssocDeploymentInstDevice")
self.assertEquals(
deploy_sup.associations['DeploymentHasInstrumentDevice'].resources,
[])
self.assertEquals(
deploy_sup.associations['DeploymentHasInstrumentDevice'].
associated_resources, [])
self.assertEquals(
deploy_sup.associations['DeploymentHasPlatformDevice'].type_,
"AssocDeploymentPlatDevice")
self.assertEquals(
deploy_sup.associations['DeploymentHasPlatformDevice'].resources,
[])
self.assertEquals(
deploy_sup.associations['DeploymentHasPlatformDevice'].
associated_resources, [])
self.assertEquals(
deploy_sup.associations['DeploymentHasInstrumentSite'].type_,
"AssocDeploymentInstSite")
self.assertEquals(
deploy_sup.associations['DeploymentHasInstrumentSite'].resources,
[])
self.assertEquals(
deploy_sup.associations['DeploymentHasInstrumentSite'].
associated_resources, [])
self.assertEquals(
deploy_sup.associations['DeploymentHasPlatformSite'].type_,
"AssocDeploymentPlatSite")
self.assertEquals(
deploy_sup.associations['DeploymentHasPlatformSite'].resources, [])
self.assertEquals(
deploy_sup.associations['DeploymentHasPlatformSite'].
associated_resources, [])
#create a deployment with metadata and an initial site and device
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site')
site_id = self.omsclient.create_platform_site(platform_site__obj)
platform_device__obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
device_id = self.imsclient.create_platform_device(platform_device__obj)
start = str(int(time.mktime(datetime.datetime(2013, 1,
1).timetuple())))
end = str(int(time.mktime(datetime.datetime(2014, 1, 1).timetuple())))
temporal_bounds = IonObject(OT.TemporalBounds,
name='planned',
start_datetime=start,
end_datetime=end)
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment',
constraint_list=[temporal_bounds])
deployment_id = self.omsclient.create_deployment(deployment_obj)
deploy_sup = self.omsclient.prepare_deployment_support(deployment_id)
self.assertEquals(
deploy_sup.associations['DeploymentHasInstrumentDevice'].resources,
[])
self.assertEquals(
deploy_sup.associations['DeploymentHasInstrumentDevice'].
associated_resources, [])
self.assertEquals(
len(deploy_sup.associations['DeploymentHasPlatformDevice'].
resources), 1)
self.assertEquals(
deploy_sup.associations['DeploymentHasPlatformDevice'].
associated_resources, [])
self.assertEquals(
deploy_sup.associations['DeploymentHasInstrumentSite'].resources,
[])
self.assertEquals(
deploy_sup.associations['DeploymentHasInstrumentSite'].
associated_resources, [])
self.assertEquals(
len(deploy_sup.associations['DeploymentHasPlatformSite'].resources
), 1)
self.assertEquals(
deploy_sup.associations['DeploymentHasPlatformSite'].
associated_resources, [])
self.omsclient.assign_site_to_deployment(site_id, deployment_id)
self.omsclient.assign_device_to_deployment(device_id, deployment_id)
deploy_sup = self.omsclient.prepare_deployment_support(deployment_id)
self.assertEquals(
deploy_sup.associations['DeploymentHasInstrumentDevice'].resources,
[])
self.assertEquals(
deploy_sup.associations['DeploymentHasInstrumentDevice'].
associated_resources, [])
self.assertEquals(
len(deploy_sup.associations['DeploymentHasPlatformDevice'].
resources), 1)
self.assertEquals(
len(deploy_sup.associations['DeploymentHasPlatformDevice'].
associated_resources), 1)
self.assertEquals(
deploy_sup.associations['DeploymentHasInstrumentSite'].resources,
[])
self.assertEquals(
deploy_sup.associations['DeploymentHasInstrumentSite'].
associated_resources, [])
self.assertEquals(
len(deploy_sup.associations['DeploymentHasPlatformSite'].resources
), 1)
self.assertEquals(
len(deploy_sup.associations['DeploymentHasPlatformSite'].
associated_resources), 1)
#delete the deployment
self.omsclient.force_delete_deployment(deployment_id)
# now try to get the deleted dp object
try:
self.omsclient.read_deployment(deployment_id)
except NotFound:
pass
else:
self.fail("deleted deployment was found during read")
#@unittest.skip("targeting")
def base_activate_deployment(self, make_assigns=False):
# Create platform site, platform device, platform model
bounds = GeospatialBounds(geospatial_latitude_limit_north=float(5),
geospatial_latitude_limit_south=float(5),
geospatial_longitude_limit_west=float(15),
geospatial_longitude_limit_east=float(15),
geospatial_vertical_min=float(0),
geospatial_vertical_max=float(1000))
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site',
constraint_list=[bounds])
platform_site_id = self.omsclient.create_platform_site(
platform_site__obj)
platform_device_obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
platform_device_id = self.imsclient.create_platform_device(
platform_device_obj)
platform_model__obj = IonObject(RT.PlatformModel,
name='PlatformModel1',
description='test platform model')
platform_model_id = self.imsclient.create_platform_model(
platform_model__obj)
# Create instrument site
#-------------------------------------------------------------------------------------
bounds = GeospatialBounds(geospatial_latitude_limit_north=float(45),
geospatial_latitude_limit_south=float(40),
geospatial_longitude_limit_west=float(-75),
geospatial_longitude_limit_east=float(-70),
geospatial_vertical_min=float(0),
geospatial_vertical_max=float(500))
instrument_site_obj = IonObject(
RT.InstrumentSite,
name='InstrumentSite1',
description='test instrument site',
reference_designator='GA01SUMO-FI003-01-CTDMO0999',
constraint_list=[bounds])
instrument_site_id = self.omsclient.create_instrument_site(
instrument_site_obj, platform_site_id)
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.psmsclient.create_stream_definition(
name='SBE37_CDM', parameter_dictionary_id=pdict_id)
# Create an instrument device
instrument_device_obj = IonObject(RT.InstrumentDevice,
name='InstrumentDevice1',
description='test instrument device')
instrument_device_id = self.imsclient.create_instrument_device(
instrument_device_obj)
self.rrclient.create_association(platform_device_id, PRED.hasDevice,
instrument_device_id)
pp_obj = IonObject(OT.PlatformPort,
reference_designator='GA01SUMO-FI003-01-CTDMO0999',
port_type=PortTypeEnum.PAYLOAD,
ip_address='1')
port_assignments = {instrument_device_id: pp_obj}
#----------------------------------------------------------------------------------------------------
# Create an instrument model
instrument_model_obj = IonObject(RT.InstrumentModel,
name='InstrumentModel1',
description='test instrument model')
instrument_model_id = self.imsclient.create_instrument_model(
instrument_model_obj)
# Create a deployment object
#----------------------------------------------------------------------------------------------------
start = str(int(time.mktime(datetime.datetime(2013, 1,
1).timetuple())))
end = str(int(time.mktime(datetime.datetime(2020, 1, 1).timetuple())))
temporal_bounds = IonObject(OT.TemporalBounds,
name='planned',
start_datetime=start,
end_datetime=end)
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment',
context=IonObject(
OT.CabledNodeDeploymentContext),
port_assignments=port_assignments,
constraint_list=[temporal_bounds])
deployment_id = self.omsclient.create_deployment(deployment_obj)
log.debug("test_create_deployment: created deployment id: %s ",
str(deployment_id))
if make_assigns:
self.imsclient.assign_platform_model_to_platform_device(
platform_model_id, platform_device_id)
self.imsclient.assign_instrument_model_to_instrument_device(
instrument_model_id, instrument_device_id)
self.omsclient.assign_platform_model_to_platform_site(
platform_model_id, platform_site_id)
self.omsclient.assign_instrument_model_to_instrument_site(
instrument_model_id, instrument_site_id)
self.omsclient.assign_site_to_deployment(platform_site_id,
deployment_id)
self.omsclient.assign_device_to_deployment(platform_device_id,
deployment_id)
ret = DotDict(instrument_site_id=instrument_site_id,
instrument_device_id=instrument_device_id,
instrument_model_id=instrument_model_id,
platform_site_id=platform_site_id,
platform_device_id=platform_device_id,
platform_model_id=platform_model_id,
deployment_id=deployment_id)
return ret
def _create_subsequent_deployment(self, prior_dep_info):
platform_device_obj = IonObject(RT.PlatformDevice,
name='PlatformDevice2',
description='test platform device')
platform_device_id = self.imsclient.create_platform_device(
platform_device_obj)
instrument_device_obj = IonObject(RT.InstrumentDevice,
name='InstrumentDevice2',
description='test instrument device')
instrument_device_id = self.imsclient.create_instrument_device(
instrument_device_obj)
self.rrclient.create_association(platform_device_id, PRED.hasDevice,
instrument_device_id)
self.imsclient.assign_platform_model_to_platform_device(
prior_dep_info.platform_model_id, platform_device_id)
self.imsclient.assign_instrument_model_to_instrument_device(
prior_dep_info.instrument_model_id, instrument_device_id)
start = str(int(time.mktime(datetime.datetime(2013, 6,
1).timetuple())))
end = str(int(time.mktime(datetime.datetime(2020, 6, 1).timetuple())))
temporal_bounds = IonObject(OT.TemporalBounds,
name='planned',
start_datetime=start,
end_datetime=end)
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment2',
description='some new deployment',
context=IonObject(
OT.CabledNodeDeploymentContext),
constraint_list=[temporal_bounds])
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.assign_site_to_deployment(
prior_dep_info.platform_site_id, deployment_id)
self.omsclient.assign_device_to_deployment(
prior_dep_info.platform_device_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ",
str(deployment_id))
ret = DotDict(instrument_device_id=instrument_device_id,
platform_device_id=platform_device_id,
deployment_id=deployment_id)
return ret
#@unittest.skip("targeting")
def test_activate_deployment_normal(self):
res = self.base_activate_deployment(make_assigns=True)
before_activate_instrument_device_obj = self.rrclient.read(
res.instrument_device_id)
self.assertNotEquals(before_activate_instrument_device_obj.lcstate,
LCS.DEPLOYED)
log.debug("activating deployment, expecting success")
self.omsclient.activate_deployment(res.deployment_id)
def assertGeospatialBoundsEquals(a, b):
self.assertEquals(a['geospatial_latitude_limit_north'],
b['geospatial_latitude_limit_north'])
self.assertEquals(a['geospatial_latitude_limit_south'],
b['geospatial_latitude_limit_south'])
self.assertEquals(a['geospatial_longitude_limit_west'],
b['geospatial_longitude_limit_west'])
self.assertEquals(a['geospatial_longitude_limit_east'],
b['geospatial_longitude_limit_east'])
def assertGeospatialBoundsNotEquals(a, b):
self.assertNotEquals(a['geospatial_latitude_limit_north'],
b['geospatial_latitude_limit_north'])
self.assertNotEquals(a['geospatial_latitude_limit_south'],
b['geospatial_latitude_limit_south'])
self.assertNotEquals(a['geospatial_longitude_limit_west'],
b['geospatial_longitude_limit_west'])
self.assertNotEquals(a['geospatial_longitude_limit_east'],
b['geospatial_longitude_limit_east'])
after_activate_instrument_device_obj = self.rrclient.read(
res.instrument_device_id)
assertGeospatialBoundsNotEquals(
before_activate_instrument_device_obj.geospatial_bounds,
after_activate_instrument_device_obj.geospatial_bounds)
deployment_obj = self.RR2.read(res.deployment_id)
self.assertEquals(deployment_obj.lcstate, LCS.DEPLOYED)
log.debug("deactivatin deployment, expecting success")
self.omsclient.deactivate_deployment(res.deployment_id)
after_deactivate_instrument_device_obj = self.rrclient.read(
res.instrument_device_id)
assertGeospatialBoundsNotEquals(
after_activate_instrument_device_obj.geospatial_bounds,
after_deactivate_instrument_device_obj.geospatial_bounds)
deployment_obj = self.RR2.read(res.deployment_id)
self.assertEquals(deployment_obj.lcstate, LCS.INTEGRATED)
def test_activate_deployment_redeploy(self):
dep_util = DeploymentUtil(self.container)
res = self.base_activate_deployment(make_assigns=True)
log.debug("activating first deployment, expecting success")
self.omsclient.activate_deployment(res.deployment_id)
deployment_obj1 = self.RR2.read(res.deployment_id)
self.assertEquals(deployment_obj1.lcstate, LCS.DEPLOYED)
next_dep_info = self._create_subsequent_deployment(res)
deployment_obj2 = self.RR2.read(next_dep_info.deployment_id)
self.assertNotEquals(deployment_obj2.lcstate, LCS.DEPLOYED)
log.debug("activating subsequent deployment, expecting success")
self.omsclient.activate_deployment(next_dep_info.deployment_id)
deployment_obj1 = self.RR2.read(res.deployment_id)
self.assertEquals(deployment_obj1.lcstate, LCS.INTEGRATED)
deployment_obj2 = self.RR2.read(next_dep_info.deployment_id)
self.assertEquals(deployment_obj2.lcstate, LCS.DEPLOYED)
dep1_tc = dep_util.get_temporal_constraint(deployment_obj1)
dep2_tc = dep_util.get_temporal_constraint(deployment_obj2)
self.assertLessEqual(float(dep1_tc.end_datetime),
float(dep2_tc.end_datetime))
log.debug("deactivating second deployment, expecting success")
self.omsclient.deactivate_deployment(next_dep_info.deployment_id)
deployment_obj2 = self.RR2.read(next_dep_info.deployment_id)
self.assertEquals(deployment_obj2.lcstate, LCS.INTEGRATED)
#@unittest.skip("targeting")
def test_activate_deployment_nomodels(self):
res = self.base_activate_deployment()
self.omsclient.assign_site_to_deployment(res.platform_site_id,
res.deployment_id)
self.omsclient.assign_device_to_deployment(res.platform_device_id,
res.deployment_id)
log.debug(
"activating deployment without site+device models, expecting fail")
self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1")
log.debug("assigning instrument site model")
self.omsclient.assign_instrument_model_to_instrument_site(
res.instrument_model_id, res.instrument_site_id)
log.debug(
"activating deployment without device models, expecting fail")
self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1")
#@unittest.skip("targeting")
def test_activate_deployment_nosite(self):
res = self.base_activate_deployment()
log.debug("assigning instrument models")
self.imsclient.assign_instrument_model_to_instrument_device(
res.instrument_model_id, res.instrument_device_id)
self.omsclient.assign_instrument_model_to_instrument_site(
res.instrument_model_id, res.instrument_site_id)
log.debug("deploying instrument device only")
self.omsclient.assign_device_to_deployment(res.instrument_device_id,
res.deployment_id)
log.debug(
"activating deployment without instrument site, expecting fail")
self.assert_deploy_fail(res.deployment_id, BadRequest)
#@unittest.skip("targeting")
def test_activate_deployment_nodevice(self):
res = self.base_activate_deployment()
log.debug("assigning platform and instrument models")
self.imsclient.assign_instrument_model_to_instrument_device(
res.instrument_model_id, res.instrument_device_id)
self.omsclient.assign_instrument_model_to_instrument_site(
res.instrument_model_id, res.instrument_site_id)
log.debug("deploying instrument site only")
self.omsclient.assign_site_to_deployment(res.instrument_site_id,
res.deployment_id)
log.debug("activating deployment without device, expecting fail")
self.assert_deploy_fail(res.deployment_id, BadRequest,
"No devices were found in the deployment")
def assert_deploy_fail(self,
deployment_id,
err_type=BadRequest,
fail_message=""):
with self.assertRaises(err_type) as cm:
self.omsclient.activate_deployment(deployment_id)
log.debug("assert_deploy_fail cm: %s", str(cm))
if fail_message:
self.assertIn(fail_message, cm.exception.message)
def test_3x3_matchups_remoteplatform(self):
self.base_3x3_matchups(IonObject(OT.RemotePlatformDeploymentContext))
def test_3x3_matchups_cabledinstrument(self):
self.base_3x3_matchups(IonObject(OT.CabledInstrumentDeploymentContext))
def test_3x3_matchups_cablednode(self):
self.base_3x3_matchups(IonObject(OT.CabledNodeDeploymentContext))
def base_3x3_matchups(self, deployment_context):
"""
This will be 1 root platform, 3 sub platforms (2 of one model, 1 of another) and 3 sub instruments each (2-to-1)
"""
deployment_context_type = type(deployment_context).__name__
instrument_model_id = [
self.RR2.create(any_old(RT.InstrumentModel)) for _ in range(6)
]
platform_model_id = [
self.RR2.create(any_old(RT.PlatformModel)) for _ in range(3)
]
instrument_device_id = [
self.RR2.create(any_old(RT.InstrumentDevice)) for _ in range(9)
]
platform_device_id = [
self.RR2.create(any_old(RT.PlatformDevice)) for _ in range(4)
]
instrument_site_id = [
self.RR2.create(
any_old(
RT.InstrumentSite, {
"reference_designator":
"GA01SUMO-FI003-0%s-CTDMO0999" % (i + 1),
"planned_uplink_port":
IonObject(
OT.PlatformPort,
reference_designator="GA01SUMO-FI003-0%s-CTDMO0999"
% (i + 1))
})) for i in range(9)
]
platform_site_id = [
self.RR2.create(
any_old(
RT.PlatformSite, {
"reference_designator":
"GA01SUMO-FI003-0%s-CTDMO0888" % (i + 1),
"planned_uplink_port":
IonObject(
OT.PlatformPort,
reference_designator="GA01SUMO-FI003-0%s-CTDMO0888"
% (i + 1))
})) for i in range(4)
]
def instrument_model_at(platform_idx, instrument_idx):
m = platform_idx * 2
if instrument_idx > 0:
m += 1
return m
def platform_model_at(platform_idx):
if platform_idx > 0:
return 1
return 0
def instrument_at(platform_idx, instrument_idx):
return platform_idx * 3 + instrument_idx
# set up the structure
for p in range(3):
m = platform_model_at(p)
self.RR2.assign_platform_model_to_platform_site_with_has_model(
platform_model_id[m], platform_site_id[p])
self.RR2.assign_platform_model_to_platform_device_with_has_model(
platform_model_id[m], platform_device_id[p])
self.RR2.assign_platform_device_to_platform_device_with_has_device(
platform_device_id[p], platform_device_id[3])
self.RR2.assign_platform_site_to_platform_site_with_has_site(
platform_site_id[p], platform_site_id[3])
for i in range(3):
m = instrument_model_at(p, i)
idx = instrument_at(p, i)
self.RR2.assign_instrument_model_to_instrument_site_with_has_model(
instrument_model_id[m], instrument_site_id[idx])
self.RR2.assign_instrument_model_to_instrument_device_with_has_model(
instrument_model_id[m], instrument_device_id[idx])
self.RR2.assign_instrument_device_to_platform_device_with_has_device(
instrument_device_id[idx], platform_device_id[p])
self.RR2.assign_instrument_site_to_platform_site_with_has_site(
instrument_site_id[idx], platform_site_id[p])
# top level models
self.RR2.assign_platform_model_to_platform_device_with_has_model(
platform_model_id[2], platform_device_id[3])
self.RR2.assign_platform_model_to_platform_site_with_has_model(
platform_model_id[2], platform_site_id[3])
# verify structure
for p in range(3):
parent_id = self.RR2.find_platform_device_id_by_platform_device_using_has_device(
platform_device_id[p])
self.assertEqual(platform_device_id[3], parent_id)
parent_id = self.RR2.find_platform_site_id_by_platform_site_using_has_site(
platform_site_id[p])
self.assertEqual(platform_site_id[3], parent_id)
for i in range(len(platform_site_id)):
self.assertEqual(
self.RR2.
find_platform_model_of_platform_device_using_has_model(
platform_device_id[i]),
self.RR2.find_platform_model_of_platform_site_using_has_model(
platform_site_id[i]))
for i in range(len(instrument_site_id)):
self.assertEqual(
self.RR2.
find_instrument_model_of_instrument_device_using_has_model(
instrument_device_id[i]),
self.RR2.
find_instrument_model_of_instrument_site_using_has_model(
instrument_site_id[i]))
# OOIReferenceDesignator format: GA01SUMO-FI003-03-CTDMO0999 (site-platform_id-port-device_id)
port_assignments = {}
for p in range(3):
ref_desig = "GA01SUMO-FI003-0%s-CTDMO0888" % (p + 1)
pp_obj = IonObject(OT.PlatformPort,
reference_designator=ref_desig,
port_type=PortTypeEnum.PAYLOAD,
ip_address=str(p))
port_assignments[platform_device_id[p]] = pp_obj
for i in range(3):
ref_desig = "GA01SUMO-FI003-0%s-CTDMO0999" % ((p * 3) + i + 1)
pp_obj = IonObject(OT.PlatformPort,
reference_designator=ref_desig,
port_type=PortTypeEnum.PAYLOAD,
ip_address=str(p))
idx = instrument_at(p, i)
port_assignments[instrument_device_id[idx]] = pp_obj
deployment_id = self.RR2.create(
any_old(
RT.Deployment, {
"context": deployment_context,
"port_assignments": port_assignments
}))
log.debug("assigning device/site to %s deployment",
deployment_context_type)
if OT.RemotePlatformDeploymentContext == deployment_context_type:
self.RR2.assign_deployment_to_platform_device_with_has_deployment(
deployment_id, platform_device_id[3])
self.RR2.assign_deployment_to_platform_site_with_has_deployment(
deployment_id, platform_site_id[3])
elif OT.CabledInstrumentDeploymentContext == deployment_context_type:
self.RR2.assign_deployment_to_instrument_device_with_has_deployment(
deployment_id, instrument_device_id[1])
self.RR2.assign_deployment_to_instrument_site_with_has_deployment(
deployment_id, instrument_site_id[1])
elif OT.CabledNodeDeploymentContext == deployment_context_type:
self.RR2.assign_deployment_to_platform_device_with_has_deployment(
deployment_id, platform_device_id[1])
self.RR2.assign_deployment_to_platform_site_with_has_deployment(
deployment_id, platform_site_id[1])
log.debug("activation of %s deployment", deployment_context_type)
self.omsclient.activate_deployment(deployment_id)
log.debug("validation of %s deployment", deployment_context_type)
if OT.RemotePlatformDeploymentContext == deployment_context_type:
# verify proper associations
for i, d in enumerate(platform_device_id):
self.assertEqual(
d,
self.RR2.
find_platform_device_id_of_platform_site_using_has_device(
platform_site_id[i]))
for i, d in enumerate(instrument_device_id):
self.assertEqual(
d,
self.RR2.
find_instrument_device_id_of_instrument_site_using_has_device(
instrument_site_id[i]))
elif OT.CabledInstrumentDeploymentContext == deployment_context_type:
self.assertEqual(
instrument_device_id[1],
self.RR2.
find_instrument_device_id_of_instrument_site_using_has_device(
instrument_site_id[1]))
elif OT.CabledNodeDeploymentContext == deployment_context_type:
expected_platforms = [1]
# verify proper associations
for i, d in enumerate(platform_device_id):
self.assertEqual(
i in expected_platforms, d in self.RR2.
find_platform_device_ids_of_platform_site_using_has_device(
platform_site_id[i]))
class TestDataProductProvenance(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.dpmsclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]:
self.dataprocessclient.deactivate_data_process(proc_id)
self.dataprocessclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
def test_get_data_product_provenance_report(self):
#Create a test device
device_obj = Device(name='Device1',
description='test instrument site')
device_id, _ = self.rrclient.create(device_obj)
self.addCleanup(self.rrclient.delete, device_id)
#Create a test DataProduct
data_product1_obj = DataProduct(name='DataProduct1',
description='test data product 1')
data_product1_id, _ = self.rrclient.create(data_product1_obj)
self.addCleanup(self.rrclient.delete, data_product1_id)
#Create a test DataProcess
data_process_obj = DataProcess(name='DataProcess',
description='test data process')
data_process_id, _ = self.rrclient.create(data_process_obj)
self.addCleanup(self.rrclient.delete, data_process_id)
#Create a second test DataProduct
data_product2_obj = DataProduct(name='DataProduct2',
description='test data product 2')
data_product2_id, _ = self.rrclient.create(data_product2_obj)
self.addCleanup(self.rrclient.delete, data_product2_id)
#Create a test DataProducer
data_producer_obj = DataProducer(name='DataProducer',
description='test data producer')
data_producer_id, rev = self.rrclient.create(data_producer_obj)
#Link the DataProcess to the second DataProduct manually
assoc_id, _ = self.rrclient.create_association(subject=data_process_id, predicate=PRED.hasInputProduct, object=data_product2_id)
self.addCleanup(self.rrclient.delete_association, assoc_id)
# Register the instrument and process. This links the device and the data process
# with their own producers
self.damsclient.register_instrument(device_id)
self.addCleanup(self.damsclient.unregister_instrument, device_id)
self.damsclient.register_process(data_process_id)
self.addCleanup(self.damsclient.unregister_process, data_process_id)
#Manually link the first DataProduct with the test DataProducer
assoc_id, _ = self.rrclient.create_association(subject=data_product1_id, predicate=PRED.hasDataProducer, object=data_producer_id)
#Get the DataProducer linked to the DataProcess (created in register_process above)
#Associate that with with DataProduct1's DataProducer
data_process_producer_ids, _ = self.rrclient.find_objects(subject=data_process_id, predicate=PRED.hasDataProducer, object_type=RT.DataProducer, id_only=True)
assoc_id, _ = self.rrclient.create_association(subject=data_process_producer_ids[0], predicate=PRED.hasParent, object=data_producer_id)
self.addCleanup(self.rrclient.delete_association, assoc_id)
#Get the DataProducer linked to the Device (created in register_instrument
#Associate that with the DataProcess's DataProducer
device_producer_ids, _ = self.rrclient.find_objects(subject=device_id, predicate=PRED.hasDataProducer, object_type=RT.DataProducer, id_only=True)
assoc_id, _ = self.rrclient.create_association(subject=data_producer_id, predicate=PRED.hasParent, object=device_producer_ids[0])
#Create the links between the Device, DataProducts, DataProcess, and all DataProducers
self.damsclient.assign_data_product(input_resource_id=device_id, data_product_id=data_product1_id)
self.addCleanup(self.damsclient.unassign_data_product, device_id, data_product1_id)
self.damsclient.assign_data_product(input_resource_id=data_process_id, data_product_id=data_product2_id)
self.addCleanup(self.damsclient.unassign_data_product, data_process_id, data_product2_id)
#Traverse through the relationships to get the links between objects
res = self.dpmsclient.get_data_product_provenance_report(data_product2_id)
#Make sure there are four keys
self.assertEqual(len(res.keys()), 4)
parent_count = 0
config_count = 0
for v in res.itervalues():
if 'parent' in v:
parent_count += 1
if 'config' in v:
config_count += 1
#Make sure there are three parents and four configs
self.assertEqual(parent_count, 3)
self.assertEqual(config_count, 4)
@unittest.skip('This test is obsolete with new framework')
def test_get_provenance(self):
#create a deployment with metadata and an initial site and device
instrument_site_obj = IonObject(RT.InstrumentSite,
name='InstrumentSite1',
description='test instrument site')
instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, "")
log.debug( 'test_get_provenance: new instrument_site_id id = %s ', str(instrument_site_id))
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel" )
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" %ex)
log.debug( 'test_get_provenance: new InstrumentModel id = %s ', str(instModel_id))
self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrument_site_id)
# Create InstrumentAgent
parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict' )
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=DRV_URI_GOOD,
stream_configurations = [parsed_config] )
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" %ex)
log.debug( 'test_get_provenance:new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_get_provenance: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
log.debug("test_get_provenance: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id)
#-------------------------------
# Create CTD Parsed data product
#-------------------------------
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
log.debug( 'test_get_provenance:Creating new CDM data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product = self.dpmsclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', ctd_parsed_data_product)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product)
#-------------------------------
# create a data product for the site to pass the OMS check.... we need to remove this check
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
log_data_product_id = self.dpmsclient.create_data_product(dp_obj, parsed_stream_def_id)
#-------------------------------
# Deploy instrument device to instrument site
#-------------------------------
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment')
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id)
self.imsclient.deploy_instrument_device(instDevice_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
self.omsclient.activate_deployment(deployment_id)
inst_device_objs, _ = self.rrclient.find_objects(subject=instrument_site_id, predicate=PRED.hasDevice, object_type=RT.InstrumetDevice, id_only=False)
log.debug("test_create_deployment: deployed device: %s ", str(inst_device_objs[0]) )
#-------------------------------
# Create the agent instance
#-------------------------------
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
#-------------------------------
# L1 Conductivity: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1ConductivityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_conductivity',
description='create the L1 conductivity data product',
module='ion.processes.data.transforms.ctd.ctd_L1_conductivity',
class_name='CTDL1ConductivityTransform')
try:
ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex)
#-------------------------------
# L1 Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1PressureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_pressure',
description='create the L1 pressure data product',
module='ion.processes.data.transforms.ctd.ctd_L1_pressure',
class_name='CTDL1PressureTransform')
try:
ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex)
#-------------------------------
# L1 Temperature: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1TemperatureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_temperature',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L1_temperature',
class_name='CTDL1TemperatureTransform')
try:
ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex)
#-------------------------------
# L2 Salinity: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition SalinityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_salinity',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new SalinityTransform data process definition: %s" %ex)
#-------------------------------
# L2 Density: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition DensityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_density',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_density',
class_name='DensityTransform')
try:
ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new DensityTransform data process definition: %s" %ex)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' )
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' )
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' )
log.debug("TestDataProductProvenance: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
log.debug("TestDataProductProvenance: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_pressure_output_dp_obj,
outgoing_stream_l0_pressure_id)
log.debug("TestDataProductProvenance: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
#-------------------------------
# L1 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(name='L1_conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id, binding='conductivity' )
outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(name='L1_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id, binding='pressure' )
outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(name='L1_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id, binding='temperature' )
log.debug("TestDataProductProvenance: create output data product L1 conductivity")
ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct,
name='L1_Conductivity',
description='transform output L1 conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_conductivity_output_dp_obj,
outgoing_stream_l1_conductivity_id)
log.debug("TestDataProductProvenance: create output data product L1 pressure")
ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Pressure',
description='transform output L1 pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_pressure_output_dp_obj,
outgoing_stream_l1_pressure_id)
log.debug("TestDataProductProvenance: create output data product L1 temperature")
ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Temperature',
description='transform output L1 temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_temperature_output_dp_obj,
outgoing_stream_l1_temperature_id)
#-------------------------------
# L2 Salinity - Density: Output Data Products
#-------------------------------
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(name='L2_salinity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id, binding='salinity' )
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(name='L2_Density', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id, binding='density' )
log.debug("TestDataProductProvenance: create output data product L2 Salinity")
ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Salinity',
description='transform output L2 salinity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_salinity_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_salinity_output_dp_obj,
outgoing_stream_l2_salinity_id)
log.debug("TestDataProductProvenance: create output data product L2 Density")
# ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
# name='L2_Density',
# description='transform output pressure',
# temporal_domain = tdom,
# spatial_domain = sdom)
#
# ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj,
# outgoing_stream_l2_density_id,
# parameter_dictionary)
contactInfo = ContactInformation()
contactInfo.individual_names_given = "Bill"
contactInfo.individual_name_family = "Smith"
contactInfo.street_address = "111 First St"
contactInfo.city = "San Diego"
contactInfo.email = "*****@*****.**"
contactInfo.phones = ["858-555-6666"]
contactInfo.country = "USA"
contactInfo.postal_code = "92123"
ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Density',
description='transform output pressure',
contacts = [contactInfo],
iso_topic_category = "my_iso_topic_category_here",
quality_control_level = "1",
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj,
outgoing_stream_l2_density_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L0 all data_process start")
try:
input_data_products = [ctd_parsed_data_product]
output_data_products = [ctd_l0_conductivity_output_dp_id, ctd_l0_pressure_output_dp_id, ctd_l0_temperature_output_dp_id]
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L0_all_dprocdef_id,
in_data_product_ids = input_data_products,
out_data_product_ids = output_data_products
)
#activate only this data process just for coverage
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
contents = "this is the lookup table contents, replace with a file..."
att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
deviceAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att)
log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment)
log.debug("TestDataProductProvenance: create L0 all data_process return")
#-------------------------------
# L1 Conductivity: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1 Conductivity data_process start")
try:
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L1_conductivity_dprocdef_id,
in_data_product_ids = [ctd_l0_conductivity_output_dp_id],
out_data_product_ids = [ctd_l1_conductivity_output_dp_id])
self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L1 Pressure: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1_Pressure data_process start")
try:
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L1_pressure_dprocdef_id,
in_data_product_ids = [ctd_l0_pressure_output_dp_id],
out_data_product_ids = [ctd_l1_pressure_output_dp_id])
self.dataprocessclient.activate_data_process(l1_pressure_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L1 Temperature: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1_Pressure data_process start")
try:
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L1_temperature_dprocdef_id,
in_data_product_ids = [ctd_l0_temperature_output_dp_id],
out_data_product_ids = [ctd_l1_temperature_output_dp_id])
self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L2 Salinity: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L2_salinity data_process start")
try:
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L2_salinity_dprocdef_id,
in_data_product_ids = [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id],
out_data_product_ids = [ctd_l2_salinity_output_dp_id])
self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L2 Density: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L2_Density data_process start")
try:
in_dp_ids = [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id]
out_dp_ids = [ctd_l2_density_output_dp_id]
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L2_density_dprocdef_id,
in_data_product_ids = in_dp_ids,
out_data_product_ids = out_dp_ids)
self.dataprocessclient.activate_data_process(l2_density_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
print 'TestDataProductProvenance: Instrument agent instance obj: = ', inst_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
# self._ia_client = ResourceAgentClient('iaclient', name=ResourceAgentClient._get_agent_process_id(instDevice_id, process=FakeProcess())
# print 'activate_instrument: got ia client %s', self._ia_client
# log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client))
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.dataprocessclient.deactivate_data_process(l2_density_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l2_salinity_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_temperature_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_pressure_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_conductivity_data_process_id)
self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id)
#-------------------------------
# Retrieve the provenance info for the ctd density data product
#-------------------------------
provenance_dict = self.dpmsclient.get_data_product_provenance(ctd_l2_density_output_dp_id)
log.debug("TestDataProductProvenance: provenance_dict %s", str(provenance_dict))
#validate that products are represented
self.assertTrue (provenance_dict[str(ctd_l1_conductivity_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l0_conductivity_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l2_density_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l1_temperature_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l0_temperature_output_dp_id)])
density_dict = (provenance_dict[str(ctd_l2_density_output_dp_id)])
self.assertEquals(density_dict['producer'], [l2_density_all_data_process_id])
#-------------------------------
# Retrieve the extended resource for this data product
#-------------------------------
extended_product = self.dpmsclient.get_data_product_extension(ctd_l2_density_output_dp_id)
self.assertEqual(1, len(extended_product.data_processes) )
self.assertEqual(3, len(extended_product.process_input_data_products) )
# log.debug("TestDataProductProvenance: DataProduct provenance_product_list %s", str(extended_product.provenance_product_list))
# log.debug("TestDataProductProvenance: DataProduct data_processes %s", str(extended_product.data_processes))
# log.debug("TestDataProductProvenance: DataProduct process_input_data_products %s", str(extended_product.process_input_data_products))
# log.debug("TestDataProductProvenance: provenance %s", str(extended_product.computed.provenance.value))
#-------------------------------
# Retrieve the extended resource for this data process
#-------------------------------
extended_process_def = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id)
# log.debug("TestDataProductProvenance: DataProcess extended_process_def %s", str(extended_process_def))
# log.debug("TestDataProductProvenance: DataProcess data_processes %s", str(extended_process_def.data_processes))
# log.debug("TestDataProductProvenance: DataProcess data_products %s", str(extended_process_def.data_products))
self.assertEqual(1, len(extended_process_def.data_processes) )
self.assertEqual(3, len(extended_process_def.output_stream_definitions) )
self.assertEqual(3, len(extended_process_def.data_products) ) #one list because of one data process
#-------------------------------
# Request the xml report
#-------------------------------
results = self.dpmsclient.get_data_product_provenance_report(ctd_l2_density_output_dp_id)
print results
#-------------------------------
# Cleanup
#-------------------------------
self.dpmsclient.delete_data_product(ctd_parsed_data_product)
self.dpmsclient.delete_data_product(log_data_product_id)
self.dpmsclient.delete_data_product(ctd_l0_conductivity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l0_pressure_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l0_temperature_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_conductivity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_pressure_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_temperature_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l2_salinity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l2_density_output_dp_id)
class TestIMSDeployAsPrimaryDevice(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
#self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
print 'started services'
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(
node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(
RT.DataProcess, None, None, True)[0]:
self.dataprocessclient.deactivate_data_process(proc_id)
self.dataprocessclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name + '_logger')
producer_definition.executable = {
'module': 'ion.processes.data.stream_granule_logger',
'class': 'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(
process_definition=producer_definition)
configuration = {
'process': {
'stream_id': stream_id,
}
}
pid = self.processdispatchclient.schedule_process(
process_definition_id=logger_procdef_id,
configuration=configuration)
return pid
def cleanupprocs(self):
stm = os.popen('ps -e | grep ion.agents.port.logger_process')
procs = stm.read()
if len(procs) > 0:
procs = procs.split()
if procs[0].isdigit():
pid = int(procs[0])
os.kill(pid, signal.SIGKILL)
stm = os.popen('ps -e | grep ion.agents.instrument.zmq_driver_process')
procs = stm.read()
if len(procs) > 0:
procs = procs.split()
if procs[0].isdigit():
pid = int(procs[0])
os.kill(pid, signal.SIGKILL)
# stm = os.popen('rm /tmp/*.pid.txt')
@unittest.skip(
"Deprecated by IngestionManagement refactor, timeout on start inst agent?"
)
def test_deploy_activate_full(self):
# ensure no processes or pids are left around by agents or Sims
#self.cleanupprocs()
self.loggerpids = []
#-------------------------------
# Create InstrumentModel
#-------------------------------
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
try:
instModel_id = self.imsclient.create_instrument_model(
instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" % ex)
#-------------------------------
# Create InstrumentAgent
#-------------------------------
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=DRV_URI_GOOD)
try:
instAgent_id = self.imsclient.create_instrument_agent(
instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" % ex)
log.debug('new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
#-------------------------------
# Create Instrument Site
#-------------------------------
instrumentSite_obj = IonObject(RT.InstrumentSite,
name='instrumentSite1',
description="SBE37IMInstrumentSite")
try:
instrumentSite_id = self.omsclient.create_instrument_site(
instrument_site=instrumentSite_obj, parent_id='')
except BadRequest as ex:
self.fail("failed to create new InstrumentSite: %s" % ex)
print 'test_deployAsPrimaryDevice: new instrumentSite id = ', instrumentSite_id
self.omsclient.assign_instrument_model_to_instrument_site(
instModel_id, instrumentSite_id)
#-------------------------------
# Logical Transform: Output Data Products
#-------------------------------
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubclient.create_stream_definition(
name='parsed', parameter_dictionary_id=parsed_pdict_id)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.pubsubclient.create_stream_definition(
name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Old InstrumentDevice
#-------------------------------
instDevice_obj = IonObject(
RT.InstrumentDevice,
name='SBE37IMDeviceYear1',
description="SBE37IMDevice for the FIRST year of deployment",
serial_number="12345")
try:
oldInstDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, oldInstDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
print 'test_deployAsPrimaryDevice: new Year 1 InstrumentDevice id = ', oldInstDevice_id
self.rrclient.execute_lifecycle_transition(oldInstDevice_id,
LCE.DEPLOY)
self.rrclient.execute_lifecycle_transition(oldInstDevice_id,
LCE.ENABLE)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='SiteDataProduct',
description='SiteDataProduct',
temporal_domain=tdom,
spatial_domain=sdom)
instrument_site_output_dp_id = self.dataproductclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
self.damsclient.assign_data_product(
input_resource_id=oldInstDevice_id,
data_product_id=instrument_site_output_dp_id)
#self.dataproductclient.activate_data_product_persistence(data_product_id=instrument_site_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(
instrument_site_output_dp_id, PRED.hasStream, None, True)
log.debug('Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(
instrument_site_output_dp_id, PRED.hasDataset, RT.Dataset, True)
log.debug('Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
pid = self.create_logger('ctd_parsed', stream_ids[0])
self.loggerpids.append(pid)
#-------------------------------
# Create Old Deployment
#-------------------------------
deployment_obj = IonObject(RT.Deployment, name='first deployment')
oldDeployment_id = self.omsclient.create_deployment(deployment_obj)
# deploy this device to the logical slot
self.imsclient.deploy_instrument_device(oldInstDevice_id,
oldDeployment_id)
self.omsclient.deploy_instrument_site(instrumentSite_id,
oldDeployment_id)
#-------------------------------
# Create InstrumentAgentInstance for OldInstrumentDevice to hold configuration information
# cmd_port=5556, evt_port=5557, comms_method="ethernet", comms_device_address=CFG.device.sbe37.host, comms_device_port=CFG.device.sbe37.port,
#-------------------------------
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
raw_config = StreamConfiguration(
stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict')
parsed_config = StreamConfiguration(
stream_name='parsed',
parameter_dictionary_name='ctd_parsed_param_dict')
instAgentInstance_obj = IonObject(
RT.InstrumentAgentInstance,
name='SBE37IMAgentInstanceYear1',
description="SBE37IMAgentInstanceYear1",
port_agent_config=port_agent_config,
stream_configurations=[raw_config, parsed_config])
oldInstAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, oldInstDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
#-------------------------------
# Create CTD Parsed as the Year 1 data product and attach to instrument
#-------------------------------
print 'Creating new CDM data product with a stream definition'
dp_obj = IonObject(RT.DataProduct,
name='ctd_parsed_year1',
description='ctd stream test year 1',
temporal_domain=tdom,
spatial_domain=sdom)
ctd_parsed_data_product_year1 = self.dataproductclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product_year1
self.damsclient.assign_data_product(
input_resource_id=oldInstDevice_id,
data_product_id=ctd_parsed_data_product_year1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(
ctd_parsed_data_product_year1, PRED.hasStream, None, True)
print 'test_deployAsPrimaryDevice: Data product streams1 = ', stream_ids
#-------------------------------
# Create New InstrumentDevice
#-------------------------------
instDevice_obj_2 = IonObject(
RT.InstrumentDevice,
name='SBE37IMDeviceYear2',
description="SBE37IMDevice for the SECOND year of deployment",
serial_number="67890")
try:
newInstDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj_2)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, newInstDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
print 'test_deployAsPrimaryDevice: new Year 2 InstrumentDevice id = ', newInstDevice_id
#set the LCSTATE
self.rrclient.execute_lifecycle_transition(newInstDevice_id,
LCE.DEPLOY)
self.rrclient.execute_lifecycle_transition(newInstDevice_id,
LCE.ENABLE)
instDevice_obj_2 = self.rrclient.read(newInstDevice_id)
log.debug(
"test_deployAsPrimaryDevice: Create New InstrumentDevice LCSTATE: %s ",
str(instDevice_obj_2.lcstate))
#-------------------------------
# Create Old Deployment
#-------------------------------
deployment_obj = IonObject(RT.Deployment, name='second deployment')
newDeployment_id = self.omsclient.create_deployment(deployment_obj)
# deploy this device to the logical slot
self.imsclient.deploy_instrument_device(newInstDevice_id,
newDeployment_id)
self.omsclient.deploy_instrument_site(instrumentSite_id,
newDeployment_id)
#-------------------------------
# Create InstrumentAgentInstance for NewInstrumentDevice to hold configuration information
#-------------------------------
port_agent_config = {
'device_addr': 'sbe37-simulator.oceanobservatories.org',
'device_port': 4004,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': 4005,
'data_port': 4006,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(
RT.InstrumentAgentInstance,
name='SBE37IMAgentInstanceYear2',
description="SBE37IMAgentInstanceYear2",
port_agent_config=port_agent_config)
newInstAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, newInstDevice_id)
#-------------------------------
# Create CTD Parsed as the Year 2 data product
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='ctd_parsed_year2',
description='ctd stream test year 2',
temporal_domain=tdom,
spatial_domain=sdom)
ctd_parsed_data_product_year2 = self.dataproductclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product_year2
self.damsclient.assign_data_product(
input_resource_id=newInstDevice_id,
data_product_id=ctd_parsed_data_product_year2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(
ctd_parsed_data_product_year2, PRED.hasStream, None, True)
print 'test_deployAsPrimaryDevice: Data product streams2 = ', stream_ids
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug(
"test_deployAsPrimaryDevice: create data process definition ctd_L0_all"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new ctd_L0_all data process definition: %s" %
ex)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(
name='L0_Conductivity', parameter_dictionary_id=parsed_pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_conductivity_id,
ctd_L0_all_dprocdef_id,
binding='conductivity')
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(
name='L0_Pressure', parameter_dictionary_id=parsed_pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_pressure_id,
ctd_L0_all_dprocdef_id,
binding='pressure')
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(
name='L0_Temperature', parameter_dictionary_id=parsed_pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_temperature_id,
ctd_L0_all_dprocdef_id,
binding='temperature')
self.out_prod_dict = {}
log.debug(
"test_deployAsPrimaryDevice: create output data product L0 conductivity"
)
ctd_l0_conductivity_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain=tdom,
spatial_domain=sdom)
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(
data_product=ctd_l0_conductivity_output_dp_obj,
stream_definition_id=parsed_stream_def_id)
self.out_prod_dict['conductivity'] = ctd_l0_conductivity_output_dp_id
#self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id)
log.debug(
"test_deployAsPrimaryDevice: create output data product L0 pressure"
)
ctd_l0_pressure_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain=tdom,
spatial_domain=sdom)
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(
data_product=ctd_l0_pressure_output_dp_obj,
stream_definition_id=parsed_stream_def_id)
self.out_prod_dict['pressure'] = ctd_l0_pressure_output_dp_id
#self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id)
log.debug(
"test_deployAsPrimaryDevice: create output data product L0 temperature"
)
ctd_l0_temperature_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain=tdom,
spatial_domain=sdom)
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(
data_product=ctd_l0_temperature_output_dp_obj,
stream_definition_id=parsed_stream_def_id)
self.out_prod_dict['temperature'] = ctd_l0_temperature_output_dp_id
#self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process, listening to Sim1 (later: logical instrument output product)
#-------------------------------
log.debug(
"test_deployAsPrimaryDevice: create L0 all data_process start")
try:
out_data_products = self.out_prod_dict.values()
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(
ctd_L0_all_dprocdef_id, [ctd_parsed_data_product_year1],
out_data_products)
self.dataprocessclient.activate_data_process(
ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug(
"test_deployAsPrimaryDevice: create L0 all data_process return")
#--------------------------------
# Activate the deployment
#--------------------------------
self.omsclient.activate_deployment(oldDeployment_id)
#-------------------------------
# Launch InstrumentAgentInstance Sim1, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=oldInstAgentInstance_id)
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=oldInstAgentInstance_id)
#wait for start
instance_obj = self.imsclient.read_instrument_agent_instance(
oldInstAgentInstance_id)
gate = AgentProcessStateGate(self.processdispatchclient.read_process,
oldInstDevice_id,
ProcessStateEnum.RUNNING)
self.assertTrue(
gate. await (30),
"The instrument agent instance (%s) did not spawn in 30 seconds" %
gate.process_id)
inst_agent1_instance_obj = self.imsclient.read_instrument_agent_instance(
oldInstAgentInstance_id)
print 'test_deployAsPrimaryDevice: Instrument agent instance obj: = ', inst_agent1_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client_sim1 = ResourceAgentClient('iaclient Sim1',
name=gate.process_id,
process=FakeProcess())
print 'activate_instrument: got _ia_client_sim1 %s', self._ia_client_sim1
log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim1 %s",
str(self._ia_client_sim1))
#-------------------------------
# Launch InstrumentAgentInstance Sim2, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=newInstAgentInstance_id)
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=newInstAgentInstance_id)
#wait for start
instance_obj = self.imsclient.read_instrument_agent_instance(
newInstAgentInstance_id)
gate = AgentProcessStateGate(self.processdispatchclient.read_process,
oldInstDevice_id,
ProcessStateEnum.RUNNING)
self.assertTrue(
gate. await (30),
"The instrument agent instance (%s) did not spawn in 30 seconds" %
gate.process_id)
inst_agent2_instance_obj = self.imsclient.read_instrument_agent_instance(
newInstAgentInstance_id)
print 'test_deployAsPrimaryDevice: Instrument agent instance obj: = ', inst_agent2_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client_sim2 = ResourceAgentClient('iaclient Sim2',
name=gate.process_id,
process=FakeProcess())
print 'activate_instrument: got _ia_client_sim2 %s', self._ia_client_sim2
log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim2 %s",
str(self._ia_client_sim2))
#-------------------------------
# Streaming Sim1 (old instrument)
#-------------------------------
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice: initialize %s", str(retval))
log.debug("(L4-CI-SA-RQ-334): Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice: return value from go_active %s",
str(reply))
self.assertTrue(reply)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client_sim1.execute_agent(cmd)
state = retval.result
log.debug(
"(L4-CI-SA-RQ-334): current state after sending go_active command %s",
str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice: run %s", str(reply))
gevent.sleep(2)
cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
retval = self._ia_client_sim1.execute_resource(cmd)
log.debug(
"test_activateInstrumentSample: return from START_AUTOSAMPLE: %s",
str(retval))
#-------------------------------
# Streaming Sim 2 (new instrument)
#-------------------------------
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client_sim2.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice: initialize_sim2 %s",
str(retval))
log.debug("(L4-CI-SA-RQ-334): Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client_sim2.execute_agent(cmd)
log.debug(
"test_deployAsPrimaryDevice: return value from go_active_sim2 %s",
str(reply))
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client_sim2.execute_agent(cmd)
state = retval.result
log.debug(
"(L4-CI-SA-RQ-334): current state after sending go_active_sim2 command %s",
str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client_sim2.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice: run %s", str(reply))
gevent.sleep(2)
cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
retval = self._ia_client_sim2.execute_resource(cmd)
log.debug(
"test_activateInstrumentSample: return from START_AUTOSAMPLE_sim2: %s",
str(retval))
gevent.sleep(10)
#-------------------------------
# Shutdown Sim1 (old instrument)
#-------------------------------
cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
retval = self._ia_client_sim1.execute_resource(cmd)
log.debug(
"test_activateInstrumentSample: return from STOP_AUTOSAMPLE: %s",
str(retval))
log.debug("test_activateInstrumentSample: calling reset ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset %s",
str(reply))
time.sleep(5)
#-------------------------------
# Shutdown Sim2 (old instrument)
#-------------------------------
cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
retval = self._ia_client_sim2.execute_resource(cmd)
log.debug(
"test_activateInstrumentSample: return from STOP_AUTOSAMPLE_sim2: %s",
str(retval))
log.debug("test_activateInstrumentSample: calling reset_sim2 ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset_sim2 %s",
str(reply))
time.sleep(5)
