def setUp(self):
self._start_container()
# self.container.start_rel_from_url(rel_url='res/deploy/r2deploy.yml')
self.container.start_rel_from_url(rel_url='res/deploy/r2eoi.yml')
self.eoas_cli = ExternalObservatoryAgentServiceClient()
self.rr_cli = ResourceRegistryServiceClient()
self.pubsub_cli = PubsubManagementServiceClient()
self.dams_cli = DataAcquisitionManagementServiceClient()
self._setup_ncom()
self._setup_hfr()
class TestIntExternalObservatoryAgentService(IonIntegrationTestCase):
def setUp(self):
self._start_container()
# self.container.start_rel_from_url(rel_url='res/deploy/r2deploy.yml')
self.container.start_rel_from_url(rel_url='res/deploy/r2eoi.yml')
self.eoas_cli = ExternalObservatoryAgentServiceClient()
self.rr_cli = ResourceRegistryServiceClient()
self.pubsub_cli = PubsubManagementServiceClient()
self.dams_cli = DataAcquisitionManagementServiceClient()
self._setup_ncom()
self._setup_hfr()
def _setup_ncom(self):
# TODO: some or all of this (or some variation) should move to DAMS
# Create and register the necessary resources/objects
# Create DataProvider
dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation())
# dprov.institution.name = "OOI CGSN"
dprov.contact.name = "Robert Weller"
dprov.contact.email = "*****@*****.**"
# Create DataSource
dsrc = DataSource(protocol_type="DAP", institution=Institution(), contact=ContactInformation())
# dsrc.connection_params["base_data_url"] = "http://ooi.whoi.edu/thredds/dodsC/"
dsrc.connection_params["base_data_url"] = ""
dsrc.contact.name="Rich Signell"
dsrc.contact.email = "*****@*****.**"
# Create ExternalDataset
dset = ExternalDataset(name="test", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
# dset.dataset_description.parameters["dataset_path"] = "ooi/AS02CPSM_R_M.nc"
dset.dataset_description.parameters["dataset_path"] = "test_data/ncom.nc"
dset.dataset_description.parameters["temporal_dimension"] = "time"
dset.dataset_description.parameters["zonal_dimension"] = "lon"
dset.dataset_description.parameters["meridional_dimension"] = "lat"
# Create ExternalDataSourceModel
dsrc_model = ExternalDataSourceModel(name="dap_model")
dsrc_model.model = "DAP"
dsrc_model.data_handler_module = "eoi.agent.handler.dap_external_data_handler"
dsrc_model.data_handler_class = "DapExternalDataHandler"
## Run everything through DAMS
#TODO: Uncomment when CRUD methods in DAMS are implemented
# self.ncom_ds_id = self.dams_cli.create_external_dataset(external_dataset=dset)
# ext_dprov_id = self.dams_cli.create_external_data_provider(external_data_provider=dprov)
# ext_dsrc_id = self.dams_cli.create_data_source(data_source=dsrc)
self.ncom_ds_id, _ = self.rr_cli.create(dset)
ext_dprov_id, _ = self.rr_cli.create(dprov)
ext_dsrc_id, _ = self.rr_cli.create(dsrc)
#TODO: this needs to be added to DAMS
ext_dsrc_model_id, _ = self.rr_cli.create(dsrc_model)
## Associate everything
self.rr_cli.create_association(self.ncom_ds_id, PRED.hasSource, ext_dsrc_id)
log.debug("Associated ExternalDataset %s with DataSource %s" % (self.ncom_ds_id, ext_dsrc_id))
self.rr_cli.create_association(ext_dsrc_id, PRED.hasProvider, ext_dprov_id)
log.debug("Associated DataSource %s with ExternalDataProvider %s" % (ext_dsrc_id, ext_dprov_id))
self.rr_cli.create_association(ext_dsrc_id, PRED.hasModel, ext_dsrc_model_id)
log.debug("Associated DataSource %s with ExternalDataSourceModel %s" % (ext_dsrc_id, ext_dsrc_model_id))
data_prod_id = self.dams_cli.register_external_data_set(self.ncom_ds_id)
log.debug("Registered ExternalDataset {%s}: DataProducer ID = %s" % (self.ncom_ds_id, data_prod_id))
def _setup_hfr(self):
# TODO: some or all of this (or some variation) should move to DAMS
# Create and register the necessary resources/objects
# Create DataProvider
dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation())
# dprov.institution.name = "HFR UCSD"
# Create DataSource
dsrc = DataSource(protocol_type="DAP", institution=Institution(), contact=ContactInformation())
dsrc.connection_params["base_data_url"] = "http://hfrnet.ucsd.edu:8080/thredds/dodsC/"
# Create ExternalDataset
dset = ExternalDataset(name="UCSD HFR", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
dset.dataset_description.parameters["dataset_path"] = "HFRNet/USEGC/6km/hourly/RTV"
dset.dataset_description.parameters["temporal_dimension"] = "time"
dset.dataset_description.parameters["zonal_dimension"] = "lon"
dset.dataset_description.parameters["meridional_dimension"] = "lat"
# Create ExternalDataSourceModel
dsrc_model = ExternalDataSourceModel(name="dap_model")
dsrc_model.model = "DAP"
dsrc_model.data_handler_module = "eoi.agent.handler.dap_external_data_handler"
dsrc_model.data_handler_class = "DapExternalDataHandler"
## Run everything through DAMS
#TODO: Uncomment when CRUD methods in DAMS are implemented
# self.ncom_ds_id = self.dams_cli.create_external_dataset(external_dataset=dset)
# ext_dprov_id = self.dams_cli.create_external_data_provider(external_data_provider=dprov)
# ext_dsrc_id = self.dams_cli.create_data_source(data_source=dsrc)
self.hfr_ds_id, _ = self.rr_cli.create(dset)
ext_dprov_id, _ = self.rr_cli.create(dprov)
ext_dsrc_id, _ = self.rr_cli.create(dsrc)
#TODO: this needs to be added to DAMS
ext_dsrc_model_id, _ = self.rr_cli.create(dsrc_model)
self.rr_cli.create_association(self.hfr_ds_id, PRED.hasSource, ext_dsrc_id)
log.debug("Associated ExternalDataset %s with DataSource %s" % (self.hfr_ds_id, ext_dsrc_id))
self.rr_cli.create_association(ext_dsrc_id, PRED.hasProvider, ext_dprov_id)
log.debug("Associated DataSource %s with ExternalDataProvider %s" % (ext_dsrc_id, ext_dprov_id))
self.rr_cli.create_association(ext_dsrc_id, PRED.hasModel, ext_dsrc_model_id)
log.debug("Associated DataSource %s with ExternalDataSourceModel %s" % (ext_dsrc_id, ext_dsrc_model_id))
data_prod_id = self.dams_cli.register_external_data_set(self.hfr_ds_id)
log.debug("Registered ExternalDataset {%s}: DataProducer ID = %s" % (self.hfr_ds_id, data_prod_id))
########## Tests ##########
# @unittest.skip("")
def test_spawn_worker(self):
log.debug("test_spawn_worker with ds_id: %s" % self.ncom_ds_id)
proc_name, pid, queue_id = self.eoas_cli.spawn_worker(self.ncom_ds_id)
log.debug("proc_name: %s\tproc_id: %s\tqueue_id: %s" % (proc_name, pid, queue_id))
self.assertEquals(proc_name, self.ncom_ds_id+'_worker')
@unittest.skip("Underlying method not yet implemented")
def test_get_worker(self, resource_id=''):
log.debug("test_spawn_worker with ds_id: %s" % self.ncom_ds_id)
proc_name, pid, queue_id = self.eoas_cli.spawn_worker(self.ncom_ds_id)
log.debug("proc_name: %s\tproc_id: %s\tqueue_id: %s" % (proc_name, pid, queue_id))
with self.assertRaises(IonException):
self.eoas_cli.get_worker(resource_id=self.ncom_ds_id)
@unittest.skip("Underlying method not yet implemented")
def test_get_capabilities(self):
log.debug("test_spawn_worker with ds_id: %s" % self.ncom_ds_id)
proc_name, pid, queue_id = self.eoas_cli.spawn_worker(self.ncom_ds_id)
log.debug("proc_name: %s\tproc_id: %s\tqueue_id: %s" % (proc_name, pid, queue_id))
with self.assertRaises(IonException):
self.eoas_cli.get_capabilities()
# @unittest.skip("")
def test_execute_single_worker(self):
ds_id = self.ncom_ds_id
log.debug("test_spawn_worker with ds_id: %s" % ds_id)
proc_name, pid, queue_id = self.eoas_cli.spawn_worker(ds_id)
log.debug("proc_name: %s\tproc_id: %s\tqueue_id: %s" % (proc_name, pid, queue_id))
# with self.assertRaises(IonException):
# self.eoas_cli.execute()
cmd = AgentCommand(command_id="111", command="get_attributes", kwargs={"ds_id":ds_id})
log.debug("Execute AgentCommand: %s" % cmd)
ret = self.eoas_cli.execute(command=cmd)
log.debug("Returned: %s" % ret)
self.assertEquals(ret.status, "SUCCESS")
self.assertTrue(type(ret.result[0]), dict)
cmd = AgentCommand(command_id="112", command="get_signature", kwargs={"ds_id":ds_id})
log.debug("Execute AgentCommand: %s" % cmd)
ret = self.eoas_cli.execute(command=cmd)
log.debug("Returned: %s" % ret)
self.assertEquals(ret.status, "SUCCESS")
self.assertTrue(type(ret.result[0]), dict)
# @unittest.skip("")
def test_execute_multi_worker(self):
log.debug("test_spawn_worker #1 with ds_id: %s" % self.ncom_ds_id)
proc_name, pid, queue_id = self.eoas_cli.spawn_worker(self.ncom_ds_id)
log.debug("proc_name: %s\tproc_id: %s\tqueue_id: %s" % (proc_name, pid, queue_id))
log.debug("test_spawn_worker #2 with ds_id: %s" % self.hfr_ds_id)
proc_name, pid, queue_id = self.eoas_cli.spawn_worker(self.hfr_ds_id)
log.debug("proc_name: %s\tproc_id: %s\tqueue_id: %s" % (proc_name, pid, queue_id))
cmd = AgentCommand(command_id="112", command="get_signature", kwargs={"ds_id":self.ncom_ds_id})
log.debug("Execute AgentCommand: %s" % cmd)
ret = self.eoas_cli.execute(command=cmd)
log.debug("Returned: %s" % ret)
self.assertEquals(ret.status, "SUCCESS")
self.assertTrue(type(ret.result[0]), dict)
cmd = AgentCommand(command_id="112", command="get_signature", kwargs={"ds_id":self.hfr_ds_id})
log.debug("Execute AgentCommand: %s" % cmd)
ret = self.eoas_cli.execute(command=cmd)
log.debug("Returned: %s" % ret)
self.assertEquals(ret.status, "SUCCESS")
self.assertTrue(type(ret.result[0]), dict)
# @unittest.skip("")
def test_execute_acquire_data(self):
ds_id = self.ncom_ds_id
log.debug("test_spawn_worker with ds_id: %s" % ds_id)
proc_name, pid, queue_id = self.eoas_cli.spawn_worker(ds_id)
log.debug("proc_name: %s\tproc_id: %s\tqueue_id: %s" % (proc_name, pid, queue_id))
cmd = AgentCommand(command_id="113", command="acquire_data", kwargs={"ds_id":ds_id})
log.debug("Execute AgentCommand: %s" % cmd)
ret = self.eoas_cli.execute(command=cmd)
log.debug("Returned: %s" % ret)
self.assertEquals(ret.status, "SUCCESS")
@unittest.skip("Underlying method not yet implemented")
def test_set_param(self):
log.debug("test_spawn_worker with ds_id: %s" % self.ncom_ds_id)
proc_name, pid, queue_id = self.eoas_cli.spawn_worker(self.ncom_ds_id)
log.debug("proc_name: %s\tproc_id: %s\tqueue_id: %s" % (proc_name, pid, queue_id))
with self.assertRaises(IonException):
self.eoas_cli.set_param(name="param", value="value")
@unittest.skip("Underlying method not yet implemented")
def test_get_param(self):
log.debug("test_spawn_worker with ds_id: %s" % self.ncom_ds_id)
proc_name, pid, queue_id = self.eoas_cli.spawn_worker(self.ncom_ds_id)
log.debug("proc_name: %s\tproc_id: %s\tqueue_id: %s" % (proc_name, pid, queue_id))
with self.assertRaises(IonException):
self.eoas_cli.get_param(name="param")
@unittest.skip("Underlying method not yet implemented")
def test_execute_agent(self):
log.debug("test_spawn_worker with ds_id: %s" % self.ncom_ds_id)
proc_name, pid, queue_id = self.eoas_cli.spawn_worker(self.ncom_ds_id)
log.debug("proc_name: %s\tproc_id: %s\tqueue_id: %s" % (proc_name, pid, queue_id))
with self.assertRaises(IonException):
self.eoas_cli.execute_agent()
@unittest.skip("Underlying method not yet implemented")
def test_set_agent_param(self):
log.debug("test_spawn_worker with ds_id: %s" % self.ncom_ds_id)
proc_name, pid, queue_id = self.eoas_cli.spawn_worker(self.ncom_ds_id)
log.debug("proc_name: %s\tproc_id: %s\tqueue_id: %s" % (proc_name, pid, queue_id))
with self.assertRaises(IonException):
self.eoas_cli.set_agent_param(name="param", value="value")
@unittest.skip("Underlying method not yet implemented")
def test_get_agent_param(self):
log.debug("test_spawn_worker with ds_id: %s" % self.ncom_ds_id)
proc_name, pid, queue_id = self.eoas_cli.spawn_worker(self.ncom_ds_id)
log.debug("proc_name: %s\tproc_id: %s\tqueue_id: %s" % (proc_name, pid, queue_id))
with self.assertRaises(IonException):
self.eoas_cli.get_agent_param(name="param")