def test_serialize_compatability(self): ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_extended_parsed() stream_def_id = self.pubsub_management.create_stream_definition('ctd extended', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) stream_id, route = self.pubsub_management.create_stream('ctd1', 'xp1', stream_definition_id=stream_def_id) self.addCleanup(self.pubsub_management.delete_stream, stream_id) sub_id = self.pubsub_management.create_subscription('sub1', stream_ids=[stream_id]) self.addCleanup(self.pubsub_management.delete_subscription, sub_id) self.pubsub_management.activate_subscription(sub_id) self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id) verified = Event() def verifier(msg, route, stream_id): for k,v in msg.record_dictionary.iteritems(): if v is not None: self.assertIsInstance(v, np.ndarray) rdt = RecordDictionaryTool.load_from_granule(msg) for k,v in rdt.iteritems(): self.assertIsInstance(rdt[k], np.ndarray) self.assertIsInstance(v, np.ndarray) verified.set() subscriber = StandaloneStreamSubscriber('sub1', callback=verifier) subscriber.start() self.addCleanup(subscriber.stop) publisher = StandaloneStreamPublisher(stream_id,route) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) ph.fill_rdt(rdt,10) publisher.publish(rdt.to_granule()) self.assertTrue(verified.wait(60))
def setup_resource(self): ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_extended_parsed() stream_def_id = self.pubsub_management.create_stream_definition('example', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) tdom, sdom = time_series_domain() dp = DataProduct(name='example') dp.spatial_domain = sdom.dump() dp.temporal_domain = tdom.dump() data_product_id = self.data_product_management.create_data_product(dp, stream_def_id) self.addCleanup(self.data_product_management.delete_data_product, data_product_id) self.data_product_management.activate_data_product_persistence(data_product_id) self.addCleanup(self.data_product_management.suspend_data_product_persistence, data_product_id) dataset_id = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0] monitor = DatasetMonitor(dataset_id) self.addCleanup(monitor.stop) rdt = ph.get_rdt(stream_def_id) ph.fill_rdt(rdt, 100) ph.publish_rdt_to_data_product(data_product_id, rdt) # Yield to other greenlets, had an issue with connectivity gevent.sleep(1) self.offering_id = dataset_id
def test_create_dataset(self): ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_extended_parsed() stream_def_id = self.pubsub_management.create_stream_definition('example', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) dp = DataProduct(name='example') data_product_id = self.data_product_management.create_data_product(dp, stream_def_id) self.addCleanup(self.data_product_management.delete_data_product, data_product_id) self.data_product_management.activate_data_product_persistence(data_product_id) self.addCleanup(self.data_product_management.suspend_data_product_persistence, data_product_id) dataset_id = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0] monitor = DatasetMonitor(dataset_id) self.addCleanup(monitor.stop) rdt = ph.get_rdt(stream_def_id) ph.fill_rdt(rdt, 100) ph.publish_rdt_to_data_product(data_product_id, rdt) self.assertTrue(monitor.event.wait(10)) # Yield to other greenlets, had an issue with connectivity gevent.sleep(1) log.debug("--------------------------------") log.debug(dataset_id) coverage_path = DatasetManagementService()._get_coverage_path(dataset_id) log.debug(coverage_path) log.debug("--------------------------------") breakpoint(locals(), globals())
def test_fdt_created_during(self): # generate a data product and check that the FDT exists ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_extended_parsed() stream_def_id = self.pubsub_management.create_stream_definition('example', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) dp = DataProduct(name='example') data_product_id = self.data_product_management.create_data_product(dp, stream_def_id) self.addCleanup(self.data_product_management.delete_data_product, data_product_id) self.data_product_management.activate_data_product_persistence(data_product_id) self.addCleanup(self.data_product_management.suspend_data_product_persistence, data_product_id) dataset_id = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0] monitor = DatasetMonitor(dataset_id) self.addCleanup(monitor.stop) rdt = ph.get_rdt(stream_def_id) ph.fill_rdt(rdt, 100) ph.publish_rdt_to_data_product(data_product_id, rdt) self.assertTrue(monitor.event.wait(10)) gevent.sleep(1) # Yield to other greenlets, had an issue with connectivity print "--------------------------------" print dataset_id coverage_path = DatasetManagementService()._get_coverage_path(dataset_id) print coverage_path print "--------------------------------"
def test_pydap(self): if not CFG.get_safe('bootstrap.use_pydap',False): raise unittest.SkipTest('PyDAP is off (bootstrap.use_pydap)') ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_extended_parsed() stream_def_id = self.pubsub_management.create_stream_definition('example', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) tdom, sdom = time_series_domain() dp = DataProduct(name='example') dp.spatial_domain = sdom.dump() dp.temporal_domain = tdom.dump() data_product_id = self.data_product_management.create_data_product(dp, stream_def_id) self.addCleanup(self.data_product_management.delete_data_product, data_product_id) self.data_product_management.activate_data_product_persistence(data_product_id) self.addCleanup(self.data_product_management.suspend_data_product_persistence, data_product_id) dataset_id = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0] monitor = DatasetMonitor(dataset_id) self.addCleanup(monitor.stop) rdt = ph.get_rdt(stream_def_id) ph.fill_rdt(rdt,10) ph.publish_rdt_to_data_product(data_product_id, rdt) self.assertTrue(monitor.event.wait(10)) gevent.sleep(1) # Yield to other greenlets, had an issue with connectivity pydap_host = CFG.get_safe('server.pydap.host','localhost') pydap_port = CFG.get_safe('server.pydap.port',8001) url = 'http://%s:%s/%s' %(pydap_host, pydap_port, dataset_id) ds = open_url(url) np.testing.assert_array_equal(ds['time'][:], np.arange(10)) untested = [] for k,v in rdt.iteritems(): if k==rdt.temporal_parameter: continue context = rdt.context(k) if isinstance(context.param_type, QuantityType): np.testing.assert_array_equal(ds[k][k][:][0], rdt[k]) elif isinstance(context.param_type, ArrayType): values = np.empty(rdt[k].shape, dtype='O') for i,obj in enumerate(rdt[k]): values[i] = str(obj) np.testing.assert_array_equal(ds[k][k][:][0], values) elif isinstance(context.param_type, ConstantType): np.testing.assert_array_equal(ds[k][k][:][0], rdt[k]) elif isinstance(context.param_type, CategoryType): np.testing.assert_array_equal(ds[k][k][:][0], rdt[k]) else: untested.append('%s (%s)' % (k,context.param_type)) if untested: raise AssertionError('Untested parameters: %s' % untested)
def test_get_data_from_FDW(self): # generate a data product and check that the FDW can get data ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_extended_parsed() stream_def_id = self.pubsub_management.create_stream_definition( 'example', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) dp = DataProduct(name='example') data_product_id = self.data_product_management.create_data_product( dp, stream_def_id) self.addCleanup(self.data_product_management.delete_data_product, data_product_id) self.data_product_management.activate_data_product_persistence( data_product_id) self.addCleanup( self.data_product_management.suspend_data_product_persistence, data_product_id) dataset_id = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0] monitor = DatasetMonitor(dataset_id) self.addCleanup(monitor.stop) rdt = ph.get_rdt(stream_def_id) ph.fill_rdt(rdt, 100) ph.publish_rdt_to_data_product(data_product_id, rdt) self.assertTrue(monitor.event.wait(10)) gevent.sleep( 1) # Yield to other greenlets, had an issue with connectivity print "--------------------------------" print dataset_id coverage_path = DatasetManagementService()._get_coverage_path( dataset_id) print coverage_path print "--------------------------------" #verify table exists in the DB (similar to above) # ....code... # check that the geoserver layer exists as above # ... code .... # make a WMS/WFS request...somet like this (or both) url = self.gs_host + '/geoserver/geonode/ows?service=WFS&version=1.0.0&request=GetFeature&typeName=geonode:ooi_' + dataset_id + '_ooi&maxFeatures=1&outputFormat=csv' r = requests.get(url) assertTrue(r.status_code == 200)
def test_create_dataset_verify_geoserver_layer(self): #generate layer and check that the service created it in geoserver ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_extended_parsed() stream_def_id = self.pubsub_management.create_stream_definition( 'example', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) dp = DataProduct(name='example') data_product_id = self.data_product_management.create_data_product( dp, stream_def_id) self.addCleanup(self.data_product_management.delete_data_product, data_product_id) self.data_product_management.activate_data_product_persistence( data_product_id) self.addCleanup( self.data_product_management.suspend_data_product_persistence, data_product_id) dataset_id = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0] monitor = DatasetMonitor(dataset_id) self.addCleanup(monitor.stop) rdt = ph.get_rdt(stream_def_id) ph.fill_rdt(rdt, 100) ph.publish_rdt_to_data_product(data_product_id, rdt) self.assertTrue(monitor.event.wait(10)) gevent.sleep( 1) # Yield to other greenlets, had an issue with connectivity log.debug("--------------------------------") log.debug(dataset_id) coverage_path = DatasetManagementService()._get_coverage_path( dataset_id) log.debug(coverage_path) log.debug("--------------------------------") # verify that the layer exists in geoserver try: r = requests.get(self.gs_rest_url + '/layers/ooi_' + dataset_id + '_ooi.xml', auth=(self.username, self.PASSWORD)) self.assertTrue(r.status_code == 200) except Exception as e: log.error("check service and layer exist...%s", e) self.assertTrue(False)
def test_get_data_from_FDW(self): # generate a data product and check that the FDW can get data ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_extended_parsed() stream_def_id = self.pubsub_management.create_stream_definition('example', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) tdom, sdom = time_series_domain() dp = DataProduct(name='example') dp.spatial_domain = sdom.dump() dp.temporal_domain = tdom.dump() data_product_id = self.data_product_management.create_data_product(dp, stream_def_id) self.addCleanup(self.data_product_management.delete_data_product, data_product_id) self.data_product_management.activate_data_product_persistence(data_product_id) self.addCleanup(self.data_product_management.suspend_data_product_persistence, data_product_id) dataset_id = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0] monitor = DatasetMonitor(dataset_id) self.addCleanup(monitor.stop) rdt = ph.get_rdt(stream_def_id) ph.fill_rdt(rdt, 100) ph.publish_rdt_to_data_product(data_product_id, rdt) self.assertTrue(monitor.event.wait(10)) gevent.sleep(1) # Yield to other greenlets, had an issue with connectivity print "--------------------------------" print dataset_id coverage_path = DatasetManagementService()._get_coverage_path(dataset_id) print coverage_path print "--------------------------------" #verify table exists in the DB (similar to above) # ....code... # check that the geoserver layer exists as above # ... code .... # make a WMS/WFS request...somet like this (or both) url = self.gs_host+'/geoserver/geonode/ows?service=WFS&version=1.0.0&request=GetFeature&typeName=geonode:ooi_' + dataset_id + '_ooi&maxFeatures=1&outputFormat=csv' r = requests.get(url) assertTrue(r.status_code == 200)
def test_create_dataset_verify_geoserver_layer(self): #generate layer and check that the service created it in geoserver ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_extended_parsed() stream_def_id = self.pubsub_management.create_stream_definition('example', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) tdom, sdom = time_series_domain() dp = DataProduct(name='example') dp.spatial_domain = sdom.dump() dp.temporal_domain = tdom.dump() data_product_id = self.data_product_management.create_data_product(dp, stream_def_id) self.addCleanup(self.data_product_management.delete_data_product, data_product_id) self.data_product_management.activate_data_product_persistence(data_product_id) self.addCleanup(self.data_product_management.suspend_data_product_persistence, data_product_id) dataset_id = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0] monitor = DatasetMonitor(dataset_id) self.addCleanup(monitor.stop) rdt = ph.get_rdt(stream_def_id) ph.fill_rdt(rdt, 100) ph.publish_rdt_to_data_product(data_product_id, rdt) self.assertTrue(monitor.event.wait(10)) gevent.sleep(1) # Yield to other greenlets, had an issue with connectivity log.debug("--------------------------------") log.debug(dataset_id) coverage_path = DatasetManagementService()._get_coverage_path(dataset_id) log.debug(coverage_path) log.debug("--------------------------------") # verify that the layer exists in geoserver try: r = requests.get(self.gs_rest_url + '/layers/ooi_' + dataset_id + '_ooi.xml', auth=(self.username, self.PASSWORD)) self.assertTrue(r.status_code == 200) except Exception as e: log.error("check service and layer exist...%s", e) self.assertTrue(False)
def test_create_dataset(self): ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_extended_parsed() stream_def_id = self.pubsub_management.create_stream_definition( 'example', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) dp = DataProduct(name='example') data_product_id = self.data_product_management.create_data_product( dp, stream_def_id) self.addCleanup(self.data_product_management.delete_data_product, data_product_id) self.data_product_management.activate_data_product_persistence( data_product_id) self.addCleanup( self.data_product_management.suspend_data_product_persistence, data_product_id) dataset_id = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0] monitor = DatasetMonitor(dataset_id) self.addCleanup(monitor.stop) rdt = ph.get_rdt(stream_def_id) ph.fill_rdt(rdt, 100) ph.publish_rdt_to_data_product(data_product_id, rdt) self.assertTrue(monitor.event.wait(10)) # Yield to other greenlets, had an issue with connectivity gevent.sleep(1) log.debug("--------------------------------") log.debug(dataset_id) coverage_path = DatasetManagementService()._get_coverage_path( dataset_id) log.debug(coverage_path) log.debug("--------------------------------") breakpoint(locals(), globals())
def test_fdt_created_during(self): # generate a data product and check that the FDT exists ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_extended_parsed() stream_def_id = self.pubsub_management.create_stream_definition( 'example', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) dp = DataProduct(name='example') data_product_id = self.data_product_management.create_data_product( dp, stream_def_id) self.addCleanup(self.data_product_management.delete_data_product, data_product_id) self.data_product_management.activate_data_product_persistence( data_product_id) self.addCleanup( self.data_product_management.suspend_data_product_persistence, data_product_id) dataset_id = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0] monitor = DatasetMonitor(dataset_id) self.addCleanup(monitor.stop) rdt = ph.get_rdt(stream_def_id) ph.fill_rdt(rdt, 100) ph.publish_rdt_to_data_product(data_product_id, rdt) self.assertTrue(monitor.event.wait(10)) gevent.sleep( 1) # Yield to other greenlets, had an issue with connectivity print "--------------------------------" print dataset_id coverage_path = DatasetManagementService()._get_coverage_path( dataset_id) print coverage_path print "--------------------------------"
def setup_resource(self): ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_extended_parsed() stream_def_id = self.pubsub_management.create_stream_definition( 'example', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) dp = DataProduct(name='example') data_product_id = self.data_product_management.create_data_product( dp, stream_def_id) self.addCleanup(self.data_product_management.delete_data_product, data_product_id) self.data_product_management.activate_data_product_persistence( data_product_id) self.addCleanup( self.data_product_management.suspend_data_product_persistence, data_product_id) dataset_id = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0] monitor = DatasetMonitor(dataset_id) self.addCleanup(monitor.stop) rdt = ph.get_rdt(stream_def_id) ph.fill_rdt(rdt, 100) ph.publish_rdt_to_data_product(data_product_id, rdt) # Yield to other greenlets, had an issue with connectivity gevent.sleep(1) self.offering_id = dataset_id
def test_pydap(self): if not CFG.get_safe('bootstrap.use_pydap', False): raise unittest.SkipTest('PyDAP is off (bootstrap.use_pydap)') ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_extended_parsed() stream_def_id = self.pubsub_management.create_stream_definition( 'example', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) tdom, sdom = time_series_domain() dp = DataProduct(name='example') dp.spatial_domain = sdom.dump() dp.temporal_domain = tdom.dump() data_product_id = self.data_product_management.create_data_product( dp, stream_def_id) self.addCleanup(self.data_product_management.delete_data_product, data_product_id) self.data_product_management.activate_data_product_persistence( data_product_id) self.addCleanup( self.data_product_management.suspend_data_product_persistence, data_product_id) dataset_id = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0] monitor = DatasetMonitor(dataset_id) self.addCleanup(monitor.stop) rdt = ph.get_rdt(stream_def_id) ph.fill_rdt(rdt, 10) ph.publish_rdt_to_data_product(data_product_id, rdt) self.assertTrue(monitor.event.wait(10)) gevent.sleep( 1) # Yield to other greenlets, had an issue with connectivity pydap_host = CFG.get_safe('server.pydap.host', 'localhost') pydap_port = CFG.get_safe('server.pydap.port', 8001) url = 'http://%s:%s/%s' % (pydap_host, pydap_port, dataset_id) for i in xrange( 3 ): # Do it three times to test that the cache doesn't corrupt the requests/responses ds = open_url(url) np.testing.assert_array_equal(ds['time'][:], np.arange(10)) untested = [] for k, v in rdt.iteritems(): if k == rdt.temporal_parameter: continue context = rdt.context(k) if isinstance(context.param_type, QuantityType): np.testing.assert_array_equal(ds[k][k][:][0], rdt[k]) elif isinstance(context.param_type, ArrayType): if context.param_type.inner_encoding is None: values = np.empty(rdt[k].shape, dtype='O') for i, obj in enumerate(rdt[k]): values[i] = str(obj) np.testing.assert_array_equal(ds[k][k][:][0], values) elif len(rdt[k].shape) > 1: values = np.empty(rdt[k].shape[0], dtype='O') for i in xrange(rdt[k].shape[0]): values[i] = ','.join( map(lambda x: str(x), rdt[k][i].tolist())) elif isinstance(context.param_type, ConstantType): np.testing.assert_array_equal(ds[k][k][:][0], rdt[k]) elif isinstance(context.param_type, CategoryType): np.testing.assert_array_equal(ds[k][k][:][0], rdt[k]) else: untested.append('%s (%s)' % (k, context.param_type)) if untested: raise AssertionError('Untested parameters: %s' % untested)
def test_pydap(self): ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_extended_parsed() stream_def_id = self.pubsub_management.create_stream_definition('example', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) dp = DataProduct(name='example') data_product_id = self.data_product_management.create_data_product(dp, stream_def_id) self.addCleanup(self.data_product_management.delete_data_product, data_product_id) self.data_product_management.activate_data_product_persistence(data_product_id) self.addCleanup(self.data_product_management.suspend_data_product_persistence, data_product_id) dataset_id = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0] monitor = DatasetMonitor(dataset_id) self.addCleanup(monitor.stop) rdt = ph.get_rdt(stream_def_id) ph.fill_rdt(rdt,10) ph.publish_rdt_to_data_product(data_product_id, rdt) self.assertTrue(monitor.wait()) gevent.sleep(1) # Yield to other greenlets, had an issue with connectivity pydap_host = CFG.get_safe('server.pydap.host','localhost') pydap_port = CFG.get_safe('server.pydap.port',8001) url = 'http://%s:%s/%s' %(pydap_host, pydap_port, data_product_id) for i in xrange(3): # Do it three times to test that the cache doesn't corrupt the requests/responses ds = open_url(url) np.testing.assert_array_equal(list(ds['data']['time']), np.arange(10)) untested = [] for k,v in rdt.iteritems(): if k==rdt.temporal_parameter: continue context = rdt.context(k) if isinstance(context.param_type, QuantityType): np.testing.assert_array_equal(list(ds['data'][k]), rdt[k]) elif isinstance(context.param_type, ArrayType): if context.param_type.inner_encoding is None: values = np.empty(rdt[k].shape, dtype='O') for i,obj in enumerate(rdt[k]): values[i] = str(obj) np.testing.assert_array_equal(list(ds['data'][k]), values) elif len(rdt[k].shape)>1: values = np.empty(rdt[k].shape[0], dtype='O') for i in xrange(rdt[k].shape[0]): values[i] = ','.join(map(lambda x : str(x), rdt[k][i].tolist())) elif isinstance(context.param_type, ConstantType): np.testing.assert_array_equal(list(ds['data'][k]), rdt[k]) elif isinstance(context.param_type, CategoryType): np.testing.assert_array_equal(list(ds['data'][k]), rdt[k].astype('|S')) else: untested.append('%s (%s)' % (k,context.param_type)) if untested: raise AssertionError('Untested parameters: %s' % untested)