def setUp(self): os.environ['FILESYSTEM_PUBLISH_ENABLED'] = '0' os.environ['FILESYSTEM_ENABLED'] = '0' os.environ['GOOGLE_PUBLISH_ENABLED'] = '1' os.environ['GOOGLE_ENABLED'] = '1' os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'credentials.json' os.environ['CONFIG'] = './example/config.yml' self.store = Datastore() self.publish_config = self.store.config.publish['handlers']['gcloud'] self.storage_config = self.store.config.storage['gcloud'] self.dataset = self.store.datasets[0] self.client = Client()
def test_publish_with_timestamp(self): config = self.store.config config['publish']['handlers']['filesystem']['with_timestamp'] = True store = Datastore(config) fp = store.datasets[0].publish(name='test_with_ts')[0] date = datetime.now().date().isoformat() self.assertIn(date, fp)
def test_gcloud_cache(self): config = self.store.config.to_dict().copy() config['publish']['handlers']['gcloud']['cache_control'] = 'no-cache' store = Datastore(self.store.config.update(config)) dataset = store.datasets[0] dataset.publish() url = 'https://runpandarun-testbucket-publish.storage.googleapis.com/my_dataset/my_dataset.csv' res = requests.get(url) self.assertEqual(res.headers['cache-control'], 'no-cache')
def test_disabled_handler(self): os.environ['FILESYSTEM_PUBLISH_ENABLED'] = '0' os.environ['CONFIG'] = './example/config.yml' store = Datastore() self.assertFalse( banal.as_bool( store.config.publish['handlers']['filesystem']['enabled'])) res = store.datasets[0].publish() self.assertListEqual([], res) # re-enable for further tests os.environ['FILESYSTEM_PUBLISH_ENABLED'] = 'true'
def test_import(self): # FIXME this test doesnt work although the implementation does !? return from runpandarun import Datastore from runpandarun.dataset import Dataset from runpandarun import datasets from runpandarun.datasets import my_dataset as _my_dataset store = Datastore() self.assertIsInstance(_my_dataset, Dataset) self.assertEqual(store.my_dataset, _my_dataset) self.assertEqual(datasets.my_dataset, _my_dataset) self.assertEqual(datasets.my_dataset, store.my_dataset)
def setUp(self): self.store = Datastore('./example/rki.yml') self.dataset = self.store.rki with open('./example/rki.json') as f: self.data = json.load(f)
def test_remote_json(self): # TODO this test will fail sooner or later... store = Datastore('./example/rki_remote.yml') ds = store.rki self.assertEqual(len(self.data['features']), len(ds.get_df()))
def setUp(self): self.store = Datastore('./example/rki_json.yml') self.dataset = self.store.rki_json
class Test(unittest.TestCase): def setUp(self): os.environ['FILESYSTEM_PUBLISH_ENABLED'] = '0' os.environ['FILESYSTEM_ENABLED'] = '0' os.environ['GOOGLE_PUBLISH_ENABLED'] = '1' os.environ['GOOGLE_ENABLED'] = '1' os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'credentials.json' os.environ['CONFIG'] = './example/config.yml' self.store = Datastore() self.publish_config = self.store.config.publish['handlers']['gcloud'] self.storage_config = self.store.config.storage['gcloud'] self.dataset = self.store.datasets[0] self.client = Client() def tearDown(self): os.environ['FILESYSTEM_PUBLISH_ENABLED'] = '1' os.environ['FILESYSTEM_ENABLED'] = '1' os.environ['GOOGLE_PUBLISH_ENABLED'] = '0' os.environ['GOOGLE_ENABLED'] = '0' # delete created google buckets for bucket in (self.publish_config['bucket'], self.storage_config['bucket']): try: bucket = self.client.get_bucket(bucket) bucket.delete(force=True) except NotFound: pass def test_1config(self): self.assertTrue(banal.as_bool(self.publish_config['enabled'])) self.assertTrue(banal.as_bool(self.storage_config['enabled'])) self.assertEqual('runpandarun-testbucket-publish', self.publish_config['bucket']) self.assertEqual('runpandarun-testbucket-storage', self.storage_config['bucket']) def test_2implicit_bucket_creation(self): # storage bucket implicit created by storage init dataset = self.store.datasets[0] bucket = dataset._storage.backend._bucket _bucket = self.client.get_bucket(self.storage_config['bucket']) self.assertTrue(_bucket.exists()) self.assertTrue(bucket.exists()) self.assertEqual(bucket.id, _bucket.id) self.assertEqual(bucket.path, _bucket.path) self.assertEqual(bucket.time_created, _bucket.time_created) # publish bucket not created yet self.assertRaises(NotFound, self.client.get_bucket, self.publish_config['bucket']) def test_gcloud_storage(self): self.store.update() ds = self.store.datasets[0] df = ds.get_df() self.assertIsInstance(df, pd.DataFrame) def test_gcloud_publish(self): ds = self.store.datasets[0] res = ds.publish() url = 'https://storage.googleapis.com/runpandarun-testbucket-publish/my_dataset/my_dataset.csv' self.assertEqual(len(res), 1) self.assertEqual(url, res[0]) # test overwrite self.assertRaises(FileExistsError, ds.publish) res = ds.publish(overwrite=True) self.assertEqual(url, res[0]) def test_gcloud_cache(self): config = self.store.config.to_dict().copy() config['publish']['handlers']['gcloud']['cache_control'] = 'no-cache' store = Datastore(self.store.config.update(config)) dataset = store.datasets[0] dataset.publish() url = 'https://runpandarun-testbucket-publish.storage.googleapis.com/my_dataset/my_dataset.csv' res = requests.get(url) self.assertEqual(res.headers['cache-control'], 'no-cache')
def setUp(self): self.store = Datastore('./example/config.yml') self.dataset = self.store.datasets[0]
def setUp(self): self.store = Datastore('./example/config.yml') self.config = self.store.config.publish['handlers']['filesystem'] self.dataset = self.store.datasets[0]