def setUp(self):
os.environ['FILESYSTEM_PUBLISH_ENABLED'] = '0'
os.environ['FILESYSTEM_ENABLED'] = '0'
os.environ['GOOGLE_PUBLISH_ENABLED'] = '1'
os.environ['GOOGLE_ENABLED'] = '1'
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'credentials.json'
os.environ['CONFIG'] = './example/config.yml'
self.store = Datastore()
self.publish_config = self.store.config.publish['handlers']['gcloud']
self.storage_config = self.store.config.storage['gcloud']
self.dataset = self.store.datasets[0]
self.client = Client()
def test_publish_with_timestamp(self):
config = self.store.config
config['publish']['handlers']['filesystem']['with_timestamp'] = True
store = Datastore(config)
fp = store.datasets[0].publish(name='test_with_ts')[0]
date = datetime.now().date().isoformat()
self.assertIn(date, fp)
def test_gcloud_cache(self):
config = self.store.config.to_dict().copy()
config['publish']['handlers']['gcloud']['cache_control'] = 'no-cache'
store = Datastore(self.store.config.update(config))
dataset = store.datasets[0]
dataset.publish()
url = 'https://runpandarun-testbucket-publish.storage.googleapis.com/my_dataset/my_dataset.csv'
res = requests.get(url)
self.assertEqual(res.headers['cache-control'], 'no-cache')
def test_disabled_handler(self):
os.environ['FILESYSTEM_PUBLISH_ENABLED'] = '0'
os.environ['CONFIG'] = './example/config.yml'
store = Datastore()
self.assertFalse(
banal.as_bool(
store.config.publish['handlers']['filesystem']['enabled']))
res = store.datasets[0].publish()
self.assertListEqual([], res)
# re-enable for further tests
os.environ['FILESYSTEM_PUBLISH_ENABLED'] = 'true'
def test_import(self):
# FIXME this test doesnt work although the implementation does !?
return
from runpandarun import Datastore
from runpandarun.dataset import Dataset
from runpandarun import datasets
from runpandarun.datasets import my_dataset as _my_dataset
store = Datastore()
self.assertIsInstance(_my_dataset, Dataset)
self.assertEqual(store.my_dataset, _my_dataset)
self.assertEqual(datasets.my_dataset, _my_dataset)
self.assertEqual(datasets.my_dataset, store.my_dataset)
def setUp(self):
self.store = Datastore('./example/rki.yml')
self.dataset = self.store.rki
with open('./example/rki.json') as f:
self.data = json.load(f)
def test_remote_json(self):
# TODO this test will fail sooner or later...
store = Datastore('./example/rki_remote.yml')
ds = store.rki
self.assertEqual(len(self.data['features']), len(ds.get_df()))
def setUp(self):
self.store = Datastore('./example/rki_json.yml')
self.dataset = self.store.rki_json
class Test(unittest.TestCase):
def setUp(self):
os.environ['FILESYSTEM_PUBLISH_ENABLED'] = '0'
os.environ['FILESYSTEM_ENABLED'] = '0'
os.environ['GOOGLE_PUBLISH_ENABLED'] = '1'
os.environ['GOOGLE_ENABLED'] = '1'
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'credentials.json'
os.environ['CONFIG'] = './example/config.yml'
self.store = Datastore()
self.publish_config = self.store.config.publish['handlers']['gcloud']
self.storage_config = self.store.config.storage['gcloud']
self.dataset = self.store.datasets[0]
self.client = Client()
def tearDown(self):
os.environ['FILESYSTEM_PUBLISH_ENABLED'] = '1'
os.environ['FILESYSTEM_ENABLED'] = '1'
os.environ['GOOGLE_PUBLISH_ENABLED'] = '0'
os.environ['GOOGLE_ENABLED'] = '0'
# delete created google buckets
for bucket in (self.publish_config['bucket'],
self.storage_config['bucket']):
try:
bucket = self.client.get_bucket(bucket)
bucket.delete(force=True)
except NotFound:
pass
def test_1config(self):
self.assertTrue(banal.as_bool(self.publish_config['enabled']))
self.assertTrue(banal.as_bool(self.storage_config['enabled']))
self.assertEqual('runpandarun-testbucket-publish',
self.publish_config['bucket'])
self.assertEqual('runpandarun-testbucket-storage',
self.storage_config['bucket'])
def test_2implicit_bucket_creation(self):
# storage bucket implicit created by storage init
dataset = self.store.datasets[0]
bucket = dataset._storage.backend._bucket
_bucket = self.client.get_bucket(self.storage_config['bucket'])
self.assertTrue(_bucket.exists())
self.assertTrue(bucket.exists())
self.assertEqual(bucket.id, _bucket.id)
self.assertEqual(bucket.path, _bucket.path)
self.assertEqual(bucket.time_created, _bucket.time_created)
# publish bucket not created yet
self.assertRaises(NotFound, self.client.get_bucket,
self.publish_config['bucket'])
def test_gcloud_storage(self):
self.store.update()
ds = self.store.datasets[0]
df = ds.get_df()
self.assertIsInstance(df, pd.DataFrame)
def test_gcloud_publish(self):
ds = self.store.datasets[0]
res = ds.publish()
url = 'https://storage.googleapis.com/runpandarun-testbucket-publish/my_dataset/my_dataset.csv'
self.assertEqual(len(res), 1)
self.assertEqual(url, res[0])
# test overwrite
self.assertRaises(FileExistsError, ds.publish)
res = ds.publish(overwrite=True)
self.assertEqual(url, res[0])
def test_gcloud_cache(self):
config = self.store.config.to_dict().copy()
config['publish']['handlers']['gcloud']['cache_control'] = 'no-cache'
store = Datastore(self.store.config.update(config))
dataset = store.datasets[0]
dataset.publish()
url = 'https://runpandarun-testbucket-publish.storage.googleapis.com/my_dataset/my_dataset.csv'
res = requests.get(url)
self.assertEqual(res.headers['cache-control'], 'no-cache')
def setUp(self):
self.store = Datastore('./example/config.yml')
self.dataset = self.store.datasets[0]
def setUp(self):
self.store = Datastore('./example/config.yml')
self.config = self.store.config.publish['handlers']['filesystem']
self.dataset = self.store.datasets[0]