class Test(TestBase):
def setUp(self):
import testbundle.bundle
from ambry.run import RunConfig
self.bundle_dir = os.path.dirname(testbundle.bundle.__file__)
self.rc = get_runconfig((os.path.join(self.bundle_dir,'warehouse-test-config.yaml'),
os.path.join(self.bundle_dir,'bundle.yaml'),
RunConfig.USER_ACCOUNTS))
self.copy_or_build_bundle()
self.bundle = Bundle()
print "Deleting: {}".format(self.rc.group('filesystem').root_dir)
ambry.util.rm_rf(self.rc.group('filesystem').root_dir)
def tearDown(self):
pass
def resolver(self,name):
if name == self.bundle.identity.name or name == self.bundle.identity.vname:
return self.bundle
else:
return False
def get_library(self, name='default'):
"""Clear out the database before the test run"""
from ambry.library import new_library
config = self.rc.library(name)
l = new_library(config, reset=True)
l.database.enable_delete = True
l.database.drop()
l.database.create()
return l
def get_warehouse(self, l, name):
from ambry.util import get_logger
from ambry.warehouse import new_warehouse
w = new_warehouse(self.rc.warehouse(name), l)
w.logger = get_logger('unit_test')
lr = self.bundle.init_log_rate(10000)
w.logger = TestLogger(lr)
w.database.enable_delete = True
w.database.delete()
w.create()
return w
def _test_local_install(self, name):
l = self.get_library('local')
l.put_bundle(self.bundle)
w = self.get_warehouse(l, name)
print "Warehouse: ", w.database.dsn
print "Library: ", l.database.dsn
w.install("source-dataset-subset-variation-tone-0.0.1")
w.install("source-dataset-subset-variation-tthree-0.0.1")
w.install("source-dataset-subset-variation-geot1-geo-0.0.1")
w = self.get_warehouse(l, 'spatialite')
print "WAREHOUSE: ", w.database.dsn
w.install("source-dataset-subset-variation-tone-0.0.1")
w.install("source-dataset-subset-variation-tthree-0.0.1")
w.install("source-dataset-subset-variation-geot1-geo-0.0.1")
def test_local_sqlite_install(self):
self._test_local_install('sqlite')
def test_local_postgres_install(self):
self._test_local_install('postgres1')
def _test_remote_install(self, name):
self.start_server(self.rc.library('server'))
l = self.get_library('client')
l.put_bundle(self.bundle)
w = self.get_warehouse(l, name)
print "WAREHOUSE: ", w.database.dsn
w.install("source-dataset-subset-variation-tone-0.0.1")
w.install("source-dataset-subset-variation-tthree-0.0.1")
w.install("source-dataset-subset-variation-geot1-geo-0.0.1")
w = self.get_warehouse(l, 'spatialite')
print "WAREHOUSE: ", w.database.dsn
w.install("source-dataset-subset-variation-tone-0.0.1")
w.install("source-dataset-subset-variation-tthree-0.0.1")
w.install("source-dataset-subset-variation-geot1-geo-0.0.1")
def test_remote_sqlite_install(self):
self._test_remote_install('sqlite')
def test_remote_postgres_install(self):
self._test_remote_install('postgres1')
def test_manifest(self):
from ambry.warehouse.manifest import Manifest
m = Manifest("""
First Line of documentation
partitions:
part1 # Comment
part2 # Comment
views:
create view foobar1 as
one
two
three;
create view foobar2 as
one
two
three;
documentation:
Foo Doc
views:
create view foobar3 as
one
two
three;
doc:
More Documentation
sql:driver1|driver2
one
two
three
sql:driver1
four
five
sql:driver2
seven
eight
""")
for view in m.views:
print "view", view
for partition in m.partitions:
print 'partition', partition
print 'doc', m.documentation
print '----'
print m.sql
def x_test_install(self):
def resolver(name):
if name == self.bundle.identity.name or name == self.bundle.identity.vname:
return self.bundle
else:
return False
def progress_cb(lr, type,name,n):
if n:
lr("{} {}: {}".format(type, name, n))
else:
self.bundle.log("{} {}".format(type, name))
from ambry.warehouse import new_warehouse
from functools import partial
print "Getting warehouse"
w = new_warehouse(self.rc.warehouse('postgres'))
print "Re-create database"
w.database.enable_delete = True
w.resolver = resolver
w.progress_cb = progress_cb
try: w.drop()
except: pass
w.create()
ps = self.bundle.partitions.all
print "{} partitions".format(len(ps))
for p in self.bundle.partitions:
lr = self.bundle.init_log_rate(10000)
w.install(p, progress_cb = partial(progress_cb, lr) )
self.assertTrue(w.has(self.bundle.identity.vname))
for p in self.bundle.partitions:
self.assertTrue(w.has(p.identity.vname))
for p in self.bundle.partitions:
w.remove(p.identity.vname)
print w.get(self.bundle.identity.name)
print w.get(self.bundle.identity.vname)
print w.get(self.bundle.identity.id_)
w.install(self.bundle)
print w.get(self.bundle.identity.name)
print w.get(self.bundle.identity.vname)
print w.get(self.bundle.identity.id_)
for p in self.bundle.partitions:
lr = self.bundle.init_log_rate(10000)
w.install(p, progress_cb = partial(progress_cb, lr))
class Test(TestBase):
def setUp(self):
import testbundle.bundle
self.bundle_dir = os.path.dirname(testbundle.bundle.__file__)
self.rc = get_runconfig((os.path.join(self.bundle_dir,'warehouse-test-config.yaml'),
os.path.join(self.bundle_dir,'bundle.yaml')))
self.copy_or_build_bundle()
self.bundle = Bundle()
print "Deleting: {}".format(self.rc.group('filesystem').root_dir)
databundles.util.rm_rf(self.rc.group('filesystem').root_dir)
def tearDown(self):
pass
def resolver(self,name):
if name == self.bundle.identity.name or name == self.bundle.identity.vname:
return self.bundle
else:
return False
class Resolver(object):
def get(self,name):
if name == self.bundle.identity.name or name == self.bundle.identity.vname:
return self.bundle
else:
return False
def get_ref(self,name):
pass
def progress_cb(self, lr, type_,name,n):
if n:
lr("{} {}: {}".format(type, name, n))
else:
self.bundle.log("{} {}".format(type_, name))
def test_create(self):
from databundles.warehouse import new_warehouse
w = new_warehouse(self.rc.warehouse('postgres'))
print "Re-create database"
w.database.enable_delete = True
w.resolver = lambda name: self.resolver(name)
lr = self.bundle.init_log_rate(10000)
w.progress_cb = lambda type_,name,n: self.progress_cb(lr, type_,name,n)
try: w.drop()
except: pass
w.create()
w.library.create()
w.install(self.bundle)
w.create_table(self.bundle.dataset.vid, "ttwo")
def x_test_install(self):
def resolver(name):
if name == self.bundle.identity.name or name == self.bundle.identity.vname:
return self.bundle
else:
return False
def progress_cb(lr, type,name,n):
if n:
lr("{} {}: {}".format(type, name, n))
else:
self.bundle.log("{} {}".format(type, name))
from databundles.warehouse import new_warehouse
from functools import partial
print "Getting warehouse"
w = new_warehouse(self.rc.warehouse('postgres'))
print "Re-create database"
w.database.enable_delete = True
w.resolver = resolver
w.progress_cb = progress_cb
try: w.drop()
except: pass
w.create()
ps = self.bundle.partitions.all
print "{} partitions".format(len(ps))
for p in self.bundle.partitions:
lr = self.bundle.init_log_rate(10000)
w.install(p, progress_cb = partial(progress_cb, lr) )
self.assertTrue(w.has(self.bundle.identity.vname))
for p in self.bundle.partitions:
self.assertTrue(w.has(p.identity.vname))
for p in self.bundle.partitions:
w.remove(p.identity.vname)
print w.get(self.bundle.identity.name)
print w.get(self.bundle.identity.vname)
print w.get(self.bundle.identity.id_)
w.install(self.bundle)
print w.get(self.bundle.identity.name)
print w.get(self.bundle.identity.vname)
print w.get(self.bundle.identity.id_)
for p in self.bundle.partitions:
lr = self.bundle.init_log_rate(10000)
w.install(p, progress_cb = partial(progress_cb, lr))
