def x_test_install(self):
def resolver(name):
if name == self.bundle.identity.name or name == self.bundle.identity.vname:
return self.bundle
else:
return False
def progress_cb(lr, type, name, n):
if n:
lr("{} {}: {}".format(type, name, n))
else:
self.bundle.log("{} {}".format(type, name))
from ambry.warehouse import new_warehouse
from functools import partial
print "Getting warehouse"
w = new_warehouse(self.rc.warehouse('postgres'))
print "Re-create database"
w.database.enable_delete = True
w.resolver = resolver
w.progress_cb = progress_cb
try:
w.drop()
except:
pass
w.create()
ps = self.bundle.partitions.all
print "{} partitions".format(len(ps))
for p in self.bundle.partitions:
lr = self.bundle.init_log_rate(10000)
w.install(p, progress_cb=partial(progress_cb, lr))
self.assertTrue(w.has(self.bundle.identity.vname))
for p in self.bundle.partitions:
self.assertTrue(w.has(p.identity.vname))
for p in self.bundle.partitions:
w.remove(p.identity.vname)
print w.get(self.bundle.identity.name)
print w.get(self.bundle.identity.vname)
print w.get(self.bundle.identity.id_)
w.install(self.bundle)
print w.get(self.bundle.identity.name)
print w.get(self.bundle.identity.vname)
print w.get(self.bundle.identity.id_)
for p in self.bundle.partitions:
lr = self.bundle.init_log_rate(10000)
w.install(p, progress_cb=partial(progress_cb, lr))
def x_test_install(self):
def resolver(name):
if name == self.bundle.identity.name or name == self.bundle.identity.vname:
return self.bundle
else:
return False
def progress_cb(lr, type,name,n):
if n:
lr("{} {}: {}".format(type, name, n))
else:
self.bundle.log("{} {}".format(type, name))
from ambry.warehouse import new_warehouse
from functools import partial
print "Getting warehouse"
w = new_warehouse(self.rc.warehouse('postgres'))
print "Re-create database"
w.database.enable_delete = True
w.resolver = resolver
w.progress_cb = progress_cb
try: w.drop()
except: pass
w.create()
ps = self.bundle.partitions.all
print "{} partitions".format(len(ps))
for p in self.bundle.partitions:
lr = self.bundle.init_log_rate(10000)
w.install(p, progress_cb = partial(progress_cb, lr) )
self.assertTrue(w.has(self.bundle.identity.vname))
for p in self.bundle.partitions:
self.assertTrue(w.has(p.identity.vname))
for p in self.bundle.partitions:
w.remove(p.identity.vname)
print w.get(self.bundle.identity.name)
print w.get(self.bundle.identity.vname)
print w.get(self.bundle.identity.id_)
w.install(self.bundle)
print w.get(self.bundle.identity.name)
print w.get(self.bundle.identity.vname)
print w.get(self.bundle.identity.id_)
for p in self.bundle.partitions:
lr = self.bundle.init_log_rate(10000)
w.install(p, progress_cb = partial(progress_cb, lr))
def get_warehouse(self, l, name, delete=True):
from ambry.warehouse import new_warehouse
waho = new_warehouse(self.rc.warehouse(name), l)
if delete:
waho.database.enable_delete = True
waho.database.delete()
waho.create()
return waho
def warehouse(self, uid):
from ambry.warehouse import new_warehouse, database_config
s = self.store(uid)
if not s:
raise NotFoundError("Did not find warehouse for uid: '{}' ".format(uid))
config = database_config(s.path)
return new_warehouse(config, self, logger=self.logger)
def warehouse(self, uid):
from ambry.warehouse import new_warehouse, database_config
s = self.store(uid)
if not s:
from ..dbexceptions import NotFoundError
raise NotFoundError("Did not find warehouse for uid: '{}' ".format(uid))
config = database_config(s.path)
return new_warehouse(config, self, logger=self.logger)
def get_warehouse(self, l, name, delete=True):
from ambry.util import get_logger
from ambry.warehouse import new_warehouse
w = new_warehouse(self.rc.warehouse(name), l)
w.logger = get_logger('unit_test')
lr = self.bundle.init_log_rate(10000)
w.logger = TestLogger(lr)
if delete:
w.database.enable_delete = True
w.database.delete()
w.create()
return w
def get_warehouse(self, l, name, delete = True):
from ambry.util import get_logger
from ambry.warehouse import new_warehouse
w = new_warehouse(self.rc.warehouse(name), l)
w.logger = get_logger('unit_test')
lr = self.bundle.init_log_rate(10000)
w.logger = TestLogger(lr)
if delete:
w.database.enable_delete = True
w.database.delete()
w.create()
return w
def _warehouse_new_from_dbc(dbc, l):
from ambry.warehouse import new_warehouse
from . import global_logger
w = new_warehouse(dbc, l, logger=global_logger)
w.create()
for c in w.configurable:
if c in dbc:
w._meta_set(c, dbc[c])
sf = l.sync_warehouse(w)
w.uid = sf.ref
return w
def _warehouse_new_from_dbc(dbc, l):
from ambry.warehouse import new_warehouse
from . import global_logger
w = new_warehouse(dbc, l, logger=global_logger)
w.create()
for c in w.configurable:
if c in dbc:
w._meta_set(c, dbc[c])
sf = l.sync_warehouse(w)
w.uid = sf.ref
return w
def build(self):
from ambry.warehouse import new_warehouse
from ambry.warehouse.manifest import Manifest
wf = self.filesystem.build_path(self.identity.vname, 'warehouse.db')
w = new_warehouse('sqlite:///{}'.format(wf), self.library, self.logger)
self.log("Installing manifest to {}".format(w.database.dsn))
w.create()
w.install_manifest(Manifest(self.filesystem.meta_path('manifest.ambry')))
self.log("Installed manifest to {}".format(w.database.dsn))
for row in w.database.connection.execute("SELECT * FROM example_combined"):
print row
return True
def warehouse_command(args, rc):
from ambry.warehouse import new_warehouse
from ..library import new_library
from . import logger
import urlparse
if args.database:
parts = urlparse.urlparse(args.database)
if parts.scheme == 'sqlite':
config = dict(service='sqlite', database=dict(dbname=parts.path, driver='sqlite'))
elif parts.scheme == 'spatialite':
config = dict(service='spatialite', database=dict(dbname=args.spatialite, driver='sqlite'))
elif parts.scheme == 'postgres':
config = dict(service='postgres',
database=dict(driver='postgres',
server=parts.hostname,
username=parts.username,
password=parts.password,
dbname=parts.path.strip('/')
))
elif parts.scheme == 'postgis':
config = dict(service='postgis',
database=dict(driver='postgis',
server=parts.hostname,
username=parts.username,
password=parts.password,
dbname=parts.path.strip('/')
))
else:
raise ValueError("Unknown database connection scheme: {}".format(parts.scheme))
else:
config = rc.warehouse(args.name)
l = new_library(rc.library(args.library_name))
l.logger = logger
w = new_warehouse(config, l)
globals()['warehouse_'+args.subcommand](args, w,rc)
def build(self):
from ambry.warehouse import new_warehouse
from ambry.warehouse.manifest import Manifest
import sqlite3
wf = self.filesystem.build_path(self.identity.vname, 'warehouse.db')
w = new_warehouse('sqlite:///{}'.format(wf), self.library, self.logger)
self.log("Installing manifest to {}".format(w.database.dsn))
w.create()
w.install_manifest(
Manifest(self.filesystem.meta_path('manifest.ambry')))
self.log("Installed manifest to {}".format(w.database.dsn))
for row in w.database.connection.execute(
"SELECT * FROM example_combined"):
print row
return True
def warehouse_command(args, rc):
from ambry.warehouse import new_warehouse
from ..library import new_library
from . import global_logger
from ambry.warehouse import database_config
l = new_library(rc.library(args.library_name))
w = None
l.logger = global_logger
config = None
#
# Special case for installing manifests, can get the database DSN from the
# manifest
if not args.database and args.subcommand == 'install':
from ..warehouse.manifest import Manifest
m = Manifest(args.term)
dsnt = "spatialite:///{}.db" if m.is_geo else "sqlite:///{}.db"
name = args.name if args.name else m.uid
dsn = dsnt.format(l.warehouse_cache.path(name, missing_ok=True))
config = database_config(dsn)
w = new_warehouse(config, l, logger=global_logger)
if not w.exists():
w.create()
elif args.database:
# Check if the string is the uid of a database in the library.
s = l.store(args.database)
if not s:
raise ConfigurationError("Could not identitfy warehouse for term '{}'".format(args.database))
config = database_config(s.path)
elif args.subcommand not in ['list', 'new', 'install', 'test', 'parse']:
raise ConfigurationError(
"Must set the id, path or dsn of the database, either on the command line or in a manifest. ")
if not w and config:
w = new_warehouse(config, l, logger=global_logger)
if not w.exists() and args.subcommand not in ['clean', 'delete']:
raise ConfigurationError("Database {} must be created first".format(w.database.dsn))
if args.subcommand == 'new':
globals()['warehouse_' + args.subcommand](args, l, rc)
else:
globals()['warehouse_' + args.subcommand](args, w, rc)
def warehouse_command(args, rc):
from ambry.warehouse import new_warehouse
from ..library import new_library
from . import global_logger
from ambry.warehouse import database_config
l = new_library(rc.library(args.library_name))
w = None
l.logger = global_logger
config = None
#
# Special case for installing manifests, can get the database DSN from the
# manifest
if not args.database and args.subcommand == 'install':
from ..warehouse.manifest import Manifest
import os.path
m = Manifest(args.term)
dsnt = "spatialite:///{}.db" if m.is_geo else "sqlite:///{}.db"
name = args.name if args.name else m.uid
dsn = dsnt.format(l.warehouse_cache.path(name, missing_ok=True))
config = database_config(dsn)
w = new_warehouse(config, l, logger=global_logger)
if not w.exists():
w.create()
database_uid = w.uid
elif args.database:
# Check if the string is the uid of a database in the library.
s = l.store(args.database)
if not s:
raise ConfigurationError(
"Could not identitfy warehouse for term '{}'".format(
args.database))
config = database_config(s.path)
elif args.subcommand not in ['list', 'new', 'install', 'test']:
raise ConfigurationError(
"Must set the id, path or dsn of the database, either on the command line or in a manifest. "
)
if not w and config:
w = new_warehouse(config, l, logger=global_logger)
if not w.exists() and not args.subcommand in ['clean', 'delete']:
raise ConfigurationError(
"Database {} must be created first".format(w.database.dsn))
if args.subcommand == 'new':
globals()['warehouse_' + args.subcommand](args, l, rc)
else:
globals()['warehouse_' + args.subcommand](args, w, rc)
