def x_test_install(self): def resolver(name): if name == self.bundle.identity.name or name == self.bundle.identity.vname: return self.bundle else: return False def progress_cb(lr, type, name, n): if n: lr("{} {}: {}".format(type, name, n)) else: self.bundle.log("{} {}".format(type, name)) from ambry.warehouse import new_warehouse from functools import partial print "Getting warehouse" w = new_warehouse(self.rc.warehouse('postgres')) print "Re-create database" w.database.enable_delete = True w.resolver = resolver w.progress_cb = progress_cb try: w.drop() except: pass w.create() ps = self.bundle.partitions.all print "{} partitions".format(len(ps)) for p in self.bundle.partitions: lr = self.bundle.init_log_rate(10000) w.install(p, progress_cb=partial(progress_cb, lr)) self.assertTrue(w.has(self.bundle.identity.vname)) for p in self.bundle.partitions: self.assertTrue(w.has(p.identity.vname)) for p in self.bundle.partitions: w.remove(p.identity.vname) print w.get(self.bundle.identity.name) print w.get(self.bundle.identity.vname) print w.get(self.bundle.identity.id_) w.install(self.bundle) print w.get(self.bundle.identity.name) print w.get(self.bundle.identity.vname) print w.get(self.bundle.identity.id_) for p in self.bundle.partitions: lr = self.bundle.init_log_rate(10000) w.install(p, progress_cb=partial(progress_cb, lr))
def x_test_install(self): def resolver(name): if name == self.bundle.identity.name or name == self.bundle.identity.vname: return self.bundle else: return False def progress_cb(lr, type,name,n): if n: lr("{} {}: {}".format(type, name, n)) else: self.bundle.log("{} {}".format(type, name)) from ambry.warehouse import new_warehouse from functools import partial print "Getting warehouse" w = new_warehouse(self.rc.warehouse('postgres')) print "Re-create database" w.database.enable_delete = True w.resolver = resolver w.progress_cb = progress_cb try: w.drop() except: pass w.create() ps = self.bundle.partitions.all print "{} partitions".format(len(ps)) for p in self.bundle.partitions: lr = self.bundle.init_log_rate(10000) w.install(p, progress_cb = partial(progress_cb, lr) ) self.assertTrue(w.has(self.bundle.identity.vname)) for p in self.bundle.partitions: self.assertTrue(w.has(p.identity.vname)) for p in self.bundle.partitions: w.remove(p.identity.vname) print w.get(self.bundle.identity.name) print w.get(self.bundle.identity.vname) print w.get(self.bundle.identity.id_) w.install(self.bundle) print w.get(self.bundle.identity.name) print w.get(self.bundle.identity.vname) print w.get(self.bundle.identity.id_) for p in self.bundle.partitions: lr = self.bundle.init_log_rate(10000) w.install(p, progress_cb = partial(progress_cb, lr))
def get_warehouse(self, l, name, delete=True): from ambry.warehouse import new_warehouse waho = new_warehouse(self.rc.warehouse(name), l) if delete: waho.database.enable_delete = True waho.database.delete() waho.create() return waho
def warehouse(self, uid): from ambry.warehouse import new_warehouse, database_config s = self.store(uid) if not s: raise NotFoundError("Did not find warehouse for uid: '{}' ".format(uid)) config = database_config(s.path) return new_warehouse(config, self, logger=self.logger)
def warehouse(self, uid): from ambry.warehouse import new_warehouse, database_config s = self.store(uid) if not s: from ..dbexceptions import NotFoundError raise NotFoundError("Did not find warehouse for uid: '{}' ".format(uid)) config = database_config(s.path) return new_warehouse(config, self, logger=self.logger)
def get_warehouse(self, l, name, delete=True): from ambry.util import get_logger from ambry.warehouse import new_warehouse w = new_warehouse(self.rc.warehouse(name), l) w.logger = get_logger('unit_test') lr = self.bundle.init_log_rate(10000) w.logger = TestLogger(lr) if delete: w.database.enable_delete = True w.database.delete() w.create() return w
def get_warehouse(self, l, name, delete = True): from ambry.util import get_logger from ambry.warehouse import new_warehouse w = new_warehouse(self.rc.warehouse(name), l) w.logger = get_logger('unit_test') lr = self.bundle.init_log_rate(10000) w.logger = TestLogger(lr) if delete: w.database.enable_delete = True w.database.delete() w.create() return w
def _warehouse_new_from_dbc(dbc, l): from ambry.warehouse import new_warehouse from . import global_logger w = new_warehouse(dbc, l, logger=global_logger) w.create() for c in w.configurable: if c in dbc: w._meta_set(c, dbc[c]) sf = l.sync_warehouse(w) w.uid = sf.ref return w
def build(self): from ambry.warehouse import new_warehouse from ambry.warehouse.manifest import Manifest wf = self.filesystem.build_path(self.identity.vname, 'warehouse.db') w = new_warehouse('sqlite:///{}'.format(wf), self.library, self.logger) self.log("Installing manifest to {}".format(w.database.dsn)) w.create() w.install_manifest(Manifest(self.filesystem.meta_path('manifest.ambry'))) self.log("Installed manifest to {}".format(w.database.dsn)) for row in w.database.connection.execute("SELECT * FROM example_combined"): print row return True
def warehouse_command(args, rc): from ambry.warehouse import new_warehouse from ..library import new_library from . import logger import urlparse if args.database: parts = urlparse.urlparse(args.database) if parts.scheme == 'sqlite': config = dict(service='sqlite', database=dict(dbname=parts.path, driver='sqlite')) elif parts.scheme == 'spatialite': config = dict(service='spatialite', database=dict(dbname=args.spatialite, driver='sqlite')) elif parts.scheme == 'postgres': config = dict(service='postgres', database=dict(driver='postgres', server=parts.hostname, username=parts.username, password=parts.password, dbname=parts.path.strip('/') )) elif parts.scheme == 'postgis': config = dict(service='postgis', database=dict(driver='postgis', server=parts.hostname, username=parts.username, password=parts.password, dbname=parts.path.strip('/') )) else: raise ValueError("Unknown database connection scheme: {}".format(parts.scheme)) else: config = rc.warehouse(args.name) l = new_library(rc.library(args.library_name)) l.logger = logger w = new_warehouse(config, l) globals()['warehouse_'+args.subcommand](args, w,rc)
def build(self): from ambry.warehouse import new_warehouse from ambry.warehouse.manifest import Manifest import sqlite3 wf = self.filesystem.build_path(self.identity.vname, 'warehouse.db') w = new_warehouse('sqlite:///{}'.format(wf), self.library, self.logger) self.log("Installing manifest to {}".format(w.database.dsn)) w.create() w.install_manifest( Manifest(self.filesystem.meta_path('manifest.ambry'))) self.log("Installed manifest to {}".format(w.database.dsn)) for row in w.database.connection.execute( "SELECT * FROM example_combined"): print row return True
def warehouse_command(args, rc): from ambry.warehouse import new_warehouse from ..library import new_library from . import global_logger from ambry.warehouse import database_config l = new_library(rc.library(args.library_name)) w = None l.logger = global_logger config = None # # Special case for installing manifests, can get the database DSN from the # manifest if not args.database and args.subcommand == 'install': from ..warehouse.manifest import Manifest m = Manifest(args.term) dsnt = "spatialite:///{}.db" if m.is_geo else "sqlite:///{}.db" name = args.name if args.name else m.uid dsn = dsnt.format(l.warehouse_cache.path(name, missing_ok=True)) config = database_config(dsn) w = new_warehouse(config, l, logger=global_logger) if not w.exists(): w.create() elif args.database: # Check if the string is the uid of a database in the library. s = l.store(args.database) if not s: raise ConfigurationError("Could not identitfy warehouse for term '{}'".format(args.database)) config = database_config(s.path) elif args.subcommand not in ['list', 'new', 'install', 'test', 'parse']: raise ConfigurationError( "Must set the id, path or dsn of the database, either on the command line or in a manifest. ") if not w and config: w = new_warehouse(config, l, logger=global_logger) if not w.exists() and args.subcommand not in ['clean', 'delete']: raise ConfigurationError("Database {} must be created first".format(w.database.dsn)) if args.subcommand == 'new': globals()['warehouse_' + args.subcommand](args, l, rc) else: globals()['warehouse_' + args.subcommand](args, w, rc)
def warehouse_command(args, rc): from ambry.warehouse import new_warehouse from ..library import new_library from . import global_logger from ambry.warehouse import database_config l = new_library(rc.library(args.library_name)) w = None l.logger = global_logger config = None # # Special case for installing manifests, can get the database DSN from the # manifest if not args.database and args.subcommand == 'install': from ..warehouse.manifest import Manifest import os.path m = Manifest(args.term) dsnt = "spatialite:///{}.db" if m.is_geo else "sqlite:///{}.db" name = args.name if args.name else m.uid dsn = dsnt.format(l.warehouse_cache.path(name, missing_ok=True)) config = database_config(dsn) w = new_warehouse(config, l, logger=global_logger) if not w.exists(): w.create() database_uid = w.uid elif args.database: # Check if the string is the uid of a database in the library. s = l.store(args.database) if not s: raise ConfigurationError( "Could not identitfy warehouse for term '{}'".format( args.database)) config = database_config(s.path) elif args.subcommand not in ['list', 'new', 'install', 'test']: raise ConfigurationError( "Must set the id, path or dsn of the database, either on the command line or in a manifest. " ) if not w and config: w = new_warehouse(config, l, logger=global_logger) if not w.exists() and not args.subcommand in ['clean', 'delete']: raise ConfigurationError( "Database {} must be created first".format(w.database.dsn)) if args.subcommand == 'new': globals()['warehouse_' + args.subcommand](args, l, rc) else: globals()['warehouse_' + args.subcommand](args, w, rc)