def run_command(args, rc):
from . import global_logger
if args.test_library:
rc.set_lirbary_database('test')
try:
from ambry.library import global_library, Library
global global_library
l = Library(rc, echo=args.echo)
global_library = l
l.logger = global_logger
l.sync_config()
except Exception as e:
if args.subcommand != 'info':
warn('Failed to instantiate library: {}'.format(e))
l = None
if args.exceptions:
raise
try:
globals()['root_' + args.subcommand](args, l, rc)
except NotFoundError as e:
if args.exceptions:
raise
fatal(e)
except Exception:
raise
def test_library(self):
db_path = '/tmp/foo/bar/library.db'
if os.path.exists(db_path):
os.remove(db_path)
config_root = fsopendir('temp://')
config_root.createfile(CONFIG_FILE)
config_file_syspath = config_root.getsyspath(CONFIG_FILE)
with open(config_file_syspath, 'w') as f:
f.write("""
library:
category: development
filesystem_root: /tmp/foo/bar
remotes:
census: s3://test.library.civicknowledge.com/census
public: s3://test.library.civicknowledge.com/public
restricted: s3://test.library.civicknowledge.com/restricted
test: s3://test.library.civicknowledge.com/test""")
config = load(config_root.getsyspath('/'))
lf = LibraryFilesystem(config)
self.assertTrue('/tmp/foo/bar', lf.root)
l = Library(config)
l.sync_config()
self.assertEqual(sorted(['test', 'restricted', 'census', 'public']),
sorted([x.short_name for x in l.remotes]))
def __init__(self,
app_config,
import_name,
static_path=None,
static_url_path=None,
static_folder='static',
template_folder='templates',
instance_path=None,
instance_relative_config=False):
from flask.ext.cache import Cache
from ambry.library import Library
from ambry.run import get_runconfig
self._initialized = False
self.csrf = CsrfProtect()
self.login_manager = LoginManager()
super(Application,
self).__init__(import_name, static_path, static_url_path,
static_folder, template_folder, instance_path,
instance_relative_config)
self.config.update(app_config)
l = Library(get_runconfig(), read_only=True, echo=False)
self.cache = Cache(config={
'CACHE_TYPE': 'filesystem',
'CACHE_DIR': l.filesystem.cache('ui')
})
self.cache.init_app(self)
def init_sqlite(self, use_proto=True):
import shutil
shutil.rmtree(self.sqlite_dir())
self.config.library.filesystem_root = self.sqlite_dir(create=False)
if use_proto:
self.build_proto()
shutil.copytree(self.proto_dir(), self.sqlite_dir(create=False))
return Library(self.config)
else:
self.sqlite_dir() # Ensure it exists
l = Library(self.config)
l.create()
return l
def init_pg(self, use_proto=True):
if use_proto:
# self.create_pg_template()
# self.build_proto()
self.create_pg(re_create=True)
else:
self.create_pg(re_create=True, template_name='template1')
l = Library(self.config)
l.create()
return l
def __call__(self, environ, start_response):
if not self._initialized:
from ambry.library import Library
from ambry.run import get_runconfig
rc = get_runconfig()
l = Library(rc, read_only=True, echo=False)
secret_key = None
if os.getenv('AMBRY_UI_SECRET'):
app.logger.info("Using secret_key from env")
secret_key = os.getenv('AMBRY_UI_SECRET')
if not secret_key and l.ui_config.secret:
app.logger.info("Using secret_key from library")
secret_key = l.ui_config.secret
if not secret_key:
from uuid import uuid4
app.logger.warn(
"SECRET_KEY was not set. Setting to a random value")
secret_key = str(
uuid4()) # Must be the same for all worker processes.
if not self.config['WTF_CSRF_SECRET_KEY']:
self.config['WTF_CSRF_SECRET_KEY'] = secret_key
self.config['SECRET_KEY'] = secret_key
title = os.getenv('AMBRY_UI_TITLE', "Ambry Data Library"),
if l.ui_config.website_title:
title = l.ui_config.website_title
self.config['website_title'] = title
self.secret_key = secret_key
self.csrf.init_app(self)
self.session_interface = ItsdangerousSessionInterface()
self.login_manager.init_app(app)
Bootstrap(app)
self._initialized = True
return super(Application, self).__call__(environ, start_response)
def test_remotes_accounts(self):
import test
from os.path import dirname, join, exists
from os import makedirs
import ambry.run
from ambry.library import Library
test_config_dir = join(dirname(test.__file__), 'test_config',
'separate_files')
config = ambry.run.load(test_config_dir, load_user=False)
if not exists(config.library.filesystem_root):
makedirs(config.library.filesystem_root)
l = Library(config)
l.sync_config(force=True)
for r in l.remotes:
print r.short_name, r.access, r.secret
def __init__(self):
from ambry.library import Library
from ambry.run import get_runconfig
rc = get_runconfig()
self.library = Library(rc, read_only=True, echo=False)
def build_proto(self):
"""Builds the prototype library, by building or injesting any bundles that don't
exist in it yet. """
from ambry.orm.exc import NotFoundError
l = Library(self._proto_config())
try:
b = l.bundle('ingest.example.com-headerstypes')
except NotFoundError:
b = self.import_bundle(l, 'ingest.example.com/headerstypes')
b.log('Build to: {}'.format(b.build_fs))
b.ingest()
b.close()
try:
b = l.bundle('ingest.example.com-stages')
except NotFoundError:
b = self.import_bundle(l, 'ingest.example.com/stages')
b.ingest()
b.close()
try:
b = l.bundle('ingest.example.com-basic')
except NotFoundError:
b = self.import_bundle(l, 'ingest.example.com/basic')
b.ingest()
b.close()
try:
b = l.bundle('build.example.com-coverage')
except NotFoundError:
b = self.import_bundle(l, 'build.example.com/coverage')
b.ingest()
b.source_schema()
b.schema()
b.build()
b.finalize()
b.close()
try:
b = l.bundle('build.example.com-generators')
except NotFoundError:
b = self.import_bundle(l, 'build.example.com/generators')
b.run()
b.finalize()
b.close()
try:
b = l.bundle('build.example.com-plot')
except NotFoundError:
b = self.import_bundle(l, 'build.example.com/plot')
b.build()
b.finalize()
b.close()
try:
b = l.bundle('build.example.com-casters')
except NotFoundError:
b = self.import_bundle(l, 'build.example.com/casters')
b.ingest()
b.source_schema()
b.schema()
b.build()
b.finalize()
b.close()
try:
b = l.bundle('build.example.com-sql')
except NotFoundError:
b = self.import_bundle(l, 'build.example.com/sql')
b.build(sources=['integers', 'integers2', 'integers3'])
def remove(self, ref):
l = Library(self._proto_config())
l.remove(ref)
