def test_library(self):
db_path = '/tmp/foo/bar/library.db'
if os.path.exists(db_path):
os.remove(db_path)
config_root = fsopendir('temp://')
config_root.createfile(CONFIG_FILE)
config_file_syspath = config_root.getsyspath(CONFIG_FILE)
with open(config_file_syspath, 'w') as f:
f.write("""
library:
category: development
filesystem_root: /tmp/foo/bar
remotes:
census: s3://test.library.civicknowledge.com/census
public: s3://test.library.civicknowledge.com/public
restricted: s3://test.library.civicknowledge.com/restricted
test: s3://test.library.civicknowledge.com/test""")
config = load(config_root.getsyspath('/'))
lf = LibraryFilesystem(config)
self.assertTrue('/tmp/foo/bar', lf.root)
l = Library(config)
l.sync_config()
self.assertEqual(sorted(['test', 'restricted', 'census', 'public']),
sorted([x.short_name for x in l.remotes]))
def test_library(self):
db_path = '/tmp/foo/bar/library.db'
if os.path.exists(db_path):
os.remove(db_path)
config_root = fsopendir('temp://')
config_root.createfile(CONFIG_FILE)
config_file_syspath = config_root.getsyspath(CONFIG_FILE)
with open(config_file_syspath, 'w') as f:
f.write("""
library:
category: development
filesystem_root: /tmp/foo/bar
remotes:
census: s3://test.library.civicknowledge.com/census
public: s3://test.library.civicknowledge.com/public
restricted: s3://test.library.civicknowledge.com/restricted
test: s3://test.library.civicknowledge.com/test""")
config = load(config_root.getsyspath('/'))
lf = LibraryFilesystem(config)
self.assertTrue('/tmp/foo/bar', lf.root)
l = Library(config)
l.sync_config()
self.assertEqual(
sorted(['test', 'restricted', 'census', 'public']),
sorted([x.short_name for x in l.remotes]))
def run_command(args, rc):
from . import global_logger
if args.test_library:
rc.set_lirbary_database('test')
try:
from ambry.library import global_library, Library
global global_library
l = Library(rc, echo=args.echo)
global_library = l
l.logger = global_logger
l.sync_config()
except Exception as e:
if args.subcommand != 'info':
warn('Failed to instantiate library: {}'.format(e))
l = None
if args.exceptions:
raise
try:
globals()['root_' + args.subcommand](args, l, rc)
except NotFoundError as e:
if args.exceptions:
raise
fatal(e)
except Exception:
raise
def run_command(args, rc):
from . import global_logger
if args.test_library:
rc.set_lirbary_database('test')
try:
from ambry.library import global_library, Library
global global_library
l = Library(rc, echo=args.echo)
global_library = l
l.logger = global_logger
l.sync_config()
except Exception as e:
if args.subcommand != 'info':
warn('Failed to instantiate library: {}'.format(e))
l = None
if args.exceptions:
raise
try:
globals()['root_' + args.subcommand](args, l, rc)
except NotFoundError as e:
if args.exceptions:
raise
fatal(e)
except Exception:
raise
def init_pg(self, use_proto=True):
if use_proto:
# self.create_pg_template()
# self.build_proto()
self.create_pg(re_create=True)
else:
self.create_pg(re_create=True, template_name='template1')
l = Library(self.config)
l.create()
return l
def init_pg(self, use_proto=True):
if use_proto:
# self.create_pg_template()
# self.build_proto()
self.create_pg(re_create=True)
else:
self.create_pg(re_create=True, template_name='template1')
l = Library(self.config)
l.create()
return l
def __init__(self,
app_config,
import_name,
static_path=None,
static_url_path=None,
static_folder='static',
template_folder='templates',
instance_path=None,
instance_relative_config=False):
from flask.ext.cache import Cache
from ambry.library import Library
from ambry.run import get_runconfig
self._initialized = False
self.csrf = CsrfProtect()
self.login_manager = LoginManager()
super(Application,
self).__init__(import_name, static_path, static_url_path,
static_folder, template_folder, instance_path,
instance_relative_config)
self.config.update(app_config)
l = Library(get_runconfig(), read_only=True, echo=False)
self.cache = Cache(config={
'CACHE_TYPE': 'filesystem',
'CACHE_DIR': l.filesystem.cache('ui')
})
self.cache.init_app(self)
def setUp(self):
self.sqlite_db = LibraryDb(driver='sqlite', dbname=SQLITE_DATABASE)
self.sqlite_db.enable_delete = True
self.sqlite_db.create_tables()
FileFactory._meta.sqlalchemy_session = self.sqlite_db.session
# use cache with dict instead of DictCache
self.cache = {'key1': 'val1'} # TODO: use valid cache
self.lib = Library(self.cache, self.sqlite_db)
def test_remotes_accounts(self):
import test
from os.path import dirname, join, exists
from os import makedirs
import ambry.run
from ambry.library import Library
test_config_dir = join(dirname(test.__file__), 'test_config', 'separate_files')
config = ambry.run.load(test_config_dir, load_user=False)
if not exists(config.library.filesystem_root):
makedirs(config.library.filesystem_root)
l = Library(config)
l.sync_config(force=True)
for r in l.remotes:
print r.short_name, r.access, r.secret
def init_sqlite(self, use_proto=True):
import shutil
shutil.rmtree(self.sqlite_dir())
self.config.library.filesystem_root = self.sqlite_dir(create=False)
if use_proto:
self.build_proto()
shutil.copytree(self.proto_dir(), self.sqlite_dir(create=False))
return Library(self.config)
else:
self.sqlite_dir() # Ensure it exists
l = Library(self.config)
l.create()
return l
def init_sqlite(self, use_proto=True):
import shutil
shutil.rmtree(self.sqlite_dir())
self.config.library.filesystem_root = self.sqlite_dir(create=False)
if use_proto:
self.build_proto()
shutil.copytree(self.proto_dir(), self.sqlite_dir(create=False))
return Library(self.config)
else:
self.sqlite_dir() # Ensure it exists
l = Library(self.config)
l.create()
return l
def test_remotes_accounts(self):
import test
from os.path import dirname, join, exists
from os import makedirs
import ambry.run
from ambry.library import Library
test_config_dir = join(dirname(test.__file__), 'test_config',
'separate_files')
config = ambry.run.load(test_config_dir, load_user=False)
if not exists(config.library.filesystem_root):
makedirs(config.library.filesystem_root)
l = Library(config)
l.sync_config(force=True)
for r in l.remotes:
print r.short_name, r.access, r.secret
def __call__(self, environ, start_response):
if not self._initialized:
from ambry.library import Library
from ambry.run import get_runconfig
rc = get_runconfig()
l = Library(rc, read_only=True, echo=False)
secret_key = None
if os.getenv('AMBRY_UI_SECRET'):
app.logger.info("Using secret_key from env")
secret_key = os.getenv('AMBRY_UI_SECRET')
if not secret_key and l.ui_config.secret:
app.logger.info("Using secret_key from library")
secret_key = l.ui_config.secret
if not secret_key:
from uuid import uuid4
app.logger.warn(
"SECRET_KEY was not set. Setting to a random value")
secret_key = str(
uuid4()) # Must be the same for all worker processes.
if not self.config['WTF_CSRF_SECRET_KEY']:
self.config['WTF_CSRF_SECRET_KEY'] = secret_key
self.config['SECRET_KEY'] = secret_key
title = os.getenv('AMBRY_UI_TITLE', "Ambry Data Library"),
if l.ui_config.website_title:
title = l.ui_config.website_title
self.config['website_title'] = title
self.secret_key = secret_key
self.csrf.init_app(self)
self.session_interface = ItsdangerousSessionInterface()
self.login_manager.init_app(app)
Bootstrap(app)
self._initialized = True
return super(Application, self).__call__(environ, start_response)
def remove(self, ref):
l = Library(self._proto_config())
l.remove(ref)
class DocCacheTest(unittest.TestCase):
def setUp(self):
self.sqlite_db = LibraryDb(driver='sqlite', dbname=SQLITE_DATABASE)
self.sqlite_db.enable_delete = True
self.sqlite_db.create_tables()
FileFactory._meta.sqlalchemy_session = self.sqlite_db.session
# use cache with dict instead of DictCache
self.cache = {'key1': 'val1'} # TODO: use valid cache
self.lib = Library(self.cache, self.sqlite_db)
def tearDown(self):
self.sqlite_db.close()
try:
os.remove(SQLITE_DATABASE)
except OSError:
pass
# _munge_key tests
def test_uses_given_key_and_deletes_it_from_kwargs(self):
dc = DocCache(self.lib)
key, args, kwargs = dc._munge_key(_key='my-key')
self.assertIn('my-key', key)
self.assertNotIn('_key', kwargs)
def test_uses_given_key_to_create_prefix(self):
dc = DocCache(self.lib)
key, args, kwargs = dc._munge_key(_key='my-key')
self.assertEquals('m/y/my-key', key)
def test_adds_given_prefix_to_the_key_and_deletes_prefix_from_kwargs(self):
dc = DocCache(self.lib)
key, args, kwargs = dc._munge_key(_key='my-key', _key_prefix='prefix')
self.assertEquals('prefix/m/y/my-key', key)
self.assertNotIn('_key_prefix', kwargs)
def test_uses_argument_names_to_create_key(self):
dc = DocCache(self.lib)
key, args, kwargs = dc._munge_key('arg1', 'arg2')
self.assertEquals('a/r/arg1_arg2', key)
# it should returns the same args
self.assertEquals(len(args), 2)
self.assertIn('arg1', args)
self.assertIn('arg2', args)
def test_uses_optional_argument_names_to_create_key(self):
dc = DocCache(self.lib)
key, args, kwargs = dc._munge_key(kwarg1='kwarg1', kwarg2='kwarg2')
self.assertEquals('k/w/kwarg1_kwarg2', key)
# it should returns the same args
self.assertEquals(len(kwargs), 2)
self.assertIn('kwarg1', kwargs)
self.assertIn('kwarg2', kwargs)
def test_uses_upper_prefixes(self):
dc = DocCache(self.lib)
dc.prefix_upper = True
key, args, kwargs = dc._munge_key(_key='TheKey')
self.assertEquals('_/T/_The_Key', key)
# .cache tests
def test_adds_return_value_to_the_cache_if_cache_is_empty(self):
dc = DocCache(self.lib)
self.assertEquals(dc._cache, {})
dc.cache(sorted, [1, 2], reverse=True)
# reverted list is in the cache.
self.assertIn([2, 1], dc._cache.values())
def test_adds_return_value_to_the_cache_if_cache_is_ignored(self):
dc = DocCache(self.lib)
dc.ignore_cache = True
# cache is empty
self.assertEquals(dc._cache, {})
dc.cache(sorted, [1, 2], reverse=True)
key = dc._munge_key([1, 2], reverse=True)[0]
self.assertIn(key, dc._cache)
dc._cache[key] = [4, 3, 2, 1]
# force adding
dc.cache(sorted, [1, 2], reverse=True)
self.assertEquals(dc._cache[key], [2, 1])
def test_returns_value_from_cache(self):
dc = DocCache(self.lib)
key = dc._munge_key([1, 2], reverse=True)[0]
dc._cache[key] = [3, 2, 1]
ret = dc.cache(sorted, [1, 2], reverse=True)
self.assertEquals(ret, [3, 2, 1])
# .clean tests
@fudge.patch(
'ckcache.dictionary.DictCache.clean')
def test_cleans_dictcache(self, fake_clean):
fake_clean.expects_call()
# force cache to use DictCache
self.lib._doc_cache = True
dc = DocCache(self.lib)
dc.clean()
def test_cleans_dictionary(self):
dc = DocCache(self.lib)
dc._cache = {'a': 'b'}
dc.clean()
self.assertEquals(dc._cache, {})
# .remove tests
def test_removes_given_call_from_cache(self):
dc = DocCache(self.lib)
key = dc._munge_key([1, 2], reverse=True)[0]
dc._cache[key] = [2, 1]
dc.remove([1, 2], reverse=True)
self.assertNotIn(key, dc._cache)
# .library_info tests
def test_adds_library_info_to_the_cache(self):
dc = DocCache(self.lib)
dc.library_info()
self.assertIn('l/i/library_info', dc._cache)
self.assertEquals(dc._cache['l/i/library_info'], self.lib.summary_dict)
# .bundle_index tests
def test_adds_bundle_index_to_the_cache(self):
self.lib.versioned_datasets = lambda: ['a', 'b']
dc = DocCache(self.lib)
dc.bundle_index()
self.assertIn('b/u/bundle_index', dc._cache)
self.assertEquals(
dc._cache['b/u/bundle_index'],
self.lib.versioned_datasets())
# .dataset tests
def test_adds_dataset_call_to_the_cache(self):
class FakeDataset(object):
dict = {
'title': 'title1',
'summary': 'summary1'}
self.lib.dataset = lambda x: FakeDataset()
dc = DocCache(self.lib)
dc.dataset('vid')
# TODO: is using _key_prefix as key suffix valid behaviour?
self.assertIn('ds/v/i/vidds', dc._cache)
self.assertEquals(dc._cache['ds/v/i/vidds'], FakeDataset().dict)
# .bundle_summary tests
def test_adds_bundle_summary_to_the_cache(self):
class FakeBundle(object):
summary_dict = {'a': 'b'}
self.lib.bundle = lambda x: FakeBundle()
dc = DocCache(self.lib)
dc.bundle_summary('vid')
# TODO: is using _key_prefix as key suffix valid behaviour?
self.assertIn('bs/v/i/vidbs', dc._cache)
self.assertEquals(dc._cache['bs/v/i/vidbs'], FakeBundle().summary_dict)
# .bundle tests
def test_adds_bundle_to_the_cache(self):
class FakeBundle(object):
dict = {'a': 'b'}
self.lib.bundle = lambda x: FakeBundle()
dc = DocCache(self.lib)
dc.bundle('vid')
self.assertIn('v/i/vid', dc._cache)
self.assertEquals(dc._cache['v/i/vid'], FakeBundle().dict)
# .partition tests
def test_adds_partition_to_the_cache(self):
class FakePartition(object):
dict = {'a': 'b'}
self.lib.partition = lambda x: FakePartition()
dc = DocCache(self.lib)
dc.partition('vid')
self.assertIn('v/i/vid', dc._cache)
self.assertEquals(dc._cache['v/i/vid'], FakePartition().dict)
# .table tests
def test_adds_table_to_the_cache(self):
class FakeTable(object):
nonull_col_dict = {
'a': 'b',
'columns': {}}
self.lib.table = lambda x: FakeTable()
dc = DocCache(self.lib)
dc.table('vid')
self.assertIn('v/i/vid', dc._cache)
self.assertEquals(dc._cache['v/i/vid'], FakeTable().nonull_col_dict)
def test_adds_warehouse_to_the_cache(self):
class FakeWarehouse(object):
dict = {'a': 'b'}
self.lib.warehouse = lambda x: FakeWarehouse()
dc = DocCache(self.lib)
dc.warehouse('vid')
self.assertIn('v/i/vid', dc._cache)
self.assertEquals(dc._cache['v/i/vid'], FakeWarehouse().dict)
def test_adds_manifest_to_the_cache(self):
class FakeManifest(object):
dict = {'a': 'b'}
def fake_manifest(vid):
return '', FakeManifest()
self.lib.manifest = fake_manifest
dc = DocCache(self.lib)
dc.manifest('vid')
self.assertIn('v/i/vid', dc._cache)
self.assertEquals(dc._cache['v/i/vid'], FakeManifest().dict)
# .table_version_map
def test_adds_table_version_map_to_the_cache(self):
class FakeTable(object):
def __init__(self, id_, vid):
self.id_ = id_
self.vid = vid
tables_no_columns = [
FakeTable('1', 'vid1'),
FakeTable('2', 'vid2'),
FakeTable('3', 'vid3'),
FakeTable('3', 'vid4')]
dc = DocCache(self.lib)
with fudge.patched_context(self.lib, 'tables_no_columns', tables_no_columns):
dc.table_version_map()
self.assertIn('t/a/table_version_map', dc._cache)
self.assertIn('1', dc._cache['t/a/table_version_map'])
self.assertIn('2', dc._cache['t/a/table_version_map'])
self.assertIn('3', dc._cache['t/a/table_version_map'])
self.assertEquals(dc._cache['t/a/table_version_map']['1'], ['vid1'])
self.assertEquals(dc._cache['t/a/table_version_map']['2'], ['vid2'])
self.assertEquals(dc._cache['t/a/table_version_map']['3'], ['vid3', 'vid4'])
def build_proto(self):
"""Builds the prototype library, by building or injesting any bundles that don't
exist in it yet. """
from ambry.orm.exc import NotFoundError
l = Library(self._proto_config())
try:
b = l.bundle('ingest.example.com-headerstypes')
except NotFoundError:
b = self.import_bundle(l, 'ingest.example.com/headerstypes')
b.log('Build to: {}'.format(b.build_fs))
b.ingest()
b.close()
try:
b = l.bundle('ingest.example.com-stages')
except NotFoundError:
b = self.import_bundle(l, 'ingest.example.com/stages')
b.ingest()
b.close()
try:
b = l.bundle('ingest.example.com-basic')
except NotFoundError:
b = self.import_bundle(l, 'ingest.example.com/basic')
b.ingest()
b.close()
try:
b = l.bundle('build.example.com-coverage')
except NotFoundError:
b = self.import_bundle(l, 'build.example.com/coverage')
b.ingest()
b.source_schema()
b.schema()
b.build()
b.finalize()
b.close()
try:
b = l.bundle('build.example.com-generators')
except NotFoundError:
b = self.import_bundle(l, 'build.example.com/generators')
b.run()
b.finalize()
b.close()
try:
b = l.bundle('build.example.com-plot')
except NotFoundError:
b = self.import_bundle(l, 'build.example.com/plot')
b.build()
b.finalize()
b.close()
try:
b = l.bundle('build.example.com-casters')
except NotFoundError:
b = self.import_bundle(l, 'build.example.com/casters')
b.ingest()
b.source_schema()
b.schema()
b.build()
b.finalize()
b.close()
try:
b = l.bundle('build.example.com-sql')
except NotFoundError:
b = self.import_bundle(l, 'build.example.com/sql')
b.build(sources=['integers', 'integers2', 'integers3'])
def remove(self, ref):
l = Library(self._proto_config())
l.remove(ref)
class AmbryAppContext(object):
"""Ambry specific objects for the application context"""
def __init__(self):
from ambry.library import Library
from ambry.run import get_runconfig
rc = get_runconfig()
self.library = Library(rc, read_only=True, echo=False)
def render(self, template, *args, **kwargs):
from flask import render_template
context = self.cc
context.update(kwargs)
context['l'] = self.library
return render_template(template, *args, **context)
@property
def cc(self):
"""Return common context values. These are primarily helper functions
that can be used from the context. """
from ambry._meta import __version__ as ambry_version
from __meta__ import __version__ as ui_version
from flask import request
def scale_font(x):
"""Scales the font for the partition title. Completely ad-hoc."""
size = 100
if len(x) > 130:
size = 90
return 'font-size: {}%;'.format(size)
return {
'scale_font': scale_font,
'ambry_version': ambry_version,
'ui_version': ui_version,
'url_for': url_for,
'from_root': lambda x: x,
'getattr': getattr,
'title': app.config.get('website_title'),
'next_page': request.path, # Actually last page, for login redirect
'session': session,
'autologin_user': app.config['LOGGED_IN_USER']
}
def bundle(self, ref):
from flask import abort
from ambry.orm.exc import NotFoundError
try:
return self.library.bundle(ref)
except NotFoundError:
abort(404)
def json(self, **kwargs):
return Response(dumps(kwargs, cls=JSONEncoder), mimetype='application/json')
def close(self):
self.library.close()
def __init__(self):
from ambry.library import Library
from ambry.run import get_runconfig
rc = get_runconfig()
self.library = Library(rc, read_only=True, echo=False)
def build_proto(self):
"""Builds the prototype library, by building or injesting any bundles that don't
exist in it yet. """
from ambry.orm.exc import NotFoundError
l = Library(self._proto_config())
try:
b = l.bundle('ingest.example.com-headerstypes')
except NotFoundError:
b = self.import_bundle(l, 'ingest.example.com/headerstypes')
b.log('Build to: {}'.format(b.build_fs))
b.ingest()
b.close()
try:
b = l.bundle('ingest.example.com-stages')
except NotFoundError:
b = self.import_bundle(l, 'ingest.example.com/stages')
b.ingest()
b.close()
try:
b = l.bundle('ingest.example.com-basic')
except NotFoundError:
b = self.import_bundle(l, 'ingest.example.com/basic')
b.ingest()
b.close()
try:
b = l.bundle('build.example.com-coverage')
except NotFoundError:
b = self.import_bundle(l, 'build.example.com/coverage')
b.ingest()
b.source_schema()
b.schema()
b.build()
b.finalize()
b.close()
try:
b = l.bundle('build.example.com-generators')
except NotFoundError:
b = self.import_bundle(l, 'build.example.com/generators')
b.run()
b.finalize()
b.close()
try:
b = l.bundle('build.example.com-plot')
except NotFoundError:
b = self.import_bundle(l, 'build.example.com/plot')
b.build()
b.finalize()
b.close()
try:
b = l.bundle('build.example.com-casters')
except NotFoundError:
b = self.import_bundle(l, 'build.example.com/casters')
b.ingest()
b.source_schema()
b.schema()
b.build()
b.finalize()
b.close()
try:
b = l.bundle('build.example.com-sql')
except NotFoundError:
b = self.import_bundle(l, 'build.example.com/sql')
b.build(sources=['integers', 'integers2', 'integers3'])
def __init__(self):
from ambry.library import Library
from ambry.run import get_runconfig
rc = get_runconfig()
self.library = Library(rc, read_only=True, echo=False)
class AmbryAppContext(object):
"""Ambry specific objects for the application context"""
def __init__(self):
from ambry.library import Library
from ambry.run import get_runconfig
rc = get_runconfig()
self.library = Library(rc, read_only=True, echo=False)
def render(self, template, *args, **kwargs):
from flask import render_template
context = self.cc
context.update(kwargs)
context['l'] = self.library
return render_template(template, *args, **context)
@property
def cc(self):
"""Return common context values. These are primarily helper functions
that can be used from the context. """
from ambry._meta import __version__ as ambry_version
from __meta__ import __version__ as ui_version
from flask import request
def scale_font(x):
"""Scales the font for the partition title. Completely ad-hoc."""
size = 100
if len(x) > 130:
size = 90
return 'font-size: {}%;'.format(size)
return {
'scale_font': scale_font,
'ambry_version': ambry_version,
'ui_version': ui_version,
'url_for': url_for,
'from_root': lambda x: x,
'getattr': getattr,
'title': app.config.get('website_title'),
'next_page':
request.path, # Actually last page, for login redirect
'session': session,
'autologin_user': app.config['LOGGED_IN_USER']
}
def bundle(self, ref):
from flask import abort
from ambry.orm.exc import NotFoundError
try:
return self.library.bundle(ref)
except NotFoundError:
abort(404)
def json(self, **kwargs):
return Response(dumps(kwargs, cls=JSONEncoder),
mimetype='application/json')
def close(self):
self.library.close()