def test_partition_2(self):
bundle = Bundle()
bundle.clean()
bundle.pre_prepare()
bundle.prepare()
bundle.post_prepare()
table = self.bundle.schema.tables[0]
p = (('time', 'time2'), ('space', 'space3'), ('table', table.name),
('grain', 'grain4'))
p += p
pids = {}
for i in range(4):
for j in range(4):
pid = self.bundle.identity.as_partition(**dict(p[i:i + j + 1]))
pids[pid.fqname] = pid
for pid in pids.values():
part = bundle.partitions.new_db_partition(**pid.dict)
part.create()
parts = bundle.partitions._find_orm(
PartitionNameQuery(vid=pid.vid)).all()
self.assertIn(pid.sname, [p.name for p in parts])
def setUp(self):
import bundles.testbundle.bundle
from ambry.run import RunConfig
import manifests, configs
self.bundle_dir = os.path.dirname(bundles.testbundle.bundle.__file__)
self.config_dir = os.path.dirname(configs.__file__)
self.rc = get_runconfig(
(os.path.join(self.config_dir, 'test.yaml'),
os.path.join(self.bundle_dir,
'bundle.yaml'), RunConfig.USER_ACCOUNTS))
self.copy_or_build_bundle()
self.bundle = Bundle()
#print "Deleting: {}".format(self.rc.group('filesystem').root)
#ambry.util.rm_rf(self.rc.group('filesystem').root)
self.m = os.path.join(os.path.dirname(manifests.__file__),
'test.ambry')
with open(self.m) as f:
self.m_contents = f.read()
def setUp(self):
super(Test, self).setUp()
self.copy_or_build_bundle()
self.bundle = Bundle()
self.bundle_dir = self.bundle.bundle_dir
def setUp(self):
super(Test, self).setUp()
self.copy_or_build_bundle()
self.bundle = Bundle()
self.bundle_dir = self.bundle.bundle_dir
def copy_or_build_bundle(self):
"""Set up a clean bundle build, either by re-building the bundle, or
by copying it from a saved bundle directory """
# For most cases, re-set the bundle by copying from a saved version. If
# the bundle doesn't exist and the saved version doesn't exist,
# build a new one.
bundle, marker, build_dir, save_dir = self.bundle_dirs()
idnt = bundle.identity
if str(idnt.name.version) != "0.0.1":
# Rebuild the bundle if the test_library.py:test_versions
# script didn't reset the bundle at the end
from ambry.util import rm_rf
rm_rf(build_dir)
rm_rf(save_dir)
idnt = Identity.from_dict({'subset': 'subset',
'vid': 'piEGPXmDC8001001',
'variation': 'variation',
'dataset': 'dataset',
'source': 'source',
'version': '0.0.1',
'id': 'diEGPXmDC8',
'revision': 1}
)
bundle.config.rewrite(
identity = idnt.ident_dict,
names = idnt.names_dict
)
if not os.path.exists(marker):
logger.info( "Build dir marker ({}) is missing".format(marker))
# There is a good reason to create a seperate instance,
# but don't remember what it is ...
bundle.clean()
bundle = Bundle()
if not os.path.exists(save_dir):
logger.info( "Save dir is missing; re-build bundle. ")
bundle.prepare()
if str(bundle.identity.name.version) != '0.0.1':
raise Exception("Can only save bundle if version is 0.0.1")
bundle.build()
bundle.close()
with open(marker, 'w') as f:
f.write(str(time.time()))
# Copy the newly built bundle to the save directory
os.system("rm -rf {1}; rsync -arv {0} {1} > /dev/null ".format(build_dir, save_dir))
# Always copy, just to be safe.
logger.info( "Copying bundle from {}".format(save_dir))
os.system("rm -rf {0}; rsync -arv {1} {0} > /dev/null ".format(build_dir, save_dir))
def bundle_dirs(self):
bundle = Bundle()
marker = bundle.filesystem.build_path('test-marker')
build_dir = bundle.filesystem.build_path(
) + '/' # Slash needed for rsync
save_dir = bundle.filesystem.build_path() + "-save/"
return bundle, marker, build_dir, save_dir
def test_wkb(self):
from shapely.wkb import dumps, loads
b = Bundle()
p = b.partitions.find(table='geot2')
for row in p.query(
"SELECT quote(AsBinary(GEOMETRY)) as wkb, quote(GEOMETRY) FROM geot2"
):
print row
def setUp(self):
import os
from ambry.run import get_runconfig, RunConfig
self.copy_or_build_bundle()
self.bundle = Bundle()
self.bundle_dir = self.bundle.bundle_dir
self.rc = get_runconfig(
(os.path.join(self.bundle_dir, 'geo-test-config.yaml'),
os.path.join(self.bundle_dir,
'bundle.yaml'), RunConfig.USER_ACCOUNTS))
def test_db_bundle(self):
from ambry.bundle import BuildBundle, DbBundle
b = BuildBundle(self.bundle_dir)
b.clean()
self.assertTrue(b.identity.id_ is not None)
self.assertEquals('source-dataset-subset-variation', b.identity.sname)
self.assertEquals('source-dataset-subset-variation-0.0.1',
b.identity.vname)
b.database.create()
db_path = b.database.path
dbb = DbBundle(db_path)
self.assertEqual("source-dataset-subset-variation", dbb.identity.sname)
self.assertEqual("source-dataset-subset-variation-0.0.1",
dbb.identity.vname)
b = Bundle()
b.database.enable_delete = True
b.clean()
b.database.create()
b = Bundle()
b.exit_on_fatal = False
b.pre_prepare()
b.prepare()
b.post_prepare()
b.pre_build()
#b.build_db_inserter()
b.build_geo()
b.post_build()
b.close()
def setUp(self):
super(Test, self).setUp() #
import bundles.testbundle.bundle
self.bundle_dir = os.path.dirname(bundles.testbundle.bundle.__file__)
self.rc = get_runconfig(
(os.path.join(self.bundle_dir, 'library-test-config.yaml'),
os.path.join(self.bundle_dir,
'bundle.yaml'), RunConfig.USER_ACCOUNTS))
self.copy_or_build_bundle()
self.bundle = Bundle()
Test.rm_rf(self.rc.group('filesystem').root)
def setUp(self):
self.copy_or_build_bundle()
self.bundle_dir = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'testbundle')
self.rc = RunConfig([
os.path.join(self.bundle_dir, 'client-test-config.yaml'),
os.path.join(self.bundle_dir, 'bundle.yaml'), RunConfig.USER_CONFIG
])
self.server_rc = RunConfig([
os.path.join(self.bundle_dir, 'server-test-config.yaml'),
RunConfig.USER_CONFIG
])
self.bundle = Bundle()
self.bundle_dir = self.bundle.bundle_dir
def test_partition_2(self):
bundle = Bundle()
bundle.clean()
bundle.pre_prepare()
bundle.prepare()
bundle.post_prepare()
table = self.bundle.schema.tables[0]
p = (('time', 'time2'), ('space', 'space3'), ('table', table.name), ('grain', 'grain4'))
p += p
pids = {}
for i in range(4):
for j in range(4):
pid = self.bundle.identity.as_partition(**dict(p[i:i + j + 1]))
pids[pid.fqname] = pid
for pid in pids.values():
part = bundle.partitions.new_db_partition(**pid.dict)
part.create()
parts = bundle.partitions._find_orm(PartitionNameQuery(vid=pid.vid)).all()
self.assertIn(pid.sname, [p.name for p in parts])
def setUp(self):
import bundles.testbundle.bundle
from ambry.run import RunConfig
import manifests, configs
self.bundle_dir = os.path.dirname( bundles.testbundle.bundle.__file__)
self.config_dir = os.path.dirname(configs.__file__)
self.rc = get_runconfig((os.path.join(self.config_dir, 'test.yaml'),
os.path.join(self.bundle_dir,'bundle.yaml'),
RunConfig.USER_ACCOUNTS))
self.copy_or_build_bundle()
self.bundle = Bundle()
#print "Deleting: {}".format(self.rc.group('filesystem').root)
#ambry.util.rm_rf(self.rc.group('filesystem').root)
self.m = os.path.join(os.path.dirname(manifests.__file__), 'test.ambry')
with open(self.m) as f:
self.m_contents = f.read()
def copy_or_build_bundle(self):
"""Set up a clean bundle build, either by re-building the bundle, or
by copying it from a saved bundle directory """
# For most cases, re-set the bundle by copying from a saved version. If
# the bundle doesn't exist and the saved version doesn't exist,
# build a new one.
bundle, marker, build_dir, save_dir = self.bundle_dirs()
idnt = bundle.identity
if str(idnt.name.version) != "0.0.1":
# Rebuild the bundle if the test_library.py:test_versions
# script didn't reset the bundle at the end
from ambry.util import rm_rf
rm_rf(build_dir)
rm_rf(save_dir)
idnt = Identity.from_dict(dict(bundle.metadata.identity))
bundle.metadata.identity = idnt.ident_dict
bundle.metadata.names = idnt.names_dict
bundle.metadata.write_to_dir()
if not os.path.exists(marker):
global_logger.info(
"Build dir marker ({}) is missing".format(marker))
# There is a good reason to create a seperate instance,
# but don't remember what it is ...
bundle.clean()
bundle = Bundle()
if not os.path.exists(save_dir):
global_logger.info("Save dir is missing; re-build bundle. ")
bundle.pre_prepare()
bundle.prepare()
bundle.post_prepare()
if str(bundle.identity.name.version) != '0.0.1':
raise Exception(
"Can only save bundle if version is 0.0.1. This one is version: {} "
.format(bundle.identity.name.version))
bundle.pre_build()
bundle.build()
bundle.post_build()
bundle.close()
with open(marker, 'w') as f:
f.write(str(time.time()))
# Copy the newly built bundle to the save directory
os.system("rm -rf {1}; rsync -arv {0} {1} > /dev/null ".format(
build_dir, save_dir))
# Always copy, just to be safe.
#global_logger.info( "Copying bundle from {}".format(save_dir))
os.system("rm -rf {0}; rsync -arv {1} {0} > /dev/null ".format(
build_dir, save_dir))
class Test(TestBase):
def setUp(self):
super(Test, self).setUp()
self.copy_or_build_bundle()
self.bundle = Bundle()
self.bundle_dir = self.bundle.bundle_dir
def test_db_bundle(self):
from ambry.bundle import BuildBundle, DbBundle
b = BuildBundle(self.bundle_dir)
b.clean()
self.assertTrue(b.identity.id_ is not None)
self.assertEquals('source-dataset-subset-variation', b.identity.sname)
self.assertEquals('source-dataset-subset-variation-0.0.1',
b.identity.vname)
b.database.create()
db_path = b.database.path
dbb = DbBundle(db_path)
self.assertEqual("source-dataset-subset-variation", dbb.identity.sname)
self.assertEqual("source-dataset-subset-variation-0.0.1",
dbb.identity.vname)
def test_paths(self):
''' Test that a build bundle and a db bundle both produce the same paths. '''
from ambry.bundle import DbBundle
b = self.bundle
db = DbBundle(b.database.path)
self.assertEqual(b.path, db.path)
self.assertTrue(os.path.exists(b.path))
self.assertEqual(b.database.path, db.database.path)
self.assertTrue(os.path.exists(b.database.path))
self.assertEqual(b.identity.path, db.identity.path)
for p in zip(b.partitions, db.partitions):
self.assertTrue(bool(p[0].path))
self.assertEqual(p[0].path, p[1].path)
self.assertTrue(bool(p[0].path))
def test_schema_direct(self):
'''Test adding tables directly to the schema'''
# If we don't explicitly set the id_, it will change for every run.
self.bundle.metadata.identity.id = 'aTest'
self.bundle.schema.clean()
with self.bundle.session:
s = self.bundle.schema
s.add_table('table 1', altname='alt name a')
s.add_table('table 2', altname='alt name b')
self.assertRaises(Exception, s.add_table, ('table 1', ))
t = s.add_table('table 3', altname='alt name')
s.add_column(t, 'col 1', altname='altname1')
s.add_column(t, 'col 2', altname='altname2')
s.add_column(t, 'col 3', altname='altname3')
#print self.bundle.schema.as_csv()
self.assertIn('tiEGPXmDC801',
[t.id_ for t in self.bundle.schema.tables])
self.assertIn('tiEGPXmDC802',
[t.id_ for t in self.bundle.schema.tables])
self.assertNotIn('cTest03', [t.id_ for t in self.bundle.schema.tables])
t = self.bundle.schema.table('table_3')
self.assertIn('ciEGPXmDC803001', [c.id_ for c in t.columns])
self.assertIn('ciEGPXmDC803002', [c.id_ for c in t.columns])
self.assertIn('ciEGPXmDC803003', [c.id_ for c in t.columns])
# Try with a nested session, b/c we need to test it somewhere ...
with self.bundle.session:
with self.bundle.session:
t = s.add_table('table 4', altname='alt name')
s.add_column(t, 'col 1', altname='altname1')
s.add_column(t, 'col 2', altname='altname2')
s.add_column(t, 'col 3', altname='altname3')
def x_test_generate_schema(self):
'''Uses the generateSchema method in the bundle'''
from ambry.orm import Column
with self.bundle.session:
s = self.bundle.schema
s.clean()
t1 = s.add_table('table1')
s.add_column(t1, name='col1', datatype=Column.DATATYPE_REAL)
s.add_column(t1, name='col2', datatype=Column.DATATYPE_INTEGER)
s.add_column(t1, name='col3', datatype=Column.DATATYPE_TEXT)
t2 = s.add_table('table2')
s.add_column(t2, name='col1')
s.add_column(t2, name='col2')
s.add_column(t2, name='col3')
t3 = s.add_table('table3')
s.add_column(t3, name='col1', datatype=Column.DATATYPE_REAL)
s.add_column(t3, name='col2', datatype=Column.DATATYPE_INTEGER)
s.add_column(t3, name='col3', datatype=Column.DATATYPE_TEXT)
def test_column_processor(self):
from ambry.orm import Column
from ambry.transform import BasicTransform, CensusTransform
self.bundle.schema.clean()
with self.bundle.session:
s = self.bundle.schema
t = s.add_table('table3')
s.add_column(t,
name='col1',
datatype=Column.DATATYPE_INTEGER,
default=-1,
illegal_value='999')
s.add_column(t, name='col2', datatype=Column.DATATYPE_TEXT)
s.add_column(t, name='col3', datatype=Column.DATATYPE_REAL)
c1 = t.column('col1')
self.assertEquals(1, BasicTransform(c1)({'col1': ' 1 '}))
with self.assertRaises(ValueError):
print "PROCESSOR '{}'".format(
CensusTransform(c1)({
'col1': ' B '
}))
self.assertEquals(1, CensusTransform(c1)({'col1': ' 1 '}))
self.assertEquals(-1, CensusTransform(c1)({'col1': ' 999 '}))
self.assertEquals(-3, CensusTransform(c1)({'col1': ' # '}))
self.assertEquals(-2, CensusTransform(c1)({'col1': ' ! '}))
def test_validator(self):
#
# Validators
#
tests = [
('tone', True, (None, 'VALUE', 0, 0)),
('tone', True, (None, 'VALUE', -1, 0)),
('tone', False, (None, 'DEFAULT', 0, 0)),
('tone', False, (None, 'DEFAULT', -1, 0)),
('ttwo', True, (None, 'DEFAULT', 0, 0)),
('ttwo', True, (None, 'DEFAULT', 0, 3.14)),
('ttwo', False, (None, 'DEFAULT', -1, 0)),
('tthree', True, (None, 'DEFAULT', 0, 0)),
('tthree', True, (None, 'DEFAULT', 0, 3.14)),
('all', True, (None, 'text1', 'text2', 1, 2, 3, 3.14)),
('all', False, (None, 'text1', 'text2', -1, -1, 3, 3.14)),
('all', False, (None, 'text1', 'text2', -1, 2, 3, 3.14)),
('all', False, (None, 'text1', 'text2', 1, -1, 3, 3.14)),
]
for i, test in enumerate(tests):
table_name, truth, row = test
table = self.bundle.schema.table(table_name)
vd = table._get_validator()
if truth:
self.assertTrue(
vd(row), "Test {} not 'true' for table '{}': {}".format(
i + 1, table_name, row))
else:
self.assertFalse(
vd(row), "Test {} not 'false' for table '{}': {}".format(
i + 1, table_name, row))
# Testing the "OR" join of multiple columns.
tests = [
('tone', True, (None, 'VALUE', 0, 0)), #1
('tone', True, (None, 'VALUE', -1, 0)),
('tone', False, (None, 'DEFAULT', 0, 0)),
('tone', False, (None, 'DEFAULT', -1, 0)),
('ttwo', True, (None, 'DEFAULT', 0, 0)), #5
('ttwo', True, (None, 'DEFAULT', 0, 3.14)),
('ttwo', False, (None, 'DEFAULT', -1, 0)),
('tthree', True, (None, 'DEFAULT', 0, 0)), #8
('tthree', True, (None, 'DEFAULT', 0, 3.14)),
('all', True, (None, 'text1', 'text2', 1, 2, 3, 3.14)), #10
('all', False, (None, 'text1', 'text2', -1, -1, 3, 3.14)), #11
('all', True, (None, 'text1', 'text2', -1, 2, 3, 3.14)), #12
('all', True, (None, 'text1', 'text2', 1, -1, 3, 3.14)), #13
]
for i, test in enumerate(tests):
table_name, truth, row = test
table = self.bundle.schema.table(table_name)
vd = table._get_validator(and_join=False)
if truth:
self.assertTrue(
vd(row), "Test {} not 'true' for table '{}': {}".format(
i + 1, table_name, row))
else:
self.assertFalse(
vd(row), "Test {} not 'false' for table '{}': {}".format(
i + 1, table_name, row))
# Test the hash functions. This test depends on the d_test values in geoschema.csv
tests = [('tone', 'A|1|', (None, 'A', 1, 2)),
('ttwo', '1|2|', (None, 'B', 1, 2)),
('tthree', 'C|2|', (None, 'C', 1, 2))]
import hashlib
for i, test in enumerate(tests):
table_name, hashed_str, row = test
table = self.bundle.schema.table(table_name)
m = hashlib.md5()
m.update(hashed_str)
self.assertEquals(int(m.hexdigest()[:14], 16), table.row_hash(row))
def test_partition(self):
from ambry.dbexceptions import ConflictError
from ambry.identity import PartitionNameQuery
from ambry.partition.csv import CsvPartition
self.bundle.clean()
self.bundle.prepare()
p = self.bundle.partitions.new_db_partition(time=10,
space=10,
data={'pid': 'pid1'})
with self.assertRaises(ConflictError):
self.bundle.partitions.new_db_partition(time=10,
space=10,
data={'pid': 'pid1'})
self.assertEqual(1, len(self.bundle.partitions.all))
p = self.bundle.partitions.find_or_new(time=10, space=10)
p.database.create(
) # Find will go to the library if the database doesn't exist.
self.assertEqual(1, len(self.bundle.partitions.all))
self.assertEquals('pid1', p.data['pid'])
p = self.bundle.partitions.find(PartitionNameQuery(time=10, space=10))
self.assertEquals('pid1', p.data['pid'])
p = self.bundle.partitions.find(time=10, space=10)
self.assertEquals('pid1', p.data['pid'])
pnq3 = PartitionNameQuery(space=10)
with self.bundle.session as s:
p = self.bundle.partitions._find_orm(pnq3).first()
p.data['foo'] = 'bar'
s.add(p)
bundle = Bundle()
p = bundle.partitions.find(pnq3)
self.assertEquals('bar', p.data['foo'])
#p = self.bundle.partitions.find(PartitionNameQuery(name='source-dataset-subset-variation-30-hdf'))
#self.assertTrue(p is not None)
#self.assertEquals('source-dataset-subset-variation-30-hdf', p.identity.sname)
#
# Create all possible combinations of partition names
#
table = self.bundle.schema.tables[0]
p = (('time', 'time2'), ('space', 'space3'), ('table', table.name),
('grain', 'grain4'))
p += p
pids = {}
for i in range(4):
for j in range(4):
pid = self.bundle.identity.as_partition(**dict(p[i:i + j + 1]))
pids[pid.fqname] = pid
with self.bundle.session as s:
s.commit()
# These two deletey bits clear out all of the old
# partitions, to avoid a conflict with the next section. We also have
# to delete the files, since create() adds a partition record to the database,
# and if one already exists, it will throw an Integrity Error.
for p in self.bundle.partitions:
if os.path.exists(p.database.path):
os.remove(p.database.path)
for p in self.bundle.dataset.partitions:
# Using SQL instead of s.delete() because we want to avoid the cascade to stored_partitions, since
# that table doesn't exist in the bundle, only in the library
s.execute("DELETE FROM partitions WHERE p_vid = :vid",
{'vid': p.vid})
#s.delete(p)
def test_partition_2(self):
bundle = Bundle()
bundle.clean()
bundle.pre_prepare()
bundle.prepare()
bundle.post_prepare()
table = self.bundle.schema.tables[0]
p = (('time', 'time2'), ('space', 'space3'), ('table', table.name),
('grain', 'grain4'))
p += p
pids = {}
for i in range(4):
for j in range(4):
pid = self.bundle.identity.as_partition(**dict(p[i:i + j + 1]))
pids[pid.fqname] = pid
for pid in pids.values():
part = bundle.partitions.new_db_partition(**pid.dict)
part.create()
parts = bundle.partitions._find_orm(
PartitionNameQuery(vid=pid.vid)).all()
self.assertIn(pid.sname, [p.name for p in parts])
def test_runconfig(self):
"""Check the the RunConfig expands the library configuration"""
from ambry.run import get_runconfig, RunConfig
rc = get_runconfig((os.path.join(self.bundle_dir,
'test-run-config.yaml'),
RunConfig.USER_CONFIG, RunConfig.USER_ACCOUNTS))
l = rc.library('library1')
self.assertEquals('database1', l['database']['_name'])
self.assertEquals('filesystem1', l['filesystem']['_name'])
self.assertEquals('filesystem2', l['filesystem']['upstream']['_name'])
self.assertEquals('filesystem3',
l['filesystem']['upstream']['upstream']['_name'])
self.assertEquals(
'devtest.sandiegodata.org',
l['filesystem']['upstream']['upstream']['account']['_name'])
def test_build_bundle(self):
import shutil
bundle = Bundle()
shutil.copyfile(bundle.filesystem.path('meta', 'schema-edit-me.csv'),
bundle.filesystem.path('meta', 'schema.csv'))
#try:
bundle.database.enable_delete = True
bundle.clean()
bundle = Bundle()
bundle.exit_on_fatal = False
bundle.pre_prepare()
bundle.prepare()
bundle.post_prepare()
bundle.pre_build()
bundle.build_db_inserter_codes()
bundle.post_build()
bundle.close()
# The second run will use the changes to the schema made in the
# first run, due to the types errors in the 'coding' table.
bundle.clean()
bundle = Bundle()
bundle.exit_on_fatal = False
bundle.pre_prepare()
bundle.prepare()
bundle.post_prepare()
bundle.pre_build()
bundle.build_db_inserter_codes()
bundle.post_build()
bundle.close()
try:
pass
finally:
# Need to clean up to ensure that we're back to a good state.
# This runs the normal build, which will be used by the other
# tests.
shutil.copyfile(
bundle.filesystem.path('meta', 'schema-edit-me.csv'),
bundle.filesystem.path('meta', 'schema.csv'))
bundle.clean()
bundle = Bundle()
bundle.exit_on_fatal = False
bundle.pre_prepare()
bundle.prepare()
bundle.post_prepare()
bundle.pre_build()
bundle.build()
bundle.post_build()
def test_simple_build(self):
import shutil
bundle = Bundle()
shutil.copyfile(bundle.filesystem.path('meta', 'schema-edit-me.csv'),
bundle.filesystem.path('meta', 'schema.csv'))
bundle.clean()
bundle = Bundle()
bundle.exit_on_fatal = False
bundle.pre_prepare()
bundle.prepare()
bundle.post_prepare()
bundle.pre_build()
bundle.build()
bundle.post_build()
def test_config_update(self):
bundle = Bundle()
bundle.update_configuration()
def test_session(self):
import uuid
b = self.bundle
uv = str(uuid.uuid4())
with b.session as s1:
with b.session as s2:
b.set_value('test', 'uuid', uv)
b.close()
self.assertEqual(uv, b.get_value('test', 'uuid').value)
uv2 = str(uuid.uuid4())
self.assertNotEqual(uv, uv2)
with b.session as s1:
with b.session as s2:
b.set_value('test', 'uuid', uv2)
self.assertEqual(uv2, b.get_value('test', 'uuid').value)
b.set_value('test', 'uuid', uv2)
def setUp(self):
self.copy_or_build_bundle()
self.bundle = Bundle()
self.bundle_dir = self.bundle.bundle_dir
class Test(TestBase):
def setUp(self):
import bundles.testbundle.bundle
from ambry.run import RunConfig
import manifests, configs
self.bundle_dir = os.path.dirname(bundles.testbundle.bundle.__file__)
self.config_dir = os.path.dirname(configs.__file__)
self.rc = get_runconfig(
(os.path.join(self.config_dir, 'test.yaml'),
os.path.join(self.bundle_dir,
'bundle.yaml'), RunConfig.USER_ACCOUNTS))
self.copy_or_build_bundle()
self.bundle = Bundle()
#print "Deleting: {}".format(self.rc.group('filesystem').root)
#ambry.util.rm_rf(self.rc.group('filesystem').root)
self.m = os.path.join(os.path.dirname(manifests.__file__),
'test.ambry')
with open(self.m) as f:
self.m_contents = f.read()
def tearDown(self):
pass
def resolver(self, name):
if name == self.bundle.identity.name or name == self.bundle.identity.vname:
return self.bundle
else:
return False
def get_library(self, name='default'):
"""Clear out the database before the test run"""
from ambry.library import new_library
config = self.rc.library(name)
l = new_library(config, reset=True)
return l
def get_warehouse(self, l, name, delete=True):
from ambry.util import get_logger
from ambry.warehouse import new_warehouse
w = new_warehouse(self.rc.warehouse(name), l)
w.logger = get_logger('unit_test')
lr = self.bundle.init_log_rate(10000)
w.logger = TestLogger(lr)
if delete:
w.database.enable_delete = True
w.database.delete()
w.create()
return w
def get_fs_cache(self, name):
from ckcache.filesystem import FsCache
import shutil
#cache_dir = os.path.join(temp_file_name(), 'warehouse-test', name)
cache_dir = os.path.join('/tmp/ambry/test-warehouse', 'warehouse-test',
name)
if os.path.exists(cache_dir):
shutil.rmtree(cache_dir)
return FsCache(cache_dir)
def _test_local_install(self, name):
l = self.get_library()
l.put_bundle(self.bundle)
w = self.get_warehouse(l, name)
print "Warehouse: ", w.database.dsn
print "Library: ", l.database.dsn
w.install("source-dataset-subset-variation-tone-0.0.1")
w.install("source-dataset-subset-variation-tthree-0.0.1")
w.install("source-dataset-subset-variation-geot1-geo-0.0.1")
w = self.get_warehouse(l, 'spatialite')
print "WAREHOUSE: ", w.database.dsn
w.install("source-dataset-subset-variation-tone-0.0.1")
w.install("source-dataset-subset-variation-tthree-0.0.1")
w.install("source-dataset-subset-variation-geot1-geo-0.0.1")
def test_local_sqlite_install(self):
self._test_local_install('sqlite')
def test_local_postgres_install(self):
self._test_local_install('postgres1')
def _test_remote_install(self, name):
self.start_server(self.rc.library('server'))
l = self.get_library('client')
l.put_bundle(self.bundle)
w = self.get_warehouse(l, name)
print "WAREHOUSE: ", w.database.dsn
w.install("source-dataset-subset-variation-tone-0.0.1")
w.install("source-dataset-subset-variation-tthree-0.0.1")
w.install("source-dataset-subset-variation-geot1-geo-0.0.1")
w = self.get_warehouse(l, 'spatialite')
print "WAREHOUSE: ", w.database.dsn
w.install("source-dataset-subset-variation-tone-0.0.1")
w.install("source-dataset-subset-variation-tthree-0.0.1")
w.install("source-dataset-subset-variation-geot1-geo-0.0.1")
def test_manifest(self):
"""Load the manifest and convert it to a string to check the round-trip"""
from ambry.warehouse.manifest import Manifest
from ambry.util import get_logger
from ambry.util import print_yaml
m = Manifest(self.m, get_logger('TL'))
self.assertEqual(self.m_contents.strip(), str(m).strip())
l = self.get_library()
l.put_bundle(self.bundle)
for k, ident in l.list().items():
print ident
w = self.get_warehouse(l, 'sqlite')
print 'Installing to ', w.database.path
w.title = "This is the Warehouse!"
w.about = "A Warehouse full of wonder"
w.install_manifest(m)
extracts = w.extract(force=True)
print print_yaml(extracts)
def test_extract(self):
l = self.get_library()
l.put_bundle(self.bundle)
w = self.get_warehouse(l, 'sqlite', delete=False)
print 'WAREHOUSE: ', w.database.dsn
#cache = new_cache('s3://warehouse.sandiegodata.org/test', run_config = get_runconfig())
extracts = w.extract(force=True)
from ambry.util import print_yaml
print_yaml(extracts)
def test_manifest_parser(self):
import pprint
lines = [
"sangis.org-business-sites-orig-businesses-geo-0.1.1",
"table from sangis.org-business-sites-orig-businesses-geo-0.1.1",
"table FROM sangis.org-business-sites-orig-businesses-geo-0.1.1",
"table1, table2 FROM sangis.org-business-sites-orig-businesses-geo-0.1.1",
"table1, table2 FROM sangis.org-business-sites-orig-businesses-geo-0.1.1 WHERE foo and bar and bas",
"table1, table2 , table3,table4 FROM sangis.org-business-sites-orig-businesses-geo-0.1.1 # Wot you got?",
]
for line in lines:
print '----', line
pprint.pprint(Manifest.parse_partition_line(line))
def test_manifest_parts(self):
from ambry.warehouse.manifest import Manifest
from ambry.util import get_logger
from old.ipython.manifest import ManifestMagicsImpl
m = Manifest('', get_logger('TL'))
mmi = ManifestMagicsImpl(m)
m_head = """
TITLE: A Test Manifest, For Testing
UID: b4303f85-7d07-471d-9bcb-6980ea1bbf18
DATABASE: spatialite:///tmp/census-race-ethnicity.db
DIR: /tmp/warehouse
"""
mmi.manifest('', m_head)
mmi.extract('foobar AS csv TO /bin/bar/bingo')
mmi.extract('foobar AS csv TO /bin/bar/bingo')
mmi.extract('foobar AS csv TO /bin/bar/bingo2')
mmi.extract('foobar AS csv TO /bin/bar/bingo')
mmi.partitions('', 'one\ntwo\nthree\nfour')
mmi.view('foo_view_1', '1234\n5678\n')
mmi.view('foo_view_2', '1234\n5678\n')
mmi.mview('foo_mview_1', '1234\n5678\n')
mmi.mview('foo_mview_2', '1234\n5678\n')
mmi.view('foo_view_1', '1234\n5678\n')
mmi.view('foo_view_2', '1234\n5678\n')
mmi.mview('foo_mview_1', '1234\n5678\n')
mmi.mview('foo_mview_2', '1234\n5678\n')
#print yaml.dump(m.sections, default_flow_style=False)
print str(m)
def test_sql_parser(self):
sql = """
SELECT
geo.state, -- comment 1
geo.county, -- comment 2
geo.tract,
geo.blkgrp,
bb.geometry,
CAST(Area(Transform(geometry,26946)) AS REAL) AS area,
CAST(b02001001 AS INTEGER) AS total_pop,
FROM d02G003_geofile AS geo
JOIN d024004_b02001_estimates AS b02001e ON geo.stusab = b02001e.stusab AND geo.logrecno = b02001e.logrecno
JOIN blockgroup_boundaries AS bb ON geo.state = bb.state AND geo.county = bb.county AND bb.tract = geo.tract AND bb.blkgrp = geo.blkgrp
WHERE geo.sumlevel = 150 AND geo.state = 6 and geo.county = 73
"""
import sqlparse
import sqlparse.sql
r = sqlparse.parse(sql)
for t in r[0].tokens:
if isinstance(t, sqlparse.sql.IdentifierList):
for i in t.get_identifiers():
print i, type(i)
#print sqlparse.format(sql, strip_comments = True, reindent = True)
def x_test_install(self):
def resolver(name):
if name == self.bundle.identity.name or name == self.bundle.identity.vname:
return self.bundle
else:
return False
def progress_cb(lr, type, name, n):
if n:
lr("{} {}: {}".format(type, name, n))
else:
self.bundle.log("{} {}".format(type, name))
from ambry.warehouse import new_warehouse
from functools import partial
print "Getting warehouse"
w = new_warehouse(self.rc.warehouse('postgres'))
print "Re-create database"
w.database.enable_delete = True
w.resolver = resolver
w.progress_cb = progress_cb
try:
w.drop()
except:
pass
w.create()
ps = self.bundle.partitions.all
print "{} partitions".format(len(ps))
for p in self.bundle.partitions:
lr = self.bundle.init_log_rate(10000)
w.install(p, progress_cb=partial(progress_cb, lr))
self.assertTrue(w.has(self.bundle.identity.vname))
for p in self.bundle.partitions:
self.assertTrue(w.has(p.identity.vname))
for p in self.bundle.partitions:
w.remove(p.identity.vname)
print w.get(self.bundle.identity.name)
print w.get(self.bundle.identity.vname)
print w.get(self.bundle.identity.id_)
w.install(self.bundle)
print w.get(self.bundle.identity.name)
print w.get(self.bundle.identity.vname)
print w.get(self.bundle.identity.id_)
for p in self.bundle.partitions:
lr = self.bundle.init_log_rate(10000)
w.install(p, progress_cb=partial(progress_cb, lr))
def test_partition(self):
from ambry.dbexceptions import ConflictError
from ambry.identity import PartitionIdentity, PartitionNameQuery
from ambry.partition.csv import CsvPartition
from ambry.partition.hdf import HdfPartition
self.bundle.clean()
self.bundle.prepare()
p = self.bundle.partitions.new_db_partition(time=10, space=10, data={'pid':'pid1'})
p = self.bundle.partitions.new_csv_partition(time=20, space=20, data={'pid':'pid2'})
self.assertIsInstance(p, CsvPartition )
p = self.bundle.partitions.find_or_new_csv(time=20, space=20)
self.assertIsInstance(p, CsvPartition)
p = self.bundle.partitions.new_hdf_partition(space=30, data={'pid':'pid3'})
self.assertIsInstance(p, HdfPartition)
p = self.bundle.partitions.find_or_new_hdf(space=30)
self.assertIsInstance(p, HdfPartition)
with self.assertRaises(ConflictError):
self.bundle.partitions.new_db_partition(time=10, space=10, data={'pid':'pid1'})
with self.assertRaises(ConflictError):
self.bundle.partitions.new_csv_partition(time=20, space=20, data={'pid':'pid21'})
with self.assertRaises(ConflictError):
self.bundle.partitions.new_hdf_partition(space=30, data={'pid':'pid31'})
self.assertEqual(3, len(self.bundle.partitions.all))
p = self.bundle.partitions.find_or_new(time=10, space=10)
p.database.create() # Find will go to the library if the database doesn't exist.
self.assertEqual(3, len(self.bundle.partitions.all))
self.assertEquals('pid1',p.data['pid'] )
p = self.bundle.partitions.find_or_new_csv(time=20, space=20)
p.database.create()
self.assertEquals('pid2',p.data['pid'] )
p = self.bundle.partitions.find_or_new_hdf(space=30)
self.assertEquals('pid3',p.data['pid'] )
p = self.bundle.partitions.find(PartitionNameQuery(time=10, space=10))
self.assertEquals('pid1',p.data['pid'] )
p = self.bundle.partitions.find(time=10, space=10)
self.assertEquals('pid1', p.data['pid'])
p = self.bundle.partitions.find(PartitionNameQuery(time=20, space=20))
self.assertEquals('pid2',p.data['pid'] )
p = self.bundle.partitions.find(time=20, space=20)
self.assertEquals('pid2',p.data['pid'] )
pnq3 = PartitionNameQuery(space=30)
p = self.bundle.partitions.find(pnq3)
self.assertEquals('pid3',p.data['pid'] )
with self.bundle.session as s:
p = self.bundle.partitions._find_orm(pnq3).first()
p.data['foo'] = 'bar'
s.add(p)
bundle = Bundle()
p = bundle.partitions.find(pnq3)
print p.data
self.assertEquals('bar',p.data['foo'] )
p = self.bundle.partitions.find(PartitionNameQuery(name='source-dataset-subset-variation-30-hdf'))
self.assertTrue(p is not None)
self.assertEquals('source-dataset-subset-variation-30-hdf', p.identity.sname)
#
# Create all possible combinations of partition names
#
table = self.bundle.schema.tables[0]
p = (('time','time2'),('space','space3'),('table',table.name),('grain','grain4'))
p += p
pids = {}
for i in range(4):
for j in range(4):
pid = self.bundle.identity.as_partition(**dict(p[i:i+j+1]))
pids[pid.fqname] = pid
with self.bundle.session as s:
# These two deletely bits clear out all of the old
# partitions, to avoid a conflict with the next section. We also have
# to delete the files, since create() adds a partition record to the database,
# and if one already exists, it will throw an Integrity Error.
for p in self.bundle.partitions:
if os.path.exists(p.database.path):
os.remove(p.database.path)
for p in self.bundle.dataset.partitions:
s.delete(p)
import pprint
pprint.pprint(sorted([ pid.fqname for pid in pids.values()]))
bundle = Bundle()
bundle.clean()
bundle.prepare()
for pid in pids.values():
part = bundle.partitions.new_db_partition(**pid.dict)
part.create()
parts = bundle.partitions._find_orm(PartitionNameQuery(vid=pid.vid)).all()
self.assertIn(pid.sname, [p.name for p in parts])
class Test(TestBase):
def setUp(self):
import bundles.testbundle.bundle
from ambry.run import RunConfig
import manifests, configs
self.bundle_dir = os.path.dirname( bundles.testbundle.bundle.__file__)
self.config_dir = os.path.dirname(configs.__file__)
self.rc = get_runconfig((os.path.join(self.config_dir, 'test.yaml'),
os.path.join(self.bundle_dir,'bundle.yaml'),
RunConfig.USER_ACCOUNTS))
self.copy_or_build_bundle()
self.bundle = Bundle()
#print "Deleting: {}".format(self.rc.group('filesystem').root)
#ambry.util.rm_rf(self.rc.group('filesystem').root)
self.m = os.path.join(os.path.dirname(manifests.__file__), 'test.ambry')
with open(self.m) as f:
self.m_contents = f.read()
def tearDown(self):
pass
def resolver(self,name):
if name == self.bundle.identity.name or name == self.bundle.identity.vname:
return self.bundle
else:
return False
def get_library(self, name='default'):
"""Clear out the database before the test run"""
from ambry.library import new_library
config = self.rc.library(name)
l = new_library(config, reset=True)
return l
def get_warehouse(self, l, name, delete = True):
from ambry.util import get_logger
from ambry.warehouse import new_warehouse
w = new_warehouse(self.rc.warehouse(name), l)
w.logger = get_logger('unit_test')
lr = self.bundle.init_log_rate(10000)
w.logger = TestLogger(lr)
if delete:
w.database.enable_delete = True
w.database.delete()
w.create()
return w
def get_fs_cache(self,name):
from ckcache.filesystem import FsCache
import shutil
#cache_dir = os.path.join(temp_file_name(), 'warehouse-test', name)
cache_dir = os.path.join('/tmp/ambry/test-warehouse', 'warehouse-test', name)
if os.path.exists(cache_dir):
shutil.rmtree(cache_dir)
return FsCache(cache_dir)
def _test_local_install(self, name):
l = self.get_library()
l.put_bundle(self.bundle)
w = self.get_warehouse(l, name)
print "Warehouse: ", w.database.dsn
print "Library: ", l.database.dsn
w.install("source-dataset-subset-variation-tone-0.0.1")
w.install("source-dataset-subset-variation-tthree-0.0.1")
w.install("source-dataset-subset-variation-geot1-geo-0.0.1")
w = self.get_warehouse(l, 'spatialite')
print "WAREHOUSE: ", w.database.dsn
w.install("source-dataset-subset-variation-tone-0.0.1")
w.install("source-dataset-subset-variation-tthree-0.0.1")
w.install("source-dataset-subset-variation-geot1-geo-0.0.1")
def test_local_sqlite_install(self):
self._test_local_install('sqlite')
def test_local_postgres_install(self):
self._test_local_install('postgres1')
def _test_remote_install(self, name):
self.start_server(self.rc.library('server'))
l = self.get_library('client')
l.put_bundle(self.bundle)
w = self.get_warehouse(l, name)
print "WAREHOUSE: ", w.database.dsn
w.install("source-dataset-subset-variation-tone-0.0.1")
w.install("source-dataset-subset-variation-tthree-0.0.1")
w.install("source-dataset-subset-variation-geot1-geo-0.0.1")
w = self.get_warehouse(l, 'spatialite')
print "WAREHOUSE: ", w.database.dsn
w.install("source-dataset-subset-variation-tone-0.0.1")
w.install("source-dataset-subset-variation-tthree-0.0.1")
w.install("source-dataset-subset-variation-geot1-geo-0.0.1")
def test_manifest(self):
"""Load the manifest and convert it to a string to check the round-trip"""
from ambry.warehouse.manifest import Manifest
from ambry.util import get_logger
from ambry.util import print_yaml
m = Manifest(self.m,get_logger('TL') )
self.assertEqual(self.m_contents.strip(), str(m).strip())
l = self.get_library()
l.put_bundle(self.bundle)
for k, ident in l.list().items():
print ident
w = self.get_warehouse(l, 'sqlite')
print 'Installing to ', w.database.path
w.title = "This is the Warehouse!"
w.about = "A Warehouse full of wonder"
w.install_manifest(m)
extracts = w.extract(force=True)
print print_yaml(extracts)
def test_extract(self):
l = self.get_library()
l.put_bundle(self.bundle)
w = self.get_warehouse(l, 'sqlite', delete=False)
print 'WAREHOUSE: ', w.database.dsn
#cache = new_cache('s3://warehouse.sandiegodata.org/test', run_config = get_runconfig())
extracts = w.extract(force = True)
from ambry.util import print_yaml
print_yaml(extracts)
def test_manifest_parser(self):
import pprint
lines = [
"sangis.org-business-sites-orig-businesses-geo-0.1.1",
"table from sangis.org-business-sites-orig-businesses-geo-0.1.1",
"table FROM sangis.org-business-sites-orig-businesses-geo-0.1.1",
"table1, table2 FROM sangis.org-business-sites-orig-businesses-geo-0.1.1",
"table1, table2 FROM sangis.org-business-sites-orig-businesses-geo-0.1.1 WHERE foo and bar and bas",
"table1, table2 , table3,table4 FROM sangis.org-business-sites-orig-businesses-geo-0.1.1 # Wot you got?",
]
for line in lines:
print '----', line
pprint.pprint( Manifest.parse_partition_line(line))
def test_manifest_parts(self):
from ambry.warehouse.manifest import Manifest
from ambry.util import get_logger
from old.ipython.manifest import ManifestMagicsImpl
m = Manifest('', get_logger('TL'))
mmi = ManifestMagicsImpl(m)
m_head = """
TITLE: A Test Manifest, For Testing
UID: b4303f85-7d07-471d-9bcb-6980ea1bbf18
DATABASE: spatialite:///tmp/census-race-ethnicity.db
DIR: /tmp/warehouse
"""
mmi.manifest('',m_head)
mmi.extract('foobar AS csv TO /bin/bar/bingo')
mmi.extract('foobar AS csv TO /bin/bar/bingo')
mmi.extract('foobar AS csv TO /bin/bar/bingo2')
mmi.extract('foobar AS csv TO /bin/bar/bingo')
mmi.partitions('','one\ntwo\nthree\nfour')
mmi.view('foo_view_1','1234\n5678\n')
mmi.view('foo_view_2', '1234\n5678\n')
mmi.mview('foo_mview_1', '1234\n5678\n')
mmi.mview('foo_mview_2', '1234\n5678\n')
mmi.view('foo_view_1', '1234\n5678\n')
mmi.view('foo_view_2', '1234\n5678\n')
mmi.mview('foo_mview_1', '1234\n5678\n')
mmi.mview('foo_mview_2', '1234\n5678\n')
#print yaml.dump(m.sections, default_flow_style=False)
print str(m)
def test_sql_parser(self):
sql = """
SELECT
geo.state, -- comment 1
geo.county, -- comment 2
geo.tract,
geo.blkgrp,
bb.geometry,
CAST(Area(Transform(geometry,26946)) AS REAL) AS area,
CAST(b02001001 AS INTEGER) AS total_pop,
FROM d02G003_geofile AS geo
JOIN d024004_b02001_estimates AS b02001e ON geo.stusab = b02001e.stusab AND geo.logrecno = b02001e.logrecno
JOIN blockgroup_boundaries AS bb ON geo.state = bb.state AND geo.county = bb.county AND bb.tract = geo.tract AND bb.blkgrp = geo.blkgrp
WHERE geo.sumlevel = 150 AND geo.state = 6 and geo.county = 73
"""
import sqlparse
import sqlparse.sql
r = sqlparse.parse(sql)
for t in r[0].tokens:
if isinstance(t, sqlparse.sql.IdentifierList):
for i in t.get_identifiers():
print i, type(i)
#print sqlparse.format(sql, strip_comments = True, reindent = True)
def x_test_install(self):
def resolver(name):
if name == self.bundle.identity.name or name == self.bundle.identity.vname:
return self.bundle
else:
return False
def progress_cb(lr, type,name,n):
if n:
lr("{} {}: {}".format(type, name, n))
else:
self.bundle.log("{} {}".format(type, name))
from ambry.warehouse import new_warehouse
from functools import partial
print "Getting warehouse"
w = new_warehouse(self.rc.warehouse('postgres'))
print "Re-create database"
w.database.enable_delete = True
w.resolver = resolver
w.progress_cb = progress_cb
try: w.drop()
except: pass
w.create()
ps = self.bundle.partitions.all
print "{} partitions".format(len(ps))
for p in self.bundle.partitions:
lr = self.bundle.init_log_rate(10000)
w.install(p, progress_cb = partial(progress_cb, lr) )
self.assertTrue(w.has(self.bundle.identity.vname))
for p in self.bundle.partitions:
self.assertTrue(w.has(p.identity.vname))
for p in self.bundle.partitions:
w.remove(p.identity.vname)
print w.get(self.bundle.identity.name)
print w.get(self.bundle.identity.vname)
print w.get(self.bundle.identity.id_)
w.install(self.bundle)
print w.get(self.bundle.identity.name)
print w.get(self.bundle.identity.vname)
print w.get(self.bundle.identity.id_)
for p in self.bundle.partitions:
lr = self.bundle.init_log_rate(10000)
w.install(p, progress_cb = partial(progress_cb, lr))
def test_build_bundle(self):
import shutil
bundle = Bundle()
shutil.copyfile(bundle.filesystem.path('meta', 'schema-edit-me.csv'),
bundle.filesystem.path('meta', 'schema.csv'))
#try:
bundle.database.enable_delete = True
bundle.clean()
bundle = Bundle()
bundle.exit_on_fatal = False
bundle.pre_prepare()
bundle.prepare()
bundle.post_prepare()
bundle.pre_build()
bundle.build_db_inserter_codes()
bundle.post_build()
bundle.close()
# The second run will use the changes to the schema made in the
# first run, due to the types errors in the 'coding' table.
bundle.clean()
bundle = Bundle()
bundle.exit_on_fatal = False
bundle.pre_prepare()
bundle.prepare()
bundle.post_prepare()
bundle.pre_build()
bundle.build_db_inserter_codes()
bundle.post_build()
bundle.close()
try:
pass
finally:
# Need to clean up to ensure that we're back to a good state.
# This runs the normal build, which will be used by the other
# tests.
shutil.copyfile(
bundle.filesystem.path('meta', 'schema-edit-me.csv'),
bundle.filesystem.path('meta', 'schema.csv'))
bundle.clean()
bundle = Bundle()
bundle.exit_on_fatal = False
bundle.pre_prepare()
bundle.prepare()
bundle.post_prepare()
bundle.pre_build()
bundle.build()
bundle.post_build()
def test_simple_build(self):
import shutil
bundle = Bundle()
shutil.copyfile(
bundle.filesystem.path('meta', 'schema-edit-me.csv'),
bundle.filesystem.path('meta', 'schema.csv'))
bundle.clean()
bundle = Bundle()
bundle.exit_on_fatal = False
bundle.pre_prepare()
bundle.prepare()
bundle.post_prepare()
bundle.pre_build()
bundle.build()
bundle.post_build()
def test_build_bundle(self):
import shutil
bundle = Bundle()
shutil.copyfile(
bundle.filesystem.path('meta', 'schema-edit-me.csv'),
bundle.filesystem.path('meta', 'schema.csv'))
# try:
bundle.database.enable_delete = True
bundle.clean()
bundle = Bundle()
bundle.exit_on_fatal = False
bundle.pre_prepare()
bundle.prepare()
bundle.post_prepare()
bundle.pre_build()
bundle.build_db_inserter_codes()
bundle.post_build()
bundle.close()
# The second run will use the changes to the schema made in the
# first run, due to the types errors in the 'coding' table.
bundle.clean()
bundle = Bundle()
bundle.exit_on_fatal = False
bundle.pre_prepare()
bundle.prepare()
bundle.post_prepare()
bundle.pre_build()
bundle.build_db_inserter_codes()
bundle.post_build()
bundle.close()
try:
pass
finally:
# Need to clean up to ensure that we're back to a good state.
# This runs the normal build, which will be used by the other
# tests.
shutil.copyfile(
bundle.filesystem.path('meta', 'schema-edit-me.csv'),
bundle.filesystem.path('meta', 'schema.csv'))
bundle.clean()
bundle = Bundle()
bundle.exit_on_fatal = False
bundle.pre_prepare()
bundle.prepare()
bundle.post_prepare()
bundle.pre_build()
bundle.build()
bundle.post_build()
def test_build_bundle_hdf(self):
bundle = Bundle()
bundle.clean()
bundle = Bundle()
bundle.exit_on_fatal = False
bundle.pre_prepare()
bundle.prepare()
bundle.post_prepare()
bundle.pre_build()
bundle.build_hdf()
bundle.post_build()
def setUp(self):
super(Test, self).setUp()
self.bundle = Bundle()
self.bundle_dir = self.bundle.bundle_dir
def test_config_update(self):
bundle = Bundle()
bundle.update_configuration()
def setUp(self):
self.copy_or_build_bundle()
self.bundle = Bundle()
self.bundle_dir = self.bundle.bundle_dir
def test_versions(self):
from ambry.run import get_runconfig
from ambry.library.query import Resolver
import shutil
idnt = self.bundle.identity
l = self.get_library()
l.purge()
orig = os.path.join(self.bundle.bundle_dir, 'bundle.yaml')
save = os.path.join(self.bundle.bundle_dir, 'bundle.yaml.save')
shutil.copyfile(orig, save)
datasets = {}
try:
for i in [1, 2, 3]:
idnt._on.revision = i
idnt.name.version_major = i
idnt.name.version_minor = i * 10
bundle = Bundle()
get_runconfig.clear() #clear runconfig cache
bundle.metadata.load_all()
bundle.metadata.identity = idnt.ident_dict
bundle.metadata.names = idnt.names_dict
bundle.metadata.write_to_dir(write_all=True)
bundle = Bundle()
bundle.clean()
bundle.pre_prepare()
bundle.prepare()
bundle.post_prepare()
bundle.pre_build()
bundle.build_small()
#bundle.build()
bundle.post_build()
bundle = Bundle()
l.put_bundle(bundle)
finally:
pass
os.rename(save, orig)
#
# Save the list of datasets for version analysis in other
# tests
#
db = l.database
for d in db.list(with_partitions=True).values():
datasets[d.vid] = d.dict
datasets[d.vid]['partitions'] = {}
for p_vid, p in d.partitions.items():
datasets[d.vid]['partitions'][p_vid] = p.dict
with open(self.bundle.filesystem.path('meta', 'version_datasets.json'),
'w') as f:
import json
f.write(json.dumps(datasets))
r = Resolver(db.session)
ref = idnt.id_
ref = "source-dataset-subset-variation-=2.20"
ip, results = r.resolve_ref_all(ref)
for row in results:
print row
def test_partition(self):
from ambry.dbexceptions import ConflictError
from ambry.identity import PartitionNameQuery
from ambry.partition.csv import CsvPartition
self.bundle.clean()
self.bundle.prepare()
p = self.bundle.partitions.new_db_partition(time=10,
space=10,
data={'pid': 'pid1'})
with self.assertRaises(ConflictError):
self.bundle.partitions.new_db_partition(time=10,
space=10,
data={'pid': 'pid1'})
self.assertEqual(1, len(self.bundle.partitions.all))
p = self.bundle.partitions.find_or_new(time=10, space=10)
p.database.create(
) # Find will go to the library if the database doesn't exist.
self.assertEqual(1, len(self.bundle.partitions.all))
self.assertEquals('pid1', p.data['pid'])
p = self.bundle.partitions.find(PartitionNameQuery(time=10, space=10))
self.assertEquals('pid1', p.data['pid'])
p = self.bundle.partitions.find(time=10, space=10)
self.assertEquals('pid1', p.data['pid'])
pnq3 = PartitionNameQuery(space=10)
with self.bundle.session as s:
p = self.bundle.partitions._find_orm(pnq3).first()
p.data['foo'] = 'bar'
s.add(p)
bundle = Bundle()
p = bundle.partitions.find(pnq3)
self.assertEquals('bar', p.data['foo'])
#p = self.bundle.partitions.find(PartitionNameQuery(name='source-dataset-subset-variation-30-hdf'))
#self.assertTrue(p is not None)
#self.assertEquals('source-dataset-subset-variation-30-hdf', p.identity.sname)
#
# Create all possible combinations of partition names
#
table = self.bundle.schema.tables[0]
p = (('time', 'time2'), ('space', 'space3'), ('table', table.name),
('grain', 'grain4'))
p += p
pids = {}
for i in range(4):
for j in range(4):
pid = self.bundle.identity.as_partition(**dict(p[i:i + j + 1]))
pids[pid.fqname] = pid
with self.bundle.session as s:
s.commit()
# These two deletey bits clear out all of the old
# partitions, to avoid a conflict with the next section. We also have
# to delete the files, since create() adds a partition record to the database,
# and if one already exists, it will throw an Integrity Error.
for p in self.bundle.partitions:
if os.path.exists(p.database.path):
os.remove(p.database.path)
for p in self.bundle.dataset.partitions:
# Using SQL instead of s.delete() because we want to avoid the cascade to stored_partitions, since
# that table doesn't exist in the bundle, only in the library
s.execute("DELETE FROM partitions WHERE p_vid = :vid",
{'vid': p.vid})
def x_test_rewrite(self):
from testbundle.bundle import Bundle
import json
from ambry.run import get_runconfig
# Prepare to rewrite the bundle.yaml file.
bundle = Bundle()
orig = os.path.join(bundle.bundle_dir,'bundle.yaml')
save = os.path.join(bundle.bundle_dir,'bundle.yaml.save')
try:
os.rename(orig,save)
print 'Write to ', orig
with open(orig,'w') as f:
f.write(json.dumps(
{
"identity":{
"dataset": "dataset1",
"id": "dfoo",
"revision": 100,
"source": "source1",
"subset": "subset1",
"variation": "variation1",
"version": "1.0.1",
"vid": "dfob001",
},
"about": {
"author": "*****@*****.**"
}
}
))
get_runconfig.clear() # clear config cache.
bundle = Bundle()
bundle.clean()
bundle.pre_prepare()
bundle.prepare()
bundle.post_prepare() # Does the rewrite, adding the 'names'
# Need to clear and reload one more time for the 'names' to appear
get_runconfig.clear() # clear config cache.
bundle = Bundle()
bundle.exit_on_fatal = False
self.assertEquals('dataset1', bundle.config.identity.dataset)
self.assertEquals('dfoo', bundle.config.identity.id)
self.assertEquals(100, bundle.config.identity.revision)
self.assertEquals("source1-dataset1-subset1-variation1-1.0.100~dfoo01C", bundle.config.names.fqname)
self.assertEquals("*****@*****.**", bundle.config.about.author)
finally:
os.rename(save, orig)
self.delete_bundle()
def test_config_update(self):
bundle = Bundle()
bundle.update_configuration()
def test_simple_build(self):
import shutil
bundle = Bundle()
shutil.copyfile(bundle.filesystem.path('meta', 'schema-edit-me.csv'),
bundle.filesystem.path('meta', 'schema.csv'))
bundle.clean()
bundle = Bundle()
bundle.exit_on_fatal = False
bundle.pre_prepare()
bundle.prepare()
bundle.post_prepare()
bundle.pre_build()
bundle.build()
bundle.post_build()
def test_bundle_build(self):
from ambry.dbexceptions import ConflictError
bundle = Bundle()
# Need to clear the library, or the Bundle's pre_prepare
# will cancel the build if this version is already installed
bundle.library.purge()
bundle.exit_on_fatal = False
bundle.clean()
bundle.database.create()
bp = bundle.partitions
with bundle.session:
bp._new_orm_partition(PartialPartitionName(time = 't1', space='s1'))
bp._new_orm_partition(PartialPartitionName(time = 't1', space='s2'))
bp._new_orm_partition(PartialPartitionName(time = 't1', space=None))
bp._new_orm_partition(PartialPartitionName(time = 't2', space='s1'))
bp._new_orm_partition(PartialPartitionName(time = 't2', space='s2'))
bp._new_orm_partition(PartialPartitionName(time = 't2', space=None))
with self.assertRaises(ConflictError):
with bundle.session:
bp._new_orm_partition(PartialPartitionName(time = 't1', space='s1'))
pnq = PartitionNameQuery(time=NameQuery.ANY, space='s1')
names = [p.vname
for p in bp._find_orm(pnq).all()]
self.assertEqual({u'source-dataset-subset-variation-t2-s1-0.0.1',
u'source-dataset-subset-variation-t1-s1-0.0.1'},
set(names))
names = [p.vname
for p in bp._find_orm(PartitionNameQuery(space=NameQuery.ANY)).all()]
self.assertEqual(6,len(names))
names = [p.vname
for p in bp._find_orm(PartitionNameQuery(time='t1',space=NameQuery.ANY)).all()]
self.assertEqual({'source-dataset-subset-variation-t1-s2-0.0.1',
'source-dataset-subset-variation-t1-0.0.1',
'source-dataset-subset-variation-t1-s1-0.0.1'},
set(names))
names = [p.vname
for p in bp._find_orm(PartitionNameQuery(time='t1',space=NameQuery.NONE)).all()]
self.assertEqual({'source-dataset-subset-variation-t1-0.0.1'},
set(names))
# Start over, use a higher level function to create the partitions
bundle.close() # Or you'll get an OperationalError
bundle = Bundle()
bundle.exit_on_fatal = False
bundle.clean()
bundle.database.create()
bp = bundle.partitions
bp._new_partition(PartialPartitionName(time = 't1', space='s1'))
self.assertEquals(1, len(bp.all))
bp._new_partition(PartialPartitionName(time = 't1', space='s2'))
self.assertEquals(2, len(bp.all))
bp._new_partition(PartialPartitionName(time = 't1', space=None))
bp._new_partition(PartialPartitionName(time = 't2', space='s1'))
bp._new_partition(PartialPartitionName(time = 't2', space='s2'))
bp._new_partition(PartialPartitionName(time = 't2', space=None))
self.assertEquals(6, len(bp.all))
names = [p.vname
for p in bp._find_orm(PartitionNameQuery(time='t1',space=NameQuery.ANY)).all()]
self.assertEqual({'source-dataset-subset-variation-t1-s2-0.0.1',
'source-dataset-subset-variation-t1-0.0.1',
'source-dataset-subset-variation-t1-s1-0.0.1'},
set(names))
# Start over, use a higher level function to create the partitions
bundle.close()
bundle = Bundle()
bundle.exit_on_fatal = False
bundle.clean()
bundle.database.create()
bp = bundle.partitions
p = bp.new_db_partition(time = 't1', space='s1')
self.assertEquals('source-dataset-subset-variation-t1-s1-0.0.1~piEGPXmDC8001001', p.identity.fqname)
p = bp.find_or_new(time = 't1', space='s2')
self.assertEquals('source-dataset-subset-variation-t1-s2-0.0.1~piEGPXmDC8002001', p.identity.fqname)
# Duplicate
p = bp.find_or_new(time = 't1', space='s2')
self.assertEquals('source-dataset-subset-variation-t1-s2-0.0.1~piEGPXmDC8002001', p.identity.fqname)
p = bp.find_or_new_geo(time = 't2', space='s1')
# Which it is depends on whether GDAL is installed.
self.assertIn(p.identity.fqname,[
'source-dataset-subset-variation-t2-s1-geo-0.0.1~piEGPXmDC8003001',
'source-dataset-subset-variation-t2-s1-0.0.1~piEGPXmDC8003001' ]
)
# Ok! Build!
bundle.close()
bundle = Bundle()
bundle.exit_on_fatal = False
bundle.clean()
bundle.pre_prepare()
bundle.prepare()
bundle.post_prepare()
bundle.pre_build()
bundle.build_db_inserter_codes()
bundle.post_build()
self.assertEquals('diEGPXmDC8001',bundle.identity.vid)
self.assertEquals('source-dataset-subset-variation',bundle.identity.sname)
self.assertEquals('source-dataset-subset-variation-0.0.1',bundle.identity.vname)
self.assertEquals('source-dataset-subset-variation-0.0.1~diEGPXmDC8001',bundle.identity.fqname)
def test_bundle_build(self):
from ambry.dbexceptions import ConflictError
bundle = Bundle()
# Need to clear the library, or the Bundle's pre_prepare
# will cancel the build if this version is already installed
bundle.library.purge()
bundle.exit_on_fatal = False
bundle.clean()
bundle.database.create()
bundle.prepare()
bp = bundle.partitions
with bundle.session:
bp._new_orm_partition(PartialPartitionName(table = 'tone', time = 't1', space='s1'))
bp._new_orm_partition(PartialPartitionName(table = 'tone', time = 't1', space='s2'))
bp._new_orm_partition(PartialPartitionName(table = 'tone', time = 't1', space=None))
bp._new_orm_partition(PartialPartitionName(table = 'tone', time = 't2', space='s1'))
bp._new_orm_partition(PartialPartitionName(table = 'tone', time = 't2', space='s2'))
bp._new_orm_partition(PartialPartitionName(table = 'tone', time = 't2', space=None))
with self.assertRaises(ConflictError):
with bundle.session:
bp._new_orm_partition(PartialPartitionName(table = 'tone',time = 't1', space='s1'))
pnq = PartitionNameQuery(table = 'tone', time=NameQuery.ANY, space='s1')
names = [p.vname
for p in bp._find_orm(pnq).all()]
self.assertEqual({u'source-dataset-subset-variation-tone-t1-s1-0.0.1',
u'source-dataset-subset-variation-tone-t2-s1-0.0.1'},
set(names))
names = [p.vname
for p in bp._find_orm(PartitionNameQuery(space=NameQuery.ANY)).all()]
self.assertEqual(6, len(names))
names = [p.vname
for p in bp._find_orm(PartitionNameQuery(table = 'tone',time='t1',space=NameQuery.ANY)).all()]
self.assertEqual({'source-dataset-subset-variation-tone-t1-s2-0.0.1',
'source-dataset-subset-variation-tone-t1-0.0.1',
'source-dataset-subset-variation-tone-t1-s1-0.0.1'},
set(names))
names = [p.vname
for p in bp._find_orm(PartitionNameQuery(table = 'tone',time='t1',space=NameQuery.NONE)).all()]
self.assertEqual({'source-dataset-subset-variation-tone-t1-0.0.1'},
set(names))
# Start over, use a higher level function to create the partitions
bundle.close() # Or you'll get an OperationalError
bundle = Bundle()
bundle.exit_on_fatal = False
bundle.clean()
bundle.database.create()
bundle.prepare()
bp = bundle.partitions
bp._new_partition(PartialPartitionName(table = 'tone',time = 't1', space='s1'))
self.assertEquals(1, len(bp.all))
bp._new_partition(PartialPartitionName(table = 'tone',time = 't1', space='s2'))
self.assertEquals(2, len(bp.all))
bp._new_partition(PartialPartitionName(table = 'tone',time = 't1', space=None))
bp._new_partition(PartialPartitionName(table = 'tone',time = 't2', space='s1'))
bp._new_partition(PartialPartitionName(table = 'tone',time = 't2', space='s2'))
bp._new_partition(PartialPartitionName(table = 'tone',time = 't2', space=None))
self.assertEquals(6, len(bp.all))
names = [p.vname
for p in bp._find_orm(PartitionNameQuery(time='t1', space=NameQuery.ANY)).all()]
self.assertEqual({'source-dataset-subset-variation-tone-t1-s2-0.0.1',
'source-dataset-subset-variation-tone-t1-0.0.1',
'source-dataset-subset-variation-tone-t1-s1-0.0.1'},
set(names))
# Start over, use a higher level function to create the partitions
bundle.close()
bundle = Bundle()
bundle.exit_on_fatal = False
bundle.clean()
bundle.database.create()
bundle.prepare()
bp = bundle.partitions
p = bp.new_db_partition(table = 'tone',time = 't1', space='s1')
self.assertEquals('source-dataset-subset-variation-tone-t1-s1-0.0.1~piEGPXmDC8001001', p.identity.fqname)
p = bp.find_or_new(table = 'tone',time = 't1', space='s2')
self.assertEquals('source-dataset-subset-variation-tone-t1-s2-0.0.1~piEGPXmDC8002001', p.identity.fqname)
# Duplicate
p = bp.find_or_new(table = 'tone',time = 't1', space='s2')
self.assertEquals('source-dataset-subset-variation-tone-t1-s2-0.0.1~piEGPXmDC8002001', p.identity.fqname)
# Ok! Build!
bundle.close()
bundle = Bundle()
bundle.exit_on_fatal = False
bundle.clean()
bundle.pre_prepare()
bundle.prepare()
bundle.post_prepare()
bundle.pre_build()
bundle.build_db_inserter_codes()
bundle.post_build()
self.assertEquals('diEGPXmDC8001', bundle.identity.vid)
self.assertEquals('source-dataset-subset-variation', bundle.identity.sname)
self.assertEquals('source-dataset-subset-variation-0.0.1', bundle.identity.vname)
self.assertEquals('source-dataset-subset-variation-0.0.1~diEGPXmDC8001', bundle.identity.fqname)
class Test(TestBase):
def setUp(self):
super(Test, self).setUp()
self.copy_or_build_bundle()
self.bundle = Bundle()
self.bundle_dir = self.bundle.bundle_dir
def test_db_bundle(self):
from ambry.bundle import BuildBundle, DbBundle
b = BuildBundle(self.bundle_dir)
b.clean()
self.assertTrue(b.identity.id_ is not None)
self.assertEquals('source-dataset-subset-variation', b.identity.sname)
self.assertEquals('source-dataset-subset-variation-0.0.1', b.identity.vname)
b.database.create()
db_path = b.database.path
dbb = DbBundle(db_path)
self.assertEqual("source-dataset-subset-variation", dbb.identity.sname)
self.assertEqual("source-dataset-subset-variation-0.0.1", dbb.identity.vname)
def test_paths(self):
""" Test that a build bundle and a db bundle both produce the same paths. """
from ambry.bundle import DbBundle
b = self.bundle
db = DbBundle(b.database.path)
self.assertEqual(b.path, db.path)
self.assertTrue(os.path.exists(b.path))
self.assertEqual(b.database.path, db.database.path)
self.assertTrue(os.path.exists(b.database.path))
self.assertEqual(b.identity.path, db.identity.path)
for p in zip(b.partitions, db.partitions):
self.assertTrue(bool(p[0].path))
self.assertEqual(p[0].path, p[1].path)
self.assertTrue(bool(p[0].path))
def test_schema_direct(self):
"""Test adding tables directly to the schema"""
# If we don't explicitly set the id_, it will change for every run.
self.bundle.metadata.identity.id = 'aTest'
self.bundle.schema.clean()
with self.bundle.session:
s = self.bundle.schema
s.add_table('table 1', altname='alt name a')
s.add_table('table 2', altname='alt name b')
self.assertRaises(Exception, s.add_table, ('table 1', ))
t = s.add_table('table 3', altname='alt name')
s.add_column(t, 'col 1', altname='altname1')
s.add_column(t, 'col 2', altname='altname2')
s.add_column(t, 'col 3', altname='altname3')
# print self.bundle.schema.as_csv()
self.assertIn('tiEGPXmDC801', [t.id_ for t in self.bundle.schema.tables])
self.assertIn('tiEGPXmDC802', [t.id_ for t in self.bundle.schema.tables])
self.assertNotIn('cTest03', [t.id_ for t in self.bundle.schema.tables])
t = self.bundle.schema.table('table_3')
self.assertIn('ciEGPXmDC803001', [c.id_ for c in t.columns])
self.assertIn('ciEGPXmDC803002', [c.id_ for c in t.columns])
self.assertIn('ciEGPXmDC803003', [c.id_ for c in t.columns])
# Try with a nested session, b/c we need to test it somewhere ...
with self.bundle.session:
with self.bundle.session:
t = s.add_table('table 4', altname='alt name')
s.add_column(t, 'col 1', altname='altname1')
s.add_column(t, 'col 2', altname='altname2')
s.add_column(t, 'col 3', altname='altname3')
def x_test_generate_schema(self):
"""Uses the generateSchema method in the bundle"""
from ambry.orm import Column
with self.bundle.session:
s = self.bundle.schema
s.clean()
t1 = s.add_table('table1')
s.add_column(t1, name='col1', datatype=Column.DATATYPE_REAL)
s.add_column(t1, name='col2', datatype=Column.DATATYPE_INTEGER)
s.add_column(t1, name='col3', datatype=Column.DATATYPE_TEXT)
t2 = s.add_table('table2')
s.add_column(t2, name='col1')
s.add_column(t2, name='col2')
s.add_column(t2, name='col3')
t3 = s.add_table('table3')
s.add_column(t3, name='col1', datatype=Column.DATATYPE_REAL)
s.add_column(t3, name='col2', datatype=Column.DATATYPE_INTEGER)
s.add_column(t3, name='col3', datatype=Column.DATATYPE_TEXT)
def test_column_processor(self):
from ambry.orm import Column
from ambry.transform import BasicTransform, CensusTransform
self.bundle.schema.clean()
with self.bundle.session:
s = self.bundle.schema
t = s.add_table('table3')
s.add_column(t, name='col1', datatype=Column.DATATYPE_INTEGER, default=-1, illegal_value='999')
s.add_column(t, name='col2', datatype=Column.DATATYPE_TEXT)
s.add_column(t, name='col3', datatype=Column.DATATYPE_REAL)
c1 = t.column('col1')
self.assertEquals(1, BasicTransform(c1)({'col1': ' 1 '}))
with self.assertRaises(ValueError):
print "PROCESSOR '{}'".format(CensusTransform(c1)({'col1': ' B '}))
self.assertEquals(1, CensusTransform(c1)({'col1': ' 1 '}))
self.assertEquals(-1, CensusTransform(c1)({'col1': ' 999 '}))
self.assertEquals(-3, CensusTransform(c1)({'col1': ' # '}))
self.assertEquals(-2, CensusTransform(c1)({'col1': ' ! '}))
def test_validator(self):
#
# Validators
#
tests = [
('tone', True, (None, 'VALUE', 0, 0)),
('tone', True, (None, 'VALUE', -1, 0)),
('tone', False, (None, 'DEFAULT', 0, 0)),
('tone', False, (None, 'DEFAULT', -1, 0)),
('ttwo', True, (None, 'DEFAULT', 0, 0)),
('ttwo', True, (None, 'DEFAULT', 0, 3.14)),
('ttwo', False, (None, 'DEFAULT', -1, 0)),
('tthree', True, (None, 'DEFAULT', 0, 0)),
('tthree', True, (None, 'DEFAULT', 0, 3.14)),
('all', True, (None, 'text1', 'text2', 1, 2, 3, 3.14)),
('all', False, (None, 'text1', 'text2', -1, -1, 3, 3.14)),
('all', False, (None, 'text1', 'text2', -1, 2, 3, 3.14)),
('all', False, (None, 'text1', 'text2', 1, -1, 3, 3.14)),
]
for i, test in enumerate(tests):
table_name, truth, row = test
table = self.bundle.schema.table(table_name)
vd = table._get_validator()
if truth:
self.assertTrue(vd(row), "Test {} not 'true' for table '{}': {}".format(i + 1, table_name, row))
else:
self.assertFalse(vd(row), "Test {} not 'false' for table '{}': {}".format(i + 1, table_name, row))
# Testing the "OR" join of multiple columns.
tests = [
('tone', True, (None, 'VALUE', 0, 0)), # 1
('tone', True, (None, 'VALUE', -1, 0)),
('tone', False, (None, 'DEFAULT', 0, 0)),
('tone', False, (None, 'DEFAULT', -1, 0)),
('ttwo', True, (None, 'DEFAULT', 0, 0)), # 5
('ttwo', True, (None, 'DEFAULT', 0, 3.14)),
('ttwo', False, (None, 'DEFAULT', -1, 0)),
('tthree', True, (None, 'DEFAULT', 0, 0)), # 8
('tthree', True, (None, 'DEFAULT', 0, 3.14)),
('all', True, (None, 'text1', 'text2', 1, 2, 3, 3.14)), # 10
('all', False, (None, 'text1', 'text2', -1, -1, 3, 3.14)), # 11
('all', True, (None, 'text1', 'text2', -1, 2, 3, 3.14)), # 12
('all', True, (None, 'text1', 'text2', 1, -1, 3, 3.14)), # 13
]
for i, test in enumerate(tests):
table_name, truth, row = test
table = self.bundle.schema.table(table_name)
vd = table._get_validator(and_join=False)
if truth:
self.assertTrue(vd(row), "Test {} not 'true' for table '{}': {}".format(i + 1, table_name, row))
else:
self.assertFalse(vd(row), "Test {} not 'false' for table '{}': {}".format(i + 1, table_name, row))
# Test the hash functions. This test depends on the d_test values in geoschema.csv
tests = [
('tone', 'A|1|', (None, 'A', 1, 2)),
('ttwo', '1|2|', (None, 'B', 1, 2)),
('tthree', 'C|2|', (None, 'C', 1, 2))]
import hashlib
for i, test in enumerate(tests):
table_name, hashed_str, row = test
table = self.bundle.schema.table(table_name)
m = hashlib.md5()
m.update(hashed_str)
self.assertEquals(int(m.hexdigest()[:14], 16), table.row_hash(row))
def test_partition(self):
from ambry.dbexceptions import ConflictError
from ambry.identity import PartitionNameQuery
self.bundle.clean()
self.bundle.prepare()
p = self.bundle.partitions.new_db_partition(table='tone', time=10, space=10, data={'pid':'pid1'})
with self.assertRaises(ConflictError):
self.bundle.partitions.new_db_partition(table='tone',time=10, space=10, data={'pid':'pid1'})
self.assertEqual(1, len(self.bundle.partitions.all))
p = self.bundle.partitions.find_or_new(table='tone',time=10, space=10)
p.database.create() # Find will go to the library if the database doesn't exist.
self.assertEqual(1, len(self.bundle.partitions.all))
self.assertEquals('pid1', p.data['pid'])
p = self.bundle.partitions.find(PartitionNameQuery(table='tone',time=10, space=10))
self.assertEquals('pid1',p.data['pid'] )
p = self.bundle.partitions.find(table='tone',time=10, space=10)
self.assertEquals('pid1', p.data['pid'])
pnq3 = PartitionNameQuery(space=10)
with self.bundle.session as s:
p = self.bundle.partitions._find_orm(pnq3).first()
p.data['foo'] = 'bar'
s.add(p)
bundle = Bundle()
p = bundle.partitions.find(pnq3)
self.assertEquals('bar', p.data['foo'])
# p = self.bundle.partitions.find(PartitionNameQuery(name='source-dataset-subset-variation-30-hdf'))
# self.assertTrue(p is not None)
# self.assertEquals('source-dataset-subset-variation-30-hdf', p.identity.sname)
#
# Create all possible combinations of partition names
#
table = self.bundle.schema.tables[0]
p = (('time', 'time2'), ('space', 'space3'), ('table', table.name), ('grain', 'grain4'))
p += p
pids = {}
for i in range(4):
for j in range(4):
pid = self.bundle.identity.as_partition(**dict(p[i:i + j + 1]))
pids[pid.fqname] = pid
with self.bundle.session as s:
s.commit()
# These two deletey bits clear out all of the old
# partitions, to avoid a conflict with the next section. We also have
# to delete the files, since create() adds a partition record to the database,
# and if one already exists, it will throw an Integrity Error.
for p in self.bundle.partitions:
if os.path.exists(p.database.path):
os.remove(p.database.path)
for p in self.bundle.dataset.partitions:
# Using SQL instead of s.delete() because we want to avoid the cascade to stored_partitions, since
# that table doesn't exist in the bundle, only in the library
s.execute("DELETE FROM partitions WHERE p_vid = :vid", {'vid': p.vid})
# s.delete(p)
def test_runconfig(self):
"""Check the the RunConfig expands the library configuration"""
from ambry.run import get_runconfig, RunConfig
rc = get_runconfig(
(os.path.join(self.bundle_dir, 'test-run-config.yaml'), RunConfig.USER_CONFIG, RunConfig.USER_ACCOUNTS))
l = rc.library('library1')
self.assertEquals('database1', l['database']['_name'])
self.assertEquals('filesystem1', l['filesystem']['_name'])
self.assertEquals('filesystem2', l['filesystem']['upstream']['_name'])
self.assertEquals('filesystem3', l['filesystem']['upstream']['upstream']['_name'])
def test_build_bundle(self):
import shutil
bundle = Bundle()
shutil.copyfile(
bundle.filesystem.path('meta', 'schema-edit-me.csv'),
bundle.filesystem.path('meta', 'schema.csv'))
# try:
bundle.database.enable_delete = True
bundle.clean()
bundle = Bundle()
bundle.exit_on_fatal = False
bundle.pre_prepare()
bundle.prepare()
bundle.post_prepare()
bundle.pre_build()
bundle.build_db_inserter_codes()
bundle.post_build()
bundle.close()
# The second run will use the changes to the schema made in the
# first run, due to the types errors in the 'coding' table.
bundle.clean()
bundle = Bundle()
bundle.exit_on_fatal = False
bundle.pre_prepare()
bundle.prepare()
bundle.post_prepare()
bundle.pre_build()
bundle.build_db_inserter_codes()
bundle.post_build()
bundle.close()
try:
pass
finally:
# Need to clean up to ensure that we're back to a good state.
# This runs the normal build, which will be used by the other
# tests.
shutil.copyfile(
bundle.filesystem.path('meta', 'schema-edit-me.csv'),
bundle.filesystem.path('meta', 'schema.csv'))
bundle.clean()
bundle = Bundle()
bundle.exit_on_fatal = False
bundle.pre_prepare()
bundle.prepare()
bundle.post_prepare()
bundle.pre_build()
bundle.build()
bundle.post_build()
def test_simple_build(self):
import shutil
bundle = Bundle()
shutil.copyfile(
bundle.filesystem.path('meta', 'schema-edit-me.csv'),
bundle.filesystem.path('meta', 'schema.csv'))
bundle.clean()
bundle = Bundle()
bundle.exit_on_fatal = False
bundle.pre_prepare()
bundle.prepare()
bundle.post_prepare()
bundle.pre_build()
bundle.build()
bundle.post_build()
def test_config_update(self):
bundle = Bundle()
bundle.update_configuration()
def test_session(self):
import uuid
b = self.bundle
uv = str(uuid.uuid4())
with b.session as s1:
with b.session as s2:
b.set_value('test', 'uuid', uv)
b.close()
self.assertEqual(uv, b.get_value('test', 'uuid').value)
uv2 = str(uuid.uuid4())
self.assertNotEqual(uv, uv2)
with b.session as s1:
with b.session as s2:
b.set_value('test', 'uuid', uv2)
self.assertEqual(uv2, b.get_value('test', 'uuid').value)
b.set_value('test', 'uuid', uv2)
