def init_db():
engine = create_engine('sqlite://')
DBSession.configure(bind=engine)
VersionedDBSession.configure(bind=engine)
Base.metadata.bind = engine
Base.metadata.create_all()
return engine
def testapp():
from webtest import TestApp
from clld.db.meta import DBSession, VersionedDBSession, Base
from clld.db.models import common
from clld_cognacy_plugin.models import Cognateset, Cognate
def main():
cfg = config.Configurator(settings={
'sqlalchemy.url': 'sqlite://',
'mako.directories': [
'clld:web/templates',
'clld_cognacy_plugin:templates'
]})
cfg.include('clld.web.app')
cfg.include('clld_cognacy_plugin')
return cfg.make_wsgi_app()
DBSession.remove()
VersionedDBSession.remove()
wsgi_app = main()
Base.metadata.bind = DBSession.bind
Base.metadata.create_all()
DBSession.add(common.Dataset(id='1', name='test app', domain='example.org'))
cs = Cognateset(id='1', name='cs: test')
lang = common.Language(id='l', latitude=2, longitude=2)
param = common.Parameter(id='l')
vs = common.ValueSet(id='vs', language=lang, parameter=param)
v = common.Value(id='v', name='abc', valueset=vs)
DBSession.add(Cognate(cognateset=cs, counterpart=v))
yield TestApp(wsgi_app)
def setup_session(config_uri, engine=None):
setup_logging(config_uri)
settings = get_appsettings(config_uri)
engine = engine or engine_from_config(settings, "sqlalchemy.")
DBSession.configure(bind=engine)
VersionedDBSession.configure(bind=engine)
Base.metadata.create_all(engine)
return Path(config_uri.split("#")[0]).resolve().parent.name
def setup_session(config_uri, engine=None):
setup_logging(config_uri)
settings = get_appsettings(config_uri)
engine = engine or engine_from_config(settings, 'sqlalchemy.')
DBSession.configure(bind=engine)
VersionedDBSession.configure(bind=engine)
Base.metadata.create_all(engine)
return path(config_uri.split('#')[0]).abspath().dirname().basename()
def setup_session(config_uri, engine=None):
setup_logging(config_uri)
settings = get_appsettings(config_uri)
engine = engine or engine_from_config(settings, 'sqlalchemy.')
DBSession.configure(bind=engine)
VersionedDBSession.configure(bind=engine)
Base.metadata.create_all(engine)
return path(config_uri.split('#')[0]).abspath().dirname().basename()
def setUp(self):
from clld.tests.fixtures import CustomLanguage
assert CustomLanguage
engine = create_engine('sqlite://')
DBSession.configure(bind=engine)
VersionedDBSession.configure(bind=engine)
Base.metadata.bind = engine
Base.metadata.create_all()
def test_Base_jsondata(db):
l = Language(id='abc', name='Name')
VersionedDBSession.add(l)
VersionedDBSession.flush()
l.update_jsondata(a=1)
assert 'a' in l.jsondata
l.update_jsondata(b=1)
assert 'b' in l.jsondata and 'a' in l.jsondata
assert 'b' in l.__json__(None)['jsondata']
def setUp(self):
from clld.tests.fixtures import CustomLanguage
assert CustomLanguage
engine = create_engine('sqlite://')
DBSession.configure(bind=engine)
VersionedDBSession.configure(bind=engine)
Base.metadata.bind = engine
Base.metadata.create_all()
def test_Base_jsondata(self):
l = Language(id='abc', name='Name')
VersionedDBSession.add(l)
VersionedDBSession.flush()
l.update_jsondata(a=1)
self.assertTrue('a' in l.jsondata)
l.update_jsondata(b=1)
self.assertTrue('b' in l.jsondata and 'a' in l.jsondata)
self.assertTrue('b' in l.__json__(None)['jsondata'])
def test_Base_jsondata(self):
l = Language(id='abc', name='Name')
VersionedDBSession.add(l)
VersionedDBSession.flush()
l.update_jsondata(a=1)
self.assertTrue('a' in l.jsondata)
l.update_jsondata(b=1)
self.assertTrue('b' in l.jsondata and 'a' in l.jsondata)
self.assertTrue('b' in l.__json__(None)['jsondata'])
def test_Base_jsondata(db):
l = Language(id='abc', name='Name')
VersionedDBSession.add(l)
VersionedDBSession.flush()
l.update_jsondata(a=1)
assert 'a' in l.jsondata
l.update_jsondata(b=1)
assert 'b' in l.jsondata and 'a' in l.jsondata
assert 'b' in l.__json__(None)['jsondata']
def data(settings):
from clld.db.meta import Base, DBSession, VersionedDBSession
engine = sa.engine_from_config(settings)
Base.metadata.create_all(bind=engine)
DBSession.configure(bind=engine)
VersionedDBSession.configure(bind=engine)
yield engine
DBSession.close()
VersionedDBSession.close()
def main(args):
with transaction.manager:
param = VersionedDBSession.query(common.Parameter) \
.filter(common.Parameter.id == '138A') \
.one()
for de in param.domain:
for k, v in VALUES.items():
if v == de.name:
VALUES[k] = de
for lid, value in [('lat', 'te'), ('dhi', 'cha'), ('chp', 'te')]:
vs = VersionedDBSession.query(common.ValueSet)\
.join(common.Language)\
.filter(common.Language.id == lid)\
.filter(common.ValueSet.parameter_pk == param.pk)\
.one()
de = VALUES[value]
assert vs.values[0].domainelement != de
vs.values[0].domainelement = de
def test_Base(db):
l = Language(id='abc', name='Name')
VersionedDBSession.add(l)
VersionedDBSession.flush()
VersionedDBSession.expunge(l)
l = Language.get('abc', session=VersionedDBSession)
assert l.name == 'Name'
assert not list(l.history())
Language().__str__()
assert repr(l) == "<Language 'abc'>"
def test_Base(self):
l = Language(id='abc', name='Name')
VersionedDBSession.add(l)
VersionedDBSession.flush()
VersionedDBSession.expunge(l)
l = Language.get('abc')
self.assertEqual(l.name, 'Name')
assert not list(l.history())
# a bit of a hack to test the human readable representations.
# we exploit the fact, that on py2, string and unicode comparison does type
# coercion, while on py3, the two methods should actually return the same string.
self.assertEqual(l.__str__(), l.__unicode__())
Language().__str__()
def test_Base(self):
l = Language(id='abc', name='Name')
VersionedDBSession.add(l)
VersionedDBSession.flush()
VersionedDBSession.expunge(l)
l = Language.get('abc')
self.assertEqual(l.name, 'Name')
assert not list(l.history())
# a bit of a hack to test the human readable representations.
# we exploit the fact, that on py2, string and unicode comparison does type
# coercion, while on py3, the two methods should actually return the same string.
self.assertEqual(l.__str__(), l.__unicode__())
Language().__str__()
def test_Base(db):
l = Language(id='abc', name='Name')
VersionedDBSession.add(l)
VersionedDBSession.flush()
VersionedDBSession.expunge(l)
l = Language.get('abc', session=VersionedDBSession)
assert l.name == 'Name'
assert not list(l.history())
# a bit of a hack to test the human readable representations.
# we exploit the fact, that on py2, string and unicode comparison does type
# coercion, while on py3, the two methods should actually return the same string.
assert l.__str__() == l.__unicode__()
Language().__str__()
assert repr(l) == "<Language 'abc'>" if PY3 else "<Language u'abc'>"
def test_Base(self):
l = Language(id='abc', name='Name')
VersionedDBSession.add(l)
VersionedDBSession.flush()
VersionedDBSession.expunge(l)
#print('pk: %s' % l.pk)
#transaction.commit()
#transaction.begin()
#l = VersionedDBSession.query(Language).get(1)
#print(l)
#l.name = 'New name'
#print('pk: %s' % l.pk)
#transaction.commit()
#transaction.begin()
l = Language.get('abc')
#print(l.version)
self.assertEqual(l.name, 'Name')
l.history()
# a bit of a hack to test the human readable representations.
# we exploit the fact, that on py2, string and unicode comparison does type
# coercion, while on py3, the two methods should actualy return the same string.
self.assertEqual(l.__str__(), l.__unicode__())
Language().__str__()
def test_Base(self):
l = Language(id='abc', name='Name')
VersionedDBSession.add(l)
VersionedDBSession.flush()
VersionedDBSession.expunge(l)
#print('pk: %s' % l.pk)
#transaction.commit()
#transaction.begin()
#l = VersionedDBSession.query(Language).get(1)
#print(l)
#l.name = 'New name'
#print('pk: %s' % l.pk)
#transaction.commit()
#transaction.begin()
l = Language.get('abc')
#print(l.version)
self.assertEqual(l.name, 'Name')
l.history()
# a bit of a hack to test the human readable representations.
# we exploit the fact, that on py2, string and unicode comparison does type
# coercion, while on py3, the two methods should actualy return the same string.
self.assertEqual(l.__str__(), l.__unicode__())
Language().__str__()
def db(db):
try:
yield db
finally:
DBSession.rollback()
VersionedDBSession.rollback()
def prime_cache(args): # pragma: no cover
"""
we use a versioned session to insert the changes in value assignment
"""
#
# compute the changes from 2008 to 2011:
#
vs2008 = get_vs2008(args)
for row in DB.execute("select * from datapoint"):
key = (row["language_id"], row["feature_id"])
old_value = vs2008.get(key)
new_value = row["value_numeric"]
if old_value and old_value != new_value:
valueset = (
VersionedDBSession.query(common.ValueSet)
.join(common.Language)
.join(common.Parameter)
.filter(common.Parameter.id == row["feature_id"])
.filter(common.Language.id == row["language_id"])
.one()
)
value = valueset.values[0]
assert value.domainelement.number == old_value
for de in valueset.parameter.domain:
if de.number == new_value:
value.domainelement = de
break
assert value.domainelement.number == new_value
valueset.updated = E2011
value.updated = E2011
VersionedDBSession.flush()
for row in reader(args.data_file("corrections_2013.tab"), namedtuples=True, newline="\r"):
valueset = (
VersionedDBSession.query(common.ValueSet)
.join(common.Language)
.join(common.Parameter)
.filter(common.Parameter.id == row.feature)
.filter(common.Language.id == row.wals_code)
.one()
)
value = valueset.values[0]
if value.domainelement.number == int(row.new):
print("**** old news", valueset.language.id, valueset.parameter.id)
continue
if value.domainelement.number != int(row.old):
print("--->", valueset.language.id, valueset.parameter.id, value.domainelement.number)
for de in valueset.parameter.domain:
if de.number == int(row.new):
value.domainelement = de
break
assert value.domainelement.number == int(row.new)
valueset.updated = E2013
value.updated = E2013
VersionedDBSession.flush()
print("corrections 2013 done")
for issue in ["0", "9", "10", "11", "13", "14", "15", "16", "17", "19", "20", "24", "26", "27", "28"]:
issue = getattr(issues, "issue" + issue)
issue(VersionedDBSession, E2013)
VersionedDBSession.flush()
transaction.commit()
transaction.begin()
#
# TODO: these must be recomputed as well, after migrations!
#
# cache number of languages for a parameter:
for parameter, valuesets in groupby(
DBSession.query(common.ValueSet).order_by(common.ValueSet.parameter_pk), lambda vs: vs.parameter
):
parameter.representation = str(len(set(v.language_pk for v in valuesets)))
print("recomputation of representation done")
transaction.commit()
transaction.begin()
# cache iso codes for languages:
for language in DBSession.query(common.Language).options(
joinedload_all(common.Language.languageidentifier, common.LanguageIdentifier.identifier)
):
iso_codes = []
for identifier in language.identifiers:
if identifier.type == common.IdentifierType.iso.value:
iso_codes.append(identifier.name)
language.iso_codes = ", ".join(sorted(set(iso_codes)))
print("ecomputation of iso codes done")
transaction.commit()
transaction.begin()
compute_language_sources()
transaction.commit()
transaction.begin()
gbs_func("update", args)
def test_Versioning(db):
from clld.db.models.common import Language, Language_data
from clld.db.meta import VersionedDBSession
l = Language(id='abc', name='Old Name', jsondata={'i': 2})
VersionedDBSession.add(l)
VersionedDBSession.flush()
assert l.version == 1
l.name = 'New Name'
l.description = 'New Description'
VersionedDBSession.flush()
assert l.version == 2
History = l.__history_mapper__.class_
res = VersionedDBSession.query(History).filter(History.pk == l.pk).all()
assert res[0].name == 'Old Name'
l.data.append(Language_data(key='k', value='v'))
VersionedDBSession.flush()
assert l.datadict()
VersionedDBSession.delete(l)
VersionedDBSession.flush()
def prime_cache(args): # pragma: no cover
"""
we use a versioned session to insert the changes in value assignment
"""
#
# compute the changes from 2008 to 2011:
#
vs2008 = get_vs2008(args)
for row in DB.execute("select * from datapoint"):
key = (row['language_id'], row['feature_id'])
old_value = vs2008.get(key)
new_value = row['value_numeric']
if old_value and old_value != new_value:
valueset = VersionedDBSession.query(common.ValueSet)\
.join(common.Language)\
.join(common.Parameter)\
.filter(common.Parameter.id == row['feature_id'])\
.filter(common.Language.id == row['language_id'])\
.one()
value = valueset.values[0]
assert value.domainelement.number == old_value
for de in valueset.parameter.domain:
if de.number == new_value:
value.domainelement = de
break
assert value.domainelement.number == new_value
valueset.updated = E2011
value.updated = E2011
VersionedDBSession.flush()
for row in reader(args.data_file('corrections_2013.tab'),
namedtuples=True,
newline='\r'):
valueset = VersionedDBSession.query(common.ValueSet)\
.join(common.Language)\
.join(common.Parameter)\
.filter(common.Parameter.id == row.feature)\
.filter(common.Language.id == row.wals_code)\
.one()
value = valueset.values[0]
if value.domainelement.number == int(row.new):
print('**** old news', valueset.language.id, valueset.parameter.id)
continue
if value.domainelement.number != int(row.old):
print('--->', valueset.language.id, valueset.parameter.id,
value.domainelement.number)
for de in valueset.parameter.domain:
if de.number == int(row.new):
value.domainelement = de
break
assert value.domainelement.number == int(row.new)
valueset.updated = E2013
value.updated = E2013
VersionedDBSession.flush()
print('corrections 2013 done')
for issue in [
'0', '9', '10', '11', '13', '14', '15', '16', '17', '19', '20',
'24', '26', '27', '28'
]:
issue = getattr(issues, 'issue' + issue)
issue(VersionedDBSession, E2013)
VersionedDBSession.flush()
transaction.commit()
transaction.begin()
#
# TODO: these must be recomputed as well, after migrations!
#
# cache number of languages for a parameter:
for parameter, valuesets in groupby(
DBSession.query(common.ValueSet).order_by(
common.ValueSet.parameter_pk), lambda vs: vs.parameter):
parameter.representation = str(
len(set(v.language_pk for v in valuesets)))
print('recomputation of representation done')
transaction.commit()
transaction.begin()
# cache iso codes for languages:
for language in DBSession.query(common.Language).options(
joinedload_all(common.Language.languageidentifier,
common.LanguageIdentifier.identifier)):
iso_codes = []
for identifier in language.identifiers:
if identifier.type == common.IdentifierType.iso.value:
iso_codes.append(identifier.name)
language.iso_codes = ', '.join(sorted(set(iso_codes)))
print('ecomputation of iso codes done')
transaction.commit()
transaction.begin()
compute_language_sources()
transaction.commit()
transaction.begin()
gbs_func('update', args)
def test_Versioning(self):
from clld.db.models.common import Language, Identifier, LanguageIdentifier
from clld.db.meta import VersionedDBSession
l = Language(id='abc', name='Old Name', jsondata={'i': 2})
VersionedDBSession.add(l)
VersionedDBSession.flush()
self.assertEqual(l.version, 1)
l.name = 'New Name'
l.description = 'New Description'
VersionedDBSession.flush()
self.assertEqual(l.version, 2)
History = l.__history_mapper__.class_
res = VersionedDBSession.query(History).filter(
History.pk == l.pk).all()
self.assertEqual(res[0].name, 'Old Name')
li = LanguageIdentifier(identifier=Identifier(id='asd', type='wals'),
language=l)
VersionedDBSession.flush()
VersionedDBSession.delete(li)
VersionedDBSession.delete(l)
VersionedDBSession.flush()
def db(db):
try:
yield db
finally:
DBSession.rollback()
VersionedDBSession.rollback()
def test_Versioning(self):
from clld.db.models.common import Language, Identifier, LanguageIdentifier
from clld.db.meta import VersionedDBSession
l = Language(id='abc', name='Old Name', jsondata={'i': 2})
VersionedDBSession.add(l)
VersionedDBSession.flush()
self.assertEqual(l.version, 1)
l.name = 'New Name'
l.description = 'New Description'
VersionedDBSession.flush()
self.assertEqual(l.version, 2)
History = l.__history_mapper__.class_
res = VersionedDBSession.query(History).filter(History.pk == l.pk).all()
self.assertEqual(res[0].name, 'Old Name')
li = LanguageIdentifier(
identifier=Identifier(id='asd', type='wals'),
language=l)
VersionedDBSession.flush()
VersionedDBSession.delete(li)
VersionedDBSession.delete(l)
VersionedDBSession.flush()
def test_Versioning(self):
from clld.db.models.common import Language, Language_data
from clld.db.meta import VersionedDBSession
l = Language(id='abc', name='Old Name', jsondata={'i': 2})
VersionedDBSession.add(l)
VersionedDBSession.flush()
self.assertEqual(l.version, 1)
l.name = 'New Name'
l.description = 'New Description'
VersionedDBSession.flush()
self.assertEqual(l.version, 2)
History = l.__history_mapper__.class_
res = VersionedDBSession.query(History).filter(History.pk == l.pk).all()
self.assertEqual(res[0].name, 'Old Name')
l.data.append(Language_data(key='k', value='v'))
VersionedDBSession.flush()
assert l.datadict()
VersionedDBSession.delete(l)
VersionedDBSession.flush()
def prime_cache(args):
"""
we use a versioned session to insert the changes in value assignment
"""
#
# compute the changes from 2008 to 2011:
#
vs2008 = get_vs2008(args)
for row in DB.execute("select * from datapoint"):
key = (row['language_id'], row['feature_id'])
old_value = vs2008.get(key)
new_value = row['value_numeric']
if old_value and old_value != new_value:
valueset = VersionedDBSession.query(common.ValueSet)\
.join(common.Language)\
.join(common.Parameter)\
.filter(common.Parameter.id == row['feature_id'])\
.filter(common.Language.id == row['language_id'])\
.one()
value = valueset.values[0]
assert value.domainelement.number == old_value
for de in valueset.parameter.domain:
if de.number == new_value:
value.domainelement = de
break
assert value.domainelement.number == new_value
valueset.updated = E2011
value.updated = E2011
VersionedDBSession.flush()
for row in reader(args.data_file('corrections_2013.tab'), namedtuples=True, newline='\r'):
valueset = VersionedDBSession.query(common.ValueSet)\
.join(common.Language)\
.join(common.Parameter)\
.filter(common.Parameter.id == row.feature)\
.filter(common.Language.id == row.wals_code)\
.one()
value = valueset.values[0]
if value.domainelement.number == int(row.new):
print '**** old news', valueset.language.id, valueset.parameter.id
continue
if value.domainelement.number != int(row.old):
print '--->', valueset.language.id, valueset.parameter.id, value.domainelement.number
for de in valueset.parameter.domain:
if de.number == int(row.new):
value.domainelement = de
break
assert value.domainelement.number == int(row.new)
valueset.updated = E2013
value.updated = E2013
VersionedDBSession.flush()
print 'corrections 2013 done'
for issue in ['0', '9', '10', '11', '13', '14', '15', '16', '17', '19', '20', '24', '26', '27', '28']:
issue = getattr(issues, 'issue' + issue)
issue(VersionedDBSession, E2013)
VersionedDBSession.flush()
transaction.commit()
transaction.begin()
#
# TODO: these must be recomputed as well, after migrations!
#
# cache number of languages for a parameter:
for parameter, valuesets in groupby(
DBSession.query(common.ValueSet).order_by(common.ValueSet.parameter_pk),
lambda vs: vs.parameter):
parameter.representation = str(len(set(v.language_pk for v in valuesets)))
print 'recomputation of representation done'
transaction.commit()
transaction.begin()
# cache iso codes for languages:
for language in DBSession.query(common.Language).options(joinedload_all(
common.Language.languageidentifier, common.LanguageIdentifier.identifier
)):
iso_codes = []
for identifier in language.identifiers:
if identifier.type == common.IdentifierType.iso.value:
iso_codes.append(identifier.name)
language.iso_codes = ', '.join(sorted(set(iso_codes)))
print 'recomputation of iso codes done'
transaction.commit()
transaction.begin()
compute_language_sources()
transaction.commit()
transaction.begin()
gbs_func('update', args)
