def test_Base_jsondata(self):
l = Language(id='abc', name='Name')
VersionedDBSession.add(l)
VersionedDBSession.flush()
l.update_jsondata(a=1)
self.assertTrue('a' in l.jsondata)
l.update_jsondata(b=1)
self.assertTrue('b' in l.jsondata and 'a' in l.jsondata)
self.assertTrue('b' in l.__json__(None)['jsondata'])
def test_Base_jsondata(db):
l = Language(id='abc', name='Name')
VersionedDBSession.add(l)
VersionedDBSession.flush()
l.update_jsondata(a=1)
assert 'a' in l.jsondata
l.update_jsondata(b=1)
assert 'b' in l.jsondata and 'a' in l.jsondata
assert 'b' in l.__json__(None)['jsondata']
def test_Base_jsondata(db):
l = Language(id='abc', name='Name')
VersionedDBSession.add(l)
VersionedDBSession.flush()
l.update_jsondata(a=1)
assert 'a' in l.jsondata
l.update_jsondata(b=1)
assert 'b' in l.jsondata and 'a' in l.jsondata
assert 'b' in l.__json__(None)['jsondata']
def test_Base_jsondata(self):
l = Language(id='abc', name='Name')
VersionedDBSession.add(l)
VersionedDBSession.flush()
l.update_jsondata(a=1)
self.assertTrue('a' in l.jsondata)
l.update_jsondata(b=1)
self.assertTrue('b' in l.jsondata and 'a' in l.jsondata)
self.assertTrue('b' in l.__json__(None)['jsondata'])
def test_Base(db):
l = Language(id='abc', name='Name')
VersionedDBSession.add(l)
VersionedDBSession.flush()
VersionedDBSession.expunge(l)
l = Language.get('abc', session=VersionedDBSession)
assert l.name == 'Name'
assert not list(l.history())
Language().__str__()
assert repr(l) == "<Language 'abc'>"
def test_Versioning(self):
from clld.db.models.common import Language, Identifier, LanguageIdentifier
from clld.db.meta import VersionedDBSession
l = Language(id='abc', name='Old Name', jsondata={'i': 2})
VersionedDBSession.add(l)
VersionedDBSession.flush()
self.assertEqual(l.version, 1)
l.name = 'New Name'
l.description = 'New Description'
VersionedDBSession.flush()
self.assertEqual(l.version, 2)
History = l.__history_mapper__.class_
res = VersionedDBSession.query(History).filter(
History.pk == l.pk).all()
self.assertEqual(res[0].name, 'Old Name')
li = LanguageIdentifier(identifier=Identifier(id='asd', type='wals'),
language=l)
VersionedDBSession.flush()
VersionedDBSession.delete(li)
VersionedDBSession.delete(l)
VersionedDBSession.flush()
def test_Versioning(self):
from clld.db.models.common import Language, Identifier, LanguageIdentifier
from clld.db.meta import VersionedDBSession
l = Language(id='abc', name='Old Name', jsondata={'i': 2})
VersionedDBSession.add(l)
VersionedDBSession.flush()
self.assertEqual(l.version, 1)
l.name = 'New Name'
l.description = 'New Description'
VersionedDBSession.flush()
self.assertEqual(l.version, 2)
History = l.__history_mapper__.class_
res = VersionedDBSession.query(History).filter(History.pk == l.pk).all()
self.assertEqual(res[0].name, 'Old Name')
li = LanguageIdentifier(
identifier=Identifier(id='asd', type='wals'),
language=l)
VersionedDBSession.flush()
VersionedDBSession.delete(li)
VersionedDBSession.delete(l)
VersionedDBSession.flush()
def test_Base(self):
l = Language(id='abc', name='Name')
VersionedDBSession.add(l)
VersionedDBSession.flush()
VersionedDBSession.expunge(l)
l = Language.get('abc')
self.assertEqual(l.name, 'Name')
assert not list(l.history())
# a bit of a hack to test the human readable representations.
# we exploit the fact, that on py2, string and unicode comparison does type
# coercion, while on py3, the two methods should actually return the same string.
self.assertEqual(l.__str__(), l.__unicode__())
Language().__str__()
def test_Base(self):
l = Language(id='abc', name='Name')
VersionedDBSession.add(l)
VersionedDBSession.flush()
VersionedDBSession.expunge(l)
l = Language.get('abc')
self.assertEqual(l.name, 'Name')
assert not list(l.history())
# a bit of a hack to test the human readable representations.
# we exploit the fact, that on py2, string and unicode comparison does type
# coercion, while on py3, the two methods should actually return the same string.
self.assertEqual(l.__str__(), l.__unicode__())
Language().__str__()
def test_Base(db):
l = Language(id='abc', name='Name')
VersionedDBSession.add(l)
VersionedDBSession.flush()
VersionedDBSession.expunge(l)
l = Language.get('abc', session=VersionedDBSession)
assert l.name == 'Name'
assert not list(l.history())
# a bit of a hack to test the human readable representations.
# we exploit the fact, that on py2, string and unicode comparison does type
# coercion, while on py3, the two methods should actually return the same string.
assert l.__str__() == l.__unicode__()
Language().__str__()
assert repr(l) == "<Language 'abc'>" if PY3 else "<Language u'abc'>"
def test_Base(self):
l = Language(id='abc', name='Name')
VersionedDBSession.add(l)
VersionedDBSession.flush()
VersionedDBSession.expunge(l)
#print('pk: %s' % l.pk)
#transaction.commit()
#transaction.begin()
#l = VersionedDBSession.query(Language).get(1)
#print(l)
#l.name = 'New name'
#print('pk: %s' % l.pk)
#transaction.commit()
#transaction.begin()
l = Language.get('abc')
#print(l.version)
self.assertEqual(l.name, 'Name')
l.history()
# a bit of a hack to test the human readable representations.
# we exploit the fact, that on py2, string and unicode comparison does type
# coercion, while on py3, the two methods should actualy return the same string.
self.assertEqual(l.__str__(), l.__unicode__())
Language().__str__()
def test_Base(self):
l = Language(id='abc', name='Name')
VersionedDBSession.add(l)
VersionedDBSession.flush()
VersionedDBSession.expunge(l)
#print('pk: %s' % l.pk)
#transaction.commit()
#transaction.begin()
#l = VersionedDBSession.query(Language).get(1)
#print(l)
#l.name = 'New name'
#print('pk: %s' % l.pk)
#transaction.commit()
#transaction.begin()
l = Language.get('abc')
#print(l.version)
self.assertEqual(l.name, 'Name')
l.history()
# a bit of a hack to test the human readable representations.
# we exploit the fact, that on py2, string and unicode comparison does type
# coercion, while on py3, the two methods should actualy return the same string.
self.assertEqual(l.__str__(), l.__unicode__())
Language().__str__()
def test_Versioning(db):
from clld.db.models.common import Language, Language_data
from clld.db.meta import VersionedDBSession
l = Language(id='abc', name='Old Name', jsondata={'i': 2})
VersionedDBSession.add(l)
VersionedDBSession.flush()
assert l.version == 1
l.name = 'New Name'
l.description = 'New Description'
VersionedDBSession.flush()
assert l.version == 2
History = l.__history_mapper__.class_
res = VersionedDBSession.query(History).filter(History.pk == l.pk).all()
assert res[0].name == 'Old Name'
l.data.append(Language_data(key='k', value='v'))
VersionedDBSession.flush()
assert l.datadict()
VersionedDBSession.delete(l)
VersionedDBSession.flush()
def test_Versioning(self):
from clld.db.models.common import Language, Language_data
from clld.db.meta import VersionedDBSession
l = Language(id='abc', name='Old Name', jsondata={'i': 2})
VersionedDBSession.add(l)
VersionedDBSession.flush()
self.assertEqual(l.version, 1)
l.name = 'New Name'
l.description = 'New Description'
VersionedDBSession.flush()
self.assertEqual(l.version, 2)
History = l.__history_mapper__.class_
res = VersionedDBSession.query(History).filter(History.pk == l.pk).all()
self.assertEqual(res[0].name, 'Old Name')
l.data.append(Language_data(key='k', value='v'))
VersionedDBSession.flush()
assert l.datadict()
VersionedDBSession.delete(l)
VersionedDBSession.flush()
def prime_cache(args): # pragma: no cover
"""
we use a versioned session to insert the changes in value assignment
"""
#
# compute the changes from 2008 to 2011:
#
vs2008 = get_vs2008(args)
for row in DB.execute("select * from datapoint"):
key = (row["language_id"], row["feature_id"])
old_value = vs2008.get(key)
new_value = row["value_numeric"]
if old_value and old_value != new_value:
valueset = (
VersionedDBSession.query(common.ValueSet)
.join(common.Language)
.join(common.Parameter)
.filter(common.Parameter.id == row["feature_id"])
.filter(common.Language.id == row["language_id"])
.one()
)
value = valueset.values[0]
assert value.domainelement.number == old_value
for de in valueset.parameter.domain:
if de.number == new_value:
value.domainelement = de
break
assert value.domainelement.number == new_value
valueset.updated = E2011
value.updated = E2011
VersionedDBSession.flush()
for row in reader(args.data_file("corrections_2013.tab"), namedtuples=True, newline="\r"):
valueset = (
VersionedDBSession.query(common.ValueSet)
.join(common.Language)
.join(common.Parameter)
.filter(common.Parameter.id == row.feature)
.filter(common.Language.id == row.wals_code)
.one()
)
value = valueset.values[0]
if value.domainelement.number == int(row.new):
print("**** old news", valueset.language.id, valueset.parameter.id)
continue
if value.domainelement.number != int(row.old):
print("--->", valueset.language.id, valueset.parameter.id, value.domainelement.number)
for de in valueset.parameter.domain:
if de.number == int(row.new):
value.domainelement = de
break
assert value.domainelement.number == int(row.new)
valueset.updated = E2013
value.updated = E2013
VersionedDBSession.flush()
print("corrections 2013 done")
for issue in ["0", "9", "10", "11", "13", "14", "15", "16", "17", "19", "20", "24", "26", "27", "28"]:
issue = getattr(issues, "issue" + issue)
issue(VersionedDBSession, E2013)
VersionedDBSession.flush()
transaction.commit()
transaction.begin()
#
# TODO: these must be recomputed as well, after migrations!
#
# cache number of languages for a parameter:
for parameter, valuesets in groupby(
DBSession.query(common.ValueSet).order_by(common.ValueSet.parameter_pk), lambda vs: vs.parameter
):
parameter.representation = str(len(set(v.language_pk for v in valuesets)))
print("recomputation of representation done")
transaction.commit()
transaction.begin()
# cache iso codes for languages:
for language in DBSession.query(common.Language).options(
joinedload_all(common.Language.languageidentifier, common.LanguageIdentifier.identifier)
):
iso_codes = []
for identifier in language.identifiers:
if identifier.type == common.IdentifierType.iso.value:
iso_codes.append(identifier.name)
language.iso_codes = ", ".join(sorted(set(iso_codes)))
print("ecomputation of iso codes done")
transaction.commit()
transaction.begin()
compute_language_sources()
transaction.commit()
transaction.begin()
gbs_func("update", args)
def prime_cache(args):
"""
we use a versioned session to insert the changes in value assignment
"""
#
# compute the changes from 2008 to 2011:
#
vs2008 = get_vs2008(args)
for row in DB.execute("select * from datapoint"):
key = (row['language_id'], row['feature_id'])
old_value = vs2008.get(key)
new_value = row['value_numeric']
if old_value and old_value != new_value:
valueset = VersionedDBSession.query(common.ValueSet)\
.join(common.Language)\
.join(common.Parameter)\
.filter(common.Parameter.id == row['feature_id'])\
.filter(common.Language.id == row['language_id'])\
.one()
value = valueset.values[0]
assert value.domainelement.number == old_value
for de in valueset.parameter.domain:
if de.number == new_value:
value.domainelement = de
break
assert value.domainelement.number == new_value
valueset.updated = E2011
value.updated = E2011
VersionedDBSession.flush()
for row in reader(args.data_file('corrections_2013.tab'), namedtuples=True, newline='\r'):
valueset = VersionedDBSession.query(common.ValueSet)\
.join(common.Language)\
.join(common.Parameter)\
.filter(common.Parameter.id == row.feature)\
.filter(common.Language.id == row.wals_code)\
.one()
value = valueset.values[0]
if value.domainelement.number == int(row.new):
print '**** old news', valueset.language.id, valueset.parameter.id
continue
if value.domainelement.number != int(row.old):
print '--->', valueset.language.id, valueset.parameter.id, value.domainelement.number
for de in valueset.parameter.domain:
if de.number == int(row.new):
value.domainelement = de
break
assert value.domainelement.number == int(row.new)
valueset.updated = E2013
value.updated = E2013
VersionedDBSession.flush()
print 'corrections 2013 done'
for issue in ['0', '9', '10', '11', '13', '14', '15', '16', '17', '19', '20', '24', '26', '27', '28']:
issue = getattr(issues, 'issue' + issue)
issue(VersionedDBSession, E2013)
VersionedDBSession.flush()
transaction.commit()
transaction.begin()
#
# TODO: these must be recomputed as well, after migrations!
#
# cache number of languages for a parameter:
for parameter, valuesets in groupby(
DBSession.query(common.ValueSet).order_by(common.ValueSet.parameter_pk),
lambda vs: vs.parameter):
parameter.representation = str(len(set(v.language_pk for v in valuesets)))
print 'recomputation of representation done'
transaction.commit()
transaction.begin()
# cache iso codes for languages:
for language in DBSession.query(common.Language).options(joinedload_all(
common.Language.languageidentifier, common.LanguageIdentifier.identifier
)):
iso_codes = []
for identifier in language.identifiers:
if identifier.type == common.IdentifierType.iso.value:
iso_codes.append(identifier.name)
language.iso_codes = ', '.join(sorted(set(iso_codes)))
print 'recomputation of iso codes done'
transaction.commit()
transaction.begin()
compute_language_sources()
transaction.commit()
transaction.begin()
gbs_func('update', args)
def prime_cache(args): # pragma: no cover
"""
we use a versioned session to insert the changes in value assignment
"""
#
# compute the changes from 2008 to 2011:
#
vs2008 = get_vs2008(args)
for row in DB.execute("select * from datapoint"):
key = (row['language_id'], row['feature_id'])
old_value = vs2008.get(key)
new_value = row['value_numeric']
if old_value and old_value != new_value:
valueset = VersionedDBSession.query(common.ValueSet)\
.join(common.Language)\
.join(common.Parameter)\
.filter(common.Parameter.id == row['feature_id'])\
.filter(common.Language.id == row['language_id'])\
.one()
value = valueset.values[0]
assert value.domainelement.number == old_value
for de in valueset.parameter.domain:
if de.number == new_value:
value.domainelement = de
break
assert value.domainelement.number == new_value
valueset.updated = E2011
value.updated = E2011
VersionedDBSession.flush()
for row in reader(args.data_file('corrections_2013.tab'),
namedtuples=True,
newline='\r'):
valueset = VersionedDBSession.query(common.ValueSet)\
.join(common.Language)\
.join(common.Parameter)\
.filter(common.Parameter.id == row.feature)\
.filter(common.Language.id == row.wals_code)\
.one()
value = valueset.values[0]
if value.domainelement.number == int(row.new):
print('**** old news', valueset.language.id, valueset.parameter.id)
continue
if value.domainelement.number != int(row.old):
print('--->', valueset.language.id, valueset.parameter.id,
value.domainelement.number)
for de in valueset.parameter.domain:
if de.number == int(row.new):
value.domainelement = de
break
assert value.domainelement.number == int(row.new)
valueset.updated = E2013
value.updated = E2013
VersionedDBSession.flush()
print('corrections 2013 done')
for issue in [
'0', '9', '10', '11', '13', '14', '15', '16', '17', '19', '20',
'24', '26', '27', '28'
]:
issue = getattr(issues, 'issue' + issue)
issue(VersionedDBSession, E2013)
VersionedDBSession.flush()
transaction.commit()
transaction.begin()
#
# TODO: these must be recomputed as well, after migrations!
#
# cache number of languages for a parameter:
for parameter, valuesets in groupby(
DBSession.query(common.ValueSet).order_by(
common.ValueSet.parameter_pk), lambda vs: vs.parameter):
parameter.representation = str(
len(set(v.language_pk for v in valuesets)))
print('recomputation of representation done')
transaction.commit()
transaction.begin()
# cache iso codes for languages:
for language in DBSession.query(common.Language).options(
joinedload_all(common.Language.languageidentifier,
common.LanguageIdentifier.identifier)):
iso_codes = []
for identifier in language.identifiers:
if identifier.type == common.IdentifierType.iso.value:
iso_codes.append(identifier.name)
language.iso_codes = ', '.join(sorted(set(iso_codes)))
print('ecomputation of iso codes done')
transaction.commit()
transaction.begin()
compute_language_sources()
transaction.commit()
transaction.begin()
gbs_func('update', args)
