def init_db(): engine = create_engine('sqlite://') DBSession.configure(bind=engine) VersionedDBSession.configure(bind=engine) Base.metadata.bind = engine Base.metadata.create_all() return engine
def testapp(): from webtest import TestApp from clld.db.meta import DBSession, VersionedDBSession, Base from clld.db.models import common from clld_cognacy_plugin.models import Cognateset, Cognate def main(): cfg = config.Configurator(settings={ 'sqlalchemy.url': 'sqlite://', 'mako.directories': [ 'clld:web/templates', 'clld_cognacy_plugin:templates' ]}) cfg.include('clld.web.app') cfg.include('clld_cognacy_plugin') return cfg.make_wsgi_app() DBSession.remove() VersionedDBSession.remove() wsgi_app = main() Base.metadata.bind = DBSession.bind Base.metadata.create_all() DBSession.add(common.Dataset(id='1', name='test app', domain='example.org')) cs = Cognateset(id='1', name='cs: test') lang = common.Language(id='l', latitude=2, longitude=2) param = common.Parameter(id='l') vs = common.ValueSet(id='vs', language=lang, parameter=param) v = common.Value(id='v', name='abc', valueset=vs) DBSession.add(Cognate(cognateset=cs, counterpart=v)) yield TestApp(wsgi_app)
def setup_session(config_uri, engine=None): setup_logging(config_uri) settings = get_appsettings(config_uri) engine = engine or engine_from_config(settings, "sqlalchemy.") DBSession.configure(bind=engine) VersionedDBSession.configure(bind=engine) Base.metadata.create_all(engine) return Path(config_uri.split("#")[0]).resolve().parent.name
def setup_session(config_uri, engine=None): setup_logging(config_uri) settings = get_appsettings(config_uri) engine = engine or engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) VersionedDBSession.configure(bind=engine) Base.metadata.create_all(engine) return path(config_uri.split('#')[0]).abspath().dirname().basename()
def setUp(self): from clld.tests.fixtures import CustomLanguage assert CustomLanguage engine = create_engine('sqlite://') DBSession.configure(bind=engine) VersionedDBSession.configure(bind=engine) Base.metadata.bind = engine Base.metadata.create_all()
def test_Base_jsondata(db): l = Language(id='abc', name='Name') VersionedDBSession.add(l) VersionedDBSession.flush() l.update_jsondata(a=1) assert 'a' in l.jsondata l.update_jsondata(b=1) assert 'b' in l.jsondata and 'a' in l.jsondata assert 'b' in l.__json__(None)['jsondata']
def test_Base_jsondata(self): l = Language(id='abc', name='Name') VersionedDBSession.add(l) VersionedDBSession.flush() l.update_jsondata(a=1) self.assertTrue('a' in l.jsondata) l.update_jsondata(b=1) self.assertTrue('b' in l.jsondata and 'a' in l.jsondata) self.assertTrue('b' in l.__json__(None)['jsondata'])
def data(settings): from clld.db.meta import Base, DBSession, VersionedDBSession engine = sa.engine_from_config(settings) Base.metadata.create_all(bind=engine) DBSession.configure(bind=engine) VersionedDBSession.configure(bind=engine) yield engine DBSession.close() VersionedDBSession.close()
def main(args): with transaction.manager: param = VersionedDBSession.query(common.Parameter) \ .filter(common.Parameter.id == '138A') \ .one() for de in param.domain: for k, v in VALUES.items(): if v == de.name: VALUES[k] = de for lid, value in [('lat', 'te'), ('dhi', 'cha'), ('chp', 'te')]: vs = VersionedDBSession.query(common.ValueSet)\ .join(common.Language)\ .filter(common.Language.id == lid)\ .filter(common.ValueSet.parameter_pk == param.pk)\ .one() de = VALUES[value] assert vs.values[0].domainelement != de vs.values[0].domainelement = de
def test_Base(db): l = Language(id='abc', name='Name') VersionedDBSession.add(l) VersionedDBSession.flush() VersionedDBSession.expunge(l) l = Language.get('abc', session=VersionedDBSession) assert l.name == 'Name' assert not list(l.history()) Language().__str__() assert repr(l) == "<Language 'abc'>"
def test_Base(self): l = Language(id='abc', name='Name') VersionedDBSession.add(l) VersionedDBSession.flush() VersionedDBSession.expunge(l) l = Language.get('abc') self.assertEqual(l.name, 'Name') assert not list(l.history()) # a bit of a hack to test the human readable representations. # we exploit the fact, that on py2, string and unicode comparison does type # coercion, while on py3, the two methods should actually return the same string. self.assertEqual(l.__str__(), l.__unicode__()) Language().__str__()
def test_Base(db): l = Language(id='abc', name='Name') VersionedDBSession.add(l) VersionedDBSession.flush() VersionedDBSession.expunge(l) l = Language.get('abc', session=VersionedDBSession) assert l.name == 'Name' assert not list(l.history()) # a bit of a hack to test the human readable representations. # we exploit the fact, that on py2, string and unicode comparison does type # coercion, while on py3, the two methods should actually return the same string. assert l.__str__() == l.__unicode__() Language().__str__() assert repr(l) == "<Language 'abc'>" if PY3 else "<Language u'abc'>"
def test_Base(self): l = Language(id='abc', name='Name') VersionedDBSession.add(l) VersionedDBSession.flush() VersionedDBSession.expunge(l) #print('pk: %s' % l.pk) #transaction.commit() #transaction.begin() #l = VersionedDBSession.query(Language).get(1) #print(l) #l.name = 'New name' #print('pk: %s' % l.pk) #transaction.commit() #transaction.begin() l = Language.get('abc') #print(l.version) self.assertEqual(l.name, 'Name') l.history() # a bit of a hack to test the human readable representations. # we exploit the fact, that on py2, string and unicode comparison does type # coercion, while on py3, the two methods should actualy return the same string. self.assertEqual(l.__str__(), l.__unicode__()) Language().__str__()
def db(db): try: yield db finally: DBSession.rollback() VersionedDBSession.rollback()
def prime_cache(args): # pragma: no cover """ we use a versioned session to insert the changes in value assignment """ # # compute the changes from 2008 to 2011: # vs2008 = get_vs2008(args) for row in DB.execute("select * from datapoint"): key = (row["language_id"], row["feature_id"]) old_value = vs2008.get(key) new_value = row["value_numeric"] if old_value and old_value != new_value: valueset = ( VersionedDBSession.query(common.ValueSet) .join(common.Language) .join(common.Parameter) .filter(common.Parameter.id == row["feature_id"]) .filter(common.Language.id == row["language_id"]) .one() ) value = valueset.values[0] assert value.domainelement.number == old_value for de in valueset.parameter.domain: if de.number == new_value: value.domainelement = de break assert value.domainelement.number == new_value valueset.updated = E2011 value.updated = E2011 VersionedDBSession.flush() for row in reader(args.data_file("corrections_2013.tab"), namedtuples=True, newline="\r"): valueset = ( VersionedDBSession.query(common.ValueSet) .join(common.Language) .join(common.Parameter) .filter(common.Parameter.id == row.feature) .filter(common.Language.id == row.wals_code) .one() ) value = valueset.values[0] if value.domainelement.number == int(row.new): print("**** old news", valueset.language.id, valueset.parameter.id) continue if value.domainelement.number != int(row.old): print("--->", valueset.language.id, valueset.parameter.id, value.domainelement.number) for de in valueset.parameter.domain: if de.number == int(row.new): value.domainelement = de break assert value.domainelement.number == int(row.new) valueset.updated = E2013 value.updated = E2013 VersionedDBSession.flush() print("corrections 2013 done") for issue in ["0", "9", "10", "11", "13", "14", "15", "16", "17", "19", "20", "24", "26", "27", "28"]: issue = getattr(issues, "issue" + issue) issue(VersionedDBSession, E2013) VersionedDBSession.flush() transaction.commit() transaction.begin() # # TODO: these must be recomputed as well, after migrations! # # cache number of languages for a parameter: for parameter, valuesets in groupby( DBSession.query(common.ValueSet).order_by(common.ValueSet.parameter_pk), lambda vs: vs.parameter ): parameter.representation = str(len(set(v.language_pk for v in valuesets))) print("recomputation of representation done") transaction.commit() transaction.begin() # cache iso codes for languages: for language in DBSession.query(common.Language).options( joinedload_all(common.Language.languageidentifier, common.LanguageIdentifier.identifier) ): iso_codes = [] for identifier in language.identifiers: if identifier.type == common.IdentifierType.iso.value: iso_codes.append(identifier.name) language.iso_codes = ", ".join(sorted(set(iso_codes))) print("ecomputation of iso codes done") transaction.commit() transaction.begin() compute_language_sources() transaction.commit() transaction.begin() gbs_func("update", args)
def test_Versioning(db): from clld.db.models.common import Language, Language_data from clld.db.meta import VersionedDBSession l = Language(id='abc', name='Old Name', jsondata={'i': 2}) VersionedDBSession.add(l) VersionedDBSession.flush() assert l.version == 1 l.name = 'New Name' l.description = 'New Description' VersionedDBSession.flush() assert l.version == 2 History = l.__history_mapper__.class_ res = VersionedDBSession.query(History).filter(History.pk == l.pk).all() assert res[0].name == 'Old Name' l.data.append(Language_data(key='k', value='v')) VersionedDBSession.flush() assert l.datadict() VersionedDBSession.delete(l) VersionedDBSession.flush()
def prime_cache(args): # pragma: no cover """ we use a versioned session to insert the changes in value assignment """ # # compute the changes from 2008 to 2011: # vs2008 = get_vs2008(args) for row in DB.execute("select * from datapoint"): key = (row['language_id'], row['feature_id']) old_value = vs2008.get(key) new_value = row['value_numeric'] if old_value and old_value != new_value: valueset = VersionedDBSession.query(common.ValueSet)\ .join(common.Language)\ .join(common.Parameter)\ .filter(common.Parameter.id == row['feature_id'])\ .filter(common.Language.id == row['language_id'])\ .one() value = valueset.values[0] assert value.domainelement.number == old_value for de in valueset.parameter.domain: if de.number == new_value: value.domainelement = de break assert value.domainelement.number == new_value valueset.updated = E2011 value.updated = E2011 VersionedDBSession.flush() for row in reader(args.data_file('corrections_2013.tab'), namedtuples=True, newline='\r'): valueset = VersionedDBSession.query(common.ValueSet)\ .join(common.Language)\ .join(common.Parameter)\ .filter(common.Parameter.id == row.feature)\ .filter(common.Language.id == row.wals_code)\ .one() value = valueset.values[0] if value.domainelement.number == int(row.new): print('**** old news', valueset.language.id, valueset.parameter.id) continue if value.domainelement.number != int(row.old): print('--->', valueset.language.id, valueset.parameter.id, value.domainelement.number) for de in valueset.parameter.domain: if de.number == int(row.new): value.domainelement = de break assert value.domainelement.number == int(row.new) valueset.updated = E2013 value.updated = E2013 VersionedDBSession.flush() print('corrections 2013 done') for issue in [ '0', '9', '10', '11', '13', '14', '15', '16', '17', '19', '20', '24', '26', '27', '28' ]: issue = getattr(issues, 'issue' + issue) issue(VersionedDBSession, E2013) VersionedDBSession.flush() transaction.commit() transaction.begin() # # TODO: these must be recomputed as well, after migrations! # # cache number of languages for a parameter: for parameter, valuesets in groupby( DBSession.query(common.ValueSet).order_by( common.ValueSet.parameter_pk), lambda vs: vs.parameter): parameter.representation = str( len(set(v.language_pk for v in valuesets))) print('recomputation of representation done') transaction.commit() transaction.begin() # cache iso codes for languages: for language in DBSession.query(common.Language).options( joinedload_all(common.Language.languageidentifier, common.LanguageIdentifier.identifier)): iso_codes = [] for identifier in language.identifiers: if identifier.type == common.IdentifierType.iso.value: iso_codes.append(identifier.name) language.iso_codes = ', '.join(sorted(set(iso_codes))) print('ecomputation of iso codes done') transaction.commit() transaction.begin() compute_language_sources() transaction.commit() transaction.begin() gbs_func('update', args)
def test_Versioning(self): from clld.db.models.common import Language, Identifier, LanguageIdentifier from clld.db.meta import VersionedDBSession l = Language(id='abc', name='Old Name', jsondata={'i': 2}) VersionedDBSession.add(l) VersionedDBSession.flush() self.assertEqual(l.version, 1) l.name = 'New Name' l.description = 'New Description' VersionedDBSession.flush() self.assertEqual(l.version, 2) History = l.__history_mapper__.class_ res = VersionedDBSession.query(History).filter( History.pk == l.pk).all() self.assertEqual(res[0].name, 'Old Name') li = LanguageIdentifier(identifier=Identifier(id='asd', type='wals'), language=l) VersionedDBSession.flush() VersionedDBSession.delete(li) VersionedDBSession.delete(l) VersionedDBSession.flush()
def test_Versioning(self): from clld.db.models.common import Language, Identifier, LanguageIdentifier from clld.db.meta import VersionedDBSession l = Language(id='abc', name='Old Name', jsondata={'i': 2}) VersionedDBSession.add(l) VersionedDBSession.flush() self.assertEqual(l.version, 1) l.name = 'New Name' l.description = 'New Description' VersionedDBSession.flush() self.assertEqual(l.version, 2) History = l.__history_mapper__.class_ res = VersionedDBSession.query(History).filter(History.pk == l.pk).all() self.assertEqual(res[0].name, 'Old Name') li = LanguageIdentifier( identifier=Identifier(id='asd', type='wals'), language=l) VersionedDBSession.flush() VersionedDBSession.delete(li) VersionedDBSession.delete(l) VersionedDBSession.flush()
def test_Versioning(self): from clld.db.models.common import Language, Language_data from clld.db.meta import VersionedDBSession l = Language(id='abc', name='Old Name', jsondata={'i': 2}) VersionedDBSession.add(l) VersionedDBSession.flush() self.assertEqual(l.version, 1) l.name = 'New Name' l.description = 'New Description' VersionedDBSession.flush() self.assertEqual(l.version, 2) History = l.__history_mapper__.class_ res = VersionedDBSession.query(History).filter(History.pk == l.pk).all() self.assertEqual(res[0].name, 'Old Name') l.data.append(Language_data(key='k', value='v')) VersionedDBSession.flush() assert l.datadict() VersionedDBSession.delete(l) VersionedDBSession.flush()
def prime_cache(args): """ we use a versioned session to insert the changes in value assignment """ # # compute the changes from 2008 to 2011: # vs2008 = get_vs2008(args) for row in DB.execute("select * from datapoint"): key = (row['language_id'], row['feature_id']) old_value = vs2008.get(key) new_value = row['value_numeric'] if old_value and old_value != new_value: valueset = VersionedDBSession.query(common.ValueSet)\ .join(common.Language)\ .join(common.Parameter)\ .filter(common.Parameter.id == row['feature_id'])\ .filter(common.Language.id == row['language_id'])\ .one() value = valueset.values[0] assert value.domainelement.number == old_value for de in valueset.parameter.domain: if de.number == new_value: value.domainelement = de break assert value.domainelement.number == new_value valueset.updated = E2011 value.updated = E2011 VersionedDBSession.flush() for row in reader(args.data_file('corrections_2013.tab'), namedtuples=True, newline='\r'): valueset = VersionedDBSession.query(common.ValueSet)\ .join(common.Language)\ .join(common.Parameter)\ .filter(common.Parameter.id == row.feature)\ .filter(common.Language.id == row.wals_code)\ .one() value = valueset.values[0] if value.domainelement.number == int(row.new): print '**** old news', valueset.language.id, valueset.parameter.id continue if value.domainelement.number != int(row.old): print '--->', valueset.language.id, valueset.parameter.id, value.domainelement.number for de in valueset.parameter.domain: if de.number == int(row.new): value.domainelement = de break assert value.domainelement.number == int(row.new) valueset.updated = E2013 value.updated = E2013 VersionedDBSession.flush() print 'corrections 2013 done' for issue in ['0', '9', '10', '11', '13', '14', '15', '16', '17', '19', '20', '24', '26', '27', '28']: issue = getattr(issues, 'issue' + issue) issue(VersionedDBSession, E2013) VersionedDBSession.flush() transaction.commit() transaction.begin() # # TODO: these must be recomputed as well, after migrations! # # cache number of languages for a parameter: for parameter, valuesets in groupby( DBSession.query(common.ValueSet).order_by(common.ValueSet.parameter_pk), lambda vs: vs.parameter): parameter.representation = str(len(set(v.language_pk for v in valuesets))) print 'recomputation of representation done' transaction.commit() transaction.begin() # cache iso codes for languages: for language in DBSession.query(common.Language).options(joinedload_all( common.Language.languageidentifier, common.LanguageIdentifier.identifier )): iso_codes = [] for identifier in language.identifiers: if identifier.type == common.IdentifierType.iso.value: iso_codes.append(identifier.name) language.iso_codes = ', '.join(sorted(set(iso_codes))) print 'recomputation of iso codes done' transaction.commit() transaction.begin() compute_language_sources() transaction.commit() transaction.begin() gbs_func('update', args)