def tearDown(self, **kwargs):
clear_all_tables = kwargs.get('clear_all_tables', False)
dirs_to_clear = kwargs.get('dirs_to_clear', [])
del_global_app_set = kwargs.get('del_global_app_set', False)
dirs_to_destroy = kwargs.get('dirs_to_destroy', [])
if clear_all_tables:
h.clear_all_tables(['language'])
else:
h.clear_all_models()
administrator = h.generate_default_administrator()
contributor = h.generate_default_contributor()
viewer = h.generate_default_viewer()
Session.add_all([administrator, contributor, viewer])
Session.commit()
for dir_path in dirs_to_clear:
h.clear_directory_of_files(getattr(self, dir_path))
for dir_name in dirs_to_destroy:
{
'user': lambda: h.destroy_all_directories('users', 'test.ini'),
'corpus': lambda: h.destroy_all_directories('corpora', 'test.ini'),
'phonology': lambda: h.destroy_all_directories('phonologies', 'test.ini'),
'morphology': lambda: h.destroy_all_directories('morphologies', 'test.ini'),
'morphological_parser': lambda: h.destroy_all_directories('morphological_parsers', 'test.ini'),
'morpheme_language_model': lambda: h.destroy_all_directories('morpheme_language_models', 'test.ini')
}.get(dir_name, lambda: None)()
if del_global_app_set:
# Perform a vacuous GET just to delete app_globals.application_settings
# to clean up for subsequent tests.
self.app.get(url('new_form'), extra_environ=self.extra_environ_admin_appset)
def test_e_cleanup(self):
"""Clean up the database after /rememberedforms tests."""
h.clear_all_models()
administrator = h.generate_default_administrator()
contributor = h.generate_default_contributor()
viewer = h.generate_default_viewer()
Session.add_all([administrator, contributor, viewer])
Session.commit()
# Perform a vacuous GET just to delete app_globals.application_settings
# to clean up for subsequent tests.
extra_environ = {'test.authentication.role': u'administrator',
'test.application_settings': True}
self.app.get(url('forms'), extra_environ=extra_environ)
def test_e_cleanup(self):
"""Clean up the database after /rememberedforms tests."""
h.clear_all_models()
administrator = h.generate_default_administrator()
contributor = h.generate_default_contributor()
viewer = h.generate_default_viewer()
Session.add_all([administrator, contributor, viewer])
Session.commit()
# Perform a vacuous GET just to delete app_globals.application_settings
# to clean up for subsequent tests.
extra_environ = {
'test.authentication.role': u'administrator',
'test.application_settings': True
}
self.app.get(url('forms'), extra_environ=extra_environ)
def test_a_initialize(self):
"""Initialize the database for /rememberedforms tests."""
h.clear_all_models()
administrator = h.generate_default_administrator()
contributor = h.generate_default_contributor()
viewer = h.generate_default_viewer()
Session.add_all([administrator, contributor, viewer])
Session.commit()
_create_test_data(self.n)
self._add_SEARCH_to_web_test_valid_methods()
# Create an application settings where the contributor is unrestricted
viewer, contributor, administrator = get_users()
application_settings = h.generate_default_application_settings()
application_settings.unrestricted_users = [contributor]
Session.add(application_settings)
Session.commit()
def test_a_initialize(self):
"""Initialize the database for /rememberedforms tests."""
h.clear_all_models()
administrator = h.generate_default_administrator()
contributor = h.generate_default_contributor()
viewer = h.generate_default_viewer()
Session.add_all([administrator, contributor, viewer])
Session.commit()
_create_test_data(self.n)
self._add_SEARCH_to_web_test_valid_methods()
# Create an application settings where the contributor is unrestricted
viewer, contributor, administrator = get_users()
application_settings = h.generate_default_application_settings()
application_settings.unrestricted_users = [contributor]
Session.add(application_settings)
Session.commit()
def tearDown(self, **kwargs):
clear_all_tables = kwargs.get('clear_all_tables', False)
dirs_to_clear = kwargs.get('dirs_to_clear', [])
del_global_app_set = kwargs.get('del_global_app_set', False)
dirs_to_destroy = kwargs.get('dirs_to_destroy', [])
if clear_all_tables:
h.clear_all_tables(['language'])
else:
h.clear_all_models()
administrator = h.generate_default_administrator()
contributor = h.generate_default_contributor()
viewer = h.generate_default_viewer()
Session.add_all([administrator, contributor, viewer])
Session.commit()
for dir_path in dirs_to_clear:
h.clear_directory_of_files(getattr(self, dir_path))
for dir_name in dirs_to_destroy:
{
'user':
lambda: h.destroy_all_directories('users', 'test.ini'),
'corpus':
lambda: h.destroy_all_directories('corpora', 'test.ini'),
'phonology':
lambda: h.destroy_all_directories('phonologies', 'test.ini'),
'morphology':
lambda: h.destroy_all_directories('morphologies', 'test.ini'),
'morphological_parser':
lambda: h.destroy_all_directories('morphological_parsers',
'test.ini'),
'morpheme_language_model':
lambda: h.destroy_all_directories('morpheme_language_models',
'test.ini')
}.get(dir_name, lambda: None)()
if del_global_app_set:
# Perform a vacuous GET just to delete app_globals.application_settings
# to clean up for subsequent tests.
self.app.get(url('new_form'),
extra_environ=self.extra_environ_admin_appset)
def test_aaa_initialize(self):
"""Initialize the database using pseudo-data generated from random lorem ipsum sentences.
These are located in ``onlinelinguisticdatabase/tests/data/corpora``.
The data contain morphologically analyzed sentences, their component
morphemes, and syntactic categories. The sentences have phrase
structure trees in bracket notation.
The test will try to load the lorem ipsum dataset from a MySQL/SQLite
dump file in ``onlinelinguisticdatabase/tests/data/corpora``. If the
dump file corresponding to ``loremipsum_path`` does not exist, it will
import the lorem ipsum data directly from the text files and create
the dump file so that future tests can run more speedily. The
``loremipsum100_path``, ``loremipsum1000_path``, ``loremipsum10000_path``
and ``loremipsum30000_path`` files are available and contain 100, 1000
and 10,000 sentences, respectively.
Setting the ``via_request`` variable to ``True`` will cause all of the
forms to be created via request, i.e., via
``self.app.post(url('forms))...``. This is much slower but may be
desirable since values for the morphological analysis attributes
will be generated.
.. note::
In order to run ``mysqldump`` with the MySQL user listed in
``test.ini``, that user must have permission to lock and update
tables::
mysql -u root -p<root_password>
grant lock tables, update on old_test.* to 'old'@'localhost';
.. warning::
Loading the .txt or .sql files with the ``via_request`` option set to
``True`` will take a very long time. This might be an argument for
separating the interface and logic components of the controllers so
that a "core" HTTP-less OLD application could be exposed. This
would facilitate the creation of models with system-generated data
and validation but without the HTTP overhead...
"""
########################################################################
# Configure lorem ipsum data set import
########################################################################
# Set ``loremipsum_path`` this to ``self.loremipsum100_path``,
# ``self.loremipsum1000_path`` or ``self.loremipsum10000_path``.
# WARNING: the larger ones will take a long time.
# Use the 10,000-sentence lorem ipsum dataset to ensure that
# very large corpora are handled correctly.
loremipsum_path = self.loremipsum100_path
# Set ``via_request`` to ``True`` to create all forms via HTTP requests.
via_request = True
self._add_SEARCH_to_web_test_valid_methods()
# Add an application settings so that morpheme references will work out right.
application_settings = h.generate_default_application_settings()
Session.add(application_settings)
Session.commit()
def create_model(line, categories, via_request=False):
"""Create a model (form or syncat) using the string in ``line``."""
model = 'Form'
elements = unicode(line).split('\t')
non_empty_elements = filter(None, elements)
try:
ol, mb, mg, ml, sc, sx = non_empty_elements
except Exception:
try:
ol, mb, mg, ml, sc = non_empty_elements
sx = u''
except Exception:
try:
model = 'SyntacticCategory'
n, t = non_empty_elements
except Exception:
return categories
if via_request:
if model == 'SyntacticCategory':
params = self.syntactic_category_create_params.copy()
params.update({
'name': n,
'type': t
})
params = json.dumps(params)
response = self.app.post(url('syntacticcategories'), params, self.json_headers,
self.extra_environ_admin)
cat_id = json.loads(response.body)['id']
categories[n] = cat_id
else:
params = self.form_create_params.copy()
params.update({
'transcription': ol,
'morpheme_break': mb,
'morpheme_gloss': mg,
'translations': [{'transcription': ml, 'grammaticality': u''}],
'syntax': sx,
'syntactic_category': categories.get(sc, u'')
})
params = json.dumps(params)
self.app.post(url('forms'), params, self.json_headers,
self.extra_environ_admin)
else:
if model == 'SyntacticCategory':
syntactic_category = model.SyntacticCategory()
syntactic_category.name = n
syntactic_category.type = t
Session.add(syntactic_category)
categories[n] = syntactic_category.id
else:
form = model.Form()
form.transcription = ol
form.morpheme_break = mb
form.morpheme_gloss = mg
translation = model.Translation()
translation.transcription = ml
form.translations.append(translation)
form.syntax = sx
form.syntacticcategory_id = categories.get(sc, None)
Session.add(form)
return categories
def add_loremipsum_to_db(loremipsum_path, via_request=False):
"""Add the contents of the file at ``loremipsum_path`` to the database."""
categories = {}
with open(loremipsum_path, 'r') as f:
i = 0
for l in f:
if i % 100 == 0:
if not via_request: Session.commit()
log.debug('%d lines processed' % i)
i = i + 1
categories = create_model(l.replace('\n', ''), categories,
via_request)
Session.commit()
loremipsum_path_no_ext = os.path.splitext(loremipsum_path)[0]
sqlalchemy_URL = self.config['sqlalchemy.url']
sqlalchemy_URL_list = sqlalchemy_URL.split(':')
olddump_script_path = os.path.join(self.test_scripts_path, 'olddump.sh')
oldload_script_path = os.path.join(self.test_scripts_path, 'oldload.sh')
RDBMS = sqlalchemy_URL_list[0]
if RDBMS == 'mysql':
mysql_dump_path = '%s_mysql.sql' % loremipsum_path_no_ext
username = sqlalchemy_URL_list[1][2:]
password = sqlalchemy_URL_list[2].split('@')[0]
dbname = sqlalchemy_URL_list[3].split('/')[1]
if os.path.exists(mysql_dump_path):
log.debug('The lorem ipsum MySQL dump file exists. Loading it...')
# Clear the current DB completely
h.clear_all_models(retain=[])
# Load the dump file to the DB
shell_script = '#!/bin/sh\nmysql -u %s -p%s %s < %s' % (
username, password, dbname, mysql_dump_path)
with open(oldload_script_path, 'w') as f:
f.write(shell_script)
os.chmod(oldload_script_path, 0744)
# Load the DB
with open(os.devnull, 'w') as f:
call([oldload_script_path], stdout=f, stderr=f)
# Destroy the load script
os.remove(oldload_script_path)
log.debug('Loaded.')
else:
log.debug('Have to import the lorem ipsum dataset from the text file and create the MySQL dump file.')
# Populate the database from the loremipusm text file and dump it
add_loremipsum_to_db(loremipsum_path, via_request=via_request)
# Write the DB dump shell script
shell_script = '#!/bin/sh\nmysqldump -u %s -p%s --no-create-info %s > %s' % (
username, password, dbname, mysql_dump_path)
with open(olddump_script_path, 'w') as f:
f.write(shell_script)
os.chmod(olddump_script_path, 0744)
# Dump the DB
with open(os.devnull, 'w') as f:
call([olddump_script_path], stdout=f, stderr=f)
# Destroy the dump script
os.remove(olddump_script_path)
log.debug('Imported and dumped.')
elif RDBMS == 'sqlite' and h.command_line_program_installed('sqlite3'):
sqlite_dump_path = '%s_sqlite.sql' % loremipsum_path_no_ext
sqlite_db = sqlalchemy_URL.split('/')[-1]
dbpath = os.path.join(self.here, sqlite_db)
if os.path.exists(sqlite_dump_path):
log.debug('The lorem ipsum SQLite dump file exists. Loading it...')
# Clear the current DB completely
h.clear_all_models(retain=[])
# Load the dump file to the DB
shell_script = '#!/bin/sh\nsqlite3 %s < %s' % (
dbpath, sqlite_dump_path)
with open(oldload_script_path, 'w') as f:
f.write(shell_script)
os.chmod(oldload_script_path, 0744)
# Load the DB
with open(os.devnull, 'w') as f:
call([oldload_script_path], stdout=f, stderr=f)
# Destroy the load script
os.remove(oldload_script_path)
log.debug('Loaded.')
else:
log.debug('Have to import the lorem ipsum dataset from the text file and create the SQLite dump file.')
# Populate the database from the loremipusm text file and dump it
add_loremipsum_to_db(loremipsum_path, via_request=via_request)
# Write the DB dump shell script
shell_script = '#!/bin/sh\nsqlite3 %s ".dump" | grep -v "^CREATE" > %s' % (
dbpath, sqlite_dump_path)
with open(olddump_script_path, 'w') as f:
f.write(shell_script)
os.chmod(olddump_script_path, 0744)
# Dump the DB
with open(os.devnull, 'w') as f:
call([olddump_script_path], stdout=f, stderr=f)
# Destroy the dump script
os.remove(olddump_script_path)
log.debug('Imported and dumped.')
forms = h.get_forms()
log.debug('Lorem ipsum data loaded. There are now %d forms in the db.' % len(forms))
# Restrict one sentential form in the db.
restricted_tag = h.generate_restricted_tag()
Session.add(restricted_tag)
Session.commit()
a_form = Session.query(model.Form).\
filter(model.Form.syntactic_category.\
has(model.SyntacticCategory.name==u'S')).first()
a_form_id = a_form.id
a_form.tags.append(restricted_tag)
Session.commit()
restricted_form = Session.query(model.Form).\
filter(model.Form.tags.any(model.Tag.name==u'restricted')).first()
assert a_form_id == restricted_form.id
def test_aaa_initialize(self):
"""Initialize the database using pseudo-data generated from random lorem ipsum sentences.
These are located in ``onlinelinguisticdatabase/tests/data/corpora``.
The data contain morphologically analyzed sentences, their component
morphemes, and syntactic categories. The sentences have phrase
structure trees in bracket notation.
The test will try to load the lorem ipsum dataset from a MySQL/SQLite
dump file in ``onlinelinguisticdatabase/tests/data/corpora``. If the
dump file corresponding to ``loremipsum_path`` does not exist, it will
import the lorem ipsum data directly from the text files and create
the dump file so that future tests can run more speedily. The
``loremipsum100_path``, ``loremipsum1000_path``, ``loremipsum10000_path``
and ``loremipsum30000_path`` files are available and contain 100, 1000
and 10,000 sentences, respectively.
Setting the ``via_request`` variable to ``True`` will cause all of the
forms to be created via request, i.e., via
``self.app.post(url('forms))...``. This is much slower but may be
desirable since values for the morphological analysis attributes
will be generated.
.. note::
In order to run ``mysqldump`` with the MySQL user listed in
``test.ini``, that user must have permission to lock and update
tables (alter and file privileges may also be required ...)::
mysql -u root -p<root_password>
grant lock tables, update on old_test.* to 'old'@'localhost';
.. warning::
Loading the .txt or .sql files with the ``via_request`` option set to
``True`` will take a very long time. This might be an argument for
separating the interface and logic components of the controllers so
that a "core" HTTP-less OLD application could be exposed. This
would facilitate the creation of models with system-generated data
and validation but without the HTTP overhead...
"""
########################################################################
# Configure lorem ipsum data set import
########################################################################
# Set ``loremipsum_path`` this to ``self.loremipsum100_path``,
# ``self.loremipsum1000_path`` or ``self.loremipsum10000_path``.
# WARNING: the larger ones will take a long time.
# Use the 10,000-sentence lorem ipsum dataset to ensure that
# very large corpora are handled correctly.
loremipsum_path = self.loremipsum100_path
# Set ``via_request`` to ``True`` to create all forms via HTTP requests.
via_request = True
self._add_SEARCH_to_web_test_valid_methods()
# Add an application settings so that morpheme references will work out right.
application_settings = h.generate_default_application_settings()
Session.add(application_settings)
Session.commit()
def create_model(line, categories, via_request=False):
"""Create a model (form or syncat) using the string in ``line``."""
model = 'Form'
elements = unicode(line).split('\t')
non_empty_elements = filter(None, elements)
try:
ol, mb, mg, ml, sc, sx = non_empty_elements
except Exception:
try:
ol, mb, mg, ml, sc = non_empty_elements
sx = u''
except Exception:
try:
model = 'SyntacticCategory'
n, t = non_empty_elements
except Exception:
return categories
if via_request:
if model == 'SyntacticCategory':
params = self.syntactic_category_create_params.copy()
params.update({'name': n, 'type': t})
params = json.dumps(params)
response = self.app.post(url('syntacticcategories'),
params, self.json_headers,
self.extra_environ_admin)
cat_id = json.loads(response.body)['id']
categories[n] = cat_id
else:
params = self.form_create_params.copy()
params.update({
'transcription':
ol,
'morpheme_break':
mb,
'morpheme_gloss':
mg,
'translations': [{
'transcription': ml,
'grammaticality': u''
}],
'syntax':
sx,
'syntactic_category':
categories.get(sc, u'')
})
params = json.dumps(params)
self.app.post(url('forms'), params, self.json_headers,
self.extra_environ_admin)
else:
if model == 'SyntacticCategory':
syntactic_category = model.SyntacticCategory()
syntactic_category.name = n
syntactic_category.type = t
Session.add(syntactic_category)
categories[n] = syntactic_category.id
else:
form = model.Form()
form.transcription = ol
form.morpheme_break = mb
form.morpheme_gloss = mg
translation = model.Translation()
translation.transcription = ml
form.translations.append(translation)
form.syntax = sx
form.syntacticcategory_id = categories.get(sc, None)
Session.add(form)
return categories
def add_loremipsum_to_db(loremipsum_path, via_request=False):
"""Add the contents of the file at ``loremipsum_path`` to the database."""
categories = {}
with open(loremipsum_path, 'r') as f:
i = 0
for l in f:
if i % 100 == 0:
if not via_request: Session.commit()
log.debug('%d lines processed' % i)
i = i + 1
categories = create_model(l.replace('\n', ''), categories,
via_request)
Session.commit()
loremipsum_path_no_ext = os.path.splitext(loremipsum_path)[0]
sqlalchemy_URL = self.config['sqlalchemy.url']
sqlalchemy_URL_list = sqlalchemy_URL.split(':')
olddump_script_path = os.path.join(self.test_scripts_path,
'olddump.sh')
oldload_script_path = os.path.join(self.test_scripts_path,
'oldload.sh')
RDBMS = sqlalchemy_URL_list[0]
if RDBMS == 'mysql':
mysql_dump_path = '%s_mysql.sql' % loremipsum_path_no_ext
username = sqlalchemy_URL_list[1][2:]
password = sqlalchemy_URL_list[2].split('@')[0]
dbname = sqlalchemy_URL_list[3].split('/')[1]
if os.path.exists(mysql_dump_path):
log.debug(
'The lorem ipsum MySQL dump file exists. Loading it...')
# Clear the current DB completely
h.clear_all_models(retain=[])
# Load the dump file to the DB
shell_script = '#!/bin/sh\nmysql -u %s -p%s %s < %s' % (
username, password, dbname, mysql_dump_path)
with open(oldload_script_path, 'w') as f:
f.write(shell_script)
os.chmod(oldload_script_path, 0744)
# Load the DB
with open(os.devnull, 'w') as f:
call([oldload_script_path], stdout=f, stderr=f)
# Destroy the load script
os.remove(oldload_script_path)
log.debug('Loaded.')
else:
log.debug(
'Have to import the lorem ipsum dataset from the text file and create the MySQL dump file.'
)
# Populate the database from the loremipusm text file and dump it
add_loremipsum_to_db(loremipsum_path, via_request=via_request)
# Write the DB dump shell script
# Note: the --single-transaction option seems to be required (on Mac MySQL 5.6 using InnoDB tables ...)
# see http://forums.mysql.com/read.php?10,108835,112951#msg-112951
shell_script = '#!/bin/sh\nmysqldump -u %s -p%s --single-transaction --no-create-info --result-file=%s %s' % (
username, password, mysql_dump_path, dbname)
with open(olddump_script_path, 'w') as f:
f.write(shell_script)
os.chmod(olddump_script_path, 0744)
# Dump the DB
with open(os.devnull, 'w') as f:
call([olddump_script_path], stdout=f, stderr=f)
# Destroy the dump script
os.remove(olddump_script_path)
log.debug('Imported and dumped.')
elif RDBMS == 'sqlite' and h.command_line_program_installed('sqlite3'):
sqlite_dump_path = '%s_sqlite.sql' % loremipsum_path_no_ext
sqlite_db = sqlalchemy_URL.split('/')[-1]
dbpath = os.path.join(self.here, sqlite_db)
if os.path.exists(sqlite_dump_path):
log.debug(
'The lorem ipsum SQLite dump file exists. Loading it...')
# Clear the current DB completely
h.clear_all_models(retain=[])
# Load the dump file to the DB
shell_script = '#!/bin/sh\nsqlite3 %s < %s' % (
dbpath, sqlite_dump_path)
with open(oldload_script_path, 'w') as f:
f.write(shell_script)
os.chmod(oldload_script_path, 0744)
# Load the DB
with open(os.devnull, 'w') as f:
call([oldload_script_path], stdout=f, stderr=f)
# Destroy the load script
os.remove(oldload_script_path)
log.debug('Loaded.')
else:
log.debug(
'Have to import the lorem ipsum dataset from the text file and create the SQLite dump file.'
)
# Populate the database from the loremipusm text file and dump it
add_loremipsum_to_db(loremipsum_path, via_request=via_request)
# Write the DB dump shell script
shell_script = '#!/bin/sh\nsqlite3 %s ".dump" | grep -v "^CREATE" > %s' % (
dbpath, sqlite_dump_path)
with open(olddump_script_path, 'w') as f:
f.write(shell_script)
os.chmod(olddump_script_path, 0744)
# Dump the DB
with open(os.devnull, 'w') as f:
call([olddump_script_path], stdout=f, stderr=f)
# Destroy the dump script
os.remove(olddump_script_path)
log.debug('Imported and dumped.')
forms = h.get_forms()
log.debug(
'Lorem ipsum data loaded. There are now %d forms in the db.' %
len(forms))
# Restrict one sentential form in the db.
restricted_tag = h.generate_restricted_tag()
Session.add(restricted_tag)
Session.commit()
a_form = Session.query(model.Form).\
filter(model.Form.syntactic_category.\
has(model.SyntacticCategory.name==u'S')).first()
a_form_id = a_form.id
a_form.tags.append(restricted_tag)
Session.commit()
restricted_form = Session.query(model.Form).\
filter(model.Form.tags.any(model.Tag.name==u'restricted')).first()
assert a_form_id == restricted_form.id
