def __init__(self, config_filename):
conf = appconfig('config:%s' % config_filename, relative_to='.')
load_environment(conf.global_conf, conf.local_conf)
self.config = config
self.Session = Session
self.model = model
self.h = h
def __init__(self, config_filename):
conf = appconfig('config:%s' % config_filename, relative_to='.')
load_environment(conf.global_conf, conf.local_conf)
self.config = config
self.Session = Session
self.model = model
self.h = h
def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
"""Create a Pylons WSGI application and return it
``global_conf``
The inherited configuration for this application. Normally from
the [DEFAULT] section of the Paste ini file.
``full_stack``
Whether this application provides a full WSGI stack (by default,
meaning it handles its own exceptions and errors). Disable
full_stack when this application is "managed" by another WSGI
middleware.
``static_files``
Whether this application serves its own static files; disable
when another web server is responsible for serving them.
``app_conf``
The application's local configuration. Normally specified in
the [app:<name>] section of the Paste ini file (where <name>
defaults to main).
"""
# Configure the Pylons environment
load_environment(global_conf, app_conf)
# The Pylons WSGI app
app = PylonsApp()
# Routing/Session/Cache Middleware
app = RoutesMiddleware(app, config['routes.map'])
app = SessionMiddleware(app, config)
app = CacheMiddleware(app, config)
# CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares)
if asbool(full_stack):
# Handle Python exceptions
app = ErrorHandler(app, global_conf, **config['pylons.errorware'])
# Display error documents for 401, 403, 404 status codes (and
# 500 when debug is disabled)
if asbool(config['debug']):
app = StatusCodeRedirect(app)
else:
app = StatusCodeRedirect(app, [400, 401, 403, 404, 500])
# Establish the Registry for this application
app = RegistryManager(app)
if asbool(static_files):
# Serve static files
static_app = StaticURLParser(config['pylons.paths']['static_files'])
app = Cascade([static_app, app])
return app
def make_app(global_conf, full_stack=False, static_files=True, **app_conf):
"""Create a Pylons WSGI application and return it
``global_conf``
The inherited configuration for this application. Normally from
the [DEFAULT] section of the Paste ini file.
``full_stack``
Whether this application provides a full WSGI stack (by default,
meaning it handles its own exceptions and errors). Disable
full_stack when this application is "managed" by another WSGI
middleware.
``static_files``
Whether this application serves its own static files; disable
when another web server is responsible for serving them.
``app_conf``
The application's local configuration. Normally specified in
the [app:<name>] section of the Paste ini file (where <name>
defaults to main).
"""
# Configure the Pylons environment
config = load_environment(global_conf, app_conf)
# The Pylons WSGI app
app = PylonsApp(config=config)
# Routing/Session Middleware
app = RoutesMiddleware(app, config['routes.map'], singleton=False)
app = SessionMiddleware(app, config)
# At some point it seems that Pylons converts the Content-Type of any
# response without a 200 OK status to 'text/html; charset=utf-8'. Well
# no more Pylons! The HTML2JSONContentType middleware zaps those
# nasty text/html content types and converts them to application/json!
app = HTML2JSONContentType(app)
if asbool(full_stack):
# Handle Python exceptions
app = ErrorHandler(app, global_conf, **config['pylons.errorware'])
# Display error documents for 401, 403, 404 status codes (and
# 500 when debug is disabled)
if asbool(config['debug']):
app = StatusCodeRedirect(app)
else:
app = StatusCodeRedirect(app, [400, 401, 403, 404, 500])
# Establish the Registry for this application
app = RegistryManager(app)
if asbool(static_files):
# Serve static files
static_app = StaticURLParser(config['pylons.paths']['static_files'])
app = Cascade([static_app, app])
app.config = config
return app
def setup_app(command, conf, vars):
"""Commands to setup onlinelinguisticdatabase."""
config = load_environment(conf.global_conf, conf.local_conf)
log.info('Environment loaded.')
Base.metadata.create_all(bind=Session.bind)
filename = os.path.split(conf.filename)[-1] # e.g., production.ini, development.ini, test.ini, ...
# Create the ``store`` directory and those for file, analysis and corpora
# objects and their subdirectories. See ``lib.utils.py`` for details.
h.create_OLD_directories(config=config)
# ISO-639-3 Language data for the languages table
log.info("Retrieving ISO-639-3 languages data.")
languages = h.get_language_objects(filename, config)
# Get default users.
log.info("Creating a default administrator, contributor and viewer.")
administrator = h.generate_default_administrator(config_filename=filename)
contributor = h.generate_default_contributor(config_filename=filename)
viewer = h.generate_default_viewer(config_filename=filename)
# If we are running tests, make sure the test db contains only language data.
if filename == 'test.ini':
# Permanently drop any existing tables
Base.metadata.drop_all(bind=Session.bind, checkfirst=True)
log.info("Existing tables dropped.")
# Create the tables if they don't already exist
Base.metadata.create_all(bind=Session.bind, checkfirst=True)
log.info('Tables created.')
Session.add_all(languages + [administrator, contributor, viewer])
Session.commit()
# Not a test: add a bunch of nice defaults.
else:
# Create the _requests_tests.py script
requests_tests_path = os.path.join(config['pylons.paths']['root'], 'tests',
'scripts', '_requests_tests.py')
# This line is problematic in production apps because the
# _requests_tests.py file is not included in the build. So, I'm
# commenting it out by default.
# copyfile(requests_tests_path, '_requests_tests.py')
# Create the tables if they don't already exist
Base.metadata.create_all(bind=Session.bind, checkfirst=True)
log.info('Tables created.')
# Get default home & help pages.
log.info("Creating default home and help pages.")
homepage = h.generate_default_home_page()
helppage = h.generate_default_help_page()
# Get default application settings.
log.info("Generating default application settings.")
application_settings = h.generate_default_application_settings()
# Get default tags and categories
log.info("Creating some useful tags and categories.")
restricted_tag = h.generate_restricted_tag()
foreign_word_tag = h.generate_foreign_word_tag()
S = h.generate_s_syntactic_category()
N = h.generate_n_syntactic_category()
V = h.generate_v_syntactic_category()
# Initialize the database
log.info("Adding defaults.")
data = [administrator, contributor, viewer, homepage, helppage,
application_settings, restricted_tag, foreign_word_tag]
if config['add_language_data'] != '0':
data += languages
if config['empty_database'] == '0':
Session.add_all(data)
Session.commit()
log.info("OLD successfully set up.")
Form.elicitor 24.3, 25.7 3.3, 5.8, 3.4 3.5, 3.5, 4.1
Form.files 29.3 4.6, 4.6 3.9, 3.8
Collection.forms 90.9 54.2 45.4
"""
from paste.deploy import appconfig
from pylons import config
from onlinelinguisticdatabase.config.environment import load_environment
import onlinelinguisticdatabase.model as model
from onlinelinguisticdatabase.model.meta import Session
import onlinelinguisticdatabase.lib.helpers as h
import timeit
conf = appconfig('config:test.ini', relative_to='.')
load_environment(conf.global_conf, conf.local_conf)
def setUp():
h.clearAllModels()
def cleanUp():
h.clearAllModels()
def createCollections(n):
createForms(n)
forms = Session.query(model.Form).all()
for i in range(1, n + 1):
createCollection(i, forms)
Session.commit()
def setup_app(command, conf, vars):
"""Place any commands to setup onlinelinguisticdatabase here"""
load_environment(conf.global_conf, conf.local_conf)
log.debug('environment loaded')
# Create the tables if they don't already exist
meta.metadata.create_all(bind=meta.engine)
log.debug('tables created')
# Create the files directory and the archived_files and researchers
# subdirectories
try:
os.mkdir('files')
except OSError:
pass
try:
os.mkdir(os.path.join('files', 'archived_files'))
except OSError:
pass
try:
os.mkdir(os.path.join('files', 'researchers'))
except OSError:
pass
# Create the analysis directory and its phonology, morphotactics,
# morphophonology, probabilitycalculator subdirectories
try:
os.mkdir('analysis')
except OSError:
pass
try:
os.mkdir(os.path.join('analysis', 'phonology'))
except OSError:
pass
try:
os.mkdir(os.path.join('analysis', 'morphotactics'))
except OSError:
pass
try:
os.mkdir(os.path.join('analysis', 'morphophonology'))
except OSError:
pass
try:
os.mkdir(os.path.join('analysis', 'probabilitycalculator'))
except OSError:
pass
# Add an administrator and some general language data
# Administrator
log.info("Creating a default administrator.")
admin = model.User()
admin.firstName = u'Admin'
admin.lastName = u'Admin'
admin.username = u'admin'
admin.email = u'*****@*****.**'
admin.password = unicode(hashlib.sha224(u'admin').hexdigest())
admin.role = {'0': u'administrator', '1': u'mirror'}[config['mirror']]
admin.collectionViewType = u'long'
admin.inputOrthography = None
admin.outputOrthography = None
admin.personalPageContent = u''
h.createResearcherDirectory(admin)
# Contributor
log.info("Creating a default contributor.")
contributor = model.User()
contributor.firstName = u'Contributor'
contributor.lastName = u'Contributor'
contributor.username = u'contributor'
contributor.email = u'*****@*****.**'
contributor.password = unicode(hashlib.sha224(u'contributor').hexdigest())
contributor.role = u'contributor'
contributor.collectionViewType = u'long'
contributor.inputOrthography = None
contributor.outputOrthography = None
contributor.personalPageContent = u''
h.createResearcherDirectory(contributor)
# Viewer
log.info("Creating a default viewer.")
viewer = model.User()
viewer.firstName = u'Viewer'
viewer.lastName = u'Viewer'
viewer.username = u'viewer'
viewer.email = u'*****@*****.**'
viewer.password = unicode(hashlib.sha224(u'viewer').hexdigest())
viewer.role = u'viewer'
viewer.collectionViewType = u'long'
viewer.inputOrthography = None
viewer.outputOrthography = None
viewer.personalPageContent = u''
h.createResearcherDirectory(viewer)
# Default Home Page
homepage = model.Page()
homepage.name = u'home'
homepage.heading = u'Welcome to the OLD'
homepage.content = u"""
The Online Linguistic Database is a web application that helps people to
document, study and learn a language.
"""
homepage.markup = u'restructuredtext'
# Default Help Page
helppage = model.Page()
helppage.name = u'help'
helppage.heading = u'OLD Application Help'
helppage.content = u"""
Welcome to the help page of this OLD application.
This page should contain content entered by your administrator.
"""
helppage.markup = u'restructuredtext'
# Enter ISO-639-3 Language data into the languages table
log.info("Retrieving ISO-639-3 languages data.")
languages = [getLanguageObject(language) for language in iso_639_3.languages]
# Default Application Settings
log.info("Generating default settings.")
orthography = u', '.join(list(string.ascii_lowercase))
applicationSettings = model.ApplicationSettings()
applicationSettings.objectLanguageName = u'Anonymous'
applicationSettings.storageOrthography = u'Orthography 1'
applicationSettings.defaultInputOrthography = u'Orthography 1'
applicationSettings.defaultOutputOrthography = u'Orthography 1'
applicationSettings.objectLanguageOrthography1 = orthography
applicationSettings.objectLanguageOrthography1Name = u'English alphabet'
applicationSettings.metaLanguageName = u'Unknown'
applicationSettings.metaLanguageOrthography = orthography
applicationSettings.headerImageName = u''
applicationSettings.colorsCSS = u'green.css'
applicationSettings.OLO1Lowercase = u'1'
applicationSettings.OLO1InitialGlottalStops = u'1'
applicationSettings.OLO2Lowercase = u'1'
applicationSettings.OLO2InitialGlottalStops = u'1'
applicationSettings.OLO3Lowercase = u'1'
applicationSettings.OLO3InitialGlottalStops = u'1'
applicationSettings.OLO4Lowercase = u'1'
applicationSettings.OLO4InitialGlottalStops = u'1'
applicationSettings.OLO5Lowercase = u'1'
applicationSettings.OLO5InitialGlottalStops = u'1'
applicationSettings.morphemeBreakIsObjectLanguageString = u'no'
applicationSettings.unrestrictedUsers = u'[]'
applicationSettings.orthographicValidation = u'None'
applicationSettings.narrPhonValidation = u'None'
applicationSettings.broadPhonValidation = u'None'
applicationSettings.morphPhonValidation = u'None'
applicationSettings.morphDelimiters = u'-,='
applicationSettings.punctuation = u""".,;:!?'"\u2018\u2019\u201C\u201D[]{}()-"""
applicationSettings.grammaticalities = u'*,#,?'
applicationSettings.narrPhonInventory = u''
applicationSettings.broadPhonInventory = u''
applicationSettings.morphPhonInventory = u''
# Default Keywords
restrictedKW = model.Keyword()
restrictedKW.name = u'restricted'
restrictedKW.description = u'''Forms tagged with the keyword 'restricted'
can only be viewed by administrators, unrestricted users and the users they were
entered by.'''
foreignWordKW = model.Keyword()
foreignWordKW.name = u'foreign word'
foreignWordKW.description = u'''Use this tag for lexical entries that are
not from the object language. For example, it might be desirable to create a
form as lexical entry for a proper noun like "John". Such a form should be
tagged as a "foreign word". This will allow forms containing "John" to have
gapless syntactic category string values. Additionally, the system ignores
foreign word transcriptions when validating user input against orthographic,
phonetic and phonemic inventories.'''
# Initialize the database
log.info("Adding defaults.")
data = [admin, contributor, viewer, homepage, helppage, applicationSettings,
restrictedKW, foreignWordKW]
if config['addLanguageData'] != '0':
data += languages
if config['emptyDatabase'] == '0':
meta.Session.add_all(data)
meta.Session.commit()
log.info("OLD successfully set up.")
def setup_app(command, conf, vars):
"""Commands to setup onlinelinguisticdatabase."""
config = load_environment(conf.global_conf, conf.local_conf)
log.info('Environment loaded.')
Base.metadata.create_all(bind=Session.bind)
filename = os.path.split(conf.filename)[
-1] # e.g., production.ini, development.ini, test.ini, ...
# Create the ``store`` directory and those for file, analysis and corpora
# objects and their subdirectories. See ``lib.utils.py`` for details.
h.create_OLD_directories(config=config)
# ISO-639-3 Language data for the languages table
log.info("Retrieving ISO-639-3 languages data.")
languages = h.get_language_objects(filename, config)
# Get default users.
log.info("Creating a default administrator, contributor and viewer.")
administrator = h.generate_default_administrator(config_filename=filename)
contributor = h.generate_default_contributor(config_filename=filename)
viewer = h.generate_default_viewer(config_filename=filename)
# If we are running tests, make sure the test db contains only language data.
if filename == 'test.ini':
# Permanently drop any existing tables
Base.metadata.drop_all(bind=Session.bind, checkfirst=True)
log.info("Existing tables dropped.")
# Create the tables if they don't already exist
Base.metadata.create_all(bind=Session.bind, checkfirst=True)
log.info('Tables created.')
Session.add_all(languages + [administrator, contributor, viewer])
Session.commit()
# Not a test: add a bunch of nice defaults.
else:
# Create the _requests_tests.py script
requests_tests_path = os.path.join(config['pylons.paths']['root'],
'tests', 'scripts',
'_requests_tests.py')
# This line is problematic in production apps because the
# _requests_tests.py file is not included in the build. So, I'm
# commenting it out by default.
# copyfile(requests_tests_path, '_requests_tests.py')
# Create the tables if they don't already exist
Base.metadata.create_all(bind=Session.bind, checkfirst=True)
log.info('Tables created.')
# Get default home & help pages.
log.info("Creating default home and help pages.")
homepage = h.generate_default_home_page()
helppage = h.generate_default_help_page()
# Get default application settings.
log.info("Generating default application settings.")
application_settings = h.generate_default_application_settings()
# Get default tags and categories
log.info("Creating some useful tags and categories.")
restricted_tag = h.generate_restricted_tag()
foreign_word_tag = h.generate_foreign_word_tag()
S = h.generate_s_syntactic_category()
N = h.generate_n_syntactic_category()
V = h.generate_v_syntactic_category()
# Initialize the database
log.info("Adding defaults.")
data = [
administrator, contributor, viewer, homepage, helppage,
application_settings, restricted_tag, foreign_word_tag
]
if config['add_language_data'] != '0':
data += languages
if config['empty_database'] == '0':
Session.add_all(data)
Session.commit()
log.info("OLD successfully set up.")
def setup_app(command, conf, vars):
"""Place any commands to setup onlinelinguisticdatabase here"""
load_environment(conf.global_conf, conf.local_conf)
log.debug('environment loaded')
# Create the tables if they don't already exist
meta.metadata.create_all(bind=meta.engine)
log.debug('tables created')
# Create the files directory and the archived_files and researchers
# subdirectories
try:
os.mkdir('files')
except OSError:
pass
try:
os.mkdir(os.path.join('files', 'archived_files'))
except OSError:
pass
try:
os.mkdir(os.path.join('files', 'researchers'))
except OSError:
pass
# Create the analysis directory and its phonology, morphotactics,
# morphophonology, probabilitycalculator subdirectories
try:
os.mkdir('analysis')
except OSError:
pass
try:
os.mkdir(os.path.join('analysis', 'phonology'))
except OSError:
pass
try:
os.mkdir(os.path.join('analysis', 'morphotactics'))
except OSError:
pass
try:
os.mkdir(os.path.join('analysis', 'morphophonology'))
except OSError:
pass
try:
os.mkdir(os.path.join('analysis', 'probabilitycalculator'))
except OSError:
pass
# Add an administrator and some general language data
# Administrator
log.info("Creating a default administrator.")
admin = model.User()
admin.firstName = u'Admin'
admin.lastName = u'Admin'
admin.username = u'admin'
admin.email = u'*****@*****.**'
admin.password = unicode(hashlib.sha224(u'admin').hexdigest())
admin.role = {'0': u'administrator', '1': u'mirror'}[config['mirror']]
admin.collectionViewType = u'long'
admin.inputOrthography = None
admin.outputOrthography = None
admin.personalPageContent = u''
h.createResearcherDirectory(admin)
# Contributor
log.info("Creating a default contributor.")
contributor = model.User()
contributor.firstName = u'Contributor'
contributor.lastName = u'Contributor'
contributor.username = u'contributor'
contributor.email = u'*****@*****.**'
contributor.password = unicode(hashlib.sha224(u'contributor').hexdigest())
contributor.role = u'contributor'
contributor.collectionViewType = u'long'
contributor.inputOrthography = None
contributor.outputOrthography = None
contributor.personalPageContent = u''
h.createResearcherDirectory(contributor)
# Viewer
log.info("Creating a default viewer.")
viewer = model.User()
viewer.firstName = u'Viewer'
viewer.lastName = u'Viewer'
viewer.username = u'viewer'
viewer.email = u'*****@*****.**'
viewer.password = unicode(hashlib.sha224(u'viewer').hexdigest())
viewer.role = u'viewer'
viewer.collectionViewType = u'long'
viewer.inputOrthography = None
viewer.outputOrthography = None
viewer.personalPageContent = u''
h.createResearcherDirectory(viewer)
# Default Home Page
homepage = model.Page()
homepage.name = u'home'
homepage.heading = u'Welcome to the OLD'
homepage.content = u"""
The Online Linguistic Database is a web application that helps people to
document, study and learn a language.
"""
homepage.markup = u'restructuredtext'
# Default Help Page
helppage = model.Page()
helppage.name = u'help'
helppage.heading = u'OLD Application Help'
helppage.content = u"""
Welcome to the help page of this OLD application.
This page should contain content entered by your administrator.
"""
helppage.markup = u'restructuredtext'
# Enter ISO-639-3 Language data into the languages table
log.info("Retrieving ISO-639-3 languages data.")
languages = [
getLanguageObject(language) for language in iso_639_3.languages
]
# Default Application Settings
log.info("Generating default settings.")
orthography = u', '.join(list(string.ascii_lowercase))
applicationSettings = model.ApplicationSettings()
applicationSettings.objectLanguageName = u'Anonymous'
applicationSettings.storageOrthography = u'Orthography 1'
applicationSettings.defaultInputOrthography = u'Orthography 1'
applicationSettings.defaultOutputOrthography = u'Orthography 1'
applicationSettings.objectLanguageOrthography1 = orthography
applicationSettings.objectLanguageOrthography1Name = u'English alphabet'
applicationSettings.metaLanguageName = u'Unknown'
applicationSettings.metaLanguageOrthography = orthography
applicationSettings.headerImageName = u''
applicationSettings.colorsCSS = u'green.css'
applicationSettings.OLO1Lowercase = u'1'
applicationSettings.OLO1InitialGlottalStops = u'1'
applicationSettings.OLO2Lowercase = u'1'
applicationSettings.OLO2InitialGlottalStops = u'1'
applicationSettings.OLO3Lowercase = u'1'
applicationSettings.OLO3InitialGlottalStops = u'1'
applicationSettings.OLO4Lowercase = u'1'
applicationSettings.OLO4InitialGlottalStops = u'1'
applicationSettings.OLO5Lowercase = u'1'
applicationSettings.OLO5InitialGlottalStops = u'1'
applicationSettings.morphemeBreakIsObjectLanguageString = u'no'
applicationSettings.unrestrictedUsers = u'[]'
applicationSettings.orthographicValidation = u'None'
applicationSettings.narrPhonValidation = u'None'
applicationSettings.broadPhonValidation = u'None'
applicationSettings.morphPhonValidation = u'None'
applicationSettings.morphDelimiters = u'-,='
applicationSettings.punctuation = u""".,;:!?'"\u2018\u2019\u201C\u201D[]{}()-"""
applicationSettings.grammaticalities = u'*,#,?'
applicationSettings.narrPhonInventory = u''
applicationSettings.broadPhonInventory = u''
applicationSettings.morphPhonInventory = u''
# Default Keywords
restrictedKW = model.Keyword()
restrictedKW.name = u'restricted'
restrictedKW.description = u'''Forms tagged with the keyword 'restricted'
can only be viewed by administrators, unrestricted users and the users they were
entered by.'''
foreignWordKW = model.Keyword()
foreignWordKW.name = u'foreign word'
foreignWordKW.description = u'''Use this tag for lexical entries that are
not from the object language. For example, it might be desirable to create a
form as lexical entry for a proper noun like "John". Such a form should be
tagged as a "foreign word". This will allow forms containing "John" to have
gapless syntactic category string values. Additionally, the system ignores
foreign word transcriptions when validating user input against orthographic,
phonetic and phonemic inventories.'''
# Initialize the database
log.info("Adding defaults.")
data = [
admin, contributor, viewer, homepage, helppage, applicationSettings,
restrictedKW, foreignWordKW
]
if config['addLanguageData'] != '0':
data += languages
if config['emptyDatabase'] == '0':
meta.Session.add_all(data)
meta.Session.commit()
log.info("OLD successfully set up.")
Form.elicitor 24.3, 25.7 3.3, 5.8, 3.4 3.5, 3.5, 4.1
Form.files 29.3 4.6, 4.6 3.9, 3.8
Collection.forms 90.9 54.2 45.4
"""
from paste.deploy import appconfig
from pylons import config
from onlinelinguisticdatabase.config.environment import load_environment
import onlinelinguisticdatabase.model as model
from onlinelinguisticdatabase.model.meta import Session
import onlinelinguisticdatabase.lib.helpers as h
import timeit
conf = appconfig('config:test.ini', relative_to='.')
load_environment(conf.global_conf, conf.local_conf)
def setUp():
h.clearAllModels()
def cleanUp():
h.clearAllModels()
def createCollections(n):
createForms(n)
forms = Session.query(model.Form).all()
for i in range(1, n + 1):
createCollection(i, forms)
