def test_read_mediagoblin_config():
# An empty file
this_conf, validation_results = config.read_mediagoblin_config(
CARROT_CONF_EMPTY, FAKE_CONFIG_SPEC)
assert this_conf['carrotapp']['carrotcake'] == False
assert this_conf['carrotapp']['num_carrots'] == 1
assert not this_conf['carrotapp'].has_key('encouragement_phrase')
assert this_conf['celery']['eat_celery_with_carrots'] == True
# A good file
this_conf, validation_results = config.read_mediagoblin_config(
CARROT_CONF_GOOD, FAKE_CONFIG_SPEC)
assert this_conf['carrotapp']['carrotcake'] == True
assert this_conf['carrotapp']['num_carrots'] == 88
assert this_conf['carrotapp']['encouragement_phrase'] == \
"I'd love it if you eat your carrots!"
assert this_conf['carrotapp']['blah_blah'] == "blah!"
assert this_conf['celery']['eat_celery_with_carrots'] == False
# A bad file
this_conf, validation_results = config.read_mediagoblin_config(
CARROT_CONF_BAD, FAKE_CONFIG_SPEC)
# These should still open but will have errors that we'll test for
# in test_generate_validation_report()
assert this_conf['carrotapp']['carrotcake'] == 'slobber'
assert this_conf['carrotapp']['num_carrots'] == 'GROSS'
assert this_conf['carrotapp']['encouragement_phrase'] == \
"586956856856"
assert this_conf['carrotapp']['blah_blah'] == "blah!"
assert this_conf['celery']['eat_celery_with_carrots'] == "pants"
def test_generate_validation_report():
# Empty
this_conf, validation_results = config.read_mediagoblin_config(
CARROT_CONF_EMPTY, FAKE_CONFIG_SPEC)
report = config.generate_validation_report(this_conf, validation_results)
assert report is None
# Good
this_conf, validation_results = config.read_mediagoblin_config(
CARROT_CONF_GOOD, FAKE_CONFIG_SPEC)
report = config.generate_validation_report(this_conf, validation_results)
assert report is None
# Bad
this_conf, validation_results = config.read_mediagoblin_config(
CARROT_CONF_BAD, FAKE_CONFIG_SPEC)
report = config.generate_validation_report(this_conf, validation_results)
assert report.startswith("""\
There were validation problems loading this config file:
--------------------------------------------------------""")
expected_warnings = [
'carrotapp:carrotcake = the value "slobber" is of the wrong type.',
'carrotapp:num_carrots = the value "GROSS" is of the wrong type.',
'celery:eat_celery_with_carrots = the value "pants" is of the wrong type.']
warnings = report.splitlines()[2:]
assert len(warnings) == 3
for warning in expected_warnings:
assert warning in warnings
def test_generate_validation_report():
# Empty
this_conf, validation_results = config.read_mediagoblin_config(
CARROT_CONF_EMPTY, FAKE_CONFIG_SPEC)
report = config.generate_validation_report(this_conf, validation_results)
assert report is None
# Good
this_conf, validation_results = config.read_mediagoblin_config(
CARROT_CONF_GOOD, FAKE_CONFIG_SPEC)
report = config.generate_validation_report(this_conf, validation_results)
assert report is None
# Bad
this_conf, validation_results = config.read_mediagoblin_config(
CARROT_CONF_BAD, FAKE_CONFIG_SPEC)
report = config.generate_validation_report(this_conf, validation_results)
assert report.startswith("""\
There were validation problems loading this config file:
--------------------------------------------------------""")
expected_warnings = [
'carrotapp:carrotcake = the value "slobber" is of the wrong type.',
'carrotapp:num_carrots = the value "GROSS" is of the wrong type.',
'celery:eat_celery_with_carrots = the value "pants" is of the wrong type.'
]
warnings = report.splitlines()[2:]
assert len(warnings) == 3
for warning in expected_warnings:
assert warning in warnings
def test_read_mediagoblin_config():
# An empty file
this_conf, validation_results = config.read_mediagoblin_config(
CARROT_CONF_EMPTY, FAKE_CONFIG_SPEC)
assert this_conf['carrotapp']['carrotcake'] == False
assert this_conf['carrotapp']['num_carrots'] == 1
assert not this_conf['carrotapp'].has_key('encouragement_phrase')
assert this_conf['celery']['eat_celery_with_carrots'] == True
# A good file
this_conf, validation_results = config.read_mediagoblin_config(
CARROT_CONF_GOOD, FAKE_CONFIG_SPEC)
assert this_conf['carrotapp']['carrotcake'] == True
assert this_conf['carrotapp']['num_carrots'] == 88
assert this_conf['carrotapp']['encouragement_phrase'] == \
"I'd love it if you eat your carrots!"
assert this_conf['carrotapp']['blah_blah'] == "blah!"
assert this_conf['celery']['eat_celery_with_carrots'] == False
# A bad file
this_conf, validation_results = config.read_mediagoblin_config(
CARROT_CONF_BAD, FAKE_CONFIG_SPEC)
# These should still open but will have errors that we'll test for
# in test_generate_validation_report()
assert this_conf['carrotapp']['carrotcake'] == 'slobber'
assert this_conf['carrotapp']['num_carrots'] == 'GROSS'
assert this_conf['carrotapp']['encouragement_phrase'] == \
"586956856856"
assert this_conf['carrotapp']['blah_blah'] == "blah!"
assert this_conf['celery']['eat_celery_with_carrots'] == "pants"
def test_setup_celery_from_config():
def _wipe_testmodule_clean(module):
vars_to_wipe = [
var for var in dir(module)
if not var.startswith('__') and not var.endswith('__')]
for var in vars_to_wipe:
delattr(module, var)
global_config, validation_result = read_mediagoblin_config(
TEST_CELERY_CONF_NOSPECIALDB)
app_config = global_config['mediagoblin']
celery_setup.setup_celery_from_config(
app_config, global_config,
'mediagoblin.tests.fake_celery_module', set_environ=False)
from mediagoblin.tests import fake_celery_module
assert fake_celery_module.SOME_VARIABLE == 'floop'
assert fake_celery_module.MAIL_PORT == 2000
assert isinstance(fake_celery_module.MAIL_PORT, int)
assert fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION == 1.3
assert isinstance(fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION, float)
assert fake_celery_module.CELERY_RESULT_PERSISTENT is True
assert fake_celery_module.CELERY_IMPORTS == [
'foo.bar.baz', 'this.is.an.import', 'mediagoblin.process_media']
assert fake_celery_module.CELERY_MONGODB_BACKEND_SETTINGS == {
'database': 'mediagoblin'}
assert fake_celery_module.CELERY_RESULT_BACKEND == 'mongodb'
assert fake_celery_module.BROKER_BACKEND == 'mongodb'
_wipe_testmodule_clean(fake_celery_module)
global_config, validation_result = read_mediagoblin_config(
TEST_CELERY_CONF_MGSPECIALDB)
app_config = global_config['mediagoblin']
celery_setup.setup_celery_from_config(
app_config, global_config,
'mediagoblin.tests.fake_celery_module', set_environ=False)
from mediagoblin.tests import fake_celery_module
assert fake_celery_module.SOME_VARIABLE == 'poolf'
assert fake_celery_module.MAIL_PORT == 2020
assert isinstance(fake_celery_module.MAIL_PORT, int)
assert fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION == 3.1
assert isinstance(fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION, float)
assert fake_celery_module.CELERY_RESULT_PERSISTENT is False
assert fake_celery_module.CELERY_IMPORTS == [
'baz.bar.foo', 'import.is.a.this', 'mediagoblin.process_media']
assert fake_celery_module.CELERY_MONGODB_BACKEND_SETTINGS == {
'database': 'captain_lollerskates',
'host': 'mongodb.example.org',
'port': 8080}
assert fake_celery_module.CELERY_RESULT_BACKEND == 'mongodb'
assert fake_celery_module.BROKER_BACKEND == 'mongodb'
assert fake_celery_module.BROKER_HOST == 'mongodb.example.org'
assert fake_celery_module.BROKER_PORT == 8080
def get_test_app(dump_old_app=True):
suicide_if_bad_celery_environ()
# Leave this imported as it sets up celery.
from mediagoblin.celery_setup import from_tests
global MGOBLIN_APP
# Just return the old app if that exists and it's okay to set up
# and return
if MGOBLIN_APP and not dump_old_app:
return MGOBLIN_APP
# Remove and reinstall user_dev directories
if os.path.exists(TEST_USER_DEV):
shutil.rmtree(TEST_USER_DEV)
for directory in USER_DEV_DIRECTORIES_TO_SETUP:
full_dir = os.path.join(TEST_USER_DEV, directory)
os.makedirs(full_dir)
# Get app config
global_config, validation_result = read_mediagoblin_config(TEST_APP_CONFIG)
app_config = global_config['mediagoblin']
# Wipe database
# @@: For now we're dropping collections, but we could also just
# collection.remove() ?
connection, db = setup_connection_and_db_from_config(app_config)
assert db.name == MEDIAGOBLIN_TEST_DB_NAME
collections_to_wipe = [
collection
for collection in db.collection_names()
if not collection.startswith('system.')]
for collection in collections_to_wipe:
db.drop_collection(collection)
# TODO: Drop and recreate indexes
# setup app and return
test_app = loadapp(
'config:' + TEST_SERVER_CONFIG)
app = TestApp(test_app)
MGOBLIN_APP = app
return app
def __init__(self, config_path, setup_celery=True):
"""
Initialize the application based on a configuration file.
Arguments:
- config_path: path to the configuration file we're opening.
- setup_celery: whether or not to setup celery during init.
(Note: setting 'celery_setup_elsewhere' also disables
setting up celery.)
"""
##############
# Setup config
##############
# Open and setup the config
global_config, validation_result = read_mediagoblin_config(config_path)
app_config = global_config['mediagoblin']
# report errors if necessary
validation_report = generate_validation_report(global_config,
validation_result)
if validation_report:
raise ImproperlyConfigured(validation_report)
##########################################
# Setup other connections / useful objects
##########################################
# Set up the database
self.connection, self.db = setup_connection_and_db_from_config(
app_config)
# Get the template environment
self.template_loader = util.get_jinja_loader(
app_config.get('user_template_path'))
# Set up storage systems
self.public_store = storage.storage_system_from_config(
app_config, 'publicstore')
self.queue_store = storage.storage_system_from_config(
app_config, 'queuestore')
# set up routing
self.routing = routing.get_mapper()
# set up staticdirector tool
if app_config.has_key('direct_remote_path'):
self.staticdirector = staticdirect.RemoteStaticDirect(
app_config['direct_remote_path'].strip())
elif app_config.has_key('direct_remote_paths'):
direct_remote_path_lines = app_config['direct_remote_paths'].strip(
).splitlines()
self.staticdirector = staticdirect.MultiRemoteStaticDirect(
dict([
line.strip().split(' ', 1)
for line in direct_remote_path_lines
]))
else:
raise ImproperlyConfigured("One of direct_remote_path or "
"direct_remote_paths must be provided")
# Setup celery, if appropriate
if setup_celery and not app_config.get('celery_setup_elsewhere'):
if os.environ.get('CELERY_ALWAYS_EAGER'):
setup_celery_from_config(app_config,
global_config,
force_celery_always_eager=True)
else:
setup_celery_from_config(app_config, global_config)
#######################################################
# Insert appropriate things into mediagoblin.mg_globals
#
# certain properties need to be accessed globally eg from
# validators, etc, which might not access to the request
# object.
#######################################################
setup_globals(
app_config=app_config,
global_config=global_config,
# TODO: No need to set these two up as globals, we could
# just read them out of mg_globals.app_config
email_sender_address=app_config['email_sender_address'],
email_debug_mode=app_config['email_debug_mode'],
# Actual, useful to everyone objects
app=self,
db_connection=self.connection,
database=self.db,
public_store=self.public_store,
queue_store=self.queue_store,
workbench_manager=WorkbenchManager(app_config['workbench_path']))
def __init__(self, config_path, setup_celery=True):
"""
Initialize the application based on a configuration file.
Arguments:
- config_path: path to the configuration file we're opening.
- setup_celery: whether or not to setup celery during init.
(Note: setting 'celery_setup_elsewhere' also disables
setting up celery.)
"""
##############
# Setup config
##############
# Open and setup the config
global_config, validation_result = read_mediagoblin_config(config_path)
app_config = global_config['mediagoblin']
# report errors if necessary
validation_report = generate_validation_report(
global_config, validation_result)
if validation_report:
raise ImproperlyConfigured(validation_report)
##########################################
# Setup other connections / useful objects
##########################################
# Set up the database
self.connection, self.db = setup_connection_and_db_from_config(
app_config)
# Get the template environment
self.template_loader = util.get_jinja_loader(
app_config.get('user_template_path'))
# Set up storage systems
self.public_store = storage.storage_system_from_config(
app_config, 'publicstore')
self.queue_store = storage.storage_system_from_config(
app_config, 'queuestore')
# set up routing
self.routing = routing.get_mapper()
# set up staticdirector tool
if app_config.has_key('direct_remote_path'):
self.staticdirector = staticdirect.RemoteStaticDirect(
app_config['direct_remote_path'].strip())
elif app_config.has_key('direct_remote_paths'):
direct_remote_path_lines = app_config[
'direct_remote_paths'].strip().splitlines()
self.staticdirector = staticdirect.MultiRemoteStaticDirect(
dict([line.strip().split(' ', 1)
for line in direct_remote_path_lines]))
else:
raise ImproperlyConfigured(
"One of direct_remote_path or "
"direct_remote_paths must be provided")
# Setup celery, if appropriate
if setup_celery and not app_config.get('celery_setup_elsewhere'):
if os.environ.get('CELERY_ALWAYS_EAGER'):
setup_celery_from_config(
app_config, global_config,
force_celery_always_eager=True)
else:
setup_celery_from_config(app_config, global_config)
#######################################################
# Insert appropriate things into mediagoblin.mg_globals
#
# certain properties need to be accessed globally eg from
# validators, etc, which might not access to the request
# object.
#######################################################
setup_globals(
app_config=app_config,
global_config=global_config,
# TODO: No need to set these two up as globals, we could
# just read them out of mg_globals.app_config
email_sender_address=app_config['email_sender_address'],
email_debug_mode=app_config['email_debug_mode'],
# Actual, useful to everyone objects
app=self,
db_connection=self.connection,
database=self.db,
public_store=self.public_store,
queue_store=self.queue_store,
workbench_manager=WorkbenchManager(app_config['workbench_path']))
def test_setup_celery_from_config():
def _wipe_testmodule_clean(module):
vars_to_wipe = [
var for var in dir(module)
if not var.startswith('__') and not var.endswith('__')
]
for var in vars_to_wipe:
delattr(module, var)
global_config, validation_result = read_mediagoblin_config(
TEST_CELERY_CONF_NOSPECIALDB)
app_config = global_config['mediagoblin']
celery_setup.setup_celery_from_config(
app_config,
global_config,
'mediagoblin.tests.fake_celery_module',
set_environ=False)
from mediagoblin.tests import fake_celery_module
assert fake_celery_module.SOME_VARIABLE == 'floop'
assert fake_celery_module.MAIL_PORT == 2000
assert isinstance(fake_celery_module.MAIL_PORT, int)
assert fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION == 1.3
assert isinstance(fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION,
float)
assert fake_celery_module.CELERY_RESULT_PERSISTENT is True
assert fake_celery_module.CELERY_IMPORTS == [
'foo.bar.baz', 'this.is.an.import', 'mediagoblin.process_media'
]
assert fake_celery_module.CELERY_MONGODB_BACKEND_SETTINGS == {
'database': 'mediagoblin'
}
assert fake_celery_module.CELERY_RESULT_BACKEND == 'mongodb'
assert fake_celery_module.BROKER_BACKEND == 'mongodb'
_wipe_testmodule_clean(fake_celery_module)
global_config, validation_result = read_mediagoblin_config(
TEST_CELERY_CONF_MGSPECIALDB)
app_config = global_config['mediagoblin']
celery_setup.setup_celery_from_config(
app_config,
global_config,
'mediagoblin.tests.fake_celery_module',
set_environ=False)
from mediagoblin.tests import fake_celery_module
assert fake_celery_module.SOME_VARIABLE == 'poolf'
assert fake_celery_module.MAIL_PORT == 2020
assert isinstance(fake_celery_module.MAIL_PORT, int)
assert fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION == 3.1
assert isinstance(fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION,
float)
assert fake_celery_module.CELERY_RESULT_PERSISTENT is False
assert fake_celery_module.CELERY_IMPORTS == [
'baz.bar.foo', 'import.is.a.this', 'mediagoblin.process_media'
]
assert fake_celery_module.CELERY_MONGODB_BACKEND_SETTINGS == {
'database': 'captain_lollerskates',
'host': 'mongodb.example.org',
'port': 8080
}
assert fake_celery_module.CELERY_RESULT_BACKEND == 'mongodb'
assert fake_celery_module.BROKER_BACKEND == 'mongodb'
assert fake_celery_module.BROKER_HOST == 'mongodb.example.org'
assert fake_celery_module.BROKER_PORT == 8080