def setup_self(check_environ_for_conf=True, module_name=OUR_MODULENAME, default_conf_file='mediagoblin.ini'): """ Transform this module into a celery config module by reading the mediagoblin config file. Set the environment variable MEDIAGOBLIN_CONFIG to specify where this config file is. By default it defaults to 'mediagoblin.ini'. Note that if celery_setup_elsewhere is set in your config file, this simply won't work. """ if check_environ_for_conf: mgoblin_conf_file = os.path.abspath( os.environ.get('MEDIAGOBLIN_CONFIG', default_conf_file)) else: mgoblin_conf_file = default_conf_file if not os.path.exists(mgoblin_conf_file): raise IOError("MEDIAGOBLIN_CONFIG not set or file does not exist") # By setting the environment variable here we should ensure that # this is the module that gets set up. os.environ['CELERY_CONFIG_MODULE'] = module_name app.MediaGoblinApp(mgoblin_conf_file, setup_celery=False) setup_celery_from_config(mg_globals.app_config, mg_globals.global_config, settings_module=module_name, set_environ=False)
def paste_app_factory(global_config, **app_config): # Get the database connection connection = mongokit.Connection( app_config.get('db_host'), app_config.get('db_port')) # Set up the storage systems. public_store = storage.storage_system_from_paste_config( app_config, 'publicstore') queue_store = storage.storage_system_from_paste_config( app_config, 'queuestore') # Set up the staticdirect system if app_config.has_key('direct_remote_path'): staticdirector = staticdirect.RemoteStaticDirect( app_config['direct_remote_path'].strip()) elif app_config.has_key('direct_remote_paths'): direct_remote_path_lines = app_config[ 'direct_remote_paths'].strip().splitlines() staticdirector = staticdirect.MultiRemoteStaticDirect( dict([line.strip().split(' ', 1) for line in direct_remote_path_lines])) else: raise ImproperlyConfigured( "One of direct_remote_path or direct_remote_paths must be provided") setup_celery_from_config(app_config, global_config) mgoblin_app = MediaGoblinApp( connection, app_config.get('db_name', 'mediagoblin'), public_store=public_store, queue_store=queue_store, staticdirector=staticdirector, user_template_path=app_config.get('local_templates')) return mgoblin_app
def setup_self(check_environ_for_conf=True, module_name=OUR_MODULENAME, default_conf_file='mediagoblin.ini'): """ Transform this module into a celery config module by reading the mediagoblin config file. Set the environment variable MEDIAGOBLIN_CONFIG to specify where this config file is. By default it defaults to 'mediagoblin.ini'. Note that if celery_setup_elsewhere is set in your config file, this simply won't work. """ if check_environ_for_conf: mgoblin_conf_file = os.path.abspath( os.environ.get('MEDIAGOBLIN_CONFIG', default_conf_file)) else: mgoblin_conf_file = default_conf_file if not os.path.exists(mgoblin_conf_file): raise IOError( "MEDIAGOBLIN_CONFIG not set or file does not exist") # By setting the environment variable here we should ensure that # this is the module that gets set up. os.environ['CELERY_CONFIG_MODULE'] = module_name app.MediaGoblinApp(mgoblin_conf_file, setup_celery=False) setup_celery_from_config( mg_globals.app_config, mg_globals.global_config, settings_module=module_name, set_environ=False)
def test_setup_celery_from_config(): def _wipe_testmodule_clean(module): vars_to_wipe = [ var for var in dir(module) if not var.startswith('__') and not var.endswith('__')] for var in vars_to_wipe: delattr(module, var) global_config, validation_result = read_mediagoblin_config( TEST_CELERY_CONF_NOSPECIALDB) app_config = global_config['mediagoblin'] celery_setup.setup_celery_from_config( app_config, global_config, 'mediagoblin.tests.fake_celery_module', set_environ=False) from mediagoblin.tests import fake_celery_module assert fake_celery_module.SOME_VARIABLE == 'floop' assert fake_celery_module.MAIL_PORT == 2000 assert isinstance(fake_celery_module.MAIL_PORT, int) assert fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION == 1.3 assert isinstance(fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION, float) assert fake_celery_module.CELERY_RESULT_PERSISTENT is True assert fake_celery_module.CELERY_IMPORTS == [ 'foo.bar.baz', 'this.is.an.import', 'mediagoblin.process_media'] assert fake_celery_module.CELERY_MONGODB_BACKEND_SETTINGS == { 'database': 'mediagoblin'} assert fake_celery_module.CELERY_RESULT_BACKEND == 'mongodb' assert fake_celery_module.BROKER_BACKEND == 'mongodb' _wipe_testmodule_clean(fake_celery_module) global_config, validation_result = read_mediagoblin_config( TEST_CELERY_CONF_MGSPECIALDB) app_config = global_config['mediagoblin'] celery_setup.setup_celery_from_config( app_config, global_config, 'mediagoblin.tests.fake_celery_module', set_environ=False) from mediagoblin.tests import fake_celery_module assert fake_celery_module.SOME_VARIABLE == 'poolf' assert fake_celery_module.MAIL_PORT == 2020 assert isinstance(fake_celery_module.MAIL_PORT, int) assert fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION == 3.1 assert isinstance(fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION, float) assert fake_celery_module.CELERY_RESULT_PERSISTENT is False assert fake_celery_module.CELERY_IMPORTS == [ 'baz.bar.foo', 'import.is.a.this', 'mediagoblin.process_media'] assert fake_celery_module.CELERY_MONGODB_BACKEND_SETTINGS == { 'database': 'captain_lollerskates', 'host': 'mongodb.example.org', 'port': 8080} assert fake_celery_module.CELERY_RESULT_BACKEND == 'mongodb' assert fake_celery_module.BROKER_BACKEND == 'mongodb' assert fake_celery_module.BROKER_HOST == 'mongodb.example.org' assert fake_celery_module.BROKER_PORT == 8080
def setup_self(setup_globals_func=setup_globals): """ Transform this module into a celery config module by reading the mediagoblin config file. Set the environment variable MEDIAGOBLIN_CONFIG to specify where this config file is at and what section it uses. By default it defaults to 'mediagoblin.ini:app:mediagoblin'. The first colon ":" is a delimiter between the filename and the config section, so in this case the filename is 'mediagoblin.ini' and the section where mediagoblin is defined is 'app:mediagoblin'. Args: - 'setup_globals_func': this is for testing purposes only. Don't set this! """ mgoblin_conf_file, mgoblin_section = os.environ.get( 'MEDIAGOBLIN_CONFIG', 'mediagoblin.ini:app:mediagoblin').split(':', 1) if not os.path.exists(mgoblin_conf_file): raise IOError("MEDIAGOBLIN_CONFIG not set or file does not exist") parser = NicerConfigParser(mgoblin_conf_file) parser.read(mgoblin_conf_file) parser._defaults.setdefault( 'here', os.path.dirname(os.path.abspath(mgoblin_conf_file))) parser._defaults.setdefault('__file__', os.path.abspath(mgoblin_conf_file)) mgoblin_section = dict(parser.items(mgoblin_section)) mgoblin_conf = dict([(section_name, dict(parser.items(section_name))) for section_name in parser.sections()]) setup_celery_from_config(mgoblin_section, mgoblin_conf, settings_module=OUR_MODULENAME, set_environ=False) connection = mongokit.Connection(mgoblin_section.get('db_host'), mgoblin_section.get('db_port')) db = connection[mgoblin_section.get('db_name', 'mediagoblin')] models.register_models(connection) # Set up the storage systems. public_store = storage.storage_system_from_paste_config( mgoblin_section, 'publicstore') queue_store = storage.storage_system_from_paste_config( mgoblin_section, 'queuestore') setup_globals_func(db_connection=connection, database=db, public_store=public_store, queue_store=queue_store)
def setup_self(setup_globals_func=setup_globals): """ Transform this module into a celery config module by reading the mediagoblin config file. Set the environment variable MEDIAGOBLIN_CONFIG to specify where this config file is at and what section it uses. By default it defaults to 'mediagoblin.ini:app:mediagoblin'. The first colon ":" is a delimiter between the filename and the config section, so in this case the filename is 'mediagoblin.ini' and the section where mediagoblin is defined is 'app:mediagoblin'. Args: - 'setup_globals_func': this is for testing purposes only. Don't set this! """ mgoblin_conf_file, mgoblin_section = os.environ.get("MEDIAGOBLIN_CONFIG", "mediagoblin.ini:app:mediagoblin").split( ":", 1 ) if not os.path.exists(mgoblin_conf_file): raise IOError("MEDIAGOBLIN_CONFIG not set or file does not exist") parser = NicerConfigParser(mgoblin_conf_file) parser.read(mgoblin_conf_file) parser._defaults.setdefault("here", os.path.dirname(os.path.abspath(mgoblin_conf_file))) parser._defaults.setdefault("__file__", os.path.abspath(mgoblin_conf_file)) mgoblin_section = dict(parser.items(mgoblin_section)) mgoblin_conf = dict([(section_name, dict(parser.items(section_name))) for section_name in parser.sections()]) setup_celery_from_config(mgoblin_section, mgoblin_conf, settings_module=OUR_MODULENAME, set_environ=False) connection = mongokit.Connection(mgoblin_section.get("db_host"), mgoblin_section.get("db_port")) db = connection[mgoblin_section.get("db_name", "mediagoblin")] models.register_models(connection) # Set up the storage systems. public_store = storage.storage_system_from_paste_config(mgoblin_section, "publicstore") queue_store = storage.storage_system_from_paste_config(mgoblin_section, "queuestore") setup_globals_func(db_connection=connection, database=db, public_store=public_store, queue_store=queue_store)
def paste_app_factory(global_config, **app_config): # Get the database connection connection = mongokit.Connection(app_config.get('db_host'), app_config.get('db_port')) # Set up the storage systems. public_store = storage.storage_system_from_paste_config( app_config, 'publicstore') queue_store = storage.storage_system_from_paste_config( app_config, 'queuestore') # Set up the staticdirect system if app_config.has_key('direct_remote_path'): staticdirector = staticdirect.RemoteStaticDirect( app_config['direct_remote_path'].strip()) elif app_config.has_key('direct_remote_paths'): direct_remote_path_lines = app_config['direct_remote_paths'].strip( ).splitlines() staticdirector = staticdirect.MultiRemoteStaticDirect( dict([ line.strip().split(' ', 1) for line in direct_remote_path_lines ])) else: raise ImproperlyConfigured( "One of direct_remote_path or direct_remote_paths must be provided" ) setup_celery_from_config(app_config, global_config) mgoblin_app = MediaGoblinApp( connection, app_config.get('db_name', 'mediagoblin'), public_store=public_store, queue_store=queue_store, staticdirector=staticdirector, user_template_path=app_config.get('local_templates')) return mgoblin_app
def test_setup_celery_from_config(): def _wipe_testmodule_clean(module): vars_to_wipe = [ var for var in dir(module) if not var.startswith('__') and not var.endswith('__')] for var in vars_to_wipe: delattr(module, var) celery_setup.setup_celery_from_config( {}, {'something': {'or': 'other'}, 'celery': {'some_variable': 'floop', 'mail_port': '2000', 'CELERYD_ETA_SCHEDULER_PRECISION': '1.3', 'celery_result_persistent': 'true', 'celery_imports': 'foo.bar.baz this.is.an.import'}}, 'mediagoblin.tests.fake_celery_module', set_environ=False) from mediagoblin.tests import fake_celery_module assert fake_celery_module.SOME_VARIABLE == 'floop' assert fake_celery_module.MAIL_PORT == 2000 assert isinstance(fake_celery_module.MAIL_PORT, int) assert fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION == 1.3 assert isinstance(fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION, float) assert fake_celery_module.CELERY_RESULT_PERSISTENT is True assert fake_celery_module.CELERY_IMPORTS == [ 'foo.bar.baz', 'this.is.an.import'] assert fake_celery_module.CELERY_MONGODB_BACKEND_SETTINGS == { 'database': 'mediagoblin'} assert fake_celery_module.CELERY_RESULT_BACKEND == 'mongodb' assert fake_celery_module.BROKER_BACKEND == 'mongodb' _wipe_testmodule_clean(fake_celery_module) celery_setup.setup_celery_from_config( {'db_host': 'mongodb.example.org', 'db_port': '8080', 'db_name': 'captain_lollerskates', 'celery_section': 'vegetable'}, {'something': {'or': 'other'}, 'vegetable': {'some_variable': 'poolf', 'mail_port': '2020', 'CELERYD_ETA_SCHEDULER_PRECISION': '3.1', 'celery_result_persistent': 'false', 'celery_imports': 'baz.bar.foo import.is.a.this'}}, 'mediagoblin.tests.fake_celery_module', set_environ=False) from mediagoblin.tests import fake_celery_module assert fake_celery_module.SOME_VARIABLE == 'poolf' assert fake_celery_module.MAIL_PORT == 2020 assert isinstance(fake_celery_module.MAIL_PORT, int) assert fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION == 3.1 assert isinstance(fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION, float) assert fake_celery_module.CELERY_RESULT_PERSISTENT is False assert fake_celery_module.CELERY_IMPORTS == [ 'baz.bar.foo', 'import.is.a.this'] assert fake_celery_module.CELERY_MONGODB_BACKEND_SETTINGS == { 'database': 'captain_lollerskates', 'host': 'mongodb.example.org', 'port': 8080} assert fake_celery_module.CELERY_RESULT_BACKEND == 'mongodb' assert fake_celery_module.BROKER_BACKEND == 'mongodb' assert fake_celery_module.BROKER_HOST == 'mongodb.example.org' assert fake_celery_module.BROKER_PORT == 8080
def __init__(self, config_path, setup_celery=True): """ Initialize the application based on a configuration file. Arguments: - config_path: path to the configuration file we're opening. - setup_celery: whether or not to setup celery during init. (Note: setting 'celery_setup_elsewhere' also disables setting up celery.) """ ############## # Setup config ############## # Open and setup the config global_config, validation_result = read_mediagoblin_config(config_path) app_config = global_config['mediagoblin'] # report errors if necessary validation_report = generate_validation_report(global_config, validation_result) if validation_report: raise ImproperlyConfigured(validation_report) ########################################## # Setup other connections / useful objects ########################################## # Set up the database self.connection, self.db = setup_connection_and_db_from_config( app_config) # Get the template environment self.template_loader = util.get_jinja_loader( app_config.get('user_template_path')) # Set up storage systems self.public_store = storage.storage_system_from_config( app_config, 'publicstore') self.queue_store = storage.storage_system_from_config( app_config, 'queuestore') # set up routing self.routing = routing.get_mapper() # set up staticdirector tool if app_config.has_key('direct_remote_path'): self.staticdirector = staticdirect.RemoteStaticDirect( app_config['direct_remote_path'].strip()) elif app_config.has_key('direct_remote_paths'): direct_remote_path_lines = app_config['direct_remote_paths'].strip( ).splitlines() self.staticdirector = staticdirect.MultiRemoteStaticDirect( dict([ line.strip().split(' ', 1) for line in direct_remote_path_lines ])) else: raise ImproperlyConfigured("One of direct_remote_path or " "direct_remote_paths must be provided") # Setup celery, if appropriate if setup_celery and not app_config.get('celery_setup_elsewhere'): if os.environ.get('CELERY_ALWAYS_EAGER'): setup_celery_from_config(app_config, global_config, force_celery_always_eager=True) else: setup_celery_from_config(app_config, global_config) ####################################################### # Insert appropriate things into mediagoblin.mg_globals # # certain properties need to be accessed globally eg from # validators, etc, which might not access to the request # object. ####################################################### setup_globals( app_config=app_config, global_config=global_config, # TODO: No need to set these two up as globals, we could # just read them out of mg_globals.app_config email_sender_address=app_config['email_sender_address'], email_debug_mode=app_config['email_debug_mode'], # Actual, useful to everyone objects app=self, db_connection=self.connection, database=self.db, public_store=self.public_store, queue_store=self.queue_store, workbench_manager=WorkbenchManager(app_config['workbench_path']))
def __init__(self, config_path, setup_celery=True): """ Initialize the application based on a configuration file. Arguments: - config_path: path to the configuration file we're opening. - setup_celery: whether or not to setup celery during init. (Note: setting 'celery_setup_elsewhere' also disables setting up celery.) """ ############## # Setup config ############## # Open and setup the config global_config, validation_result = read_mediagoblin_config(config_path) app_config = global_config['mediagoblin'] # report errors if necessary validation_report = generate_validation_report( global_config, validation_result) if validation_report: raise ImproperlyConfigured(validation_report) ########################################## # Setup other connections / useful objects ########################################## # Set up the database self.connection, self.db = setup_connection_and_db_from_config( app_config) # Get the template environment self.template_loader = util.get_jinja_loader( app_config.get('user_template_path')) # Set up storage systems self.public_store = storage.storage_system_from_config( app_config, 'publicstore') self.queue_store = storage.storage_system_from_config( app_config, 'queuestore') # set up routing self.routing = routing.get_mapper() # set up staticdirector tool if app_config.has_key('direct_remote_path'): self.staticdirector = staticdirect.RemoteStaticDirect( app_config['direct_remote_path'].strip()) elif app_config.has_key('direct_remote_paths'): direct_remote_path_lines = app_config[ 'direct_remote_paths'].strip().splitlines() self.staticdirector = staticdirect.MultiRemoteStaticDirect( dict([line.strip().split(' ', 1) for line in direct_remote_path_lines])) else: raise ImproperlyConfigured( "One of direct_remote_path or " "direct_remote_paths must be provided") # Setup celery, if appropriate if setup_celery and not app_config.get('celery_setup_elsewhere'): if os.environ.get('CELERY_ALWAYS_EAGER'): setup_celery_from_config( app_config, global_config, force_celery_always_eager=True) else: setup_celery_from_config(app_config, global_config) ####################################################### # Insert appropriate things into mediagoblin.mg_globals # # certain properties need to be accessed globally eg from # validators, etc, which might not access to the request # object. ####################################################### setup_globals( app_config=app_config, global_config=global_config, # TODO: No need to set these two up as globals, we could # just read them out of mg_globals.app_config email_sender_address=app_config['email_sender_address'], email_debug_mode=app_config['email_debug_mode'], # Actual, useful to everyone objects app=self, db_connection=self.connection, database=self.db, public_store=self.public_store, queue_store=self.queue_store, workbench_manager=WorkbenchManager(app_config['workbench_path']))
def test_setup_celery_from_config(): def _wipe_testmodule_clean(module): vars_to_wipe = [ var for var in dir(module) if not var.startswith('__') and not var.endswith('__') ] for var in vars_to_wipe: delattr(module, var) global_config, validation_result = read_mediagoblin_config( TEST_CELERY_CONF_NOSPECIALDB) app_config = global_config['mediagoblin'] celery_setup.setup_celery_from_config( app_config, global_config, 'mediagoblin.tests.fake_celery_module', set_environ=False) from mediagoblin.tests import fake_celery_module assert fake_celery_module.SOME_VARIABLE == 'floop' assert fake_celery_module.MAIL_PORT == 2000 assert isinstance(fake_celery_module.MAIL_PORT, int) assert fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION == 1.3 assert isinstance(fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION, float) assert fake_celery_module.CELERY_RESULT_PERSISTENT is True assert fake_celery_module.CELERY_IMPORTS == [ 'foo.bar.baz', 'this.is.an.import', 'mediagoblin.process_media' ] assert fake_celery_module.CELERY_MONGODB_BACKEND_SETTINGS == { 'database': 'mediagoblin' } assert fake_celery_module.CELERY_RESULT_BACKEND == 'mongodb' assert fake_celery_module.BROKER_BACKEND == 'mongodb' _wipe_testmodule_clean(fake_celery_module) global_config, validation_result = read_mediagoblin_config( TEST_CELERY_CONF_MGSPECIALDB) app_config = global_config['mediagoblin'] celery_setup.setup_celery_from_config( app_config, global_config, 'mediagoblin.tests.fake_celery_module', set_environ=False) from mediagoblin.tests import fake_celery_module assert fake_celery_module.SOME_VARIABLE == 'poolf' assert fake_celery_module.MAIL_PORT == 2020 assert isinstance(fake_celery_module.MAIL_PORT, int) assert fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION == 3.1 assert isinstance(fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION, float) assert fake_celery_module.CELERY_RESULT_PERSISTENT is False assert fake_celery_module.CELERY_IMPORTS == [ 'baz.bar.foo', 'import.is.a.this', 'mediagoblin.process_media' ] assert fake_celery_module.CELERY_MONGODB_BACKEND_SETTINGS == { 'database': 'captain_lollerskates', 'host': 'mongodb.example.org', 'port': 8080 } assert fake_celery_module.CELERY_RESULT_BACKEND == 'mongodb' assert fake_celery_module.BROKER_BACKEND == 'mongodb' assert fake_celery_module.BROKER_HOST == 'mongodb.example.org' assert fake_celery_module.BROKER_PORT == 8080
def test_setup_celery_from_config(): def _wipe_testmodule_clean(module): vars_to_wipe = [ var for var in dir(module) if not var.startswith('__') and not var.endswith('__') ] for var in vars_to_wipe: delattr(module, var) celery_setup.setup_celery_from_config( {}, { 'something': { 'or': 'other' }, 'celery': { 'some_variable': 'floop', 'mail_port': '2000', 'CELERYD_ETA_SCHEDULER_PRECISION': '1.3', 'celery_result_persistent': 'true', 'celery_imports': 'foo.bar.baz this.is.an.import' } }, 'mediagoblin.tests.fake_celery_module', set_environ=False) from mediagoblin.tests import fake_celery_module assert fake_celery_module.SOME_VARIABLE == 'floop' assert fake_celery_module.MAIL_PORT == 2000 assert isinstance(fake_celery_module.MAIL_PORT, int) assert fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION == 1.3 assert isinstance(fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION, float) assert fake_celery_module.CELERY_RESULT_PERSISTENT is True assert fake_celery_module.CELERY_IMPORTS == [ 'foo.bar.baz', 'this.is.an.import' ] assert fake_celery_module.CELERY_MONGODB_BACKEND_SETTINGS == { 'database': 'mediagoblin' } assert fake_celery_module.CELERY_RESULT_BACKEND == 'mongodb' assert fake_celery_module.BROKER_BACKEND == 'mongodb' _wipe_testmodule_clean(fake_celery_module) celery_setup.setup_celery_from_config( { 'db_host': 'mongodb.example.org', 'db_port': '8080', 'db_name': 'captain_lollerskates', 'celery_section': 'vegetable' }, { 'something': { 'or': 'other' }, 'vegetable': { 'some_variable': 'poolf', 'mail_port': '2020', 'CELERYD_ETA_SCHEDULER_PRECISION': '3.1', 'celery_result_persistent': 'false', 'celery_imports': 'baz.bar.foo import.is.a.this' } }, 'mediagoblin.tests.fake_celery_module', set_environ=False) from mediagoblin.tests import fake_celery_module assert fake_celery_module.SOME_VARIABLE == 'poolf' assert fake_celery_module.MAIL_PORT == 2020 assert isinstance(fake_celery_module.MAIL_PORT, int) assert fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION == 3.1 assert isinstance(fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION, float) assert fake_celery_module.CELERY_RESULT_PERSISTENT is False assert fake_celery_module.CELERY_IMPORTS == [ 'baz.bar.foo', 'import.is.a.this' ] assert fake_celery_module.CELERY_MONGODB_BACKEND_SETTINGS == { 'database': 'captain_lollerskates', 'host': 'mongodb.example.org', 'port': 8080 } assert fake_celery_module.CELERY_RESULT_BACKEND == 'mongodb' assert fake_celery_module.BROKER_BACKEND == 'mongodb' assert fake_celery_module.BROKER_HOST == 'mongodb.example.org' assert fake_celery_module.BROKER_PORT == 8080