Esempio n. 1
0
def test_setup_celery_from_config():
    def _wipe_testmodule_clean(module):
        vars_to_wipe = [
            var for var in dir(module)
            if not var.startswith('__') and not var.endswith('__')]
        for var in vars_to_wipe:
            delattr(module, var)

    global_config, validation_result = read_mediagoblin_config(
        TEST_CELERY_CONF_NOSPECIALDB)
    app_config = global_config['mediagoblin']

    celery_setup.setup_celery_from_config(
        app_config, global_config,
        'mediagoblin.tests.fake_celery_module', set_environ=False)

    from mediagoblin.tests import fake_celery_module
    assert fake_celery_module.SOME_VARIABLE == 'floop'
    assert fake_celery_module.MAIL_PORT == 2000
    assert isinstance(fake_celery_module.MAIL_PORT, int)
    assert fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION == 1.3
    assert isinstance(fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION, float)
    assert fake_celery_module.CELERY_RESULT_PERSISTENT is True
    assert fake_celery_module.CELERY_IMPORTS == [
        'foo.bar.baz', 'this.is.an.import', 'mediagoblin.processing.task', \
        'mediagoblin.notifications.task', 'mediagoblin.submit.task']
    assert fake_celery_module.CELERY_RESULT_BACKEND == 'database'
    assert fake_celery_module.CELERY_RESULT_DBURI == (
        'sqlite:///' +
        pkg_resources.resource_filename('mediagoblin.tests', 'celery.db'))

    assert fake_celery_module.BROKER_TRANSPORT == 'sqlalchemy'
    assert fake_celery_module.BROKER_URL == (
        'sqlite:///' +
        pkg_resources.resource_filename('mediagoblin.tests', 'kombu.db'))
Esempio n. 2
0
def test_setup_celery_from_config():
    def _wipe_testmodule_clean(module):
        vars_to_wipe = [
            var for var in dir(module)
            if not var.startswith('__') and not var.endswith('__')
        ]
        for var in vars_to_wipe:
            delattr(module, var)

    global_config, validation_result = read_mediagoblin_config(
        TEST_CELERY_CONF_NOSPECIALDB)
    app_config = global_config['mediagoblin']

    celery_setup.setup_celery_from_config(
        app_config,
        global_config,
        'mediagoblin.tests.fake_celery_module',
        set_environ=False)

    from mediagoblin.tests import fake_celery_module
    assert fake_celery_module.SOME_VARIABLE == 'floop'
    assert fake_celery_module.MAIL_PORT == 2000
    assert isinstance(fake_celery_module.MAIL_PORT, int)
    assert fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION == 1.3
    assert isinstance(fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION,
                      float)
    assert fake_celery_module.CELERY_RESULT_PERSISTENT is True
    assert fake_celery_module.CELERY_IMPORTS == [
        'foo.bar.baz', 'this.is.an.import', 'mediagoblin.processing.task',
        'mediagoblin.notifications.task'
    ]
    assert fake_celery_module.CELERY_RESULT_BACKEND == 'database'
    assert fake_celery_module.CELERY_RESULT_DBURI == (
        'sqlite:///' +
        pkg_resources.resource_filename('mediagoblin.tests', 'celery.db'))

    assert fake_celery_module.BROKER_TRANSPORT == 'sqlalchemy'
    assert fake_celery_module.BROKER_URL == (
        'sqlite:///' +
        pkg_resources.resource_filename('mediagoblin.tests', 'kombu.db'))
Esempio n. 3
0
def setup_self(check_environ_for_conf=True, module_name=OUR_MODULENAME,
               default_conf_file=None):
    """
    Transform this module into a celery config module by reading the
    mediagoblin config file.  Set the environment variable
    MEDIAGOBLIN_CONFIG to specify where this config file is.

    By default it defaults to 'mediagoblin.ini'.

    Note that if celery_setup_elsewhere is set in your config file,
    this simply won't work.
    """
    if not default_conf_file:
        if os.path.exists(os.path.abspath('mediagoblin_local.ini')):
            default_conf_file = 'mediagoblin_local.ini'
        else:
            default_conf_file = 'mediagoblin.ini'

    if check_environ_for_conf:
        mgoblin_conf_file = os.path.abspath(
            os.environ.get('MEDIAGOBLIN_CONFIG', default_conf_file))
    else:
        mgoblin_conf_file = default_conf_file

    if not os.path.exists(mgoblin_conf_file):
        raise IOError(
            "MEDIAGOBLIN_CONFIG not set or file does not exist")

    # By setting the environment variable here we should ensure that
    # this is the module that gets set up.
    os.environ['CELERY_CONFIG_MODULE'] = module_name
    app.MediaGoblinApp(mgoblin_conf_file, setup_celery=False)

    setup_celery_from_config(
        mg_globals.app_config, mg_globals.global_config,
        settings_module=module_name,
        set_environ=False)
Esempio n. 4
0
def setup_self(check_environ_for_conf=True,
               module_name=OUR_MODULENAME,
               default_conf_file=None):
    """
    Transform this module into a celery config module by reading the
    mediagoblin config file.  Set the environment variable
    MEDIAGOBLIN_CONFIG to specify where this config file is.

    By default it defaults to 'mediagoblin.ini'.

    Note that if celery_setup_elsewhere is set in your config file,
    this simply won't work.
    """
    if not default_conf_file:
        if os.path.exists(os.path.abspath('mediagoblin_local.ini')):
            default_conf_file = 'mediagoblin_local.ini'
        else:
            default_conf_file = 'mediagoblin.ini'

    if check_environ_for_conf:
        mgoblin_conf_file = os.path.abspath(
            os.environ.get('MEDIAGOBLIN_CONFIG', default_conf_file))
    else:
        mgoblin_conf_file = default_conf_file

    if not os.path.exists(mgoblin_conf_file):
        raise OSError("MEDIAGOBLIN_CONFIG not set or file does not exist")

    # By setting the environment variable here we should ensure that
    # this is the module that gets set up.
    os.environ['CELERY_CONFIG_MODULE'] = module_name
    app.MediaGoblinApp(mgoblin_conf_file, setup_celery=False)

    setup_celery_from_config(mg_globals.app_config,
                             mg_globals.global_config,
                             settings_module=module_name,
                             set_environ=False)
Esempio n. 5
0
    def __init__(self, config_path, setup_celery=True):
        """
        Initialize the application based on a configuration file.

        Arguments:
         - config_path: path to the configuration file we're opening.
         - setup_celery: whether or not to setup celery during init.
           (Note: setting 'celery_setup_elsewhere' also disables
           setting up celery.)
        """
        _log.info("GNU MediaGoblin %s main server starting", __version__)
        _log.debug("Using config file %s", config_path)
        ##############
        # Setup config
        ##############

        # Open and setup the config
        global_config, app_config = setup_global_and_app_config(config_path)

        media_type_warning()

        setup_crypto()

        ##########################################
        # Setup other connections / useful objects
        ##########################################

        # Setup Session Manager, not needed in celery
        self.session_manager = session.SessionManager()

        # load all available locales
        setup_locales()

        # Set up plugins -- need to do this early so that plugins can
        # affect startup.
        _log.info("Setting up plugins.")
        setup_plugins()

        # Set up the database
        self.db = setup_database(app_config['run_migrations'])

        # Quit app if need to run dbupdate
        check_db_up_to_date()

        # Register themes
        self.theme_registry, self.current_theme = register_themes(app_config)

        # Get the template environment
        self.template_loader = get_jinja_loader(
            app_config.get('local_templates'),
            self.current_theme,
            PluginManager().get_template_paths()
            )

        # Check if authentication plugin is enabled and respond accordingly.
        self.auth = check_auth_enabled()
        if not self.auth:
            app_config['allow_comments'] = False

        # Set up storage systems
        self.public_store, self.queue_store = setup_storage()

        # set up routing
        self.url_map = get_url_map()

        # set up staticdirector tool
        self.staticdirector = get_staticdirector(app_config)

        # Setup celery, if appropriate
        if setup_celery and not app_config.get('celery_setup_elsewhere'):
            if os.environ.get('CELERY_ALWAYS_EAGER', 'false').lower() == 'true':
                setup_celery_from_config(
                    app_config, global_config,
                    force_celery_always_eager=True)
            else:
                setup_celery_from_config(app_config, global_config)

        #######################################################
        # Insert appropriate things into mediagoblin.mg_globals
        #
        # certain properties need to be accessed globally eg from
        # validators, etc, which might not access to the request
        # object.
        #######################################################

        setup_globals(app=self)

        # Workbench *currently* only used by celery, so this only
        # matters in always eager mode :)
        setup_workbench()

        # instantiate application meddleware
        self.meddleware = [common.import_component(m)(self)
                           for m in meddleware.ENABLED_MEDDLEWARE]