def main(self, args, db, global_config):
options = self.parser.parse_args(args.args_for_alembic)
# This is useful for people running commands involving plugins, such
# as building autogenerated revisions.
if args.with_plugins:
plugins = global_config.get('plugins', {}).keys()
for plugin in plugins:
try:
import_component('{0}.models:MODELS'.format(plugin))
except ImportError:
# It doesn't really matter if there's no models to import
# here.
pass
# This code is inspired by a hack in Alembic, but isn't the same really.
# Regardless, Alembic is Expat licensed.
if not hasattr(options, "cmd"):
print(
"* Only use this command if you know what you are doing! *\n"
"If not, use the 'gmg dbupdate' command instead.\n\n"
"(You can also pass --with-plugins here.)"
"Alembic help:\n")
self.parser.print_help()
return
else:
Session = sessionmaker(bind=db.engine)
session = Session()
cfg = build_alembic_config(global_config, options, session)
self.run_cmd(cfg, options)
def main_cli():
parser = argparse.ArgumentParser(
description='GNU MediaGoblin utilities.')
parser.add_argument(
'-cf', '--conf_file', default=None,
help=(
"Config file used to set up environment. "
"Default to mediagoblin_local.ini if readable, "
"otherwise mediagoblin.ini"))
subparsers = parser.add_subparsers(help='sub-command help')
for command_name, command_struct in SUBCOMMAND_MAP.iteritems():
if 'help' in command_struct:
subparser = subparsers.add_parser(
command_name, help=command_struct['help'])
else:
subparser = subparsers.add_parser(command_name)
setup_func = import_component(command_struct['setup'])
exec_func = import_component(command_struct['func'])
setup_func(subparser)
subparser.set_defaults(func=exec_func)
args = parser.parse_args()
args.orig_conf_file = args.conf_file
if args.conf_file is None:
if os.path.exists('mediagoblin_local.ini') \
and os.access('mediagoblin_local.ini', os.R_OK):
args.conf_file = 'mediagoblin_local.ini'
else:
args.conf_file = 'mediagoblin.ini'
args.func(args)
def main(self, args, db, global_config):
options = self.parser.parse_args(args.args_for_alembic)
# This is useful for people running commands involving plugins, such
# as building autogenerated revisions.
if args.with_plugins:
plugins = global_config.get('plugins', {}).keys()
for plugin in plugins:
try:
import_component('{0}.models:MODELS'.format(plugin))
except ImportError:
# It doesn't really matter if there's no models to import
# here.
pass
# This code is inspired by a hack in Alembic, but isn't the same really.
# Regardless, Alembic is Expat licensed.
if not hasattr(options, "cmd"):
print("* Only use this command if you know what you are doing! *\n"
"If not, use the 'gmg dbupdate' command instead.\n\n"
"(You can also pass --with-plugins here.)"
"Alembic help:\n")
self.parser.print_help()
return
else:
Session = sessionmaker(bind=db.engine)
session = Session()
cfg = build_alembic_config(global_config, options, session)
self.run_cmd(cfg, options)
def main_cli():
parser = argparse.ArgumentParser(
description='GNU MediaGoblin utilities.')
parser.add_argument(
'-cf', '--conf_file', default=None,
help=(
"Config file used to set up environment. "
"Default to mediagoblin_local.ini if readable, "
"otherwise mediagoblin.ini"))
subparsers = parser.add_subparsers(help='sub-command help')
for command_name, command_struct in SUBCOMMAND_MAP.iteritems():
if 'help' in command_struct:
subparser = subparsers.add_parser(
command_name, help=command_struct['help'])
else:
subparser = subparsers.add_parser(command_name)
setup_func = import_component(command_struct['setup'])
exec_func = import_component(command_struct['func'])
setup_func(subparser)
subparser.set_defaults(func=exec_func)
args = parser.parse_args()
args.orig_conf_file = args.conf_file
if args.conf_file is None:
if os.path.exists('mediagoblin_local.ini') \
and os.access('mediagoblin_local.ini', os.R_OK):
args.conf_file = 'mediagoblin_local.ini'
else:
args.conf_file = 'mediagoblin.ini'
args.func(args)
def gather_database_data(plugins):
"""
Gather all database data relevant to the extensions we have
installed so we can do migrations and table initialization.
Returns a list of DatabaseData objects.
"""
managed_dbdata = []
# Add main first
from mediagoblin.db.models import MODELS as MAIN_MODELS
from mediagoblin.db.migrations import MIGRATIONS as MAIN_MIGRATIONS
from mediagoblin.db.models import FOUNDATIONS as MAIN_FOUNDATIONS
managed_dbdata.append(
DatabaseData(u'__main__', MAIN_MODELS, MAIN_FOUNDATIONS,
MAIN_MIGRATIONS))
for plugin in plugins:
try:
models = import_component('{0}.models:MODELS'.format(plugin))
except ImportError as exc:
_log.debug('No models found for {0}: {1}'.format(plugin, exc))
models = []
except AttributeError as exc:
_log.warning('Could not find MODELS in {0}.models, have you \
forgotten to add it? ({1})'.format(plugin, exc))
models = []
try:
migrations = import_component(
'{0}.migrations:MIGRATIONS'.format(plugin))
except ImportError as exc:
_log.debug('No migrations found for {0}: {1}'.format(plugin, exc))
migrations = {}
except AttributeError as exc:
_log.debug('Could not find MIGRATIONS in {0}.migrations, have you \
forgotten to add it? ({1})'.format(plugin, exc))
migrations = {}
try:
foundations = import_component(
'{0}.models:FOUNDATIONS'.format(plugin))
except ImportError as exc:
_log.debug('No foundations found for {0}: {1}'.format(plugin, exc))
foundations = {}
except AttributeError as exc:
_log.debug('Could not find FOUNDATIONS in {0}.models, have you \
forgotten to add it? ({1})'.format(plugin, exc))
foundations = {}
if models:
managed_dbdata.append(
DatabaseData(plugin, models, foundations, migrations))
return managed_dbdata
def gather_database_data(plugins):
"""
Gather all database data relevant to the extensions we have
installed so we can do migrations and table initialization.
Returns a list of DatabaseData objects.
"""
managed_dbdata = []
# Add main first
from mediagoblin.db.models import MODELS as MAIN_MODELS
from mediagoblin.db.migrations import MIGRATIONS as MAIN_MIGRATIONS
from mediagoblin.db.models import FOUNDATIONS as MAIN_FOUNDATIONS
managed_dbdata.append(
DatabaseData(
u'__main__', MAIN_MODELS, MAIN_FOUNDATIONS, MAIN_MIGRATIONS))
for plugin in plugins:
try:
models = import_component('{0}.models:MODELS'.format(plugin))
except ImportError as exc:
_log.debug('No models found for {0}: {1}'.format(
plugin,
exc))
models = []
except AttributeError as exc:
_log.warning('Could not find MODELS in {0}.models, have you \
forgotten to add it? ({1})'.format(plugin, exc))
models = []
try:
migrations = import_component('{0}.migrations:MIGRATIONS'.format(
plugin))
except ImportError as exc:
_log.debug('No migrations found for {0}: {1}'.format(
plugin,
exc))
migrations = {}
except AttributeError as exc:
_log.debug('Could not find MIGRATIONS in {0}.migrations, have you \
forgotten to add it? ({1})'.format(plugin, exc))
migrations = {}
try:
foundations = import_component('{0}.models:FOUNDATIONS'.format(plugin))
except ImportError as exc:
foundations = {}
except AttributeError as exc:
foundations = {}
if models:
managed_dbdata.append(
DatabaseData(plugin, models, foundations, migrations))
return managed_dbdata
def main_cli():
parser = argparse.ArgumentParser(
description='GNU MediaGoblin utilities.')
parser.add_argument(
'-cf', '--conf_file', default=None,
help=(
"Config file used to set up environment. "
"Default to mediagoblin_local.ini if readable, "
"otherwise mediagoblin.ini"))
subparsers = parser.add_subparsers(help='sub-command help')
for command_name, command_struct in six.iteritems(SUBCOMMAND_MAP):
if 'help' in command_struct:
subparser = subparsers.add_parser(
command_name, help=command_struct['help'])
else:
subparser = subparsers.add_parser(command_name)
setup_func = import_component(command_struct['setup'])
exec_func = import_component(command_struct['func'])
setup_func(subparser)
subparser.set_defaults(func=exec_func)
args = parser.parse_args()
args.orig_conf_file = args.conf_file
if args.conf_file is None:
if os.path.exists('mediagoblin_local.ini') \
and os.access('mediagoblin_local.ini', os.R_OK):
args.conf_file = 'mediagoblin_local.ini'
else:
args.conf_file = 'mediagoblin.ini'
# This is a hopefully TEMPORARY hack for adding a mediagoblin.ini
# if none exists, to make up for a deficiency as we are migrating
# our docs to the new "no mediagoblin.ini by default" workflow.
# Normally, the docs should provide instructions for this, but
# since 0.7.1 docs say "install from master!" and yet we removed
# mediagoblin.ini, we need to cover our bases...
parent_directory = os.path.split(os.path.abspath(args.conf_file))[0]
if os.path.split(args.conf_file)[1] == 'mediagoblin.ini' \
and not os.path.exists(args.conf_file) \
and os.path.exists(
os.path.join(
parent_directory, 'mediagoblin.example.ini')):
# Do the copy-over of the mediagoblin config for the user
_log.warning(
"No mediagoblin.ini found and no other path given, "
"so making one for you.")
shutil.copy(
os.path.join(parent_directory, "mediagoblin.example.ini"),
os.path.join(parent_directory, "mediagoblin.ini"))
args.func(args)
def main_cli():
parser = argparse.ArgumentParser(description='GNU MediaGoblin utilities.')
parser.add_argument('-cf',
'--conf_file',
default=None,
help=("Config file used to set up environment. "
"Default to mediagoblin_local.ini if readable, "
"otherwise mediagoblin.ini"))
subparsers = parser.add_subparsers(help='sub-command help')
for command_name, command_struct in six.iteritems(SUBCOMMAND_MAP):
if 'help' in command_struct:
subparser = subparsers.add_parser(command_name,
help=command_struct['help'])
else:
subparser = subparsers.add_parser(command_name)
setup_func = import_component(command_struct['setup'])
exec_func = import_component(command_struct['func'])
setup_func(subparser)
subparser.set_defaults(func=exec_func)
args = parser.parse_args()
args.orig_conf_file = args.conf_file
if args.conf_file is None:
if os.path.exists('mediagoblin_local.ini') \
and os.access('mediagoblin_local.ini', os.R_OK):
args.conf_file = 'mediagoblin_local.ini'
else:
args.conf_file = 'mediagoblin.ini'
# This is a hopefully TEMPORARY hack for adding a mediagoblin.ini
# if none exists, to make up for a deficiency as we are migrating
# our docs to the new "no mediagoblin.ini by default" workflow.
# Normally, the docs should provide instructions for this, but
# since 0.7.1 docs say "install from master!" and yet we removed
# mediagoblin.ini, we need to cover our bases...
parent_directory = os.path.split(os.path.abspath(args.conf_file))[0]
if os.path.split(args.conf_file)[1] == 'mediagoblin.ini' \
and not os.path.exists(args.conf_file) \
and os.path.exists(
os.path.join(
parent_directory, 'mediagoblin.example.ini')):
# Do the copy-over of the mediagoblin config for the user
_log.warning("No mediagoblin.ini found and no other path given, "
"so making one for you.")
shutil.copy(os.path.join(parent_directory, "mediagoblin.example.ini"),
os.path.join(parent_directory, "mediagoblin.ini"))
try:
args.func(args)
except AttributeError: # no subcommand or no func of subcommand
parser.print_help()
def endpoint_to_controller(rule):
endpoint = rule.endpoint
view_func = rule.gmg_controller
_log.debug('endpoint: {0} view_func: {1}'.format(endpoint, view_func))
# import the endpoint, or if it's already a callable, call that
if isinstance(view_func, six.string_types):
view_func = import_component(view_func)
rule.gmg_controller = view_func
return view_func
def endpoint_to_controller(rule):
endpoint = rule.endpoint
view_func = rule.gmg_controller
_log.debug('endpoint: {0} view_func: {1}'.format(endpoint, view_func))
# import the endpoint, or if it's already a callable, call that
if isinstance(view_func, basestring):
view_func = import_component(view_func)
rule.gmg_controller = view_func
return view_func
def get_fail_exception(self):
"""
Get the exception that's appropriate for this error
"""
if self.fail_error:
try:
return common.import_component(self.fail_error)
except ImportError:
# TODO(breton): fail_error should give some hint about why it
# failed. fail_error is used as a path to import().
# Unfortunately, I didn't know about that and put general error
# message there. Maybe it's for the best, because for admin,
# we could show even some raw python things. Anyway, this
# should be properly resolved. Now we are in a freeze, that's
# why I simply catch ImportError.
return None
def get_fail_exception(self):
"""
Get the exception that's appropriate for this error
"""
if self.fail_error:
try:
return common.import_component(self.fail_error)
except ImportError:
# TODO(breton): fail_error should give some hint about why it
# failed. fail_error is used as a path to import().
# Unfortunately, I didn't know about that and put general error
# message there. Maybe it's for the best, because for admin,
# we could show even some raw python things. Anyway, this
# should be properly resolved. Now we are in a freeze, that's
# why I simply catch ImportError.
return None
def media_data_init(self, **kwargs):
"""
Initialize or update the contents of a media entry's media_data row
"""
media_data = self.media_data
if media_data is None:
# Get the correct table:
table = import_component(self.media_type + '.models:DATA_MODEL')
# No media data, so actually add a new one
media_data = table(**kwargs)
# Get the relationship set up.
media_data.get_media_entry = self
else:
# Update old media data
for field, value in kwargs.iteritems():
setattr(media_data, field, value)
def media_data_init(self, **kwargs):
"""
Initialize or update the contents of a media entry's media_data row
"""
media_data = self.media_data
if media_data is None:
# Get the correct table:
table = import_component(self.media_type + '.models:DATA_MODEL')
# No media data, so actually add a new one
media_data = table(**kwargs)
# Get the relationship set up.
media_data.get_media_entry = self
else:
# Update old media data
for field, value in kwargs.iteritems():
setattr(media_data, field, value)
def storage_system_from_config(config_section):
"""
Utility for setting up a storage system from a config section.
Note that a special argument may be passed in to
the config_section which is "storage_class" which will provide an
import path to a storage system. This defaults to
"mediagoblin.storage:BasicFileStorage" if otherwise undefined.
Arguments:
- config_section: dictionary of config parameters
Returns:
An instantiated storage system.
Example:
storage_system_from_config(
{'base_url': '/media/',
'base_dir': '/var/whatever/media/'})
Will return:
BasicFileStorage(
base_url='/media/',
base_dir='/var/whatever/media')
"""
# This construct is needed, because dict(config) does
# not replace the variables in the config items.
config_params = dict(config_section.iteritems())
if 'storage_class' in config_params:
storage_class = config_params['storage_class']
config_params.pop('storage_class')
else:
storage_class = 'mediagoblin.storage.filestorage:BasicFileStorage'
storage_class = common.import_component(storage_class)
return storage_class(**config_params)
def storage_system_from_config(config_section):
"""
Utility for setting up a storage system from a config section.
Note that a special argument may be passed in to
the config_section which is "storage_class" which will provide an
import path to a storage system. This defaults to
"mediagoblin.storage:BasicFileStorage" if otherwise undefined.
Arguments:
- config_section: dictionary of config parameters
Returns:
An instantiated storage system.
Example:
storage_system_from_config(
{'base_url': '/media/',
'base_dir': '/var/whatever/media/'})
Will return:
BasicFileStorage(
base_url='/media/',
base_dir='/var/whatever/media')
"""
# This construct is needed, because dict(config) does
# not replace the variables in the config items.
config_params = dict(six.iteritems(config_section))
if 'storage_class' in config_params:
storage_class = config_params['storage_class']
config_params.pop('storage_class')
else:
storage_class = 'mediagoblin.storage.filestorage:BasicFileStorage'
storage_class = common.import_component(storage_class)
return storage_class(**config_params)
def run_foundations(db, global_config):
"""
Gather foundations data and run it.
"""
from mediagoblin.db.models import FOUNDATIONS as MAIN_FOUNDATIONS
all_foundations = [(u"__main__", MAIN_FOUNDATIONS)]
Session = sessionmaker(bind=db.engine)
session = Session()
plugins = global_config.get('plugins', {})
for plugin in plugins:
try:
foundations = import_component(
'{0}.models:FOUNDATIONS'.format(plugin))
all_foundations.append((plugin, foundations))
except ImportError as exc:
continue
except AttributeError as exc:
continue
for name, foundations in all_foundations:
populate_table_foundations(session, foundations, name)
def run_foundations(db, global_config):
"""
Gather foundations data and run it.
"""
from mediagoblin.db.models import FOUNDATIONS as MAIN_FOUNDATIONS
all_foundations = [("__main__", MAIN_FOUNDATIONS)]
Session = sessionmaker(bind=db.engine)
session = Session()
plugins = global_config.get('plugins', {})
for plugin in plugins:
try:
foundations = import_component(
'{}.models:FOUNDATIONS'.format(plugin))
all_foundations.append((plugin, foundations))
except ImportError as exc:
continue
except AttributeError as exc:
continue
for name, foundations in all_foundations:
populate_table_foundations(session, foundations, name)
def __init__(self, config_path, setup_celery=True):
"""
Initialize the application based on a configuration file.
Arguments:
- config_path: path to the configuration file we're opening.
- setup_celery: whether or not to setup celery during init.
(Note: setting 'celery_setup_elsewhere' also disables
setting up celery.)
"""
_log.info("GNU MediaGoblin %s main server starting", __version__)
_log.debug("Using config file %s", config_path)
##############
# Setup config
##############
# Open and setup the config
global_config, app_config = setup_global_and_app_config(config_path)
media_type_warning()
setup_crypto()
##########################################
# Setup other connections / useful objects
##########################################
# Setup Session Manager, not needed in celery
self.session_manager = session.SessionManager()
# load all available locales
setup_locales()
# Set up plugins -- need to do this early so that plugins can
# affect startup.
_log.info("Setting up plugins.")
setup_plugins()
# Set up the database
self.db = setup_database(app_config['run_migrations'])
# Quit app if need to run dbupdate
check_db_up_to_date()
# Register themes
self.theme_registry, self.current_theme = register_themes(app_config)
# Get the template environment
self.template_loader = get_jinja_loader(
app_config.get('local_templates'),
self.current_theme,
PluginManager().get_template_paths()
)
# Check if authentication plugin is enabled and respond accordingly.
self.auth = check_auth_enabled()
if not self.auth:
app_config['allow_comments'] = False
# Set up storage systems
self.public_store, self.queue_store = setup_storage()
# set up routing
self.url_map = get_url_map()
# set up staticdirector tool
self.staticdirector = get_staticdirector(app_config)
# Setup celery, if appropriate
if setup_celery and not app_config.get('celery_setup_elsewhere'):
if os.environ.get('CELERY_ALWAYS_EAGER', 'false').lower() == 'true':
setup_celery_from_config(
app_config, global_config,
force_celery_always_eager=True)
else:
setup_celery_from_config(app_config, global_config)
#######################################################
# Insert appropriate things into mediagoblin.mg_globals
#
# certain properties need to be accessed globally eg from
# validators, etc, which might not access to the request
# object.
#######################################################
setup_globals(app=self)
# Workbench *currently* only used by celery, so this only
# matters in always eager mode :)
setup_workbench()
# instantiate application meddleware
self.meddleware = [common.import_component(m)(self)
for m in meddleware.ENABLED_MEDDLEWARE]
def media_data_ref(self):
return import_component(self.media_type + '.models:BACKREF_NAME')
def test_import_component():
imported_func = common.import_component(
'mediagoblin.tests.test_util:_import_component_testing_method')
result = imported_func('hooobaladoobala')
expected = u"'hooobaladoobala' is the silliest string I've ever seen"
assert result == expected
def get_fail_exception(self):
"""
Get the exception that's appropriate for this error
"""
if self.fail_error:
return common.import_component(self.fail_error)
def get_fail_exception(self):
"""
Get the exception that's appropriate for this error
"""
if self.fail_error:
return common.import_component(self.fail_error)
def media_data_ref(self):
return import_component(self.media_type + '.models:BACKREF_NAME')
def test_import_component():
imported_func = common.import_component(
'mediagoblin.tests.test_util:_import_component_testing_method')
result = imported_func('hooobaladoobala')
expected = u"'hooobaladoobala' is the silliest string I've ever seen"
assert result == expected
def call_backend(self, environ, start_response):
request = Request(environ)
## Compatibility webob -> werkzeug
request.GET = request.args
request.accept_language = request.accept_languages
request.accept = request.accept_mimetypes
## Routing / controller loading stuff
path_info = request.path
route_match = self.routing.match(path_info)
# By using fcgi, mediagoblin can run under a base path
# like /mediagoblin/. request.path_info contains the
# path inside mediagoblin. If the something needs the
# full path of the current page, that should include
# the basepath.
# Note: urlgen and routes are fine!
request.full_path = environ["SCRIPT_NAME"] + request.path
# python-routes uses SCRIPT_NAME. So let's use that too.
# The other option would be:
# request.full_path = environ["SCRIPT_URL"]
# Fix up environ for urlgen
# See bug: https://bitbucket.org/bbangert/routes/issue/55/cache_hostinfo-breaks-on-https-off
if environ.get('HTTPS', '').lower() == 'off':
environ.pop('HTTPS')
## Attach utilities to the request object
request.matchdict = route_match
request.urlgen = routes.URLGenerator(self.routing, environ)
# Do we really want to load this via middleware? Maybe?
request.session = request.environ['beaker.session']
# Attach self as request.app
# Also attach a few utilities from request.app for convenience?
request.app = self
request.locale = translate.get_locale_from_request(request)
request.template_env = template.get_jinja_env(
self.template_loader, request.locale)
request.db = self.db
request.staticdirect = self.staticdirector
mg_request.setup_user_in_request(request)
# No matching page?
if route_match is None:
# Try to do see if we have a match with a trailing slash
# added and if so, redirect
if not path_info.endswith('/') \
and request.method == 'GET' \
and self.routing.match(path_info + '/'):
new_path_info = path_info + '/'
if request.GET:
new_path_info = '%s?%s' % (
new_path_info, urllib.urlencode(request.GET))
redirect = exc.HTTPFound(location=new_path_info)
return request.get_response(redirect)(environ, start_response)
# Okay, no matches. 404 time!
request.matchdict = {} # in case our template expects it
return render_404(request)(environ, start_response)
# import the controller, or if it's already a callable, call that
route_controller = route_match['controller']
if isinstance(route_controller, unicode) \
or isinstance(route_controller, str):
controller = common.import_component(route_match['controller'])
else:
controller = route_match['controller']
# pass the request through our meddleware classes
for m in self.meddleware:
response = m.process_request(request, controller)
if response is not None:
return response(environ, start_response)
request.start_response = start_response
# get the response from the controller
response = controller(request)
# pass the response through the meddleware
for m in self.meddleware[::-1]:
m.process_response(request, response)
return response(environ, start_response)
