def main():
if len(sys.argv) < 2:
print("Usage: exporter postgresql://<sqlalchemy_conn>")
return 1
engine = sqlalchemy.create_engine(sys.argv[1])
Sessionmaker = sqlalchemy.orm.sessionmaker(bind=engine)
l_session = Sessionmaker()
config = flask.Config(".")
config.from_envvar("GOGDB_CONFIG")
n_db = gogdb.core.storage.Storage(config["STORAGE_PATH"])
exported_ids = set()
l_products = l_session.query(legacy_model.Product)
for l_prod in l_products:
print("Converting", l_prod.id)
n_prod = convert_product(l_prod)
n_prices = convert_prices(l_prod)
n_changes = convert_changelog(l_prod)
exported_ids.add(n_prod.id)
n_db.product.save(n_prod, n_prod.id)
n_db.prices.save(n_prices, n_prod.id)
n_db.changelog.save(n_changes, n_prod.id)
n_db.ids.save(list(exported_ids))
def get_bundle_config(
self,
bundle: Bundle,
env: Union[DEV, PROD, STAGING, TEST],
) -> flask.Config:
bundle_config_module = self.import_bundle_module(bundle)
base_config = getattr(bundle_config_module, BASE_CONFIG_CLASS, None)
env_config = getattr(bundle_config_module, ENV_CONFIG_CLASSES[env],
None)
if (isinstance(bundle, AppBundle) and
(not base_config or not issubclass(base_config, AppBundleConfig))):
raise Exception(
f"Could not find an AppBundleConfig subclass named "
f"{BASE_CONFIG_CLASS} in your app bundle's "
f"{self.get_module_name(bundle)} module.")
merged = flask.Config(None)
for config in [base_config, env_config]:
if config:
merged.from_object(config)
if isinstance(bundle, AppBundle) and 'SECRET_KEY' not in merged:
raise Exception(
f"The `SECRET_KEY` config option is required. Please "
f"set it in your app bundle's {BASE_CONFIG_CLASS} class.")
return merged
def app(request, tmpdir_factory):
"""Session-wide test `Flask` application."""
cfg = flask.Config(p.join(p.dirname(tmaps.__file__), p.pardir))
if not 'TMAPS_SETTINGS_TEST' in os.environ:
print('No test config specified by the environment variable'
' `TMAPS_SETTINGS_TEST`! Make sure that this config'
' exists and that it specifies a database different'
' from the one used normally! The database will be wiped'
' during testing!')
sys.exit(1)
else:
print('Loading server config from: %s' %
os.environ['TMAPS_SETTINGS_TEST'])
cfg.from_envvar('TMAPS_SETTINGS_TEST')
cfg['GC3PIE_SESSION_DIR'] = str(tmpdir_factory.mktemp('gc3pie'))
cfg['TMAPS_STORAGE'] = str(tmpdir_factory.mktemp('experiments'))
app = create_app(config_overrides=cfg)
# Establish an application context before running the tests.
ctx = app.app_context()
ctx.push()
yield app
ctx.pop()
def load_server_config():
server_config = flask.Config('.')
if 'PROD' in os.environ:
server_config.from_object('prod_config')
else:
server_config.from_object('settings.config')
return server_config
def __init__(self, import_name=__package__, config=None, **kwargs):
if not getattr(self, 'settings'):
self.settings = flask.Config('.')
super(NewsroomNewsAPI, self).__init__(import_name=import_name,
config=config,
**kwargs)
def main():
config = flask.Config(".")
config.from_envvar("GOGDB_CONFIG")
db = storage.Storage(config["STORAGE_PATH"])
logging.basicConfig()
logger.setLevel(config.get("UPDATER_LOGLEVEL", logging.NOTSET))
logging.getLogger("UpdateDB.session").setLevel(config.get("SESSION_LOGLEVEL", logging.NOTSET))
tasks = sys.argv[1:]
if not tasks:
eprint("Updater missing task argument: [all, download, index, startpage, charts]")
exit(1)
if "all" in tasks:
tasks = ["download", "index", "startpage", "charts"]
if "download" in tasks:
asyncio.run(download_main(db, config))
processors = []
if "index" in tasks:
processors.append(IndexDbProcessor(db))
if "startpage" in tasks:
processors.append(StartpageProcessor(db))
if "charts" in tasks:
processors.append(ChartsProcessor(db))
if processors:
asyncio.run(processors_main(db, processors))
def app_config(
app,
settings="fence.settings",
root_dir=None,
config_path=None,
file_name=None,
):
"""
Set up the config for the Flask app.
"""
if root_dir is None:
root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
logger.info("Loading settings...")
# not using app.config.from_object because we don't want all the extra flask cfg
# vars inside our singleton when we pass these through in the next step
settings_cfg = flask.Config(app.config.root_path)
settings_cfg.from_object(settings)
# dump the settings into the config singleton before loading a configuration file
config.update(dict(settings_cfg))
# load the configuration file, this overwrites anything from settings/local_settings
config.load(
config_path=config_path,
search_folders=CONFIG_SEARCH_FOLDERS,
file_name=file_name,
)
# load all config back into flask app config for now, we should PREFER getting config
# directly from the fence config singleton in the code though.
app.config.update(**config._configs)
_setup_arborist_client(app)
_setup_audit_service_client(app)
_setup_data_endpoint_and_boto(app)
_load_keys(app, root_dir)
_set_authlib_cfgs(app)
app.prometheus_counters = {}
if config["ENABLE_PROMETHEUS_METRICS"]:
logger.info("Enabling Prometheus metrics...")
_setup_prometheus(app)
else:
logger.info("Prometheus metrics are NOT enabled.")
app.storage_manager = StorageManager(config["STORAGE_CREDENTIALS"],
logger=logger)
app.debug = config["DEBUG"]
# Following will update logger level, propagate, and handlers
get_logger(__name__,
log_level="debug" if config["DEBUG"] is True else "info")
_setup_oidc_clients(app)
with app.app_context():
_check_aws_creds_and_region(app)
_check_azure_storage(app)
def load_default_config() -> flask.Config:
"""Loads configuration with default values
:rtype: flask.Config
"""
config = flask.Config(os.path.dirname(__file__))
config.from_object(DefaultConfiguration)
return config
def test_config_update(self):
config = flask.Config(os.path.abspath(os.path.dirname(__file__)))
config.update({"X": True})
self.assertTrue(config.get("X"))
config.update({"X": False})
self.assertFalse(config.get("X"))
config.from_object(TestConfig())
self.assertTrue(config.get("X"))
def get_app(config=None):
"""
App factory.
:param dict config: configuration that can override config
from `settings.py`
:return: a new SuperdeskEve app instance
"""
app_config = flask.Config(".")
# get content api default conf
app_config.from_object("content_api.app.settings")
# set some required fields
app_config.update({"DOMAIN": {"upload": {}}, "SOURCES": {}})
try:
# override from settings module, but only things defined in default config
import settings as server_settings
for key in dir(server_settings):
if key.isupper() and key in app_config:
app_config[key] = getattr(server_settings, key)
except ImportError:
pass # if exists
if config:
app_config.update(config)
media_storage = SuperdeskGridFSMediaStorage
if app_config.get("AMAZON_CONTAINER_NAME"):
from superdesk.storage import AmazonMediaStorage
media_storage = AmazonMediaStorage
app = Eve(
auth=SubscriberTokenAuth,
settings=app_config,
data=SuperdeskDataLayer,
media=media_storage,
json_encoder=MongoJSONEncoder,
validator=SuperdeskValidator,
)
app.notification_client = None
set_error_handlers(app)
for module_name in app.config.get("CONTENTAPI_INSTALLED_APPS", []):
app_module = importlib.import_module(module_name)
try:
app_module.init_app(app)
except AttributeError:
pass
app.sentry = SuperdeskSentry(app)
return app
def get_app(config=None):
"""
App factory.
:param dict config: configuration that can override config
from `settings.py`
:return: a new SuperdeskEve app instance
"""
app_config = flask.Config('.')
# default config
app_config.from_object('prod_api.app.settings')
# https://docs.python-eve.org/en/stable/config.html#domain-configuration
app_config.update({'DOMAIN': {'upload': {}}})
# override from instance settings module, but only things defined in default config
try:
import settings as server_settings
for key in dir(server_settings):
if key.isupper() and key in app_config:
app_config[key] = getattr(server_settings, key)
except ImportError:
pass
if config:
app_config.update(config)
# media storage
media_storage = SuperdeskGridFSMediaStorage
if app_config.get('AMAZON_CONTAINER_NAME'):
from superdesk.storage import AmazonMediaStorage
media_storage = AmazonMediaStorage
app = Eve(
settings=app_config,
data=SuperdeskDataLayer,
media=media_storage,
json_encoder=MongoJSONEncoder,
validator=SuperdeskValidator
)
set_error_handlers(app)
for module_name in app.config.get('PRODAPI_INSTALLED_APPS', []):
app_module = importlib.import_module(module_name)
try:
init_app = app_module.init_app
except AttributeError:
pass
else:
init_app(app)
app.sentry = SuperdeskSentry(app)
return app
def load_config(self):
# Override Eve.load_config in order to get default_settings
if not getattr(self, 'settings'):
self.settings = flask.Config('.')
super(NewsroomApp, self).load_config()
self.config.setdefault('DOMAIN', {})
self.config.setdefault('SOURCES', {})
self.load_app_config()
async def main():
config = flask.Config(".")
config.from_envvar("GOGDB_CONFIG")
db = Storage(config["STORAGE_PATH"])
ids = await db.ids.load()
worker_tasks = [
asyncio.create_task(cleanup_worker(db, ids, worker_num))
for worker_num in range(8)
]
await asyncio.gather(*worker_tasks, return_exceptions=False)
def __init__(self,
import_name=__package__,
config=None,
testing=False,
**kwargs):
"""Override __init__ to do Newsroom specific config and still be able
to create an instance using ``app = Newsroom()``
"""
self.sidenavs = []
self.settings_apps = []
self.download_formatters = {}
self.extensions = {}
self.theme_folder = 'theme'
self.sections = []
self.dashboards = []
self._testing = testing
self._general_settings = {}
app_config = os.path.join(NEWSROOM_DIR, 'default_settings.py')
# get content api default conf
if config:
app_config = flask.Config(app_config)
app_config.from_object('content_api.app.settings')
app_config.update(config)
super(Newsroom, self).__init__(
import_name,
data=SuperdeskDataLayer,
auth=SessionAuth,
settings=app_config,
template_folder=os.path.join(NEWSROOM_DIR, 'templates'),
static_folder=os.path.join(NEWSROOM_DIR, 'static'),
json_encoder=MongoJSONEncoder,
**kwargs)
self.json_encoder = MongoJSONEncoder
if self.config.get('AMAZON_CONTAINER_NAME'):
self.media = AmazonMediaStorage(self)
else:
self.media = SuperdeskGridFSMediaStorage(self)
self._setup_jinja()
self._setup_limiter()
self._setup_babel()
init_celery(self)
newsroom.app = self
self._setup_blueprints(self.config['BLUEPRINTS'])
self._setup_apps(self.config['CORE_APPS'])
self._setup_apps(self.config.get('INSTALLED_APPS', []))
self._setup_webpack()
self._setup_email()
self._setup_cache()
self._setup_error_handlers()
self._setup_theme()
def load_config_from_env(cls, accepted_keys: list = None) -> flask.Config:
"""Loads configuration from system's environment
If param `accepted_keys` is not instance of `list` - NO keys will be accepted.
:param list accepted_keys: List of keys which can be accepted
:rtype: flask.Config
"""
if not isinstance(accepted_keys, list):
accepted_keys = []
config = flask.Config(os.path.dirname(__file__))
for key in accepted_keys:
if key in os.environ:
config[key] = cls.parse_string_value(os.environ[key])
return config
def init_app():
httpkom.init_app(httpkom_app)
# Use Flask's Config class to read config, as Quart's config
# handling is not identical to Flask's and it didn't work with our
# config files. Flask will parse the config file as it was a
# Python file, so you can use lists for example. In Quart the list
# became a string.
#
# Hack: Parse with Flask and then convert (via a dict) to Quart.
config = flask.Config(jskom_app.config.root_path)
config.from_object(default_settings)
if 'JSKOM_SETTINGS' in os.environ:
log.info(
"Using config file specified by JSKOM_SETTINGS environment variable: %s",
os.environ['JSKOM_SETTINGS'])
config.from_envvar('JSKOM_SETTINGS')
else:
log.info(
"No environment variable JSKOM_SETTINGS found, using default settings."
)
# Import config to Quart's app object.
config_dict = dict(config)
jskom_app.config.from_mapping(config_dict)
# Initialize assets
# Flask Assets and webassets jinja integration does not work with Quart.
assets.debug = jskom_app.debug
jskom_app.config['ASSETS_URLS'] = build_assets(
assets, ['js_libs', 'js_angular', 'js_jskom', 'css_jskom'])
if not jskom_app.debug and jskom_app.config['LOG_FILE'] is not None:
# keep 7 days of logs, rotated every midnight
file_handler = TimedRotatingFileHandler(jskom_app.config['LOG_FILE'],
when='midnight',
interval=1,
backupCount=7)
file_handler.setFormatter(
logging.Formatter('%(asctime)s %(levelname)s: %(message)s '
'[in %(pathname)s:%(lineno)d]'))
file_handler.setLevel(jskom_app.config['LOG_LEVEL'])
jskom_app.logger.addHandler(file_handler)
jskom_app.logger.setLevel(jskom_app.config['LOG_LEVEL'])
jskom_app.logger.info("Finished setting up file logger.")
def load_config_from_dotenv(cls,
env_file: str,
accepted_keys: list = None) -> flask.Config:
"""Loads configuration variables from `.env`-file (or similar)
If param `accepted_keys` is not instance of `list` - ALL keys will be accepted.
:param string env_file: Path to custom file with configuration variables
:param list of string accepted_keys: List of keys which can be accepted
:rtype: flask.Config
"""
config = flask.Config(os.path.dirname(__file__))
all_keys_accepted = not isinstance(accepted_keys, list)
for key, value in dotenv.dotenv_values(env_file).items():
if key.isupper() and (all_keys_accepted or key in accepted_keys):
config[key] = cls.parse_string_value(value)
return config
def get_app(config=None):
abs_path = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
# flask config
app_config = flask.Config(abs_path)
app_config.from_object('settings.base')
if config:
app_config.update(config)
# create eve blog app instance
app = BlogEve(settings=app_config)
# register blueprints with resources inside
_register_blueprints(app)
return app
def app_config(app,
settings="fence.settings",
root_dir=None,
config_path=None,
file_name=None):
"""
Set up the config for the Flask app.
"""
if root_dir is None:
root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
logger.info("Loading settings...")
# not using app.config.from_object because we don't want all the extra flask cfg
# vars inside our singleton when we pass these through in the next step
settings_cfg = flask.Config(app.config.root_path)
settings_cfg.from_object(settings)
# dump the settings into the config singleton before loading a configuration file
config.update(dict(settings_cfg))
# load the configuration file, this overwrites anything from settings/local_settings
config.load(config_path, file_name)
# load all config back into flask app config for now, we should PREFER getting config
# directly from the fence config singleton in the code though.
app.config.update(**config._configs)
_setup_arborist_client(app)
_setup_data_endpoint_and_boto(app)
_load_keys(app, root_dir)
_set_authlib_cfgs(app)
app.storage_manager = StorageManager(config["STORAGE_CREDENTIALS"],
logger=logger)
app.debug = config["DEBUG"]
# Following will update logger level, propagate, and handlers
get_logger(__name__,
log_level="debug" if config["DEBUG"] == True else "info")
_setup_oidc_clients(app)
def __init__(self, import_name=__package__, config=None, **kwargs):
if not getattr(self, 'settings'):
self.settings = flask.Config('.')
super(NewsroomNewsAPI, self).__init__(import_name=import_name,
config=config,
**kwargs)
template_folder = os.path.abspath(
os.path.join(API_DIR, '../../templates'))
self.add_template_filter(datetime_short)
self.add_template_filter(datetime_long)
self.add_template_filter(date_header)
self.add_template_filter(plain_text)
self.add_template_filter(time_short)
self.add_template_filter(date_short)
self.add_template_filter(word_count)
self.add_template_filter(char_count)
self.jinja_loader = jinja2.ChoiceLoader([
jinja2.FileSystemLoader(template_folder),
])
def get_app(config=None, media_storage=None, config_object=None):
"""App factory.
:param config: configuration that can override config from `default_settings.py`
:param media_storage: media storage class to use
:param config_object: config object to load (can be module name, module or an object)
:return: a new SuperdeskEve app instance
"""
abs_path = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
app_config = flask.Config(abs_path)
app_config.from_object('superdesk.factory.default_settings')
app_config.setdefault('APP_ABSPATH', abs_path)
app_config.setdefault('DOMAIN', {})
app_config.setdefault('SOURCES', {})
if config_object:
app_config.from_object(config_object)
try:
app_config.update(config or {})
except TypeError:
app_config.from_object(config)
if not media_storage and app_config.get('AMAZON_CONTAINER_NAME'):
media_storage = AmazonMediaStorage
elif not media_storage:
media_storage = SuperdeskGridFSMediaStorage
app = eve.Eve(data=SuperdeskDataLayer,
auth=TokenAuth,
media=media_storage,
settings=app_config,
json_encoder=MongoJSONEncoder,
validator=SuperdeskValidator,
template_folder=os.path.join(abs_path, 'templates'))
superdesk.app = app
custom_loader = jinja2.ChoiceLoader([
app.jinja_loader,
jinja2.FileSystemLoader([abs_path + '/../templates'])
])
app.jinja_loader = custom_loader
app.mail = Mail(app)
app.sentry = SuperdeskSentry(app)
@app.errorhandler(SuperdeskError)
def client_error_handler(error):
"""Return json error response.
:param error: an instance of :attr:`superdesk.SuperdeskError` class
"""
return send_response(None,
(error.to_dict(), None, None, error.status_code))
@app.errorhandler(500)
def server_error_handler(error):
"""Log server errors."""
app.sentry.captureException()
superdesk.logger.exception(error)
return_error = SuperdeskApiError.internalError()
return client_error_handler(return_error)
init_celery(app)
for module_name in app.config.get('INSTALLED_APPS', []):
app_module = importlib.import_module(module_name)
try:
app_module.init_app(app)
except AttributeError:
pass
for resource in superdesk.DOMAIN:
app.register_resource(resource, superdesk.DOMAIN[resource])
for blueprint in superdesk.BLUEPRINTS:
prefix = app.api_prefix or None
app.register_blueprint(blueprint, url_prefix=prefix)
for name, jinja_filter in superdesk.JINJA_FILTERS.items():
app.jinja_env.filters[name] = jinja_filter
# we can only put mapping when all resources are registered
app.data.init_elastic(app)
# instantiate registered provider classes (leave non-classes intact)
for key, provider in registered_feeding_services.items():
registered_feeding_services[key] = provider() if isinstance(
provider, type) else provider
configure_logging(app.config['LOG_CONFIG_FILE'])
return app
def get_app(config=None,
media_storage=None,
config_object=None,
init_elastic=None):
"""App factory.
:param config: configuration that can override config from ``default_settings.py``
:param media_storage: media storage class to use
:param config_object: config object to load (can be module name, module or an object)
:param init_elastic: obsolete config - kept there for BC
:return: a new SuperdeskEve app instance
"""
abs_path = SUPERDESK_PATH
app_config = flask.Config(abs_path)
app_config.from_object('superdesk.default_settings')
app_config.setdefault('APP_ABSPATH', abs_path)
app_config.setdefault('DOMAIN', {})
app_config.setdefault('SOURCES', {})
if config_object:
app_config.from_object(config_object)
try:
app_config.update(config or {})
except TypeError:
app_config.from_object(config)
if not media_storage and app_config.get('AMAZON_CONTAINER_NAME'):
media_storage = AmazonMediaStorage
elif not media_storage:
media_storage = SuperdeskGridFSMediaStorage
app = SuperdeskEve(data=SuperdeskDataLayer,
auth=TokenAuth,
media=media_storage,
settings=app_config,
json_encoder=SuperdeskJSONEncoder,
validator=SuperdeskValidator,
template_folder=os.path.join(abs_path, 'templates'))
app.jinja_options = {'autoescape': False}
app.json_encoder = SuperdeskJSONEncoder # seems like eve param doesn't set it on flask
# init client_config with default config
app.client_config = {
'content_expiry_minutes': app.config.get('CONTENT_EXPIRY_MINUTES', 0),
'ingest_expiry_minutes': app.config.get('INGEST_EXPIRY_MINUTES', 0)
}
superdesk.app = app
custom_loader = jinja2.ChoiceLoader([
jinja2.FileSystemLoader('templates'),
jinja2.FileSystemLoader(os.path.join(SUPERDESK_PATH, 'templates')),
])
app.jinja_loader = custom_loader
app.mail = Mail(app)
app.sentry = SuperdeskSentry(app)
# setup babel
app.config.setdefault('BABEL_TRANSLATION_DIRECTORIES',
os.path.join(SUPERDESK_PATH, 'translations'))
app.babel_tzinfo = None
app.babel_locale = None
app.babel_translations = None
babel = Babel(app, configure_jinja=False)
@babel.localeselector
def get_locale():
user = getattr(g, 'user', {})
user_language = user.get('language',
app.config.get('DEFAULT_LANGUAGE', 'en'))
try:
# Attempt to load the local using Babel.parse_local
parse_locale(user_language.replace('-', '_'))
except ValueError:
# If Babel fails to recognise the locale, then use the default language
user_language = app.config.get('DEFAULT_LANGUAGE', 'en')
return user_language.replace('-', '_')
set_error_handlers(app)
@app.after_request
def after_request(response):
# fixing previous media prefixes if defined
if app.config['MEDIA_PREFIXES_TO_FIX'] and app.config['MEDIA_PREFIX']:
current_prefix = app.config['MEDIA_PREFIX'].rstrip('/').encode()
for prefix in app.config['MEDIA_PREFIXES_TO_FIX']:
response.data = response.data.replace(
prefix.rstrip('/').encode(), current_prefix)
return response
init_celery(app)
installed = set()
def install_app(module_name):
if module_name in installed:
return
installed.add(module_name)
app_module = importlib.import_module(module_name)
if hasattr(app_module, 'init_app'):
app_module.init_app(app)
for module_name in app.config.get('CORE_APPS', []):
install_app(module_name)
for module_name in app.config.get('INSTALLED_APPS', []):
install_app(module_name)
for resource in superdesk.DOMAIN:
app.register_resource(resource, superdesk.DOMAIN[resource])
for name, jinja_filter in superdesk.JINJA_FILTERS.items():
app.jinja_env.filters[name] = jinja_filter
configure_logging(app.config['LOG_CONFIG_FILE'])
return app
def get_app(config=None, media_storage=None, config_object=None, init_elastic=None):
"""App factory.
:param config: configuration that can override config from ``default_settings.py``
:param media_storage: media storage class to use
:param config_object: config object to load (can be module name, module or an object)
:param init_elastic: obsolete config - kept there for BC
:return: a new SuperdeskEve app instance
"""
abs_path = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
app_config = flask.Config(abs_path)
app_config.from_object('superdesk.default_settings')
app_config.setdefault('APP_ABSPATH', abs_path)
app_config.setdefault('DOMAIN', {})
app_config.setdefault('SOURCES', {})
if config_object:
app_config.from_object(config_object)
try:
app_config.update(config or {})
except TypeError:
app_config.from_object(config)
if not media_storage and app_config.get('AMAZON_CONTAINER_NAME'):
media_storage = AmazonMediaStorage
elif not media_storage:
media_storage = SuperdeskGridFSMediaStorage
app = SuperdeskEve(
data=SuperdeskDataLayer,
auth=TokenAuth,
media=media_storage,
settings=app_config,
json_encoder=MongoJSONEncoder,
validator=SuperdeskValidator,
template_folder=os.path.join(abs_path, 'templates'))
app.jinja_options = {'autoescape': False}
# init client_config with default config
app.client_config = {
'content_expiry_minutes': app.config.get('CONTENT_EXPIRY_MINUTES', 0),
'ingest_expiry_minutes': app.config.get('INGEST_EXPIRY_MINUTES', 0)
}
superdesk.app = app
custom_loader = jinja2.ChoiceLoader([
jinja2.FileSystemLoader('templates'),
jinja2.FileSystemLoader(os.path.join(
os.path.dirname(os.path.dirname(os.path.realpath(__file__))), 'templates'))])
app.jinja_loader = custom_loader
app.mail = Mail(app)
app.sentry = SuperdeskSentry(app)
@app.errorhandler(SuperdeskError)
def client_error_handler(error):
"""Return json error response.
:param error: an instance of :attr:`superdesk.SuperdeskError` class
"""
return send_response(None, (error.to_dict(), None, None, error.status_code))
@app.errorhandler(403)
def server_forbidden_handler(error):
return send_response(None, ({'code': 403, 'error': error.response}, None, None, 403))
@app.errorhandler(500)
def server_error_handler(error):
"""Log server errors."""
return_error = SuperdeskApiError.internalError(error)
return client_error_handler(return_error)
@app.after_request
def after_request(response):
# fixing previous media prefixes if defined
if app.config['MEDIA_PREFIXES_TO_FIX'] and app.config['MEDIA_PREFIX']:
current_prefix = app.config['MEDIA_PREFIX'].rstrip('/').encode()
for prefix in app.config['MEDIA_PREFIXES_TO_FIX']:
response.data = response.data.replace(
prefix.rstrip('/').encode(), current_prefix
)
return response
init_celery(app)
installed = set()
def install_app(module_name):
if module_name in installed:
return
installed.add(module_name)
app_module = importlib.import_module(module_name)
if hasattr(app_module, 'init_app'):
app_module.init_app(app)
for module_name in app.config.get('CORE_APPS', []):
install_app(module_name)
for module_name in app.config.get('INSTALLED_APPS', []):
install_app(module_name)
for resource in superdesk.DOMAIN:
app.register_resource(resource, superdesk.DOMAIN[resource])
for name, jinja_filter in superdesk.JINJA_FILTERS.items():
app.jinja_env.filters[name] = jinja_filter
# instantiate registered provider classes (leave non-classes intact)
for key, provider in registered_feeding_services.items():
registered_feeding_services[key] = provider() if isinstance(provider, type) else provider
configure_logging(app.config['LOG_CONFIG_FILE'])
return app
def get_app(config=None,
media_storage=None,
config_object=None,
init_elastic=True):
"""App factory.
:param config: configuration that can override config from ``default_settings.py``
:param media_storage: media storage class to use
:param config_object: config object to load (can be module name, module or an object)
:param init_elastic: should it init elastic indexes or not
:return: a new SuperdeskEve app instance
"""
abs_path = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
app_config = flask.Config(abs_path)
app_config.from_object('superdesk.default_settings')
app_config.setdefault('APP_ABSPATH', abs_path)
app_config.setdefault('DOMAIN', {})
app_config.setdefault('SOURCES', {})
if config_object:
app_config.from_object(config_object)
try:
app_config.update(config or {})
except TypeError:
app_config.from_object(config)
if not media_storage and app_config.get('AMAZON_CONTAINER_NAME'):
media_storage = AmazonMediaStorage
elif not media_storage:
media_storage = SuperdeskGridFSMediaStorage
app = SuperdeskEve(data=SuperdeskDataLayer,
auth=TokenAuth,
media=media_storage,
settings=app_config,
json_encoder=MongoJSONEncoder,
validator=SuperdeskValidator,
template_folder=os.path.join(abs_path, 'templates'))
app.init_indexes = init_elastic
app.jinja_options = {'autoescape': False}
superdesk.app = app
custom_loader = jinja2.ChoiceLoader([
jinja2.FileSystemLoader('templates'),
jinja2.FileSystemLoader(
os.path.join(
os.path.dirname(os.path.dirname(os.path.realpath(__file__))),
'templates'))
])
app.jinja_loader = custom_loader
app.mail = Mail(app)
app.sentry = SuperdeskSentry(app)
@app.errorhandler(SuperdeskError)
def client_error_handler(error):
"""Return json error response.
:param error: an instance of :attr:`superdesk.SuperdeskError` class
"""
return send_response(None,
(error.to_dict(), None, None, error.status_code))
@app.errorhandler(500)
def server_error_handler(error):
"""Log server errors."""
return_error = SuperdeskApiError.internalError(error)
return client_error_handler(return_error)
init_celery(app)
installed = set()
def install_app(module_name):
if module_name in installed:
return
installed.add(module_name)
app_module = importlib.import_module(module_name)
if hasattr(app_module, 'init_app'):
app_module.init_app(app)
for module_name in app.config.get('CORE_APPS', []):
install_app(module_name)
for module_name in app.config.get('INSTALLED_APPS', []):
install_app(module_name)
for resource in superdesk.DOMAIN:
app.register_resource(resource, superdesk.DOMAIN[resource])
for name, jinja_filter in superdesk.JINJA_FILTERS.items():
app.jinja_env.filters[name] = jinja_filter
if not app_config.get('SUPERDESK_TESTING', False) and init_elastic:
# we can only put mapping when all resources are registered
app.data.init_elastic(app)
# instantiate registered provider classes (leave non-classes intact)
for key, provider in registered_feeding_services.items():
registered_feeding_services[key] = provider() if isinstance(
provider, type) else provider
configure_logging(app.config['LOG_CONFIG_FILE'])
return app
def main():
args = parse_arguments()
# get database information
sys.path.append(args.path)
# replicate cfg loading done in flask app to maintain backwards compatibility
# TODO (DEPRECATE LOCAL_SETTINGS): REMOVE this when putting cfg in
# settings/local_settings is deprecated
import flask
settings_cfg = flask.Config(".")
settings_cfg.from_object("fence.settings")
config.update(dict(settings_cfg))
# END - TODO (DEPRECATE LOCAL_SETTINGS): REMOVE
config.load(search_folders=CONFIG_SEARCH_FOLDERS)
DB = os.environ.get("FENCE_DB") or config.get("DB")
# attempt to get from settings, this is backwards-compatibility for integration
# tests
if DB is None:
try:
from fence.settings import DB
except ImportError:
pass
BASE_URL = os.environ.get("BASE_URL") or config.get("BASE_URL")
ROOT_DIR = os.environ.get("ROOT_DIR") or os.path.dirname(
os.path.dirname(os.path.realpath(__file__)))
dbGaP = os.environ.get("dbGaP") or config.get("dbGaP")
if not isinstance(dbGaP, list):
dbGaP = [dbGaP]
STORAGE_CREDENTIALS = os.environ.get("STORAGE_CREDENTIALS") or config.get(
"STORAGE_CREDENTIALS")
usersync = config.get("USERSYNC", {})
sync_from_visas = usersync.get("sync_from_visas", False)
fallback_to_dbgap_sftp = usersync.get("fallback_to_dbgap_sftp", False)
arborist = None
if args.arborist:
arborist = ArboristClient(
arborist_base_url=args.arborist,
logger=get_logger("user_syncer.arborist_client"),
authz_provider="user-sync",
)
if args.action == "create":
yaml_input = args.__dict__["yaml-file-path"]
create_sample_data(DB, yaml_input)
elif args.action == "client-create":
confidential = not args.public
create_client_action(
DB,
username=args.username,
client=args.client,
urls=args.urls,
auto_approve=args.auto_approve,
grant_types=args.grant_types,
confidential=confidential,
arborist=arborist,
policies=args.policies,
allowed_scopes=args.allowed_scopes,
)
elif args.action == "client-modify":
modify_client_action(
DB,
client=args.client,
delete_urls=args.delete_urls,
urls=args.urls,
name=args.name,
description=args.description,
set_auto_approve=args.set_auto_approve,
unset_auto_approve=args.unset_auto_approve,
arborist=arborist,
policies=args.policies,
allowed_scopes=args.allowed_scopes,
append=args.append,
)
elif args.action == "client-delete":
delete_client_action(DB, args.client)
elif args.action == "client-list":
list_client_action(DB)
elif args.action == "user-delete":
delete_users(DB, args.users)
elif args.action == "expired-service-account-delete":
delete_expired_service_accounts(DB)
elif args.action == "bucket-access-group-verify":
verify_bucket_access_group(DB)
elif args.action == "sync":
sync_users(
dbGaP,
STORAGE_CREDENTIALS,
DB,
projects=args.project_mapping,
is_sync_from_dbgap_server=str2bool(args.sync_from_dbgap),
sync_from_local_csv_dir=args.csv_dir,
sync_from_local_yaml_file=args.yaml,
folder=args.folder,
arborist=arborist,
sync_from_visas=sync_from_visas,
fallback_to_dbgap_sftp=fallback_to_dbgap_sftp,
)
elif args.action == "dbgap-download-access-files":
download_dbgap_files(
dbGaP,
STORAGE_CREDENTIALS,
DB,
folder=args.folder,
)
elif args.action == "google-manage-keys":
remove_expired_google_service_account_keys(DB)
elif args.action == "google-init":
google_init(DB)
elif args.action == "google-manage-user-registrations":
verify_user_registration(DB)
elif args.action == "google-manage-account-access":
remove_expired_google_accounts_from_proxy_groups(DB)
elif args.action == "google-bucket-create":
# true if true provided, false if anything else provided, leave as
# None if not provided at all (policy will remain unchanged)
if args.public and args.public.lower().strip() == "true":
args.public = True
elif args.public is not None:
args.public = False
create_or_update_google_bucket(
DB,
args.unique_name,
storage_class=args.storage_class,
public=args.public,
requester_pays=args.requester_pays,
google_project_id=args.google_project_id,
project_auth_id=args.project_auth_id,
access_logs_bucket=args.access_logs_bucket,
allowed_privileges=args.allowed_privileges,
)
elif args.action == "google-logging-bucket-create":
create_google_logging_bucket(
args.unique_name,
storage_class=args.storage_class,
google_project_id=args.google_project_id,
)
elif args.action == "link-external-bucket":
link_external_bucket(DB, name=args.bucket_name)
elif args.action == "link-bucket-to-project":
link_bucket_to_project(
DB,
bucket_id=args.bucket_id,
bucket_provider=args.bucket_provider,
project_auth_id=args.project_auth_id,
)
elif args.action == "google-list-authz-groups":
google_list_authz_groups(DB)
elif args.action == "token-create":
keys_path = getattr(args, "keys-dir", os.path.join(ROOT_DIR, "keys"))
keypairs = keys.load_keypairs(keys_path)
# Default to the most recent one, but try to find the keypair with
# matching ``kid`` to the argument provided.
keypair = keypairs[-1]
kid = getattr(args, "kid")
if kid:
for try_keypair in keypairs:
if try_keypair.kid == kid:
keypair = try_keypair
break
jwt_creator = JWTCreator(
DB,
BASE_URL,
kid=keypair.kid,
private_key=keypair.private_key,
username=args.username,
scopes=args.scopes,
expires_in=args.exp,
)
token_type = str(args.type).strip().lower()
if token_type == "access_token" or token_type == "access":
print(jwt_creator.create_access_token().token)
elif token_type == "refresh_token" or token_type == "refresh":
print(jwt_creator.create_refresh_token().token)
else:
print('invalid token type "{}"; expected "access" or "refresh"'.
format(token_type))
sys.exit(1)
elif args.action == "force-link-google":
exp = force_update_google_link(
DB,
username=args.username,
google_email=args.google_email,
expires_in=args.expires_in,
)
print(exp)
elif args.action == "notify-problem-users":
notify_problem_users(DB, args.emails, args.auth_ids,
args.check_linking, args.google_project_id)
elif args.action == "migrate":
migrate_database(DB)
elif args.action == "update-visas":
update_user_visas(
DB,
chunk_size=args.chunk_size,
concurrency=args.concurrency,
thread_pool_size=args.thread_pool_size,
buffer_size=args.buffer_size,
)
'''
Global configuration module for Partridge
'''
import os
import json
import flask
config = flask.Config({})
if( os.getenv("PARTRIDGE_CONF")):
#try and load config from env directory
config.from_envvar("PARTRIDGE_CONF")
for loc in (os.getcwd(),
os.path.expanduser("~/.config/"),
"/etc/"):
try:
source = os.path.join(loc,"partridge.cfg")
config.from_pyfile(source)
except IOError:
pass
def generate_conf():
conf = flask.Config('')
conf.from_object('dci.settings')
conf.from_object(os.environ.get('DCI_SETTINGS_MODULE'))
return conf
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
from dci.stores import filesystem, swift
import flask
import sqlalchemy
# this is an application global variable
CONFIG = flask.Config('')
CONFIG.from_object('dci.settings')
CONFIG.from_object(os.environ.get('DCI_SETTINGS_MODULE'))
# todo(yassine): remove the param used by client's CI.
def generate_conf(param=None):
return CONFIG
def get_engine():
sa_engine = sqlalchemy.create_engine(
CONFIG['SQLALCHEMY_DATABASE_URI'],
pool_size=CONFIG['SQLALCHEMY_POOL_SIZE'],
max_overflow=CONFIG['SQLALCHEMY_MAX_OVERFLOW'],
encoding='utf8',
def get_app(config=None):
"""
App factory.
:param dict config: configuration that can override config
from `settings.py`
:return: a new SuperdeskEve app instance
"""
app_config = flask.Config(".")
# default config
app_config.from_object("prod_api.app.settings")
# https://docs.python-eve.org/en/stable/config.html#domain-configuration
app_config.update({"DOMAIN": {"upload": {}}})
# override from instance settings module, but only things defined in default config
try:
import settings as server_settings # type: ignore
for key in dir(server_settings):
if key.isupper() and key in app_config:
app_config[key] = getattr(server_settings, key)
except ImportError:
pass
if config:
app_config.update(config)
# media storage
media_storage = get_media_storage_class(app_config)
# auth
auth = None
if app_config["PRODAPI_AUTH_ENABLED"]:
auth = JWTAuth
app = Eve(
auth=auth,
settings=app_config,
data=SuperdeskDataLayer,
media=media_storage,
json_encoder=MongoJSONEncoder,
validator=SuperdeskValidator,
)
app.notification_client = None
set_error_handlers(app)
for module_name in app.config.get("PRODAPI_INSTALLED_APPS", []):
app_module = importlib.import_module(module_name)
try:
init_app = app_module.init_app
except AttributeError:
pass
else:
init_app(app)
app.sentry = SuperdeskSentry(app)
return app
def load_server_config():
server_config = flask.Config('.')
init_flask_config(server_config)
return server_config