def main():
args = parser.parse_args()
fileConfig(args.config)
logger = getLogger('ocdsapi')
loader = plaster.get_loader(args.config, protocols=['wsgi'])
settings = loader.get_settings('app:main')
es = Elasticsearch([settings.get("elasticsearch.url")])
index = settings.get("elasticsearch.index", 'releases')
es.indices.delete(index=index, ignore=[400, 404])
es.indices.create(index=index, ignore=400)
mapping = settings.get("elasticsearch.mapping")
if mapping:
with open(mapping) as _in:
mapping = load(_in)
es.indices.put_mapping(
doc_type='Tender',
index=index,
body=mapping
)
session = get_db_session(settings)
rules = get_merge_rules(settings.get('api.schema'))
page = 1
while True:
page = SqlalchemyOrmPage(
(session
.query(Record)
.order_by(Record.date.desc())),
page,
1000
)
records = []
for record in page.items:
record = prepare_record(
record,
[{
"id": r.release_id,
"date": r.date,
"ocid": r.id
} for r in record.releases],
rules
)
if record:
records.append({
'_index': index,
'_type': 'Tender',
'_id': record['ocid'],
'_source': {'ocds': record['compiledRelease']}
})
try:
resp = bulk(es, records)
logger.info(f"Response from elasticsearch: {resp}")
except ElasticsearchException as e:
logger.error(f"Failed to index bulk with error: {repr(e)}")
break
page = page.next_page
if not page:
logger.info("Done indexing")
break
def setUp(self):
from pyramid_sample import main
config_loader = plaster.get_loader(os.environ.get(
'ini_file', 'development.ini'),
protocols=['wsgi'])
app = main({}, **config_loader.get_settings('app:main'))
from webtest import TestApp
self.testapp = TestApp(app)
def get_wsgi_app(config_uri: str, defaults: dict) -> router.Router:
"""Return a Websauna WSGI application given a configuration uri.
:param config_uri: Configuration uri, i.e.: websauna/conf/development.ini.
:param defaults: Extra options to be passed to the app.
:return: A Websauna WSGI Application
"""
config_uri = prepare_config_uri(config_uri)
loader = plaster.get_loader(config_uri)
return loader.get_wsgi_app(defaults=defaults)
def pytest_configure(config):
global settings
inifile = config.inifile
if not type(inifile) is str:
inifile = str(inifile)
loader = plaster.get_loader(inifile)
for section in loader.get_sections():
settings[section] = _get_section_settings(section, loader)
def test_get_wsgi_filter(self):
import fakeapp.apps
app_filter_factory = self.loader.get_wsgi_filter('filt')
other_loader = plaster.get_loader('sample_configs/basic_app.ini',
protocols=['wsgi'])
app = other_loader.get_wsgi_app()
app_filter = app_filter_factory(app)
assert isinstance(app_filter, fakeapp.apps.CapFilter)
assert app_filter.method_to_call == 'lower'
assert app_filter.app is fakeapp.apps.basic_app
def test_get_wsgi_filter(self):
import fakeapp.apps
app_filter_factory = self.loader.get_wsgi_filter('filt')
other_loader = plaster.get_loader(
'sample_configs/basic_app.ini', protocols=['wsgi'])
app = other_loader.get_wsgi_app()
app_filter = app_filter_factory(app)
assert isinstance(app_filter, fakeapp.apps.CapFilter)
assert app_filter.method_to_call == 'lower'
assert app_filter.app is fakeapp.apps.basic_app
def get_celery_config(self, config_file: str) -> dict:
"""Return celery configuration, from given config_file"""
ws_config = 'ws://{0}'.format(config_file)
loader = plaster.get_loader(ws_config)
settings = loader.get_settings('app:main')
# TODO: We have ugly app:main hardcode hack here
value = settings.get('websauna.celery_config')
if not value:
raise RuntimeError(
"Could not find websauna.celery_config in {}".format(ini_file))
return value
def test_get_wsgi_filter(self):
import fakeapp.apps
app_filter_factory = self.loader.get_wsgi_filter()
other_loader = plaster.get_loader(
"pastedeploy+ini:sample_configs/basic_app.ini#main",
protocols=["wsgi"],
)
app = other_loader.get_wsgi_app()
app_filter = app_filter_factory(app)
assert isinstance(app_filter, fakeapp.apps.CapFilter)
assert app_filter.method_to_call == "lower"
assert app_filter.app is fakeapp.apps.basic_app
def loadapp(config_uri, **global_conf):
loader = plaster.get_loader(config_uri)
section = loader.uri.fragment or 'app'
config = loader.get_settings(section)
scheme, resource = config.pop('use').split(':', 1)
assert scheme in {'egg', 'wheel', 'package'}
if "#" in resource:
pkg, name = resource.split('#')
else:
pkg, name = resource, "main"
ep = pkg_resources.get_entry_map(pkg)['plaster.app_factory'][name]
app = ep.load()(**config)
return app
def get_celery_config(self, config_file: str) -> str:
"""Return celery configuration, from given config_file.
:param config_file: Websauna configuration file.
:return: Celery configuration, as a string.
"""
value = None
loader = plaster.get_loader(config_file)
settings = loader.get_settings('app:main')
secrets_file = settings.get('websauna.secrets_file')
if secrets_file:
secrets = read_ini_secrets(secrets_file)
value = secrets.get('app:main.websauna.celery_config', '')
value = value if value else settings.get('websauna.celery_config')
if not value:
raise RuntimeError('Could not find websauna.celery_config in {ini_file}'.format(ini_file=ini_file))
return value
def read_configuration(self, **kwargs) -> dict:
"""BaseLoader API override to load Celery config from Pyramid INI file.
We need to be able to do this without ramping up full Websauna,
because that's the order of the events Celery worker wants.
This way we avoid circular dependencies during Celery worker start up.
"""
loader = plaster.get_loader(ini_file) # global
settings = loader.get_settings('app:main')
try:
value = self.get_celery_config(settings)
except RuntimeError as exc:
raise RuntimeError('Bad or missing Celery configuration in "%s": %s' % (ini_file, exc))
config = parse_celery_config(value, settings=settings)
return config
def __init__(self, global_config: dict, settings: t.Optional[dict] = None):
"""
:param global_config: Dictionary as passed to WSGI entry point.
:param settings: DEPRECATED. Extra settings as passed to WSGI entry point. TODO: How to handle these?
"""
import plaster
# Check if Python and Pyramid versions are the required
check_python_pyramid_requirements()
if not settings:
config = global_config['__file__']
if not config.startswith('ws://'):
config = 'ws://{0}'.format(config)
loader = plaster.get_loader(config)
# Read [app] section
settings = loader.get_settings('app:main')
#: This is the refer ence to the config file which started our process. We need to later pass it to Notebook.
settings["websauna.global_config"] = global_config
self.global_config = global_config
#: Reference to Celery app instance
self.celery = None
self.settings = expandvars_dict(settings)
self.config = self.create_configurator()
self.config.registry.static_asset_policy = self.static_asset_policy = self.create_static_asset_policy(
)
#: Flag to tell if we need to do sanity check for redis sessiosn
self._has_redis_sessions = False
#: This flag keeps state if the initializer has been run or not.
self._already_run = False
# Exposed Websauna features
self.config.registry.features = set()
# Secrets file
self.secrets = self.read_secrets()
def pserve_file_config(self, filename, global_conf=None):
here = os.path.abspath(os.path.dirname(filename))
loader = plaster.get_loader(filename)
if 'plserve' not in loader.get_sections():
return
items = loader.get_settings('plshell')
watch_files = aslist(items.get('watch_files', ''), flatten=False)
# track file paths relative to the ini file
resolver = AssetResolver(package=None)
for file in watch_files:
if ':' in file:
file = resolver.resolve(file).abspath()
elif not os.path.isabs(file):
file = os.path.join(here, file)
self.watch_files.append(os.path.abspath(file))
def pshell_file_config(self, filename):
loader = plaster.get_loader(filename)
if 'plshell' not in loader.get_sections():
return
items = loader.get_settings('plshell')
resolver = DottedNameResolver(None)
self.loaded_objects = {}
self.object_help = {}
self.setup = None
for k, v in items:
if k == 'setup':
self.setup = v
elif k == 'default_shell':
self.preferred_shells = [x.lower() for x in aslist(v)]
else:
self.loaded_objects[k] = resolver.maybe_resolve(v)
self.object_help[k] = v
def ini_settings(request: FixtureRequest, test_config_path) -> dict:
"""Load INI settings for test run from py.test command line.
Example:
py.test yourpackage -s --ini=test.ini
:return: A dictionary representing the key/value pairs in an ``app`` section within the file represented by ``config_uri``
"""
# Setup Python logging from the INI
# Add Websauna loader
if not test_config_path.startswith('ws://'):
test_config_path = 'ws://{0}'.format(test_config_path)
loader = plaster.get_loader(test_config_path)
# Read [app] section
settings = loader.get_settings('app:main')
# To pass the config filename itself forward
settings["_ini_file"] = test_config_path
return settings
def webdav_app(config_uri):
config_uri = '{}#webdav'.format(config_uri)
plaster.setup_logging(config_uri)
loader = plaster.get_loader(config_uri, protocols=['wsgi'])
return loader.get_wsgi_app()
def loader(self, fake_packages, monkeypatch):
monkeypatch.chdir(here)
self.loader = plaster.get_loader(test_config_relpath,
protocols=['wsgi'])
def loader(self, fake_packages):
self.loader = plaster.get_loader(self.config_uri, protocols=['wsgi'])
def loader(self, fake_packages, monkeypatch):
monkeypatch.chdir(here)
self.loader = plaster.get_loader(self.config_uri, protocols=["wsgi"])
def webdav_app(config_uri: str) -> WsgiDAVApp:
plaster.setup_logging(config_uri)
loader = plaster.get_loader(config_uri, protocols=["wsgi"])
return loader.get_wsgi_app(name=WEBDAV_APP_NAME)
def loader(self, fake_packages, monkeypatch):
monkeypatch.chdir(here)
self.loader = plaster.get_loader(
test_config_relpath, protocols=["wsgi"]
)
def setup_logging(config_uri, disable_existing_loggers=False):
"""Include-aware Python logging setup from INI config file.
"""
config_uri = prepare_config_uri(config_uri)
loader = plaster.get_loader(config_uri, protocols=['wsgi'])
loader.setup_logging(disable_existing_loggers=disable_existing_loggers)
def caldav_app(config_uri: str) -> RadicaleApplication:
plaster.setup_logging(config_uri)
loader = plaster.get_loader(config_uri, protocols=["wsgi"])
return loader.get_wsgi_app(name=CALDAV_APP_NAME)
def webdav_app(config_uri):
plaster.setup_logging(config_uri)
loader = plaster.get_loader(config_uri, protocols=['wsgi'])
return loader.get_wsgi_app(name=WEBDAV_APP_NAME)
def loader(self, fake_packages):
self.loader = plaster.get_loader(self.config_uri)
def get_wsgi_loader(self):
"""get wsgi loader from self's config"""
loader = plaster.get_loader(self.config, protocols=["wsgi"])
return loader
def get_config_loader(config_uri):
"""
Find a ``plaster.ILoader`` object supporting the "wsgi" protocol.
"""
return plaster.get_loader(config_uri, protocols=['wsgi'])
def _get_server(self, config_uri: str):
loader = plaster.get_loader(config_uri, protocols=['wsgi'])
return loader.get_wsgi_server(name=WEBDAV_APP_NAME)
def loader(self, fake_packages, monkeypatch):
monkeypatch.chdir(here)
self.loader = plaster.get_loader('sample_configs/test_filter_with.ini',
protocols=['wsgi'])
def main(input_args=None):
if not input_args:
input_args = sys.argv[1:]
logging.lastResort = logging.StreamHandler(stream=sys.stderr)
logging.lastResort.setFormatter(
Formatter('%(asctime)s %(levelname)-5.5s [%(name)s:%(lineno)s][%(threadName)s] %(message)s'))
now = datetime.now()
logger.debug("Startup in main", now)
def do_at_exit():
new_now = datetime.now()
logger.info("exiting... [%s]", new_now - now)
atexit.register(do_at_exit)
parser = argparse.ArgumentParser(parents=[]) # db_dump.args.argument_parser()])
parser.add_argument('--config-uri', '-c', help="Provide config_uri for configuration via plaster.",
action=ConfigAction)
parser.add_argument('--section', default='app:main')
# parser.add_argument('--test-app', '-t', help="Test the application", action="store_true")
parser.add_argument('--entry-point', '-e', help="Specify entry point", action="store")
parser.add_argument('--list-entry-points', '-l', help="List entry points", action="store_true")
parser.add_argument('--virtual-assets', action="store_true")
parser.add_argument('--pipe', action="store_true")
parser.add_argument('--cmd', action="append")
parser.add_argument('--package', action="store")
args = parser.parse_args(input_args)
config_uri = args.config_uri
setup_logging(config_uri)
settings = get_appsettings(config_uri)
# what's this ??
output_path = settings.get('heptet_app.process_views_output_path', '.')
config = {'output_path': output_path }
# we need to do this automatically
setup = setup_jsonencoder()
setup()
package = "heptet_app"
if args.package:
logger.info("selecting package %r", args.package)
# this doesn't seem to help us right now
package = args.package
loader = plaster.get_loader(config_uri)
app = loader.get_wsgi_app()
registry = app.registry
root_factory = registry.queryUtility(IRootFactory)
root = root_factory()
asset_mgr, proc_context, template_env = initialize(args, registry, config)
# here we get our entry points
entry_points = list(registry.getUtilitiesFor(IEntryPoint))
if not entry_points:
logger.critical("Configuration error: no entry points.")
exit(255)
if args.list_entry_points:
for name, ep in entry_points:
print(name)
exit(0)
if args.cmd:
for cmd in args.cmd:
exec_command(registry, proc_context, cmd)
exit(0)
if args.pipe:
_run_pipe(registry, proc_context)
# handle a specified entry point (untested)
if args.entry_point:
for name, ep in entry_points:
if name == args.entry_point:
process_views(registry, config, proc_context, [(name, ep)])
for k, v in asset_mgr.asset_path.items():
logger.critical("%r = %s", k[0].key, v)
exit(0)
# we should be able to remove request and registry?
process_views(registry, config, proc_context, entry_points)
d = {}
v: PurePath
curdir = PurePath("./")
if args.virtual_assets:
vd = {}
for k, v in asset_mgr.asset_content.items():
vd[k[0].key] = {'content': v}
d[k[0].key] = "./" + curdir.joinpath(v).as_posix()
json.dump(vd, fp=sys.stdout)
else:
for k, v in asset_mgr.assets.items():
d[k.key] = "./" + Path(v).as_posix()
with open("entry_point.json", 'w') as f:
json.dump(d, fp=f, indent=4, sort_keys=True)
f.close()
def loader(self, fake_packages):
self.loader = plaster.get_loader(self.config_uri, protocols=['wsgi'])
def loader(self, fake_packages):
self.loader = plaster.get_loader(self.config_uri)
def _makeOne(self, uri=None):
if uri is None:
uri = test_config_relpath
return plaster.get_loader(uri)
def loader(self, fake_packages, monkeypatch):
monkeypatch.chdir(here)
self.loader = plaster.get_loader(self.config_uri, protocols=['wsgi'])
def _get_server(self, config_uri: str):
loader = plaster.get_loader(config_uri, protocols=["wsgi"])
return loader.get_wsgi_server(name=CALDAV_APP_NAME)
def loader(self, fake_packages, monkeypatch):
monkeypatch.chdir(here)
self.loader = plaster.get_loader(
"sample_configs/test_filter_with.ini", protocols=["wsgi"]
)
