def main(): args = parser.parse_args() fileConfig(args.config) logger = getLogger('ocdsapi') loader = plaster.get_loader(args.config, protocols=['wsgi']) settings = loader.get_settings('app:main') es = Elasticsearch([settings.get("elasticsearch.url")]) index = settings.get("elasticsearch.index", 'releases') es.indices.delete(index=index, ignore=[400, 404]) es.indices.create(index=index, ignore=400) mapping = settings.get("elasticsearch.mapping") if mapping: with open(mapping) as _in: mapping = load(_in) es.indices.put_mapping( doc_type='Tender', index=index, body=mapping ) session = get_db_session(settings) rules = get_merge_rules(settings.get('api.schema')) page = 1 while True: page = SqlalchemyOrmPage( (session .query(Record) .order_by(Record.date.desc())), page, 1000 ) records = [] for record in page.items: record = prepare_record( record, [{ "id": r.release_id, "date": r.date, "ocid": r.id } for r in record.releases], rules ) if record: records.append({ '_index': index, '_type': 'Tender', '_id': record['ocid'], '_source': {'ocds': record['compiledRelease']} }) try: resp = bulk(es, records) logger.info(f"Response from elasticsearch: {resp}") except ElasticsearchException as e: logger.error(f"Failed to index bulk with error: {repr(e)}") break page = page.next_page if not page: logger.info("Done indexing") break
def setUp(self): from pyramid_sample import main config_loader = plaster.get_loader(os.environ.get( 'ini_file', 'development.ini'), protocols=['wsgi']) app = main({}, **config_loader.get_settings('app:main')) from webtest import TestApp self.testapp = TestApp(app)
def get_wsgi_app(config_uri: str, defaults: dict) -> router.Router: """Return a Websauna WSGI application given a configuration uri. :param config_uri: Configuration uri, i.e.: websauna/conf/development.ini. :param defaults: Extra options to be passed to the app. :return: A Websauna WSGI Application """ config_uri = prepare_config_uri(config_uri) loader = plaster.get_loader(config_uri) return loader.get_wsgi_app(defaults=defaults)
def pytest_configure(config): global settings inifile = config.inifile if not type(inifile) is str: inifile = str(inifile) loader = plaster.get_loader(inifile) for section in loader.get_sections(): settings[section] = _get_section_settings(section, loader)
def test_get_wsgi_filter(self): import fakeapp.apps app_filter_factory = self.loader.get_wsgi_filter('filt') other_loader = plaster.get_loader('sample_configs/basic_app.ini', protocols=['wsgi']) app = other_loader.get_wsgi_app() app_filter = app_filter_factory(app) assert isinstance(app_filter, fakeapp.apps.CapFilter) assert app_filter.method_to_call == 'lower' assert app_filter.app is fakeapp.apps.basic_app
def test_get_wsgi_filter(self): import fakeapp.apps app_filter_factory = self.loader.get_wsgi_filter('filt') other_loader = plaster.get_loader( 'sample_configs/basic_app.ini', protocols=['wsgi']) app = other_loader.get_wsgi_app() app_filter = app_filter_factory(app) assert isinstance(app_filter, fakeapp.apps.CapFilter) assert app_filter.method_to_call == 'lower' assert app_filter.app is fakeapp.apps.basic_app
def get_celery_config(self, config_file: str) -> dict: """Return celery configuration, from given config_file""" ws_config = 'ws://{0}'.format(config_file) loader = plaster.get_loader(ws_config) settings = loader.get_settings('app:main') # TODO: We have ugly app:main hardcode hack here value = settings.get('websauna.celery_config') if not value: raise RuntimeError( "Could not find websauna.celery_config in {}".format(ini_file)) return value
def test_get_wsgi_filter(self): import fakeapp.apps app_filter_factory = self.loader.get_wsgi_filter() other_loader = plaster.get_loader( "pastedeploy+ini:sample_configs/basic_app.ini#main", protocols=["wsgi"], ) app = other_loader.get_wsgi_app() app_filter = app_filter_factory(app) assert isinstance(app_filter, fakeapp.apps.CapFilter) assert app_filter.method_to_call == "lower" assert app_filter.app is fakeapp.apps.basic_app
def loadapp(config_uri, **global_conf): loader = plaster.get_loader(config_uri) section = loader.uri.fragment or 'app' config = loader.get_settings(section) scheme, resource = config.pop('use').split(':', 1) assert scheme in {'egg', 'wheel', 'package'} if "#" in resource: pkg, name = resource.split('#') else: pkg, name = resource, "main" ep = pkg_resources.get_entry_map(pkg)['plaster.app_factory'][name] app = ep.load()(**config) return app
def get_celery_config(self, config_file: str) -> str: """Return celery configuration, from given config_file. :param config_file: Websauna configuration file. :return: Celery configuration, as a string. """ value = None loader = plaster.get_loader(config_file) settings = loader.get_settings('app:main') secrets_file = settings.get('websauna.secrets_file') if secrets_file: secrets = read_ini_secrets(secrets_file) value = secrets.get('app:main.websauna.celery_config', '') value = value if value else settings.get('websauna.celery_config') if not value: raise RuntimeError('Could not find websauna.celery_config in {ini_file}'.format(ini_file=ini_file)) return value
def read_configuration(self, **kwargs) -> dict: """BaseLoader API override to load Celery config from Pyramid INI file. We need to be able to do this without ramping up full Websauna, because that's the order of the events Celery worker wants. This way we avoid circular dependencies during Celery worker start up. """ loader = plaster.get_loader(ini_file) # global settings = loader.get_settings('app:main') try: value = self.get_celery_config(settings) except RuntimeError as exc: raise RuntimeError('Bad or missing Celery configuration in "%s": %s' % (ini_file, exc)) config = parse_celery_config(value, settings=settings) return config
def __init__(self, global_config: dict, settings: t.Optional[dict] = None): """ :param global_config: Dictionary as passed to WSGI entry point. :param settings: DEPRECATED. Extra settings as passed to WSGI entry point. TODO: How to handle these? """ import plaster # Check if Python and Pyramid versions are the required check_python_pyramid_requirements() if not settings: config = global_config['__file__'] if not config.startswith('ws://'): config = 'ws://{0}'.format(config) loader = plaster.get_loader(config) # Read [app] section settings = loader.get_settings('app:main') #: This is the refer ence to the config file which started our process. We need to later pass it to Notebook. settings["websauna.global_config"] = global_config self.global_config = global_config #: Reference to Celery app instance self.celery = None self.settings = expandvars_dict(settings) self.config = self.create_configurator() self.config.registry.static_asset_policy = self.static_asset_policy = self.create_static_asset_policy( ) #: Flag to tell if we need to do sanity check for redis sessiosn self._has_redis_sessions = False #: This flag keeps state if the initializer has been run or not. self._already_run = False # Exposed Websauna features self.config.registry.features = set() # Secrets file self.secrets = self.read_secrets()
def pserve_file_config(self, filename, global_conf=None): here = os.path.abspath(os.path.dirname(filename)) loader = plaster.get_loader(filename) if 'plserve' not in loader.get_sections(): return items = loader.get_settings('plshell') watch_files = aslist(items.get('watch_files', ''), flatten=False) # track file paths relative to the ini file resolver = AssetResolver(package=None) for file in watch_files: if ':' in file: file = resolver.resolve(file).abspath() elif not os.path.isabs(file): file = os.path.join(here, file) self.watch_files.append(os.path.abspath(file))
def pshell_file_config(self, filename): loader = plaster.get_loader(filename) if 'plshell' not in loader.get_sections(): return items = loader.get_settings('plshell') resolver = DottedNameResolver(None) self.loaded_objects = {} self.object_help = {} self.setup = None for k, v in items: if k == 'setup': self.setup = v elif k == 'default_shell': self.preferred_shells = [x.lower() for x in aslist(v)] else: self.loaded_objects[k] = resolver.maybe_resolve(v) self.object_help[k] = v
def ini_settings(request: FixtureRequest, test_config_path) -> dict: """Load INI settings for test run from py.test command line. Example: py.test yourpackage -s --ini=test.ini :return: A dictionary representing the key/value pairs in an ``app`` section within the file represented by ``config_uri`` """ # Setup Python logging from the INI # Add Websauna loader if not test_config_path.startswith('ws://'): test_config_path = 'ws://{0}'.format(test_config_path) loader = plaster.get_loader(test_config_path) # Read [app] section settings = loader.get_settings('app:main') # To pass the config filename itself forward settings["_ini_file"] = test_config_path return settings
def webdav_app(config_uri): config_uri = '{}#webdav'.format(config_uri) plaster.setup_logging(config_uri) loader = plaster.get_loader(config_uri, protocols=['wsgi']) return loader.get_wsgi_app()
def loader(self, fake_packages, monkeypatch): monkeypatch.chdir(here) self.loader = plaster.get_loader(test_config_relpath, protocols=['wsgi'])
def loader(self, fake_packages): self.loader = plaster.get_loader(self.config_uri, protocols=['wsgi'])
def loader(self, fake_packages, monkeypatch): monkeypatch.chdir(here) self.loader = plaster.get_loader(self.config_uri, protocols=["wsgi"])
def webdav_app(config_uri: str) -> WsgiDAVApp: plaster.setup_logging(config_uri) loader = plaster.get_loader(config_uri, protocols=["wsgi"]) return loader.get_wsgi_app(name=WEBDAV_APP_NAME)
def loader(self, fake_packages, monkeypatch): monkeypatch.chdir(here) self.loader = plaster.get_loader( test_config_relpath, protocols=["wsgi"] )
def setup_logging(config_uri, disable_existing_loggers=False): """Include-aware Python logging setup from INI config file. """ config_uri = prepare_config_uri(config_uri) loader = plaster.get_loader(config_uri, protocols=['wsgi']) loader.setup_logging(disable_existing_loggers=disable_existing_loggers)
def caldav_app(config_uri: str) -> RadicaleApplication: plaster.setup_logging(config_uri) loader = plaster.get_loader(config_uri, protocols=["wsgi"]) return loader.get_wsgi_app(name=CALDAV_APP_NAME)
def webdav_app(config_uri): plaster.setup_logging(config_uri) loader = plaster.get_loader(config_uri, protocols=['wsgi']) return loader.get_wsgi_app(name=WEBDAV_APP_NAME)
def loader(self, fake_packages): self.loader = plaster.get_loader(self.config_uri)
def get_wsgi_loader(self): """get wsgi loader from self's config""" loader = plaster.get_loader(self.config, protocols=["wsgi"]) return loader
def get_config_loader(config_uri): """ Find a ``plaster.ILoader`` object supporting the "wsgi" protocol. """ return plaster.get_loader(config_uri, protocols=['wsgi'])
def _get_server(self, config_uri: str): loader = plaster.get_loader(config_uri, protocols=['wsgi']) return loader.get_wsgi_server(name=WEBDAV_APP_NAME)
def loader(self, fake_packages, monkeypatch): monkeypatch.chdir(here) self.loader = plaster.get_loader('sample_configs/test_filter_with.ini', protocols=['wsgi'])
def main(input_args=None): if not input_args: input_args = sys.argv[1:] logging.lastResort = logging.StreamHandler(stream=sys.stderr) logging.lastResort.setFormatter( Formatter('%(asctime)s %(levelname)-5.5s [%(name)s:%(lineno)s][%(threadName)s] %(message)s')) now = datetime.now() logger.debug("Startup in main", now) def do_at_exit(): new_now = datetime.now() logger.info("exiting... [%s]", new_now - now) atexit.register(do_at_exit) parser = argparse.ArgumentParser(parents=[]) # db_dump.args.argument_parser()]) parser.add_argument('--config-uri', '-c', help="Provide config_uri for configuration via plaster.", action=ConfigAction) parser.add_argument('--section', default='app:main') # parser.add_argument('--test-app', '-t', help="Test the application", action="store_true") parser.add_argument('--entry-point', '-e', help="Specify entry point", action="store") parser.add_argument('--list-entry-points', '-l', help="List entry points", action="store_true") parser.add_argument('--virtual-assets', action="store_true") parser.add_argument('--pipe', action="store_true") parser.add_argument('--cmd', action="append") parser.add_argument('--package', action="store") args = parser.parse_args(input_args) config_uri = args.config_uri setup_logging(config_uri) settings = get_appsettings(config_uri) # what's this ?? output_path = settings.get('heptet_app.process_views_output_path', '.') config = {'output_path': output_path } # we need to do this automatically setup = setup_jsonencoder() setup() package = "heptet_app" if args.package: logger.info("selecting package %r", args.package) # this doesn't seem to help us right now package = args.package loader = plaster.get_loader(config_uri) app = loader.get_wsgi_app() registry = app.registry root_factory = registry.queryUtility(IRootFactory) root = root_factory() asset_mgr, proc_context, template_env = initialize(args, registry, config) # here we get our entry points entry_points = list(registry.getUtilitiesFor(IEntryPoint)) if not entry_points: logger.critical("Configuration error: no entry points.") exit(255) if args.list_entry_points: for name, ep in entry_points: print(name) exit(0) if args.cmd: for cmd in args.cmd: exec_command(registry, proc_context, cmd) exit(0) if args.pipe: _run_pipe(registry, proc_context) # handle a specified entry point (untested) if args.entry_point: for name, ep in entry_points: if name == args.entry_point: process_views(registry, config, proc_context, [(name, ep)]) for k, v in asset_mgr.asset_path.items(): logger.critical("%r = %s", k[0].key, v) exit(0) # we should be able to remove request and registry? process_views(registry, config, proc_context, entry_points) d = {} v: PurePath curdir = PurePath("./") if args.virtual_assets: vd = {} for k, v in asset_mgr.asset_content.items(): vd[k[0].key] = {'content': v} d[k[0].key] = "./" + curdir.joinpath(v).as_posix() json.dump(vd, fp=sys.stdout) else: for k, v in asset_mgr.assets.items(): d[k.key] = "./" + Path(v).as_posix() with open("entry_point.json", 'w') as f: json.dump(d, fp=f, indent=4, sort_keys=True) f.close()
def _makeOne(self, uri=None): if uri is None: uri = test_config_relpath return plaster.get_loader(uri)
def loader(self, fake_packages, monkeypatch): monkeypatch.chdir(here) self.loader = plaster.get_loader(self.config_uri, protocols=['wsgi'])
def _get_server(self, config_uri: str): loader = plaster.get_loader(config_uri, protocols=["wsgi"]) return loader.get_wsgi_server(name=CALDAV_APP_NAME)
def loader(self, fake_packages, monkeypatch): monkeypatch.chdir(here) self.loader = plaster.get_loader( "sample_configs/test_filter_with.ini", protocols=["wsgi"] )