def load_wr_config():
config = load_overlay_config('WR_CONFIG',
'pkg://webrecorder/config/wr.yaml',
'WR_USER_CONFIG', '')
init_props(config)
return config
def load_wr_config():
config = load_overlay_config('WR_CONFIG', 'pkg://webrecorder/config/wr.yaml', 'WR_USER_CONFIG', '')
config['dyn_stats_key_templ'] = {
'rec': 'r:{rec}:<sesh_id>:stats:',
'coll': 'c:{coll}:<sesh_id>:stats:'
}
config['dyn_ref_templ'] = {
'rec': 'r:{rec}:<sesh_id>:ref:',
'coll': 'c:{coll}:<sesh_id>:ref:',
}
return config
def __init__(self, config_file='./config.yaml', custom_config=None):
config = load_yaml_config(DEFAULT_CONFIG)
if config_file:
try:
file_config = load_overlay_config('PYWB_CONFIG_FILE',
config_file)
config.update(file_config)
except Exception as e:
if not custom_config:
custom_config = {'debug': True}
print(e)
if custom_config:
if 'collections' in custom_config and 'collections' in config:
custom_config['collections'].update(config['collections'])
if 'proxy' in custom_config and 'proxy' in config:
custom_config['proxy'].update(config['proxy'])
config.update(custom_config)
super(WarcServer, self).__init__(debug=config.get('debug', False))
self.config = config
self.root_dir = self.config.get('collections_root', '')
self.index_paths = self.init_paths('index_paths')
self.archive_paths = self.init_paths('archive_paths', self.root_dir)
self.acl_paths = self.init_paths('acl_paths')
self.default_access = self.config.get('default_access')
self.rules_file = self.config.get('rules_file', '')
self.auto_handler = None
if self.config.get('enable_auto_colls', True):
self.auto_handler = self.load_auto_colls()
self.fixed_routes = self.load_colls()
for name, route in iteritems(self.fixed_routes):
if route == self.auto_handler:
self.add_route('/' + name,
route,
path_param_name='param.coll',
default_value='*')
else:
self.add_route('/' + name, route)
if self.auto_handler:
self.add_route('/<path_param_value>',
self.auto_handler,
path_param_name='param.coll')
def load_wr_config():
return load_overlay_config('WR_CONFIG', 'pkg://webrecorder/config/wr.yaml',
'WR_USER_CONFIG', '')
def __init__(self, config_file='./config.yaml', custom_config=None):
config = load_yaml_config(DEFAULT_CONFIG)
if config_file:
try:
file_config = load_overlay_config('PYWB_CONFIG_FILE', config_file)
config.update(file_config)
except Exception as e:
if not custom_config:
custom_config = {'debug': True}
print(e)
if custom_config:
if 'collections' in custom_config and 'collections' in config:
custom_config['collections'].update(config['collections'])
if 'proxy' in custom_config and 'proxy' in config:
custom_config['proxy'].update(config['proxy'])
if 'recorder' in custom_config and 'recorder' in config:
if isinstance(custom_config['recorder'], str):
custom_config['recorder'] = {'source_coll': custom_config['recorder']}
if isinstance(config['recorder'], str):
config['recorder'] = {'source_coll': config['recorder']}
config['recorder'].update(custom_config['recorder'])
custom_config['recorder'] = config['recorder']
config.update(custom_config)
super(WarcServer, self).__init__(debug=config.get('debug', False))
self.config = config
recorder_config = self.config.get('recorder') or {}
if isinstance(recorder_config, dict) and recorder_config.get('dedup_policy'):
self.dedup_index_url = recorder_config.get('dedup_index_url', WarcServer.DEFAULT_DEDUP_URL)
if self.dedup_index_url and not self.dedup_index_url.startswith('redis://'):
raise Exception("The dedup_index_url must start with \"redis://\". Only Redis-based dedup index is supported at this time.")
else:
self.dedup_index_url = None
self.root_dir = self.config.get('collections_root', '')
self.index_paths = self.init_paths('index_paths')
self.archive_paths = self.init_paths('archive_paths', self.root_dir)
self.acl_paths = self.init_paths('acl_paths')
self.default_access = self.config.get('default_access')
self.rules_file = self.config.get('rules_file', '')
if 'certificates' in self.config:
certs_config = self.config['certificates']
DefaultAdapters.live_adapter = PywbHttpAdapter(max_retries=Retry(3),
cert_reqs=certs_config.get('cert_reqs', 'CERT_NONE'),
ca_cert_dir=certs_config.get('ca_cert_dir'))
DefaultAdapters.remote_adapter = PywbHttpAdapter(max_retries=Retry(3),
cert_reqs=certs_config.get('cert_reqs', 'CERT_NONE'),
ca_cert_dir=certs_config.get('ca_cert_dir'))
self.auto_handler = None
if self.config.get('enable_auto_colls', True):
self.auto_handler = self.load_auto_colls()
self.fixed_routes = self.load_colls()
for name, route in iteritems(self.fixed_routes):
if route == self.auto_handler:
self.add_route('/' + name, route, path_param_name='param.coll', default_value='*')
else:
self.add_route('/' + name, route)
if self.auto_handler:
self.add_route('/<path_param_value>', self.auto_handler, path_param_name='param.coll')
def load_wr_config():
config = load_overlay_config('WR_CONFIG', 'pkg://webrecorder/config/wr.yaml', 'WR_USER_CONFIG', '')
init_props(config)
return config