def load_all(self, webarchives_path):
wa_file = load(webarchives_path)
with closing(wa_file):
for doc in yaml.load_all(wa_file):
webarchives = doc['webarchives']
for pk, webarchive in webarchives.items():
self.load_archive(pk, webarchive)
def _do_download_cdxj(self, cdxj_key, output_key):
lock_key = None
try:
rec_info_key = cdxj_key.rsplit(':', 1)[0] + ':info'
cdxj_filename = self.redis.hget(rec_info_key, self.INDEX_FILE_KEY)
if not cdxj_filename:
logging.debug('No index for ' + rec_info_key)
return
lock_key = cdxj_key + ':_'
logging.debug('Downloading for {0} file {1}'.format(
rec_info_key, cdxj_filename))
attempts = 0
if not self.redis.set(lock_key, 1, nx=True):
logging.warning('Already downloading, skipping')
lock_key = None
return
while attempts < 10:
fh = None
try:
fh = load(cdxj_filename)
buff = fh.read()
for cdxj_line in buff.splitlines():
self.redis.zadd(output_key, 0, cdxj_line)
break
except Exception as e:
import traceback
traceback.print_exc()
logging.error('Could not load: ' + cdxj_filename)
attempts += 1
finally:
if fh:
fh.close()
if self.COLL_CDXJ_TTL > 0:
self.redis.expire(output_key, self.COLL_CDXJ_TTL)
except Exception as e:
logging.error('Error downloading cache: ' + str(e))
import traceback
traceback.print_exc()
finally:
if lock_key:
self.redis.delete(lock_key)
def _do_download_cdxj(self, cdxj_key, output_key):
lock_key = None
try:
rec_info_key = cdxj_key.rsplit(':', 1)[0] + ':info'
cdxj_filename = self.redis.hget(rec_info_key, self.INDEX_FILE_KEY)
if not cdxj_filename:
logger.debug('CDX Sync: No index for ' + rec_info_key)
return
lock_key = cdxj_key + ':_'
logger.debug('CDX Sync: Downloading for {0} file {1}'.format(
rec_info_key, cdxj_filename))
attempts = 0
if not self.redis.set(
lock_key, 1, ex=self.COMMIT_WAIT_SECS, nx=True):
logger.warning(
'CDX Sync: Already downloading, skipping: {0}'.format(
cdxj_filename))
lock_key = None
return
while attempts < 10:
fh = None
try:
fh = load(cdxj_filename)
buff = fh.read()
for cdxj_line in buff.splitlines():
self.redis.zadd(output_key, 0, cdxj_line)
break
except Exception as e:
traceback.print_exc()
logger.error('CDX Sync: Could not load: ' + cdxj_filename)
attempts += 1
finally:
if fh:
fh.close()
self.reset_cdxj_ttl(output_key)
except Exception as e:
logger.error('CDX Sync: Error downloading cache: ' + str(e))
traceback.print_exc()
finally:
if lock_key:
self.redis.delete(lock_key)
def get_full_version():
full_version = 'unknown'
curr_app = sys.argv[0].rsplit(os.path.sep)[-1]
try:
# standalone app, read baked-in _full_version
if getattr(sys, 'frozen', False):
from pywb.utils.loaders import load
full_version = load(
'pkg://webrecorder/config/_full_version').read()
full_version = full_version.decode('utf-8').format(curr_app)
else:
# generate full_version dynamically
full_version = get_version_str()
except:
pass
return (full_version % curr_app)
def __init__(self,
paths=['templates', '.', '/'],
packages=['pywb'],
assets_path=None,
globals=None,
overlay=None,
extensions=None,
env_template_params_key='pywb.template_params',
env_template_dir_key='pywb.templates_dir'):
self._init_filters()
loader = ChoiceLoader(self._make_loaders(paths, packages))
self.env_template_params_key = env_template_params_key
self.env_template_dir_key = env_template_dir_key
extensions = extensions or []
if assets_path:
extensions.append(AssetsExtension)
if overlay:
jinja_env = overlay.jinja_env.overlay(loader=loader,
trim_blocks=True,
extensions=extensions)
else:
jinja_env = RelEnvironment(loader=loader,
trim_blocks=True,
extensions=extensions)
jinja_env.filters.update(self.filters)
if globals:
jinja_env.globals.update(globals)
self.jinja_env = jinja_env
# init assets
if assets_path:
assets_loader = YAMLLoader(load(assets_path))
assets_env = assets_loader.load_environment()
assets_env.resolver = PkgResResolver()
jinja_env.assets_environment = assets_env
def __init__(self,
paths=None,
packages=None,
assets_path=None,
globals=None,
overlay=None,
extensions=None,
env_template_params_key='pywb.template_params',
env_template_dir_key='pywb.templates_dir'):
"""Construct a new JinjaEnv.
:param list[str] paths: List of paths to search for templates
:param list[str] packages: List of assets package names
:param str assets_path: Path to a yaml file containing assets
:param dict[str, str] globals: Dictionary of additional globals available during template rendering
:param overlay:
:param list extensions: List of webassets extension classes
:param str env_template_params_key: The full pywb package key for the template params
:param str env_template_dir_key: The full pywb package key for the template directory
"""
if paths is None:
paths = ['templates', '.', '/']
if packages is None:
packages = ['pywb']
self._init_filters()
loader = ChoiceLoader(self._make_loaders(paths, packages))
self.env_template_params_key = env_template_params_key
self.env_template_dir_key = env_template_dir_key
extensions = extensions or []
if assets_path:
extensions.append(AssetsExtension)
if overlay:
jinja_env = overlay.jinja_env.overlay(
loader=loader,
autoescape=select_autoescape(),
trim_blocks=True,
extensions=extensions)
else:
jinja_env = RelEnvironment(loader=loader,
autoescape=select_autoescape(),
trim_blocks=True,
extensions=extensions)
jinja_env.filters.update(self.filters)
if globals:
jinja_env.globals.update(globals)
self.jinja_env = jinja_env
# init assets
if assets_path:
assets_loader = YAMLLoader(load(assets_path))
assets_env = assets_loader.load_environment()
assets_env.resolver = PkgResResolver()
jinja_env.assets_environment = assets_env
self.default_locale = ''
