def load_all(self, webarchives_path): wa_file = load(webarchives_path) with closing(wa_file): for doc in yaml.load_all(wa_file): webarchives = doc['webarchives'] for pk, webarchive in webarchives.items(): self.load_archive(pk, webarchive)
def _do_download_cdxj(self, cdxj_key, output_key): lock_key = None try: rec_info_key = cdxj_key.rsplit(':', 1)[0] + ':info' cdxj_filename = self.redis.hget(rec_info_key, self.INDEX_FILE_KEY) if not cdxj_filename: logging.debug('No index for ' + rec_info_key) return lock_key = cdxj_key + ':_' logging.debug('Downloading for {0} file {1}'.format( rec_info_key, cdxj_filename)) attempts = 0 if not self.redis.set(lock_key, 1, nx=True): logging.warning('Already downloading, skipping') lock_key = None return while attempts < 10: fh = None try: fh = load(cdxj_filename) buff = fh.read() for cdxj_line in buff.splitlines(): self.redis.zadd(output_key, 0, cdxj_line) break except Exception as e: import traceback traceback.print_exc() logging.error('Could not load: ' + cdxj_filename) attempts += 1 finally: if fh: fh.close() if self.COLL_CDXJ_TTL > 0: self.redis.expire(output_key, self.COLL_CDXJ_TTL) except Exception as e: logging.error('Error downloading cache: ' + str(e)) import traceback traceback.print_exc() finally: if lock_key: self.redis.delete(lock_key)
def _do_download_cdxj(self, cdxj_key, output_key): lock_key = None try: rec_info_key = cdxj_key.rsplit(':', 1)[0] + ':info' cdxj_filename = self.redis.hget(rec_info_key, self.INDEX_FILE_KEY) if not cdxj_filename: logger.debug('CDX Sync: No index for ' + rec_info_key) return lock_key = cdxj_key + ':_' logger.debug('CDX Sync: Downloading for {0} file {1}'.format( rec_info_key, cdxj_filename)) attempts = 0 if not self.redis.set( lock_key, 1, ex=self.COMMIT_WAIT_SECS, nx=True): logger.warning( 'CDX Sync: Already downloading, skipping: {0}'.format( cdxj_filename)) lock_key = None return while attempts < 10: fh = None try: fh = load(cdxj_filename) buff = fh.read() for cdxj_line in buff.splitlines(): self.redis.zadd(output_key, 0, cdxj_line) break except Exception as e: traceback.print_exc() logger.error('CDX Sync: Could not load: ' + cdxj_filename) attempts += 1 finally: if fh: fh.close() self.reset_cdxj_ttl(output_key) except Exception as e: logger.error('CDX Sync: Error downloading cache: ' + str(e)) traceback.print_exc() finally: if lock_key: self.redis.delete(lock_key)
def get_full_version(): full_version = 'unknown' curr_app = sys.argv[0].rsplit(os.path.sep)[-1] try: # standalone app, read baked-in _full_version if getattr(sys, 'frozen', False): from pywb.utils.loaders import load full_version = load( 'pkg://webrecorder/config/_full_version').read() full_version = full_version.decode('utf-8').format(curr_app) else: # generate full_version dynamically full_version = get_version_str() except: pass return (full_version % curr_app)
def __init__(self, paths=['templates', '.', '/'], packages=['pywb'], assets_path=None, globals=None, overlay=None, extensions=None, env_template_params_key='pywb.template_params', env_template_dir_key='pywb.templates_dir'): self._init_filters() loader = ChoiceLoader(self._make_loaders(paths, packages)) self.env_template_params_key = env_template_params_key self.env_template_dir_key = env_template_dir_key extensions = extensions or [] if assets_path: extensions.append(AssetsExtension) if overlay: jinja_env = overlay.jinja_env.overlay(loader=loader, trim_blocks=True, extensions=extensions) else: jinja_env = RelEnvironment(loader=loader, trim_blocks=True, extensions=extensions) jinja_env.filters.update(self.filters) if globals: jinja_env.globals.update(globals) self.jinja_env = jinja_env # init assets if assets_path: assets_loader = YAMLLoader(load(assets_path)) assets_env = assets_loader.load_environment() assets_env.resolver = PkgResResolver() jinja_env.assets_environment = assets_env
def __init__(self, paths=None, packages=None, assets_path=None, globals=None, overlay=None, extensions=None, env_template_params_key='pywb.template_params', env_template_dir_key='pywb.templates_dir'): """Construct a new JinjaEnv. :param list[str] paths: List of paths to search for templates :param list[str] packages: List of assets package names :param str assets_path: Path to a yaml file containing assets :param dict[str, str] globals: Dictionary of additional globals available during template rendering :param overlay: :param list extensions: List of webassets extension classes :param str env_template_params_key: The full pywb package key for the template params :param str env_template_dir_key: The full pywb package key for the template directory """ if paths is None: paths = ['templates', '.', '/'] if packages is None: packages = ['pywb'] self._init_filters() loader = ChoiceLoader(self._make_loaders(paths, packages)) self.env_template_params_key = env_template_params_key self.env_template_dir_key = env_template_dir_key extensions = extensions or [] if assets_path: extensions.append(AssetsExtension) if overlay: jinja_env = overlay.jinja_env.overlay( loader=loader, autoescape=select_autoescape(), trim_blocks=True, extensions=extensions) else: jinja_env = RelEnvironment(loader=loader, autoescape=select_autoescape(), trim_blocks=True, extensions=extensions) jinja_env.filters.update(self.filters) if globals: jinja_env.globals.update(globals) self.jinja_env = jinja_env # init assets if assets_path: assets_loader = YAMLLoader(load(assets_path)) assets_env = assets_loader.load_environment() assets_env.resolver = PkgResResolver() jinja_env.assets_environment = assets_env self.default_locale = ''