def load(*entries): """Initialize references, load previous state.""" global __orig_refs, __seen_refs, __entry_map __seen_refs = defaultdict(set) __orig_refs = cache.memoize('references') or defaultdict(set) __entry_map = dict((hash(entry), entry) for entry in chain(*entries))
def paginate(lst, ipp, salt="", orphans=0): """paginate(lst, ipp, func=lambda x: x, salt=None, orphans=0) Yields a triple ((next, current, previous), list of entries, has changed) of a paginated entrylist. It will first filter by the specified function, then split the ist into several sublists and check wether the list or an entry has changed. :param lst: the entrylist containing Entry instances. :param ipp: items per page :param salt: uses as additional identifier in memoize :param orphans: avoid N orphans on last page >>> for x, values, _, paginate(entryrange(20), 6, orphans=2): ... print(x, values) (None, 0, 1), [entries 1..6] (0, 1, 2), [entries 7..12] (1, 2, None), [entries 12..20]""" # detect removed or newly added entries modified = cache.memoize('paginate-' + salt, hash(*lst)) # slice into batches res = list(batch(lst, ipp)) if len(res) >= 2 and len(res[-1]) <= orphans: res[-2].extend(res[-1]) res.pop(-1) j = len(res) for i, entries in enumerate(res): i += 1 next = None if i == 1 else i-1 curr = i prev = None if i >= j else i+1 yield (next, curr, prev), entries, modified or any(e.modified for e in entries)
def modified(self): return hash(self) != cache.memoize(self.__class__.__name__)
def compile(conf, env): """The compilation process.""" if env.options.force: cache.clear(conf.get("cache_dir")) # time measurement ctime = time.time() # populate env and corrects some conf things data = initialize(conf, env) # load pages/entries and store them in env rv = dict(zip(["entrylist", "pages", "translations", "drafts"], map(HashableList, readers.load(conf)))) entrylist, pages = rv["entrylist"], rv["pages"] translations, drafts = rv["translations"], rv["drafts"] # load references refs.load(entrylist, pages, translations, drafts) data.update(rv) env.globals.update(rv) # here we store all found filter and their aliases ns = defaultdict(set) # [<class head_offset.Headoffset at 0x1014882c0>, <class html.HTML at 0x101488328>,...] aflist = filters.get_filters() # ... and get all configured views _views = views.get_views() # filters found in all entries, views and conf.py (skip translations, has no items) found = sum((x.filters for x in chain(entrylist, pages, drafts, _views, [conf])), []) for val in found: # first we for `no` and get the function name and arguments f = val[2:] if val.startswith("no") else val fname, fargs = f.split("+")[:1][0], f.split("+")[1:] try: # initialize the filter with its function name and arguments fx = aflist[fname](conf, env, val, *fargs) if val.startswith("no"): fx = filters.disable(fx) except ValueError: try: fx = aflist[val.split("+")[:1][0]](conf, env, val, *fargs) except ValueError: raise AcrylamidException("no such filter: %s" % val) ns[fx].add(val) # include actual used filters to trigger modified state env.filters = HashableList(ns.keys()) for entry in chain(entrylist, pages, drafts): for v in _views: # a list that sorts out conflicting and duplicated filters flst = filters.FilterList() # filters found in this specific entry plus views and conf.py found = entry.filters + v.filters + data["conf"]["filters"] for fn in found: fx, _ = next((k for k in ns.iteritems() if fn in k[1])) if fx not in flst: flst.append(fx) # sort them ascending because we will pop within filters.add entry.filters.add(sorted(flst, key=lambda k: (-k.priority, k.name)), context=v) # lets offer a last break to populate tags and such for v in _views: env = v.context(conf, env, data) # now teh real thing! for v in _views: for entry in chain(entrylist, pages, translations, drafts): entry.context = v for var in "entrylist", "pages", "translations", "drafts": data[var] = HashableList(filter(v.condition, locals()[var])) if v.condition else locals()[var] tt = time.time() for buf, path in v.generate(conf, env, data): try: helpers.mkfile( buf, path, time.time() - tt, ns=v.name, force=env.options.force, dryrun=env.options.dryrun ) except UnicodeError: log.exception(path) finally: buf.close() tt = time.time() # copy modified/missing assets to output assets.compile(conf, env) # wait for unfinished hooks hooks.shutdown() # save conf/environment hash and new/changed/unchanged references helpers.memoize("Configuration", hash(conf)) helpers.memoize("Environment", hash(env)) refs.save() # remove abandoned cache files cache.shutdown() # print a short summary log.info( "%i new, %i updated, %i skipped [%.2fs]", event.count("create"), event.count("update"), event.count("identical") + event.count("skip"), time.time() - ctime, )
def append(entry, *references): """Appenf `references` to `entry`.""" global __seen_refs for ref in references: __seen_refs[hash(entry)].add(hash(ref))
def references(entry): """Return hash for entry and the referenced entries' hashes.""" global __seen_refs return hash(entry), __seen_refs.get(hash(entry), set())
def compile(conf, env): """The compilation process.""" if env.options.force: cache.clear(conf.get('cache_dir')) # time measurement ctime = time.time() # populate env and corrects some conf things data = initialize(conf, env) # load pages/entries and store them in env rv = dict(zip(['entrylist', 'pages', 'translations', 'drafts'], map(HashableList, readers.load(conf)))) entrylist, pages = rv['entrylist'], rv['pages'] translations, drafts = rv['translations'], rv['drafts'] # load references refs.load(entrylist, pages, translations, drafts) data.update(rv) env.globals.update(rv) # here we store all found filter and their aliases ns = defaultdict(set) # get available filter list, something like with obj.get-function # list = [<class head_offset.Headoffset at 0x1014882c0>, <class html.HTML at 0x101488328>,...] aflist = filters.get_filters() # ... and get all configured views _views = views.get_views() # filters found in all entries, views and conf.py (skip translations, has no items) found = sum((x.filters for x in chain(entrylist, pages, drafts, _views, [conf])), []) for val in found: # first we for `no` and get the function name and arguments f = val[2:] if val.startswith('no') else val fname, fargs = f.split('+')[:1][0], f.split('+')[1:] try: # initialize the filter with its function name and arguments fx = aflist[fname](conf, env, val, *fargs) if val.startswith('no'): fx = filters.disable(fx) except ValueError: try: fx = aflist[val.split('+')[:1][0]](conf, env, val, *fargs) except ValueError: raise AcrylamidException('no such filter: %s' % val) ns[fx].add(val) for entry in chain(entrylist, pages, drafts): for v in _views: # a list that sorts out conflicting and duplicated filters flst = filters.FilterList() # filters found in this specific entry plus views and conf.py found = entry.filters + v.filters + data['conf']['filters'] for fn in found: fx, _ = next((k for k in ns.iteritems() if fn in k[1])) if fx not in flst: flst.append(fx) # sort them ascending because we will pop within filters.add entry.filters.add(sorted(flst, key=lambda k: (-k.priority, k.name)), context=v) # lets offer a last break to populate tags and such for v in _views: env = v.context(conf, env, data) # now teh real thing! for v in _views: for entry in chain(entrylist, pages, translations, drafts): entry.context = v for var in 'entrylist', 'pages', 'translations', 'drafts': data[var] = HashableList(filter(v.condition, locals()[var])) \ if v.condition else locals()[var] tt = time.time() for buf, path in v.generate(conf, env, data): try: helpers.mkfile(buf, path, time.time()-tt, **env.options.__dict__) finally: buf.close() tt = time.time() # copy modified/missing assets to output assets.compile(conf, env) # save conf/environment hash and new/changed/unchanged references helpers.memoize('Configuration', hash(conf)) helpers.memoize('Environment', hash(env)) refs.save() # remove abandoned cache files cache.shutdown() # print a short summary log.info('%i new, %i updated, %i skipped [%.2fs]', event.count('create'), event.count('update'), event.count('identical') + event.count('skip'), time.time() - ctime)
def compile(conf, env): """The compilation process.""" hooks.initialize(conf, env) hooks.run(conf, env, 'pre') if env.options.force: cache.clear(conf.get('cache_dir')) # time measurement ctime = time.time() # populate env and corrects some conf things data = initialize(conf, env) # load pages/entries and store them in env rv = dict( zip(['entrylist', 'pages', 'translations', 'drafts'], map(HashableList, readers.load(conf)))) entrylist, pages = rv['entrylist'], rv['pages'] translations, drafts = rv['translations'], rv['drafts'] # load references refs.load(entrylist, pages, translations, drafts) data.update(rv) env.globals.update(rv) # here we store all found filter and their aliases ns = defaultdict(set) # [<class head_offset.Headoffset at 0x1014882c0>, <class html.HTML at 0x101488328>,...] aflist = filters.get_filters() # ... and get all configured views _views = views.get_views() # filters found in all entries, views and conf.py (skip translations, has no items) found = sum((x.filters for x in chain(entrylist, pages, drafts, _views, [conf])), []) for val in found: # first we for `no` and get the function name and arguments f = val[2:] if val.startswith('no') else val fname, fargs = f.split('+')[:1][0], f.split('+')[1:] try: # initialize the filter with its function name and arguments fx = aflist[fname](conf, env, val, *fargs) if val.startswith('no'): fx = filters.disable(fx) except ValueError: try: fx = aflist[val.split('+')[:1][0]](conf, env, val, *fargs) except ValueError: raise AcrylamidException('no such filter: %s' % val) ns[fx].add(val) # include actual used filters to trigger modified state env.filters = HashableList(iterkeys(ns)) for entry in chain(entrylist, pages, drafts): for v in _views: # a list that sorts out conflicting and duplicated filters flst = filters.FilterList() # filters found in this specific entry plus views and conf.py found = entry.filters + v.filters + data['conf']['filters'] for fn in found: fx, _ = next((k for k in iteritems(ns) if fn in k[1])) if fx not in flst: flst.append(fx) # sort them ascending because we will pop within filters.add entry.filters.add(sorted(flst, key=lambda k: (-k.priority, k.name)), context=v) # lets offer a last break to populate tags and such for v in _views: env = v.context(conf, env, data) # now teh real thing! for v in _views: for entry in chain(entrylist, pages, translations, drafts): entry.context = v for var in 'entrylist', 'pages', 'translations', 'drafts': data[var] = HashableList(filter(v.condition, locals()[var])) \ if v.condition else locals()[var] tt = time.time() for buf, path in v.generate(conf, env, data): try: helpers.mkfile(buf, path, time.time() - tt, ns=v.name, force=env.options.force, dryrun=env.options.dryrun) except UnicodeError: log.exception(path) finally: buf.close() tt = time.time() # copy modified/missing assets to output assets.compile(conf, env) # wait for unfinished hooks hooks.shutdown() # run post hooks (blocks) hooks.run(conf, env, 'post') # save conf/environment hash and new/changed/unchanged references helpers.memoize('Configuration', hash(conf)) helpers.memoize('Environment', hash(env)) refs.save() # remove abandoned cache files cache.shutdown() # print a short summary log.info('%i new, %i updated, %i skipped [%.2fs]', event.count('create'), event.count('update'), event.count('identical') + event.count('skip'), time.time() - ctime)