def ressources(): i = Indexing() ressources = [] for h, freq in i.ressources: domain_freq = i.ressources_number_domains(h) context = lookyloo.context.find_known_content(h) capture_uuid, url_uuid, hostnode_uuid = i.get_hash_uuids(h) ressources.append((h, freq, domain_freq, context.get(h), capture_uuid, url_uuid, hostnode_uuid)) return render_template('ressources.html', ressources=ressources)
def main(): parser = argparse.ArgumentParser(description='Rebuild the redis cache.') parser.add_argument( '--rebuild_pickles', default=False, action='store_true', help= 'Delete and rebuild the pickles. Count 20s/pickle, it can take a very long time.' ) args = parser.parse_args() lookyloo = Lookyloo() if args.rebuild_pickles: lookyloo.rebuild_all() else: lookyloo.rebuild_cache() indexing = Indexing() indexing.clear_indexes() # This call will rebuild all the caches as needed. lookyloo.sorted_capture_cache()
def main(): parser = argparse.ArgumentParser(description='Rebuild the redis cache.') parser.add_argument( '--rebuild_pickles', default=False, action='store_true', help= 'Delete and rebuild the pickles. Count 20s/pickle, it can take a very long time.' ) args = parser.parse_args() lookyloo = Lookyloo() if args.rebuild_pickles: lookyloo.rebuild_all() else: lookyloo.rebuild_cache() indexing = Indexing() indexing.clear_indexes() for capture_uuid in lookyloo.capture_uuids: index = True try: tree = lookyloo.get_crawled_tree(capture_uuid) except Exception as e: print(capture_uuid, e) continue if lookyloo.is_public_instance: cache = lookyloo.capture_cache(capture_uuid) if cache.get('no_index') is not None: index = False # NOTE: these two methods do nothing if we just generated the pickle if index: indexing.index_cookies_capture(tree) indexing.index_body_hashes_capture(tree)
def cookies_lookup(): i = Indexing() cookies_names = [(name, freq, i.cookies_names_number_domains(name)) for name, freq in i.cookies_names] return render_template('cookies.html', cookies_names=cookies_names)
def categories(): i = Indexing() print(i.categories) return render_template('categories.html', categories=i.categories)