def run_flush_extract_cache(opts, **kwargs):
"""
Flush the extraction cache.
"""
URLCache.flush()
ExtractCache.flush()
ThumbnailCache.flush()
echo('Extraction caches flushed.', no_color=opts.no_color)
def run_flush_extract_cache(opts, **kwargs):
"""
Flush the extraction cache.
"""
from newslynx.models import URLCache, ExtractCache, ThumbnailCache
URLCache.flush()
ExtractCache.flush()
ThumbnailCache.flush()
log.info('Extraction caches flushed.')
def run_flush_extract_cache(opts, **kwargs):
"""
Flush the extraction cache.
"""
from newslynx.models import URLCache, ExtractCache, ThumbnailCache
URLCache.flush()
ExtractCache.flush()
ThumbnailCache.flush()
log.info('Extraction caches flushed.')
def flush_work_cache():
URLCache.flush()
ExtractCache.flush()
ThumbnailCache.flush()
patch_all()
from gevent.pool import Pool
from newslynx.lib import dates
from newslynx.lib import url
from newslynx.lib import text
from newslynx.lib import html
from newslynx.lib import stats
from newslynx.models import URLCache, ThumbnailCache
from newslynx import settings
from newslynx.exc import RequestError
from newslynx.constants import METRIC_FACET_KEYS
# the url cache object
url_cache = URLCache()
thumbnail_cache = ThumbnailCache()
# a pool to multithread url_cache.
url_cache_pool = Pool(settings.URL_CACHE_POOL_SIZE)
def prepare_links(links=[], domains=[]):
"""
Prepare links to be tested against content items.
"""
if len(domains):
_links = []
for l in links:
if any([d in l for d in domains]):
_links.append(l)
links = _links