def pytest_runtest_setup(item):
marker = item.get_marker('cassandra')
if marker is not None:
if not database.setup():
pytest.skip('No connection to Cassandra')
marker = item.get_marker('elasticsearch')
if marker is not None:
if not use_es:
pytest.skip('No connection to Elasticsearch')
def pytest_runtest_setup(item):
marker = item.get_marker('cassandra')
if marker is not None:
if not database.setup():
pytest.skip('No connection to Cassandra')
marker = item.get_marker('elasticsearch')
if marker is not None:
if not use_es:
pytest.skip('No connection to Elasticsearch')
def try_n_times(n, action, *args, **kwargs):
for _ in xrange(n):
try:
return action(*args, **kwargs)
except Exception as e:
logger.exception(e)
time.sleep(15)
connection_open = setup(force=True, sync=False)
logger.info("Trying again... Cassandra connection open: {}".format(connection_open))
if e:
raise e
import requests
from celery import Celery
from scrapi import util
from scrapi import events
from scrapi import database
from scrapi import settings
from scrapi import registry
from scrapi import processing
from scrapi.util import timestamp
app = Celery()
app.config_from_object(settings)
database.setup()
logger = logging.getLogger(__name__)
@app.task
@events.creates_task(events.HARVESTER_RUN)
def run_harvester(harvester_name, days_back=1):
logger.info('Running harvester "{}"'.format(harvester_name))
normalization = begin_normalization.s(harvester_name)
start_harvest = harvest.si(harvester_name, timestamp(), days_back=days_back)
# Form and start a celery chain
(start_harvest | normalization).apply_async()
from datetime import date, timedelta
from celery import Celery
from scrapi import util
from scrapi import events
from scrapi import database
from scrapi import settings
from scrapi import registry
from scrapi import processing
from scrapi.util import timestamp
app = Celery()
app.config_from_object(settings)
database.setup()
logger = logging.getLogger(__name__)
def task_autoretry(*args_task, **kwargs_task):
def actual_decorator(func):
@app.task(*args_task, **kwargs_task)
@functools.wraps(func)
def wrapper(*args, **kwargs):
try:
func(*args, **kwargs)
except kwargs_task.get('autoretry_on', Exception) as exc:
logger.info('Retrying with exception {}'.format(exc))
wrapper.retry(exc=exc)
return wrapper
return actual_decorator