예제 #1
0
def pytest_runtest_setup(item):
    marker = item.get_marker('cassandra')
    if marker is not None:
        if not database.setup():
            pytest.skip('No connection to Cassandra')

    marker = item.get_marker('elasticsearch')
    if marker is not None:
        if not use_es:
            pytest.skip('No connection to Elasticsearch')
예제 #2
0
def pytest_runtest_setup(item):
    marker = item.get_marker('cassandra')
    if marker is not None:
        if not database.setup():
            pytest.skip('No connection to Cassandra')

    marker = item.get_marker('elasticsearch')
    if marker is not None:
        if not use_es:
            pytest.skip('No connection to Elasticsearch')
예제 #3
0
def try_n_times(n, action, *args, **kwargs):
    for _ in xrange(n):
        try:
            return action(*args, **kwargs)
        except Exception as e:
            logger.exception(e)
            time.sleep(15)
            connection_open = setup(force=True, sync=False)
            logger.info("Trying again... Cassandra connection open: {}".format(connection_open))
    if e:
        raise e
예제 #4
0
파일: tasks.py 프로젝트: Eleonore9/scrapi
import requests
from celery import Celery

from scrapi import util
from scrapi import events
from scrapi import database
from scrapi import settings
from scrapi import registry
from scrapi import processing
from scrapi.util import timestamp


app = Celery()
app.config_from_object(settings)

database.setup()
logger = logging.getLogger(__name__)


@app.task
@events.creates_task(events.HARVESTER_RUN)
def run_harvester(harvester_name, days_back=1):
    logger.info('Running harvester "{}"'.format(harvester_name))

    normalization = begin_normalization.s(harvester_name)
    start_harvest = harvest.si(harvester_name, timestamp(), days_back=days_back)

    # Form and start a celery chain
    (start_harvest | normalization).apply_async()

예제 #5
0
from datetime import date, timedelta

from celery import Celery

from scrapi import util
from scrapi import events
from scrapi import database
from scrapi import settings
from scrapi import registry
from scrapi import processing
from scrapi.util import timestamp

app = Celery()
app.config_from_object(settings)

database.setup()
logger = logging.getLogger(__name__)


def task_autoretry(*args_task, **kwargs_task):
    def actual_decorator(func):
        @app.task(*args_task, **kwargs_task)
        @functools.wraps(func)
        def wrapper(*args, **kwargs):
            try:
                func(*args, **kwargs)
            except kwargs_task.get('autoretry_on', Exception) as exc:
                logger.info('Retrying with exception {}'.format(exc))
                wrapper.retry(exc=exc)
        return wrapper
    return actual_decorator