command.extend(['--hostname', hostname])
if autoreload:
command.extend(['--autoreload'])
app.worker_main(command)
@task
def harvester(harvester_name, async=False, start=None, end=None):
''' Runs a specific harvester '''
from scrapi import settings
settings.CELERY_ALWAYS_EAGER = not async
from scrapi import registry
from scrapi.tasks import run_harvester
from dateutil.parser import parse
if not registry.get(harvester_name):
raise ValueError('No such harvesters {}'.format(harvester_name))
end = parse(end).date() if end else date.today()
start = parse(start).date() if start else end - timedelta(settings.DAYS_BACK)
run_harvester.delay(harvester_name, start_date=start, end_date=end)
@task
def harvesters(async=False, start=None, end=None):
''' Runs all harvesters '''
from scrapi import settings
settings.CELERY_ALWAYS_EAGER = not async
from scrapi import registry
from scrapi.tasks import run_harvester
command.extend(['--hostname', hostname])
if autoreload:
command.extend(['--autoreload'])
app.worker_main(command)
@task
def harvester(harvester_name, async=False, start=None, end=None):
''' Runs a specific harvester '''
from scrapi import settings
settings.CELERY_ALWAYS_EAGER = not async
from scrapi import registry
from scrapi.tasks import run_harvester
from dateutil.parser import parse
if not registry.get(harvester_name):
raise ValueError('No such harvesters {}'.format(harvester_name))
end = parse(end).date() if end else date.today()
start = parse(start).date() if start else end - timedelta(settings.DAYS_BACK)
run_harvester.delay(harvester_name, start_date=start, end_date=end)
@task
def harvesters(async=False, start=None, end=None):
''' Runs all harvesters '''
from scrapi import settings
settings.CELERY_ALWAYS_EAGER = not async
from scrapi import registry
from scrapi.tasks import run_harvester