def _handle_successful_job(self, job):
"""Handle successufl jobs"""
result = job.result
task_id = job.kwargs['task_id']
try:
task = self.registry.get(task_id)
except NotFoundError:
logger.warning(
"Task %s not found; related job #%s will not be rescheduled",
task_id, job.id)
return
job_args = self._build_job_arguments(task)
job_args['fetch_from_cache'] = False
if result.nitems > 0:
from_date = unixtime_to_datetime(result.max_date)
job_args['backend_args']['from_date'] = from_date
if result.offset:
job_args['backend_args']['offset'] = result.offset
delay = task.sched_args['delay']
job_id = self._scheduler.schedule_job_task(Q_UPDATING_JOBS,
task_id,
job_args,
delay=delay)
logger.info("Job #%s (task: %s, old job: %s) re-scheduled", job_id,
task_id, job.id)
def run(self, backend_args, archive_args=None, resume=False):
"""Run the backend with the given parameters.
The method will run the backend assigned to this job,
storing the fetched items in a Redis queue. The ongoing
status of the job, can be accessed through the property
`result`. When `resume` is set, the job will start from
the last execution, overwriting 'from_date' and 'offset'
parameters, if needed.
Setting to `True` the parameter `fetch_from_archive`, items can
be fetched from the archive assigned to this job.
Any exception during the execution of the process will
be raised.
:param backend_args: parameters used to un the backend
:param archive_args: archive arguments
:param resume: fetch items starting where the last
execution stopped
"""
args = backend_args.copy()
if archive_args:
self.initialize_archive_manager(archive_args['archive_path'])
if not resume:
self._result = JobResult(self.job_id,
self.task_id,
self.backend,
self.category,
None,
None,
0,
offset=None,
nresumed=0)
else:
if self.result.max_date:
args['from_date'] = unixtime_to_datetime(self.result.max_date)
if self.result.offset:
args['offset'] = self.result.offset
self._result.nresumed += 1
for item in self._execute(args, archive_args):
self.conn.rpush(self.qitems, pickle.dumps(item))
self._result.nitems += 1
self._result.last_uuid = item['uuid']
if not self.result.max_date or self.result.max_date < item[
'updated_on']:
self._result.max_date = item['updated_on']
if 'offset' in item:
self._result.offset = item['offset']
def test_dates(self):
"""Check if it converts some timestamps to datetime objects."""
date = unixtime_to_datetime(0)
expected = datetime.datetime(1970,
1,
1,
0,
0,
0,
tzinfo=dateutil.tz.tzutc())
self.assertIsInstance(date, datetime.datetime)
self.assertEqual(date, expected)
date = unixtime_to_datetime(1426868155.0)
expected = datetime.datetime(2015,
3,
20,
16,
15,
55,
tzinfo=dateutil.tz.tzutc())
self.assertIsInstance(date, datetime.datetime)
self.assertEqual(date, expected)
def run(self, backend_args, archive_args=None, resume=False):
"""Run the backend with the given parameters.
The method will run the backend assigned to this job,
storing the fetched items in a Redis queue. The ongoing
status of the job, can be accessed through the property
`result`. When `resume` is set, the job will start from
the last execution, overwriting 'from_date' and 'offset'
parameters, if needed.
Setting to `True` the parameter `fetch_from_archive`, items can
be fetched from the archive assigned to this job.
Any exception during the execution of the process will
be raised.
:param backend_args: parameters used to un the backend
:param archive_args: archive arguments
:param resume: fetch items starting where the last
execution stopped
"""
args = backend_args.copy()
if archive_args:
self.initialize_archive_manager(archive_args['archive_path'])
if not resume:
self._result = JobResult(self.job_id, self.task_id, self.backend, self.category,
None, None, 0, offset=None,
nresumed=0)
else:
if self.result.max_date:
args['from_date'] = unixtime_to_datetime(self.result.max_date)
if self.result.offset:
args['offset'] = self.result.offset
self._result.nresumed += 1
for item in self._execute(args, archive_args):
self.conn.rpush(self.qitems, pickle.dumps(item))
self._result.nitems += 1
self._result.last_uuid = item['uuid']
if not self.result.max_date or self.result.max_date < item['updated_on']:
self._result.max_date = item['updated_on']
if 'offset' in item:
self._result.offset = item['offset']
def _handle_successful_job(self, job):
"""Handle successufl jobs"""
result = job.result
task_id = job.kwargs['task_id']
try:
task = self.registry.get(task_id)
except NotFoundError:
logger.warning("Task %s not found; related job #%s will not be rescheduled",
task_id, job.id)
return
archive_task = task.archive_args.get('fetch_from_archive', False)
if archive_task:
logger.info("Job #%s (task: %s) successfully finished", job.id, task_id)
return
job_args = self._build_job_arguments(task)
if result.nitems > 0:
from_date = unixtime_to_datetime(result.max_date)
job_args['backend_args']['from_date'] = from_date
if result.offset:
job_args['backend_args']['offset'] = result.offset
delay = task.sched_args['delay']
job_id = self._scheduler.schedule_job_task(Q_UPDATING_JOBS,
task_id, job_args,
delay=delay)
logger.info("Job #%s (task: %s, old job: %s) re-scheduled",
job_id, task_id, job.id)