from collections import deque
from ocelot.lib import logging
from ocelot.pipeline.exceptions import StopProcessingException
from ocelot.services.pipeline_schedule import PipelineScheduleService
from ocelot.services.task import TaskService
from ocelot.services.task_connection import TaskConnectionService
from ocelot.services.mappers.pipeline import PipelineMapper
from ocelot.services.repositories.pipeline import PipelineRepository
log = logging.getLogger('ocelot.pipelines')
class PipelineService(object):
@classmethod
def fetch_pipeline_by_id(cls, id):
"""Returns PipelineEntity by id.
:param str id:
:returns PipelineEntity:
"""
return PipelineMapper.to_entity(
PipelineRepository.fetch_pipeline_by_id(id),
)
@classmethod
def run_pipeline_by_id(cls, id):
"""Runs a pipeline by id.
:param str id:
"""
from ocelot.lib import logging
from ocelot.services.constants.task import TASK_MAP
from ocelot.services.mappers.task import TaskMapper
from ocelot.services.repositories.task import TaskRepository
log = logging.getLogger('ocelot.tasks')
class TaskService(object):
@classmethod
def fetch_task_by_id(cls, id):
"""Fetch TaskEntity by id.
:param str id:
:returns TaskEntity:
"""
return TaskMapper.to_entity(
TaskRepository.fetch_task_by_id(id),
)
@classmethod
def process_task_with_data(cls, id, data):
"""Instantiates and runs task by ID.
:param str id:
:param object data:
:returns object: response from task class
"""
task_entity = cls.fetch_task_by_id(id)
from datetime import datetime, timedelta
from croniter import croniter
from ocelot.lib import logging
from ocelot.services.constants.pipeline_schedule import PipelineScheduleTypes
from ocelot.services.exceptions import ResourceNotFoundException
from ocelot.services.mappers.pipeline_schedule import PipelineScheduleMapper
from ocelot.services.repositories.pipeline_schedule import PipelineScheduleRepository
log = logging.getLogger('ocelot.pipeline_schedules')
class PipelineScheduleService(object):
@classmethod
def fetch_schedule_for_pipeline(cls, pipeline_id):
"""Returns a PipelineScheduleEntity for a pipeline_id.
:param str pipeline_id:
:returns PipelineScheduleEntity:
:raises ResourceNotFoundException: if not found
"""
return PipelineScheduleMapper.to_entity(
PipelineScheduleRepository.fetch_schedule_for_pipeline(
pipeline_id, ), )
@classmethod
def fetch_schedules_to_run(cls):
"""Return a list of PipelineScheduleEntities that need to run.
:returns list: PipelineScheduleEntity
from datetime import datetime, timedelta
from croniter import croniter
from ocelot.lib import logging
from ocelot.services.constants.pipeline_schedule import PipelineScheduleTypes
from ocelot.services.exceptions import ResourceNotFoundException
from ocelot.services.mappers.pipeline_schedule import PipelineScheduleMapper
from ocelot.services.repositories.pipeline_schedule import PipelineScheduleRepository
log = logging.getLogger('ocelot.pipeline_schedules')
class PipelineScheduleService(object):
@classmethod
def fetch_schedule_for_pipeline(cls, pipeline_id):
"""Returns a PipelineScheduleEntity for a pipeline_id.
:param str pipeline_id:
:returns PipelineScheduleEntity:
:raises ResourceNotFoundException: if not found
"""
return PipelineScheduleMapper.to_entity(
PipelineScheduleRepository.fetch_schedule_for_pipeline(
pipeline_id,
),
)
@classmethod
def fetch_schedules_to_run(cls):
"""Return a list of PipelineScheduleEntities that need to run.
def process(self, data):
"""Logs the data to a logger.
:param object data:
"""
logging.getLogger(self.log_name).info(data)
import time
from ocelot.lib import logging
from ocelot.services import datastores
from ocelot.services.pipeline import PipelineService
from ocelot.services.pipeline_schedule import PipelineScheduleService
log = logging.getLogger('ocelot.scheduler')
SLEEP_SECONDS = 10
if __name__ == '__main__':
datastores.create_tables()
datastores.initialize()
log.info('Starting scheduler')
try:
while True:
pipeline_schedules = PipelineScheduleService.fetch_schedules_to_run()
log.info('Found {} pipelines to run'.format(len(pipeline_schedules)))
for schedule in PipelineScheduleService.fetch_schedules_to_run():
PipelineService.run_pipeline_by_id(schedule.pipeline_id)
time.sleep(SLEEP_SECONDS)
except KeyboardInterrupt:
pass
log.info('Shutting down scheduler')
