from collections import deque from ocelot.lib import logging from ocelot.pipeline.exceptions import StopProcessingException from ocelot.services.pipeline_schedule import PipelineScheduleService from ocelot.services.task import TaskService from ocelot.services.task_connection import TaskConnectionService from ocelot.services.mappers.pipeline import PipelineMapper from ocelot.services.repositories.pipeline import PipelineRepository log = logging.getLogger('ocelot.pipelines') class PipelineService(object): @classmethod def fetch_pipeline_by_id(cls, id): """Returns PipelineEntity by id. :param str id: :returns PipelineEntity: """ return PipelineMapper.to_entity( PipelineRepository.fetch_pipeline_by_id(id), ) @classmethod def run_pipeline_by_id(cls, id): """Runs a pipeline by id. :param str id: """
from ocelot.lib import logging from ocelot.services.constants.task import TASK_MAP from ocelot.services.mappers.task import TaskMapper from ocelot.services.repositories.task import TaskRepository log = logging.getLogger('ocelot.tasks') class TaskService(object): @classmethod def fetch_task_by_id(cls, id): """Fetch TaskEntity by id. :param str id: :returns TaskEntity: """ return TaskMapper.to_entity( TaskRepository.fetch_task_by_id(id), ) @classmethod def process_task_with_data(cls, id, data): """Instantiates and runs task by ID. :param str id: :param object data: :returns object: response from task class """ task_entity = cls.fetch_task_by_id(id)
from datetime import datetime, timedelta from croniter import croniter from ocelot.lib import logging from ocelot.services.constants.pipeline_schedule import PipelineScheduleTypes from ocelot.services.exceptions import ResourceNotFoundException from ocelot.services.mappers.pipeline_schedule import PipelineScheduleMapper from ocelot.services.repositories.pipeline_schedule import PipelineScheduleRepository log = logging.getLogger('ocelot.pipeline_schedules') class PipelineScheduleService(object): @classmethod def fetch_schedule_for_pipeline(cls, pipeline_id): """Returns a PipelineScheduleEntity for a pipeline_id. :param str pipeline_id: :returns PipelineScheduleEntity: :raises ResourceNotFoundException: if not found """ return PipelineScheduleMapper.to_entity( PipelineScheduleRepository.fetch_schedule_for_pipeline( pipeline_id, ), ) @classmethod def fetch_schedules_to_run(cls): """Return a list of PipelineScheduleEntities that need to run. :returns list: PipelineScheduleEntity
from datetime import datetime, timedelta from croniter import croniter from ocelot.lib import logging from ocelot.services.constants.pipeline_schedule import PipelineScheduleTypes from ocelot.services.exceptions import ResourceNotFoundException from ocelot.services.mappers.pipeline_schedule import PipelineScheduleMapper from ocelot.services.repositories.pipeline_schedule import PipelineScheduleRepository log = logging.getLogger('ocelot.pipeline_schedules') class PipelineScheduleService(object): @classmethod def fetch_schedule_for_pipeline(cls, pipeline_id): """Returns a PipelineScheduleEntity for a pipeline_id. :param str pipeline_id: :returns PipelineScheduleEntity: :raises ResourceNotFoundException: if not found """ return PipelineScheduleMapper.to_entity( PipelineScheduleRepository.fetch_schedule_for_pipeline( pipeline_id, ), ) @classmethod def fetch_schedules_to_run(cls): """Return a list of PipelineScheduleEntities that need to run.
def process(self, data): """Logs the data to a logger. :param object data: """ logging.getLogger(self.log_name).info(data)
import time from ocelot.lib import logging from ocelot.services import datastores from ocelot.services.pipeline import PipelineService from ocelot.services.pipeline_schedule import PipelineScheduleService log = logging.getLogger('ocelot.scheduler') SLEEP_SECONDS = 10 if __name__ == '__main__': datastores.create_tables() datastores.initialize() log.info('Starting scheduler') try: while True: pipeline_schedules = PipelineScheduleService.fetch_schedules_to_run() log.info('Found {} pipelines to run'.format(len(pipeline_schedules))) for schedule in PipelineScheduleService.fetch_schedules_to_run(): PipelineService.run_pipeline_by_id(schedule.pipeline_id) time.sleep(SLEEP_SECONDS) except KeyboardInterrupt: pass log.info('Shutting down scheduler')