Ejemplo n.º 1
0
    def test_pipeline_finished_success(self, backend_name, session):
        event = events_pb2.PipelineStarted(
            uuid="test1",
            timestamp=timestamp_from_json_string("2017-02-02T14:25:43.511Z"))
        model.Event.write_event(session, event)

        event = events_pb2.PipelineFinished(
            uuid="test1",
            timestamp=timestamp_from_json_string("2017-02-02T14:26:43.511Z"),
            duration_ms=100,
            result=events_pb2.SUCCESS)
        model.Event.write_event(session, event)

        projection_backend = get_projection_backend(backend_name, session)
        projector = Projector(projection_backend,
                              EventSQLAlchemyBackend(session))
        projector.process_events()

        pipeline = projection_backend.get_one_by_external_id(
            "test1", model.Pipeline)
        assert pipeline is not None
        assert pipeline.external_id == 'test1'
        assert pipeline.currently_running == False
        assert pipeline.result == "SUCCESS"
        assert pipeline.started_running_at == dateutil.parser.parse(
            "2017-02-02T14:25:43.511Z")
        assert pipeline.finished_running_at == dateutil.parser.parse(
            "2017-02-02T14:26:43.511Z")
Ejemplo n.º 2
0
    def test_forwards_pipeline_started(self, amqp_connection):
        ts = timestamp_from_json_string("2012-04-23T18:25:43.511Z")
        event = events_pb2.PipelineStarted(uuid="abcdef", timestamp=ts)
        headers = {"proto-message-type": 'PipelineStarted'}
        received_msgs = publish_msg_and_get_from_other_queue(
            amqp_connection, event,
            queue_configs.EXTERNAL_QUEUES['pipeline.started'],
            queue_configs.INTERNAL_QUEUES['pipeline.started'], headers)
        assert len(received_msgs) == 1
        msg = received_msgs[0]
        assert msg.content_type == 'application/vnd.google.protobuf'
        assert msg.headers == {"proto-message-type": 'PipelineStarted'}
        assert msg.delivery_info['routing_key'] == 'pipeline.started'

        data = events_pb2.PipelineStarted()
        data.ParseFromString(msg.body)
        assert data.uuid == 'abcdef'
Ejemplo n.º 3
0
def pipeline_started(uuid=None):
    """ Example payload:
    {
        "timestamp": "2012-04-23T18:25:43.511Z"
    }
    """
    payload = request.get_json(force=True)

    rabbitmq = get_rabbitmq()
    ts = timestamp_from_json_string(payload['timestamp'])
    event = events_pb2.PipelineStarted(uuid=uuid, timestamp=ts)

    queue = queue_configs.EXTERNAL_QUEUES['pipeline.started']
    publish_to_queue(rabbitmq, event, queue)
    return " [x] Sent PipelineStarted"
Ejemplo n.º 4
0
    def test_pipeline_currently_running(self, backend_name, session):
        ts = timestamp_from_json_string("2017-02-02T14:25:43.511Z")
        event = events_pb2.PipelineStarted(uuid="test1", timestamp=ts)
        model.Event.write_event(session, event)

        projection_backend = get_projection_backend(backend_name, session)
        projector = Projector(projection_backend,
                              EventSQLAlchemyBackend(session))
        projector.process_events()
        print(projection_backend.get_all(model.Pipeline))
        pipeline = projection_backend.get_one_by_external_id(
            "test1", model.Pipeline)
        assert pipeline is not None
        assert pipeline.external_id == 'test1'
        assert pipeline.currently_running == True
        assert pipeline.result == None
Ejemplo n.º 5
0
    def _generate_pipeline_events(self):
        from ngcd_common import events_pb2
        from google.protobuf.timestamp_pb2 import Timestamp
        from google.protobuf.json_format import MessageToDict
        import random
        from datetime import timedelta
        import pytz

        pipeline_id = 0
        times = self.fake.time_series(start_date="-5d",
                                      end_date="now",
                                      precision=timedelta(minutes=1),
                                      distrib=None,
                                      tzinfo=pytz.utc)
        for i in range(0, 10):
            time = next(times)[0].isoformat()
            if i % 2 == 0:
                # Generate started event
                ts = Timestamp()
                ts.FromJsonString(time)
                event_pb = events_pb2.PipelineStarted(uuid=str(pipeline_id),
                                                      timestamp=ts)
                event_model = Event(id=self._next_global_id(),
                                    type='PipelineStarted',
                                    body=MessageToDict(
                                        event_pb,
                                        including_default_value_fields=True),
                                    event_origin_time=time)

                yield event_model
                yield from self._generate_pipeline_stages(times, pipeline_id)
            else:
                # Generate finished event
                ts = Timestamp()
                ts.FromJsonString(time)
                event_pb = events_pb2.PipelineFinished(
                    uuid=str(pipeline_id),
                    timestamp=ts,
                    result=random.choice([
                        events_pb2.SUCCESS, events_pb2.FAILURE,
                        events_pb2.ABORTED
                    ]),
                    duration_ms=random.randint(100, 10000))
                event_model = Event(id=self._next_global_id(),
                                    type='PipelineFinished',
                                    body=MessageToDict(
                                        event_pb,
                                        including_default_value_fields=True),
                                    event_origin_time=time)

                pipeline_id = pipeline_id + 1

                yield event_model

        # Create a started one, to be able to check in-progress
        time = next(times)[0].isoformat()
        ts = Timestamp()
        ts.FromJsonString(time)
        event_pb = events_pb2.PipelineStarted(uuid=str(pipeline_id),
                                              timestamp=ts)
        event_model = Event(id=self._next_global_id(),
                            type='PipelineStarted',
                            body=MessageToDict(
                                event_pb, including_default_value_fields=True),
                            event_origin_time=time)

        yield event_model
        yield from self._generate_pipeline_stages(times, pipeline_id)
Ejemplo n.º 6
0
    def test_stage_only_applies_to_same_pipeline(self, backend_name, session):
        # First pipeline
        event = events_pb2.PipelineStarted(
            uuid="test1",
            timestamp=timestamp_from_json_string("2017-02-02T14:25:43.511Z"))
        model.Event.write_event(session, event)

        event = events_pb2.PipelineFinished(
            uuid="test1",
            timestamp=timestamp_from_json_string("2017-02-02T14:26:43.511Z"),
            duration_ms=100,
            result=events_pb2.SUCCESS)
        model.Event.write_event(session, event)

        # Second pipeline
        event = events_pb2.PipelineStarted(
            uuid="test2",
            timestamp=timestamp_from_json_string("2017-02-02T14:25:43.511Z"))
        model.Event.write_event(session, event)

        event = events_pb2.PipelineFinished(
            uuid="test2",
            timestamp=timestamp_from_json_string("2017-02-02T14:26:43.511Z"),
            duration_ms=100,
            result=events_pb2.SUCCESS)
        model.Event.write_event(session, event)

        # Correct Stage
        event = events_pb2.PipelineStageStarted(
            uuid="stage1",
            pipeline_uuid='test1',
            timestamp=timestamp_from_json_string("2017-02-02T14:25:43.511Z"))
        model.Event.write_event(session, event)

        event = events_pb2.PipelineStageFinished(
            uuid="stage1",
            pipeline_uuid='test1',
            timestamp=timestamp_from_json_string("2017-02-02T14:26:43.511Z"),
            duration_ms=100,
            result=events_pb2.SUCCESS)
        model.Event.write_event(session, event)

        # Wrong Stage
        event = events_pb2.PipelineStageStarted(
            uuid="stage1",
            pipeline_uuid='test2',
            timestamp=timestamp_from_json_string("2017-02-02T14:25:43.511Z"))
        model.Event.write_event(session, event)

        event = events_pb2.PipelineStageFinished(
            uuid="stage1",
            pipeline_uuid='test2',
            timestamp=timestamp_from_json_string("2017-02-02T14:26:43.511Z"),
            duration_ms=1000,
            result=events_pb2.ABORTED)
        model.Event.write_event(session, event)

        projection_backend = get_projection_backend(backend_name, session)
        projector = Projector(projection_backend,
                              EventSQLAlchemyBackend(session))
        projector.process_events()

        pipeline_stage = projection_backend.get_one_by_filter(
            model.PipelineStage, {
                "pipeline_id": "test1",
                "external_id": "stage1"
            })
        assert pipeline_stage is not None
        assert pipeline_stage.external_id == 'stage1'
        assert pipeline_stage.pipeline_id == 'test1'
        assert pipeline_stage.currently_running == False
        assert pipeline_stage.result == "SUCCESS"
        assert pipeline_stage.started_running_at == dateutil.parser.parse(
            "2017-02-02T14:25:43.511Z")
        assert pipeline_stage.finished_running_at == dateutil.parser.parse(
            "2017-02-02T14:26:43.511Z")