def test_pipeline_finished_success(self, backend_name, session):
event = events_pb2.PipelineStarted(
uuid="test1",
timestamp=timestamp_from_json_string("2017-02-02T14:25:43.511Z"))
model.Event.write_event(session, event)
event = events_pb2.PipelineFinished(
uuid="test1",
timestamp=timestamp_from_json_string("2017-02-02T14:26:43.511Z"),
duration_ms=100,
result=events_pb2.SUCCESS)
model.Event.write_event(session, event)
projection_backend = get_projection_backend(backend_name, session)
projector = Projector(projection_backend,
EventSQLAlchemyBackend(session))
projector.process_events()
pipeline = projection_backend.get_one_by_external_id(
"test1", model.Pipeline)
assert pipeline is not None
assert pipeline.external_id == 'test1'
assert pipeline.currently_running == False
assert pipeline.result == "SUCCESS"
assert pipeline.started_running_at == dateutil.parser.parse(
"2017-02-02T14:25:43.511Z")
assert pipeline.finished_running_at == dateutil.parser.parse(
"2017-02-02T14:26:43.511Z")
def test_forwards_pipeline_started(self, amqp_connection):
ts = timestamp_from_json_string("2012-04-23T18:25:43.511Z")
event = events_pb2.PipelineStarted(uuid="abcdef", timestamp=ts)
headers = {"proto-message-type": 'PipelineStarted'}
received_msgs = publish_msg_and_get_from_other_queue(
amqp_connection, event,
queue_configs.EXTERNAL_QUEUES['pipeline.started'],
queue_configs.INTERNAL_QUEUES['pipeline.started'], headers)
assert len(received_msgs) == 1
msg = received_msgs[0]
assert msg.content_type == 'application/vnd.google.protobuf'
assert msg.headers == {"proto-message-type": 'PipelineStarted'}
assert msg.delivery_info['routing_key'] == 'pipeline.started'
data = events_pb2.PipelineStarted()
data.ParseFromString(msg.body)
assert data.uuid == 'abcdef'
def pipeline_started(uuid=None):
""" Example payload:
{
"timestamp": "2012-04-23T18:25:43.511Z"
}
"""
payload = request.get_json(force=True)
rabbitmq = get_rabbitmq()
ts = timestamp_from_json_string(payload['timestamp'])
event = events_pb2.PipelineStarted(uuid=uuid, timestamp=ts)
queue = queue_configs.EXTERNAL_QUEUES['pipeline.started']
publish_to_queue(rabbitmq, event, queue)
return " [x] Sent PipelineStarted"
def test_pipeline_currently_running(self, backend_name, session):
ts = timestamp_from_json_string("2017-02-02T14:25:43.511Z")
event = events_pb2.PipelineStarted(uuid="test1", timestamp=ts)
model.Event.write_event(session, event)
projection_backend = get_projection_backend(backend_name, session)
projector = Projector(projection_backend,
EventSQLAlchemyBackend(session))
projector.process_events()
print(projection_backend.get_all(model.Pipeline))
pipeline = projection_backend.get_one_by_external_id(
"test1", model.Pipeline)
assert pipeline is not None
assert pipeline.external_id == 'test1'
assert pipeline.currently_running == True
assert pipeline.result == None
def _generate_pipeline_events(self):
from ngcd_common import events_pb2
from google.protobuf.timestamp_pb2 import Timestamp
from google.protobuf.json_format import MessageToDict
import random
from datetime import timedelta
import pytz
pipeline_id = 0
times = self.fake.time_series(start_date="-5d",
end_date="now",
precision=timedelta(minutes=1),
distrib=None,
tzinfo=pytz.utc)
for i in range(0, 10):
time = next(times)[0].isoformat()
if i % 2 == 0:
# Generate started event
ts = Timestamp()
ts.FromJsonString(time)
event_pb = events_pb2.PipelineStarted(uuid=str(pipeline_id),
timestamp=ts)
event_model = Event(id=self._next_global_id(),
type='PipelineStarted',
body=MessageToDict(
event_pb,
including_default_value_fields=True),
event_origin_time=time)
yield event_model
yield from self._generate_pipeline_stages(times, pipeline_id)
else:
# Generate finished event
ts = Timestamp()
ts.FromJsonString(time)
event_pb = events_pb2.PipelineFinished(
uuid=str(pipeline_id),
timestamp=ts,
result=random.choice([
events_pb2.SUCCESS, events_pb2.FAILURE,
events_pb2.ABORTED
]),
duration_ms=random.randint(100, 10000))
event_model = Event(id=self._next_global_id(),
type='PipelineFinished',
body=MessageToDict(
event_pb,
including_default_value_fields=True),
event_origin_time=time)
pipeline_id = pipeline_id + 1
yield event_model
# Create a started one, to be able to check in-progress
time = next(times)[0].isoformat()
ts = Timestamp()
ts.FromJsonString(time)
event_pb = events_pb2.PipelineStarted(uuid=str(pipeline_id),
timestamp=ts)
event_model = Event(id=self._next_global_id(),
type='PipelineStarted',
body=MessageToDict(
event_pb, including_default_value_fields=True),
event_origin_time=time)
yield event_model
yield from self._generate_pipeline_stages(times, pipeline_id)
def test_stage_only_applies_to_same_pipeline(self, backend_name, session):
# First pipeline
event = events_pb2.PipelineStarted(
uuid="test1",
timestamp=timestamp_from_json_string("2017-02-02T14:25:43.511Z"))
model.Event.write_event(session, event)
event = events_pb2.PipelineFinished(
uuid="test1",
timestamp=timestamp_from_json_string("2017-02-02T14:26:43.511Z"),
duration_ms=100,
result=events_pb2.SUCCESS)
model.Event.write_event(session, event)
# Second pipeline
event = events_pb2.PipelineStarted(
uuid="test2",
timestamp=timestamp_from_json_string("2017-02-02T14:25:43.511Z"))
model.Event.write_event(session, event)
event = events_pb2.PipelineFinished(
uuid="test2",
timestamp=timestamp_from_json_string("2017-02-02T14:26:43.511Z"),
duration_ms=100,
result=events_pb2.SUCCESS)
model.Event.write_event(session, event)
# Correct Stage
event = events_pb2.PipelineStageStarted(
uuid="stage1",
pipeline_uuid='test1',
timestamp=timestamp_from_json_string("2017-02-02T14:25:43.511Z"))
model.Event.write_event(session, event)
event = events_pb2.PipelineStageFinished(
uuid="stage1",
pipeline_uuid='test1',
timestamp=timestamp_from_json_string("2017-02-02T14:26:43.511Z"),
duration_ms=100,
result=events_pb2.SUCCESS)
model.Event.write_event(session, event)
# Wrong Stage
event = events_pb2.PipelineStageStarted(
uuid="stage1",
pipeline_uuid='test2',
timestamp=timestamp_from_json_string("2017-02-02T14:25:43.511Z"))
model.Event.write_event(session, event)
event = events_pb2.PipelineStageFinished(
uuid="stage1",
pipeline_uuid='test2',
timestamp=timestamp_from_json_string("2017-02-02T14:26:43.511Z"),
duration_ms=1000,
result=events_pb2.ABORTED)
model.Event.write_event(session, event)
projection_backend = get_projection_backend(backend_name, session)
projector = Projector(projection_backend,
EventSQLAlchemyBackend(session))
projector.process_events()
pipeline_stage = projection_backend.get_one_by_filter(
model.PipelineStage, {
"pipeline_id": "test1",
"external_id": "stage1"
})
assert pipeline_stage is not None
assert pipeline_stage.external_id == 'stage1'
assert pipeline_stage.pipeline_id == 'test1'
assert pipeline_stage.currently_running == False
assert pipeline_stage.result == "SUCCESS"
assert pipeline_stage.started_running_at == dateutil.parser.parse(
"2017-02-02T14:25:43.511Z")
assert pipeline_stage.finished_running_at == dateutil.parser.parse(
"2017-02-02T14:26:43.511Z")