def test_combine_retrigger(self):
params = {
"bucket":
"bucket",
"log":
"log",
"functions": {
"step": {
"file": "application"
}
},
"pipeline": [{
"name": "step",
}, {
"name": "step",
}, {
"name": "step",
}]
}
key = "0/123.400000-13/1-1/1-1-1-suffix.log"
mock = MockScheduler("fifo", params, 0, 3)
mock.add_job(mock.log_queue, key)
# Stage 1. 3 bins, each with 2 files
children = []
for i in range(1, 4):
for j in range(1, 3):
bucket_key = "1/123.400000-13/{0:d}-3/{1:d}-1-2-suffix.new".format(
i, j)
child_key = "2/123.400000-13/{0:d}-3/{1:d}-1-2-suffix.new".format(
i, j)
log_key = "1/123.400000-13/{0:d}-3/{1:d}-1-2-suffix.log".format(
i, j)
mock.add_to_queue(log_key)
children.append(scheduler.payload(params["bucket"],
bucket_key))
mock.add_children_payloads(
log_key, [scheduler.payload(params["bucket"], child_key)])
mock.add_children_payloads(key, children)
# Stage 2. 1 bin, 3 files
for i in range(1, 4):
bucket_key = "2/123.400000-13/1-1/{0:d}-1-3-suffix.new".format(i)
for j in range(1, 3):
log_key = "2/123.400000-13/{0:d}-3/{1:d}-1-2-suffix.log".format(
i, j)
child_key = "1/123.400000-13/{0:d}-3/{1:d}-1-2-suffix.new".format(
i, j)
if i != 1 or j != 1:
mock.add_to_queue(log_key)
mock.listen()
self.assertEqual(len(mock.tasks), 1)
self.assertEqual(len(mock.tasks[0].invokes), 1)
self.assertEqual(
mock.tasks[0].invokes[0]["Records"][0]["s3"]["object"]["key"],
"2/123.400000-13/1-3/1-1-2-suffix.new")
self.assertTrue(mock.tasks[0].running)
def test_steps(self):
params = {
"bucket":
"bucket",
"log":
"log",
"functions": {
"step": {
"file": "application"
}
},
"pipeline": [{
"name": "step",
}, {
"name": "step",
}, {
"name": "step",
}]
}
key = "0/123.400000-13/1-1/1-1-1-suffix.log"
mock = MockScheduler("fifo", params, 600, 3)
mock.add_job(mock.log_queue, key)
children = []
for i in range(1, 4):
bucket_key = "1/123.400000-13/1-1/{0:d}-1-3-suffix.new".format(i)
log_key = "1/123.400000-13/1-1/{0:d}-1-3-suffix.log".format(i)
mock.add_to_queue(log_key)
children.append(scheduler.payload(params["bucket"], bucket_key))
mock.add_children_payloads(key, children)
for i in range(1, 4):
parent_log_key = "1/123.400000-13/1-1/{0:d}-1-3-suffix.log".format(
i)
children = []
for j in range(1, 3):
log_key = "2/123.400000-13/{1:d}-2/{0:d}-1-3-suffix.log".format(
i, j)
bucket_key = "2/123.400000-13/{1:d}-2/{0:d}-1-3-suffix.new".format(
i, j)
mock.add_to_queue(log_key)
mock.add_children_payloads(log_key, [])
children.append(scheduler.payload(params["bucket"],
bucket_key))
mock.add_children_payloads(parent_log_key, children)
mock.listen()
self.assertEqual(len(mock.tasks), 1)
self.assertEqual(len(mock.tasks[0].invokes), 0)
self.assertFalse(mock.tasks[0].running)
def test_pause(self):
params = {
"bucket": "bucket",
"log": "log",
"functions": {
"step": {
"file": "application"
}
},
"pipeline": [{
"name": "step",
}, {
"name": "step",
}]
}
mock = MockScheduler("fifo", params, 0, 2)
key = "0/123.400000-13/1-1/1-1-1-suffix.new"
log = "0/123.400000-13/1-1/1-1-1-suffix.log"
ctime = time.time()
job = scheduler.Job("source_bucket", "destination_bucket", key,
ctime - 2)
job.pause = [ctime, job.start_time + 10]
mock.add_jobs([job])
time.sleep(2)
children = []
for i in range(1, 4):
bucket_key = "1/123.400000-13/1-1/{0:d}-1-3-suffix.new".format(i)
children.append(scheduler.payload(params["bucket"], bucket_key))
mock.add_children_payloads(log, children)
mock.listen()
self.assertEqual(len(mock.tasks[0].invokes), 0)
def test_retrigger(self):
params = {
"bucket": "bucket",
"log": "log",
"functions": {
"step": {
"file": "application"
}
},
"pipeline": [{
"name": "step",
}, {
"name": "step",
}]
}
key = "0/123.400000-13/1-1/1-1-1-suffix.log"
mock = MockScheduler("fifo", params, 0, 2)
mock.add_job(mock.log_queue, key)
children = []
for i in range(1, 4):
bucket_key = "1/123.400000-13/1-1/{0:d}-1-3-suffix.new".format(i)
children.append(scheduler.payload(params["bucket"], bucket_key))
mock.add_children_payloads(key, children)
mock.listen()
self.assertEqual(len(mock.tasks), 1)
self.assertEqual(len(mock.tasks[0].invokes), 3)
for i in range(3):
self.assertEqual(
mock.tasks[0].invokes[i]["Records"][0]["s3"]["object"]["key"],
"1/123.400000-13/1-1/{0:d}-1-3-suffix.new".format(i + 1))
self.assertTrue(mock.tasks[0].running)
def add_job(self, queue, key):
queue["messages"].append({
"Body":
json.dumps(scheduler.payload(self.params["bucket"], key))
})
self.queue[0] = [MockObject(key)]