def test_fifo(self):
job1 = scheduler.Job("source_bucket", "destination_bucket", "key1", 10)
job2 = scheduler.Job("source_bucket", "destination_bucket", "key2", 0)
job3 = scheduler.Job("source_bucket", "destination_bucket", "key3", 20)
orders = scheduler.simulation_order([job1, job2, job3], "fifo", 0.0, 3)
expected = [job2, job1, job3]
for i in range(len(expected)):
self.assertEqual(orders[i][1], expected[i].start_time)
self.assertEqual(orders[i][0], expected[i])
def test_deadline(self):
job1 = scheduler.Job("source_bucket", "destination_bucket", "key1", 0,
None)
job2 = scheduler.Job("source_bucket", "destination_bucket", "key2", 10,
40)
job3 = scheduler.Job("source_bucket", "destination_bucket", "key3", 20,
60)
orders = scheduler.simulation_order([job1, job2, job3], "deadline",
30.0, 2)
expected = [job1, job2, job3]
for i in range(len(expected)):
self.assertEqual(orders[i][1], expected[i].start_time)
self.assertEqual(orders[i][0], expected[i])
self.assertEqual(orders[0][0].pause, [20.0, 40.0])
def test_pause(self):
params = {
"bucket": "bucket",
"log": "log",
"functions": {
"step": {
"file": "application"
}
},
"pipeline": [{
"name": "step",
}, {
"name": "step",
}]
}
mock = MockScheduler("fifo", params, 0, 2)
key = "0/123.400000-13/1-1/1-1-1-suffix.new"
log = "0/123.400000-13/1-1/1-1-1-suffix.log"
ctime = time.time()
job = scheduler.Job("source_bucket", "destination_bucket", key,
ctime - 2)
job.pause = [ctime, job.start_time + 10]
mock.add_jobs([job])
time.sleep(2)
children = []
for i in range(1, 4):
bucket_key = "1/123.400000-13/1-1/{0:d}-1-3-suffix.new".format(i)
children.append(scheduler.payload(params["bucket"], bucket_key))
mock.add_children_payloads(log, children)
mock.listen()
self.assertEqual(len(mock.tasks[0].invokes), 0)
def create_jobs(source_bucket, destination_bucket, policy, prefix, num_jobs,
job_duration, offset, relative):
s3 = boto3.resource("s3")
bucket = s3.Bucket(source_bucket)
objs = list(bucket.objects.filter(Prefix=prefix))
objs = list(filter(lambda obj: not obj.key.endswith("/"), objs))
now = 0 if relative else time.time() + 10
if policy == "deadline":
jobs = []
for i in range(num_jobs):
obj = objs[i]
start_time = now + i * offset
deadline = now + job_duration + (2 * (num_jobs - 1) - i) * offset
jobs.append(
scheduler.Job(source_bucket,
destination_bucket,
obj.key,
start_time=start_time,
deadline=deadline))
elif policy == "priority":
jobs = []
for i in range(num_jobs):
obj = objs[i]
start_time = now + i * offset
if i % 3 == 1:
priority = 10
else:
priority = 1
jobs.append(
scheduler.Job(source_bucket,
destination_bucket,
obj.key,
start_time=start_time,
priority=priority))
jobs = scheduler.simulation_order(jobs, policy, job_duration, 1)
jobs = list(map(lambda j: j[0], jobs))
for job in jobs:
print(job)
return jobs