def test_status(reader, writer, ledger):
job = Job('somejob', reader, writer, ledger)
assert job.status == IN_PROGRESS
items = job.serve(5)
assert job.status == IN_PROGRESS
items.extend(job.serve(100))
job.receive(items, True)
assert job.status == COMPLETE
def test_serve_continuation(reader, writer, ledger, pre_served_size):
job = Job('somejob', reader, writer, ledger)
for i in range(pre_served_size):
ledger.set(job.ledger_id(i), 'fake value')
items = job.serve(50)
n_items = CSV_SIZE - pre_served_size
assert len(items) == n_items
def test_receive_cont(job):
n_served = 10
n_received = 5
job_name = job.name
ledger = job.ledger
items = job.serve(n_served)
job.receive(items[:n_received], False)
reader = CsvReader({'input_file_path': job.reader.file_path})
writer = CsvWriter({'output_file_path': job.writer.file_path})
del job
# we processed half the served items and we are making a new job
cont_job = Job(job_name, reader, writer, ledger, cont=True)
# requesting the second half of the items
print('serving')
items = cont_job.serve(n_served - n_received)
cont_job.receive(items, False)
# counting in the ledger
received = [
x for x in ledger.scan_iter(f'{job_name}*')
if ledger.get(x).decode('utf8') == RECEIVED
]
served = [
x for x in ledger.scan_iter(f'{job_name}*')
if ledger.get(x).decode('utf8') == SERVED
]
assert len(received) == n_served
assert len(cont_job.received) == n_served
assert len(served) == 0
def test_serve_batch(reader, writer, ledger, batch_size):
job = Job('somejob', reader, writer, ledger)
items = job.serve(batch_size)
n_items = batch_size if batch_size < 30 else 30
assert len(items) == n_items