def test_status(reader, writer, ledger): job = Job('somejob', reader, writer, ledger) assert job.status == IN_PROGRESS items = job.serve(5) assert job.status == IN_PROGRESS items.extend(job.serve(100)) job.receive(items, True) assert job.status == COMPLETE
def test_serve_continuation(reader, writer, ledger, pre_served_size): job = Job('somejob', reader, writer, ledger) for i in range(pre_served_size): ledger.set(job.ledger_id(i), 'fake value') items = job.serve(50) n_items = CSV_SIZE - pre_served_size assert len(items) == n_items
def test_receive_cont(job): n_served = 10 n_received = 5 job_name = job.name ledger = job.ledger items = job.serve(n_served) job.receive(items[:n_received], False) reader = CsvReader({'input_file_path': job.reader.file_path}) writer = CsvWriter({'output_file_path': job.writer.file_path}) del job # we processed half the served items and we are making a new job cont_job = Job(job_name, reader, writer, ledger, cont=True) # requesting the second half of the items print('serving') items = cont_job.serve(n_served - n_received) cont_job.receive(items, False) # counting in the ledger received = [ x for x in ledger.scan_iter(f'{job_name}*') if ledger.get(x).decode('utf8') == RECEIVED ] served = [ x for x in ledger.scan_iter(f'{job_name}*') if ledger.get(x).decode('utf8') == SERVED ] assert len(received) == n_served assert len(cont_job.received) == n_served assert len(served) == 0
def test_serve_batch(reader, writer, ledger, batch_size): job = Job('somejob', reader, writer, ledger) items = job.serve(batch_size) n_items = batch_size if batch_size < 30 else 30 assert len(items) == n_items