def job_processer(num_jobs, objects, job_action, concur, kwargs=None,
opt=None):
"""Process all jobs in batches.
:param num_jobs:
:param objects:
:param job_action:
:param concur:
:param payload:
"""
count = 0
batch_size = basic.batcher(num_files=num_jobs)
for work in basic.batch_gen(data=objects,
batch_size=batch_size,
count=num_jobs):
count += 1
report.reporter(msg='Job Count %s' % count)
work_q = basic_queue(work)
with spinner(work_q=work_q):
worker_proc(job_action=job_action,
concurrency=concur,
queue=work_q,
opt=opt,
kwargs=kwargs)
basic.stupid_hack(wait=1)
work_q.close()
def job_processer(num_jobs, objects, job_action, concur, kwargs=None,
opt=None):
"""Process all jobs in batches.
:param num_jobs:
:param objects:
:param job_action:
:param concur:
:param payload:
"""
count = 0
batch_size = basic.batcher(num_files=num_jobs)
while objects:
count += 1
report.reporter(msg='Job Count %s' % count)
work = [
objects.pop(objects.index(obj)) for obj in objects[0:batch_size]
]
work_q = basic_queue(work)
with spinner(work_q=work_q):
worker_proc(
job_action=job_action,
concurrency=concur,
queue=work_q,
opt=opt,
kwargs=kwargs
)
basic.stupid_hack(wait=.2)
work_q.close()
def test_batcher_with_more_files(self, mock_reporter):
return_batch_size = basic_utils.batcher(2)
self.assertEqual(return_batch_size, 1)
self.assertTrue(mock_reporter.called)
def test_batcher_with_more_files(self, mock_reporter):
return_batch_size = basic_utils.batcher(2)
self.assertEqual(return_batch_size, 1)
self.assertTrue(mock_reporter.called)