def test_multipush(self):
'''Tests behavior when multiple servers are pushing jobs simultaneously.'''
total_completed = RawValue('i')
total_completed.value = 0
start_workers, kill_workers = testing_lib.construct_worker_pool(config.num_local_workers(), config.WORKER_ADDRESSES, send_jobs, (), on_recv_result, (total_completed,), num_pushers=config.NUM_PUSHERS)
start_workers()
completion = testing_lib.check_for_completion(total_completed, config.NUM_JOBS * config.NUM_PUSHERS, get_timeout(len(config.WORKER_ADDRESSES)))
kill_workers()
if not completion:
self.fail('Not all jobs received: %d / %d' % (total_completed.value, config.NUM_JOBS * config.NUM_PUSHERS))
def test_files(self):
'''Tests that workers can correctly store files in a central location.'''
total_completed = RawValue('i')
total_completed.value = 0
if not os.path.exists(os.path.join(os.path.dirname(__file__), 'testing_files/')):
os.mkdir('testing_files')
start_workers, kill_workers = testing_lib.construct_worker_pool(config.num_local_workers(), config.WORKER_ADDRESSES, send_jobs, (), on_recv_result, (total_completed,))
start_workers()
if not testing_lib.check_for_completion(total_completed, config.NUM_FILES, get_timeout(len(config.WORKER_ADDRESSES))):
self.fail('Not all jobs received: %d / %d' % (total_completed.value, config.NUM_FILES))
kill_workers()
os.rmdir('testing_files')
def test_mem(self):
'''Tests that memory required to transfer jobs does not exceed the given amount.'''
total_completed = RawValue('i')
print 'Constructing test string (this could take some time).'
test_str = generate_str(config.STR_LEN)
print 'Done. Starting test.'
total_completed.value = 0
start_workers, kill_workers = testing_lib.construct_worker_pool(config.num_local_workers(), config.WORKER_ADDRESSES, send_jobs, (test_str,), on_recv_result, (total_completed,))
start_workers()
completion = testing_lib.check_for_completion(total_completed, config.NUM_STRINGS, get_timeout())
kill_workers()
if not completion:
self.fail('Not all jobs received: %d / %d' % (total_completed.value, config.NUM_STRINGS))
def test_multiprocessing(self):
'''Tests that jobs will execute properly on multiple processes simultaneously.'''
total_completed = RawValue('i')
job_processors = MultiQueue()
total_completed.value = 0
start_workers, kill_workers = testing_lib.construct_worker_pool(config.num_local_workers(), config.WORKER_ADDRESSES, send_jobs, (), on_recv_result, (total_completed, job_processors))
start_workers()
completion = testing_lib.check_for_completion(total_completed, config.NUM_JOBS, get_timeout(len(config.WORKER_ADDRESSES)))
kill_workers()
if not completion:
self.fail('Not all jobs received: %d / %d' % (total_completed.value, config.NUM_JOBS))
if not check_load_balance(job_processors):
self.fail('Not all workers utilized.')
def test_threading(self):
'''Tests that jobs will execute properly on multiple threads simultaneously.'''
total_completed = RawValue('i')
job_processors = MultiQueue()
def push(vent_port, sink_port, worker_pool):
worker, close, run_job = parallel.construct_worker(worker_pool, {'vent_port': vent_port, 'sink_port': sink_port})
for i in range(config.NUM_JOBS):
run_job(wait_job, (config.WAIT_TIME))
worker(on_recv_result, (total_completed, job_processors))
total_completed.value = 0
port = 5000
thread.start_new_thread(push, (port, port + 1, config.WORKER_ADDRESSES))
for i in range(config.num_local_workers() - 1):
port += 2
thread.start_new_thread(testing_lib.work, (port, port + 1, config.WORKER_ADDRESSES))
if not testing_lib.check_for_completion(total_completed, config.NUM_JOBS, get_timeout(len(config.WORKER_ADDRESSES))):
self.fail('Not all jobs received: %d / %d' % (total_completed.value, len(config.WORKER_ADDRESSES)))
if not check_load_balance(job_processors):
self.fail('Not all workers utilized.')