def test_multipush(self):
        '''Tests behavior when multiple servers are pushing jobs simultaneously.'''
        total_completed = RawValue('i')
        total_completed.value = 0

        start_workers, kill_workers = testing_lib.construct_worker_pool(config.num_local_workers(), config.WORKER_ADDRESSES, send_jobs, (), on_recv_result, (total_completed,), num_pushers=config.NUM_PUSHERS)
        start_workers()
        completion = testing_lib.check_for_completion(total_completed, config.NUM_JOBS * config.NUM_PUSHERS, get_timeout(len(config.WORKER_ADDRESSES)))
        kill_workers()
        if not completion:
            self.fail('Not all jobs received: %d / %d' % (total_completed.value, config.NUM_JOBS * config.NUM_PUSHERS))
Example #2
0
    def test_files(self):
        '''Tests that workers can correctly store files in a central location.'''
        total_completed = RawValue('i')
        total_completed.value = 0
        if not os.path.exists(os.path.join(os.path.dirname(__file__), 'testing_files/')):
            os.mkdir('testing_files')

        start_workers, kill_workers = testing_lib.construct_worker_pool(config.num_local_workers(), config.WORKER_ADDRESSES, send_jobs, (), on_recv_result, (total_completed,))
        start_workers()
        if not testing_lib.check_for_completion(total_completed, config.NUM_FILES, get_timeout(len(config.WORKER_ADDRESSES))):
            self.fail('Not all jobs received: %d / %d' % (total_completed.value, config.NUM_FILES))
        kill_workers()
        os.rmdir('testing_files')
Example #3
0
    def test_mem(self):
        '''Tests that memory required to transfer jobs does not exceed the given amount.'''
        total_completed = RawValue('i')
        print 'Constructing test string (this could take some time).'
        test_str = generate_str(config.STR_LEN)
        print 'Done. Starting test.'
        total_completed.value = 0

        start_workers, kill_workers = testing_lib.construct_worker_pool(config.num_local_workers(), config.WORKER_ADDRESSES, send_jobs, (test_str,), on_recv_result, (total_completed,))
        start_workers()
        completion = testing_lib.check_for_completion(total_completed, config.NUM_STRINGS, get_timeout())
        kill_workers()
        if not completion:
            self.fail('Not all jobs received: %d / %d' % (total_completed.value, config.NUM_STRINGS))
    def test_multiprocessing(self):
        '''Tests that jobs will execute properly on multiple processes simultaneously.'''
        total_completed = RawValue('i')
        job_processors = MultiQueue()
        total_completed.value = 0

        start_workers, kill_workers = testing_lib.construct_worker_pool(config.num_local_workers(), config.WORKER_ADDRESSES, send_jobs, (), on_recv_result, (total_completed, job_processors))
        start_workers()
        completion = testing_lib.check_for_completion(total_completed, config.NUM_JOBS, get_timeout(len(config.WORKER_ADDRESSES)))
        kill_workers()
        if not completion:
            self.fail('Not all jobs received: %d / %d' % (total_completed.value, config.NUM_JOBS))
        if not check_load_balance(job_processors):
            self.fail('Not all workers utilized.')
    def test_threading(self):
        '''Tests that jobs will execute properly on multiple threads simultaneously.'''
        total_completed = RawValue('i')
        job_processors = MultiQueue()
        def push(vent_port, sink_port, worker_pool):
            worker, close, run_job = parallel.construct_worker(worker_pool, {'vent_port': vent_port, 'sink_port': sink_port})
            for i in range(config.NUM_JOBS):
                run_job(wait_job, (config.WAIT_TIME))
            worker(on_recv_result, (total_completed, job_processors))

        total_completed.value = 0
        port = 5000
        thread.start_new_thread(push, (port, port + 1, config.WORKER_ADDRESSES))
        for i in range(config.num_local_workers() - 1):
            port += 2
            thread.start_new_thread(testing_lib.work, (port, port + 1, config.WORKER_ADDRESSES))
        if not testing_lib.check_for_completion(total_completed, config.NUM_JOBS, get_timeout(len(config.WORKER_ADDRESSES))):
            self.fail('Not all jobs received: %d / %d' % (total_completed.value, len(config.WORKER_ADDRESSES)))
        if not check_load_balance(job_processors):
            self.fail('Not all workers utilized.')