def test_multipush(self): '''Tests behavior when multiple servers are pushing jobs simultaneously.''' total_completed = RawValue('i') total_completed.value = 0 start_workers, kill_workers = testing_lib.construct_worker_pool(config.num_local_workers(), config.WORKER_ADDRESSES, send_jobs, (), on_recv_result, (total_completed,), num_pushers=config.NUM_PUSHERS) start_workers() completion = testing_lib.check_for_completion(total_completed, config.NUM_JOBS * config.NUM_PUSHERS, get_timeout(len(config.WORKER_ADDRESSES))) kill_workers() if not completion: self.fail('Not all jobs received: %d / %d' % (total_completed.value, config.NUM_JOBS * config.NUM_PUSHERS))
def test_volume(self): '''Tests the ability to handle various traffic patterns (trickle, normal, and spike by default).''' total_completed = RawValue('i') for pattern in config.USAGE_PATTERNS: total_completed.value = 0 total_jobs = pattern.sets * pattern.set_reps start_workers, kill_workers = testing_lib.construct_worker_pool(len(config.WORKER_ADDRESSES), config.WORKER_ADDRESSES, send_jobs, (pattern,), on_recv_result, (total_completed,)) start_workers() if not testing_lib.check_for_completion(total_completed, total_jobs, get_timeout(pattern, len(config.WORKER_ADDRESSES))): self.fail('Failed on usage pattern: %s' % str(pattern)) kill_workers()
def test_files(self): '''Tests that workers can correctly store files in a central location.''' total_completed = RawValue('i') total_completed.value = 0 if not os.path.exists(os.path.join(os.path.dirname(__file__), 'testing_files/')): os.mkdir('testing_files') start_workers, kill_workers = testing_lib.construct_worker_pool(config.num_local_workers(), config.WORKER_ADDRESSES, send_jobs, (), on_recv_result, (total_completed,)) start_workers() if not testing_lib.check_for_completion(total_completed, config.NUM_FILES, get_timeout(len(config.WORKER_ADDRESSES))): self.fail('Not all jobs received: %d / %d' % (total_completed.value, config.NUM_FILES)) kill_workers() os.rmdir('testing_files')
def test_mem(self): '''Tests that memory required to transfer jobs does not exceed the given amount.''' total_completed = RawValue('i') print 'Constructing test string (this could take some time).' test_str = generate_str(config.STR_LEN) print 'Done. Starting test.' total_completed.value = 0 start_workers, kill_workers = testing_lib.construct_worker_pool(config.num_local_workers(), config.WORKER_ADDRESSES, send_jobs, (test_str,), on_recv_result, (total_completed,)) start_workers() completion = testing_lib.check_for_completion(total_completed, config.NUM_STRINGS, get_timeout()) kill_workers() if not completion: self.fail('Not all jobs received: %d / %d' % (total_completed.value, config.NUM_STRINGS))
def test_multiprocessing(self): '''Tests that jobs will execute properly on multiple processes simultaneously.''' total_completed = RawValue('i') job_processors = MultiQueue() total_completed.value = 0 start_workers, kill_workers = testing_lib.construct_worker_pool(config.num_local_workers(), config.WORKER_ADDRESSES, send_jobs, (), on_recv_result, (total_completed, job_processors)) start_workers() completion = testing_lib.check_for_completion(total_completed, config.NUM_JOBS, get_timeout(len(config.WORKER_ADDRESSES))) kill_workers() if not completion: self.fail('Not all jobs received: %d / %d' % (total_completed.value, config.NUM_JOBS)) if not check_load_balance(job_processors): self.fail('Not all workers utilized.')
def test_threading(self): '''Tests that jobs will execute properly on multiple threads simultaneously.''' total_completed = RawValue('i') job_processors = MultiQueue() def push(vent_port, sink_port, worker_pool): worker, close, run_job = parallel.construct_worker(worker_pool, {'vent_port': vent_port, 'sink_port': sink_port}) for i in range(config.NUM_JOBS): run_job(wait_job, (config.WAIT_TIME)) worker(on_recv_result, (total_completed, job_processors)) total_completed.value = 0 port = 5000 thread.start_new_thread(push, (port, port + 1, config.WORKER_ADDRESSES)) for i in range(config.num_local_workers() - 1): port += 2 thread.start_new_thread(testing_lib.work, (port, port + 1, config.WORKER_ADDRESSES)) if not testing_lib.check_for_completion(total_completed, config.NUM_JOBS, get_timeout(len(config.WORKER_ADDRESSES))): self.fail('Not all jobs received: %d / %d' % (total_completed.value, len(config.WORKER_ADDRESSES))) if not check_load_balance(job_processors): self.fail('Not all workers utilized.')
}, { "id": "adnpptot", "type": "", "description": "daily average NPP", "scale": 1000.0, "offset": 0.0, "unit": "kgC m-2 d-1", "metricName": "daily-average-NPP" }] folder = DATA_HOME + '/IBIS_Data/5b9012e4c29ca433443dcfab/outputs' srcSuffix = '.daily.txt' distSuffix = '.month.txt' counter = RawValue('i') counter.value = 0 def convert2Month(i): # try: counter.value += 1 print('counter: %s %5.2f%% site index: %s' % (str(counter.value), counter.value * 100 / 40595, str(i))) srcPath = '%s/%s%s' % (folder, str(i), srcSuffix) distPath = '%s/%s%s' % (folder, str(i), distSuffix) if path.exists(distPath): fsize = path.getsize(distPath) if fsize > 100000: # 100k rerun = False else: rerun = True
i = INT(0); v1=RawValue('i', 0) v2=RawValue('i', 0) p2 = INTProcess(i, v1, v2); p2.start() p2.join() print i.getI() print i.getV() print v1.value print v2.value v = RawValue('i', 0) l = Lock() nbrProcs = 10 procs = [LockProcess(v, l, False) for j in range(nbrProcs)] for j in range(nbrProcs): procs[j].start() for j in range(nbrProcs): procs[j].join() print v.value v.value = 0 procs2 = [LockProcess(v, l, True) for j in range(nbrProcs)] for j in range(nbrProcs): procs2[j].start() for j in range(nbrProcs): procs2[j].join() print v.value
import csv import pandas as pd import re import linecache import time import matplotlib.pyplot as plt import matplotlib from multiprocessing import Pool, sharedctypes, RawArray, RawValue from MS.IBIS import * from MS.LPJ import * from MS.Biome_BGC import * from CMIP import * plt.switch_backend('tkagg') counter = RawValue('i') counter.value = 0 failed_counter = RawValue('i') failed_counter.value = 0 test = False options, args = getopt.getopt(sys.argv[1:], '', ['model=', 'test=']) for opt in options: if (opt[0] == '--model'): if opt[1] == 'IBIS': ms_daily = IBIS('daily') ms_month = IBIS('month') elif opt[1] == 'Biome-BGC': ms_daily = Biome_BGC('daily') ms_month = Biome_BGC('month') elif opt[1] == 'LPJ': ms_daily = LPJ('daily') ms_month = LPJ('month')