def test_multipush(self):
'''Tests behavior when multiple servers are pushing jobs simultaneously.'''
total_completed = RawValue('i')
total_completed.value = 0
start_workers, kill_workers = testing_lib.construct_worker_pool(config.num_local_workers(), config.WORKER_ADDRESSES, send_jobs, (), on_recv_result, (total_completed,), num_pushers=config.NUM_PUSHERS)
start_workers()
completion = testing_lib.check_for_completion(total_completed, config.NUM_JOBS * config.NUM_PUSHERS, get_timeout(len(config.WORKER_ADDRESSES)))
kill_workers()
if not completion:
self.fail('Not all jobs received: %d / %d' % (total_completed.value, config.NUM_JOBS * config.NUM_PUSHERS))
def test_volume(self):
'''Tests the ability to handle various traffic patterns (trickle, normal, and spike by default).'''
total_completed = RawValue('i')
for pattern in config.USAGE_PATTERNS:
total_completed.value = 0
total_jobs = pattern.sets * pattern.set_reps
start_workers, kill_workers = testing_lib.construct_worker_pool(len(config.WORKER_ADDRESSES), config.WORKER_ADDRESSES, send_jobs, (pattern,), on_recv_result, (total_completed,))
start_workers()
if not testing_lib.check_for_completion(total_completed, total_jobs, get_timeout(pattern, len(config.WORKER_ADDRESSES))):
self.fail('Failed on usage pattern: %s' % str(pattern))
kill_workers()
def test_files(self):
'''Tests that workers can correctly store files in a central location.'''
total_completed = RawValue('i')
total_completed.value = 0
if not os.path.exists(os.path.join(os.path.dirname(__file__), 'testing_files/')):
os.mkdir('testing_files')
start_workers, kill_workers = testing_lib.construct_worker_pool(config.num_local_workers(), config.WORKER_ADDRESSES, send_jobs, (), on_recv_result, (total_completed,))
start_workers()
if not testing_lib.check_for_completion(total_completed, config.NUM_FILES, get_timeout(len(config.WORKER_ADDRESSES))):
self.fail('Not all jobs received: %d / %d' % (total_completed.value, config.NUM_FILES))
kill_workers()
os.rmdir('testing_files')
def test_mem(self):
'''Tests that memory required to transfer jobs does not exceed the given amount.'''
total_completed = RawValue('i')
print 'Constructing test string (this could take some time).'
test_str = generate_str(config.STR_LEN)
print 'Done. Starting test.'
total_completed.value = 0
start_workers, kill_workers = testing_lib.construct_worker_pool(config.num_local_workers(), config.WORKER_ADDRESSES, send_jobs, (test_str,), on_recv_result, (total_completed,))
start_workers()
completion = testing_lib.check_for_completion(total_completed, config.NUM_STRINGS, get_timeout())
kill_workers()
if not completion:
self.fail('Not all jobs received: %d / %d' % (total_completed.value, config.NUM_STRINGS))
def test_multiprocessing(self):
'''Tests that jobs will execute properly on multiple processes simultaneously.'''
total_completed = RawValue('i')
job_processors = MultiQueue()
total_completed.value = 0
start_workers, kill_workers = testing_lib.construct_worker_pool(config.num_local_workers(), config.WORKER_ADDRESSES, send_jobs, (), on_recv_result, (total_completed, job_processors))
start_workers()
completion = testing_lib.check_for_completion(total_completed, config.NUM_JOBS, get_timeout(len(config.WORKER_ADDRESSES)))
kill_workers()
if not completion:
self.fail('Not all jobs received: %d / %d' % (total_completed.value, config.NUM_JOBS))
if not check_load_balance(job_processors):
self.fail('Not all workers utilized.')
def test_threading(self):
'''Tests that jobs will execute properly on multiple threads simultaneously.'''
total_completed = RawValue('i')
job_processors = MultiQueue()
def push(vent_port, sink_port, worker_pool):
worker, close, run_job = parallel.construct_worker(worker_pool, {'vent_port': vent_port, 'sink_port': sink_port})
for i in range(config.NUM_JOBS):
run_job(wait_job, (config.WAIT_TIME))
worker(on_recv_result, (total_completed, job_processors))
total_completed.value = 0
port = 5000
thread.start_new_thread(push, (port, port + 1, config.WORKER_ADDRESSES))
for i in range(config.num_local_workers() - 1):
port += 2
thread.start_new_thread(testing_lib.work, (port, port + 1, config.WORKER_ADDRESSES))
if not testing_lib.check_for_completion(total_completed, config.NUM_JOBS, get_timeout(len(config.WORKER_ADDRESSES))):
self.fail('Not all jobs received: %d / %d' % (total_completed.value, len(config.WORKER_ADDRESSES)))
if not check_load_balance(job_processors):
self.fail('Not all workers utilized.')
}, {
"id": "adnpptot",
"type": "",
"description": "daily average NPP",
"scale": 1000.0,
"offset": 0.0,
"unit": "kgC m-2 d-1",
"metricName": "daily-average-NPP"
}]
folder = DATA_HOME + '/IBIS_Data/5b9012e4c29ca433443dcfab/outputs'
srcSuffix = '.daily.txt'
distSuffix = '.month.txt'
counter = RawValue('i')
counter.value = 0
def convert2Month(i):
# try:
counter.value += 1
print('counter: %s %5.2f%% site index: %s' %
(str(counter.value), counter.value * 100 / 40595, str(i)))
srcPath = '%s/%s%s' % (folder, str(i), srcSuffix)
distPath = '%s/%s%s' % (folder, str(i), distSuffix)
if path.exists(distPath):
fsize = path.getsize(distPath)
if fsize > 100000: # 100k
rerun = False
else:
rerun = True
i = INT(0);
v1=RawValue('i', 0)
v2=RawValue('i', 0)
p2 = INTProcess(i, v1, v2);
p2.start()
p2.join()
print i.getI()
print i.getV()
print v1.value
print v2.value
v = RawValue('i', 0)
l = Lock()
nbrProcs = 10
procs = [LockProcess(v, l, False) for j in range(nbrProcs)]
for j in range(nbrProcs): procs[j].start()
for j in range(nbrProcs): procs[j].join()
print v.value
v.value = 0
procs2 = [LockProcess(v, l, True) for j in range(nbrProcs)]
for j in range(nbrProcs): procs2[j].start()
for j in range(nbrProcs): procs2[j].join()
print v.value
import csv
import pandas as pd
import re
import linecache
import time
import matplotlib.pyplot as plt
import matplotlib
from multiprocessing import Pool, sharedctypes, RawArray, RawValue
from MS.IBIS import *
from MS.LPJ import *
from MS.Biome_BGC import *
from CMIP import *
plt.switch_backend('tkagg')
counter = RawValue('i')
counter.value = 0
failed_counter = RawValue('i')
failed_counter.value = 0
test = False
options, args = getopt.getopt(sys.argv[1:], '', ['model=', 'test='])
for opt in options:
if (opt[0] == '--model'):
if opt[1] == 'IBIS':
ms_daily = IBIS('daily')
ms_month = IBIS('month')
elif opt[1] == 'Biome-BGC':
ms_daily = Biome_BGC('daily')
ms_month = Biome_BGC('month')
elif opt[1] == 'LPJ':
ms_daily = LPJ('daily')
ms_month = LPJ('month')
