def __init__(self, pname, target, input_queue_args, result_queue_args):
self.t = SimpleTimer(name='manager_' + pname)
input_queues = {}
nowait_queues = {}
result_queues = {}
for name, size in input_queue_args.iteritems():
if size is None:
input_queues[name] = mp.Queue()
elif size is 0:
nowait_queues[name] = mp.Queue()
else:
input_queues[name] = mp.Queue(maxsize=size)
for name, size in result_queue_args.iteritems():
if size is None:
result_queues[name] = mp.Queue()
else:
result_queues[name] = mp.Queue(maxsize=size)
is_running = mp.Value('b', True)
args = (target, input_queues, nowait_queues, result_queues, is_running)
print args
self.process = mp.Process(name=pname, target=do_work, args=args)
self.process.start()
self.client = WorkerClient(*args)
def __init__(self, target, input_queues, nowait_queues, result_queues,
is_running, *queues):
self.target = target
self.input_queues = input_queues
self.nowait_queues = nowait_queues
self.result_queues = result_queues
self.is_running = is_running
self.t = SimpleTimer(use_process_name=True)
def init(self):
self.t = SimpleTimer(use_process_name=True)
# Number of iterations after which to reset target for boids to move at.
# Needs to be run more often in 2D than in 3D.
self.new_target_iter = 45
self.i = 0
self.init_boids()
self.worker.add_result('boids', self.boids.copy())
self.worker.add_result('big_boids', self.big_boids.copy())
def __init__(self,
on_un_reply_msg_retry_func=None,
max_count=3,
key_expires=60):
self.timer = SimpleTimer(self.on_exire_keys)
self.msg_dict = {}
self.max_count = max_count
self.imei_seq = {}
self.on_un_reply_msg_retry_func = on_un_reply_msg_retry_func
self.key_expires = key_expires
if self.on_un_reply_msg_retry_func != None:
self.timer.start()
class UnreplyMsgMgr:
def __init__(self,
on_un_reply_msg_retry_func=None,
max_count=3,
key_expires=60):
self.timer = SimpleTimer(self.on_exire_keys)
self.msg_dict = {}
self.max_count = max_count
self.imei_msg_dict = {}
self.on_un_reply_msg_retry_func = on_un_reply_msg_retry_func
self.key_expires = key_expires
if self.on_un_reply_msg_retry_func != None:
self.timer.start()
def set_on_un_reply_msg_retry_func(self, on_un_reply_msg_retry_func):
self.on_un_reply_msg_retry_func = on_un_reply_msg_retry_func
if not self.timer.is_started():
self.timer.start()
def add_unreply_msg(self, sn, imei, msg):
logger.debug("add_unreply_msg sn:%s, imei:%s msg:%s", sn, imei, msg)
seq = self.get_seq_by_sn(sn)
msg_and_count = self.msg_dict.get((imei, seq), None)
if msg_and_count is None:
self.msg_dict[(imei, seq)] = [msg, self.max_count]
seqs = self.imei_msg_dict.get(imei, None)
if seqs is None:
self.imei_msg_dict[imei] = set(seq)
else:
seqs.add(seq)
self.timer.add_key((imei, seq), self.key_expires)
else:
logger.info("on add_unreply_msg imei:%s seq:%s exist", imei, seq)
def get_seq_by_sn(self, sn):
return sn[-4:]
def delete_unreply_msg(self, sn, imei):
logger.debug("delete_unreply_msg sn:%s,imei:%s ", sn, imei)
seq = self.get_seq_by_sn(sn)
msg_and_count = self.msg_dict.get((imei, seq), None)
if msg_and_count is None:
logger.warning("on delete_unreply_msg imei:%s seq:%s not exist",
imei, seq)
else:
del self.msg_dict[(imei, seq)]
seqs = self.imei_msg_dict.get(imei, None)
if seqs is not None:
try:
seqs.remove(seq)
except Exception, e:
logger.exception(e)
def run_boids(dims, boid_q, big_boid_q, is_running):
global __is_running
__is_running = True
boids, big_boids = create_boids(dims)
# Increase to have more frames per velocity change. This slows down and smooths visualisation.
smoothness = 3
t = SimpleTimer()
i = 0
# Number of iterations after which to reset target for boids to move at.
# Needs to be run more often in 2D than in 3D.
new_target_iter = dims * 10
while __is_running and is_running.value:
# apply rules that govern velocity
boids.update_velocity()
big_boids.update_velocity()
for _ in xrange(smoothness):
# move with a fixed velocity
boids.move(1.0 / smoothness)
big_boids.move(1.0 / smoothness)
# copy the boids datastructure to avoid simultanious modification
t.print_time("putting boids (copies) in queue")
boid_q.put(boids.copy())
big_boid_q.put(big_boids.copy())
i += 1
if i % new_target_iter == 0:
t.print_time("set new position")
boids.set_random_direction()
class BoidSimulation(worker.Worker):
def init(self):
self.t = SimpleTimer(use_process_name=True)
# Number of iterations after which to reset target for boids to move at.
# Needs to be run more often in 2D than in 3D.
self.new_target_iter = 45
self.i = 0
self.init_boids()
self.worker.add_result('boids', self.boids.copy())
self.worker.add_result('big_boids', self.big_boids.copy())
def init_boids(self):
N_BIG_BOIDS = 0
self.boids, self.big_boids = create_boids_3D(num_boids,
N_BIG_BOIDS,
use_process=True)
def iteration(self, input, input_nowait):
self.t.print_time("viewer.run_boids(): top of loop")
if 'escape' in input_nowait:
escapes = input_nowait['escape']
for near, far in escapes:
self.boids.add_escapes_between(near, far)
if len(escapes) > 0:
self.i = 0
self.boids.move(1.0)
self.big_boids.move(1.0)
self.t.print_time("viewer.run_boids(): moved boids")
self.boids.update_velocity()
self.big_boids.update_velocity()
self.t.print_time("viewer.run_boids(): velocity computed")
self.i += 1
if self.i % self.new_target_iter == 0:
self.boids.clear_escapes()
self.t.print_time("viewer.run_boids(): placed boids (copies) in queue")
return {'boids': self.boids.copy(), 'big_boids': self.big_boids.copy()}
def finalize(self):
print "finalizing boids"
self.boids.finalize()
class ImeiTimer:
def __init__(self, imei_timeout=6 * 60, check_timeout=10, check_num=5):
self.real_timer = SimpleTimer(None, check_num, check_timeout)
self.imei_timeout = imei_timeout
def set_on_imeis_expire(self, on_imeis_expire_func):
self.real_timer.set_on_keys_expire(on_imeis_expire_func)
def add_imei(self, imei):
self.real_timer.add_key(imei, self.imei_timeout)
def start(self):
self.real_timer.start()
def run_server_monitoring():
"""This checks the CPU and Memory performance of all designated servers every X seconds.
:return: Void.
"""
database_api = DatabaseSimpleCommands()
timer = SimpleTimer()
pulse_rate_in_seconds = 3.0
start_time = time.time()
global keep_monitoring_servers
while keep_monitoring_servers:
timer.start_timer()
# CPU/MEMORY Monitor here.
global all_servers
for server in all_servers:
# Only poll the server if it is alive.
if server.get_is_alive():
'''
From 'top's man page.
-----------------------------------------------------------------------------------------------------------------------------------
-n :Number-of-iterations limit as: -n number
Specifies the maximum number of iterations, or frames, top should produce before ending.
-----------------------------------------------------------------------------------------------------------------------------------
-b :Batch-mode operation
Starts top in 'Batch' mode, which could be useful for sending output from top to other programs or to a file. In this
mode, top will not accept input and runs until the iterations limit you've set with the '-n' command-line option or until
killed.
-----------------------------------------------------------------------------------------------------------------------------------
-p :Monitor-PIDs mode as: -pN1 -pN2 ... or -pN1,N2,N3 ...
Monitor only processes with specified process IDs. This option can be given up to 20 times, or you can provide a comma
delimited list with up to 20 pids. Co-mingling both approaches is permitted.
A pid value of zero will be treated as the process id of the top program itself once it is running.
This is a command-line option only and should you wish to return to normal operation, it is not necessary to quit and
restart top -- just issue any of these interactive commands: '=', 'u' or 'U'.
The 'p', 'u' and 'U' command-line options are mutually exclusive.
------------------------------------------------------------------------------------------------------------------------------------
'''
p = subprocess.Popen(['top', '-p', str(server.get_pid()), '-n', '1', '-b'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate()
top_output = out.decode('ascii')
#print(top_output)
if 'utarsuno' in top_output:
#print(top_output[top_output.index('utarsuno'):len(top_output) - 1])
output_split = top_output[top_output.index('utarsuno'):len(top_output) - 1].split()
#print('CPU: ' + str(output_split[7]) + '\tMEMORY: ' + str(output_split[8]))
database_api.insert_row_into_table(daa.GLOBAL_TABLE_SERVER_MONITOR.get_table_name(), daa.GLOBAL_TABLE_SERVER_MONITOR.get_all_column_names(),
["'" + server.get_name() + "'", "'" + simple_timer.get_now_timestamp_as_string() + "'", str(output_split[7]), str(output_split[8]), 'true'])
else:
server.set_is_alive(False)
# The server is dead here
database_api.insert_row_into_table(daa.GLOBAL_TABLE_SERVER_MONITOR.get_table_name(), daa.GLOBAL_TABLE_SERVER_MONITOR.get_all_column_names(),
["'" + server.get_name() + "'", "'" + simple_timer.get_now_timestamp_as_string() + "'", '0.0', '0.0', 'false'])
else:
# The server is dead here
database_api.insert_row_into_table(daa.GLOBAL_TABLE_SERVER_MONITOR.get_table_name(), daa.GLOBAL_TABLE_SERVER_MONITOR.get_all_column_names(),
["'" + server.get_name() + "'", "'" + simple_timer.get_now_timestamp_as_string() + "'", '0.0', '0.0', 'false'])
#if server == scraper_server:
#print('Turning on the scraper server.')
#scraper_server.launch_server()
timer.end_timer()
timer.print_time()
time.sleep(pulse_rate_in_seconds - ((time.time() - start_time) % pulse_rate_in_seconds))
database_api.terminate()
print('Server monitoring has terminated!')
class WorkerClient(object):
def __init__(self, target, input_queues, nowait_queues, result_queues,
is_running, *queues):
self.target = target
self.input_queues = input_queues
self.nowait_queues = nowait_queues
self.result_queues = result_queues
self.is_running = is_running
self.t = SimpleTimer(use_process_name=True)
def work(self):
self.t.reset()
self.target.setWorkerClient(self)
self.target.init()
try:
self.target.run()
except Exception as e:
print e
self.t.print_time("finished running")
# empty input queue
clear_queues(self.input_queues)
clear_queues(self.nowait_queues)
close_queues(self.result_queues)
self.t.print_time("finalizing target")
self.target.finalize()
self.t.print_time("finalized target")
def get_all_input(self):
return dict((k, self.get_input(k)) for k in self.input_queues.keys())
def get_all_nowait(self):
nowait_inputs = {}
for key in self.nowait_queues.keys():
nowait_inputs[key] = []
try:
while True:
value = self.get_input_nowait(key)
if value is None:
nowait_inputs[key] = None
break
nowait_inputs[key].append(value)
except:
continue
return nowait_inputs
def has_input_queue(self, queue):
return queue in self.input_queues
def has_input(self, queue):
return not self.input_queues[queue].empty()
def add_result(self, queue, value):
self.result_queues[queue].put(value)
def add_all_results(self, results):
for queue, result in results.iteritems():
self.add_result(queue, result)
def get_input(self, queue):
value = self.input_queues[queue].get()
# don't finalize final queue
if value is None:
del self.input_queues[queue]
return value
def get_input_nowait(self, queue):
value = self.nowait_queues[queue].get_nowait()
# don't finalize final queue
if value is None:
del self.nowait_queues[queue]
return value
def continue_run(self):
return self.is_running.value
boids = bds.get_result('boids')
big_boids = bds.get_result('big_boids')
if with_shadow_model:
shadow_bds = worker.WorkerProcess(
'unaltered boids', ShadowBoidSimulation(boids, big_boids), {}, {
'boids': 2,
'big_boids': 2
})
shadow_boids = shadow_bds.get_result('boids')
shadow_big_boids = shadow_bds.get_result('big_boids')
else:
shadow_boids = shadow_big_boids = None
t = SimpleTimer(name="main")
t.print_time('Starting 3D interface')
glgame = GLPyGame3D(settings, interactions_file)
while bds.continue_run():
t.print_time('calling process_events()')
points = process_events(glgame, bds, boids, big_boids, shadow_boids,
shadow_big_boids)
if glgame.animate and bds.continue_run():
t.print_time('getting boids from queue')
boids = bds.get_result('boids')
import numpy as np
import dolfin as df
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from math import sin
from simple_timer import SimpleTimer
benchmark = SimpleTimer()
"""
The goal is to compute the values of f(r, t) = sin(x) * sin(t) on a mesh for
different points in time. The cost of updating f a couple of times is measured.
"""
def time_dolfin(mesh, ts):
"""
Uses a dolfin expression to compute the values of f on the mesh and the times in ts.
"""
S = df.FunctionSpace(mesh, "CG", 1)
expr = df.Expression("sin(x[0]) * sin(t)", t=0.0)
with benchmark:
for t in ts:
expr.t = t
f = df.interpolate(expr, S)
return benchmark.elapsed, f.vector().array()
class UnreplyMsgMgr2:
def __init__(self,
on_un_reply_msg_retry_func=None,
max_count=3,
key_expires=60):
self.timer = SimpleTimer(self.on_exire_keys)
self.msg_dict = {}
self.max_count = max_count
self.imei_seq = {}
self.on_un_reply_msg_retry_func = on_un_reply_msg_retry_func
self.key_expires = key_expires
if self.on_un_reply_msg_retry_func != None:
self.timer.start()
def set_on_un_reply_msg_retry_func(self, on_un_reply_msg_retry_func):
self.on_un_reply_msg_retry_func = on_un_reply_msg_retry_func
if not self.timer.is_started():
self.timer.start()
def add_unreply_msg(self, sn, imei, msg, msg_type):
logger.debug("add_unreply_msg sn:%s, imei:%s msg:%s msg_type:%s", sn,
imei, msg, msg_type)
seq = self.get_seq_by_sn(sn)
#msg_and_count = self.msg_dict.get((imei, msg_type), None)
self.msg_dict[(imei, msg_type)] = [msg, self.max_count, seq]
sub_imei_seq = self.imei_seq.get(imei, None)
if sub_imei_seq is None:
sub_imei_seq = {seq: msg_type}
else:
sub_imei_seq[seq] = msg_type
self.imei_seq[imei] = sub_imei_seq
self.timer.add_key((imei, seq), self.key_expires)
def get_seq_by_sn(self, sn):
return sn[-4:]
def delete_unreply_msg(self, sn, imei):
logger.debug("delete_unreply_msg sn:%s,imei:%s ", sn, imei)
seq = self.get_seq_by_sn(sn)
msg_and_count, msg_type = self._get_msg_and_count_by_seq(imei, seq)
if msg_and_count is not None:
if msg_and_count[2] == seq:
logger.info("delete_unreply_msg seq:%s imei:%s find ", seq,
imei)
del self.msg_dict[(imei, msg_type)]
else:
logger.warning(
"delete_unreply_msg seq:%s imei:%s not find may be removed",
seq, imei)
del self.imei_seq[imei][seq]
def get_un_reply_msg(self, imei):
logger.debug("get_un_reply_msg imei:%s ", imei)
msgs = []
sub_imei_seq = self.imei_seq.get(imei, None)
if sub_imei_seq is not None:
msg_types = sub_imei_seq.values()
msg_types = set(msg_types)
for msg_type in msg_types:
msg_and_count = self.msg_dict.get((imei, msg_type), None)
if msg_and_count is not None:
msgs.append((msg_and_count[2], msg_and_count[0]))
#del self.msg_dict[(imei, msg_type)]
return msgs
def _get_msg_and_count_by_seq(self, imei, seq):
logger.debug("_get_msg_and_count_by_seq imei:%s seq:%s", imei, seq)
msg_count = None
msg_type = None
sub_imei_seq = self.imei_seq.get(imei, None)
if sub_imei_seq is not None:
msg_type = sub_imei_seq.get(seq, None)
if msg_type is not None:
msg_count = self.msg_dict.get((imei, msg_type), None)
return msg_count, msg_type
def on_exire_keys(self, keys):
need_retry_msgs = []
for imei, seq in keys:
msg_and_count, msg_type = self._get_msg_and_count_by_seq(imei, seq)
if msg_and_count is None:
logger.warning("on on_exire_keys imei:%s seq:%s not exist",
imei, seq)
continue
if msg_and_count[2] == seq:
if msg_and_count[1] > 0:
need_retry_msgs.append(
(seq, imei, msg_and_count[0],
self.max_count - msg_and_count[1] + 1))
msg_and_count[1] = msg_and_count[1] - 1
self.msg_dict[(imei, msg_type)] = msg_and_count
else:
logger.warning(
"on on_exire_keys imei:%s seq:%s count:%d need client reconnect",
imei, seq, msg_and_count[1])
else:
logger.warning(
"on on_exire_keys imei:%s seq:%s not eq seq:%s may be replaced",
imei, seq, msg_and_count[2])
if len(need_retry_msgs) > 0:
self.on_un_reply_msg_retry_func(need_retry_msgs)
for retry_msg in need_retry_msgs:
self.timer.add_key((retry_msg[1], retry_msg[0]), self.key_expires)
def calculate_velocity(self):
t = SimpleTimer(use_process_name=True)
tmp_velocity = np.zeros((self.size, self.dimensions))
self.adjacency_list
t.print_time("computed adjacency list")
if self.use_process:
self.velocity_worker.add_input(
'matrix', (self.velocity, self.adjacency_list))
self.position_worker.add_input(
'matrix', (self.position, self.adjacency_list))
t.print_time("sent message to rule process")
else:
tmp_velocity += self.converge_velocity_neighbors()
t.print_time("converge velocity")
tmp_velocity += self.converge_position_neighbors()
t.print_time("converge position")
tmp_velocity += self.avoid_neighbors()
t.print_time("avoid neighbor")
# self.velocity += self.approach_position(self.center, self.rule1_factor)
if self.enforce_bounds:
tmp_velocity += self.ruleBounds()
t.print_time("avoid bounds")
if self.in_random_direction:
tmp_velocity += self.ruleDirection()
t.print_time("random direction")
for e in self.escapes:
tmp_velocity += self.escape_position(e, self.escape_threshold)
t.print_time("avoid escapes")
for bb in self.big_boids.position:
tmp_velocity += self.escape_position(bb, self.escape_threshold)
# self.velocity += (np.random.random((self.size, self.dimensions)) - 0.5)*0.00005
if self.use_process:
tmp_velocity += self.velocity_worker.get_result('converged')
tmp_velocity += self.position_worker.get_result('converged')
t.print_time("applied process results")
self.velocity += tmp_velocity
self.apply_min_max_velocity()
t.print_time("apply min max")
self.update_redness()
def compress_binary_data(data):
# 0x0: Get the length of data.
length_of_data = len(data)
number_of_zeros = 0
number_of_ones = 0
for b in data:
if b is 1:
number_of_ones += 1
else:
number_of_zeros += 1
print('Number of 0s: ' + str(number_of_zeros))
print('Number of 1s: ' + str(number_of_ones))
percentage_of_digits_hit_needed = 0.15
timer = SimpleTimer()
print('Obtaining linear functions...', end='')
timer.start_timer()
linear_functions = linear.get_formulas_for_linear_functions(percentage_of_digits_hit_needed, data, length_of_data, number_of_ones)
timer.end_timer()
timer.print_time()
print('Obtaining quadratic functions...', end='')
timer.start_timer()
quadratic_functions = quadratic.get_formulas_for_quadratic_functions(percentage_of_digits_hit_needed, data, length_of_data, number_of_ones)
timer.end_timer()
timer.print_time()
all_functions = []
print('Simplifying the linear functions.')
print('Functions that work: ')
for f in linear_functions:
print(f)
all_functions.append(f)
for f in quadratic_functions:
print(f)
all_functions.append(f)
print('Testing all combinations of functions to make a universal one...')
return '1 LOLOL'
def RunAnalysis(dataset, pheno_covar):
"""Run the actual analysis on all valid loci for each phenotype
:param dataset: GWAS parser object
:param pheno_covar: holds all of the variables
This acts as a standard iterator, returning a single MVResult for
each locus/phenotype combination.
Missing is evaluated as anything missing in any of the
phenotype, covariate(s) or genotype
"""
covar_missing = numpy.empty(dataset.ind_count, dtype=bool)
covar_missing[:] = False
total_covar_count = len(pheno_covar.covariate_data)
iteration_count = []
unsolved = []
pcount = 2 + total_covar_count
std = get_standardizer()
for snp in dataset:
for y in pheno_covar:
st = SimpleTimer()
(pheno, covariates, nonmissing) = y.get_variables(
(snp.genotype_data == DataParser.missing_storage))
genotypes = snp.genotype_data[nonmissing]
try:
pvalt, estimates, pvalues, se, v = RunMeanVar(
pheno, genotypes, covariates)
lmgeno = genotypes
for c in covariates:
lmgeno = lmgeno + c
lm = scipy.stats.linregress(pheno, lmgeno)[3]
betavars, se, pvalues = y.destandardize(estimates,
se,
pvalues=pvalues,
v=v,
nonmissing=nonmissing)
betas = betavars[0:pcount]
betase = se[0:pcount]
vars = betavars[pcount:]
varse = se[pcount:]
# Check for invalid output
for var in betavars + se + list(pvalues):
if numpy.isnan(var):
raise NanInResult()
nonmissing_ct = numpy.sum(nonmissing)
result = MVResult(
snp.chr,
snp.pos,
snp.rsid,
snp.major_allele,
snp.minor_allele,
dataset.get_effa_freq(genotypes),
non_miss_count=nonmissing_ct,
ph_label=y.get_phenotype_name(),
p_mvtest=pvalt,
beta_values=list(betas) + list(vars),
pvalues=pvalues,
stderrors=list(betase) + list(varse),
maf=snp.maf,
covar_labels=y.get_covariate_names(),
lm=lm,
runtime=st.runtime(),
)
result.blin = estimates[0:pcount]
result.bvar = estimates[pcount:]
yield result
except NanInResult as e:
print >> sys.stderr, "\t".join([
str(x) for x in [
snp.chr, snp.pos, snp.rsid,
y.get_phenotype_name(),
"%d" % (numpy.sum(nonmissing)), snp.major_allele,
snp.minor_allele, snp.allele_count2, "NAN-Found",
"MAF=%0.4f" % (snp.maf)
]
])
except ValueError as e:
print >> sys.stderr, "\t".join([
str(x) for x in [
snp.chr, snp.pos, snp.rsid,
y.get_phenotype_name(),
"%d" % (numpy.sum(nonmissing)), snp.major_allele,
snp.minor_allele, snp.allele_count2, "Unsolvable",
"MAF=%0.4f" % (snp.maf)
]
])
except UnsolvedLocus as e:
print >> sys.stderr, "\t".join([
str(x) for x in [
snp.chr, snp.pos, snp.rsid,
y.get_phenotype_name(),
"%d" % (numpy.sum(nonmissing)), snp.major_allele,
snp.minor_allele, snp.allele_count2, "Unsolved",
"MAF=%0.4f" % (snp.maf)
]
])
unsolved.append(snp)
if len(unsolved) > 0:
print >> sys.stderr, "Total unsolvable loci: ", len(unsolved)
class WorkerProcess(object):
def __init__(self, pname, target, input_queue_args, result_queue_args):
self.t = SimpleTimer(name='manager_' + pname)
input_queues = {}
nowait_queues = {}
result_queues = {}
for name, size in input_queue_args.iteritems():
if size is None:
input_queues[name] = mp.Queue()
elif size is 0:
nowait_queues[name] = mp.Queue()
else:
input_queues[name] = mp.Queue(maxsize=size)
for name, size in result_queue_args.iteritems():
if size is None:
result_queues[name] = mp.Queue()
else:
result_queues[name] = mp.Queue(maxsize=size)
is_running = mp.Value('b', True)
args = (target, input_queues, nowait_queues, result_queues, is_running)
print args
self.process = mp.Process(name=pname, target=do_work, args=args)
self.process.start()
self.client = WorkerClient(*args)
def has_result_queue(self, queue):
return queue in self.client.result_queues
def get_result(self, queue):
value = self.client.result_queues[queue].get()
# don't finalize final queue
if value is None:
del self.client.result_queues[queue]
return value
def add_input(self, queue, value):
self.client.input_queues[queue].put(value)
def add_input_nowait(self, queue, value):
self.client.nowait_queues[queue].put(value)
def finalize(self):
self.t.reset()
self.stop_running()
# send final call
self.t.print_time("final iter value")
close_queues(self.client.input_queues)
self.t.print_time("final nowait value")
close_queues(self.client.nowait_queues)
# empty result queue
clear_queues(self.client.result_queues, empty_first=True)
self.process.join()
def continue_run(self):
return self.client.is_running.value
def stop_running(self):
self.client.is_running.value = False
def __init__(self, imei_timeout=6 * 60, check_timeout=10, check_num=5):
self.real_timer = SimpleTimer(None, check_num, check_timeout)
self.imei_timeout = imei_timeout
def RunAnalysis(dataset, pheno_covar):
"""Run the actual analysis on all valid loci for each phenotype
:param dataset: GWAS parser object
:param pheno_covar: holds all of the variables
This acts as a standard iterator, returning a single MVResult for
each locus/phenotype combination.
Missing is evaluated as anything missing in any of the
phenotype, covariate(s) or genotype
"""
covar_missing = numpy.empty(dataset.ind_count, dtype=bool)
covar_missing[:] = False
total_covar_count = len(pheno_covar.covariate_data)
iteration_count = []
unsolved = []
pcount = 2+total_covar_count
std = get_standardizer()
for snp in dataset:
for y in pheno_covar:
st = SimpleTimer()
(pheno, covariates, nonmissing) = y.get_variables((snp.genotype_data==DataParser.missing_storage))
genotypes = snp.genotype_data[nonmissing]
try:
pvalt, estimates, pvalues, se, v = RunMeanVar(pheno, genotypes, covariates)
lmgeno = genotypes
for c in covariates:
lmgeno = lmgeno + c
lm = scipy.stats.linregress(pheno, lmgeno)[3]
betavars, se, pvalues = y.destandardize(estimates, se, pvalues=pvalues,v=v, nonmissing=nonmissing)
betas = betavars[0:pcount]
betase = se[0:pcount]
vars = betavars[pcount:]
varse = se[pcount:]
# Check for invalid output
for var in betavars + se + list(pvalues):
if numpy.isnan(var):
raise NanInResult()
nonmissing_ct = numpy.sum(nonmissing)
result = MVResult(
snp.chr,
snp.pos,
snp.rsid,
snp.major_allele,
snp.minor_allele,
dataset.get_effa_freq(genotypes),
non_miss_count=nonmissing_ct,
ph_label=y.get_phenotype_name(),
p_mvtest=pvalt,
beta_values=list(betas)+list(vars),
pvalues=pvalues,
stderrors=list(betase) + list(varse),
maf=snp.maf,
covar_labels=y.get_covariate_names(),
lm=lm,
runtime = st.runtime(),
)
result.blin = estimates[0:pcount]
result.bvar = estimates[pcount:]
yield result
except NanInResult as e:
print >> sys.stderr, "\t".join([str(x) for x in [
snp.chr,
snp.pos,
snp.rsid,
y.get_phenotype_name(),
"%d" % (numpy.sum(nonmissing)),
snp.major_allele,
snp.minor_allele,
snp.allele_count2,
"NAN-Found",
"MAF=%0.4f" % (snp.maf)]])
except ValueError as e:
print >> sys.stderr, "\t".join([str(x) for x in [
snp.chr,
snp.pos,
snp.rsid,
y.get_phenotype_name(),
"%d" % (numpy.sum(nonmissing)),
snp.major_allele,
snp.minor_allele,
snp.allele_count2,
"Unsolvable",
"MAF=%0.4f" % (snp.maf)]])
except UnsolvedLocus as e:
print >> sys.stderr, "\t".join([str(x) for x in [
snp.chr,
snp.pos,
snp.rsid,
y.get_phenotype_name(),
"%d" % (numpy.sum(nonmissing)),
snp.major_allele,
snp.minor_allele,
snp.allele_count2,
"Unsolved",
"MAF=%0.4f" % (snp.maf)]])
unsolved.append(snp)
if len(unsolved)>0:
print >> sys.stderr, "Total unsolvable loci: ", len(unsolved)