def find_maximum_depth(self):
grace.status('Finding maximum depth')
if self.strand_specific:
iterator = legion.interleave([
self.iter_over(lambda item: item.ambiguous_depths[0], zeros=False),
self.iter_over(lambda item: item.ambiguous_depths[1], zeros=False),
])
else:
iterator = self.iter_over_unstranded(lambda item: item.ambiguous_depths, zeros=False)
maximum = 1
norm_maximum = 1.0
normalize = self.normalizer()
n = 0
#for name, pos, depths in iterator:
# if n % 1000000 == 0:
# grace.status('Finding maximum depth %s %s' % (name, grace.pretty_number(pos)))
# n += 1
#
# maximum = max(maximum,max(depths))
# norm_maximum = max(norm_maximum,max(normalize(depths)))
futures = [ ]
for name in self.chromosome_names:
for i, depth in enumerate(self.depths):
if self.strand_specific:
#this = futures.append(max(
# max(depth[name].ambiguous_depths[0]),
# max(depth[name].ambiguous_depths[1])
#)
futures.append( (name,i,legion.future(max, depth[name].ambiguous_depths[0])) )
futures.append( (name,i,legion.future(max, depth[name].ambiguous_depths[1])) )
else:
#this = max(iter_add(depth[name].ambiguous_depths[0],depth[name].ambiguous_depths[1]))
futures.append( (name,i,legion.future(lambda item: max(iter_add(item[0],item[1])), depth[name].ambiguous_depths)) )
for name, i, future in futures:
grace.status('Finding maximum depth %s %s' % (name, self.sample_names[i]))
this = future()
maximum = max(maximum, this)
norm_maximum = max(maximum, self.norm_mult[i] * this)
self.maximum = maximum
self.norm_maximum = norm_maximum
grace.status('')
def find_maximum_depth(self):
grace.status('Finding maximum depth')
#if self.strand_specific:
# iterator = itertools.chain(
# self.iter_over(lambda item: item.ambiguous_depths[0], zeros=False),
# self.iter_over(lambda item: item.ambiguous_depths[1], zeros=False),
# )
#else:
# iterator = self.iter_over_unstranded(lambda item: item.ambiguous_depths, zeros=False)
maximum = 1
norm_maximum = 1.0
normalize = self.normalizer()
n = 0
#for name, pos, depths in iterator:
# if n % 1000000 == 0:
# grace.status('Finding maximum depth %s %s' % (name, grace.pretty_number(pos)))
# n += 1
#
# maximum = max(maximum,max(depths))
# norm_maximum = max(norm_maximum,max(normalize(depths)))
futures = [ ]
for name in self.chromosome_names:
for i, depth in enumerate(self.depths):
if self.strand_specific:
#this = futures.append(max(
# max(depth[name].ambiguous_depths[0]),
# max(depth[name].ambiguous_depths[1])
#)
futures.append( (name,i,legion.future(max, depth[name].ambiguous_depths[0])) )
futures.append( (name,i,legion.future(max, depth[name].ambiguous_depths[1])) )
else:
#this = max(iter_add(depth[name].ambiguous_depths[0],depth[name].ambiguous_depths[1]))
futures.append( (name,i,legion.future(lambda item: max(iter_add(item[0],item[1])), depth[name].ambiguous_depths)) )
for name, i, future in futures:
grace.status('Finding maximum depth %s %s' % (name, self.sample_names[i]))
this = future()
maximum = max(maximum, this)
norm_maximum = max(maximum, self.norm_mult[i] * this)
self.maximum = maximum
self.norm_maximum = norm_maximum
grace.status('')
def improve(comment,
constrainer,
scorer,
start_x,
ftol=1e-4,
xtol=1e-6,
initial_accuracy=0.001,
monitor=lambda x, y: None):
pool_size = legion.coordinator().get_cores()
worker_futs = [
legion.coordinator().new_future() for i in xrange(pool_size)
]
reply_futs = []
workers = [legion.future(worker, scorer, fut) for fut in worker_futs]
last_t = 0.0
try:
best = start_x
c_score = constrainer(best)
if c_score:
best_score = (c_score, 0.0)
else:
best_score = (0.0, scorer(best))
n_good = 0
n_real = 0
i = 0
jobs = []
pool_size = int(len(best) * 5) #5
print len(best), 'parameters, pool size', pool_size
currents = [(best, best_score)]
done = False
while not done or reply_futs:
t = time.time()
if t > last_t + 20.0:
def rep(x):
if x[0]: return 'C%.6f' % x[0]
return '%.6f' % x[1]
grace.status(
'%s %s %d %d %d %d %s' %
(rep(best_score), rep(max(item[1] for item in currents)),
len(currents), n_good, n_real, i, comment))
if best_score[0] == 0:
monitor(best, [item[0] for item in currents])
last_t = time.time()
have_score = False
if not done and worker_futs:
new = make_update([item[0] for item in currents],
initial_accuracy,
len(currents) < pool_size)
c_score = constrainer(new)
if c_score:
have_score = True
new_score = (c_score, 0.0)
else:
reply_fut = legion.coordinator().new_future()
worker_fut = worker_futs.pop(0)
legion.coordinator().deliver_future(
worker_fut, (new, reply_fut))
reply_futs.append((new, reply_fut))
if not have_score:
if not reply_futs or (not done and worker_futs):
continue
new, reply_fut = reply_futs.pop(0)
new_score, worker_fut = legion.coordinator().get_future(
reply_fut)
new_score = (0.0, new_score)
worker_futs.append(worker_fut)
if new_score[0] == 0.0:
n_real += 1
l = sorted(item[1][1] for item in currents)
if pool_size < len(l):
c = l[pool_size]
else:
c = 1e30
cutoff = (best_score[0], c)
if new_score <= cutoff:
currents = [item for item in currents if item[1] <= cutoff]
currents.append((new, new_score))
n_good += 1
if new_score < best_score:
best_score = new_score
best = new
if len(currents) >= pool_size and best_score[0] == 0.0:
xspan = 0.0
for i in xrange(len(start_x)):
xspan = max(
xspan,
max(item[0][i]
for item in currents) - min(item[0][i]
for item in currents))
fspan = (max(item[1] for item in currents)[1] - best_score[1])
if xspan < xtol or (n_good >= 5000 and fspan < ftol):
done = True
i += 1
grace.status('')
print '%s %.5f\n' % (comment, best_score[1])
finally:
#pool.terminate()
pass
while worker_futs:
fut = worker_futs.pop(0)
legion.coordinator().deliver_future(fut, None)
for item in workers:
item()
return best
def improve(comment, constrainer, scorer, start_x, ftol=1e-4, xtol=1e-6, initial_accuracy=0.001, monitor = lambda x,y: None):
pool_size = legion.coordinator().get_cores()
worker_futs = [ legion.coordinator().new_future() for i in xrange(pool_size) ]
reply_futs = [ ]
workers = [
legion.future(worker,scorer,fut)
for fut in worker_futs
]
last_t = 0.0
try:
best = start_x
c_score = constrainer(best)
if c_score:
best_score = (c_score, 0.0)
else:
best_score = (0.0, scorer(best))
n_good = 0
n_real = 0
i = 0
jobs = [ ]
pool_size = int(len(best)*5)
print len(best),'parameters, pool size', pool_size
currents = [ (best, best_score) ]
done = False
while not done or reply_futs:
t = time.time()
if t > last_t+20.0:
def rep(x):
if x[0]: return 'C%.6f' % x[0]
return '%.6f' % x[1]
grace.status('%s %s %d %d %d %d %s'%(rep(best_score), rep(max(item[1] for item in currents)), len(currents), n_good, n_real, i, comment))
if best_score[0] == 0:
monitor(best, [ item[0] for item in currents ])
last_t = time.time()
have_score = False
if not done and worker_futs:
new = make_update([item[0] for item in currents], initial_accuracy, len(currents) < pool_size)
c_score = constrainer(new)
if c_score:
have_score = True
new_score = (c_score, 0.0)
else:
reply_fut = legion.coordinator().new_future()
worker_fut = worker_futs.pop(0)
legion.coordinator().deliver_future(worker_fut, (new, reply_fut))
reply_futs.append( (new, reply_fut) )
if not have_score:
if not reply_futs or (not done and worker_futs):
continue
new, reply_fut = reply_futs.pop(0)
new_score, worker_fut = legion.coordinator().get_future(reply_fut)
new_score = (0.0, new_score)
worker_futs.append(worker_fut)
if new_score[0] == 0.0:
n_real += 1
l = sorted( item[1][1] for item in currents )
if pool_size < len(l):
c = l[pool_size]
else:
c = 1e30
cutoff = (best_score[0], c)
if new_score <= cutoff:
currents = [ item for item in currents if item[1] <= cutoff ]
currents.append((new,new_score))
n_good += 1
if new_score < best_score:
best_score = new_score
best = new
if len(currents) >= pool_size and best_score[0] == 0.0:
xspan = 0.0
for i in xrange(len(start_x)):
xspan = max(xspan,
max(item[0][i] for item in currents) -
min(item[0][i] for item in currents)
)
fspan = (max(item[1] for item in currents)[1]-best_score[1])
if xspan < xtol or (n_good >= 5000 and fspan < ftol):
done = True
i += 1
grace.status('')
print '%s %.5f\n' % (comment, best_score[1])
finally:
#pool.terminate()
pass
while worker_futs:
fut = worker_futs.pop(0)
legion.coordinator().deliver_future(fut, None)
for item in workers:
item()
return best
