def find_maximum_depth(self): grace.status('Finding maximum depth') if self.strand_specific: iterator = legion.interleave([ self.iter_over(lambda item: item.ambiguous_depths[0], zeros=False), self.iter_over(lambda item: item.ambiguous_depths[1], zeros=False), ]) else: iterator = self.iter_over_unstranded(lambda item: item.ambiguous_depths, zeros=False) maximum = 1 norm_maximum = 1.0 normalize = self.normalizer() n = 0 #for name, pos, depths in iterator: # if n % 1000000 == 0: # grace.status('Finding maximum depth %s %s' % (name, grace.pretty_number(pos))) # n += 1 # # maximum = max(maximum,max(depths)) # norm_maximum = max(norm_maximum,max(normalize(depths))) futures = [ ] for name in self.chromosome_names: for i, depth in enumerate(self.depths): if self.strand_specific: #this = futures.append(max( # max(depth[name].ambiguous_depths[0]), # max(depth[name].ambiguous_depths[1]) #) futures.append( (name,i,legion.future(max, depth[name].ambiguous_depths[0])) ) futures.append( (name,i,legion.future(max, depth[name].ambiguous_depths[1])) ) else: #this = max(iter_add(depth[name].ambiguous_depths[0],depth[name].ambiguous_depths[1])) futures.append( (name,i,legion.future(lambda item: max(iter_add(item[0],item[1])), depth[name].ambiguous_depths)) ) for name, i, future in futures: grace.status('Finding maximum depth %s %s' % (name, self.sample_names[i])) this = future() maximum = max(maximum, this) norm_maximum = max(maximum, self.norm_mult[i] * this) self.maximum = maximum self.norm_maximum = norm_maximum grace.status('')
def find_maximum_depth(self): grace.status('Finding maximum depth') #if self.strand_specific: # iterator = itertools.chain( # self.iter_over(lambda item: item.ambiguous_depths[0], zeros=False), # self.iter_over(lambda item: item.ambiguous_depths[1], zeros=False), # ) #else: # iterator = self.iter_over_unstranded(lambda item: item.ambiguous_depths, zeros=False) maximum = 1 norm_maximum = 1.0 normalize = self.normalizer() n = 0 #for name, pos, depths in iterator: # if n % 1000000 == 0: # grace.status('Finding maximum depth %s %s' % (name, grace.pretty_number(pos))) # n += 1 # # maximum = max(maximum,max(depths)) # norm_maximum = max(norm_maximum,max(normalize(depths))) futures = [ ] for name in self.chromosome_names: for i, depth in enumerate(self.depths): if self.strand_specific: #this = futures.append(max( # max(depth[name].ambiguous_depths[0]), # max(depth[name].ambiguous_depths[1]) #) futures.append( (name,i,legion.future(max, depth[name].ambiguous_depths[0])) ) futures.append( (name,i,legion.future(max, depth[name].ambiguous_depths[1])) ) else: #this = max(iter_add(depth[name].ambiguous_depths[0],depth[name].ambiguous_depths[1])) futures.append( (name,i,legion.future(lambda item: max(iter_add(item[0],item[1])), depth[name].ambiguous_depths)) ) for name, i, future in futures: grace.status('Finding maximum depth %s %s' % (name, self.sample_names[i])) this = future() maximum = max(maximum, this) norm_maximum = max(maximum, self.norm_mult[i] * this) self.maximum = maximum self.norm_maximum = norm_maximum grace.status('')
def improve(comment, constrainer, scorer, start_x, ftol=1e-4, xtol=1e-6, initial_accuracy=0.001, monitor=lambda x, y: None): pool_size = legion.coordinator().get_cores() worker_futs = [ legion.coordinator().new_future() for i in xrange(pool_size) ] reply_futs = [] workers = [legion.future(worker, scorer, fut) for fut in worker_futs] last_t = 0.0 try: best = start_x c_score = constrainer(best) if c_score: best_score = (c_score, 0.0) else: best_score = (0.0, scorer(best)) n_good = 0 n_real = 0 i = 0 jobs = [] pool_size = int(len(best) * 5) #5 print len(best), 'parameters, pool size', pool_size currents = [(best, best_score)] done = False while not done or reply_futs: t = time.time() if t > last_t + 20.0: def rep(x): if x[0]: return 'C%.6f' % x[0] return '%.6f' % x[1] grace.status( '%s %s %d %d %d %d %s' % (rep(best_score), rep(max(item[1] for item in currents)), len(currents), n_good, n_real, i, comment)) if best_score[0] == 0: monitor(best, [item[0] for item in currents]) last_t = time.time() have_score = False if not done and worker_futs: new = make_update([item[0] for item in currents], initial_accuracy, len(currents) < pool_size) c_score = constrainer(new) if c_score: have_score = True new_score = (c_score, 0.0) else: reply_fut = legion.coordinator().new_future() worker_fut = worker_futs.pop(0) legion.coordinator().deliver_future( worker_fut, (new, reply_fut)) reply_futs.append((new, reply_fut)) if not have_score: if not reply_futs or (not done and worker_futs): continue new, reply_fut = reply_futs.pop(0) new_score, worker_fut = legion.coordinator().get_future( reply_fut) new_score = (0.0, new_score) worker_futs.append(worker_fut) if new_score[0] == 0.0: n_real += 1 l = sorted(item[1][1] for item in currents) if pool_size < len(l): c = l[pool_size] else: c = 1e30 cutoff = (best_score[0], c) if new_score <= cutoff: currents = [item for item in currents if item[1] <= cutoff] currents.append((new, new_score)) n_good += 1 if new_score < best_score: best_score = new_score best = new if len(currents) >= pool_size and best_score[0] == 0.0: xspan = 0.0 for i in xrange(len(start_x)): xspan = max( xspan, max(item[0][i] for item in currents) - min(item[0][i] for item in currents)) fspan = (max(item[1] for item in currents)[1] - best_score[1]) if xspan < xtol or (n_good >= 5000 and fspan < ftol): done = True i += 1 grace.status('') print '%s %.5f\n' % (comment, best_score[1]) finally: #pool.terminate() pass while worker_futs: fut = worker_futs.pop(0) legion.coordinator().deliver_future(fut, None) for item in workers: item() return best
def improve(comment, constrainer, scorer, start_x, ftol=1e-4, xtol=1e-6, initial_accuracy=0.001, monitor = lambda x,y: None): pool_size = legion.coordinator().get_cores() worker_futs = [ legion.coordinator().new_future() for i in xrange(pool_size) ] reply_futs = [ ] workers = [ legion.future(worker,scorer,fut) for fut in worker_futs ] last_t = 0.0 try: best = start_x c_score = constrainer(best) if c_score: best_score = (c_score, 0.0) else: best_score = (0.0, scorer(best)) n_good = 0 n_real = 0 i = 0 jobs = [ ] pool_size = int(len(best)*5) print len(best),'parameters, pool size', pool_size currents = [ (best, best_score) ] done = False while not done or reply_futs: t = time.time() if t > last_t+20.0: def rep(x): if x[0]: return 'C%.6f' % x[0] return '%.6f' % x[1] grace.status('%s %s %d %d %d %d %s'%(rep(best_score), rep(max(item[1] for item in currents)), len(currents), n_good, n_real, i, comment)) if best_score[0] == 0: monitor(best, [ item[0] for item in currents ]) last_t = time.time() have_score = False if not done and worker_futs: new = make_update([item[0] for item in currents], initial_accuracy, len(currents) < pool_size) c_score = constrainer(new) if c_score: have_score = True new_score = (c_score, 0.0) else: reply_fut = legion.coordinator().new_future() worker_fut = worker_futs.pop(0) legion.coordinator().deliver_future(worker_fut, (new, reply_fut)) reply_futs.append( (new, reply_fut) ) if not have_score: if not reply_futs or (not done and worker_futs): continue new, reply_fut = reply_futs.pop(0) new_score, worker_fut = legion.coordinator().get_future(reply_fut) new_score = (0.0, new_score) worker_futs.append(worker_fut) if new_score[0] == 0.0: n_real += 1 l = sorted( item[1][1] for item in currents ) if pool_size < len(l): c = l[pool_size] else: c = 1e30 cutoff = (best_score[0], c) if new_score <= cutoff: currents = [ item for item in currents if item[1] <= cutoff ] currents.append((new,new_score)) n_good += 1 if new_score < best_score: best_score = new_score best = new if len(currents) >= pool_size and best_score[0] == 0.0: xspan = 0.0 for i in xrange(len(start_x)): xspan = max(xspan, max(item[0][i] for item in currents) - min(item[0][i] for item in currents) ) fspan = (max(item[1] for item in currents)[1]-best_score[1]) if xspan < xtol or (n_good >= 5000 and fspan < ftol): done = True i += 1 grace.status('') print '%s %.5f\n' % (comment, best_score[1]) finally: #pool.terminate() pass while worker_futs: fut = worker_futs.pop(0) legion.coordinator().deliver_future(fut, None) for item in workers: item() return best