def main(max_fib=30): if max_fib > 30: raise Exception for i in range(0, max_fib): print fib(i+90) print mean([0,2,4,5,6,7,5,9])
def write_means(account_values, years_until_donation_date, outfile): regular_mean = util.mean(account_values["regular"]) margin_mean = util.mean(account_values["margin"]) matched_401k_mean = util.mean(account_values["matched401k"]) outfile.write("\n") outfile.write("Mean regular = ${:,}\n".format(int(round(regular_mean,0)))) outfile.write("Mean margin = ${:,}\n".format(int(round(margin_mean,0)))) outfile.write("Mean matched 401k = ${:,}\n".format(int(round(matched_401k_mean,0)))) outfile.write("Mean value per year of margin over regular = ${:,}\n".format(int(round( (margin_mean-regular_mean)/years_until_donation_date, 0))))
def residual(self, max_row_variance=None): """computes the residual for this matrix, if max_row_variance is given, result is normalized by the row variance""" d_rows = util.row_means(self.values) d_cols = util.column_means(self.values) d_all = util.mean(d_rows) tmp = self.values + d_all - util.r_outer(d_rows, d_cols, operator.add) average = util.mean(np.abs(tmp)) if max_row_variance is not None: row_var = self.row_variance() if np.isnan(row_var) or row_var > max_row_variance: row_var = max_row_variance average = average / row_var return average
def write_means(account_values, years_until_donation_date, outfile): regular_mean = util.mean(account_values["regular"]) margin_mean = util.mean(account_values["margin"]) matched_401k_mean = util.mean(account_values["matched401k"]) outfile.write("\n") outfile.write("Mean regular = ${:,}\n".format(int(round(regular_mean, 0)))) outfile.write("Mean margin = ${:,}\n".format(int(round(margin_mean, 0)))) outfile.write("Mean matched 401k = ${:,}\n".format( int(round(matched_401k_mean, 0)))) outfile.write( "Mean value per year of margin over regular = ${:,}\n".format( int( round((margin_mean - regular_mean) / years_until_donation_date, 0))))
def histogram(d, bins=100, x_label='', title='', alpha=1, label='', stats_on_label=True): stats_text = " | " + r'$\mu=' + str(round(mean(d), 2)) + r',\ \sigma=' + str(round(sstdev(d), 3)) + r'$' weights = np.ones_like(d)/float(len(d)) plt.hist(d, bins, weights=weights, alpha=alpha, label=label + (stats_text if stats_on_label else '')) plt.xlabel(x_label) plt.ylabel('Frequency') plt.title(title + (stats_text if not stats_on_label else ''))
def train(m, p=None): d = DataLoader(BSD3000(), o.batch_size, num_workers=o.num_workers) optimizer = torch.optim.Adam(m.parameters(), lr=o.lr) iter_num = len(d) num = 0 losss = [] stage = 1 if not p else p.stage + 1 for epoch in range(o.epoch): for i in tqdm(d): g, y, k, s = [x.to(o.device) for x in i] x = y optimizer.zero_grad() out = m(x) log("out", out) loss = npsnr(out, g) loss.backward() optimizer.step() losss.append(loss.detach().item()) assert not isnan(losss[-1]) print("stage", stage, "epoch", epoch + 1) log("loss", mean(losss[-5:])) num += 1 # if num > (o.epoch * iter_num - 4): if num % 50 == 1: show(torch.cat((y[0, 0], g[0, 0], out[0, 0]), 1), # save=f"save/{stage:02}{epoch:02}.png", ) plt.clf() plt.plot(range(len(losss)), losss) plt.xlabel("batch") plt.ylabel("loss") plt.title(f"{iter_num} iter x {o.epoch} epoch") plt.savefig(f"save/{stage:02}loss.png")
def average_linkage_distance(self, x, y): """ The method to determine the distance between one cluster an another item/cluster. The distance equals to the *average* (mean) distance from any member of one cluster to any member of the other cluster. :param x: first cluster/item. :param y: second cluster/item. """ # create a flat list of all the items in <x> if not isinstance(x, Cluster): x = [x] else: x = x.fullyflatten() # create a flat list of all the items in <y> if not isinstance(y, Cluster): y = [y] else: y = y.fullyflatten() distances = [] for k in x: for l in y: distances.append(self.distance(k, l)) return mean(distances)
def get_mean(x, y, z, samplesize=100, start_at=1000): length = len(x) peaks = [] for i in range(start_at, length - samplesize, 10): peaks.append(reduce_data(x, y, z, start=i, stop=i + samplesize)[1]) plt.plot(np.array(peaks)) plt.show() return util.mean(peaks)
def scatter_plot(x, y, alpha=1, x_label='', y_label='', title='', sample=0): if sample > 0: x, y = zip(*random.sample(list(zip(x, y)), sample)) fit = np.polyfit(x,y,1) fit_fn = np.poly1d(fit) plt.plot(x,y, 'k.', alpha=alpha) plt.plot([min(x), max(x)], fit_fn([min(x), max(x)]), '-g', alpha=1.0, label = 'R^2=' + str(r_square(fit_fn, x, y))) plt.xlabel(x_label) plt.ylabel(y_label) plt.title(title + " | " + r'$\mu=' + str(round(mean(y), 2)) + r',\ \sigma=' + str(round(sstdev(y), 3)) + r'$' )
def avg_trend(filename): avgs = [] orig_data = read_data(filename) for i in range(1, len(orig_data)+1): avgs.append(util.mean(calc_avg_score(orig_data, i))) plt.cla() plt.plot(avgs, '-*') plt.title('avg trend with years of source data') save_name = os.path.join(BASE_DIR, 'figures/avg_trend_by_years.png') plt.savefig(save_name, dpi=96)
def markerReliabilityCheck(self, markerCount, alphaWeight=1, betaWeight=5, lengthWeight=2): markers = {} for i in range(markerCount): ms = super(Tobo, self).see_cube() for m in ms: mCode = str(m.info.code) if mCode in markers: markers[mCode]["marker"].append(m) markers[mCode]["alpha"].append(m.centre.rot_y) markers[mCode]["beta"].append(m.orientation.rot_y) markers[mCode]["length"].append(m.centre.dist) else: markers.update({ mCode: { "marker": [m], "alpha": [m.centre.rot_y], "beta": [m.orientation.rot_y], "length": [m.centre.dist] } }) for key in list(markers): mDict = markers[key] sqrtN = math.sqrt(len(mDict["marker"])) alphaXBar = util.mean(mDict["alpha"]) alphaSd = util.stdev(mDict["alpha"]) / sqrtN betaXBar = util.mean(mDict["beta"]) betaSd = util.stdev(mDict["beta"]) / sqrtN lengthXBar = util.mean(mDict["length"]) lengthSd = util.stdev(mDict["length"]) / sqrtN error = (alphaWeight * alphaSd + betaWeight * betaSd + lengthWeight * lengthSd) / (alphaWeight + betaWeight + lengthWeight) for m in mDict["marker"]: m.error = error
def interpolate_null(geneset_size_enrichments, size_skip, go_size): # find range size_less = go_size size_more = go_size for d in range(size_skip+1): if not size_less in geneset_size_enrichments: size_less -= 1 if not size_more in geneset_size_enrichments: size_more += 1 # compute interpolation weights max_dist = 1+max(go_size-size_less, size_more-go_size) w_less = float(max_dist - (go_size-size_less)) w_more = float(max_dist - (size_more-go_size)) # compute mean, sd mean = (w_less*util.mean(geneset_size_enrichments[size_less]) + w_more*util.mean(geneset_size_enrichments[size_more])) / (w_less+w_more) sd = (w_less*util.sd(geneset_size_enrichments[size_less]) + w_more*util.sd(geneset_size_enrichments[size_more])) / (w_less+w_more) return mean, sd
def train(self, training_dataset): """ Calculates the mean and stdev of each feature in each class :param training_dataset: A n*m matrix of n features and m training items :return: a dictionary containing lists of the expected value and standard deviation for each feature in each class """ feature_summary = {"class_1": [], "class_2": []} for cls, feature_matrix in training_dataset.iteritems(): for feature in feature_matrix: feature_summary[cls].append((util.mean(feature), util.stdev(feature))) self.training_params = feature_summary return feature_summary
def _get_mean_concreteness(sample, pos_tag): """ Return the mean concreteness for the list of tokens. Inputs: pos_tag: str. the pos tag you want to compute conrcreteness for. Returns: mean_concr: float. the mean concreteness score. is None if there are no tokens with concreteness scores (after filtering by pos_tag and punctuation). """ # Get spacy encoded story spacy_encoded_story = sample.get_spacy_annotated_story() # Get all story tokens with given pos_tag toks = [t for t in util.flatten(spacy_encoded_story) if t.pos_ == pos_tag] # list of spacy Tokens # Convert to lemmas, lowercased, excluding any containing punctuation lemmas = [t.lemma_.lower() for t in toks] # list of strings lemmas = [l for l in lemmas if not util.contains_punc(l)] # If there are no remaining lemmas, return None if len(lemmas) == 0: return None # Init word2conc if necessary if word2conc is None: _init_conc() # Get concreteness ratings concr_ratings = [(t, word2conc[t]) for t in lemmas if t in word2conc] # list of (string, float) pairs # Calculate coverage (i.e. how many of the lemmas we had concreteness ratings for) concr_cov = len(concr_ratings) / len(lemmas) # Calculate mean if len(concr_ratings) == 0: mean_concr = None else: mean_concr = util.mean([score for (token, score) in concr_ratings]) # Cache the list of lemmas, the individual ratings, and the overall coverage sample.cache['concreteness_stats_{}'.format(pos_tag)] = { '{}_lemmas'.format(pos_tag): lemmas, 'concreteness_ratings': concr_ratings, 'concreteness_coverage': concr_cov, } # Return the mean concreteness return mean_concr
def interpolate_null(geneset_size_enrichments, size_skip, go_size): # find range size_less = go_size size_more = go_size for d in range(size_skip + 1): if not size_less in geneset_size_enrichments: size_less -= 1 if not size_more in geneset_size_enrichments: size_more += 1 # compute interpolation weights max_dist = 1 + max(go_size - size_less, size_more - go_size) w_less = float(max_dist - (go_size - size_less)) w_more = float(max_dist - (size_more - go_size)) # compute mean, sd mean = (w_less * util.mean(geneset_size_enrichments[size_less]) + w_more * util.mean(geneset_size_enrichments[size_more])) / (w_less + w_more) sd = (w_less * util.sd(geneset_size_enrichments[size_less]) + w_more * util.sd(geneset_size_enrichments[size_more])) / (w_less + w_more) return mean, sd
def amplitude_mean(N=100): amps = lambda: util.images().pad(100, 100).map(transform.amplitude) logger.info("Calculating mean amplitude") mean_a = util.mean(amps(), np.zeros((100, 100))) # distance d = np.linalg.norm dist = (d(amp - mean_a) for amp in util.progress(amps())) logger.info(f"Finding {N} farthest away from the mean") oddities = nlargest(N, zip(dist, util.image_ids())) logger.info(f"Finished amplitude mean outliers") return oddities
def centered_mean(N=100): centered = lambda: util.images().pad(100, 100).map(transform.center) logger.info("Calculating mean centered image") mean_c = util.mean(centered(), np.zeros((100, 100))) # distance d = np.linalg.norm dist = (d(img - mean_c) for img in util.progress(centered())) logger.info(f"Finding {N} farthest away from the mean") oddities = nlargest(N, zip(dist, util.image_ids())) logger.info(f"Finished centered mean outliers") return oddities
def histogram_mean(N=100): hists = lambda: util.images().map(transform.smoothen).map(transform.grayscale) logger.info("Calculating mean histogram") mean_h = util.mean(hists(), np.zeros((256))) # distance d = np.linalg.norm d = util.max_correlation1d dist = (d(hist, mean_h) for hist in util.progress(hists())) logger.info(f"Finding {N} farthest away from the mean") oddities = nlargest(N, zip(dist, util.image_ids())) logger.info(f"Finished histogram mean outliers") return oddities
def item_filter(per_item_preds, baseline_val): ''' Decides whether an item should be considered for inclusion in the recommendations. This version requires that three conditions be met: 1. The item must have a reasonably high predicted rating (per_item_mean > 3.) 2. The predictions must indicate that the user will rate the item higher than its baseline rating (lift > .2) 3. The predictions must not be too uncertain regarding the positive lift (conf > .75) ''' N = len(per_item_preds) mn = mean(per_item_preds) conf = float(sum([1 for p in per_item_preds if p - baseline_val > 0.])) / N lift = mn - baseline_val return mn > 3. and lift > .2 and conf > .75
def mean(N=100): images = lambda: util.images().pad(100, 100) logger.info("Calculating mean image") mean = util.mean(images()) # max-correlation of mean d = util.max_correlation d = np.linalg.norm maxcorr = (d(img - mean) for img in util.progress(images())) logger.info(f"Finding {N} with least correlation with the mean") oddities = nlargest(N, zip(maxcorr, util.image_ids())) logger.info(f"Finished maxcorr outliers") return oddities
def main(): usage = 'usage: %prog [options] <gtf file> <bam file>' parser = OptionParser(usage) parser.add_option('-i', dest='intersect_done', default=False, action='store_true', help='intersectBed is already done [Default: %default]') parser.add_option('-o', dest='output_prefix', help='Prefix for the intersectBed intermediate file [Default: %default]') (options,args) = parser.parse_args() if len(args) != 2: parser.error('Must provide gtf file and bam file') else: gtf_file = args[0] bam_file = args[1] if options.output_prefix: ib_file = '%s_reads_genes.gff' % options.output_prefix else: ib_file = 'reads_genes.gff' if not options.intersect_done: # overlap genes w/ aligned reads p = subprocess.Popen('intersectBed -s -wo -abam -bed -a %s -b %s > %s' % (bam_file,gtf_file,ib_file), shell=True) os.waitpid(p.pid,0) # count transcriptome alignments per read read_aligns = {} for line in open(ib_file): a = line.split('\t') chrom = a[0] start = int(a[1]) read_id = a[3] read_aligns.setdefault(read_id,set()).add((chrom,start)) # hash reads by gene gene_reads = {} for line in open(ib_file): a = line.split('\t') read_id = a[3] gene_id = gff.gtf_kv(a[14])['transcript_id'] gene_reads.setdefault(gene_id,[]).append(read_id) # print gene stats for gene_id in gene_reads: align_counts = [len(read_aligns[read_id]) for read_id in gene_reads[gene_id]] multi_count = float(len([ac for ac in align_counts if ac > 1])) cols = (gene_id, len(align_counts), util.mean(align_counts), multi_count/float(len(align_counts))) print '%-15s %7d %7.2f %7.2f' % cols
def rotated_mean(N=100): images = lambda: util.images().pad(100, 100) angles = lambda: util.images().pad(100, 100).map(transform.PCA_angle) rotated = lambda: util.seq(transform.rotate(img, ang) for img, ang in zip(images(), angles())) logger.info("Calculating mean rotated image") mean_r = util.mean(rotated(), np.zeros((100, 100))) # distance d = np.linalg.norm dist = (d(img - mean_r) for img in util.progress(rotated())) logger.info(f"Finding {N} farthest away from the mean") oddities = nlargest(N, zip(dist, util.image_ids())) logger.info(f"Finished rotated mean outliers") return oddities
def frequency_mean(N=100): phases = lambda: util.images().pad(100, 100).map(transform.phase) logger.info("Calculating mean phase") mean_p = util.mean(phases(), np.zeros((100, 100))) # mean_p = transform.shape(mean_p, shape="circle", mask_value=0, radius=25) # distance d = np.linalg.norm d = util.max_correlation dist = (-d(phase, mean_p) for phase in util.progress(phases())) logger.info(f"Finding {N} farthest away from the mean") oddities = nlargest(N, zip(dist, util.image_ids())) logger.info(f"Finished phase mean outliers") return oddities
def mean_log_unigramprob(sample): """ Returns the mean log unigram probability of the words in story_text. Note that we measure word unigram probability case-blind. """ tokens = sample['story_text'].strip().lower().split( ) # lowercase the story text first if word2unigramprob is None: print( "\nInitializing word2unigramprob for mean_log_unigramprob metric..." ) _init_word2unigramprob() unigram_probs = [ word2unigramprob[t] for t in tokens if t in word2unigramprob ] # if len(unigram_probs) < len(tokens): # print("WARNING: the following tokens are OOV for word2unigramprob: ", [t for t in tokens if t not in word2unigramprob]) log_unigram_probs = [np.log(p) for p in unigram_probs] return util.mean(log_unigram_probs)
def train(m, p=None): d = DataLoader(BSD3000(), o.batch_size, num_workers=o.num_workers, shuffle=True) optimizer = torch.optim.Adam(m.parameters(), lr=o.lr) iter_num = len(d) num = 0 losss = [] mse = torch.nn.MSELoss() stage = 1 if not p else p.stage + 1 for epoch in range(o.epoch): for i in tqdm(d): g, y, k, s = [x.to(o.device) for x in i] k = k.flip(1, 2) x = y if p: with torch.no_grad(): x = p([x, y, k, s]) optimizer.zero_grad() out = m([x, y, k, s]) log("out", out) out = crop(out, k) loss = npsnr(out, g) loss.backward() optimizer.step() losss.append(loss.detach().item()) assert not isnan(losss[-1]) print("stage", stage, "epoch", epoch + 1) log("loss", mean(losss[-5:])) num += 1 if num > (o.epoch * iter_num - 4): # if num % 6 == 1: show(torch.cat((crop(y, k)[0, 0], g[0, 0], out[0, 0]), 1), # save=f"save/{stage:02}{epoch:02}.png", ) plt.clf() plt.plot([i + 1 for i in range(len(losss))], losss) plt.xlabel("batch") plt.ylabel("loss") plt.title(f"{iter_num} iter x {o.epoch} epoch") plt.savefig(f"save/{stage:02}loss.png")
def train(m, p=None): d = DataLoader(BSD3000(noise=False, edgetaper=False), o.batch_size, num_workers=o.num_workers) optimizer = torch.optim.Adam(m.parameters(), lr=o.lr) iter_num = len(d) num = 0 losss = [] stage = 1 if not p else p.stage + 1 for epoch in range(o.epoch): for i in tqdm(d): g, y, k, s = [x.to(o.device) for x in i] k = k.flip(1, 2) x = torch.tensor(y, requires_grad=True) if p: with torch.no_grad(): x = p([x, y, k, s]) optimizer.zero_grad() out = m([x, y, k, s]) log("out", out) out = center_crop(out, *g.shape[-2:]) loss = npsnr(out, g) loss.backward() optimizer.step() losss.append(loss.detach().item()) assert not isnan(losss[-1]) print("stage", stage, "epoch", epoch + 1) log("loss", mean(losss[-5:])) num += 1 # if num > (o.epoch * iter_num - 4): if num % 20 == 0: show( torch.cat((center_crop( y, *g.shape[-2:])[0, 0], g[0, 0], out[0, 0]), 1), save=f"save/{stage:02}{epoch:02}.png", ) plt.clf() plt.plot(range(len(losss)), losss) plt.xlabel("batch") plt.ylabel("loss") plt.title(f"{iter_num} iter x {o.epoch} epoch") plt.savefig(f"save/{stage:02}loss.png")
def recommend(): # make predictions query = request.json preds = analysis.predict(query) # compute the rating change from baseline for each item item_scores = [] for m in query: if query[m] == None: per_item_preds = [float(p[m]) for p in preds.distribution] if item_filter(per_item_preds, baselines[m]): # print ITEM_NAMES[m], lift, per_item_mean, per_item_std per_item_mean = mean(per_item_preds) lift = per_item_mean - baselines[m] item_scores.append((m, lift)) # sort by rating change and return those items with an increase item_scores.sort(key=item_sorter, reverse=True) result = [(ITEM_NAMES[r[0]], r[1]) for r in item_scores] return json.dumps(result)
def calcFitness(population, topper_count=10): ''' Bereken de fitness voor de hele populatie en selecteer een aantal toppers ''' population_with_fitness = [] fitness_list = [] for solution in population: fit = fitness(solution) population_with_fitness.append((fit, solution)) fitness_list.append(fit) average.append(mean(fitness_list)) minima.append(min(fitness_list)) # sort the population by reversed fitness and get the first `topper_count` sorted_list = sorted(population_with_fitness, key=lambda x: x[0]) toppers = [f[1] for f in sorted_list[0:topper_count]] return toppers, fitness_list
def find_info_gain(self, X, y, attr): ''' Find the information gain for a given attribute ''' # Find out if its a numerical or categorical isCategorical = type(X[1][attr]) == str if isCategorical: value_set = set([x[attr] for x in X]) if len(value_set) == 1: # only one value for entire list return None elif len(value_set) == 2: attribute_values = [list(value_set)[0]] else: attribute_values = [x for x in value_set] else: # calculate mean, median of the values value_set = set([x[attr] for x in X]) if len(value_set) == 1: return None values = [x for x in value_set] # calculate info gain on median and mean attribute_values = [mean(values), median(values)] max_info_gain = 0 val = None X_left = None X_right = None Y_left = None Y_right = None for each in attribute_values: x_left, x_right, y_left, y_right = partition_classes( X, y, attr, each) info_gain = information_gain(y, [y_left, y_right]) if info_gain > max_info_gain: max_info_gain = info_gain val = each X_left, X_right, Y_left, Y_right = x_left, x_right, y_left, y_right return isCategorical, val, max_info_gain, X_left, X_right, Y_left, Y_right
if len(sys.argv[1:]) > 2: samples = int(sys.argv[3]) else: samples = 1000 if len(sys.argv[1:]) > 4: control_file = sys.argv[5] else: control_file = False raw_data, settings = parser.parse_analysis_results(data_file) data = raw_data log.info('Actual Top Parents Jaccard Mean (+- SD): %s (+- %s)', mean([r.top_parents_jaccard for r in data]), sstdev([r.top_parents_jaccard for r in data])) log.info('Actual Top Parents Tversky Mean (+- SD): %s (+- %s)', mean([r.top_parents_tversky for r in data]), sstdev([r.top_parents_tversky for r in data])) log.info('Actual Age Mean (+- SD): %s (+- %s)', mean([r.age for r in data]), sstdev([r.age for r in data])) log.info('') if control_file: raw_control_data, control_settings = parser.parse_analysis_results(control_file) assert settings == control_settings control_data = raw_control_data log.info('Control Top Parents Jaccard Mean (+- SD): %s (+- %s)', mean([r.top_parents_jaccard for r in control_data]), sstdev([r.top_parents_jaccard for r in control_data])) log.info('Control Top Parents Tversky Mean (+- SD): %s (+- %s)', mean([r.top_parents_tversky for r in control_data]), sstdev([r.top_parents_tversky for r in control_data])) log.info('Control Age Mean (+- SD): %s (+- %s)', mean([r.age for r in control_data]), sstdev([r.age for r in control_data])) log.info('') all_data = data + control_data
def mean(self): """returns the mean value""" return util.mean(self.values)
def hunt_choices(self, cur_round, my_food, my_rep, m, reputations): return hunt_with_threshold(reputations, util.mean(reputations))
def uploads(self, requests, peers, history): """ requests -- a list of the requests for this peer for this round peers -- available info about all the peers history -- history for all previous rounds returns: list of Upload objects. In each round, this will be called after requests(). """ ################################################################## ########### updating d_js ######################################## ################################################################## alpha = 0.2 gamma = 0.1 round = history.current_round() self.bandwidthHistory.append(self.up_bw) if round == 0: for peer in peers: self.downloadRate[peer.id] = 1 self.uploadRate[peer.id] = 1 self.slots[peer.id] = 4 self.downloadUploadRatio[peer.id] = self.downloadRate[peer.id]/self.uploadRate[peer.id] else: for peer in peers: for download in history.downloads[round-1]: if peer.id == download.from_id: self.downloadRate[peer.id] = download.blocks if download.blocks == 0: print "!!!!!! %s uploaded %s block(s)" % (peer.id, download.blocks) self.slots[peer.id] = mean(self.bandwidthHistory)/float(self.downloadRate[peer.id]) # Find how to find out max and min bw or infer from personal history if round >= 3: peer_download = 0 for download2 in history.downloads[round-2]: if peer.id == download2.from_id: for download3 in history.downloads[round-3]: if peer.id == download3.from_id: peer_download += 1 if peer_download > 0: self.uploadRate[peer.id] *= 1 - gamma break if len(peer.available_pieces) > 0: av_pieces = float(len(peer.available_pieces)) rnd = float(round) slots = float(self.slots[peer.id]) self.downloadRate[peer.id] = av_pieces/(rnd * slots) #self.downloadRate[peer.id] = float((len(peer.available_pieces)/float(round)))/float(self.slots[peer.id]) self.uploadRate[peer.id] *= 1 + alpha if self.downloadRate[peer.id] == 0: print str(peer.id) + ": " + str(peer.available_pieces) print "Peer %s has %s available pieces" % (peer.id, len(peer.available_pieces)) print "downloadUploadRatio" print self.downloadUploadRatio ########### updating current ratio ############################### if round == 0: for peer in peers: self.downloadUploadRatio[peer.id] = 1 else: for peer in peers: self.downloadUploadRatio[peer.id] = self.downloadRate[peer.id]/self.uploadRate[peer.id] ################################################################### print "download rates" print self.downloadRate print "Upload rates" print self.uploadRate print "donwload upload ratio" print self.downloadUploadRatio print "slots" print self.slots logging.debug("%s again. It's round %d." % ( self.id, round)) # One could look at other stuff in the history too here. # For example, history.downloads[round-1] (if round != 0, of course) # has a list of Download objects for each Download to this peer in # the previous round. if len(requests) == 0: logging.debug("No one wants my pieces!") chosen = [] bws = [] uploads = [] else: logging.debug("Still here: uploading to a random peer") # change my internal state for no reason self.dummy_state["cake"] = "pie" ########### Building upload list ################################# sumUpload = 0 chosen = {} downloadUploadRatio_tmp = {} # creating list with ratios for only peers in requests for request in requests: downloadUploadRatio_tmp[request.requester_id] = self.downloadUploadRatio[request.requester_id] print self.downloadUploadRatio[request.requester_id] while (sumUpload <= len(peers) and len(downloadUploadRatio_tmp) > 0): peer_id = max(downloadUploadRatio_tmp, key = downloadUploadRatio_tmp.get) chosen[peer_id] = downloadUploadRatio_tmp.pop(peer_id) sumUpload += self.uploadRate[peer_id] # print "sumUpload of %s" % (peer_id) # print sumUpload # print downloadUploadRatio_tmp[peer_id] # print self.uploadRate[peer_id] """ Calculate the total proportional BW allocated to other peers """ totalUploadBW = 0 for choice in chosen: totalUploadBW += chosen[choice] # print chosen[choice] """ Make each BW as a proportion of totalUploadBW """ for choice in chosen: chosen[choice] = 100 * float(chosen[choice]) / float(totalUploadBW) # print "Vector of choices for this round:" # print chosen """ Now need to divide our BW as integers according to chosen vector """ peerWeights = [value for (key, value) in sorted(chosen.items())] peerNames = sorted(chosen) # print "original chosen: %s" % (chosen) # print "names: %s" % (peerNames) # print "weights: %s" % (peerWeights) # request = random.choice(requests) # chosen = [request.requester_id] # Evenly "split" my upload bandwidth among the one chosen requester # bws = even_split(self.up_bw, len(chosen)) bws = proportional_split(self.up_bw, peerWeights) # create actual uploads out of the list of peer ids and bandwidths uploads = [Upload(self.id, peer_id, bw) for (peer_id, bw) in zip(chosen, bws)] return uploads
def analysis(data, settings, flat_age_matrix=[], flat_sim_matrix=[], alpha=0.03): # value_keys = ["complete_term_jaccard", "top_term_jaccard", "top_gene_jaccard", "top_parents_jaccard"] value_keys = ["top_gene_jaccard", "top_parents_jaccard"] data_values = [[row._asdict()[k] for k in value_keys] for row in data] means = [mean([x[i] for x in data_values]) for i in range(len(value_keys))] stds = [sstdev([x[i] for x in data_values]) for i in range(len(value_keys))] medians = [median([x[i] for x in data_values]) for i in range(len(value_keys))] log.info("N: %s", len(data_values)) genes_found, genes_missed = len([genes[1] for row in data for genes in row.genes_found]),len([genes for row in data for genes in row.genes_missed]) log.info("genes_found: %s, genes_missed: %s", genes_found, genes_missed) total = genes_found + genes_missed log.info("genes_found: %s, genes_missed: %s", round(genes_found / total, 2), round(genes_missed / total, 2)) #log.info("distinct unknown genes: %s", len(unknown)) fig_num = 0 for i, value_key in enumerate(value_keys): log.info("%s: mean=%s, std=%s, median=%s", value_key, means[i], stds[i], medians[i]) f = plt.figure(fig_num) fig_num +=1 d = [x[i] for x in data_values] weights = np.ones_like(d)/float(len(d)) plt.hist(d, 100, weights=weights, alpha=0.5, label='Actual') weights = np.ones_like(flat_sim_matrix)/float(len(flat_sim_matrix)) plt.hist(flat_sim_matrix, 100, weights=weights, alpha=0.5, label='Randomized') plt.xlabel('Similarity') plt.ylabel(value_keys[i]) plt.title('Histogram of ' + value_key + " | " + r'$\mu=' + str(round(means[i], 2)) + r',\ \sigma=' + str(round(stds[i], 2)) + r'$') plt.legend(loc='upper right') f.show() f = plt.figure(fig_num) fig_num +=1 x = [r.age for r in data] y = [r._asdict()[value_key] for r in data] fit = np.polyfit(x,y,1) fit_fn = np.poly1d(fit) plt.plot(x,y, 'k.', [min(x), max(x)], fit_fn([min(x), max(x)]), '--g') plt.xlabel('Age') plt.ylabel("Similarity Index") plt.title("Ancestors of " + value_key + " vs Age" + " | " + r'$\mu=' + str(round(means[i], 2)) + r',\ \sigma=' + str(round(stds[i], 2)) + r'$' ) f.show() f = plt.figure(fig_num) fig_num +=1 x = flat_age_matrix y = flat_sim_matrix fit = np.polyfit(x,y,1) fit_fn = np.poly1d(fit) plt.plot(x,y, 'k.', [min(x), max(x)], fit_fn([min(x), max(x)]), '--g', alpha=alpha) plt.xlabel('Age') plt.ylabel("Similarity Index") plt.title("Randomized Ancestors of " + value_key + " vs Age" + " | " + r'$\mu=' + str(round(means[i], 2)) + r',\ \sigma=' + str(round(stds[i], 2)) + r'$' ) f.show() f = plt.figure(fig_num) fig_num +=1 d = [x[4] for x in data] weights = np.ones_like(d)/float(len(d)) plt.hist(d, 100, weights=weights, alpha=0.5, label='Actual') weights = np.ones_like(flat_age_matrix)/float(len(flat_age_matrix)) plt.hist(flat_age_matrix, 100, weights=weights, alpha=0.5, label='Randomized') plt.xlabel('Age') f.show() raw_input() plt.close("all")
from util import mean, deviation, cv, pearson, SIZES, timing_plot # Задаём диапазон и объём каждой выборки и количество выборок RANGE, SIZE, COUNT = 10000, 100, 10 # Вычисляем теоретические значения характеристик theoretical_mean = RANGE / 2 theoretical_deviation = (((RANGE + 1)**2 - 1) / 12)**(1 / 2) theoretical_cv = theoretical_deviation / theoretical_mean # Генерируем выборки linear_samples = [linear(RANGE, SIZE, r) for r in range(1, COUNT + 1)] tausworth_samples = [tausworth(RANGE, SIZE) for _ in range(COUNT)] # Вычисляем выборочные средние linear_means = [mean(sample) for sample in linear_samples] tausworth_means = [mean(sample) for sample in tausworth_samples] # Вычисляем выборочные стандартные отклонения linear_deviations = [deviation(sample) for sample in linear_samples] tausworth_deviations = [deviation(sample) for sample in tausworth_samples] # Вычисляем выборочные коэффициенты вариации linear_cvs = [cv(sample) for sample in linear_samples] tausworth_cvs = [cv(sample) for sample in tausworth_samples] print('\nТеоретические значения характеристик:') print(' Математическое ожидание:', colored('%.2f' % theoretical_mean, 'yellow')) print(' Стандартное отклонение:', colored('%.2f' % theoretical_deviation, 'yellow'))
def test_mean(self): result = util.mean(alist) self.assertEquals(result, 3.0)
def main(args): usage_msg = "Usage: %prog [options] PeerClass1[,cnt] PeerClass2[,cnt2] ..." parser = OptionParser(usage=usage_msg) def usage(msg): print "Error: %s\n" % msg parser.print_help() sys.exit() parser.add_option("--loglevel", dest="loglevel", default="info", help="Set the logging level: 'debug' or 'info'") parser.add_option("--mech", dest="mechanism", default="gsp", help="Set the mechanim: 'gsp' or 'vcg' or 'switch'") parser.add_option("--num-rounds", dest="num_rounds", default=48, type="int", help="Set number of rounds") parser.add_option("--min-val", dest="min_val", default=25, type="int", help="Min per-click value, in cents") parser.add_option("--max-val", dest="max_val", default=175, type="int", help="Max per-click value, in cents") parser.add_option("--budget", dest="budget", default=500000, type="int", help="Total budget, in cents") parser.add_option("--reserve", dest="reserve", default=0, type="int", help="Reserve price, in cents") parser.add_option("--perms", dest="max_perms", default=120, type="int", help="Max number of value permutations to run. Set to 1 for debugging.") parser.add_option("--iters", dest="iters", default=1, type="int", help="Number of different value draws to sample. Set to 1 for debugging.") parser.add_option("--seed", dest="seed", default=None, type="int", help="seed for random numbers") (options, args) = parser.parse_args() # leftover args are class names: # e.g. "Truthful BBAgent CleverBidder Fred" if len(args) == 0: # default agents_to_run = ['Truthful', 'Truthful', 'Truthful'] else: agents_to_run = parse_agents(args) configure_logging(options.loglevel) if options.seed != None: random.seed(options.seed) # Add some more config options options.agent_class_names = agents_to_run options.agent_classes = load_modules(options.agent_class_names) options.dropoff = 0.75 logging.info("Starting simulation...") n = len(agents_to_run) totals = dict((id, 0) for id in range(n)) total_revenues = [] approx = math.factorial(n) > options.max_perms if approx: num_perms = options.max_perms logging.warning( "Running approximation: taking %d samples of value permutations" % options.max_perms) else: num_perms = math.factorial(n) av_value=range(0,n) total_spent = [0 for i in range(n)] ## iters = no. of samples to take for i in range(options.iters): values = get_utils(n, options) logging.info("==== Iteration %d / %d. Values %s ====" % (i, options.iters, values)) ## Create permutations (permutes the random values, and assigns them to agents) if approx: perms = [shuffled(values) for i in range(options.max_perms)] else: perms = itertools.permutations(values) total_rev = 0 ## Iterate over permutations for vals in perms: options.agent_values = list(vals) values = dict(zip(range(n), list(vals))) ## Runs simulation ### history = sim(options) ### simulation ends. stats = Stats(history, values) # Print stats in console? # logging.info(stats) for id in range(n): totals[id] += stats.total_utility(id) total_spent[id] += history.agents_spent[id] total_rev += stats.total_revenue() total_revenues.append(total_rev / float(num_perms)) ## total_spent = total amount of money spent by agents, for all iterations, all permutations, all rounds # Averages are over all the value permutations considered N = float(num_perms) * options.iters logging.info("%s\t\t%s\t\t%s" % ("#" * 15, "RESULTS", "#" * 15)) logging.info("") for a in range(n): logging.info("Stats for Agent %d, %s" % (a, agents_to_run[a]) ) logging.info("Average spend $%.2f (daily)" % (0.01 *total_spent[a]/N) ) logging.info("Average utility $%.2f (daily)" % (0.01 * totals[a]/N)) logging.info("-" * 40) logging.info("\n") m = mean(total_revenues) std = stddev(total_revenues) logging.warning("Average daily revenue (stddev): $%.2f ($%.2f)" % (0.01 * m, 0.01*std))
def pred_group_prob(self, group, xdata): x = [xdata[i] for i in group.index] probs = self.pred_prob(x) return mean(probs)
def _mean(data, threshold=0, default=0): return mean((e for e in data if e > threshold), default)
stats['unique_genes_missed'] = len(set(genes_missed)) # exit() # age, sim, studies = create_random_null_similarity(data, settings, True) # analysis(data, settings, age, sim) # # log.info("Perfectly Matching Studies in Randomized Data: %s", len([studies[i] for i,a in enumerate(age) if a ==0 and sim[i]==1])) # log.info("Perfectly Matching Studies in Randomized Data With same PMID: %s", len([studies[i] for i,a in enumerate(age) if a ==0 and sim[i]==1 and data[studies[i][0]].pmid == data[studies[i][1]].pmid])) # f = plt.figure() # scatter_plot(age, sim, alpha=0.002) # f.show() # # raw_input() # exit() actual_m = mean([r.top_parents_jaccard for r in data]) actual_sd = sstdev([r.top_parents_jaccard for r in data]) actual_m_tversky = mean([r.top_parents_tversky for r in data]) actual_sd_tversky = sstdev([r.top_parents_tversky for r in data]) control_m = mean([r.top_parents_jaccard for r in control_data]) control_sd = sstdev([r.top_parents_jaccard for r in control_data]) control_m_tversky = mean([r.top_parents_tversky for r in control_data]) control_sd_tversky = sstdev([r.top_parents_tversky for r in control_data]) # Add actual data to permutation test # av_sims.append(actual_m) # sd_sims.append(actual_sd) # all_sims += [r.top_parents_jaccard for r in data] # size_sims += [len(r.top_parents_current) for r in data]
def test_mean_with_nans(self): """tests the mean() function""" array = np.array([2.0, 3.0, np.nan, 1.0]) result = util.mean(array) self.assertAlmostEqual(2.0, result)
def main(args): usage_msg = "Usage: %prog [options] PeerClass1[,cnt] PeerClass2[,cnt2] ..." parser = OptionParser(usage=usage_msg) def usage(msg): print "Error: %s\n" % msg parser.print_help() sys.exit() parser.add_option("--loglevel", dest="loglevel", default="info", help="Set the logging level: 'debug' or 'info'") parser.add_option("--mech", dest="mechanism", default="gsp", help="Set the mechanim: 'gsp' or 'vcg' or 'switch'") parser.add_option("--num-rounds", dest="num_rounds", default=48, type="int", help="Set number of rounds") parser.add_option("--min-val", dest="min_val", default=25, type="int", help="Min per-click value, in cents") parser.add_option("--max-val", dest="max_val", default=175, type="int", help="Max per-click value, in cents") parser.add_option("--budget", dest="budget", default=500000, type="int", help="Total budget, in cents") parser.add_option("--reserve", dest="reserve", default=0, type="int", help="Reserve price, in cents") parser.add_option("--perms", dest="max_perms", default=120, type="int", help="Max number of value permutations to run. Set to 1 for debugging.") parser.add_option("--iters", dest="iters", default=1, type="int", help="Number of different value draws to sample. Set to 1 for debugging.") parser.add_option("--seed", dest="seed", default=None, type="int", help="seed for random numbers") (options, args) = parser.parse_args() # leftover args are class names: # e.g. "Truthful BBAgent CleverBidder Fred" if len(args) == 0: # default agents_to_run = ['Truthful', 'Truthful', 'Truthful'] else: agents_to_run = parse_agents(args) configure_logging(options.loglevel) if options.seed != None: random.seed(options.seed) # Add some more config options options.agent_class_names = agents_to_run options.agent_classes = load_modules(options.agent_class_names) options.dropoff = 0.75 logging.info("Starting simulation...") n = len(agents_to_run) totals = dict((id, 0) for id in range(n)) total_revenues = [] approx = math.factorial(n) > options.max_perms if approx: num_perms = options.max_perms logging.warning( "Running approximation: taking %d samples of value permutations" % options.max_perms) else: num_perms = math.factorial(n) av_value=range(0,n) total_spent = [0 for i in range(n)] """ Additional statistics to follow """ agents_spent_stats = [] ### For spending statistics information agents_util_stats = [] ### For utility statistics for i in range(n): agents_spent_stats.append([]) agents_util_stats.append([]) """ ------------------------------- """ ## iters = no. of samples to take for i in range(options.iters): values = get_utils(n, options) logging.info("==== Iteration %d / %d. Values %s ====" % (i, options.iters, values)) ## Create permutations (permutes the random values, and assigns them to agents) if approx: perms = [shuffled(values) for i in range(options.max_perms)] else: perms = itertools.permutations(values) total_rev = 0 ## Iterate over permutations for vals in perms: options.agent_values = list(vals) values = dict(zip(range(n), list(vals))) ## Runs simulation ### history = sim(options) ### simulation ends. stats = Stats(history, values) # Print stats in console? # logging.info(stats) for id in range(n): totals[id] += stats.total_utility(id) total_spent[id] += history.agents_spent[id] """ Additional stats ---------------- """ agents_spent_stats[id].append(history.agents_spent[id]) agents_util_stats[id].append(stats.total_utility(id)) """ --------------------------------- """ total_rev += stats.total_revenue() total_revenues.append(total_rev / float(num_perms)) ## total_spent = total amount of money spent by agents, for all iterations, all permutations, all rounds # Averages are over all the value permutations considered N = float(num_perms) * options.iters print("Total permutations: %s") % num_perms logging.info("%s\t\t%s\t\t%s" % ("#" * 15, "RESULTS", "#" * 15)) logging.info("") for a in range(n): logging.info("Stats for Agent %d, %s" % (a, agents_to_run[a]) ) logging.info("Average spend $%.2f (daily)" % (0.01 * total_spent[a]/N)) logging.info("Average utility $%.2f (daily)" % (0.01 * totals[a]/N)) logging.info("-" * 40) logging.info("\n") m = mean(total_revenues) std = stddev(total_revenues) logging.warning("Average daily revenue (stddev): $%.2f ($%.2f)" % (0.01 * m, 0.01*std)) logging.debug("agents_spent_stats:") logging.debug(agents_spent_stats) logging.debug("agents_util_stats:") logging.debug(agents_util_stats) """ Implementing output into csv """ filename = 'Stats/bb_only_reserve' + str(options.reserve) + '_seed' + str(options.seed) + '_iters' + str(options.iters) + '.csv' with open(filename, 'wb') as csvfile: spamwriter = csv.writer(csvfile, delimiter=';', quotechar='|', quoting=csv.QUOTE_MINIMAL) spamwriter.writerow(['Number of iterations'] + [options.iters]+['']+['']+['']+['']) spamwriter.writerow(['Reserve'] + [options.reserve]+['']+['']+['']+['']) spamwriter.writerow(['Seed'] + [options.seed]+['']+['']+['']+['']) spamwriter.writerow(['Agent #'] + ['Agent type'] + ['Average spent'] + ['SD spent'] + ['Average Utility'] + ['SD Utility']) for a in range(n): spamwriter.writerow([a] + [agents_to_run[a]] + [0.01*sum(agents_spent_stats[a])/N] + [0.01*stddev(agents_spent_stats[a])] + [0.01*sum(agents_util_stats[a])/N] + [0.01*stddev(agents_util_stats[a])/N]) spamwriter.writerow([n] + ['Seller'] + [0.01 * m] + [0.01 * std] + [0] + [0]) #for t in range(47, 48): #for a in agents: #print a,"'s added values is", av_value[a.id] print options
axes[i, j].plot(*line(angles[k]), color='k', lw=1.2) with Figure(2, 5) as fig: axes = get_axes(2, 5, fig) arrow(axes[..., 2]) for col_index in [0, 1, 3, 4]: clean(axes[..., col_index]) square(axes[..., col_index]) angles = lambda: (util.images().pad().map(transform.PCA_angle)) # block 1 img_mean = util.mean(util.images().pad(100, 100)) img_3 = [ util.pad(util.image(i), 100, 100) for i in ["1377", "1590", "9728"] ] samples = [img_mean] + img_3 block_imshow(1, axes, samples) # block 2 angle_mean = util.angle_mean(angles()) angle_3 = list(map(transform.PCA_angle, img_3)) transformed_samples = [angle_mean] + angle_3 block_circle(2, axes, transformed_samples)
def run(self): if not self.quiet: print 'starting with ACO with %d ants for %d iterations' % ( self.ant_count, self.iterations) maze = self.maze self.iteration_best_trail = [] # initialize ants for k in range(self.ant_count): self.ants.append(Ant(maze, maze.start)) global_best = iteration_best = None for i in range(self.iterations): if not self.quiet: print '\nIteration: %d, Q: %d, max_steps: %d' % ( i, self.Q, self.ant_max_steps) if self.multiprocessing: # Make ants do their steps. self.ants = self.pool.map_async( ant_loop_apply, itertools.izip(self.ants, [self.ant_max_steps] * self.ant_count)).get(9999999) else: print 'Stepping...' for ant in self.ants: i = 0 while not ant.done and len(ant.trail) < self.ant_max_steps: ant.step() i += 1 if i % 1000 == 1: self.visualizer.update('stepping: %d' % i) if not ant.done: print 'moving to next ant, this one stuck in', ant.position done_ants = [a for a in self.ants if a is not None and a.done] if not self.quiet: print '%d out of %d ants finished within %d steps.' % ( len(done_ants), self.ant_count, self.ant_max_steps) if self.optimize_ants: # optimize the trails for these ants opts = [] for ant in done_ants: opts.append(ant.optimize_trail(quiet=self.quiet)) if not self.quiet: print 'Optimisation reduced trail langth with an average of', mean( opts) # disable the dead ends found by the ant if self.maze_elimination: for ant in self.ants: if ant is not None: for p in ant.disable_positions: self.maze.disable_at(p) # select the best ant: if len(done_ants) > 0: iteration_best = min(done_ants) # if global_best becomes invalid, forget it. # if global_best is not None: # global_best.maze = self.maze # if not global_best.is_valid(): # global_best = None # if not self.quiet: # print 'Forgot global best!' if global_best is None: global_best = iteration_best.clone() else: global_best = min([iteration_best, global_best]).clone() # update pheromone in the maze, for unique positions deltas = np.zeros((self.maze.height, self.maze.width)) if global_best is not None: deltas = self.delta_matrix(global_best) if iteration_best is not None and global_best is not iteration_best: deltas += self.delta_matrix(iteration_best) # only update if iteration returned something. if iteration_best is not None: self.iteration_best_trail.append(len(iteration_best.trail)) else: self.iteration_best_trail.append(None) maze.update_tau(delta_tau=deltas, evaporation=self.evaporation) # update ant_max_steps to the max value of this iteration if len(done_ants) > 3: if self.update_max_steps: try: self.ant_max_steps = min( self.ant_max_steps, max( len(x.trail) for x in done_ants if len(x.trail) < self.ant_max_steps)) except: pass if self.update_Q: self.Q = min(min(len(x.trail) for x in self.ants), self.Q) if not self.quiet: if iteration_best is not None and global_best is not None: print 'Best ant: %d, iteration best: %d' % (len( global_best.trail), len(iteration_best.trail)) else: print 'None of the ants finished stepping' # reset ants for ant in self.ants: ant.reset(maze) if self.visualize: self.visualizer.update('Pheromone level iteration %d' % i) self.visualizer.save('%dth_iteration.png' % i) if self.multiprocessing: self.interrupt() self.global_best = global_best return global_best
def main(args): usage_msg = "Usage: %prog [options] PeerClass1[,cnt] PeerClass2[,cnt2] ..." parser = OptionParser(usage=usage_msg) def usage(msg): print "Error: %s\n" % msg parser.print_help() sys.exit() parser.add_option("--loglevel", dest="loglevel", default="info", help="Set the logging level: 'debug' or 'info'") parser.add_option("--mech", dest="mechanism", default="gsp", help="Set the mechanim: 'gsp' or 'vcg' or 'switch'") parser.add_option("--num-rounds", dest="num_rounds", default=48, type="int", help="Set number of rounds") parser.add_option("--min-val", dest="min_val", default=25, type="int", help="Min per-click value, in cents") parser.add_option("--max-val", dest="max_val", default=175, type="int", help="Max per-click value, in cents") parser.add_option("--budget", dest="budget", default=500000, type="int", help="Total budget, in cents") parser.add_option("--reserve", dest="reserve", default=0, type="int", help="Reserve price, in cents") parser.add_option( "--perms", dest="max_perms", default=120, type="int", help="Max number of value permutations to run. Set to 1 for debugging." ) parser.add_option( "--iters", dest="iters", default=1, type="int", help= "Number of different value draws to sample. Set to 1 for debugging.") parser.add_option("--seed", dest="seed", default=None, type="int", help="seed for random numbers") (options, args) = parser.parse_args() # leftover args are class names: # e.g. "Truthful BBAgent CleverBidder Fred" if len(args) == 0: # default agents_to_run = ['Truthful', 'Truthful', 'Truthful'] else: agents_to_run = parse_agents(args) configure_logging(options.loglevel) if options.seed != None: random.seed(options.seed) # Add some more config options options.agent_class_names = agents_to_run options.agent_classes = load_modules(options.agent_class_names) options.dropoff = 0.75 logging.info("Starting simulation...") n = len(agents_to_run) totals = dict((id, 0) for id in range(n)) total_revenues = [] approx = math.factorial(n) > options.max_perms if approx: num_perms = options.max_perms logging.warning( "Running approximation: taking %d samples of value permutations" % options.max_perms) else: num_perms = math.factorial(n) av_value = range(0, n) total_spent = [0 for i in range(n)] ## iters = no. of samples to take for i in range(options.iters): values = get_utils(n, options) logging.info("==== Iteration %d / %d. Values %s ====" % (i, options.iters, values)) ## Create permutations (permutes the random values, and assigns them to agents) if approx: perms = [shuffled(values) for i in range(options.max_perms)] else: perms = itertools.permutations(values) total_rev = 0 ## Iterate over permutations for vals in perms: options.agent_values = list(vals) values = dict(zip(range(n), list(vals))) ## Runs simulation ### history = sim(options) ### simulation ends. stats = Stats(history, values) # Print stats in console? # logging.info(stats) for id in range(n): totals[id] += stats.total_utility(id) total_spent[id] += history.agents_spent[id] total_rev += stats.total_revenue() total_revenues.append(total_rev / float(num_perms)) ## total_spent = total amount of money spent by agents, for all iterations, all permutations, all rounds # Averages are over all the value permutations considered N = float(num_perms) * options.iters logging.info("%s\t\t%s\t\t%s" % ("#" * 15, "RESULTS", "#" * 15)) logging.info("") for a in range(n): logging.info("Stats for Agent %d, %s" % (a, agents_to_run[a])) logging.info("Average spend $%.2f (daily)" % (0.01 * total_spent[a] / N)) logging.info("Average utility $%.2f (daily)" % (0.01 * totals[a] / N)) logging.info("-" * 40) logging.info("\n") m = mean(total_revenues) std = stddev(total_revenues) logging.warning("Average daily revenue (stddev): $%.2f ($%.2f)" % (0.01 * m, 0.01 * std)) ############### # ERASE LATER # ############### return (0.01 * m, 0.01 * std, list(map(lambda x: 0.01 * x / N, total_spent)), list(map(lambda x: 0.01 * totals[x] / N, totals.keys())))
def computeConfidence(meanList, sdList): return util.mean([sd/val for val,sd in zip(meanList,sdList)])
def _ratio(*data): _len = [len(e) for e in data] return mean(_len) / max(_len)
def run(self): if not self.quiet: print 'starting with ACO with %d ants for %d iterations' % ( self.ant_count, self.iterations ) maze = self.maze self.iteration_best_trail = [] # initialize ants for k in range(self.ant_count): self.ants.append(Ant(maze, maze.start)) global_best = iteration_best = None for i in range(self.iterations): if not self.quiet: print '\nIteration: %d, Q: %d, max_steps: %d' % (i, self.Q, self.ant_max_steps) if self.multiprocessing: # Make ants do their steps. self.ants = self.pool.map_async( ant_loop_apply, itertools.izip(self.ants, [self.ant_max_steps] * self.ant_count) ).get(9999999) else: print 'Stepping...' for ant in self.ants: i = 0 while not ant.done and len(ant.trail) < self.ant_max_steps: ant.step() i += 1 if i % 1000 == 1: self.visualizer.update('stepping: %d' % i) if not ant.done: print 'moving to next ant, this one stuck in', ant.position done_ants = [a for a in self.ants if a is not None and a.done] if not self.quiet: print '%d out of %d ants finished within %d steps.' % ( len(done_ants), self.ant_count, self.ant_max_steps ) if self.optimize_ants: # optimize the trails for these ants opts = [] for ant in done_ants: opts.append(ant.optimize_trail(quiet=self.quiet)) if not self.quiet: print 'Optimisation reduced trail langth with an average of', mean(opts) # disable the dead ends found by the ant if self.maze_elimination: for ant in self.ants: if ant is not None: for p in ant.disable_positions: self.maze.disable_at(p) # select the best ant: if len(done_ants) > 0: iteration_best = min(done_ants) # if global_best becomes invalid, forget it. # if global_best is not None: # global_best.maze = self.maze # if not global_best.is_valid(): # global_best = None # if not self.quiet: # print 'Forgot global best!' if global_best is None: global_best = iteration_best.clone() else: global_best = min([iteration_best, global_best]).clone() # update pheromone in the maze, for unique positions deltas = np.zeros((self.maze.height, self.maze.width)) if global_best is not None: deltas = self.delta_matrix(global_best) if iteration_best is not None and global_best is not iteration_best: deltas += self.delta_matrix(iteration_best) # only update if iteration returned something. if iteration_best is not None: self.iteration_best_trail.append(len(iteration_best.trail)) else: self.iteration_best_trail.append(None) maze.update_tau(delta_tau=deltas, evaporation=self.evaporation) # update ant_max_steps to the max value of this iteration if len(done_ants) > 3: if self.update_max_steps: try: self.ant_max_steps = min( self.ant_max_steps, max(len(x.trail) for x in done_ants if len(x.trail) < self.ant_max_steps) ) except: pass if self.update_Q: self.Q = min(min(len(x.trail) for x in self.ants), self.Q) if not self.quiet: if iteration_best is not None and global_best is not None: print 'Best ant: %d, iteration best: %d' % ( len(global_best.trail), len(iteration_best.trail) ) else: print 'None of the ants finished stepping' # reset ants for ant in self.ants: ant.reset(maze) if self.visualize: self.visualizer.update('Pheromone level iteration %d' % i) self.visualizer.save('%dth_iteration.png' % i) if self.multiprocessing: self.interrupt() self.global_best = global_best return global_best
def uploads(self, requests, peers, history): ################################################################## ########### updating d_js ######################################## ################################################################## alpha = 0.2 gamma = 0.1 round = history.current_round() self.bandwidthHistory.append(self.up_bw) if round == 0: bw_list = even_split(self.up_bw,len(peers)) for peer,i in zip(peers,range(len(peers))): self.downloadRate[peer.id] = (self.conf.max_up_bw - self.conf.min_up_bw)/(4*2) if bw_list[i] == 0: self.uploadRate[peer.id] = 0.5 else: self.uploadRate[peer.id] = bw_list[i] self.slots[peer.id] = 4 self.downloadUploadRatio[peer.id] = 1 else: for peer in peers: self.downloadRate[peer.id] = 0 for download in history.downloads[round-1]: if peer.id == download.from_id: self.downloadRate[peer.id] += download.blocks if download.blocks == 0: print "!!!!!! %s uploaded %s block(s)" % (peer.id, download.blocks) self.slots[peer.id] = mean(self.bandwidthHistory)/float(self.downloadRate[peer.id]) # Find how to find out max and min bw or infer from personal history if round >= 3: peer_download = 0 for download2 in history.downloads[round-2]: if peer.id == download2.from_id: for download3 in history.downloads[round-3]: if peer.id == download3.from_id: peer_download += 1 if peer_download > 0: self.uploadRate[peer.id] *= 1 - gamma break if len(peer.available_pieces) > 0: av_pieces = float(len(peer.available_pieces)) rnd = float(round) slots = float(self.slots[peer.id]) self.downloadRate[peer.id] = av_pieces/(rnd * self.conf.blocks_per_piece * slots) self.uploadRate[peer.id] *= 1 + alpha self.downloadUploadRatio[peer.id] = self.downloadRate[peer.id]/self.uploadRate[peer.id] if len(requests) == 0: logging.debug("No one wants my pieces!") chosen = [] bws = [] uploads = [] else: logging.debug("Still here: uploading to a random peer") # change my internal state for no reason self.dummy_state["cake"] = "pie" ################################################################## ########### Building upload list ################################# ################################################################## sumUpload = 0 chosen = {} downloadUploadRatio_tmp = {} # creating list with ratios for only peers in requests for request in requests: downloadUploadRatio_tmp[request.requester_id] = self.downloadUploadRatio[request.requester_id] while (sumUpload <= self.up_bw and len(downloadUploadRatio_tmp) > 0): peer_id = max(downloadUploadRatio_tmp, key = downloadUploadRatio_tmp.get) chosen[peer_id] = downloadUploadRatio_tmp.pop(peer_id) sumUpload += self.uploadRate[peer_id] """ Calculate the total proportional BW allocated to other peers """ totalUploadBW = 0 for choice in chosen: totalUploadBW += chosen[choice] """ Make each BW as a proportion of totalUploadBW """ if float(totalUploadBW) == 0: uploads = [] else: for choice in chosen: chosen[choice] = 100 * float(chosen[choice]) / float(totalUploadBW) """ Now need to divide our BW as integers according to chosen vector """ peerWeights = [value for (key, value) in sorted(chosen.items())] peerNames = sorted(chosen) bws = proportional_split(self.up_bw, peerWeights) # create actual uploads out of the list of peer ids and bandwidths uploads = [Upload(self.id, peer_id, bw) for (peer_id, bw) in zip(chosen, bws)] return uploads
def uploads(self, requests, peers, history): """ requests -- a list of the requests for this peer for this round peers -- available info about all the peers history -- history for all previous rounds returns: list of Upload objects. In each round, this will be called after requests(). """ ################################################################## ########### updating d_js ######################################## ################################################################## alpha = 0.2 gamma = 0.1 round = history.current_round() self.bandwidthHistory.append(self.up_bw) if round == 0: bw_list = even_split(self.up_bw,len(peers)) for peer,i in zip(peers,range(len(peers))): self.downloadRate[peer.id] = 1 if bw_list[i] == 0: self.uploadRate[peer.id] = 0.5 else: self.uploadRate[peer.id] = bw_list[i] self.slots[peer.id] = 4 self.downloadUploadRatio[peer.id] = 1 else: for peer in peers: for download in history.downloads[round-1]: if peer.id == download.from_id: self.downloadRate[peer.id] = download.blocks if download.blocks == 0: print "!!!!!! %s uploaded %s block(s)" % (peer.id, download.blocks) self.slots[peer.id] = mean(self.bandwidthHistory)/float(self.downloadRate[peer.id]) # Find how to find out max and min bw or infer from personal history if round >= 3: peer_download = 0 for download2 in history.downloads[round-2]: if peer.id == download2.from_id: for download3 in history.downloads[round-3]: if peer.id == download3.from_id: peer_download += 1 if peer_download > 0: self.uploadRate[peer.id] *= 1 - gamma break if len(peer.available_pieces) > 0: av_pieces = float(len(peer.available_pieces)) rnd = float(round) slots = float(self.slots[peer.id]) self.downloadRate[peer.id] = av_pieces/(rnd * self.conf.blocks_per_piece * slots) self.uploadRate[peer.id] *= 1 + alpha #if self.downloadRate[peer.id] == 0: # print str(peer.id) + ": " + str(peer.available_pieces) # print "Peer %s has %s available pieces" % (peer.id, len(peer.available_pieces)) self.downloadUploadRatio[peer.id] = self.downloadRate[peer.id]/self.uploadRate[peer.id] logging.debug("%s again. It's round %d." % ( self.id, round)) ########### Dynamic Optimistic Unchoking ################################# ########### Eallocate each 3 rounds ###################################### ### Choose peer to uchoke every 3 round ### Choose # of peers divided by x: x = 3 if round % 3 ==0: self.optUnchokedPeers = random.sample(peers, len(peers)/x) ### Initial share to uchoke: a = 0.5 availPiecesShare = float(sum(self.pieces))/float(self.conf.num_pieces*self.conf.blocks_per_piece) ### Allocate BW to opt unchoking bwToOptUnchoking = (a*self.up_bw - availPiecesShare * self.up_bw * a) + 0.001 ### Divide this BW among number of neighbors divided by x bwToOptUnchoking = bwToOptUnchoking/(len(peers)/x) optUnchokedAllocation = {} for peer in self.optUnchokedPeers: optUnchokedAllocation[peer.id] = float(100 * bwToOptUnchoking) /(float((a*self.up_bw - availPiecesShare * self.up_bw * a)) +0.001) up_bw_available = self.up_bw - bwToOptUnchoking*(len(peers)/x) # Removing optimistically unchoked peers from consideration peers_tmp = list(peers) for peer in self.optUnchokedPeers: if peer in peers_tmp: peers_tmp.remove(peer) ######################################################################### ########### Building upload list ######################################## ######################################################################### if len(requests) == 0: logging.debug("No one wants my pieces!") chosen = [] bws = [] uploads = [] else: sumUpload = 0 chosen = {} downloadUploadRatio_tmp = {} # creating list with ratios for only peers in requests for request in requests: for peer in peers_tmp: if request.requester_id == peer.id: downloadUploadRatio_tmp[request.requester_id] = self.downloadUploadRatio[request.requester_id] for key in downloadUploadRatio_tmp: if downloadUploadRatio_tmp[key] in self.optUnchokedPeers: downloadUploadRatio_tmp.pop(key) needed = lambda i: self.pieces[i] < self.conf.blocks_per_piece needed_pieces = filter(needed, range(len(self.pieces))) while (sumUpload <= up_bw_available * 0.75 and len(downloadUploadRatio_tmp) > 0): peer_id = max(downloadUploadRatio_tmp, key = downloadUploadRatio_tmp.get) chosen[peer_id] = downloadUploadRatio_tmp.pop(peer_id) sumUpload += self.uploadRate[peer_id] """ Calculate the total proportional BW allocated to other peers """ totalUploadBW = 0 for choice in chosen: totalUploadBW += chosen[choice] """ Make each BW as a proportion of totalUploadBW """ if (float(totalUploadBW) * len(optUnchokedAllocation) == 0): uploads = [] else: for choice in chosen: chosen[choice] = 100 * float(chosen[choice]) / float(totalUploadBW) ### Connecting optimistic unchoking list to tyrant list # chosen.update(optUnchokedAllocation) # print "Vector of choices for this round:" # print chosen """ Now need to divide our BW as integers according to chosen vector """ peerWeights = [value for (key, value) in sorted(chosen.items())] peerNames = sorted(chosen) # print "original chosen: %s" % (chosen) # print "names: %s" % (peerNames) # print "weights: %s" % (peerWeights) bws = proportional_split(int(math.floor(up_bw_available)), peerWeights) # create actual uploads out of the list of peer ids and bandwidths uploads = [Upload(self.id, peer_id, bw) for (peer_id, bw) in zip(chosen, bws)] peerWeights = [value for (key, value) in sorted(optUnchokedAllocation.items())] peerNames = sorted(optUnchokedAllocation) bws = proportional_split(self.up_bw - int(up_bw_available), peerWeights) uploads2 = [Upload(self.id, peer_id, bw) for (peer_id, bw) in zip(peerNames, bws)] uploads = uploads + uploads2 if (round + 1) % 5 == 0: request = random.choice(requests) chosen = [request.requester_id] # Evenly "split" my upload bandwidth among the one chosen requester bws = even_split(self.up_bw, len(chosen)) #uploads = uploads.append([Upload(self.id, peer_id, bw) # for (peer_id, bw) in zip(optUnchokedAllocation, bws)]) return uploads
def coeff_var(row_values): """computes the coefficient of variation""" sigma = util.r_stddev(row_values) mu = util.mean(row_values) return sigma / mu