def main(max_fib=30): if max_fib > 30: raise Exception for i in range(0, max_fib): print fib(i+90) print mean([0,2,4,5,6,7,5,9])
def write_means(account_values, years_until_donation_date, outfile):
regular_mean = util.mean(account_values["regular"])
margin_mean = util.mean(account_values["margin"])
matched_401k_mean = util.mean(account_values["matched401k"])
outfile.write("\n")
outfile.write("Mean regular = ${:,}\n".format(int(round(regular_mean,0))))
outfile.write("Mean margin = ${:,}\n".format(int(round(margin_mean,0))))
outfile.write("Mean matched 401k = ${:,}\n".format(int(round(matched_401k_mean,0))))
outfile.write("Mean value per year of margin over regular = ${:,}\n".format(int(round( (margin_mean-regular_mean)/years_until_donation_date, 0))))
def residual(self, max_row_variance=None):
"""computes the residual for this matrix, if max_row_variance is given,
result is normalized by the row variance"""
d_rows = util.row_means(self.values)
d_cols = util.column_means(self.values)
d_all = util.mean(d_rows)
tmp = self.values + d_all - util.r_outer(d_rows, d_cols, operator.add)
average = util.mean(np.abs(tmp))
if max_row_variance is not None:
row_var = self.row_variance()
if np.isnan(row_var) or row_var > max_row_variance:
row_var = max_row_variance
average = average / row_var
return average
def write_means(account_values, years_until_donation_date, outfile):
regular_mean = util.mean(account_values["regular"])
margin_mean = util.mean(account_values["margin"])
matched_401k_mean = util.mean(account_values["matched401k"])
outfile.write("\n")
outfile.write("Mean regular = ${:,}\n".format(int(round(regular_mean, 0))))
outfile.write("Mean margin = ${:,}\n".format(int(round(margin_mean, 0))))
outfile.write("Mean matched 401k = ${:,}\n".format(
int(round(matched_401k_mean, 0))))
outfile.write(
"Mean value per year of margin over regular = ${:,}\n".format(
int(
round((margin_mean - regular_mean) / years_until_donation_date,
0))))
def residual(self, max_row_variance=None):
"""computes the residual for this matrix, if max_row_variance is given,
result is normalized by the row variance"""
d_rows = util.row_means(self.values)
d_cols = util.column_means(self.values)
d_all = util.mean(d_rows)
tmp = self.values + d_all - util.r_outer(d_rows, d_cols, operator.add)
average = util.mean(np.abs(tmp))
if max_row_variance is not None:
row_var = self.row_variance()
if np.isnan(row_var) or row_var > max_row_variance:
row_var = max_row_variance
average = average / row_var
return average
def histogram(d, bins=100, x_label='', title='', alpha=1, label='', stats_on_label=True):
stats_text = " | " + r'$\mu=' + str(round(mean(d), 2)) + r',\ \sigma=' + str(round(sstdev(d), 3)) + r'$'
weights = np.ones_like(d)/float(len(d))
plt.hist(d, bins, weights=weights, alpha=alpha, label=label + (stats_text if stats_on_label else ''))
plt.xlabel(x_label)
plt.ylabel('Frequency')
plt.title(title + (stats_text if not stats_on_label else ''))
def train(m, p=None):
d = DataLoader(BSD3000(), o.batch_size, num_workers=o.num_workers)
optimizer = torch.optim.Adam(m.parameters(), lr=o.lr)
iter_num = len(d)
num = 0
losss = []
stage = 1 if not p else p.stage + 1
for epoch in range(o.epoch):
for i in tqdm(d):
g, y, k, s = [x.to(o.device) for x in i]
x = y
optimizer.zero_grad()
out = m(x)
log("out", out)
loss = npsnr(out, g)
loss.backward()
optimizer.step()
losss.append(loss.detach().item())
assert not isnan(losss[-1])
print("stage", stage, "epoch", epoch + 1)
log("loss", mean(losss[-5:]))
num += 1
# if num > (o.epoch * iter_num - 4):
if num % 50 == 1:
show(torch.cat((y[0, 0], g[0, 0], out[0, 0]), 1),
# save=f"save/{stage:02}{epoch:02}.png",
)
plt.clf()
plt.plot(range(len(losss)), losss)
plt.xlabel("batch")
plt.ylabel("loss")
plt.title(f"{iter_num} iter x {o.epoch} epoch")
plt.savefig(f"save/{stage:02}loss.png")
def average_linkage_distance(self, x, y):
"""
The method to determine the distance between one cluster an another
item/cluster. The distance equals to the *average* (mean) distance
from any member of one cluster to any member of the other cluster.
:param x: first cluster/item.
:param y: second cluster/item.
"""
# create a flat list of all the items in <x>
if not isinstance(x, Cluster):
x = [x]
else:
x = x.fullyflatten()
# create a flat list of all the items in <y>
if not isinstance(y, Cluster):
y = [y]
else:
y = y.fullyflatten()
distances = []
for k in x:
for l in y:
distances.append(self.distance(k, l))
return mean(distances)
def get_mean(x, y, z, samplesize=100, start_at=1000):
length = len(x)
peaks = []
for i in range(start_at, length - samplesize, 10):
peaks.append(reduce_data(x, y, z, start=i, stop=i + samplesize)[1])
plt.plot(np.array(peaks))
plt.show()
return util.mean(peaks)
def scatter_plot(x, y, alpha=1, x_label='', y_label='', title='', sample=0):
if sample > 0:
x, y = zip(*random.sample(list(zip(x, y)), sample))
fit = np.polyfit(x,y,1)
fit_fn = np.poly1d(fit)
plt.plot(x,y, 'k.', alpha=alpha)
plt.plot([min(x), max(x)], fit_fn([min(x), max(x)]), '-g', alpha=1.0, label = 'R^2=' + str(r_square(fit_fn, x, y)))
plt.xlabel(x_label)
plt.ylabel(y_label)
plt.title(title + " | " + r'$\mu=' + str(round(mean(y), 2)) + r',\ \sigma=' + str(round(sstdev(y), 3)) + r'$' )
def avg_trend(filename):
avgs = []
orig_data = read_data(filename)
for i in range(1, len(orig_data)+1):
avgs.append(util.mean(calc_avg_score(orig_data, i)))
plt.cla()
plt.plot(avgs, '-*')
plt.title('avg trend with years of source data')
save_name = os.path.join(BASE_DIR, 'figures/avg_trend_by_years.png')
plt.savefig(save_name, dpi=96)
def markerReliabilityCheck(self,
markerCount,
alphaWeight=1,
betaWeight=5,
lengthWeight=2):
markers = {}
for i in range(markerCount):
ms = super(Tobo, self).see_cube()
for m in ms:
mCode = str(m.info.code)
if mCode in markers:
markers[mCode]["marker"].append(m)
markers[mCode]["alpha"].append(m.centre.rot_y)
markers[mCode]["beta"].append(m.orientation.rot_y)
markers[mCode]["length"].append(m.centre.dist)
else:
markers.update({
mCode: {
"marker": [m],
"alpha": [m.centre.rot_y],
"beta": [m.orientation.rot_y],
"length": [m.centre.dist]
}
})
for key in list(markers):
mDict = markers[key]
sqrtN = math.sqrt(len(mDict["marker"]))
alphaXBar = util.mean(mDict["alpha"])
alphaSd = util.stdev(mDict["alpha"]) / sqrtN
betaXBar = util.mean(mDict["beta"])
betaSd = util.stdev(mDict["beta"]) / sqrtN
lengthXBar = util.mean(mDict["length"])
lengthSd = util.stdev(mDict["length"]) / sqrtN
error = (alphaWeight * alphaSd + betaWeight * betaSd +
lengthWeight * lengthSd) / (alphaWeight + betaWeight +
lengthWeight)
for m in mDict["marker"]:
m.error = error
def interpolate_null(geneset_size_enrichments, size_skip, go_size):
# find range
size_less = go_size
size_more = go_size
for d in range(size_skip+1):
if not size_less in geneset_size_enrichments:
size_less -= 1
if not size_more in geneset_size_enrichments:
size_more += 1
# compute interpolation weights
max_dist = 1+max(go_size-size_less, size_more-go_size)
w_less = float(max_dist - (go_size-size_less))
w_more = float(max_dist - (size_more-go_size))
# compute mean, sd
mean = (w_less*util.mean(geneset_size_enrichments[size_less]) + w_more*util.mean(geneset_size_enrichments[size_more])) / (w_less+w_more)
sd = (w_less*util.sd(geneset_size_enrichments[size_less]) + w_more*util.sd(geneset_size_enrichments[size_more])) / (w_less+w_more)
return mean, sd
def train(self, training_dataset):
"""
Calculates the mean and stdev of each feature in each class
:param training_dataset: A n*m matrix of n features and m training items
:return: a dictionary containing lists of the expected value and standard deviation for each feature in each class
"""
feature_summary = {"class_1": [], "class_2": []}
for cls, feature_matrix in training_dataset.iteritems():
for feature in feature_matrix:
feature_summary[cls].append((util.mean(feature), util.stdev(feature)))
self.training_params = feature_summary
return feature_summary
def _get_mean_concreteness(sample, pos_tag):
"""
Return the mean concreteness for the list of tokens.
Inputs:
pos_tag: str. the pos tag you want to compute conrcreteness for.
Returns:
mean_concr: float. the mean concreteness score. is None if there are no tokens
with concreteness scores (after filtering by pos_tag and punctuation).
"""
# Get spacy encoded story
spacy_encoded_story = sample.get_spacy_annotated_story()
# Get all story tokens with given pos_tag
toks = [t for t in util.flatten(spacy_encoded_story)
if t.pos_ == pos_tag] # list of spacy Tokens
# Convert to lemmas, lowercased, excluding any containing punctuation
lemmas = [t.lemma_.lower() for t in toks] # list of strings
lemmas = [l for l in lemmas if not util.contains_punc(l)]
# If there are no remaining lemmas, return None
if len(lemmas) == 0:
return None
# Init word2conc if necessary
if word2conc is None:
_init_conc()
# Get concreteness ratings
concr_ratings = [(t, word2conc[t]) for t in lemmas
if t in word2conc] # list of (string, float) pairs
# Calculate coverage (i.e. how many of the lemmas we had concreteness ratings for)
concr_cov = len(concr_ratings) / len(lemmas)
# Calculate mean
if len(concr_ratings) == 0:
mean_concr = None
else:
mean_concr = util.mean([score for (token, score) in concr_ratings])
# Cache the list of lemmas, the individual ratings, and the overall coverage
sample.cache['concreteness_stats_{}'.format(pos_tag)] = {
'{}_lemmas'.format(pos_tag): lemmas,
'concreteness_ratings': concr_ratings,
'concreteness_coverage': concr_cov,
}
# Return the mean concreteness
return mean_concr
def interpolate_null(geneset_size_enrichments, size_skip, go_size):
# find range
size_less = go_size
size_more = go_size
for d in range(size_skip + 1):
if not size_less in geneset_size_enrichments:
size_less -= 1
if not size_more in geneset_size_enrichments:
size_more += 1
# compute interpolation weights
max_dist = 1 + max(go_size - size_less, size_more - go_size)
w_less = float(max_dist - (go_size - size_less))
w_more = float(max_dist - (size_more - go_size))
# compute mean, sd
mean = (w_less * util.mean(geneset_size_enrichments[size_less]) + w_more *
util.mean(geneset_size_enrichments[size_more])) / (w_less + w_more)
sd = (w_less * util.sd(geneset_size_enrichments[size_less]) + w_more *
util.sd(geneset_size_enrichments[size_more])) / (w_less + w_more)
return mean, sd
def amplitude_mean(N=100):
amps = lambda: util.images().pad(100, 100).map(transform.amplitude)
logger.info("Calculating mean amplitude")
mean_a = util.mean(amps(), np.zeros((100, 100)))
# distance
d = np.linalg.norm
dist = (d(amp - mean_a) for amp in util.progress(amps()))
logger.info(f"Finding {N} farthest away from the mean")
oddities = nlargest(N, zip(dist, util.image_ids()))
logger.info(f"Finished amplitude mean outliers")
return oddities
def centered_mean(N=100):
centered = lambda: util.images().pad(100, 100).map(transform.center)
logger.info("Calculating mean centered image")
mean_c = util.mean(centered(), np.zeros((100, 100)))
# distance
d = np.linalg.norm
dist = (d(img - mean_c) for img in util.progress(centered()))
logger.info(f"Finding {N} farthest away from the mean")
oddities = nlargest(N, zip(dist, util.image_ids()))
logger.info(f"Finished centered mean outliers")
return oddities
def histogram_mean(N=100):
hists = lambda: util.images().map(transform.smoothen).map(transform.grayscale)
logger.info("Calculating mean histogram")
mean_h = util.mean(hists(), np.zeros((256)))
# distance
d = np.linalg.norm
d = util.max_correlation1d
dist = (d(hist, mean_h) for hist in util.progress(hists()))
logger.info(f"Finding {N} farthest away from the mean")
oddities = nlargest(N, zip(dist, util.image_ids()))
logger.info(f"Finished histogram mean outliers")
return oddities
def item_filter(per_item_preds, baseline_val):
'''
Decides whether an item should be considered for inclusion in the
recommendations. This version requires that three conditions be met:
1. The item must have a reasonably high predicted rating (per_item_mean > 3.)
2. The predictions must indicate that the user will rate the item higher than
its baseline rating (lift > .2)
3. The predictions must not be too uncertain regarding the positive
lift (conf > .75)
'''
N = len(per_item_preds)
mn = mean(per_item_preds)
conf = float(sum([1 for p in per_item_preds if p - baseline_val > 0.])) / N
lift = mn - baseline_val
return mn > 3. and lift > .2 and conf > .75
def mean(N=100):
images = lambda: util.images().pad(100, 100)
logger.info("Calculating mean image")
mean = util.mean(images())
# max-correlation of mean
d = util.max_correlation
d = np.linalg.norm
maxcorr = (d(img - mean) for img in util.progress(images()))
logger.info(f"Finding {N} with least correlation with the mean")
oddities = nlargest(N, zip(maxcorr, util.image_ids()))
logger.info(f"Finished maxcorr outliers")
return oddities
def main():
usage = 'usage: %prog [options] <gtf file> <bam file>'
parser = OptionParser(usage)
parser.add_option('-i', dest='intersect_done', default=False, action='store_true', help='intersectBed is already done [Default: %default]')
parser.add_option('-o', dest='output_prefix', help='Prefix for the intersectBed intermediate file [Default: %default]')
(options,args) = parser.parse_args()
if len(args) != 2:
parser.error('Must provide gtf file and bam file')
else:
gtf_file = args[0]
bam_file = args[1]
if options.output_prefix:
ib_file = '%s_reads_genes.gff' % options.output_prefix
else:
ib_file = 'reads_genes.gff'
if not options.intersect_done:
# overlap genes w/ aligned reads
p = subprocess.Popen('intersectBed -s -wo -abam -bed -a %s -b %s > %s' % (bam_file,gtf_file,ib_file), shell=True)
os.waitpid(p.pid,0)
# count transcriptome alignments per read
read_aligns = {}
for line in open(ib_file):
a = line.split('\t')
chrom = a[0]
start = int(a[1])
read_id = a[3]
read_aligns.setdefault(read_id,set()).add((chrom,start))
# hash reads by gene
gene_reads = {}
for line in open(ib_file):
a = line.split('\t')
read_id = a[3]
gene_id = gff.gtf_kv(a[14])['transcript_id']
gene_reads.setdefault(gene_id,[]).append(read_id)
# print gene stats
for gene_id in gene_reads:
align_counts = [len(read_aligns[read_id]) for read_id in gene_reads[gene_id]]
multi_count = float(len([ac for ac in align_counts if ac > 1]))
cols = (gene_id, len(align_counts), util.mean(align_counts), multi_count/float(len(align_counts)))
print '%-15s %7d %7.2f %7.2f' % cols
def rotated_mean(N=100):
images = lambda: util.images().pad(100, 100)
angles = lambda: util.images().pad(100, 100).map(transform.PCA_angle)
rotated = lambda: util.seq(transform.rotate(img, ang) for img, ang in zip(images(), angles()))
logger.info("Calculating mean rotated image")
mean_r = util.mean(rotated(), np.zeros((100, 100)))
# distance
d = np.linalg.norm
dist = (d(img - mean_r) for img in util.progress(rotated()))
logger.info(f"Finding {N} farthest away from the mean")
oddities = nlargest(N, zip(dist, util.image_ids()))
logger.info(f"Finished rotated mean outliers")
return oddities
def frequency_mean(N=100):
phases = lambda: util.images().pad(100, 100).map(transform.phase)
logger.info("Calculating mean phase")
mean_p = util.mean(phases(), np.zeros((100, 100)))
# mean_p = transform.shape(mean_p, shape="circle", mask_value=0, radius=25)
# distance
d = np.linalg.norm
d = util.max_correlation
dist = (-d(phase, mean_p) for phase in util.progress(phases()))
logger.info(f"Finding {N} farthest away from the mean")
oddities = nlargest(N, zip(dist, util.image_ids()))
logger.info(f"Finished phase mean outliers")
return oddities
def mean_log_unigramprob(sample):
"""
Returns the mean log unigram probability of the words in story_text.
Note that we measure word unigram probability case-blind.
"""
tokens = sample['story_text'].strip().lower().split(
) # lowercase the story text first
if word2unigramprob is None:
print(
"\nInitializing word2unigramprob for mean_log_unigramprob metric..."
)
_init_word2unigramprob()
unigram_probs = [
word2unigramprob[t] for t in tokens if t in word2unigramprob
]
# if len(unigram_probs) < len(tokens):
# print("WARNING: the following tokens are OOV for word2unigramprob: ", [t for t in tokens if t not in word2unigramprob])
log_unigram_probs = [np.log(p) for p in unigram_probs]
return util.mean(log_unigram_probs)
def train(m, p=None):
d = DataLoader(BSD3000(),
o.batch_size,
num_workers=o.num_workers,
shuffle=True)
optimizer = torch.optim.Adam(m.parameters(), lr=o.lr)
iter_num = len(d)
num = 0
losss = []
mse = torch.nn.MSELoss()
stage = 1 if not p else p.stage + 1
for epoch in range(o.epoch):
for i in tqdm(d):
g, y, k, s = [x.to(o.device) for x in i]
k = k.flip(1, 2)
x = y
if p:
with torch.no_grad():
x = p([x, y, k, s])
optimizer.zero_grad()
out = m([x, y, k, s])
log("out", out)
out = crop(out, k)
loss = npsnr(out, g)
loss.backward()
optimizer.step()
losss.append(loss.detach().item())
assert not isnan(losss[-1])
print("stage", stage, "epoch", epoch + 1)
log("loss", mean(losss[-5:]))
num += 1
if num > (o.epoch * iter_num - 4):
# if num % 6 == 1:
show(torch.cat((crop(y, k)[0, 0], g[0, 0], out[0, 0]), 1),
# save=f"save/{stage:02}{epoch:02}.png",
)
plt.clf()
plt.plot([i + 1 for i in range(len(losss))], losss)
plt.xlabel("batch")
plt.ylabel("loss")
plt.title(f"{iter_num} iter x {o.epoch} epoch")
plt.savefig(f"save/{stage:02}loss.png")
def train(m, p=None):
d = DataLoader(BSD3000(noise=False, edgetaper=False),
o.batch_size,
num_workers=o.num_workers)
optimizer = torch.optim.Adam(m.parameters(), lr=o.lr)
iter_num = len(d)
num = 0
losss = []
stage = 1 if not p else p.stage + 1
for epoch in range(o.epoch):
for i in tqdm(d):
g, y, k, s = [x.to(o.device) for x in i]
k = k.flip(1, 2)
x = torch.tensor(y, requires_grad=True)
if p:
with torch.no_grad():
x = p([x, y, k, s])
optimizer.zero_grad()
out = m([x, y, k, s])
log("out", out)
out = center_crop(out, *g.shape[-2:])
loss = npsnr(out, g)
loss.backward()
optimizer.step()
losss.append(loss.detach().item())
assert not isnan(losss[-1])
print("stage", stage, "epoch", epoch + 1)
log("loss", mean(losss[-5:]))
num += 1
# if num > (o.epoch * iter_num - 4):
if num % 20 == 0:
show(
torch.cat((center_crop(
y, *g.shape[-2:])[0, 0], g[0, 0], out[0, 0]), 1),
save=f"save/{stage:02}{epoch:02}.png",
)
plt.clf()
plt.plot(range(len(losss)), losss)
plt.xlabel("batch")
plt.ylabel("loss")
plt.title(f"{iter_num} iter x {o.epoch} epoch")
plt.savefig(f"save/{stage:02}loss.png")
def recommend():
# make predictions
query = request.json
preds = analysis.predict(query)
# compute the rating change from baseline for each item
item_scores = []
for m in query:
if query[m] == None:
per_item_preds = [float(p[m]) for p in preds.distribution]
if item_filter(per_item_preds, baselines[m]):
# print ITEM_NAMES[m], lift, per_item_mean, per_item_std
per_item_mean = mean(per_item_preds)
lift = per_item_mean - baselines[m]
item_scores.append((m, lift))
# sort by rating change and return those items with an increase
item_scores.sort(key=item_sorter, reverse=True)
result = [(ITEM_NAMES[r[0]], r[1]) for r in item_scores]
return json.dumps(result)
def calcFitness(population, topper_count=10):
'''
Bereken de fitness voor de hele populatie en selecteer een aantal toppers
'''
population_with_fitness = []
fitness_list = []
for solution in population:
fit = fitness(solution)
population_with_fitness.append((fit, solution))
fitness_list.append(fit)
average.append(mean(fitness_list))
minima.append(min(fitness_list))
# sort the population by reversed fitness and get the first `topper_count`
sorted_list = sorted(population_with_fitness, key=lambda x: x[0])
toppers = [f[1] for f in sorted_list[0:topper_count]]
return toppers, fitness_list
def find_info_gain(self, X, y, attr):
'''
Find the information gain for a given attribute
'''
# Find out if its a numerical or categorical
isCategorical = type(X[1][attr]) == str
if isCategorical:
value_set = set([x[attr] for x in X])
if len(value_set) == 1:
# only one value for entire list
return None
elif len(value_set) == 2:
attribute_values = [list(value_set)[0]]
else:
attribute_values = [x for x in value_set]
else:
# calculate mean, median of the values
value_set = set([x[attr] for x in X])
if len(value_set) == 1:
return None
values = [x for x in value_set]
# calculate info gain on median and mean
attribute_values = [mean(values), median(values)]
max_info_gain = 0
val = None
X_left = None
X_right = None
Y_left = None
Y_right = None
for each in attribute_values:
x_left, x_right, y_left, y_right = partition_classes(
X, y, attr, each)
info_gain = information_gain(y, [y_left, y_right])
if info_gain > max_info_gain:
max_info_gain = info_gain
val = each
X_left, X_right, Y_left, Y_right = x_left, x_right, y_left, y_right
return isCategorical, val, max_info_gain, X_left, X_right, Y_left, Y_right
if len(sys.argv[1:]) > 2:
samples = int(sys.argv[3])
else:
samples = 1000
if len(sys.argv[1:]) > 4:
control_file = sys.argv[5]
else:
control_file = False
raw_data, settings = parser.parse_analysis_results(data_file)
data = raw_data
log.info('Actual Top Parents Jaccard Mean (+- SD): %s (+- %s)', mean([r.top_parents_jaccard for r in data]), sstdev([r.top_parents_jaccard for r in data]))
log.info('Actual Top Parents Tversky Mean (+- SD): %s (+- %s)', mean([r.top_parents_tversky for r in data]), sstdev([r.top_parents_tversky for r in data]))
log.info('Actual Age Mean (+- SD): %s (+- %s)', mean([r.age for r in data]), sstdev([r.age for r in data]))
log.info('')
if control_file:
raw_control_data, control_settings = parser.parse_analysis_results(control_file)
assert settings == control_settings
control_data = raw_control_data
log.info('Control Top Parents Jaccard Mean (+- SD): %s (+- %s)', mean([r.top_parents_jaccard for r in control_data]), sstdev([r.top_parents_jaccard for r in control_data]))
log.info('Control Top Parents Tversky Mean (+- SD): %s (+- %s)', mean([r.top_parents_tversky for r in control_data]), sstdev([r.top_parents_tversky for r in control_data]))
log.info('Control Age Mean (+- SD): %s (+- %s)', mean([r.age for r in control_data]), sstdev([r.age for r in control_data]))
log.info('')
all_data = data + control_data
def mean(self):
"""returns the mean value"""
return util.mean(self.values)
def hunt_choices(self, cur_round, my_food, my_rep, m, reputations):
return hunt_with_threshold(reputations, util.mean(reputations))
def uploads(self, requests, peers, history):
"""
requests -- a list of the requests for this peer for this round
peers -- available info about all the peers
history -- history for all previous rounds
returns: list of Upload objects.
In each round, this will be called after requests().
"""
##################################################################
########### updating d_js ########################################
##################################################################
alpha = 0.2
gamma = 0.1
round = history.current_round()
self.bandwidthHistory.append(self.up_bw)
if round == 0:
for peer in peers:
self.downloadRate[peer.id] = 1
self.uploadRate[peer.id] = 1
self.slots[peer.id] = 4
self.downloadUploadRatio[peer.id] = self.downloadRate[peer.id]/self.uploadRate[peer.id]
else:
for peer in peers:
for download in history.downloads[round-1]:
if peer.id == download.from_id:
self.downloadRate[peer.id] = download.blocks
if download.blocks == 0: print "!!!!!! %s uploaded %s block(s)" % (peer.id, download.blocks)
self.slots[peer.id] = mean(self.bandwidthHistory)/float(self.downloadRate[peer.id]) # Find how to find out max and min bw or infer from personal history
if round >= 3:
peer_download = 0
for download2 in history.downloads[round-2]:
if peer.id == download2.from_id:
for download3 in history.downloads[round-3]:
if peer.id == download3.from_id:
peer_download += 1
if peer_download > 0:
self.uploadRate[peer.id] *= 1 - gamma
break
if len(peer.available_pieces) > 0:
av_pieces = float(len(peer.available_pieces))
rnd = float(round)
slots = float(self.slots[peer.id])
self.downloadRate[peer.id] = av_pieces/(rnd * slots)
#self.downloadRate[peer.id] = float((len(peer.available_pieces)/float(round)))/float(self.slots[peer.id])
self.uploadRate[peer.id] *= 1 + alpha
if self.downloadRate[peer.id] == 0:
print str(peer.id) + ": " + str(peer.available_pieces)
print "Peer %s has %s available pieces" % (peer.id, len(peer.available_pieces))
print "downloadUploadRatio"
print self.downloadUploadRatio
########### updating current ratio ###############################
if round == 0:
for peer in peers:
self.downloadUploadRatio[peer.id] = 1
else:
for peer in peers:
self.downloadUploadRatio[peer.id] = self.downloadRate[peer.id]/self.uploadRate[peer.id]
###################################################################
print "download rates"
print self.downloadRate
print "Upload rates"
print self.uploadRate
print "donwload upload ratio"
print self.downloadUploadRatio
print "slots"
print self.slots
logging.debug("%s again. It's round %d." % (
self.id, round))
# One could look at other stuff in the history too here.
# For example, history.downloads[round-1] (if round != 0, of course)
# has a list of Download objects for each Download to this peer in
# the previous round.
if len(requests) == 0:
logging.debug("No one wants my pieces!")
chosen = []
bws = []
uploads = []
else:
logging.debug("Still here: uploading to a random peer")
# change my internal state for no reason
self.dummy_state["cake"] = "pie"
########### Building upload list #################################
sumUpload = 0
chosen = {}
downloadUploadRatio_tmp = {}
# creating list with ratios for only peers in requests
for request in requests:
downloadUploadRatio_tmp[request.requester_id] = self.downloadUploadRatio[request.requester_id]
print self.downloadUploadRatio[request.requester_id]
while (sumUpload <= len(peers) and len(downloadUploadRatio_tmp) > 0):
peer_id = max(downloadUploadRatio_tmp, key = downloadUploadRatio_tmp.get)
chosen[peer_id] = downloadUploadRatio_tmp.pop(peer_id)
sumUpload += self.uploadRate[peer_id]
# print "sumUpload of %s" % (peer_id)
# print sumUpload
# print downloadUploadRatio_tmp[peer_id]
# print self.uploadRate[peer_id]
""" Calculate the total proportional BW allocated to other peers """
totalUploadBW = 0
for choice in chosen:
totalUploadBW += chosen[choice]
# print chosen[choice]
""" Make each BW as a proportion of totalUploadBW """
for choice in chosen:
chosen[choice] = 100 * float(chosen[choice]) / float(totalUploadBW)
# print "Vector of choices for this round:"
# print chosen
""" Now need to divide our BW as integers according to chosen vector """
peerWeights = [value for (key, value) in sorted(chosen.items())]
peerNames = sorted(chosen)
# print "original chosen: %s" % (chosen)
# print "names: %s" % (peerNames)
# print "weights: %s" % (peerWeights)
# request = random.choice(requests)
# chosen = [request.requester_id]
# Evenly "split" my upload bandwidth among the one chosen requester
# bws = even_split(self.up_bw, len(chosen))
bws = proportional_split(self.up_bw, peerWeights)
# create actual uploads out of the list of peer ids and bandwidths
uploads = [Upload(self.id, peer_id, bw)
for (peer_id, bw) in zip(chosen, bws)]
return uploads
def analysis(data, settings, flat_age_matrix=[], flat_sim_matrix=[], alpha=0.03):
# value_keys = ["complete_term_jaccard", "top_term_jaccard", "top_gene_jaccard", "top_parents_jaccard"]
value_keys = ["top_gene_jaccard", "top_parents_jaccard"]
data_values = [[row._asdict()[k] for k in value_keys] for row in data]
means = [mean([x[i] for x in data_values]) for i in range(len(value_keys))]
stds = [sstdev([x[i] for x in data_values]) for i in range(len(value_keys))]
medians = [median([x[i] for x in data_values]) for i in range(len(value_keys))]
log.info("N: %s", len(data_values))
genes_found, genes_missed = len([genes[1] for row in data for genes in row.genes_found]),len([genes for row in data for genes in row.genes_missed])
log.info("genes_found: %s, genes_missed: %s", genes_found, genes_missed)
total = genes_found + genes_missed
log.info("genes_found: %s, genes_missed: %s", round(genes_found / total, 2), round(genes_missed / total, 2))
#log.info("distinct unknown genes: %s", len(unknown))
fig_num = 0
for i, value_key in enumerate(value_keys):
log.info("%s: mean=%s, std=%s, median=%s", value_key, means[i], stds[i], medians[i])
f = plt.figure(fig_num)
fig_num +=1
d = [x[i] for x in data_values]
weights = np.ones_like(d)/float(len(d))
plt.hist(d, 100, weights=weights, alpha=0.5, label='Actual')
weights = np.ones_like(flat_sim_matrix)/float(len(flat_sim_matrix))
plt.hist(flat_sim_matrix, 100, weights=weights, alpha=0.5, label='Randomized')
plt.xlabel('Similarity')
plt.ylabel(value_keys[i])
plt.title('Histogram of ' + value_key + " | " + r'$\mu=' + str(round(means[i], 2)) + r',\ \sigma=' + str(round(stds[i], 2)) + r'$')
plt.legend(loc='upper right')
f.show()
f = plt.figure(fig_num)
fig_num +=1
x = [r.age for r in data]
y = [r._asdict()[value_key] for r in data]
fit = np.polyfit(x,y,1)
fit_fn = np.poly1d(fit)
plt.plot(x,y, 'k.', [min(x), max(x)], fit_fn([min(x), max(x)]), '--g')
plt.xlabel('Age')
plt.ylabel("Similarity Index")
plt.title("Ancestors of " + value_key + " vs Age" + " | " + r'$\mu=' + str(round(means[i], 2)) + r',\ \sigma=' + str(round(stds[i], 2)) + r'$' )
f.show()
f = plt.figure(fig_num)
fig_num +=1
x = flat_age_matrix
y = flat_sim_matrix
fit = np.polyfit(x,y,1)
fit_fn = np.poly1d(fit)
plt.plot(x,y, 'k.', [min(x), max(x)], fit_fn([min(x), max(x)]), '--g', alpha=alpha)
plt.xlabel('Age')
plt.ylabel("Similarity Index")
plt.title("Randomized Ancestors of " + value_key + " vs Age" + " | " + r'$\mu=' + str(round(means[i], 2)) + r',\ \sigma=' + str(round(stds[i], 2)) + r'$' )
f.show()
f = plt.figure(fig_num)
fig_num +=1
d = [x[4] for x in data]
weights = np.ones_like(d)/float(len(d))
plt.hist(d, 100, weights=weights, alpha=0.5, label='Actual')
weights = np.ones_like(flat_age_matrix)/float(len(flat_age_matrix))
plt.hist(flat_age_matrix, 100, weights=weights, alpha=0.5, label='Randomized')
plt.xlabel('Age')
f.show()
raw_input()
plt.close("all")
from util import mean, deviation, cv, pearson, SIZES, timing_plot
# Задаём диапазон и объём каждой выборки и количество выборок
RANGE, SIZE, COUNT = 10000, 100, 10
# Вычисляем теоретические значения характеристик
theoretical_mean = RANGE / 2
theoretical_deviation = (((RANGE + 1)**2 - 1) / 12)**(1 / 2)
theoretical_cv = theoretical_deviation / theoretical_mean
# Генерируем выборки
linear_samples = [linear(RANGE, SIZE, r) for r in range(1, COUNT + 1)]
tausworth_samples = [tausworth(RANGE, SIZE) for _ in range(COUNT)]
# Вычисляем выборочные средние
linear_means = [mean(sample) for sample in linear_samples]
tausworth_means = [mean(sample) for sample in tausworth_samples]
# Вычисляем выборочные стандартные отклонения
linear_deviations = [deviation(sample) for sample in linear_samples]
tausworth_deviations = [deviation(sample) for sample in tausworth_samples]
# Вычисляем выборочные коэффициенты вариации
linear_cvs = [cv(sample) for sample in linear_samples]
tausworth_cvs = [cv(sample) for sample in tausworth_samples]
print('\nТеоретические значения характеристик:')
print(' Математическое ожидание:', colored('%.2f' % theoretical_mean,
'yellow'))
print(' Стандартное отклонение:',
colored('%.2f' % theoretical_deviation, 'yellow'))
def test_mean(self):
result = util.mean(alist)
self.assertEquals(result, 3.0)
def main(args):
usage_msg = "Usage: %prog [options] PeerClass1[,cnt] PeerClass2[,cnt2] ..."
parser = OptionParser(usage=usage_msg)
def usage(msg):
print "Error: %s\n" % msg
parser.print_help()
sys.exit()
parser.add_option("--loglevel",
dest="loglevel", default="info",
help="Set the logging level: 'debug' or 'info'")
parser.add_option("--mech",
dest="mechanism", default="gsp",
help="Set the mechanim: 'gsp' or 'vcg' or 'switch'")
parser.add_option("--num-rounds",
dest="num_rounds", default=48, type="int",
help="Set number of rounds")
parser.add_option("--min-val",
dest="min_val", default=25, type="int",
help="Min per-click value, in cents")
parser.add_option("--max-val",
dest="max_val", default=175, type="int",
help="Max per-click value, in cents")
parser.add_option("--budget",
dest="budget", default=500000, type="int",
help="Total budget, in cents")
parser.add_option("--reserve",
dest="reserve", default=0, type="int",
help="Reserve price, in cents")
parser.add_option("--perms",
dest="max_perms", default=120, type="int",
help="Max number of value permutations to run. Set to 1 for debugging.")
parser.add_option("--iters",
dest="iters", default=1, type="int",
help="Number of different value draws to sample. Set to 1 for debugging.")
parser.add_option("--seed",
dest="seed", default=None, type="int",
help="seed for random numbers")
(options, args) = parser.parse_args()
# leftover args are class names:
# e.g. "Truthful BBAgent CleverBidder Fred"
if len(args) == 0:
# default
agents_to_run = ['Truthful', 'Truthful', 'Truthful']
else:
agents_to_run = parse_agents(args)
configure_logging(options.loglevel)
if options.seed != None:
random.seed(options.seed)
# Add some more config options
options.agent_class_names = agents_to_run
options.agent_classes = load_modules(options.agent_class_names)
options.dropoff = 0.75
logging.info("Starting simulation...")
n = len(agents_to_run)
totals = dict((id, 0) for id in range(n))
total_revenues = []
approx = math.factorial(n) > options.max_perms
if approx:
num_perms = options.max_perms
logging.warning(
"Running approximation: taking %d samples of value permutations"
% options.max_perms)
else:
num_perms = math.factorial(n)
av_value=range(0,n)
total_spent = [0 for i in range(n)]
## iters = no. of samples to take
for i in range(options.iters):
values = get_utils(n, options)
logging.info("==== Iteration %d / %d. Values %s ====" % (i, options.iters, values))
## Create permutations (permutes the random values, and assigns them to agents)
if approx:
perms = [shuffled(values) for i in range(options.max_perms)]
else:
perms = itertools.permutations(values)
total_rev = 0
## Iterate over permutations
for vals in perms:
options.agent_values = list(vals)
values = dict(zip(range(n), list(vals)))
## Runs simulation ###
history = sim(options)
### simulation ends.
stats = Stats(history, values)
# Print stats in console?
# logging.info(stats)
for id in range(n):
totals[id] += stats.total_utility(id)
total_spent[id] += history.agents_spent[id]
total_rev += stats.total_revenue()
total_revenues.append(total_rev / float(num_perms))
## total_spent = total amount of money spent by agents, for all iterations, all permutations, all rounds
# Averages are over all the value permutations considered
N = float(num_perms) * options.iters
logging.info("%s\t\t%s\t\t%s" % ("#" * 15, "RESULTS", "#" * 15))
logging.info("")
for a in range(n):
logging.info("Stats for Agent %d, %s" % (a, agents_to_run[a]) )
logging.info("Average spend $%.2f (daily)" % (0.01 *total_spent[a]/N) )
logging.info("Average utility $%.2f (daily)" % (0.01 * totals[a]/N))
logging.info("-" * 40)
logging.info("\n")
m = mean(total_revenues)
std = stddev(total_revenues)
logging.warning("Average daily revenue (stddev): $%.2f ($%.2f)" % (0.01 * m, 0.01*std))
def pred_group_prob(self, group, xdata):
x = [xdata[i] for i in group.index]
probs = self.pred_prob(x)
return mean(probs)
def _mean(data, threshold=0, default=0):
return mean((e for e in data if e > threshold), default)
stats['unique_genes_missed'] = len(set(genes_missed))
# exit()
# age, sim, studies = create_random_null_similarity(data, settings, True)
# analysis(data, settings, age, sim)
#
# log.info("Perfectly Matching Studies in Randomized Data: %s", len([studies[i] for i,a in enumerate(age) if a ==0 and sim[i]==1]))
# log.info("Perfectly Matching Studies in Randomized Data With same PMID: %s", len([studies[i] for i,a in enumerate(age) if a ==0 and sim[i]==1 and data[studies[i][0]].pmid == data[studies[i][1]].pmid]))
# f = plt.figure()
# scatter_plot(age, sim, alpha=0.002)
# f.show()
#
# raw_input()
# exit()
actual_m = mean([r.top_parents_jaccard for r in data])
actual_sd = sstdev([r.top_parents_jaccard for r in data])
actual_m_tversky = mean([r.top_parents_tversky for r in data])
actual_sd_tversky = sstdev([r.top_parents_tversky for r in data])
control_m = mean([r.top_parents_jaccard for r in control_data])
control_sd = sstdev([r.top_parents_jaccard for r in control_data])
control_m_tversky = mean([r.top_parents_tversky for r in control_data])
control_sd_tversky = sstdev([r.top_parents_tversky for r in control_data])
# Add actual data to permutation test
# av_sims.append(actual_m)
# sd_sims.append(actual_sd)
# all_sims += [r.top_parents_jaccard for r in data]
# size_sims += [len(r.top_parents_current) for r in data]
def test_mean_with_nans(self):
"""tests the mean() function"""
array = np.array([2.0, 3.0, np.nan, 1.0])
result = util.mean(array)
self.assertAlmostEqual(2.0, result)
def main(args):
usage_msg = "Usage: %prog [options] PeerClass1[,cnt] PeerClass2[,cnt2] ..."
parser = OptionParser(usage=usage_msg)
def usage(msg):
print "Error: %s\n" % msg
parser.print_help()
sys.exit()
parser.add_option("--loglevel",
dest="loglevel", default="info",
help="Set the logging level: 'debug' or 'info'")
parser.add_option("--mech",
dest="mechanism", default="gsp",
help="Set the mechanim: 'gsp' or 'vcg' or 'switch'")
parser.add_option("--num-rounds",
dest="num_rounds", default=48, type="int",
help="Set number of rounds")
parser.add_option("--min-val",
dest="min_val", default=25, type="int",
help="Min per-click value, in cents")
parser.add_option("--max-val",
dest="max_val", default=175, type="int",
help="Max per-click value, in cents")
parser.add_option("--budget",
dest="budget", default=500000, type="int",
help="Total budget, in cents")
parser.add_option("--reserve",
dest="reserve", default=0, type="int",
help="Reserve price, in cents")
parser.add_option("--perms",
dest="max_perms", default=120, type="int",
help="Max number of value permutations to run. Set to 1 for debugging.")
parser.add_option("--iters",
dest="iters", default=1, type="int",
help="Number of different value draws to sample. Set to 1 for debugging.")
parser.add_option("--seed",
dest="seed", default=None, type="int",
help="seed for random numbers")
(options, args) = parser.parse_args()
# leftover args are class names:
# e.g. "Truthful BBAgent CleverBidder Fred"
if len(args) == 0:
# default
agents_to_run = ['Truthful', 'Truthful', 'Truthful']
else:
agents_to_run = parse_agents(args)
configure_logging(options.loglevel)
if options.seed != None:
random.seed(options.seed)
# Add some more config options
options.agent_class_names = agents_to_run
options.agent_classes = load_modules(options.agent_class_names)
options.dropoff = 0.75
logging.info("Starting simulation...")
n = len(agents_to_run)
totals = dict((id, 0) for id in range(n))
total_revenues = []
approx = math.factorial(n) > options.max_perms
if approx:
num_perms = options.max_perms
logging.warning(
"Running approximation: taking %d samples of value permutations"
% options.max_perms)
else:
num_perms = math.factorial(n)
av_value=range(0,n)
total_spent = [0 for i in range(n)]
""" Additional statistics to follow """
agents_spent_stats = [] ### For spending statistics information
agents_util_stats = [] ### For utility statistics
for i in range(n):
agents_spent_stats.append([])
agents_util_stats.append([])
""" ------------------------------- """
## iters = no. of samples to take
for i in range(options.iters):
values = get_utils(n, options)
logging.info("==== Iteration %d / %d. Values %s ====" % (i, options.iters, values))
## Create permutations (permutes the random values, and assigns them to agents)
if approx:
perms = [shuffled(values) for i in range(options.max_perms)]
else:
perms = itertools.permutations(values)
total_rev = 0
## Iterate over permutations
for vals in perms:
options.agent_values = list(vals)
values = dict(zip(range(n), list(vals)))
## Runs simulation ###
history = sim(options)
### simulation ends.
stats = Stats(history, values)
# Print stats in console?
# logging.info(stats)
for id in range(n):
totals[id] += stats.total_utility(id)
total_spent[id] += history.agents_spent[id]
""" Additional stats ---------------- """
agents_spent_stats[id].append(history.agents_spent[id])
agents_util_stats[id].append(stats.total_utility(id))
""" --------------------------------- """
total_rev += stats.total_revenue()
total_revenues.append(total_rev / float(num_perms))
## total_spent = total amount of money spent by agents, for all iterations, all permutations, all rounds
# Averages are over all the value permutations considered
N = float(num_perms) * options.iters
print("Total permutations: %s") % num_perms
logging.info("%s\t\t%s\t\t%s" % ("#" * 15, "RESULTS", "#" * 15))
logging.info("")
for a in range(n):
logging.info("Stats for Agent %d, %s" % (a, agents_to_run[a]) )
logging.info("Average spend $%.2f (daily)" % (0.01 * total_spent[a]/N))
logging.info("Average utility $%.2f (daily)" % (0.01 * totals[a]/N))
logging.info("-" * 40)
logging.info("\n")
m = mean(total_revenues)
std = stddev(total_revenues)
logging.warning("Average daily revenue (stddev): $%.2f ($%.2f)" % (0.01 * m, 0.01*std))
logging.debug("agents_spent_stats:")
logging.debug(agents_spent_stats)
logging.debug("agents_util_stats:")
logging.debug(agents_util_stats)
""" Implementing output into csv """
filename = 'Stats/bb_only_reserve' + str(options.reserve) + '_seed' + str(options.seed) + '_iters' + str(options.iters) + '.csv'
with open(filename, 'wb') as csvfile:
spamwriter = csv.writer(csvfile, delimiter=';',
quotechar='|', quoting=csv.QUOTE_MINIMAL)
spamwriter.writerow(['Number of iterations'] + [options.iters]+['']+['']+['']+[''])
spamwriter.writerow(['Reserve'] + [options.reserve]+['']+['']+['']+[''])
spamwriter.writerow(['Seed'] + [options.seed]+['']+['']+['']+[''])
spamwriter.writerow(['Agent #'] +
['Agent type'] +
['Average spent'] +
['SD spent'] +
['Average Utility'] +
['SD Utility'])
for a in range(n):
spamwriter.writerow([a] +
[agents_to_run[a]] +
[0.01*sum(agents_spent_stats[a])/N] +
[0.01*stddev(agents_spent_stats[a])] +
[0.01*sum(agents_util_stats[a])/N] +
[0.01*stddev(agents_util_stats[a])/N])
spamwriter.writerow([n] +
['Seller'] +
[0.01 * m] + [0.01 * std] + [0] + [0])
#for t in range(47, 48):
#for a in agents:
#print a,"'s added values is", av_value[a.id]
print options
axes[i, j].plot(*line(angles[k]), color='k', lw=1.2)
with Figure(2, 5) as fig:
axes = get_axes(2, 5, fig)
arrow(axes[..., 2])
for col_index in [0, 1, 3, 4]:
clean(axes[..., col_index])
square(axes[..., col_index])
angles = lambda: (util.images().pad().map(transform.PCA_angle))
# block 1
img_mean = util.mean(util.images().pad(100, 100))
img_3 = [
util.pad(util.image(i), 100, 100) for i in ["1377", "1590", "9728"]
]
samples = [img_mean] + img_3
block_imshow(1, axes, samples)
# block 2
angle_mean = util.angle_mean(angles())
angle_3 = list(map(transform.PCA_angle, img_3))
transformed_samples = [angle_mean] + angle_3
block_circle(2, axes, transformed_samples)
def run(self):
if not self.quiet:
print 'starting with ACO with %d ants for %d iterations' % (
self.ant_count, self.iterations)
maze = self.maze
self.iteration_best_trail = []
# initialize ants
for k in range(self.ant_count):
self.ants.append(Ant(maze, maze.start))
global_best = iteration_best = None
for i in range(self.iterations):
if not self.quiet:
print '\nIteration: %d, Q: %d, max_steps: %d' % (
i, self.Q, self.ant_max_steps)
if self.multiprocessing:
# Make ants do their steps.
self.ants = self.pool.map_async(
ant_loop_apply,
itertools.izip(self.ants, [self.ant_max_steps] *
self.ant_count)).get(9999999)
else:
print 'Stepping...'
for ant in self.ants:
i = 0
while not ant.done and len(ant.trail) < self.ant_max_steps:
ant.step()
i += 1
if i % 1000 == 1:
self.visualizer.update('stepping: %d' % i)
if not ant.done:
print 'moving to next ant, this one stuck in', ant.position
done_ants = [a for a in self.ants if a is not None and a.done]
if not self.quiet:
print '%d out of %d ants finished within %d steps.' % (
len(done_ants), self.ant_count, self.ant_max_steps)
if self.optimize_ants:
# optimize the trails for these ants
opts = []
for ant in done_ants:
opts.append(ant.optimize_trail(quiet=self.quiet))
if not self.quiet:
print 'Optimisation reduced trail langth with an average of', mean(
opts)
# disable the dead ends found by the ant
if self.maze_elimination:
for ant in self.ants:
if ant is not None:
for p in ant.disable_positions:
self.maze.disable_at(p)
# select the best ant:
if len(done_ants) > 0:
iteration_best = min(done_ants)
# if global_best becomes invalid, forget it.
# if global_best is not None:
# global_best.maze = self.maze
# if not global_best.is_valid():
# global_best = None
# if not self.quiet:
# print 'Forgot global best!'
if global_best is None:
global_best = iteration_best.clone()
else:
global_best = min([iteration_best, global_best]).clone()
# update pheromone in the maze, for unique positions
deltas = np.zeros((self.maze.height, self.maze.width))
if global_best is not None:
deltas = self.delta_matrix(global_best)
if iteration_best is not None and global_best is not iteration_best:
deltas += self.delta_matrix(iteration_best)
# only update if iteration returned something.
if iteration_best is not None:
self.iteration_best_trail.append(len(iteration_best.trail))
else:
self.iteration_best_trail.append(None)
maze.update_tau(delta_tau=deltas, evaporation=self.evaporation)
# update ant_max_steps to the max value of this iteration
if len(done_ants) > 3:
if self.update_max_steps:
try:
self.ant_max_steps = min(
self.ant_max_steps,
max(
len(x.trail) for x in done_ants
if len(x.trail) < self.ant_max_steps))
except:
pass
if self.update_Q:
self.Q = min(min(len(x.trail) for x in self.ants), self.Q)
if not self.quiet:
if iteration_best is not None and global_best is not None:
print 'Best ant: %d, iteration best: %d' % (len(
global_best.trail), len(iteration_best.trail))
else:
print 'None of the ants finished stepping'
# reset ants
for ant in self.ants:
ant.reset(maze)
if self.visualize:
self.visualizer.update('Pheromone level iteration %d' % i)
self.visualizer.save('%dth_iteration.png' % i)
if self.multiprocessing:
self.interrupt()
self.global_best = global_best
return global_best
def main(args):
usage_msg = "Usage: %prog [options] PeerClass1[,cnt] PeerClass2[,cnt2] ..."
parser = OptionParser(usage=usage_msg)
def usage(msg):
print "Error: %s\n" % msg
parser.print_help()
sys.exit()
parser.add_option("--loglevel",
dest="loglevel",
default="info",
help="Set the logging level: 'debug' or 'info'")
parser.add_option("--mech",
dest="mechanism",
default="gsp",
help="Set the mechanim: 'gsp' or 'vcg' or 'switch'")
parser.add_option("--num-rounds",
dest="num_rounds",
default=48,
type="int",
help="Set number of rounds")
parser.add_option("--min-val",
dest="min_val",
default=25,
type="int",
help="Min per-click value, in cents")
parser.add_option("--max-val",
dest="max_val",
default=175,
type="int",
help="Max per-click value, in cents")
parser.add_option("--budget",
dest="budget",
default=500000,
type="int",
help="Total budget, in cents")
parser.add_option("--reserve",
dest="reserve",
default=0,
type="int",
help="Reserve price, in cents")
parser.add_option(
"--perms",
dest="max_perms",
default=120,
type="int",
help="Max number of value permutations to run. Set to 1 for debugging."
)
parser.add_option(
"--iters",
dest="iters",
default=1,
type="int",
help=
"Number of different value draws to sample. Set to 1 for debugging.")
parser.add_option("--seed",
dest="seed",
default=None,
type="int",
help="seed for random numbers")
(options, args) = parser.parse_args()
# leftover args are class names:
# e.g. "Truthful BBAgent CleverBidder Fred"
if len(args) == 0:
# default
agents_to_run = ['Truthful', 'Truthful', 'Truthful']
else:
agents_to_run = parse_agents(args)
configure_logging(options.loglevel)
if options.seed != None:
random.seed(options.seed)
# Add some more config options
options.agent_class_names = agents_to_run
options.agent_classes = load_modules(options.agent_class_names)
options.dropoff = 0.75
logging.info("Starting simulation...")
n = len(agents_to_run)
totals = dict((id, 0) for id in range(n))
total_revenues = []
approx = math.factorial(n) > options.max_perms
if approx:
num_perms = options.max_perms
logging.warning(
"Running approximation: taking %d samples of value permutations" %
options.max_perms)
else:
num_perms = math.factorial(n)
av_value = range(0, n)
total_spent = [0 for i in range(n)]
## iters = no. of samples to take
for i in range(options.iters):
values = get_utils(n, options)
logging.info("==== Iteration %d / %d. Values %s ====" %
(i, options.iters, values))
## Create permutations (permutes the random values, and assigns them to agents)
if approx:
perms = [shuffled(values) for i in range(options.max_perms)]
else:
perms = itertools.permutations(values)
total_rev = 0
## Iterate over permutations
for vals in perms:
options.agent_values = list(vals)
values = dict(zip(range(n), list(vals)))
## Runs simulation ###
history = sim(options)
### simulation ends.
stats = Stats(history, values)
# Print stats in console?
# logging.info(stats)
for id in range(n):
totals[id] += stats.total_utility(id)
total_spent[id] += history.agents_spent[id]
total_rev += stats.total_revenue()
total_revenues.append(total_rev / float(num_perms))
## total_spent = total amount of money spent by agents, for all iterations, all permutations, all rounds
# Averages are over all the value permutations considered
N = float(num_perms) * options.iters
logging.info("%s\t\t%s\t\t%s" % ("#" * 15, "RESULTS", "#" * 15))
logging.info("")
for a in range(n):
logging.info("Stats for Agent %d, %s" % (a, agents_to_run[a]))
logging.info("Average spend $%.2f (daily)" %
(0.01 * total_spent[a] / N))
logging.info("Average utility $%.2f (daily)" %
(0.01 * totals[a] / N))
logging.info("-" * 40)
logging.info("\n")
m = mean(total_revenues)
std = stddev(total_revenues)
logging.warning("Average daily revenue (stddev): $%.2f ($%.2f)" %
(0.01 * m, 0.01 * std))
###############
# ERASE LATER #
###############
return (0.01 * m, 0.01 * std, list(map(lambda x: 0.01 * x / N,
total_spent)),
list(map(lambda x: 0.01 * totals[x] / N, totals.keys())))
def computeConfidence(meanList, sdList):
return util.mean([sd/val for val,sd in zip(meanList,sdList)])
def _ratio(*data):
_len = [len(e) for e in data]
return mean(_len) / max(_len)
def run(self):
if not self.quiet:
print 'starting with ACO with %d ants for %d iterations' % (
self.ant_count, self.iterations
)
maze = self.maze
self.iteration_best_trail = []
# initialize ants
for k in range(self.ant_count):
self.ants.append(Ant(maze, maze.start))
global_best = iteration_best = None
for i in range(self.iterations):
if not self.quiet:
print '\nIteration: %d, Q: %d, max_steps: %d' % (i, self.Q, self.ant_max_steps)
if self.multiprocessing:
# Make ants do their steps.
self.ants = self.pool.map_async(
ant_loop_apply, itertools.izip(self.ants, [self.ant_max_steps] * self.ant_count)
).get(9999999)
else:
print 'Stepping...'
for ant in self.ants:
i = 0
while not ant.done and len(ant.trail) < self.ant_max_steps:
ant.step()
i += 1
if i % 1000 == 1:
self.visualizer.update('stepping: %d' % i)
if not ant.done:
print 'moving to next ant, this one stuck in', ant.position
done_ants = [a for a in self.ants if a is not None and a.done]
if not self.quiet:
print '%d out of %d ants finished within %d steps.' % (
len(done_ants),
self.ant_count,
self.ant_max_steps
)
if self.optimize_ants:
# optimize the trails for these ants
opts = []
for ant in done_ants:
opts.append(ant.optimize_trail(quiet=self.quiet))
if not self.quiet:
print 'Optimisation reduced trail langth with an average of', mean(opts)
# disable the dead ends found by the ant
if self.maze_elimination:
for ant in self.ants:
if ant is not None:
for p in ant.disable_positions:
self.maze.disable_at(p)
# select the best ant:
if len(done_ants) > 0:
iteration_best = min(done_ants)
# if global_best becomes invalid, forget it.
# if global_best is not None:
# global_best.maze = self.maze
# if not global_best.is_valid():
# global_best = None
# if not self.quiet:
# print 'Forgot global best!'
if global_best is None:
global_best = iteration_best.clone()
else:
global_best = min([iteration_best, global_best]).clone()
# update pheromone in the maze, for unique positions
deltas = np.zeros((self.maze.height, self.maze.width))
if global_best is not None:
deltas = self.delta_matrix(global_best)
if iteration_best is not None and global_best is not iteration_best:
deltas += self.delta_matrix(iteration_best)
# only update if iteration returned something.
if iteration_best is not None:
self.iteration_best_trail.append(len(iteration_best.trail))
else:
self.iteration_best_trail.append(None)
maze.update_tau(delta_tau=deltas, evaporation=self.evaporation)
# update ant_max_steps to the max value of this iteration
if len(done_ants) > 3:
if self.update_max_steps:
try:
self.ant_max_steps = min(
self.ant_max_steps,
max(len(x.trail) for x in done_ants if len(x.trail) < self.ant_max_steps)
)
except:
pass
if self.update_Q:
self.Q = min(min(len(x.trail) for x in self.ants), self.Q)
if not self.quiet:
if iteration_best is not None and global_best is not None:
print 'Best ant: %d, iteration best: %d' % (
len(global_best.trail),
len(iteration_best.trail)
)
else:
print 'None of the ants finished stepping'
# reset ants
for ant in self.ants:
ant.reset(maze)
if self.visualize:
self.visualizer.update('Pheromone level iteration %d' % i)
self.visualizer.save('%dth_iteration.png' % i)
if self.multiprocessing:
self.interrupt()
self.global_best = global_best
return global_best
def uploads(self, requests, peers, history):
##################################################################
########### updating d_js ########################################
##################################################################
alpha = 0.2
gamma = 0.1
round = history.current_round()
self.bandwidthHistory.append(self.up_bw)
if round == 0:
bw_list = even_split(self.up_bw,len(peers))
for peer,i in zip(peers,range(len(peers))):
self.downloadRate[peer.id] = (self.conf.max_up_bw - self.conf.min_up_bw)/(4*2)
if bw_list[i] == 0:
self.uploadRate[peer.id] = 0.5
else:
self.uploadRate[peer.id] = bw_list[i]
self.slots[peer.id] = 4
self.downloadUploadRatio[peer.id] = 1
else:
for peer in peers:
self.downloadRate[peer.id] = 0
for download in history.downloads[round-1]:
if peer.id == download.from_id:
self.downloadRate[peer.id] += download.blocks
if download.blocks == 0: print "!!!!!! %s uploaded %s block(s)" % (peer.id, download.blocks)
self.slots[peer.id] = mean(self.bandwidthHistory)/float(self.downloadRate[peer.id]) # Find how to find out max and min bw or infer from personal history
if round >= 3:
peer_download = 0
for download2 in history.downloads[round-2]:
if peer.id == download2.from_id:
for download3 in history.downloads[round-3]:
if peer.id == download3.from_id:
peer_download += 1
if peer_download > 0:
self.uploadRate[peer.id] *= 1 - gamma
break
if len(peer.available_pieces) > 0:
av_pieces = float(len(peer.available_pieces))
rnd = float(round)
slots = float(self.slots[peer.id])
self.downloadRate[peer.id] = av_pieces/(rnd * self.conf.blocks_per_piece * slots)
self.uploadRate[peer.id] *= 1 + alpha
self.downloadUploadRatio[peer.id] = self.downloadRate[peer.id]/self.uploadRate[peer.id]
if len(requests) == 0:
logging.debug("No one wants my pieces!")
chosen = []
bws = []
uploads = []
else:
logging.debug("Still here: uploading to a random peer")
# change my internal state for no reason
self.dummy_state["cake"] = "pie"
##################################################################
########### Building upload list #################################
##################################################################
sumUpload = 0
chosen = {}
downloadUploadRatio_tmp = {}
# creating list with ratios for only peers in requests
for request in requests:
downloadUploadRatio_tmp[request.requester_id] = self.downloadUploadRatio[request.requester_id]
while (sumUpload <= self.up_bw and len(downloadUploadRatio_tmp) > 0):
peer_id = max(downloadUploadRatio_tmp, key = downloadUploadRatio_tmp.get)
chosen[peer_id] = downloadUploadRatio_tmp.pop(peer_id)
sumUpload += self.uploadRate[peer_id]
""" Calculate the total proportional BW allocated to other peers """
totalUploadBW = 0
for choice in chosen:
totalUploadBW += chosen[choice]
""" Make each BW as a proportion of totalUploadBW """
if float(totalUploadBW) == 0:
uploads = []
else:
for choice in chosen:
chosen[choice] = 100 * float(chosen[choice]) / float(totalUploadBW)
""" Now need to divide our BW as integers according to chosen vector """
peerWeights = [value for (key, value) in sorted(chosen.items())]
peerNames = sorted(chosen)
bws = proportional_split(self.up_bw, peerWeights)
# create actual uploads out of the list of peer ids and bandwidths
uploads = [Upload(self.id, peer_id, bw)
for (peer_id, bw) in zip(chosen, bws)]
return uploads
def uploads(self, requests, peers, history):
"""
requests -- a list of the requests for this peer for this round
peers -- available info about all the peers
history -- history for all previous rounds
returns: list of Upload objects.
In each round, this will be called after requests().
"""
##################################################################
########### updating d_js ########################################
##################################################################
alpha = 0.2
gamma = 0.1
round = history.current_round()
self.bandwidthHistory.append(self.up_bw)
if round == 0:
bw_list = even_split(self.up_bw,len(peers))
for peer,i in zip(peers,range(len(peers))):
self.downloadRate[peer.id] = 1
if bw_list[i] == 0:
self.uploadRate[peer.id] = 0.5
else:
self.uploadRate[peer.id] = bw_list[i]
self.slots[peer.id] = 4
self.downloadUploadRatio[peer.id] = 1
else:
for peer in peers:
for download in history.downloads[round-1]:
if peer.id == download.from_id:
self.downloadRate[peer.id] = download.blocks
if download.blocks == 0: print "!!!!!! %s uploaded %s block(s)" % (peer.id, download.blocks)
self.slots[peer.id] = mean(self.bandwidthHistory)/float(self.downloadRate[peer.id]) # Find how to find out max and min bw or infer from personal history
if round >= 3:
peer_download = 0
for download2 in history.downloads[round-2]:
if peer.id == download2.from_id:
for download3 in history.downloads[round-3]:
if peer.id == download3.from_id:
peer_download += 1
if peer_download > 0:
self.uploadRate[peer.id] *= 1 - gamma
break
if len(peer.available_pieces) > 0:
av_pieces = float(len(peer.available_pieces))
rnd = float(round)
slots = float(self.slots[peer.id])
self.downloadRate[peer.id] = av_pieces/(rnd * self.conf.blocks_per_piece * slots)
self.uploadRate[peer.id] *= 1 + alpha
#if self.downloadRate[peer.id] == 0:
# print str(peer.id) + ": " + str(peer.available_pieces)
# print "Peer %s has %s available pieces" % (peer.id, len(peer.available_pieces))
self.downloadUploadRatio[peer.id] = self.downloadRate[peer.id]/self.uploadRate[peer.id]
logging.debug("%s again. It's round %d." % (
self.id, round))
########### Dynamic Optimistic Unchoking #################################
########### Eallocate each 3 rounds ######################################
### Choose peer to uchoke every 3 round
### Choose # of peers divided by x:
x = 3
if round % 3 ==0:
self.optUnchokedPeers = random.sample(peers, len(peers)/x)
### Initial share to uchoke:
a = 0.5
availPiecesShare = float(sum(self.pieces))/float(self.conf.num_pieces*self.conf.blocks_per_piece)
### Allocate BW to opt unchoking
bwToOptUnchoking = (a*self.up_bw - availPiecesShare * self.up_bw * a) + 0.001
### Divide this BW among number of neighbors divided by x
bwToOptUnchoking = bwToOptUnchoking/(len(peers)/x)
optUnchokedAllocation = {}
for peer in self.optUnchokedPeers:
optUnchokedAllocation[peer.id] = float(100 * bwToOptUnchoking) /(float((a*self.up_bw - availPiecesShare * self.up_bw * a)) +0.001)
up_bw_available = self.up_bw - bwToOptUnchoking*(len(peers)/x)
# Removing optimistically unchoked peers from consideration
peers_tmp = list(peers)
for peer in self.optUnchokedPeers:
if peer in peers_tmp:
peers_tmp.remove(peer)
#########################################################################
########### Building upload list ########################################
#########################################################################
if len(requests) == 0:
logging.debug("No one wants my pieces!")
chosen = []
bws = []
uploads = []
else:
sumUpload = 0
chosen = {}
downloadUploadRatio_tmp = {}
# creating list with ratios for only peers in requests
for request in requests:
for peer in peers_tmp:
if request.requester_id == peer.id:
downloadUploadRatio_tmp[request.requester_id] = self.downloadUploadRatio[request.requester_id]
for key in downloadUploadRatio_tmp:
if downloadUploadRatio_tmp[key] in self.optUnchokedPeers:
downloadUploadRatio_tmp.pop(key)
needed = lambda i: self.pieces[i] < self.conf.blocks_per_piece
needed_pieces = filter(needed, range(len(self.pieces)))
while (sumUpload <= up_bw_available * 0.75 and len(downloadUploadRatio_tmp) > 0):
peer_id = max(downloadUploadRatio_tmp, key = downloadUploadRatio_tmp.get)
chosen[peer_id] = downloadUploadRatio_tmp.pop(peer_id)
sumUpload += self.uploadRate[peer_id]
""" Calculate the total proportional BW allocated to other peers """
totalUploadBW = 0
for choice in chosen:
totalUploadBW += chosen[choice]
""" Make each BW as a proportion of totalUploadBW """
if (float(totalUploadBW) * len(optUnchokedAllocation) == 0):
uploads = []
else:
for choice in chosen:
chosen[choice] = 100 * float(chosen[choice]) / float(totalUploadBW)
### Connecting optimistic unchoking list to tyrant list
# chosen.update(optUnchokedAllocation)
# print "Vector of choices for this round:"
# print chosen
""" Now need to divide our BW as integers according to chosen vector """
peerWeights = [value for (key, value) in sorted(chosen.items())]
peerNames = sorted(chosen)
# print "original chosen: %s" % (chosen)
# print "names: %s" % (peerNames)
# print "weights: %s" % (peerWeights)
bws = proportional_split(int(math.floor(up_bw_available)), peerWeights)
# create actual uploads out of the list of peer ids and bandwidths
uploads = [Upload(self.id, peer_id, bw)
for (peer_id, bw) in zip(chosen, bws)]
peerWeights = [value for (key, value) in sorted(optUnchokedAllocation.items())]
peerNames = sorted(optUnchokedAllocation)
bws = proportional_split(self.up_bw - int(up_bw_available), peerWeights)
uploads2 = [Upload(self.id, peer_id, bw)
for (peer_id, bw) in zip(peerNames, bws)]
uploads = uploads + uploads2
if (round + 1) % 5 == 0:
request = random.choice(requests)
chosen = [request.requester_id]
# Evenly "split" my upload bandwidth among the one chosen requester
bws = even_split(self.up_bw, len(chosen))
#uploads = uploads.append([Upload(self.id, peer_id, bw)
# for (peer_id, bw) in zip(optUnchokedAllocation, bws)])
return uploads
def coeff_var(row_values):
"""computes the coefficient of variation"""
sigma = util.r_stddev(row_values)
mu = util.mean(row_values)
return sigma / mu
def mean(self):
"""returns the mean value"""
return util.mean(self.values)
