def read_individually_downloaded_sw_file(file_name):
with open(file_name) as f:
reader = csv.reader(f)
rows = [r for r in reader]
details = rows[13][0]
# print details
site_id, site_info, lat, lng, elev = re.match('Site (\d+) (.+) Lat:(.*) Long:(.*) Elev:(.*)', details).groups()
lat = float(lat)
lng = float(lng)
elev = float(elev)
headers = rows[15]
date_i = 0; assert headers[date_i] == "Datetime"
level_i = 1; assert headers[level_i] == "Water Level (m) Mean"
flow_i = 3; assert headers[flow_i] == "Discharge (Ml/d) Mean"
old_flow_i = 5; assert headers[old_flow_i] == "Discharge (Ml/d) Mean"
start_i = 16
while rows[start_i][level_i] == '':
start_i += 1
end_i = len(rows)
while len(rows[end_i-1]) == 0:
end_i -= 1
# print "start_i-end-i", start_i, '-', end_i
dates = numpy.array([datetime.datetime.strptime(r[date_i], "%d/%m/%Y %H:%M:%S") for r in rows[start_i:end_i]])
levels = utils.interpolate( numpy.array([utils.as_float(r[level_i]) for r in rows[start_i:end_i]]) )
flows = utils.interpolate( numpy.array([utils.as_float(r[flow_i]) for r in rows[start_i:end_i]]) )
return site_id, site_info, lat, lng, elev, dates, levels, flows
def score_histogram(question, p, tol=1e-3):
# TODO pc is added to pu then renormalized, see https://discord.com/channels/694850840200216657/694850840707596320/798616689918738453
# TODO pc is actually calculated by logistic best fit
# Normalized resolution
x_norm = get_norm_resolution(question)
# Is question range open or closed?
pc = get_histogram(question)['c'].values.tolist()
assert len(pc) == len(p)
n_bins = len(p) - 1
dx = 1 / n_bins
closed = abs(sum(pc) * dx - 1) < tol # for lack of anything better
# Community distribution
pc_star = interpolate(pc, x_norm, closed)
# Uniform distribution
pu = 1 / n_bins if closed else (1 - .15) / n_bins
# Own prediction
p_star = interpolate(p, x_norm, closed)
# # predictions
N = question['number_of_predictions']
my_sc = scoring_(p_star, pc_star, pu, N)
comm_sc = scoring_(pc_star, pc_star, pu, N)
return my_sc - comm_sc
def main(args):
with open(args.data_dir+'/ptb.vocab.json', 'r') as file:
vocab = json.load(file)
w2i, i2w = vocab['w2i'], vocab['i2w']
model = SentenceVAE(
vocab_size=len(w2i),
sos_idx=w2i['<sos>'],
eos_idx=w2i['<eos>'],
pad_idx=w2i['<pad>'],
unk_idx=w2i['<unk>'],
max_sequence_length=args.max_sequence_length,
embedding_size=args.embedding_size,
rnn_type=args.rnn_type,
hidden_size=args.hidden_size,
word_dropout=args.word_dropout,
embedding_dropout=args.embedding_dropout,
latent_size=args.latent_size,
num_layers=args.num_layers,
bidirectional=args.bidirectional
)
if not os.path.exists(args.load_checkpoint):
raise FileNotFoundError(args.load_checkpoint)
model.load_state_dict(torch.load(args.load_checkpoint))
print("Model loaded from %s"%(args.load_checkpoint))
if torch.cuda.is_available():
model = model.cuda()
model.eval()
# samples, z = model.inference(n=args.num_samples)
# print('----------SAMPLES----------')
# print(idx2word(samples, i2w=i2w, pad_idx=w2i['<pad>']))
z1 = torch.randn([args.latent_size]).numpy()
z2 = torch.randn([args.latent_size]).numpy()
z = to_var(torch.from_numpy(interpolate(start=z1, end=z2, steps=8)).float())
samples, _ = model.inference(z=z)
print('-------INTERPOLATION-------')
print(idx2word(samples, i2w=i2w, pad_idx=w2i['<pad>']))
model.load_state_dict(torch.load('bin/2019-May-16-04:24:16/E10.pytorch'))
z = to_var(torch.from_numpy(interpolate(start=z1, end=z2, steps=8)).float())
samples, _ = model.inference(z=z)
print('-------INTERPOLATION-------')
print(idx2word(samples, i2w=i2w, pad_idx=w2i['<pad>']))
def do_dynamics(self):
current_dynamic = 0 # default to forte
cresc_goal = -1
cresc_duration = 0
cresc_begin = 0
cresc_into = 0
for i, note in enumerate(self.notes):
if not self.is_note(note):
continue
dyn = self.get_details(note, "dynamic")
if dyn:
test_dyn = self.dynamic_string_to_float(dyn[0][0])
if test_dyn is not None:
current_dynamic = test_dyn
cresc_goal = -1
cresc_duration = 0.0
cresc_details = self.get_details(note, "cresc")
if not cresc_details:
cresc_details = self.get_details(note, "decresc")
if cresc_details:
cresc_begin = float(current_dynamic)
cresc_into = 0.0
cresc_duration = note.duration
for later_note in self.notes[i+1:]:
if not self.is_note(later_note):
continue
dyn = self.get_details(later_note, "dynamic")
if dyn:
test_dyn = self.dynamic_string_to_float(dyn[0][0])
if test_dyn is not None:
cresc_goal = test_dyn
break
cresc_duration += later_note.duration
if cresc_goal >= 0:
current_dynamic = utils.interpolate(
cresc_into,
0.0, cresc_begin,
cresc_duration, cresc_goal)
cresc_into += note.duration
end_dynamic = utils.interpolate(
cresc_into,
0.0, cresc_begin,
cresc_duration, cresc_goal)
self.set_dynamic(note.begin.physical, current_dynamic)
if cresc_goal >= 0:
note.end.physical.string_number = note.begin.physical.string_number
note.end.physical.finger_position = note.begin.physical.finger_position
self.set_dynamic(note.end.physical, end_dynamic)
if cresc_into >= cresc_duration:
cresc_goal = -1
def _tile_flow(config: Configuration) -> None:
'''
The "tile" flow attempts to synthesize a tileable output texture from a given input texture.
Args:
config: Configuration specifying the parameters of the flow.
'''
with torch.no_grad():
autoencoder, svbrdf, lights, viewer, camera, overlap, input_path, output_path = config.load_tile_flow(
)
autoencoder.eval()
# It is assumed that the dimensions of the input image will be accepted by the network.
input_image = image.load(path=input_path, encoding='sRGB')
input_distance = utils.create_radial_distance_field(
num_rows=autoencoder.dimensions['Texture']['Input'][0],
num_cols=autoencoder.dimensions['Texture']['Input'][1])
input_batch = torch.cat([input_image, input_distance],
dim=2).unsqueeze(0).permute(0, 3, 1, 2)
# As long as the perceptive field of an output pixel is less than the size of the latent field, a tileable
# output texture can be obtained by decoding a tiling of the latent field (interpolated for good measure).
latent_tiles_row = autoencoder.encode(input_batch).expand(
3, -1, -1, -1).permute(0, 2, 3, 1)
latent_field_row = utils.interpolate(latent_tiles_row,
overlap=overlap).expand(
3, -1, -1, -1)
latent_field = utils.interpolate(
latent_field_row.transpose(1, 2),
overlap=overlap).transpose(0, 1).unsqueeze(0).permute(0, 3, 1, 2)
# The center crop of the output image will be tileable as long as the latent field was smoothly convolved.
output = autoencoder.decode(latent_field)
output_row_padding = output.size(
2) // 2 - autoencoder.dimensions['Texture']['Output'][0] // 2
output_col_padding = output.size(
3) // 2 - autoencoder.dimensions['Texture']['Output'][1] // 2
cropped_output = output[:, :, output_row_padding:-output_row_padding,
output_col_padding:-output_col_padding]
# The fully-convolutional nature of the SVBRDF decoder trivializes the creation of textures with arbitrary sizes.
normals, svbrdf.parameters = SVBRDFAutoencoder.interpret(
cropped_output)
_shade_render_save(normals=normals,
svbrdf=svbrdf,
lights=lights,
viewer=viewer,
camera=camera,
path=output_path)
def compute_point_on_multiline(self, points):
if self.point_index < len(points) - 1:
return utils.interpolate(points[self.point_index],
points[self.point_index + 1],
self.line_param)
else:
return points[-1]
def disc_step(model, opt, gen_img, style_img):
model.train()
opt.zero_grad()
d_real, _ = model(style_img)
d_gen, _ = model(gen_img)
if model.distance.startswith('wgan'):
# Wasserstein Distance
dist = d_gen - d_real
if model.distance.endswith('-gp'):
# Gradient Penalty
x = utils.interpolate(gen_img, style_img)
gp = model.disc_gp(x)
loss = dist + 10 * gp
else:
assert model.distance.endswith('-sn')
loss = dist
else:
assert 'sn' in model.distance, model.distance
# Spectral norm
real_loss = F.binary_cross_entropy_with_logits(d_real,
torch.ones_like(d_real))
gen_loss = F.binary_cross_entropy_with_logits(d_gen,
torch.zeros_like(d_gen))
loss = real_loss + gen_loss
loss.backward()
opt.step()
return loss.item()
def test(self):
src_z = self._sample_z()
dst_z = self._sample_z()
counter = 0.0
cap = cv2.VideoCapture(0)
while cap.isOpened():
with torch.no_grad():
z = interpolate(src_z, dst_z, counter)
out = self.generator(z, self.x, self.y, self.r)
out = out.view(-1, self.dim_x, self.dim_y, self.dim_c).permute(
(0, 3, 1, 2))
out = out.squeeze(0)
out = (out + 1.0) * 0.5
out = out.cpu().numpy().transpose(1, 2, 0)
out = out[:, :, ::-1]
cv2.imshow('Result', out)
counter += 0.005
if counter >= 1.0:
counter = 0.0
src_z = dst_z
dst_z = self._sample_z()
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cv2.destroyAllWindows()
def compute_point_on_multiline(self, points):
if self.point_index < len(points) - 1:
return utils.interpolate(points[self.point_index],
points[self.point_index + 1],
self.line_param)
else:
return points[-1]
def plot_h_vs_ic(L,sigmas=interpolate(0.1,10,100),max_h=None,M=None,xfunc=lambda ps:2*L):
if max_h is None:
print "generating samples"
pss = [simplexify_sample(4**L,sigma=sigma)
for sigma in tqdm(sigmas)]
else:
pss = []
while len(pss) < trials:
ps = sample(L)
if h(ps) < max_h:
pss.append(ps)
print len(pss)
print "computing M"
if M is None:
M = marginalization_matrix(L)
icq_s = map(lambda ps:ic(ps,M),tqdm(pss))
print "computing entropy"
icp_s = map(lambda ps:2*L - h_np(ps),tqdm(pss))
# print "computing total mi"
# mis = map(lambda ps:total_mi(ps,M),tqdm(pss))
# print "computing columnwise entropies"
# hqs = map(lambda ps:psfm_entropy(ps,M),tqdm(pss))
# plt.scatter(hs,hqs)
plt.scatter(icp_s,icq_s)
#plt.plot([0,2*L],[2*L,0])
#plt.plot([0,2*L],[0,2*L])
# plt.plot([0,2],[0,4])
# plt.plot([0,2],[0,2*L])
# print pearsonr(ics,hs)
# print spearmanr(ics,hs)
plt.plot([0,2*L],[0,2*L])
plt.plot(*pl(lambda icp:L*icp+2*(L-L**2),[2*(L-1),2*L]),color='b')
plt.xlabel("Distribution IC")
plt.ylabel("PSFM IC")
plt.title("Distribution vs. Columnwise IC, Length=%s" % L)
def forward(self, input, mixup_lambda=None):
"""
Input: (batch_size, data_length)"""
x, frames_num = self.preprocess(input, mixup_lambda=mixup_lambda)
# Output shape (batch size, channels, time, frequency)
x = self.cnn_feature_extractor(x)
# Aggregate in frequency axis
x = torch.mean(x, dim=3)
x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)
x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)
x = x1 + x2
x = F.dropout(x, p=0.5, training=self.training)
x = x.transpose(1, 2)
x = F.relu_(self.fc1(x))
x = x.transpose(1, 2)
x = F.dropout(x, p=0.5, training=self.training)
(clipwise_output, norm_att, segmentwise_output) = self.att_block(x)
segmentwise_output = segmentwise_output.transpose(1, 2)
# Get framewise output
framewise_output = interpolate(segmentwise_output,
self.interpolate_ratio)
framewise_output = pad_framewise_output(framewise_output, frames_num)
output_dict = {
'framewise_output': framewise_output,
'clipwise_output': clipwise_output
}
return output_dict
def expected_entropy(K,alpha=1,num_points=1000):
"""Compute expected entropy of uniform prior on probability simplex on
K elements through numerical integration"""
def phi(x):
return -x*log(x) if x > 0 else 0
def integrand(x):
if x == 0 or x == 1:
return 0
else:
return phi(x)*x**(alpha-1)*(1-x)**((K-1)*alpha-1)
#prefactor_ref = K*gamma(K*alpha)/(gamma(alpha)*gamma((K-1)*alpha))
log_prefactor = log(K) + gammaln(K*alpha) - (gammaln((alpha)) + gammaln((K-1)*alpha))
try :
prefactor = exp(log_prefactor)
except:
raise Exception("expected entropy failed on:",K,alpha)
#print prefactor,prefactor_ref
#prefactor = K/(gamma(alpha))*(alpha*(K-1))**alpha
#print prefactor,approx_prefactor
# f = lambda x:(alpha*x*log(x)*K-(x+alpha)*log(x)+x-1)/(x*(x-1)*log(x))
# f2 = lambda x:(alpha*x*log(x)*K-(x+alpha)*log(x)+x-1)/(x*(x-1)*log(x))
# xmax = 1#bisect_interval(f,10**-200,1/2.0)
# prudent_num_points = int(10/xmax)
# if num_points < prudent_num_points:
# num_points = prudent_num_points
# print "updated num_points to:",num_points
#integral_ref = mean(integrand(x) for x in interpolate(0,1,num_points))
#xs = interpolate(0,1,num_points)
#print "computing xs"
xs = [10**x for x in interpolate(-100,0,num_points)]
#print len(xs)
#print "min x:",xs[1]
ys = map(integrand,xs)
integral_trap = np.trapz(ys,xs)
return prefactor*integral_trap/log(2)
def explore_coupling_const(iterations=1000000):
"""Given 3 state system, explore spin probabilities as function of coupling strength"""
N = 10
x0 = [0] * N
hs = [log(1000000)] * N
def hamil(xs, J):
return dot(xs, hs) + J * (xs[0] + sum([xi * xj for (xi, xj) in pairs(xs)]))
Js = interpolate(-16, -8 + 1, 20)
def proposal(xs):
return [int(random.random() < 0.5) for i in range(N)]
results = []
for J in Js:
chain = mh(f=lambda xs: -hamil(xs, J), proposal=proposal, x0=x0, use_log=True, iterations=iterations)
ps = map(mean, transpose(chain))
results.append((J, ps))
Js, pss = transpose(results)
pss = transpose(pss)
colors = "bgrcmyk"
for i, ps in enumerate(pss):
color = colors[i % len(colors)]
plt.plot(Js, ps, marker="o", linestyle="", color=color)
errs = [p + 1.96 * sqrt(p * (1 - p) / iterations) ** (i + 1) + p ** (i + 1) for p in pss[0]]
print i, errs
plt.plot(Js, [p ** (i + 1) for p in pss[0]])
# plt.errorbar(Js,[p**(i+1) for p in pss[0]],yerr=errs,
# marker='',linestyle='--',color=color)
plt.plot(Js, [1.0 / iterations for J in Js])
# plt.semilogy()
return results
def integrate_improp(f, int_points=1000):
"""Integrate f from a to infinity using transform u = 1/(1-t)"""
interp_start = 0
interp_stop = 1 - 1.0 / int_points
return mean(
f(t / (1 - t)) * 1 / (1 - t)**2
for t in (interpolate(interp_start, interp_stop, int_points)))
def read_bulk_downloaded_sw_file(dir_name, site_id, data_types):
# get details of all sites
site_details = {}
with open(os.path.join(dir_name, "Site Details.csv")) as f:
reader = csv.DictReader(f)
for row in reader:
site_details[row["Site Id"]] = row
data_types_values = {}
for data_type in data_types:
file_name = os.path.join(dir_name, site_id+"."+data_type+".csv")
with open(file_name) as f:
reader = csv.reader(f)
rows = [r for r in reader]
headers = rows[2]
date_i = 0; assert headers[date_i] == "Date"
data_i = 1; assert headers[data_i] == "Mean"
start_i = 3
dates = numpy.array([datetime.datetime.strptime(r[date_i], "%H:%M:%S %d/%m/%Y") for r in rows[start_i:]])
data = utils.interpolate( numpy.array([utils.as_float(r[data_i]) for r in rows[start_i:]]) )
data_types_values[data_type] ={
"type": data_type,
"dates": dates,
"data": data
}
return site_details[site_id], data_types_values
def do_dynamics(self):
current_dynamic = 0 # default to forte
cresc_goal = -1
cresc_duration = 0
cresc_begin = 0
cresc_into = 0
for i, note in enumerate(self.notes):
if not self.is_note(note):
continue
dyn = self.get_details(note, "dynamic")
if dyn:
test_dyn = self.dynamic_string_to_float(dyn[0][0])
if test_dyn is not None:
current_dynamic = test_dyn
cresc_goal = -1
cresc_duration = 0.0
cresc_details = self.get_details(note, "cresc")
if not cresc_details:
cresc_details = self.get_details(note, "decresc")
if cresc_details:
cresc_begin = float(current_dynamic)
cresc_into = 0.0
cresc_duration = note.duration
for later_note in self.notes[i + 1:]:
if not self.is_note(later_note):
continue
dyn = self.get_details(later_note, "dynamic")
if dyn:
test_dyn = self.dynamic_string_to_float(dyn[0][0])
if test_dyn is not None:
cresc_goal = test_dyn
break
cresc_duration += later_note.duration
if cresc_goal >= 0:
current_dynamic = utils.interpolate(cresc_into, 0.0,
cresc_begin,
cresc_duration, cresc_goal)
cresc_into += note.duration
end_dynamic = utils.interpolate(cresc_into, 0.0, cresc_begin,
cresc_duration, cresc_goal)
self.set_dynamic(note.begin.physical, current_dynamic)
if cresc_goal >= 0:
note.end.physical.string_number = note.begin.physical.string_number
note.end.physical.finger_position = note.begin.physical.finger_position
self.set_dynamic(note.end.physical, end_dynamic)
if cresc_into >= cresc_duration:
cresc_goal = -1
def expected_entropy_ref(K,num_points):
"""Compute expected entropy of uniform prior on probability simplex on
K elements through numerical integration"""
def f(x):
return -x*log(x) if x > 0 else 0
def integrand(x):
return f(x)*(1-x)**(K-2)
return K*(K-1)*mean(integrand(x) for x in interpolate(0,1,num_points))/log(2)
def compute_insciption_point(self):
points = self.get_all_points()
if self.inscription_point < len(points) - 1:
return utils.interpolate(points[self.inscription_point],
points[self.inscription_point + 1],
self.inscription_param)
else:
return self.points[self.inscription_point]
def compute_insciption_point(self):
points = self.get_all_points()
if self.inscription_point < len(points) - 1:
return utils.interpolate(points[self.inscription_point],
points[self.inscription_point + 1],
self.inscription_param)
else:
return self.points[self.inscription_point]
def main(args):
with open(args.data_dir + '/ptb.vocab.json', 'r') as file:
vocab = json.load(file)
w2i, i2w = vocab['w2i'], vocab['i2w']
model = SentenceVAE(vocab_size=len(w2i),
sos_idx=w2i['<sos>'],
eos_idx=w2i['<eos>'],
pad_idx=w2i['<pad>'],
unk_idx=w2i['<unk>'],
max_sequence_length=args.max_sequence_length,
embedding_size=args.embedding_size,
rnn_type=args.rnn_type,
hidden_size=args.hidden_size,
word_dropout=args.word_dropout,
embedding_dropout=args.embedding_dropout,
latent_size=args.latent_size,
num_layers=args.num_layers,
bidirectional=args.bidirectional)
if not os.path.exists(args.load_checkpoint):
raise FileNotFoundError(args.load_checkpoint)
model.load_state_dict(torch.load(args.load_checkpoint))
print("Model loaded from %s" % args.load_checkpoint)
if torch.cuda.is_available():
model = model.cuda()
model.eval()
# samples, z = model.inference(n=args.num_samples)
# print('----------SAMPLES----------')
# print(*idx2word(samples, i2w=i2w, pad_idx=w2i['<pad>']), sep='\n')
# z_ = torch.randn([args.latent_size]).numpy()
# input_sent = "the n stock specialist firms on the big board floor the buyers and sellers of last resort who were criticized after the n crash once again could n't handle the selling pressure"
input_sent = "looking for a job was one of the most anxious periods of my life and is for most people"
batch_input = torch.LongTensor([[w2i[i]
for i in input_sent.split()]]).cuda()
batch_len = torch.LongTensor([len(input_sent.split())]).cuda()
input_mean = model(batch_input, batch_len, output_mean=True)
z_ = input_mean.cpu().detach().numpy()
print(z_.shape)
# z2 = torch.randn([args.latent_size]).numpy()
for i in range(args.latent_size):
print(f"-------Dimension {i}------")
z1, z2 = z_.copy(), z_.copy()
z1[i] -= 0.5
z2[i] += 0.5
z = to_var(
torch.from_numpy(interpolate(start=z1, end=z2, steps=5)).float())
samples, _ = model.inference(z=z)
print('-------INTERPOLATION-------')
print(*idx2word(samples, i2w=i2w, pad_idx=w2i['<pad>']), sep='\n')
def validate_dirichlet_sample(Ks = [2,5,10,20,50,100],N=1000):
alphas = [10**i for i in interpolate(-5,0,100)]
for K in Ks:
print K
plt.scatter(*pl(lambda a:mean(h_np(dirichlet_sample(K,a)) for i in xrange(N)),alphas))
plt.plot(*pl(lambda alpha:expected_entropy(K,alpha=alpha),alphas),label="%s pred" % K)
plt.xlabel("alpha")
plt.ylabel("Entropy (bits)")
plt.semilogx()
plt.legend()
def forward(self, x):
# conv-(norm)-act-(interp)
x = self.conv1(x)
if self.use_norm:
x = self.norm1(x)
x = self.actv1(x)
if self.factor != 1:
x = interpolate(x, self.factor)
return x
def circular_transport(p,q,iterations=1000):
"""return circular arc from p to q"""
pcirc = normalize(p)
qcirc = normalize(q)
theta = acos(pcirc.dot(qcirc))
k = normalize(np.cross(pcirc,qcirc))
assert is_normalized(k),np.linalg.norm(k)
bik = normalize(np.cross(k,pcirc))
assert is_normalized(bik)
return [(cos(t))*pcirc + (sin(t))*bik+k*k.dot(pcirc)*(1-cos(t)) for t in interpolate(0,theta,iterations)]
def circular_transport(p,q,iterations=1000):
"""return circular arc from p to q"""
pcirc = normalize(p)
qcirc = normalize(q)
theta = acos(pcirc.dot(qcirc))
k = normalize(np.cross(pcirc,qcirc))
assert is_normalized(k),np.linalg.norm(k)
bik = normalize(np.cross(k,pcirc))
assert is_normalized(bik)
return [(cos(t))*pcirc + (sin(t))*bik+k*k.dot(pcirc)*(1-cos(t)) for t in interpolate(0,theta,iterations)]
def interpolateKeyData(a, b, weight):
key = KeyFrame()
key.translation = euclid.Vector3(
*utils.interpolate3d(a.translation, b.translation, weight))
key.rotation = euclid.Vector3(
*utils.interpolate3d(a.rotation, b.rotation, weight))
key.scale = euclid.Vector3(*utils.interpolate3d(a.scale, b.scale, weight))
key.visible = a.visible
key.transparency = utils.interpolate(a.transparency, b.transparency,
weight)
return key
def test_isocontour_projection_hypothesis():
"""
Conjecture: entropy isocontours given by projection of centered
circles on unit sphere onto probability simplex.
Conclusion: doesn't work
"""
K = 3
p = simplexify_sample(K)
n = project_to_sphere(np.array([1.0/K]*K))
theta = find_theta(p,n) # in radians
ps = [np.asarray(general_rotation_matrix(n,theta).dot(p))[0] for theta in interpolate(0,2*pi,100)]
def get_interpolations(vae, sample_start, sample_end, args):
model = vae['model']
tokenizer = vae['tokenizer']
w2i = vae['w2i']
i2w = vae['i2w']
# Initialize semantic loss
# sl = Semantic_Loss()
start_encode = tokenizer.encode(sample_start)
end_encode = tokenizer.encode(sample_end)
with torch.no_grad():
z1 = model._encode(**start_encode)
z1_hidden = z1['z'].cpu()[0]
z2 = model._encode(**end_encode)
z2_hidden = z2['z'].cpu()[0]
z_hidden = to_var(torch.from_numpy(interpolate(start=z1_hidden, end=z2_hidden, steps=args.steps)).float())
if args.rnn_type == "lstm":
z1_cell_state = z1['z_cell_state'].cpu()[0].squeeze()
z2_cell_state = z2['z_cell_state'].cpu()[0].squeeze()
# print(z1_cell_state.shape)
z_cell_states = \
to_var(torch.from_numpy(interpolate(start=z1_cell_state, end=z2_cell_state, steps=args.steps)).float())
samples, _ = model.inference(z=z_hidden, z_cell_state=z_cell_states)
else:
samples, _ = model.inference(z=z_hidden, z_cell_state=None)
# print('-------INTERPOLATION-------')
interpolated_sentences = idx2word(samples, i2w=i2w, pad_idx=w2i['<pad>'])
# For each sentence, get the perplexity and show it
# for sentence in interpolated_sentences:
# print(sentence + "\t\t" + str(sl.get_perplexity(sentence)))
# print(sentence)
return interpolated_sentences
def closest_obs(obs_code, dates, closest_names, closest_ids, closest_locations, zipped_sites_dir, bom_obs_types):
print dates[0], dates[-1]
for closest_i in range(len(closest_names)):
climate_data = get_bom_climate(zipped_sites_dir, closest_ids[closest_i])
if obs_code in climate_data and utils.intersection_size([climate_data[obs_code]["dates"], dates]) > 5 * 365:
closest_dates = climate_data[obs_code]["dates"]
closest_data = utils.interpolate(climate_data[obs_code]["values"])
print "closest", bom_obs_types[obs_code], closest_names[closest_i], closest_ids[
closest_i
], closest_locations[closest_i]
return closest_i, closest_dates, closest_data
print "ERROR: NO closest", bom_obs_types[obs_code]
return 0, [], []
def main(args):
with open(args.data_dir + '/poems.vocab.json', 'r') as file:
vocab = json.load(file)
w2i, i2w = vocab['w2i'], vocab['i2w']
model = SentenceVAE(vocab_size=len(w2i),
sos_idx=w2i['<sos>'],
eos_idx=w2i['<eos>'],
pad_idx=w2i['<pad>'],
unk_idx=w2i['<unk>'],
max_sequence_length=args.max_sequence_length,
embedding_size=args.embedding_size,
rnn_type=args.rnn_type,
hidden_size=args.hidden_size,
word_dropout=args.word_dropout,
embedding_dropout=args.embedding_dropout,
latent_size=args.latent_size,
num_layers=args.num_layers,
bidirectional=args.bidirectional,
condition_size=0)
if not os.path.exists(args.load_checkpoint):
raise FileNotFoundError(args.load_checkpoint)
model.load_state_dict(
torch.load(args.load_checkpoint, map_location=torch.device('cpu')))
print("Model loaded from %s" % (args.load_checkpoint))
if torch.cuda.is_available():
model = model.cuda()
model.eval()
samples, z = model.inference(n=args.num_samples)
print('----------SAMPLES----------')
print(*idx2word(samples, i2w=i2w, pad_idx=w2i['<pad>']), sep='\n')
# while True:
# samples, z = model.inference(n=1, condition=torch.Tensor([[1, 0, 0, 0, 0, 0, 0]]).cuda())
# poem = idx2word(samples, i2w=i2w, pad_idx=w2i['<pad>'])[0]
# if 'love' in poem:
# breakpoint()
z1 = torch.randn([args.latent_size]).numpy()
z2 = torch.randn([args.latent_size]).numpy()
z = to_var(
torch.from_numpy(interpolate(start=z1, end=z2, steps=8)).float())
# samples, _ = model.inference(z=z, condition=torch.Tensor([[1, 0, 0, 0, 0, 0, 0], [1, 0, 0, 0, 0, 0, 0], [1, 0, 0, 0, 0, 0, 0], [1, 0, 0, 0, 0, 0, 0], [1, 0, 0, 0, 0, 0, 0], [1, 0, 0, 0, 0, 0, 0], [1, 0, 0, 0, 0, 0, 0], [1, 0, 0, 0, 0, 0, 0], [1, 0, 0, 0, 0, 0, 0], [1, 0, 0, 0, 0, 0, 0]]).cuda())
samples, _ = model.inference(z=z)
print('-------INTERPOLATION-------')
print(*idx2word(samples, i2w=i2w, pad_idx=w2i['<pad>']), sep='\n')
def rerankScores(bm25Weight, baselineScores, classifierScores, classifierRankings, rerankedFile, writeFile=False, qrelsDict=QRELS, printUnjudged=False):
if writeFile:
out = open(rerankedFile, "w")
allP10s = []
unjudged = 0.0
for qid in QUERY_RANGE:
queryQrels = dict(qrelsDict[qid])
queryScores = baselineScores[qid]
queryTopicModel = defaultdict(float) # TOPIC_MODEL_SCORES[qid]
queryDoc2Vec = defaultdict(float) #DOC2VEC_SCORES[qid]
queryClfRankings = classifierRankings[qid]
queryClfScores = classifierScores[qid]
if opts.fusion == "interpolation":
rerankedScores = [(did, utils.interpolate(queryScores[did][1], queryClfScores[did][0], bm25Weight)) for did in queryScores.keys()]
elif opts.fusion == "rrf":
rerankedScores = [(did, rrf(queryScores[did][0], queryClfRankings[did], bm25Weight)) for did in queryScores.keys()]
elif opts.fusion == "borda":
rerankedScores = [(did, borda(queryScores[did][0], queryClfRankings[did], bm25Weight)) for did in queryScores.keys()]
rerankedScores.sort(key = lambda docScore : docScore[1], reverse = True)
top10Docs = set([doc for (doc, _) in rerankedScores[:10]])
relDocs = set([did for (did, rel) in queryQrels.items() if rel > 0])
p10 = float(len(top10Docs & relDocs)) / 10.0
allP10s.append(p10)
judgedDocs = set(queryQrels.keys())
unjudged += float(len(top10Docs - judgedDocs)) / 10.0
if writeFile:
rank = 1
for did, score in rerankedScores:
out.write("%d Q0 %s %3d %f SUMMARY" % (qid, did, rank, score))
out.write("\t#\t%f\t%d\t%f\t%s\t%f\t%f\t\t%s\n" %
(bm25Weight,
queryScores[did][0],
queryScores[did][1],
classifierStr(queryClfScores[did]),
queryTopicModel.get(did, -1),
queryDoc2Vec.get(did, -1),
qrelStr(queryQrels.get(did))))
rank += 1
if writeFile:
out.close()
if printUnjudged:
print "Weight = %.2f Unjudged = %.2f, P10 = %.2f" % (bm25Weight, unjudged / len(QUERY_RANGE), np.mean(allP10s))
return np.mean(allP10s), allP10s
def main():
G = 1000
mu_ep = 0
sigma_ep = 1
eps = gaussians(mu_ep,sigma_ep,G)
mus = interpolate(-100,10,100)
plt.plot(*pl(lambda mu:mean_occ(eps,mu),mus),label="Mean occ")
plt.plot(*pl(lambda mu:G/(1+exp(-0.75*mu)),mus),label="predicted occ")
plt.plot(*pl(lambda mu:sd_occ(eps,mu),mus),label="Sd occ")
plt.plot(*pl(lambda mu:entropy(eps,mu),mus),label="Entropy (bits)")
plt.plot([mu_ep,mu_ep],[0,G],linestyle='--')
plt.plot([mus[0],mus[-1]],[G/2,G/2],linestyle='--')
plt.xlabel("mu")
plt.legend()
plt.show()
def main(args):
# load checkpoint
if not os.path.exists(args.load_checkpoint):
raise FileNotFoundError(args.load_checkpoint)
saved_dir_name = args.load_checkpoint.split('/')[2]
params_path = './saved_vae_models/' + saved_dir_name + '/model_params.json'
if not os.path.exists(params_path):
raise FileNotFoundError(params_path)
# load params
params = load_model_params_from_checkpoint(params_path)
# create and load model
model = SentenceVae(**params)
print(model)
model.load_state_dict(torch.load(args.load_checkpoint))
print("Model loaded from %s" % args.load_checkpoint)
if torch.cuda.is_available():
model = model.cuda()
model.eval()
# load the vocab of the chosen dataset
if (model.dataset == 'yelp'):
print("Yelp dataset used!")
with open(args.data_dir + '/yelp/yelp.vocab.json', 'r') as file:
vocab = json.load(file)
w2i, i2w = vocab['w2i'], vocab['i2w']
samples, z = model.inference(n=args.num_samples)
print('----------SAMPLES----------')
print(*idx2word(samples, i2w=i2w, pad_idx=w2i['<pad>']), sep='\n')
z1 = torch.randn([params['latent_size']]).numpy()
z2 = torch.randn([params['latent_size']]).numpy()
z = to_var(
torch.from_numpy(interpolate(start=z1, end=z2, steps=8)).float())
samples, _ = model.inference(z=z)
print('-------INTERPOLATION-------')
print(*idx2word(samples, i2w=i2w, pad_idx=w2i['<pad>']), sep='\n')
def mu_summary_stat_experiment():
"""Can we correlate copy number with a summary statistic?"""
trials = 100
ep_mu = -2
ep_sigma = 5
G = 100
ts = []
copies = []
eps = [random.gauss(ep_mu,ep_sigma) for i in range(G)]
mus = interpolate(-10,10,1000)
eta = mean(eps)
gamma = 1.0/variance(eps)
print gamma
plt.plot(*pl(lambda mu:mean_occ(eps,mu),mus))
plt.plot(*pl(lambda mu:G*fd(eta,mu,beta=gamma),mus))
plt.plot(*pl(lambda x:G/2.0,mus))
def lanefind(image):
#image = mpimg.imread(r"G:\SelfDrivingcarsPractice\LaneFinding\data\images\solidWhiteRight.jpg")
gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
ysize, xsize = gray.shape
#apply the gaussian blur
kernel_size = 5
blurred_gray = cv2.GaussianBlur(gray, (kernel_size, kernel_size), 0)
#apply the canny edge detection
high_threshold = 150
low_threshold = 100
edges = cv2.Canny(blurred_gray, low_threshold, high_threshold)
#get the region of interest
mask = np.zeros_like(edges)
ignore_mask_color = 1
vertices = np.array([[(0, ysize), (5 * xsize / 11, ysize / 2),
(6 * xsize / 11, ysize / 2), (xsize, ysize)]],
dtype=np.int32)
cv2.fillPoly(mask, vertices, ignore_mask_color)
masked_edges = cv2.bitwise_and(edges, mask)
#applying Hough transform to draw lines
rho = 1
theta = np.pi / 180
threshold = 20
min_line_length = 30
max_line_gap = 20
line_image = np.zeros_like(image)
lines = cv2.HoughLinesP(masked_edges, rho, theta, threshold, np.array([]),
min_line_length, max_line_gap)
filtered_lines_0 = utils.slopefilter(lines)
filtered_lines_1 = utils.lanelinefilter(filtered_lines_0, ysize, xsize)
filtered_lines_2 = utils.extend_lines(filtered_lines_1, ysize)
final_lines = utils.interpolate(filtered_lines_2, xsize, ysize)
for line in final_lines:
for x1, y1, x2, y2 in line:
cv2.line(line_image, (x1, y1), (x2, y2), (255, 0, 0), 10)
color_edges = np.dstack((masked_edges, masked_edges, masked_edges))
combo = cv2.addWeighted(image, 1, line_image, 0.3, 0)
return combo
def plot_h_vs_ic_entropy_prior(num_cols):
def perturb(ps):
qs = np.array([random.gauss(0,1) for p in ps])
min_p = min(ps)
qs *= min_p/(np.linalg.norm(qs))
ps_prime = project_to_simplex(ps + qs)
assert abs(sum(ps_prime) - 1) < 10**16
assert np.all(ps_prime > 0)
return ps_prime
M = 100
return weighted_ensemble(q=perturb,
f=h,
init_states = [sample(num_cols) for i in range(M)],
bins = interpolate(0,2*num_cols,1000),
M=M,
timesteps=100,
tau=1,verbose=1)
def plot_h_vs_ic_entropy_prior(num_cols):
def perturb(ps):
qs = np.array([random.gauss(0,1) for p in ps])
min_p = min(ps)
qs *= min_p/(np.linalg.norm(qs))
ps_prime = project_to_simplex(ps + qs)
assert abs(sum(ps_prime) - 1) < 10**16
assert np.all(ps_prime > 0)
return ps_prime
M = 100
return weighted_ensemble(q=perturb,
f=h,
init_states = [sample(num_cols) for i in range(M)],
bins = interpolate(0,2*num_cols,1000),
M=M,
timesteps=100,
tau=1,verbose=1)
def writeTestFeatures(bm25Scores, clfScores, outFile):
print "Wrinting test features"
out = open(outFile, "w")
didsOut = open(outFile + ".doc-ids.txt", "w")
for qid in QUERY_RANGE:
queryQrels = dict(QRELS[qid])
for did, bm25Score in bm25Scores[qid].items():
relevance = queryQrels.get(did, 0)
# relevance = min(1, relevance)
clfScore = clfScores[qid][did][0]
fused = utils.interpolate(bm25Score, clfScore, BM25_WEIGHT)
out.write("%d qid:%d\t\t1:%f\t\t2:%f\t\t3:%f\t\t# %d\n" % (relevance, qid, bm25Score, clfScore, fused, did))
didsOut.write("%d\n" % did)
out.close()
didsOut.close()
def disc_step(disc, opt, gen_img, style_img):
disc.train()
opt.zero_grad()
# Wasserstein Distance
d_real, _ = disc(style_img)
d_gen, _ = disc(gen_img)
disc_loss = d_gen - d_real
# Gradient Penalty
x = utils.interpolate(gen_img, style_img)
gp = disc.disc_gp(x)
loss = disc_loss + 10 * gp
loss.backward()
opt.step()
return disc_loss.item(), gp.item()
def profile():
try:
body = group(json.loads(request.data))
interpolated_path = []
for line in body:
interpolated_path.append(interpolate(line[0], line[1]))
result = []
for line in interpolated_path:
for point in line:
alt = int(get_altitude(point[1], point[0]))
result.append([*point, alt])
print(result)
return make_response(json.dumps(result), 200)
except:
return Response(status=404)
def get_simple_force(self, st, finger_position, dyn):
# TODO: this function is icky
low_index = 0
high_index = 0
# ASSUME: FINGER_MIDIS is already sorted
fm = utils.pos2midi(finger_position)
for i, val in enumerate(basic_training.FINGER_MIDIS):
if fm >= val:
low_index = i
high_index = i+1
if high_index >= len(basic_training.FINGER_MIDIS):
high_index = len(basic_training.FINGER_MIDIS) - 1
force = utils.interpolate(fm,
basic_training.FINGER_MIDIS[low_index],
self.controller_params[st][int(dyn+0.5)].get_attack_force(low_index),
basic_training.FINGER_MIDIS[high_index],
self.controller_params[st][int(dyn+0.5)].get_attack_force(high_index))
#print force
return force
def flow_diagram(L,band,band_tol,sigmas=interpolate(0.0005,0.01,10),steps=1,show=False,mu=10**-3,disp_fact=0.1):
"""Gather trajectories as they pass through ICp band"""
print "sampling"
pss = [dirichlet_sample(4**L,sigma) for sigma in tqdm(sigmas)]
Mut = mutation_matrix_ref(mu,L,mode='discrete')
Mar = marginalization_matrix(L)
band_bin = []
delta_icps = []
delta_icqs = []
def mut_dist(ps):
mut_ps = Mut.dot(ps)
return h_np(ps) - h_np(mut_ps)
all_ps = []
def icp(ps):
return 2*L-h_np(ps)
def icq(ps):
return ic(ps,M=Mar)
for sigma in tqdm((sigmas)):
ps = dirichlet_sample(4**L,sigma)
all_ps.append(ps)
for i,step in enumerate(xrange(steps)):
ps = Mut.dot(ps)
if i % 10 == 0: # collect every tenth step
all_ps.append(ps.copy())
cmap = plt.get_cmap('jet')
divergences = {}
for p in tqdm(all_ps):
pp = Mut.dot(p)
#print p,pp
x,y = icp(p),icq(p)
dx, dy = disp_fact*(icp(pp)-x),disp_fact*(icq(pp)-y)
d = abs(dx) #sqrt(dx**2 + dy**2)
plt.arrow(x,y,dx,dy,color=cmap(100/0.7*d))
divergences[(x,y)] = d
plt.plot([0,2*L-2],[0,0],color='b')
plt.plot([0,2],[0,2*L],color='b')
plt.plot([2,2*L],[2*L,2*L],color='b')
plt.plot([2*L-2,2*L],[0,2*L],color='b')
plt.xlabel("ICp")
plt.ylabel("ICq")
# plt.show()
return divergences
def main(args):
with open(args.data_dir + '/snli_yelp/snli_yelp.vocab.json', 'r') as file:
vocab = json.load(file)
w2i, i2w = vocab['w2i'], vocab['i2w']
if not os.path.exists(args.load_checkpoint):
raise FileNotFoundError(args.load_checkpoint)
if not os.path.exists(args.load_params):
raise FileNotFoundError(args.load_params)
# load params
params = load_model_params_from_checkpoint(args.load_params)
# create model
model = SentenceVaeMultiTask(**params)
print(model)
model.load_state_dict(torch.load(args.load_checkpoint))
print("Model loaded from %s" % args.load_checkpoint)
if torch.cuda.is_available():
model = model.cuda()
model.eval()
# print(params['latent_size'])
# exit()
samples, z = model.inference(n=args.num_samples)
print('----------SAMPLES----------')
print(*idx2word(samples, i2w=i2w, pad_idx=w2i['<pad>']), sep='\n')
z1 = torch.randn([params['latent_size']]).numpy()
z2 = torch.randn([params['latent_size']]).numpy()
z = to_var(
torch.from_numpy(interpolate(start=z1, end=z2, steps=8)).float())
samples, _ = model.inference(z=z)
print('-------INTERPOLATION-------')
print(*idx2word(samples, i2w=i2w, pad_idx=w2i['<pad>']), sep='\n')
def pipeline(image):
#image = cv2.imread(r"G:\SelfDrivingcarsPractice\LaneFinding\data\images\solidWhiteRight.jpg")
ysize = image.shape[0]
xsize = image.shape[1]
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
kernel_size = 5
blurred_gray = cv2.GaussianBlur(gray, (kernel_size, kernel_size), 0)
high_threshold = 90
low_threshold = 30
edges = cv2.Canny(blurred_gray, low_threshold, high_threshold)
mask = np.zeros_like(edges)
ignore_mask_color = 255
vertices = np.array([[(0, ysize), (5 * xsize / 11, ysize / 2),
(6 * xsize / 11, ysize / 2), (xsize, ysize)]],
dtype=np.int32)
cv2.fillPoly(mask, vertices, ignore_mask_color)
masked_edges = cv2.bitwise_and(edges, mask)
rho = 1
theta = np.pi / 180
threshold = 60
min_line_length = 100
max_line_gap = 50
line_image = np.zeros_like(image)
lines = cv2.HoughLinesP(masked_edges, rho, theta, threshold, np.array([]),
min_line_length, max_line_gap)
filtered_lines_0 = utils.slopefilter(lines)
filtered_lines_1 = utils.lanelinefilter(filtered_lines_0, ysize, xsize)
filtered_lines_2 = utils.extend_lines(filtered_lines_1, ysize)
filtered_lines_3 = utils.interpolate(filtered_lines_2, xsize, ysize)
for line in filtered_lines_3:
for x1, y1, x2, y2 in line:
cv2.line(line_image, (x1, y1), (x2, y2), (0, 0, 255), 10)
combo = cv2.addWeighted(image, 1, line_image, 0.3, 0)
return combo
def _runTest(self, browser_config, test_config, setup):
minidump_dir = os.path.join(setup.profile_dir, 'minidumps')
counters = test_config.get(self.platform_type + 'counters', [])
resolution = test_config['resolution']
# add the mainthread_io to the environment variable, as defined
# in test.py configs
here = os.path.dirname(os.path.realpath(__file__))
if test_config['mainthread']:
mainthread_io = os.path.join(here, "mainthread_io.log")
setup.env['MOZ_MAIN_THREAD_IO_LOG'] = mainthread_io
test_config['url'] = utils.interpolate(
test_config['url'],
profile=setup.profile_dir,
firefox=browser_config['browser_path']
)
# setup global (cross-cycle) counters:
# shutdown, responsiveness
global_counters = {}
if browser_config.get('xperf_path'):
for c in test_config.get('xperf_counters', []):
global_counters[c] = []
if test_config['shutdown']:
global_counters['shutdown'] = []
if test_config.get('responsiveness') and \
platform.system() != "Darwin":
# ignore osx for now as per bug 1245793
setup.env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1'
setup.env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = '20'
setup.env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = '10'
global_counters['responsiveness'] = []
# instantiate an object to hold test results
test_results = results.TestResults(
test_config,
global_counters
)
for i in range(test_config['cycles']):
LOG.info("Running cycle %d/%d for %s test..."
% (i+1, test_config['cycles'], test_config['name']))
# remove the browser error file
mozfile.remove(browser_config['error_filename'])
# reinstall any file whose stability we need to ensure across
# the cycles
if test_config.get('reinstall', ''):
for keep in test_config['reinstall']:
origin = os.path.join(test_config['profile_path'],
keep)
dest = os.path.join(setup.profile_dir, keep)
LOG.debug("Reinstalling %s on top of %s"
% (origin, dest))
shutil.copy(origin, dest)
# Run the test
timeout = test_config.get('timeout', 7200) # 2 hours default
if setup.sps_profile:
# When profiling, give the browser some extra time
# to dump the profile.
timeout += 5 * 60
command_args = utils.GenerateBrowserCommandLine(
browser_config["browser_path"],
browser_config["extra_args"],
setup.profile_dir,
test_config['url'],
profiling_info=(setup.sps_profile.profiling_info
if setup.sps_profile else None)
)
mainthread_error_count = 0
if test_config['setup']:
# Generate bcontroller.json for xperf
talosconfig.generateTalosConfig(command_args,
browser_config,
test_config)
subprocess.call(
['python'] + test_config['setup'].split(),
)
counter_management = None
if counters:
counter_management = CounterManagement(
browser_config['process'],
counters,
resolution
)
try:
pcontext = run_browser(
command_args,
minidump_dir,
timeout=timeout,
env=setup.env,
# start collecting counters as soon as possible
on_started=(counter_management.start
if counter_management else None),
)
finally:
if counter_management:
counter_management.stop()
if test_config['mainthread']:
rawlog = os.path.join(here, "mainthread_io.log")
if os.path.exists(rawlog):
processedlog = \
os.path.join(here, 'mainthread_io.json')
xre_path = \
os.path.dirname(browser_config['browser_path'])
mtio_py = os.path.join(here, 'mainthreadio.py')
command = ['python', mtio_py, rawlog,
processedlog, xre_path]
mtio = subprocess.Popen(command,
env=os.environ.copy(),
stdout=subprocess.PIPE)
output, stderr = mtio.communicate()
for line in output.split('\n'):
if line.strip() == "":
continue
print line
mainthread_error_count += 1
mozfile.remove(rawlog)
if test_config['cleanup']:
# HACK: add the pid to support xperf where we require
# the pid in post processing
talosconfig.generateTalosConfig(command_args,
browser_config,
test_config,
pid=pcontext.pid)
subprocess.call(
[sys.executable] + test_config['cleanup'].split()
)
# For startup tests, we launch the browser multiple times
# with the same profile
for fname in ('sessionstore.js', '.parentlock',
'sessionstore.bak'):
mozfile.remove(os.path.join(setup.profile_dir, fname))
# check for xperf errors
if os.path.exists(browser_config['error_filename']) or \
mainthread_error_count > 0:
raise TalosRegression(
"Talos has found a regression, if you have questions"
" ask for help in irc on #perf"
)
# add the results from the browser output
test_results.add(
'\n'.join(pcontext.output),
counter_results=(counter_management.results()
if counter_management
else None)
)
if setup.sps_profile:
setup.sps_profile.symbolicate(i)
self.check_for_crashes(browser_config, minidump_dir,
test_config['name'])
# include global (cross-cycle) counters
test_results.all_counter_results.extend(
[{key: value} for key, value in global_counters.items()]
)
for c in test_results.all_counter_results:
for key, value in c.items():
LOG.debug("COUNTER %r: %s" % (key, value))
# return results
return test_results
def _runTest(self, browser_config, test_config, setup):
counters = test_config.get(self.platform_type + "counters", [])
resolution = test_config["resolution"]
# add the mainthread_io to the environment variable, as defined
# in test.py configs
here = os.path.dirname(os.path.realpath(__file__))
if test_config["mainthread"]:
mainthread_io = os.path.join(here, "mainthread_io.log")
setup.env["MOZ_MAIN_THREAD_IO_LOG"] = mainthread_io
test_config["url"] = utils.interpolate(
test_config["url"], profile=setup.profile_dir, firefox=browser_config["browser_path"]
)
# setup global (cross-cycle) counters:
# shutdown, responsiveness
global_counters = {}
if browser_config.get("xperf_path"):
for c in test_config.get("xperf_counters", []):
global_counters[c] = []
if test_config["shutdown"]:
global_counters["shutdown"] = []
if test_config.get("responsiveness") and platform.system() != "Linux":
# ignore responsiveness tests on linux until we fix
# Bug 710296
setup.env["MOZ_INSTRUMENT_EVENT_LOOP"] = "1"
setup.env["MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD"] = "20"
setup.env["MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL"] = "10"
global_counters["responsiveness"] = []
# instantiate an object to hold test results
test_results = results.TestResults(test_config, global_counters)
for i in range(test_config["cycles"]):
logging.info("Running cycle %d/%d for %s test...", i + 1, test_config["cycles"], test_config["name"])
# remove the browser error file
mozfile.remove(browser_config["error_filename"])
# reinstall any file whose stability we need to ensure across
# the cycles
if test_config.get("reinstall", ""):
for keep in test_config["reinstall"]:
origin = os.path.join(test_config["profile_path"], keep)
dest = os.path.join(setup.profile_dir, keep)
logging.debug("Reinstalling %s on top of %s", origin, dest)
shutil.copy(origin, dest)
# Run the test
timeout = test_config.get("timeout", 7200) # 2 hours default
if setup.sps_profile:
# When profiling, give the browser some extra time
# to dump the profile.
timeout += 5 * 60
command_args = utils.GenerateBrowserCommandLine(
browser_config["browser_path"],
browser_config["extra_args"],
setup.profile_dir,
test_config["url"],
profiling_info=(setup.sps_profile.profiling_info if setup.sps_profile else None),
)
mainthread_error_count = 0
if test_config["setup"]:
# Generate bcontroller.json for xperf
talosconfig.generateTalosConfig(command_args, browser_config, test_config)
subprocess.call(["python"] + test_config["setup"].split())
mm_httpd = None
if test_config["name"] == "media_tests":
from startup_test.media import media_manager
mm_httpd = media_manager.run_server(os.path.dirname(os.path.realpath(__file__)))
counter_management = None
if counters:
counter_management = CounterManagement(browser_config["process"], counters, resolution)
try:
pcontext = run_browser(
command_args,
timeout=timeout,
env=setup.env,
# start collecting counters as soon as possible
on_started=(counter_management.start if counter_management else None),
)
finally:
if counter_management:
counter_management.stop()
if mm_httpd:
mm_httpd.stop()
if test_config["mainthread"]:
rawlog = os.path.join(here, "mainthread_io.log")
if os.path.exists(rawlog):
processedlog = os.path.join(here, "mainthread_io.json")
xre_path = os.path.dirname(browser_config["browser_path"])
mtio_py = os.path.join(here, "mainthreadio.py")
command = ["python", mtio_py, rawlog, processedlog, xre_path]
mtio = subprocess.Popen(command, env=os.environ.copy(), stdout=subprocess.PIPE)
output, stderr = mtio.communicate()
for line in output.split("\n"):
if line.strip() == "":
continue
print line
mainthread_error_count += 1
mozfile.remove(rawlog)
if test_config["cleanup"]:
# HACK: add the pid to support xperf where we require
# the pid in post processing
talosconfig.generateTalosConfig(command_args, browser_config, test_config, pid=pcontext.pid)
subprocess.call([sys.executable] + test_config["cleanup"].split())
# For startup tests, we launch the browser multiple times
# with the same profile
for fname in ("sessionstore.js", ".parentlock", "sessionstore.bak"):
mozfile.remove(os.path.join(setup.profile_dir, fname))
# check for xperf errors
if os.path.exists(browser_config["error_filename"]) or mainthread_error_count > 0:
raise TalosRegression(
"Talos has found a regression, if you have questions" " ask for help in irc on #perf"
)
# add the results from the browser output
test_results.add(
"\n".join(pcontext.output),
counter_results=(counter_management.results() if counter_management else None),
)
if setup.sps_profile:
setup.sps_profile.symbolicate(i)
self.check_for_crashes(browser_config, setup.profile_dir, test_config["name"])
# include global (cross-cycle) counters
test_results.all_counter_results.extend([{key: value} for key, value in global_counters.items()])
for c in test_results.all_counter_results:
for key, value in c.items():
logging.debug("COUNTER %r: %s", key, value)
# return results
return test_results
def interpolate_edge(edge, ts=0.5, size=(1080, 810)):
""" Return interpolated set of images from dataset with that lie on the edge. """
global n_classes, X, y, pp, X_tr, X_inv
if type(ts) is not list:
ts = [ts]
return [to_image(invert(interpolate(X_tr[edge[0]], X_tr[edge[1]], t / 100.0)), size) for t in ts]
def run_tests(config, browser_config):
"""Runs the talos tests on the given configuration and generates a report.
"""
# get the test data
tests = config['tests']
tests = useBaseTestDefaults(config.get('basetest', {}), tests)
paths = ['profile_path', 'tpmanifest', 'extensions', 'setup', 'cleanup']
for test in tests:
# Check for profile_path, tpmanifest and interpolate based on Talos
# root https://bugzilla.mozilla.org/show_bug.cgi?id=727711
# Build command line from config
for path in paths:
if test.get(path):
test[path] = utils.interpolate(test[path])
if test.get('tpmanifest'):
test['tpmanifest'] = \
os.path.normpath('file:/%s' % (urllib.quote(test['tpmanifest'],
'/\\t:\\')))
if not test.get('url'):
# build 'url' for tptest
test['url'] = buildCommandLine(test)
test['url'] = utils.interpolate(test['url'])
test['setup'] = utils.interpolate(test['setup'])
test['cleanup'] = utils.interpolate(test['cleanup'])
# pass --no-remote to firefox launch, if --develop is specified
# we do that to allow locally the user to have another running firefox
# instance
if browser_config['develop']:
browser_config['extra_args'] = '--no-remote'
# with addon signing for production talos, we want to develop without it
if browser_config['develop'] or browser_config['branch_name'] == 'Try':
browser_config['preferences']['xpinstall.signatures.required'] = False
browser_config['extensions'] = [os.path.dirname(i)
for i in browser_config['extensions']]
# set defaults
title = config.get('title', '')
testdate = config.get('testdate', '')
if browser_config['e10s'] and not title.endswith(".e"):
# we are running in e10s mode
title = "%s.e" % (title,)
# get the process name from the path to the browser
if not browser_config['process']:
browser_config['process'] = \
os.path.basename(browser_config['browser_path'])
# fix paths to substitute
# `os.path.dirname(os.path.abspath(__file__))` for ${talos}
# https://bugzilla.mozilla.org/show_bug.cgi?id=705809
browser_config['extensions'] = [utils.interpolate(i)
for i in browser_config['extensions']]
browser_config['bcontroller_config'] = \
utils.interpolate(browser_config['bcontroller_config'])
# normalize browser path to work across platforms
browser_config['browser_path'] = \
os.path.normpath(browser_config['browser_path'])
binary = browser_config["browser_path"]
version_info = mozversion.get_version(binary=binary)
browser_config['browser_name'] = version_info['application_name']
browser_config['browser_version'] = version_info['application_version']
browser_config['buildid'] = version_info['application_buildid']
try:
browser_config['repository'] = version_info['application_repository']
browser_config['sourcestamp'] = version_info['application_changeset']
except KeyError:
if not browser_config['develop']:
print "unable to find changeset or repository: %s" % version_info
sys.exit()
else:
browser_config['repository'] = 'develop'
browser_config['sourcestamp'] = 'develop'
# get test date in seconds since epoch
if testdate:
date = int(time.mktime(time.strptime(testdate,
'%a, %d %b %Y %H:%M:%S GMT')))
else:
date = int(time.time())
LOG.debug("using testdate: %d" % date)
LOG.debug("actual date: %d" % int(time.time()))
# results container
talos_results = TalosResults()
# results links
if not browser_config['develop']:
results_urls = dict(
# another hack; datazilla stands for Perfherder
# and do not require url, but a non empty dict is required...
datazilla_urls=['local.json'],
)
else:
# local mode, output to files
results_urls = dict(datazilla_urls=[os.path.abspath('local.json')])
talos_results.check_output_formats(results_urls)
httpd = setup_webserver(browser_config['webserver'])
httpd.start()
testname = None
# run the tests
timer = utils.Timer()
LOG.suite_start(tests=[test['name'] for test in tests])
try:
for test in tests:
testname = test['name']
LOG.test_start(testname)
mytest = TTest()
talos_results.add(mytest.runTest(browser_config, test))
LOG.test_end(testname, status='OK')
except TalosRegression as exc:
LOG.error("Detected a regression for %s" % testname)
# by returning 1, we report an orange to buildbot
# http://docs.buildbot.net/latest/developer/results.html
LOG.test_end(testname, status='FAIL', message=unicode(exc),
stack=traceback.format_exc())
return 1
except Exception as exc:
# NOTE: if we get into this condition, talos has an internal
# problem and cannot continue
# this will prevent future tests from running
LOG.test_end(testname, status='ERROR', message=unicode(exc),
stack=traceback.format_exc())
# indicate a failure to buildbot, turn the job red
return 2
finally:
LOG.suite_end()
httpd.stop()
LOG.info("Completed test suite (%s)" % timer.elapsed())
# output results
if results_urls:
talos_results.output(results_urls)
if browser_config['develop']:
print ("Thanks for running Talos locally. Results are in %s"
% (results_urls['datazilla_urls']))
# we will stop running tests on a failed test, or we will return 0 for
# green
return 0
heu_acc_errors = pickle.load(open('heu_acc_error.pickle', 'r'))
occurrences = pickle.load(open('triple_occurrences.pickle', 'r'))
# Compute average values, from the
# accumulated averages
for i, count in enumerate(occurrences):
if count > 0:
ext_acc_errors[i] = ext_acc_errors[i] / count
max_ent_acc_errors[i] = max_ent_acc_errors[i] / count
ind_acc_errors[i] = ind_acc_errors[i] / count
heu_acc_errors[i] = heu_acc_errors[i] / count
baseline_acc_errors[i] = baseline_acc_errors[i] / count
interpolate(ext_acc_errors)
interpolate(max_ent_acc_errors)
interpolate(ind_acc_errors)
interpolate(heu_acc_errors)
interpolate(baseline_acc_errors)
max_count = 100
plot(range(max_count), max_ent_acc_errors[:max_count], color='blue')
plot(range(max_count), ext_acc_errors[:max_count], color='red')
# plot(range(max_count), ind_acc_errors[:max_count], color='green')
# plot(range(max_count), heu_acc_errors[:max_count], color='yellow')
# plot(range(max_count), baseline_acc_errors[:max_count], color='purple')
def calc_avg_errors(output_folder):
def format(self, val, *args, **kwds): #if val is None: return "-" # FIXME: When/where did this happen? res, interp_val = utils.interpolate(val, self.curve) if not res: return "<out-of-range: %s>" % (interp_val) return CustomFormatter.format(self, interp_val)
def get_position(self, position):
return utils.interpolate(
position,
utils.vector_add(position, self.element.default_size),
-0.5)
def _runTest(self, browser_config, test_config, setup):
minidump_dir = os.path.join(setup.profile_dir, 'minidumps')
counters = test_config.get('%s_counters' % self._get_counter_prefix(), [])
resolution = test_config['resolution']
# add the mainthread_io to the environment variable, as defined
# in test.py configs
here = os.path.dirname(os.path.realpath(__file__))
if test_config['mainthread']:
mainthread_io = os.path.join(here, 'mainthread_io.log')
setup.env['MOZ_MAIN_THREAD_IO_LOG'] = mainthread_io
if browser_config['disable_stylo']:
if browser_config['stylothreads']:
raise TalosError('--disable-stylo conflicts with --stylo-threads')
if browser_config['enable_stylo']:
raise TalosError('--disable-stylo conflicts with --enable-stylo')
# As we transition to Stylo, we need to set env vars and output data properly
if browser_config['enable_stylo']:
setup.env['STYLO_FORCE_ENABLED'] = '1'
if browser_config['disable_stylo']:
setup.env['STYLO_FORCE_DISABLED'] = '1'
# During the Stylo transition, measure different number of threads
if browser_config.get('stylothreads', 0) > 0:
setup.env['STYLO_THREADS'] = str(browser_config['stylothreads'])
# set url if there is one (i.e. receiving a test page, not a manifest/pageloader test)
if test_config.get('url', None) is not None:
test_config['url'] = utils.interpolate(
test_config['url'],
profile=setup.profile_dir,
firefox=browser_config['browser_path']
)
# setup global (cross-cycle) responsiveness counters
global_counters = {}
if browser_config.get('xperf_path'):
for c in test_config.get('xperf_counters', []):
global_counters[c] = []
if test_config.get('responsiveness') and \
platform.system() != 'Darwin':
# ignore osx for now as per bug 1245793
setup.env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1'
setup.env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = '20'
setup.env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = '10'
global_counters['responsiveness'] = []
setup.env['JSGC_DISABLE_POISONING'] = '1'
setup.env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '1'
# if using mitmproxy we must allow access to 'external' sites
if browser_config.get('mitmproxy', False):
LOG.info('Using mitmproxy so setting MOZ_DISABLE_NONLOCAL_CONNECTIONS to 0')
setup.env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '0'
# instantiate an object to hold test results
test_results = results.TestResults(
test_config,
global_counters,
browser_config.get('framework')
)
for i in range(test_config['cycles']):
LOG.info('Running cycle %d/%d for %s test...'
% (i+1, test_config['cycles'], test_config['name']))
# remove the browser error file
mozfile.remove(browser_config['error_filename'])
# reinstall any file whose stability we need to ensure across
# the cycles
if test_config.get('reinstall', ''):
for keep in test_config['reinstall']:
origin = os.path.join(test_config['profile_path'],
keep)
dest = os.path.join(setup.profile_dir, keep)
LOG.debug('Reinstalling %s on top of %s'
% (origin, dest))
shutil.copy(origin, dest)
# Run the test
timeout = test_config.get('timeout', 7200) # 2 hours default
if setup.gecko_profile:
# When profiling, give the browser some extra time
# to dump the profile.
timeout += 5 * 60
# store profiling info for pageloader; too late to add it as browser pref
setup.env["TPPROFILINGINFO"] = json.dumps(setup.gecko_profile.profiling_info)
command_args = utils.GenerateBrowserCommandLine(
browser_config['browser_path'],
browser_config['extra_args'],
setup.profile_dir,
test_config['url'],
profiling_info=(setup.gecko_profile.profiling_info
if setup.gecko_profile else None)
)
mainthread_error_count = 0
if test_config['setup']:
# Generate bcontroller.json for xperf
talosconfig.generateTalosConfig(command_args,
browser_config,
test_config)
subprocess.call(
['python'] + test_config['setup'].split(),
)
counter_management = None
if counters:
counter_management = CounterManagement(
browser_config['process'],
counters,
resolution
)
try:
pcontext = run_browser(
command_args,
minidump_dir,
timeout=timeout,
env=setup.env,
# start collecting counters as soon as possible
on_started=(counter_management.start
if counter_management else None),
debug=browser_config['debug'],
debugger=browser_config['debugger'],
debugger_args=browser_config['debugger_args']
)
except Exception:
self.check_for_crashes(browser_config, minidump_dir,
test_config['name'])
raise
finally:
if counter_management:
counter_management.stop()
if test_config['mainthread']:
rawlog = os.path.join(here, 'mainthread_io.log')
if os.path.exists(rawlog):
processedlog = \
os.path.join(here, 'mainthread_io.json')
xre_path = \
os.path.dirname(browser_config['browser_path'])
mtio_py = os.path.join(here, 'mainthreadio.py')
command = ['python', mtio_py, rawlog,
processedlog, xre_path]
mtio = subprocess.Popen(command,
env=os.environ.copy(),
stdout=subprocess.PIPE)
output, stderr = mtio.communicate()
for line in output.split('\n'):
if line.strip() == '':
continue
print(line)
mainthread_error_count += 1
mozfile.remove(rawlog)
if test_config['cleanup']:
# HACK: add the pid to support xperf where we require
# the pid in post processing
talosconfig.generateTalosConfig(command_args,
browser_config,
test_config,
pid=pcontext.pid)
subprocess.call(
[sys.executable] + test_config['cleanup'].split()
)
# For startup tests, we launch the browser multiple times
# with the same profile
for fname in ('sessionstore.js', '.parentlock',
'sessionstore.bak'):
mozfile.remove(os.path.join(setup.profile_dir, fname))
# check for xperf errors
if os.path.exists(browser_config['error_filename']) or \
mainthread_error_count > 0:
raise TalosRegression(
'Talos has found a regression, if you have questions'
' ask for help in irc on #perf'
)
# add the results from the browser output
if not run_in_debug_mode(browser_config):
test_results.add(
'\n'.join(pcontext.output),
counter_results=(counter_management.results()
if counter_management
else None)
)
if setup.gecko_profile:
setup.gecko_profile.symbolicate(i)
self.check_for_crashes(browser_config, minidump_dir,
test_config['name'])
# include global (cross-cycle) counters
test_results.all_counter_results.extend(
[{key: value} for key, value in global_counters.items()]
)
for c in test_results.all_counter_results:
for key, value in c.items():
LOG.debug('COUNTER %r: %s' % (key, value))
# if running against a code-coverage instrumented build, move the
# produced gcda files to a folder where they will be collected later
if browser_config.get('code_coverage', False):
setup.collect_or_clean_ccov()
# return results
return test_results
def integrate_improp(f, int_points=1000):
"""Integrate f from a to infinity using transform u = 1/(1-t)"""
interp_start = 0
interp_stop = 1 - 1.0 / int_points
return mean(f(t / (1 - t)) * 1 / (1 - t) ** 2 for t in (interpolate(interp_start, interp_stop, int_points)))
def run_tests(config, browser_config):
"""Runs the talos tests on the given configuration and generates a report.
"""
# get the test data
tests = config['tests']
tests = useBaseTestDefaults(config.get('basetest', {}), tests)
paths = ['profile_path', 'tpmanifest', 'extensions', 'setup', 'cleanup']
for test in tests:
# Check for profile_path, tpmanifest and interpolate based on Talos
# root https://bugzilla.mozilla.org/show_bug.cgi?id=727711
# Build command line from config
for path in paths:
if test.get(path):
test[path] = utils.interpolate(test[path])
if test.get('tpmanifest'):
test['tpmanifest'] = \
os.path.normpath('file:/%s' % (urllib.quote(test['tpmanifest'],
'/\\t:\\')))
if not test.get('url'):
# build 'url' for tptest
test['url'] = buildCommandLine(test)
test['url'] = utils.interpolate(test['url'])
test['setup'] = utils.interpolate(test['setup'])
test['cleanup'] = utils.interpolate(test['cleanup'])
# pass --no-remote to firefox launch, if --develop is specified
if browser_config['develop']:
browser_config['extra_args'] = '--no-remote'
# set defaults
title = config.get('title', '')
testdate = config.get('testdate', '')
if browser_config['e10s'] and not title.endswith(".e"):
# we are running in e10s mode
title = "%s.e" % (title,)
# get the process name from the path to the browser
if not browser_config['process']:
browser_config['process'] = \
os.path.basename(browser_config['browser_path'])
# fix paths to substitute
# `os.path.dirname(os.path.abspath(__file__))` for ${talos}
# https://bugzilla.mozilla.org/show_bug.cgi?id=705809
browser_config['extensions'] = [utils.interpolate(i)
for i in browser_config['extensions']]
browser_config['bcontroller_config'] = \
utils.interpolate(browser_config['bcontroller_config'])
# normalize browser path to work across platforms
browser_config['browser_path'] = \
os.path.normpath(browser_config['browser_path'])
binary = browser_config["browser_path"]
version_info = mozversion.get_version(binary=binary)
browser_config['browser_name'] = version_info['application_name']
browser_config['browser_version'] = version_info['application_version']
browser_config['buildid'] = version_info['application_buildid']
try:
browser_config['repository'] = version_info['application_repository']
browser_config['sourcestamp'] = version_info['application_changeset']
except KeyError:
if not browser_config['develop']:
print "unable to find changeset or repository: %s" % version_info
sys.exit()
else:
browser_config['repository'] = 'develop'
browser_config['sourcestamp'] = 'develop'
# get test date in seconds since epoch
if testdate:
date = int(time.mktime(time.strptime(testdate,
'%a, %d %b %Y %H:%M:%S GMT')))
else:
date = int(time.time())
logging.debug("using testdate: %d", date)
logging.debug("actual date: %d", int(time.time()))
# results container
talos_results = TalosResults(title=title,
date=date,
browser_config=browser_config)
# results links
if not browser_config['develop']:
results_urls = dict(
# hardcoded, this will be removed soon anyway.
results_urls=['http://graphs.mozilla.org/server/collect.cgi'],
# another hack; datazilla stands for Perfherder
# and do not require url, but a non empty dict is required...
datazilla_urls=['local.json'],
)
else:
# local mode, output to files
results_urls = dict(
results_urls=[os.path.abspath('local.out')],
datazilla_urls=[os.path.abspath('local.json')]
)
talos_results.check_output_formats(results_urls)
# setup a webserver, if --develop is specified
httpd = None
if browser_config['develop']:
httpd = setup_webserver(browser_config['webserver'])
if httpd:
httpd.start()
# run the tests
timer = utils.Timer()
logging.info("Starting test suite %s", title)
for test in tests:
testname = test['name']
testtimer = utils.Timer()
logging.info("Starting test %s", testname)
try:
mytest = TTest()
if mytest:
talos_results.add(mytest.runTest(browser_config, test))
else:
logging.error("Error found while running %s", testname)
except TalosRegression:
logging.error("Detected a regression for %s", testname)
if httpd:
httpd.stop()
# by returning 1, we report an orange to buildbot
# http://docs.buildbot.net/latest/developer/results.html
return 1
except (TalosCrash, TalosError):
# NOTE: if we get into this condition, talos has an internal
# problem and cannot continue
# this will prevent future tests from running
traceback.print_exception(*sys.exc_info())
if httpd:
httpd.stop()
# indicate a failure to buildbot, turn the job red
return 2
logging.info("Completed test %s (%s)", testname, testtimer.elapsed())
logging.info("Completed test suite (%s)", timer.elapsed())
# stop the webserver if running
if httpd:
httpd.stop()
# output results
if results_urls:
talos_results.output(results_urls)
if browser_config['develop']:
print
print ("Thanks for running Talos locally. Results are in"
" %s and %s" % (results_urls['results_urls'],
results_urls['datazilla_urls']))
# we will stop running tests on a failed test, or we will return 0 for
# green
return 0
def run_tests(config, browser_config):
"""Runs the talos tests on the given configuration and generates a report.
"""
# get the test data
tests = config['tests']
tests = useBaseTestDefaults(config.get('basetest', {}), tests)
paths = ['profile_path', 'tpmanifest', 'extensions', 'setup', 'cleanup']
for test in tests:
# Check for profile_path, tpmanifest and interpolate based on Talos
# root https://bugzilla.mozilla.org/show_bug.cgi?id=727711
# Build command line from config
for path in paths:
if test.get(path):
test[path] = utils.interpolate(test[path])
if test.get('tpmanifest'):
test['tpmanifest'] = \
os.path.normpath('file:/%s' % (urllib.quote(test['tpmanifest'],
'/\\t:\\')))
test['preferences']['talos.tpmanifest'] = test['tpmanifest']
# if using firstNonBlankPaint, set test preference for it
# so that the browser pref will be turned on (in ffsetup)
if test.get('fnbpaint', False):
LOG.info("Test is using firstNonBlankPaint, browser pref will be turned on")
test['preferences']['dom.performance.time_to_non_blank_paint.enabled'] = True
test['setup'] = utils.interpolate(test['setup'])
test['cleanup'] = utils.interpolate(test['cleanup'])
if not test.get('profile', False):
test['profile'] = config.get('profile')
# pass --no-remote to firefox launch, if --develop is specified
# we do that to allow locally the user to have another running firefox
# instance
if browser_config['develop']:
browser_config['extra_args'] = '--no-remote'
# Pass subtests filter argument via a preference
if browser_config['subtests']:
browser_config['preferences']['talos.subtests'] = browser_config['subtests']
# If --code-coverage files are expected, set flag in browser config so ffsetup knows
# that it needs to delete any ccov files resulting from browser initialization
# NOTE: This is only supported in production; local setup of ccov folders and
# data collection not supported yet, so if attempting to run with --code-coverage
# flag locally, that is not supported yet
if config.get('code_coverage', False):
if browser_config['develop']:
raise TalosError('Aborting: talos --code-coverage flag is only '
'supported in production')
else:
browser_config['code_coverage'] = True
# set defaults
testdate = config.get('testdate', '')
# get the process name from the path to the browser
if not browser_config['process']:
browser_config['process'] = \
os.path.basename(browser_config['browser_path'])
# fix paths to substitute
# `os.path.dirname(os.path.abspath(__file__))` for ${talos}
# https://bugzilla.mozilla.org/show_bug.cgi?id=705809
browser_config['extensions'] = [utils.interpolate(i)
for i in browser_config['extensions']]
browser_config['bcontroller_config'] = \
utils.interpolate(browser_config['bcontroller_config'])
# normalize browser path to work across platforms
browser_config['browser_path'] = \
os.path.normpath(browser_config['browser_path'])
binary = browser_config["browser_path"]
version_info = mozversion.get_version(binary=binary)
browser_config['browser_name'] = version_info['application_name']
browser_config['browser_version'] = version_info['application_version']
browser_config['buildid'] = version_info['application_buildid']
try:
browser_config['repository'] = version_info['application_repository']
browser_config['sourcestamp'] = version_info['application_changeset']
except KeyError:
if not browser_config['develop']:
print("Abort: unable to find changeset or repository: %s" % version_info)
sys.exit(1)
else:
browser_config['repository'] = 'develop'
browser_config['sourcestamp'] = 'develop'
# get test date in seconds since epoch
if testdate:
date = int(time.mktime(time.strptime(testdate,
'%a, %d %b %Y %H:%M:%S GMT')))
else:
date = int(time.time())
LOG.debug("using testdate: %d" % date)
LOG.debug("actual date: %d" % int(time.time()))
# results container
talos_results = TalosResults()
# results links
if not browser_config['develop'] and not config['gecko_profile']:
results_urls = dict(
# another hack; datazilla stands for Perfherder
# and do not require url, but a non empty dict is required...
output_urls=['local.json'],
)
else:
# local mode, output to files
results_urls = dict(output_urls=[os.path.abspath('local.json')])
httpd = setup_webserver(browser_config['webserver'])
httpd.start()
# legacy still required for perfherder data
talos_results.add_extra_option('e10s')
# stylo is another option for testing
if config['enable_stylo']:
talos_results.add_extra_option('stylo')
if config['disable_stylo']:
talos_results.add_extra_option('stylo_disabled')
# measuring the difference of a a certain thread level
if config.get('stylothreads', 0) > 0:
talos_results.add_extra_option('%s_thread' % config['stylothreads'])
if config['gecko_profile']:
talos_results.add_extra_option('geckoProfile')
# some tests use mitmproxy to playback pages
mitmproxy_recordings_list = config.get('mitmproxy', False)
if mitmproxy_recordings_list is not False:
# needed so can tell talos ttest to allow external connections
browser_config['mitmproxy'] = True
# start mitmproxy playback; this also generates the CA certificate
mitmdump_path = config.get('mitmdumpPath', False)
if mitmdump_path is False:
# cannot continue, need path for mitmdump playback tool
raise TalosError('Aborting: mitmdumpPath not provided on cmd line but is required')
mitmproxy_recording_path = os.path.join(here, 'mitmproxy')
mitmproxy_proc = mitmproxy.start_mitmproxy_playback(mitmdump_path,
mitmproxy_recording_path,
mitmproxy_recordings_list.split(),
browser_config['browser_path'])
# install the generated CA certificate into Firefox
# mitmproxy cert setup needs path to mozharness install; mozharness has set this
# value in the SCRIPTSPATH env var for us in mozharness/mozilla/testing/talos.py
scripts_path = os.environ.get('SCRIPTSPATH')
LOG.info('scripts_path: %s' % str(scripts_path))
mitmproxy.install_mitmproxy_cert(mitmproxy_proc,
browser_config['browser_path'],
str(scripts_path))
testname = None
# run the tests
timer = utils.Timer()
LOG.suite_start(tests=[test['name'] for test in tests])
try:
for test in tests:
testname = test['name']
LOG.test_start(testname)
if not test.get('url'):
# set browser prefs for pageloader test setings (doesn't use cmd line args / url)
test['url'] = None
set_tp_preferences(test, browser_config)
mytest = TTest()
# some tests like ts_paint return multiple results in a single iteration
if test.get('firstpaint', False) or test.get('userready', None):
# we need a 'testeventmap' to tell us which tests each event should map to
multi_value_result = None
separate_results_list = []
test_event_map = test.get('testeventmap', None)
if test_event_map is None:
raise TalosError("Need 'testeventmap' in test.py for %s" % test.get('name'))
# run the test
multi_value_result = mytest.runTest(browser_config, test)
if multi_value_result is None:
raise TalosError("Abort: no results returned for %s" % test.get('name'))
# parse out the multi-value results, and 'fake it' to appear like separate tests
separate_results_list = convert_to_separate_test_results(multi_value_result,
test_event_map)
# now we have three separate test results, store them
for test_result in separate_results_list:
talos_results.add(test_result)
# some tests like bloom_basic run two separate tests and then compare those values
# we want the results in perfherder to only be the actual difference between those
# and store the base and reference test replicates in results.json for upload
elif test.get('base_vs_ref', False):
# run the test, results will be reported for each page like two tests in the suite
base_and_reference_results = mytest.runTest(browser_config, test)
# now compare each test, and create a new test object for the comparison
talos_results.add(make_comparison_result(base_and_reference_results))
else:
# just expecting regular test - one result value per iteration
talos_results.add(mytest.runTest(browser_config, test))
LOG.test_end(testname, status='OK')
except TalosRegression as exc:
LOG.error("Detected a regression for %s" % testname)
# by returning 1, we report an orange to buildbot
# http://docs.buildbot.net/latest/developer/results.html
LOG.test_end(testname, status='FAIL', message=str(exc),
stack=traceback.format_exc())
return 1
except Exception as exc:
# NOTE: if we get into this condition, talos has an internal
# problem and cannot continue
# this will prevent future tests from running
LOG.test_end(testname, status='ERROR', message=str(exc),
stack=traceback.format_exc())
# indicate a failure to buildbot, turn the job red
return 2
finally:
LOG.suite_end()
httpd.stop()
LOG.info("Completed test suite (%s)" % timer.elapsed())
# if mitmproxy was used for page playback, stop it
if mitmproxy_recordings_list is not False:
mitmproxy.stop_mitmproxy_playback(mitmproxy_proc)
# output results
if results_urls and not browser_config['no_upload_results']:
talos_results.output(results_urls)
if browser_config['develop'] or config['gecko_profile']:
print("Thanks for running Talos locally. Results are in %s"
% (results_urls['output_urls']))
# we will stop running tests on a failed test, or we will return 0 for
# green
return 0
