def il_max_avg_obstacle_clearance():
name = f'samples/{inspect.currentframe().f_code.co_name}.pkl'
sampler.sample(N=10000, alpha=0.1, prior_file='samples/il_prior.pkl', N_sigma=1000,
behavior_func=behavior.obstacle_clearance,
env=environment.RBF2dGymEnv(use_lidar=True), env_kernel=kernel.RBF2dEnvKernelNormal(),
controller=controller.ILController(), controller_kernel=kernel.TransitionKernel(),
target_type='maximal', save=name)
def rrt_max_avg_obstacle_clearance(prior_file='samples/rrt_prior.pkl'):
name = f'samples/{inspect.currentframe().f_code.co_name}.pkl'
sampler.sample(N=10000, alpha=0.1, prior_file=prior_file, N_sigma=1000,
behavior_func=behavior.obstacle_clearance,
env=environment.RBF2dGymEnv(use_lidar=False), env_kernel=kernel.RBF2dEnvKernelNormal(),
controller=controller.RRTController(), controller_kernel=kernel.RRTKernelNormal([-1, -1], [1, 1]),
target_type='maximal', save=name)
def test_sample(self):
settings.DJANGO_SAMPLER_USE_COST = True
sampler.sample('sql', 'SELECT 1', 1, [])
query = models.Query.objects.get()
self.assertEquals('sql', query.query_type)
self.assertEquals('SELECT 1', query.query)
def il_min_legibility(prior_file='samples/il_prior.pkl'):
name = f'samples/{inspect.currentframe().f_code.co_name}.pkl'
sampler.sample(N=10000, alpha=0.1, prior_file=prior_file, N_sigma=1000,
behavior_func=behavior.neg_behavior(behavior.legibility),
env=environment.RBF2dGymEnv(use_lidar=True), env_kernel=kernel.RBF2dEnvKernelNormal(),
controller=controller.ILController(), controller_kernel=kernel.TransitionKernel(),
target_type='maximal', save=name)
def rrt_min_abs_jerkiness():
name = f'samples/{inspect.currentframe().f_code.co_name}.pkl'
sampler.sample(N=10000, alpha=0.1, prior_file='samples/rrt_prior.pkl', N_sigma=1000,
env=environment.RBF2dGymEnv(use_lidar=False), controller=controller.RRTController(),
behavior_func=behavior.min_jerkiness, env_kernel=kernel.RBF2dEnvKernelNormal(),
controller_kernel=kernel.RRTKernelNormal([-1, -1], [1, 1]),
target_type='match', target_behavior=0, save=name)
def create_sample_deltas(sample_size=20000):
'''
This function read the delta data and save a sample of it
'''
nodelta_file = "/mnt/h/FeatureData/all_nodelta_feature_data.csv"
nodelta_sample_file = "../data/sampled_nodelta_feature_data.csv"
sampler.sample(nodelta_file, nodelta_sample_file, sample_size)
def rrt_min_legibility():
name = f'samples/{inspect.currentframe().f_code.co_name}.pkl'
sampler.sample(N=10000, alpha=0.1, prior_file='samples/rrt_prior.pkl', N_sigma=1000,
behavior_func=behavior.neg_behavior(behavior.legibility),
env=environment.RBF2dGymEnv(use_lidar=False), env_kernel=kernel.RBF2dEnvKernelNormal(),
controller=controller.RRTController(), controller_kernel=kernel.RRTKernelNormal([-1, -1], [1, 1]),
target_type='maximal', save=name)
def il_min_abs_jerkiness():
name = f'samples/{inspect.currentframe().f_code.co_name}.pkl'
sampler.sample(N=10000, alpha=0.1, prior_file='samples/il_prior.pkl', N_sigma=1000,
behavior_func=behavior.min_jerkiness,
env=environment.RBF2dGymEnv(use_lidar=True), env_kernel=kernel.RBF2dEnvKernelNormal(),
controller=controller.ILController(), controller_kernel=kernel.TransitionKernel(),
target_type='match', target_behavior=0, save=name)
def sample_preview():
if len(sys.argv) < 3:
print(
'Please specify the path to the WikiNews pages-articles XML to sample from.'
)
return
sampler.sample(sys.argv[2], preview=True)
def ds_min_avg_obstacle_clearance():
name = f'samples/{inspect.currentframe().f_code.co_name}.pkl'
sampler.sample(N=10000, alpha=0.1, prior_file='samples/ds_prior.pkl', N_sigma=1000,
behavior_func=behavior.obstacle_clearance,
env=environment.RBF2dGymEnv(time_limit=500, oob_termination=False, use_lidar=False),
env_kernel=kernel.RBF2dEnvKernelNormal(),
controller=controller.DSController(), controller_kernel=kernel.TransitionKernel(),
target_type='match', target_behavior=0, save=name)
def _find_FC(var_type,
sd,
pauroc_goal,
start_fc,
test,
max_iter=500,
precision=0.003,
scale=0.05):
"""
gradient descent based method for finding the smallest fold change, given the standard deviation of
the sample noise, that produces the desired partial AUROC score
:param var_type: either "uniform", "gamma" or "trend". describes how variance is sampled
:param sd: mean standard deviation of sample noise
:param pauroc_goal: desired partial AUROC value
:param start_fc: where to start looking
:param test: statistical tests that has the shape test(ctrl, exp) and returns an array of p-values
:param max_iter: maximum number of iterations
:param precision: how close the pAUROC with the estimated fold change should be to the desired
pAUROC
:param scale: multiply the step size by
:return: approximate fold change that produces the the desired pAUROC score given the SD
None if max_iter is exceeded
"""
fc = start_fc
if var_type == "uniform":
simulator = lambda fold_change: sample(
var_type, pep_var=sd**2, fold_change=fold_change)
else:
simulator = lambda fold_change: sample(
var_type, beta=(ALPHA - 1) * sd**2, fold_change=fold_change)
for x in range(max_iter):
# simulate data
ctrl, exp, is_changed = simulator(fc)
p_vals = test(ctrl, exp)
# calculate pauroc
predicted = -np.log(p_vals)
fpr, tpr, _ = roc_curve(is_changed, predicted)
try:
idx = next(i for i, v in enumerate(fpr) if v > FDR)
except StopIteration:
idx = len(fpr) - 1
t_fpr, t_tpr = fpr[:idx + 1], tpr[:idx + 1]
t_fpr[-1] = FDR
pauroc = auc(t_fpr, t_tpr) / FDR
if (abs(pauroc_goal - pauroc) < precision):
return fc
# update fold change
fc += scale * (0.75 - pauroc)
def density_scatter(ctrl_data):
"""
Scatterplot of variance vs log2 mean intensity
used for figure 1ABCD
:param ctrl_data: pd.DataFrame containing control data
:return: plot
"""
matplotlib.rc("font", size=15)
f, (ax1, ax2, ax3, ax4) = plt.subplots(1, 4, figsize=(15, 5))
def do_plot(data, ax):
means = np.mean(data.values, axis=1)
vars = np.var(data.values, axis=1)
plot_density_scatter(means, vars, ax)
# Real
rs = np.random.choice(ctrl_data.shape[0], NUM_PEPTIDES, replace=False)
slct_data = ctrl_data.iloc[rs]
# Simulated
# only NUM_ROWS avg_ctrl data provided -> will be based on real data (reordered but none omitted)
ctrl_u, _, _ = sample("uniform", num_changed=0)
ctrl_g, _, _ = sample("gamma", num_changed=0)
ctrl_t, _, _ = sample("trend", num_changed=0)
do_plot(ctrl_u, ax1)
do_plot(ctrl_g, ax2)
do_plot(ctrl_t, ax3)
do_plot(slct_data, ax4)
ax1.set_ylabel("Peptide variance")
ax1.set_title("Uniform variance", fontsize=15)
ax2.set_title("Inverse gamma variance", fontsize=15)
ax3.set_title("Intensity dependent \ninverse gamma variance", fontsize=15)
ax4.set_title("Empirical Data", fontsize=15)
ax1.set_xlabel("Mean $log2$ peptide intensity")
ax2.set_xlabel("Mean $log2$ peptide intensity")
ax3.set_xlabel("Mean $log2$ peptide intensity")
ax4.set_xlabel("Mean $log2$ peptide intensity")
f.tight_layout()
f.subplots_adjust(right=0.94)
# Legend
make_colorbar(ax4)
return f
def triangulate(path):
mesh = sampler.sample(path)
print('>> mesh samples with chew93_Surface')
for f_index in range(mesh.number_of_faces()):
global dihedral_min, dihedral_max
dihedral_max = 0.0
dihedral_min = np.pi
print('face', f_index + 1, 'of', mesh.number_of_faces(), '. . .')
face_mesh = mesh.face_meshes[f_index]
meta_block = mesh.meta_blocks[f_index]
meta_block[MeshkD.MM_PRIO] = PRIORITY_FACTOR
meta_block[MeshkD.MM_SIZE] = USE_SIZE_TEST
meta_block[MeshkD.MM_ADMS] = APPROX_DIST_MULTI_SAMPLING
meta_block[MeshkD.MV_ADTH] = DISTANCE_THRESHOLD
chew93_Surface(face_mesh, meta_block)
print('\rrefining mesh - done (after', meta_block[MeshkD.NV_REFI],
'iterations)')
if meta_block[MeshkD.NV_REFI] > 0:
print('dihedral range: (',
np.rad2deg(dihedral_min),
', ',
np.rad2deg(dihedral_max),
')',
sep='')
mesh.reset_bounding_boxes()
return mesh
def err_bars_peptide_roc(var_type, **kwargs):
"""
Runs multiple rounds of simulations
Summarizes overall ROC scores
:param var_type: either "uniform", "gamma" or "trend". describes how variance is sampled
:param kwargs: optional arguments to be passed to sampler
:return: numpy.ndarray (N_RUNS x number of tests x 3), arr[i][j] = (AUC, PRC, pAUC)
"""
start = time.time()
res = np.zeros((N_RUNS, len(TESTS), 3), dtype=np.float32)
for i in xrange(N_RUNS):
if i % 50 == 0:
print "At iteration %d" % i
ctrl, exp, is_changed = sample(var_type, **kwargs)
p_vals = do_all_tests(ctrl, exp)
res[i, :, 0], res[i, :,
1], res[i, :,
2] = roc_prc_scores(is_changed, p_vals)
end = time.time()
print end - start
return res
def err_bars_peptide_fdr(var_type, **kwargs):
"""
Runs multiple rounds of simulations
Summarizes overall FDR scores
:param var_type: either "uniform", "gamma" or "trend". describes how variance is sampled
:param kwargs: optional arguments to be passed to sampler
:return: numpy.ndarray (N_RUNS x number of tests x 4), arr[i][j] = (FP_raw, TP_raw, FP_adj, TP_adj)
"""
start = time.time()
res = np.zeros((N_RUNS, len(TESTS), 4), dtype=np.float32)
for i in xrange(N_RUNS):
if i % 50 == 0:
print "At iteration %d" % i
ctrl, exp, is_changed = sample(var_type, **kwargs)
p_vals = do_all_tests(ctrl, exp)
res[i, :,
0], res[i, :,
1], res[i, :,
2], res[i, :,
3] = power_analysis(is_changed, p_vals)
# avoid memory filling up with unused variables
del ctrl, exp, is_changed
end = time.time()
print end - start
return res
def test_2_sample_ten(self): "Test size after 1 sample() run" print() mon_crit = 'corpusName' # (allow +/- 1 result length for roundup error) hit_index = sampler.sample(10, [mon_crit]) self.assertTrue(len(hit_index) <= 11) self.assertTrue(len(hit_index) >= 9)
def get(self, page=None):
if page in METALS:
point = sample()
level = point[METALS.index(page)]
level = int((level * 20) + 80) / 100.0 # .8 - 1.
log.info("%s: %s" % (page, level))
return self.text(str(level))
# return self.text(str(1.0))
return self.text("ANIMAS: %s" % (METALS, ))
def get(self, page=None):
if page in METALS:
point = sample()
level = point[METALS.index(page)]
level = int((level * 20) + 80) / 100.0 # .8 - 1.
log.info("%s: %s" % (page, level))
return self.text(str(level))
# return self.text(str(1.0))
return self.text("ANIMAS: %s" % (METALS,))
def run_LSPI():
epsilon = 0.1 #convergence criterion, should this be dynamic?
iteration = 0
max_iterations = 100
distance = float('Inf')
first_time = True
pi = []
distances = []
sample_n = 10 # num of episodes to simulate
while iteration < max_iterations and distance > epsilon:
samples = sampler.sample(
sample_n, pi
) # get samples from simulation (need to sample from these according to prob. dist.?)
phi = basis.calculate_bases(samples)
k = np.size(
phi, 1) #dimensions of basis phi (needs to be altered for matrix)
if first_time:
pi = policy.zero_policy(
k) # initial policy with initial weights zero
old_pi = copy.copy(pi) #old weights of pi
LSQ.calc_weights(
samples, k, phi, pi
) ## Least squares approximation of Q function, calculate and set new weights for pi based on samples
l1 = len(old_pi.weights)
l2 = len(pi.weights)
#compare weights of old_pi and new weights of pi (based on new samples)
if l1 == l2:
diff = old_pi.weights - pi.weights
Linf_norm = np.linalg.norm(diff, np.inf)
L2_norm = np.linalg.norm(diff)
else:
Linf_norm = np.absolute(
np.linalg.norm(old_pi.weights, inf) -
np.linalg.norm(pi.weights, inf))
L2_norm = np.absolute(
np.linalg.norm(old_pi.weights) - np.linalg.norm(pi.weights))
distance = L2_norm #print Linf_norm
distances.append(distance)
print("Distance between weights is " + str(distance) +
" at iteration " + str(iteration))
iteration += 1
first_time = False
return pi, distances
def test_3_query_trace(self):
"Check if we kept coherent trace of query 'crit:val'"
print()
mon_crit = 'corpusName'
hit_index = sampler.sample(1, [mon_crit])
# read index[some_id]['_q']
id0 = next(iter(hit_index))
first_query = hit_index[id0]['_q']
left_side = first_query.split(':')[0]
self.assertEqual(left_side, mon_crit)
def plot_example_roc_curves(var_type):
"""
Figure 1EFG, example curves
:param var_type: type of distribution to use to sample peptide variance, either "uniform", "gamma" or "trend"
:return: plot
"""
matplotlib.rc("font", size=11)
ctrl, exp, is_changed = sample(var_type)
pvals = do_all_tests(ctrl, exp)
return plot_roc_curves(is_changed, pvals)
def rrt_max_illegibility():
name = f'samples/{inspect.currentframe().f_code.co_name}.pkl'
samples = sample(N=10000,
alpha=0.1,
prior_file='samples/rrt_prior.pkl',
N_sigma=1000,
behavior_func=behavior.illegibility_behavior,
env=PandaEnv(),
env_kernel=ReachingEnvKernelNormal(),
controller=RRTController(),
controller_kernel=RRTKernelNormal(),
target_type='maximal',
save=name)
def volcano_multipanel_example(var_type):
"""
Generate panel comparing volcano plots
Compare uniform and inverse gamma
Corresponds to Figure 4B of manuscript
:param var_type: type of distribution to use to sample peptide variance, either "uniform", "gamma" or "trend"
:return: plot
"""
matplotlib.rc("font", size=20)
ctrl, exp, is_changed = sample(var_type)
pvals = do_all_tests(ctrl, exp)
return volcano_multipanel(pvals, ctrl, exp, is_changed)
def mesher(path):
mesh = sampler.sample(path)
print('>> mesh samples with chew93_Surface')
for face_index in range(mesh.number_of_faces()):
print('face', face_index + 1, 'of', mesh.number_of_faces(), '. . .')
vertices, wire_meshes, triangles, _ = mesh.face_meshes[face_index]
triangles.clear()
triangles += chew93_Surface(vertices, wire_meshes)
mesh.reset_bounding_boxes()
return mesh
def ds_improved_min_end_distance():
name = f'samples/{inspect.currentframe().f_code.co_name}.pkl'
samples = sample(N=10000,
alpha=0.1,
prior_file='samples/ds_improved_prior.pkl',
N_sigma=1000,
behavior_func=behavior.ee_distance_behavior,
env=PandaEnv(),
env_kernel=ReachingEnvKernelNormal(),
controller=DSController(typ='improved'),
controller_kernel=TransitionKernel(),
target_type='match',
target_behavior=0,
save=name)
def rrt_min_ee_distance():
name = f'samples/{inspect.currentframe().f_code.co_name}.pkl'
samples = sample(N=10000,
alpha=0.1,
prior_file='samples/rrt_prior.pkl',
N_sigma=1000,
behavior_func=behavior.ee_distance_behavior,
env=PandaEnv(),
env_kernel=ReachingEnvKernelNormal(),
controller=RRTController(),
controller_kernel=RRTKernelNormal(),
target_type='match',
target_behavior=0,
save=name)
def barplot_multipanel(var_type):
"""
Compute FP, TP, FN, TN for raw and adj p-values
Corresponds to Figure 4C in manuscript
:param var_type: type of distribution to use to sample peptide variance, either "uniform", "gamma" or "trend"
:return: plot
"""
matplotlib.rc("font", size=15)
f, axarr = plt.subplots(2, 2, sharey="row", sharex=True, figsize=(5, 7))
ctrl, exp, is_changed = sample(var_type)
pvals = do_all_tests(ctrl, exp)
pvals.drop("fold change", axis=1, inplace=True)
# Edit column titles for pretty printing
pvals.columns = [
(LABEL_MAPPING[l] if l in LABEL_MAPPING else l).replace('\n', ' ')
for l in list(pvals.columns)
]
# Adjust pvals
pvals_a = pd.DataFrame.from_dict(
OrderedDict([(col, multipletests(pvals[col], 0.05, method="fdr_bh")[1])
for col in pvals.columns]))
# Plot
barplot_accuracy(pvals, is_changed, axarr[0][0], axarr[1][0])
barplot_accuracy(pvals_a, is_changed, axarr[0][1], axarr[1][1])
axarr[1][0].set_ylabel("Count")
axarr[0][0].set_title("Raw p-values")
axarr[0][1].set_title("BH adjusted")
f.tight_layout()
# Resize ax
for ax in axarr[0]:
box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width, box.height * 0.5])
# Add legend
handles, labels = axarr[0][1].get_legend_handles_labels()
plt.figlegend(handles, labels, loc="upper center", ncol=2)
return f
def main(n, Jth):
# V = set([xinit])
V = set()
print("Sampling...")
COST = dict()
V |= sample(n, (0., 0., 0.), XDIM, YDIM, ZDIM, COST, ax=None)
# sample_pairs: a set with elements (state0, state1)
Npair = int(n*(n-1)/20)
sample_pairs = sample_data(V, Npair)
## --- COST roadmap --- ##
# create a dictionary COST with entries: (state0, state1) : (opt_cost, opt_time)
print("Constructing (sampled) COST roadmap...")
for p in sample_pairs:
tau = find_tau(*p)
J_opt = Jstar(*p, tau)
COST[p] = (J_opt, tau)
# create svm.SVC object which classifies pairs of states with C and Jth
classifier = trainsvm(COST, C, Jth)
# add every possible edge to COST
print("Completing COST roadmap...")
sys.setrecursionlimit(2*n*n)
N = len(V)*(len(V)-1)
i = 1
for u in V:
for v in V - set([u]):
if i % 1000 == 1:
print(" Processing {} / {}".format(i, N))
cost(u, v, COST) # Computes and stores cost in COST
if i % 50000 == 0:
save_object(COST, 'COST_full_n{}_J{}.pkl'.format(int(n), int(Jth)))
i += 1
V = precompute_reachable_sets(V, COST, classifier, Jth)
print("Saving objects.")
save_object(V, 'V_n{}_J{}.pkl'.format(int(n), int(Jth)))
save_object(COST, 'COST_full_n{}_J{}.pkl'.format(int(n), int(Jth)))
save_object(classifier, 'classifier{}node_J{}.pkl'.format(n, int(Jth)))
print("Success! n: {}, Jth: {}".format(n, int(Jth)))
def batchProcess(self, batch, showSeg):
self.lm.cleargrads()
# reduce
for b in batch:
self.ds.reduceSeg(b)
# inVoc and V
V = len([w for w in self.ds.cObs.uni if self.ds.cObs.uni[w] > 0])
inVoc = self.ds.getInVoc(self.samplingSizeK, mode='uniform')
# set ngram in batch
sampler.setNgram([self.ds.idData[b][1:-1] for b in batch], self.lm,
self.ds, inVoc)
for b in batch:
if showSeg:
print('red<', '_'.join(self.ds.getSegedLine(b)))
self.ds.segData[b] = sampler.sample(self.ds.idData[b][1:-1],
self.lm, self.ds)
if showSeg:
print('add>', '_'.join(self.ds.getSegedLine(b)))
# add
for b in batch:
self.ds.addSeg(b)
# recalc inVoc and V
V = len([w for w in self.ds.cObs.uni if self.ds.cObs.uni[w] > 0])
# calc loss
segedLines = [
module.segmentIdLine(self.ds.idData[b],
[1] + self.ds.segData[b] + [1]) for b in batch
]
# BOS Padding
segedLines = [[segedLine[0] for _ in range(n - 1)] + segedLine
for segedLine in segedLines]
loss = self.lm.getLoss(segedLines, self.ds.getInVoc())
loss.backward()
self.opt.update()
def pvalue_multipanel():
"""
Generate panel comparing p-value distributions
Compare uniform, inverse gamma and trend
Corresponds to Figure 2BDF of manuscript
:return: plot
"""
matplotlib.rc("font", size=15)
f, axarr = plt.subplots(3,
len(TESTS) + 1,
sharex="col",
sharey="row",
figsize=(20, 15))
for i, var_type in enumerate(["uniform", "gamma", "trend"]):
ctrl, exp, _ = sample(var_type, num_changed=0)
pvals = do_all_tests(ctrl, exp)
pvals.drop("fold change", axis=1, inplace=True)
# Edit column titles for pretty printing
pvals.columns = [
LABEL_MAPPING[l] if l in LABEL_MAPPING else l
for l in list(pvals.columns)
]
pvals["ModT \n(2-s., robust)"] = modT(ctrl,
exp,
robust=True,
run_2sample=True)
pvals["ModT \n(1-s., robust)"] = modT(ctrl, exp, robust=True)
plot_pvalue_dist(pvals, axarr[i])
for ax in axarr[i]:
ax.set_xlabel("" if i < 2 else "p-value")
f.tight_layout()
plt.subplots_adjust(hspace=0.4)
return f
def main(args):
# If args['hetero'] is True, g would be a heterogeneous graph.
# Otherwise, it will be a list of homogeneous graphs.
g, features, labels, num_classes, train_idx, val_idx, test_idx, train_mask, \
val_mask, test_mask = load_data(args['dataset'])
if hasattr(torch, 'BoolTensor'):
train_mask = train_mask.bool()
val_mask = val_mask.bool()
test_mask = test_mask.bool()
# features = features.to(args['device'])
features = [f.to(args['device']) for f in features]
labels = labels.to(args['device'])
train_mask = train_mask.to(args['device'])
val_mask = val_mask.to(args['device'])
test_mask = test_mask.to(args['device'])
if args['hetero']:
from model_hetero import SS_HAN
model = SS_HAN(muti_meta_paths=
[[['pa', 'ap'], ['pf', 'fp']],
[['ap', 'pa']],
[['fp', 'pf']]],
in_size=features[0].shape[1],
hidden_size=args['hidden_units'],
out_size=num_classes,
num_heads=args['num_heads'],
dropout=args['dropout']).to(args['device'])
g = g.to(args['device'])
else:
from model import HAN
model = HAN(num_meta_paths=len(g),
in_size=features.shape[1],
hidden_size=args['hidden_units'],
out_size=num_classes,
num_heads=args['num_heads'],
dropout=args['dropout']).to(args['device'])
g = [graph.to(args['device']) for graph in g]
stopper = EarlyStopping(patience=args['patience'])
# loss_fcn = F.binary_cross_entropy_with_logits
loss_fcn = torch.nn.BCEWithLogitsLoss()
optimizer = optim.Adam(model.parameters(), lr=args['lr'],
weight_decay=args['weight_decay'])
# lr_scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.95)
print('*****************************Pre-training Starting*************************************')
for epoch in range(args['pretrain_epochs']):
model.train()
for idx in range(args['batch_size']):
embeddings = model(g, features)
pos_edge_index, neg_edge_index = sample(g, 1)
link_logits = model.calculate_loss(embeddings, pos_edge_index, neg_edge_index)
link_labels = get_link_labels(pos_edge_index, neg_edge_index)
loss = loss_fcn(link_logits, link_labels)
link_probs = link_logits.sigmoid().detach().numpy()
acc = roc_auc_score(link_labels, link_probs)
optimizer.zero_grad()
loss.backward()
optimizer.step()
# print('link_labels : {}'.format(link_labels))
# print('link_probs : {}'.format(link_probs))
print('epoch: {} || batch_size : {} || loss: {} || accuracy: {}'.format(epoch, idx, loss, acc))
# lr_scheduler.step()
early_stop = stopper.step(model, epoch, loss.item(), acc)
if early_stop:
break
filename = './model/ss-han_{}_{:02f}_{:02f}'.format(epoch, loss, acc)
torch.save(model.state_dict(), filename)
print('*****************************Pre-training Ending*************************************')
print('\n')
print('*****************************Fine-tuning Starting*************************************')
# freeze the pretrained parameter
for parms in model.parameters():
parms.requires_grad = False
from model_hetero import Classifier
classifier = Classifier(in_size=args['hidden_units']*args['num_heads'][-1],
hidden_size=128,
out_size=num_classes)
loss_fcn = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(classifier.parameters(), lr=args['lr'],
weight_decay=args['weight_decay'])
for epoch in range(args['fine-tuning_epochs']):
model.train()
embeddings = model(g, features)
output = classifier(embeddings[0])
loss = loss_fcn(output[train_mask], labels[train_mask])
optimizer.zero_grad()
loss.backward()
optimizer.step()
train_acc, train_micro_f1, train_macro_f1 = score(output[train_mask], labels[train_mask])
val_loss, val_acc, val_micro_f1, val_macro_f1 \
= evaluate(model, classifier, g, features, labels, val_mask, loss_fcn)
print('Epoch {:d} | Train Loss {:.4f} | Train Micro f1 {:.4f} | Train Macro f1 {:.4f} | '
'Val Loss {:.4f} | Val Micro f1 {:.4f} | Val Macro f1 {:.4f}'.format(
epoch + 1, loss.item(), train_micro_f1, train_macro_f1, val_loss.item(), val_micro_f1, val_macro_f1))
print('*****************************Fine-tuning Ending*************************************')
test_loss, test_acc, test_micro_f1, test_macro_f1 \
= evaluate(model, classifier, g, features, labels, val_mask, loss_fcn)
print('Test loss {:.4f} | Test Micro f1 {:.4f} | Test Macro f1 {:.4f}'.format(
test_loss.item(), test_micro_f1, test_macro_f1))
def sample_preview():
if len(sys.argv) < 3:
print('Please specify the path to the WikiNews pages-articles XML to sample from.')
return
sampler.sample(sys.argv[2], preview=True)
import entropy
import os
import sys
def postfn(params):
# print('postfn, params:',params)
entropy.loadCosts('../data/costMatrixSea.csv')
sites = '../data/cities_weights.csv'
experiment = entropy.Experiment(0, params[0], params[1], params[2])
simSites = entropy.runEntropy(experiment, sites, False)
return simSites
sites = '../data/cities_weights.csv'
data = entropy.loadHistoricalSites(sites)
eps = threshold.LinearEps(15, 200, 150)
priors = sampler.TophatPrior([0,0,0],[2,2,10])
sampler = sampler.Sampler(N=200, Y=data, postfn=postfn, dist=entropy.distRelative, threads=16)
for pool in sampler.sample(priors, eps):
print("T: {0}, eps: {1:>.4f}, ratio: {2:>.4f}".format(pool.t, pool.eps, pool.ratio))
for i, (mean, std) in enumerate(zip(np.mean(pool.thetas, axis=0), np.std(pool.thetas, axis=0))):
print(u" theta[{0}]: {1:>.4f} \u00B1 {2:>.4f}".format(i, mean,std))
np.savetxt("result_"+str('%.2f')%pool.eps+'.csv', pool.thetas, delimiter=";", fmt='%1.5f')
print(pool.thetas)
np.savetxt("foo.csv", pool.thetas, delimiter=";", fmt='%1.5f')
def generate0(charset, policy, length, f):
n = len(charset)
while 1:
password = ''.join(charset[sample(n, f)] for _ in xrange(length))
if policy is None or policy.accept(password):
return password
