def test_mlp_pickle():
X = np.random.standard_normal((10, 2))
Z = np.random.standard_normal((10, 1))
X, Z = theano_floatx(X, Z)
mlp = Mlp(2, [10], 1, ['tanh'], 'identity', 'squared', max_iter=2)
climin.initialize.randomize_normal(mlp.parameters.data, 0, 1)
mlp.fit(X, Z)
Y = mlp.predict(X)
pickled = cPickle.dumps(mlp)
mlp2 = cPickle.loads(pickled)
Y2 = mlp2.predict(X)
assert np.allclose(Y, Y2)
def test_mlp_pickle():
X = np.random.standard_normal((10, 2))
Z = np.random.standard_normal((10, 1))
X, Z = theano_floatx(X, Z)
mlp = Mlp(2, [10], 1, ['tanh'], 'identity', 'squared', max_iter=2)
climin.initialize.randomize_normal(mlp.parameters.data, 0, 1)
mlp.fit(X, Z)
Y = mlp.predict(X)
pickled = cPickle.dumps(mlp)
mlp2 = cPickle.loads(pickled)
Y2 = mlp2.predict(X)
assert np.allclose(Y, Y2)
def test_mlp_predict():
X = np.random.standard_normal((10, 2))
X, = theano_floatx(X)
mlp = Mlp(2, [10], 1, ['tanh'], 'identity', 'squared', max_iter=10)
mlp.predict(X)
info['n_iter'] += n_iter
row = '%(n_iter)i\t%(time)g\t%(loss)f\t%(val_loss)f\t%(mae_train)s\t%(rmse_train)s\t%(mae_test)s\t%(rmse_test)s' % info
#row = '%(n_iter)i\t%(mae_train)s' % info
results = open('result.txt', 'a')
print row
results.write(row + '\n')
results.close()
with open('pars.pkl', 'wb') as fp:
cp.dump((info['n_iter'], info['best_pars']), fp)
m.parameters.data[...] = info['best_pars']
with open('best_pars.pkl', 'wb') as bp:
cp.dump(info['best_pars'], bp)
Y = m.predict(m.transformedData(X))
TY = m.predict(TX)
output_train = Y * np.std(train_labels, axis=0) + np.mean(train_labels, axis=0)
output_test = TY * np.std(train_labels, axis=0) + np.mean(train_labels, axis=0)
print 'TRAINING SET\n'
print('MAE: %s kcal/mol' % np.abs(output_train - train_labels).mean(axis=0))
print('RMSE: %s kcal/mol' %
np.square(output_train - train_labels).mean(axis=0)**.5)
print 'TESTING SET\n'
print('MAE: %s kcal/mol' % np.abs(output_test - test_labels).mean(axis=0))
print('RMSE: %s kcal/mol' %
np.square(output_test - test_labels).mean(axis=0)**.5)
row = (
"%(n_iter)i\t%(time)g\t%(loss)f\t%(val_loss)f\t%(mae_train)g\t%(rmse_train)g\t%(mae_test)g\t%(rmse_test)g"
% info
)
results = open("result_gpu.txt", "a")
print row
results.write(row + "\n")
results.close()
m.parameters.data[...] = info["best_pars"]
cp.dump(info["best_pars"], open("best_pars.pkl", "w"))
Y = m.predict(m.transformedData(X))
TY = m.predict(TX)
output_train = Y * np.std(train_labels) + np.mean(train_labels)
output_test = TY * np.std(train_labels) + np.mean(train_labels)
print "TRAINING SET\n"
print ("MAE: %5.2f kcal/mol" % np.abs(output_train - train_labels).mean(axis=0))
print ("RMSE: %5.2f kcal/mol" % np.square(output_train - train_labels).mean(axis=0) ** 0.5)
print "TESTING SET\n"
print ("MAE: %5.2f kcal/mol" % np.abs(output_test - test_labels).mean(axis=0))
print ("RMSE: %5.2f kcal/mol" % np.square(output_test - test_labels).mean(axis=0) ** 0.5)
def run_mlp(arch, func, step, batch, X, Z, TX, TZ, wd, opt):
batch_size = batch
#max_iter = max_passes * X.shape[ 0] / batch_size
max_iter = 100000
n_report = X.shape[0] / batch_size
weights = []
input_size = len(X[0])
train_labels = Z
test_labels = TZ
stop = climin.stops.AfterNIterations(max_iter)
pause = climin.stops.ModuloNIterations(n_report)
optimizer = opt, {'step_rate': step}
typ = 'plain'
if typ == 'plain':
m = Mlp(input_size, arch, 1, X, Z, hidden_transfers=func, out_transfer='identity', loss='squared', optimizer=optimizer, batch_size=batch_size, max_iter=max_iter)
elif typ == 'fd':
m = FastDropoutNetwork(2099, [400, 100], 1, X, Z, TX, TZ,
hidden_transfers=['tanh', 'tanh'], out_transfer='identity', loss='squared',
p_dropout_inpt=.1,
p_dropout_hiddens=.2,
optimizer=optimizer, batch_size=batch_size, max_iter=max_iter)
climin.initialize.randomize_normal(m.parameters.data, 0, 1 / np.sqrt(m.n_inpt))
# Transform the test data
#TX = m.transformedData(TX)
TX = np.array([m.transformedData(TX) for _ in range(10)]).mean(axis=0)
losses = []
print 'max iter', max_iter
m.init_weights()
X, Z, TX, TZ = [breze.learn.base.cast_array_to_local_type(i) for i in (X, Z, TX, TZ)]
for layer in m.mlp.layers:
weights.append(m.parameters[layer.weights])
weight_decay = ((weights[0]**2).sum()
+ (weights[1]**2).sum()
+ (weights[2]**2).sum()
+ (weights[3]**2).sum()
)
weight_decay /= m.exprs['inpt'].shape[0]
m.exprs['true_loss'] = m.exprs['loss']
c_wd = wd
m.exprs['loss'] = m.exprs['loss'] + c_wd * weight_decay
'''
weight_decay = ((m.parameters.in_to_hidden**2).sum()
+ (m.parameters.hidden_to_out**2).sum()
+ (m.parameters.hidden_to_hidden_0**2).sum())
weight_decay /= m.exprs['inpt'].shape[0]
m.exprs['true_loss'] = m.exprs['loss']
c_wd = 0.1
m.exprs['loss'] = m.exprs['loss'] + c_wd * weight_decay
'''
mae = T.abs_((m.exprs['output'] * np.std(train_labels) + np.mean(train_labels))- m.exprs['target']).mean()
f_mae = m.function(['inpt', 'target'], mae)
rmse = T.sqrt(T.square((m.exprs['output'] * np.std(train_labels) + np.mean(train_labels))- m.exprs['target']).mean())
f_rmse = m.function(['inpt', 'target'], rmse)
start = time.time()
# Set up a nice printout.
keys = '#', 'seconds', 'loss', 'val loss', 'mae_train', 'rmse_train', 'mae_test', 'rmse_test'
max_len = max(len(i) for i in keys)
header = '\t'.join(i for i in keys)
print header
print '-' * len(header)
results = open('result.txt', 'a')
results.write(header + '\n')
results.write('-' * len(header) + '\n')
results.close()
for i, info in enumerate(m.powerfit((X, Z), (TX, TZ), stop, pause)):
if info['n_iter'] % n_report != 0:
continue
passed = time.time() - start
losses.append((info['loss'], info['val_loss']))
info.update({
'time': passed,
'mae_train': f_mae(m.transformedData(X), train_labels),
'rmse_train': f_rmse(m.transformedData(X), train_labels),
'mae_test': f_mae(TX, test_labels),
'rmse_test': f_rmse(TX, test_labels)
})
row = '%(n_iter)i\t%(time)g\t%(loss)f\t%(val_loss)f\t%(mae_train)g\t%(rmse_train)g\t%(mae_test)g\t%(rmse_test)g' % info
results = open('result.txt','a')
print row
results.write(row + '\n')
results.close()
m.parameters.data[...] = info['best_pars']
cp.dump(info['best_pars'], open('best_pars.pkl', 'w'))
Y = m.predict(m.transformedData(X))
TY = m.predict(TX)
output_train = Y * np.std(train_labels) + np.mean(train_labels)
output_test = TY * np.std(train_labels) + np.mean(train_labels)
print 'TRAINING SET\n'
print('MAE: %5.2f kcal/mol'%np.abs(output_train - train_labels).mean(axis=0))
print('RMSE: %5.2f kcal/mol'%np.square(output_train - train_labels).mean(axis=0) ** .5)
print 'TESTING SET\n'
print('MAE: %5.2f kcal/mol'%np.abs(output_test - test_labels).mean(axis=0))
print('RMSE: %5.2f kcal/mol'%np.square(output_test - test_labels).mean(axis=0) ** .5)
mae_train = np.abs(output_train - train_labels).mean(axis=0)
rmse_train = np.square(output_train - train_labels).mean(axis=0) ** .5
mae_test = np.abs(output_test - test_labels).mean(axis=0)
rmse_test = np.square(output_test - test_labels).mean(axis=0) ** .5
results = open('result.txt', 'a')
results.write('Training set:\n')
results.write('MAE:\n')
results.write("%5.2f" %mae_train)
results.write('\nRMSE:\n')
results.write("%5.2f" %rmse_train)
results.write('\nTesting set:\n')
results.write('MAE:\n')
results.write("%5.2f" %mae_test)
results.write('\nRMSE:\n')
results.write("%5.2f" %rmse_test)
results.close()
def run_mlp(func, step, momentum, X, Z, TX, TZ, wd, opt, counter):
print func, step, momentum, wd, opt, counter
seed = 3453
np.random.seed(seed)
batch_size = 25
# max_iter = max_passes * X.shape[ 0] / batch_size
max_iter = 25000000
n_report = X.shape[0] / batch_size
weights = []
input_size = len(X[0])
stop = climin.stops.AfterNIterations(max_iter)
pause = climin.stops.ModuloNIterations(n_report)
optimizer = opt, {"step_rate": step, "momentum": momentum}
typ = "plain"
if typ == "plain":
m = Mlp(
input_size,
[400, 100],
1,
X,
Z,
hidden_transfers=func,
out_transfer="identity",
loss="squared",
optimizer=optimizer,
batch_size=batch_size,
max_iter=max_iter,
)
elif typ == "fd":
m = FastDropoutNetwork(
2099,
[400, 100],
1,
X,
Z,
TX,
TZ,
hidden_transfers=["tanh", "tanh"],
out_transfer="identity",
loss="squared",
p_dropout_inpt=0.1,
p_dropout_hiddens=0.2,
optimizer=optimizer,
batch_size=batch_size,
max_iter=max_iter,
)
# climin.initialize.randomize_normal(m.parameters.data, 0, 1 / np.sqrt(m.n_inpt))
# Transform the test data
# TX = m.transformedData(TX)
TX = np.array([m.transformedData(TX) for _ in range(10)]).mean(axis=0)
print TX.shape
losses = []
print "max iter", max_iter
m.init_weights()
X, Z, TX, TZ = [breze.learn.base.cast_array_to_local_type(i) for i in (X, Z, TX, TZ)]
for layer in m.mlp.layers:
weights.append(m.parameters[layer.weights])
weight_decay = (weights[0] ** 2).sum() + (weights[1] ** 2).sum() + (weights[2] ** 2).sum()
weight_decay /= m.exprs["inpt"].shape[0]
m.exprs["true_loss"] = m.exprs["loss"]
c_wd = wd
m.exprs["loss"] = m.exprs["loss"] + c_wd * weight_decay
mae = T.abs_((m.exprs["output"] * np.std(train_labels) + np.mean(train_labels)) - m.exprs["target"]).mean()
f_mae = m.function(["inpt", "target"], mae)
rmse = T.sqrt(
T.square((m.exprs["output"] * np.std(train_labels) + np.mean(train_labels)) - m.exprs["target"]).mean()
)
f_rmse = m.function(["inpt", "target"], rmse)
start = time.time()
# Set up a nice printout.
keys = "#", "seconds", "loss", "val loss", "mae_train", "rmse_train", "mae_test", "rmse_test"
max_len = max(len(i) for i in keys)
header = "\t".join(i for i in keys)
print header
print "-" * len(header)
results = open("result_hp.txt", "a")
results.write(header + "\n")
results.write("-" * len(header) + "\n")
results.close()
EXP_DIR = os.getcwd()
base_path = os.path.join(EXP_DIR, "pars_hp" + str(counter) + ".pkl")
n_iter = 0
if os.path.isfile(base_path):
with open("pars_hp" + str(counter) + ".pkl", "rb") as tp:
n_iter, best_pars = cp.load(tp)
m.parameters.data[...] = best_pars
for i, info in enumerate(m.powerfit((X, Z), (TX, TZ), stop, pause)):
if info["n_iter"] % n_report != 0:
continue
passed = time.time() - start
losses.append((info["loss"], info["val_loss"]))
info.update(
{
"time": passed,
"mae_train": f_mae(m.transformedData(X), train_labels),
"rmse_train": f_rmse(m.transformedData(X), train_labels),
"mae_test": f_mae(TX, test_labels),
"rmse_test": f_rmse(TX, test_labels),
}
)
info["n_iter"] += n_iter
row = (
"%(n_iter)i\t%(time)g\t%(loss)f\t%(val_loss)f\t%(mae_train)g\t%(rmse_train)g\t%(mae_test)g\t%(rmse_test)g"
% info
)
results = open("result_hp.txt", "a")
print row
results.write(row + "\n")
results.close()
with open("pars_hp" + str(counter) + ".pkl", "wb") as fp:
cp.dump((info["n_iter"], info["best_pars"]), fp)
with open("hps" + str(counter) + ".pkl", "wb") as tp:
cp.dump((func, step, momentum, wd, opt, counter, info["n_iter"]), tp)
m.parameters.data[...] = info["best_pars"]
cp.dump(info["best_pars"], open("best_pars.pkl", "wb"))
Y = m.predict(m.transformedData(X))
TY = m.predict(TX)
output_train = Y * np.std(train_labels) + np.mean(train_labels)
output_test = TY * np.std(train_labels) + np.mean(train_labels)
print "TRAINING SET\n"
print ("MAE: %5.2f kcal/mol" % np.abs(output_train - train_labels).mean(axis=0))
print ("RMSE: %5.2f kcal/mol" % np.square(output_train - train_labels).mean(axis=0) ** 0.5)
print "TESTING SET\n"
print ("MAE: %5.2f kcal/mol" % np.abs(output_test - test_labels).mean(axis=0))
print ("RMSE: %5.2f kcal/mol" % np.square(output_test - test_labels).mean(axis=0) ** 0.5)
mae_train = np.abs(output_train - train_labels).mean(axis=0)
rmse_train = np.square(output_train - train_labels).mean(axis=0) ** 0.5
mae_test = np.abs(output_test - test_labels).mean(axis=0)
rmse_test = np.square(output_test - test_labels).mean(axis=0) ** 0.5
results = open("result_hp.txt", "a")
results.write("Training set:\n")
results.write("MAE:\n")
results.write("%5.2f" % mae_train)
results.write("\nRMSE:\n")
results.write("%5.2f" % rmse_train)
results.write("\nTesting set:\n")
results.write("MAE:\n")
results.write("%5.2f" % mae_test)
results.write("\nRMSE:\n")
results.write("%5.2f" % rmse_test)
results.close()
class Predictor:
# initialize the object
def __init__(self):
with open('config.txt', 'r') as config_f:
for line in config_f:
if not line.find('mode='):
self.mode = line.replace('mode=', '').replace('\n', '')
if not line.find('robust='):
self.robust = line.replace('robust=', '').replace('\n', '')
print 'mode=%s\nrobustness=%s' %(self.mode, self.robust)
if self.robust == 'majority':
self.pred_count = 0
self.predictions = np.zeros((13,))
if self.robust == 'markov':
self.markov = Markov_Chain()
self.last_state = 0
self.current_state = 0
if self.robust == 'markov_2nd':
self.markov = Markov_Chain_2nd()
self.pre_last_state = 0
self.last_state = 0
self.current_state = 0
self.sample_count = 0
self.sample = []
if self.mode == 'cnn':
self.bin_cm = 10
self.max_x_cm = 440
self.min_x_cm = 70
self.max_y_cm = 250
self.max_z_cm = 200
self.nr_z_intervals = 2
self.x_range = (self.max_x_cm - self.min_x_cm)/self.bin_cm
self.y_range = self.max_y_cm*2/self.bin_cm
self.z_range = self.nr_z_intervals
self.input_size = 3700
self.output_size = 13
self.n_channels = 2
self.im_width = self.y_range
self.im_height = self.x_range
print 'initializing cnn model.'
self.model = Cnn(self.input_size, [16, 32], [200, 200], self.output_size, ['tanh', 'tanh'], ['tanh', 'tanh'],
'softmax', 'cat_ce', image_height=self.im_height, image_width=self.im_width,
n_image_channel=self.n_channels, pool_size=[2, 2], filter_shapes=[[5, 5], [5, 5]], batch_size=1)
self.model.parameters.data[...] = cp.load(open('./best_cnn_pars.pkl', 'rb'))
if self.mode == 'crafted':
self.input_size = 156
self.output_size = 13
self.means = cp.load(open('means_crafted.pkl', 'rb'))
self.stds = cp.load(open('stds_crafted.pkl', 'rb'))
print 'initializing crafted features model.'
self.model = Mlp(self.input_size, [1000, 1000], self.output_size, ['tanh', 'tanh'], 'softmax', 'cat_ce',
batch_size=1)
self.model.parameters.data[...] = cp.load(open('./best_crafted_pars.pkl', 'rb'))
# this is just a trick to make the internal C-functions compile before the first real sample arrives
compile_sample = np.random.random((1,self.input_size))
self.model.predict(compile_sample)
print 'starting to listen to topic.'
self.listener()
# build the full samples from the arriving point clouds
def build_samples(self, sample_part):
for point in read_points(sample_part):
self.sample.append(point)
self.sample_count += 1
if self.sample_count == 6:
if self.mode == 'cnn':
self.cnn_predict()
if self.mode == 'crafted':
self.crafted_predict()
self.sample = []
self.sample_count = 0
# start listening to the point cloud topic
def listener(self):
rospy.init_node('listener', anonymous=True)
rospy.Subscriber("/USArray_pc", PointCloud2, self.build_samples)
rospy.spin()
# let the model predict the output
def cnn_predict(self):
grid = np.zeros((self.z_range, self.x_range, self.y_range))
for point in self.sample:
if point[0]*100 < self.min_x_cm or point[0]*100 > self.max_x_cm-1 or point[1]*100 > self.max_y_cm-1 or point[1]*100 < -self.max_y_cm:
continue
x = (int(point[0]*100) - self.min_x_cm) / self.bin_cm
y = (int(point[1]*100) + self.max_y_cm) / self.bin_cm
z = int(point[2]*100) > (self.max_z_cm / self.nr_z_intervals)
pow = point[4]
if grid[z][x][y] != 0:
if grid[z][x][y] < pow:
grid[z][x][y] = pow
else:
grid[z][x][y] = pow
grid = np.reshape(grid,(1,-1))
self.output_prediction(self.model.predict(grid))
# let the model predict the output
def crafted_predict(self):
vec = np.zeros((156,), dtype=np.float32)
area_points = [[] for _ in np.arange(12)]
area_counts = np.zeros(12)
area_x_means = np.zeros(12)
area_y_means = np.zeros(12)
area_z_means = np.zeros(12)
area_highest = np.zeros(12)
area_highest_pow = np.zeros(12)
area_pow_means = np.zeros(12)
area_x_vars = np.zeros(12)
area_y_vars = np.zeros(12)
area_z_vars = np.zeros(12)
area_xy_covars = np.zeros(12)
area_xz_covars = np.zeros(12)
area_yz_covars = np.zeros(12)
bad = False
for qpoint in self.sample:
# need to substract -1 since the function returns the value starting with 1
label = determine_label((float(qpoint[0]), float(qpoint[1]), float(qpoint[2])))-1
area_points[label].append(qpoint)
area_counts[label] += 1
if float(qpoint[2]) > area_highest[label]:
area_highest[label] = float(qpoint[2])
if float(qpoint[4]) > area_highest_pow[label]:
area_highest_pow[label] = float(qpoint[4])
for area in np.arange(12):
for point in area_points[area]:
area_x_means[area] += float(point[0])
area_y_means[area] += float(point[1])
area_z_means[area] += float(point[2])
area_pow_means[area] += float(point[4])
if area_counts[area] > 0:
area_x_means[area] /= area_counts[area]
area_y_means[area] /= area_counts[area]
area_z_means[area] /= area_counts[area]
area_pow_means[area] /= area_pow_means[area]
for point in area_points[area]:
area_x_vars[area] += (float(point[0]) - area_x_means[area])**2
area_y_vars[area] += (float(point[1]) - area_y_means[area])**2
area_z_vars[area] += (float(point[2]) - area_z_means[area])**2
# if there is only one point, we assume the uncorrected estimator and implicitly divide by one
if area_counts[area] > 1:
area_x_vars[area] *= 1/(area_counts[area]-1)
area_y_vars[area] *= 1/(area_counts[area]-1)
area_z_vars[area] *= 1/(area_counts[area]-1)
for point in area_points[area]:
area_xy_covars[area] += (float(point[0]) - area_x_means[area])*(float(point[1]) - area_y_means[area])
area_xz_covars[area] += (float(point[0]) - area_x_means[area])*(float(point[2]) - area_z_means[area])
area_yz_covars[area] += (float(point[1]) - area_y_means[area])*(float(point[2]) - area_z_means[area])
# if there is only one point, we assume the uncorrected estimator and implicitly divide by one
if area_counts[area] > 1:
area_xy_covars[area] *= 1/(area_counts[area]-1)
area_xz_covars[area] *= 1/(area_counts[area]-1)
area_yz_covars[area] *= 1/(area_counts[area]-1)
for area in np.arange(12):
vec[area*11] = area_counts[area]
vec[area*11+1] = area_x_means[area]
vec[area*11+2] = area_y_means[area]
vec[area*11+3] = area_z_means[area]
vec[area*11+4] = area_x_vars[area]
vec[area*11+5] = area_y_vars[area]
vec[area*11+6] = area_z_vars[area]
vec[area*11+7] = area_xy_covars[area]
vec[area*11+8] = area_xz_covars[area]
vec[area*11+9] = area_yz_covars[area]
vec[area*11+10] = area_highest[area]
vec[area*11+11] = area_highest_pow[area]
vec[area*11+12] = area_pow_means[area]
vec = np.reshape(vec, (1, 156))
vec -= self.means
vec /= self.stds
self.output_prediction(self.model.predict(vec))
# create the output
def output_prediction(self, probabilites):
if self.robust == 'majority':
prediction = np.argmax(probabilites)
# majority vote among the last three predictions
self.predictions[prediction] += 1
self.pred_count += 1
if self.pred_count == 3:
print 'majority prediction: %d' %np.argmax(self.predictions)
self.pred_count = 0
self.predictions = np.zeros((13,))
if self.robust == 'markov':
markov_probs = self.markov.transition_table[self.last_state]
probabilites *= markov_probs
probabilites /= np.sum(probabilites)
prediction = np.argmax(probabilites)
print 'markov prediction: %d' %prediction
self.last_state = prediction
if self.robust == 'markov_2nd':
markov_probs = self.markov.transition_table[self.pre_last_state][self.last_state]
probabilites *= markov_probs
probabilites /= np.sum(probabilites)
prediction = np.argmax(probabilites)
print 'markov 2nd order prediction: %d' %prediction
self.pre_last_state = self.last_state
self.last_state = prediction
if self.robust == 'off':
prediction = np.argmax(probabilites)
print 'fast prediction: %d' %prediction
