def main(args):
if len(args) < 1:
sys.stderr.write("Error - one required argument: <data directory>\n")
sys.exit(-1)
working_dir = args[0]
# print("Reading data...")
Y, X = ctk_io.read_multitask_liblinear(
working_dir) # ('data_testing/multitask_assertion/train_and_test')
stopper = nn_models.get_early_stopper()
num_examples, dimension = X.shape
num_y_examples, num_labels = Y.shape
assert num_examples == num_y_examples
#print("Data has %d examples and dimension %d" % (num_examples, dimension) )
#print("Output has %d dimensions" % (num_labels) )
X = np.reshape(X, (num_examples, 11, dimension / 11))
Y_adj, indices = ctk_io.flatten_outputs(Y)
#print("After reshaping the data has shape %s" % (str(X.shape)))
for label_ind in range(0, Y.shape[1]):
num_outputs = indices[label_ind + 1] - indices[label_ind]
model = nn_models.get_cnn_model(X.shape, num_outputs)
#print("For label ind %d, grabbing indices from %d to %d" % (label_ind, int(indices[label_ind]), int(indices[label_ind+1])))
train_y = Y_adj[:, int(indices[label_ind]):int(indices[label_ind + 1])]
#if(train_y.shape[-1] == 1):
# print("Number of values=1 is %d" % (train_y.sum()))
#print("Shape of y is %s, shape of X is %s, max value in y is %f and min is %f" % (str(train_y.shape), str(X.shape), train_y.max(), train_y.min()) )
model.fit(X,
train_y,
nb_epoch=nb_epoch,
batch_size=batch_size,
verbose=1,
validation_split=0.2,
callbacks=[stopper])
model.summary()
json_string = model.to_json()
open(os.path.join(working_dir, 'model_%d.json' % label_ind),
'w').write(json_string)
model.save_weights(os.path.join(working_dir,
'model_%d.h5' % label_ind),
overwrite=True)
#print("This model has %d layers and layer 3 has %d weights" % (len(model.layers), len(model.layers[3].get_weights()) ) )
#print("The weight of the first layer at index 50 is %f" % model.layers[3].get_weights()[50])
sys.exit(0)
def run_one_eval(epochs, config, train_x, train_y, valid_x, valid_y,
vocab_size, output_dims_list, weights):
print("Running with config: %s" % (config))
np.random.seed(1337)
stopper = nn_models.get_early_stopper()
model = nn_models.get_multitask_cnn(train_x.shape,
vocab_size,
output_dims_list,
conv_layers=config['filters'],
fc_layers=config['layers'],
embed_dim=config['embed_dim'],
filter_widths=config['widths'])
history = model.fit(train_x,
train_y,
nb_epoch=max(2, epochs),
batch_size=config['batch_size'],
verbose=1,
validation_data=(valid_x, valid_y),
callbacks=[stopper])
pred_y = model.predict(valid_x)
tp = [
calc_tp(valid_y[i], pred_y[i], output_dims_list[i])
for i in range(len(pred_y))
]
fp = [
calc_fp(valid_y[i], pred_y[i], output_dims_list[i])
for i in range(len(pred_y))
]
fn = [
calc_fn(valid_y[i], pred_y[i], output_dims_list[i])
for i in range(len(pred_y))
]
if sum(tp) > 0:
print("tp = %s" % tp)
if sum(fp) > 0:
print("fp = %s" % fp)
if sum(fn) > 0:
print("fn = %s" % fn)
recalls = [
0 if tp[i] == 0 else float(tp[i]) / (tp[i] + fn[i])
for i in range(len(pred_y))
]
precision = [
0 if tp[i] == 0 else float(tp[i]) / (tp[i] + fp[i])
for i in range(len(pred_y))
]
f1 = [calc_f1(recalls[i], precision[i]) for i in range(len(pred_y))]
loss = 1 - np.mean(f1)
print("Returning loss: %f" % (loss))
#loss = history.history['val_loss'][-1]
return loss
def main(args):
if len(args) < 1:
sys.stderr.write("Error - one required argument: <data directory>\n")
sys.exit(-1)
working_dir = args[0]
# print("Reading data...")
Y, X = ctk_io.read_multitask_liblinear(working_dir) # ('data_testing/multitask_assertion/train_and_test')
stopper = nn_models.get_early_stopper()
num_examples, dimension = X.shape
num_y_examples, num_labels = Y.shape
assert num_examples == num_y_examples
#print("Data has %d examples and dimension %d" % (num_examples, dimension) )
#print("Output has %d dimensions" % (num_labels) )
X = np.reshape(X, (num_examples, 11, dimension / 11))
Y_adj, indices = ctk_io.flatten_outputs(Y)
#print("After reshaping the data has shape %s" % (str(X.shape)))
for label_ind in range(0, Y.shape[1]):
num_outputs = indices[label_ind+1] - indices[label_ind]
model = nn_models.get_cnn_model(X.shape, num_outputs)
#print("For label ind %d, grabbing indices from %d to %d" % (label_ind, int(indices[label_ind]), int(indices[label_ind+1])))
train_y = Y_adj[:, int(indices[label_ind]):int(indices[label_ind+1])]
#if(train_y.shape[-1] == 1):
# print("Number of values=1 is %d" % (train_y.sum()))
#print("Shape of y is %s, shape of X is %s, max value in y is %f and min is %f" % (str(train_y.shape), str(X.shape), train_y.max(), train_y.min()) )
model.fit(X, train_y,
nb_epoch=nb_epoch,
batch_size=batch_size,
verbose=1,
validation_split=0.2,
callbacks=[stopper])
model.summary()
json_string = model.to_json()
open(os.path.join(working_dir, 'model_%d.json' % label_ind), 'w').write(json_string)
model.save_weights(os.path.join(working_dir, 'model_%d.h5' % label_ind), overwrite=True)
#print("This model has %d layers and layer 3 has %d weights" % (len(model.layers), len(model.layers[3].get_weights()) ) )
#print("The weight of the first layer at index 50 is %f" % model.layers[3].get_weights()[50])
sys.exit(0)
def run_one_eval(epochs, config, train_x, train_y, valid_x, valid_y, vocab_size, output_dims_list, weights):
print("Running with config: %s" % (config) )
np.random.seed(1337)
stopper = nn_models.get_early_stopper()
model = nn_models.get_multitask_cnn(train_x.shape, vocab_size, output_dims_list, conv_layers=config['filters'], fc_layers=config['layers'], embed_dim=config['embed_dim'], filter_widths=config['widths'])
history = model.fit(train_x, train_y,
nb_epoch=max(2, epochs),
batch_size=config['batch_size'],
verbose=1,
validation_data=(valid_x, valid_y),
callbacks=[stopper])
pred_y = model.predict(valid_x)
tp = [calc_tp(valid_y[i], pred_y[i], output_dims_list[i]) for i in range(len(pred_y))]
fp = [calc_fp(valid_y[i], pred_y[i], output_dims_list[i]) for i in range(len(pred_y))]
fn = [calc_fn(valid_y[i], pred_y[i], output_dims_list[i]) for i in range(len(pred_y))]
if sum(tp) > 0:
print("tp = %s" % tp)
if sum(fp) > 0:
print("fp = %s" % fp)
if sum(fn) > 0:
print("fn = %s" % fn)
recalls = [0 if tp[i] == 0 else float(tp[i]) / (tp[i] + fn[i]) for i in range(len(pred_y))]
precision = [0 if tp[i] == 0 else float(tp[i]) / (tp[i] + fp[i]) for i in range(len(pred_y))]
f1 = [calc_f1(recalls[i], precision[i]) for i in range(len(pred_y))]
loss = 1 - np.mean(f1)
print("Returning loss: %f" % (loss) )
#loss = history.history['val_loss'][-1]
return loss
def main(args):
if len(args) < 1:
sys.stderr.write("Error - one required argument: <working_dir>\n")
sys.exit(-1)
working_dir = args[0]
### Extract existing model:
print("Extracting existing model")
with ZipFile(os.path.join(working_dir, 'script.model'), 'r') as myzip:
myzip.extract('model.h5', working_dir)
myzip.extract('alphabets.pkl', working_dir)
(feature_alphabet, label_alphabet) = pickle.load( open(os.path.join(working_dir, 'alphabets.pkl'), 'r' ) )
label_lookup = {val:key for (key,val) in label_alphabet.iteritems()}
model = load_model(os.path.join(working_dir, "model.h5"))
#config = model.get_config()
#model = Container.from_config(config)
## Find the model params needed by CNN method and get a cnn with one extra FC layer:
# nn_models.get_cnn_model(X_array.shape, len(feature_alphabet), num_outputs, conv_layers=convs, fc_layers=layers,
# embed_dim=embed_dim, filter_widths=width)
print("Building new model with extra layer")
convs = []
dense = []
for layer in model.layers:
if 'convolution' in layer.name:
convs.append(layer)
if 'dense' in layer.name:
dense.append(layer)
filters = [x.filter_length for x in convs]
nb_filters = (convs[0].nb_filter,)
fc_widths = [x.output_dim for x in dense]
fc_widths.append(fc_widths[-1] //2)
new_model = nn_models.get_cnn_model(model.layers[0].input_shape, model.layers[1].input_dim, model.layers[-1].output_dim,
conv_layers=nb_filters, fc_layers=fc_widths, embed_dim=model.layers[1].output_dim, filter_widths=filters )
## Just so i don't accidentally try to refer to this later
del model
## Change the name of the output layer so that we don't try to read those weights in -- we will have a different number of parameters:
#new_model.layers[-1].name = "NewOutput"
## Load as many weights as possible taking advantage of consistently named layers:
new_model.load_weights(os.path.join(working_dir, "model.h5"), by_name=True)
## Re-load data and retrain model:
print("Reading data...")
Y, label_alphabet, X_array, feature_alphabet = ctk_io.read_token_sequence_data(working_dir)
Y_array = np.array(Y)
#print("Shape of X is %s and Y is %s" % (str(X.shape), str(Y.shape)))
num_examples, dimension = X_array.shape
num_outputs = 1 if len(label_alphabet) == 2 else len(label_alphabet)
num_y_examples = len(Y)
Y_adj, indices = ctk_io.flatten_outputs(Y_array)
out_counts = Y_adj.sum(0)
stopper = nn_models.get_early_stopper()
print("Retraining model")
new_model.fit(X_array, Y_adj,
nb_epoch=nb_epoch,
batch_size=batch_size,
verbose=1,
validation_split=0.2) #,
#callbacks=[stopper]) #,
#class_weight=class_weights)
new_model.summary()
new_model.save(os.path.join(working_dir, 'new_model.h5'), overwrite=True)
with ZipFile(os.path.join(working_dir, 'extended.model'), 'w') as myzip:
myzip.write(os.path.join(working_dir, 'new_model.h5'), 'model.h5')
myzip.write(os.path.join(working_dir, 'alphabets.pkl'), 'alphabets.pkl')
def main(args):
if len(args) < 1:
sys.stderr.write("Error - one required argument: <working_dir>\n")
sys.exit(-1)
working_dir = args[0]
### Extract existing model:
print("Extracting existing model")
with ZipFile(os.path.join(working_dir, 'script.model'), 'r') as myzip:
myzip.extract('model.h5', working_dir)
myzip.extract('alphabets.pkl', working_dir)
(feature_alphabet, label_alphabet) = pickle.load(
open(os.path.join(working_dir, 'alphabets.pkl'), 'r'))
label_lookup = {val: key for (key, val) in label_alphabet.iteritems()}
model = load_model(os.path.join(working_dir, "model.h5"))
#config = model.get_config()
#model = Container.from_config(config)
## Find the model params needed by CNN method and get a cnn with one extra FC layer:
# nn_models.get_cnn_model(X_array.shape, len(feature_alphabet), num_outputs, conv_layers=convs, fc_layers=layers,
# embed_dim=embed_dim, filter_widths=width)
print("Building new model with extra layer")
convs = []
dense = []
for layer in model.layers:
if 'convolution' in layer.name:
convs.append(layer)
if 'dense' in layer.name:
dense.append(layer)
filters = [x.filter_length for x in convs]
nb_filters = (convs[0].nb_filter, )
fc_widths = [x.output_dim for x in dense]
#fc_widths.append(fc_widths[-1] //2)
fc_widths.append(fc_widths[-1])
new_model = nn_models.get_cnn_model(model.layers[0].input_shape,
model.layers[1].input_dim,
model.layers[-1].output_dim,
conv_layers=nb_filters,
fc_layers=fc_widths,
embed_dim=model.layers[1].output_dim,
filter_widths=filters)
## Just so i don't accidentally try to refer to this later
del model
## Change the name of the output layer so that we don't try to read those weights in -- we will have a different number of parameters:
#new_model.layers[-1].name = "NewOutput"
## Load as many weights as possible taking advantage of consistently named layers:
new_model.load_weights(os.path.join(working_dir, "model.h5"), by_name=True)
## Re-load data and retrain model:
print("Reading data...")
Y, label_alphabet, X_array, feature_alphabet = ctk_io.read_token_sequence_data(
working_dir)
Y_array = np.array(Y)
#print("Shape of X is %s and Y is %s" % (str(X.shape), str(Y.shape)))
num_examples, dimension = X_array.shape
num_outputs = 1 if len(label_alphabet) == 2 else len(label_alphabet)
num_y_examples = len(Y)
Y_adj, indices = ctk_io.flatten_outputs(Y_array)
out_counts = Y_adj.sum(0)
stopper = nn_models.get_early_stopper()
print("Retraining model")
new_model.fit(X_array,
Y_adj,
nb_epoch=nb_epoch,
batch_size=batch_size,
verbose=1,
validation_split=0.2) #,
#callbacks=[stopper]) #,
#class_weight=class_weights)
new_model.summary()
new_model.save(os.path.join(working_dir, 'new_model.h5'), overwrite=True)
with ZipFile(os.path.join(working_dir, 'extended.model'), 'w') as myzip:
myzip.write(os.path.join(working_dir, 'new_model.h5'), 'model.h5')
myzip.write(os.path.join(working_dir, 'alphabets.pkl'),
'alphabets.pkl')
def main(args):
if len(args) < 1:
sys.stderr.write("Error - one required argument: <data directory>\n")
sys.exit(-1)
working_dir = args[0]
print("Reading data...")
Y, outcome_map, outcome_list, X, feature_alphabet = ctk_io.read_multitask_token_sequence_data(working_dir) # ('data_testing/multitask_assertion/train_and_test')
print("Shape of X is %s and Y is %s" % (str(X.shape), str(Y.shape)))
num_examples, dimension = X.shape
num_y_examples, num_labels = Y.shape
assert num_examples == num_y_examples
#print("Data has %d examples and dimension %d" % (num_examples, dimension) )
#print("Output has %d dimensions" % (num_labels) )
#X = np.reshape(X, (num_examples, 11, dimension / 11))
Y_adj, indices = ctk_io.flatten_outputs(Y)
stopper = nn_models.get_early_stopper()
output_dims_list = []
y_list = []
for i in range(len(indices)-1):
label_dims = indices[i+1] - indices[i]
output_dims_list.append(label_dims)
if label_dims == 1:
y_list.append(Y_adj[:, indices[i]])
else:
y_list.append(Y_adj[:, indices[i]:indices[i+1]])
print("Dimensions of label %d are %s" % (i, str(y_list[-1].shape) ) )
model = nn_models.get_multitask_cnn(X.shape, len(feature_alphabet), output_dims_list, conv_layers=filters, fc_layers=layers,
embed_dim=embed_dim, filter_widths=widths)
#model = nn_models.get_multitask_mlp(X.shape, len(feature_alphabet), output_dims_list, fc_layers=layers, embed_dim=embed_dim)
model.fit(X, y_list,
nb_epoch=nb_epoch,
batch_size=batch_size,
verbose=1,
validation_split=0.2,
callbacks=[stopper])
model.summary()
json_string = model.to_json()
open(os.path.join(working_dir, 'model_0.json'), 'w').write(json_string)
model.save_weights(os.path.join(working_dir, 'model_0.h5'), overwrite=True)
#script_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
fn = open(os.path.join(working_dir, 'alphabets.pkl'), 'w')
pickle.dump( (feature_alphabet, outcome_map, outcome_list), fn)
fn.close()
with ZipFile(os.path.join(working_dir, 'script.model'), 'w') as myzip:
myzip.write(os.path.join(working_dir, 'model_0.json'), 'model_0.json')
myzip.write(os.path.join(working_dir, 'model_0.h5'), 'model_0.h5')
myzip.write(os.path.join(working_dir, 'alphabets.pkl'), 'alphabets.pkl')