def main(args): if len(args) < 1: sys.stderr.write("Error - one required argument: <data directory>\n") sys.exit(-1) working_dir = args[0] # print("Reading data...") Y, X = ctk_io.read_multitask_liblinear( working_dir) # ('data_testing/multitask_assertion/train_and_test') stopper = nn_models.get_early_stopper() num_examples, dimension = X.shape num_y_examples, num_labels = Y.shape assert num_examples == num_y_examples #print("Data has %d examples and dimension %d" % (num_examples, dimension) ) #print("Output has %d dimensions" % (num_labels) ) X = np.reshape(X, (num_examples, 11, dimension / 11)) Y_adj, indices = ctk_io.flatten_outputs(Y) #print("After reshaping the data has shape %s" % (str(X.shape))) for label_ind in range(0, Y.shape[1]): num_outputs = indices[label_ind + 1] - indices[label_ind] model = nn_models.get_cnn_model(X.shape, num_outputs) #print("For label ind %d, grabbing indices from %d to %d" % (label_ind, int(indices[label_ind]), int(indices[label_ind+1]))) train_y = Y_adj[:, int(indices[label_ind]):int(indices[label_ind + 1])] #if(train_y.shape[-1] == 1): # print("Number of values=1 is %d" % (train_y.sum())) #print("Shape of y is %s, shape of X is %s, max value in y is %f and min is %f" % (str(train_y.shape), str(X.shape), train_y.max(), train_y.min()) ) model.fit(X, train_y, nb_epoch=nb_epoch, batch_size=batch_size, verbose=1, validation_split=0.2, callbacks=[stopper]) model.summary() json_string = model.to_json() open(os.path.join(working_dir, 'model_%d.json' % label_ind), 'w').write(json_string) model.save_weights(os.path.join(working_dir, 'model_%d.h5' % label_ind), overwrite=True) #print("This model has %d layers and layer 3 has %d weights" % (len(model.layers), len(model.layers[3].get_weights()) ) ) #print("The weight of the first layer at index 50 is %f" % model.layers[3].get_weights()[50]) sys.exit(0)
def run_one_eval(epochs, config, train_x, train_y, valid_x, valid_y, vocab_size, output_dims_list, weights): print("Running with config: %s" % (config)) np.random.seed(1337) stopper = nn_models.get_early_stopper() model = nn_models.get_multitask_cnn(train_x.shape, vocab_size, output_dims_list, conv_layers=config['filters'], fc_layers=config['layers'], embed_dim=config['embed_dim'], filter_widths=config['widths']) history = model.fit(train_x, train_y, nb_epoch=max(2, epochs), batch_size=config['batch_size'], verbose=1, validation_data=(valid_x, valid_y), callbacks=[stopper]) pred_y = model.predict(valid_x) tp = [ calc_tp(valid_y[i], pred_y[i], output_dims_list[i]) for i in range(len(pred_y)) ] fp = [ calc_fp(valid_y[i], pred_y[i], output_dims_list[i]) for i in range(len(pred_y)) ] fn = [ calc_fn(valid_y[i], pred_y[i], output_dims_list[i]) for i in range(len(pred_y)) ] if sum(tp) > 0: print("tp = %s" % tp) if sum(fp) > 0: print("fp = %s" % fp) if sum(fn) > 0: print("fn = %s" % fn) recalls = [ 0 if tp[i] == 0 else float(tp[i]) / (tp[i] + fn[i]) for i in range(len(pred_y)) ] precision = [ 0 if tp[i] == 0 else float(tp[i]) / (tp[i] + fp[i]) for i in range(len(pred_y)) ] f1 = [calc_f1(recalls[i], precision[i]) for i in range(len(pred_y))] loss = 1 - np.mean(f1) print("Returning loss: %f" % (loss)) #loss = history.history['val_loss'][-1] return loss
def main(args): if len(args) < 1: sys.stderr.write("Error - one required argument: <data directory>\n") sys.exit(-1) working_dir = args[0] # print("Reading data...") Y, X = ctk_io.read_multitask_liblinear(working_dir) # ('data_testing/multitask_assertion/train_and_test') stopper = nn_models.get_early_stopper() num_examples, dimension = X.shape num_y_examples, num_labels = Y.shape assert num_examples == num_y_examples #print("Data has %d examples and dimension %d" % (num_examples, dimension) ) #print("Output has %d dimensions" % (num_labels) ) X = np.reshape(X, (num_examples, 11, dimension / 11)) Y_adj, indices = ctk_io.flatten_outputs(Y) #print("After reshaping the data has shape %s" % (str(X.shape))) for label_ind in range(0, Y.shape[1]): num_outputs = indices[label_ind+1] - indices[label_ind] model = nn_models.get_cnn_model(X.shape, num_outputs) #print("For label ind %d, grabbing indices from %d to %d" % (label_ind, int(indices[label_ind]), int(indices[label_ind+1]))) train_y = Y_adj[:, int(indices[label_ind]):int(indices[label_ind+1])] #if(train_y.shape[-1] == 1): # print("Number of values=1 is %d" % (train_y.sum())) #print("Shape of y is %s, shape of X is %s, max value in y is %f and min is %f" % (str(train_y.shape), str(X.shape), train_y.max(), train_y.min()) ) model.fit(X, train_y, nb_epoch=nb_epoch, batch_size=batch_size, verbose=1, validation_split=0.2, callbacks=[stopper]) model.summary() json_string = model.to_json() open(os.path.join(working_dir, 'model_%d.json' % label_ind), 'w').write(json_string) model.save_weights(os.path.join(working_dir, 'model_%d.h5' % label_ind), overwrite=True) #print("This model has %d layers and layer 3 has %d weights" % (len(model.layers), len(model.layers[3].get_weights()) ) ) #print("The weight of the first layer at index 50 is %f" % model.layers[3].get_weights()[50]) sys.exit(0)
def run_one_eval(epochs, config, train_x, train_y, valid_x, valid_y, vocab_size, output_dims_list, weights): print("Running with config: %s" % (config) ) np.random.seed(1337) stopper = nn_models.get_early_stopper() model = nn_models.get_multitask_cnn(train_x.shape, vocab_size, output_dims_list, conv_layers=config['filters'], fc_layers=config['layers'], embed_dim=config['embed_dim'], filter_widths=config['widths']) history = model.fit(train_x, train_y, nb_epoch=max(2, epochs), batch_size=config['batch_size'], verbose=1, validation_data=(valid_x, valid_y), callbacks=[stopper]) pred_y = model.predict(valid_x) tp = [calc_tp(valid_y[i], pred_y[i], output_dims_list[i]) for i in range(len(pred_y))] fp = [calc_fp(valid_y[i], pred_y[i], output_dims_list[i]) for i in range(len(pred_y))] fn = [calc_fn(valid_y[i], pred_y[i], output_dims_list[i]) for i in range(len(pred_y))] if sum(tp) > 0: print("tp = %s" % tp) if sum(fp) > 0: print("fp = %s" % fp) if sum(fn) > 0: print("fn = %s" % fn) recalls = [0 if tp[i] == 0 else float(tp[i]) / (tp[i] + fn[i]) for i in range(len(pred_y))] precision = [0 if tp[i] == 0 else float(tp[i]) / (tp[i] + fp[i]) for i in range(len(pred_y))] f1 = [calc_f1(recalls[i], precision[i]) for i in range(len(pred_y))] loss = 1 - np.mean(f1) print("Returning loss: %f" % (loss) ) #loss = history.history['val_loss'][-1] return loss
def main(args): if len(args) < 1: sys.stderr.write("Error - one required argument: <working_dir>\n") sys.exit(-1) working_dir = args[0] ### Extract existing model: print("Extracting existing model") with ZipFile(os.path.join(working_dir, 'script.model'), 'r') as myzip: myzip.extract('model.h5', working_dir) myzip.extract('alphabets.pkl', working_dir) (feature_alphabet, label_alphabet) = pickle.load( open(os.path.join(working_dir, 'alphabets.pkl'), 'r' ) ) label_lookup = {val:key for (key,val) in label_alphabet.iteritems()} model = load_model(os.path.join(working_dir, "model.h5")) #config = model.get_config() #model = Container.from_config(config) ## Find the model params needed by CNN method and get a cnn with one extra FC layer: # nn_models.get_cnn_model(X_array.shape, len(feature_alphabet), num_outputs, conv_layers=convs, fc_layers=layers, # embed_dim=embed_dim, filter_widths=width) print("Building new model with extra layer") convs = [] dense = [] for layer in model.layers: if 'convolution' in layer.name: convs.append(layer) if 'dense' in layer.name: dense.append(layer) filters = [x.filter_length for x in convs] nb_filters = (convs[0].nb_filter,) fc_widths = [x.output_dim for x in dense] fc_widths.append(fc_widths[-1] //2) new_model = nn_models.get_cnn_model(model.layers[0].input_shape, model.layers[1].input_dim, model.layers[-1].output_dim, conv_layers=nb_filters, fc_layers=fc_widths, embed_dim=model.layers[1].output_dim, filter_widths=filters ) ## Just so i don't accidentally try to refer to this later del model ## Change the name of the output layer so that we don't try to read those weights in -- we will have a different number of parameters: #new_model.layers[-1].name = "NewOutput" ## Load as many weights as possible taking advantage of consistently named layers: new_model.load_weights(os.path.join(working_dir, "model.h5"), by_name=True) ## Re-load data and retrain model: print("Reading data...") Y, label_alphabet, X_array, feature_alphabet = ctk_io.read_token_sequence_data(working_dir) Y_array = np.array(Y) #print("Shape of X is %s and Y is %s" % (str(X.shape), str(Y.shape))) num_examples, dimension = X_array.shape num_outputs = 1 if len(label_alphabet) == 2 else len(label_alphabet) num_y_examples = len(Y) Y_adj, indices = ctk_io.flatten_outputs(Y_array) out_counts = Y_adj.sum(0) stopper = nn_models.get_early_stopper() print("Retraining model") new_model.fit(X_array, Y_adj, nb_epoch=nb_epoch, batch_size=batch_size, verbose=1, validation_split=0.2) #, #callbacks=[stopper]) #, #class_weight=class_weights) new_model.summary() new_model.save(os.path.join(working_dir, 'new_model.h5'), overwrite=True) with ZipFile(os.path.join(working_dir, 'extended.model'), 'w') as myzip: myzip.write(os.path.join(working_dir, 'new_model.h5'), 'model.h5') myzip.write(os.path.join(working_dir, 'alphabets.pkl'), 'alphabets.pkl')
def main(args): if len(args) < 1: sys.stderr.write("Error - one required argument: <working_dir>\n") sys.exit(-1) working_dir = args[0] ### Extract existing model: print("Extracting existing model") with ZipFile(os.path.join(working_dir, 'script.model'), 'r') as myzip: myzip.extract('model.h5', working_dir) myzip.extract('alphabets.pkl', working_dir) (feature_alphabet, label_alphabet) = pickle.load( open(os.path.join(working_dir, 'alphabets.pkl'), 'r')) label_lookup = {val: key for (key, val) in label_alphabet.iteritems()} model = load_model(os.path.join(working_dir, "model.h5")) #config = model.get_config() #model = Container.from_config(config) ## Find the model params needed by CNN method and get a cnn with one extra FC layer: # nn_models.get_cnn_model(X_array.shape, len(feature_alphabet), num_outputs, conv_layers=convs, fc_layers=layers, # embed_dim=embed_dim, filter_widths=width) print("Building new model with extra layer") convs = [] dense = [] for layer in model.layers: if 'convolution' in layer.name: convs.append(layer) if 'dense' in layer.name: dense.append(layer) filters = [x.filter_length for x in convs] nb_filters = (convs[0].nb_filter, ) fc_widths = [x.output_dim for x in dense] #fc_widths.append(fc_widths[-1] //2) fc_widths.append(fc_widths[-1]) new_model = nn_models.get_cnn_model(model.layers[0].input_shape, model.layers[1].input_dim, model.layers[-1].output_dim, conv_layers=nb_filters, fc_layers=fc_widths, embed_dim=model.layers[1].output_dim, filter_widths=filters) ## Just so i don't accidentally try to refer to this later del model ## Change the name of the output layer so that we don't try to read those weights in -- we will have a different number of parameters: #new_model.layers[-1].name = "NewOutput" ## Load as many weights as possible taking advantage of consistently named layers: new_model.load_weights(os.path.join(working_dir, "model.h5"), by_name=True) ## Re-load data and retrain model: print("Reading data...") Y, label_alphabet, X_array, feature_alphabet = ctk_io.read_token_sequence_data( working_dir) Y_array = np.array(Y) #print("Shape of X is %s and Y is %s" % (str(X.shape), str(Y.shape))) num_examples, dimension = X_array.shape num_outputs = 1 if len(label_alphabet) == 2 else len(label_alphabet) num_y_examples = len(Y) Y_adj, indices = ctk_io.flatten_outputs(Y_array) out_counts = Y_adj.sum(0) stopper = nn_models.get_early_stopper() print("Retraining model") new_model.fit(X_array, Y_adj, nb_epoch=nb_epoch, batch_size=batch_size, verbose=1, validation_split=0.2) #, #callbacks=[stopper]) #, #class_weight=class_weights) new_model.summary() new_model.save(os.path.join(working_dir, 'new_model.h5'), overwrite=True) with ZipFile(os.path.join(working_dir, 'extended.model'), 'w') as myzip: myzip.write(os.path.join(working_dir, 'new_model.h5'), 'model.h5') myzip.write(os.path.join(working_dir, 'alphabets.pkl'), 'alphabets.pkl')
def main(args): if len(args) < 1: sys.stderr.write("Error - one required argument: <data directory>\n") sys.exit(-1) working_dir = args[0] print("Reading data...") Y, outcome_map, outcome_list, X, feature_alphabet = ctk_io.read_multitask_token_sequence_data(working_dir) # ('data_testing/multitask_assertion/train_and_test') print("Shape of X is %s and Y is %s" % (str(X.shape), str(Y.shape))) num_examples, dimension = X.shape num_y_examples, num_labels = Y.shape assert num_examples == num_y_examples #print("Data has %d examples and dimension %d" % (num_examples, dimension) ) #print("Output has %d dimensions" % (num_labels) ) #X = np.reshape(X, (num_examples, 11, dimension / 11)) Y_adj, indices = ctk_io.flatten_outputs(Y) stopper = nn_models.get_early_stopper() output_dims_list = [] y_list = [] for i in range(len(indices)-1): label_dims = indices[i+1] - indices[i] output_dims_list.append(label_dims) if label_dims == 1: y_list.append(Y_adj[:, indices[i]]) else: y_list.append(Y_adj[:, indices[i]:indices[i+1]]) print("Dimensions of label %d are %s" % (i, str(y_list[-1].shape) ) ) model = nn_models.get_multitask_cnn(X.shape, len(feature_alphabet), output_dims_list, conv_layers=filters, fc_layers=layers, embed_dim=embed_dim, filter_widths=widths) #model = nn_models.get_multitask_mlp(X.shape, len(feature_alphabet), output_dims_list, fc_layers=layers, embed_dim=embed_dim) model.fit(X, y_list, nb_epoch=nb_epoch, batch_size=batch_size, verbose=1, validation_split=0.2, callbacks=[stopper]) model.summary() json_string = model.to_json() open(os.path.join(working_dir, 'model_0.json'), 'w').write(json_string) model.save_weights(os.path.join(working_dir, 'model_0.h5'), overwrite=True) #script_dir = os.path.dirname(os.path.realpath(sys.argv[0])) fn = open(os.path.join(working_dir, 'alphabets.pkl'), 'w') pickle.dump( (feature_alphabet, outcome_map, outcome_list), fn) fn.close() with ZipFile(os.path.join(working_dir, 'script.model'), 'w') as myzip: myzip.write(os.path.join(working_dir, 'model_0.json'), 'model_0.json') myzip.write(os.path.join(working_dir, 'model_0.h5'), 'model_0.h5') myzip.write(os.path.join(working_dir, 'alphabets.pkl'), 'alphabets.pkl')