def main(): # Load the parameters from json file args = parser.parse_args() json_path = os.path.join(args.model_dir, 'params.json') params = Params(json_path) # Set the logger set_logger(os.path.join(args.model_dir, 'train.log')) # Create the input data pipeline logging.info('Creating the dataset...') data_dir = args.data_dir valid_data_dir = os.path.join(data_dir, 'valid') # Get the filenames and labels from the test set valid_filenames, valid_labels = get_filenames_and_labels( valid_data_dir, params) params.valid_size = len(valid_filenames) params.num_labels = len(set(valid_labels)) # Create the two iterators over the two datasets valid_inputs = input_fn(False, valid_filenames, valid_labels, params) # Define the model logging.info("Creating the model...") model_spec = model_fn('eval', valid_inputs, params, reuse=False) logging.info("Starting evaluation") evaluate(model_spec, args.model_dir, params, args.restore_from)
def main(): # Load the parameters from json file args = parser.parse_args() json_path = os.path.join(args.model_dir, 'params.json') assert os.path.isfile(json_path), 'No json configuration file found at {}'.format(json_path) params = Params(json_path) # Set the logger set_logger(os.path.join(args.model_dir, 'train.log')) if not os.path.exists(args.restore_from): os.makedirs(args.restore_from) # Create the input data pipeline logging.info('Creating the datasets...') data_dir = args.data_dir train_data_dir = os.path.join(data_dir, 'train') valid_data_dir = os.path.join(data_dir, 'valid') # Get the filenames and labels from the train and valid sets train_filenames, train_labels = get_filenames_and_labels( train_data_dir, params) valid_filenames, valid_labels = get_filenames_and_labels( valid_data_dir, params) params.train_size = len(train_filenames) params.valid_size = len(valid_filenames) params.num_labels = len(set(train_labels)) # Create the two iterators over the two datasets train_inputs = input_fn(True, train_filenames, train_labels, params) valid_inputs = input_fn(False, valid_filenames, valid_labels, params) # Define the model logging.info('Creating the model...') train_model_spec = model_fn('train', train_inputs, params) valid_model_spec = model_fn('eval', valid_inputs, params, reuse=True) # Train the model logging.info('Starting training for {} epoch(s)'.format( params.num_epochs)) train_and_evaluate(train_model_spec, valid_model_spec, args.model_dir, params, args.restore_from)
def train(): # Set the logger set_logger(os.path.join(params['model_dir'], 'train.log')) # log params logging.info(params) # Load vacabulary vocab = tf.contrib.lookup.index_table_from_file(vocab_path, num_oov_buckets=1) # Create the input data pipeline logging.info('Creating the datasets...') train_input_words = load_dataset_from_text(data_dir, train_input_filename, vocab) train_context_words = load_dataset_from_text(data_dir, train_context_filename, vocab) # Create the iterator over the dataset train_inputs = input_fn('train', train_input_words, train_context_words, params) eval_inputs = input_fn('eval', train_input_words, train_context_words, params) logging.info("- done") # Define the model logging.info('Creating the model...') train_model_spec = model_fn('train', train_inputs, params, reuse=tf.AUTO_REUSE) eval_model_spec = model_fn('eval', eval_inputs, params, reuse=True) logging.info('- done.') # Train the model logging.info('Starting training for {} epochs'.format( params['num_epochs'])) normalized_embedding_matrix = train_and_evaluate(train_model_spec, eval_model_spec, params) save_dict_to_json(params, params['model_dir'] + '/params.json') pd.DataFrame(normalized_embedding_matrix).to_csv(os.path.join( params['model_dir'], 'normalized_embedding_matrix.tsv'), index=False, header=None, sep='\t')
def funct(x): # Set the random seed for the whole graph tf.set_random_seed(230) # Load the parameters args = parser.parse_args() json_path = os.path.join(args.model_dir, 'params.json') assert os.path.isfile(json_path), "No json configuration file found at {}".format(json_path) params = Params(json_path) # Set the logger set_logger(os.path.join(args.data_dir, 'predict.log')) # Create the input data pipeline data_dir = args.data_dir test_data_dir = os.path.join(data_dir) # Get the filenames from the test set test_filenames = [os.path.join(test_data_dir, 'predict.jpg') ] test_labels = [x] # print(test_labels) # specify the size of the evaluation set params.eval_size = len(test_filenames) # create the iterator over the dataset test_inputs = input_fn(False, test_filenames, test_labels, params) # Define the model model_spec = model_fn('eval', test_inputs, params, reuse=tf.AUTO_REUSE) evaluate(model_spec, args.model_dir, params, args.restore_from)
#TODO: check and load if there's the best weights so far # model_dir_has_best_weights = os.path.isdir(os.path.join(args.model_dir, "best_weights")) #set logger set_logger(os.path.join(args.model_dir, 'train.log')) #train/test split train_fpaths, test_fpaths, train_targets, test_targets = \ get_train_test_split(args.json_path, args.data_dir, train_size=args.train_size) params.train_size = len(train_fpaths) params.test_size = len(test_fpaths) logging.info("Creating the dataset...") train_inputs = input_fn(True, train_fpaths, train_targets, params) test_inputs = input_fn(False, test_fpaths, test_targets, params) logging.info("Creating the model...") train_model_spec = model_fn(True, train_inputs, params) test_model_spec = model_fn(False, test_inputs, params, reuse=True) logging.info("train set predict...") predict(train_model_spec, args.model_save_dir, params, args.restore_from) logging.info("test set predict...") predict(test_model_spec, args.model_save_dir, params, args.restore_from) end_time = time.time() logging.info("Elapsed training time is {:.2f} secs".format(end_time - start_time))
from train import import_names_and_labels parser = argparse.ArgumentParser() parser.add_argument("--model_dir", default="experiments/07_full_images") parser.add_argument("--data_dir", default="data/kaggle") parser.add_argument("--restore_from", default="best_weights") parser.add_argument("--set", default="test") if __name__ == "__main__": tf.set_random_seed(230) args = parser.parse_args() json_path = os.path.join(args.model_dir, "params.json") params = Params(json_path) params.evaluate() set_logger(os.path.join(args.model_dir, 'evaluate.log')) logging.info("Creating the dataset...") data_dir = args.data_dir image_dir = os.path.join(data_dir, "images") names, labels = import_names_and_labels(data_dir, "test", params.num_labels) params.eval_size = len(names) inputs = input_fn("test", image_dir, names, labels, params) logging.info("Creating the model...") model_spec = model_fn("eval", inputs, params) logging.info("Evaluating...") evaluate(model_spec, args.model_dir, params, args.restore_from)
def train(param_file): # load params from param file params = Params(param_file) # determine model dir if params.model_dir == 'same': # this allows the model dir to be inferred from params.json file path params.model_dir = os.path.dirname(param_file) if not os.path.isdir(params.model_dir): raise ValueError("Specified model directory does not exist: {}".format( params.model_dir)) # Set the logger, delete old log file if overwrite param is set to yes train_dir = os.path.join(params.model_dir, 'train') if not os.path.isdir(train_dir): os.mkdir(train_dir) log_path = os.path.join(train_dir, 'train.log') if os.path.isfile(log_path) and params.overwrite: os.remove(log_path) set_logger(log_path) logging.info("Using model directory {}".format(params.model_dir)) logging.info("Using logging file {}".format(log_path)) logging.info("Using TensorFlow version {}".format(tf.__version__)) # Make sure data directory exists if not os.path.isdir(params.data_dir): raise ValueError("Specified data directory does not exist: {}".format( params.data_dir)) logging.info("Using data directory {}".format(params.data_dir)) # determine distribution strategy for multi GPU training if params.dist_strat.lower() == 'mirrored': logging.info("Using Mirrored distribution strategy") params.strategy = tf.distribute.MirroredStrategy() # adjust batch size and learning rate to compensate for mirrored replicas # batch size is multiplied by num replicas params.batch_size = params.batch_size * params.strategy.num_replicas_in_sync logging.info("Batch size adjusted to {} for {} replicas".format( params.batch_size, params.strategy.num_replicas_in_sync)) # initial learning rate is multiplied by squre root of replicas # params.learning_rate[0] = params.learning_rate[0] * np.sqrt(params.strategy.num_replicas_in_sync) # logging.info( # "Initial learning rate adjusted by a factor of {} (root {} for {} replicas)".format( # np.sqrt(params.strategy.num_replicas_in_sync), params.strategy.num_replicas_in_sync, # params.strategy.num_replicas_in_sync)) else: params.strategy = tf.distribute.get_strategy() # Determine checkpoint directories and determine current epoch checkpoint_path = os.path.join(params.model_dir, 'checkpoints') latest_ckpt = None if not os.path.isdir(checkpoint_path): os.mkdir(checkpoint_path) checkpoints = glob(checkpoint_path + '/*.hdf5') if checkpoints and not params.overwrite: latest_ckpt = max(checkpoints, key=os.path.getctime) completed_epochs = int( os.path.splitext(os.path.basename(latest_ckpt).split('epoch_')[1]) [0].split('_')[0]) logging.info("Checkpoint exists for epoch {}".format(completed_epochs)) else: completed_epochs = 0 # generate dataset objects for model inputs train_inputs = patch_input_fn(params, mode='train') eval_inputs = patch_input_fn(params, mode='eval') # Check for existing model and load if exists, otherwise create from scratch if latest_ckpt and not params.overwrite: logging.info("Creating the model to resume checkpoint") model = model_fn( params ) # recreating model from scratech may be neccesary if custom loss function is used logging.info( "Loading model weights checkpoint file {}".format(latest_ckpt)) model.load_weights(latest_ckpt) else: # Define the model from scratch logging.info("Creating the model...") model = model_fn(params) # SET CALLBACKS FOR TRAINING FUNCTION # define learning rate schedule callback for model learning_rate = LearningRateScheduler( learning_rate_picker(params.learning_rate, params.learning_rate_decay)) # checkpoint save callback if not os.path.isdir(checkpoint_path): os.mkdir(checkpoint_path) # save validation loss in name if evaluation files are passed, else use train loss if params.train_fract < 1.: ckpt = os.path.join(checkpoint_path, 'epoch_{epoch:02d}_valloss_{val_loss:.4f}.hdf5') else: ckpt = os.path.join(checkpoint_path, 'epoch_{epoch:02d}_trainloss_{loss:.4f}.hdf5') checkpoint = ModelCheckpoint(ckpt, monitor='val_loss', verbose=1, save_weights_only=False, save_best_only=False, mode='auto', save_freq='epoch') # tensorboard callback tensorboard = TensorBoard( log_dir=params.model_dir, histogram_freq=0, write_graph=True, write_images=False, update_freq=(params.samples_per_epoch // params.batch_size) // 100, # write losses/metrics 100x per epoch profile_batch=2, embeddings_freq=0, embeddings_metadata=None) # combine callbacks for the model train_callbacks = [learning_rate, checkpoint, tensorboard] # TRAINING logging.info("Training for {} total epochs starting at epoch {}".format( params.num_epochs, completed_epochs + 1)) model.fit(train_inputs, epochs=params.num_epochs, initial_epoch=completed_epochs, steps_per_epoch=params.samples_per_epoch // params.batch_size, callbacks=train_callbacks, validation_data=eval_inputs, shuffle=False, verbose=1) logging.info( "Successfully trained model for {} epochs ({} total epochs)".format( params.num_epochs - completed_epochs, params.num_epochs))
# Get paths for vocabularies and dataset path_vocab = os.path.join(args.data_dir, 'vocab{}'.format(params.min_freq)) params.vocab_path = path_vocab path_test_queries = os.path.join(args.data_dir, 'dev/queries.txt') path_test_articles = os.path.join(args.data_dir, 'dev/articles.txt') # Load Vocabularies vocab = tf.contrib.lookup.index_table_from_file( path_vocab, num_oov_buckets=num_oov_buckets, key_column_index=0) # Create the input data pipeline logging.info("Creating the dataset...") test_queries = load_dataset_from_text(path_test_queries, vocab, params) test_articles = load_dataset_from_text(path_test_articles, vocab, params) # Specify other parameters for the dataset and the model params.eval_size = params.test_size params.id_pad_word = vocab.lookup(tf.constant(params.pad_word)) # Create iterator over the test set inputs = input_fn('eval', test_queries, test_articles, params) logging.info("- done.") # Define the model logging.info("Creating the model...") model_spec = model_fn('eval', inputs, params, reuse=False) logging.info("- done.") logging.info("Starting evaluation") evaluate(model_spec, args.model_dir, params, args.restore_from)
params.vocab_fname, num_oov_buckets=params.num_oov_buckets) char_vocab_rev = tf.contrib.lookup.index_to_string_table_from_file( params.vocab_fname, default_value='<unk>') vocab_size = char_vocab.size() params.pad_token_id = char_vocab.lookup(tf.constant(params.pad_token)) params.start_token_id = char_vocab.lookup(tf.constant(params.start_token)) params.end_token_id = char_vocab.lookup(tf.constant(params.end_token)) with tf.Session() as sess: sess.run(tf.tables_initializer()) params.vocab_size_val = vocab_size.eval() train_dataset = load_dataset_from_file(params.train_fname, char_vocab, params) train_inputs = input_fn('train', train_dataset, params) train_model_spec = model_fn('train', train_inputs, params, reuse=False) test_inputs = test_input_fn('test', char_vocab, params) test_model_spec = model_fn('test', test_inputs, params, reuse=True) train_test(train_model_spec, test_model_spec, params, char_vocab_rev) texts = ['ff', 'av'] preds, pred_logits = test_sess(texts, test_model_spec, params, char_vocab_rev, restore_from=os.path.join( params.model_dir, 'last_weights')) for text, t_preds, t_pred_logits in zip(texts, preds, pred_logits):
dataset[split] = dict() for field in dataset_fields: with open(f'./data/nsmc/{split}/{field}.txt') as f: lines = f.readlines() dataset[split][field] = tuple([line.strip() for line in lines]) with open('./params/dataset_params.json') as f: data_params = json.load(f) with open('./params/model_params.json') as f: model_params = json.load(f) with open('./params/training_params.json') as f: training_params = json.load(f) model = model_fn(data_params, model_params) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) training_sentences = input_fn(dataset['train']['sentences'], data_params) training_labels = np.asarray( [int(label) for label in dataset['train']['labels']]) dev_sentences = input_fn(dataset['dev']['sentences'], data_params) dev_labels = np.asarray([int(label) for label in dataset['dev']['labels']]) print(training_sentences.shape, training_sentences[1]) print(training_labels.shape, training_labels[1]) print(type(dev_sentences), dev_sentences.dtype) batch_size = training_params['batch_size']
logging.info("Loading pretrained Word2Vec...") word2vec = gensim.models.KeyedVectors.load_word2vec_format( args.word2vec_file, binary=True) logging.info("- done.") logging.info("Building datasets...") with open(os.path.join(args.data_dir, "symbols.txt")) as f_symbols: symbols = [line.strip() for line in f_symbols] train_signal_map = { symbol: load_signal(path_train, path_corpus, symbol) for symbol in symbols[:2] } dev_signal_map = { symbol: load_signal(path_dev, path_corpus, symbol) for symbol in symbols[:2] } params.eval_size = params.dev_size params.buffer_size = params.train_size train_inputs = input_fn("train", train_signal_map, word2vec, params) eval_inputs = input_fn("eval", dev_signal_map, word2vec, params) logging.info("- done.") logging.info("Creating the model...") train_model_spec = model_fn("train", train_inputs, params) eval_model_spec = model_fn("eval", eval_inputs, params, reuse=True) logging.info("- done.") logging.info("Starting training for {} epoch(s)".format(params.num_epochs)) train_and_evaluate(train_model_spec, eval_model_spec, args.model_dir, params, args.restore_dir)
tf.set_random_seed(230) # TODO: Load the parameters config_args = parse_args() # Create the inputs data pipeline data_dir = 'dataset/cone/' model_dir = 'experiments/basic_model' restore_from = 'best_weights' image_dir = os.path.join(data_dir, 'Images') label_dir = os.path.join(data_dir, 'Labels') test_filenames, test_labels = get_filenames_and_labels( image_dir, label_dir, 'test') # create ssd labels test_size = len(test_filenames) preset = get_preset_by_name('ssdmobilenet160') test_labels = create_labels(preset, test_size, 2, test_labels) print("[INFO] Test labels Shape:", test_labels.shape) # Create the two iterators over the two datasets test_inputs = input_fn(False, test_filenames, test_labels, config_args) # Define the model model_specs = model_fn('eval', test_inputs, preset, config_args) evaluate(model_specs, model_dir, config_args, restore_from)
def predict(params, pred_dirs, out_dir, mask=None, checkpoint='last'): # load latest checkpoint checkpoint_path = os.path.join(params.model_dir, 'checkpoints') checkpoints = glob(checkpoint_path + '/*.hdf5') if checkpoints: # load best or last checkpoint # determine last by timestamp if checkpoint == 'last': ckpt = max(checkpoints, key=os.path.getctime) # determine best by minimum loss value in filename elif checkpoint == 'best': try: vals = [ float(item[0:-5].split('_')[-1]) for item in checkpoints ] ckpt = checkpoints[np.argmin(vals)] except: line1 = "Could not determine 'best' checkpoint based on checkpoint filenames. " line2 = "Use 'last' or pass a specific checkpoint filename to the checkpoint argument." logging.error(line1 + line2) raise ValueError(line1 + line2) elif os.path.isfile( os.path.join(my_params.model_dir, "checkpoints/{}.hdf5".format(checkpoint))): ckpt = os.path.join(my_params.model_dir, "checkpoints/{}.hdf5".format(checkpoint)) else: raise ValueError("Did not understand checkpoint value: {}".format( args.checkpoint)) # net_builder input layer uses train_dims, so set these to infer dims to allow different size inference params.train_dims = params.infer_dims # batch size for inference is hard-coded to 1 params.batch_size = 1 # recreate the model using infer dims as input dims logging.info("Creating the model...") model = model_fn(params) # load weights from last checkpoint logging.info("Loading '{}' checkpoint from {}...".format( checkpoint, ckpt)) model.load_weights(ckpt) else: raise ValueError( "No model checkpoints found at {}".format(checkpoint_path)) # infer directories in a loop niis_out = [] for pred_dir in pred_dirs: # define expected output file name to check if output prediction already exists model_name = os.path.basename(params.model_dir) name_prefix = os.path.basename(pred_dir) pred_out = os.path.join( out_dir, name_prefix + '_predictions_' + model_name + '.nii.gz') # if output doesn't already exist, then predict and make nii if not os.path.isfile(pred_out): # Create the inference dataset structure infer_inputs = patch_input_fn(params=params, mode='infer', infer_dir=pred_dir) # predict predictions = model.predict(infer_inputs) # save nii pred_out = predictions_2_nii(predictions, pred_dir, out_dir, params, mask=mask) else: logging.info( "Predictions already exist and will not be overwritten: {}". format(pred_out)) # update list of output niis niis_out.append(pred_out) return niis_out
seed=1234, name='train', params=params) logging.info("\nReading the TFRecords validation files...") validation_inputs = input_fn(tfrecord_dir=os.path.join( args.data_dir, 'validation'), mean_npz=args.preproc_file, n_images=None, is_training=False, seed=1234, name='validation', params=params) # Define the model logging.info("\nCreating the model...") train_model_spec = model_fn(train_inputs, params, is_training=True) eval_model_spec = model_fn(validation_inputs, params, is_training=False) # Train and evaluate the model logging.info("\nStarting training for {} epoch(s)".format( params.num_epochs)) train_and_evaluate(train_model_spec, eval_model_spec, args.model_dir, params, args.restore_from) # Create and save the confusion matrices for the train and evaluation datasets for mode in ['train', 'validation']: logging.info( 'Creating the confusion matrix for {} data...'.format(mode)) logging.info( '\tRetrieving information from the labels and predictions files')
def predict(inp, target, params, restore_from, config=None,\ model_dir='./ie590_project/experiments/ex1', model_save_dir='./ie590_project/experiments/ex1/model_save/1'): """predict target values given input file paths Args: inp: (list) a string list of image files paths; 2D -> [sample_size, number_of_channels] model_spec: (dict) model specifications of tf Ops params: (Params or str) Params object or params.joson path tar: (list) a float list of target values restore_from: (str) ckpt or directory name where ckpts are located for restoring ... Return: out: (list) a list of precicted target values; have exactly same dimension as target """ assert len(inp) == len(target) iterator_init_op = model_spec['iterator_init_op'] update_metrics_op = model_spec['update_metrics_op'] metrics = model_spec['metrics'] metrics_init_op = model_spec['metrics_init_op'] predictions = model_spec['predictions'] saver = tf.compat.v1.train.Saver() if type(params) is str: assert os.path.isfile( params), "params.json does not exits at {}".format(params) params = Params(params) params.load(params.update) # load parameters params.inp_size = len(inp) set_logger(os.path.join(model_dir, 'train.log')) logging.info("Creating the dataset...") inputs = input_fn(False, inp, target, params) logging.info("Creating the model...") model_spec = model_fn(False, inputs, params) logging.info("Calculating predictions...") with tf.compat.v1.Session(config=config) as sess: sess.run(model_spec['variable_init_op']) save_path = os.path.join(model_save_dir, restore_from) if os.path.isdir(save_path): save_path = tf.train.latest_checkpoint( save_path ) # If restore_from is a directory, get the latest ckpt saver.restore(sess, save_path) num_steps = (params.inp_size + params.batch_size - 1) // params.batch_size sess.run([iterator_init_op, metrics_init_op]) if len(np.shape(target)) == 1: out = np.empty(np.shape(target))[:, np.newaxis] else: out = np.empty(np.shape(target)) for i in tqdm(range(num_steps)): _, predictions_eval = sess.run([update_metrics_op, predictions]) if i < num_steps - 1: out[i * params.batch_size:(i + 1) * params.batch_size, :] = predictions_eval else: out[i * params.batch_size:, :] = predictions_eval return out
# Load the parameters args = parser.parse_args() json_path = os.path.join(args.model_dir, 'params.json') assert os.path.isfile( json_path), "No json configuration file found at {}".format(json_path) params = Params(json_path) # Set the logger set_logger(os.path.join(args.model_dir, 'evaluate.log')) # Get paths for dataset path_test_prices = os.path.join(args.data_dir, 'test_inputs.pkl') path_test_deltas = os.path.join(args.data_dir, 'test_labels.pkl') # Create the input data pipeline logging.info("Creating the dataset...") data_dir = args.data_dir test_prices, test_deltas = load_prices_and_deltas(path_test_prices, path_test_deltas, params) # Create the two iterators over the two datasets test_inputs = input_fn('test', test_prices, test_deltas, params) logging.info("- done.") # Define the model logging.info("Creating the model...") model_spec = model_fn('test', test_inputs, params, reuse=False) logging.info("Starting evaluation") evaluate(model_spec, args.model_dir, params, args.restore_from)
test_flow_filenames = construct_optical_flow_filenames( test_filenames, params.volume_depth) test_inputs = input_temporal_fn(False, test_flow_filenames, test_labels, params) # Free up the memory del test_flow_filenames del test_filenames del test_labels else: test_inputs = input_two_stream_fn(False, test_filenames, test_flow_filenames, test_labels, params) # Free up the memory del test_flow_filenames del test_filenames del test_labels # Define the model logging.info("Creating the model...") train_model_spec = model_fn('train', train_inputs, params, args.stream) test_model_spec = model_fn('test', test_inputs, params, args.stream, reuse=True) # Train the model logging.info("Starting training for {} epoch(s)".format(params.num_epochs)) train_and_evaluate(train_model_spec, test_model_spec, args.model_dir, params, args.restore_from)
eras = tf.contrib.lookup.index_table_from_file(path_era_tags) # Create the input data pipeline reviews = load_dataset_from_text(path_reviews, words) review_eras = load_dataset_from_text(path_eras, eras, isLabels=True) # Specify other parameters for the dataset and the model params_era.id_pad_word = words.lookup(tf.constant(params_era.pad_word)) params_era.id_pad_tag = words.lookup(tf.constant(params_era.pad_tag)) # Create the iterator over the test set inputs_era = input_fn('eval', reviews, review_eras, params_era) # Define the model print('Creating era models...') model_spec_era = model_fn('eval', inputs_era, params_era, reuse=False) print('Done') print(era_model_path) print(path_words) print(path_era_tags) print(path_reviews) print(path_eras) print(os.path.join(args.model_dir, args.restore_from)) # Evaluate the model... # evaluate(model-spec, args.model_dir, params, args.restore_from) # initialize saver to restore model saver = tf.train.Saver()
if f.endswith('.jpg') ] eval_filenames = [ os.path.join(dev_data_dir, f) for f in os.listdir(dev_data_dir) if f.endswith('.jpg') ] # Labels will be between 0 and 5 included (6 classes in total) train_labels = [int(f.split('/')[-1][0]) for f in train_filenames] eval_labels = [int(f.split('/')[-1][0]) for f in eval_filenames] # print("train_labels = "+str(train_labels)) # print("eval_labels ="+str(eval_labels)) # Specify the sizes of the dataset we train on and evaluate on params.train_size = len(train_filenames) params.eval_size = len(eval_filenames) # Create the two iterators over the two datasets train_inputs = input_fn(True, train_filenames, train_labels, params) eval_inputs = input_fn(False, eval_filenames, eval_labels, params) # Define the model logging.info("Creating the model...") train_model_spec = model_fn('train', train_inputs, params) eval_model_spec = model_fn('eval', eval_inputs, params, reuse=True) # Train the model logging.info("Starting training for {} epoch(s)".format(params.num_epochs)) train_and_evaluate(train_model_spec, eval_model_spec, args.model_dir, params, args.restore_from)
# Create the input data pipeline reviews = load_dataset_from_text(path_reviews,words) review_sentiments = load_dataset_from_text(path_sentiments,sentiments, isLabels=True) # Specify other parameters for the dataset and the model params_sentiment.id_pad_word = words.lookup(tf.constant(params_sentiment.pad_word)) params_sentiment.id_pad_tag = words.lookup(tf.constant(params_sentiment.pad_tag)) # Create the iterator over the test set inputs_sentiment = input_fn('eval', reviews, review_sentiments, params_sentiment) # Define the model print('Creating sentiment and era models...') model_spec_sentiment = model_fn('eval', inputs_sentiment, params_sentiment, reuse=False) print('Done') # Evaluate the model... # evaluate(model-spec, args.model_dir, params, args.restore_from) # initialize saver to restore model saver = tf.train.Saver() with tf.Session() as sess: # Initialize lookup tables for both models sess.run(model_spec_sentiment['variable_init_op']) # Reload weights from the weights subdirectory save_path = os.path.join(args.model_dir, args.restore_from) if os.path.isdir(save_path):
continue hparam_spec = params[k] if hparam_spec["scale"] == "log": val = 10**np.random.uniform(*hparam_spec["range"]) elif hparam_spec["scale"] == "linear": val = np.random.uniform(*hparam_spec["range"]) ret[k.replace("_hparam", "")] = val yield ret if __name__ == "__main__": params = params_util.load_params() tf.set_random_seed(230) with open("data/vocab.json") as f: vocab_json = json.load(f) vocab_list = vocab_as_sorted_list(vocab_json) vocab = tf.contrib.lookup.index_table_from_tensor( tf.constant(vocab_list), default_value=params["vocab_unk_idx"]) glove_weights = np.load("data/glove.npy") tweets, lens = load_tweets_naive("data/dev/text.npy", params["max_len"]) labels = load_labels_naive("data/dev/labels.npy") inputs = make_inputs(vocab, glove_weights, params["max_len"]) train_model = model_fn("train", inputs, params) eval_model = model_fn("eval", inputs, params, reuse=True) if params.get("restore"): print(params) evaluate_model(inputs, params, eval_model, params["test_set_size"])
][0] dev_filenames = os.listdir(dev_data_dir) dev_features_data = [ os.path.join(dev_data_dir, f) for f in dev_filenames if f == 'X.npy' ][0] dev_labels_data = [ os.path.join(dev_data_dir, f) for f in dev_filenames if f == "Y.npy" ][0] # Specify the sizes of the dataset we train on and evaluate on train_data_loaded = np.load(train_features_data, mmap_mode='r') params.train_size = train_data_loaded.shape[0] dev_data_loaded = np.load(dev_features_data, mmap_mode='r') params.eval_size = dev_data_loaded.shape[0] # Create the two iterators over the two datasets train_inputs = input_fn(train_features_data, train_labels_data, params) dev_inputs = input_fn(dev_features_data, dev_labels_data, params) # Define the model logging.info("Creating the model...") train_model_spec = model_fn('train', train_inputs, params) dev_model_spec = model_fn('eval', dev_inputs, params, reuse=True) # Train the model logging.info("Starting training for {} epoch(s)".format(params.num_epochs)) train_and_evaluate(train_model_spec, dev_model_spec, args.model_dir, params, args.restore_from)