def __init__(self): """Create the chatbot Initializes a tensorflow session, initialzes vocabulary and builds a model with a batch size of 1 for decoding 1 sentence at a time. """ self.sess = tf.Session() vocab_path = os.path.join(params.data_dir, "vocab%d" % params.vocab_size) self.vocab, self.rev_vocab = data_utils.initialize_vocabulary(vocab_path) self.model = model_utils.create_model(self.sess, True) self.model.batch_size = 1 # Respond 1 sentence at a time.
def save_variables(): with tf.Session() as sess: model = create_model(sess, True) model_params = sess.run(tf.all_variables()) for i in xrange(5, len(VARIABLES)): if "decoder_embedding" not in VARIABLES[i]: print(VARIABLES[i]) to_save = model_params[i].tolist() with open(SAVE_DIR + VARIABLES[i], 'w') as test_file: test_file.write(json.dumps(to_save)) upload_variable(VARIABLES[i])
def __init__(self): """Create the chatbot Initializes a tensorflow session, initialzes vocabulary and builds a model with a batch size of 1 for decoding 1 sentence at a time. """ self.sess = tf.Session() vocab_path = os.path.join(params.data_dir, "vocab%d" % params.vocab_size) self.vocab, self.rev_vocab = data_utils.initialize_vocabulary( vocab_path) self.model = model_utils.create_model(self.sess, True) self.model.batch_size = 1 # Respond 1 sentence at a time.
def main(): parser = argparse.ArgumentParser() parser.add_argument('--base_dir', default='') parser.add_argument('--data', default='./training_data/', type=str, help='path to dataset contains inputs and targets') parser.add_argument('--log_dir', default='tacotron2', type=str, help='path to save alignment and checkpoint') parser.add_argument('--restore_from', default=None, type=str, help='the checkpoint restored from the log_dir you set') parser.add_argument('--summary_interval', type=int, default=250, help='Steps between running summary ops') parser.add_argument('--checkpoint_interval', type=int, default=2500, help='Steps between writing checkpoints') parser.add_argument('--eval_interval', type=int, default=5000, help='Steps between eval on test data') parser.add_argument('--name', default='tacotron2', type=str, help='name of the experiment') args = parser.parse_args() logging.basicConfig(format='%(asctime)s %(filename)s %(levelname)s %(message)s', datefmt='%a, %d %b %Y %H:%M:%S', level=logging.DEBUG, stream=sys.stdout) step = 0 exp_name = args.name model = create_model() if args.restore_from is not None: checkpoint_path = os.path.join(args.log_dir, os.path.join("checkpoint", args.restore_from)) if os.path.exists(checkpoint_path): model.load_state_dict(torch.load(checkpoint_path)) step = int(checkpoint_path.split('/')[-1].split('_')[-1].split(".")[0]) exp_name = checkpoint_path.split('/')[-1].split("_")[0] else: logging.error(f'checkpoint path:{checkpoint_path} does\'t exist!') os.environ["CUDA_VISIBEL_DEVICES"] = hp.gpu_ids device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") if torch.cuda.device_count() > 1: model = nn.DataParallel(model) model.to(device) train(args, model, exp_name, step, checkpoint_path, device)
def return_line_entity(FLAGS): # evaluate logger.info("Evaluate Line") # load data config = load_config(FLAGS.resource.config_file) with open(FLAGS.resource.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) # start tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.resource.ckpt_path, load_word2vec, config, id_to_char, False) while True: line = input("请输入测试句子:") result = model.return_sentence_pred_evaluate( sess, input_from_line(line, char_to_id), id_to_tag) print(result)
def extract_from_html_dir(self, html_dir_path): map = { "公告id": [], "甲方": [], "乙方": [], "项目名称": [], "合同名称": [], "合同金额上限": [], "合同金额下限": [], "联合体成员": [] } config = load_config(FLAGS.resource.config_file2) with open(FLAGS.resource.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.resource.ckpt_dir, load_word2vec, config, id_to_char, False) trans = model.trans.eval() for html_id in tqdm(os.listdir(html_dir_path)): self._extract_from_html_dir(html_dir_path, html_id, map, sess, trans, model, id_to_tag, tag_to_id, char_to_id) dataframe = pd.DataFrame(data=map, columns=[ "公告id", "甲方", "乙方", "项目名称", "合同名称", "合同金额上限", "合同金额下限", "联合体成员" ], dtype=None, copy=False) if os.path.exists('ht_result.csv'): os.remove('ht_result.csv') dataframe.to_csv("ht_result.csv", encoding="utf_8_sig")
def train(FLAGS): # load data sets train_sentences = load_sentences(FLAGS.resource.train_file, FLAGS.trainer.zeros) test_sentences = load_sentences(FLAGS.resource.test_file, FLAGS.trainer.zeros) update_tag_scheme( train_sentences, FLAGS.model.tag_schema) # Use selected tagging scheme (IOB / IOBES) update_tag_scheme(test_sentences, FLAGS.model.tag_schema) # create maps if not exist if not os.path.isfile(FLAGS.resource.map_file): if FLAGS.trainer.pre_emb: # create dictionary for word dico_chars_train = char_mapping(train_sentences, FLAGS.trainer.lower)[0] _, char_to_id, id_to_char = augment_with_pretrained( dico_chars_train.copy(), FLAGS.resource.emb_file, list( itertools.chain.from_iterable([[w[0] for w in s] for s in test_sentences]))) else: _, char_to_id, id_to_char = char_mapping(train_sentences, FLAGS.trainer.lower) _, tag_to_id, id_to_tag = tag_mapping( train_sentences) # Create a dictionary and a mapping for tags with open(FLAGS.resource.map_file, "wb") as f: pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f) else: with open(FLAGS.resource.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) # prepare data, get a collection of list containing index train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id, FLAGS.trainer.lower) test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id, FLAGS.trainer.lower) train_manager = BatchManager(train_data, FLAGS.trainer.batch_size) test_manager = BatchManager(test_data, 100) # make path for store log and model if not exist if os.path.isfile(FLAGS.resource.config_file2): config = load_config(FLAGS.resource.config_file2) else: config = _config_model(FLAGS, char_to_id, tag_to_id) save_config(config, FLAGS.resource.config_file2) print_config(config, logger) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True steps_per_epoch = train_manager.len_data with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.resource.ckpt_dir, load_word2vec, config, id_to_char) logger.info("Start raining") loss = [] for i in range(FLAGS.trainer.max_epoch): for batch in train_manager.iter_batch(shuffle=True): step, batch_loss = model.run_step(sess, True, batch) loss.append(batch_loss) if step % FLAGS.trainer.steps_check == 0: iteration = step // steps_per_epoch + 1 logger.info("iteration:{} step:{}/{}, " "NER loss:{:>9.6f}".format( iteration, step % steps_per_epoch, steps_per_epoch, np.mean(loss))) loss = [] best = return_f1(FLAGS, sess, model, "test", test_manager, id_to_tag) if best: save_model(sess, model, FLAGS.resource.ckpt_dir)
def train(): slack.connection.notify(text='Training SpeakEasy!', ) # Prepare reddit data. print("Preparing data in %s" % params.data_dir) sys.stdout.flush() data_train, data_dev, _ = data_utils.prepare_data(params.data_dir, params.vocab_size) # CONFIG FOR GPU (won't work on mac): # config = tf.ConfigProto() # config.gpu_options.allocator_type = 'BFC' # with tf.Session(config=config) as sess: with tf.Session() as sess: # Create model. print("Creating %s model with %d layers of %d units." % (params.model_type, params.num_layers, params.size)) sys.stdout.flush() if params.buckets: print("Using bucketed model.") sys.stdout.flush() model = model_utils.create_model(sess, False) # Set up event logging. NOTE: added this, this is not finished merged_summaries = tf.merge_all_summaries() # writer = tf.train.SummaryWriter(params.train_dir, sess.graph_def) # Read data into buckets and compute their sizes. print("Reading development and training data (limit: %d)." % params.max_train_data_size) sys.stdout.flush() dev_set, _ = read_data(data_dev) train_set, epoch = read_data(data_train, params.max_train_data_size) if params.buckets: train_bucket_sizes = [ len(train_set[b]) for b in xrange(len(buckets)) ] train_total_size = float(sum(train_bucket_sizes)) # A bucket scale is a list of increasing numbers from 0 to 1 that we'll use # to select a bucket. Length of [scale[i], scale[i+1]] is proportional to # the size if i-th training bucket, as used later. train_buckets_scale = [ sum(train_bucket_sizes[:i + 1]) / train_total_size for i in xrange(len(train_bucket_sizes)) ] # This is the training loop. step_time, loss = 0.0, 0.0 current_step = 0 previous_losses = [] while True: # while current_step <= epoch * 7.5: # Get a batch and make a step. start_time = time.time() if params.buckets: # Choose a bucket according to data distribution. We pick a random number # in [0, 1] and use the corresponding interval in train_buckets_scale. random_number_01 = np.random.random_sample() bucket_id = min([ i for i in xrange(len(train_buckets_scale)) if train_buckets_scale[i] > random_number_01 ]) encoder_inputs, decoder_inputs, target_weights = model.get_batch( train_set, bucket_id=bucket_id) summaries, _, step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs, target_weights, False, bucket_id=bucket_id) else: encoder_inputs, decoder_inputs, target_weights = model.get_batch( train_set, bucket_id=None) summaries, _, step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs, target_weights, False, bucket_id=None) step_time += (time.time() - start_time) / params.steps_per_checkpoint loss += step_loss / params.steps_per_checkpoint current_step += 1 if current_step % epoch == 0 and current_step >= epoch * 5: sess.run([model.learning_rate_decay_op]) # Once in a while, we save checkpoint, print statistics, and run evals. if current_step % params.steps_per_checkpoint == 0: # Print statistics for the previous epoch. perplexity = math.exp(loss) if loss < 300 else float('inf') log_line = ( "global step %d learning rate %.4f step-time %.2f perplexity %.2f" % (model.global_step.eval(), model.learning_rate.eval(), step_time, perplexity)) print(log_line) sys.stdout.flush() slack.connection.notify(text=log_line, ) previous_losses.append(loss) # Save checkpoint and zero timer and loss. checkpoint_path = os.path.join( params.train_dir, "speakEasy_vocab%d_size%d_%s.ckpt" % params.vocab_size, params.size, params.train_data) model.saver.save(sess, checkpoint_path, global_step=model.global_step) step_time, loss = 0.0, 0.0 if params.buckets: # Run evals on development set and print their perplexity. for bucket_id in xrange(len(_buckets) - 1): encoder_inputs, decoder_inputs, target_weights = model.get_batch( dev_set, bucket_id) _, _, eval_loss, _ = model.step(sess, encoder_inputs, decoder_inputs, target_weights, True, bucket_id=bucket_id) eval_ppx = math.exp( eval_loss) if eval_loss < 300 else float('inf') log_line = "eval: bucket %d perplexity %.2f" % ( bucket_id, eval_ppx) print(" %s" % log_line) slack.connection.notify(text=log_line, ) sys.stdout.flush()
from utils.dataloader import DataLoader import torch from model import model_utils from optimizer.optimizer import NoamOpt from train.trainer import Trainer hidden_size = 256 num_encoder = 6 num_decoder = 6 n_head = 8 pf_dim = 1024 drop_out = 0.5 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') device = 'cpu' dataloader = DataLoader(device) train_iterator, valid_iterator, test_iterator = dataloader.load_data(64) model = model_utils.create_model(dataloader.src_vocab_size(), dataloader.trg_vocab_size(), hidden_size, num_encoder, num_decoder, n_head, pf_dim, drop_out, dataloader.get_pad_idx(), device) print(model_utils.count_parameters(model)) model_utils.init(model) optimizer = NoamOpt(hidden_size , 1, 2000, torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9)) trainer = Trainer(train_iterator, valid_iterator, model, optimizer, dataloader.get_pad_idx(), device) trainer.train(5) # for i, batch in enumerate(train_iterator): # src = batch.src.permute(1, 0).to(device) # trg = batch.trg.permute(1, 0).to(device)
def train(): slack.connection.notify( text='Training SpeakEasy!', ) # Prepare reddit data. print("Preparing data in %s" % params.data_dir) sys.stdout.flush() data_train, data_dev, _ = data_utils.prepare_data(params.data_dir, params.vocab_size) # CONFIG FOR GPU (won't work on mac): # config = tf.ConfigProto() # config.gpu_options.allocator_type = 'BFC' # with tf.Session(config=config) as sess: with tf.Session() as sess: # Create model. print("Creating %s model with %d layers of %d units." % (params.model_type, params.num_layers, params.size)) sys.stdout.flush() if params.buckets: print("Using bucketed model.") sys.stdout.flush() model = model_utils.create_model(sess, False) # Set up event logging. NOTE: added this, this is not finished merged_summaries = tf.merge_all_summaries() # writer = tf.train.SummaryWriter(params.train_dir, sess.graph_def) # Read data into buckets and compute their sizes. print ("Reading development and training data (limit: %d)." % params.max_train_data_size) sys.stdout.flush() dev_set, _ = read_data(data_dev) train_set, epoch = read_data(data_train, params.max_train_data_size) if params.buckets: train_bucket_sizes = [len(train_set[b]) for b in xrange(len(buckets))] train_total_size = float(sum(train_bucket_sizes)) # A bucket scale is a list of increasing numbers from 0 to 1 that we'll use # to select a bucket. Length of [scale[i], scale[i+1]] is proportional to # the size if i-th training bucket, as used later. train_buckets_scale = [sum(train_bucket_sizes[:i + 1]) / train_total_size for i in xrange(len(train_bucket_sizes))] # This is the training loop. step_time, loss = 0.0, 0.0 current_step = 0 previous_losses = [] while True: # while current_step <= epoch * 7.5: # Get a batch and make a step. start_time = time.time() if params.buckets: # Choose a bucket according to data distribution. We pick a random number # in [0, 1] and use the corresponding interval in train_buckets_scale. random_number_01 = np.random.random_sample() bucket_id = min([i for i in xrange(len(train_buckets_scale)) if train_buckets_scale[i] > random_number_01]) encoder_inputs, decoder_inputs, target_weights = model.get_batch(train_set, bucket_id=bucket_id) summaries, _, step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs, target_weights, False, bucket_id=bucket_id) else: encoder_inputs, decoder_inputs, target_weights = model.get_batch(train_set, bucket_id=None) summaries, _, step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs, target_weights, False, bucket_id=None) step_time += (time.time() - start_time) / params.steps_per_checkpoint loss += step_loss / params.steps_per_checkpoint current_step += 1 if current_step % epoch == 0 and current_step >= epoch * 5: sess.run([model.learning_rate_decay_op]) # Once in a while, we save checkpoint, print statistics, and run evals. if current_step % params.steps_per_checkpoint == 0: # Print statistics for the previous epoch. perplexity = math.exp(loss) if loss < 300 else float('inf') log_line = ("global step %d learning rate %.4f step-time %.2f perplexity %.2f" % (model.global_step.eval(), model.learning_rate.eval(),step_time, perplexity)) print(log_line) sys.stdout.flush() slack.connection.notify( text=log_line, ) previous_losses.append(loss) # Save checkpoint and zero timer and loss. checkpoint_path = os.path.join(params.train_dir, "speakEasy_vocab%d_size%d_%s.ckpt" % params.vocab_size, params.size, params.train_data) model.saver.save(sess, checkpoint_path, global_step=model.global_step) step_time, loss = 0.0, 0.0 if params.buckets: # Run evals on development set and print their perplexity. for bucket_id in xrange(len(_buckets)-1): encoder_inputs, decoder_inputs, target_weights = model.get_batch(dev_set, bucket_id) _, _, eval_loss, _ = model.step(sess, encoder_inputs, decoder_inputs, target_weights, True, bucket_id=bucket_id) eval_ppx = math.exp(eval_loss) if eval_loss < 300 else float('inf') log_line = "eval: bucket %d perplexity %.2f" % (bucket_id, eval_ppx) print(" %s" % log_line) slack.connection.notify( text=log_line, ) sys.stdout.flush()
def train(data_path_source_dir_: str, training_params: dict, model_params: dict): """ Sequence all the steps in the helper scripts in accordance to the parameter values given to do the following: - Create generator objects that can feed 2D single channel images to a keras.model fit() method - Create a 2D Unet - Fit the Unet and store the information of its progress and best performing versions - Obtain IoU on train and holdout - Store information about the pixel values predicted by the model on a sample of 5 images :param data_path_source_dir_: Path to directory where the image data is stored assuming existence of imageTr/Ts and labelTr subdirs :param training_params: Dictionary with parameters of the different functions used for training :param model_params: Dictionary with parameters of the different functions used to create and compile the Unet :return: """ logging.info('-------------------TRAIN-------------------') model_object_storing_dir = training_params['model_object_storing_dir'] os.makedirs(model_object_storing_dir, exist_ok=True) preprocesing_params_: dict = training_params["preprocesing_params"] # Get data generators for train and holdout logging.info('Get data generators for train and holdout') train_data_generator, holdout_data_generator, tr_fold_0_df_cancer_info, holdout_fold_0_df_cancer_info = \ training_utils.prepare_train_holdout_generators( training_params=training_params, data_path_source_dir_=data_path_source_dir_) # Build model logging.info('Build model') model = model_utils.create_model( resize_dim_=preprocesing_params_['resize_dim'], lr_=training_params['learning_rate'], loss_function_name_=training_params['loss_function_name'], object_storing_dir_=model_object_storing_dir, num_filters_first_level_=training_params['num_filters_first_level'], **training_params['loss_function_params'] ) # Set the callbacks my_callbacks = [ tf.keras.callbacks.LearningRateScheduler(training_utils.scheduler, verbose=1), tf.keras.callbacks.ModelCheckpoint( filepath=f'./{model_object_storing_dir}' + '/model.{epoch:02d}-{val_loss:.2f}.h5', verbose=1), tf.keras.callbacks.TensorBoard(log_dir=f'./{model_object_storing_dir}/logs'), ] # Train the model logging.info('Train the model') model.fit(train_data_generator, validation_data=holdout_data_generator, epochs=training_params['num_epoch'], callbacks=my_callbacks) # Save the models training history logging.info("Save the model's training history") pd.DataFrame(model.history.history).to_csv(os.path.join(model_object_storing_dir, 'training_history.csv')) # Store last version of the model logging.info("Store last version of the model") model.save(f'./{model_object_storing_dir}/end_of_training_version') # Store metrics for the train and holdout sets logging.info('Store metrics for the train and holdout sets') store_metrics(df_cancer_info=tr_fold_0_df_cancer_info, dataset_type='train', model=model, model_object_storing_dir=model_object_storing_dir, model_params=model_params, preprocesing_params_=preprocesing_params_) store_metrics(df_cancer_info=holdout_fold_0_df_cancer_info, dataset_type='holdout', model=model, model_object_storing_dir=model_object_storing_dir, model_params=model_params, preprocesing_params_=preprocesing_params_)