def __init__(self): """Create the chatbot Initializes a tensorflow session, initialzes vocabulary and builds a model with a batch size of 1 for decoding 1 sentence at a time. """ self.sess = tf.Session() vocab_path = os.path.join(params.data_dir, "vocab%d" % params.vocab_size) self.vocab, self.rev_vocab = data_utils.initialize_vocabulary(vocab_path) self.model = model_utils.create_model(self.sess, True) self.model.batch_size = 1 # Respond 1 sentence at a time.
def save_variables():
with tf.Session() as sess:
model = create_model(sess, True)
model_params = sess.run(tf.all_variables())
for i in xrange(5, len(VARIABLES)):
if "decoder_embedding" not in VARIABLES[i]:
print(VARIABLES[i])
to_save = model_params[i].tolist()
with open(SAVE_DIR + VARIABLES[i], 'w') as test_file:
test_file.write(json.dumps(to_save))
upload_variable(VARIABLES[i])
def __init__(self):
"""Create the chatbot
Initializes a tensorflow session, initialzes vocabulary and builds a model with a batch size of 1 for decoding 1 sentence at a time.
"""
self.sess = tf.Session()
vocab_path = os.path.join(params.data_dir,
"vocab%d" % params.vocab_size)
self.vocab, self.rev_vocab = data_utils.initialize_vocabulary(
vocab_path)
self.model = model_utils.create_model(self.sess, True)
self.model.batch_size = 1 # Respond 1 sentence at a time.
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--base_dir', default='')
parser.add_argument('--data', default='./training_data/', type=str,
help='path to dataset contains inputs and targets')
parser.add_argument('--log_dir', default='tacotron2', type=str, help='path to save alignment and checkpoint')
parser.add_argument('--restore_from', default=None, type=str,
help='the checkpoint restored from the log_dir you set')
parser.add_argument('--summary_interval', type=int, default=250,
help='Steps between running summary ops')
parser.add_argument('--checkpoint_interval', type=int, default=2500,
help='Steps between writing checkpoints')
parser.add_argument('--eval_interval', type=int, default=5000,
help='Steps between eval on test data')
parser.add_argument('--name', default='tacotron2', type=str,
help='name of the experiment')
args = parser.parse_args()
logging.basicConfig(format='%(asctime)s %(filename)s %(levelname)s %(message)s',
datefmt='%a, %d %b %Y %H:%M:%S', level=logging.DEBUG,
stream=sys.stdout)
step = 0
exp_name = args.name
model = create_model()
if args.restore_from is not None:
checkpoint_path = os.path.join(args.log_dir, os.path.join("checkpoint", args.restore_from))
if os.path.exists(checkpoint_path):
model.load_state_dict(torch.load(checkpoint_path))
step = int(checkpoint_path.split('/')[-1].split('_')[-1].split(".")[0])
exp_name = checkpoint_path.split('/')[-1].split("_")[0]
else:
logging.error(f'checkpoint path:{checkpoint_path} does\'t exist!')
os.environ["CUDA_VISIBEL_DEVICES"] = hp.gpu_ids
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
if torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
model.to(device)
train(args, model, exp_name, step, checkpoint_path, device)
def return_line_entity(FLAGS):
# evaluate
logger.info("Evaluate Line")
# load data
config = load_config(FLAGS.resource.config_file)
with open(FLAGS.resource.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# start
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.resource.ckpt_path,
load_word2vec, config, id_to_char, False)
while True:
line = input("请输入测试句子:")
result = model.return_sentence_pred_evaluate(
sess, input_from_line(line, char_to_id), id_to_tag)
print(result)
def extract_from_html_dir(self, html_dir_path):
map = {
"公告id": [],
"甲方": [],
"乙方": [],
"项目名称": [],
"合同名称": [],
"合同金额上限": [],
"合同金额下限": [],
"联合体成员": []
}
config = load_config(FLAGS.resource.config_file2)
with open(FLAGS.resource.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.resource.ckpt_dir,
load_word2vec, config, id_to_char, False)
trans = model.trans.eval()
for html_id in tqdm(os.listdir(html_dir_path)):
self._extract_from_html_dir(html_dir_path, html_id, map, sess,
trans, model, id_to_tag, tag_to_id,
char_to_id)
dataframe = pd.DataFrame(data=map,
columns=[
"公告id", "甲方", "乙方", "项目名称", "合同名称",
"合同金额上限", "合同金额下限", "联合体成员"
],
dtype=None,
copy=False)
if os.path.exists('ht_result.csv'):
os.remove('ht_result.csv')
dataframe.to_csv("ht_result.csv", encoding="utf_8_sig")
def train(FLAGS):
# load data sets
train_sentences = load_sentences(FLAGS.resource.train_file,
FLAGS.trainer.zeros)
test_sentences = load_sentences(FLAGS.resource.test_file,
FLAGS.trainer.zeros)
update_tag_scheme(
train_sentences,
FLAGS.model.tag_schema) # Use selected tagging scheme (IOB / IOBES)
update_tag_scheme(test_sentences, FLAGS.model.tag_schema)
# create maps if not exist
if not os.path.isfile(FLAGS.resource.map_file):
if FLAGS.trainer.pre_emb: # create dictionary for word
dico_chars_train = char_mapping(train_sentences,
FLAGS.trainer.lower)[0]
_, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(), FLAGS.resource.emb_file,
list(
itertools.chain.from_iterable([[w[0] for w in s]
for s in test_sentences])))
else:
_, char_to_id, id_to_char = char_mapping(train_sentences,
FLAGS.trainer.lower)
_, tag_to_id, id_to_tag = tag_mapping(
train_sentences) # Create a dictionary and a mapping for tags
with open(FLAGS.resource.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.resource.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# prepare data, get a collection of list containing index
train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id,
FLAGS.trainer.lower)
test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id,
FLAGS.trainer.lower)
train_manager = BatchManager(train_data, FLAGS.trainer.batch_size)
test_manager = BatchManager(test_data, 100)
# make path for store log and model if not exist
if os.path.isfile(FLAGS.resource.config_file2):
config = load_config(FLAGS.resource.config_file2)
else:
config = _config_model(FLAGS, char_to_id, tag_to_id)
save_config(config, FLAGS.resource.config_file2)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.resource.ckpt_dir,
load_word2vec, config, id_to_char)
logger.info("Start raining")
loss = []
for i in range(FLAGS.trainer.max_epoch):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.trainer.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch,
steps_per_epoch, np.mean(loss)))
loss = []
best = return_f1(FLAGS, sess, model, "test", test_manager,
id_to_tag)
if best:
save_model(sess, model, FLAGS.resource.ckpt_dir)
def train():
slack.connection.notify(text='Training SpeakEasy!', )
# Prepare reddit data.
print("Preparing data in %s" % params.data_dir)
sys.stdout.flush()
data_train, data_dev, _ = data_utils.prepare_data(params.data_dir,
params.vocab_size)
# CONFIG FOR GPU (won't work on mac):
# config = tf.ConfigProto()
# config.gpu_options.allocator_type = 'BFC'
# with tf.Session(config=config) as sess:
with tf.Session() as sess:
# Create model.
print("Creating %s model with %d layers of %d units." %
(params.model_type, params.num_layers, params.size))
sys.stdout.flush()
if params.buckets: print("Using bucketed model.")
sys.stdout.flush()
model = model_utils.create_model(sess, False)
# Set up event logging. NOTE: added this, this is not finished
merged_summaries = tf.merge_all_summaries()
# writer = tf.train.SummaryWriter(params.train_dir, sess.graph_def)
# Read data into buckets and compute their sizes.
print("Reading development and training data (limit: %d)." %
params.max_train_data_size)
sys.stdout.flush()
dev_set, _ = read_data(data_dev)
train_set, epoch = read_data(data_train, params.max_train_data_size)
if params.buckets:
train_bucket_sizes = [
len(train_set[b]) for b in xrange(len(buckets))
]
train_total_size = float(sum(train_bucket_sizes))
# A bucket scale is a list of increasing numbers from 0 to 1 that we'll use
# to select a bucket. Length of [scale[i], scale[i+1]] is proportional to
# the size if i-th training bucket, as used later.
train_buckets_scale = [
sum(train_bucket_sizes[:i + 1]) / train_total_size
for i in xrange(len(train_bucket_sizes))
]
# This is the training loop.
step_time, loss = 0.0, 0.0
current_step = 0
previous_losses = []
while True:
# while current_step <= epoch * 7.5:
# Get a batch and make a step.
start_time = time.time()
if params.buckets:
# Choose a bucket according to data distribution. We pick a random number
# in [0, 1] and use the corresponding interval in train_buckets_scale.
random_number_01 = np.random.random_sample()
bucket_id = min([
i for i in xrange(len(train_buckets_scale))
if train_buckets_scale[i] > random_number_01
])
encoder_inputs, decoder_inputs, target_weights = model.get_batch(
train_set, bucket_id=bucket_id)
summaries, _, step_loss, _ = model.step(sess,
encoder_inputs,
decoder_inputs,
target_weights,
False,
bucket_id=bucket_id)
else:
encoder_inputs, decoder_inputs, target_weights = model.get_batch(
train_set, bucket_id=None)
summaries, _, step_loss, _ = model.step(sess,
encoder_inputs,
decoder_inputs,
target_weights,
False,
bucket_id=None)
step_time += (time.time() -
start_time) / params.steps_per_checkpoint
loss += step_loss / params.steps_per_checkpoint
current_step += 1
if current_step % epoch == 0 and current_step >= epoch * 5:
sess.run([model.learning_rate_decay_op])
# Once in a while, we save checkpoint, print statistics, and run evals.
if current_step % params.steps_per_checkpoint == 0:
# Print statistics for the previous epoch.
perplexity = math.exp(loss) if loss < 300 else float('inf')
log_line = (
"global step %d learning rate %.4f step-time %.2f perplexity %.2f"
% (model.global_step.eval(), model.learning_rate.eval(),
step_time, perplexity))
print(log_line)
sys.stdout.flush()
slack.connection.notify(text=log_line, )
previous_losses.append(loss)
# Save checkpoint and zero timer and loss.
checkpoint_path = os.path.join(
params.train_dir,
"speakEasy_vocab%d_size%d_%s.ckpt" % params.vocab_size,
params.size, params.train_data)
model.saver.save(sess,
checkpoint_path,
global_step=model.global_step)
step_time, loss = 0.0, 0.0
if params.buckets:
# Run evals on development set and print their perplexity.
for bucket_id in xrange(len(_buckets) - 1):
encoder_inputs, decoder_inputs, target_weights = model.get_batch(
dev_set, bucket_id)
_, _, eval_loss, _ = model.step(sess,
encoder_inputs,
decoder_inputs,
target_weights,
True,
bucket_id=bucket_id)
eval_ppx = math.exp(
eval_loss) if eval_loss < 300 else float('inf')
log_line = "eval: bucket %d perplexity %.2f" % (
bucket_id, eval_ppx)
print(" %s" % log_line)
slack.connection.notify(text=log_line, )
sys.stdout.flush()
from utils.dataloader import DataLoader
import torch
from model import model_utils
from optimizer.optimizer import NoamOpt
from train.trainer import Trainer
hidden_size = 256
num_encoder = 6
num_decoder = 6
n_head = 8
pf_dim = 1024
drop_out = 0.5
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
device = 'cpu'
dataloader = DataLoader(device)
train_iterator, valid_iterator, test_iterator = dataloader.load_data(64)
model = model_utils.create_model(dataloader.src_vocab_size(), dataloader.trg_vocab_size(), hidden_size, num_encoder, num_decoder, n_head, pf_dim,
drop_out, dataloader.get_pad_idx(), device)
print(model_utils.count_parameters(model))
model_utils.init(model)
optimizer = NoamOpt(hidden_size , 1, 2000, torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9))
trainer = Trainer(train_iterator, valid_iterator, model, optimizer, dataloader.get_pad_idx(), device)
trainer.train(5)
# for i, batch in enumerate(train_iterator):
# src = batch.src.permute(1, 0).to(device)
# trg = batch.trg.permute(1, 0).to(device)
def train():
slack.connection.notify(
text='Training SpeakEasy!',
)
# Prepare reddit data.
print("Preparing data in %s" % params.data_dir)
sys.stdout.flush()
data_train, data_dev, _ = data_utils.prepare_data(params.data_dir, params.vocab_size)
# CONFIG FOR GPU (won't work on mac):
# config = tf.ConfigProto()
# config.gpu_options.allocator_type = 'BFC'
# with tf.Session(config=config) as sess:
with tf.Session() as sess:
# Create model.
print("Creating %s model with %d layers of %d units." % (params.model_type, params.num_layers, params.size))
sys.stdout.flush()
if params.buckets: print("Using bucketed model.")
sys.stdout.flush()
model = model_utils.create_model(sess, False)
# Set up event logging. NOTE: added this, this is not finished
merged_summaries = tf.merge_all_summaries()
# writer = tf.train.SummaryWriter(params.train_dir, sess.graph_def)
# Read data into buckets and compute their sizes.
print ("Reading development and training data (limit: %d)." % params.max_train_data_size)
sys.stdout.flush()
dev_set, _ = read_data(data_dev)
train_set, epoch = read_data(data_train, params.max_train_data_size)
if params.buckets:
train_bucket_sizes = [len(train_set[b]) for b in xrange(len(buckets))]
train_total_size = float(sum(train_bucket_sizes))
# A bucket scale is a list of increasing numbers from 0 to 1 that we'll use
# to select a bucket. Length of [scale[i], scale[i+1]] is proportional to
# the size if i-th training bucket, as used later.
train_buckets_scale = [sum(train_bucket_sizes[:i + 1]) / train_total_size for i in xrange(len(train_bucket_sizes))]
# This is the training loop.
step_time, loss = 0.0, 0.0
current_step = 0
previous_losses = []
while True:
# while current_step <= epoch * 7.5:
# Get a batch and make a step.
start_time = time.time()
if params.buckets:
# Choose a bucket according to data distribution. We pick a random number
# in [0, 1] and use the corresponding interval in train_buckets_scale.
random_number_01 = np.random.random_sample()
bucket_id = min([i for i in xrange(len(train_buckets_scale)) if train_buckets_scale[i] > random_number_01])
encoder_inputs, decoder_inputs, target_weights = model.get_batch(train_set, bucket_id=bucket_id)
summaries, _, step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs, target_weights, False, bucket_id=bucket_id)
else:
encoder_inputs, decoder_inputs, target_weights = model.get_batch(train_set, bucket_id=None)
summaries, _, step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs, target_weights, False, bucket_id=None)
step_time += (time.time() - start_time) / params.steps_per_checkpoint
loss += step_loss / params.steps_per_checkpoint
current_step += 1
if current_step % epoch == 0 and current_step >= epoch * 5:
sess.run([model.learning_rate_decay_op])
# Once in a while, we save checkpoint, print statistics, and run evals.
if current_step % params.steps_per_checkpoint == 0:
# Print statistics for the previous epoch.
perplexity = math.exp(loss) if loss < 300 else float('inf')
log_line = ("global step %d learning rate %.4f step-time %.2f perplexity %.2f" %
(model.global_step.eval(), model.learning_rate.eval(),step_time, perplexity))
print(log_line)
sys.stdout.flush()
slack.connection.notify(
text=log_line,
)
previous_losses.append(loss)
# Save checkpoint and zero timer and loss.
checkpoint_path = os.path.join(params.train_dir, "speakEasy_vocab%d_size%d_%s.ckpt" % params.vocab_size, params.size, params.train_data)
model.saver.save(sess, checkpoint_path, global_step=model.global_step)
step_time, loss = 0.0, 0.0
if params.buckets:
# Run evals on development set and print their perplexity.
for bucket_id in xrange(len(_buckets)-1):
encoder_inputs, decoder_inputs, target_weights = model.get_batch(dev_set, bucket_id)
_, _, eval_loss, _ = model.step(sess, encoder_inputs, decoder_inputs, target_weights, True, bucket_id=bucket_id)
eval_ppx = math.exp(eval_loss) if eval_loss < 300 else float('inf')
log_line = "eval: bucket %d perplexity %.2f" % (bucket_id, eval_ppx)
print(" %s" % log_line)
slack.connection.notify(
text=log_line,
)
sys.stdout.flush()
def train(data_path_source_dir_: str, training_params: dict, model_params: dict):
"""
Sequence all the steps in the helper scripts in accordance to the parameter values given to do the following:
- Create generator objects that can feed 2D single channel images to a keras.model fit() method
- Create a 2D Unet
- Fit the Unet and store the information of its progress and best performing versions
- Obtain IoU on train and holdout
- Store information about the pixel values predicted by the model on a sample of 5 images
:param data_path_source_dir_: Path to directory where the image data is stored assuming existence of
imageTr/Ts and labelTr subdirs
:param training_params: Dictionary with parameters of the different functions used for training
:param model_params: Dictionary with parameters of the different functions used to create and compile the Unet
:return:
"""
logging.info('-------------------TRAIN-------------------')
model_object_storing_dir = training_params['model_object_storing_dir']
os.makedirs(model_object_storing_dir, exist_ok=True)
preprocesing_params_: dict = training_params["preprocesing_params"]
# Get data generators for train and holdout
logging.info('Get data generators for train and holdout')
train_data_generator, holdout_data_generator, tr_fold_0_df_cancer_info, holdout_fold_0_df_cancer_info = \
training_utils.prepare_train_holdout_generators(
training_params=training_params, data_path_source_dir_=data_path_source_dir_)
# Build model
logging.info('Build model')
model = model_utils.create_model(
resize_dim_=preprocesing_params_['resize_dim'],
lr_=training_params['learning_rate'],
loss_function_name_=training_params['loss_function_name'],
object_storing_dir_=model_object_storing_dir,
num_filters_first_level_=training_params['num_filters_first_level'],
**training_params['loss_function_params']
)
# Set the callbacks
my_callbacks = [
tf.keras.callbacks.LearningRateScheduler(training_utils.scheduler, verbose=1),
tf.keras.callbacks.ModelCheckpoint(
filepath=f'./{model_object_storing_dir}' + '/model.{epoch:02d}-{val_loss:.2f}.h5', verbose=1),
tf.keras.callbacks.TensorBoard(log_dir=f'./{model_object_storing_dir}/logs'),
]
# Train the model
logging.info('Train the model')
model.fit(train_data_generator, validation_data=holdout_data_generator,
epochs=training_params['num_epoch'], callbacks=my_callbacks)
# Save the models training history
logging.info("Save the model's training history")
pd.DataFrame(model.history.history).to_csv(os.path.join(model_object_storing_dir, 'training_history.csv'))
# Store last version of the model
logging.info("Store last version of the model")
model.save(f'./{model_object_storing_dir}/end_of_training_version')
# Store metrics for the train and holdout sets
logging.info('Store metrics for the train and holdout sets')
store_metrics(df_cancer_info=tr_fold_0_df_cancer_info, dataset_type='train', model=model,
model_object_storing_dir=model_object_storing_dir, model_params=model_params,
preprocesing_params_=preprocesing_params_)
store_metrics(df_cancer_info=holdout_fold_0_df_cancer_info, dataset_type='holdout', model=model,
model_object_storing_dir=model_object_storing_dir, model_params=model_params,
preprocesing_params_=preprocesing_params_)
