def __init__(self, slackbot=None):
config = Config()
self.api = jenkins.Jenkins(config.open_api.jenkins.URL,
username=config.open_api.jenkins.USERNAME,
password=config.open_api.jenkins.PASSWORD)
self.token = config.open_api.jenkins.TOKEN
if slackbot is None:
self.slackbot = SlackerAdapter()
else:
self.slackbot = slackbot
def main(mode):
params = tf.contrib.training.HParams(**Config.model.to_dict())
run_config = tf.contrib.learn.RunConfig(
model_dir=Config.train.model_dir,
save_checkpoints_steps=Config.train.save_checkpoints_steps)
tf.contrib.learn.learn_runner.run(experiment_fn=experiment_fn,
run_config=run_config,
schedule=mode,
hparams=params)
tf.logging._logger.setLevel(logging.INFO)
Config(args.config)
print("Config: ", Config)
if Config.description:
print("Config Description")
for key, value in Config.description.items():
print(f" - {key}: {value}")
main(args.mode)
def predict(image_file):
Config('ocr-dev')
vocab = load_vocab()
with tf.Session() as sess:
servo_dir = '/Users/dhu/code/paragraph_classfication/logs/ocr/export/Servo'
latest_dir = max([
os.path.join(servo_dir, d) for d in os.listdir(servo_dir)
if os.path.isdir(os.path.join(servo_dir, d))
],
key=os.path.getmtime)
tf.saved_model.loader.load(sess, ["serve"], latest_dir)
with open(image_file, 'rb') as f:
image_bytes = f.read()
classes, scores, tf_images = sess.run([
"class:0", "scores:0", 'map/TensorArrayStack/TensorArrayGatherV3:0'
],
feed_dict={
"image_bytes:0":
[image_bytes]
})
print(vocab.reverse(classes[0] + 1))
cv2.imshow('img', tf_images[0])
cv2.waitKey()
def setUp(self):
Config("config_example")
print(Config)
run_config = tf.contrib.learn.RunConfig(
model_dir=Config.train.model_dir,
save_checkpoints_steps=Config.train.save_checkpoints_steps)
tf.contrib.learn.learn_runner.run(experiment_fn=experiment.experiment_fn,
run_config=run_config,
schedule=mode,
hparams=params)
if __name__ == '__main__':
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--config',
type=str,
default='config',
help='config file name')
parser.add_argument('--mode',
type=str,
default='train',
help='Mode (train/test/train_and_evaluate)')
args = parser.parse_args()
tf.logging._logger.setLevel(logging.INFO)
Config(args.config)
print("Config: ", Config)
main(args.mode)
# Return function and hook
return train_inputs, iterator_initializer_hook
return get_inputs()
def ids_to_str(vocab, ids):
return "".join([vocab.reverse(cid) for cid in ids if cid != PAD_ID])
def sentence2id(vocab, sentence):
return [vocab.get(ch) for ch in sentence]
def _pad_input(input_, size):
return input_ + [PAD_ID] * (size - len(input_))
if __name__ == "__main__":
Config('text')
read_file()
# exit()
vocab = load_vocab()
Config.data.vocab_size = len(vocab)
train_data, test_data = make_train_and_test_set()
data, label = next(read_data(train_data, 10, 40))
for j in range(10):
print(ids_to_str(vocab, data[j]))
print(label[j])
parser.add_argument('--savecheck',
type=int,
default=0,
help="Number of steps before model is saved")
parser.add_argument('--evalcheck',
type=int,
default=0,
help="Number of steps before model is evaluated")
parser.add_argument('--modeldir',
type=str,
default="",
help="Where to save the model")
args = parser.parse_args()
tf.logging._logger.setLevel(logging.INFO)
# Print Config setting
Config(args.config)
re_config(args)
print("Config: ", Config)
if Config.get("description", None):
print("Config Description")
for key, value in Config.description.items():
print(f" - {key}: {value}")
# After terminated Notification to Slack
atexit.register(utils.send_message_to_slack, config_name=args.config)
main(args.mode)
from hbconfig import Config
import os
from utils.data_loader import DataLoader
from model.dmn_plus import DMN_PLUS
if __name__ == '__main__':
config_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config')
params_path = os.path.join(config_path, 'bAbi_task1.yml')
Config(params_path)
print("Config: ", Config)
print(params_path)
data_loader = DataLoader(
task_path=Config.data.task_path,
task_id=Config.data.task_id,
task_test_id=Config.data.task_id,
w2v_dim=Config.model.embed_dim,
use_pretrained=Config.model.use_pretrained
)
data = data_loader.make_train_and_test_set()
train_input, train_question, train_answer, train_input_mask = data['train']
vocab_size = len(data_loader.vocab)
#(self, embedding_input, input_mask, embedding_question, vocab_size, params)
model = DMN_PLUS(train_input, train_input_mask, train_question, vocab_size, Config)
def __init__(self):
Config('configs/en_word.yml')
self._dl = DataLoader()
self._g = Generator(self._dl)
def __init__(self):
super(CopyNet, self).__init__()
Config('configs/en_word.yml')
self._dl = DataLoader()
self._g = Generator(self._dl)
def train(Config):
setSeed(Config.train.seed, Config.use_gpu)
# load data
dataPath = os.path.join(Config.data.data_base_path)
if Config.eval.do_eval:
testData = Corpus(os.path.join(dataPath, 'test.txt'), Config.data.use_normalized_word)
trainData = Corpus(os.path.join(dataPath, 'train.txt'), Config.data.use_normalized_word)
if Config.use_dev:
devData = Corpus(os.path.join(dataPath, 'dev.txt'), Config.data.use_normalized_word)
if Config.data.get('word_embedding', False):
trainData.words.update(testData.words)
trainData.words.update(devData.words)
trainData.bwbigrams.update(testData.bwbigrams)
trainData.bwbigrams.update(devData.bwbigrams)
trainData.fwbigrams.update(testData.fwbigrams)
trainData.fwbigrams.update(devData.fwbigrams)
# gazetter
gazetters = []
if Config.model.get("graph_emb", None) is not None and Config.model.graph_emb.get('gazetter', None) is not None:
for name in Config.model.graph_emb.gazetter.to_dict():
if name != 'get_tag':
gaItem = Config.model.graph_emb.gazetter.get(name)
gazetters.append(Gazetteer(name, gaItem['path'], Config.data.use_normalized_word, gaItem['emb_dim'], gaItem['method'], gaItem['space'], gaItem['match_ignore_case'], embedding=gaItem.get('embedding', None)))
gazetters[-1].matchCorpus(testData)
gazetters[-1].matchCorpus(devData)
gazetters[-1].matchCorpus(trainData)
# generate train data meta
print("Generating corpus meta...")
trainMeta = CorpusMeta(trainData, Config)
trainMeta.updateMaxSentLength(devData)
trainMeta.updateMaxSentLength(testData)
trainMeta.gazetters = gazetters
# initialize model
print("Initializing model...")
layerUtils = LayerUtils(Config, trainMeta)
layerHelper = LayerHelper(Config, layerUtils)
modelHelper = ModelHelper(Config, layerHelper)
if Config.model.get('load_from_pretrain', False):
model = modelHelper.loadModel()
else:
model = modelHelper.getModel()
if Config.use_gpu:
model.cuda()
trainer = modelHelper.getTrainer(model)
if len(Config.gpu_num) > 1:
trainer = nn.DataParallel(trainer, device_ids=device_ids)
# genenerate batch input
print("Generating batch input...")
if Config.use_dev:
if len(Config.gpu_num) > 1:
devRawSentenceBatch = model.module.getRawSentenceBatches(devData, trainMeta, Config.train.batch_size)
else:
devRawSentenceBatch = model.getRawSentenceBatches(devData, trainMeta, Config.train.batch_size)
if Config.eval.do_eval:
if len(Config.gpu_num) > 1:
testRawSentenceBatch = model.module.getRawSentenceBatches(testData, trainMeta, Config.train.batch_size)
else:
testRawSentenceBatch = model.getRawSentenceBatches(testData, trainMeta, Config.train.batch_size)
logFolder = os.path.join(Config.log_folder, Config.model.model_name)
if not os.path.exists(logFolder):
os.makedirs(logFolder)
# train
trainLog = open(os.path.join(logFolder, "train.log"), 'w', encoding='utf-8')
trainLog.write(Config.__str__()+'\n')
print(Config.__str__())
trainStart = time.time()
bestF1 = -1
bestEpoch = -1
for epoch in range(1, Config.train.epoch + 1):
random.shuffle(trainData.utterances)
if len(Config.gpu_num) > 1:
batchInput = model.module.generateBatchInput(trainData, trainMeta, Config.train.batch_size)
else:
batchInput = model.generateBatchInput(trainData, trainMeta, Config.train.batch_size)
if Config.train.optimizer == "sgd":
trainer = modelHelper.lrDecay(trainer, epoch)
model.train()
model.zero_grad()
sampleLoss = 0
sampleCount = 0
tempStart = time.time()
epochStart = time.time()
epochLoss = 0
for batchItem in batchInput:
if Config.use_gpu:
batchItem = move2cuda(batchItem)
if len(Config.gpu_num) > 1:
loss = model(batchItem, negMode=True).sum()
else:
loss = model.negLogLikelihoodLoss(batchItem)
sampleLoss += loss.data
epochLoss += float(loss.data)
sampleCount += len(batchItem[0])
loss.backward()
trainer.step()
model.zero_grad()
if sampleCount >= Config.train.report_frequence:
tempTime = time.time()
tempCost = tempTime - tempStart
tempStart = tempTime
print("Process {} sentences. Loss: {:.2f}. Time: {:.2f}".format(sampleCount, loss/sampleCount, tempCost))
trainLog.write("Process {} sentences. Loss: {:.2f}. Time: {:.2f}".format(sampleCount, loss/sampleCount, tempCost) + '\n')
if sampleLoss > 1e8 or str(sampleLoss) == "nan":
print("ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT....")
trainLog.write("ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT...." + '\n')
exit(1)
sys.stdout.flush()
sampleLoss = 0
sampleCount = 0
epochCost = time.time() - epochStart
if Config.use_dev:
if len(Config.gpu_num) > 1:
devBatchInput = model.module.generateBatchInput(devData, trainMeta, Config.train.batch_size)
else:
devBatchInput = model.generateBatchInput(devData, trainMeta, Config.train.batch_size)
if Config.eval.do_eval:
if len(Config.gpu_num) > 1:
testBatchInpit = model.module.generateBatchInput(testData, trainMeta, Config.train.batch_size)
else:
testBatchInpit = model.generateBatchInput(testData, trainMeta, Config.train.batch_size)
if Config.use_dev and epoch % Config.train.dev_epoch_frequence == 0:
model.eval()
ps, pl, rs, rl, fs, fl = evaluate(model, devBatchInput, trainMeta.tag2Idx, trainMeta.idx2Tag, rawSentenceBatch=devRawSentenceBatch, useGpu=Config.use_gpu)
if fs >= bestF1:
bestF1 = fs
bestEpoch = epoch
modelHelper.saveModel(model, epoch)
print("Epoch {}. Loss: {:.2f}. Time: {:.2f}".format(epoch, epochLoss, epochCost))
print('Dev P: {:.2f} R: {:.2f} F1: {:.2f}'.format(ps, rs, fs))
ps, pl, rs, rl, fs, fl = evaluate(model, testBatchInpit, trainMeta.tag2Idx, trainMeta.idx2Tag, rawSentenceBatch=testRawSentenceBatch, useGpu=Config.use_gpu)
print('Test P: {:.2f} R: {:.2f} F1: {:.2f}'.format(ps, rs, fs))
trainLog.write("Epoch {}. Loss: {:.2f}. Dev P: {:.2f} R: {:.2f} F1: {:.2f}. Time: {:.2f}".format(
epoch, epochLoss, ps, rs, fs, epochCost) + '\n')
else:
print("Epoch {}. Loss: {:.2f}. Time: {:.2f}".format(epoch, epochLoss, epochCost))
trainLog.write("Epoch {}. Loss: {:.2f}. Time: {:.2f}".format(epoch, epochLoss, epochCost) + '\n')
sys.stdout.flush()
if bestF1 == 0:
modelHelper.saveModel(model, 0)
print("Finish training. Best epoch {}. F1 {:.2f}. Time {:.2f}".format(bestEpoch, bestF1, time.time() - trainStart))
trainLog.write("Finish training. Time {:.2f}".format(time.time() - trainStart) + '\n')
trainLog.close()
if Config.eval.do_eval:
model = modelHelper.loadModel(bestEpoch)
if Config.use_gpu:
model.cuda()
testBatchInpit = model.generateBatchInput(testData, trainMeta, 20)
testRawSentenceBatch = model.getRawSentenceBatches(testData, trainMeta, 20)
testLog = open(os.path.join(logFolder, 'test.log'), 'w', encoding='utf-8')
strictFalse = open(os.path.join(logFolder, 'strictFalse'), 'w', encoding='utf-8')
looseFalse = open(os.path.join(logFolder, 'looseFalse'), 'w', encoding='utf-8')
rawOutput = open(os.path.join(logFolder, 'testOutput'), 'w', encoding='utf-8')
goldOutput = open(os.path.join(logFolder, 'testGold'), 'w', encoding='utf-8')
goldEntityOutput = open(os.path.join(logFolder, 'testGoldEntities'), 'w', encoding='utf-8')
predEntityOutput = open(os.path.join(logFolder, 'testPredEntities'), 'w', encoding='utf-8')
ps, pl, rs, rl, fs, fl = evaluate(model, testBatchInpit, trainMeta.tag2Idx, trainMeta.idx2Tag, (
strictFalse, looseFalse, rawOutput, goldEntityOutput, predEntityOutput, goldOutput), trainMeta.idx2Word,
testRawSentenceBatch, useGpu=Config.use_gpu)
print("Test P: {:.2f} R: {:.2f} F1: {:.2f}".format(ps, rs, fs))
testLog.write("Test P: {:.2f} R: {:.2f} F1: {:.2f}".format(ps, rs, fs) + '\n')
strictFalse.close()
looseFalse.close()
testLog.close()
