def __init__(self, slackbot=None): config = Config() self.api = jenkins.Jenkins(config.open_api.jenkins.URL, username=config.open_api.jenkins.USERNAME, password=config.open_api.jenkins.PASSWORD) self.token = config.open_api.jenkins.TOKEN if slackbot is None: self.slackbot = SlackerAdapter() else: self.slackbot = slackbot
def main(mode): params = tf.contrib.training.HParams(**Config.model.to_dict()) run_config = tf.contrib.learn.RunConfig( model_dir=Config.train.model_dir, save_checkpoints_steps=Config.train.save_checkpoints_steps) tf.contrib.learn.learn_runner.run(experiment_fn=experiment_fn, run_config=run_config, schedule=mode, hparams=params) tf.logging._logger.setLevel(logging.INFO) Config(args.config) print("Config: ", Config) if Config.description: print("Config Description") for key, value in Config.description.items(): print(f" - {key}: {value}") main(args.mode)
def predict(image_file): Config('ocr-dev') vocab = load_vocab() with tf.Session() as sess: servo_dir = '/Users/dhu/code/paragraph_classfication/logs/ocr/export/Servo' latest_dir = max([ os.path.join(servo_dir, d) for d in os.listdir(servo_dir) if os.path.isdir(os.path.join(servo_dir, d)) ], key=os.path.getmtime) tf.saved_model.loader.load(sess, ["serve"], latest_dir) with open(image_file, 'rb') as f: image_bytes = f.read() classes, scores, tf_images = sess.run([ "class:0", "scores:0", 'map/TensorArrayStack/TensorArrayGatherV3:0' ], feed_dict={ "image_bytes:0": [image_bytes] }) print(vocab.reverse(classes[0] + 1)) cv2.imshow('img', tf_images[0]) cv2.waitKey()
def setUp(self): Config("config_example") print(Config)
run_config = tf.contrib.learn.RunConfig( model_dir=Config.train.model_dir, save_checkpoints_steps=Config.train.save_checkpoints_steps) tf.contrib.learn.learn_runner.run(experiment_fn=experiment.experiment_fn, run_config=run_config, schedule=mode, hparams=params) if __name__ == '__main__': parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--config', type=str, default='config', help='config file name') parser.add_argument('--mode', type=str, default='train', help='Mode (train/test/train_and_evaluate)') args = parser.parse_args() tf.logging._logger.setLevel(logging.INFO) Config(args.config) print("Config: ", Config) main(args.mode)
# Return function and hook return train_inputs, iterator_initializer_hook return get_inputs() def ids_to_str(vocab, ids): return "".join([vocab.reverse(cid) for cid in ids if cid != PAD_ID]) def sentence2id(vocab, sentence): return [vocab.get(ch) for ch in sentence] def _pad_input(input_, size): return input_ + [PAD_ID] * (size - len(input_)) if __name__ == "__main__": Config('text') read_file() # exit() vocab = load_vocab() Config.data.vocab_size = len(vocab) train_data, test_data = make_train_and_test_set() data, label = next(read_data(train_data, 10, 40)) for j in range(10): print(ids_to_str(vocab, data[j])) print(label[j])
parser.add_argument('--savecheck', type=int, default=0, help="Number of steps before model is saved") parser.add_argument('--evalcheck', type=int, default=0, help="Number of steps before model is evaluated") parser.add_argument('--modeldir', type=str, default="", help="Where to save the model") args = parser.parse_args() tf.logging._logger.setLevel(logging.INFO) # Print Config setting Config(args.config) re_config(args) print("Config: ", Config) if Config.get("description", None): print("Config Description") for key, value in Config.description.items(): print(f" - {key}: {value}") # After terminated Notification to Slack atexit.register(utils.send_message_to_slack, config_name=args.config) main(args.mode)
from hbconfig import Config import os from utils.data_loader import DataLoader from model.dmn_plus import DMN_PLUS if __name__ == '__main__': config_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config') params_path = os.path.join(config_path, 'bAbi_task1.yml') Config(params_path) print("Config: ", Config) print(params_path) data_loader = DataLoader( task_path=Config.data.task_path, task_id=Config.data.task_id, task_test_id=Config.data.task_id, w2v_dim=Config.model.embed_dim, use_pretrained=Config.model.use_pretrained ) data = data_loader.make_train_and_test_set() train_input, train_question, train_answer, train_input_mask = data['train'] vocab_size = len(data_loader.vocab) #(self, embedding_input, input_mask, embedding_question, vocab_size, params) model = DMN_PLUS(train_input, train_input_mask, train_question, vocab_size, Config)
def __init__(self): Config('configs/en_word.yml') self._dl = DataLoader() self._g = Generator(self._dl)
def __init__(self): super(CopyNet, self).__init__() Config('configs/en_word.yml') self._dl = DataLoader() self._g = Generator(self._dl)
def train(Config): setSeed(Config.train.seed, Config.use_gpu) # load data dataPath = os.path.join(Config.data.data_base_path) if Config.eval.do_eval: testData = Corpus(os.path.join(dataPath, 'test.txt'), Config.data.use_normalized_word) trainData = Corpus(os.path.join(dataPath, 'train.txt'), Config.data.use_normalized_word) if Config.use_dev: devData = Corpus(os.path.join(dataPath, 'dev.txt'), Config.data.use_normalized_word) if Config.data.get('word_embedding', False): trainData.words.update(testData.words) trainData.words.update(devData.words) trainData.bwbigrams.update(testData.bwbigrams) trainData.bwbigrams.update(devData.bwbigrams) trainData.fwbigrams.update(testData.fwbigrams) trainData.fwbigrams.update(devData.fwbigrams) # gazetter gazetters = [] if Config.model.get("graph_emb", None) is not None and Config.model.graph_emb.get('gazetter', None) is not None: for name in Config.model.graph_emb.gazetter.to_dict(): if name != 'get_tag': gaItem = Config.model.graph_emb.gazetter.get(name) gazetters.append(Gazetteer(name, gaItem['path'], Config.data.use_normalized_word, gaItem['emb_dim'], gaItem['method'], gaItem['space'], gaItem['match_ignore_case'], embedding=gaItem.get('embedding', None))) gazetters[-1].matchCorpus(testData) gazetters[-1].matchCorpus(devData) gazetters[-1].matchCorpus(trainData) # generate train data meta print("Generating corpus meta...") trainMeta = CorpusMeta(trainData, Config) trainMeta.updateMaxSentLength(devData) trainMeta.updateMaxSentLength(testData) trainMeta.gazetters = gazetters # initialize model print("Initializing model...") layerUtils = LayerUtils(Config, trainMeta) layerHelper = LayerHelper(Config, layerUtils) modelHelper = ModelHelper(Config, layerHelper) if Config.model.get('load_from_pretrain', False): model = modelHelper.loadModel() else: model = modelHelper.getModel() if Config.use_gpu: model.cuda() trainer = modelHelper.getTrainer(model) if len(Config.gpu_num) > 1: trainer = nn.DataParallel(trainer, device_ids=device_ids) # genenerate batch input print("Generating batch input...") if Config.use_dev: if len(Config.gpu_num) > 1: devRawSentenceBatch = model.module.getRawSentenceBatches(devData, trainMeta, Config.train.batch_size) else: devRawSentenceBatch = model.getRawSentenceBatches(devData, trainMeta, Config.train.batch_size) if Config.eval.do_eval: if len(Config.gpu_num) > 1: testRawSentenceBatch = model.module.getRawSentenceBatches(testData, trainMeta, Config.train.batch_size) else: testRawSentenceBatch = model.getRawSentenceBatches(testData, trainMeta, Config.train.batch_size) logFolder = os.path.join(Config.log_folder, Config.model.model_name) if not os.path.exists(logFolder): os.makedirs(logFolder) # train trainLog = open(os.path.join(logFolder, "train.log"), 'w', encoding='utf-8') trainLog.write(Config.__str__()+'\n') print(Config.__str__()) trainStart = time.time() bestF1 = -1 bestEpoch = -1 for epoch in range(1, Config.train.epoch + 1): random.shuffle(trainData.utterances) if len(Config.gpu_num) > 1: batchInput = model.module.generateBatchInput(trainData, trainMeta, Config.train.batch_size) else: batchInput = model.generateBatchInput(trainData, trainMeta, Config.train.batch_size) if Config.train.optimizer == "sgd": trainer = modelHelper.lrDecay(trainer, epoch) model.train() model.zero_grad() sampleLoss = 0 sampleCount = 0 tempStart = time.time() epochStart = time.time() epochLoss = 0 for batchItem in batchInput: if Config.use_gpu: batchItem = move2cuda(batchItem) if len(Config.gpu_num) > 1: loss = model(batchItem, negMode=True).sum() else: loss = model.negLogLikelihoodLoss(batchItem) sampleLoss += loss.data epochLoss += float(loss.data) sampleCount += len(batchItem[0]) loss.backward() trainer.step() model.zero_grad() if sampleCount >= Config.train.report_frequence: tempTime = time.time() tempCost = tempTime - tempStart tempStart = tempTime print("Process {} sentences. Loss: {:.2f}. Time: {:.2f}".format(sampleCount, loss/sampleCount, tempCost)) trainLog.write("Process {} sentences. Loss: {:.2f}. Time: {:.2f}".format(sampleCount, loss/sampleCount, tempCost) + '\n') if sampleLoss > 1e8 or str(sampleLoss) == "nan": print("ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT....") trainLog.write("ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT...." + '\n') exit(1) sys.stdout.flush() sampleLoss = 0 sampleCount = 0 epochCost = time.time() - epochStart if Config.use_dev: if len(Config.gpu_num) > 1: devBatchInput = model.module.generateBatchInput(devData, trainMeta, Config.train.batch_size) else: devBatchInput = model.generateBatchInput(devData, trainMeta, Config.train.batch_size) if Config.eval.do_eval: if len(Config.gpu_num) > 1: testBatchInpit = model.module.generateBatchInput(testData, trainMeta, Config.train.batch_size) else: testBatchInpit = model.generateBatchInput(testData, trainMeta, Config.train.batch_size) if Config.use_dev and epoch % Config.train.dev_epoch_frequence == 0: model.eval() ps, pl, rs, rl, fs, fl = evaluate(model, devBatchInput, trainMeta.tag2Idx, trainMeta.idx2Tag, rawSentenceBatch=devRawSentenceBatch, useGpu=Config.use_gpu) if fs >= bestF1: bestF1 = fs bestEpoch = epoch modelHelper.saveModel(model, epoch) print("Epoch {}. Loss: {:.2f}. Time: {:.2f}".format(epoch, epochLoss, epochCost)) print('Dev P: {:.2f} R: {:.2f} F1: {:.2f}'.format(ps, rs, fs)) ps, pl, rs, rl, fs, fl = evaluate(model, testBatchInpit, trainMeta.tag2Idx, trainMeta.idx2Tag, rawSentenceBatch=testRawSentenceBatch, useGpu=Config.use_gpu) print('Test P: {:.2f} R: {:.2f} F1: {:.2f}'.format(ps, rs, fs)) trainLog.write("Epoch {}. Loss: {:.2f}. Dev P: {:.2f} R: {:.2f} F1: {:.2f}. Time: {:.2f}".format( epoch, epochLoss, ps, rs, fs, epochCost) + '\n') else: print("Epoch {}. Loss: {:.2f}. Time: {:.2f}".format(epoch, epochLoss, epochCost)) trainLog.write("Epoch {}. Loss: {:.2f}. Time: {:.2f}".format(epoch, epochLoss, epochCost) + '\n') sys.stdout.flush() if bestF1 == 0: modelHelper.saveModel(model, 0) print("Finish training. Best epoch {}. F1 {:.2f}. Time {:.2f}".format(bestEpoch, bestF1, time.time() - trainStart)) trainLog.write("Finish training. Time {:.2f}".format(time.time() - trainStart) + '\n') trainLog.close() if Config.eval.do_eval: model = modelHelper.loadModel(bestEpoch) if Config.use_gpu: model.cuda() testBatchInpit = model.generateBatchInput(testData, trainMeta, 20) testRawSentenceBatch = model.getRawSentenceBatches(testData, trainMeta, 20) testLog = open(os.path.join(logFolder, 'test.log'), 'w', encoding='utf-8') strictFalse = open(os.path.join(logFolder, 'strictFalse'), 'w', encoding='utf-8') looseFalse = open(os.path.join(logFolder, 'looseFalse'), 'w', encoding='utf-8') rawOutput = open(os.path.join(logFolder, 'testOutput'), 'w', encoding='utf-8') goldOutput = open(os.path.join(logFolder, 'testGold'), 'w', encoding='utf-8') goldEntityOutput = open(os.path.join(logFolder, 'testGoldEntities'), 'w', encoding='utf-8') predEntityOutput = open(os.path.join(logFolder, 'testPredEntities'), 'w', encoding='utf-8') ps, pl, rs, rl, fs, fl = evaluate(model, testBatchInpit, trainMeta.tag2Idx, trainMeta.idx2Tag, ( strictFalse, looseFalse, rawOutput, goldEntityOutput, predEntityOutput, goldOutput), trainMeta.idx2Word, testRawSentenceBatch, useGpu=Config.use_gpu) print("Test P: {:.2f} R: {:.2f} F1: {:.2f}".format(ps, rs, fs)) testLog.write("Test P: {:.2f} R: {:.2f} F1: {:.2f}".format(ps, rs, fs) + '\n') strictFalse.close() looseFalse.close() testLog.close()