def image_def(image: str, model=None): """ Returns the image as Clothing so it can be used for matching :param image: the str with the location of the file :param model: the test model, if one is not provided it will be generated in function :return: the src file classified as a clothing item """ class_names = [ 'T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot' ] if model is None: model = build_model() __image = rgb2gray(io.imread(image)) __image = transform.resize(__image, (1, 28, 28), mode='symmetric') __image = model.predict(__image) predict = np.argmax(__image) clothing_type = class_names[predict] if isinstance(image, str): _image = PIL.Image.open(image) else: _image = image pattern = detect_pattern(_image) color = detect_clothing_color(_image) return Clothing(clothing_type, pattern, color)
def first_run(): model = model_builder.build_model() model = model_trainer.train_model(model, x_train, y_train) result = model_tester.test_model(model, x_test, y_test) model.save("./nn_models/mnist_nn_v1.h5")
def _build_model(audio_file,train_wav_file,name_surname,duration, token): classpath= os.path.join(settings.BASE_DIR, 'plugins_script',"speaker_extractor","lium_spkdiarization-8.4.1.jar" ) ubm= os.path.join(settings.MEDIA_ROOT,"models/audio/globals/ubm.gmm") start=0 audio_file="/tmp/prova.wav" gmm_path= build_model (classpath, ubm,audio_file, start, duration, train_wav_file, name_surname, token) return gmm_path
def _build_model(audio_file,train_wav_file,name_surname,duration, token): classpath= os.path.join(get_base_dir(), 'plugins_script',"speaker_extractor","lium_spkdiarization-8.4.1.jar" ) ubm= os.path.join(get_media_root(),"models/audio/globals/ubm.gmm") start=0 audio_file="/tmp/prova.wav" gmm_path= build_model (classpath, ubm,audio_file, start, duration, train_wav_file, name_surname, token) return gmm_path
def run(params): """Run training Args: params: Parameters for training Returns: None """ logging.info(f'params: {params}') strategy = distribution_utils.get_distribution_strategy( params.get('tpu_address')) batch_size = distribution_utils.update_batch_size(strategy, params['batch_size']) with strategy.scope(): model = model_builder.build_model() input_image_size = model.input_shape[1] # Build dataset train_image_paths, train_scores = data_loader.read_csv(params['train_csv'], params['image_dir'], is_training=True) validation_image_paths, validation_scores = data_loader.read_csv( params['validation_csv'], params['image_dir'], is_training=False) train_dataset = data_loader.build_dataset(train_image_paths, train_scores, is_training=True, batch_size=batch_size, target_size=input_image_size) validation_dataset = data_loader.build_dataset( validation_image_paths, validation_scores, is_training=False, batch_size=batch_size, target_size=input_image_size) train_dataset = strategy.experimental_distribute_dataset(train_dataset) validation_dataset = strategy.experimental_distribute_dataset( validation_dataset) loss_fn = loss_builder.build_loss_fn( loss_name=params['loss'], trainable_variables=model.trainable_variables) train(model=model, loss_fn=loss_fn, strategy=strategy, epochs=params['epochs'], batch_size=batch_size, train_dataset=train_dataset, validation_dataset=validation_dataset, checkpoint_dir=params['checkpoint_dir'], log_dir=params['log_dir'])
def build_model(self): if self.input_pix is None: input_shape = (256, 256, 1) output_shape = (256, 256, 1) else: input_shape = self.input_pix.shape[1:] output_shape = self.output_pix.shape[1:] self.model = build_model(input_shape, output_shape) self.model.summary() optimizer = Adam(lr=1e-3) self.model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
def train_model(opt, logger): logger.info('---START---') # initialize for reproduce np.random.seed(opt.seed) # load data logger.info('---LOAD DATA---') opt, training, training_snli, validation, test_matched, test_mismatched = load_data(opt) if not opt.skip_train: logger.info('---TRAIN MODEL---') for train_counter in range(opt.max_epochs): if train_counter == 0: model = build_model(opt) else: model = load_model_local(opt) np.random.seed(train_counter) lens = len(training_snli[-1]) perm = np.random.permutation(lens) idx = perm[:int(lens * 0.2)] train_data = [np.concatenate((training[0], training_snli[0][idx])), np.concatenate((training[1], training_snli[1][idx])), np.concatenate((training[2], training_snli[2][idx]))] csv_logger = CSVLogger('{}{}.csv'.format(opt.log_dir, opt.model_name), append=True) cp_filepath = opt.save_dir + "cp-" + opt.model_name + "-" + str(train_counter) + "-{val_acc:.2f}.h5" cp = ModelCheckpoint(cp_filepath, monitor='val_acc', save_best_only=True, save_weights_only=True) callbacks = [cp, csv_logger] model.fit(train_data[:-1], train_data[-1], batch_size=opt.batch_size, epochs=1, validation_data=(validation[:-1], validation[-1]), callbacks=callbacks) save_model_local(opt, model) else: logger.info('---LOAD MODEL---') model = load_model_local(opt) # predict logger.info('---TEST MODEL---') preds_matched = model.predict(test_matched[:-1], batch_size=128, verbose=1) preds_mismatched = model.predict(test_mismatched[:-1], batch_size=128, verbose=1) save_preds_matched_to_csv(preds_matched, test_mismatched[-1], opt) save_preds_mismatched_to_csv(preds_mismatched, test_mismatched[-1], opt)
from model_builder import build_model for n in range(0, k): # god I love python trainFolds = folds[:n] + folds[n + 1:] train = [] for fold in trainFolds: train.extend(fold) test = folds[n] trainGen = listGenerator(train) testGen = listGenerator(test) model = build_model(input_length, line_length, "distcnn", "malware_only") print("Compiling Model and Training, test fold =", n) print() model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) print(model.summary()) csv_logger = CSVLogger(r".\networks\kfold" + str(n) + r".log") filepath = r".\networks\kfold" + str(n) + r"-{epoch:02d}.hdf5" checkpoint = ModelCheckpoint(filepath) model.fit_generator(
def train(cfg): # startup_prog = fluid.Program() # train_prog = fluid.Program() drop_last = True dataset = SegDataset( file_list=cfg.DATASET.TRAIN_FILE_LIST, mode=ModelPhase.TRAIN, shuffle=True, data_dir=cfg.DATASET.DATA_DIR) def data_generator(): if args.use_mpio: data_gen = dataset.multiprocess_generator( num_processes=cfg.DATALOADER.NUM_WORKERS, max_queue_size=cfg.DATALOADER.BUF_SIZE) else: data_gen = dataset.generator() batch_data = [] for b in data_gen: batch_data.append(b) if len(batch_data) == (cfg.BATCH_SIZE // cfg.NUM_TRAINERS): for item in batch_data: yield item[0], item[1], item[2] batch_data = [] # If use sync batch norm strategy, drop last batch if number of samples # in batch_data is less then cfg.BATCH_SIZE to avoid NCCL hang issues if not cfg.TRAIN.SYNC_BATCH_NORM: for item in batch_data: yield item[0], item[1], item[2] # Get device environment # places = fluid.cuda_places() if args.use_gpu else fluid.cpu_places() # place = places[0] gpu_id = int(os.environ.get('FLAGS_selected_gpus', 0)) place = fluid.CUDAPlace(gpu_id) if args.use_gpu else fluid.CPUPlace() places = fluid.cuda_places() if args.use_gpu else fluid.cpu_places() # Get number of GPU dev_count = cfg.NUM_TRAINERS if cfg.NUM_TRAINERS > 1 else len(places) print_info("#Device count: {}".format(dev_count)) # Make sure BATCH_SIZE can divided by GPU cards assert cfg.BATCH_SIZE % dev_count == 0, ( 'BATCH_SIZE:{} not divisble by number of GPUs:{}'.format( cfg.BATCH_SIZE, dev_count)) # If use multi-gpu training mode, batch data will allocated to each GPU evenly batch_size_per_dev = cfg.BATCH_SIZE // dev_count print_info("batch_size_per_dev: {}".format(batch_size_per_dev)) data_loader, loss, lr, pred, grts, masks, image = build_model( phase=ModelPhase.TRAIN) data_loader.set_sample_generator( data_generator, batch_size=batch_size_per_dev, drop_last=drop_last) exe = fluid.Executor(place) cfg.update_from_file(args.teacher_cfg_file) # teacher_arch = teacher_cfg.architecture teacher_program = fluid.Program() teacher_startup_program = fluid.Program() with fluid.program_guard(teacher_program, teacher_startup_program): with fluid.unique_name.guard(): _, teacher_loss, _, _, _, _, _ = build_model( teacher_program, teacher_startup_program, phase=ModelPhase.TRAIN, image=image, label=grts, mask=masks) exe.run(teacher_startup_program) teacher_program = teacher_program.clone(for_test=True) ckpt_dir = cfg.SLIM.KNOWLEDGE_DISTILL_TEACHER_MODEL_DIR assert ckpt_dir is not None print('load teacher model:', ckpt_dir) if os.path.exists(ckpt_dir): try: fluid.load(teacher_program, os.path.join(ckpt_dir, 'model'), exe) except: fluid.io.load_params(exe, ckpt_dir, main_program=teacher_program) # cfg = load_config(FLAGS.config) cfg.update_from_file(args.cfg_file) data_name_map = { 'image': 'image', 'label': 'label', 'mask': 'mask', } merge(teacher_program, fluid.default_main_program(), data_name_map, place) distill_pairs = [[ 'teacher_bilinear_interp_2.tmp_0', 'bilinear_interp_0.tmp_0' ]] def distill(pairs, weight): """ Add 3 pairs of distillation losses, each pair of feature maps is the input of teacher and student's yolov3_loss respectively """ loss = l2_loss(pairs[0][0], pairs[0][1]) weighted_loss = loss * weight return weighted_loss distill_loss = distill(distill_pairs, 0.1) cfg.update_from_file(args.cfg_file) optimizer = solver.Solver(None, None) all_loss = loss + distill_loss lr = optimizer.optimise(all_loss) exe.run(fluid.default_startup_program()) exec_strategy = fluid.ExecutionStrategy() # Clear temporary variables every 100 iteration if args.use_gpu: exec_strategy.num_threads = fluid.core.get_cuda_device_count() exec_strategy.num_iteration_per_drop_scope = 100 build_strategy = fluid.BuildStrategy() build_strategy.fuse_all_reduce_ops = False build_strategy.fuse_all_optimizer_ops = False build_strategy.fuse_elewise_add_act_ops = True if cfg.NUM_TRAINERS > 1 and args.use_gpu: dist_utils.prepare_for_multi_process(exe, build_strategy, fluid.default_main_program()) exec_strategy.num_threads = 1 if cfg.TRAIN.SYNC_BATCH_NORM and args.use_gpu: if dev_count > 1: # Apply sync batch norm strategy print_info("Sync BatchNorm strategy is effective.") build_strategy.sync_batch_norm = True else: print_info( "Sync BatchNorm strategy will not be effective if GPU device" " count <= 1") compiled_train_prog = fluid.CompiledProgram( fluid.default_main_program()).with_data_parallel( loss_name=all_loss.name, exec_strategy=exec_strategy, build_strategy=build_strategy) # Resume training begin_epoch = cfg.SOLVER.BEGIN_EPOCH if cfg.TRAIN.RESUME_MODEL_DIR: begin_epoch = load_checkpoint(exe, fluid.default_main_program()) # Load pretrained model elif os.path.exists(cfg.TRAIN.PRETRAINED_MODEL_DIR): load_pretrained_weights(exe, fluid.default_main_program(), cfg.TRAIN.PRETRAINED_MODEL_DIR) else: print_info( 'Pretrained model dir {} not exists, training from scratch...'. format(cfg.TRAIN.PRETRAINED_MODEL_DIR)) #fetch_list = [avg_loss.name, lr.name] fetch_list = [ loss.name, 'teacher_' + teacher_loss.name, distill_loss.name, lr.name ] if args.debug: # Fetch more variable info and use streaming confusion matrix to # calculate IoU results if in debug mode np.set_printoptions( precision=4, suppress=True, linewidth=160, floatmode="fixed") fetch_list.extend([pred.name, grts.name, masks.name]) cm = ConfusionMatrix(cfg.DATASET.NUM_CLASSES, streaming=True) if args.use_vdl: if not args.vdl_log_dir: print_info("Please specify the log directory by --vdl_log_dir.") exit(1) from visualdl import LogWriter log_writer = LogWriter(args.vdl_log_dir) # trainer_id = int(os.getenv("PADDLE_TRAINER_ID", 0)) # num_trainers = int(os.environ.get('PADDLE_TRAINERS_NUM', 1)) step = 0 all_step = cfg.DATASET.TRAIN_TOTAL_IMAGES // cfg.BATCH_SIZE if cfg.DATASET.TRAIN_TOTAL_IMAGES % cfg.BATCH_SIZE and drop_last != True: all_step += 1 all_step *= (cfg.SOLVER.NUM_EPOCHS - begin_epoch + 1) avg_loss = 0.0 avg_t_loss = 0.0 avg_d_loss = 0.0 best_mIoU = 0.0 timer = Timer() timer.start() if begin_epoch > cfg.SOLVER.NUM_EPOCHS: raise ValueError( ("begin epoch[{}] is larger than cfg.SOLVER.NUM_EPOCHS[{}]").format( begin_epoch, cfg.SOLVER.NUM_EPOCHS)) if args.use_mpio: print_info("Use multiprocess reader") else: print_info("Use multi-thread reader") for epoch in range(begin_epoch, cfg.SOLVER.NUM_EPOCHS + 1): data_loader.start() while True: try: if args.debug: # Print category IoU and accuracy to check whether the # traning process is corresponed to expectation loss, lr, pred, grts, masks = exe.run( program=compiled_train_prog, fetch_list=fetch_list, return_numpy=True) cm.calculate(pred, grts, masks) avg_loss += np.mean(np.array(loss)) step += 1 if step % args.log_steps == 0: speed = args.log_steps / timer.elapsed_time() avg_loss /= args.log_steps category_acc, mean_acc = cm.accuracy() category_iou, mean_iou = cm.mean_iou() print_info(( "epoch={} step={} lr={:.5f} loss={:.4f} acc={:.5f} mIoU={:.5f} step/sec={:.3f} | ETA {}" ).format(epoch, step, lr[0], avg_loss, mean_acc, mean_iou, speed, calculate_eta(all_step - step, speed))) print_info("Category IoU: ", category_iou) print_info("Category Acc: ", category_acc) if args.use_vdl: log_writer.add_scalar('Train/mean_iou', mean_iou, step) log_writer.add_scalar('Train/mean_acc', mean_acc, step) log_writer.add_scalar('Train/loss', avg_loss, step) log_writer.add_scalar('Train/lr', lr[0], step) log_writer.add_scalar('Train/step/sec', speed, step) sys.stdout.flush() avg_loss = 0.0 cm.zero_matrix() timer.restart() else: # If not in debug mode, avoid unnessary log and calculate loss, t_loss, d_loss, lr = exe.run( program=compiled_train_prog, fetch_list=fetch_list, return_numpy=True) avg_loss += np.mean(np.array(loss)) avg_t_loss += np.mean(np.array(t_loss)) avg_d_loss += np.mean(np.array(d_loss)) step += 1 if step % args.log_steps == 0 and cfg.TRAINER_ID == 0: avg_loss /= args.log_steps avg_t_loss /= args.log_steps avg_d_loss /= args.log_steps speed = args.log_steps / timer.elapsed_time() print(( "epoch={} step={} lr={:.5f} loss={:.4f} teacher loss={:.4f} distill loss={:.4f} step/sec={:.3f} | ETA {}" ).format(epoch, step, lr[0], avg_loss, avg_t_loss, avg_d_loss, speed, calculate_eta(all_step - step, speed))) if args.use_vdl: log_writer.add_scalar('Train/loss', avg_loss, step) log_writer.add_scalar('Train/lr', lr[0], step) log_writer.add_scalar('Train/speed', speed, step) sys.stdout.flush() avg_loss = 0.0 avg_t_loss = 0.0 avg_d_loss = 0.0 timer.restart() except fluid.core.EOFException: data_loader.reset() break except Exception as e: print(e) if (epoch % cfg.TRAIN.SNAPSHOT_EPOCH == 0 or epoch == cfg.SOLVER.NUM_EPOCHS) and cfg.TRAINER_ID == 0: ckpt_dir = save_checkpoint(fluid.default_main_program(), epoch) if args.do_eval: print("Evaluation start") _, mean_iou, _, mean_acc = evaluate( cfg=cfg, ckpt_dir=ckpt_dir, use_gpu=args.use_gpu, use_mpio=args.use_mpio) if args.use_vdl: log_writer.add_scalar('Evaluate/mean_iou', mean_iou, step) log_writer.add_scalar('Evaluate/mean_acc', mean_acc, step) if mean_iou > best_mIoU: best_mIoU = mean_iou update_best_model(ckpt_dir) print_info("Save best model {} to {}, mIoU = {:.4f}".format( ckpt_dir, os.path.join(cfg.TRAIN.MODEL_SAVE_DIR, 'best_model'), mean_iou)) # Use VisualDL to visualize results if args.use_vdl and cfg.DATASET.VIS_FILE_LIST is not None: visualize( cfg=cfg, use_gpu=args.use_gpu, vis_file_list=cfg.DATASET.VIS_FILE_LIST, vis_dir="visual", ckpt_dir=ckpt_dir, log_writer=log_writer) if cfg.TRAINER_ID == 0: ckpt_dir = save_checkpoint(fluid.default_main_program(), epoch) # save final model if cfg.TRAINER_ID == 0: save_checkpoint(fluid.default_main_program(), 'final')
def query(model_name, experiment_name, inputs, target): MODEL_NAME = experiment_name description = model_builder.load_description(MODEL_PATH, MODEL_NAME) net = model_builder.build_model(model_name, description) net.load(MODEL_PATH, MODEL_NAME, description) # net.model.summary() # print net.model.get_layer(name="embedding_2").get_weights()[0] print net.role_vocabulary print("unk_word_id", net.unk_word_id) print("missing_word_id", net.missing_word_id) # net.set_0_bias() net.model.summary() propbank_map = { "subj" : "A0", "obj" : "A1", "ARG0" : "A0", "ARG1" : "A1", "ARG2" : "A2", } # tr_map = { # "A0": numpy.asarray([[net.role_vocabulary["A0"]]], dtype=numpy.int64), # "A1": numpy.asarray([[net.role_vocabulary["A1"]]], dtype=numpy.int64), # "A2": numpy.asarray([[net.role_vocabulary["<UNKNOWN>"]]], dtype=numpy.int64) # } # net.word_vocabulary["<NOTHING>"] = net.missing_word_id # net.role_vocabulary["<UNKNOWN>"] = net.unk_role_id reverse_vocabulary = utils.get_reverse_map(net.word_vocabulary) reverse_role_vocabulary = utils.get_reverse_map(net.role_vocabulary) print reverse_role_vocabulary raw_words = dict((reverse_role_vocabulary[r], reverse_vocabulary[net.missing_word_id]) for r in net.role_vocabulary.values()) # print raw_words raw_words.update(inputs) # print raw_words # print len(raw_words) assert len(raw_words) == len(net.role_vocabulary) # print repr(raw_words) # n = int(sys.argv[3]) t_r = [net.role_vocabulary.get(r, net.unk_role_id) for r in target.keys()] t_w = [net.word_vocabulary.get(w, net.unk_word_id) for w in target.values()] input_roles_words = {} for r, w in raw_words.items(): input_roles_words[net.role_vocabulary[r]] = utils.input_word_index(net.word_vocabulary, w, net.unk_word_id, warn_unk=True) print input_roles_words, t_r input_roles_words.pop(t_r[0]) # default_roles_words = dict((r, net.missing_word_id) for r in (net.role_vocabulary.values())) # default_roles_words.update(input_roles_words) # input_roles_words = default_roles_words x_w_i = numpy.asarray([input_roles_words.values()], dtype=numpy.int64) x_r_i = numpy.asarray([input_roles_words.keys()], dtype=numpy.int64) y_w_i = numpy.asarray(t_w, dtype=numpy.int64) y_r_i = numpy.asarray(t_r, dtype=numpy.int64) topN=20 predicted_word_indices = net.top_words(x_w_i, x_r_i, y_w_i, y_r_i, topN) # print predicted_word_indices # print len(predicted_word_indices) print(x_w_i, x_r_i, y_w_i, y_r_i) p_w = net.p_words(x_w_i, x_r_i, y_w_i, y_r_i, batch_size=1, verbose=0)[0] print ('p_t_w: ', p_w) resultlist = predicted_word_indices # print resultlist for i, t_w_i in enumerate(resultlist): t_w = net.word_vocabulary.get(t_w_i, net.unk_word_id) y_w_i = numpy.asarray([t_w_i], dtype=numpy.int64) p = net.p_words(x_w_i, x_r_i, y_w_i, y_r_i, batch_size=1, verbose=0)[0] n = numpy.round(p / 0.005) fb = numpy.floor(n) hb = n % 2 print u"{:<5} {:7.6f} {:<20} ".format(i+1, float(p), reverse_vocabulary[int(t_w_i)]) + u"\u2588" * int(fb) + u"\u258C" * int(hb)
def _prepare_model(self, config): print('[Logging] preparing model...') self.model = build_model(self._model_name, config['models'], self._cuda, self.data_utils) self.lr = config['lr']
def eval_bicknell_switch(model_name, experiment_name, evaluation, model=None, print_result=True, switch_test=False): MODEL_NAME = experiment_name if model: net = model else: description = model_builder.load_description(MODEL_PATH, MODEL_NAME) net = model_builder.build_model(model_name, description) net.load(MODEL_PATH, MODEL_NAME, description) bias = net.set_0_bias() if print_result: print net.role_vocabulary eval_data_file = os.path.join(RF_EVAL_PATH, evaluation + '.txt') result_file = os.path.join(MODEL_PATH, MODEL_NAME + '_' + evaluation + '.txt') probs = [] baseline = [] oov_count = 0 if print_result: print eval_data_file print "=" * 60 dataset = numpy.genfromtxt(eval_data_file, dtype=str, delimiter='\t', usecols=[0, 1, 2, 3, 4]) samples = [] i = 0 while True: d = dataset[i] d2 = dataset[i + 1] A0 = d[0][:-2] V = d[1][:-2] assert d2[0][:-2] == A0 assert d2[1][:-2] == V if d[3] == 'yes': assert d2[3] == 'no' A1_correct = d[2][:-2] A1_incorrect = d2[2][:-2] b_correct = d[4] b_incorrect = d2[4] else: assert d[3] == 'no' A1_correct = d2[2][:-2] A1_incorrect = d[2][:-2] b_correct = d2[4] b_incorrect = d[4] if A1_correct not in net.word_vocabulary or A1_incorrect not in net.word_vocabulary: if A1_correct not in net.word_vocabulary and print_result: print "%s MISSING FROM VOCABULARY. SKIPPING..." % A1_correct if A1_incorrect not in net.word_vocabulary and print_result: print "%s MISSING FROM VOCABULARY. SKIPPING..." % A1_incorrect else: roles = net.role_vocabulary.values() del roles[net.unk_role_id] input_roles_words = dict((r, net.missing_word_id) for r in (roles)) input_roles_words[ net.role_vocabulary["A0"]] = utils.input_word_index( net.word_vocabulary, A0, net.unk_word_id, warn_unk=True) input_roles_words[ net.role_vocabulary["V"]] = utils.input_word_index( net.word_vocabulary, V, net.unk_word_id, warn_unk=True) sample = ( numpy.asarray( [input_roles_words.values(), input_roles_words.values()], dtype=numpy.int64), # x_w_i numpy.asarray( [input_roles_words.keys(), input_roles_words.keys()], dtype=numpy.int64), # x_r_i numpy.asarray([ net.word_vocabulary[A1_correct], net.word_vocabulary[A1_incorrect] ], dtype=numpy.int64 ), # y_i (1st is correct and 2nd is incorrect numpy.asarray( [net.role_vocabulary["A1"], net.role_vocabulary["A1"]], dtype=numpy.int64), # y_r_i [b_correct, b_incorrect], # bicknell scores "\"" + A0 + " " + V + "\"", # context [A1_correct, A1_incorrect]) samples.append(sample) i += 2 if i > len(dataset) - 2: break num_samples = len(samples) num_correct = 0 num_total = 0 if print_result: print "context", "correct", "incorrect", "P(correct)", "P(incorrect)", "bicnell_correct", "bicnell_incorrect" result_list = [] for x_w_i, x_r_i, y_w_i, y_r_i, bicknell, context, a1 in samples: p = net.p_words(x_w_i, x_r_i, y_w_i, y_r_i) p_correct = p[0] p_incorrect = p[1] if print_result: print context, a1[0], a1[1], p_correct, p_incorrect, bicknell[ 0], bicknell[1] if p_correct > p_incorrect: result_list.append(1) else: result_list.append(0) num_correct += p_correct > p_incorrect num_total += 1 assert num_total == num_samples accuracy = float(num_correct) / float(num_samples) if print_result: print "Number of lines %d" % num_samples print "Baseline Lenci11 is 43/64=0.671875" print "Final score of theano model is %d/%d=%.6f" % ( num_correct, num_samples, accuracy) print result_list if switch_test and print_result: print "\nSwitch A0/A1 TEST" input_words = [] input_roles = [] for i in range(1): roles = net.role_vocabulary.values() print net.unk_role_id roles.remove(net.unk_role_id) input_role_word_pairs = dict( (r, net.missing_word_id) for r in roles) input_role_word_pairs[ net.role_vocabulary["V"]] = utils.input_word_index( net.word_vocabulary, "buy", net.unk_word_id, warn_unk=True) input_words.append(input_role_word_pairs.values()) input_roles.append(input_role_word_pairs.keys()) man = utils.input_word_index(net.word_vocabulary, "man", net.unk_word_id, warn_unk=True) car = utils.input_word_index(net.word_vocabulary, "car", net.unk_word_id, warn_unk=True) a1 = net.role_vocabulary["A1"] a0 = net.role_vocabulary["A0"] a0_test = ( numpy.asarray(input_words, dtype=numpy.int64), numpy.asarray(input_roles, dtype=numpy.int64), numpy.asarray([man, car], dtype=numpy.int64), numpy.asarray([a0], dtype=numpy.int64), ) x_w_i, x_r_i, y_w_i, y_r_i = a0_test p0 = net.p_words(x_w_i, x_r_i, y_w_i, y_r_i) print p0 a1_test = ( numpy.asarray(input_words, dtype=numpy.int64), numpy.asarray(input_roles, dtype=numpy.int64), numpy.asarray([man, car], dtype=numpy.int64), numpy.asarray([a1], dtype=numpy.int64), ) x_w_i, x_r_i, y_w_i, y_r_i = a1_test p1 = net.p_words(x_w_i, x_r_i, y_w_i, y_r_i) print p1 print "man buy", p0[0] print "buy man", p1[0] print "car buy", p0[1] print "buy car", p1[1] net.set_bias(bias) return num_correct, num_samples, accuracy
type=str, default="cifar10") args = parser.parse_args() #fixing errors which occur on HPC try: _create_unverified_https_context = ssl._create_unverified_context except AttributeError: # Legacy Python that doesn't verify HTTPS certificates by default pass else: # Handle target environment that doesn't support HTTPS verification ssl._create_default_https_context = _create_unverified_https_context model = build_model(th=0.1, dv=args.device_variation, std_dev=args.device_std_dev, add_zero_pad=(not args.dataset == "cifar10")) opt = tf.keras.optimizers.SGD(learning_rate=1) model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) if ("mnist" in args.dataset): model.build(input_shape=tf.TensorShape([None, 28, 28, 1])) else: model.build(input_shape=tf.TensorShape([None, 32, 32, 3])) print(model.summary()) dataset, dataset_test = build_dataset(args.dataset) loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False) base_path = os.path.join( os.path.abspath("./results/"), args.prefix,
def main(opt, device_id, batch_queue=None, semaphore=None): # NOTE: It's important that ``opt`` has been validated and updated # at this point. configure_process(opt, device_id) init_logger(opt.log_file) assert len(opt.accum_count) == len(opt.accum_steps), \ 'Number of accum_count values must match number of accum_steps' # Load checkpoint if we resume from a previous training. if opt.train_from: logger.info('Loading checkpoint from %s' % opt.train_from) checkpoint = torch.load(opt.train_from, map_location=lambda storage, loc: storage) model_opt = ArgumentParser.ckpt_model_opts(checkpoint["opt"]) ArgumentParser.update_model_opts(model_opt) ArgumentParser.validate_model_opts(model_opt) logger.info('Loading vocab from checkpoint at %s.' % opt.train_from) vocab = checkpoint['vocab'] else: checkpoint = None model_opt = opt vocab = torch.load(opt.data + '.vocab.pt') if opt.teacher_model_path: logger.info('Loading teacher model from {path}'.format( path=opt.teacher_model_path)) teacher_model_ckpt = torch.load( opt.teacher_model_path, map_location=lambda storage, loc: storage) teacher_model_opt = ArgumentParser.ckpt_model_opts( teacher_model_ckpt['opt']) ArgumentParser.update_model_opts(teacher_model_opt) ArgumentParser.validate_model_opts(teacher_model_opt) logger.info('Loading vocab from checkpoint at {path}'.format( path=opt.teacher_model_path)) teacher_vocab = teacher_model_ckpt['vocab'] # check for code where vocab is saved instead of fields # (in the future this will be done in a smarter way) if old_style_vocab(vocab): fields = load_old_vocab(vocab, opt.model_type, dynamic_dict=opt.copy_attn) else: fields = vocab teacher_fields = teacher_vocab if opt.teacher_model_path else None # patch for fields that may be missing in old data/model # patch_fields(opt, fields) # Report src and tgt vocab sizes, including for features report_vocab_size(fields) if teacher_fields is not None: report_vocab_size(teacher_fields) # Build model. fields_opt = {"original": fields, "teacher": teacher_fields} model = custom_builder.build_model(model_opt, opt, fields_opt, checkpoint) # model = build_model(model_opt, opt, fields, checkpoint) teacher_model = build_model( teacher_model_opt, teacher_model_opt, teacher_fields, teacher_model_ckpt) if opt.teacher_model_path else None n_params, enc, dec = _tally_parameters(model) logger.info('encoder: %d' % enc) logger.info('decoder: %d' % dec) logger.info('* number of parameters: %d' % n_params) _check_save_model_path(opt) if teacher_model is not None: n_params, enc, dec = _tally_parameters(teacher_model) logger.info('encoder: %d' % enc) logger.info('decoder: %d' % dec) logger.info('* number of parameters: %d' % n_params) _check_save_model_path(teacher_model_opt) # Build optimizer. optim = Optimizer.from_opt(model, opt, checkpoint=checkpoint) # Build model saver # model_saver = build_model_saver(model_opt, opt, model, fields, optim) model_saver = custom_model_saver.build_model_saver(model_opt, opt, model, fields_opt, optim) tgt_field = dict(teacher_fields)["tgt"].base_field if teacher_model is not None \ else dict(fields)["tgt"].base_field sos_id = tgt_field.vocab.stoi[tgt_field.init_token] if teacher_model is not None and opt.word_sampling: sampler = Emulator(teacher_model, teacher_fields, device_id, max_length=50, random_sampling_topk=5) else: sampler = None if teacher_model is not None: trainer = build_trainer(opt, device_id, model, teacher_fields, optim, model_saver, teacher_model=teacher_model, emulator=sampler) else: trainer = build_trainer(opt, device_id, model, fields, optim, model_saver, teacher_model=teacher_model, emulator=sampler) if batch_queue is None: if len(opt.data_ids) > 1: train_shards = [] for train_id in opt.data_ids: shard_base = "train_" + train_id train_shards.append(shard_base) train_iter = build_dataset_iter_multiple(train_shards, fields, opt) else: if opt.data_ids[0] is not None: shard_base = "train_" + opt.data_ids[0] else: shard_base = "train" train_iter = build_dataset_iter(shard_base, fields, opt) else: assert semaphore is not None, \ "Using batch_queue requires semaphore as well" def _train_iter(): while True: batch = batch_queue.get() semaphore.release() yield batch train_iter = _train_iter() valid_iter = build_dataset_iter("valid", fields, opt, is_train=False) if len(opt.gpu_ranks): logger.info('Starting training on GPU: %s' % opt.gpu_ranks) else: logger.info('Starting training on CPU, could be very slow') train_steps = opt.train_steps if opt.single_pass and train_steps > 0: logger.warning("Option single_pass is enabled, ignoring train_steps.") train_steps = 0 trainer.train(train_iter, train_steps, sos_id=sos_id, save_checkpoint_steps=opt.save_checkpoint_steps, valid_iter=valid_iter, valid_steps=opt.valid_steps) if trainer.report_manager.tensorboard_writer is not None: trainer.report_manager.tensorboard_writer.close()
def evaluate(model_name, experiment_name, test_name, batch_size, VR_SP_SRL=True, bootstrapping=False, majority_baseline=False): MODEL_NAME = experiment_name # repr_file = os.path.join(MODEL_PATH, 'confusionM_' + MODEL_NAME) description = model_builder.load_description(MODEL_PATH, MODEL_NAME) net = model_builder.build_model(model_name, description) net.load(MODEL_PATH, MODEL_NAME, description) n_roles = len(net.role_vocabulary) reverse_word_vocabulary = utils.get_reverse_map(net.word_vocabulary) reverse_role_vocabulary = utils.get_reverse_map(net.role_vocabulary) # net.set_0_bias() print net.role_vocabulary print("unk_word_id", net.unk_word_id) print("missing_word_id", net.missing_word_id) net.model.summary() # print net.model.metrics_names test_sample_size = 0 with open(EVAL_PATH + test_name, 'r') as lines: for l in lines: test_sample_size += 1 print(test_sample_size) test_steps = test_sample_size / float(batch_size) # test_steps = test_sample_size # # DEBUG # test_steps = 10 print 'Testing ' + test_name + ' ...' print 'VR_SP_SRL: ' + str(VR_SP_SRL) test_start = time.clock() # if re.search('NNRF_1e8', experiment_name) or re.search('MTRF_dev', experiment_name): # test_gen = get_minibatch(DATA_PATH + "NN_test", net.unk_word_id, net.unk_role_id, net.missing_word_id, # n_roles, random=False, batch_size=batch_size) # else: # test_gen = generator(DATA_PATH + "NN_test", model_name, net.unk_word_id, net.unk_role_id, net.missing_word_id, # n_roles, random=False, batch_size=batch_size) # # Test the model # test_result = net.model.evaluate_generator( # generator = test_gen, # steps = test_steps, # max_q_size = 1, # workers = 1, # pickle_safe = False # ) # print ('test_result', test_result) # Compute confusion matrix metrics_names = net.model.metrics_names result_dict = {(x, 0) for x in metrics_names} batch_n = 0 confusionM = np.zeros((n_roles, n_roles), dtype='int32') ppl_role_list = dict() result_list = [] output_list = [] for ([i_w, i_r, t_w, t_r], _) in data_gen(EVAL_PATH + test_name, model_name, net, batch_size, VR_SP_SRL=VR_SP_SRL): # zeros = np.zeros(t_r.shape) result_role = net.predict_role(i_w, i_r, t_w, t_r, batch_size) # word_emb, avg_emb, event_emb = net.avg_emb.predict([i_w, i_r, t_w, t_r], batch_size) # print word_emb.shape, avg_emb.shape, event_emb.shape # assert np.multiply(word_emb[0][0], avg_emb[0])[0] == event_emb[0][0][0] # assert np.multiply(word_emb[0][0], avg_emb[0])[1] == event_emb[0][0][1] # test role prediction of MTRF_dev, result: role prediction is useless # print i_r # print t_r.reshape(-1) # print result_role # result_word_likelihood = net.predict(i_w, i_r, t_w, t_r, batch_size)[0] # neg_log_likelihoods = -np.log(result_word_likelihood) # for i, row in enumerate(neg_log_likelihoods, start=0): # target_word = t_w[i][0] # target_role = t_r[i][0] # neg_log_likelihood = row[target_word] # ppl_role_list.setdefault(target_role, []).append(neg_log_likelihood) # print i_w, i_r, t_w, t_r for i, true_r in enumerate(t_r, start=0): # if reverse_role_vocabulary.get(t_r[0][0], '<unknown>') == 'AM-LOC': # print ("input words", [reverse_word_vocabulary.get(w, '<unknown>') for w in i_w[0]]) # print ("input roles", [reverse_role_vocabulary.get(r, '<unknown>') for r in i_r[0]]) # print ("target word", [reverse_word_vocabulary.get(w, '<unknown>') for w in t_w[0]]) # print ("target role", [reverse_role_vocabulary.get(r, '<unknown>') for r in t_r[0]]) # print ("predicted role", [reverse_role_vocabulary.get(result_role[i], '<unknown>') for r in t_r[0]]) # print '' confusionM[true_r, result_role[i]] += 1 if true_r == result_role[i]: result_list.append(1) output_list.append((true_r, result_role[i])) batch_n += 1 if batch_n % 100 == 0: print(batch_n) if batch_n >= test_steps: break # ppl_role = dict() # for k, v in ppl_role_list.items(): # neg_log_likelihood_role = np.mean(np.array(v)) # ppl_role[k] = np.exp(neg_log_likelihood_role) # obtain ZeroR baseline print confusionM majority = 1 if majority_baseline == True: for i in range(7): confusionM[i][majority] = confusionM[i][:].sum() confusionM[i][majority - 1] = 0 confusionM[i][majority + 1:] = 0 print confusionM dir_P, dir_R, dir_F1, precision, recall, F1 = stats(net, confusionM) print "Dir: %.2f \t %.2f \t %.2f" % (dir_P, dir_R, dir_F1) # np.savetxt('confusionM_' + experiment_name + '.' + test_name.strip('.dat') + '.csv', confusionM, delimiter = ',') # np.savetxt('output_' + experiment_name + '.' + test_name.strip('.dat') + '.csv', output_list, delimiter = ',') # with open(repr_file, 'w') as f_out: # f_out.write('[') # for i in range(n_roles): # f_out.write('[') # for j in range(n_roles): # f_out.write(str(confusionM[i][j]) + ", ") # f_out.write('] \n') # f_out.write(']') # print "Loss(neg_log_likelihood) by role: " # for r in ppl_role.keys(): # print (reverse_role_vocabulary[r], np.log(ppl_role[r])) print("Result by role: ") for r in range(len(precision)): print('%s: \t %.2f \t %.2f \t %.2f' % (reverse_role_vocabulary[r], precision[r], recall[r], F1[r])) test_end = time.clock() print 'test time: %f, sps: %f' % (test_end - test_start, test_steps * batch_size / (test_end - test_start)) if bootstrapping: P_mean, P_std, R_mean, R_std, F1_mean, F1_std = bootstrap( experiment_name, test_name, net, n_roles, output_list=output_list) return P_mean, P_std, R_mean, R_std, F1_mean, F1_std
val_ds = dataset_builder_fn(config["dataset"]["val_file_name"]) test_ds = dataset_builder_fn(config["dataset"]["test_file_name"]) decayed_lr = tf.train.cosine_decay( learning_rate=config["train"]["learning_rate"], global_step=global_step, decay_steps=config['train']['decay_step'], alpha=config['train']['decay_alpha']) optimizer = tf.train.AdamOptimizer(decayed_lr) anchor_num_per_output = len(config["anchor"]["scales"]) * len( config["anchor"]["aspect_ratio"]) od_model = model_builder.build_model(config["dataset"]["num_classes"], anchor_num_per_output) if config["save"]["load_model"]: od_model.load_weights(config["save"]["model_dir"]) compute_loss_fn = functools.partial( model_builder.compute_loss, num_classes=config["dataset"]["num_classes"], c_weight=config["train"]["classificaiton_loss_weight"], r_weight=config["train"]["regression_loss_weight"], neg_label_value=config["dataset"]["neg_label_value"], ignore_label_value=config["dataset"]["ignore_label_value"], negative_ratio=config["train"]["negative_ratio"]) train_loss_sum = 0 train_c_loss_sum = 0
def eval_MNR_LOC(model_name, experiment_name, evaluation, model=None, print_result=True, skip_header=False): MODEL_NAME = experiment_name if model: net = model else: description = model_builder.load_description(MODEL_PATH, MODEL_NAME) net = model_builder.build_model(model_name, description) net.load(MODEL_PATH, MODEL_NAME, description) bias = net.set_0_bias() if print_result: print net.role_vocabulary tr_map = { "ARG2": "A2", "ARG3": "A3", "ARGM-MNR": "AM-MNR", "ARGM-LOC": "AM-LOC", } if evaluation == "AM-MNR": eval_data_file = os.path.join(RV_EVAL_PATH, 'McRaeInstr-fixed' + '.txt') remove_suffix = False elif evaluation == 'AM-LOC': eval_data_file = os.path.join(RV_EVAL_PATH, 'McRaeLoc-fixed' + '.txt') remove_suffix = True else: sys.exit('No such evaluation!!!') result_file = os.path.join(MODEL_PATH, MODEL_NAME + '_' + evaluation + '.txt') probs = [] baseline = [] oov_count = 0 r_i = net.role_vocabulary["V"] if print_result: print eval_data_file, evaluation print "=" * 60 with open(eval_data_file, 'r') as f, \ open(result_file, 'w') as f_out: for i, line in enumerate(f): if i == 0 and skip_header: # print line.strip() + "\tP(instrument|verb)" continue #skip header line = line.strip() w, tw, temp1, temp2 = line.split( )[:4] # input word, target word, other stuff w = w[:-2] if remove_suffix else w tw = tw[:-2] if remove_suffix else tw w = wnl.lemmatize(w.lower(), wn.VERB) tw = wnl.lemmatize(tw.lower(), wn.NOUN) w_i = net.word_vocabulary.get(w, net.unk_word_id) tw_i = net.word_vocabulary.get(tw, net.unk_word_id) if evaluation == "AM-MNR": r = temp2 else: r = temp1 # tr_i = net.role_vocabulary.get(evaluation, net.unk_role_id) tr_i = net.role_vocabulary.get(tr_map[r], net.unk_role_id) y_r_i = numpy.asarray([tr_i], dtype=numpy.int64) if tw_i == net.unk_word_id: oov_count += 1 print w, tw f_out.write(line + "\tnan\n") continue b = float(line.split()[-1 if remove_suffix else -2]) baseline.append(b) input_roles_words = dict( (r, net.missing_word_id) for r in (net.role_vocabulary.values() + [net.unk_role_id])) input_roles_words[r_i] = w_i input_roles_words.pop(tr_i, None) x_w_i = numpy.asarray([input_roles_words.values()], dtype=numpy.int64) x_r_i = numpy.asarray([input_roles_words.keys()], dtype=numpy.int64) y_w_i = numpy.asarray([tw_i], dtype=numpy.int64) p = net.p_words(x_w_i, x_r_i, y_w_i, y_r_i) # pr = net.p_roles(x_w_i, x_r_i, y_w_i, y_r_i) probs.append(p) f_out.write(line + "\t%s\n" % p) rho, p_value = spearmanr(baseline, probs) rating = len(probs) if print_result: print "Spearman correlation: %f; 2-tailed p-value: %f" % (rho, p_value) print "Num ratings: %d (%d out of vocabulary)" % (rating, oov_count) net.set_bias(bias) return rho, p_value, oov_count, probs, baseline
def pd_themfit(model_name, experiment_name, df, predict_role='V', input_roles='all_available_args', function='filler_prob', n=5, debug=False): """ Adds a column to a pandas df with a role filler probability. For each row in the pandas df, calculates the probability that a particular role filler will fill a particular role, given a set of input roles and fillers (from that row). Keyword arguments: model_name -- The name of the model experiment_name -- The name of the model plus the name of the experiment, separated by '_' df -- The pandas dataframe. Must include columns for all propbank labels in predict_role and input_roles predict_role -- the target role (in propbank labels) for which the filler will be predicted (default: 'V') input_roles -- the set of roles (in propbank labels) that should be used as inputs (default: 'all_args') """ possible_roles = set( ['A0', 'A1', 'AM-LOC', 'AM-TMP', 'AM-MNR', '<UNKNOWN>', 'V']) try: assert predict_role in df.columns assert predict_role in possible_roles if input_roles != 'all_available_args': for r in input_roles: assert r in df.columns assert r in possible_roles except: print("NOT ALL ROLES ARE AVAILABLE AS DF COLUMNS") MODEL_NAME = experiment_name description = model_builder.load_description(MODEL_PATH, MODEL_NAME) net = model_builder.build_model(model_name, description) net.load(MODEL_PATH, MODEL_NAME, description) bias = net.set_0_bias() # net.model.summary() # print net.model.get_layer(name="embedding_2").get_weights()[0] # If no <UNKNOWN> role in the role vocabulary, add it. if net.role_vocabulary.get("<UNKNOWN>", -1) == -1: net.role_vocabulary["<UNKNOWN>"] = len(net.role_vocabulary) - 1 print("Role vocabulary", net.role_vocabulary) print("unk_word_id", net.unk_word_id) print("missing_word_id", net.missing_word_id) reverse_vocabulary = utils.get_reverse_map(net.word_vocabulary) reverse_role_vocabulary = utils.get_reverse_map(net.role_vocabulary) print("Reverse role vocabulary", reverse_role_vocabulary) raw_words = dict( (reverse_role_vocabulary[r], reverse_vocabulary[net.missing_word_id]) for r in net.role_vocabulary.values()) if input_roles == 'all_available_args': possible_roles.remove(predict_role) input_roles = possible_roles.intersection(set(df.columns)) all_roles = input_roles all_roles.add(predict_role) df = df.apply( lambda x: process_row(predict_role=predict_role, role_fillers={i: x[i] for i in all_roles}, model=net, raw_word_list=raw_words, function=function, n=n, debug=debug), axis=1) return df
def eval_greenberg(model_name, experiment_name, evaluation, model=None, print_result=True): MODEL_NAME = experiment_name if model: net = model else: description = model_builder.load_description(MODEL_PATH, MODEL_NAME) net = model_builder.build_model(model_name, description) net.load(MODEL_PATH, MODEL_NAME, description) bias = net.set_0_bias() if print_result: print(net.role_vocabulary) # Updated to python3 syntax (team1-change) eval_data_file = os.path.join(RV_EVAL_PATH, evaluation + '.txt') result_file = os.path.join(MODEL_PATH, MODEL_NAME + '_' + evaluation + '.txt') probs = [] baseline = [] oov_count = 0 tr = "A1" r_i = net.role_vocabulary["V"] tr_i = net.role_vocabulary["A1"] y_r_i = numpy.asarray([tr_i], dtype=numpy.int64) if print_result: print(eval_data_file) # Updated to python3 syntax (team1-change) print("=" * 60) # Updated to python3 syntax (team1-change) with open(eval_data_file, 'r') as f, \ open(result_file, 'w') as f_out: for line in f: line = line.strip().lower() w, tw = line.split()[:2] # input word, target word, other stuff w = w[:-2].strip() tw = tw[:-2].strip() w = wnl.lemmatize(w, wn.VERB) tw = wnl.lemmatize(tw, wn.NOUN) # a hack to fix some words # tw = word_fix.get(tw, tw) w_i = net.word_vocabulary.get(w, net.unk_word_id) tw_i = net.word_vocabulary.get(tw, net.unk_word_id) if tw_i == net.unk_word_id: oov_count += 1 print(w, tr, tw) # Updated to python3 syntax (team1-change) f_out.write(line + "\tnan\n") continue b = float(line.split()[-1]) sample = dict( (r, net.missing_word_id) for r in (net.role_vocabulary.values() + [net.unk_role_id])) sample[r_i] = w_i sample.pop(tr_i, None) x_w_i = numpy.asarray([sample.values()], dtype=numpy.int64) x_r_i = numpy.asarray([sample.keys()], dtype=numpy.int64) y_w_i = numpy.asarray([tw_i], dtype=numpy.int64) p = net.p_words(x_w_i, x_r_i, y_w_i, y_r_i) # pr = net.p_roles(x_w_i, x_r_i, y_w_i, y_r_i) if tw_i == net.unk_word_id: print('OOV: %s' % w, b, p) baseline.append(b) probs.append(p) f_out.write(line + "\t%s\n" % p) rho, p_value = spearmanr(baseline, probs) if print_result: print( f"Spearman correlation of {evaluation}: {rho}; 2-tailed p-value: {p_value}" ) # Updated to python3 syntax and f-string (team1-change) print(f"Num ratings: {len(probs)} ({oov_count} out of vocabulary)" ) # Updated to python3 syntax and f-string (team1-change) net.set_bias(bias) return rho, p_value, oov_count, probs, baseline
input_data_list, label_data_list = dp.get_all_data() print(f"train data size: {len(input_data_list)}") print(f"sample input_data shape: {input_data_list[0].shape}") print(f"sample label data: {label_data_list[0]}") valdp = ExistingDataProvider(val_imgdir, val_annotdir, model_input_size) val_data = valdp.get_all_data() print(f"validation data size: {len(val_data[0])}") model = build_model() # model.summary() modeljson = model.to_json() model_arch_save_path = os.path.join(ckpt_dirpath, "model_arch.json") with open(model_arch_save_path,'w') as fd: fd.write(modeljson) print("start model compile") # model.compile(optimizer = tf.optimizers.Adam(0.001), loss = "mse") metric_list=[ tf.keras.losses.MSE
def main(config: DictConfig) -> None: warnings.filterwarnings('ignore') print(OmegaConf.to_yaml(config)) torch.manual_seed(config.train.seed) torch.cuda.manual_seed_all(config.train.seed) np.random.seed(config.train.seed) random.seed(config.train.seed) use_cuda = config.train.cuda and torch.cuda.is_available() device = torch.device('cuda' if use_cuda else 'cpu') char2id, id2char = load_label(config.train.label_path, config.train.blank_id) train_audio_paths, train_transcripts, valid_audio_paths, valid_transcripts = load_dataset( config.train.dataset_path, config.train.mode) train_dataset = SpectrogramDataset( config.train.audio_path, train_audio_paths, train_transcripts, config.audio.sampling_rate, config.audio.n_mfcc if config.audio.feature_extraction == 'mfcc' else config.audio.n_mel, config.audio.frame_length, config.audio.frame_stride, config.audio.extension, config.audio.feature_extraction, config.audio.normalize, config.audio.spec_augment, config.audio.freq_mask_parameter, config.audio.num_time_mask, config.audio.num_freq_mask, config.train.sos_id, config.train.eos_id, ) train_sampler = BucketingSampler(train_dataset, batch_size=config.train.batch_size) train_loader = AudioDataLoader( train_dataset, batch_sampler=train_sampler, num_workers=config.train.num_workers, ) valid_dataset = SpectrogramDataset( config.train.audio_path, valid_audio_paths, valid_transcripts, config.audio.sampling_rate, config.audio.n_mfcc if config.audio.feature_extraction == 'mfcc' else config.audio.n_mel, config.audio.frame_length, config.audio.frame_stride, config.audio.extension, config.audio.feature_extraction, config.audio.normalize, config.audio.spec_augment, config.audio.freq_mask_parameter, config.audio.num_time_mask, config.audio.num_freq_mask, config.train.sos_id, config.train.eos_id, ) valid_sampler = BucketingSampler(valid_dataset, batch_size=config.train.batch_size) valid_loader = AudioDataLoader( valid_dataset, batch_sampler=valid_sampler, num_workers=config.train.num_workers, ) model = build_model(config, device) optimizer = optim.Adam(model.parameters(), lr=config.train.lr) print('Start Train !!!') for epoch in range(0, config.train.epochs): train(config, model, device, train_loader, valid_loader, train_sampler, optimizer, epoch, id2char, epoch) if epoch % 2 == 0: torch.save( model, os.path.join(os.getcwd(), config.train.model_save_path + str(epoch) + '.pt'))
def eval_GS(model_name, experiment_name, eval_file_name, model=None, print_result=True, verb_baseline=False): MODEL_NAME = experiment_name eval_file = os.path.join(EVAL_PATH, eval_file_name) result_file = os.path.join(MODEL_PATH, MODEL_NAME + '_' + eval_file_name) if model: net = model else: description = model_builder.load_description(MODEL_PATH, MODEL_NAME) net = model_builder.build_model(model_name, description) net.load(MODEL_PATH, MODEL_NAME, description) sent_layer = 'context_embedding' sent_model = Model(inputs=net.model.input, outputs=net.model.get_layer(sent_layer).output) # if print_result: # sent_model.summary() n_input_length = len(net.role_vocabulary) - 1 print net.role_vocabulary scores = [] similarities = [] original_sim_f = [] similarities_f = [] lo_similarities = [] hi_similarities = [] records = [] print("Embedding: " + experiment_name) print("=" * 60) print("\n") print("sentence1\tsentence2\taverage_score\tembedding_cosine") print("-" * 60) with open(eval_file, 'r') as f, \ open(result_file, 'w') as f_out: first = True for line in f: # skip header if first: first = False continue s = line.split() sentence = " ".join(s[1:5]) score = float(s[5]) hilo = s[6].upper() # verb subject object landmark # A1 - object; A0 - subject V1, A0, A1, V2 = sentence.split() V1 = wnl.lemmatize(V1, wn.VERB) A0 = wnl.lemmatize(A0, wn.NOUN) A1 = wnl.lemmatize(A1, wn.NOUN) V2 = wnl.lemmatize(V2, wn.VERB) V1_i = net.word_vocabulary.get(V1, net.unk_word_id) A0_i = net.word_vocabulary.get(A0, net.unk_word_id) A1_i = net.word_vocabulary.get(A1, net.unk_word_id) V2_i = net.word_vocabulary.get(V2, net.unk_word_id) # if np.array([V1_i, A0_i, A1_i, V2_i]).any() == net.unk_word_id: # print 'OOV: ', A0, A1, V1, V2 V_ri = net.role_vocabulary['V'] A0_ri = net.role_vocabulary['A0'] A1_ri = net.role_vocabulary['A1'] sent1_x = dict((r, net.missing_word_id) for r in (net.role_vocabulary.values())) sent2_x = dict((r, net.missing_word_id) for r in (net.role_vocabulary.values())) sent1_x.pop(n_input_length) sent2_x.pop(n_input_length) sent1_x[V_ri] = V1_i sent2_x[V_ri] = V2_i if not verb_baseline: sent1_x[A0_ri] = A0_i sent1_x[A1_ri] = A1_i sent2_x[A0_ri] = A0_i sent2_x[A1_ri] = A1_i zeroA = np.array([0]) s1_w = np.array(sent1_x.values()).reshape((1, n_input_length)) s1_r = np.array(sent1_x.keys()).reshape((1, n_input_length)) s2_w = np.array(sent2_x.values()).reshape((1, n_input_length)) s2_r = np.array(sent2_x.keys()).reshape((1, n_input_length)) if re.search('NNRF', model_name): sent1_emb = sent_model.predict([s1_w, s1_r, zeroA]) sent2_emb = sent_model.predict([s2_w, s2_r, zeroA]) else: sent1_emb = sent_model.predict([s1_w, s1_r, zeroA, zeroA]) sent2_emb = sent_model.predict([s2_w, s2_r, zeroA, zeroA]) # Baseline #sent1_emb = V1_i #sent2_emb = V2_i # Compositional # sent1_emb = V1_i + A0_i + A1_i # sent2_emb = V2_i + A0_i + A1_i #sent1_emb = V1_i * A0_i * A1_i #sent2_emb = V2_i * A0_i * A1_i similarity = -(cosine(sent1_emb, sent2_emb) - 1.0 ) # convert distance to similarity if hilo == "HIGH": hi_similarities.append(similarity) elif hilo == "LOW": lo_similarities.append(similarity) else: raise Exception("Unknown hilo value %s" % hilo) if (V1, A0, A1, V2) not in records: records.append((V1, A0, A1, V2)) # print "\"%s %s %s\"\t\"%s %s %s\"\t%.2f\t%.2f \n" % (A0, V1, A1, A0, V2, A1, score, similarity) scores.append(score) similarities.append(similarity) f_out.write("\"%s %s %s\"\t\"%s %s %s\"\t %.2f \t %.2f \n" % (A0, V1, A1, A0, V2, A1, score, similarity)) print("-" * 60) correlation, pvalue = spearmanr(scores, similarities) if print_result: print("Total number of samples: %d" % len(scores) ) #Added paranthesis to the print statements (team1-change) print("Spearman correlation: %.4f; 2-tailed p-value: %.10f" % (correlation, pvalue) ) #Added paranthesis to the print statements (team1-change) print("High: %.2f; Low: %.2f" % (np.mean(hi_similarities), np.mean(lo_similarities)) ) #Added paranthesis to the print statements (team1-change) # import pylab # pylab.scatter(scores, similarities) # pylab.show() return correlation
annotations_test = tools.load_annotations('test', opts) # Create data loaders: data_train = data_loader(annotations_train, opts) data_val = data_loader(annotations_val, opts) data_test = data_loader(annotations_test, opts) # Create the computational graph, or import it: if opts.restore_model: # Load meta-graph: checkpoint = tools.get_checkpoint(opts) metamodel_file = checkpoint + '.meta' saver = tf.train.import_meta_graph(metamodel_file) else: # Create network and loss: gradients = model_builder.build_model(opts) saver = tf.train.Saver() # Start clock: start = time.time() with tf.Session(config=tf_config) as sess: if opts.restore_model: checkpoint = tools.get_checkpoint(opts) saver.restore(sess, checkpoint) else: sess.run(tf.global_variables_initializer()) # Train: if opts.train:
# batch_size = 1 steps_per_epoch = num_samples/batch_size valid_steps = num_samples_valid/batch_size # should be this epochs = 100 from model_builder import build_model # model = build_model(input_length, line_length, "cnn", "malware_only") # model = build_model(input_length, line_length, "convlstm", "malware_only") # model = build_model(input_length, line_length, "minconvrnn", "malware_only") # model = build_model(input_length, line_length, "distcnn", "malware_only") model = build_model(input_length, line_length, "distcnn", "binary") # model = build_model(input_length, line_length, "distcnn", "binary", extras=["noembed"]) # model = load_model(r"D:\Research\3_Kaggle\kaggle_networks\93. NOPOOL net based on 79\KaggleConv-15.hdf5", # model = load_model(r".\networks\rnn binary final nets\addconv\KaggleConv-09.hdf5", # custom_objects={'DecayingConvLSTM2D':MinConvRNN, # 'window_size': window_size , # 'ELU': elu, # } # ) print("Compiling Model and Training") print()
def eval_pado_mcrae(model_name, experiment_name, evaluation, model=None, print_result=True): MODEL_NAME = experiment_name if model: net = model else: description = model_builder.load_description(MODEL_PATH, MODEL_NAME) net = model_builder.build_model(model_name, description) net.load(MODEL_PATH, MODEL_NAME, description) bias = net.set_0_bias() # net.model.summary() # print net.model.get_layer(name="embedding_2").get_weights()[0] # If no <UNKNOWN> role in the role vocabulary, add it. if net.role_vocabulary.get("<UNKNOWN>", -1) == -1: net.role_vocabulary["<UNKNOWN>"] = len(net.role_vocabulary) - 1 if print_result: print net.role_vocabulary print("unk_word_id", net.unk_word_id) print("missing_word_id", net.missing_word_id) propbank_map = { "subj" : "A0", "obj" : "A1", "ARG0" : "A0", "ARG1" : "A1", } tr_map = { "A0": numpy.asarray([net.role_vocabulary["A0"]], dtype=numpy.int64), "A1": numpy.asarray([net.role_vocabulary["A1"]], dtype=numpy.int64), "<UNKNOWN>": numpy.asarray([net.role_vocabulary["<UNKNOWN>"]], dtype=numpy.int64) } if "A2" not in net.role_vocabulary.keys(): propbank_map["ARG2"] = "<UNKNOWN>" tr_map["A2"] = numpy.asarray([net.role_vocabulary["<UNKNOWN>"]], dtype=numpy.int64) else: propbank_map["ARG2"] = "A2" tr_map["A2"] = numpy.asarray([net.role_vocabulary["A2"]], dtype=numpy.int64) fixed = False if evaluation == "pado": eval_data_file = os.path.join(EVAL_PATH, 'pado_plausibility_pb.txt') elif evaluation == 'mcrae': eval_data_file = os.path.join(EVAL_PATH, 'mcrae_agent_patient_more.txt') else: fixed = True if evaluation == 'pado_fixed': eval_data_file = os.path.join(RV_EVAL_PATH, 'Pado-AsadFixes.txt') elif evaluation == 'mcrae_fixed': eval_data_file = os.path.join(RV_EVAL_PATH, 'McRaeNN-fixed.txt') else: eval_data_file = os.path.join(COMP_EVAL_PATH, 'compare-pado.txt') result_file = os.path.join(MODEL_PATH, MODEL_NAME + '_' + evaluation + '.txt') r_i = net.role_vocabulary["V"] probs = {} baseline = {} oov_count = {} blist=[] plist = [] if print_result: print eval_data_file print "="* 60 with open(eval_data_file, 'r') as f, \ open(result_file, 'w') as f_out: for i, line in enumerate(f): line = line.strip() if line == "": continue w, tw, tr = line.split()[:3] # input word, target word, role w = w[:-2] if fixed else w tw = tw[:-2] if fixed else tw w = wnl.lemmatize(w, wn.VERB) tw = wnl.lemmatize(tw, wn.NOUN) w_i = net.word_vocabulary.get(w, net.unk_word_id) tw_i = net.word_vocabulary.get(tw, net.unk_word_id) tr_i = net.role_vocabulary.get(propbank_map[tr], net.unk_role_id) if tw_i == net.unk_word_id: print w, tr, tw oov_count[tr] = oov_count.get(tr, 0) + 1 f_out.write(line + "\tnan\n") continue b = float(line.split()[3]) baseline.setdefault(tr, []).append(b) blist.append(b) sample = dict((r, net.missing_word_id) for r in (net.role_vocabulary.values() + [net.unk_role_id])) sample[r_i] = w_i sample.pop(net.role_vocabulary[propbank_map[tr]], None) x_w_i = numpy.asarray([sample.values()], dtype=numpy.int64) x_r_i = numpy.asarray([sample.keys()], dtype=numpy.int64) y_w_i = numpy.asarray([tw_i]) y_r_i = tr_map[propbank_map[tr]] s = net.p_words(x_w_i, x_r_i, y_w_i, y_r_i) # pr = net.p_roles(x_w_i, x_r_i, y_w_i, y_r_i) plist.append(s) probs.setdefault(tr, []).append(s) f_out.write(line + "\t%s\n" % s) result = dict() for r, b in baseline.iteritems(): p = probs[r] rho, p_value = spearmanr(b, p) rating = len(p) oov = oov_count.get(r, 0) result[r] = round(rho, 4) if print_result: print "=" * 60 print "ROLE: %s" % r print "-" * 60 print "Spearman correlation: %f; 2-tailed p-value: %f" % (rho, p_value) print "Num ratings: %d (%d out of vocabulary)" % (rating, oov) rho, p_value = spearmanr(blist, plist) result['all'] = round(rho, 4) if print_result: print "Spearman correlation of %s: %f; 2-tailed p-value: %f" % (evaluation, rho, p_value) net.set_bias(bias) return result, plist, blist
def main(): batch_size = 5 num_epochs = 100 df_train, df_val, df_test = load_data() class_map, class_index, class_name = form_class_map() train_set = ImageDataset(df_train) train_dataloader = DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=8) val_set = ImageDataset(df_val) val_dataloader = DataLoader(val_set, batch_size=batch_size, shuffle=False, num_workers=8) test_set = ImageDataset(df_test) test_dataloader = DataLoader(test_set, batch_size=batch_size, shuffle=False, num_workers=8) model = build_model('UNet') model.apply(init_weights) model.to(DEVICE) criterion = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(model.parameters(), lr=1e-3, weight_decay=1e-5) n_params = calc_n_params(model) print("No. of parameters:", n_params) frame_rate = test_inf_speed(model, test_set, test_dataloader) print("Frame rate: {:.2f} FPS (frame per second)".format(frame_rate)) df = pd.DataFrame(columns=class_name + ["Train", "Val"]) model.train() for epoch in range(num_epochs): model = train(model, train_dataloader, criterion, optimizer, epoch) start = time.time() val_loss, val_acc, val_iou = validate(model, val_dataloader, criterion, class_index) train_loss, train_acc, _ = validate(model, train_dataloader, criterion, class_index) end = time.time() print('Validate: {:.1f}s'.format(end - start)) print( 'Train Loss: {:.4f}\tTrain Accuracy: {:.4f}\tVal Loss: {:.4f}\tVal Accuracy: {:.4f}' .format(train_loss, train_acc, val_loss, val_acc)) df.loc[len(df.index)] = val_iou + [train_acc, val_acc] print('Val IOU:') for i in range(len(class_index)): print('{}: {:.4f}\t'.format(class_name[i], val_iou[i]), end='') if i % 6 == 5 or i == len(class_index) - 1: print() print('mIOU: {:.4f}'.format(np.mean(val_iou))) df.to_csv("iou.csv", header=True, index=False, columns=class_name) df.to_csv("acc.csv", header=True, index=False, columns=["Train", "Val"]) visualize_results(df, class_name) test(model, test_dataloader, class_map) torch.save( { 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), }, "Model_" + str(num_epochs))
class_names_string = class_names_string + class_name + ", " else: class_names_string = class_names_string + class_name num_classes = len(label_values) config = tf.ConfigProto() config.gpu_options.allow_growth = True sess=tf.Session(config=config) # Compute your softmax cross entropy loss net_input = tf.placeholder(tf.float32,shape=[None,None,None,3]) net_output = tf.placeholder(tf.float32,shape=[None,None,None,num_classes]) network, init_fn = model_builder.build_model(args.model, frontend=args.frontend, net_input=net_input, num_classes=num_classes, is_training=True) loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=network, labels=net_output)) opt = tf.train.RMSPropOptimizer(learning_rate=0.0001, decay=0.995).minimize(loss, var_list=[var for var in tf.trainable_variables()]) saver=tf.train.Saver(max_to_keep=1000) sess.run(tf.global_variables_initializer()) utils.count_params() # If a pre-trained ResNet is required, load the weights. # This must be done AFTER the variables are initialized with sess.run(tf.global_variables_initializer()) if init_fn is not None: init_fn(sess)
def train(cfg): startup_prog = fluid.Program() train_prog = fluid.Program() if args.enable_ce: startup_prog.random_seed = 1000 train_prog.random_seed = 1000 drop_last = True dataset = SegDataset(file_list=cfg.DATASET.TRAIN_FILE_LIST, mode=ModelPhase.TRAIN, shuffle=True, data_dir=cfg.DATASET.DATA_DIR) def data_generator(): if args.use_mpio: data_gen = dataset.multiprocess_generator( num_processes=cfg.DATALOADER.NUM_WORKERS, max_queue_size=cfg.DATALOADER.BUF_SIZE) else: data_gen = dataset.generator() batch_data = [] for b in data_gen: batch_data.append(b) if len(batch_data) == (cfg.BATCH_SIZE // cfg.NUM_TRAINERS): for item in batch_data: yield item[0], item[1], item[2] batch_data = [] # If use sync batch norm strategy, drop last batch if number of samples # in batch_data is less then cfg.BATCH_SIZE to avoid NCCL hang issues if not cfg.TRAIN.SYNC_BATCH_NORM: for item in batch_data: yield item[0], item[1], item[2] # Get device environment # places = fluid.cuda_places() if args.use_gpu else fluid.cpu_places() # place = places[0] gpu_id = int(os.environ.get('FLAGS_selected_gpus', 0)) place = fluid.CUDAPlace(gpu_id) if args.use_gpu else fluid.CPUPlace() places = fluid.cuda_places() if args.use_gpu else fluid.cpu_places() # Get number of GPU dev_count = cfg.NUM_TRAINERS if cfg.NUM_TRAINERS > 1 else len(places) print_info("#Device count: {}".format(dev_count)) # Make sure BATCH_SIZE can divided by GPU cards assert cfg.BATCH_SIZE % dev_count == 0, ( 'BATCH_SIZE:{} not divisble by number of GPUs:{}'.format( cfg.BATCH_SIZE, dev_count)) # If use multi-gpu training mode, batch data will allocated to each GPU evenly batch_size_per_dev = cfg.BATCH_SIZE // dev_count print_info("batch_size_per_dev: {}".format(batch_size_per_dev)) config_info = {'input_size': 769, 'output_size': 1, 'block_num': 7} config = ([(cfg.SLIM.NAS_SPACE_NAME, config_info)]) factory = SearchSpaceFactory() space = factory.get_search_space(config) port = cfg.SLIM.NAS_PORT server_address = (cfg.SLIM.NAS_ADDRESS, port) sa_nas = SANAS(config, server_addr=server_address, search_steps=cfg.SLIM.NAS_SEARCH_STEPS, is_server=cfg.SLIM.NAS_IS_SERVER) for step in range(cfg.SLIM.NAS_SEARCH_STEPS): arch = sa_nas.next_archs()[0] start_prog = fluid.Program() train_prog = fluid.Program() data_loader, avg_loss, lr, pred, grts, masks = build_model( train_prog, start_prog, arch=arch, phase=ModelPhase.TRAIN) cur_flops = flops(train_prog) print('current step:', step, 'flops:', cur_flops) data_loader.set_sample_generator(data_generator, batch_size=batch_size_per_dev, drop_last=drop_last) exe = fluid.Executor(place) exe.run(start_prog) exec_strategy = fluid.ExecutionStrategy() # Clear temporary variables every 100 iteration if args.use_gpu: exec_strategy.num_threads = fluid.core.get_cuda_device_count() exec_strategy.num_iteration_per_drop_scope = 100 build_strategy = fluid.BuildStrategy() if cfg.NUM_TRAINERS > 1 and args.use_gpu: dist_utils.prepare_for_multi_process(exe, build_strategy, train_prog) exec_strategy.num_threads = 1 if cfg.TRAIN.SYNC_BATCH_NORM and args.use_gpu: if dev_count > 1: # Apply sync batch norm strategy print_info("Sync BatchNorm strategy is effective.") build_strategy.sync_batch_norm = True else: print_info( "Sync BatchNorm strategy will not be effective if GPU device" " count <= 1") compiled_train_prog = fluid.CompiledProgram( train_prog).with_data_parallel(loss_name=avg_loss.name, exec_strategy=exec_strategy, build_strategy=build_strategy) # Resume training begin_epoch = cfg.SOLVER.BEGIN_EPOCH if cfg.TRAIN.RESUME_MODEL_DIR: begin_epoch = load_checkpoint(exe, train_prog) # Load pretrained model elif os.path.exists(cfg.TRAIN.PRETRAINED_MODEL_DIR): print_info('Pretrained model dir: ', cfg.TRAIN.PRETRAINED_MODEL_DIR) load_vars = [] load_fail_vars = [] def var_shape_matched(var, shape): """ Check whehter persitable variable shape is match with current network """ var_exist = os.path.exists( os.path.join(cfg.TRAIN.PRETRAINED_MODEL_DIR, var.name)) if var_exist: var_shape = parse_shape_from_file( os.path.join(cfg.TRAIN.PRETRAINED_MODEL_DIR, var.name)) return var_shape == shape return False for x in train_prog.list_vars(): if isinstance(x, fluid.framework.Parameter): shape = tuple(fluid.global_scope().find_var( x.name).get_tensor().shape()) if var_shape_matched(x, shape): load_vars.append(x) else: load_fail_vars.append(x) fluid.io.load_vars(exe, dirname=cfg.TRAIN.PRETRAINED_MODEL_DIR, vars=load_vars) for var in load_vars: print_info("Parameter[{}] loaded sucessfully!".format( var.name)) for var in load_fail_vars: print_info( "Parameter[{}] don't exist or shape does not match current network, skip" " to load it.".format(var.name)) print_info( "{}/{} pretrained parameters loaded successfully!".format( len(load_vars), len(load_vars) + len(load_fail_vars))) else: print_info( 'Pretrained model dir {} not exists, training from scratch...'. format(cfg.TRAIN.PRETRAINED_MODEL_DIR)) fetch_list = [avg_loss.name, lr.name] global_step = 0 all_step = cfg.DATASET.TRAIN_TOTAL_IMAGES // cfg.BATCH_SIZE if cfg.DATASET.TRAIN_TOTAL_IMAGES % cfg.BATCH_SIZE and drop_last != True: all_step += 1 all_step *= (cfg.SOLVER.NUM_EPOCHS - begin_epoch + 1) avg_loss = 0.0 timer = Timer() timer.start() if begin_epoch > cfg.SOLVER.NUM_EPOCHS: raise ValueError( ("begin epoch[{}] is larger than cfg.SOLVER.NUM_EPOCHS[{}]" ).format(begin_epoch, cfg.SOLVER.NUM_EPOCHS)) if args.use_mpio: print_info("Use multiprocess reader") else: print_info("Use multi-thread reader") best_miou = 0.0 for epoch in range(begin_epoch, cfg.SOLVER.NUM_EPOCHS + 1): data_loader.start() while True: try: loss, lr = exe.run(program=compiled_train_prog, fetch_list=fetch_list, return_numpy=True) avg_loss += np.mean(np.array(loss)) global_step += 1 if global_step % args.log_steps == 0 and cfg.TRAINER_ID == 0: avg_loss /= args.log_steps speed = args.log_steps / timer.elapsed_time() print(( "epoch={} step={} lr={:.5f} loss={:.4f} step/sec={:.3f} | ETA {}" ).format(epoch, global_step, lr[0], avg_loss, speed, calculate_eta(all_step - global_step, speed))) sys.stdout.flush() avg_loss = 0.0 timer.restart() except fluid.core.EOFException: data_loader.reset() break except Exception as e: print(e) if epoch > cfg.SLIM.NAS_START_EVAL_EPOCH: ckpt_dir = save_checkpoint(exe, train_prog, '{}_tmp'.format(port)) _, mean_iou, _, mean_acc = evaluate(cfg=cfg, arch=arch, ckpt_dir=ckpt_dir, use_gpu=args.use_gpu, use_mpio=args.use_mpio) if best_miou < mean_iou: print('search step {}, epoch {} best iou {}'.format( step, epoch, mean_iou)) best_miou = mean_iou sa_nas.reward(float(best_miou))
def evaluate(model_name, experiment_name, batch_size): MODEL_NAME = experiment_name repr_file = os.path.join(MODEL_PATH, 'confusionM_' + MODEL_NAME) description = model_builder.load_description(MODEL_PATH, MODEL_NAME) net = model_builder.build_model(model_name, description) net.load(MODEL_PATH, MODEL_NAME, description) n_roles = len(net.role_vocabulary) print(net.role_vocabulary) #Added () to print (team1-change) print("unk_word_id", net.unk_word_id) print("missing_word_id", net.missing_word_id) net.model.summary() print(net.model.metrics_names) #Added () to print (team1-change) reverse_role_vocabulary = utils.get_reverse_map(net.role_vocabulary) test_sample_size = config.OCT_TEST_SIZE test_steps = test_sample_size / batch_size # # DEBUG # test_steps = 10 print('Testing...') #Added () to print (team1-change) test_start = time.process_time() #Changed from time.clock() (team1-change) # Always use generator in Keras if re.search('NAME_WHICH_YOU_NEED_OLD_BATCHER', experiment_name): test_gen = get_minibatch(DATA_PATH + "NN_test", net.unk_word_id, net.unk_role_id, net.missing_word_id, n_roles, random=False, batch_size=batch_size) else: test_gen = generator(DATA_PATH + "NN_test", model_name, net.unk_word_id, net.unk_role_id, net.missing_word_id, n_roles, random=False, batch_size=batch_size) # Test the model test_result = net.model.evaluate_generator(generator=test_gen, steps=test_steps, max_q_size=1, workers=1, pickle_safe=False) print('test_result', test_result) # Compute confusion matrix metrics_names = net.model.metrics_names result_dict = {(x, 0) for x in metrics_names} batch_n = 0 confusionM = np.zeros((n_roles, n_roles), dtype='int32') ppl_role_list = dict() ppl_role = dict() result_list = [] for ([i_w, i_r, t_w, t_r], _) in generator(DATA_PATH + "NN_test", model_name, net.unk_word_id, net.unk_role_id, net.missing_word_id, n_roles, random=False, batch_size=batch_size): result_role = net.predict_role(i_w, i_r, t_w, t_r, batch_size) result_word_likelihood = net.predict(i_w, i_r, t_w, t_r, batch_size)[0] neg_log_likelihoods = -np.log(result_word_likelihood) for i, row in enumerate(neg_log_likelihoods, start=0): target_word = t_w[i][0] target_role = t_r[i][0] neg_log_likelihood = row[target_word] ppl_role_list.setdefault(target_role, []).append(neg_log_likelihood) for i, true_r in enumerate(t_r, start=0): confusionM[true_r, result_role[i]] += 1 if true_r == result_role[i]: result_list.append(1) batch_n += 1 print(batch_n) #Added () to print (team1-change) if batch_n >= test_steps: break for k, v in ppl_role_list.items(): neg_log_likelihood_role = np.mean(np.array(v)) ppl_role[k] = np.exp(neg_log_likelihood_role) print("Confusion Matrix: ") #Added () to print (team1-change) print(" A0, A1, LOC, TMP, MNR, V, <UNKNOWN>" ) #Added () to print (team1-change) print(confusionM) #Added () to print (team1-change) np.savetxt('confusionM_' + experiment_name + '.csv', confusionM, delimiter=',') np.savetxt('result_list_' + experiment_name + '.csv', result_list, delimiter=',') stats(net, confusionM) print("Loss(neg_log_likelihood) by role: " ) #Added () to print (team1-change) for r in ppl_role.keys(): print(reverse_role_vocabulary[r], np.log(ppl_role[r])) print("PPL by role: ") #Added () to print (team1-change) for r in ppl_role.keys(): print(reverse_role_vocabulary[r], ppl_role[r]) with open(repr_file, 'w') as f_out: f_out.write('[') for i in range(n_roles): f_out.write('[') for j in range(n_roles): f_out.write(str(confusionM[i][j]) + ", ") f_out.write('] \n') f_out.write(']') test_end = time.process_time() #Changed from time.clock() (team1-change) print('test time: %f, sps: %f' % (test_end - test_start, test_steps * batch_size / (test_end - test_start))) #Added () to print (team1-change)
def main(): parser = argparse.ArgumentParser(description="Run QA-CLEF-System") parser.add_argument('--preprocess',action="store_true") parser.add_argument('--train',action="store_true") parser.add_argument('--answeronly',action='store_true') parser.add_argument('--selftest',action='store_true') parser.add_argument('--data',nargs = '+',default=[2011],type=int) parser.add_argument('--test',nargs = '+',default=[2012],type=int) parser.add_argument('--forcedownload',action='store_true') parser.add_argument('--preprocessonly',action='store_true') parser.add_argument('--ngram', type=int, default=3) parser.add_argument('--threshold', type=float, default=0.5) parser.add_argument('--report',action='store_true') args = parser.parse_args() process_args(args) data = [] for edition in args.data + args.test: _data = qacache.find_data(edition) if args.preprocess or _data is None: input_check([edition],args.forcedownload) _data = input_parse([edition]) print >> sys.stderr, 'preprocessing ' + str(edition) + '-data' _data = preprocessing.preprocess(_data) qacache.store_preprocessed_data(edition,_data[0]) else: print >> sys.stderr, str(edition) + '-data is found on cache/' + str(edition) + '-prerocessed.txt' data.append(_data) if args.preprocessonly: print >> sys.stderr, 'Preprocess-only task is done.' sys.exit(0) # build-model print >> sys.stderr, 'Building model...' training_model = model_builder.build_model(data[:len(args.data)]) test_model = model_builder.build_model(data[-len(args.test):]) if len(args.test) != 0 and not args.selftest else [] # scoring print >> sys.stderr, 'Unweighted Feature Scoring...' training_model and scoring.score(training_model) test_model and scoring.score(test_model) # training weight = qacache.stored_weight() if args.train or weight is None: print >> sys.stderr, 'Training...' weight = train(training_model) else: print >> sys.stderr, 'Weight is found on cache/weight.txt' # weighted_scoring print >> sys.stderr, 'Weighted Feature Scoring...' final = scoring.weighted_scoring(training_model if args.selftest else test_model, weight) # answer selection select_answer(final,args.threshold) # evaluation result = evaluate(final) qacache.write_json(final,'final.txt',indent=True) if args.report: report(final, args.test if not args.selftest else args.data,weight) print "Result: %f" % result
def evaluate(cfg, ckpt_dir=None, use_gpu=False, use_mpio=False, **kwargs): np.set_printoptions(precision=5, suppress=True) startup_prog = fluid.Program() test_prog = fluid.Program() dataset = SegDataset( file_list=cfg.DATASET.VAL_FILE_LIST, mode=ModelPhase.EVAL, data_dir=cfg.DATASET.DATA_DIR) def data_generator(): #TODO: check is batch reader compatitable with Windows if use_mpio: data_gen = dataset.multiprocess_generator( num_processes=cfg.DATALOADER.NUM_WORKERS, max_queue_size=cfg.DATALOADER.BUF_SIZE) else: data_gen = dataset.generator() for b in data_gen: yield b[0], b[1], b[2] data_loader, avg_loss, pred, grts, masks = build_model( test_prog, startup_prog, phase=ModelPhase.EVAL, arch=kwargs['arch']) data_loader.set_sample_generator( data_generator, drop_last=False, batch_size=cfg.BATCH_SIZE) # Get device environment places = fluid.cuda_places() if use_gpu else fluid.cpu_places() place = places[0] dev_count = len(places) print("#Device count: {}".format(dev_count)) exe = fluid.Executor(place) exe.run(startup_prog) test_prog = test_prog.clone(for_test=True) ckpt_dir = cfg.TEST.TEST_MODEL if not ckpt_dir else ckpt_dir if not os.path.exists(ckpt_dir): raise ValueError('The TEST.TEST_MODEL {} is not found'.format(ckpt_dir)) if ckpt_dir is not None: print('load test model:', ckpt_dir) try: fluid.load(test_prog, os.path.join(ckpt_dir, 'model'), exe) except: fluid.io.load_params(exe, ckpt_dir, main_program=test_prog) # Use streaming confusion matrix to calculate mean_iou np.set_printoptions( precision=4, suppress=True, linewidth=160, floatmode="fixed") conf_mat = ConfusionMatrix(cfg.DATASET.NUM_CLASSES, streaming=True) fetch_list = [avg_loss.name, pred.name, grts.name, masks.name] num_images = 0 step = 0 all_step = cfg.DATASET.TEST_TOTAL_IMAGES // cfg.BATCH_SIZE + 1 timer = Timer() timer.start() data_loader.start() while True: try: step += 1 loss, pred, grts, masks = exe.run( test_prog, fetch_list=fetch_list, return_numpy=True) loss = np.mean(np.array(loss)) num_images += pred.shape[0] conf_mat.calculate(pred, grts, masks) _, iou = conf_mat.mean_iou() _, acc = conf_mat.accuracy() speed = 1.0 / timer.elapsed_time() print( "[EVAL]step={} loss={:.5f} acc={:.4f} IoU={:.4f} step/sec={:.2f} | ETA {}" .format(step, loss, acc, iou, speed, calculate_eta(all_step - step, speed))) timer.restart() sys.stdout.flush() except fluid.core.EOFException: break category_iou, avg_iou = conf_mat.mean_iou() category_acc, avg_acc = conf_mat.accuracy() print("[EVAL]#image={} acc={:.4f} IoU={:.4f}".format( num_images, avg_acc, avg_iou)) print("[EVAL]Category IoU:", category_iou) print("[EVAL]Category Acc:", category_acc) print("[EVAL]Kappa:{:.4f}".format(conf_mat.kappa())) return category_iou, avg_iou, category_acc, avg_acc
mean, variance = tf.nn.moments(inputs, axes=(0, 1, 2, 3), keep_dims=True, name="normalize_moments") Gamma = tf.constant(1.0, name="scale_factor", shape=mean.shape, dtype=tf.float32) Beta = tf.constant(0.0,name="offset_factor", shape=mean.shape, dtype=tf.float32) data = tf.nn.batch_normalization(inputs, mean, variance, offset=Beta, scale=Gamma, variance_epsilon=1e-3) ''' normilizing is must at first ''' result = build_model(data=data, labels=None, model_name='r2plus1d', model_depth=18, num_labels=400, num_channels=3, crop_size=112, clip_length=24, loss_scale=1.0, is_test = 1, ) rgb_saver = tf.train.Saver() global_init = tf.global_variables_initializer() feed_dict={} all_activations = {'final': []} nodes = tf.get_collection('source_node') feature = None for var in nodes: if 'final_avg/AvgPool3D:0' in var.name: feature = var
print("Model -->", args.model) print("Crop Height -->", args.crop_height) print("Crop Width -->", args.crop_width) print("Num Classes -->", num_classes) print("Image -->", args.image) # Initializing network config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) net_input = tf.placeholder(tf.float32, shape=[None, None, None, 3]) net_output = tf.placeholder(tf.float32, shape=[None, None, None, num_classes]) network, _ = model_builder.build_model(args.model, net_input=net_input, num_classes=num_classes, is_training=False) sess.run(tf.global_variables_initializer()) print('Loading model checkpoint weights') saver = tf.train.Saver(max_to_keep=1000) saver.restore(sess, args.checkpoint_path) print("Testing image " + args.image) loaded_image = utils.load_image(args.image) resized_image = cv2.resize(loaded_image, (args.crop_width, args.crop_width)) input_image = np.expand_dims(np.float32( resized_image[:args.crop_height, :args.crop_width]), axis=0) / 255.0
from model_builder import build_model, regex_baseline from model_tester import execute_test import sys import profile logistic = NeuralLogistic(restarts = 2, regularization = 0.00) nn = NeuralNetwork([(8, 'logistic'), (8, 'tanh'), (None, 'softmax')], 'maxent', regularization = 1e0, restarts = 30, max_iter = 10000, init_size = 1, step_size = 1e-2) #nn = NeuralNetwork([(None, 'softmax')], 'maxent', include_offset = True, max_iter = 5000) me = MixtureOfExperts([NeuralLogistic(regularization = 1e-1), NeuralLogistic(regularization = 1e-1), NeuralLogistic(regularization = 1e-1)], NeuralLogistic(regularization = 1e-1), max_iter = 25) #NeuralNetwork([(3, 'tanh'), (None, 'softmax')], 'maxent')) adaboost = AdaBoostClassifier(n_estimators = 500) mix_ex_args = {'experts' :[NeuralLogistic(regularization = 1e-1), NeuralLogistic(regularization = 1e-1), NeuralLogistic(regularization = 1e-1)], 'gate' : NeuralLogistic(regularization = 1e-1), 'max_iter': 15} model = build_model(regex_baseline, method = 'me', model_args = mix_ex_args) print execute_test(model, 25, 5)