def main(): config = get_config() if config['train'] and not config['resume']: for key in ['folder_log', 'folder_out']: if os.path.exists(config[key]): raise FileExistsError(config[key]) os.makedirs(config[key]) with open(os.path.join(config['folder_out'], 'config.yaml'), 'w') as f: yaml.safe_dump(config, f) data_loaders, image_shape = get_data_loaders(config) config['image_shape'] = image_shape net = get_model(config) if config['train']: train_model(config, data_loaders, net) test_model(config, data_loaders, net) return
def test_model(images_path, labels_path, model_path, remove_tmp_if_exist=True): '''' Test model where the graph was exported in pb format ''' workspace_path = '/home/osboxes/Desktop/ML/Final_Project/Workspace/test_model_data' output_path = '/home/osboxes/PycharmProjects/ML_final_project/workspace' if os.path.exists(workspace_path) and remove_tmp_if_exist: shutil.rmtree(workspace_path) dataset_utils.prepare_dataset(images_path, scale_image_size=(28, 28), normalize_values=False, change_to_gray=False, smooth_factor=1, in_memory=False, target_path=workspace_path) run_model.test_model(workspace_path, labels_path, model_path, output_path)
def main(): gpus = tf.config.experimental.list_physical_devices('GPU') for gpu in gpus: tf.config.experimental.set_memory_growth(gpu, True) config = get_config() if config['train'] and not config['resume']: for key in ['folder_log', 'folder_out']: if os.path.exists(config[key]): raise FileExistsError(config[key]) os.makedirs(config[key]) with open(os.path.join(config['folder_out'], 'config.yaml'), 'w') as f: yaml.safe_dump(config, f) strategy = tf.distribute.MirroredStrategy() data_loaders = get_data_loaders(strategy, config) net = get_model(config) if config['train']: train_model(strategy, config, data_loaders, net) test_model(strategy, config, data_loaders, net) return
def self_test_test(if_show_qual_rslt=False): """ Self test the model with toy data, testing part. """ # Load and prepare fake data buckets, bucket_collapsed, bucket_sizes, train_tuple_b, word2id, \ word_embeddings = fake_data_with_buckets() mydata = MyData(train_tuple_b, train_tuple_b, train_tuple_b, word2id, word_embeddings) assert(mydata.train_buckets == buckets) assert(mydata.train_bucket_sizes == bucket_sizes) assert(mydata.valid_buckets == buckets) assert(mydata.valid_bucket_sizes == bucket_sizes) buckets_t = zip(*buckets) # Create model with vocabulary of 10, 2 small buckets config = SelfTestConfig() config.vocab_size = 10 config.max_question_length = max(buckets_t[1]) config.max_sentence_num = max(buckets_t[0]) config.max_sentence_length = max(buckets_t[2]) config.data_type = data_type() config.batch_size = 2 config.init_scale = np.sqrt(6.0 / (config.word_embed_size + config.rnn_hidden_size)) with tf.Session() as session: # Set random seed to fixed number tf.set_random_seed(FLAGS.tf_random_seed) random.seed(FLAGS.random_seed) np.random.seed(FLAGS.random_seed) initializer = tf.random_uniform_initializer(-config.init_scale, config.init_scale) embedding_initializer = tf.constant_initializer(word_embeddings, dtype=config.data_type) with tf.variable_scope("model", reuse=False, initializer=initializer): m_test = MIL_AnswerTrigger(config=config, buckets=buckets, is_training=False, embedding_initializer=embedding_initializer) test_model(FLAGS, session, m_test, config, mydata, if_test=True, if_show_qual_rslt=if_show_qual_rslt, if_load_ckpt=True)
def main(): config = get_config() if config['train'] and not config['resume']: for key in ['folder_log', 'folder_out']: if os.path.exists(config[key]): raise FileExistsError(config[key]) os.makedirs(config[key]) with open(os.path.join(config['folder_out'], 'config.yaml'), 'w') as f: yaml.safe_dump(config, f) data_loaders, image_shape = get_data_loaders(config) config['image_shape'] = image_shape if 'crop_shape' not in config: config['crop_shape'] = [ val if idx == 0 else val // 2 for idx, val in enumerate(image_shape) ] net = get_model(config) net_gen = None if config['path_pretrain'] is None else get_model(config) if config['train']: train_model(config, data_loaders, net, net_gen) test_model(config, data_loaders, net) return
def test(if_show_qual_rslt=False): """Test the model.""" buckets = BUCKETS train_tuple_b, valid_tuple_b, test_tuple_b, word2id, word_embeddings = \ prep_data() mydata = MyData(train_tuple_b, valid_tuple_b, test_tuple_b, word2id, word_embeddings) assert(mydata.test_buckets == buckets) buckets_t = zip(*buckets) # Create model config = get_config() config.vocab_size = len(word2id) config.max_sentence_num = max(buckets_t[0]) config.max_question_length = max(buckets_t[1]) config.max_sentence_length = max(buckets_t[2]) config.data_type = data_type() config.batch_size = 1 config.init_scale = np.sqrt(6.0 / (config.word_embed_size + config.rnn_hidden_size)) print_config(config) with tf.Session() as session: # Set random seed to fixed number tf.set_random_seed(FLAGS.tf_random_seed) random.seed(FLAGS.random_seed) np.random.seed(FLAGS.random_seed) initializer = tf.random_uniform_initializer(-config.init_scale, config.init_scale) embedding_initializer = tf.constant_initializer(word_embeddings, dtype=config.data_type) with tf.variable_scope("model", reuse=None, initializer=initializer): m = MIL_AnswerTrigger(config=config, buckets=buckets, is_training=False, embedding_initializer=embedding_initializer) test_model(FLAGS, session, m, config, mydata, if_test=True, if_show_qual_rslt=if_show_qual_rslt, if_load_ckpt=True)
def main(): parser = argparse.ArgumentParser(description='Trainning model ... ') # ''' modify here accordingly ... ''' # id_process = os.getpid() time_current = datetime.datetime.now().isoformat() # # parser.add_argument('-fd', '--FileData', required=False, help='Path of the dataset') # parser.add_argument('-fp', '--FilePretrain', required=True, help='File of pretrained model') parser.add_argument('-mt', '--MapTest', required=False, help='Test Map') #parser.add_argument( # '-sr', '--SaveResults', required=False, # help='Save results ? True or False' #) parser.add_argument('--saveresults', dest='saveresults', action='store_true') parser.add_argument('--no-saveresults', dest='saveresults', action='store_false') parser.set_defaults(saveresults=False) # args = parser.parse_args() # #print "args.saveresults : ", args.saveresults # if args.FileData == None: args.FileData = None # assert (args.FilePretrain != None) args.PathPretrain = os.path.abspath(args.PathPretrain) if args.MapTest == None: args.MapTest = 'l' else: args.MapTest = str(args.MapTest) if args.saveresults == False: file_save = None else: pretrain = args.FilePretrain tag_save = pretrain.split('track_')[1].split('/model.pkl')[0] file_save = './results/result_' + tag_save + '.pkl' # print("PID is : %s" % str(id_process)) print("TIME is : %s" % time_current) # print("FileData is : %s" % args.FileData) print("FilePretrain is : %s" % args.FilePretrain) print("MapTest is : %s" % str(args.MapTest)) print("Save Results ? : %s" % file_save) # dict_args = { 'PID': id_process, 'TIME': time_current, 'FileData': args.FileData, 'FilePretrain': args.FilePretrain, 'MapTest': args.MapTest } # input_tester = { 'path_rawdata': args.FileData, 'path_model': args.FilePretrain, 'args': dict_args, 'map_test': args.MapTest, 'file_save': file_save } # run_model.test_model(input_tester)
args = get_arguments() # dataloaders, args.image_planes, args.image_full_height, args.image_full_width = get_dataloaders(args) dataloaders, image_shapes = get_data_loaders(args) args.image_planes, args.image_full_height, args.image_full_width = image_shapes args.image_crop_height = args.image_full_height // 2 args.image_crop_width = args.image_full_width // 2 if args.train: if not os.path.exists(args.folder): os.mkdir(args.folder) exclude_keys = [ 'path_config', 'folder', 'train', 'file_args', 'file_log', 'file_model', 'file_result_base', ] args_dict = { key: val for key, val in args.__dict__.items() if key not in exclude_keys } with open(os.path.join(args.folder, args.file_args), 'w') as f: yaml.safe_dump(args_dict, f) net = get_model(args).cuda() train_model(args, net, dataloaders) else: net = get_model(args, os.path.join(args.folder, args.file_model)).cuda() test_model(args, net, dataloaders)
map_sequence = ['grid', 'l', 'jelly'] if __name__ == '__main__': print('Start Map tester ...') # declare the parser parser = argparse.ArgumentParser(description='test the hong yuan moel') # add the argument # parser.add_argument('-log_file', help='log file content', required = True) # parser.add_argument('-save_file_path', help='save file path', required = True) parser.add_argument('-model_path', help='model path', required=True) parser.add_argument('-seed', help='random seed', required=True) parser.add_argument('-test_map', help='map_for_test', required=True) parser.add_argument('-file_save', help='file save path', required=True) # parse the argument args = parser.parse_args() # input_trainer['log_file'] = args.log_file # input_trainer['save_file_path'] = args.save_file_path input_tester['seed'] = int(args.seed) input_tester['map_test'] = map_sequence[int(args.test_map)] input_tester['path_model'] = [args.model_path] input_tester['file_save'] = args.file_save # input_trainer['map_train'] = [map_sequence[idx] for idx in range(3) if idx != int(args.test_map)] print(input_tester) run_model.test_model(input_tester)
def main(): parser = argparse.ArgumentParser(description='Testing model ... ') id_process = os.getpid() time_current = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S') tag_model = '_PID=' + str(id_process) + '_TIME=' + time_current path_track = os.path.abspath('./testing/track' + tag_model + '/') if not os.path.exists(path_track): os.makedirs(path_track) file_log = os.path.join(path_track, 'log.txt') file_results = os.path.join(path_track, 'results.txt') # # The directory from which to find the data. default in data_processers is './data/' parser.add_argument('-fd', '--FileData', required=False, help='Path of the data-set') # parser.add_argument( '-fp', '--FilePretrain', required=True, # TODO change to True help='File of pretrained model') parser.add_argument('-mt', '--MapTest', required=True, choices=['map_1', 'map_2', 'map_3'], type=str, help='Test Map') parser.add_argument('-sr', '--SaveResults', required=False, action='store_true', help='Save Results (True/False, default is false)') parser.add_argument('-sb', '--SizeBeam', required=False, choices=range(1, 20), default=4, type=int, help='Size of Beam (Integer, default is 4)') parser.add_argument( '-lnf', '--LengthNormalizationFactor', required=False, default=0.5, help='Length Normalization Factor [0.5-0.7] (0.5 is the default)') args = parser.parse_args() args.id_process = id_process args.time_current = time_current assert (args.FilePretrain is not None) args.PathPretrain = os.path.abspath(args.FilePretrain) args.MapTest = str(args.MapTest) assert isinstance(args.LengthNormalizationFactor, float) assert 0.5 <= args.LengthNormalizationFactor <= 0.7 # according to 'Google's neural machine translation system: Bridging the gap between human and machine translation' assert isinstance(args.SizeBeam, int), "Size of Beam is not an int" args.SaveResultsPath = file_results create_logger(file_results, 'testing results log') logger = create_logger(file_log, 'testing log') logger.info(args) run_model.test_model(args)