def get_data(batch_size, train_augs, test_augs=None): cifar10_train = gluon.data.vision.CIFAR10( train=True, transform=get_transform(train_augs)) cifar10_test = gluon.data.vision.CIFAR10( train=False, transform=get_transform(test_augs)) train_data = utils.DataLoader(cifar10_train, batch_size, shuffle=True) test_data = utils.DataLoader(cifar10_test, batch_size, shuffle=True) return (train_data, test_data)
def load_dataset(data_dir, model_params, inference_mode=False): tf.logging.info('loaddataset-开始数据处理=================================================') source = model_params.source #source.npy target = model_params.target #target.npy source_data = np.load(os.path.join(data_dir, source)) target_data = np.load(os.path.join(data_dir, target)) tf.logging.info('打印原始输入长度 %i.',len(source_data)) tf.logging.info('打印目标输入长度 %i.',len(target_data)) num_points = 68 model_params.max_seq_len = num_points tf.logging.info('model_params.max_seq_len %i.', model_params.max_seq_len) #并打印出来 eval_model_params = sketch_rnn_model.copy_hparams(model_params) #讲model的参数复制给评价模型 eval_model_params.use_input_dropout = 0 eval_model_params.use_recurrent_dropout = 0 #并修改一些参数 eval_model_params.use_output_dropout = 0 eval_model_params.is_training = 1 if inference_mode: # = fales eval_model_params.batch_size = 1 eval_model_params.is_training = 0 sample_model_params = sketch_rnn_model.copy_hparams(eval_model_params) #将参数复制给sample模型 sample_model_params.batch_size = 1 # only sample one at a time sample_model_params.max_seq_len = 1 # sample one point at a time #随机打乱后依次取出一个batch的训练集、测试集、验证集数据 #数据的x,y做过normalize处理,且不足Nmax的补充为(0,0,0,0,1) tf.logging.info('正式处理数据,输入网络') indices = np.random.permutation(range(0, len(source_data)))[0:model_params.batch_size] #为保证source和target同顺序 source_set = utils.DataLoader( source_data, indices, model_params.batch_size, max_seq_length=model_params.max_seq_len, random_scale_factor=model_params.random_scale_factor) target_set = utils.DataLoader( target_data, indices, model_params.batch_size, max_seq_length=model_params.max_seq_len, random_scale_factor=model_params.random_scale_factor) factor_source = source_set.calculate_normalizing_scale_factor() source_set.normalize(factor_source)#再对数据做normalize factor_target = target_set.calculate_normalizing_scale_factor() target_set.normalize(factor_target)#再对数据做normalize tf.logging.info('source normalizing_scale_factor is %4.4f.',factor_source) tf.logging.info('target normalizing_scale_factor is %4.4f.',factor_target) result = [source_set, target_set, model_params, eval_model_params, sample_model_params] return result
def infer(self, src): self.build(mode="infer") data_loader = utils.DataLoader(src) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) saver = tf.train.Saver(tf.global_variables(), max_to_keep=1) train_writer = tf.summary.FileWriter('summary_ckpt/', sess.graph) ckpt = tf.train.latest_checkpoint(utils.checkpoint_dir) if ckpt: saver.restore(sess, ckpt) print('restore from checkpoint{0}'.format(ckpt)) print( '=============================begin infer=============================' ) train_cost = 0 last_step = 0 while data_loader.epoch < 1: batch_inputs = data_loader.get_imgs(self.batch_size) feed_dict = {self.inputs: batch_inputs} decoded = sess.run(self.dense_decoded, feed_dict) print(decoded) with open("result.txt", "a") as f: for s in decoded: for c in s: if c > 0: f.write(utils.decode_maps[c]) f.write("\n")
def __init__(self, system, output_dir): self.data = utils.DataLoader() self._system = system self._output_dir = output_dir self._plot_counter = 0 for f in os.listdir(self._output_dir): os.remove(os.path.join(self._output_dir, f))
def train(): device = 'cuda' if torch.cuda.is_available() and is_gpu else 'cpu' print("Using Device -", device) print("Start Training Process...\n") train_dataset = utils.tranform_train_dataset(train_dir) trainloader = utils.DataLoader(train_dataset, 64) valid_dataset = utils.transform_valid_test_dataset(valid_dir) validloader = utils.DataLoader(valid_dataset, 64) test_dataset = utils.transform_valid_test_dataset(test_dir) testloader = utils.DataLoader(test_dataset, 64) fc_model = OrderedDict([ ('fc1', nn.Linear(25088, hidden_layer_1)), ('relu', nn.ReLU()), ('dropout1', nn.Dropout(dropout)), ('fc2', nn.Linear(hidden_layer_1, hidden_layer_1)), ('relu', nn.ReLU()), #('fc3',nn.Linear(256,256)), #('relu',nn.ReLU()), ('fc4', nn.Linear(hidden_layer_1, 102)), ('output', nn.LogSoftmax(dim=1)) ]) model = utils.build_network(fc_model, network, dropout, device) criterion = nn.NLLLoss() optimizer = optim.Adam(model.classifier.parameters(), lr=learning_rate) model = utils.train(model, trainloader, epochs, learning_rate, criterion, optimizer, device, validloader) model.class_to_idx = train_dataset.class_to_idx utils.save_checkpoint(model, optimizer, epochs, save_dir, network, hidden_layer_1, dropout, learning_rate) print("End Training Process...\n") print("Start Test Process...\n") utils.test(model, testloader, criterion, device) print("End Test Process...\n")
def transform_draws(draws): max_seq_len = 137 test_set = utils.DataLoader( draws, 100, max_seq_length=max_seq_len, random_scale_factor=0.15, augment_stroke_prob=0.1) normalizing_scale_factor = test_set.calculate_normalizing_scale_factor() test_set.normalize(normalizing_scale_factor) test_set.strokes = [utils.to_big_strokes( stroke, max_seq_len) for stroke in test_set.strokes] test_set.strokes = [ np.insert(stroke, 0, [0, 0, 1, 0, 0], axis=0) for stroke in test_set.strokes] return test_set.strokes
def _make_inputs_plot(self): GRID_X, GRID_Y = 1, 3 time = self._system.timeline generator = self._system.get_block(self._system.GENERATOR) modulator = self._system.get_block(self._system.MODULATOR) plt.figure(figsize=(8, 12)) plt.subplot(GRID_Y, GRID_X, 1) plt.plot(time, generator.output) plt.title(self._build_title("Sygnał modulujący")) self._add_std_figure_formatting('s', 'V') plt.subplot(GRID_Y, GRID_X, 2) self._plot_single_spectrum(generator.output) plt.title(self._build_title("Widmo sygnału modulującego")) plt.xlim(0, utils.DataLoader().modulating_freq * 3) plt.subplot(GRID_Y, GRID_X, 3) plt.title(self._build_title("Widmo amplitudowe sygnału zmodulowanego")) self._plot_single_spectrum(modulator.output) plt.tight_layout() self._save_plt("input")
def load_dataset(data_dir, model_params, inference_mode=False, contain_labels=False): """Loads the .npz file, and splits the set into train/valid/test.""" # normalizes the x and y columns usint the training set. # applies same scaling factor to valid and test set. # contain_labels: set to True to return labels for classification tasks, default as False datasets = [] if isinstance(model_params.data_set, list): datasets = model_params.data_set else: datasets = [model_params.data_set] train_strokes = None valid_strokes = None test_strokes = None label_index = 0 class_num = len(datasets) for dataset in datasets: # Get input data data_filepath = os.path.join(data_dir, "sketch", dataset) onv_left_filepath = os.path.join(data_dir, "onv_9936_thick", dataset) onv_right_filepath = os.path.join(data_dir, "onv_9936_thick_right", dataset) if data_dir.startswith('http://') or data_dir.startswith('https://'): tf.logging.info('Downloading %s', data_filepath) response = requests.get(data_filepath) data = np.load(StringIO(response.content)) else: tf.logging.info('Getting data from %s', data_filepath) data = np.load(data_filepath) # load this into dictionary tf.logging.info('Getting left onv from %s', onv_left_filepath) onv_left = np.load(onv_left_filepath) tf.logging.info('Getting right onv from %s', onv_right_filepath) onv_right = np.load(onv_right_filepath) train_size = len(onv_left['train']) valid_size = len(onv_left['valid']) test_size = len(onv_left['test']) tf.logging.info('Loaded {}/{}/{} from {}'.format( train_size, valid_size, test_size, dataset)) # set labels for classification task cur_train_labels = np.zeros((train_size, class_num)) cur_valid_labels = np.zeros((valid_size, class_num)) cur_test_labels = np.zeros((test_size, class_num)) cur_train_labels[:, label_index] = 1 cur_valid_labels[:, label_index] = 1 cur_test_labels[:, label_index] = 1 #print ("label_index", label_index, cur_train_labels[0]) if train_strokes is None: train_strokes = data['train'][0:train_size] valid_strokes = data['valid'][0:valid_size] test_strokes = data['test'][0:test_size] train_onvs_left = onv_left['train'][0:train_size] valid_onvs_left = onv_left['valid'][0:valid_size] test_onvs_left = onv_left['test'][0:test_size] train_onvs_right = onv_right['train'][0:train_size] valid_onvs_right = onv_right['valid'][0:valid_size] test_onvs_right = onv_right['test'][0:test_size] train_labels = cur_train_labels[0:train_size] valid_labels = cur_valid_labels[0:valid_size] test_labels = cur_valid_labels[0:test_size] else: train_strokes = np.concatenate((train_strokes, data['train'][0:train_size])) valid_strokes = np.concatenate((valid_strokes, data['valid'][0:valid_size])) test_strokes = np.concatenate((test_strokes, data['test'][0:test_size])) train_onvs_left = np.concatenate((train_onvs_left, onv_left['train'][0:train_size])) valid_onvs_left = np.concatenate((valid_onvs_left, onv_left['valid'][0:valid_size])) test_onvs_left = np.concatenate((test_onvs_left, onv_left['test'][0:test_size])) train_onvs_right = np.concatenate((train_onvs_right, onv_right['train'][0:train_size])) valid_onvs_right = np.concatenate((valid_onvs_right, onv_right['valid'][0:valid_size])) test_onvs_right = np.concatenate((test_onvs_right, onv_right['test'][0:test_size])) train_labels = np.concatenate((train_labels, cur_train_labels[0:train_size])) valid_labels = np.concatenate((valid_labels, cur_valid_labels[0:valid_size])) test_labels = np.concatenate((test_labels, cur_test_labels[0:test_size])) label_index+=1 all_strokes = np.concatenate((train_strokes, valid_strokes, test_strokes)) num_points = 0 for stroke in all_strokes: num_points += len(stroke) avg_len = num_points / len(all_strokes) tf.logging.info('Dataset combined: {} ({}/{}/{}), avg len {}'.format( len(all_strokes), len(train_strokes), len(valid_strokes), len(test_strokes), int(avg_len))) # calculate the max strokes we need. max_seq_len = utils.get_max_len(all_strokes) # overwrite the hps with this calculation. model_params.max_seq_len = max_seq_len tf.logging.info('model_params.max_seq_len %i.', model_params.max_seq_len) eval_model_params = sketch_rnn_model.copy_hparams(model_params) eval_model_params.use_input_dropout = 0 eval_model_params.use_recurrent_dropout = 0 eval_model_params.use_output_dropout = 0 eval_model_params.is_training = 1 if inference_mode: eval_model_params.batch_size = 1 eval_model_params.is_training = 0 sample_model_params = sketch_rnn_model.copy_hparams(eval_model_params) sample_model_params.batch_size = 1 # only sample one at a time sample_model_params.max_seq_len = 1 # sample one point at a time train_set = utils.DataLoader( train_strokes, model_params.batch_size, max_seq_length=model_params.max_seq_len, random_scale_factor=model_params.random_scale_factor, augment_stroke_prob=model_params.augment_stroke_prob) normalizing_scale_factor = train_set.calculate_normalizing_scale_factor() train_set.normalize(normalizing_scale_factor) valid_set = utils.DataLoader( valid_strokes, eval_model_params.batch_size, max_seq_length=eval_model_params.max_seq_len, random_scale_factor=0.0, augment_stroke_prob=0.0) valid_set.normalize(normalizing_scale_factor) test_set = utils.DataLoader( test_strokes, eval_model_params.batch_size, max_seq_length=eval_model_params.max_seq_len, random_scale_factor=0.0, augment_stroke_prob=0.0) test_set.normalize(normalizing_scale_factor) tf.logging.info('normalizing_scale_factor %4.4f.', normalizing_scale_factor) # onv preprocess print ("unique", np.unique(train_onvs_left)) train_onvs_left = train_onvs_left / 255.0 valid_onvs_left = valid_onvs_left / 255.0 test_onvs_left = test_onvs_left / 255.0 train_onvs_right = train_onvs_right / 255.0 valid_onvs_right = valid_onvs_right / 255.0 test_onvs_right = test_onvs_right / 255.0 if not contain_labels: result = [ train_set, valid_set, test_set, model_params, eval_model_params, sample_model_params, train_onvs_left, valid_onvs_left, test_onvs_left, train_onvs_right, valid_onvs_right, test_onvs_right ] else: #return labels for classification tasks result = [ train_set, valid_set, test_set, model_params, eval_model_params, sample_model_params, train_onvs_left, valid_onvs_left, test_onvs_left, train_onvs_right, valid_onvs_right, test_onvs_right, train_labels, valid_labels, test_labels ] return result
def _format_spectrum(self): yLimit = -70 plt.xlim(0, utils.DataLoader().carrier_freq * 2) plt.ylim((yLimit, 0)) self._add_std_figure_formatting('Hz', 'dB')
def main(args): CDF, AHs = utils.DataLoader(args.infile, Single=args.Single) print("main function")
def main(): # Parse the arguments received from command line parser = argparse.ArgumentParser(description="Train a social LSTM") parser.add_argument( "modelParams", type=str, help= "Path to the file or folder with the parameters of the experiments", ) parser.add_argument( "-l", "--logLevel", help="logging level of the logger. Default is INFO", metavar="level", type=str, ) parser.add_argument( "-f", "--logFolder", help= "path to the folder where to save the logs. If None, logs are only printed in stderr", type=str, metavar="path", ) args = parser.parse_args() if os.path.isdir(args.modelParams): names_experiments = os.listdir(args.modelParams) experiments = [ os.path.join(args.modelParams, experiment) for experiment in names_experiments ] else: experiments = [args.modelParams] for experiment in experiments: # Load the parameters hparams = utils.YParams(experiment) # Define the logger setLogger(hparams, args, PHASE) remainSpaces = 29 - len(hparams.name) logging.info( "\n" + "--------------------------------------------------------------------------------\n" + "| Training experiment: " + hparams.name + " " * remainSpaces + "|\n" + "--------------------------------------------------------------------------------\n" ) trajectory_size = hparams.obsLen + hparams.predLen logging.info("Loading the training datasets...") train_loader = utils.DataLoader( hparams.dataPath, hparams.trainDatasets, hparams.trainMaps, hparams.semanticMaps, hparams.trainMapping, hparams.homography, num_labels=hparams.numLabels, delimiter=hparams.delimiter, skip=hparams.skip, max_num_ped=hparams.maxNumPed, trajectory_size=trajectory_size, neighborood_size=hparams.neighborhoodSize, ) logging.info("Loading the validation datasets...") val_loader = utils.DataLoader( hparams.dataPath, hparams.validationDatasets, hparams.validationMaps, hparams.semanticMaps, hparams.validationMapping, hparams.homography, num_labels=hparams.numLabels, delimiter=hparams.delimiter, skip=hparams.skip, max_num_ped=hparams.maxNumPed, trajectory_size=trajectory_size, neighborood_size=hparams.neighborhoodSize, ) logging.info( "Creating the training and validation dataset pipeline...") dataset = utils.TrajectoriesDataset( train_loader, val_loader=val_loader, batch=False, shuffle=hparams.shuffle, prefetch_size=hparams.prefetchSize, ) hparams.add_hparam("learningRateSteps", train_loader.num_sequences) logging.info("Creating the model...") start = time.time() model = SocialModel(dataset, hparams, phase=PHASE) end = time.time() - start logging.debug("Model created in {:.2f}s".format(end)) # Define the path to where save the model and the checkpoints if hparams.modelFolder: save_model = True model_folder = os.path.join(hparams.modelFolder, hparams.name) if not os.path.exists(model_folder): os.makedirs(model_folder) os.makedirs(os.path.join(model_folder, "checkpoints")) model_path = os.path.join(model_folder, hparams.name) checkpoints_path = os.path.join(model_folder, "checkpoints", hparams.name) # Create the saver saver = tf.train.Saver() # Zero padding padding = len(str(train_loader.num_sequences)) # ============================ START TRAINING ============================ with tf.Session() as sess: logging.info( "\n" + "--------------------------------------------------------------------------------\n" + "| Start training |\n" + "--------------------------------------------------------------------------------\n" ) # Initialize all the variables in the graph sess.run(tf.global_variables_initializer()) for epoch in range(hparams.epochs): logging.info("Starting epoch {}".format(epoch + 1)) # ==================== TRAINING PHASE ==================== # Initialize the iterator of the training dataset sess.run(dataset.init_train) for sequence in range(train_loader.num_sequences): start = time.time() loss, _ = sess.run([model.loss, model.train_optimizer]) end = time.time() - start logging.info( "{:{width}d}/{} epoch: {} time/Batch = {:.2f}s. Loss = {:.4f}" .format( sequence + 1, train_loader.num_sequences, epoch + 1, end, loss, width=padding, )) # ==================== VALIDATION PHASE ==================== logging.info(" ========== Validation ==========") # Initialize the iterator of the validation dataset sess.run(dataset.init_val) loss_val = 0 for _ in range(val_loader.num_sequences): loss = sess.run(model.loss) loss_val += loss mean_val = loss_val / val_loader.num_sequences logging.info("Epoch: {}. Validation loss = {:.4f}".format( epoch + 1, mean_val)) # Save the model if save_model: logging.info("Saving model...") saver.save( sess, checkpoints_path, global_step=epoch + 1, write_meta_graph=False, ) logging.info("Model saved...") # Save the final model if save_model: saver.save(sess, model_path) tf.reset_default_graph()
def load_dataset(data_dir, model_params, inference_mode=False): """Loads the .npz file, and splits the set into train/valid/test.""" # normalizes the x and y columns using the training set. # applies same scaling factor to valid and test set. if isinstance(model_params.data_set, list): datasets = model_params.data_set else: datasets = [model_params.data_set] train_strokes = None valid_strokes = None test_strokes = None png_paths_map = {'train': [], 'valid': [], 'test': []} for dataset in datasets: if data_dir.startswith('http://') or data_dir.startswith('https://'): data_filepath = '/'.join([data_dir, dataset]) print('Downloading %s' % data_filepath) response = requests.get(data_filepath) data = np.load(six.BytesIO(response.content), encoding='latin') else: data_filepath = os.path.join(data_dir, 'npz', dataset) if six.PY3: data = np.load(data_filepath, encoding='latin1') else: data = np.load(data_filepath) print('Loaded {}/{}/{} from {}'.format(len(data['train']), len(data['valid']), len(data['test']), dataset)) if train_strokes is None: train_strokes = data[ 'train'] # [N (#sketches),], each with [S (#points), 3] valid_strokes = data['valid'] test_strokes = data['test'] else: train_strokes = np.concatenate((train_strokes, data['train'])) valid_strokes = np.concatenate((valid_strokes, data['valid'])) test_strokes = np.concatenate((test_strokes, data['test'])) splits = ['train', 'valid', 'test'] for split in splits: for im_idx in range(len(data[split])): png_path = os.path.join( data_dir, 'png', dataset[:-4], split, str(model_params.img_H) + 'x' + str(model_params.img_W), str(im_idx) + '.png') png_paths_map[split].append(png_path) all_strokes = np.concatenate((train_strokes, valid_strokes, test_strokes)) num_points = 0 for stroke in all_strokes: num_points += len(stroke) avg_len = num_points / len(all_strokes) print('Dataset combined: {} ({}/{}/{}), avg len {}'.format( len(all_strokes), len(train_strokes), len(valid_strokes), len(test_strokes), int(avg_len))) assert len(train_strokes) == len(png_paths_map['train']) assert len(valid_strokes) == len(png_paths_map['valid']) assert len(test_strokes) == len(png_paths_map['test']) # calculate the max strokes we need. max_seq_len = utils.get_max_len(all_strokes) # overwrite the hps with this calculation. model_params.max_seq_len = max_seq_len print('model_params.max_seq_len %i.' % model_params.max_seq_len) eval_model_params = sketch_rnn_model.copy_hparams(model_params) eval_model_params.use_input_dropout = 0 eval_model_params.use_recurrent_dropout = 0 eval_model_params.use_output_dropout = 0 eval_model_params.is_training = 1 if inference_mode: eval_model_params.batch_size = 1 eval_model_params.is_training = 0 sample_model_params = sketch_rnn_model.copy_hparams(eval_model_params) sample_model_params.batch_size = 1 # only sample one at a time sample_model_params.max_seq_len = 1 # sample one point at a time train_set = utils.DataLoader( train_strokes, png_paths_map['train'], model_params.img_H, model_params.img_W, model_params.batch_size, max_seq_length=model_params.max_seq_len, random_scale_factor=model_params.random_scale_factor, augment_stroke_prob=model_params.augment_stroke_prob) normalizing_scale_factor = train_set.calculate_normalizing_scale_factor() train_set.normalize(normalizing_scale_factor) valid_set = utils.DataLoader(valid_strokes, png_paths_map['valid'], eval_model_params.img_H, eval_model_params.img_W, eval_model_params.batch_size, max_seq_length=eval_model_params.max_seq_len, random_scale_factor=0.0, augment_stroke_prob=0.0) valid_set.normalize(normalizing_scale_factor) test_set = utils.DataLoader(test_strokes, png_paths_map['test'], eval_model_params.img_H, eval_model_params.img_W, eval_model_params.batch_size, max_seq_length=eval_model_params.max_seq_len, random_scale_factor=0.0, augment_stroke_prob=0.0) test_set.normalize(normalizing_scale_factor) print('normalizing_scale_factor %4.4f.' % normalizing_scale_factor) result = [ train_set, valid_set, test_set, model_params, eval_model_params, sample_model_params ] return result
tag_scores = model(words_idx_tensor, pos_idx_tensor) predicted_mst, _ = decode_mst(energy=tag_scores.detach().cpu(), length=tag_scores.shape[0], has_labels=False) tags.append(predicted_mst[1:]) return tags # create data sets data_dir = "HW2-files/" path_train = data_dir + "train.labeled" path_test = data_dir + "test.labeled" paths_list = [path_train, path_test] word_cnt, word_dict, pos_dict = utils.get_vocabs(paths_list) train = utils.PosDataset(word_cnt, word_dict, pos_dict, data_dir, 'train') train_dataloader = utils.DataLoader(train, shuffle=True) test = utils.PosDataset(word_cnt, word_dict, pos_dict, data_dir, 'test') test_dataloader = utils.DataLoader(test, shuffle=False) word_vocab_size = len(train.word2idx) tag_vocab_size = len(train.pos_idx_mappings) use_cuda = torch.cuda.is_available() device = torch.device("cuda:0" if use_cuda else "cpu") # create and load trained model base_model = basic_model.DnnDependencyParser(basic_model.WORD_EMBEDDING_DIM, basic_model.POS_EMBEDDING_DIM, basic_model.HIDDEN_DIM, word_vocab_size, tag_vocab_size).to(device) if path.exists("./basic_model_best_params"):
def load_dataset(data_dir, datasets, inference_mode=False): """Loads the .npz file, and splits the set into train/valid/test.""" # normalizes the x and y columns usint the training set. # applies same scaling factor to valid and test set. train_strokes = None valid_strokes = None test_strokes = None for dataset in datasets: data_filepath = os.path.join(data_dir, dataset) if data_dir.startswith('http://') or data_dir.startswith('https://'): tf.logging.info('Downloading %s', data_filepath) response = requests.get(data_filepath) data = np.load(StringIO(response.content)) else: if six.PY3: data = np.load(data_filepath, encoding='latin1') else: data = np.load(data_filepath) tf.logging.info('Loaded {}/{}/{} from {}'.format( len(data['train']), len(data['valid']), len(data['test']), dataset)) if train_strokes is None: train_strokes = data['train'] valid_strokes = data['valid'] test_strokes = data['test'] else: train_strokes = np.concatenate((train_strokes, data['train'])) valid_strokes = np.concatenate((valid_strokes, data['valid'])) test_strokes = np.concatenate((test_strokes, data['test'])) all_strokes = np.concatenate((train_strokes, valid_strokes, test_strokes)) num_points = 0 for stroke in all_strokes: num_points += len(stroke) avg_len = num_points / len(all_strokes) tf.logging.info('Dataset combined: {} ({}/{}/{}), avg len {}'.format( len(all_strokes), len(train_strokes), len(valid_strokes), len(test_strokes), int(avg_len))) # calculate the max strokes we need. max_seq_len = utils.get_max_len(all_strokes) tf.logging.info('model_params.max_seq_len %i.', max_seq_len) train_set = utils.DataLoader(train_strokes, random_scale_factor=0.1, augment_stroke_prob=0.1) normalizing_scale_factor = train_set.calculate_normalizing_scale_factor() train_set.normalize(normalizing_scale_factor) valid_set = utils.DataLoader(valid_strokes, random_scale_factor=0.0, augment_stroke_prob=0.0) valid_set.normalize(normalizing_scale_factor) test_set = utils.DataLoader(test_strokes, random_scale_factor=0.0, augment_stroke_prob=0.0) test_set.normalize(normalizing_scale_factor) tf.logging.info('normalizing_scale_factor %4.4f.', normalizing_scale_factor) result = [train_set, valid_set, test_set] return result
def load_dataset(sketch_data_dir, photo_data_dir, model_params, inference_mode=False): """Loads the .npz file, and splits the set into train/test.""" # normalizes the x and y columns using the training set. # applies same scaling factor to test set. if isinstance(model_params.data_set, list): datasets = model_params.data_set else: datasets = [model_params.data_set] train_strokes = None test_strokes = None train_image_paths = [] test_image_paths = [] for dataset in datasets: if model_params.data_type == 'QMUL': train_data_filepath = os.path.join(sketch_data_dir, dataset, 'train_svg_sim_spa_png.h5') test_data_filepath = os.path.join(sketch_data_dir, dataset, 'test_svg_sim_spa_png.h5') train_data_dict = utils.load_hdf5(train_data_filepath) test_data_dict = utils.load_hdf5(test_data_filepath) train_sketch_data = utils.reassemble_data( train_data_dict['image_data'], train_data_dict['data_offset'] ) # list of [N_sketches], each [N_points, 4] train_photo_names = train_data_dict[ 'image_base_name'] # [N_sketches, 1], byte train_photo_paths = [ os.path.join(photo_data_dir, train_photo_names[i, 0].decode() + '.png') for i in range(train_photo_names.shape[0]) ] # [N_sketches], str test_sketch_data = utils.reassemble_data( test_data_dict['image_data'], test_data_dict['data_offset'] ) # list of [N_sketches], each [N_points, 4] test_photo_names = test_data_dict[ 'image_base_name'] # [N_sketches, 1], byte test_photo_paths = [ os.path.join(photo_data_dir, test_photo_names[i, 0].decode() + '.png') for i in range(test_photo_names.shape[0]) ] # [N_sketches], str # transfer stroke-4 to stroke-3 train_sketch_data = utils.to_normal_strokes_4to3(train_sketch_data) test_sketch_data = utils.to_normal_strokes_4to3( test_sketch_data) # [N_sketches,], each with [N_points, 3] if train_strokes is None: train_strokes = train_sketch_data test_strokes = test_sketch_data else: train_strokes = np.concatenate( (train_strokes, train_sketch_data)) test_strokes = np.concatenate((test_strokes, test_sketch_data)) elif model_params.data_type == 'QuickDraw': data_filepath = os.path.join(sketch_data_dir, dataset, 'npz', 'sketchrnn_' + dataset + '.npz') if six.PY3: data = np.load(data_filepath, encoding='latin1') else: data = np.load(data_filepath) if train_strokes is None: train_strokes = data[ 'train'] # [N_sketches,], each with [N_points, 3] test_strokes = data['test'] else: train_strokes = np.concatenate((train_strokes, data['train'])) test_strokes = np.concatenate((test_strokes, data['test'])) train_photo_paths = [ os.path.join( sketch_data_dir, dataset, 'png', 'train', str(model_params.image_size) + 'x' + str(model_params.image_size), str(im_idx) + '.png') for im_idx in range(len(data['train'])) ] test_photo_paths = [ os.path.join( sketch_data_dir, dataset, 'png', 'test', str(model_params.image_size) + 'x' + str(model_params.image_size), str(im_idx) + '.png') for im_idx in range(len(data['test'])) ] else: raise Exception('Unknown data type:', model_params.data_type) print('Loaded {}/{} from {} {}'.format(len(train_photo_paths), len(test_photo_paths), model_params.data_type, dataset)) train_image_paths += train_photo_paths test_image_paths += test_photo_paths all_strokes = np.concatenate((train_strokes, test_strokes)) num_points = 0 for stroke in all_strokes: num_points += len(stroke) avg_len = num_points / len(all_strokes) print('Dataset combined: {} ({}/{}), avg len {}'.format( len(all_strokes), len(train_strokes), len(test_strokes), int(avg_len))) assert len(train_image_paths) == len(train_strokes) assert len(test_image_paths) == len(test_strokes) # calculate the max strokes we need. max_seq_len = utils.get_max_len(all_strokes) # overwrite the hps with this calculation. model_params.max_seq_len = max_seq_len print('model_params.max_seq_len %i.' % model_params.max_seq_len) eval_model_params = sketch_p2s_model.copy_hparams(model_params) eval_model_params.use_input_dropout = 0 eval_model_params.use_recurrent_dropout = 0 eval_model_params.use_output_dropout = 0 eval_model_params.is_training = 1 if inference_mode: eval_model_params.batch_size = 1 eval_model_params.is_training = 0 sample_model_params = sketch_p2s_model.copy_hparams(eval_model_params) sample_model_params.batch_size = 1 # only sample one at a time sample_model_params.max_seq_len = 1 # sample one point at a time train_set = utils.DataLoader( train_strokes, train_image_paths, model_params.image_size, model_params.image_size, model_params.batch_size, max_seq_length=model_params.max_seq_len, random_scale_factor=model_params.random_scale_factor, augment_stroke_prob=model_params.augment_stroke_prob) normalizing_scale_factor = train_set.calculate_normalizing_scale_factor() train_set.normalize(normalizing_scale_factor) # valid_set = utils.DataLoader( # valid_strokes, # eval_model_params.batch_size, # max_seq_length=eval_model_params.max_seq_len, # random_scale_factor=0.0, # augment_stroke_prob=0.0) # valid_set.normalize(normalizing_scale_factor) test_set = utils.DataLoader(test_strokes, test_image_paths, model_params.image_size, model_params.image_size, eval_model_params.batch_size, max_seq_length=eval_model_params.max_seq_len, random_scale_factor=0.0, augment_stroke_prob=0.0) test_set.normalize(normalizing_scale_factor) print('normalizing_scale_factor %4.4f.' % normalizing_scale_factor) result = [ train_set, None, test_set, model_params, eval_model_params, sample_model_params ] return result
test[k].shape[2])) test_norm.append(img_norm(imag)) test_ds = (nd.concat(*test_norm, dim=0).astype('float32'), ids) print("max/min train: %f\t%f" % (nd.max(train_ds[0]).asscalar(), nd.min(train_ds[0]).asscalar())) print("max/min valid: %f\t%f" % (nd.max(valid_ds[0]).asscalar(), nd.min(valid_ds[0]).asscalar())) print("max/min test: %f\t%f" % (nd.max(test_ds[0]).asscalar(), nd.min(test_ds[0]).asscalar())) print("finish gen train/valid dataset") batch_size = 128 train_data = utils.DataLoader(train_ds, batch_size, shuffle=True) valid_data = utils.DataLoader(valid_ds, batch_size, shuffle=False) test_data = utils.TestDataLoader(test_ds, batch_size) ctx = utils.try_gpu() num_epochs = 100 learning_rate = .001 net = Net_vgg10() net.initialize(init=init.Xavier(), ctx=ctx) # net.hybridize() print("Start training on ", ctx) sys.stdout.flush() train(net, train_data, valid_data, test_data, batch_size, num_epochs, learning_rate, ctx)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--data_path', type=str) parser.add_argument('--src_column', type=str) parser.add_argument('--tgt_column', type=str) parser.add_argument('--batch_size', type=int) parser.add_argument('--src_maxlen', type=int) parser.add_argument('--tgt_maxlen', type=int) parser.add_argument('--model_file', type=str) parser.add_argument('--loss_file', type=str) parser.add_argument('--word2vec_path', type=str) parser.add_argument('--embedding_dim', type=int) parser.add_argument('--attention', action='store_true') args = parser.parse_args() batch_size = args.batch_size if args.batch_size is not None else 32 src_maxlen = args.src_maxlen if args.src_maxlen is not None else 100 tgt_maxlen = args.tgt_maxlen if args.tgt_maxlen is not None else 100 model_file = args.model_file if args.model_file is not None else 'tmp.model' loss_file = args.loss_file if args.loss_file is not None else 'tmp.png' data_path = args.data_path if args.data_path is not None else './data/train.csv' embedding_dim = args.embedding_dim if args.embedding_dim is not None else 200 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print('Loading data...') train = utils.DataBuilder() train.add_data_from_csv(data_path, 'src', 'tgt', preprocess=False) train.drop_long_seq(src_maxlen, tgt_maxlen) # make dictionary src_vocab, tgt_vocab = train.build_vocab() with open('./cache/src.vocab', 'wb') as f: pickle.dump(src_vocab, f) with open('./cache/tgt.vocab', 'wb') as f: pickle.dump(tgt_vocab, f) print('vocabulary size in source is', src_vocab.size) print('vocabulary size in target is', tgt_vocab.size) src_embedding_matrix = None tgt_embedding_matrix = None unknown_set = set() # use pre-trained word2vec embedding as Embedding Layer # if a word is not in word2vec model, its embedding initializes from uniform random number. if args.word2vec_path is not None: print('Loading word2vec model...') word2vec = Word2Vec.load(args.word2vec_path) assert embedding_dim == word2vec.size, 'embedding dim unmatched. args:{}, word2vec:{}'.format( embedding_dim, word2vec.size) src_embedding_matrix, src_unknown_set = utils.get_embedding_matrix( src_vocab, word2vec) tgt_embedding_matrix, tgt_unknown_set = utils.get_embedding_matrix( tgt_vocab, word2vec) unknown_set = src_unknown_set | tgt_unknown_set def replace_unknown(text): text = utils.replace_unknown(text, unknown_set) return text train.data = train.data.applymap(replace_unknown) src, tgt = train.make_id_array(src_maxlen, tgt_maxlen) src_lengths = train.data['src'].str.split().apply(len) src_lengths = np.array(src_lengths).astype( 'int32') - 1 #'cause delete <EOS> later. src = src[:, :-1] # <EOS> delete tgt = tgt[:, 1:] # <BOS> delete #dump unknown words as json file with codecs.open('./cache/unknown.json', 'w', 'utf-8') as f: unknown_list = list(unknown_set) dump = json.dumps(unknown_list, ensure_ascii=False) f.write(dump) # not to include <PAD> in loss calculation criterion = nn.CrossEntropyLoss(ignore_index=utils.Vocab.pad_id) params = { 'src_num_vocab': src_vocab.size, 'tgt_num_vocab': tgt_vocab.size, 'embedding_dim': embedding_dim, 'hidden_size': 512, 'src_embedding_matrix': src_embedding_matrix, 'tgt_embedding_matrix': tgt_embedding_matrix } with open('./cache/params.json', 'w') as f: json.dump(params, f) if args.attention: model = seq2seq.GlobalAttentionEncoderDecoder(**params).to(device) else: model = seq2seq.EncoderDecoder(**params).to(device) train_src, valid_src, train_tgt, valid_tgt, train_src_lengths, valid_src_lengths = train_test_split( src, tgt, src_lengths, test_size=0.1) train_dataloader = utils.DataLoader(train_src, train_tgt, train_src_lengths, batch_size=batch_size) valid_dataloader = utils.DataLoader(valid_src, valid_tgt, valid_src_lengths, batch_size=batch_size) print('Start Training') losses = train_iters(model, criterion, train_dataloader, valid_dataloader, epochs=30, model_file=model_file) plt.figure(figsize=(16, 6)) plt.ylim(0, max(losses) + 1) plt.plot(losses) plt.savefig(FIGURE_PATH + loss_file) print('Finished!')
def load_dataset(data_dir, model_params, testing_mode=False): """Loads the .npz file, and splits the set into train/valid/test.""" # normalizes the x and y columns using scale_factor. dataset = model_params.data_set data_filepath = os.path.join(data_dir, dataset) data = np.load(data_filepath, allow_pickle=True, encoding='latin1') # target data train_strokes = data['train'] valid_strokes = data['valid'] test_strokes = data['test'] all_strokes = np.concatenate((train_strokes, valid_strokes, test_strokes)) # standard data (reference data in paper) std_train_strokes = data['std_train'] std_valid_strokes = data['std_valid'] std_test_strokes = data['std_test'] all_std_trokes = np.concatenate( (std_train_strokes, std_valid_strokes, std_test_strokes)) print('Dataset combined: %d (train=%d/validate=%d/test=%d)' % (len(all_strokes), len(train_strokes), len(valid_strokes), len(test_strokes))) # calculate the max strokes we need. max_seq_len = utils.get_max_len(all_strokes) max_std_seq_len = utils.get_max_len(all_std_trokes) # overwrite the hps with this calculation. model_params.max_seq_len = max(max_seq_len, max_std_seq_len) print('model_params.max_seq_len set to %d.' % model_params.max_seq_len) eval_model_params = copy_hparams(model_params) eval_model_params.rnn_dropout_keep_prob = 1.0 eval_model_params.is_training = True if testing_mode: # for testing eval_model_params.batch_size = 1 eval_model_params.is_training = False # sample mode train_set = utils.DataLoader( train_strokes, model_params.batch_size, max_seq_length=model_params.max_seq_len, random_scale_factor=model_params.random_scale_factor, augment_stroke_prob=model_params.augment_stroke_prob) normalizing_scale_factor = model_params.scale_factor train_set.normalize(normalizing_scale_factor) valid_set = utils.DataLoader(valid_strokes, eval_model_params.batch_size, max_seq_length=eval_model_params.max_seq_len, random_scale_factor=0.0, augment_stroke_prob=0.0) valid_set.normalize(normalizing_scale_factor) test_set = utils.DataLoader(test_strokes, eval_model_params.batch_size, max_seq_length=eval_model_params.max_seq_len, random_scale_factor=0.0, augment_stroke_prob=0.0) test_set.normalize(normalizing_scale_factor) # process the reference dataset std_train_set = utils.DataLoader( std_train_strokes, model_params.batch_size, max_seq_length=model_params.max_seq_len, random_scale_factor=model_params.random_scale_factor, augment_stroke_prob=model_params.augment_stroke_prob) std_train_set.normalize(normalizing_scale_factor) std_valid_set = utils.DataLoader( std_valid_strokes, eval_model_params.batch_size, max_seq_length=eval_model_params.max_seq_len, random_scale_factor=0.0, augment_stroke_prob=0.0) std_valid_set.normalize(normalizing_scale_factor) std_test_set = utils.DataLoader( std_test_strokes, eval_model_params.batch_size, max_seq_length=eval_model_params.max_seq_len, random_scale_factor=0.0, augment_stroke_prob=0.0) std_test_set.normalize(normalizing_scale_factor) result = [ train_set, valid_set, test_set, std_train_set, std_valid_set, std_test_set, model_params, eval_model_params ] return result
def main(): parser = argparse.ArgumentParser() parser.add_argument('--batch_size', type=int) parser.add_argument( '--src_maxlen', type=int, help='this should be equal to its counterpart in training') parser.add_argument( '--tgt_maxlen', type=int, help='this should be equal to its counterpart in training') parser.add_argument( '--model_file', type=str, help='this should be equal to its counterpart in training') parser.add_argument('--data_path', type=str, help='test data') parser.add_argument( '--sample_file', type=str, help='csv file to write inference results in') parser.add_argument('--attention', action='store_true') args = parser.parse_args() batch_size = args.batch_size if args.batch_size is not None else 20 src_maxlen = args.src_maxlen if args.src_maxlen is not None else 100 tgt_maxlen = args.tgt_maxlen if args.tgt_maxlen is not None else 100 model_file = args.model_file if args.model_file is not None else 'tmp.model' data_path = args.data_path if args.data_path is not None else './data/test.csv' sample_file = args.sample_file if args.sample_file is not None else 'tmp.csv' device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') test = utils.DataBuilder() test.add_data_from_csv(data_path, 'src', 'tgt', preprocess=False) test.drop_long_seq(src_maxlen, tgt_maxlen) with open('./cache/src.vocab', 'rb') as f: test.src_vocab = pickle.load(f) with open('./cache/tgt.vocab', 'rb') as f: test.tgt_vocab = pickle.load(f) with open('./cache/unknown.json', 'r') as f: unknown_list = json.loads(f.read(), encoding='utf-8') unknown_set = set(unknown_list) def replace_unknown(text): text = utils.replace_unknown(text, unknown_set) return text test.data = test.data.applymap(replace_unknown) src, tgt = test.make_id_array(src_maxlen, tgt_maxlen) src_lengths = test.data['src'].str.split().apply(len) src_lengths = np.array(src_lengths).astype('int32') - 1 # 'cause delete <EOS> token later. src = src[:, :-1] # delete <EOS> tgt = tgt[:, 1:] # delete <BOS> test_dataloader = utils.DataLoader( src, tgt, src_lengths, batch_size=batch_size, shuffle=False) with open('./cache/params.json', 'r') as f: params = json.load(f) assert isinstance(params, dict) if args.attention: model = seq2seq.GlobalAttentionBeamEncoderDecoder(**params).to(device) else: model = seq2seq.EncoderDecoder(**params).to(device) print('Loading model...') model.load_state_dict( torch.load(MODEL_PATH + model_file, map_location=device)) with open(SAMPLE_PATH + sample_file, 'w') as f: bleu = 0.0 for batch_X, batch_Y, X_length in test_dataloader: tgt_length = batch_Y.size(1) y_pred = model.sample(batch_X, X_length, tgt_length) X = batch_X.tolist() Y_true = batch_Y.tolist() Y_pred = y_pred.tolist() for x, y_true, y_pred in zip(X, Y_true, Y_pred): x = test.src_vocab.ids2seq(x) y_true = test.tgt_vocab.ids2seq(y_true) y_pred = test.tgt_vocab.ids2seq(y_pred) x = ' '.join(x) y_true = ' '.join(y_true) y_pred = ' '.join(y_pred) # print(x) # print(y_true) # print(y_pred) bleu += sentence_bleu([y_true], y_pred) f.write(x + ',' + y_true + ',' + y_pred + '\n') bleu /= test_dataloader.size print('BLEU score is {:.2f}'.format(bleu))
def train(self, src): self.build(mode="train") data_loader = utils.DataLoader(src) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) saver = tf.train.Saver(tf.global_variables(), max_to_keep=1) train_writer = tf.summary.FileWriter('summary_ckpt/', sess.graph) ckpt = tf.train.latest_checkpoint(utils.checkpoint_dir) if ckpt: saver.restore(sess, ckpt) print('restore from checkpoint{0}'.format(ckpt)) print( '=============================begin training=============================' ) train_cost = 0 last_step = 0 while data_loader.epoch < 100: batch_inputs, batch_labels, sparse_labels = data_loader.next_batch( self.batch_size) feed_dict = { self.inputs: batch_inputs, self.labels: batch_labels } summary_str, loss, step, _ = sess.run([ self.merged_summay, self.loss, self.global_step, self.train_op ], feed_dict) # calculate the cost train_cost += loss train_writer.add_summary(summary_str, step) print(step, loss) if step % 100 == 1: accuracy, decoded = sess.run( [self.accuracy, self.dense_decoded], feed_dict) print(step, accuracy, train_cost / (step - last_step)) with open("accuracy.txt", "w") as f: f.write( str(step) + ' ' + str(accuracy) + ' ' + str(train_cost / (step - last_step)) + '\n') for s in decoded: for c in s: if c > 0: f.write(utils.decode_maps[c]) f.write("\n") f.write("===================================\n") for s in sparse_labels: for c in s: if c > 0: f.write(utils.decode_maps[c]) f.write("\n") last_step = step train_cost = 0 if step % 100 == 0: saver.save(sess, os.path.join(utils.checkpoint_dir, 'ocr-model'), global_step=step) saver.save(sess, os.path.join(utils.checkpoint_dir, 'ocr-model'), global_step=step)
def main(): parser = argparse.ArgumentParser( description="Sample new trajectories with a social LSTM") parser.add_argument( "modelParams", type=str, help= "Path to the file or folder with the parameters of the experiments", ) parser.add_argument( "-l", "--logLevel", help="logging level of the logger. Default is INFO", metavar="level", type=str, ) parser.add_argument( "-f", "--logFolder", help= "path to the folder where to save the logs. If None, logs are only printed in stderr", metavar="path", type=str, ) parser.add_argument( "-ns", "--noSaveCoordinates", help="Flag to not save the predicted and ground truth coordinates", action="store_true", ) args = parser.parse_args() if os.path.isdir(args.modelParams): names_experiments = os.listdir(args.modelParams) experiments = [ os.path.join(args.modelParams, experiment) for experiment in names_experiments ] else: experiments = [args.modelParams] # Table will show the metrics of each experiment results = BeautifulTable() results.column_headers = ["Name experiment", "ADE", "FDE"] for experiment in experiments: # Load the parameters hparams = utils.YParams(experiment) # Define the logger setLogger(hparams, args, PHASE) remainSpaces = 29 - len(hparams.name) logging.info( "\n" + "--------------------------------------------------------------------------------\n" + "| Sampling experiment: " + hparams.name + " " * remainSpaces + "|\n" + "--------------------------------------------------------------------------------\n" ) trajectory_size = hparams.obsLen + hparams.predLen saveCoordinates = False if args.noSaveCoordinates is True: saveCoordinates = False elif hparams.saveCoordinates: saveCoordinates = hparams.saveCoordinates if saveCoordinates: coordinates_path = os.path.join("coordinates", hparams.name) if not os.path.exists("coordinates"): os.makedirs("coordinates") logging.info("Loading the test datasets...") test_loader = utils.DataLoader( hparams.dataPath, hparams.testDatasets, hparams.testMaps, hparams.semanticMaps, hparams.testMapping, hparams.homography, num_labels=hparams.numLabels, delimiter=hparams.delimiter, skip=hparams.skip, max_num_ped=hparams.maxNumPed, trajectory_size=trajectory_size, neighborood_size=hparams.neighborhoodSize, ) logging.info("Creating the test dataset pipeline...") dataset = utils.TrajectoriesDataset( test_loader, val_loader=None, batch=False, shuffle=hparams.shuffle, prefetch_size=hparams.prefetchSize, ) logging.info("Creating the model...") start = time.time() model = SocialModel(dataset, hparams, phase=PHASE) end = time.time() - start logging.debug("Model created in {:.2f}s".format(end)) # Define the path to the file that contains the variables of the model model_folder = os.path.join(hparams.modelFolder, hparams.name) model_path = os.path.join(model_folder, hparams.name) # Create a saver saver = tf.train.Saver() # Add to the computation graph the evaluation functions ade_sequence = utils.average_displacement_error( model.new_pedestrians_coordinates[-hparams.predLen:], model.pedestrians_coordinates[-hparams.predLen:], model.num_peds_frame, ) fde_sequence = utils.final_displacement_error( model.new_pedestrians_coordinates[-1], model.pedestrians_coordinates[-1], model.num_peds_frame, ) ade = 0 fde = 0 coordinates_predicted = [] coordinates_gt = [] peds_in_sequence = [] # Zero padding padding = len(str(test_loader.num_sequences)) # ============================ START SAMPLING ============================ with tf.Session() as sess: # Restore the model trained saver.restore(sess, model_path) # Initialize the iterator of the sample dataset sess.run(dataset.init_train) logging.info( "\n" + "--------------------------------------------------------------------------------\n" + "| Start sampling |\n" + "--------------------------------------------------------------------------------\n" ) for seq in range(test_loader.num_sequences): logging.info("Sample trajectory number {:{width}d}/{}".format( seq + 1, test_loader.num_sequences, width=padding)) ade_value, fde_value, coordinates_pred_value, coordinates_gt_value, num_peds = sess.run( [ ade_sequence, fde_sequence, model.new_pedestrians_coordinates, model.pedestrians_coordinates, model.num_peds_frame, ]) ade += ade_value fde += fde_value coordinates_predicted.append(coordinates_pred_value) coordinates_gt.append(coordinates_gt_value) peds_in_sequence.append(num_peds) ade = ade / test_loader.num_sequences fde = fde / test_loader.num_sequences logging.info("Sampling finished. ADE: {:.4f} FDE: {:.4f}".format( ade, fde)) results.append_row([hparams.name, ade, fde]) if saveCoordinates: coordinates_predicted = np.array(coordinates_predicted) coordinates_gt = np.array(coordinates_gt) saveCoords( coordinates_predicted, coordinates_gt, peds_in_sequence, hparams.predLen, coordinates_path, ) tf.reset_default_graph() logging.info("\n{}".format(results))
from sklearn.preprocessing import scale import statsmodels.api as sm import m_models as mm import numpy as np import utils as ut import os from tqdm import tqdm as tqdm dataset_name = 'boston' X, y = ut.DataLoader(name=dataset_name) X = scale(X) y = scale(y) X = sm.add_constant(X) backup_folder = "backup/" if not os.path.exists(backup_folder): os.mkdir(backup_folder) datasets = [] sample_sizes = [28, 50, 100, 120, 140, 160, 180, 200, 220, 240] name = dataset_name data = { 'name': name, 'X': np.array(X), 'y': np.array(y), 'folder_path': backup_folder + name + "/", 'backup': {
def load_dataset(data_dir, model_params, inference_mode=False): """Loads the .npz file, and splits the set into train/valid/test.""" # normalizes the x and y columns usint the training set. # applies same scaling factor to valid and test set. datasets = [] if isinstance(model_params.data_set, list): datasets = model_params.data_set else: datasets = [model_params.data_set] train_strokes = None valid_strokes = None test_strokes = None for dataset in datasets: data_filepath = os.path.join(data_dir, dataset) if data_dir.startswith('http://') or data_dir.startswith('https://'): tf.logging.info('Downloading %s', data_filepath) response = requests.get(data_filepath) data = np.load(StringIO(response.content)) else: data = np.load(data_filepath) # load this into dictionary tf.logging.info('Loaded {}/{}/{} from {}'.format( len(data['train']), len(data['valid']), len(data['test']), dataset)) if train_strokes is None: train_strokes = data['train'] valid_strokes = data['valid'] test_strokes = data['test'] else: train_strokes = np.concatenate((train_strokes, data['train'])) valid_strokes = np.concatenate((valid_strokes, data['valid'])) test_strokes = np.concatenate((test_strokes, data['test'])) all_strokes = np.concatenate((train_strokes, valid_strokes, test_strokes)) num_points = 0 for stroke in all_strokes: num_points += len(stroke) avg_len = num_points / len(all_strokes) tf.logging.info('Dataset combined: {} ({}/{}/{}), avg len {}'.format( len(all_strokes), len(train_strokes), len(valid_strokes), len(test_strokes), int(avg_len))) # calculate the max strokes we need. max_seq_len = utils.get_max_len(all_strokes) # overwrite the hps with this calculation. model_params.max_seq_len = max_seq_len tf.logging.info('model_params.max_seq_len %i.', model_params.max_seq_len) eval_model_params = sketch_rnn_model.copy_hparams(model_params) eval_model_params.use_input_dropout = 0 eval_model_params.use_recurrent_dropout = 0 eval_model_params.use_output_dropout = 0 eval_model_params.is_training = 1 if inference_mode: eval_model_params.batch_size = 1 eval_model_params.is_training = 0 sample_model_params = sketch_rnn_model.copy_hparams(eval_model_params) sample_model_params.batch_size = 1 # only sample one at a time sample_model_params.max_seq_len = 1 # sample one point at a time train_set = utils.DataLoader( train_strokes, model_params.batch_size, max_seq_length=model_params.max_seq_len, random_scale_factor=model_params.random_scale_factor, augment_stroke_prob=model_params.augment_stroke_prob) normalizing_scale_factor = train_set.calculate_normalizing_scale_factor() train_set.normalize(normalizing_scale_factor) print('Length original', len(train_strokes), len(valid_strokes), len(test_strokes)) valid_set = utils.DataLoader(valid_strokes, eval_model_params.batch_size, max_seq_length=eval_model_params.max_seq_len, random_scale_factor=0.0, augment_stroke_prob=0.0) valid_set.normalize(normalizing_scale_factor) test_set = utils.DataLoader(test_strokes, eval_model_params.batch_size, max_seq_length=eval_model_params.max_seq_len, random_scale_factor=0.0, augment_stroke_prob=0.0) test_set.normalize(normalizing_scale_factor) tf.logging.info('normalizing_scale_factor %4.4f.', normalizing_scale_factor) result = [ train_set, valid_set, test_set, model_params, eval_model_params, sample_model_params ] return result
def load_datasets(data_dir, model_params, inference_mode=False): """Load and preprocess data""" data = utils.load_dataset(data_dir) train_strokes = data['train'] valid_strokes = data['valid'] test_strokes = data['test'] all_strokes = np.concatenate((train_strokes, valid_strokes, test_strokes)) num_points = 0 for stroke in all_strokes: num_points += len(stroke) avg_len = num_points / len(all_strokes) tf.logging.info('{} Shapes / {} Total points'.format(len(all_strokes), num_points)) tf.logging.info('Dataset combined: {} ({}/{}/{}), avg len {}'.format( len(all_strokes), len(train_strokes), len(valid_strokes), len(test_strokes), int(avg_len))) # calculate the max strokes we need. max_seq_len = utils.get_max_len(all_strokes) # overwrite the hps with this calculation. model_params.max_seq_len = max_seq_len tf.logging.info('model_params.max_seq_len %i.', model_params.max_seq_len) eval_model_params = derender_model.copy_hparams(model_params) eval_model_params.use_input_dropout = 0 eval_model_params.use_recurrent_dropout = 0 eval_model_params.use_output_dropout = 0 eval_model_params.is_training = 1 if inference_mode: eval_model_params.batch_size = 1 eval_model_params.is_training = 0 sample_model_params = derender_model.copy_hparams(eval_model_params) sample_model_params.batch_size = 1 # only sample one at a time sample_model_params.max_seq_len = 1 # sample one point at a time train_set = utils.DataLoader( train_strokes, model_params.batch_size, max_seq_length=model_params.max_seq_len, random_scale_factor=model_params.random_scale_factor, augment_stroke_prob=model_params.augment_stroke_prob) normalizing_scale_factor = train_set.calculate_normalizing_scale_factor() train_set.normalize(normalizing_scale_factor) valid_set = utils.DataLoader( valid_strokes, eval_model_params.batch_size, max_seq_length=eval_model_params.max_seq_len, random_scale_factor=0.0, augment_stroke_prob=0.0) valid_set.normalize(normalizing_scale_factor) test_set = utils.DataLoader( test_strokes, eval_model_params.batch_size, max_seq_length=eval_model_params.max_seq_len, random_scale_factor=0.0, augment_stroke_prob=0.0) test_set.normalize(normalizing_scale_factor) tf.logging.info('normalizing_scale_factor %4.4f.', normalizing_scale_factor) result = [ train_set, valid_set, test_set, model_params, eval_model_params, sample_model_params ] return result
def load_dataset(data_dir, model_params, inference_mode=False): """Loads the .npz file, and splits the set into train/valid/test.""" # normalizes the x and y columns using the training set. # applies same scaling factor to valid and test set. if isinstance(model_params.data_set, list): datasets = model_params.data_set else: datasets = [model_params.data_set] train_strokes = None valid_strokes = None test_strokes = None train_data = [] valid_data = [] test_data = [] dataset_lengths = [] all_strokes = [] for i, dataset in enumerate(datasets): data_filepath = os.path.join(data_dir, dataset) if six.PY3: tmp_data = np.load(data_filepath, encoding='latin1', allow_pickle=True) else: tmp_data = np.load(data_filepath, allow_pickle=True) all_strokes = np.concatenate((all_strokes, tmp_data['train'], tmp_data['valid'], tmp_data['test'])) max_seq_len = utils.get_max_len(all_strokes) model_params.max_seq_len = max_seq_len print('Max sequence length: ', max_seq_len) for i, dataset in enumerate(datasets): data_filepath = os.path.join(data_dir, dataset) if six.PY3: data = np.load(data_filepath, encoding='latin1', allow_pickle=True) else: data = np.load(data_filepath, allow_pickle=True) logger.info('Loaded {}/{}/{} from {}'.format(len(data['train']), len(data['valid']), len(data['test']), dataset)) train_strokes = data['train'] valid_strokes = data['valid'] test_strokes = data['test'] train_set = utils.DataLoader( train_strokes, model_params.batch_size, max_seq_length=max_seq_len, random_scale_factor=model_params.random_scale_factor, augment_stroke_prob=model_params.augment_stroke_prob) normalizing_scale_factor = train_set.calculate_normalizing_scale_factor( ) train_set.normalize(normalizing_scale_factor) train_set.strokes = [ utils.to_big_strokes(stroke, max_seq_len) for stroke in train_set.strokes ] train_set.strokes = [ np.insert(stroke, 0, [0, 0, 1, 0, 0], axis=0) for stroke in train_set.strokes ] valid_set = utils.DataLoader( valid_strokes, model_params.batch_size, max_seq_length=max_seq_len, random_scale_factor=model_params.random_scale_factor, augment_stroke_prob=model_params.augment_stroke_prob) valid_set.normalize(normalizing_scale_factor) valid_set.strokes = [ utils.to_big_strokes(stroke, max_seq_len) for stroke in valid_set.strokes ] valid_set.strokes = [ np.insert(stroke, 0, [0, 0, 1, 0, 0], axis=0) for stroke in valid_set.strokes ] test_set = utils.DataLoader( test_strokes, model_params.batch_size, max_seq_length=max_seq_len, random_scale_factor=model_params.random_scale_factor, augment_stroke_prob=model_params.augment_stroke_prob) test_set.normalize(normalizing_scale_factor) test_set.strokes = [ utils.to_big_strokes(stroke, max_seq_len) for stroke in test_set.strokes ] test_set.strokes = [ np.insert(stroke, 0, [0, 0, 1, 0, 0], axis=0) for stroke in test_set.strokes ] train_sketches = [{ 'dataset': dataset, 'draw': sketch } for sketch in train_set.strokes] valid_sketches = [{ 'dataset': dataset, 'draw': sketch } for sketch in valid_set.strokes] test_sketches = [{ 'dataset': dataset, 'draw': sketch } for sketch in test_set.strokes] train_data.append(train_sketches) valid_data.append(valid_sketches) test_data.append(test_sketches) return [train_data, valid_data, test_data]
def __init__(self): self.data = utils.DataLoader() self._data = utils.DataLoader() self._system = None