def test(self): """Forward function used in test time""" for j, input_BP2 in enumerate(self.input_BP2): input_BP2 = self.obtain_shape_net_semantic(input_BP2) input_BP1 = self.obtain_shape_net_semantic(self.input_BP1) img_gen, flow_fields, masks = self.net_G(self.input_P1, input_BP1, input_BP2) for i in range(self.input_P1.size(0)): util.mkdir(self.opt.results_dir) img_path = os.path.join(self.opt.results_dir, self.image_paths[i]) util.mkdir(img_path) source_name = os.path.join(img_path, 'source.png') img_numpy = util.tensor2im(self.input_P1[i]) util.save_image(img_numpy, source_name) gt_name = os.path.join(img_path, 'gt_' + str(j).zfill(4) + '.png') img_numpy = util.tensor2im(self.input_P2[j][i]) util.save_image(img_numpy, gt_name) gen_name = os.path.join(img_path, 'result_' + str(j).zfill(4) + '.png') img_numpy = util.tensor2im(img_gen[i]) util.save_image(img_numpy, gen_name) print(gen_name)
def __init__(self, opt): self.opt = opt self.is_train = opt.is_train self.save_dir = os.path.join(opt.checkpoints_dir, opt.name) self.optimize_modules = [] mkdir(self.save_dir) self.initialize()
def save_generated_images(sampler, N, params): """Save N generated images.""" import time mkdir(params.results_dir) expt_dir = join_and_create_dir(params.results_dir, params.expt_name) sample_dir = join_and_create_dir(expt_dir, 'samples') a_dir = join_and_create_dir(sample_dir, 'A') b_dir = join_and_create_dir(sample_dir, 'B') comp_dir = join_and_create_dir(sample_dir, 'composed') print a_dir z = np.random.normal(loc=0.0, scale=1.0, size=(N, params.latent_dim)) a, b = sampler.predict(z, batch_size=1, verbose=False) start = time.time() a, b = sampler.predict(z, batch_size=1, verbose=False) end = time.time() print 'Took {0} seconds to generate {1} images.'.format(end - start, N) for i, (ai, bi) in enumerate(zip(a, b)): img = compose_imgs(ai, bi, is_a_binary=params.is_a_binary, is_b_binary=params.is_b_binary) ai = convert_to_rgb(ai, is_binary=params.is_a_binary) bi = convert_to_rgb(bi, is_binary=params.is_b_binary) filename = '{0}.png'.format(i) plt.imsave(open(os.path.join(a_dir, filename), 'wb+'), ai) plt.imsave(open(os.path.join(b_dir, filename), 'wb+'), bi) plt.imsave(open(os.path.join(comp_dir, filename), 'wb+'), img)
def __init__(self, opt): BaseModel.__init__(self, opt) self.loss_names = ['mpjpe'] self.model_names = ['G'] self.FloatTensor = torch.cuda.FloatTensor if len(self.gpu_ids)>0 \ else torch.FloatTensor self.ByteTensor = torch.cuda.ByteTensor if len(self.gpu_ids)>0 \ else torch.ByteTensor self.net_G = network.define_g(opt, filename='generator', structure_nc=opt.structure_nc, channels=256, layers=4) if len(opt.gpu_ids) > 1: self.net_G = torch.nn.DataParallel(self.net_G, device_ids=self.gpu_ids) self.convert2skeleton = openpose_utils.tensor2skeleton() if self.isTrain: self.L2loss = torch.nn.MSELoss() self.optimizer_G = torch.optim.Adam(itertools.chain( filter(lambda p: p.requires_grad, self.net_G.parameters())), lr=opt.lr, betas=(0.0, 0.999)) self.optimizers.append(self.optimizer_G) else: self.L2loss = torch.nn.MSELoss() util.mkdir(os.path.join(self.opt.results_dir)) self.setup(opt)
def val(opt): image_1_path = opt.image1_path image_2_path = opt.image2_path A_img = Image.open(image_1_path).convert('RGB') B_img = Image.open(image_2_path).convert('RGB') trans = transform() A = trans(A_img).unsqueeze(0) B = trans(B_img).unsqueeze(0) # dataset = create_dataset(opt) # create a dataset given opt.dataset_mode and other options model = create_model( opt) # create a model given opt.model and other options model.setup( opt) # regular setup: load and print networks; create schedulers save_path = opt.results_dir mkdir(save_path) model.eval() data = {} data['A'] = A data['B'] = B data['A_paths'] = [image_1_path] model.set_input(data) # unpack data from data loader pred = model.test(val=False) # run inference return pred img_path = [image_1_path] # get image paths save_images(pred, save_path, img_path)
def preprocess_data(self): # mkdir if needed util.mkdir(self._output_path) t = time.time() for current_index in range(self._num_entries): file_stem = self._file_stems[current_index] jpg_path = '%s/%s.jpg' % (self._input_path, file_stem) json_path = '%s/%s.json' % (self._input_path, file_stem) if os.path.exists(json_path): with open(json_path, 'r') as f: json_data = ujson.load(f) entry = { 'full_image': cv.imread(jpg_path, cv.IMREAD_GRAYSCALE), 'json_data': json_data } preprocessed_entry = self.preprocess_entry(entry) if preprocessed_entry is not None: self._save_pickle(preprocessed_entry, os.path.join(self._output_path, '%s.pickle' % file_stem)) if current_index % 1000 == 0 and current_index != 0: print('preprocessed %s entries in %s' % (current_index, (time.time()-t))) t = time.time()
def val(opt): dataset = create_dataset(opt) # create a dataset given opt.dataset_mode and other options model = create_model(opt) # create a model given opt.model and other options model.setup(opt) # regular setup: load and print networks; create schedulers save_path = os.path.join(opt.checkpoints_dir, opt.name, '%s_%s' % (opt.phase, opt.epoch)) mkdir(save_path) model.eval() # create a logging file to store training losses log_name = os.path.join(opt.checkpoints_dir, opt.name, 'val1_log.txt') with open(log_name, "a") as log_file: now = time.strftime("%c") log_file.write('================ val acc (%s) ================\n' % now) running_metrics = AverageMeter() for i, data in enumerate(dataset): if i >= opt.num_test: # only apply our model to opt.num_test images. break model.set_input(data) # unpack data from data loader score = model.test(val=True) # run inference return confusion_matrix running_metrics.update(score) visuals = model.get_current_visuals() # get image results img_path = model.get_image_paths() # get image paths if i % 5 == 0: # save images to an HTML file print('processing (%04d)-th image... %s' % (i, img_path)) save_visuals(visuals,save_path,img_path[0]) score = running_metrics.get_scores() print_current_acc(log_name, opt.epoch, score)
def parse(self): if not self.initialized: self.initialize() self.opt = self.parser.parse_args() str_ids = self.opt.gpu_ids.split(',') self.opt.gpu_ids = [] for str_id in str_ids: id = int(str_id) if id >= 0: self.opt.gpu_ids.append(id) # set gpu ids if len(self.opt.gpu_ids) > 0: torch.cuda.set_device(self.opt.gpu_ids[0]) args = vars(self.opt) print('------------ Options -------------') for k, v in sorted(args.items()): print('%s: %s' % (str(k), str(v))) print('-------------- End ----------------') # save to the disk expr_dir = os.path.join(self.opt.results_dir, self.opt.name) util.mkdir(expr_dir) file_name = os.path.join(expr_dir, 'opt.txt') with open(file_name, 'wt') as opt_file: opt_file.write('------------ Options -------------\n') for k, v in sorted(args.items()): opt_file.write('%s: %s\n' % (str(k), str(v))) opt_file.write('-------------- End ----------------\n') return self.opt
def stroke_extraction_with_loop(data_path): img_list = os.listdir(data_path) mkdir(args.new_path) img_count = 0 for img_file in img_list: file_path = '{}/{}'.format(data_path, img_file) try: img = cv2.imread(file_path) except: print('problematic file:', file_path) continue if img is None: print('problematic file:', file_path) continue else: img_stroke = high_pass_filter(img) old_file_location = 'full' new_file_location = 'sketch' new_path = file_path.replace(old_file_location, new_file_location) cv2.imwrite(new_path, img_stroke) img_count += 1 if img_count % 1000 == 0: print('{} images processed!'.format(img_count), end='\r') print('finished processing {} images !'.format(img_count))
def parse(self): """Parse our options, create checkpoints directory suffix, and set up gpu device.""" opt = self.gather_options() opt.isTrain = self.isTrain # train or test # process opt.suffix if opt.suffix: suffix = ( '_' + opt.suffix.format(**vars(opt))) if opt.suffix != '' else '' opt.name = opt.name + suffix # self.print_options(opt) os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpu_ids for op, value in opt.__dict__.items(): print('{:>25}: {:<30}'.format(op, str(value))) backup_dir = osp.join(opt.checkpoints_dir, opt.name) util.mkdir(backup_dir) logger = Logger( osp.join(backup_dir, 'log' + time.strftime(".%m_%d_%H:%M:%S") + '.txt')) opt.backup = backup_dir util.save_opt(opt, backup_dir) self.opt = opt return self.opt, logger
def get_ids(): dic_ids = {} mkdir(os.path.join(PATH_DATASET, 'raw')) fnames = sorted(os.listdir(PATH_DATASET)) for fname in fnames: if not fname.endswith(SUFFIX): continue fname_tmp = fname.split('.')[0] prefix, patient_str, timepoint_str, modality = fname_tmp.split('_') patient_id, timepoint_id = int(patient_str), int(timepoint_str) # patient_id, timepoint_id = int(patient_id), int(timepoint_id) if patient_id in dic_ids and timepoint_id in dic_ids[patient_id]: continue # if there is new patient id and timepoint id, we get all the modalities and masks based on the constants # and we will move the files into the 'raw' subdirectory if patient_id not in dic_ids: dic_ids[patient_id] = {} dic_ids[patient_id][timepoint_id] = {'modalities':{}, 'mask':{}} for mod in MODALITIES+MASKS: fname_modality = '_'.join((prefix, patient_str, timepoint_str, mod)) + '.' + SUFFIX path_modality_src = os.path.join(PATH_DATASET, fname_modality) path_modality_dst = os.path.join(PATH_DATASET, 'raw', fname_modality) category = 'modalities' if mod in MODALITIES else 'mask' assert os.path.exists(path_modality_src) shutil.move(path_modality_src, path_modality_dst) dic_ids[patient_id][timepoint_id][category][mod] = path_modality_dst fname_json = os.path.join(PATH_DATASET, 'ids.json') with open(fname_json, 'w') as f: json.dump(dic_ids, f, indent=2) return dic_ids
def __getitem__(self, index): row = self.data_df.iloc[index] edge_img = io.imread(row.edge_path) org_h, org_w = row.orig_H, row.orig_W upper_x, upper_y = int(row.upper_X * 256 / org_w), int(row.upper_Y * 256 / org_h) upper_w, upper_h = int(row.upper_W * 256 / org_w), int(row.upper_H * 256 / org_h) shoulder1_x, shoulder1_y = int(row.shoulder1_x * 256 / org_w), int(row.shoulder1_y * 256 / org_h) shoulder2_x, shoulder2_y = int(row.shoulder2_x * 256 / org_w), int(row.shoulder2_y * 256 / org_h) sleeve1_x, sleeve1_y = int(row.sleeve1_x * 256 / org_w), int(row.sleeve1_y * 256 / org_h) sleeve2_x, sleeve2_y = int(row.sleeve2_x * 256 / org_w), int(row.sleeve2_y * 256 / org_h) mask = np.zeros((256, 256)) mask[shoulder1_y:, sleeve1_x - 15:shoulder1_x + 15] = 1 mask[shoulder2_y:, shoulder2_x - 15:sleeve2_x + 15] = 1 croppedSleeve = edge_img*mask strlist = row.edge_path.split('/') croppedSleeveImgName = strlist[-1] edge_img = torch.Tensor(edge_img) edge_img = edge_img.repeat(3, 1, 1) croppedSleeve = torch.Tensor(croppedSleeve) croppedSleeve = croppedSleeve.repeat(3, 1, 1) path = './result/croppedSleeve/' util.mkdir(path) vutils.save_image( croppedSleeve.detach(), '%s/%s' % (path, croppedSleeveImgName), normalize=True ) return 0
def save_result(self): self.opt.results_dir = "./results" """Save the results to the disk""" util.mkdir(self.opt.results_dir) img_name = self.fname.split('/')[-1] # save the original image original_name = '%s_%s' % ('original', img_name) original_path = os.path.join(self.opt.results_dir, original_name) img_original = util.tensor2im(self.img_truth) util.save_image(img_original, original_path) # save the mask mask_name = '%s_%d_%s' % ('mask', self.PaintPanel.iteration, img_name) mask_path = os.path.join(self.opt.results_dir, mask_name) img_mask = util.tensor2im(self.img_c) ret, img_mask = cv2.threshold(img_mask, 150, 255, cv2.THRESH_BINARY) util.save_image(img_mask, mask_path) # save the results result_name = '%s_%d_%s' % ('result', self.PaintPanel.iteration, img_name) result_path = os.path.join(self.opt.results_dir, result_name) img_result = TF.to_pil_image(self.img_out) img_result = np.array(img_result) util.save_image(img_result, result_path)
def save_result(self): """Save the results to the disk""" util.mkdir(self.opt.results_dir) img_name = self.fname.split('/')[-1] data_name = self.opt.img_file.split('/')[-1].split('.')[0] # save the original image original_name = '%s_%s_%s' % ('original', data_name, img_name) original_path = os.path.join(self.opt.results_dir, original_name) img_original = util.tensor2im(self.img_truth) util.save_image(img_original, original_path) # save the mask mask_name = '%s_%s_%d_%s' % ('mask', data_name, self.PaintPanel.iteration, img_name) mask_path = os.path.join(self.opt.results_dir, mask_name) img_mask = util.tensor2im(self.img_m) util.save_image(img_mask, mask_path) # save the results result_name = '%s_%s_%d_%s' % ('result', data_name, self.PaintPanel.iteration, img_name) result_path = os.path.join(self.opt.results_dir, result_name) img_result = util.tensor2im(self.img_out) util.save_image(img_result, result_path)
def save_checkpoints(model, train_results, dev_results, test_results, save_path): util.mkdir(save_path) model.save(save_path) results_fname = save_path + '/results.csv' save_results(model, train_results, dev_results, test_results, results_fname)
def test_selected_curve(self): ''' given a training and a reference image, synthesize the intermediate results with the selected interpolation curve. :return: ''' util.mkdir(self.args.save_dir) n_branch = len(self.args.attr.split(',')) + 1 test_model = model.model_deploy( n_branch=n_branch, model_path=self.args.model_path, label=self.args.pth_label).eval().cuda() _, test_dataset = self.load_dataset() loader = DataLoader(test_dataset, batch_size=self.args.batch_size, shuffle=False) for i, data in enumerate(loader): img, _ = data img = util.toVariable(img).cuda() idx = torch.randperm(img.size(0)).cuda() img_ref = img[idx] img_out = [img] v = torch.zeros(img.size(0), n_branch) for j in self.args.branch_list: v[:, j:j + 1] = 1 v = util.toVariable(v).cuda() out_now = test_model(img, img_ref, v) img_out += [out_now] img_out += [img_ref] img_out = torch.cat(img_out) img_out = tensorWriter.untransformTensor(img_out.data.cpu()) tensorWriter.writeTensor('{}/{}.jpg'.format(self.args.save_dir, i), img_out, nRow=img.size(0))
def load_picture_pipeline(self, file_path: str): mkdir('picture') tpk_list = self.tpwds[file_path] picture_url = [(self.tpwd_map[file_path][tpk]['picUrl'], idx) for idx, tpk in enumerate(tpk_list) if tpk in self.tpwd_map[file_path]] picture_url = [(ii, idx) for ii, idx in picture_url if not os.path.exists('picture/{}.jpg'.format(idx))] echo(1, 'Load {} picture Begin'.format(len(picture_url))) pp = [self.tpwd_exec.submit(self.load_picture, ii, jj) for ii, jj in picture_url] return pp
def attribute_manipulation(self): ''' perform attribute manipulation :return: ''' from data.attributeDataset import Dataset_testing_filtered, Dataset_testing util.mkdir(self.args.save_dir) n_branch = len(self.args.attr.split( ',')) + 1 # n groupped attribute + 1 residual attribute. test_model = model.model_deploy( n_branch=n_branch, model_path=self.args.model_path, label=self.args.pth_label).eval().cuda() if self.args.test_folder is None: _, test_dataset = self.load_dataset() else: image_list = glob.glob(self.args.test_folder + '/*.jpg') test_dataset = Dataset_testing(image_list) ref_dataset = Dataset_testing_filtered(self.args.data_dir, self.args.filter_target_attr, n_samples=self.args.n_ref) loader = DataLoader(test_dataset, batch_size=self.args.batch_size, shuffle=True) ref_loader = DataLoader(ref_dataset, batch_size=1, shuffle=True) # for data, ref_data in zip(loader, ref_loader): img_out = [tmp for tmp in ref_loader] img_out = torch.cat(img_out) img_out = tensorWriter.untransformTensor(img_out.data.cpu()) tensorWriter.writeTensor('{}/reference.jpg'.format(self.args.save_dir), img_out, nRow=1) for i, data in enumerate(loader): img, _ = data img = util.toVariable(img).cuda() img_out = [img] for ref_data in ref_loader: print('proceesing the {}-th batch'.format(i)) img_ref = ref_data img_ref = util.toVariable(img_ref).cuda() v = torch.zeros(img.size(0), n_branch) v[:, self.args.branch_idx:self.args.branch_idx + 1] = self.args.strength v = util.toVariable(v).cuda() img_ref_now = img_ref.expand_as(img) out_now = test_model(img, img_ref_now, v) img_out += [out_now] # img_out += [img_ref] img_out = torch.cat(img_out) img_out = tensorWriter.untransformTensor(img_out.data.cpu()) tensorWriter.writeTensor('{}/{}.jpg'.format(self.args.save_dir, i), img_out, nRow=img.size(0)) i += 1 if i > self.args.n_test: break
def __init__(self, file_list, output_dir, params, output_file=None): self.file_list = file_list util.mkdir(output_dir) self.params = params if output_file: self.output = open(output_dir + '/' + output_file, 'w') else: self.output = open(output_dir + '/' + str(int(time.time())), 'w') print 'reduce params:', self.params
def inference_single_audio(opt, path_label, model): # opt.path_label = path_label dataloader = data.create_dataloader(opt) processed_file_savepath = dataloader.dataset.get_processed_file_savepath() idx = 0 if opt.driving_pose: video_names = [ 'Input_', 'G_Pose_Driven_', 'Pose_Source_', 'Mouth_Source_' ] else: video_names = ['Input_', 'G_Fix_Pose_', 'Mouth_Source_'] save_paths = [] for name in video_names: save_path = os.path.join(processed_file_savepath, name) util.mkdir(save_path) save_paths.append(save_path) for data_i in tqdm(dataloader): # print('==============', i, '===============') fake_image_original_pose_a, fake_image_driven_pose_a = model.forward( data_i, mode='inference') for num in range(len(fake_image_driven_pose_a)): util.save_torch_img( data_i['input'][num], os.path.join(save_paths[0], video_names[0] + str(idx) + '.jpg')) if opt.driving_pose: util.save_torch_img( fake_image_driven_pose_a[num], os.path.join(save_paths[1], video_names[1] + str(idx) + '.jpg')) util.save_torch_img( data_i['driving_pose_frames'][num], os.path.join(save_paths[2], video_names[2] + str(idx) + '.jpg')) else: util.save_torch_img( fake_image_original_pose_a[num], os.path.join(save_paths[1], video_names[1] + str(idx) + '.jpg')) util.save_torch_img( data_i['target'][num], os.path.join(save_paths[-1], video_names[-1] + str(idx) + '.jpg')) idx += 1 if opt.gen_video: for i, video_name in enumerate(video_names): img2video(processed_file_savepath, video_name, save_paths[i]) video_concat(processed_file_savepath, 'concat', video_names, dataloader.dataset.audio_path) print('results saved...' + processed_file_savepath) del dataloader return
def main(): # pylint: disable=too-many-locals args = get_args() n_runs = 50 ouput_path = os.path.join( args.checkpoint_path, args.task, args.language, args.model, args.representation) results_fname = os.path.join(ouput_path, 'all_results.tsv') done_fname = os.path.join(ouput_path, 'finished.txt') _, _, _, n_classes, _ = \ get_data_loaders(args.data_path, args.task, args.language, 'onehot', 1, 1) curr_iter = util.file_len(results_fname) - 1 skip_first = False util.mkdir(ouput_path) if curr_iter == -1: res_columns = ['run', 'hidden_size', 'nlayers', 'dropout', 'embedding_size', 'alpha', 'max_rank', 'shuffle_labels', 'shuffle_input', 'train_loss', 'dev_loss', 'test_loss', 'train_acc', 'dev_acc', 'test_acc', 'train_norm', 'dev_norm', 'test_norm', 'train_rank', 'dev_rank', 'test_rank'] append_result(results_fname, res_columns) curr_iter = 0 if args.shuffle_labels: skip_first = curr_iter % 3 curr_iter = int(curr_iter / 3) search = get_hyperparameters_search(n_runs, args.representation, n_classes) for i, hyper in tqdm(enumerate(search[curr_iter:]), initial=curr_iter, total=n_runs): run_count = curr_iter + i hyperparameters = get_hyperparameters(hyper) if not skip_first: run_experiment(run_count, hyper, hyperparameters, args, results_fname) if args.shuffle_labels and skip_first <= 1: run_experiment(run_count, hyper, hyperparameters, args, results_fname, shuffle_labels=True) if args.shuffle_labels: run_experiment(run_count, hyper, hyperparameters, args, results_fname, shuffle_labels=True, shuffle_input=True) skip_first = 0 write_done(done_fname)
def test(self): output_skeleton = self.net_G(self.input_skeleton) if self.opt.write_image: current_map = self.convert2skeleton(self.current_skeleton, kp_form='COCO_17') obtained_map = self.convert2skeleton(output_skeleton, kp_form='human36m_17') for j in range(len(current_map)): short_path = ntpath.basename(self.image_paths[j][0]) name = os.path.splitext(short_path)[0] util.mkdir(self.results_dir_base) gt = self.gen_images[0,j*3:j*3+3,...].detach().cpu().numpy() gt = np.transpose(((gt+1)/2 * 255), (1,2,0)) in_img_name = '%s_%s.%s' % (name, 'skeleton_in', 'png') img_path = os.path.join(self.results_dir_base, in_img_name) gt_ = np.copy(gt) gt_[current_map[j]!=0]=0 skeleton_in = (current_map[j]+gt_).astype(np.uint8) util.save_image(skeleton_in, img_path) print(img_path) out_img_name = '%s_%s.%s' % (name, 'skeleton_out', 'png') img_path = os.path.join(self.results_dir_base, out_img_name) gt_ = np.copy(gt) gt_[obtained_map[j]!=0]=0 skeleton_out = (obtained_map[j]+gt_).astype(np.uint8) util.save_image(skeleton_out, img_path) output_skeleton = (output_skeleton+1)/2 skeleton_pad = torch.ones(1,17).type_as(output_skeleton) for j in range(output_skeleton.shape[-1]): skeleton = output_skeleton[0,:,j].view(2,17) skeleton[0,:] = skeleton[0,:]*self.shape[-1][0] skeleton[1,:] = skeleton[1,:]*self.shape[0][0] skeleton = torch.cat((skeleton,skeleton_pad), 0) skeleton = skeleton[[1,0,2],:] skeleton = skeleton.transpose(1,0).contiguous().view(-1).detach().cpu().numpy() people = dict() people['pose_keypoints_2d']=skeleton.tolist() output_dict=dict() output_dict["version"]="Video to Pose 2D" output_dict["people"]=[people] short_path = ntpath.basename(self.image_paths[j][0]) name = os.path.splitext(short_path)[0] util.mkdir(self.results_dir_base) name = '%s.%s' % (name, 'json') name = os.path.join(self.results_dir_base, name) with open(name, 'w') as f: json.dump(output_dict, f)
def save_feature_map(self, feature_map, save_path, name, add): if feature_map.dim() == 4: feature_map = feature_map[0] name = name[0] feature_map = feature_map.detach().cpu().numpy() for i in range(feature_map.shape[0]): img = feature_map[i,:,:] img = self.motify_outlier(img) name_ = os.path.join(save_path, os.path.splitext(os.path.basename(name))[0], add+'modify'+str(i)+'.png') util.mkdir(os.path.dirname(name_)) plt.imsave(name_, img, cmap='viridis')
def save_all_conditional(vae, unet, it_train, it_val, it_test, params): """Save all the results of the pix2pix model.""" mkdir(params.results_dir) expt_dir = join_and_create_dir(params.results_dir, params.expt_name) train_dir = join_and_create_dir(expt_dir, params.train_dir) val_dir = join_and_create_dir(expt_dir, params.val_dir) test_dir = join_and_create_dir(expt_dir, params.test_dir) save_conditional(vae, unet, it_train, train_dir, params) save_conditional(vae, unet, it_val, val_dir, params) save_conditional(vae, unet, it_test, test_dir, params)
def parse(self): if not self.initialized: self.initialize() self.opt, unknown = self.parser.parse_known_args() if self.is_given_config_path(): self.opt = self.parse_config_file() self.raise_if_no_dataroot() self.opt.is_train = self.is_train # train or test str_ids = self.opt.gpu_ids.split(',') self.opt.gpu_ids = [] for str_id in str_ids: id = int(str_id) if id >= 0: self.opt.gpu_ids.append(id) # set gpu ids if len(self.opt.gpu_ids) > 0: torch.cuda.set_device(self.opt.gpu_ids[0]) args = vars(self.opt) if self.opt.seed is not None: import numpy as np import random torch.manual_seed(self.opt.seed) np.random.seed(self.opt.seed) random.seed(self.opt.seed) if self.opt.export_folder: self.opt.export_folder = os.path.join(self.opt.checkpoints_dir, self.opt.name, self.opt.export_folder) util.mkdir(self.opt.export_folder) if self.is_train: print('------------ Options -------------') for k, v in sorted(args.items()): print('%s: %s' % (str(k), str(v))) print('-------------- End ----------------') # save to the disk expr_dir = os.path.join(self.opt.checkpoints_dir, self.opt.name) util.mkdir(expr_dir) file_name = os.path.join(expr_dir, 'opt.txt') with open(file_name, 'wt') as opt_file: opt_file.write('------------ Options -------------\n') for k, v in sorted(args.items()): opt_file.write('%s: %s\n' % (str(k), str(v))) opt_file.write('-------------- End ----------------\n') return self.opt
def __init__(self, model, option=opt.opt()): super(optimizer, self).__init__() self._default_opt() self.opt.merge_opt(option) self._get_model(model) self._get_aux_nets() self._define_optim() # self.writer = curves.writer(log_dir=self.opt.save_dir + '/log') util.mkdir(self.opt.save_dir + '/log') self.writer = tensorboardX.SummaryWriter(log_dir=self.opt.save_dir + '/log') if self.opt.continue_train: self.load()
def load_csdn_img_load(self, img_url: str, article_id: int, idx: int): img_dir = '{}{}/'.format(data_dir, article_id) img_path = '{}{}.png'.format(img_dir, idx) if os.path.exists(img_path): return req = proxy_req(img_url, 2) if type(req) == bool or req is None: if can_retry(img_url): self.load_csdn_img_load(img_url, article_id, idx) return mkdir(img_dir) with open(img_path, 'wb') as f: f.write(req.content)
def schedule(self): logging.info('push predict dispatch main process start') for job in self.conf.get("jobs"): print 'process job:%s' % job['job_name'] __import__(job['map_module']) mapper_inst = getattr(sys.modules[job['map_module']], job['mapper']) reducer_inst = None if job.get('reducer', 'NONE') != 'NONE': __import__(job['reduce_module']) reducer_inst = getattr(sys.modules[job['reduce_module']], job['reducer']) #map input_files = [ job['input_dir'] + '/' + file_ for file_ in os.listdir(job['input_dir']) ] print ','.join(input_files) sys.stdout.flush() ps = [] if job['map_num'] > len(input_files): job['map_num'] = len(input_files) file_num = len(input_files) / job['map_num'] if len( input_files ) % job['map_num'] == 0 else len(input_files) / job['map_num'] + 1 util.mkdir(job['map_output_dir'], delete_if_exist=True) for i in range(job['map_num']): p = mapper_inst(i, input_files[i * file_num:(i + 1) * file_num], job['map_output_dir'], job.get("map_params", {})) p.daemon = True p.start() ps.append(p) for p in ps: p.join() print 'map output_dir:', job['map_output_dir'] #reduce output_file = job.get('output_file', None) if reducer_inst: input_files = [ job['map_output_dir'] + '/' + file_ for file_ in os.listdir(job['map_output_dir']) ] reducer = reducer_inst(input_files, job['output_dir'], job.get("reduce_params", {}), output_file) reducer.run() print 'reduce output_dir: ', job['output_dir']
def forward_dataset(model, criterion, data_loader, opt, labels=None): sum_batch = 0 accuracy = list() avg_loss = list() for i, data in enumerate(data_loader): if opt.mode == "Train": if random.random() > opt.validate_ratio: continue if opt.mode == "Test": logging.info("test %s/%s image" %(i, len(data_loader))) sum_batch += 1 inputs, targets = data output, loss, loss_list = forward_batch(model, criterion, inputs, targets, opt, "Validate") batch_accuracy, batch_predictions = calc_accuracy(output, targets, opt.score_thres, opt.top_k) if opt.mode == "Test": save_test_path = os.path.join(opt.test_dir, "plot") util.mkdir(save_test_path) if labels != None: targets_str = ",".join([labels[ii]["__name__"] for ii, t in enumerate(list(map(lambda x: x.numpy()[0], targets))) if t == 1]) predict_str =",".join([labels[ii]["__name__"] for ii, t in enumerate(list(map(lambda x: x[0], batch_predictions))) if t == 1]) plt_title = f"Correct: {targets_str}, Predict: {predict_str}" plt.imshow(inputs[0].permute(1,2,0)) plt.title(plt_title) plt.savefig(os.path.join(save_test_path, f"test{i:09d}.png"), dpi=200, bbox_inches="tight", pad_inches=0.1) plt.clf() # accumulate accuracy if len(accuracy) == 0: accuracy = copy.deepcopy(batch_accuracy) for index, item in enumerate(batch_accuracy): for k,v in item.items(): accuracy[index][k]["ratio"] = v["ratio"] else: for index, item in enumerate(batch_accuracy): for k,v in item.items(): accuracy[index][k]["ratio"] += v["ratio"] # accumulate loss if len(avg_loss) == 0: avg_loss = copy.deepcopy(loss_list) else: for index, loss in enumerate(loss_list): avg_loss[index] += loss # average on batches for index, item in enumerate(accuracy): for k,v in item.items(): accuracy[index][k]["ratio"] /= float(sum_batch) for index in range(len(avg_loss)): avg_loss[index] /= float(sum_batch) return accuracy, avg_loss
def save_results(self, save_data, data_name='none', data_ext='jpg'): """Save the training or testing results to disk""" img_paths = self.get_image_paths() for i in range(save_data.size(0)): print('process image ...... %s' % img_paths[i]) short_path = ntpath.basename(img_paths[i]) # get image path name = os.path.splitext(short_path)[0] img_name = '%s_%s.%s' % (name, data_name, data_ext) util.mkdir(self.opt.results_dir) img_path = os.path.join(self.opt.results_dir, img_name) img_numpy = util.tensor2im(save_data[i].data) util.save_image(img_numpy, img_path)