def plot_mean_var_cl(fol, root_fol, subject, sms, run, labels_names, k_type='triangular', sim=False): # if not op.isfile(input_fname): # shutil.move(op.join(fol, 'mean_var_cl_{}.npz'.format(k_type)), input_fname) input_fname = op.join(fol, 'mean_var_cl{}_{}.npz'.format('_sim' if sim else '', k_type)) d = np.load(input_fname) mean_cl, var_cl, hs_tr, hs_ms = d['mean_cl'], d['var_cl'], d['hs_tr'], d['hs_ms'] for cl_vector, cl_name in zip([mean_cl + var_cl, mean_cl, var_cl], ['AIC', 'AIC_mean', 'AIC_var']): for label_ind in range(mean_cl.shape[0]): cl = cl_vector[label_ind] label_name = labels_names[label_ind] figs_fol = op.join(root_fol, 'figures', 'mean_var{}_cl'.format('_sim' if sim else ''), label_name, cl_name) utils.make_dir(figs_fol) fig_fname = op.join(figs_fol, '{}_{}_{}_{}.jpg'.format(subject, sms, cl_name, label_name)) # if op.isfile(fig_fname): # continue ind = np.arange(len(hs_ms)) if np.any(np.isnan(cl)): ind = ind[~np.isnan(cl)] cl = cl[ind] fig = plt.figure() width = 0.35 # plt.bar(ind, cl, width=width) plt.scatter(ind, cl, marker='o', facecolors='none') # plt.xticks(ind + width / 2, hs_ms[ind]) plt.xlim([-0.5, len(cl) + 0.5]) plt.title('{} {} {} {} {}{}'.format(cl_name, subject, label_name, sms, run, ' sim' if sim else '')) plt.xlabel('window-width (s)') # plt.text(ind[cl.argmin()], cl.min() * 0.97 + .03 * cl.max(), '*', fontsize=14) plt.scatter(ind[cl.argmin()], cl.min(), marker='o') plt.plot(ind, cl, '--') # utils.maximize_figure(plt) # plt.tight_layout() # plt.show() plt.savefig(fig_fname, dpi=100) plt.close()
def compare_vector_mean_var_cl(subject, sms, run, fol, root_fol, k_types=['triangular']): for k_type in k_types: # d = np.load(op.join(fol, 'vector_mean_cl_{}_{}-{}.npz'.format(k_type))) d = np.load(op.join(fol, 'vector_mean_cl_{}_{}-{}.npz'.format(k_type))) # vector_mean_cov_cl_triangular_mean_laus125 cl, hs_ms = d['mean_cl'], d['hs_ms'] d = np.load(op.join(fol, 'vector_mean_cl_sim_{}_{}-{}.npz'.format(k_type))) cl_sim, hs_ms_sim = d['mean_cl'], d['hs_ms'] fig = plt.figure() width = 0.35 ind = np.arange(len(hs_ms)) # plt.bar(ind, cl, width=width) plt.scatter(ind, cl, marker='o', facecolors='none', label='real data') plt.scatter(ind, cl_sim, marker='^', facecolors='none', label='simulated data') # plt.xticks(ind + width / 2, hs_ms) plt.title('{} {} {} {}{}'.format('AIC mean', subject, sms, run, ' sim' if sim else '')) plt.xlabel('window-width (s)') # plt.text(cl.argmin(), cl.min() * 0.97 + .03 * cl.max(), '*', fontsize=14) plt.legend() plt.scatter(cl.argmin(), cl.min(), marker='o') plt.scatter(cl_sim.argmin(), cl_sim.min(), marker='^') plt.plot(ind, cl, '--') plt.plot(ind, cl_sim, '--') plt.xlim([-0.5, len(cl) + 0.5]) utils.maximize_figure(plt) plt.tight_layout() # plt.show() figures_fol = op.join(root_fol, 'figures', 'cl_mean_comparison_figures') utils.make_dir(figures_fol) plt.savefig(op.join(figures_fol, 'vector_mean_cl_{}_{}_{}.jpg'.format(subject, sms, run)), dpi=200) plt.close()
def solve_labels_collision(subject, subjects_dir, atlas, backup_atlas, n_jobs=1): now = time.time() print('Read labels') labels = utils.read_labels_parallel(subject, subjects_dir, atlas, n_jobs) backup_labels_fol = op.join(subjects_dir, subject, 'label', backup_atlas) labels_fol = op.join(subjects_dir, subject, 'label', atlas) if op.isdir(backup_labels_fol): shutil.rmtree(backup_labels_fol) os.rename(labels_fol, backup_labels_fol) utils.make_dir(labels_fol) hemis_verts, labels_hemi, pia_verts = {}, {}, {} print('Read surface ({:.2f}s)'.format(time.time() - now)) for hemi in HEMIS: surf_fname = op.join(subjects_dir, subject, 'surf', '{}.pial'.format(hemi)) hemis_verts[hemi], _ = mne.surface.read_surface(surf_fname) labels_hemi[hemi] = [l for l in labels if l.hemi == hemi] print('Calc centroids ({:.2f}s)'.format(time.time() - now)) centroids = calc_labels_centroids(labels_hemi, hemis_verts) for hemi in HEMIS: print('Calc vertices labeling for {} ({:.2f}s)'.format(hemi, time.time() - now)) hemi_centroids_dist = cdist(hemis_verts[hemi], centroids[hemi]) vertices_labels_indices = np.argmin(hemi_centroids_dist, axis=1) labels_hemi_chunks = utils.chunks(list(enumerate(labels_hemi[hemi])), len(labels_hemi[hemi]) / n_jobs) params = [(labels_hemi_chunk, atlas, vertices_labels_indices, hemis_verts, labels_fol) for labels_hemi_chunk in labels_hemi_chunks] print('Save labels for {} ({:.2f}s)'.format(hemi, time.time() - now)) utils.run_parallel(_save_new_labels_parallel, params, n_jobs)
def inpaint_double_train(img_files): img_file_1, img_file_2, fold = img_files img_1 = cv2.imread(join(TRAIN_DIR, 'images', img_file_1), cv2.IMREAD_GRAYSCALE) trg_1 = cv2.imread(join(TRAIN_DIR, 'masks', img_file_1), cv2.IMREAD_GRAYSCALE) img_2 = cv2.imread(join(TRAIN_DIR, 'images', img_file_2), cv2.IMREAD_GRAYSCALE) trg_2 = cv2.imread(join(TRAIN_DIR, 'masks', img_file_2), cv2.IMREAD_GRAYSCALE) if not np.sum(img_1) or not np.sum(img_2): return img_1 = np.pad(img_1, PAD_WIDTH, mode='constant') trg_1 = np.pad(trg_1, PAD_WIDTH, mode='constant') img_2 = np.pad(img_2, PAD_WIDTH, mode='constant') trg_2 = np.pad(trg_2, PAD_WIDTH, mode='constant') img = np.concatenate([img_1, img_2], axis=1) trg = np.concatenate([trg_1, trg_2], axis=1) img = (inpaint_biharmonic(img, MASK_INPAINT) * 255).astype(np.uint8) trg = inpaint_biharmonic(trg, MASK_INPAINT) trg = np.where(trg > TARGET_THRESHOLD, 255, 0) img_file = img_file_1[:-4] + '_' + img_file_2[:-4] + '.png' make_dir(join(TRAIN_DIR, 'images' + SAVE_NAME, 'fold_%d' % fold)) make_dir(join(TRAIN_DIR, 'masks' + SAVE_NAME, 'fold_%d' % fold)) cv2.imwrite( join(TRAIN_DIR, 'images' + SAVE_NAME, 'fold_%d' % fold, img_file), img) cv2.imwrite( join(TRAIN_DIR, 'masks' + SAVE_NAME, 'fold_%d' % fold, img_file), trg) return img, trg, img_file
def plot_vector_mean_var(subject, sms, run, ys, names, label_ids, fol, tr, hs_plot, k_type='triangular', sim=False, overwrite=False, ax=None, plot_legend=True, xlim=None): from matplotlib.ticker import MultipleLocator, FormatStrFormatter from itertools import cycle majorLocator = MultipleLocator(100) majorFormatter = FormatStrFormatter('%d') minorLocator = MultipleLocator(50) d = np.load(op.join(fol, 'vector_mean_var{}_{}.npz'.format('_sim' if sim else '', k_type))) means, vars, hs_tr, hs_ms = d['means'], d['vars'], d['hs_tr'], d['hs_ms'] boynton_colors = cycle(["red", "green", "magenta", "yellow", "pink", "orange", "brown", "gray"]) t0 = int(2 * (max(hs_tr) - 1)) # t = range(ys.shape[1])[t0:] t = np.arange(ys.shape[1])[t0:] * tr / 1000 print(sms, tr, t[0], t[-1], ys.shape[1]) utils.make_dir(op.join(fol, 'pca_labels')) for y, label_name, label_ind in zip(ys, names, label_ids): fig_fname = op.join(fol, 'pca_labels', '{}_{}.jpg'.format(label_name, k_type)) # if op.isfile(fig_fname) and not overwrite: # continue if ax is None: fig, ax = plt.subplots() lines = [] l, = ax.plot(t, y[t0:], 'b', label='simulated' if sim else 'real') lines.append(l) for h_s in hs_plot: h_ind = np.where(np.array(hs_ms) == h_s)[0][0] h_ms = hs_ms[h_ind] color = next(boynton_colors) mean = means[h_ind, label_ind, t0:] error = np.power(vars[h_ind, label_ind, t0:], 0.5) l, = ax.plot(t, mean, color, label='w {:d}s'.format(h_ms)) lines.append(l) ax.fill_between(t, mean - error, mean + error, alpha=0.2, edgecolor=color, facecolor=color) if plot_legend: ax.legend(handles=lines, bbox_to_anchor=(1.05, 1.1)) ax.set_xlabel('Time (s)') if ax is None: ax.set_title('{} {} {} {}'.format(subject, sms, run, label_name)) else: ax.set_title('{}'.format(sms.replace('_', ' '))) utils.maximize_figure(plt) plt.tight_layout() ax.xaxis.set_major_locator(majorLocator) ax.xaxis.set_major_formatter(majorFormatter) ax.xaxis.set_minor_locator(minorLocator) if not xlim is None: ax.set_xlim(xlim) if ax is None: plt.savefig(fig_fname, dpi=200) plt.close()
def combine_mean_var_sim_plots(root_fol, labels_names): figure_fol = op.join(root_fol, 'figures') utils.make_dir(op.join(figure_fol, 'smss_per_label_compare')) now = time.time() for ind, label in enumerate(labels_names): utils.time_to_go(now, ind, len(labels_names), 5) figure_real_data = op.join(figure_fol, 'smss_per_label', '{}.png'.format(label)) figure_sim_data = op.join(figure_fol, 'smss_per_label_sim', '{}.png'.format(label)) new_image_fname = op.join(figure_fol, 'smss_per_label_compare', '{}.png'.format(label)) utils.combine_two_images(figure_real_data, figure_sim_data, new_image_fname)
def __init__(self, model_dir, max_to_keep=5): """ Save parameters of the model Args: dir: 모델을 저장할 폴더 max_to_keep: 모델의 저장할 갯수 (그 이상이면 가장 이전 것을 자동으로 지운다.) """ self.path = os.path.join(model_dir, 'model.ckpt') self.saver = tf.train.Saver(max_to_keep=max_to_keep, var_list=tf.get_collection( tf.GraphKeys.GLOBAL_VARIABLES)) # Folder 생성 utils.make_dir(model_dir)
def __init__(self, child_network_definition, CellGeneratorInstance, weight_dict, weight_directory="./", opt_loss='categorical_crossentropy', opt=SGD(lr=0.001, decay=1e-6, nesterov=True), opt_metrics=['accuracy']): self.child_network_definition = child_network_definition """ child_network_definition is like ["N","N","R","N","N","R"] where N is for normal and R for reduction """ self.CG = CellGeneratorInstance self.opt_loss = opt_loss self.opt = opt self.opt_metrics = opt_metrics self.model = self.generate_child_network() self.model_dict = self.generate_model_dict() self.weight_dict = weight_dict self.weight_directory = make_dir(weight_directory) self.graph = tf.get_default_graph()
def plot_mean_var(k_type='triangular', sim=False, labels_names=None, label_ids=None): from collections import defaultdict, OrderedDict import itertools xlim = None #[200, 300] figures_fol = op.join(root_fol, 'figures', 'smss_per_label{}{}'.format( '_window' if not xlim is None else '', '_sim' if sim else '')) utils.make_dir(figures_fol) gen, data = {}, {} hs_plot = [5, 10, 15, 25] if labels_names is None: labels_names = utils.load(op.join(root_fol, 'labels_names.pkl')) label_ids = range(len(labels_names)) for fol, subject, sms, run in utils.sms_generator(root_fol): print(fol, subject, sms, run) if subject not in gen: gen[subject] = OrderedDict() if sms not in gen[subject]: gen[subject][sms] = [] gen[subject][sms].append((fol, run)) if not sim: d = np.load(op.join(fol, 'labels_data_{}_{}.npz'.format(atlas, measure))) data[fol] = d['data'] else: import scipy.io as sio d_sim = sio.loadmat(op.join(fol, 'fmri_timecourse_sim.mat')) data[fol] = d_sim['timecourse_use_sim'].T # labels_names = [0] now = time.time() for subject, (label_id, label_name) in itertools.product(gen.keys(), zip(label_ids, labels_names)): utils.time_to_go(now, label_id, len(labels_names), 5) fig, axes = plt.subplots(2, 2, sharex='col', sharey='row', figsize=(12, 8)) axs = list(itertools.chain(*axes)) # fig.suptitle('{} {} {}'.format(subject, label_name, 'sim' if sim else '')) for index, (ax, sms) in enumerate(zip(axs, gen[subject].keys())): fol, run = gen[subject][sms][0] tr = utils.load(op.join(fol, 'tr.pkl')) print(tr) ys = np.array([data[fol][label_id]]) plot_vector_mean_var(subject, sms, run, ys, [label_name], [label_id], fol, tr, hs_plot, k_type, sim, overwrite=False, ax=ax, plot_legend=index==1, xlim=None) utils.maximize_figure(plt) plt.tight_layout() plt.savefig(op.join(figures_fol, label_name)) plt.close()
def pred_test_fold(model_path, fold): predictor = Predictor(model_path) prob_df = pd.read_csv(config.SAMPLE_SUBM_PATH) prob_df.rename(columns={'rle_mask': 'prob'}, inplace=True) fold_prediction_dir = join(PREDICTION_DIR, f'fold_{fold}', 'test') make_dir(fold_prediction_dir) for i, row in prob_df.iterrows(): image_path = join(config.TEST_DIR, 'images'+IMAGES_NAME, row.id + '.png') image = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE) segm, prob = predictor(image) row.prob = prob segm_save_path = join(fold_prediction_dir, row.id + '.png') cv2.imwrite(segm_save_path, segm) prob_df.to_csv(join(fold_prediction_dir, 'probs.csv'), index=False)
def generate(alignments: dict, output_dir='./') -> None: ''' Generate a summary text and json file for the alignments made Inputs: alignments: dict containing the alignments made. The key of each entry is the name of the file appended with scan number, and the values should be Alignments kwargs: output_dir: str path to the output directory. Default=./ Outputs: None ''' output_dir = make_valid_dir_string(output_dir) make_dir(output_dir) json_file(alignments, output_dir) tsv_file(alignments, output_dir)
def _init_save(self): fourcc = cv2.VideoWriter_fourcc(*'DIVX') video_dir = utils.abs_path('..', 'model_rec') utils.make_dir(video_dir) if self._base_opt.video_file_name_base: video_file_name = '{}_{}'.format( self._base_opt.name, self._base_opt.video_file_name_base) else: dt_str = utils.cur_datetime() video_file_name = 'source_{}_{:.0f}x{:.0f}_{:.0f}.avi'.format( dt_str, self._base_opt.width, self._base_opt.height, self._base_opt.fps) video_file_path = os.path.join(video_dir, video_file_name) print('Write to file', video_file_path) self._writer = cv2.VideoWriter( video_file_path, fourcc, self._base_opt.fps, (self._base_opt.width, self._base_opt.height), False)
def plot_vector_mean_cov(fol, root_fol, subject, sms, run, measure, atlas, k_type='triangular', sim=False, subplot=False, ax=None, write_x_label=True): input_fname = op.join(fol, 'vector_mean_cov_cl{}_{}_{}_{}.npz'.format('_sim' if sim else '', k_type, measure, atlas)) if not op.isfile(input_fname): print("Can't find {}!".format(input_fname)) return # print('Loading {}'.format(input_fname)) d = np.load(input_fname) # mean_cl, col_cl, cl, hs_tr = d['mean_cl'], d['cov_cl'], d['cl'], d['hs_tr'] mean_ll_cl, mean_pan_cl, cov_ll_cl, cov_pan_cl, hs_tr = d['mean_ll_cl'], d['mean_pan_cl'], d['cov_ll_cl'],\ d['cov_pan_cl'], d['hs_tr'] cl = mean_ll_cl + 1 * mean_pan_cl + cov_ll_cl + 1 * cov_pan_cl # for cl, cl_name in zip([mean_cl, col_cl, cl], ['AIC mean', 'AIC cov', 'AIC']): for cl, cl_name in zip([cl], ['AIC']): if not subplot: fig = plt.figure() ax = plt.subplot() ind = np.arange(len(hs_tr)) ax.scatter(ind, cl, marker='o', s=1, facecolors='none') if write_x_label: ax.set_xlabel('window-width (s)') # plt.text(cl.argmin(), cl.min() * 0.97 + .03 * cl.max(), '*', fontsize=14) cl_argmin, cl_min = np.nanargmin(cl), np.nanmin(cl) cl_min_exist = 0 < cl_argmin < len(cl) - 1 if cl_min_exist: ax.scatter(cl_argmin, cl_min, marker='o', s=30) ax.set_xlim([-0.5, len(cl) + 0.5]) ax.plot(ind, cl, '--') if subplot: ax.set_title('{} {}'.format(sms.replace('_', ' '), '({}s)'.format(cl_argmin) if cl_min_exist else '(no AIC min)')) if not subplot: title = '{} {} {} {} {} {}{} {}'.format( subject, measure, atlas, cl_name, sms, run, ' sim' if sim else '', 'best window length: {}s'.format(cl_argmin) if cl_min_exist else '(no AIC min)') print(title) plt.title(title) utils.maximize_figure(plt) plt.tight_layout() figures_fol = op.join(root_fol, 'figures', 'mean_cov_figures{}'.format('_sim' if sim else '')) utils.make_dir(figures_fol) figure_fname = op.join(figures_fol, 'vector_{}_{}_{}_{}{}.jpg'.format( cl_name.replace(' ', '_'), subject, run, sms, '_sim' if sim else '')) print('Saving the figure {}'.format(figure_fname)) plt.savefig(figure_fname, dpi=200) plt.close()
def run(opt): path_pairs = [] src_path = os.path.abspath(opt.src) dst_path = os.path.abspath(opt.dst) utils.make_dir(dst_path) if os.path.isfile(src_path): dir_path, file_name = os.path.split(src_path) src_files = [(os.path.join(dir_path, file_name), file_name)] else: src_files = utils.list_files(src_path) for src_file_path, src_file_name in src_files: dst_file_name = os.path.splitext(src_file_name)[0] + '.gif' dst_file_path = os.path.join(dst_path, dst_file_name) path_pairs.append((src_file_path, dst_file_path)) for fpath_src, fpath_dst in path_pairs: cmd = 'ffmpeg -y -i {} {}'.format(fpath_src, fpath_dst) print('Exec:', cmd) subprocess.run(cmd)
def store_spoof_data(data, del_action, add_action, spoof_col_name, source): """ Parameters ---------- data del_action add_action spoof_col_name source """ imputed_data = spoofing_imputation(data, del_action, add_action, spoof_col_name) ticker_exchange = str(imputed_data['Ticker'].iloc[0]) + '_' + str( imputed_data['Exchange'].iloc[0]) + '.csv' file_name = source / str(imputed_data['Date'].iloc[0]) / ticker_exchange utils.make_dir(file_name) with open(file_name, "w+") as f: imputed_data.to_csv(f, index=False)
def pred_val_fold(model_path, fold): predictor = Predictor(model_path) folds_df = pd.read_csv(TRAIN_FOLDS_PATH) fold_df = folds_df[folds_df.fold == fold] fold_prediction_dir = join(PREDICTION_DIR, f'fold_{fold}', 'val') make_dir(fold_prediction_dir) prob_dict = {'id': [], 'prob': []} for i, row in fold_df.iterrows(): image = cv2.imread(row.image_path, cv2.IMREAD_GRAYSCALE) segm, prob = predictor(image) segm_save_path = join(fold_prediction_dir, row.id + '.png') cv2.imwrite(segm_save_path, segm) prob_dict['id'].append(row.id) prob_dict['prob'].append(prob) prob_df = pd.DataFrame(prob_dict) prob_df.to_csv(join(fold_prediction_dir, 'probs.csv'), index=False)
def to_jpg(exrfile, jpgfile): ''' exrfile을 jpgfile로 변환합니다. ''' utils.make_dir(os.path.dirname(jpgfile)) color_ch = channel_list[:3] File = OpenEXR.InputFile(exrfile) PixType = Imath.PixelType(Imath.PixelType.FLOAT) DW = File.header()['dataWindow'] Size = (DW.max.x - DW.min.x + 1, DW.max.y - DW.min.y + 1) rgb = [np.fromstring(File.channel(c, PixType), dtype=np.float32) \ for c in color_ch] for i in range(len(color_ch)): rgb[i] = np.where(rgb[i] <= 0.0031308, (rgb[i] * 12.92) * 255.0, (1.055 * (rgb[i]**(1.0 / 2.4)) - 0.055) * 255.0) rgb8 = [Image.frombytes("F", Size, c.tostring()).convert("L") for c in rgb] Image.merge("RGB", rgb8).save(jpgfile, "JPEG", quality=95)
def write(path, data): assert data.ndim == 3, "In write(), data ndim must be 3, but " + str( data.shape) utils.make_dir(os.path.dirname(path)) # 어차피 color외에는 보지도 않으니 color만 저장 ch_list = channel_list[:3] h, w, _ = data.shape header = OpenEXR.Header(w, h) header['compression'] = Imath.Compression( Imath.Compression.PIZ_COMPRESSION) header['channels'] = {c: Imath.Channel(_ch_precision(c)) for c in ch_list} out = OpenEXR.OutputFile(path, header) ch_data = {ch: data[:, :, index].\ astype(_ch_np_precision(ch_list[index])).tostring() for index, ch in enumerate(ch_list)} out.writePixels(ch_data)
def main(argc): option = build_option() global_step = tf.train.get_or_create_global_step() denoising_model = eval("model." + option.model_name)( n_input_channel=option.n_input_channel, n_output_channel=option.n_output_channel, selected_loss=option.loss, start_lr=option.start_lr, lr_decay_step=option.lr_decay_step, lr_decay_rate=option.lr_decay_rate, global_step=global_step) config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: # Model write model.write(sess, option.text_dir + '/model.txt') # Saver # ========================================================================== saver = Saver( os.path.join(option.checkpoint_dir, option.experiment_name)) tf.global_variables_initializer().run() saver.restore(sess) # ========================================================================== # Saver # ========================================================================== train_dir = os.path.join(option.logs_dir, option.experiment_name, 'train') valid_dir = os.path.join(option.logs_dir, option.experiment_name, 'valid') utils.make_dir(train_dir) utils.make_dir(valid_dir) train_writer = tf.summary.FileWriter(train_dir, sess.graph) valid_writer = tf.summary.FileWriter(valid_dir, sess.graph) tf.summary.scalar('loss', denoising_model.loss) tf.summary.scalar('learning_rate', denoising_model.lr) merged_summary = tf.summary.merge_all() # ========================================================================== # valid data # ========================================================================== valid_noisy_img, valid_reference = \ next_batch_tensor( tfrecord_path = option.tfrecord_valid_path, shape = [option.img_height, option.img_width, option.n_input_channel, option.n_output_channel], repeat = 10000 ) # ========================================================================== for epoch in range(option.n_epoch): train_noisy_img, train_reference = \ next_batch_tensor( tfrecord_path = option.tfrecord_train_path, shape = [option.patch_size, option.patch_size, option.n_input_channel, option.n_output_channel], batch_size = option.batch_size, shuffle_buffer= option.shuffle_buffer, prefetch_size = option.prefetch_size, ) while True: # 더이상 읽을 것이 없으면 빠져 나오고 다음 epoch으로 try: noisy_img, reference = sess.run( [train_noisy_img, train_reference]) except tf.errors.OutOfRangeError as e: print("Done") break step = sess.run(global_step) denoising_model.optimize(sess, noisy_img, reference) # 일정 주기마다 logging if step % option.log_period == 0: loss_val = denoising_model.get_loss( sess, noisy_img, reference) print(f'epoch {epoch}, step {step}] loss : {loss_val}') summary = sess.run(merged_summary, feed_dict={ denoising_model.noisy_img: noisy_img, denoising_model.reference: reference }) train_writer.add_summary(summary, step) denoised_img = denoising_model.get_pred( sess, noisy_img, reference) for b in range(option.n_save_patch_img): exr.save_debug_img( option.debug_image_dir + f'/{option.experiment_name}' + f'/train/denoised/{step}_{b}', denoised_img[b, :, :, :]) exr.save_debug_img( option.debug_image_dir + f'/{option.experiment_name}' + f'/train/noisy/{step}_{b}', noisy_img[b, :, :, :]) exr.save_debug_img( option.debug_image_dir + f'/{option.experiment_name}' + f'/train/refer/{step}_{b}', reference[b, :, :, :]) # 일정주기마다 모델 저장 if step % option.save_period == 0: saver.save(sess, step) if step % option.valid_period == 0: print( "=====================================================================" ) noisy_img, reference = sess.run( [valid_noisy_img, valid_reference]) loss_val, denoised_img, summary = \ sess.run([denoising_model.loss, denoising_model.denoised_img, merged_summary], feed_dict={denoising_model.noisy_img:noisy_img, denoising_model.reference:reference}) print(" Test ] loss ", loss_val) valid_writer.add_summary(summary, step) debug_dir = os.path.join(option.debug_image_dir, option.experiment_name, 'valid') exr.save_debug_img(debug_dir + f'/valid/{step}', denoised_img[0, :, :, :]) print( "=====================================================================" ) tf.reset_default_graph()
tensor = torch.stack(tensors, dim=0) tensor = tensor.to(self.model.device) country_tensor = torch.stack(country_tensors, dim=0) country_tensor = country_tensor.to(self.model.device) with torch.no_grad(): probs = self.model.predict((tensor, country_tensor)) return probs if __name__ == "__main__": print("Model", MODEL_PATH) print("Prediction folder", PREDICT_DIR) make_dir(PREDICT_DIR) draw_transform = DrawTransform(DRAW_SIZE, DRAW_PAD, DRAW_LINE_WIDTH, TIME_COLOR) image_trns = ImageTransform(False, SCALE_SIZE) test_df = pd.read_csv(config.TEST_SIMPLIFIED_PATH) sample_subm = pd.read_csv(config.SAMPLE_SUBMISSION) predictor = Predictor(MODEL_PATH, draw_transform, image_trns) drawings = [] key_ids = [] pred_words = [] pred_key_ids = [] probs_lst = [] for i, row in tqdm.tqdm(test_df.iterrows(), total=test_df.shape[0]): drawing = eval(row.drawing)
def main(_): # Denoising Model. src\base_model.py과 src\model.py를 참조할 것 net = DenoisingNet(input_shape=[None, None, INPUT_CH], output_shape=[None, None, OUTPUT_CH], loss_func='L1', start_lr=1e-6, lr_decay_step=10000, lr_decay_rate=1.0) sess = tf.Session() # debug용 폴더들 생성 utils.make_dir(TRAIN_SUMMARY_DIR) utils.make_dir(VALID_SUMMARY_DIR) utils.make_dir(CKPT_DIR) utils.make_dir(NOISY_IMAGE_DIR) utils.make_dir(REFER_IMAGE_DIR) utils.make_dir(DENOISED_IMG_DIR) # Saver # ========================================================================= saver = tf.train.Saver() print('Saver initialized') recent_ckpt_job_path = tf.train.latest_checkpoint(CKPT_DIR) if recent_ckpt_job_path is None: sess.run(tf.global_variables_initializer()) print("Initializing variables...") else: saver.restore(sess, recent_ckpt_job_path) print("Restoring...", recent_ckpt_job_path) # ========================================================================= # Summary # ========================================================================= train_writer = tf.summary.FileWriter(TRAIN_SUMMARY_DIR, sess.graph) valid_writer = tf.summary.FileWriter(VALID_SUMMARY_DIR, sess.graph) # ========================================================================= # 검증용 데이터 tensor 생성 src\dataset 참조 # ========================================================================= valid_noisy_img, valid_reference = \ next_batch_tensor( tfrecord_path = VALID_TFRECORD, shape = [IMG_HEIGHT, IMG_WIDTH, N_NOISY_FEATURE, N_REFER_FEATURE], repeat = 1000 ) # ========================================================================= #merged_summary = tf.summary.merge_all() lst = [] # 학습 시작 Epoch for epoch in count(): # 학습용 데이터 tensor 생성 src\dataset 참조 train_noisy_img, train_reference = \ next_batch_tensor( tfrecord_path = TRAIN_TFRECORD, shape = [PATCH_SIZE, PATCH_SIZE, N_NOISY_FEATURE, N_REFER_FEATURE], batch_size = BATCH_SIZE, shuffle_buffer = BATCH_SIZE * 2, ) # 학습용 데이터 다 읽을 때 까지 while True: # 더이상 읽을 것이 없으면 빠져 나오고 다음 epoch으로 try: # 학습용 데이터를 위에 설정한 베치사이즈만큼 가져온다. noisy_img, reference = sess.run( [train_noisy_img, train_reference]) except tf.errors.OutOfRangeError as e: print("Done") break # 데이터로 학습 _, step, lr = sess.run([net.train_op, net.global_step, net.lr], feed_dict={ net.inputs: noisy_img, net.refers: reference }) # 일정 if step % LOG_PERIOD == LOG_PERIOD - 1: loss = sess.run([net.loss], feed_dict={ net.inputs: noisy_img, net.refers: reference }) # train_writer.add_summary(summary, step + 1) print(f'epoch:{epoch:2d} step:{step+1} loss:{loss}') if step % SAVE_PERIOD == SAVE_PERIOD - 1: saver.save(sess, os.path.join(CKPT_DIR, "model.ckpt"), global_step=step + 1) if step % VALID_PERIOD == VALID_PERIOD - 1: noisy_img, reference = sess.run( [valid_noisy_img, valid_reference]) loss, denoised_img = sess.run([net.loss, net.outputs], feed_dict={ net.inputs: noisy_img, net.refers: reference }) print(" Test ] Loss ", loss) #valid_writer.add_summary(summary, step + 1) rp = REFER_IMAGE_DIR + f'{step+1}' dp = DENOISED_IMG_DIR + f'{step+1}' np = NOISY_IMAGE_DIR + f'{step+1}' write_exr(reference[0, :, :, :3], rp + ".exr") write_exr(denoised_img[0, :, :, :3], dp + ".exr") write_exr(noisy_img[0, :, :, :3], np + ".exr") to_jpg(rp + ".exr", rp + ".jpg") to_jpg(dp + ".exr", dp + ".jpg") to_jpg(np + ".exr", np + ".jpg") os.remove(rp + ".exr") os.remove(dp + ".exr") os.remove(np + ".exr") sess.close() tf.reset_default_graph()
def main(config): CASE_NUM = config['case_num'] DATASET = config['dataset'] NORMALIZATION = config['normalization'] BATCH_SIZE = config['batch_size'] MAX_EPOCH = config['max_epoch'] OPTIM_TYPE = config['optimizer'] LR = config['learning_rate'] LR_STEP = config['lr_step'] LR_DECAY = config['lr_decay'] L2_DECAY = config['l2_decay'] TB_STATE = config['use_tensorboard'] MODEL_NAME = config['model_name'] ALPHA = config['alpha'] BETA = config['beta'] GAMMA = config['gamma'] PHI = config['phi'] LOSS_FN = config['loss_fn'] KERNEL_SIZE = config['kernel_size'] result_dir = make_dir(RESULT_ROOT_DIR, str(CASE_NUM), overwrite=args.overwrite) ckpt_path = result_dir + '/' + 'checkpoint.pt' # =============================================== Select data and construct data_fname, data_dim = select_data(DATASET) data_path = '../data/' + data_fname data_train = NLUDataset(data_path, mode='train', normalization=NORMALIZATION, random_seed=42) dataloader_train = DataLoader(data_train, batch_size=BATCH_SIZE, shuffle=True, num_workers=4) data_valid = NLUDataset(data_path, mode='valid', normalization=NORMALIZATION, random_seed=42) dataloader_valid = DataLoader(data_valid, batch_size=BATCH_SIZE, shuffle=True, num_workers=4) data_test = NLUDataset(data_path, mode='test', normalization=NORMALIZATION, random_seed=42) dataloader_test = DataLoader(data_test, batch_size=BATCH_SIZE, shuffle=True, num_workers=4) num_train_samples = data_train.__len__() classes = data_train.labels num_classes = len(classes) # =============================================== Initialize model and optimizer device = ('cuda' if torch.cuda.is_available() else 'cpu') if device == 'cuda': print('Using GPU, %s' % torch.cuda.get_device_name(0)) net = select_model(MODEL_NAME, data_dim, KERNEL_SIZE, num_classes, ALPHA, BETA, PHI) net.to(device) loss_fn = select_loss(LOSS_FN) optimizer = select_optimizer(OPTIM_TYPE, net.parameters(), LR, L2_DECAY) scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=LR_STEP, gamma=LR_DECAY) # =============================================== Train it = 0 train_losses, valid_losses, valid_accs = {}, {}, {} best_validation_acc = 0 log_term = 5 for epoch in range(MAX_EPOCH): #------------------------------------------------ One epoch start one_epoch_start = time.time() print('Epoch {} / Learning Rate: {:.0e}'.format( epoch, scheduler.get_lr()[0])) #------------------------------------------------ Train train_losses, it, net, optimizer, scheduler \ = train_1epoch(dataloader_train, device, train_losses, it, net, loss_fn, optimizer, scheduler, log_every=log_term) #------------------------------------------------ Validation valid_acc, valid_loss = evaluate(dataloader_valid, device, net, loss_fn) valid_losses[it] = valid_loss valid_accs[it] = valid_acc #------------------------------------------------ Save model saved = '' if valid_acc > best_validation_acc: best_validation_acc = valid_acc saved = save_ckpt(ckpt_path, net, best_validation_acc) print('Epoch {} / Valid loss: {:.4f}, Valid acc: {:.4f} {}'.format( epoch, valid_loss, valid_acc, saved)) #------------------------------------------------ One epoch end curr_time = time.time() print("One epoch time = %.2f s" % (curr_time - one_epoch_start)) print('#------------------------------------------------------#') save_train_log(result_dir, train_losses, valid_losses, valid_accs, best_validation_acc) # =============================================== Test net, best_validation_acc = load_ckpt(ckpt_path, net) test_acc, test_loss = evaluate(dataloader_test, device, net, loss_fn) return test_acc
from collections import defaultdict from itertools import combinations from random import shuffle from src.utils import make_dir from src import config from src.config import TRAIN_DIR ORIG_SIZE = (101, 101) SAVE_SIZE = (109, 109) SAVE_NAME = '_double_109' TRAIN_FOLDS_PATH = '/workdir/data/train_folds_148.csv' TARGET_THRESHOLD = 0.7 N_WORKERS = mp.cpu_count() make_dir(join(TRAIN_DIR, 'images' + SAVE_NAME)) make_dir(join(TRAIN_DIR, 'masks' + SAVE_NAME)) diff = (SAVE_SIZE - np.array(ORIG_SIZE)) pad_left = diff // 2 pad_right = diff - pad_left PAD_WIDTH = ((pad_left[0], pad_right[0]), (pad_left[0], pad_right[1])) print('Pad width:', PAD_WIDTH) MASK_INPAINT = np.zeros(ORIG_SIZE, dtype=np.uint8) MASK_INPAINT = np.pad(MASK_INPAINT, PAD_WIDTH, mode='constant', constant_values=255) MASK_INPAINT = np.concatenate([MASK_INPAINT, MASK_INPAINT], axis=1)
def copy_figures(subject, sms, run, fol, root_fol, label_name, k_type='triangular'): utils.make_dir(op.join(root_fol, 'mean_var_figures')) shutil.copy(op.join(fol, 'pca_labels', '{}_{}.jpg'.format(label_name, k_type)), op.join(root_fol, 'mean_var_figures', '{}_{}_{}_{}_{}.jpg'.format(subject, sms, run, label_name, k_type)))
FOLD_DIRS = [join(p, 'fold_%d' % f) for p in PREDICTION_DIRS for f in FOLDS] PREDICTION_DIRS = [ '/workdir/data/predictions/fpn-lovasz-se-resnext50-006-after-001/', ] FOLDS = [0, 1, 2, 3, 4] FOLD_DIRS += [join(p, 'fold_%d' % f) for p in PREDICTION_DIRS for f in FOLDS] segm_thresh = 0.4 prob_thresh = 0.5 SAVE_NAME = 'mean-005-0.4' MEAN_PREDICTION_DIR = f'/workdir/data/predictions/{SAVE_NAME}' make_dir(join(MEAN_PREDICTION_DIR, 'masks')) def get_mean_probs_df(): probs_df_lst = [] for fold_dir in FOLD_DIRS: probs_df = pd.read_csv(join(fold_dir, 'test', 'probs.csv'), index_col='id') probs_df_lst.append(probs_df) mean_probs_df = probs_df_lst[0].copy() for probs_df in probs_df_lst[1:]: mean_probs_df.prob += probs_df.prob mean_probs_df.prob /= len(probs_df_lst) return mean_probs_df
def build_option(): parser = ArgumentParser() parser.register('type', 'bool', _bool) parser.add_argument('-ex', '--experiment_name', type=str, default='test_1', help='실험 이름. 이것에 따라 log와 checkpoint 이름 결정') # Model # ============================================================================ parser.add_argument('-m', '--model_name', type=str, default="VGGNet", help='model 이름(VGGNet, ResNet') parser.add_argument('-ls', '--loss', type=str, default="L1", help='model 학습할 때 쓰일 loss') parser.add_argument('-sl', '--start_lr', type=float, default=0.001, help='model 학습할 때 처음 learning rate 값') parser.add_argument('-lds', '--lr_decay_step', type=int, default=4000, help='learning rate decay할 주기(step)') parser.add_argument('-ldr', '--lr_decay_rate', type=float, default=0.8, help='learning rate decay 비율') # ============================================================================ # Path # ============================================================================ parser.add_argument('-l', '--logs_dir', type=str, default='./logs', help='tensorflow model log를 저장할 폴더') parser.add_argument('-c', '--checkpoint_dir', type=str, default='./checkpoint', help='model의 paramter를 저장할 폴더') parser.add_argument('-t', '--tfrecord_train_path', type=str, default='./data/tfrecord/train_40.tfrecord', help="train set을 읽을 tfrecord 파일 이름과 경로") parser.add_argument('-e', '--tfrecord_test_path', type=str, default='./data/tfrecord/test.tfrecord', help="test set을 읽을 tfrecord 파일 이름과 경로") parser.add_argument('-v', '--tfrecord_valid_path', type=str, default='./data/tfrecord/valid.tfrecord', help="valid set을 읽을 tfrecord 파일 이름과 경로") parser.add_argument('-d', '--debug_image_dir', type=str, default='./debug/image', help="학습하면서 나오는 이미지들을 저장하는 폴더") # ============================================================================ # Dataset # ============================================================================ parser.add_argument('-nic', '--n_input_channel', type=int, default=66, help='이미지가 가지고 있는 채널 수') parser.add_argument('-noc', '--n_output_channel', type=int, default=3, help='이미지가 가지고 있는 채널 수') parser.add_argument('-wi', '--img_width', type=int, default=1280, help='이미지 가로 크기') parser.add_argument('-he', '--img_height', type=int, default=720, help='이미지 세로 크기') parser.add_argument('-p', '--patch_size', type=int, default=40, help="학습용 패치 크기") parser.add_argument('-b', '--batch_size', type=int, default=32, help="minibatch 크기") parser.add_argument('-sf', '--shuffle_buffer', type=int, default=100, help="tfrecord 파일을 읽을 때 섞기 위한 shuffle buffer 크기") parser.add_argument('-ps', '--prefetch_size', type=int, default=1, help="미리 읽는 거라는데 잘 모르겠음") # ============================================================================ # Training # ============================================================================ parser.add_argument('-ep', '--n_epoch', type=int, default=1000000000, help='epoch 수. 보통은 그냥 크게!') parser.add_argument('-lp', '--log_period', type=int, default=100, help='일정 주기마다 logging(print, log)') parser.add_argument('-vp', '--valid_period', type=int, default=3000, help='일정 주기마다 모델 parameter 저장') parser.add_argument('-sp', '--save_period', type=int, default=5000, help='일정 주기마다 모델 parameter 저장') parser.add_argument('-nsp', '--n_save_patch_img', type=int, default=2, help='학습 중 저장할 patch 개수') # ============================================================================ option = parser.parse_args() option.text_dir = os.path.join(option.debug_image_dir, option.experiment_name) make_dir(option.text_dir) f = open(option.text_dir + '/option.txt', 'w') for op in vars(option): f.write(op + ': ' + str(getattr(option, op)) + '\n') f.close() return option
def set_args(args) -> dict: # check if we use the params file or the user arguments use_params = stringtobool(args.config) config = Config() spectra_folder = args.spectra_folder if not use_params else config[ 'spectra_dir'] database_file = args.database_file if not use_params else config[ 'database_file'] output_dir = args.output_dir if not use_params else config['output_dir'] min_peptide_len = args.min_peptide_len if not use_params else config[ 'min_peptide_len'] max_peptide_len = args.max_peptide_len if not use_params else config[ 'max_peptide_len'] ppm_tolerance = args.tolerance if not use_params else config[ 'ppm_tolerance'] precursor_tolerance = args.precursor_tolerance if not use_params else config[ 'precursor_tolerance'] verbose = stringtobool( args.verbose) if not use_params else config['verbose'] peak_filter = args.peak_filter if not use_params else config['num_peaks'] relative_abundance_filter = args.rel_abund_filter if not use_params else config[ 'relative_abundance'] digest = args.digest if not use_params else config['digest'] cores = args.cores if not use_params else config['cores'] n = args.n if not use_params else config['top_results'] debug = config['debug'] truth_set = config['truth_set'] ############## Argument checking ################ if not utils.is_dir(spectra_folder): print( f'Error: {spectra_folder} is not a real path. Path to directory with spectra files is necessary.' ) sys.exit(0) if not utils.is_fasta(database_file) or not utils.is_file(database_file): print( f'Error: {database_file} is not a valid .fasta file. .fasta file needed.' ) sys.exit(0) # make the output directory output_dir = utils.make_valid_dir_string(output_dir) utils.make_dir(output_dir) return { 'spectra_folder': spectra_folder, 'database_file': database_file, 'output_dir': output_dir, 'min_peptide_len': min_peptide_len, 'max_peptide_len': max_peptide_len, 'tolerance': ppm_tolerance, 'precursor_tolerance': precursor_tolerance, 'verbose': verbose, 'peak_filter': peak_filter, 'relative_abundance_filter': relative_abundance_filter, 'digest': digest, 'DEBUG': debug, 'cores': cores, 'n': n, 'truth_set': truth_set }
from src.transforms import SimpleDepthTransform, SaltTransform, CenterCrop from src.argus_models import SaltMetaModel from src.transforms import HorizontalFlip from src.utils import RLenc, make_dir from src import config EXPERIMENT_NAME = 'mos-fpn-lovasz-se-resnext50-001' FOLDS = list(range(config.N_FOLDS)) ORIG_IMAGE_SIZE = (101, 101) PRED_IMAGE_SIZE = (128, 128) TRANSFORM_MODE = 'crop' TRAIN_FOLDS_PATH = '/workdir/data/train_folds_148_mos_emb_1.csv' IMAGES_NAME = '148' FOLDS_DIR = f'/workdir/data/experiments/{EXPERIMENT_NAME}' PREDICTION_DIR = f'/workdir/data/predictions/{EXPERIMENT_NAME}' make_dir(PREDICTION_DIR) SEGM_THRESH = 0.5 PROB_THRESH = 0.5 class Predictor: def __init__(self, model_path): self.model = load_model(model_path) self.model.nn_module.final = torch.nn.Sigmoid() # self.model.nn_module.eval() self.depth_trns = SimpleDepthTransform() self.crop_trns = CenterCrop(ORIG_IMAGE_SIZE) self.trns = SaltTransform(PRED_IMAGE_SIZE, False, TRANSFORM_MODE)
def compare_mean_var_cl(fol, root_fol, subject, sms, run, labels_names, hs, top_hs=-1, labels_ids=None, hs_sim=None, k_type='triangular'): from scipy.signal import argrelextrema input_fname = op.join(fol, 'mean_var_cl_{}_{}-{}.npz'.format(k_type, hs[0], hs[1])) if not op.isfile(input_fname): print('{} does not exist!'.format(input_fname)) return d = np.load(input_fname) utils.print_modif_time(input_fname) # cl_vector = d['mean_cl'] + d['var_cl'] hs_tr, hs_ms = d['hs_tr'], d['hs_ms'] if hs_sim is None: hs_sim = hs sim_fname = op.join(fol, 'mean_var_sim_cl_{}_{}-{}.npz'.format(k_type, hs_sim[0], hs_sim[1])) if op.isfile(sim_fname): d_sim = np.load(sim_fname) utils.print_modif_time(sim_fname) no_sim_data = False else: print("Can't find sim data!") d_sim = np.load(input_fname) no_sim_data = True # cl_vector_sim = d_sim['mean_cl'] + d_sim['var_cl'] hs_tr_sim, hs_ms_sim = d['hs_tr'], d['hs_ms'] # cl_name = 'AIC' # fig, axes = plt.subplots(2, 2, sharex='col', sharey='row', figsize=(12, 8)) # axs = list(itertools.chain(*axes)) root_figs_fol = op.join(root_fol, 'figures', subject, 'mean_var_cl_compare_{}_{}'.format(hs[0], top_hs))# hs[-1])) utils.make_dir(root_figs_fol) if labels_ids is None: labels_ids = range(len(labels_names)) for cl_name, cl_vector, cl_vector_sim in zip(['AIC', 'AIC_mean', 'AIC_var'], [d['mean_cl'] + d['var_cl'], d['mean_cl'], d['var_cl']], [d_sim['mean_cl'] + d_sim['var_cl'], d_sim['mean_cl'], d_sim['var_cl']]): for ind, label_ind in enumerate(labels_ids): fig = plt.figure() cl_real = cl_vector[label_ind] cl_sim = cl_vector_sim[label_ind] label_name = labels_names[ind] figs_fol = op.join(root_figs_fol, label_name, cl_name) utils.make_dir(figs_fol) fig_fname = op.join(figs_fol, '{}_{}_{}_{}_{}.jpg'.format(subject, sms, run, label_name, cl_name)) # if op.isfile(fig_fname): # continue # ind_real = np.arange(len(hs_ms)) # ind_sim = np.arange(len(hs_ms_sim)) ind_real = hs_ms[:top_hs] ind_sim = hs_ms_sim[:top_hs] # print('ind real len: {}, ind sim len: {}'.format(len(ind_real), len(ind_sim))) # print('cl real shape: {}, cl sim shape: {}'.format(cl_real.shape, cl_sim.shape)) lines = [] if no_sim_data: itr = zip([cl_real], ['o'], [ind_real], ['real'], ['b']) else: itr = zip([cl_real, cl_sim], ['o', '^'], [ind_real, ind_sim], ['real', 'sim'], ['b', 'g']) for cl, marker, ind, label, c in itr: # for cl, marker, ind, label in zip([cl_real], ['o'], [ind_real], ['real']): cl = cl[:top_hs] if np.any(np.isnan(cl)): ind = ind[~np.isnan(cl)] cl = cl[~np.isnan(cl)] # plt.bar(ind, cl, width=width) # plt.scatter(ind, cl, marker=marker, facecolors='none', label=label) # for cl, ind, c, label in zip([cl_real, cl_sim], [ind_real, ind_sim], ['b', 'g'], ['real', 'sim']): # for cl, ind in zip([cl_real], [ind_real]): # plt.text(ind[cl.argmin()], cl.min() * 0.97 + .03 * cl.max(), '*', fontsize=14) # plt.scatter(ind[cl.argmin()], cl.min(), marker='o') ind = ind[:len(cl)] l, = plt.plot(ind, cl, '--', label=label) lines.append(l) # plt.legend() minm = argrelextrema(cl, np.less) # (array([2, 5, 7]),) for cl_min in minm[0]: plt.scatter(ind[cl_min], cl[cl_min], marker='o', c=c) # plt.xlim([-0.5, len(cl_real) + 0.5]) plt.legend(handles=lines) plt.xlim([-0.5, max(ind_real) + 0.5]) # plt.title('{} {} {} {} {}'.format(cl_name, subject, label_name, sms, run)) plt.title('{}'.format(sms.replace('_', ' '))) plt.ylabel(cl_name) plt.xlabel('window-width (s)') # utils.maximize_figure(plt) # plt.tight_layout() # plt.show() print('Saving figure {}'.format(fig_fname)) plt.savefig(fig_fname, dpi=100) plt.close()
def main(): # ==================== # Run # ==================== outDir = make_dir(os.path.join(os.path.dirname(__file__), "output")) modelDir = make_dir(os.path.join(outDir, "models")) aniDir = make_dir(os.path.join(outDir, "animations")) logFile = os.path.join(outDir, "log.csv") env = Environment(n_good=0, n_bad=110) agent = DQNAgent(env=env, sensors=Sensors(n_sectors=4, sector_radius=1.0)) save_models = True save_animations = True n_episodes = 1000 n_iter_max = 2000 n_reward_max = 0 loss = -1 # track training loss for episode in range(n_episodes): # Generate episode state = env.reset() # initialize environment obs = agent.observe(state, env) # observe states, rewards, i = [], [], 0 while i < n_iter_max: action = agent.get_action(obs) # follow epsilon-greedy policy state_next, reward, done = env.step(state, action) # evolve obs_next = agent.observe(state, env) # observe agent.memorize((obs, action, reward, obs_next, done)) # memorize rewards.append(reward) states.append(state_next) state, obs = state_next, obs_next # transition i += 1 if done: break # terminate # Track highly successful episodes n_reward_max += (sum(rewards) >= 2000) # Print progress print( "[ep {}/{}] iter={}/{}, rew={:.0f}, nrewmax={}, mem={}, eps={:.3f}, loss={:.2f}" .format(episode + 1, n_episodes, i, n_iter_max, sum(rewards), n_reward_max, len(agent.memory), agent.epsilon, loss), flush=True) if (episode == 0 or n_reward_max % 5 == 1): # Save model if save_models: modelFile = os.path.join( modelDir, "model_ep={}_rew={}.h5".format(episode + 1, int(sum(rewards)))) print(" -> saving agent model = {}".format(modelFile), flush=True) agent.save_model(modelFile) # Save animation if save_animations: aniFile = os.path.join( aniDir, "ani_ep={}_rew={}.mp4".format(episode + 1, int(sum(rewards)))) print(" -> saving animation = {}".format(aniFile), flush=True) env.make_animation(states, env, agent.sensors, save_ani=True, filename=aniFile) n_reward_max += 0 if (episode == 0) else 1 # Train agent loss = agent.train() # automatically adjusts epsilon # Save log header = ["episode", "iter", "reward", "loss", "epsilon", "time"] values = [ episode + 1, i, sum(rewards), loss, agent.epsilon, datetime.datetime.now().strftime("%B %d %Y %I:%M:%S %p") ] with open(logFile, ('w' if episode == 0 else 'a')) as f: writer = csv.writer(f) if episode == 0: writer.writerow(header) writer.writerow(values) if (episode + 1) % 20 == 0 or (episode == n_episodes - 1): df = pd.read_csv(logFile) if df.shape[0] > 50: save_plot(["episode", "reward"], df, color=(13 / 255, 28 / 255, 164 / 255), n_bins=50) save_plot(["episode", "loss"], df, color=(195 / 255, 0 / 255, 0 / 255), n_bins=50)
import cv2 import numpy as np import pandas as pd from os.path import join from src import config from src.utils import RLenc, make_dir from pipeline.mean_submission import MEAN_PREDICTION_DIR from mosaic.postprocess import postprocess mean_path = join(MEAN_PREDICTION_DIR, 'masks') postprocess_path = join(MEAN_PREDICTION_DIR, 'postprocessed') make_dir(postprocess_path) if __name__ == "__main__": print('Make postprocessed submission') postprocess(mean_path, postprocess_path) sample_submition = pd.read_csv(config.SAMPLE_SUBM_PATH) for i, row in sample_submition.iterrows(): pred_name = row.id + '.png' pred = cv2.imread(join(postprocess_path, pred_name), cv2.IMREAD_GRAYSCALE) pred = pred > 0 rle_mask = RLenc(pred.astype(np.uint8)) row.rle_mask = rle_mask sample_submition.to_csv(join(MEAN_PREDICTION_DIR, 'submission.csv'), index=False)