def plot_mean_var_cl(fol, root_fol, subject, sms, run, labels_names, k_type='triangular', sim=False):
# if not op.isfile(input_fname):
# shutil.move(op.join(fol, 'mean_var_cl_{}.npz'.format(k_type)), input_fname)
input_fname = op.join(fol, 'mean_var_cl{}_{}.npz'.format('_sim' if sim else '', k_type))
d = np.load(input_fname)
mean_cl, var_cl, hs_tr, hs_ms = d['mean_cl'], d['var_cl'], d['hs_tr'], d['hs_ms']
for cl_vector, cl_name in zip([mean_cl + var_cl, mean_cl, var_cl], ['AIC', 'AIC_mean', 'AIC_var']):
for label_ind in range(mean_cl.shape[0]):
cl = cl_vector[label_ind]
label_name = labels_names[label_ind]
figs_fol = op.join(root_fol, 'figures', 'mean_var{}_cl'.format('_sim' if sim else ''), label_name, cl_name)
utils.make_dir(figs_fol)
fig_fname = op.join(figs_fol, '{}_{}_{}_{}.jpg'.format(subject, sms, cl_name, label_name))
# if op.isfile(fig_fname):
# continue
ind = np.arange(len(hs_ms))
if np.any(np.isnan(cl)):
ind = ind[~np.isnan(cl)]
cl = cl[ind]
fig = plt.figure()
width = 0.35
# plt.bar(ind, cl, width=width)
plt.scatter(ind, cl, marker='o', facecolors='none')
# plt.xticks(ind + width / 2, hs_ms[ind])
plt.xlim([-0.5, len(cl) + 0.5])
plt.title('{} {} {} {} {}{}'.format(cl_name, subject, label_name, sms, run, ' sim' if sim else ''))
plt.xlabel('window-width (s)')
# plt.text(ind[cl.argmin()], cl.min() * 0.97 + .03 * cl.max(), '*', fontsize=14)
plt.scatter(ind[cl.argmin()], cl.min(), marker='o')
plt.plot(ind, cl, '--')
# utils.maximize_figure(plt)
# plt.tight_layout()
# plt.show()
plt.savefig(fig_fname, dpi=100)
plt.close()
def compare_vector_mean_var_cl(subject, sms, run, fol, root_fol, k_types=['triangular']):
for k_type in k_types:
# d = np.load(op.join(fol, 'vector_mean_cl_{}_{}-{}.npz'.format(k_type)))
d = np.load(op.join(fol, 'vector_mean_cl_{}_{}-{}.npz'.format(k_type)))
# vector_mean_cov_cl_triangular_mean_laus125
cl, hs_ms = d['mean_cl'], d['hs_ms']
d = np.load(op.join(fol, 'vector_mean_cl_sim_{}_{}-{}.npz'.format(k_type)))
cl_sim, hs_ms_sim = d['mean_cl'], d['hs_ms']
fig = plt.figure()
width = 0.35
ind = np.arange(len(hs_ms))
# plt.bar(ind, cl, width=width)
plt.scatter(ind, cl, marker='o', facecolors='none', label='real data')
plt.scatter(ind, cl_sim, marker='^', facecolors='none', label='simulated data')
# plt.xticks(ind + width / 2, hs_ms)
plt.title('{} {} {} {}{}'.format('AIC mean', subject, sms, run, ' sim' if sim else ''))
plt.xlabel('window-width (s)')
# plt.text(cl.argmin(), cl.min() * 0.97 + .03 * cl.max(), '*', fontsize=14)
plt.legend()
plt.scatter(cl.argmin(), cl.min(), marker='o')
plt.scatter(cl_sim.argmin(), cl_sim.min(), marker='^')
plt.plot(ind, cl, '--')
plt.plot(ind, cl_sim, '--')
plt.xlim([-0.5, len(cl) + 0.5])
utils.maximize_figure(plt)
plt.tight_layout()
# plt.show()
figures_fol = op.join(root_fol, 'figures', 'cl_mean_comparison_figures')
utils.make_dir(figures_fol)
plt.savefig(op.join(figures_fol, 'vector_mean_cl_{}_{}_{}.jpg'.format(subject, sms, run)), dpi=200)
plt.close()
def solve_labels_collision(subject, subjects_dir, atlas, backup_atlas, n_jobs=1):
now = time.time()
print('Read labels')
labels = utils.read_labels_parallel(subject, subjects_dir, atlas, n_jobs)
backup_labels_fol = op.join(subjects_dir, subject, 'label', backup_atlas)
labels_fol = op.join(subjects_dir, subject, 'label', atlas)
if op.isdir(backup_labels_fol):
shutil.rmtree(backup_labels_fol)
os.rename(labels_fol, backup_labels_fol)
utils.make_dir(labels_fol)
hemis_verts, labels_hemi, pia_verts = {}, {}, {}
print('Read surface ({:.2f}s)'.format(time.time() - now))
for hemi in HEMIS:
surf_fname = op.join(subjects_dir, subject, 'surf', '{}.pial'.format(hemi))
hemis_verts[hemi], _ = mne.surface.read_surface(surf_fname)
labels_hemi[hemi] = [l for l in labels if l.hemi == hemi]
print('Calc centroids ({:.2f}s)'.format(time.time() - now))
centroids = calc_labels_centroids(labels_hemi, hemis_verts)
for hemi in HEMIS:
print('Calc vertices labeling for {} ({:.2f}s)'.format(hemi, time.time() - now))
hemi_centroids_dist = cdist(hemis_verts[hemi], centroids[hemi])
vertices_labels_indices = np.argmin(hemi_centroids_dist, axis=1)
labels_hemi_chunks = utils.chunks(list(enumerate(labels_hemi[hemi])), len(labels_hemi[hemi]) / n_jobs)
params = [(labels_hemi_chunk, atlas, vertices_labels_indices, hemis_verts, labels_fol) for labels_hemi_chunk in labels_hemi_chunks]
print('Save labels for {} ({:.2f}s)'.format(hemi, time.time() - now))
utils.run_parallel(_save_new_labels_parallel, params, n_jobs)
def inpaint_double_train(img_files):
img_file_1, img_file_2, fold = img_files
img_1 = cv2.imread(join(TRAIN_DIR, 'images', img_file_1),
cv2.IMREAD_GRAYSCALE)
trg_1 = cv2.imread(join(TRAIN_DIR, 'masks', img_file_1),
cv2.IMREAD_GRAYSCALE)
img_2 = cv2.imread(join(TRAIN_DIR, 'images', img_file_2),
cv2.IMREAD_GRAYSCALE)
trg_2 = cv2.imread(join(TRAIN_DIR, 'masks', img_file_2),
cv2.IMREAD_GRAYSCALE)
if not np.sum(img_1) or not np.sum(img_2):
return
img_1 = np.pad(img_1, PAD_WIDTH, mode='constant')
trg_1 = np.pad(trg_1, PAD_WIDTH, mode='constant')
img_2 = np.pad(img_2, PAD_WIDTH, mode='constant')
trg_2 = np.pad(trg_2, PAD_WIDTH, mode='constant')
img = np.concatenate([img_1, img_2], axis=1)
trg = np.concatenate([trg_1, trg_2], axis=1)
img = (inpaint_biharmonic(img, MASK_INPAINT) * 255).astype(np.uint8)
trg = inpaint_biharmonic(trg, MASK_INPAINT)
trg = np.where(trg > TARGET_THRESHOLD, 255, 0)
img_file = img_file_1[:-4] + '_' + img_file_2[:-4] + '.png'
make_dir(join(TRAIN_DIR, 'images' + SAVE_NAME, 'fold_%d' % fold))
make_dir(join(TRAIN_DIR, 'masks' + SAVE_NAME, 'fold_%d' % fold))
cv2.imwrite(
join(TRAIN_DIR, 'images' + SAVE_NAME, 'fold_%d' % fold, img_file), img)
cv2.imwrite(
join(TRAIN_DIR, 'masks' + SAVE_NAME, 'fold_%d' % fold, img_file), trg)
return img, trg, img_file
def plot_vector_mean_var(subject, sms, run, ys, names, label_ids, fol, tr, hs_plot, k_type='triangular', sim=False,
overwrite=False, ax=None, plot_legend=True, xlim=None):
from matplotlib.ticker import MultipleLocator, FormatStrFormatter
from itertools import cycle
majorLocator = MultipleLocator(100)
majorFormatter = FormatStrFormatter('%d')
minorLocator = MultipleLocator(50)
d = np.load(op.join(fol, 'vector_mean_var{}_{}.npz'.format('_sim' if sim else '', k_type)))
means, vars, hs_tr, hs_ms = d['means'], d['vars'], d['hs_tr'], d['hs_ms']
boynton_colors = cycle(["red", "green", "magenta", "yellow", "pink", "orange", "brown", "gray"])
t0 = int(2 * (max(hs_tr) - 1))
# t = range(ys.shape[1])[t0:]
t = np.arange(ys.shape[1])[t0:] * tr / 1000
print(sms, tr, t[0], t[-1], ys.shape[1])
utils.make_dir(op.join(fol, 'pca_labels'))
for y, label_name, label_ind in zip(ys, names, label_ids):
fig_fname = op.join(fol, 'pca_labels', '{}_{}.jpg'.format(label_name, k_type))
# if op.isfile(fig_fname) and not overwrite:
# continue
if ax is None:
fig, ax = plt.subplots()
lines = []
l, = ax.plot(t, y[t0:], 'b', label='simulated' if sim else 'real')
lines.append(l)
for h_s in hs_plot:
h_ind = np.where(np.array(hs_ms) == h_s)[0][0]
h_ms = hs_ms[h_ind]
color = next(boynton_colors)
mean = means[h_ind, label_ind, t0:]
error = np.power(vars[h_ind, label_ind, t0:], 0.5)
l, = ax.plot(t, mean, color, label='w {:d}s'.format(h_ms))
lines.append(l)
ax.fill_between(t, mean - error, mean + error,
alpha=0.2, edgecolor=color, facecolor=color)
if plot_legend:
ax.legend(handles=lines, bbox_to_anchor=(1.05, 1.1))
ax.set_xlabel('Time (s)')
if ax is None:
ax.set_title('{} {} {} {}'.format(subject, sms, run, label_name))
else:
ax.set_title('{}'.format(sms.replace('_', ' ')))
utils.maximize_figure(plt)
plt.tight_layout()
ax.xaxis.set_major_locator(majorLocator)
ax.xaxis.set_major_formatter(majorFormatter)
ax.xaxis.set_minor_locator(minorLocator)
if not xlim is None:
ax.set_xlim(xlim)
if ax is None:
plt.savefig(fig_fname, dpi=200)
plt.close()
def combine_mean_var_sim_plots(root_fol, labels_names):
figure_fol = op.join(root_fol, 'figures')
utils.make_dir(op.join(figure_fol, 'smss_per_label_compare'))
now = time.time()
for ind, label in enumerate(labels_names):
utils.time_to_go(now, ind, len(labels_names), 5)
figure_real_data = op.join(figure_fol, 'smss_per_label', '{}.png'.format(label))
figure_sim_data = op.join(figure_fol, 'smss_per_label_sim', '{}.png'.format(label))
new_image_fname = op.join(figure_fol, 'smss_per_label_compare', '{}.png'.format(label))
utils.combine_two_images(figure_real_data, figure_sim_data, new_image_fname)
def __init__(self, model_dir, max_to_keep=5):
""" Save parameters of the model
Args:
dir: 모델을 저장할 폴더
max_to_keep: 모델의 저장할 갯수
(그 이상이면 가장 이전 것을 자동으로 지운다.)
"""
self.path = os.path.join(model_dir, 'model.ckpt')
self.saver = tf.train.Saver(max_to_keep=max_to_keep,
var_list=tf.get_collection(
tf.GraphKeys.GLOBAL_VARIABLES))
# Folder 생성
utils.make_dir(model_dir)
def __init__(self,
child_network_definition,
CellGeneratorInstance,
weight_dict,
weight_directory="./",
opt_loss='categorical_crossentropy',
opt=SGD(lr=0.001, decay=1e-6, nesterov=True),
opt_metrics=['accuracy']):
self.child_network_definition = child_network_definition
"""
child_network_definition is like ["N","N","R","N","N","R"]
where N is for normal and R for reduction
"""
self.CG = CellGeneratorInstance
self.opt_loss = opt_loss
self.opt = opt
self.opt_metrics = opt_metrics
self.model = self.generate_child_network()
self.model_dict = self.generate_model_dict()
self.weight_dict = weight_dict
self.weight_directory = make_dir(weight_directory)
self.graph = tf.get_default_graph()
def plot_mean_var(k_type='triangular', sim=False, labels_names=None, label_ids=None):
from collections import defaultdict, OrderedDict
import itertools
xlim = None #[200, 300]
figures_fol = op.join(root_fol, 'figures', 'smss_per_label{}{}'.format(
'_window' if not xlim is None else '', '_sim' if sim else ''))
utils.make_dir(figures_fol)
gen, data = {}, {}
hs_plot = [5, 10, 15, 25]
if labels_names is None:
labels_names = utils.load(op.join(root_fol, 'labels_names.pkl'))
label_ids = range(len(labels_names))
for fol, subject, sms, run in utils.sms_generator(root_fol):
print(fol, subject, sms, run)
if subject not in gen:
gen[subject] = OrderedDict()
if sms not in gen[subject]:
gen[subject][sms] = []
gen[subject][sms].append((fol, run))
if not sim:
d = np.load(op.join(fol, 'labels_data_{}_{}.npz'.format(atlas, measure)))
data[fol] = d['data']
else:
import scipy.io as sio
d_sim = sio.loadmat(op.join(fol, 'fmri_timecourse_sim.mat'))
data[fol] = d_sim['timecourse_use_sim'].T
# labels_names = [0]
now = time.time()
for subject, (label_id, label_name) in itertools.product(gen.keys(), zip(label_ids, labels_names)):
utils.time_to_go(now, label_id, len(labels_names), 5)
fig, axes = plt.subplots(2, 2, sharex='col', sharey='row', figsize=(12, 8))
axs = list(itertools.chain(*axes))
# fig.suptitle('{} {} {}'.format(subject, label_name, 'sim' if sim else ''))
for index, (ax, sms) in enumerate(zip(axs, gen[subject].keys())):
fol, run = gen[subject][sms][0]
tr = utils.load(op.join(fol, 'tr.pkl'))
print(tr)
ys = np.array([data[fol][label_id]])
plot_vector_mean_var(subject, sms, run, ys, [label_name], [label_id], fol, tr, hs_plot, k_type, sim,
overwrite=False, ax=ax, plot_legend=index==1, xlim=None)
utils.maximize_figure(plt)
plt.tight_layout()
plt.savefig(op.join(figures_fol, label_name))
plt.close()
def pred_test_fold(model_path, fold):
predictor = Predictor(model_path)
prob_df = pd.read_csv(config.SAMPLE_SUBM_PATH)
prob_df.rename(columns={'rle_mask': 'prob'}, inplace=True)
fold_prediction_dir = join(PREDICTION_DIR, f'fold_{fold}', 'test')
make_dir(fold_prediction_dir)
for i, row in prob_df.iterrows():
image_path = join(config.TEST_DIR, 'images'+IMAGES_NAME, row.id + '.png')
image = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE)
segm, prob = predictor(image)
row.prob = prob
segm_save_path = join(fold_prediction_dir, row.id + '.png')
cv2.imwrite(segm_save_path, segm)
prob_df.to_csv(join(fold_prediction_dir, 'probs.csv'), index=False)
def generate(alignments: dict, output_dir='./') -> None:
'''
Generate a summary text and json file for the alignments made
Inputs:
alignments: dict containing the alignments made. The key of each entry is the name
of the file appended with scan number, and the values should be Alignments
kwargs:
output_dir: str path to the output directory. Default=./
Outputs:
None
'''
output_dir = make_valid_dir_string(output_dir)
make_dir(output_dir)
json_file(alignments, output_dir)
tsv_file(alignments, output_dir)
def _init_save(self):
fourcc = cv2.VideoWriter_fourcc(*'DIVX')
video_dir = utils.abs_path('..', 'model_rec')
utils.make_dir(video_dir)
if self._base_opt.video_file_name_base:
video_file_name = '{}_{}'.format(
self._base_opt.name, self._base_opt.video_file_name_base)
else:
dt_str = utils.cur_datetime()
video_file_name = 'source_{}_{:.0f}x{:.0f}_{:.0f}.avi'.format(
dt_str, self._base_opt.width, self._base_opt.height,
self._base_opt.fps)
video_file_path = os.path.join(video_dir, video_file_name)
print('Write to file', video_file_path)
self._writer = cv2.VideoWriter(
video_file_path, fourcc, self._base_opt.fps,
(self._base_opt.width, self._base_opt.height), False)
def plot_vector_mean_cov(fol, root_fol, subject, sms, run, measure, atlas, k_type='triangular', sim=False,
subplot=False, ax=None, write_x_label=True):
input_fname = op.join(fol, 'vector_mean_cov_cl{}_{}_{}_{}.npz'.format('_sim' if sim else '', k_type, measure, atlas))
if not op.isfile(input_fname):
print("Can't find {}!".format(input_fname))
return
# print('Loading {}'.format(input_fname))
d = np.load(input_fname)
# mean_cl, col_cl, cl, hs_tr = d['mean_cl'], d['cov_cl'], d['cl'], d['hs_tr']
mean_ll_cl, mean_pan_cl, cov_ll_cl, cov_pan_cl, hs_tr = d['mean_ll_cl'], d['mean_pan_cl'], d['cov_ll_cl'],\
d['cov_pan_cl'], d['hs_tr']
cl = mean_ll_cl + 1 * mean_pan_cl + cov_ll_cl + 1 * cov_pan_cl
# for cl, cl_name in zip([mean_cl, col_cl, cl], ['AIC mean', 'AIC cov', 'AIC']):
for cl, cl_name in zip([cl], ['AIC']):
if not subplot:
fig = plt.figure()
ax = plt.subplot()
ind = np.arange(len(hs_tr))
ax.scatter(ind, cl, marker='o', s=1, facecolors='none')
if write_x_label:
ax.set_xlabel('window-width (s)')
# plt.text(cl.argmin(), cl.min() * 0.97 + .03 * cl.max(), '*', fontsize=14)
cl_argmin, cl_min = np.nanargmin(cl), np.nanmin(cl)
cl_min_exist = 0 < cl_argmin < len(cl) - 1
if cl_min_exist:
ax.scatter(cl_argmin, cl_min, marker='o', s=30)
ax.set_xlim([-0.5, len(cl) + 0.5])
ax.plot(ind, cl, '--')
if subplot:
ax.set_title('{} {}'.format(sms.replace('_', ' '), '({}s)'.format(cl_argmin) if cl_min_exist else '(no AIC min)'))
if not subplot:
title = '{} {} {} {} {} {}{} {}'.format(
subject, measure, atlas, cl_name, sms, run, ' sim' if sim else '',
'best window length: {}s'.format(cl_argmin) if cl_min_exist else '(no AIC min)')
print(title)
plt.title(title)
utils.maximize_figure(plt)
plt.tight_layout()
figures_fol = op.join(root_fol, 'figures', 'mean_cov_figures{}'.format('_sim' if sim else ''))
utils.make_dir(figures_fol)
figure_fname = op.join(figures_fol, 'vector_{}_{}_{}_{}{}.jpg'.format(
cl_name.replace(' ', '_'), subject, run, sms, '_sim' if sim else ''))
print('Saving the figure {}'.format(figure_fname))
plt.savefig(figure_fname, dpi=200)
plt.close()
def run(opt):
path_pairs = []
src_path = os.path.abspath(opt.src)
dst_path = os.path.abspath(opt.dst)
utils.make_dir(dst_path)
if os.path.isfile(src_path):
dir_path, file_name = os.path.split(src_path)
src_files = [(os.path.join(dir_path, file_name), file_name)]
else:
src_files = utils.list_files(src_path)
for src_file_path, src_file_name in src_files:
dst_file_name = os.path.splitext(src_file_name)[0] + '.gif'
dst_file_path = os.path.join(dst_path, dst_file_name)
path_pairs.append((src_file_path, dst_file_path))
for fpath_src, fpath_dst in path_pairs:
cmd = 'ffmpeg -y -i {} {}'.format(fpath_src, fpath_dst)
print('Exec:', cmd)
subprocess.run(cmd)
def store_spoof_data(data, del_action, add_action, spoof_col_name, source):
"""
Parameters
----------
data
del_action
add_action
spoof_col_name
source
"""
imputed_data = spoofing_imputation(data, del_action, add_action,
spoof_col_name)
ticker_exchange = str(imputed_data['Ticker'].iloc[0]) + '_' + str(
imputed_data['Exchange'].iloc[0]) + '.csv'
file_name = source / str(imputed_data['Date'].iloc[0]) / ticker_exchange
utils.make_dir(file_name)
with open(file_name, "w+") as f:
imputed_data.to_csv(f, index=False)
def pred_val_fold(model_path, fold):
predictor = Predictor(model_path)
folds_df = pd.read_csv(TRAIN_FOLDS_PATH)
fold_df = folds_df[folds_df.fold == fold]
fold_prediction_dir = join(PREDICTION_DIR, f'fold_{fold}', 'val')
make_dir(fold_prediction_dir)
prob_dict = {'id': [], 'prob': []}
for i, row in fold_df.iterrows():
image = cv2.imread(row.image_path, cv2.IMREAD_GRAYSCALE)
segm, prob = predictor(image)
segm_save_path = join(fold_prediction_dir, row.id + '.png')
cv2.imwrite(segm_save_path, segm)
prob_dict['id'].append(row.id)
prob_dict['prob'].append(prob)
prob_df = pd.DataFrame(prob_dict)
prob_df.to_csv(join(fold_prediction_dir, 'probs.csv'), index=False)
def to_jpg(exrfile, jpgfile):
''' exrfile을 jpgfile로 변환합니다. '''
utils.make_dir(os.path.dirname(jpgfile))
color_ch = channel_list[:3]
File = OpenEXR.InputFile(exrfile)
PixType = Imath.PixelType(Imath.PixelType.FLOAT)
DW = File.header()['dataWindow']
Size = (DW.max.x - DW.min.x + 1, DW.max.y - DW.min.y + 1)
rgb = [np.fromstring(File.channel(c, PixType), dtype=np.float32) \
for c in color_ch]
for i in range(len(color_ch)):
rgb[i] = np.where(rgb[i] <= 0.0031308, (rgb[i] * 12.92) * 255.0,
(1.055 * (rgb[i]**(1.0 / 2.4)) - 0.055) * 255.0)
rgb8 = [Image.frombytes("F", Size, c.tostring()).convert("L") for c in rgb]
Image.merge("RGB", rgb8).save(jpgfile, "JPEG", quality=95)
def write(path, data):
assert data.ndim == 3, "In write(), data ndim must be 3, but " + str(
data.shape)
utils.make_dir(os.path.dirname(path))
# 어차피 color외에는 보지도 않으니 color만 저장
ch_list = channel_list[:3]
h, w, _ = data.shape
header = OpenEXR.Header(w, h)
header['compression'] = Imath.Compression(
Imath.Compression.PIZ_COMPRESSION)
header['channels'] = {c: Imath.Channel(_ch_precision(c)) for c in ch_list}
out = OpenEXR.OutputFile(path, header)
ch_data = {ch: data[:, :, index].\
astype(_ch_np_precision(ch_list[index])).tostring()
for index, ch in enumerate(ch_list)}
out.writePixels(ch_data)
def main(argc):
option = build_option()
global_step = tf.train.get_or_create_global_step()
denoising_model = eval("model." + option.model_name)(
n_input_channel=option.n_input_channel,
n_output_channel=option.n_output_channel,
selected_loss=option.loss,
start_lr=option.start_lr,
lr_decay_step=option.lr_decay_step,
lr_decay_rate=option.lr_decay_rate,
global_step=global_step)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
# Model write
model.write(sess, option.text_dir + '/model.txt')
# Saver
# ==========================================================================
saver = Saver(
os.path.join(option.checkpoint_dir, option.experiment_name))
tf.global_variables_initializer().run()
saver.restore(sess)
# ==========================================================================
# Saver
# ==========================================================================
train_dir = os.path.join(option.logs_dir, option.experiment_name,
'train')
valid_dir = os.path.join(option.logs_dir, option.experiment_name,
'valid')
utils.make_dir(train_dir)
utils.make_dir(valid_dir)
train_writer = tf.summary.FileWriter(train_dir, sess.graph)
valid_writer = tf.summary.FileWriter(valid_dir, sess.graph)
tf.summary.scalar('loss', denoising_model.loss)
tf.summary.scalar('learning_rate', denoising_model.lr)
merged_summary = tf.summary.merge_all()
# ==========================================================================
# valid data
# ==========================================================================
valid_noisy_img, valid_reference = \
next_batch_tensor(
tfrecord_path = option.tfrecord_valid_path,
shape = [option.img_height, option.img_width,
option.n_input_channel, option.n_output_channel],
repeat = 10000
)
# ==========================================================================
for epoch in range(option.n_epoch):
train_noisy_img, train_reference = \
next_batch_tensor(
tfrecord_path = option.tfrecord_train_path,
shape = [option.patch_size, option.patch_size,
option.n_input_channel, option.n_output_channel],
batch_size = option.batch_size,
shuffle_buffer= option.shuffle_buffer,
prefetch_size = option.prefetch_size,
)
while True:
# 더이상 읽을 것이 없으면 빠져 나오고 다음 epoch으로
try:
noisy_img, reference = sess.run(
[train_noisy_img, train_reference])
except tf.errors.OutOfRangeError as e:
print("Done")
break
step = sess.run(global_step)
denoising_model.optimize(sess, noisy_img, reference)
# 일정 주기마다 logging
if step % option.log_period == 0:
loss_val = denoising_model.get_loss(
sess, noisy_img, reference)
print(f'epoch {epoch}, step {step}] loss : {loss_val}')
summary = sess.run(merged_summary,
feed_dict={
denoising_model.noisy_img:
noisy_img,
denoising_model.reference: reference
})
train_writer.add_summary(summary, step)
denoised_img = denoising_model.get_pred(
sess, noisy_img, reference)
for b in range(option.n_save_patch_img):
exr.save_debug_img(
option.debug_image_dir +
f'/{option.experiment_name}' +
f'/train/denoised/{step}_{b}',
denoised_img[b, :, :, :])
exr.save_debug_img(
option.debug_image_dir +
f'/{option.experiment_name}' +
f'/train/noisy/{step}_{b}', noisy_img[b, :, :, :])
exr.save_debug_img(
option.debug_image_dir +
f'/{option.experiment_name}' +
f'/train/refer/{step}_{b}', reference[b, :, :, :])
# 일정주기마다 모델 저장
if step % option.save_period == 0:
saver.save(sess, step)
if step % option.valid_period == 0:
print(
"====================================================================="
)
noisy_img, reference = sess.run(
[valid_noisy_img, valid_reference])
loss_val, denoised_img, summary = \
sess.run([denoising_model.loss, denoising_model.denoised_img, merged_summary],
feed_dict={denoising_model.noisy_img:noisy_img,
denoising_model.reference:reference})
print(" Test ] loss ", loss_val)
valid_writer.add_summary(summary, step)
debug_dir = os.path.join(option.debug_image_dir,
option.experiment_name, 'valid')
exr.save_debug_img(debug_dir + f'/valid/{step}',
denoised_img[0, :, :, :])
print(
"====================================================================="
)
tf.reset_default_graph()
tensor = torch.stack(tensors, dim=0)
tensor = tensor.to(self.model.device)
country_tensor = torch.stack(country_tensors, dim=0)
country_tensor = country_tensor.to(self.model.device)
with torch.no_grad():
probs = self.model.predict((tensor, country_tensor))
return probs
if __name__ == "__main__":
print("Model", MODEL_PATH)
print("Prediction folder", PREDICT_DIR)
make_dir(PREDICT_DIR)
draw_transform = DrawTransform(DRAW_SIZE, DRAW_PAD, DRAW_LINE_WIDTH,
TIME_COLOR)
image_trns = ImageTransform(False, SCALE_SIZE)
test_df = pd.read_csv(config.TEST_SIMPLIFIED_PATH)
sample_subm = pd.read_csv(config.SAMPLE_SUBMISSION)
predictor = Predictor(MODEL_PATH, draw_transform, image_trns)
drawings = []
key_ids = []
pred_words = []
pred_key_ids = []
probs_lst = []
for i, row in tqdm.tqdm(test_df.iterrows(), total=test_df.shape[0]):
drawing = eval(row.drawing)
def main(_):
# Denoising Model. src\base_model.py과 src\model.py를 참조할 것
net = DenoisingNet(input_shape=[None, None, INPUT_CH],
output_shape=[None, None, OUTPUT_CH],
loss_func='L1',
start_lr=1e-6,
lr_decay_step=10000,
lr_decay_rate=1.0)
sess = tf.Session()
# debug용 폴더들 생성
utils.make_dir(TRAIN_SUMMARY_DIR)
utils.make_dir(VALID_SUMMARY_DIR)
utils.make_dir(CKPT_DIR)
utils.make_dir(NOISY_IMAGE_DIR)
utils.make_dir(REFER_IMAGE_DIR)
utils.make_dir(DENOISED_IMG_DIR)
# Saver
# =========================================================================
saver = tf.train.Saver()
print('Saver initialized')
recent_ckpt_job_path = tf.train.latest_checkpoint(CKPT_DIR)
if recent_ckpt_job_path is None:
sess.run(tf.global_variables_initializer())
print("Initializing variables...")
else:
saver.restore(sess, recent_ckpt_job_path)
print("Restoring...", recent_ckpt_job_path)
# =========================================================================
# Summary
# =========================================================================
train_writer = tf.summary.FileWriter(TRAIN_SUMMARY_DIR, sess.graph)
valid_writer = tf.summary.FileWriter(VALID_SUMMARY_DIR, sess.graph)
# =========================================================================
# 검증용 데이터 tensor 생성 src\dataset 참조
# =========================================================================
valid_noisy_img, valid_reference = \
next_batch_tensor(
tfrecord_path = VALID_TFRECORD,
shape = [IMG_HEIGHT, IMG_WIDTH, N_NOISY_FEATURE, N_REFER_FEATURE],
repeat = 1000
)
# =========================================================================
#merged_summary = tf.summary.merge_all()
lst = []
# 학습 시작 Epoch
for epoch in count():
# 학습용 데이터 tensor 생성 src\dataset 참조
train_noisy_img, train_reference = \
next_batch_tensor(
tfrecord_path = TRAIN_TFRECORD,
shape = [PATCH_SIZE, PATCH_SIZE, N_NOISY_FEATURE, N_REFER_FEATURE],
batch_size = BATCH_SIZE,
shuffle_buffer = BATCH_SIZE * 2,
)
# 학습용 데이터 다 읽을 때 까지
while True:
# 더이상 읽을 것이 없으면 빠져 나오고 다음 epoch으로
try:
# 학습용 데이터를 위에 설정한 베치사이즈만큼 가져온다.
noisy_img, reference = sess.run(
[train_noisy_img, train_reference])
except tf.errors.OutOfRangeError as e:
print("Done")
break
# 데이터로 학습
_, step, lr = sess.run([net.train_op, net.global_step, net.lr],
feed_dict={
net.inputs: noisy_img,
net.refers: reference
})
# 일정
if step % LOG_PERIOD == LOG_PERIOD - 1:
loss = sess.run([net.loss],
feed_dict={
net.inputs: noisy_img,
net.refers: reference
})
# train_writer.add_summary(summary, step + 1)
print(f'epoch:{epoch:2d} step:{step+1} loss:{loss}')
if step % SAVE_PERIOD == SAVE_PERIOD - 1:
saver.save(sess,
os.path.join(CKPT_DIR, "model.ckpt"),
global_step=step + 1)
if step % VALID_PERIOD == VALID_PERIOD - 1:
noisy_img, reference = sess.run(
[valid_noisy_img, valid_reference])
loss, denoised_img = sess.run([net.loss, net.outputs],
feed_dict={
net.inputs: noisy_img,
net.refers: reference
})
print(" Test ] Loss ", loss)
#valid_writer.add_summary(summary, step + 1)
rp = REFER_IMAGE_DIR + f'{step+1}'
dp = DENOISED_IMG_DIR + f'{step+1}'
np = NOISY_IMAGE_DIR + f'{step+1}'
write_exr(reference[0, :, :, :3], rp + ".exr")
write_exr(denoised_img[0, :, :, :3], dp + ".exr")
write_exr(noisy_img[0, :, :, :3], np + ".exr")
to_jpg(rp + ".exr", rp + ".jpg")
to_jpg(dp + ".exr", dp + ".jpg")
to_jpg(np + ".exr", np + ".jpg")
os.remove(rp + ".exr")
os.remove(dp + ".exr")
os.remove(np + ".exr")
sess.close()
tf.reset_default_graph()
def main(config):
CASE_NUM = config['case_num']
DATASET = config['dataset']
NORMALIZATION = config['normalization']
BATCH_SIZE = config['batch_size']
MAX_EPOCH = config['max_epoch']
OPTIM_TYPE = config['optimizer']
LR = config['learning_rate']
LR_STEP = config['lr_step']
LR_DECAY = config['lr_decay']
L2_DECAY = config['l2_decay']
TB_STATE = config['use_tensorboard']
MODEL_NAME = config['model_name']
ALPHA = config['alpha']
BETA = config['beta']
GAMMA = config['gamma']
PHI = config['phi']
LOSS_FN = config['loss_fn']
KERNEL_SIZE = config['kernel_size']
result_dir = make_dir(RESULT_ROOT_DIR,
str(CASE_NUM),
overwrite=args.overwrite)
ckpt_path = result_dir + '/' + 'checkpoint.pt'
# =============================================== Select data and construct
data_fname, data_dim = select_data(DATASET)
data_path = '../data/' + data_fname
data_train = NLUDataset(data_path,
mode='train',
normalization=NORMALIZATION,
random_seed=42)
dataloader_train = DataLoader(data_train,
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=4)
data_valid = NLUDataset(data_path,
mode='valid',
normalization=NORMALIZATION,
random_seed=42)
dataloader_valid = DataLoader(data_valid,
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=4)
data_test = NLUDataset(data_path,
mode='test',
normalization=NORMALIZATION,
random_seed=42)
dataloader_test = DataLoader(data_test,
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=4)
num_train_samples = data_train.__len__()
classes = data_train.labels
num_classes = len(classes)
# =============================================== Initialize model and optimizer
device = ('cuda' if torch.cuda.is_available() else 'cpu')
if device == 'cuda': print('Using GPU, %s' % torch.cuda.get_device_name(0))
net = select_model(MODEL_NAME, data_dim, KERNEL_SIZE, num_classes, ALPHA,
BETA, PHI)
net.to(device)
loss_fn = select_loss(LOSS_FN)
optimizer = select_optimizer(OPTIM_TYPE, net.parameters(), LR, L2_DECAY)
scheduler = optim.lr_scheduler.StepLR(optimizer,
step_size=LR_STEP,
gamma=LR_DECAY)
# =============================================== Train
it = 0
train_losses, valid_losses, valid_accs = {}, {}, {}
best_validation_acc = 0
log_term = 5
for epoch in range(MAX_EPOCH):
#------------------------------------------------ One epoch start
one_epoch_start = time.time()
print('Epoch {} / Learning Rate: {:.0e}'.format(
epoch,
scheduler.get_lr()[0]))
#------------------------------------------------ Train
train_losses, it, net, optimizer, scheduler \
= train_1epoch(dataloader_train, device, train_losses, it, net, loss_fn, optimizer, scheduler, log_every=log_term)
#------------------------------------------------ Validation
valid_acc, valid_loss = evaluate(dataloader_valid, device, net,
loss_fn)
valid_losses[it] = valid_loss
valid_accs[it] = valid_acc
#------------------------------------------------ Save model
saved = ''
if valid_acc > best_validation_acc:
best_validation_acc = valid_acc
saved = save_ckpt(ckpt_path, net, best_validation_acc)
print('Epoch {} / Valid loss: {:.4f}, Valid acc: {:.4f} {}'.format(
epoch, valid_loss, valid_acc, saved))
#------------------------------------------------ One epoch end
curr_time = time.time()
print("One epoch time = %.2f s" % (curr_time - one_epoch_start))
print('#------------------------------------------------------#')
save_train_log(result_dir, train_losses, valid_losses, valid_accs,
best_validation_acc)
# =============================================== Test
net, best_validation_acc = load_ckpt(ckpt_path, net)
test_acc, test_loss = evaluate(dataloader_test, device, net, loss_fn)
return test_acc
from collections import defaultdict
from itertools import combinations
from random import shuffle
from src.utils import make_dir
from src import config
from src.config import TRAIN_DIR
ORIG_SIZE = (101, 101)
SAVE_SIZE = (109, 109)
SAVE_NAME = '_double_109'
TRAIN_FOLDS_PATH = '/workdir/data/train_folds_148.csv'
TARGET_THRESHOLD = 0.7
N_WORKERS = mp.cpu_count()
make_dir(join(TRAIN_DIR, 'images' + SAVE_NAME))
make_dir(join(TRAIN_DIR, 'masks' + SAVE_NAME))
diff = (SAVE_SIZE - np.array(ORIG_SIZE))
pad_left = diff // 2
pad_right = diff - pad_left
PAD_WIDTH = ((pad_left[0], pad_right[0]), (pad_left[0], pad_right[1]))
print('Pad width:', PAD_WIDTH)
MASK_INPAINT = np.zeros(ORIG_SIZE, dtype=np.uint8)
MASK_INPAINT = np.pad(MASK_INPAINT,
PAD_WIDTH,
mode='constant',
constant_values=255)
MASK_INPAINT = np.concatenate([MASK_INPAINT, MASK_INPAINT], axis=1)
def copy_figures(subject, sms, run, fol, root_fol, label_name, k_type='triangular'):
utils.make_dir(op.join(root_fol, 'mean_var_figures'))
shutil.copy(op.join(fol, 'pca_labels', '{}_{}.jpg'.format(label_name, k_type)),
op.join(root_fol, 'mean_var_figures', '{}_{}_{}_{}_{}.jpg'.format(subject, sms, run, label_name, k_type)))
FOLD_DIRS = [join(p, 'fold_%d' % f) for p in PREDICTION_DIRS for f in FOLDS]
PREDICTION_DIRS = [
'/workdir/data/predictions/fpn-lovasz-se-resnext50-006-after-001/',
]
FOLDS = [0, 1, 2, 3, 4]
FOLD_DIRS += [join(p, 'fold_%d' % f) for p in PREDICTION_DIRS for f in FOLDS]
segm_thresh = 0.4
prob_thresh = 0.5
SAVE_NAME = 'mean-005-0.4'
MEAN_PREDICTION_DIR = f'/workdir/data/predictions/{SAVE_NAME}'
make_dir(join(MEAN_PREDICTION_DIR, 'masks'))
def get_mean_probs_df():
probs_df_lst = []
for fold_dir in FOLD_DIRS:
probs_df = pd.read_csv(join(fold_dir, 'test', 'probs.csv'),
index_col='id')
probs_df_lst.append(probs_df)
mean_probs_df = probs_df_lst[0].copy()
for probs_df in probs_df_lst[1:]:
mean_probs_df.prob += probs_df.prob
mean_probs_df.prob /= len(probs_df_lst)
return mean_probs_df
def build_option():
parser = ArgumentParser()
parser.register('type', 'bool', _bool)
parser.add_argument('-ex',
'--experiment_name',
type=str,
default='test_1',
help='실험 이름. 이것에 따라 log와 checkpoint 이름 결정')
# Model
# ============================================================================
parser.add_argument('-m',
'--model_name',
type=str,
default="VGGNet",
help='model 이름(VGGNet, ResNet')
parser.add_argument('-ls',
'--loss',
type=str,
default="L1",
help='model 학습할 때 쓰일 loss')
parser.add_argument('-sl',
'--start_lr',
type=float,
default=0.001,
help='model 학습할 때 처음 learning rate 값')
parser.add_argument('-lds',
'--lr_decay_step',
type=int,
default=4000,
help='learning rate decay할 주기(step)')
parser.add_argument('-ldr',
'--lr_decay_rate',
type=float,
default=0.8,
help='learning rate decay 비율')
# ============================================================================
# Path
# ============================================================================
parser.add_argument('-l',
'--logs_dir',
type=str,
default='./logs',
help='tensorflow model log를 저장할 폴더')
parser.add_argument('-c',
'--checkpoint_dir',
type=str,
default='./checkpoint',
help='model의 paramter를 저장할 폴더')
parser.add_argument('-t',
'--tfrecord_train_path',
type=str,
default='./data/tfrecord/train_40.tfrecord',
help="train set을 읽을 tfrecord 파일 이름과 경로")
parser.add_argument('-e',
'--tfrecord_test_path',
type=str,
default='./data/tfrecord/test.tfrecord',
help="test set을 읽을 tfrecord 파일 이름과 경로")
parser.add_argument('-v',
'--tfrecord_valid_path',
type=str,
default='./data/tfrecord/valid.tfrecord',
help="valid set을 읽을 tfrecord 파일 이름과 경로")
parser.add_argument('-d',
'--debug_image_dir',
type=str,
default='./debug/image',
help="학습하면서 나오는 이미지들을 저장하는 폴더")
# ============================================================================
# Dataset
# ============================================================================
parser.add_argument('-nic',
'--n_input_channel',
type=int,
default=66,
help='이미지가 가지고 있는 채널 수')
parser.add_argument('-noc',
'--n_output_channel',
type=int,
default=3,
help='이미지가 가지고 있는 채널 수')
parser.add_argument('-wi',
'--img_width',
type=int,
default=1280,
help='이미지 가로 크기')
parser.add_argument('-he',
'--img_height',
type=int,
default=720,
help='이미지 세로 크기')
parser.add_argument('-p',
'--patch_size',
type=int,
default=40,
help="학습용 패치 크기")
parser.add_argument('-b',
'--batch_size',
type=int,
default=32,
help="minibatch 크기")
parser.add_argument('-sf',
'--shuffle_buffer',
type=int,
default=100,
help="tfrecord 파일을 읽을 때 섞기 위한 shuffle buffer 크기")
parser.add_argument('-ps',
'--prefetch_size',
type=int,
default=1,
help="미리 읽는 거라는데 잘 모르겠음")
# ============================================================================
# Training
# ============================================================================
parser.add_argument('-ep',
'--n_epoch',
type=int,
default=1000000000,
help='epoch 수. 보통은 그냥 크게!')
parser.add_argument('-lp',
'--log_period',
type=int,
default=100,
help='일정 주기마다 logging(print, log)')
parser.add_argument('-vp',
'--valid_period',
type=int,
default=3000,
help='일정 주기마다 모델 parameter 저장')
parser.add_argument('-sp',
'--save_period',
type=int,
default=5000,
help='일정 주기마다 모델 parameter 저장')
parser.add_argument('-nsp',
'--n_save_patch_img',
type=int,
default=2,
help='학습 중 저장할 patch 개수')
# ============================================================================
option = parser.parse_args()
option.text_dir = os.path.join(option.debug_image_dir,
option.experiment_name)
make_dir(option.text_dir)
f = open(option.text_dir + '/option.txt', 'w')
for op in vars(option):
f.write(op + ': ' + str(getattr(option, op)) + '\n')
f.close()
return option
def set_args(args) -> dict:
# check if we use the params file or the user arguments
use_params = stringtobool(args.config)
config = Config()
spectra_folder = args.spectra_folder if not use_params else config[
'spectra_dir']
database_file = args.database_file if not use_params else config[
'database_file']
output_dir = args.output_dir if not use_params else config['output_dir']
min_peptide_len = args.min_peptide_len if not use_params else config[
'min_peptide_len']
max_peptide_len = args.max_peptide_len if not use_params else config[
'max_peptide_len']
ppm_tolerance = args.tolerance if not use_params else config[
'ppm_tolerance']
precursor_tolerance = args.precursor_tolerance if not use_params else config[
'precursor_tolerance']
verbose = stringtobool(
args.verbose) if not use_params else config['verbose']
peak_filter = args.peak_filter if not use_params else config['num_peaks']
relative_abundance_filter = args.rel_abund_filter if not use_params else config[
'relative_abundance']
digest = args.digest if not use_params else config['digest']
cores = args.cores if not use_params else config['cores']
n = args.n if not use_params else config['top_results']
debug = config['debug']
truth_set = config['truth_set']
############## Argument checking ################
if not utils.is_dir(spectra_folder):
print(
f'Error: {spectra_folder} is not a real path. Path to directory with spectra files is necessary.'
)
sys.exit(0)
if not utils.is_fasta(database_file) or not utils.is_file(database_file):
print(
f'Error: {database_file} is not a valid .fasta file. .fasta file needed.'
)
sys.exit(0)
# make the output directory
output_dir = utils.make_valid_dir_string(output_dir)
utils.make_dir(output_dir)
return {
'spectra_folder': spectra_folder,
'database_file': database_file,
'output_dir': output_dir,
'min_peptide_len': min_peptide_len,
'max_peptide_len': max_peptide_len,
'tolerance': ppm_tolerance,
'precursor_tolerance': precursor_tolerance,
'verbose': verbose,
'peak_filter': peak_filter,
'relative_abundance_filter': relative_abundance_filter,
'digest': digest,
'DEBUG': debug,
'cores': cores,
'n': n,
'truth_set': truth_set
}
from src.transforms import SimpleDepthTransform, SaltTransform, CenterCrop
from src.argus_models import SaltMetaModel
from src.transforms import HorizontalFlip
from src.utils import RLenc, make_dir
from src import config
EXPERIMENT_NAME = 'mos-fpn-lovasz-se-resnext50-001'
FOLDS = list(range(config.N_FOLDS))
ORIG_IMAGE_SIZE = (101, 101)
PRED_IMAGE_SIZE = (128, 128)
TRANSFORM_MODE = 'crop'
TRAIN_FOLDS_PATH = '/workdir/data/train_folds_148_mos_emb_1.csv'
IMAGES_NAME = '148'
FOLDS_DIR = f'/workdir/data/experiments/{EXPERIMENT_NAME}'
PREDICTION_DIR = f'/workdir/data/predictions/{EXPERIMENT_NAME}'
make_dir(PREDICTION_DIR)
SEGM_THRESH = 0.5
PROB_THRESH = 0.5
class Predictor:
def __init__(self, model_path):
self.model = load_model(model_path)
self.model.nn_module.final = torch.nn.Sigmoid() #
self.model.nn_module.eval()
self.depth_trns = SimpleDepthTransform()
self.crop_trns = CenterCrop(ORIG_IMAGE_SIZE)
self.trns = SaltTransform(PRED_IMAGE_SIZE, False, TRANSFORM_MODE)
def compare_mean_var_cl(fol, root_fol, subject, sms, run, labels_names, hs, top_hs=-1, labels_ids=None,
hs_sim=None, k_type='triangular'):
from scipy.signal import argrelextrema
input_fname = op.join(fol, 'mean_var_cl_{}_{}-{}.npz'.format(k_type, hs[0], hs[1]))
if not op.isfile(input_fname):
print('{} does not exist!'.format(input_fname))
return
d = np.load(input_fname)
utils.print_modif_time(input_fname)
# cl_vector = d['mean_cl'] + d['var_cl']
hs_tr, hs_ms = d['hs_tr'], d['hs_ms']
if hs_sim is None:
hs_sim = hs
sim_fname = op.join(fol, 'mean_var_sim_cl_{}_{}-{}.npz'.format(k_type, hs_sim[0], hs_sim[1]))
if op.isfile(sim_fname):
d_sim = np.load(sim_fname)
utils.print_modif_time(sim_fname)
no_sim_data = False
else:
print("Can't find sim data!")
d_sim = np.load(input_fname)
no_sim_data = True
# cl_vector_sim = d_sim['mean_cl'] + d_sim['var_cl']
hs_tr_sim, hs_ms_sim = d['hs_tr'], d['hs_ms']
# cl_name = 'AIC'
# fig, axes = plt.subplots(2, 2, sharex='col', sharey='row', figsize=(12, 8))
# axs = list(itertools.chain(*axes))
root_figs_fol = op.join(root_fol, 'figures', subject, 'mean_var_cl_compare_{}_{}'.format(hs[0], top_hs))# hs[-1]))
utils.make_dir(root_figs_fol)
if labels_ids is None:
labels_ids = range(len(labels_names))
for cl_name, cl_vector, cl_vector_sim in zip(['AIC', 'AIC_mean', 'AIC_var'],
[d['mean_cl'] + d['var_cl'], d['mean_cl'], d['var_cl']],
[d_sim['mean_cl'] + d_sim['var_cl'], d_sim['mean_cl'], d_sim['var_cl']]):
for ind, label_ind in enumerate(labels_ids):
fig = plt.figure()
cl_real = cl_vector[label_ind]
cl_sim = cl_vector_sim[label_ind]
label_name = labels_names[ind]
figs_fol = op.join(root_figs_fol, label_name, cl_name)
utils.make_dir(figs_fol)
fig_fname = op.join(figs_fol, '{}_{}_{}_{}_{}.jpg'.format(subject, sms, run, label_name, cl_name))
# if op.isfile(fig_fname):
# continue
# ind_real = np.arange(len(hs_ms))
# ind_sim = np.arange(len(hs_ms_sim))
ind_real = hs_ms[:top_hs]
ind_sim = hs_ms_sim[:top_hs]
# print('ind real len: {}, ind sim len: {}'.format(len(ind_real), len(ind_sim)))
# print('cl real shape: {}, cl sim shape: {}'.format(cl_real.shape, cl_sim.shape))
lines = []
if no_sim_data:
itr = zip([cl_real], ['o'], [ind_real], ['real'], ['b'])
else:
itr = zip([cl_real, cl_sim], ['o', '^'], [ind_real, ind_sim], ['real', 'sim'], ['b', 'g'])
for cl, marker, ind, label, c in itr:
# for cl, marker, ind, label in zip([cl_real], ['o'], [ind_real], ['real']):
cl = cl[:top_hs]
if np.any(np.isnan(cl)):
ind = ind[~np.isnan(cl)]
cl = cl[~np.isnan(cl)]
# plt.bar(ind, cl, width=width)
# plt.scatter(ind, cl, marker=marker, facecolors='none', label=label)
# for cl, ind, c, label in zip([cl_real, cl_sim], [ind_real, ind_sim], ['b', 'g'], ['real', 'sim']):
# for cl, ind in zip([cl_real], [ind_real]):
# plt.text(ind[cl.argmin()], cl.min() * 0.97 + .03 * cl.max(), '*', fontsize=14)
# plt.scatter(ind[cl.argmin()], cl.min(), marker='o')
ind = ind[:len(cl)]
l, = plt.plot(ind, cl, '--', label=label)
lines.append(l)
# plt.legend()
minm = argrelextrema(cl, np.less) # (array([2, 5, 7]),)
for cl_min in minm[0]:
plt.scatter(ind[cl_min], cl[cl_min], marker='o', c=c)
# plt.xlim([-0.5, len(cl_real) + 0.5])
plt.legend(handles=lines)
plt.xlim([-0.5, max(ind_real) + 0.5])
# plt.title('{} {} {} {} {}'.format(cl_name, subject, label_name, sms, run))
plt.title('{}'.format(sms.replace('_', ' ')))
plt.ylabel(cl_name)
plt.xlabel('window-width (s)')
# utils.maximize_figure(plt)
# plt.tight_layout()
# plt.show()
print('Saving figure {}'.format(fig_fname))
plt.savefig(fig_fname, dpi=100)
plt.close()
def main():
# ====================
# Run
# ====================
outDir = make_dir(os.path.join(os.path.dirname(__file__), "output"))
modelDir = make_dir(os.path.join(outDir, "models"))
aniDir = make_dir(os.path.join(outDir, "animations"))
logFile = os.path.join(outDir, "log.csv")
env = Environment(n_good=0, n_bad=110)
agent = DQNAgent(env=env, sensors=Sensors(n_sectors=4, sector_radius=1.0))
save_models = True
save_animations = True
n_episodes = 1000
n_iter_max = 2000
n_reward_max = 0
loss = -1 # track training loss
for episode in range(n_episodes):
# Generate episode
state = env.reset() # initialize environment
obs = agent.observe(state, env) # observe
states, rewards, i = [], [], 0
while i < n_iter_max:
action = agent.get_action(obs) # follow epsilon-greedy policy
state_next, reward, done = env.step(state, action) # evolve
obs_next = agent.observe(state, env) # observe
agent.memorize((obs, action, reward, obs_next, done)) # memorize
rewards.append(reward)
states.append(state_next)
state, obs = state_next, obs_next # transition
i += 1
if done: break # terminate
# Track highly successful episodes
n_reward_max += (sum(rewards) >= 2000)
# Print progress
print(
"[ep {}/{}] iter={}/{}, rew={:.0f}, nrewmax={}, mem={}, eps={:.3f}, loss={:.2f}"
.format(episode + 1, n_episodes, i, n_iter_max, sum(rewards),
n_reward_max, len(agent.memory), agent.epsilon, loss),
flush=True)
if (episode == 0 or n_reward_max % 5 == 1):
# Save model
if save_models:
modelFile = os.path.join(
modelDir,
"model_ep={}_rew={}.h5".format(episode + 1,
int(sum(rewards))))
print(" -> saving agent model = {}".format(modelFile),
flush=True)
agent.save_model(modelFile)
# Save animation
if save_animations:
aniFile = os.path.join(
aniDir,
"ani_ep={}_rew={}.mp4".format(episode + 1,
int(sum(rewards))))
print(" -> saving animation = {}".format(aniFile),
flush=True)
env.make_animation(states,
env,
agent.sensors,
save_ani=True,
filename=aniFile)
n_reward_max += 0 if (episode == 0) else 1
# Train agent
loss = agent.train() # automatically adjusts epsilon
# Save log
header = ["episode", "iter", "reward", "loss", "epsilon", "time"]
values = [
episode + 1, i,
sum(rewards), loss, agent.epsilon,
datetime.datetime.now().strftime("%B %d %Y %I:%M:%S %p")
]
with open(logFile, ('w' if episode == 0 else 'a')) as f:
writer = csv.writer(f)
if episode == 0:
writer.writerow(header)
writer.writerow(values)
if (episode + 1) % 20 == 0 or (episode == n_episodes - 1):
df = pd.read_csv(logFile)
if df.shape[0] > 50:
save_plot(["episode", "reward"],
df,
color=(13 / 255, 28 / 255, 164 / 255),
n_bins=50)
save_plot(["episode", "loss"],
df,
color=(195 / 255, 0 / 255, 0 / 255),
n_bins=50)
import cv2
import numpy as np
import pandas as pd
from os.path import join
from src import config
from src.utils import RLenc, make_dir
from pipeline.mean_submission import MEAN_PREDICTION_DIR
from mosaic.postprocess import postprocess
mean_path = join(MEAN_PREDICTION_DIR, 'masks')
postprocess_path = join(MEAN_PREDICTION_DIR, 'postprocessed')
make_dir(postprocess_path)
if __name__ == "__main__":
print('Make postprocessed submission')
postprocess(mean_path, postprocess_path)
sample_submition = pd.read_csv(config.SAMPLE_SUBM_PATH)
for i, row in sample_submition.iterrows():
pred_name = row.id + '.png'
pred = cv2.imread(join(postprocess_path, pred_name),
cv2.IMREAD_GRAYSCALE)
pred = pred > 0
rle_mask = RLenc(pred.astype(np.uint8))
row.rle_mask = rle_mask
sample_submition.to_csv(join(MEAN_PREDICTION_DIR, 'submission.csv'),
index=False)
