def get_model(model_path):
model_folder_name = os.path.split(model_path)[-2]
if '0-to-0' in model_folder_name:
n_input_channel = 2
elif '0-to-inf' in model_folder_name:
n_input_channel = 3
else:
raise ValueError("MODEL PATH not valid")
#%% Define Model ###
model = ClassifyOnSegment(input_shape=(288, 384, n_input_channel),
model_structure='pspnet',
model_path=tempfile.mkdtemp(),
encoder_weights='imagenet',
n_segment_classes=4,
segment_class_weights=[1., 1., 1., 1.],
n_classify_classes=4,
classify_class_weights=[1., 1., 1., 1.],
eval_fn=evaluate_confusion_mat)
model.load(model_path)
return model
valid_name_file,
valid_X_filenames,
segment_y_files=valid_y_filenames,
segment_w_files=valid_w_filenames,
classify_label_file=valid_label_file,
**kwargs)
assert len(set(train_gen.names[i] for i in train_gen.selected_inds) & \
set(valid_gen.names[i] for i in valid_gen.selected_inds)) == 0
print("Initiate Model", flush=True)
model = ClassifyOnSegment(
input_shape=(288, 384, 3),
model_structure='pspnet',
model_path=model_dir,
encoder_weights='imagenet',
n_segment_classes=2,
n_classify_classes=2)
### Training ###
# print("Start Training", flush=True)
# model.fit(train_gen,
# valid_gen=valid_gen,
# verbose=2,
# n_epochs=40)
### Validation ###
print("=========================")
print("Loading from: %s" % os.path.join(model_dir, 'bkp.model'))
}
n_fs = len([f for f in os.listdir(data_path) if f.startswith('X')])
X_filenames = [os.path.join(data_path, 'X_%d.pkl' % i) for i in range(n_fs)]
y_filenames = [os.path.join(data_path, 'y_%d.pkl' % i) for i in range(n_fs)]
w_filenames = [os.path.join(data_path, 'w_%d.pkl' % i) for i in range(n_fs)]
name_file = os.path.join(data_path, 'names.pkl')
valid_gen = CustomGenerator(X_filenames,
y_filenames,
w_filenames,
name_file,
**kwargs)
model = ClassifyOnSegment(
input_shape=(288, 384, 2),
model_structure='pspnet',
model_path='.',
encoder_weights='imagenet',
n_segment_classes=2,
n_classify_classes=2)
model_path = '/oak/stanford/groups/jamesz/zqwu/iPSC_data/model_save/0-to-0_random/bkp.model'
model.load(model_path)
fls = pickle.load(open(os.path.join(DATA_ROOT, 'D%d_fl.pkl' % day), 'rb'))
labels = pickle.load(open(os.path.join(DATA_ROOT, 'D%d_labels.pkl' % day), 'rb'))
preds = []
classify_preds = []
for batch in valid_gen:
pred = model.predict(batch)
preds.append(pred[0])
os.path.join(VALID_DIR, 'segment_continuous_y_%d.pkl' % i)
for i in range(n_fs)
]
w_filenames = [
os.path.join(VALID_DIR, 'segment_continuous_w_%d.pkl' % i)
for i in range(n_fs)
]
name_file = os.path.join(VALID_DIR, 'names.pkl')
label_file = os.path.join(VALID_DIR, 'classify_continuous_labels.pkl')
valid_gen = CustomGenerator(name_file,
X_filenames,
segment_y_files=y_filenames,
segment_w_files=w_filenames,
classify_label_file=label_file,
**kwargs)
### Training ###
model = ClassifyOnSegment(input_shape=(288, 384, 2),
model_structure='pspnet',
model_path=MODEL_DIR,
encoder_weights='imagenet',
n_segment_classes=4,
n_classify_classes=4,
eval_fn=evaluate_confusion_mat)
# model.load(os.path.join(MODEL_DIR, 'bkp.model'))
model.fit(train_gen, valid_gen=valid_gen, n_epochs=200, verbose=2)
model.save(os.path.join(MODEL_DIR, 'pspnet_random_0-to-0_0.model'))
def main(args):
input_folder = args.input_folder
output_folder = args.output_folder
model_path = args.model_path
well_setting = args.well_setting
target_day = args.target_day
intermediate_save_dir = tempfile.mkdtemp()
# if `well_setting` is given, corner views will be removed
preprocess_filter = partial(
PREPROCESS_FILTER,
well_setting=well_setting) if len(well_setting) > 0 else lambda x: True
# Load all phase contrast (identified by keyword "Phase" in the file name)
# and all GFP (identifier by keyword "GFP") images under the folder recursively.
# Phase contrast/GFP of the same well will be matched into pairs automatically
pairs = load_all_pairs(path=input_folder)
print("Checking input data")
# Function below checks all the identified phase contrast/GFP files and shows stats
check_pairs_by_day(pairs)
# Process all input files, save into a temp folder
preprocess(pairs,
output_path=intermediate_save_dir,
preprocess_filter=preprocess_filter,
target_size=(384, 288),
well_setting=well_setting,
linear_align=False,
shuffle=True,
seed=123,
labels=[])
n_fs = len([
f for f in os.listdir(intermediate_save_dir)
if f.startswith('X_') and f.endswith('.pkl')
])
X_filenames = [
os.path.join(intermediate_save_dir, 'X_%d.pkl' % i)
for i in range(n_fs)
]
name_file = os.path.join(intermediate_save_dir, 'names.pkl')
# Generator instance looping through all input files
test_gen = CustomGenerator(
name_file,
X_filenames,
batch_size=8,
augment=False,
batch_with_name=True,
shuffle_inds=False,
n_segment_classes=None,
segment_class_weights=None,
segment_extra_weights=None,
n_classify_classes=None,
classify_class_weights=None,
)
# Identify model type: 0-to-0 / 0-to-inf
n_input_channel = 2 if '0-to-0' in model_path else 3
model = ClassifyOnSegment(input_shape=(288, 384, n_input_channel),
model_structure='pspnet',
model_path=tempfile.mkdtemp(),
encoder_weights='imagenet',
n_segment_classes=4,
segment_class_weights=[1., 1., 1., 1.],
n_classify_classes=4,
classify_class_weights=[1., 1., 1., 1.])
# Load model weights
model.load(model_path)
# If it is 0-to-inf prediction, define the placeholder for endpoint (default at 18)
input_transform = None
if n_input_channel == 3:
input_transform = partial(augment_fixed_end, end=target_day)
# Calculate predictions and save to designated folder
# refer to `collect_predictions.py` for the function below
collect_preds(test_gen,
model,
output_folder,
input_transform=input_transform)
]
w_filenames = [
os.path.join(data_path, 'random_valid_w_%d.pkl' % i) for i in range(n_fs)
]
name_file = os.path.join(data_path, 'random_valid_names.pkl')
label_file = os.path.join(data_path, 'random_valid_labels.pkl')
ref_valid_gen = CustomGenerator(X_filenames,
y_filenames,
w_filenames,
name_file,
label_file=label_file,
**kwargs)
model = ClassifyOnSegment(input_shape=(288, 384, 3),
model_structure='pspnet',
model_path='.',
encoder_weights='imagenet',
n_segment_classes=2,
n_classify_classes=2)
model.load(
'/oak/stanford/groups/jamesz/zqwu/iPSC_data/model_save/bkp/pspnet_random_0-to-inf_1.model'
)
seg_preds = []
cl_preds = []
for batch in valid_gen:
seg_pred, cl_pred = model.model.predict(batch[0])
seg_pred = scipy.special.softmax(seg_pred, -1)
cl_pred = scipy.special.softmax(cl_pred, -1)
seg_preds.append(seg_pred)
cl_preds.append(cl_pred)
]
w_filenames = [
os.path.join(data_path, 'random_valid_w_%d.pkl' % i) for i in range(n_fs)
]
name_file = os.path.join(data_path, 'random_valid_names.pkl')
label_file = os.path.join(data_path, 'random_valid_labels.pkl')
ref_valid_gen = CustomGenerator(X_filenames,
y_filenames,
w_filenames,
name_file,
label_file=label_file,
**kwargs)
model = ClassifyOnSegment(input_shape=(288, 384, 2),
model_structure='pspnet',
model_path='.',
encoder_weights='imagenet',
n_segment_classes=2,
n_classify_classes=2)
model.load(
'/oak/stanford/groups/jamesz/zqwu/iPSC_data/model_save/0-to-0_random/bkp.model'
)
seg_preds = []
cl_preds = []
for batch in valid_gen:
seg_pred, cl_pred = model.model.predict(batch[0])
seg_pred = scipy.special.softmax(seg_pred, -1)
cl_pred = scipy.special.softmax(cl_pred, -1)
seg_preds.append(seg_pred)
cl_preds.append(cl_pred)