def get_model(self, verbose=False):
"""Return centers detected when object detection and photometry
is done on input image with SEP.
Args:
image: Image (single band) of galaxy to perform measurement on.
Returns:
centers: x and y coordinates of detected centroids
"""
# set detection threshold to 5 times std of image
self.model_name = model_name
self.model_path = os.path.join(MODEL_DIR, self.model_file_name)
self.norm = [
1.9844158727667542, 413.83759806375525, 51.2789974336363,
1038.4760551905683
]
#[1.6361405416087091, 416.16687641284665, 63.16814480535191, 2346.133101333463]
count = 4000
catalog_name = os.path.join(DATA_PATH, 'OneDegSq.fits')
resid_model = btk_utils.Resid_btk_model(self.model_name,
self.model_path,
MODEL_DIR,
training=False,
images_per_gpu=1)
if verbose:
resid_model.config.display()
resid_model.make_resid_model(catalog_name,
count=count,
max_number=2,
norm_val=self.norm)
self.resid_model = resid_model
def detection_coadd(Args):
"""Test performance for btk input blends"""
norm = [
1.9844158727667542, 413.83759806375525, 51.2789974336363,
1038.4760551905683
]
count = 15 #4000 # 40000
catalog_name = os.path.join(DATA_PATH, 'OneDegSq.fits')
# Define parameters for mrcnn model with btk here
resid_model = btk_utils.Resid_btk_model(Args.model_name,
Args.model_path,
MODEL_DIR,
training=False,
images_per_gpu=1)
# Load parametrs for dataset and load model
meas_params = btk_utils.Scarlet_resid_params(detect_coadd=True)
resid_model.make_resid_model(catalog_name,
count=count,
max_number=2,
norm_val=norm,
meas_params=meas_params)
results = []
# np.random.seed(0)
for im_id in range(count):
iter_detected, sep_detected, true = resid_model.get_detections(im_id)
for i in range(len(true)):
it_det, it_undet, it_spur = btk.compute_metrics.evaluate_detection(
iter_detected[i], true[i])
# print(it_det, it_undet, it_spur)
if len(sep_detected[i]) == 0:
sep_det, sep_undet, sep_spur = 0, len(true[i]), 0
else:
unique_sep_det_cent = np.unique(sep_detected[i], axis=0)
sep_det, sep_undet, sep_spur = btk.compute_metrics.evaluate_detection(
unique_sep_det_cent, true[i])
# print(sep_det, sep_undet, sep_spur)
results.append([
len(true[i]), it_det, it_undet, it_spur, sep_det, sep_undet,
sep_spur
])
arr_results = np.array(results).T
print("Results: ", np.sum(arr_results, axis=1))
save_file_name = f"sep_det_results_2gal_coadd_temp.txt"
np.savetxt(save_file_name, arr_results)
def get_model(self, verbose=False):
# catalog_name = os.path.join(DATA_PATH, 'OneDegSq.fits')
resid_model = btk_utils.Resid_btk_model(self.model_name,
self.model_file_name,
MODEL_DIR,
training=False,
images_per_gpu=BATCH_SIZE)
# Load parameters for dataset and load model
resid_model.config.TRAIN_BN = False
resid_model.config.BACKBONE = 'resnet41'
resid_model.config.SKIP_P2_RPN = True
resid_model.config.BACKBONE_STRIDES = [8, 16, 32, 64]
resid_model.config.RPN_ANCHOR_SCALES = (8, 16, 32, 64)
resid_model.config.WEIGHT_DECAY = 0.001
resid_model.config.STEPS_PER_EPOCH = 1000
resid_model.config.VALIDATION_STEPS = 20
resid_model.config.VALIDATION_STEPS = 10
sampling_function = btk_utils.group_sampling_function
obs_function = btk_utils.custom_obs_condition
selection_function = None # btk_utils.custom_selection_function
layers = 'all'
norm = [0., 1., 0., 1.] # [0, 1, 0, 1]
input_pull = True
input_model_mapping = True
max_number = 10
print("Evaluate model:", self.model_name)
resid_model.config.display()
# resid_model.make_resid_model(catalog_name, count=BATCH_SIZE)
resid_model.model = model_btk.MaskRCNN(
mode="inference",
config=resid_model.config,
model_dir=resid_model.output_dir)
resid_model.model.load_weights(self.model_file_name, by_name=True)
self.resid_model = resid_model
def main(Args):
"""Train group blends with btk"""
norm = [1.9844158727667542, 413.83759806375525,
51.2789974336363, 1038.4760551905683]
input_pull = False
input_model_mapping = False
max_number = 2
count = 40000
catalog_name = os.path.join(DATA_PATH, 'OneDegSq.fits')
# Define parameters for mrcnn model with btk here
resid_model = btk_utils.Resid_btk_model(
Args.model_name, Args.model_path, MODEL_DIR, training=True,
images_per_gpu=4, validation_for_training=True)
# Load parameters for dataset and load model
resid_model.config.WEIGHT_DECAY = 0.001
resid_model.config.STEPS_PER_EPOCH = 1000
resid_model.config.VALIDATION_STEPS = 20
sampling_function = None
layers = 'all'
if Args.model_name == 'model1':
resid_model.config.BACKBONE = 'resnet41'
elif Args.model_name == 'model2':
resid_model.config.SKIP_P2_RPN = True
resid_model.config.BACKBONE_STRIDES = [8, 16, 32, 64]
resid_model.config.RPN_ANCHOR_SCALES = (8, 16, 32, 64)
elif Args.model_name == 'model3':
resid_model.config.BACKBONE = 'resnet41'
resid_model.config.SKIP_P2_RPN = True
resid_model.config.BACKBONE_STRIDES = [8, 16, 32, 64]
resid_model.config.RPN_ANCHOR_SCALES = (8, 16, 32, 64)
elif Args.model_name == 'model4':
resid_model.config.TRAIN_BN = None
resid_model.config.BACKBONE = 'resnet41'
resid_model.config.SKIP_P2_RPN = True
resid_model.config.BACKBONE_STRIDES = [8, 16, 32, 64]
resid_model.config.RPN_ANCHOR_SCALES = (8, 16, 32, 64)
elif Args.model_name == 'model5':
resid_model.config.TRAIN_BN = None
resid_model.config.BACKBONE = 'resnet35'
resid_model.config.SKIP_P2_RPN = True
resid_model.config.BACKBONE_STRIDES = [8, 16, 32, 64]
resid_model.config.RPN_ANCHOR_SCALES = (8, 16, 32, 64)
elif Args.model_name == 'model4_large':
resid_model.config.TRAIN_BN = False
resid_model.config.BACKBONE = 'resnet41'
resid_model.config.SKIP_P2_RPN = True
resid_model.config.BACKBONE_STRIDES = [8, 16, 32, 64]
resid_model.config.RPN_ANCHOR_SCALES = (8, 16, 32, 64)
sampling_function = btk_utils.resid_general_sampling_function_large
layers = '4+' # '3+'
elif Args.model_name == 'model6':
resid_model.config.TRAIN_BN = False
resid_model.config.BACKBONE = 'resnet41'
resid_model.config.SKIP_P2_RPN = True
resid_model.config.BACKBONE_STRIDES = [8, 16, 32, 64]
resid_model.config.RPN_ANCHOR_SCALES = (8, 16, 32, 64)
sampling_function = btk_utils.resid_general_sampling_function_large
layers = 'all'
norm = [0, 1, 51.2789974336363, 1038.4760551905683]
input_pull = True
elif Args.model_name == 'model7':
resid_model.config.TRAIN_BN = False
resid_model.config.BACKBONE = 'resnet41'
resid_model.config.SKIP_P2_RPN = True
resid_model.config.BACKBONE_STRIDES = [8, 16, 32, 64]
resid_model.config.RPN_ANCHOR_SCALES = (8, 16, 32, 64)
sampling_function = btk_utils.resid_general_sampling_function_large
layers = 'all'
norm = [0, 1, 0, 1]
input_pull = True
input_model_mapping = True
elif Args.model_name == 'model8': # stretch = 0.1, Q = 3
resid_model.config.TRAIN_BN = None
resid_model.config.BACKBONE = 'resnet41'
resid_model.config.SKIP_P2_RPN = True
resid_model.config.BACKBONE_STRIDES = [8, 16, 32, 64]
resid_model.config.RPN_ANCHOR_SCALES = (8, 16, 32, 64)
sampling_function = btk_utils.resid_general_sampling_function_large
layers = 'all'
norm = [0, 1, 0, 1]
input_pull = True
input_model_mapping = True
elif Args.model_name == 'model9': # stretch = 2000, Q = 0.5
resid_model.config.TRAIN_BN = None
resid_model.config.BACKBONE = 'resnet41'
resid_model.config.SKIP_P2_RPN = True
resid_model.config.BACKBONE_STRIDES = [8, 16, 32, 64]
resid_model.config.RPN_ANCHOR_SCALES = (8, 16, 32, 64)
sampling_function = btk_utils.resid_general_sampling_function_large
layers = 'all'
norm = [0, 1, 0, 1]
input_pull = True
input_model_mapping = True
elif Args.model_name == 'model10': # stretch = 2000, Q = 0.5
resid_model.config.TRAIN_BN = None
resid_model.config.BACKBONE = 'resnet41'
resid_model.config.SKIP_P2_RPN = True
resid_model.config.BACKBONE_STRIDES = [8, 16, 32, 64]
resid_model.config.RPN_ANCHOR_SCALES = (8, 16, 32, 64)
sampling_function = btk_utils.resid_general_sampling_function_large
layers = 'all'
norm = [0., 1., 0, 1.] # [0, 1, 0, 1]
input_pull = True
input_model_mapping = True
elif Args.model_name == 'model10_again': # stretch = 2000, Q = 0.5
resid_model.config.TRAIN_BN = None
resid_model.config.BACKBONE = 'resnet41'
resid_model.config.SKIP_P2_RPN = True
resid_model.config.BACKBONE_STRIDES = [8, 16, 32, 64]
resid_model.config.RPN_ANCHOR_SCALES = (8, 16, 32, 64)
sampling_function = btk_utils.resid_general_sampling_function_large
layers = 'all'
norm = [0., 1.45, 0, 1.] # [0, 1, 0, 1]
input_pull = True
input_model_mapping = True
elif Args.model_name == 'model10_again2': # stretch = 2000, Q = 0.5
resid_model.config.TRAIN_BN = None
resid_model.config.BACKBONE = 'resnet41'
resid_model.config.SKIP_P2_RPN = True
resid_model.config.BACKBONE_STRIDES = [8, 16, 32, 64]
resid_model.config.RPN_ANCHOR_SCALES = (8, 16, 32, 64)
sampling_function = btk_utils.resid_general_sampling_function_large
layers = 'all'
norm = [0., 1.45, 0, 1.] # [0, 1, 0, 1]
input_pull = True
input_model_mapping = True
elif Args.model_name == 'model10_again3': # stretch = 2000, Q = 0.5
resid_model.config.TRAIN_BN = False
resid_model.config.BACKBONE = 'resnet41'
resid_model.config.SKIP_P2_RPN = True
resid_model.config.BACKBONE_STRIDES = [8, 16, 32, 64]
resid_model.config.RPN_ANCHOR_SCALES = (8, 16, 32, 64)
sampling_function = btk_utils.resid_general_sampling_function_large
layers = 'all'
norm = [0., 1.45, 0, 1.] # [0, 1, 0, 1]
input_pull = True
input_model_mapping = True
elif Args.model_name == 'model10_2': # stretch = 2000, Q = 0.5
resid_model.config.TRAIN_BN = False
resid_model.config.BACKBONE = 'resnet41'
resid_model.config.SKIP_P2_RPN = True
resid_model.config.BACKBONE_STRIDES = [8, 16, 32, 64]
resid_model.config.RPN_ANCHOR_SCALES = (8, 16, 32, 64)
sampling_function = btk_utils.resid_general_sampling_function_large
layers = 'all'
norm = [0., 1.45, 0., 1.] # [0, 1, 0, 1]
input_pull = True
input_model_mapping = True
elif Args.model_name == 'model11': # stretch = 2000, Q = 0.5
resid_model.config.TRAIN_BN = False
resid_model.config.BACKBONE = 'resnet41'
resid_model.config.SKIP_P2_RPN = True
resid_model.config.BACKBONE_STRIDES = [8, 16, 32, 64]
resid_model.config.RPN_ANCHOR_SCALES = (8, 16, 32, 64)
sampling_function = btk_utils.resid_general_sampling_function_large
layers = 'all'
norm = [0., 1., 0., 1.] # [0, 1, 0, 1]
input_pull = True
input_model_mapping = True
elif Args.model_name == 'model11_2': # stretch = 2000, Q = 0.5
resid_model.config.TRAIN_BN = False
resid_model.config.BACKBONE = 'resnet41'
resid_model.config.SKIP_P2_RPN = True
resid_model.config.BACKBONE_STRIDES = [8, 16, 32, 64]
resid_model.config.RPN_ANCHOR_SCALES = (8, 16, 32, 64)
sampling_function = btk_utils.resid_general_sampling_function_large
layers = 'all'
norm = [0., 1., 0., 1.] # [0, 1, 0, 1]
input_pull = True
input_model_mapping = True
elif Args.model_name == 'model12': # stretch = 2000, Q = 0.5
resid_model.config.TRAIN_BN = False
resid_model.config.BACKBONE = 'resnet41'
resid_model.config.SKIP_P2_RPN = True
resid_model.config.BACKBONE_STRIDES = [8, 16, 32, 64]
resid_model.config.RPN_ANCHOR_SCALES = (8, 16, 32, 64)
sampling_function = btk_utils.resid_general_sampling_function_large
layers = 'all'
norm = [0., 1.45, 0, 1.] # [0, 1, 0, 1]
input_pull = True
input_model_mapping = True
max_number = 6
elif Args.model_name == 'model12_again': # stretch = 2000, Q = 0.5 # larger learning rate
resid_model.config.TRAIN_BN = False
resid_model.config.BACKBONE = 'resnet41'
resid_model.config.SKIP_P2_RPN = True
resid_model.config.BACKBONE_STRIDES = [8, 16, 32, 64]
resid_model.config.RPN_ANCHOR_SCALES = (8, 16, 32, 64)
sampling_function = btk_utils.resid_general_sampling_function_large
layers = 'all'
norm = [0., 1.45, 0, 1.] # [0, 1, 0, 1]
input_pull = True
input_model_mapping = True
max_number = 10 # changed from 6 to 10 for run 4
elif Args.model_name == 'model12_again2': # stretch = 2000, Q = 0.5 # larger learning rate val set reduced to 10
resid_model.config.TRAIN_BN = False
resid_model.config.BACKBONE = 'resnet41'
resid_model.config.SKIP_P2_RPN = True
resid_model.config.BACKBONE_STRIDES = [8, 16, 32, 64]
resid_model.config.RPN_ANCHOR_SCALES = (8, 16, 32, 64)
sampling_function = btk_utils.resid_general_sampling_function_large
layers = 'all'
norm = [0., 1.45, 0, 1.] # [0, 1, 0, 1]
input_pull = True
input_model_mapping = True
max_number = 6
resid_model.config.VALIDATION_STEPS = 10
else:
raise AttributeError("model not found", Args.model_name)
print("Train in model:", Args.model_name)
resid_model.config.display()
resid_model.make_resid_model(catalog_name, count=count,
max_number=max_number, augmentation=True,
norm_val=norm, input_pull=input_pull,
sampling_function=sampling_function,
input_model_mapping=input_model_mapping)
learning_rate = resid_model.config.LEARNING_RATE/10.
np.random.seed(Args.epochs)
history = resid_model.model.train(resid_model.dataset,
resid_model.dataset_val,
learning_rate=learning_rate,
epochs=Args.epochs,
layers=layers)
name = Args.model_name + '_run2'
with open(name + ".dill", 'wb') as handle:
dill.dump(history.history, handle)
learning_rate = resid_model.config.LEARNING_RATE/10.
np.random.seed(Args.epochs + 10)
history = resid_model.model.train(resid_model.dataset,
resid_model.dataset_val,
learning_rate=learning_rate,
epochs=Args.epochs+10,
layers=layers)
name = Args.model_name + '_run3'
with open(name + ".dill", 'wb') as handle:
dill.dump(history.history, handle)
def main(Args):
"""Train group blends with btk"""
norm = [0., 1, 0, 1]
input_pull = True
input_model_mapping = True
max_number = 10
count = 40000
train_catalog_name = os.path.join(DATA_PATH,
'train_dc2_i_30min_snr_1.fits')
val_catalog_name = os.path.join(DATA_PATH, 'val_dc2_i_30min_snr_1.fits')
train_wld_catalog_name = os.path.join(DATA_PATH,
'train_group_i_30min_snr_1.fits')
val_wld_catalog_name = os.path.join(DATA_PATH,
'val_group_i_30min_snr_1.fits')
# Define parameters for mrcnn model with btk here
resid_model = btk_utils.Resid_btk_model(Args.model_name,
Args.model_path,
MODEL_DIR,
training=True,
images_per_gpu=4,
validation_for_training=True,
i_mag_lim=27)
# Load parameters for dataset and load model
resid_model.config.WEIGHT_DECAY = 0.001
resid_model.config.STEPS_PER_EPOCH = 1000
resid_model.config.VALIDATION_STEPS = 20
sampling_function = None
obs_function = None
selection_function = None
layers = 'all'
multiprocess = False
if Args.model_name == 'model1_dc2':
resid_model.config.TRAIN_BN = False
resid_model.config.BACKBONE = 'resnet41'
resid_model.config.SKIP_P2_RPN = True
resid_model.config.BACKBONE_STRIDES = [8, 16, 32, 64]
resid_model.config.RPN_ANCHOR_SCALES = (8, 16, 32, 64)
sampling_function = btk_utils.group_sampling_function
obs_function = btk_utils.custom_obs_condition
selection_function = None # btk_utils.custom_selection_function
layers = 'all'
norm = [0., 1., 0., 1.] # [0, 1, 0, 1]
input_pull = True
input_model_mapping = True
max_number = 10
resid_model.config.VALIDATION_STEPS = 10
elif Args.model_name == 'model2_dc2':
resid_model.config.TRAIN_BN = None
resid_model.config.BACKBONE = 'resnet41'
resid_model.config.SKIP_P2_RPN = True
resid_model.config.BACKBONE_STRIDES = [8, 16, 32, 64]
resid_model.config.RPN_ANCHOR_SCALES = (8, 16, 32, 64)
sampling_function = btk_utils.group_sampling_function_numbered
obs_function = btk_utils.custom_obs_condition
selection_function = None # btk_utils.custom_selection_function
layers = 'all'
norm = [0., 1., 0., 1.] # [0, 1, 0, 1]
input_pull = True
input_model_mapping = True
max_number = 10
resid_model.config.VALIDATION_STEPS = 10
multiprocess = True
else:
raise AttributeError("model not found", Args.model_name)
print("Train in model:", Args.model_name)
resid_model.config.display()
resid_model.make_resid_model(train_catalog_name,
count=count,
max_number=max_number,
augmentation=True,
norm_val=norm,
input_pull=input_pull,
sampling_function=sampling_function,
obs_condition=obs_function,
input_model_mapping=input_model_mapping,
selection_function=selection_function,
wld_catalog_name=train_wld_catalog_name,
val_wld_catalog_name=val_wld_catalog_name,
val_catalog_name=val_catalog_name,
multiprocess=multiprocess)
learning_rate = resid_model.config.LEARNING_RATE / 10.
np.random.seed(Args.epochs)
history = resid_model.model.train(resid_model.dataset,
resid_model.dataset_val,
learning_rate=learning_rate,
epochs=Args.epochs,
layers=layers)
name = Args.model_name + '_run1'
with open(name + ".dill", 'wb') as handle:
dill.dump(history.history, handle)
learning_rate = resid_model.config.LEARNING_RATE / 15.
np.random.seed(Args.epochs + 10)
history = resid_model.model.train(resid_model.dataset,
resid_model.dataset_val,
learning_rate=learning_rate,
epochs=Args.epochs + 10,
layers=layers)
name = Args.model_name + '_run2'
with open(name + ".dill", 'wb') as handle:
dill.dump(history.history, handle)