def set_all_to_image_coords(folder):
for scanfiles in utils.image_iterator(folder):
fits = scanfiles[0]
regions = scanfiles[1]
for band in range(4):
if fits[band] and regions[band]:
image_methods.set_region_type_to_image(regions[band],
fits[band])
def test_image_iterator(self):
for scanfiles in utils.image_iterator(FITS_FOLDER):
for band in range(3):
fits_file = scanfiles[0][band]
if fits_file == "":
continue
reg_file = scanfiles[1][band]
imarray, header = image_methods.get_image_data(fits_file)
reg_mask = image_methods.get_region_masks(fits_file, reg_file)
def add_annotations(folder, im_type="all_bands", reannotate=False):
if im_type == "all_bands":
iterator = ((x[0][i], x[1][i]) for i in range(4) \
for x in utils.image_iterator(folder))
else:
iterator = utils.image_iterator(folder)
print("\n\n\n\n\nAdding annotations\n\n\n\n\n\n\n\n\n\n\n")
for scanfiles in iterator:
if im_type == "composite":
fits, reg = scanfiles[0], scanfiles[2][0]
if any(fits) and (reg == "" or reannotate):
fname = str([x for x in fits if x != ""][0])
reg_name = f"{fname[:fname.find('-')]}-comp.reg"
size = IMSIZE
if reg == "":
reg_string = image_methods.annotate_composite(fits)
else:
reg_string = image_methods.annotate_composite(fits, reg)
else:
reg_string = utils.get_text_in_file(reg)
else:
if im_type == "band_3":
fits, reg = scanfiles[0][2], scanfiles[1][2]
else:
fits, reg = scanfiles
if (reg == "" or reannotate) and fits != "":
if reg == "":
reg_string = image_methods.annotate_image(fits)
else:
reg_string = image_methods.annotate_image(fits, reg)
else:
reg_string = utils.get_text_in_file(reg)
reg_name = f"{fits[:-5]}.reg"
size = image_methods.get_image_data(fits)[0].shape[0]
with open(reg_name, "w") as f:
f.write(reg_string)
print("GOING TO REGION")
image_methods.annotate_region_with_size(reg_name, size)
def test_untrained_comet_detection(self):
for images in utils.image_iterator(f"{BASE_DIR}/not_trained_comets"):
fits = images[0]
arrs = [image_methods.get_image_data(f)[0] for f in fits]
preprod = [
image_methods.preprocess(arr, resize=False) for arr in arrs
]
resized = [image_methods.resize_image(pre, 512) for pre in preprod]
comp = image_methods.get_composite_image(resized)
image_methods.show(comp)
answer = image_methods.detect_comet(preprod,
do_show=True,
im_type="composite")
print(answer)
def test_pipeline(self):
x = utils.image_iterator(FITS_FOLDER)
for _ in range(160):
scanfiles = next(x)
print(scanfiles[0])
ims = scanfiles[0]
ims = [image_methods.get_image_data(im)[0] for im in ims]
origs = ims.copy()
image_methods.show_many(np.array(origs))
preprod = [image_methods.preprocess(im, resize=False) for im in ims]
resized = [image_methods.resize_image(im, 512) for im in preprod]
image_methods.show_many(np.array(origs), row_2=np.array(resized))
composite = image_methods.get_composite_image(resized)
image_methods.show(composite, use_log=False)
image_methods.detect_comet(composite,
do_show=True,
im_type="composite")
def create_full_record(input_folder,
output_path,
im_type="all_bands",
classes=["comet"]):
output_folder = output_path[:output_path.rindex("/")]
utils.make_folder_if_doesnt_exist(output_folder)
if os.path.exists(output_path):
os.remove(output_path)
writer = tf.python_io.TFRecordWriter(output_path)
count = 0
for scanfiles in utils.image_iterator(input_folder):
if im_type != "all_bands":
if im_type == "composite":
fits_files = [f for f in scanfiles[0] if f != ""]
elif im_type == "band_3":
fits_files = [scanfiles[0][2]]
if fits_files == [""]:
continue
reg_files = [scanfiles[1][2]]
print(count)
print(fits_files)
print(reg_files)
rec = create_image_record(fits_files, reg_files, classes)
writer.write(rec.SerializeToString())
count += 1
else:
# iterates over the 4 bands
for band in range(4):
fits_file = [scanfiles[0][band]]
reg_file = [scanfiles[1][band]]
if fits_file != [""]:
print(count)
print(fits_file)
print(reg_file)
rec = create_image_record(fits_file, reg_file, classes)
writer.write(rec.SerializeToString())
count += 1
writer.close()
print(count)
def clean_up_standardized_folder(folder, add_annotations=True):
"""
add_annotations is whether or not the program should prompt if missing
annotations
takes a standardized folder structure:
Folder:
scanframe:
scanframe-w{band}(blah).fits
OR
scanframe-w{band}(blah).jpg
...
AND
scan(frame)-w{band}(blah).reg
OR
None
...
...
and cleans up filenames and missing files
Folder:
scanframe:
scanframe-w{band}.fits
...
AND
scanframe-w{band}.reg
...
...
---
NOTE: do not include w1, w2, w3, or w4 in the folder name. It must only
be where it is expected, after the scanframe
"""
for images in utils.image_iterator(folder):
im_files = images[0]
reg_files = images[1]
comp_reg = images[2][0]
size = 512
if comp_reg == "":
im_files = [f for f in im_files if f != ""]
j = im_files[0].rfind("-")
comp_reg = f"{im_files[0][:j]}-comp.reg"
comp_reg_string = image_methods.annotate_composite(im_files,
size=size)
with open(comp_reg, "w") as f:
f.write(comp_reg_string)
image_methods.annotate_region_with_size(comp_reg, size)
for band in range(4):
im = im_files[band]
if im == "":
continue
index = im.find(f"w{band + 1}")
if index == -1:
index = im.find(".") - 2
if im.endswith(".jpg"):
image_methods.convert_jpg_to_fits(im)
im = f"{im[:-4]}.fits"
new_im = f"{im[:index + 2]}.fits"
im_files[band] = new_im
os.rename(im, new_im)
reg = reg_files[band]
new_reg = f"{im[:index + 2]}.reg"
if reg == "" and add_annotations:
if comp_reg != "":
print("Composite region for reference:")
print(utils.get_text_in_file(comp_reg))
reg_string = image_methods.annotate_image(new_im)
with open(new_reg, "w") as f:
f.write(reg_string)
else:
reg_files[band] = new_reg
if os.path.exists(reg):
os.rename(reg, new_reg)
if add_annotations:
image_methods.set_region_type_to_image(new_reg, new_im)
size = image_methods.get_image_data(new_im)[0].shape[0]
if add_annotations:
image_methods.annotate_region_with_size(new_reg, size)
def get_normed_data(image_folder, file_to_save=None, file_for_orig=None,
mode="comet", reg_files = True):
"""
IMAGE_FOLDER: (str) path to original WISE images
FILE_TO_SAVE: (str) path to put numpy file of preprocessed data or None
FILE_FOR_ORIG: (str) path to put numpy file of original data or None
MODE: (str) in ["comet", "one_star", "mul_star", "defect"]
REG_FILES: (bool) whether or not there are reg files for preprocessing
---
RETURNS: (4d np.array) images zoomed in around the region file and
preprocessed and sorted by band
images[2] is an array of images from NEOWISE band 3
images[1, 9, 0, 3] is pixel [0, 3] of the tenth
image on band 2
---
SAVES TO FILE: (.npy) the returned images array
(.npy) the original images array from the IMAGE_FOLDER
"""
# start out with lists (really, they should always be lists, but saving as
# .npy files is convenient)
images = [[], [], [], []]
normal_images = [[], [], [], []]
# num is used for debugging only, to isolate specific images
num = 0
# iterates over the fits and reg files
for scanfiles in utils.image_iterator(image_folder):
# iterates over the 4 bands
for band in range(4):
fits_file = scanfiles[0][band]
if reg_files:
reg_file = scanfiles[1][band]
print(f"processing {fits_file}")
# if there is no fits file corresponding to this band, skip this
if fits_file != "":
imarray, header = get_image_data(fits_file)
# keep reference to the original image
normal_images[band].append(imarray)
# get all region masks in the image with corresponding mode
region_masks = []
if reg_files:
region_masks = get_region_masks(fits_file, reg_file, mode)
for reg_mask in region_masks:
# process image here
im = get_bbox_subimage(imarray, reg_mask)
im = preprocess(im)
images[band].append(im)
num += 1
# np.array for easy saving
images = [np.array(ims) for ims in images]
images = np.array(images)
# only save to file if a path is given, otherwise don't worry about it
if file_to_save:
np.save(file_to_save, images)
if file_for_orig:
np.save(file_for_orig, normal_images)
return images