def test_upscale_factor(outdir, img):
for vec in np.random.random((20, 2)):
print 'Testing shift vector of {}'.format(vec)
upscale_imgs = []
upscales = (50, 100, 200, 300, 400)
for i in upscales:
print 'Running upscale factor {}'.format(i)
img_cpy_str = np.copy(img)
img_cpy_str = shift_gal.shift_img(img_cpy_str,
vec,
i,
check_count=False)
upscale_imgs.append(np.copy(img_cpy_str))
org_stars, star_diffs = [], []
p = os.path.join(outdir, 'temp.fits')
load_gals.save_fits(img, p)
for s in load_gals.get_sextractor_points(p):
if s.class_prob > class_prob:
try:
org_stars.append(find_center.estimate_center(img, s))
except:
pass
os.remove(p)
for up_img in upscale_imgs:
load_gals.save_fits(up_img, p)
stars = []
for s in load_gals.get_sextractor_points(p):
if s.class_prob > class_prob:
try:
stars.append(find_center.estimate_center(up_img, s))
except:
pass
src, trg = shift_gal.find_like_points(org_stars, stars)
total_dist = sum([
np.sqrt((s.x + vec[0] - t.x)**2 + (s.y + vec[1] - t.y)**2)
for s, t in zip(src, trg)
])
star_diffs.append(total_dist / len(src))
os.remove(p)
plt.figure()
plt.plot(upscales, star_diffs)
plt.title('Mean Difference in Star Location {}'.format(vec))
plt.xlabel('Upscale Factor')
plt.ylabel('Mean Location Error')
plt.savefig(
os.path.join(outdir,
'upscale_single_shift_star_diff_{}.png'.format(vec)))
def test_delta_error(outdir, gal, vectors):
"""Shifts and image by delta and then determines what delta is without knowing
it and returns the distance between the two"""
org = np.pad(np.copy(gal[0].data), 2, 'constant')
img = np.copy(org)
p = os.path.join(outdir, 'temp.fits')
load_gals.save_fits(gal, p, isObj=True)
org_stars = []
for s in load_gals.get_sextractor_points(p):
if s.class_prob > class_prob:
try:
org_stars.append(find_center.estimate_center(org, s))
except:
pass
tmp_gal = load_gals.load_fits(p, returnObj=True)
shifted_imgs = [
np.copy(
shift_gal.shift_img(img, vector, upscale_factor,
check_count=False)) for vector in vectors
]
gal = galaxy.Galaxy({
'src': gal,
'trg': tmp_gal
}, {
'src': org_stars,
'trg': []
}, None, 'test')
estimated_vectors = []
for simg in shifted_imgs:
tmp_gal[0].data = simg
gal.gal_dict.update({'trg': tmp_gal})
load_gals.save_fits(tmp_gal, p, isObj=True)
stars = [
s for s in load_gals.get_sextractor_points(p)
if s.class_prob > class_prob
]
gal.stars_dict.update({'trg': stars})
vecs, _ = shift_gal.get_galaxy_vectors(gal, 'src', org_stars, 2)
estimated_vectors.append(vecs['trg'])
return np.array([
np.sqrt((v[0] + ev[0])**2 + (v[1] + ev[1])**2)
for v, ev in zip(vectors, estimated_vectors)
])
def process_galaxy(galaxy, out_dir, border_size, save_type, min_stars_template,
min_stars_all, upscale_factor, crop_images, run_in_parallel,
max_memory, compressOutput):
try:
# set up output directory
p = os.path.join(out_dir, galaxy.name)
if os.path.exists(p): shutil.rmtree(p)
os.mkdir(p)
global tsv_out
print('--- Processing galaxy {} ---'.format(galaxy.name))
print 'Fitting models to all stars found...'
fit_all_stars(galaxy)
template_color = find_template_gal_and_stars(galaxy,
min_stars_template)
template_cpy = np.copy(galaxy.images(template_color))
print('Reference waveband chosen is {} with {} stars'.format(
template_color, len(galaxy.stars_dict[template_color])))
shift_vectors = get_galaxy_vectors(galaxy, template_color,
min_stars_all)
# crop the images to be only the galaxy
if crop_images:
left, right, top, bottom = galaxy.crop_images_to_galaxy()
print 'Cropped images down to include only the galaxy | X: ({}, {}) | Y: ({}, {})'.format(
left, right, top, bottom)
# add a border around them to preserve flux on edges
b_size = int(galaxy.width * border_size)
b_size = 1 if b_size < 1 else b_size
galaxy.add_borders(b_size)
print 'Added border to all images of size {}'.format(b_size)
# save averaged image for testing purposes
avg = np.zeros(galaxy.images('i').shape)
for img in (galaxy.images('g'), galaxy.images('i'), galaxy.images('r'),
galaxy.images('z')):
avg += (img / 4)
load_gals.save_fits(avg, os.path.join(p, 'average_pre_shift.fits'))
# shift the images
shift_imgs = shift_wavebands(galaxy, shift_vectors, template_color,
upscale_factor, run_in_parallel,
max_memory)
def len_stars(stars):
return 'NULL' if stars is None else len(stars)
def print_stars(stars):
return 'NULL' if stars is None else tuple(
(s.info() for s in stars))
def print_vec(vec):
return 'NULL' if vec is None else str(tuple(vec))
# save shifed average for testing purposes
avg = np.zeros(galaxy.images('i').shape)
for img in shift_imgs.values():
avg += (img / 4)
load_gals.save_fits(avg, os.path.join(p, 'average_shift.fits'))
# save output images and info
save_output(p, galaxy, shift_imgs, shift_vectors, save_type,
compressOutput)
stars = galaxy.stars_dict
tsv_print(str(galaxy.name), min_stars_template,
min_stars_all, upscale_factor, template_color,
print_vec(shift_vectors['g']), print_vec(shift_vectors['i']),
print_vec(shift_vectors['r']), print_vec(shift_vectors['u']),
print_vec(shift_vectors['z']), len_stars(stars['g']),
len_stars(stars['i']), len_stars(stars['r']),
len_stars(stars['u']), len_stars(stars['z']),
print_stars(stars['g']), print_stars(stars['i']),
print_stars(stars['r']), print_stars(stars['u']),
print_stars(stars['z']))
except NotEnoughViableWavebandsError:
print 'Not enough viable wavebands ({} of {} needed), skipping galaxy'.format(
num_viable, min_wavebands)
try:
shutil.rmtree(p)
except:
pass
except NoViableTemplateError:
print 'No viable waveband template could be found, skipping galaxy'
try:
shutil.rmtree(p)
except:
pass
except FluxNotPreservedError:
print 'Error in flux preservation in shifted images, skipping galaxy'
try:
shutil.rmtree(p)
except:
pass
except ImageTooLargeError:
print 'Input image is too large to be upscaled, try lowering the upscale factor or increasing available memory, skipping galaxy'
try:
shutil.rmtree(p)
except:
pass
except CroppingError:
print 'Error cropping galaxy images, skipping galaxy'
try:
shutil.rmtree(p)
except:
pass
# if any other unknown error appears, clean up then exit
except NoError:
print 'Other exception encountered {}, skipping galaxy'.format(e)
try:
shutil.rmtree(p)
except:
pass
finally:
galaxy.close()
try:
for dir in glob.glob('temp*'):
shutil.rmtree(dir)
except:
pass
def test_positional_error(outdir, gal, vectors):
"""Shifts the image only once and checks the positional error"""
print '\nRunning single shift tests...\n'
img = np.pad(np.copy(gal[0].data), 2, 'constant')
cycle_imgs = []
org = np.copy(img)
for vector in vectors:
print 'Running delta = {}'.format(vector)
img = shift_gal.shift_img(img,
vector,
upscale_factor,
check_count=False)
cycle_imgs.append(np.copy(img[2:-2, 2:-2]))
img = np.copy(org)
dist = [np.sqrt(v[0]**2 + v[1]**2) for v in vectors]
org_stars, star_diffs = [], []
p = os.path.join(outdir, str(np.random.ranf()) + 'temp.fits')
org = org[2:-2, 2:-2]
load_gals.save_fits(gal, p, isObj=True)
for s in load_gals.get_sextractor_points(p):
if s.class_prob > class_prob:
try:
org_stars.append(find_center.estimate_center(org, s))
except:
pass
avg_num_stars = 0
try:
for i in range(len(vectors)):
gal[0].data = cycle_imgs[i]
load_gals.save_fits(gal, p, isObj=True)
stars = []
for s in load_gals.get_sextractor_points(p):
if s.class_prob > class_prob:
try:
stars.append(
find_center.estimate_center(cycle_imgs[i], s))
except:
pass
src, trg = shift_gal.find_like_points(org_stars, stars)
src, trg = shift_gal.filter_stars(src, trg)
total_dist = [
np.sqrt((s.x + vectors[i][0] - t.x)**2 +
(s.y + vectors[i][1] - t.y)**2)
for s, t in zip(src, trg)
]
total_dist = np.sum(total_dist)
star_diffs.append(-1 if len(src) == 0 else total_dist / len(src))
avg_num_stars += len(src)
print star_diffs[-1], len(src)
if star_diffs[-1] == -1: return None, None
except load_gals.SextractorError:
print 'Source Extractor error, skipping....'
return None, None
finally:
os.remove(p)
return np.array(star_diffs), avg_num_stars / float(len(vectors))
def test_flux_error(outdir, gal, vectors):
"""Shifts the galaxy back and forth and checks flux and positional error"""
print '\nRunning double shift tests...\n'
#img = np.pad(gal[0].data[137:187, 137:187], 2, 'constant')
img = np.pad(gal[0].data, 2, 'constant')
outdir = os.path.abspath(outdir)
r = np.arange(0, 1.1, 0.1)
cycle_imgs, mean_diff, mean_diff_gal, mean_diff_bg = [], [], [], []
org = np.copy(img)
seg_img = load_gals.get_seg_img(org)
gal_val = seg_img[int(seg_img.shape[0] / 2.0), int(seg_img.shape[1] / 2.0)]
for vector in vectors:
print 'Running delta = {}'.format(vector)
img = shift_gal.shift_img(img,
vector,
upscale_factor,
check_count=False)
img = shift_gal.shift_img(img,
vector * -1,
upscale_factor,
check_count=False)
diff = np.abs(org - img)
mean_diff.append(np.mean(diff))
mean_diff_gal.append(np.mean(diff[seg_img == gal_val]))
mean_diff_bg.append(np.mean(diff[seg_img == 0]))
if vector[0] == 0.5:
diff = img - org
load_gals.save_fits(
img, os.path.join(outdir, 'shifted_{}.fits'.format(vector)))
load_gals.save_fits(
diff, os.path.join(outdir, 'residual_{}.fits'.format(vector)))
print '\n---Range in Flux of Original, Doubly-Shifted, and Residual Images---'
print 'Original: ({}, {})'.format(
np.min(org), np.max(org))
print 'Original (Galaxy Pixels): ({}, {})'.format(
np.min(org[seg_img == gal_val]),
np.max(org[seg_img == gal_val]))
print 'Original (Background Pixels): ({}, {})\n'.format(
np.min(org[seg_img == 0]), np.max(org[seg_img == 0]))
print 'Shifted: ({}, {})'.format(
np.min(img), np.max(img))
print 'Shifted (Galaxy Pixels): ({}, {})'.format(
np.min(img[seg_img == gal_val]),
np.max(img[seg_img == gal_val]))
print 'Shifted (Background Pixels): ({}, {})\n'.format(
np.min(img[seg_img == 0]), np.max(img[seg_img == 0]))
print 'Residual: ({}, {})'.format(
np.min(diff), np.max(diff))
print 'Residual (Galaxy Pixels): ({}, {})'.format(
np.min(diff[seg_img == gal_val]),
np.max(diff[seg_img == gal_val]))
print 'Residual (Background Pixels): ({}, {})\n'.format(
np.min(diff[seg_img == 0]), np.max(diff[seg_img == 0]))
cycle_imgs.append(np.copy(img))
img = np.copy(org)
print 'Saving graphs...'
dist = [np.sqrt(s**2 + s**2) for s in r]
return mean_diff, mean_diff_gal, mean_diff_bg
for i in range(len(cycle_imgs)):
plt.figure()
plt.scatter(org.flatten(), np.abs(org - cycle_imgs[i]).flatten(), s=2)
plt.title('Initial Flux Value vs Resulting Error ({}, {})'.format(
r[i], r[i]))
plt.xlabel('Initial Flux')
plt.ylabel('Absolute Error in Flux')
plt.ylim(-1, 12)
plt.savefig(
os.path.join(outdir, 'cycle_flux_pixel_error_{}.png'.format(r[i])))
plt.figure()
cutoff = 5
plt.scatter(
org[org > cutoff].flatten(),
100 *
np.abs(np.abs(org - cycle_imgs[i]) / org)[org > cutoff].flatten(),
s=2)
plt.title('Initial Flux Value vs Resulting Error ({}, {})'.format(
r[i], r[i]))
plt.xlabel('Initial Flux')
plt.ylabel('Percent Error in Flux')
plt.ylim(-3, 60)
plt.savefig(
os.path.join(outdir, 'cycle_flux_pixel_error_{}.png'.format(r[i])))
org_stars, star_diffs = [], []
p = os.path.join(outdir, 'temp.fits')
load_gals.save_fits(org, p)
for s in load_gals.get_sextractor_points(p):
if s.class_prob > class_prob:
try:
org_stars.append(find_center.estimate_center(org, s))
except:
pass
for i in range(len(r)):
load_gals.save_fits(cycle_imgs[i], p)
stars = []
for s in load_gals.get_sextractor_points(p):
if s.class_prob > class_prob:
try:
stars.append(find_center.estimate_center(cycle_imgs[i], s))
except:
pass
src, trg = shift_gal.find_like_points(org_stars, stars)
total_dist = sum([
np.sqrt((i.x - j.x)**2 + (i.y - j.y)**2) for i, j in zip(src, trg)
])
star_diffs.append(total_dist / len(src))
os.remove(p)
plt.figure()
plt.plot(dist, star_diffs)
plt.title('Mean Difference in Star Location')
plt.xlabel('Shift Distance in Pixels')
plt.ylabel('Mean Location Error')
plt.savefig(os.path.join(outdir, 'cycle_star_diff.png'),
bbox_inches='tight')
return dist, star_diffs