def __init__(self, cmdline_options=None, hdulist=None):
# set command-line options to a safe value
if (cmdline_options is None):
cmdline_options = set_default_options()
self.options = cmdline_options
self.filtername = None
self.ota = None
self.fpl = None
self.detector_glow = "yes"
self.binning = 1
self.logger = logging.getLogger("Calibrations")
if (hdulist is not None):
self.filtername = hdulist[0].header['FILTER']
self.fpl = podi_focalplanelayout.FocalPlaneLayout(inp=hdulist)
self.binning = get_binning(hdulist[0].header)
self.filter_level = get_filter_level(hdulist[0].header)
self.mastercal_dir = "%s/mastercals" % (sitesetup.exec_dir)
hdulist[0].header['FPPOS'] = 'xy%02d' % (new)
# chaneg all observing date forward by 3 years
date_format = "%Y-%m-%dT%H:%M:%S.%f"
date_obs = datetime.datetime.strptime(
hdulist[0].header['DATE-OBS'], date_format)
date_new = date_obs + datetime.timedelta(days=3 * 365)
hdulist[0].header['DATE-OBS'] = date_new.strftime(date_format)
# print date_new.strftime(date_format)
#print hdulist[0].header['DATE-OBS'], date_obs, date_new
hdulist[0].header['MJD-OBS'] += 3 * 365 # change time
fpl = podi_focalplanelayout.FocalPlaneLayout(hdulist[0])
obsid_timefmt = "%Y%m%dT%H%M%S"
#localtime = date_new - timedelta(hours=7)
#new_obsid = "%s.%d" % (date_new.strftime(obsid_timefmt), dither_number)
#print obsid
new_obsid = "%s%04d%s" % (obsid[0], int(obsid[1:5]) + 3, obsid[5:])
#print new_obsid
# Change the OTA-ID to simulate a more realistic pattern for broken cells
hdulist[0].header["OTA_ID"] = "%d" % (
fpl.get_otaid_from_position(new))
hdulist[0].header['OBSID'] = new_obsid[1:]
hdulist[0].header['FILENAME'] = "%s.%02d.fits" % (new_obsid, new)
def make_fringing_template(input_filelist, outputfile, return_hdu=False,
skymode='local', operation="nanmedian.bn",
bpm_dir=None, wipe_cells=None, ocdclean=False,
):
"""
Create a fringe template from the given list of suitable input frames.
For each frame, compute the sky-level and the sky-countrate. Frames with
sky-countrates exceeding a filter-specific level are ignored, as the sky is
very likely contaminated by stray light. This also eliminates frames with
background gradients. Each frame is then background-subtracted and
normalized by its background-level, leaving only the fringe amplitude
behind.
To eliminate sources, all data for a given extension are then
median-combined. Once this is complete for all available extensions, the
resulting fringe maps are written to the output FITS file.
"""
logger = logging.getLogger("MakeFringeTemplate")
# First loop over all filenames and make sure all files exist
hdu_filelist = []
for filename in input_filelist:
if (os.path.isfile(filename)):
with pyfits.open(filename) as hdulist:
exptime = hdulist[0].header['EXPTIME']
filter = hdulist[0].header['FILTER']
skylevel = hdulist[0].header['SKYLEVEL']
if (filter in avg_sky_countrates):
max_skylevel = 2 * avg_sky_countrates[filter] * exptime
if (skylevel > max_skylevel):
logger.info("Frame %s exceeds sky-level limitation (%.1f vs %.1f cts/s)" % (
filename, skylevel/exptime, max_skylevel))
continue
hdu_filelist.append(filename) #pyfits.open(file))
if (len(hdu_filelist) <= 0):
stdout_write("No existing files found in input list, hence nothing to do!\n")
return
# Read the input parameters
# Note that file headers are copied from the first file
# Create the primary extension of the output file
ref_hdulist = pyfits.open(hdu_filelist[0]) #hdu_filelist[0]
primhdu = pyfits.PrimaryHDU(header=ref_hdulist[0].header)
fpl = podi_focalplanelayout.FocalPlaneLayout(ref_hdulist)
# Add PrimaryHDU to list of OTAs that go into the output file
out_hdulist = [primhdu]
filtername = primhdu.header['FILTER']
if (outputfile == "auto"):
outputfile = "fringes__%s.fits" % (filtername)
print("Output file=",outputfile)
#
# Prepare all input frames, just like for the illumination correction, i.e.
# - mask out sources
# - eliminate OTAs used for guiding
# - wipe out certain cells
#
queue = multiprocessing.JoinableQueue()
return_queue = multiprocessing.Queue()
logger.info("Preparing all individual fringe template frames")
number_files_sent_off = 0
for fitsfile in hdu_filelist:
logger.debug("Queuing %s" % (fitsfile))
queue.put((fitsfile))
number_files_sent_off += 1
additional_sextractor_options = """
-FILTER N
-DETECT_THRESH 3
-BACK_SIZE"""
conf_file = "%s/config/fringing.conf" % (sitesetup.exec_dir)
processes = []
for i in range(sitesetup.number_cpus):
logger.debug("Starting process #%d" % (i+1))
p = multiprocessing.Process(target=podi_illumcorr.compute_illumination_frame,
kwargs = {'queue': queue,
'return_queue': return_queue,
'tmp_dir': sitesetup.swarp_singledir,
'redo': True,
'mask_guide_otas': True, #mask_guide_otas,
'mask_regions': None, #mask_regions,
'bpm_dir': bpm_dir,
'wipe_cells': wipe_cells,
'ocdclean': ocdclean,
'apply_correction': False,
'conf_file': conf_file,
},
# args=(queue, return_queue),
)
queue.put(None)
p.start()
processes.append(p)
masked_list = []
for i in range(number_files_sent_off):
masked_frame = return_queue.get()
if (masked_frame is not None):
masked_list.append(masked_frame)
for p in processes:
p.join()
logger.info("All files prepared, combining ...")
#
# Now loop over all extensions and compute the mean
#
for extid, ext in enumerate(ref_hdulist):
# Check what OTA we are dealing with
if (not is_image_extension(ext)):
continue
data_blocks = []
extname = ext.name
if (filtername in fpl.otas_for_photometry):
useful_otas = fpl.otas_for_photometry[filtername]
ota_id = int(extname[3:5])
if (not ota_id in useful_otas):
continue
stdout_write("\rCombining frames for OTA %s (#%2d/%2d) ..." % (extname, extid, len(ref_hdulist)))
# Now open all the other files, look for the right extension, and copy their image data to buffer
#for file_number in range(0, len(filelist)):
for filename in masked_list: # hdu_filelist:
try:
hdulist = pyfits.open(filename)
this_hdu = hdulist[extname]
except:
continue
# Skip all OTAs that are marked as video/guide OTAs
cellmode = this_hdu.header['CELLMODE']
if (cellmode.find("V") >= 0):
continue
skylevel = this_hdu.header['SKY_MEDI']
if (skymode == 'global'):
skylevel = hdulist[0].header['SKYLEVEL']
if ("EXPTIME" in hdulist[0].header):
exptime = hdulist[0].header['EXPTIME']
filter = hdulist[0].header['FILTER']
if (filter in avg_sky_countrates):
max_skylevel = 2 * avg_sky_countrates[filter] * exptime
if (skylevel > max_skylevel):
stdout_write(" (%.1f)" % (skylevel/exptime))
continue
fringing = (this_hdu.data - skylevel) / skylevel
stdout_write(" %.1f" % (skylevel/exptime))
data_blocks.append(fringing)
# delete the data block to free some memory, since we won't need it anymore
hdulist.close()
stdout_write(" combining ...")
#combined = podi_imcombine.imcombine_data(data_blocks, "nanmedian")
combined = podi_imcombine.imcombine_data(data_blocks, operation) #"nanmedian.bn")
# Create new ImageHDU
# Insert the imcombine'd frame into the output HDU
# Copy all headers from the reference HDU
# stdout_write(" creating HDU ...")
hdu = pyfits.ImageHDU(header=ext.header, data=combined)
# Append the new HDU to the list of result HDUs
out_hdulist.append(hdu)
stdout_write(" done!\n")
del hdu
# Add the assumed skylevel countrate to primary header so we can use it
# when it comes to correcting actual data
out_hdulist[0].header["SKYCNTRT"] = (avg_sky_countrates[filter], "average countrate of dark sky")
return_hdu = False
out_hdu = pyfits.HDUList(out_hdulist)
if (not return_hdu and outputfile != None):
stdout_write(" writing results to file %s ..." % (outputfile))
clobberfile(outputfile)
out_hdu.writeto(outputfile, clobber=True)
out_hdu.close()
del out_hdu
del out_hdulist
stdout_write(" done!\n")
elif (return_hdu):
stdout_write(" returning HDU for further processing ...\n")
return out_hdu
else:
stdout_write(" couldn't write output file, no filename given!\n")
return
def measure_focus_ota(filename, n_stars=5):
"""
Obtain a focus measurement from teh specified filename. To do so,
1) run source extractor to get FWHM measurements and positions for all sources
2) group sources into vertical sequences as expected from the function
of the focus tool
3) assign physical focus positions to each measurement
4) return final catalog back to master so a final, combined focus curve
can be assembled.
"""
# print"\n\n\nworking on file ",filename,"\n\n\n"
try:
hdulist = pyfits.open(filename)
except IOError:
logger.debug("Can't open file %s" % (filename))
return None
except:
podi_logging.log_exception()
return None
obsid = hdulist[0].header['OBSID']
ota = hdulist[1].header['WN_OTAX'] * 10 + hdulist[1].header['WN_OTAY']
ota_id = hdulist[0].header['OTA_ID']
logger = logging.getLogger("MeasureFocusOTA: %s(%02d)" % (obsid, ota))
logger.info("Starting work ...")
obsid = hdulist[0].header['OBSID']
ota = int(hdulist[0].header['FPPOS'][2:4])
# Check the object name to see if if contains the information about the exposure
focus_positions = numpy.arange(n_stars)[::-1] + 1.
real_focus_positions = False
object_name = hdulist[0].header['OBJECT']
if (object_name.startswith("Focus Center")):
# This looks like it might be the right format
try:
items = object_name.split()
# Check all items
if (len(items) == 7 and items[0] == "Focus"
and items[1] == "Center" and items[3] == "NStep"
and items[5] == "DStep"):
n_stars = int(items[4])
focus_center = float(items[2])
focus_step = float(items[6])
focus_start = focus_center - (n_stars - 1) / 2 * focus_step
focus_positions = numpy.arange(
n_stars,
dtype=numpy.float32)[::-1] * focus_step + focus_start
logger.debug(
"Infom from header: N=%d, center=%.0f, step=%.0f, start=%.0f"
% (n_stars, focus_center, focus_Step, focus_step,
focus_start))
logger.debug("Focus positions: %s" % (str(focus_positions)))
real_focus_positions = True
except:
pass
# Run SourceExtractor on the file
sex_config = "%s/config/focus.sexconf" % (sitesetup.exec_dir)
sex_param = "%s/config/focus.sexparam" % (sitesetup.exec_dir)
catfile = "%s/tmp.%s_OTA%02d.cat" % (sitesetup.scratch_dir, obsid, ota)
sex_cmd = "%(sexcmd)s -c %(sex_config)s -PARAMETERS_NAME %(sex_param)s -CATALOG_NAME %(catfile)s %(filename)s" % {
"sexcmd": sitesetup.sextractor,
"sex_config": sex_config,
"sex_param": sex_param,
"catfile": catfile,
"filename": filename,
"redirect": sitesetup.sex_redirect,
}
# print "\n"*10,sex_cmd,"\n"*10
# Run source extractor
# catfile = "/tmp//tmp.pid4383.20121008T221836.0_OTA33.cat"
if (not os.path.isfile(catfile)):
logger.debug("Running source extractor to search for stars")
start_time = time.time()
try:
ret = subprocess.Popen(sex_cmd.split(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(sex_stdout, sex_stderr) = ret.communicate()
if (ret.returncode != 0):
logger.warning(
"Sextractor might have a problem, check the log")
logger.info("Stdout=\n" + sex_stdout)
logger.info("Stderr=\n" + sex_stderr)
except OSError as e:
podi_logging.log_exception()
print >> sys.stderr, "Execution failed:", e
end_time = time.time()
logger.debug("SourceExtractor finished after %.2f seconds" %
(end_time - start_time))
else:
logger.debug("Source catalog already exists, re-using the old file")
#
# delete the tmp catalog
#
#
# Load the source catalog.
# Handle cases of non-existing or empty catalogs
#
logger.debug("loading the source catalog from %s" % (catfile))
try:
source_cat = numpy.loadtxt(catfile)
except IOError:
logger.warning("The Sextractor catalog is empty, ignoring this OTA")
source_cat = None
return None
if (source_cat.shape[0] <= 0):
# no sources found
return None
# print "\n\n total sources in raw file",source_cat.shape,"\n\n"
logger.debug("Found %d sources in raw SourceExtractor catalog %s" %
(source_cat.shape[0], catfile))
#
# Now convert all X/Y values to proper OTA X/Y coordinates based on their
# extension number
#
corr_cat = None
# print "extensions:", source_cat[:,SXFocusColumn['extension']]
for i in range(len(hdulist)):
if (not is_image_extension(hdulist[i])):
# print "skipping extension",i,", this is not an image extension"
continue
# get cell_x, cell_y from this extension
cell_x = hdulist[i].header['WN_CELLX']
cell_y = hdulist[i].header['WN_CELLY']
x1, c2, y1, y2 = cell2ota__get_target_region(cell_x, cell_y, 1)
# Create a mask for all sources in this cell
in_this_cell = (source_cat[:, SXFocusColumn['extension']] == i)
if (numpy.sum(in_this_cell) <= 0):
logger.debug("Couldn't find any sources in cell %d,%d" %
(cell_x, cell_y))
continue
# print "found",numpy.sum(in_this_cell),"sources for cell",cell_x, cell_y, " adding", x1, y1
cell_cat = source_cat[in_this_cell]
cell_cat[:, SXFocusColumn['x']] += x1
cell_cat[:, SXFocusColumn['y']] += y1
#
# get overscan-level data for this cell
#
binning = get_binning(hdulist[i].header)
overscan_data = extract_biassec_from_cell(hdulist[i].data, binning)
overscan_level = numpy.mean(overscan_data)
cell_cat[:, SXFocusColumn['background']] -= overscan_level
corr_cat = cell_cat if corr_cat is None else numpy.append(
corr_cat, cell_cat, axis=0)
#
# Attach to each measurement what detector lot it came from
#
fpl = podi_focalplanelayout.FocalPlaneLayout(hdulist)
detector_lot = fpl.get_detector_generation(ota_id)
corr_cat[:, SXFocusColumn['ota_lot']] = detector_lot
#
# Also override the extension number in the source catalog with the
# position in the overall focal plane
#
corr_cat[:, SXFocusColumn['extension']] = ota
# print "\n\n\n\ntotal corrected catalog:",corr_cat.shape
# save the source catalog
#numpy.savetxt("focus_cat.ota%02d" % (ota), source_cat)
#numpy.savetxt("focus_cat2.ota%02d" % (ota), corr_cat)
logger.debug("done fixing the pixel coordinates")
# only select bright enough sources
bright_enough = corr_cat[:, SXFocusColumn['mag_auto']] < -10
corr_cat = corr_cat[bright_enough]
#numpy.savetxt("focus_cat3.ota%02d" % (ota), corr_cat)
#dummy_test = open("dummy.test", "w")
# Now try to match up stars in a sequence
all_angles, all_distances = [], []
for s1 in range(corr_cat.shape[0]):
# Assume this is the middle star in the sequence
# Find all stars above and below it in a cone
# compute the angle to all other stars in the catalog
dx = corr_cat[:, SXFocusColumn['x']] - corr_cat[s1, 2]
dy = corr_cat[:, SXFocusColumn['y']] - corr_cat[s1, 3]
d_total = numpy.hypot(dx, dy)
in_cone = numpy.fabs(dx / dy) < 0.1
# Need to be at most n_stars * 10'' and at least 5''
close_enough = (d_total <
(n_stars * 10. / 0.11)) & (d_total > 5 / 0.11)
good_so_far = in_cone & close_enough
if (numpy.sum(good_so_far) <= 0):
continue
candidates = corr_cat[good_so_far]
# if (numpy.sum(candidates) < n_stars):
# # Only use full sequences
# continue
# are the magnitudes comparable
delta_mag = numpy.fabs(candidates[:,SXFocusColumn['mag_auto']] \
- corr_cat[s1,SXFocusColumn['mag_auto']])
similar_brightness = delta_mag < 1
#print >>dummy_test, "#", candidates.shape[0], numpy.sum(similar_brightness)
#print similar_brightness
if (numpy.sum(similar_brightness) <= 0):
continue
good_candidates = candidates[similar_brightness]
# Now sort the data with increasing y values
si = numpy.argsort(good_candidates[:, SXFocusColumn['y']])
sorted_candidates = good_candidates[si]
#numpy.savetxt(dummy_test, sorted_candidates)
#print >>dummy_test, "\n\n\n"
# Now compute the slope and distance between each point and each point
# above it
for p1, p2 in itertools.combinations(range(sorted_candidates.shape[0]),
2):
angle = numpy.arctan2(
sorted_candidates[p1, 2] - sorted_candidates[p2, 2],
sorted_candidates[p1, 3] - sorted_candidates[p2, 3])
distance = numpy.sqrt(
(sorted_candidates[p1, 2] - sorted_candidates[p2, 2])**2 +
(sorted_candidates[p1, 3] - sorted_candidates[p2, 3])**2)
all_angles.append(angle)
all_distances.append(distance)
#dummy_test.close()
# Once we are through with the first iteration find the best-fitting angle
all_angles = numpy.array(all_angles)
all_distances = numpy.array(all_distances)
# Find the best or rather most frequently occuring angle
all_angles[all_angles < 0] += 2 * math.pi
#numpy.savetxt("dummy.angles", all_angles)
#numpy.savetxt("dummy.distances", all_distances)
filtered_angles = three_sigma_clip(all_angles)
if (filtered_angles is None or filtered_angles.ndim < 1
or filtered_angles.shape[0] <= 0):
return None
angle_width = scipy.stats.scoreatpercentile(filtered_angles, [16, 84])
angle_width = scipy.stats.scoreatpercentile(filtered_angles, [5, 95])
logger.debug(
"Found median angle %f [%f ...%f]" %
(numpy.degrees(numpy.median(filtered_angles)),
numpy.degrees(angle_width[0]), numpy.degrees(angle_width[1])))
#
# Now we can do another proper search for all stars
# This time, only search for complete series (#stars as specified)
#
#focus_stars = open("focus_stars", "w")
all_candidates = []
for s1 in range(corr_cat.shape[0]):
# Find all stars above and below it in a cone
# compute the angle to all other stars in the catalog
dx = corr_cat[:, SXFocusColumn['x']] - corr_cat[s1, 2]
dy = corr_cat[:, SXFocusColumn['y']] - corr_cat[s1, 3]
angles = numpy.arctan2(dx, dy)
angles[angles < 0] += 2 * math.pi
d_total = numpy.hypot(dx, dy)
#print numpy.degrees(angle_width), numpy.degrees(angles)
#print angle_width[0], angle_width[1]
in_cone1 = (angles > angle_width[0]) & (angles < angle_width[1])
in_cone2 = (angles + math.pi > angle_width[0]) & (angles + math.pi <
angle_width[1])
in_cone = in_cone1 | in_cone2
# print angles[in_cone]
#print angles[in_cone][0], angles[in_cone][0] > angle_width[0], angles[in_cone][0] < angle_width[1]
close_enough = (d_total <
((n_stars + 1) * 10. / 0.11)) & (d_total > 5 / 0.11)
similar_brightness = numpy.fabs(corr_cat[:,SXFocusColumn['mag_auto']] \
- corr_cat[s1,SXFocusColumn['mag_auto']]) < 1
good = in_cone & close_enough & similar_brightness
good[s1] = True
# print s1, ":", numpy.sum(in_cone), numpy.sum(close_enough), numpy.sum(similar_brightness), numpy.sum(good)
if (numpy.sum(good) <= 1):
continue
candidates = corr_cat[good]
# print "# canddates =", candidates.shape[0]
if (not candidates.shape[0] == n_stars):
# Only use full sequences
continue
pass
# print "found match:",s1
# Now we have a set with the right number of stars, matching the overall
# angle, and with similar brightnesses
# sort them top to bottom
si = numpy.argsort(candidates[:, 3])
#numpy.savetxt(focus_stars, candidates[:,3])
#numpy.savetxt(focus_stars, si)
sorted_candidates = candidates[si]
# print "XXX", sorted_candidates.shape, sorted_candidates[:,0].shape, focus_positions.shape, n_stars, candidates.shape[0]
sorted_candidates[:, 0] = focus_positions
#numpy.savetxt(focus_stars, sorted_candidates)
#numpy.savetxt(focus_stars, numpy.degrees(angles[good]))
#numpy.savetxt(focus_stars, in_cone[good])
#numpy.savetxt(focus_stars, d_total[good])
#numpy.savetxt(focus_stars, (corr_cat[:,10] - corr_cat[s1,10])[good])
#print >>focus_stars, "\n\n\n\n"
all_candidates.append(sorted_candidates)
#focus_stars.close()
all_candidates = numpy.array(all_candidates)
logger.debug(str(all_candidates.shape))
#xxx = open("steps", "w")
# Now compute the distances from each star to the previous
step_vectors = []
for i in range(1, n_stars):
# logger.debug("Candidates: %d %s\n%s" % (all_candidates.ndim, str(all_candidates.shape), str(all_candidates)))
if (all_candidates.ndim < 1 or all_candidates.shape[0] <= 0):
# We ran out of candidates
logger.debug("We ran out of viable candidates after %s stars" %
(i))
return None
steps = all_candidates[:, i, 2:4] - all_candidates[:, i - 1, 2:4]
#numpy.savetxt(xxx, steps)
#print >>xxx, "\n\n\n\n"
logger.debug("Computing average step size, star %d" % (i))
logger.debug("Steps-X:\n%s" % (str(steps[:, 0])))
logger.debug("Steps-y:\n%s" % (str(steps[:, 1])))
clean_dx = three_sigma_clip(steps[:, 0])
clean_dy = three_sigma_clip(steps[:, 1])
# Check if both clean_dx and clean_dy are not empty
logger.debug("clean-dx=%s" % (str(clean_dx)))
logger.debug("clean-dy=%s" % (str(clean_dy)))
if (clean_dx.ndim < 1 or clean_dx.shape[0] <= 0 or clean_dy.ndim < 1
or clean_dy.shape[0] <= 0):
logger.debug("Can't find a clean dx/dy shift in iteration %d" %
(i))
return None
dx = numpy.median(clean_dx)
dy = numpy.median(clean_dy)
distx = scipy.stats.scoreatpercentile(clean_dx, [16, 84])
disty = scipy.stats.scoreatpercentile(clean_dy, [16, 84])
sigma_x = 0.5 * (distx[1] - distx[0])
sigma_y = 0.5 * (disty[1] - disty[0])
step_vectors.append([dx, dy, sigma_x, sigma_y])
good_steps = (steps[:,0] > (dx - 3*sigma_x)) & (steps[:,0] < (dx + 3*sigma_x)) \
& (steps[:,1] > (dy - 3*sigma_y)) & (steps[:,1] < (dy + 3*sigma_y))
logger.debug("before step-matching #%d: %s" %
(i, str(all_candidates.shape)))
all_candidates = all_candidates[good_steps]
logger.debug("after step-matching: #%d: %s" %
(i, str(all_candidates.shape)))
logger.debug("%s: %s" % (filename, str(step_vectors)))
# final_focus = open("final_focus", "w")
# for i in range(all_candidates.shape[0]):
# numpy.savetxt(final_focus, all_candidates[i])
# print >>final_focus, "\n\n\n\n\n"
# final_focus.close()
logger.debug("Found %d focus stars" % (all_candidates.shape[0]))
return all_candidates, real_focus_positions
logger.debug("Returning final FITS table catalog")
return None
def normalize_flatfield(filename,
outputfile,
binning_x=8,
binning_y=8,
repeats=3,
batchmode_hdu=None,
normalize_otas=None):
logger = logging.getLogger("NormFlatField")
logger.debug("Starting to normalize %s" % (str(batchmode_hdu)))
if (batchmode_hdu is not None):
hdulist = batchmode_hdu
else:
hdulist = pyfits.open(filename)
filter = hdulist[0].header['FILTER']
fpl = podi_focalplanelayout.FocalPlaneLayout(hdulist)
list_of_otas_to_normalize = fpl.get_science_area_otas(
filter, include_vignetted=False)
if (normalize_otas is not None):
list_of_otas_to_normalize = normalize_otas
logger.info("Using these OTAs to normalize overall flux:\n%s" %
(", ".join(["%02d" % ota
for ota in list_of_otas_to_normalize])))
flatfield_data = numpy.zeros(shape=(len(list_of_otas_to_normalize) * 4096 *
4096 // (binning_x * binning_y)),
dtype=numpy.float32)
flatfield_data[:] = numpy.NaN
# also prepare to store the global gain value
gain_sum = 0
gain_count = 0
datapos = 0
for extension in range(1, len(hdulist)): #hdulist[1:]:
if (not is_image_extension(hdulist[extension])):
continue
fppos = int(hdulist[extension].header['FPPOS'][2:4])
#print list_of_otas_to_normalize
try:
index = list_of_otas_to_normalize.index(fppos)
except:
# We didn't find this OTA in the list, so skip it
hdulist[extension].header["FF_NORM"] = (False,
"Used in normalization")
extension += 1
continue
hdulist[extension].header["FF_NORM"] = (True, "Used in normalization")
gain_ota = hdulist[extension].header['GAIN']
gain_ota_count = hdulist[extension].header['NGAIN']
gain_sum += gain_ota * gain_ota_count
gain_count += gain_ota_count
# We now know that we should include this OTA in the
# calculation of the flat-field normalization
logger.debug("Adding OTA %02d to flat-field ..." % fppos)
#flatfield_data = numpy.concatenate((flatfield_data, extension.data.flatten()))
#flatfield_data[extension,:,:] = extension.data
if (binning_x > 1 or binning_y > 1):
sx, sy = hdulist[extension].data.shape[0], hdulist[
extension].data.shape[1]
bx, by = sx // binning_x, sy // binning_y
one_d = numpy.reshape(hdulist[extension].data,
(by, binning_y, bx, binning_x)).mean(
axis=-1).mean(axis=1).flatten()
else:
one_d = hdulist[extension].data.flatten()
flatfield_data[datapos:datapos + one_d.shape[0]] = one_d
datapos += one_d.shape[0]
#print datapos
del one_d
# Remove all remaining NaN values and atruncate the array to the values actually used
finite = numpy.isfinite(flatfield_data[:datapos])
flatfield_data = flatfield_data[:datapos][finite]
# Now we are through all flatfields, compute the median value
logger.debug(" computing median ...")
sigma_min, sigma_max = -1e5, 1e6
for i in range(repeats):
valid = (flatfield_data > sigma_min) & (flatfield_data < sigma_max)
ff_median_level = numpy.median(flatfield_data[valid])
ff_std = numpy.std(flatfield_data[valid])
sigma_min = ff_median_level - 2 * ff_std
sigma_max = ff_median_level + 3 * ff_std
#print i, numpy.sum(valid), datapos, ff_median_level, ff_std, sigma_min, sigma_max
if (ff_median_level <= 0):
logger.error("Something went wrong or this is no flatfield frame")
ff_median_level = 1.0
#stdout_write("\b\b\b(% 7.1f) ..." % (ff_median_level))
logger.debug("Found median level % 7.1f ADU, normalizing ..." %
(ff_median_level))
# Now normalize all OTAs with the median flatfield level
#stdout_write(" normalizing ...")
# Create a new HDU list for the normalized output
# hdu_out = [] #pyfits.PrimaryHDU(header=hdulist[0].header)]
# for extension in range(0, len(hdulist)):
# if (not is_image_extension(hdulist[extension])):
# hdu_out.append(hdulist[extension])
# continue
# data = hdulist[extension].data.copy()
# data /= ff_median_level
# data[data < 0.1] = numpy.NaN
# new_hdu = pyfits.ImageHDU(data=data, header=hdulist[extension].header)
# #hdulist[extension].data /= ff_median_level
# #hdulist[extension].data[hdulist[extension].data < 0.1] = numpy.NaN
# new_hdu.header.add_history("FF-level: %.1f" % (ff_median_level))
# hdu_out.append(new_hdu)
# hdulist = pyfits.HDUList(hdu_out)
hdu_out = [] #pyfits.PrimaryHDU(header=hdulist[0].header)]
for extension in hdulist:
if (not is_image_extension(extension)):
continue
# data = hdulist[extension].data.copy()
# data /= ff_median_level
# data[data < 0.1] = numpy.NaN
# new_hdu = pyfits.ImageHDU(data=data, header=hdulist[extension].header)
extension.data /= ff_median_level
extension.data[extension.data < 0.1] = numpy.NaN
#hdulist[extension].data /= ff_median_level
#hdulist[extension].data[hdulist[extension].data < 0.1] = numpy.NaN
# new_hdu.header.add_history("FF-level: %.1f" % (ff_median_level))
# hdu_out.append(new_hdu)
# hdulist = pyfits.HDUList(hdu_out)
#
# compute the global gain value and store it in primary header
#
logger.debug("Computing global gain value (sum=%.1f, #=%d)" %
(gain_sum, gain_count))
global_gain = gain_sum / gain_count if (gain_count > 0) else -1
hdulist[0].header['GAIN'] = global_gain if (gain_count > 0) else -1.
hdulist[0].header['NGAIN'] = gain_count
logger.debug("writing results to file (%s) ..." % (outputfile))
clobberfile(outputfile)
hdulist.writeto(outputfile, overwrite=True)
logger.info("done!")
def create_quickview(filename, output_directory, verbose=False, clobber=True):
logger = logging.getLogger("QuickView")
create_otalevel = cmdline_arg_isset("-otalevel")
scaling = cmdline_arg_set_or_default("-scaling", None)
if (not scaling in ['linear', 'log', 'sqrt', 'arcsinh', 'asinh']):
scaling = 'sqrt'
hdulist = pyfits.open(filename)
filter = hdulist[0].header['FILTER']
obsid = hdulist[0].header['OBSID']
object = hdulist[0].header['OBJECT'].replace(' ', '_').replace(',', '_')
fullframe_image_filename = "%s/%s_%s.%s.jpg" % (output_directory, obsid,
object, scaling)
if (os.path.isfile(fullframe_image_filename) and not clobber):
# File exists and we were asked not to overwrite anything
stdout_write("\nFile (%s) exists, skipping ...\n" % (filename))
return
if (verbose):
stdout_write("\nWorking on file %s (%s, %s) ...\n" %
(filename, object, filter))
fpl = podi_focalplanelayout.FocalPlaneLayout(hdulist)
try:
list_of_otas_to_normalize = fpl.otas_to_normalize_ff[filter]
except:
list_of_otas_to_normalize = fpl.central_2x2
# Allocate some memory to hold the data we need to determine the
# most suitable intensity levels
binned_data = numpy.zeros(shape=(13 * 512 * 512), dtype=numpy.float32)
binned_data[:] = numpy.NaN
#
#
#
available_ota_coords = []
for ext in hdulist:
if (not is_image_extension(ext)):
continue
try:
ota = ext.header['OTA']
except:
continue
x = int(math.floor(ota / 10.))
y = int(ota % 10)
available_ota_coords.append((x, y))
datapos = 0
# if (verbose):
# stdout_write(" Finding contrast: Reading OTA")
logger.info("Finding best contrast")
for extension in range(1, len(hdulist)):
if (not is_image_extension(hdulist[extension])):
continue
if ('CELLMODE' in hdulist[extension].header
and hdulist[extension].header['CELLMODE'].find("V") > 0):
logger.info("Skipping guide-OTA %s" %
(hdulist[extension].header['EXTNAME']))
continue
fppos = int(hdulist[extension].header['FPPOS'][2:4])
try:
index = list_of_otas_to_normalize.index(fppos)
except:
# We didn't find this OTA in the list, so skip it
hdulist[extension].header['FF_NORM'] = (False,
"Used in normalization")
extension += 1
continue
logger.debug("Reading OTA %02d" % (fppos))
#stdout_write("\rReading OTA %02d" % (fppos))
# if (verbose):
# stdout_write(" %02d" % (fppos))
# Rebin the frame 8x8 to make it more manageable
binned = numpy.reshape(hdulist[extension].data,
(512, 8, 512, 8)).mean(axis=-1).mean(axis=1)
one_d = binned.flatten()
binned_data[datapos:datapos + one_d.shape[0]] = one_d
datapos += one_d.shape[0]
del one_d
del binned
#if (verbose):
# stdout_write(" - done!\n")
#
# Now we are through all OTA/extensions, compute the median value and stds
# so we can scale the frames accordingly
#
#if (verbose):
# stdout_write(" Finding best intensity levels ...")
median = 0
std = 1e8
binned_data = binned_data[0:datapos]
for looper in range(3):
valid = (binned_data > (median - std)) & (binned_data <
(median + 3 * std))
#print numpy.sum(valid)
median = numpy.median(binned_data[valid])
std = numpy.std(binned_data[valid])
#print median, std, median-std, median+3*std
# Compute the intensity levels, making sure not to exceed the maximum possible range
min_level = numpy.max([median - 1 * std, 0])
max_level = numpy.min([median + 8 * std, 60000])
# stdout_write(" using %d ... %d\n" % (min_level, max_level))
logger.info("Using intensity scale %d ... %d" % (min_level, max_level))
#
# Now that we have all the scaling factors, go ahead and create the preview images
#
# Create space to hold the full 8x8 OTA focal plane
full_focalplane = numpy.zeros(shape=(4096, 4096))
# if (verbose):
# stdout_write(" Creating jpeg for OTA")
logger.info("Creating jpeg for OTAs and full focalplane")
for extension in range(1, len(hdulist)):
if (not is_image_extension(hdulist[extension])):
continue
fppos = int(hdulist[extension].header['FPPOS'][2:4])
logger.info("Creating JPG for extension %s" %
(hdulist[extension].name))
#stdout_write("\r Creating jpegs (%02d) ..." % fppos)
# if (verbose):
# stdout_write(" %02d" % (fppos))
fp_x = fppos % 10
fp_y = math.floor(fppos / 10)
hdulist[extension].data[numpy.isnan(hdulist[extension].data)] = 0
binned = numpy.reshape(hdulist[extension].data,
(512, 8, 512, 8)).mean(axis=-1).mean(axis=1)
greyscale = (binned - min_level) / (max_level - min_level)
greyscale[greyscale < 0] = 0
greyscale[greyscale >= 1] = 1
print "ASINH?", (scaling in ['arcsinh', 'asinh'])
if (scaling == 'sqrt'):
greyscale = numpy.sqrt(greyscale)
elif (scaling in ['arcsinh', 'asinh']):
greyscale = numpy.arcsinh(greyscale)
elif (scaling == 'log'):
greyscale = numpy.log10(greyscale + 1) / numpy.log10(2.0)
else: # (scaling == 'linear')
pass
ffp_x = fp_x * 512
ffp_y = fp_y * 512
full_focalplane[ffp_x:ffp_x + 512, ffp_y:ffp_y +
512] = greyscale[:, :] #.astype(numpy.int)
#image = Image.fromarray(numpy.uint8(greyscale*255))
#image_filename = "%s/%s_%s.%02d.jpg" % (output_directory, obsid, object, fppos)
#image.transpose(Image.FLIP_TOP_BOTTOM).save(image_filename, "JPEG")
#del image
if (create_otalevel):
#
# Mark all overexposed pixels in a different color
#
channel_r = greyscale + 0
channel_g = greyscale + 0
channel_b = greyscale + 0
channel_r[binned > limit_overexposed] = overexposed[0]
channel_g[binned > limit_overexposed] = overexposed[1]
channel_b[binned > limit_overexposed] = overexposed[2]
im_r = Image.fromarray(numpy.uint8(channel_r * 255))
im_g = Image.fromarray(numpy.uint8(channel_g * 255))
im_b = Image.fromarray(numpy.uint8(channel_b * 255))
im_rgb = Image.merge('RGB', (im_r, im_g, im_b))
image_filename = "%s/%s_%s.%02d.rgb-%s.jpg" % (
output_directory, obsid, object, fppos, scaling)
im_rgb.transpose(Image.FLIP_TOP_BOTTOM).save(
image_filename, "JPEG")
# Delete all temporary images to keep memory demands low
del im_r
del im_g
del im_b
del im_rgb
#
# Prepare the preview for the full focal plane
#
#stdout_write("\r Creating jpegs (full-frame) ...")
# if (verbose):
# stdout_write(" full-frame")
image = Image.fromarray(numpy.uint8(full_focalplane * 255))
# Add lines to indicate detector borders. Make sure to make them wider than
# just one pixel, otherwise the lines completely disappear if displaying the
# image zoomed out
draw = ImageDraw.Draw(image)
for i in range(1, 8):
for linewidth in range(-5, 0):
draw.line(
(0, i * 512 + linewidth, image.size[0], i * 512 + linewidth),
fill=128)
draw.line(
(i * 512 + linewidth, 0, i * 512 + linewidth, image.size[1]),
fill=128)
if (crossout_missing_otas):
# Now loop over all OTAs and mark the ones that do not exist
for y in range(8):
for x in range(8):
tuple = (x, y)
try:
index = available_ota_coords.index(tuple)
except:
# We only get here if the OTA is not listed as available
# cross it out in the full focal plane image
#print "Strokign out ",tuple
for lw in range(-5, 0):
draw.line((x * 512 + lw, y * 512, (x + 1) * 512 + lw,
(y + 1) * 512),
fill=128)
draw.line((x * 512 + lw, (y + 1) * 512,
(x + 1) * 512 + lw, y * 512),
fill=128)
image = image.transpose(Image.FLIP_TOP_BOTTOM)
watermark_OTA = True
dx, dy = 150, 150
if (watermark_OTA):
image = image.convert('RGBA')
draw = ImageDraw.Draw(image)
font = ImageFont.truetype('/usr/share/./fonts/truetype/DroidSans.ttf',
size=200)
for x, y in itertools.product(range(8), repeat=2):
try:
index = available_ota_coords.index((x, y))
except:
continue
draw.text((x * 512 + dx, (7 - y) * 512 + dy),
"%02d" % (x * 10 + y),
font=font,
fill=(0, 64, 128, 255))
image.save(fullframe_image_filename, "JPEG")
del image
logger.info("Writing file to %s" % (fullframe_image_filename))
# stdout_write(" - done!\n")
logger.info("all done!\n")