def workerprocess___qr_mastercals(queue, options):
#print "QR MasterCals worker process started, ready for action..."
logger = logging.getLogger("QRMasterCals")
logger.info("MasterCal Listener started")
while (True):
try:
# print "\n\nWaiting for stuff to do\n\n"
task = queue.get()
except (KeyboardInterrupt, SystemExit) as e:
# print "worker received termination notice"
# Ignore the shut-down command here, and wait for the official
# shutdown command from main task
continue
if (task is None):
logger.info("Shutting down worker")
queue.task_done()
break
params = task
# print params
try:
handle_mastercals_request(params, logger, options)
except:
podi_logging.log_exception()
pass
# Mark this task as done, this means we are ready for the next one.
queue.task_done()
continue
return
def workerprocess___qr_mastercals(queue, options):
#print "QR MasterCals worker process started, ready for action..."
logger = logging.getLogger("QRMasterCals")
logger.info("MasterCal Listener started")
while (True):
try:
# print "\n\nWaiting for stuff to do\n\n"
task = queue.get()
except (KeyboardInterrupt, SystemExit) as e:
# print "worker received termination notice"
# Ignore the shut-down command here, and wait for the official
# shutdown command from main task
continue
if (task is None):
logger.info("Shutting down worker")
queue.task_done()
break
params = task
# print params
try:
handle_mastercals_request(params, logger, options)
except:
podi_logging.log_exception()
pass
# Mark this task as done, this means we are ready for the next one.
queue.task_done()
continue
return
def create_saturation_catalog_ota(filename, output_dir, verbose=True,
return_numpy_catalog=False, saturation_limit=65535):
"""
Create a saturation table for a given OTA exposure.
Parameters
----------
filename : string
Filename of the OTA FITS file.
output_dir : string
If return_numpy_catalog is not set, write the saturation catalog into
this directory.
return_numpy_catalog : bool
If set, return the results as numpy array instead of writing individual
files to disk.
Returns
-------
None - if no saturated pixels are found in this frame
ndarray, extname - if return_numpy_catalog is set
Nothing - if return_numpy_array is not set
"""
logger = logging.getLogger("OTASatCat")
# Open filename
logger.debug("Input filename: %s" % (filename))
try:
hdulist = pyfits.open(filename)
except IOError:
logger.debug("Can't open file %s" % (filename))
return None
except:
podi_logging.log_exception()
return None
mjd = hdulist[0].header['MJD-OBS']
obsid = hdulist[0].header['OBSID']
ota = int(hdulist[0].header['FPPOS'][2:4])
datatype = hdulist[0].header['FILENAME'][0]
logger = logging.getLogger("CreateSatCat: %s, OTA %02d" % (obsid, ota))
logger.debug("Starting work")
full_coords = numpy.zeros(shape=(0,4)) #, dtype=numpy.int16)
saturated_pixels_total = 0
for ext in range(1, len(hdulist)):
if (not is_image_extension(hdulist[ext])):
continue
# Find all saturated pixels (values >= 65K)
data = hdulist[ext].data
saturated = (data >= saturation_limit)
# print hdulist[ext].header['EXTNAME'], data.shape, numpy.max(data)
# Skip this cell if no pixels are saturated
number_saturated_pixels = numpy.sum(saturated)
if (number_saturated_pixels <= 0):
continue
saturated_pixels_total += number_saturated_pixels
wn_cellx = hdulist[ext].header['WN_CELLX']
wn_celly = hdulist[ext].header['WN_CELLY']
# logger.debug("number of saturated pixels in cell %d,%d: %d" % (wn_cellx, wn_celly, number_saturated_pixels))
# Do some book-keeping preparing for the masking
rows, cols = numpy.indices(data.shape)
saturated_rows = rows[saturated]
saturated_cols = cols[saturated]
#print saturated_rows.shape, saturated_cols.shape
coordinates = numpy.zeros(shape=(number_saturated_pixels,4))
coordinates[:,0] = wn_cellx
coordinates[:,1] = wn_celly
coordinates[:,2] = saturated_cols[:]
coordinates[:,3] = saturated_rows[:]
full_coords = numpy.append(full_coords, coordinates, axis=0) #coordinates if full_coords == None else
final_cat = numpy.array(full_coords, dtype=numpy.dtype('int16'))
if (saturated_pixels_total <= 0):
logger.debug("No saturated pixels found, well done!")
return None
logger.debug("Found %d saturated pixels, preparing catalog" % (saturated_pixels_total))
# Now define the columns for the table
columns = [\
pyfits.Column(name='CELL_X', format='I', array=final_cat[:, 0]),
pyfits.Column(name='CELL_Y', format='I', array=final_cat[:, 1]),
pyfits.Column(name='X', format='I', array=final_cat[:, 2]),
pyfits.Column(name='Y', format='I', array=final_cat[:, 3])
]
# Create the table extension
coldefs = pyfits.ColDefs(columns)
tbhdu = pyfits.BinTableHDU.from_columns(coldefs)
extension_name = "OTA%02d.SATPIX" % (ota)
tbhdu.name = extension_name
if (return_numpy_catalog):
logger.debug("Returning results as numpy catalog")
return final_cat, extension_name
# Also copy the primary header into the new catalog
primhdu = pyfits.PrimaryHDU(header=hdulist[0].header)
# Create a HDUList for output
out_hdulist = pyfits.HDUList([primhdu, tbhdu])
# And create the output file
output_filename = "%s/%s%s.%02d.saturated.fits" % (output_dir, datatype, obsid, ota)
stdout_write("Writing output: %s\n" % (output_filename))
clobberfile(output_filename)
out_hdulist.writeto(output_filename, clobber=True)
if (verbose):
print("some of the saturated pixels:\n",final_cat[0:10,:])
#numpy.savetxt("test", final_cat)
#print full_coords.shape
logger.debug("Retuning final FITS table catalog")
return final_cat
def create_saturation_catalog(filename, output_dir, verbose=True, mp=False, redo=False,
saturation_limit=65535):
"""
Create catalogs listing all saturated pixels to enable handling saturation
and persistency effects later-on.
The main purpose of this file is to call create_saturation_catalog_ota,
possibly wrapped for with mp_create_saturation_table for parallel
processing.
Parameters
----------
filename : string
One file of the exposure. This file is mainly used to obtain the
necessary information to create all the other filenames for this
exposure.
output_dir : string
Directory to hold all the saturation catalogs. This is the directory
that will be fed into collectcells via the -persistency command line
flag.
mp : bool - not used
redo : bool
Recreate the saturation catalog if it already exists
Returns
-------
"""
logger = logging.getLogger("CreateSaturationCatalog")
logger.info("Creating saturation mask for %s ..." % (filename))
if (os.path.isfile(filename)):
# This is one of the OTA fits files
# extract the necessary information to generate the
# names of all the other filenames
try:
hdulist = pyfits.open(filename)
except IOError:
logger.warning("\rProblem opening file %s...\n" % (filename))
return
except:
podi_logging.log_exception()
hdr_filename = hdulist[0].header['FILENAME']
hdr_items = hdr_filename.split('.')
basename = "%s.%s" % (hdr_items[0], hdr_items[1])
hdulist.close()
# Split the input filename to extract the directory part
directory, dummy = os.path.split(filename)
elif (os.path.isdir(filename)):
# As a safety precaution, if the first parameter is the directory containing
# the files, extract just the ID string to be used for this script
if (filename[-1] == "/"):
filename = filename[:-1]
basedir, basename = os.path.split(filename)
directory = filename
else:
logger.error("Input %s is neither a file nor a directory!" % (filename))
logger.error("Aborting operation due to illegal input.")
return
output_filename = "%s/%s.saturated.fits" % (output_dir, basename)
logger.debug("Output saturation catalog: %s" % (output_filename))
if (os.path.isfile(output_filename) and not redo):
logger.debug("File (%s) exists, skipping!" % (output_filename))
return
# Setup parallel processing
queue = multiprocessing.JoinableQueue()
return_queue = multiprocessing.JoinableQueue()
#return_queue = multiprocessing.Queue()
number_jobs_queued = 0
first_fits_file = None
ota_list = []
for (ota_x, ota_y) in itertools.product(range(8),repeat=2):
ota = ota_x * 10 + ota_y
filename = "%s/%s.%02d.fits" % (directory, basename, ota)
if (not os.path.isfile(filename)):
filename = "%s/%s.%02d.fits.fz" % (directory, basename, ota)
if (not os.path.isfile(filename)):
continue
queue.put( (filename, saturation_limit) )
number_jobs_queued += 1
# Remember the very first FITS file we find. This will serve as the primary HDU
if (first_fits_file is None):
# Create a primary HDU from the first found fits-file
try:
firsthdu = pyfits.open(filename)
except IOError:
logger.warning("Problem opening FITS file %s" % (filename))
continue
logger.debug("Copying general information from file %s" % (filename))
ota_list.append(pyfits.PrimaryHDU(header=firsthdu[0].header))
firsthdu.close()
firsthdu = None
first_fits_file = filename
if (first_fits_file is None):
logger.warning("Couldn't find a valid FITS file, thus nothing to do")
return
# Now start all the workers
logger.debug("Starting worker processes")
processes = []
for i in range(sitesetup.number_cpus):
p = multiprocessing.Process(target=mp_create_saturation_catalog, args=(queue, return_queue, False))
p.start()
processes.append(p)
time.sleep(0.01)
# Tell all workers to shut down when no more data is left to work on
logger.debug("Sending shutdown command to worker processes")
for i in range(len(processes)):
if (verbose): stdout_write("Sending quit command!\n")
queue.put( (None) )
logger.debug("Collecting catalogs for each OTA")
for i in range(number_jobs_queued):
if (verbose): print("reading return ",i)
cat_name = return_queue.get()
if (cat_name is not None):
final_cat, extension_name = cat_name
columns = [
pyfits.Column(name='CELL_X', format='I', array=final_cat[:, 0]),
pyfits.Column(name='CELL_Y', format='I', array=final_cat[:, 1]),
pyfits.Column(name='X', format='I', array=final_cat[:, 2]),
pyfits.Column(name='Y', format='I', array=final_cat[:, 3])
]
# Create the table extension
coldefs = pyfits.ColDefs(columns)
tbhdu = pyfits.BinTableHDU.from_columns(coldefs)
tbhdu.name = extension_name
ota_list.append(tbhdu)
return_queue.task_done()
# Join each process to make thre they terminate(d) correctly
logger.debug("Joining process to ensure proper termination")
for p in processes:
p.join()
hdulist = pyfits.HDUList(ota_list)
output_filename = "%s/%s.saturated.fits" % (output_dir, basename)
clobberfile(output_filename)
logger.debug("Writing output file %s" % (output_filename))
hdulist.writeto(output_filename, clobber=True)
logger.debug("all done!")
return
def read_fits_catalog(fn, extension, flatten=True):
logger = logging.getLogger("ReadFITScat")
logger.debug("Opening FITS catalog from %s" % (fn))
if (type(fn) is str):
hdulist = pyfits.open(fn)
else:
hdulist = fn
if (type(extension) is list):
ext_list = extension
else:
ext_list = [extension]
return_tables = []
for ext_id in ext_list:
try:
ext = hdulist[ext_id]
except KeyError:
logger.warning("Extension %s not found in %s" % (ext_id, fn))
hdulist.info()
return_tables.append(None)
continue
n_fields = ext.header['TFIELDS']
n_rows = ext.header['NAXIS2']
# table = numpy.empty((n_rows, n_fields))
logger.debug("Reading data for %d fields" % (n_fields))
# print "Reading data for %d fields" % (n_fields)
table = []
for f in range(n_fields):
fd = ext.data.field(f)
if (fd.ndim > 2 and flatten):
logger.warning("Unable to handle catalog field %s" %
(ext.header['TTYPE%d' % (f + 1)]))
continue
if (fd.ndim > 1 and flatten):
for c2 in range(fd.shape[1]):
table.append(fd[:, c2])
else:
table.append(fd)
try:
if (flatten):
table = numpy.array(table).T
logger.debug("Table data: %s" % (str(table.shape)))
else:
logger.debug("Table data: %d columns" % (len(table)))
# np_table = numpy.array(table)
# print table
except:
podi_logging.log_exception()
pass
# print table.shape
# print table[1:3]
# print ext.header
return_tables.append(table)
if (type(extension) is not list):
return return_tables[0]
return return_tables
def worker_slave(queue):
"""
This function handles all work, either running collectcells locally or
remotely via ssh. Files to reduce are read from a queue.
"""
logger = logging.getLogger("SAMPWorker")
logger.info("Worker process started, ready for action...")
if (not setup.use_ssh):
# If we reduce frames locally, prepare the QR logging.
options['clobber'] = False
while (True):
try:
# print "\n\nWaiting for stuff to do\n\n"
task = queue.get()
except (KeyboardInterrupt, SystemExit) as e:
# print "worker received termination notice"
# Ignore the shut-down command here, and wait for the official
# shutdown command from main task
continue
if (task is None):
logger.info("Shutting down worker")
queue.task_done()
break
filename, object_name, obsid = task
logger.info("starting work on file %s" % (filename))
ccopts = ""
if (len(sys.argv) > 2):
# There are some parameters to be forwarded to collectcells
ccopts = " ".join(sys.argv[1:])
# print "ccopts=",ccopts
if (cmdline_arg_isset("-dryrun")):
logger.info("DRYRUN: Sending off file %s for reduction" % (filename))
# print "task done!"
queue.task_done()
continue
if (object_name.lower().find("focus") >= 0):
#
# This is most likely a focus exposure
#
n_stars = int(cmdline_arg_set_or_default("-nstars", 7))
logger.info("New focus exposure to analyze (with %d stars)" % (n_stars))
if (setup.use_ssh):
remote_inputfile = setup.translate_filename_local2remote(filename)
kw = {
'user': setup.ssh_user,
'host': setup.ssh_host,
'filename': remote_inputfile,
'podidir': setup.remote_podi_dir,
'outdir': setup.output_dir,
'nstars': n_stars,
}
ssh_command = "ssh %(user)s@%(host)s %(podidir)s/podi_focus.py -nstars=%(nstars)d %(filename)s %(outdir)s" % kw
logger.info("Out-sourcing work to %(user)s@%(host)s via ssh" % kw)
process = subprocess.Popen(ssh_command.split(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True)
_stdout, _stderr = process.communicate()
process.stdout.close()
process.stderr.close()
try:
process.terminate()
except:
pass
#print "*"*80
if (not _stdout == "" and _stdout is not None):
logger.info("Received STDOUT:\n%s" % (_stdout))
#print "*"*80
if (not _stderr == "" and _stderr is not None):
logger.info("Received STDERR:\n%s" % (_stderr))
#print _stderr
#print "*"*80
else:
# Run locally
logger.info("Analyzing focus sequence (%s) locally" % (filename))
podi_focus.get_focus_measurement(filename, n_stars=n_stars, output_dir=setup.output_dir)
logger.info("Done with analysis")
# Now check if we are supposed to open/display the focus plot
if (setup.focus_display is not None):
remote_filename = "%s/%s_focus.png" % (setup.output_dir, obsid)
local_filename = setup.translate_filename_remote2local(filename, remote_filename)
cmd = "%s %s &" % (setup.focus_display, local_filename)
logger.info("Opening and displaying plot")
os.system(cmd)
else:
#
# This is NOT a focus exposure
#
if (setup.use_ssh):
# This is not a focus exposure, to treat it as a normal science exposure
remote_inputfile = setup.translate_filename_local2remote(filename)
kw = {
'user': setup.ssh_user,
'host': setup.ssh_host,
'collectcells': setup.ssh_executable,
'options': ccopts,
'filename': remote_inputfile,
'outputfile': setup.output_format,
}
ssh_command = "ssh %(user)s@%(host)s %(collectcells)s %(filename)s %(outputfile)s %(options)s -noclobber" % kw
process = subprocess.Popen(ssh_command.split(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True)
_stdout, _stderr = process.communicate()
process.stdout.close()
process.stderr.close()
try:
process.terminate()
except:
pass
#print "*"*80
if (not _stdout == "" and _stdout is not None):
logger.info("Received STDOUT:\n%s" % (_stdout))
#print "*"*80
if (not _stderr == "" and _stderr is not None):
logger.info("Received STDERR:\n%s" % (_stderr))
#print _stderr
#print "*"*80
else:
logger.info("Running collectcells (%s)" % (filename))
podi_collectcells.collectcells_with_timeout(input=filename,
outputfile=setup.output_format,
options=options,
timeout=300,
process_tracker=process_tracker)
#
# If requested, also send the command to ds9
#
local_filename = setup.translate_filename_remote2local(filename, setup.output_format)
if (cmdline_arg_isset("-forward2ds9")):
forward2ds9_option = cmdline_arg_set_or_default("-forward2ds9", "image")
if (forward2ds9_option == "irafmosaic"):
cmd = "mosaicimage iraf %s" % (local_filename)
else:
cmd = "fits %s" % (local_filename)
logger.info("Forwarding file to ds9")
logger.debug("filename: %s" % (filename))
logger.debug("remote file: %s" % (remote_inputfile))
logger.debug("local file: %s" % (local_filename))
try:
cli1 = sampy.SAMPIntegratedClient(metadata = metadata)
cli1.connect()
cli1.enotify_all(mtype='ds9.set', cmd='frame 2')
cli1.enotify_all(mtype='ds9.set', cmd='scale scope global')
cli1.enotify_all(mtype='ds9.set', cmd=cmd)
cli1.disconnect()
except Exception as err:
logger.error("Problems sending message to ds9: %s" % err)
podi_logging.log_exception()
pass
# By default, also open the psf diagnostic plot, if available
psf_plot_fn = local_filename[:-5]+".psf.png"
if (os.path.isfile(psf_plot_fn)):
cmd = "%s %s &" % (setup.focus_display, psf_plot_fn)
logger.info("Opening and displaying PSF diagnostic plot (%s)" % (psf_plot_fn))
os.system(cmd)
#
# Once the file is reduced, mark the current task as done.
#
logger.info("task done!")
queue.task_done()
print("Terminating worker process...")
return
def get_focus_measurement(filename, n_stars=5, output_dir="./", mp=False):
"""
Parameters
----------
filename : string
One file of the exposure. This file is mainly used to obtain the
necessary information to create all the other filenames for this
exposure.
output_dir : string
Directory to hold all the saturation catalogs. This is the directory
that will be fed into collectcells via the -persistency command line
flag.
Returns
-------
"""
logger = logging.getLogger("MeasureFocus")
logger.info("Starting focus measurement for %s (%d *)..." %
(filename, n_stars))
if (os.path.isfile(filename)):
# This is one of the OTA fits files
# extract the necessary information to generate the
# names of all the other filenames
try:
hdulist = pyfits.open(filename)
except IOError:
logger.warning("\rProblem opening file %s...\n" % (filename))
return
except:
podi_logging.log_exception()
hdr_filename = hdulist[0].header['FILENAME']
hdr_items = hdr_filename.split('.')
basename = "%s.%s" % (hdr_items[0], hdr_items[1])
hdulist.close()
# Split the input filename to extract the directory part
directory, dummy = os.path.split(filename)
elif (os.path.isdir(filename)):
# As a safety precaution, if the first parameter is the directory containing
# the files, extract just the ID string to be used for this script
if (filename[-1] == "/"):
filename = filename[:-1]
basedir, basename = os.path.split(filename)
directory = filename
# Setup parallel processing
queue = multiprocessing.JoinableQueue()
return_queue = multiprocessing.JoinableQueue()
number_jobs_queued = 0
obsid = None
for (ota_x, ota_y) in itertools.product(range(8), repeat=2):
ota = ota_x * 10 + ota_y
filename = "%s/%s.%02d.fits" % (directory, basename, ota)
if (not os.path.isfile(filename)):
filename = "%s/%s.%02d.fits.fz" % (directory, basename, ota)
if (not os.path.isfile(filename)):
continue
logger.debug("Adding file %s to task list" % (filename))
if (obsid is None):
hdulist = pyfits.open(filename)
obsid = hdulist[0].header['OBSID']
hdulist.close()
queue.put((filename, n_stars))
number_jobs_queued += 1
# break
# Now start all the workers
logger.debug("Starting worker processes")
processes = []
for i in range(sitesetup.number_cpus):
p = multiprocessing.Process(target=mp_measure_focus,
args=(queue, return_queue, False))
p.start()
processes.append(p)
time.sleep(0.01)
# Tell all workers to shut down when no more data is left to work on
queue.put((None))
logger.info("Collecting catalogs for each OTA")
all_foci = None
real_numbers = False
for i in range(number_jobs_queued):
returned = return_queue.get()
if (returned is None):
continue
focus_positions, found_real_numbers = returned
if (found_real_numbers):
real_numbers = True
logger.debug("Received %d focus positions" %
(focus_positions.shape[0]))
all_foci = focus_positions if all_foci is None else numpy.append(
all_foci, focus_positions, axis=0)
#print cat_name
return_queue.task_done()
# Join each process to make sure they terminate(d) correctly
logger.debug("Joining process to ensure proper termination")
for p in processes:
p.join()
#print all_foci, all_foci.ndim, all_foci.shape
if (all_foci is None or all_foci.ndim < 2 or all_foci.shape[0] <= 0):
logger.error("Couldn't find any star patterns!")
return
logger.info("Found a grand total of %d focus positions" %
(all_foci.shape[0]))
# print all_foci.shape
with open("allfoci", "w") as f:
for s in range(all_foci.shape[0]):
numpy.savetxt(f, all_foci[s, :, :])
print >> f, "\n"
# numpy.savetxt("allfoci", all_foci.reshape((-1,all_foci.shape[2])))
#
# Eliminate all focus measurements that could be affected by the low-light
# CTE problem
#
min_background = numpy.min(all_foci[:, :, SXFocusColumn['background']],
axis=1)
detector_lots = all_foci[:, 0, SXFocusColumn['ota_lot']]
bad = (detector_lots < 7) & (min_background < 100)
logger.info("Excluding %d focus samples that might have CTE issues" %
(numpy.sum(bad)))
all_foci = all_foci[~bad]
stats = get_mean_focuscurve(all_foci)
pfit, uncert, fwhm_median, fwhm_std, fwhm_cleaned, best_focus_position, best_focus = stats
plotfilename = "%s/%s_focus.png" % (output_dir, obsid)
create_focus_plot(all_foci, stats, basename, plotfilename, real_numbers)
logger.debug("all done!")
return
def imcombine(input_filelist, outputfile, operation, return_hdu=False,
subtract=None, scale=None, gather_all_otas=True):
logger = logging.getLogger("ImCombine")
# First loop over all filenames and make sure all files exist
filelist = []
for file in input_filelist:
if (os.path.isfile(file)):
filelist.append(file)
if (len(filelist) <= 0):
logger.error("No existing files found in input list, hence nothing to do!\n")
return None
logger.debug("Stacking the following files:\n -- %s" % ("\n -- ".join(filelist)))
# elif (len(filelist) == 1):
# # stdout_write("Only 1 file to combine, save the hassle and copy the file!\n")
# hdulist = pyfits.open(filelist[0])
# if (return_hdu):
# return hdulist
# hdulist.writeto(outputfile, clobber=True)
# return
# Read the input parameters
# Note that file headers are copied from the first file
reference_filename = filelist[0]
ref_hdulist = pyfits.open(reference_filename)
# Create the primary extension of the output file
# Copy all headers from the reference HDU
primhdu = pyfits.PrimaryHDU(header=ref_hdulist[0].header)
# Add PrimaryHDU to list of OTAs that go into the output file
out_hdulist = [primhdu]
ref_hdulist.close()
del ref_hdulist
#
# Compile a list of OTAs that will be in the output frame
#
otas_found = []
ota_sizes = {}
ref_header = {}
if (gather_all_otas):
logger.info("Checking all files to compile comprehensive list of available OTAs")
master_associations = None
for fn in filelist:
hdulist = pyfits.open(fn)
for ext in hdulist:
if (is_image_extension(ext)):
try:
otas_found.append(ext.name)
if (ext.name not in ota_sizes):
ota_sizes[ext.name] = ext.data.shape
ref_header[ext.name] = ext.header
except:
podi_logging.log_exception()
#
# Collect all individual association tables to create
# a master association table combining all input data
#
assoc_table = podi_associations.read_associations(hdulist)
if (assoc_table is None):
logger.info("No association data available")
assoc_table = {'input_simple': [fn]}
if (master_associations is None):
master_associations = assoc_table
else:
master_associations = podi_associations.collect_reduction_files_used(
master_associations, assoc_table)
hdulist.close()
del hdulist
if (not gather_all_otas):
break
otas_to_combine = set(otas_found)
ota_counter = collections.Counter(otas_found)
#print otas_to_combine
#print ota_counter
#
# Now loop over all extensions and compute the mean
#
# for cur_ext in range(0, len(ref_hdulist)):
for cur_ext, extname in enumerate(otas_to_combine):
data_blocks = []
# Check what OTA we are dealing with
# if (not is_image_extension(ref_hdulist[cur_ext])):
# continue
# ref_fppos = ref_hdulist[cur_ext].name #header['EXTNAME']
#stdout_write("\rCombining frames for OTA %s (#% 2d/% 2d) ..." % (ref_fppos, cur_ext+1, len(ref_hdulist)))
#logger.debug("Combining frames for OTA %s (#% 2d/% 2d) ..." % (ref_fppos, cur_ext+1, len(ref_hdulist)))
logger.info("Combining frames for OTA %s (#% 2d/% 2d) ..." % (extname, cur_ext+1, len(otas_to_combine)))
#
# Add some weird-looking construct to move the memory allocation and actual
# imcombine into a separate process. This has to do with if/how/when python releases
# memory (or not), causing a massive short-term memory leak.
# With the additional process, the memory is onwed by the other process and the memory
# is freed once we destroy this helper process.
#
return_queue = multiprocessing.JoinableQueue()
#worker_args=(ref_fppos, filelist, ref_hdulist[cur_ext].data.shape, operation, return_queue, verbose)
kw_args = {
'extension': extname,
'filelist': filelist,
'shape': (ota_sizes[extname][0], ota_sizes[extname][1], ota_counter[extname]), #ref_hdulist[cur_ext].data.shape,
'operation': operation,
'queue': return_queue,
'verbose': verbose,
'subtract': subtract,
'scale': scale,
}
# p = multiprocessing.Process(target=imcombine_subprocess, args=worker_args)
p = multiprocessing.Process(target=imcombine_subprocess, kwargs=kw_args)
p.start()
combined = return_queue.get()
p.terminate()
del p
if (verbose): stdout_write(" done, creating fits extension ...")
logger.debug("done with computing, creating fits extension ...")
# Create new ImageHDU, insert the imcombined's data and copy the
# header from the reference frame
hdu = pyfits.ImageHDU(data=combined, header=ref_header[extname])
# Append the new HDU to the list of result HDUs
out_hdulist.append(hdu)
if (verbose): stdout_write(" done\n")
#
# At this point, add a fits table listing all filenames that went into this combination
#
filenames_only = []
for file in filelist:
dirname, filename = os.path.split(file)
filenames_only.append(filename)
logger.debug("Adding association data to output file")
out_hdulist[0].header["NCOMBINE"] = (len(filenames_only), "number of combined files")
columns = [
pyfits.Column(name='filenumber', format="I4", array=range(1,len(filenames_only)+1)),
pyfits.Column(name='filename', format="A100", array=filenames_only),
]
coldefs = pyfits.ColDefs(columns)
tablehdu = pyfits.BinTableHDU.from_columns(coldefs)
tablehdu.header["EXTNAME"] = "FILELIST"
out_hdulist.append(tablehdu)
assoc_tbhdu = podi_associations.create_association_table(
master_associations)
out_hdulist.append(assoc_tbhdu)
#
# All work done now, prepare to return the data or write it to disk
#
out_hdu = pyfits.HDUList(out_hdulist)
if (not return_hdu and outputfile != None):
logger.debug(" writing results to file %s ..." % (outputfile))
clobberfile(outputfile)
try:
out_hdu.writeto(outputfile, clobber=True, checksum=True)
except TypeError:
# this most likely is this error:
# TypeError: object of type 'NoneType' has no len()
# related to the checksum calculation
clobberfile(outputfile)
out_hdu.writeto(outputfile, clobber=True)
except pyfits.VerifyError:
logger.warning("Encountered FITS verification error, writing anyway")
try:
out_hdu.writeto(outputfile, clobber=True, checksum=False, output_verify='ignore')
except:
raise
except:
raise
out_hdu.close()
del out_hdu
del out_hdulist
stdout_write(" done!\n")
elif (return_hdu):
logger.debug(" returning HDU for further processing ...")
return out_hdu
else:
logger.debug(" couldn't write output file, no filename given!")
return None
hdr = hdulist[ota].header.copy()
hdr['NAXIS'] = 2
hdr['NAXIS1'] = 4096
hdr['NAXIS2'] = 4096
hdr['CRVAL1'] = math.fmod(hdr['CRVAL1'] + wcs_offset[0], 360.0)
hdr['CRVAL2'] += wcs_offset[1]
wcs = astWCS.WCS(hdr, mode='pyfits')
wcs.updateFromHeader()
x, y = corners[ota]
print numpy.array(wcs.pix2wcs(x, y))
radec_ref[idx, :] = numpy.array(wcs.pix2wcs(x, y))
except:
# ignore OTAs that are missing
# this implies we need at least ONE of the 4 central OTAs to
# make this correction
podi_logging.log_exception()
pass
ref_point = bottleneck.nanmean(radec_ref, axis=0)
# ref_point += wcs_offset
print "REF:", ref_point
crval = numpy.array(
[hdulist[1].header['CRVAL1'], hdulist[1].header['CRVAL2']])
d_crval = ref_point - crval
print "current CRVAL:", hdulist[1].header['CRVAL1'], hdulist[1].header[
'CRVAL2']
#
def measure_focus_ota(filename, n_stars=5):
"""
Obtain a focus measurement from teh specified filename. To do so,
1) run source extractor to get FWHM measurements and positions for all sources
2) group sources into vertical sequences as expected from the function
of the focus tool
3) assign physical focus positions to each measurement
4) return final catalog back to master so a final, combined focus curve
can be assembled.
"""
# print"\n\n\nworking on file ",filename,"\n\n\n"
try:
hdulist = pyfits.open(filename)
except IOError:
logger.debug("Can't open file %s" % (filename))
return None
except:
podi_logging.log_exception()
return None
obsid = hdulist[0].header['OBSID']
ota = hdulist[1].header['WN_OTAX'] * 10 + hdulist[1].header['WN_OTAY']
ota_id = hdulist[0].header['OTA_ID']
logger = logging.getLogger("MeasureFocusOTA: %s(%02d)" % (obsid, ota))
logger.info("Starting work ...")
obsid = hdulist[0].header['OBSID']
ota = int(hdulist[0].header['FPPOS'][2:4])
# Check the object name to see if if contains the information about the exposure
focus_positions = numpy.arange(n_stars)[::-1]+1.
real_focus_positions = False
object_name = hdulist[0].header['OBJECT']
if (object_name.startswith("Focus Center")):
# This looks like it might be the right format
try:
items = object_name.split()
# Check all items
if (len(items) == 7 and
items[0] == "Focus" and
items[1] == "Center" and
items[3] == "NStep" and
items[5] == "DStep"):
n_stars = int(items[4])
focus_center = float(items[2])
focus_step = float(items[6])
focus_start = focus_center - (n_stars-1)/2*focus_step
focus_positions = numpy.arange(n_stars, dtype=numpy.float32)[::-1] * focus_step + focus_start
logger.debug("Infom from header: N=%d, center=%.0f, step=%.0f, start=%.0f" % (
n_stars, focus_center, focus_Step, focus_step, focus_start))
logger.debug("Focus positions: %s" % (str(focus_positions)))
real_focus_positions = True
except:
pass
# Run SourceExtractor on the file
sex_config = "%s/config/focus.sexconf" % (sitesetup.exec_dir)
sex_param = "%s/config/focus.sexparam" % (sitesetup.exec_dir)
catfile = "%s/tmp.%s_OTA%02d.cat" % (sitesetup.scratch_dir, obsid, ota)
sex_cmd = "%(sexcmd)s -c %(sex_config)s -PARAMETERS_NAME %(sex_param)s -CATALOG_NAME %(catfile)s %(filename)s" % {
"sexcmd": sitesetup.sextractor,
"sex_config": sex_config,
"sex_param": sex_param,
"catfile": catfile,
"filename": filename,
"redirect": sitesetup.sex_redirect,
}
# print "\n"*10,sex_cmd,"\n"*10
# Run source extractor
# catfile = "/tmp//tmp.pid4383.20121008T221836.0_OTA33.cat"
if (not os.path.isfile(catfile)):
logger.debug("Running source extractor to search for stars")
start_time = time.time()
try:
ret = subprocess.Popen(sex_cmd.split(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(sex_stdout, sex_stderr) = ret.communicate()
if (ret.returncode != 0):
logger.warning("Sextractor might have a problem, check the log")
logger.info("Stdout=\n"+sex_stdout)
logger.info("Stderr=\n"+sex_stderr)
except OSError as e:
podi_logging.log_exception()
print >>sys.stderr, "Execution failed:", e
end_time = time.time()
logger.debug("SourceExtractor finished after %.2f seconds" % (end_time - start_time))
else:
logger.debug("Source catalog already exists, re-using the old file")
#
# delete the tmp catalog
#
#
# Load the source catalog.
# Handle cases of non-existing or empty catalogs
#
logger.debug("loading the source catalog from %s" % (catfile))
try:
source_cat = numpy.loadtxt(catfile)
except IOError:
logger.warning("The Sextractor catalog is empty, ignoring this OTA")
source_cat = None
return None
if (source_cat.shape[0] <= 0):
# no sources found
return None
# print "\n\n total sources in raw file",source_cat.shape,"\n\n"
logger.debug("Found %d sources in raw SourceExtractor catalog %s" % (source_cat.shape[0], catfile))
#
# Now convert all X/Y values to proper OTA X/Y coordinates based on their
# extension number
#
corr_cat = None
# print "extensions:", source_cat[:,SXFocusColumn['extension']]
for i in range(len(hdulist)):
if (not is_image_extension(hdulist[i])):
# print "skipping extension",i,", this is not an image extension"
continue
# get cell_x, cell_y from this extension
cell_x = hdulist[i].header['WN_CELLX']
cell_y = hdulist[i].header['WN_CELLY']
x1, c2, y1, y2 = cell2ota__get_target_region(cell_x, cell_y, 1)
# Create a mask for all sources in this cell
in_this_cell = (source_cat[:,SXFocusColumn['extension']] == i)
if (numpy.sum(in_this_cell) <= 0):
logger.debug("Couldn't find any sources in cell %d,%d" % (cell_x, cell_y))
continue
# print "found",numpy.sum(in_this_cell),"sources for cell",cell_x, cell_y, " adding", x1, y1
cell_cat = source_cat[in_this_cell]
cell_cat[:,SXFocusColumn['x']] += x1
cell_cat[:,SXFocusColumn['y']] += y1
#
# get overscan-level data for this cell
#
binning = get_binning(hdulist[i].header)
overscan_data = extract_biassec_from_cell(hdulist[i].data, binning)
overscan_level = numpy.mean(overscan_data)
cell_cat[:, SXFocusColumn['background']] -= overscan_level
corr_cat = cell_cat if corr_cat is None else numpy.append(corr_cat, cell_cat, axis=0)
#
# Attach to each measurement what detector lot it came from
#
fpl = podi_focalplanelayout.FocalPlaneLayout(hdulist)
detector_lot = fpl.get_detector_generation(ota_id)
corr_cat[:, SXFocusColumn['ota_lot']] = detector_lot
#
# Also override the extension number in the source catalog with the
# position in the overall focal plane
#
corr_cat[:, SXFocusColumn['extension']] = ota
# print "\n\n\n\ntotal corrected catalog:",corr_cat.shape
# save the source catalog
#numpy.savetxt("focus_cat.ota%02d" % (ota), source_cat)
#numpy.savetxt("focus_cat2.ota%02d" % (ota), corr_cat)
logger.debug("done fixing the pixel coordinates")
# only select bright enough sources
bright_enough = corr_cat[:,SXFocusColumn['mag_auto']] < -10
corr_cat = corr_cat[bright_enough]
#numpy.savetxt("focus_cat3.ota%02d" % (ota), corr_cat)
#dummy_test = open("dummy.test", "w")
# Now try to match up stars in a sequence
all_angles, all_distances = [], []
for s1 in range(corr_cat.shape[0]):
# Assume this is the middle star in the sequence
# Find all stars above and below it in a cone
# compute the angle to all other stars in the catalog
dx = corr_cat[:,SXFocusColumn['x']] - corr_cat[s1,2]
dy = corr_cat[:,SXFocusColumn['y']] - corr_cat[s1,3]
d_total = numpy.hypot(dx, dy)
in_cone = numpy.fabs(dx/dy) < 0.1
# Need to be at most n_stars * 10'' and at least 5''
close_enough = (d_total < (n_stars * 10. / 0.11)) & (d_total > 5 / 0.11)
good_so_far = in_cone & close_enough
if (numpy.sum(good_so_far) <= 0):
continue
candidates = corr_cat[good_so_far]
# if (numpy.sum(candidates) < n_stars):
# # Only use full sequences
# continue
# are the magnitudes comparable
delta_mag = numpy.fabs(candidates[:,SXFocusColumn['mag_auto']] \
- corr_cat[s1,SXFocusColumn['mag_auto']])
similar_brightness = delta_mag < 1
#print >>dummy_test, "#", candidates.shape[0], numpy.sum(similar_brightness)
#print similar_brightness
if (numpy.sum(similar_brightness) <= 0):
continue
good_candidates = candidates[similar_brightness]
# Now sort the data with increasing y values
si = numpy.argsort(good_candidates[:,SXFocusColumn['y']])
sorted_candidates = good_candidates[si]
#numpy.savetxt(dummy_test, sorted_candidates)
#print >>dummy_test, "\n\n\n"
# Now compute the slope and distance between each point and each point
# above it
for p1, p2 in itertools.combinations(range(sorted_candidates.shape[0]), 2):
angle = numpy.arctan2(sorted_candidates[p1,2] - sorted_candidates[p2,2],
sorted_candidates[p1,3] - sorted_candidates[p2,3])
distance = numpy.sqrt( (sorted_candidates[p1,2] - sorted_candidates[p2,2])**2
+ (sorted_candidates[p1,3] - sorted_candidates[p2,3])**2 )
all_angles.append(angle)
all_distances.append(distance)
#dummy_test.close()
# Once we are through with the first iteration find the best-fitting angle
all_angles = numpy.array(all_angles)
all_distances = numpy.array(all_distances)
# Find the best or rather most frequently occuring angle
all_angles[all_angles < 0] += 2*math.pi
#numpy.savetxt("dummy.angles", all_angles)
#numpy.savetxt("dummy.distances", all_distances)
filtered_angles = three_sigma_clip(all_angles)
if (filtered_angles is None or
filtered_angles.ndim < 1 or
filtered_angles.shape[0] <= 0):
return None
angle_width = scipy.stats.scoreatpercentile(filtered_angles, [16,84])
angle_width = scipy.stats.scoreatpercentile(filtered_angles, [5,95])
logger.debug("Found median angle %f [%f ...%f]" % (
numpy.degrees(numpy.median(filtered_angles)),
numpy.degrees(angle_width[0]), numpy.degrees(angle_width[1])
)
)
#
# Now we can do another proper search for all stars
# This time, only search for complete series (#stars as specified)
#
#focus_stars = open("focus_stars", "w")
all_candidates = []
for s1 in range(corr_cat.shape[0]):
# Find all stars above and below it in a cone
# compute the angle to all other stars in the catalog
dx = corr_cat[:,SXFocusColumn['x']] - corr_cat[s1,2]
dy = corr_cat[:,SXFocusColumn['y']] - corr_cat[s1,3]
angles = numpy.arctan2(dx, dy)
angles[angles < 0] += 2*math.pi
d_total = numpy.hypot(dx, dy)
#print numpy.degrees(angle_width), numpy.degrees(angles)
#print angle_width[0], angle_width[1]
in_cone1 = (angles > angle_width[0]) & (angles < angle_width[1])
in_cone2 = (angles+math.pi > angle_width[0]) & (angles+math.pi < angle_width[1])
in_cone = in_cone1 | in_cone2
# print angles[in_cone]
#print angles[in_cone][0], angles[in_cone][0] > angle_width[0], angles[in_cone][0] < angle_width[1]
close_enough = (d_total < ((n_stars+1) * 10. / 0.11)) & (d_total > 5 / 0.11)
similar_brightness = numpy.fabs(corr_cat[:,SXFocusColumn['mag_auto']] \
- corr_cat[s1,SXFocusColumn['mag_auto']]) < 1
good = in_cone & close_enough & similar_brightness
good[s1] = True
# print s1, ":", numpy.sum(in_cone), numpy.sum(close_enough), numpy.sum(similar_brightness), numpy.sum(good)
if (numpy.sum(good) <= 1):
continue
candidates = corr_cat[good]
# print "# canddates =", candidates.shape[0]
if (not candidates.shape[0] == n_stars):
# Only use full sequences
continue
pass
# print "found match:",s1
# Now we have a set with the right number of stars, matching the overall
# angle, and with similar brightnesses
# sort them top to bottom
si = numpy.argsort(candidates[:,3])
#numpy.savetxt(focus_stars, candidates[:,3])
#numpy.savetxt(focus_stars, si)
sorted_candidates = candidates[si]
# print "XXX", sorted_candidates.shape, sorted_candidates[:,0].shape, focus_positions.shape, n_stars, candidates.shape[0]
sorted_candidates[:,0] = focus_positions
#numpy.savetxt(focus_stars, sorted_candidates)
#numpy.savetxt(focus_stars, numpy.degrees(angles[good]))
#numpy.savetxt(focus_stars, in_cone[good])
#numpy.savetxt(focus_stars, d_total[good])
#numpy.savetxt(focus_stars, (corr_cat[:,10] - corr_cat[s1,10])[good])
#print >>focus_stars, "\n\n\n\n"
all_candidates.append(sorted_candidates)
#focus_stars.close()
all_candidates = numpy.array(all_candidates)
logger.debug(str(all_candidates.shape))
#xxx = open("steps", "w")
# Now compute the distances from each star to the previous
step_vectors = []
for i in range(1, n_stars):
# logger.debug("Candidates: %d %s\n%s" % (all_candidates.ndim, str(all_candidates.shape), str(all_candidates)))
if (all_candidates.ndim < 1 or
all_candidates.shape[0] <= 0):
# We ran out of candidates
logger.debug("We ran out of viable candidates after %s stars" % (i))
return None
steps = all_candidates[:,i,2:4] - all_candidates[:,i-1,2:4]
#numpy.savetxt(xxx, steps)
#print >>xxx, "\n\n\n\n"
logger.debug("Computing average step size, star %d" % (i))
logger.debug("Steps-X:\n%s" % (str(steps[:,0])))
logger.debug("Steps-y:\n%s" % (str(steps[:,1])))
clean_dx = three_sigma_clip(steps[:,0])
clean_dy = three_sigma_clip(steps[:,1])
# Check if both clean_dx and clean_dy are not empty
logger.debug("clean-dx=%s" % (str(clean_dx)))
logger.debug("clean-dy=%s" % (str(clean_dy)))
if (clean_dx.ndim < 1 or clean_dx.shape[0] <= 0 or
clean_dy.ndim < 1 or clean_dy.shape[0] <= 0):
logger.debug("Can't find a clean dx/dy shift in iteration %d" % (i))
return None
dx = numpy.median(clean_dx)
dy = numpy.median(clean_dy)
distx = scipy.stats.scoreatpercentile(clean_dx, [16,84])
disty = scipy.stats.scoreatpercentile(clean_dy, [16,84])
sigma_x = 0.5 * (distx[1] - distx[0])
sigma_y = 0.5 * (disty[1] - disty[0])
step_vectors.append([dx, dy, sigma_x, sigma_y])
good_steps = (steps[:,0] > (dx - 3*sigma_x)) & (steps[:,0] < (dx + 3*sigma_x)) \
& (steps[:,1] > (dy - 3*sigma_y)) & (steps[:,1] < (dy + 3*sigma_y))
logger.debug("before step-matching #%d: %s" % (i, str(all_candidates.shape)))
all_candidates = all_candidates[good_steps]
logger.debug("after step-matching: #%d: %s" % (i, str(all_candidates.shape)))
logger.debug("%s: %s" % (filename, str(step_vectors)))
# final_focus = open("final_focus", "w")
# for i in range(all_candidates.shape[0]):
# numpy.savetxt(final_focus, all_candidates[i])
# print >>final_focus, "\n\n\n\n\n"
# final_focus.close()
logger.debug("Found %d focus stars" % (all_candidates.shape[0]))
return all_candidates, real_focus_positions
logger.debug("Returning final FITS table catalog")
return None
def create_psf_profiles(infilename, outdir_base, width):
#
# Create an output directory to hold output files
#
_, output_dir = os.path.split(infilename)
# output_dir, _ = os.path.split(os.path.abspath(infilename))
if (infilename.endswith(".fits")):
output_dir = output_dir[:-5]
output_dir = "%s/%s" % (outdir_base, output_dir)
try:
os.mkdir(output_dir)
except:
pass
# Open the file
logger.info("Opening input file %s" % (infilename))
hdulist = pyfits.open(infilename)
is_odi_frame = ('INSTRUME' in hdulist[0].header and
hdulist[0].header['INSTRUME'] == 'podi')
# columns = ['RA', 'DEC', 'X', 'Y', 'OTA', 'BACKGROUND', 'FLAGS', 'MAG_D60']
columns = ['RA', 'DEC', 'X', 'Y', 'OTA', 'FWHM_IMAGE', 'BACKGROUND', 'FLAGS', 'MAG_D60']
raw_cols = ['ra', 'dec', 'x', 'y', 'ota', 'fwhm_image', 'background', 'flags', 'mag_aper_6.0']
col_format = ['%11.6f', '%11.6f', '%8.2f', '%8.2f', '%3d', '%6.2f', '%8.2f', '%6x', '%7.3f']
if (is_odi_frame):
logger.info("This frame IS a ODI frame!")
# Catalog of ODI sources
table = hdulist['CAT.ODI']
# convert table into a n-D numpy catalog, with ra/dec, x/y, and magnitudes
n_sources = table.data.field(0).shape[0]
#logger.info("Found %d stars in catalog" %(n_sources))
starcat = numpy.empty((n_sources, len(columns)))
for idx, colname in enumerate(columns):
starcat[:,idx] = table.data.field(colname)
else:
# If it's not a ODI frame, we need to run source extractor first
logger.info("This is not a ODI frame, running source extractor")
fitsfile = infilename
catfile = infilename[:-5]+".cat"
sex_config_file = "%s/config/wcsfix.sex" % (sitesetup.exec_dir)
parameters_file = "%s/config/wcsfix.sexparam" % (sitesetup.exec_dir)
sexcmd = "%s -c %s -PARAMETERS_NAME %s -CATALOG_NAME %s %s" % (
sitesetup.sextractor, sex_config_file, parameters_file, catfile,
fitsfile)
logger.debug("Sextractor command:\n%s" % (sexcmd))
if (not os.path.isfile(catfile)):
logger.debug("Running SourceExtractor")
start_time = time.time()
try:
ret = subprocess.Popen(sexcmd.split(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(sex_stdout, sex_stderr) = ret.communicate()
#os.system(sexcmd)
if (ret.returncode != 0):
logger.warning("Sextractor might have a problem, check the log")
logger.debug("Stdout=\n"+sex_stdout)
logger.debug("Stderr=\n"+sex_stderr)
except OSError as e:
podi_logging.log_exception()
print >>sys.stderr, "Execution failed:", e
end_time = time.time()
logger.debug("SourceExtractor returned after %.3f seconds" % (end_time - start_time))
else:
logger.debug("Reusing existing source catalog!")
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
source_cat = numpy.loadtxt(catfile)
except IOError:
logger.error("The Sextractor catalog is empty, ignoring this OTA")
sys.exit(0)
# Now translate the raw sextractor catalog into the format we need here
n_sources = source_cat.shape[0]
starcat = numpy.empty((n_sources, len(raw_cols)))
for idx, colname in enumerate(raw_cols):
starcat[:,idx] = source_cat[:, SXcolumn[colname]]
# Write fake header to file
# assume zeropoint of 25 mag, corrected for exposure time
# find brightest star, assume its 12th mag
hdulist[0].header['PHOTZP_X'] = 10. - numpy.min(starcat[:,-1])
print hdulist[0].header['PHOTZP_X']
#
# Also assing ODI-compatible extension names
#
for ext in range(1, len(hdulist)):
hdulist[ext].name = "OTA%02d.SCI" % (ext)
hdulist.info()
# Convert all instrumental magnitudes into calibrated ones
starcat[:,-1] += hdulist[0].header['PHOTZP_X'] # no correction for exptime etc.
logger.info("Found a total of %4d stars" % (starcat.shape[0]))
#
# Eliminate all stars with nearby sources
#
pixelscale_center = 0.1/3600.
with warnings.catch_warnings():
warnings.simplefilter("ignore")
wcs = astWCS.WCS(hdulist[1].header, mode='pyfits')
pixelscale_center = wcs.getPixelSizeDeg() * 3600. # this is arcsec per pixel
within_range_pixels = 5.0 / pixelscale_center
logger.debug("nearby stars: if within %.2f pixels" % (within_range_pixels))
kdtree = scipy.spatial.cKDTree(starcat[:,2:4]) # use x/y
# need to look for at least 2 neighbors, since every source also shows up as its one neighbor
nearest_neighbor, i = kdtree.query(x=starcat[:,2:4], k=2, p=2,
distance_upper_bound=within_range_pixels)
numpy.savetxt("nn.dump", nearest_neighbor)
neighbor_count = numpy.sum(numpy.isfinite(nearest_neighbor), axis=1) - 1.0
numpy.savetxt("nnc.dump", neighbor_count, '%d')
# now only pick stars with no neighbor star within 5 arcsec
starcat = starcat[neighbor_count < 1]
logger.info("Eliminating % 4d stars due to nearby neighbors" % (numpy.sum(neighbor_count >= 1)))
#print nearest_neighbor
# Now exclude all stars with flags, indicating something might be wrong
starcat = starcat[starcat[:,-2] == 0]
logger.info("Found %d good stars with no flags" % (starcat.shape[0]))
# Print the top of the catalog just for illustrative purposes
numpy.savetxt(sys.stdout, starcat[:5,:], col_format)
#
# try to isolate non-stellar sources.
# this likely only works if most sources are stars, i.e. not in galaxy clusters
#
fwhms = starcat[:, columns.index('FWHM_IMAGE')]
filtered_fwhm, is_star = three_sigma_clip(fwhms, return_mask=True)
#print "All sources:\n",starcat[:,columns.index('FWHM_IMAGE')]
#print "\n\nFWHM like stars:\n",starcat[:,columns.index('FWHM_IMAGE')][is_star]
#print "\n\nFWHM not stars:\n",starcat[:,columns.index('FWHM_IMAGE')][~is_star]
# Apply star-only selection
starcat = starcat[is_star]
#
# Now select average profiles for each of the magitude ranges
#
mag_ranges = [(14,15), (15,16), (16,17)] #(18,18.5), (20,20.5)]
exclude_outlying_otas = is_odi_frame
if (exclude_outlying_otas):
exclude = (starcat[:,columns.index('OTA')] < 22) | \
(starcat[:,columns.index('OTA')] > 44)
starcat = starcat[~exclude]
readme_file = "%s/README" % (output_dir)
readme = write_readme_file(hdulist, readme_file, os.path.abspath(infilename), starcat)
for mag_min, mag_max in mag_ranges:
#print mag_min, mag_max
# Select only stars in the right magnitude range
in_mag_range = (starcat[:,-1] >= mag_min) & (starcat[:,-1] <= mag_max)
n_stars_in_mag_range = numpy.sum(in_mag_range)
if (n_stars_in_mag_range < 0):
logger.warning("too few stars (%d) in magnitude range %.2f -- %.2f, skipping" % (
n_stars_in_mag_range, mag_min, mag_max))
continue
# Also select only isolated stars
logger.info("Found %5d stars in magnitude range %.2f -- %.2f, searching for isolated ones" % (
n_stars_in_mag_range, mag_min, mag_max))
#
# Cherry-pick stars with the right magnitudes
#
selected_cat = starcat[in_mag_range]
#numpy.savetxt(sys.stdout, selected_cat, col_format)
print "---"
print >>readme, """
==> Magnitude range %6.2f to %.2f mag (source count: %d):
---------------------------------------------------------------------------------
Ra Dec X Y OTA FWHM Background Flags Magnitude
---------------------------------------------------------------------------------\
""" % (
mag_min, mag_max, selected_cat.shape[0])
numpy.savetxt(readme, selected_cat, [
'%12.6f', '%12.6f', '%10.3f', '%10.3f', '%5d', '%6.2f', '%11.3f', '%6x', '%10.4f'])
print >>readme, " ================================================================================="
#
# Now create the average star profile
#
all_r = numpy.array([])
all_data = numpy.array([])
# Loop over OTAs
otas = set(list(selected_cat[:,columns.index('OTA')]))
profile_file = open("%s/profile__%5.2f--%5.2f.dat" % (output_dir, mag_min, mag_max), "w")
star_id = 0
for ota in otas:
in_this_ota = selected_cat[:,columns.index('OTA')] == ota
ota_cat = selected_cat[in_this_ota]
ota_extname = "OTA%02d.SCI" % (ota)
data = hdulist[ota_extname].data.T
with warnings.catch_warnings():
warnings.simplefilter("ignore")
wcs = astWCS.WCS(hdulist[ota_extname].header, mode='pyfits')
pixelscale = wcs.getPixelSizeDeg() * 3600 # this is arcsec per pixel
for star in range(ota_cat.shape[0]):
star_id += 1
fx = ota_cat[star,columns.index('X')] - 1.
fy = ota_cat[star,columns.index('Y')] - 1.
r, cutout = get_profile(data, center_x=fx, center_y=fy,
mx=0, my=0, width=width,
mode='radial',
normalize=False,
)
# Subtract background as determined by Sextractor
cutout -= ota_cat[star, columns.index('BACKGROUND')]
# convert magnitude into raw flux, and scale star by flux
inst_mag = ota_cat[star, -1] - hdulist[0].header['PHOTZP_X']
flux = math.pow(10, -0.4*inst_mag)
cutout /= flux
print >>profile_file
print >>profile_file, "# Source % 4d of % 4d:" % (star_id, selected_cat.shape[0])
print >>profile_file, "# Ra/Dec: %13.7f %13.7f" % (
ota_cat[star,columns.index('RA')], ota_cat[star,columns.index('DEC')])
print >>profile_file, "# X/Y: %10.5f %10.5f @ OTA %d" % (
ota_cat[star,columns.index('X')], ota_cat[star,columns.index('Y')], ota_cat[star,columns.index('OTA')])
within_width = r < width
good_r = r[within_width]
good_data = cutout[within_width]
good_arcsec = good_r * pixelscale
buffer = numpy.empty((good_r.shape[0], 3))
buffer[:,0] = good_r
buffer[:,1] = good_data
buffer[:,2] = good_arcsec
numpy.savetxt(profile_file, buffer)
# numpy.append(good_r, good_data, axis=1))
#numpy.append([within_width], cutout.reshape(-1,1)[within_width], axis=1))
print >>profile_file
profile_file.close()
readme.close()
def worker_slave(queue):
"""
This function handles all work, either running collectcells locally or
remotely via ssh. Files to reduce are read from a queue.
"""
logger = logging.getLogger("SAMPWorker")
logger.info("Worker process started, ready for action...")
if (not setup.use_ssh):
# If we reduce frames locally, prepare the QR logging.
options['clobber'] = False
while (True):
try:
# print "\n\nWaiting for stuff to do\n\n"
task = queue.get()
except (KeyboardInterrupt, SystemExit) as e:
# print "worker received termination notice"
# Ignore the shut-down command here, and wait for the official
# shutdown command from main task
continue
if (task is None):
logger.info("Shutting down worker")
queue.task_done()
break
filename, object_name, obsid = task
logger.info("starting work on file %s" % (filename))
ccopts = ""
if (len(sys.argv) > 2):
# There are some parameters to be forwarded to collectcells
ccopts = " ".join(sys.argv[1:])
# print "ccopts=",ccopts
if (cmdline_arg_isset("-dryrun")):
logger.info("DRYRUN: Sending off file %s for reduction" %
(filename))
# print "task done!"
queue.task_done()
continue
if (object_name.lower().find("focus") >= 0):
#
# This is most likely a focus exposure
#
n_stars = int(cmdline_arg_set_or_default("-nstars", 7))
logger.info("New focus exposure to analyze (with %d stars)" %
(n_stars))
if (setup.use_ssh):
remote_inputfile = setup.translate_filename_local2remote(
filename)
kw = {
'user': setup.ssh_user,
'host': setup.ssh_host,
'filename': remote_inputfile,
'podidir': setup.remote_podi_dir,
'outdir': setup.output_dir,
'nstars': n_stars,
}
ssh_command = "ssh %(user)s@%(host)s %(podidir)s/podi_focus.py -nstars=%(nstars)d %(filename)s %(outdir)s" % kw
logger.info("Out-sourcing work to %(user)s@%(host)s via ssh" %
kw)
process = subprocess.Popen(ssh_command.split(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True)
_stdout, _stderr = process.communicate()
process.stdout.close()
process.stderr.close()
try:
process.terminate()
except:
pass
#print "*"*80
if (not _stdout == "" and _stdout is not None):
logger.info("Received STDOUT:\n%s" % (_stdout))
#print "*"*80
if (not _stderr == "" and _stderr is not None):
logger.info("Received STDERR:\n%s" % (_stderr))
#print _stderr
#print "*"*80
else:
# Run locally
logger.info("Analyzing focus sequence (%s) locally" %
(filename))
podi_focus.get_focus_measurement(filename,
n_stars=n_stars,
output_dir=setup.output_dir)
logger.info("Done with analysis")
# Now check if we are supposed to open/display the focus plot
if (setup.focus_display is not None):
remote_filename = "%s/%s_focus.png" % (setup.output_dir, obsid)
local_filename = setup.translate_filename_remote2local(
filename, remote_filename)
cmd = "%s %s &" % (setup.focus_display, local_filename)
logger.info("Opening and displaying plot")
os.system(cmd)
else:
#
# This is NOT a focus exposure
#
if (setup.use_ssh):
# This is not a focus exposure, to treat it as a normal science exposure
remote_inputfile = setup.translate_filename_local2remote(
filename)
kw = {
'user': setup.ssh_user,
'host': setup.ssh_host,
'collectcells': setup.ssh_executable,
'options': ccopts,
'filename': remote_inputfile,
'outputfile': setup.output_format,
}
ssh_command = "ssh %(user)s@%(host)s %(collectcells)s %(filename)s %(outputfile)s %(options)s -noclobber" % kw
process = subprocess.Popen(ssh_command.split(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True)
_stdout, _stderr = process.communicate()
process.stdout.close()
process.stderr.close()
try:
process.terminate()
except:
pass
#print "*"*80
if (not _stdout == "" and _stdout is not None):
logger.info("Received STDOUT:\n%s" % (_stdout))
#print "*"*80
if (not _stderr == "" and _stderr is not None):
logger.info("Received STDERR:\n%s" % (_stderr))
#print _stderr
#print "*"*80
else:
logger.info("Running collectcells (%s)" % (filename))
podi_collectcells.collectcells_with_timeout(
input=filename,
outputfile=setup.output_format,
options=options,
timeout=300,
process_tracker=process_tracker)
#
# If requested, also send the command to ds9
#
local_filename = setup.translate_filename_remote2local(
filename, setup.output_format)
if (cmdline_arg_isset("-forward2ds9")):
forward2ds9_option = cmdline_arg_set_or_default(
"-forward2ds9", "image")
if (forward2ds9_option == "irafmosaic"):
cmd = "mosaicimage iraf %s" % (local_filename)
else:
cmd = "fits %s" % (local_filename)
logger.info("Forwarding file to ds9")
logger.debug("filename: %s" % (filename))
logger.debug("remote file: %s" % (remote_inputfile))
logger.debug("local file: %s" % (local_filename))
try:
cli1 = sampy.SAMPIntegratedClient(metadata=metadata)
cli1.connect()
cli1.enotify_all(mtype='ds9.set', cmd='frame 2')
cli1.enotify_all(mtype='ds9.set', cmd='scale scope global')
cli1.enotify_all(mtype='ds9.set', cmd=cmd)
cli1.disconnect()
except Exception as err:
logger.error("Problems sending message to ds9: %s" % err)
podi_logging.log_exception()
pass
# By default, also open the psf diagnostic plot, if available
psf_plot_fn = local_filename[:-5] + ".psf.png"
if (os.path.isfile(psf_plot_fn)):
cmd = "%s %s &" % (setup.focus_display, psf_plot_fn)
logger.info("Opening and displaying PSF diagnostic plot (%s)" %
(psf_plot_fn))
os.system(cmd)
#
# Once the file is reduced, mark the current task as done.
#
logger.info("task done!")
queue.task_done()
print("Terminating worker process...")
return
def combine_pupilghost_slices(out_filename, filelist, op='sigclipmean'):
logger = logging.getLogger("CombinePG")
print filelist
#
# Gather information about rotation angles and center positions
# Also collect the association data.
#
rotangles = numpy.ones(len(filelist)) * -9999
headers = [None] * len(filelist)
#
# Prepare the association data
#
assoc_table = {}
logger.info("Reading data from input files")
for idx, fn in enumerate(filelist):
hdulist = pyfits.open(fn)
# Read the center and angle keywords from the "COMBINED" extension
comb_hdu = hdulist['COMBINED']
_ra = comb_hdu.header['ROTANGLE']
rotangles[idx] = _ra if _ra > 0 else _ra + 360.
headers[idx] = comb_hdu.header
logger.debug("%s: %.2f" % (fn, _ra))
# comb_hdu.header['CNTRF%03d' % (norm_angle)] = (centerf_str[:-1], "PG center, fixed r [px]")
# comb_hdu.header['CNTRV%03d' % (norm_angle)] = (centerv_str[:-1], "PG center, var. r [px]")
# comb_hdu.header['ALPHA%03d' % (norm_angle)] = (d_angle_str[:-1], "OTA angle [arcmin]")
# comb_hdu.header['OTAORDER'] = ota_str[:-1]
this_assoc = {'pupilghost-slice': fn }
assoc_table = podi_associations.collect_reduction_files_used(assoc_table, this_assoc)
# Read the assocation table of this frame
in_assoc = podi_associations.read_associations(hdulist)
if (in_assoc is not None):
logger.debug("Found assocations:\n%s" % (str(in_assoc)))
assoc_table = podi_associations.collect_reduction_files_used(assoc_table, in_assoc)
# return
#
# Now sort the rotator angles from smallest to largest
#
angle_sort = numpy.argsort(rotangles)
#
# Combine all frames
#
logger.info("Stacking all slices into master pupilghost template")
combined_hdulist = podi_imcombine.imcombine(
filelist,
outputfile=None,
operation=op, #nanmean.bn',
return_hdu=True,
subtract=None, scale=None)
print combined_hdulist
combined = combined_hdulist['COMBINED']
combined.header['STACK_OP'] = op
#
# Add the sorted keywords back into the resulting file
#
logger.info("Adding metadata")
primhdu = pyfits.PrimaryHDU()
try:
prev_hdr = None
for header, label in [
('CNTRF%03d', "center, fixed radius"),
('CNTRV%03d', "center, var. radius"),
('ALPHA%03d', "angle mismatch [arcmin]"),
('NORM_%03d', "sector normalizations"),
]:
first_hdr = None
for i in range(rotangles.shape[0]):
idx = angle_sort[i]
rotangle = rotangles[idx]
round_angle = headers[idx]['RNDANGLE']
keyname = header % round_angle
#logger.info("Adding key: %s" % (keyname))
combined.header[keyname] = headers[idx][keyname]
#primhdu.header[keyname] = headers[idx][keyname]
first_hdr = keyname if first_hdr is None else first_hdr
if (prev_hdr is None):
print "adding header",keyname," somewhere"
primhdu.header.append((keyname, headers[idx][keyname]))
prev_hdr = keyname
else:
print "adding header",keyname,"after",prev_hdr
primhdu.header.insert(prev_hdr, (keyname, headers[idx][keyname]), after=True)
prev_hdr = keyname
add_fits_header_title(primhdu.header, label, first_hdr)
combined.header['OTAORDER'] = headers[0]['OTAORDER']
except:
podi_logging.log_exception()
pass
print assoc_table
assoc_hdu = podi_associations.create_association_table(assoc_table)
out_hdulist = [primhdu, combined, assoc_hdu]
for name in ['PROFILE', 'RAWPROFILE']:
try:
logger.info("Adding in extension %s" % (name))
out_hdulist.append(combined_hdulist[name])
except:
logger.warning("Unable to find extension %s" % (name))
pass
combined_hdulist.writeto("dummy.fits", clobber=True)
logger.info("Writing output to %s" % (out_filename))
out_hdulist = pyfits.HDUList(out_hdulist) #[primhdu, combined, assoc_hdu])
out_hdulist.writeto(out_filename, clobber=True)
logger.info("Work complete!")
def imcombine(input_filelist,
outputfile,
operation,
return_hdu=False,
subtract=None,
scale=None,
gather_all_otas=True):
logger = logging.getLogger("ImCombine")
# First loop over all filenames and make sure all files exist
filelist = []
for file in input_filelist:
if (os.path.isfile(file)):
filelist.append(file)
if (len(filelist) <= 0):
logger.error(
"No existing files found in input list, hence nothing to do!\n")
return None
logger.debug("Stacking the following files:\n -- %s" %
("\n -- ".join(filelist)))
# elif (len(filelist) == 1):
# # stdout_write("Only 1 file to combine, save the hassle and copy the file!\n")
# hdulist = pyfits.open(filelist[0])
# if (return_hdu):
# return hdulist
# hdulist.writeto(outputfile, clobber=True)
# return
# Read the input parameters
# Note that file headers are copied from the first file
reference_filename = filelist[0]
ref_hdulist = pyfits.open(reference_filename)
# Create the primary extension of the output file
# Copy all headers from the reference HDU
primhdu = pyfits.PrimaryHDU(header=ref_hdulist[0].header)
# Add PrimaryHDU to list of OTAs that go into the output file
out_hdulist = [primhdu]
ref_hdulist.close()
del ref_hdulist
#
# Compile a list of OTAs that will be in the output frame
#
otas_found = []
ota_sizes = {}
ref_header = {}
if (gather_all_otas):
logger.info(
"Checking all files to compile comprehensive list of available OTAs"
)
master_associations = None
for fn in filelist:
hdulist = pyfits.open(fn)
for ext in hdulist:
if (is_image_extension(ext)):
try:
otas_found.append(ext.name)
if (ext.name not in ota_sizes):
ota_sizes[ext.name] = ext.data.shape
ref_header[ext.name] = ext.header
except:
podi_logging.log_exception()
#
# Collect all individual association tables to create
# a master association table combining all input data
#
assoc_table = podi_associations.read_associations(hdulist)
if (assoc_table is None):
logger.info("No association data available")
assoc_table = {'input_simple': [fn]}
if (master_associations is None):
master_associations = assoc_table
else:
master_associations = podi_associations.collect_reduction_files_used(
master_associations, assoc_table)
hdulist.close()
del hdulist
if (not gather_all_otas):
break
otas_to_combine = set(otas_found)
ota_counter = collections.Counter(otas_found)
#print otas_to_combine
#print ota_counter
#
# Now loop over all extensions and compute the mean
#
# for cur_ext in range(0, len(ref_hdulist)):
for cur_ext, extname in enumerate(otas_to_combine):
data_blocks = []
# Check what OTA we are dealing with
# if (not is_image_extension(ref_hdulist[cur_ext])):
# continue
# ref_fppos = ref_hdulist[cur_ext].name #header['EXTNAME']
#stdout_write("\rCombining frames for OTA %s (#% 2d/% 2d) ..." % (ref_fppos, cur_ext+1, len(ref_hdulist)))
#logger.debug("Combining frames for OTA %s (#% 2d/% 2d) ..." % (ref_fppos, cur_ext+1, len(ref_hdulist)))
logger.info("Combining frames for OTA %s (#% 2d/% 2d) ..." %
(extname, cur_ext + 1, len(otas_to_combine)))
#
# Add some weird-looking construct to move the memory allocation and actual
# imcombine into a separate process. This has to do with if/how/when python releases
# memory (or not), causing a massive short-term memory leak.
# With the additional process, the memory is onwed by the other process and the memory
# is freed once we destroy this helper process.
#
return_queue = multiprocessing.JoinableQueue()
#worker_args=(ref_fppos, filelist, ref_hdulist[cur_ext].data.shape, operation, return_queue, verbose)
kw_args = {
'extension':
extname,
'filelist':
filelist,
'shape': (ota_sizes[extname][0], ota_sizes[extname][1],
ota_counter[extname]), #ref_hdulist[cur_ext].data.shape,
'operation':
operation,
'queue':
return_queue,
'verbose':
verbose,
'subtract':
subtract,
'scale':
scale,
}
# p = multiprocessing.Process(target=imcombine_subprocess, args=worker_args)
p = multiprocessing.Process(target=imcombine_subprocess,
kwargs=kw_args)
p.start()
combined = return_queue.get()
p.terminate()
del p
if (verbose): stdout_write(" done, creating fits extension ...")
logger.debug("done with computing, creating fits extension ...")
# Create new ImageHDU, insert the imcombined's data and copy the
# header from the reference frame
hdu = pyfits.ImageHDU(data=combined, header=ref_header[extname])
# Append the new HDU to the list of result HDUs
out_hdulist.append(hdu)
if (verbose): stdout_write(" done\n")
#
# At this point, add a fits table listing all filenames that went into this combination
#
filenames_only = []
for file in filelist:
dirname, filename = os.path.split(file)
filenames_only.append(filename)
logger.debug("Adding association data to output file")
out_hdulist[0].header["NCOMBINE"] = (len(filenames_only),
"number of combined files")
columns = [
pyfits.Column(name='filenumber',
format="I4",
array=range(1,
len(filenames_only) + 1)),
pyfits.Column(name='filename', format="A100", array=filenames_only),
]
coldefs = pyfits.ColDefs(columns)
tablehdu = pyfits.BinTableHDU.from_columns(coldefs)
tablehdu.header["EXTNAME"] = "FILELIST"
out_hdulist.append(tablehdu)
assoc_tbhdu = podi_associations.create_association_table(
master_associations)
out_hdulist.append(assoc_tbhdu)
#
# All work done now, prepare to return the data or write it to disk
#
out_hdu = pyfits.HDUList(out_hdulist)
if (not return_hdu and outputfile != None):
logger.debug(" writing results to file %s ..." % (outputfile))
clobberfile(outputfile)
try:
out_hdu.writeto(outputfile, clobber=True, checksum=True)
except TypeError:
# this most likely is this error:
# TypeError: object of type 'NoneType' has no len()
# related to the checksum calculation
clobberfile(outputfile)
out_hdu.writeto(outputfile, clobber=True)
except pyfits.VerifyError:
logger.warning(
"Encountered FITS verification error, writing anyway")
try:
out_hdu.writeto(outputfile,
clobber=True,
checksum=False,
output_verify='ignore')
except:
raise
except:
raise
out_hdu.close()
del out_hdu
del out_hdulist
stdout_write(" done!\n")
elif (return_hdu):
logger.debug(" returning HDU for further processing ...")
return out_hdu
else:
logger.debug(" couldn't write output file, no filename given!")
return None
def imcombine_sharedmem_data(shmem_buffer, operation, sizes):
size_x, size_y, n_frames = sizes
shmem_results = SharedMemory(ctypes.c_float, (size_x, size_y))
# multiprocessing.RawArray(ctypes.c_float, size_x*size_y)
logger = logging.getLogger("CombineMgr")
#
# Set up the parallel processing environment
#
queue = multiprocessing.JoinableQueue()
return_queue = multiprocessing.Queue()
# Now compute median/average/sum/etc
# buffer = shmem_as_ndarray(shmem_buffer).reshape((size_x, size_y, n_frames))
buffer = shmem_buffer.to_ndarray()
for line in range(buffer.shape[0]):
#print "Adding line",line,"to queue"
queue.put(line)
lines_done = numpy.zeros((buffer.shape[0]), dtype=numpy.bool)
lines_read = 0
#result_buffer = numpy.zeros(shape=(buffer.shape[0], buffer.shape[1]), dtype=numpy.float32)
processes = []
for i in range(number_cpus):
worker_args = (queue, return_queue, shmem_buffer, shmem_results,
size_x, size_y, n_frames, operation)
p = multiprocessing.Process(target=parallel_compute, args=worker_args)
p.start()
processes.append(p)
while (lines_read < buffer.shape[0]
and numpy.sum(lines_done) < buffer.shape[0]):
try:
line = return_queue.get(timeout=5)
lines_read += 1
try:
lines_done[line] = True
except:
pass
except Queue.Empty:
logger.error("Encountered timeout while combinging data")
# something bad has happened to one of the workers
# find one of the lines that has not been processed yet
missing_lines = (numpy.arange(buffer.shape[0]))[~lines_done]
logger.info("Re-queuing %d lines for processing" %
(missing_lines.shape[0]))
for line in missing_lines:
queue.put(line)
except:
podi_logging.log_exception()
# Tell all workers to shut down when no more data is left to work on
logger.debug("telling all workers to shut down!")
for i in range(number_cpus):
logger.debug("telling all worker %d to shut down!" % (i))
queue.put((None))
# Once all command are sent out to the workers, join them to speed things up
logger.debug("Terminating workers!")
for p in processes:
p.terminate()
p.join(timeout=1)
results = numpy.copy(
shmem_results.to_ndarray()) #).reshape((size_x, size_y)))
shmem_results.free()
del shmem_results
del queue
del buffer
return results
def create_saturation_catalog(filename,
output_dir,
verbose=True,
mp=False,
redo=False,
saturation_limit=65535):
"""
Create catalogs listing all saturated pixels to enable handling saturation
and persistency effects later-on.
The main purpose of this file is to call create_saturation_catalog_ota,
possibly wrapped for with mp_create_saturation_table for parallel
processing.
Parameters
----------
filename : string
One file of the exposure. This file is mainly used to obtain the
necessary information to create all the other filenames for this
exposure.
output_dir : string
Directory to hold all the saturation catalogs. This is the directory
that will be fed into collectcells via the -persistency command line
flag.
mp : bool - not used
redo : bool
Recreate the saturation catalog if it already exists
Returns
-------
"""
logger = logging.getLogger("CreateSaturationCatalog")
logger.info("Creating saturation mask for %s ..." % (filename))
if (os.path.isfile(filename)):
# This is one of the OTA fits files
# extract the necessary information to generate the
# names of all the other filenames
try:
hdulist = pyfits.open(filename)
except IOError:
logger.warning("\rProblem opening file %s...\n" % (filename))
return
except:
podi_logging.log_exception()
hdr_filename = hdulist[0].header['FILENAME']
hdr_items = hdr_filename.split('.')
basename = "%s.%s" % (hdr_items[0], hdr_items[1])
hdulist.close()
# Split the input filename to extract the directory part
directory, dummy = os.path.split(filename)
elif (os.path.isdir(filename)):
# As a safety precaution, if the first parameter is the directory containing
# the files, extract just the ID string to be used for this script
if (filename[-1] == "/"):
filename = filename[:-1]
basedir, basename = os.path.split(filename)
directory = filename
else:
logger.error("Input %s is neither a file nor a directory!" %
(filename))
logger.error("Aborting operation due to illegal input.")
return
output_filename = "%s/%s.saturated.fits" % (output_dir, basename)
logger.debug("Output saturation catalog: %s" % (output_filename))
if (os.path.isfile(output_filename) and not redo):
logger.debug("File (%s) exists, skipping!" % (output_filename))
return
# Setup parallel processing
queue = multiprocessing.JoinableQueue()
return_queue = multiprocessing.JoinableQueue()
#return_queue = multiprocessing.Queue()
number_jobs_queued = 0
first_fits_file = None
ota_list = []
for (ota_x, ota_y) in itertools.product(range(8), repeat=2):
ota = ota_x * 10 + ota_y
filename = "%s/%s.%02d.fits" % (directory, basename, ota)
if (not os.path.isfile(filename)):
filename = "%s/%s.%02d.fits.fz" % (directory, basename, ota)
if (not os.path.isfile(filename)):
continue
queue.put((filename, saturation_limit))
number_jobs_queued += 1
# Remember the very first FITS file we find. This will serve as the primary HDU
if (first_fits_file is None):
# Create a primary HDU from the first found fits-file
try:
firsthdu = pyfits.open(filename)
except IOError:
logger.warning("Problem opening FITS file %s" % (filename))
continue
logger.debug("Copying general information from file %s" %
(filename))
ota_list.append(pyfits.PrimaryHDU(header=firsthdu[0].header))
firsthdu.close()
firsthdu = None
first_fits_file = filename
if (first_fits_file is None):
logger.warning("Couldn't find a valid FITS file, thus nothing to do")
return
# Now start all the workers
logger.debug("Starting worker processes")
processes = []
for i in range(sitesetup.number_cpus):
p = multiprocessing.Process(target=mp_create_saturation_catalog,
args=(queue, return_queue, False))
p.start()
processes.append(p)
time.sleep(0.01)
# Tell all workers to shut down when no more data is left to work on
logger.debug("Sending shutdown command to worker processes")
for i in range(len(processes)):
if (verbose): stdout_write("Sending quit command!\n")
queue.put((None))
logger.debug("Collecting catalogs for each OTA")
for i in range(number_jobs_queued):
if (verbose): print("reading return ", i)
cat_name = return_queue.get()
if (cat_name is not None):
final_cat, extension_name = cat_name
columns = [
pyfits.Column(name='CELL_X', format='I', array=final_cat[:,
0]),
pyfits.Column(name='CELL_Y', format='I', array=final_cat[:,
1]),
pyfits.Column(name='X', format='I', array=final_cat[:, 2]),
pyfits.Column(name='Y', format='I', array=final_cat[:, 3])
]
# Create the table extension
coldefs = pyfits.ColDefs(columns)
tbhdu = pyfits.BinTableHDU.from_columns(coldefs)
tbhdu.name = extension_name
ota_list.append(tbhdu)
return_queue.task_done()
# Join each process to make thre they terminate(d) correctly
logger.debug("Joining process to ensure proper termination")
for p in processes:
p.join()
hdulist = pyfits.HDUList(ota_list)
output_filename = "%s/%s.saturated.fits" % (output_dir, basename)
clobberfile(output_filename)
logger.debug("Writing output file %s" % (output_filename))
hdulist.writeto(output_filename, overwrite=True)
logger.debug("all done!")
return
def create_saturation_catalog_ota(filename,
output_dir,
verbose=True,
return_numpy_catalog=False,
saturation_limit=65535):
"""
Create a saturation table for a given OTA exposure.
Parameters
----------
filename : string
Filename of the OTA FITS file.
output_dir : string
If return_numpy_catalog is not set, write the saturation catalog into
this directory.
return_numpy_catalog : bool
If set, return the results as numpy array instead of writing individual
files to disk.
Returns
-------
None - if no saturated pixels are found in this frame
ndarray, extname - if return_numpy_catalog is set
Nothing - if return_numpy_array is not set
"""
logger = logging.getLogger("OTASatCat")
# Open filename
logger.debug("Input filename: %s" % (filename))
try:
hdulist = pyfits.open(filename)
except IOError:
logger.debug("Can't open file %s" % (filename))
return None
except:
podi_logging.log_exception()
return None
mjd = hdulist[0].header['MJD-OBS']
obsid = hdulist[0].header['OBSID']
ota = int(hdulist[0].header['FPPOS'][2:4])
datatype = hdulist[0].header['FILENAME'][0]
logger = logging.getLogger("CreateSatCat: %s, OTA %02d" % (obsid, ota))
logger.debug("Starting work")
full_coords = numpy.zeros(shape=(0, 4)) #, dtype=numpy.int16)
saturated_pixels_total = 0
for ext in range(1, len(hdulist)):
if (not is_image_extension(hdulist[ext])):
continue
# Find all saturated pixels (values >= 65K)
data = hdulist[ext].data
saturated = (data >= saturation_limit)
# print hdulist[ext].header['EXTNAME'], data.shape, numpy.max(data)
# Skip this cell if no pixels are saturated
number_saturated_pixels = numpy.sum(saturated)
if (number_saturated_pixels <= 0):
continue
saturated_pixels_total += number_saturated_pixels
wn_cellx = hdulist[ext].header['WN_CELLX']
wn_celly = hdulist[ext].header['WN_CELLY']
# logger.debug("number of saturated pixels in cell %d,%d: %d" % (wn_cellx, wn_celly, number_saturated_pixels))
# Do some book-keeping preparing for the masking
rows, cols = numpy.indices(data.shape)
saturated_rows = rows[saturated]
saturated_cols = cols[saturated]
#print saturated_rows.shape, saturated_cols.shape
coordinates = numpy.zeros(shape=(number_saturated_pixels, 4))
coordinates[:, 0] = wn_cellx
coordinates[:, 1] = wn_celly
coordinates[:, 2] = saturated_cols[:]
coordinates[:, 3] = saturated_rows[:]
full_coords = numpy.append(
full_coords, coordinates,
axis=0) #coordinates if full_coords == None else
final_cat = numpy.array(full_coords, dtype=numpy.dtype('int16'))
if (saturated_pixels_total <= 0):
logger.debug("No saturated pixels found, well done!")
return None
logger.debug("Found %d saturated pixels, preparing catalog" %
(saturated_pixels_total))
# Now define the columns for the table
columns = [\
pyfits.Column(name='CELL_X', format='I', array=final_cat[:, 0]),
pyfits.Column(name='CELL_Y', format='I', array=final_cat[:, 1]),
pyfits.Column(name='X', format='I', array=final_cat[:, 2]),
pyfits.Column(name='Y', format='I', array=final_cat[:, 3])
]
# Create the table extension
coldefs = pyfits.ColDefs(columns)
tbhdu = pyfits.BinTableHDU.from_columns(coldefs)
extension_name = "OTA%02d.SATPIX" % (ota)
tbhdu.name = extension_name
if (return_numpy_catalog):
logger.debug("Returning results as numpy catalog")
return final_cat, extension_name
# Also copy the primary header into the new catalog
primhdu = pyfits.PrimaryHDU(header=hdulist[0].header)
# Create a HDUList for output
out_hdulist = pyfits.HDUList([primhdu, tbhdu])
# And create the output file
output_filename = "%s/%s%s.%02d.saturated.fits" % (output_dir, datatype,
obsid, ota)
stdout_write("Writing output: %s\n" % (output_filename))
clobberfile(output_filename)
out_hdulist.writeto(output_filename, overwrite=True)
if (verbose):
print("some of the saturated pixels:\n", final_cat[0:10, :])
#numpy.savetxt("test", final_cat)
#print full_coords.shape
logger.debug("Retuning final FITS table catalog")
return final_cat
def get_focus_measurement(filename, n_stars=5, output_dir="./", mp=False):
"""
Parameters
----------
filename : string
One file of the exposure. This file is mainly used to obtain the
necessary information to create all the other filenames for this
exposure.
output_dir : string
Directory to hold all the saturation catalogs. This is the directory
that will be fed into collectcells via the -persistency command line
flag.
Returns
-------
"""
logger = logging.getLogger("MeasureFocus")
logger.info("Starting focus measurement for %s (%d *)..." % (filename, n_stars))
if (os.path.isfile(filename)):
# This is one of the OTA fits files
# extract the necessary information to generate the
# names of all the other filenames
try:
hdulist = pyfits.open(filename)
except IOError:
logger.warning("\rProblem opening file %s...\n" % (filename))
return
except:
podi_logging.log_exception()
hdr_filename = hdulist[0].header['FILENAME']
hdr_items = hdr_filename.split('.')
basename = "%s.%s" % (hdr_items[0], hdr_items[1])
hdulist.close()
# Split the input filename to extract the directory part
directory, dummy = os.path.split(filename)
elif (os.path.isdir(filename)):
# As a safety precaution, if the first parameter is the directory containing
# the files, extract just the ID string to be used for this script
if (filename[-1] == "/"):
filename = filename[:-1]
basedir, basename = os.path.split(filename)
directory = filename
# Setup parallel processing
queue = multiprocessing.JoinableQueue()
return_queue = multiprocessing.JoinableQueue()
number_jobs_queued = 0
obsid = None
for (ota_x, ota_y) in itertools.product(range(8), repeat=2):
ota = ota_x * 10 + ota_y
filename = "%s/%s.%02d.fits" % (directory, basename, ota)
if (not os.path.isfile(filename)):
filename = "%s/%s.%02d.fits.fz" % (directory, basename, ota)
if (not os.path.isfile(filename)):
continue
logger.debug("Adding file %s to task list" % (filename))
if (obsid is None):
hdulist = pyfits.open(filename)
obsid = hdulist[0].header['OBSID']
hdulist.close()
queue.put( (filename, n_stars) )
number_jobs_queued += 1
# break
# Now start all the workers
logger.debug("Starting worker processes")
processes = []
for i in range(sitesetup.number_cpus):
p = multiprocessing.Process(target=mp_measure_focus, args=(queue, return_queue, False))
p.start()
processes.append(p)
time.sleep(0.01)
# Tell all workers to shut down when no more data is left to work on
queue.put( (None) )
logger.info("Collecting catalogs for each OTA")
all_foci = None
real_numbers = False
for i in range(number_jobs_queued):
returned = return_queue.get()
if (returned is None):
continue
focus_positions, found_real_numbers = returned
if (found_real_numbers):
real_numbers = True
logger.debug("Received %d focus positions" % (focus_positions.shape[0]))
all_foci = focus_positions if all_foci is None else numpy.append(all_foci, focus_positions, axis=0)
#print cat_name
return_queue.task_done()
# Join each process to make sure they terminate(d) correctly
logger.debug("Joining process to ensure proper termination")
for p in processes:
p.join()
#print all_foci, all_foci.ndim, all_foci.shape
if (all_foci is None or
all_foci.ndim < 2 or
all_foci.shape[0] <= 0):
logger.error("Couldn't find any star patterns!")
return
logger.info("Found a grand total of %d focus positions" % (all_foci.shape[0]))
# print all_foci.shape
with open("allfoci", "w") as f:
for s in range(all_foci.shape[0]):
numpy.savetxt(f, all_foci[s,:,:])
print >>f, "\n"
# numpy.savetxt("allfoci", all_foci.reshape((-1,all_foci.shape[2])))
#
# Eliminate all focus measurements that could be affected by the low-light
# CTE problem
#
min_background = numpy.min(all_foci[:,:,SXFocusColumn['background']], axis=1)
detector_lots = all_foci[:,0,SXFocusColumn['ota_lot']]
bad = (detector_lots < 7) & (min_background < 100)
logger.info("Excluding %d focus samples that might have CTE issues" % (numpy.sum(bad)))
all_foci = all_foci[~bad]
stats = get_mean_focuscurve(all_foci)
pfit, uncert, fwhm_median, fwhm_std, fwhm_cleaned, best_focus_position, best_focus = stats
plotfilename = "%s/%s_focus.png" % (output_dir, obsid)
create_focus_plot(all_foci, stats, basename, plotfilename, real_numbers)
logger.debug("all done!")
return
def imcombine_sharedmem_data(shmem_buffer, operation, sizes):
size_x, size_y, n_frames = sizes
shmem_results = SharedMemory(ctypes.c_float, (size_x,size_y))
# multiprocessing.RawArray(ctypes.c_float, size_x*size_y)
logger = logging.getLogger("CombineMgr")
#
# Set up the parallel processing environment
#
queue = multiprocessing.JoinableQueue()
return_queue = multiprocessing.Queue()
# Now compute median/average/sum/etc
# buffer = shmem_as_ndarray(shmem_buffer).reshape((size_x, size_y, n_frames))
buffer = shmem_buffer.to_ndarray()
for line in range(buffer.shape[0]):
#print "Adding line",line,"to queue"
queue.put(line)
lines_done = numpy.zeros((buffer.shape[0]), dtype=numpy.bool)
lines_read = 0
#result_buffer = numpy.zeros(shape=(buffer.shape[0], buffer.shape[1]), dtype=numpy.float32)
processes = []
for i in range(number_cpus):
worker_args = (queue, return_queue,
shmem_buffer, shmem_results,
size_x, size_y, n_frames, operation)
p = multiprocessing.Process(target=parallel_compute, args=worker_args)
p.start()
processes.append(p)
while (lines_read < buffer.shape[0] and numpy.sum(lines_done) < buffer.shape[0]):
try:
line = return_queue.get(timeout=5)
lines_read += 1
try:
lines_done[line] = True
except:
pass
except Queue.Empty:
logger.error("Encountered timeout while combinging data")
# something bad has happened to one of the workers
# find one of the lines that has not been processed yet
missing_lines = (numpy.arange(buffer.shape[0]))[~lines_done]
logger.info("Re-queuing %d lines for processing" % (missing_lines.shape[0]))
for line in missing_lines:
queue.put(line)
except:
podi_logging.log_exception()
# Tell all workers to shut down when no more data is left to work on
logger.debug("telling all workers to shut down!")
for i in range(number_cpus):
logger.debug("telling all worker %d to shut down!" % (i))
queue.put((None))
# Once all command are sent out to the workers, join them to speed things up
logger.debug("Terminating workers!")
for p in processes:
p.terminate()
p.join(timeout=1)
results = numpy.copy(shmem_results.to_ndarray()) #).reshape((size_x, size_y)))
shmem_results.free()
del shmem_results
del queue
del buffer
return results
#
grpids = {}
print "Gathering exposure groupings"
for ext in hdulist[1:]:
grpid = ext.header['COGRPID']
extname = ext.name
if not grpid in grpids:
grpids[grpid] = []
grpids[grpid].append(extname)
print "Found %d groups" % (len(grpids))
# Now go through each GRPID, and extract the relevant extensions to a separate file
print "Extracting and stacking individual objects"
for idx, grpid in enumerate(grpids):
try:
do_work(hdulist, grpids, grpid)
except:
podi_logging.log_exception()
logger.error("There was a problem processing %d" % (grpid))
pass
break
if (idx > 5):
break
podi_logging.shutdown_logging(options)
def measure_focus_ota(filename, n_stars=5):
"""
Obtain a focus measurement from teh specified filename. To do so,
1) run source extractor to get FWHM measurements and positions for all sources
2) group sources into vertical sequences as expected from the function
of the focus tool
3) assign physical focus positions to each measurement
4) return final catalog back to master so a final, combined focus curve
can be assembled.
"""
# print"\n\n\nworking on file ",filename,"\n\n\n"
try:
hdulist = pyfits.open(filename)
except IOError:
logger.debug("Can't open file %s" % (filename))
return None
except:
podi_logging.log_exception()
return None
obsid = hdulist[0].header['OBSID']
ota = hdulist[1].header['WN_OTAX'] * 10 + hdulist[1].header['WN_OTAY']
ota_id = hdulist[0].header['OTA_ID']
logger = logging.getLogger("MeasureFocusOTA: %s(%02d)" % (obsid, ota))
logger.info("Starting work ...")
obsid = hdulist[0].header['OBSID']
ota = int(hdulist[0].header['FPPOS'][2:4])
# Check the object name to see if if contains the information about the exposure
focus_positions = numpy.arange(n_stars)[::-1] + 1.
real_focus_positions = False
object_name = hdulist[0].header['OBJECT']
if (object_name.startswith("Focus Center")):
# This looks like it might be the right format
try:
items = object_name.split()
# Check all items
if (len(items) == 7 and items[0] == "Focus"
and items[1] == "Center" and items[3] == "NStep"
and items[5] == "DStep"):
n_stars = int(items[4])
focus_center = float(items[2])
focus_step = float(items[6])
focus_start = focus_center - (n_stars - 1) / 2 * focus_step
focus_positions = numpy.arange(
n_stars,
dtype=numpy.float32)[::-1] * focus_step + focus_start
logger.debug(
"Infom from header: N=%d, center=%.0f, step=%.0f, start=%.0f"
% (n_stars, focus_center, focus_Step, focus_step,
focus_start))
logger.debug("Focus positions: %s" % (str(focus_positions)))
real_focus_positions = True
except:
pass
# Run SourceExtractor on the file
sex_config = "%s/config/focus.sexconf" % (sitesetup.exec_dir)
sex_param = "%s/config/focus.sexparam" % (sitesetup.exec_dir)
catfile = "%s/tmp.%s_OTA%02d.cat" % (sitesetup.scratch_dir, obsid, ota)
sex_cmd = "%(sexcmd)s -c %(sex_config)s -PARAMETERS_NAME %(sex_param)s -CATALOG_NAME %(catfile)s %(filename)s" % {
"sexcmd": sitesetup.sextractor,
"sex_config": sex_config,
"sex_param": sex_param,
"catfile": catfile,
"filename": filename,
"redirect": sitesetup.sex_redirect,
}
# print "\n"*10,sex_cmd,"\n"*10
# Run source extractor
# catfile = "/tmp//tmp.pid4383.20121008T221836.0_OTA33.cat"
if (not os.path.isfile(catfile)):
logger.debug("Running source extractor to search for stars")
start_time = time.time()
try:
ret = subprocess.Popen(sex_cmd.split(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(sex_stdout, sex_stderr) = ret.communicate()
if (ret.returncode != 0):
logger.warning(
"Sextractor might have a problem, check the log")
logger.info("Stdout=\n" + sex_stdout)
logger.info("Stderr=\n" + sex_stderr)
except OSError as e:
podi_logging.log_exception()
print >> sys.stderr, "Execution failed:", e
end_time = time.time()
logger.debug("SourceExtractor finished after %.2f seconds" %
(end_time - start_time))
else:
logger.debug("Source catalog already exists, re-using the old file")
#
# delete the tmp catalog
#
#
# Load the source catalog.
# Handle cases of non-existing or empty catalogs
#
logger.debug("loading the source catalog from %s" % (catfile))
try:
source_cat = numpy.loadtxt(catfile)
except IOError:
logger.warning("The Sextractor catalog is empty, ignoring this OTA")
source_cat = None
return None
if (source_cat.shape[0] <= 0):
# no sources found
return None
# print "\n\n total sources in raw file",source_cat.shape,"\n\n"
logger.debug("Found %d sources in raw SourceExtractor catalog %s" %
(source_cat.shape[0], catfile))
#
# Now convert all X/Y values to proper OTA X/Y coordinates based on their
# extension number
#
corr_cat = None
# print "extensions:", source_cat[:,SXFocusColumn['extension']]
for i in range(len(hdulist)):
if (not is_image_extension(hdulist[i])):
# print "skipping extension",i,", this is not an image extension"
continue
# get cell_x, cell_y from this extension
cell_x = hdulist[i].header['WN_CELLX']
cell_y = hdulist[i].header['WN_CELLY']
x1, c2, y1, y2 = cell2ota__get_target_region(cell_x, cell_y, 1)
# Create a mask for all sources in this cell
in_this_cell = (source_cat[:, SXFocusColumn['extension']] == i)
if (numpy.sum(in_this_cell) <= 0):
logger.debug("Couldn't find any sources in cell %d,%d" %
(cell_x, cell_y))
continue
# print "found",numpy.sum(in_this_cell),"sources for cell",cell_x, cell_y, " adding", x1, y1
cell_cat = source_cat[in_this_cell]
cell_cat[:, SXFocusColumn['x']] += x1
cell_cat[:, SXFocusColumn['y']] += y1
#
# get overscan-level data for this cell
#
binning = get_binning(hdulist[i].header)
overscan_data = extract_biassec_from_cell(hdulist[i].data, binning)
overscan_level = numpy.mean(overscan_data)
cell_cat[:, SXFocusColumn['background']] -= overscan_level
corr_cat = cell_cat if corr_cat is None else numpy.append(
corr_cat, cell_cat, axis=0)
#
# Attach to each measurement what detector lot it came from
#
fpl = podi_focalplanelayout.FocalPlaneLayout(hdulist)
detector_lot = fpl.get_detector_generation(ota_id)
corr_cat[:, SXFocusColumn['ota_lot']] = detector_lot
#
# Also override the extension number in the source catalog with the
# position in the overall focal plane
#
corr_cat[:, SXFocusColumn['extension']] = ota
# print "\n\n\n\ntotal corrected catalog:",corr_cat.shape
# save the source catalog
#numpy.savetxt("focus_cat.ota%02d" % (ota), source_cat)
#numpy.savetxt("focus_cat2.ota%02d" % (ota), corr_cat)
logger.debug("done fixing the pixel coordinates")
# only select bright enough sources
bright_enough = corr_cat[:, SXFocusColumn['mag_auto']] < -10
corr_cat = corr_cat[bright_enough]
#numpy.savetxt("focus_cat3.ota%02d" % (ota), corr_cat)
#dummy_test = open("dummy.test", "w")
# Now try to match up stars in a sequence
all_angles, all_distances = [], []
for s1 in range(corr_cat.shape[0]):
# Assume this is the middle star in the sequence
# Find all stars above and below it in a cone
# compute the angle to all other stars in the catalog
dx = corr_cat[:, SXFocusColumn['x']] - corr_cat[s1, 2]
dy = corr_cat[:, SXFocusColumn['y']] - corr_cat[s1, 3]
d_total = numpy.hypot(dx, dy)
in_cone = numpy.fabs(dx / dy) < 0.1
# Need to be at most n_stars * 10'' and at least 5''
close_enough = (d_total <
(n_stars * 10. / 0.11)) & (d_total > 5 / 0.11)
good_so_far = in_cone & close_enough
if (numpy.sum(good_so_far) <= 0):
continue
candidates = corr_cat[good_so_far]
# if (numpy.sum(candidates) < n_stars):
# # Only use full sequences
# continue
# are the magnitudes comparable
delta_mag = numpy.fabs(candidates[:,SXFocusColumn['mag_auto']] \
- corr_cat[s1,SXFocusColumn['mag_auto']])
similar_brightness = delta_mag < 1
#print >>dummy_test, "#", candidates.shape[0], numpy.sum(similar_brightness)
#print similar_brightness
if (numpy.sum(similar_brightness) <= 0):
continue
good_candidates = candidates[similar_brightness]
# Now sort the data with increasing y values
si = numpy.argsort(good_candidates[:, SXFocusColumn['y']])
sorted_candidates = good_candidates[si]
#numpy.savetxt(dummy_test, sorted_candidates)
#print >>dummy_test, "\n\n\n"
# Now compute the slope and distance between each point and each point
# above it
for p1, p2 in itertools.combinations(range(sorted_candidates.shape[0]),
2):
angle = numpy.arctan2(
sorted_candidates[p1, 2] - sorted_candidates[p2, 2],
sorted_candidates[p1, 3] - sorted_candidates[p2, 3])
distance = numpy.sqrt(
(sorted_candidates[p1, 2] - sorted_candidates[p2, 2])**2 +
(sorted_candidates[p1, 3] - sorted_candidates[p2, 3])**2)
all_angles.append(angle)
all_distances.append(distance)
#dummy_test.close()
# Once we are through with the first iteration find the best-fitting angle
all_angles = numpy.array(all_angles)
all_distances = numpy.array(all_distances)
# Find the best or rather most frequently occuring angle
all_angles[all_angles < 0] += 2 * math.pi
#numpy.savetxt("dummy.angles", all_angles)
#numpy.savetxt("dummy.distances", all_distances)
filtered_angles = three_sigma_clip(all_angles)
if (filtered_angles is None or filtered_angles.ndim < 1
or filtered_angles.shape[0] <= 0):
return None
angle_width = scipy.stats.scoreatpercentile(filtered_angles, [16, 84])
angle_width = scipy.stats.scoreatpercentile(filtered_angles, [5, 95])
logger.debug(
"Found median angle %f [%f ...%f]" %
(numpy.degrees(numpy.median(filtered_angles)),
numpy.degrees(angle_width[0]), numpy.degrees(angle_width[1])))
#
# Now we can do another proper search for all stars
# This time, only search for complete series (#stars as specified)
#
#focus_stars = open("focus_stars", "w")
all_candidates = []
for s1 in range(corr_cat.shape[0]):
# Find all stars above and below it in a cone
# compute the angle to all other stars in the catalog
dx = corr_cat[:, SXFocusColumn['x']] - corr_cat[s1, 2]
dy = corr_cat[:, SXFocusColumn['y']] - corr_cat[s1, 3]
angles = numpy.arctan2(dx, dy)
angles[angles < 0] += 2 * math.pi
d_total = numpy.hypot(dx, dy)
#print numpy.degrees(angle_width), numpy.degrees(angles)
#print angle_width[0], angle_width[1]
in_cone1 = (angles > angle_width[0]) & (angles < angle_width[1])
in_cone2 = (angles + math.pi > angle_width[0]) & (angles + math.pi <
angle_width[1])
in_cone = in_cone1 | in_cone2
# print angles[in_cone]
#print angles[in_cone][0], angles[in_cone][0] > angle_width[0], angles[in_cone][0] < angle_width[1]
close_enough = (d_total <
((n_stars + 1) * 10. / 0.11)) & (d_total > 5 / 0.11)
similar_brightness = numpy.fabs(corr_cat[:,SXFocusColumn['mag_auto']] \
- corr_cat[s1,SXFocusColumn['mag_auto']]) < 1
good = in_cone & close_enough & similar_brightness
good[s1] = True
# print s1, ":", numpy.sum(in_cone), numpy.sum(close_enough), numpy.sum(similar_brightness), numpy.sum(good)
if (numpy.sum(good) <= 1):
continue
candidates = corr_cat[good]
# print "# canddates =", candidates.shape[0]
if (not candidates.shape[0] == n_stars):
# Only use full sequences
continue
pass
# print "found match:",s1
# Now we have a set with the right number of stars, matching the overall
# angle, and with similar brightnesses
# sort them top to bottom
si = numpy.argsort(candidates[:, 3])
#numpy.savetxt(focus_stars, candidates[:,3])
#numpy.savetxt(focus_stars, si)
sorted_candidates = candidates[si]
# print "XXX", sorted_candidates.shape, sorted_candidates[:,0].shape, focus_positions.shape, n_stars, candidates.shape[0]
sorted_candidates[:, 0] = focus_positions
#numpy.savetxt(focus_stars, sorted_candidates)
#numpy.savetxt(focus_stars, numpy.degrees(angles[good]))
#numpy.savetxt(focus_stars, in_cone[good])
#numpy.savetxt(focus_stars, d_total[good])
#numpy.savetxt(focus_stars, (corr_cat[:,10] - corr_cat[s1,10])[good])
#print >>focus_stars, "\n\n\n\n"
all_candidates.append(sorted_candidates)
#focus_stars.close()
all_candidates = numpy.array(all_candidates)
logger.debug(str(all_candidates.shape))
#xxx = open("steps", "w")
# Now compute the distances from each star to the previous
step_vectors = []
for i in range(1, n_stars):
# logger.debug("Candidates: %d %s\n%s" % (all_candidates.ndim, str(all_candidates.shape), str(all_candidates)))
if (all_candidates.ndim < 1 or all_candidates.shape[0] <= 0):
# We ran out of candidates
logger.debug("We ran out of viable candidates after %s stars" %
(i))
return None
steps = all_candidates[:, i, 2:4] - all_candidates[:, i - 1, 2:4]
#numpy.savetxt(xxx, steps)
#print >>xxx, "\n\n\n\n"
logger.debug("Computing average step size, star %d" % (i))
logger.debug("Steps-X:\n%s" % (str(steps[:, 0])))
logger.debug("Steps-y:\n%s" % (str(steps[:, 1])))
clean_dx = three_sigma_clip(steps[:, 0])
clean_dy = three_sigma_clip(steps[:, 1])
# Check if both clean_dx and clean_dy are not empty
logger.debug("clean-dx=%s" % (str(clean_dx)))
logger.debug("clean-dy=%s" % (str(clean_dy)))
if (clean_dx.ndim < 1 or clean_dx.shape[0] <= 0 or clean_dy.ndim < 1
or clean_dy.shape[0] <= 0):
logger.debug("Can't find a clean dx/dy shift in iteration %d" %
(i))
return None
dx = numpy.median(clean_dx)
dy = numpy.median(clean_dy)
distx = scipy.stats.scoreatpercentile(clean_dx, [16, 84])
disty = scipy.stats.scoreatpercentile(clean_dy, [16, 84])
sigma_x = 0.5 * (distx[1] - distx[0])
sigma_y = 0.5 * (disty[1] - disty[0])
step_vectors.append([dx, dy, sigma_x, sigma_y])
good_steps = (steps[:,0] > (dx - 3*sigma_x)) & (steps[:,0] < (dx + 3*sigma_x)) \
& (steps[:,1] > (dy - 3*sigma_y)) & (steps[:,1] < (dy + 3*sigma_y))
logger.debug("before step-matching #%d: %s" %
(i, str(all_candidates.shape)))
all_candidates = all_candidates[good_steps]
logger.debug("after step-matching: #%d: %s" %
(i, str(all_candidates.shape)))
logger.debug("%s: %s" % (filename, str(step_vectors)))
# final_focus = open("final_focus", "w")
# for i in range(all_candidates.shape[0]):
# numpy.savetxt(final_focus, all_candidates[i])
# print >>final_focus, "\n\n\n\n\n"
# final_focus.close()
logger.debug("Found %d focus stars" % (all_candidates.shape[0]))
return all_candidates, real_focus_positions
logger.debug("Returning final FITS table catalog")
return None
def run_swarp(input_list, outputimage, combine="MEDIAN"):
logger = logging.getLogger("Swarp")
#
#
# Now run swarp to create the stack
# Use the default.swarp from QR as the fallback swarp option
#
#
# make sure the input is a list and not just a single filename
if (not type(input_list) == list):
input_list = [input_list]
# trim the .fits from the filename (if exists), we'll add that back in later
if (outputimage.endswith(".fits")):
outputimage = outputimage[:-5]
dic = {
'swarp_default': "%s/.config/swarp.default" % (qr_dir),
'img_out': "%s.fits" % (outputimage),
'img_w_out': "%s.weight.fits" % (outputimage),
'resample_dir': sitesetup.scratch_dir,
'inputfile': " ".join(input_list),
'combinetype': combine.upper(),
}
swarp_opts = """
-c %(swarp_default)s
-IMAGEOUT_NAME %(img_out)s
-WEIGHTOUT_NAME %(img_w_out)s
-PIXEL_SCALE 0
-PIXELSCALE_TYPE MEDIAN
-COMBINE Y
-COMBINE_TYPE %(combinetype)s
-CENTER_TYPE ALL
-RESAMPLE_DIR %(resample_dir)s
-SUBTRACT_BACK Y
-FSCALE_KEYWORD XXXXXXXX
-FSCALE_DEFAULT 1.0
-WEIGHT_TYPE NONE
-WEIGHT_SUFFIX .weight.fits
-RESCALE_WEIGHTS N
-DELETE_TMPFILES Y
%(inputfile)s
""" % dic
swarp_cmd = "%s %s" % (sitesetup.swarp_exec, swarp_opts)
logger.info("Creating %s" % (dic['img_out']))
logger.debug(" ".join(swarp_cmd.split()))
try:
ret = subprocess.Popen(swarp_cmd.split(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(swarp_stdout, swarp_stderr) = ret.communicate()
# logger.debug("swarp stdout:\n"+swarp_stdout)
# if (len(swarp_stderr) > 0 and ret.returncode != 0):
# logger.warning("swarp stderr:\n"+swarp_stderr)
# else:
# logger.debug("swarp stderr:\n"+swarp_stderr)
#print "\n".join(swarp_stderr)
logger.info("done, swarp returned (ret-code: %d)!" % ret.returncode)
except OSError as e:
podi_logging.log_exception()
print >>sys.stderr, "Execution failed:", e
return
def combine_pupilghost_slices(out_filename, filelist, op='sigclipmean'):
logger = logging.getLogger("CombinePG")
print filelist
#
# Gather information about rotation angles and center positions
# Also collect the association data.
#
rotangles = numpy.ones(len(filelist)) * -9999
headers = [None] * len(filelist)
#
# Prepare the association data
#
assoc_table = {}
logger.info("Reading data from input files")
for idx, fn in enumerate(filelist):
hdulist = pyfits.open(fn)
# Read the center and angle keywords from the "COMBINED" extension
comb_hdu = hdulist['COMBINED']
_ra = comb_hdu.header['ROTANGLE']
rotangles[idx] = _ra if _ra > 0 else _ra + 360.
headers[idx] = comb_hdu.header
logger.debug("%s: %.2f" % (fn, _ra))
# comb_hdu.header['CNTRF%03d' % (norm_angle)] = (centerf_str[:-1], "PG center, fixed r [px]")
# comb_hdu.header['CNTRV%03d' % (norm_angle)] = (centerv_str[:-1], "PG center, var. r [px]")
# comb_hdu.header['ALPHA%03d' % (norm_angle)] = (d_angle_str[:-1], "OTA angle [arcmin]")
# comb_hdu.header['OTAORDER'] = ota_str[:-1]
this_assoc = {'pupilghost-slice': fn}
assoc_table = podi_associations.collect_reduction_files_used(
assoc_table, this_assoc)
# Read the assocation table of this frame
in_assoc = podi_associations.read_associations(hdulist)
if (in_assoc is not None):
logger.debug("Found assocations:\n%s" % (str(in_assoc)))
assoc_table = podi_associations.collect_reduction_files_used(
assoc_table, in_assoc)
# return
#
# Now sort the rotator angles from smallest to largest
#
angle_sort = numpy.argsort(rotangles)
#
# Combine all frames
#
logger.info("Stacking all slices into master pupilghost template")
combined_hdulist = podi_imcombine.imcombine(
filelist,
outputfile=None,
operation=op, #nanmean.bn',
return_hdu=True,
subtract=None,
scale=None)
print combined_hdulist
combined = combined_hdulist['COMBINED']
combined.header['STACK_OP'] = op
#
# Add the sorted keywords back into the resulting file
#
logger.info("Adding metadata")
primhdu = pyfits.PrimaryHDU()
try:
prev_hdr = None
for header, label in [
('CNTRF%03d', "center, fixed radius"),
('CNTRV%03d', "center, var. radius"),
('ALPHA%03d', "angle mismatch [arcmin]"),
('NORM_%03d', "sector normalizations"),
]:
first_hdr = None
for i in range(rotangles.shape[0]):
idx = angle_sort[i]
rotangle = rotangles[idx]
round_angle = headers[idx]['RNDANGLE']
keyname = header % round_angle
#logger.info("Adding key: %s" % (keyname))
combined.header[keyname] = headers[idx][keyname]
#primhdu.header[keyname] = headers[idx][keyname]
first_hdr = keyname if first_hdr is None else first_hdr
if (prev_hdr is None):
print "adding header", keyname, " somewhere"
primhdu.header.append((keyname, headers[idx][keyname]))
prev_hdr = keyname
else:
print "adding header", keyname, "after", prev_hdr
primhdu.header.insert(prev_hdr,
(keyname, headers[idx][keyname]),
after=True)
prev_hdr = keyname
add_fits_header_title(primhdu.header, label, first_hdr)
combined.header['OTAORDER'] = headers[0]['OTAORDER']
except:
podi_logging.log_exception()
pass
print assoc_table
assoc_hdu = podi_associations.create_association_table(assoc_table)
out_hdulist = [primhdu, combined, assoc_hdu]
for name in ['PROFILE', 'RAWPROFILE']:
try:
logger.info("Adding in extension %s" % (name))
out_hdulist.append(combined_hdulist[name])
except:
logger.warning("Unable to find extension %s" % (name))
pass
combined_hdulist.writeto("dummy.fits", overwrite=True)
logger.info("Writing output to %s" % (out_filename))
out_hdulist = pyfits.HDUList(out_hdulist) #[primhdu, combined, assoc_hdu])
out_hdulist.writeto(out_filename, overwrite=True)
logger.info("Work complete!")
def read_fits_catalog(fn, extension, flatten=True):
logger = logging.getLogger("ReadFITScat")
logger.debug("Opening FITS catalog from %s" % (fn))
if (type(fn) is str):
hdulist = pyfits.open(fn)
else:
hdulist = fn
if (type(extension) is list):
ext_list = extension
else:
ext_list = [extension]
return_tables = []
for ext_id in ext_list:
try:
ext = hdulist[ext_id]
except KeyError:
logger.warning("Extension %s not found in %s" % (ext_id, fn))
hdulist.info()
return_tables.append(None)
continue
n_fields = ext.header['TFIELDS']
n_rows = ext.header['NAXIS2']
# table = numpy.empty((n_rows, n_fields))
logger.debug("Reading data for %d fields" % (n_fields))
# print "Reading data for %d fields" % (n_fields)
table = []
for f in range(n_fields):
fd = ext.data.field(f)
if (fd.ndim > 2 and flatten):
logger.warning("Unable to handle catalog field %s" % (
ext.header['TTYPE%d' % (f + 1)]))
continue
if (fd.ndim > 1 and flatten):
for c2 in range(fd.shape[1]):
table.append(fd[:, c2])
else:
table.append(fd)
try:
if (flatten):
table = numpy.array(table).T
logger.debug("Table data: %s" % (str(table.shape)))
else:
logger.debug("Table data: %d columns" % (len(table)))
# np_table = numpy.array(table)
# print table
except:
podi_logging.log_exception()
pass
# print table.shape
# print table[1:3]
# print ext.header
return_tables.append(table)
if (type(extension) is not list):
return return_tables[0]
return return_tables
