def get_app_phot_target(image, plot=False, store=True, wcsin="logical", fwhm=2, box=4, ra=None, dec=None):
'''
coords: files:
wcsin: can be "world", "logic"
'''
# Load packages; splot is in the onedspec package, which is in noao.
# The special keyword _doprint=0 turns off displaying the tasks
# when loading a package.
with warnings.catch_warnings():
warnings.simplefilter("ignore")
fxn()
iraf.noao(_doprint=0)
iraf.digiphot(_doprint=0)
iraf.apphot(_doprint=0)
iraf.unlearn("apphot")
impf = pf.open(image)
wcs = pywcs.WCS(impf[0].header)
#Check that actually the object is within this frame.
if (ra is None or dec is None):
if (fitsutils.has_par(image, "OBJRA") and fitsutils.has_par(image, "OBJRA")):
ra, dec = cc.hour2deg(fitsutils.get_par(image, 'OBJRA'), fitsutils.get_par(image, 'OBJDEC'))
else:
ra, dec = cc.hour2deg(fitsutils.get_par(image, 'RA'), fitsutils.get_par(image, 'DEC'))
pra, pdec = get_xy_coords(image, ra, dec)
else:
if(wcsin == "logical"):
pra, pdec = ra, dec
else:
#Using new method to derive the X, Y pixel coordinates, as pywcs does not seem to be working well.
pra, pdec = get_xy_coords(image, ra, dec)
#pra, pdec = wcs.wcs_sky2pix(ra, dec, 1)
#pra, pdec = wcs.wcs_sky2pix(np.array([ra, dec], ndmin=2), 1)[0]
shape = impf[0].data.shape
if (pra > 0) and (pra < shape[0]) and (pdec > 0) and (pdec < shape[1]):
pass
else:
print image, "ERROR! Object coordinates are outside this frame. Skipping any aperture photometry!!"
print pra, pdec, shape
return
imdir = os.path.dirname(image)
imname = os.path.basename(image)
plotdir = os.path.join(imdir, "photometry")
if not os.path.isdir(plotdir):
os.makedirs(plotdir)
out_name = os.path.join(plotdir, imname + ".seq.mag")
clean_name = os.path.join(plotdir, imname + ".objapp.mag")
fwhm_value = fwhm
nsrc, fwhm_value, ellip = sextractor.get_image_pars(image)
if np.isnan(fwhm_value):
fwhm_value=99
fitsutils.update_par(image, 'FWHM', fwhm_value)
if (fitsutils.has_par(image, 'AIRMASS')):
airmass_value = fitsutils.get_par(image, 'AIRMASS')
else:
airmass_value = 1.3
exptime = fitsutils.get_par(image, 'EXPTIME')
gain = fitsutils.get_par(image, 'GAIN')
#print "FWHM", fwhm_value
aperture_rad = math.ceil(float(fwhm_value)*3) # Set aperture radius to three times the PSF radius
sky_rad= math.ceil(aperture_rad*4)
#print aperture_rad, sky_rad
print "Saving coodinates for the object in pixels",pra,pdec
coords = "/tmp/coords.dat"
np.savetxt("/tmp/coords.dat", np.array([[pra, pdec]]), fmt="%.4f %.4f")
if (plot):
zmin, zmax = zscale.zscale(impf[0].data)
im = plt.imshow(impf[0].data, vmin=zmin, vmax=zmax, origin="bottom")
plt.scatter(pra, pdec, marker="o", s=100, facecolor="none")
plt.savefig(os.path.join(plotdir, imname+".png"))
plt.clf()
if os.path.isfile(out_name): os.remove(out_name)
if os.path.isfile(clean_name): os.remove(clean_name)
iraf.noao.digiphot.apphot.qphot(image = image,\
cbox = box ,\
annulus = sky_rad ,\
dannulus = 15. ,\
aperture = str(aperture_rad),\
coords = coords ,\
output = out_name ,\
plotfile = "" ,\
zmag = 0. ,\
exposure = "exptime" ,\
airmass = "airmass" ,\
filter = "filter" ,\
obstime = "DATE" ,\
epadu = gain ,\
interactive = "no" ,\
radplots = "yes" ,\
verbose = "no" ,\
graphics = "stdgraph" ,\
display = "stdimage" ,\
icommands = "" ,\
wcsin = "logical",
wcsout = "logical",
gcommands = "")
#iraf.noao.digiphot.apphot.phot(image=image, cbox=5., annulus=12.4, dannulus=10., salgori = "centroid", aperture=9.3,wcsin="world",wcsout="tv", interac = "no", coords=coords, output=out_name)
iraf.txdump(out_name, "id,image,xcenter,ycenter,xshift,yshift,fwhm,msky,stdev,mag,merr", "yes", Stdout=clean_name)
ma = np.genfromtxt(clean_name, comments="#", dtype=[("id","<f4"), ("image","|S20"), ("X","<f4"), ("Y","<f4"), ("Xshift","<f4"), ("Yshift","<f4"),("fwhm","<f4"), ("ph_mag","<f4"), ("stdev","<f4"), ("fit_mag","<f4"), ("fiterr","<f4")])
if (ma.size > 0):
if (ma.size==1):
ma = np.array([ma])
m = ma[~np.isnan(ma["fit_mag"])]
else:
print "Only one object found!"
m = np.array([ma])
insmag = np.round(ma['fit_mag'][0] , 3)
insmagerr = np.round(ma['fiterr'][0], 3)
if (fitsutils.has_par(image, "ZEROPT")):
mag = insmag + float(fitsutils.get_par(image, "ZEROPT"))
magerr = np.sqrt(insmagerr**2+ float(fitsutils.get_par(image, "ZEROPTU"))**2)
if np.isnan(mag):
mag, magerr = 0, 0
insmag, insmagerr = 0,0
fitsutils.update_par(image, "INSMAG", "%.3f"%insmag )
fitsutils.update_par(image, "INSMAGER", "%.3f"%insmagerr)
fitsutils.update_par(image, "APPMAG", np.round(mag, 3) )
fitsutils.update_par(image, "APPMAGER", np.round(magerr, 3))
if (plot):
X = int(ma["X"][0])
Y = int(ma["Y"][0])
pra = int(pra)
pdec = int(pdec)
plt.scatter(X, Y, marker="o", s=100, facecolor="none", edgecolor="red")
plt.colorbar(im)
plt.savefig(os.path.join(plotdir, imname+".png"))
plt.clf()
zmin, zmax = zscale.zscale(impf[0].data.T[X-50:X+50,Y-50:Y+50].T)
im = plt.imshow(impf[0].data.T[pra-50:pra+50,pdec-50:pdec+50].T, vmin=zmin, vmax=zmax, interpolation="none", origin="bottom", extent=(-50,50,-50,50))
c1 = plt.Circle( (pra-X, pdec-Y), edgecolor="k", facecolor="none", radius=aperture_rad, label="Initial position")
c11 = plt.Circle( (pra-X, pdec-Y), edgecolor="k", facecolor="none", radius=sky_rad)
c2 = plt.Circle( (0, 0), edgecolor="orange", facecolor="none", radius=aperture_rad, label="Adjusted centroid")
c22 = plt.Circle( (0, 0), edgecolor="orange", facecolor="none", radius=sky_rad)
plt.gca().add_artist(c1)
plt.gca().add_artist(c11)
plt.gca().add_artist(c2)
plt.gca().add_artist(c22)
plt.colorbar(im)
myhandles = []
markers = ["o", "o"]
labels = ["Initial position", "Adjusted centroid"]
cols = ["k", "orange"]
for i in np.arange(len(markers)):
myhandles.append(mlines.Line2D([], [], mec=cols[i], mfc="none", marker=markers[i], ls="None", markersize=10, label=labels[i]))
plt.legend(handles=myhandles, loc="lower left", labelspacing=0.3, fontsize=11, numpoints=1, frameon=False, ncol=5, bbox_to_anchor=(0.0, 0.00), fancybox=False, shadow=True)
plt.title("MIN: %.0f MAX: %.0f"%(np.nanmin(impf[0].data.T[X-50:X+50,Y-50:Y+50]), np.nanmax(impf[0].data.T[X-50:X+50,Y-50:Y+50])))
plt.savefig(os.path.join(plotdir, imname+"_zoom.png"))
plt.clf()
def start_listening_loop():
'''
Start accepting connections from pylos.
'''
#Set address
ip = _host
port = _port
#Log into a file
FORMAT = '%(asctime)-15s %(levelname)s [%(name)s] %(message)s'
root_dir = _logpath
now = datetime.datetime.utcnow()
timestamp=datetime.datetime.isoformat(now)
timestamp=timestamp.split("T")[0]
logging.basicConfig(format=FORMAT, filename=os.path.join(root_dir, "listener_{0}.log".format(timestamp)), level=logging.INFO)
logger = logging.getLogger('listener')
#bind socket
s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
try:
s.bind((ip,port))
except socket.error as e:
#If there is already a listener there, it will generate an error.
#In this case we just return.
return
s.listen(10)
logger.info( "Starting a new listener. Hello World!")
#create continous while loop to listen for request
#Exit the loop at 11:00AM, as the new day will start.
while True:
connection,caddress = s.accept()
time.sleep(1)
cmd = "touch %s"%_alivefile
subprocess.call(cmd, shell=True)
while True:
subprocess.call(cmd, shell=True)
data = connection.recv(2048)
logger.info( "Incoming command: %s " % data)
now = datetime.datetime.utcnow()
timestamp=datetime.datetime.isoformat(now)
timestamp=timestamp.split("T")[0]
logging.basicConfig(format=FORMAT, filename=os.path.join(root_dir, "listener_{0}.log".format(timestamp)), level=logging.INFO)
logger = logging.getLogger('listener')
command = data.split(",")[0]
if "FOCUSIFU" in command:
logger.info( "Finding the best focus for the IFU.")
lfiles = data.split(",")[1:]
for i, lf in enumerate(lfiles):
lf = lf.replace("raw", "phot")
lfiles[i] = lf
#Wait until the image is available.
while (not os.path.isfile(lfiles[-1])):
time.sleep(0.5)
focus, sigma = sextractor.get_focus_ifu(lfiles, plot=True)
connection.sendall("%.2f,%.2f\n"%(focus, sigma))
logger.info("Selected focus ifu: %.2f,%.2f\n"%(focus, sigma))
elif "FOCUS" in command:
logger.info( "Finding the best focus for RC.")
lfiles = data.split(",")[1:]
for i, lf in enumerate(lfiles):
lf = lf.replace("raw", "phot")
lfiles[i] = lf
#Wait until the image is available.
while (not os.path.isfile(lfiles[-1])):
time.sleep(0.5)
focus, sigma = sextractor.get_focus(lfiles, plot=True)
connection.sendall("%.2f,%.2f\n"%(focus, sigma))
logger.info("Selected focus: %.2f,%.2f\n"%(focus, sigma))
elif "SAO" in command:
try:
logger.info( "Looking for a nice SAO star.")
name, ra, dec = sao.get_sao()
connection.sendall("%d,%s,%s,%s\n"%(0,name, ra, dec))
logger.info("Found star. Returning: %d,%s,%s,%s\n"%(0,name, ra, dec))
except Exception as e:
logger.error(str(sys.exc_info()[0]))
logger.error(e)
connection.sendall("%d,%s,%s,%s\n"%(-1,"null", (datetime.datetime.utcnow()[4]+3)*15, 40))
elif "STATS" in command:
logger.info( "Finding the statistics for the image.")
lfile = data.split(",")[1]
lfile = lfile.strip()
lfile = lfile.replace("raw", "phot")
while (not os.path.isfile(lfile)):
time.sleep(0.5)
try:
nsources, fwhm, ellipticity, bkg = sextractor.get_image_pars(lfile)
connection.sendall("%d,%d,%d,%.3f\n"%(0, nsources, fwhm, ellipticity, bkg))
except Exception as e:
logger.error(str(sys.exc_info()[0]))
logger.error(e)
connection.sendall("%d,%d,%d,%.3f\n"%(-1, 0, 0, 0, 0))
elif "FWHM" in command:
logger.info( "Finding the FWHM for the last 3 images.")
mydir = timestamp.replace("-","")
statsfile = os.path.join("%s/%s/stats"%(_photpath, mydir), "stats_%s.log"%mydir)
if (os.path.isfile(statsfile)):
stats = np.genfromtxt(statsfile, dtype=None, delimiter=",")
if len(stats) > 3:
connection.sendall("%d,%.2f\n"%(0, np.median(stats["f4"][-4:])))
logger.info( "Sent (%d,%.2f)"%(0, np.median(stats["f4"][-4:])) )
else:
connection.sendall("%d,%.2f\n"%(0, np.median(stats["f4"])))
logger.info( "Sent (%d,%.2f)"%(0, np.median(stats["f4"])) )
else:
logger.error("Stats file %s does not exists!"%statsfile)
connection.sendall("%d,%.2f\n"%(-1, 0))
elif "OFFSET" in command:
try:
isABstr = data.split(",")[1]
isAB = (isABstr =="AB")
#astrometry = int(data.split(",")[2])
astrometry = 0
image = data.split(",")[2].rstrip()
logger.info("Get Offsets AB=%s for image %s. Astrometry active=%s"%(isAB,image,astrometry))
#if(astrometry==0):
astrofile = os.path.basename(image)
date = astrofile.split("_")[0].replace("rc","")
astrofile = astrofile.replace("rc", "a_rc").replace(".new", ".fits")
endpath = "%s/%s/%s"%(_photpath, date, astrofile)
if (not os.path.isdir(os.path.dirname(endpath))):
os.makedirs(os.path.dirname(endpath))
os.system('scp %s [email protected]:%s %s'%(_rsa, image, endpath))
astro = False
#Only run astrometry if image is unavailable.
if (not os.path.isfile(endpath)):
astro=True
logger.error("Astrometry resolved image %s could not be copied into %s. Setting astrometry to True."%(image, endpath))
endpath = os.path.join(os.path.dirname(endpath), os.path.basename(image))
res = recenter_ifu.main(endpath, isAB, astro=astro, plot=True)
retcode = res[0]
offsets = res
logger.info( "OFFSETS %s Return code %d"%(offsets,retcode))
if(isAB):
connection.sendall("%d,%s,%s,%s,%s\n"%(retcode,offsets[1], offsets[2], offsets[3], offsets[4]))
else:
connection.sendall("%d,%s,%s\n"%(retcode,offsets[1], offsets[2]))
except Exception as e:
logger.error( "Error occurred when processing command " + str(data))
logger.error(str(sys.exc_info()[0]))
logger.error(e)
connection.sendall("%d,%s,%s\n"%(-1,0,0))
elif "PING" in command:
try:
logger.info( "Received a PING command. Responding...")
connection.sendall("PONG\n")
logger.info("PONG sent.\n")
except Exception as e:
logger.error(str(sys.exc_info()[0]))
logger.error(e)
else:
logger.error( "Unknown command: "%command)
break
#Exit and restart the listener if the program is running for more than 12 hours and
#it is later than 10AM, so we don't disrupt the night scheduler.
'''if (datetime.datetime.utcnow()-now).total_seconds() > 12*3600. and (datetime.datetime.utcnow()).hour>17:
def reduce_image(image,
flatdir=None,
biasdir=None,
cosmic=False,
astrometry=True,
channel='rc',
target_dir='reduced',
overwrite=False):
'''
Applies Flat field and bias calibrations to the image.
Steps:
1. - Solve astrometry on the entire image.
2. - Computes cosmic ray rejectionon the entire image.
3. - Compute master bias (if it does not exist) and de-bias the image.
4. - Separate the image into 4 filters.
5. - Compute flat field for each filter (if it does not exist) and apply flat fielding on the image.
6. - Compute the image zeropoint.
'''
logger.info("Reducing image %s" % image)
print "Reducing image ", image
image = os.path.abspath(image)
imname = os.path.basename(image).replace(".fits", "")
try:
objectname = fitsutils.get_par(image, "NAME").replace(
" ", "") + "_" + fitsutils.get_par(image, "FILTER")
except:
logger.error("ERROR, image " + image +
" does not have a NAME or a FILTER!!!")
return
print "For object", objectname
logger.info("For object %s" % objectname)
#Change to image directory
mydir = os.path.dirname(image)
if mydir == "": mydir = "."
mydir = os.path.abspath(mydir)
os.chdir(mydir)
#Create destination directory
if (not os.path.isdir(target_dir)):
os.makedirs(target_dir)
#If we don't want to overwrite the already extracted images, we check wether they exist.
if (not overwrite):
existing = True
for band in ['u', 'g', 'r', 'i']:
destfile = os.path.join(
target_dir,
imname + "_f_b_a_%s_%s_0.fits" % (objectname, band))
logger.info( "Looking if file %s exists: %s"%( destfile, \
(os.path.isfile(destfile) ) ) )
existing = existing and (os.path.isfile(destfile))
if existing:
return []
#Initialize the basic parameters.
init_header_reduced(image)
astro = ""
if (astrometry):
logger.info("Solving astometry for the whole image...")
img = solve_astrometry(image)
if (os.path.isfile(img)):
astro = "a_"
fitsutils.update_par(img, "IQWCS", 1)
else:
logger.error("ASTROMETRY DID NOT SOLVE ON IMAGE %s" % image)
img = image
#Update noise parameters needed for cosmic reection
if (fitsutils.get_par(img, "ADCSPEED") == 2):
fitsutils.update_par(img, "RDNOISE", 20.)
else:
fitsutils.update_par(img, "RDNOISE", 4.)
if (cosmic):
logger.info("Correcting for cosmic rays...")
# Correct for cosmics each filter
cleanimg = clean_cosmic(os.path.join(os.path.abspath(mydir), img))
img = cleanimg
#Compute BIAS
if (biasdir is None or biasdir == ""): biasdir = "."
create_masterbias(biasdir)
bias_slow = os.path.join(biasdir, "Bias_%s_%s.fits" % (channel, 'slow'))
bias_fast = os.path.join(biasdir, "Bias_%s_%s.fits" % (channel, 'fast'))
# Running IRAF to DE-BIAS
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
#Compute flat field
if (flatdir is None or flatdir == ""): flatdir = "."
create_masterflat(flatdir, biasdir)
#New names for the object.
debiased = os.path.join(os.path.dirname(img), "b_" + os.path.basename(img))
logger.info("Creating debiased file, %s" % debiased)
if ( (fitsutils.get_par(img, "ADCSPEED")==0.1 and not os.path.isfile(bias_slow)) \
or (fitsutils.get_par(img, "ADCSPEED")==2 and not os.path.isfile(bias_fast)) ):
logger.warn(
"Master bias not found! Tryting to copy from reference folder...")
copy_ref_calib(mydir, "Bias")
if ( (fitsutils.get_par(img, "ADCSPEED")==0.1 and not os.path.isfile(bias_slow)) \
or (fitsutils.get_par(img, "ADCSPEED")==2 and not os.path.isfile(bias_fast)) ):
logger.error("Bias not found in reference folder")
return
#Clean first
if (os.path.isfile(debiased)):
os.remove(debiased)
#Debias
if (fitsutils.get_par(img, "ADCSPEED") == 2):
iraf.imarith(img, "-", bias_fast, debiased)
fitsutils.update_par(debiased, "BIASFILE", bias_fast)
fitsutils.update_par(debiased, "RDNOISE", 20.)
else:
iraf.imarith(img, "-", bias_slow, debiased)
fitsutils.update_par(debiased, "BIASFILE", bias_slow)
fitsutils.update_par(debiased, "RDNOISE", 4.)
#Set negative counts to zero
hdu = fits.open(debiased)
header = hdu[0].header
hdu[0].data[hdu[0].data < 0] = 0
hdu.writeto(debiased, clobber=True)
#Slicing the image for flats
slice_names = slice_rc(debiased)
print "Creating sliced files, ", slice_names
#Remove un-sliced image
os.remove(debiased)
# DE-flat each filter and store under object name
for i, debiased_f in enumerate(slice_names):
b = fitsutils.get_par(debiased_f, 'filter')
deflatted = os.path.join(
os.path.dirname(image), target_dir,
imname + "_f_b_" + astro + objectname + "_%s.fits" % b)
#Flat to be used for that filter
flat = os.path.join(flatdir, "Flat_%s_%s_norm.fits" % (channel, b))
if (not os.path.isfile(flat)):
logger.warn("Master flat not found in %s" % flat)
copy_ref_calib(mydir, "Flat")
continue
else:
logger.info("Using flat %s" % flat)
#Cleans the deflatted file if exists
if (os.path.isfile(deflatted)):
os.remove(deflatted)
if (os.path.isfile(debiased_f) and os.path.isfile(flat)):
logger.info("Storing de-flatted %s as %s" %
(debiased_f, deflatted))
time.sleep(1)
iraf.imarith(debiased_f, "/", flat, deflatted)
else:
logger.error(
"SOMETHING IS WRONG. Error when dividing %s by the flat field %s!"
% (debiased_f, flat))
#Removes the de-biased file
os.remove(debiased_f)
logger.info(
"Updating header with original filename and flat field used.")
fitsutils.update_par(deflatted, "ORIGFILE", os.path.basename(image))
fitsutils.update_par(deflatted, "FLATFILE", flat)
slice_names[i] = deflatted
#Moving files to the target directory
for image in slice_names:
bkg = get_median_bkg(image)
fitsutils.update_par(image, "SKYBKG", bkg)
#Get basic statistics for the image
nsrc, fwhm, ellip, bkg = sextractor.get_image_pars(image)
plot_image(image)
logger.info(
"Sextractor statistics: nscr %d, fwhm (arcsec) %.2f, ellipticity %.2f"
% (nsrc, fwhm, ellip))
print "Sextractor statistics: nscr %d, fwhm (arcsec) %.2f, ellipticity %.2f" % (
nsrc, fwhm, ellip)
dic = {
"FWHM": np.round(fwhm, 3),
"FWHMPIX": np.round(fwhm / 0.394, 3),
"NSRC": nsrc,
"ELLIP": np.round(ellip, 3)
}
#Update the seeing information from sextractor
fitsutils.update_pars(image, dic)
#Compute the zeropoints
for image in slice_names:
zeropoint.calibrate_zeropoint(image)
return slice_names
def reduce_image(image, flatdir=None, biasdir=None, cosmic=False, astrometry=True, channel='rc', target_dir='reduced', overwrite=False):
'''
Applies Flat field and bias calibrations to the image.
Steps:
1. - Solve astrometry on the entire image.
2. - Computes cosmic ray rejectionon the entire image.
3. - Compute master bias (if it does not exist) and de-bias the image.
4. - Separate the image into 4 filters.
5. - Compute flat field for each filter (if it does not exist) and apply flat fielding on the image.
6. - Compute the image zeropoint.
'''
logger.info("Reducing image %s"% image)
print "Reducing image ", image
image = os.path.abspath(image)
imname = os.path.basename(image).replace(".fits", "")
try:
objectname = fitsutils.get_par(image, "NAME").replace(" ","")+"_"+fitsutils.get_par(image, "FILTER")
except:
logger.error( "ERROR, image "+ image + " does not have a NAME or a FILTER!!!")
return
print "For object", objectname
logger.info( "For object %s"% objectname)
#Change to image directory
mydir = os.path.dirname(image)
if mydir=="": mydir = "."
mydir = os.path.abspath(mydir)
os.chdir(mydir)
#Create destination directory
if (not os.path.isdir(target_dir)):
os.makedirs(target_dir)
#If we don't want to overwrite the already extracted images, we check wether they exist.
if (not overwrite):
existing = True
for band in ['u', 'g', 'r', 'i']:
destfile = os.path.join(target_dir, imname + "_f_b_a_%s_%s_0.fits"%(objectname, band))
logger.info( "Looking if file %s exists: %s"%( destfile, \
(os.path.isfile(destfile) ) ) )
existing = existing and (os.path.isfile( destfile ) )
if existing:
return []
#Initialize the basic parameters.
init_header_reduced(image)
astro = ""
if (astrometry):
logger.info( "Solving astometry for the whole image...")
img = solve_astrometry(image)
if (os.path.isfile(img)):
astro="a_"
fitsutils.update_par(img, "IQWCS", 1)
else:
logger.error( "ASTROMETRY DID NOT SOLVE ON IMAGE %s"% image)
img = image
#Update noise parameters needed for cosmic reection
if (fitsutils.get_par(img, "ADCSPEED")==2):
fitsutils.update_par(img, "RDNOISE", 20.)
else:
fitsutils.update_par(img, "RDNOISE", 4.)
if (cosmic):
logger.info( "Correcting for cosmic rays...")
# Correct for cosmics each filter
cleanimg = clean_cosmic(os.path.join(os.path.abspath(mydir), img))
img = cleanimg
#Get basic statistics for the image
nsrc, fwhm, ellip, bkg = sextractor.get_image_pars(img)
logger.info( "Sextractor statistics: nscr %d, fwhm (pixel) %.2f, ellipticity %.2f"% (nsrc, fwhm, ellip))
print "Sextractor statistics: nscr %d, fwhm (pixel) %.2f, ellipticity %.2f"% (nsrc, fwhm, ellip)
dic = {"SEEPIX": fwhm/0.394, "NSRC":nsrc, "ELLIP":ellip}
#Update the seeing information from sextractor
fitsutils.update_pars(img, dic)
#Compute BIAS
if (biasdir is None or biasdir==""): biasdir = "."
create_masterbias(biasdir)
bias_slow = os.path.join(biasdir, "Bias_%s_%s.fits"%(channel, 'slow'))
bias_fast = os.path.join(biasdir, "Bias_%s_%s.fits"%(channel, 'fast'))
# Running IRAF to DE-BIAS
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
#Compute flat field
if (flatdir is None or flatdir==""): flatdir = "."
create_masterflat(flatdir, biasdir)
#New names for the object.
debiased = os.path.join(os.path.dirname(img), "b_" + os.path.basename(img))
logger.info( "Creating debiased file, %s"%debiased)
if ( (fitsutils.get_par(img, "ADCSPEED")==0.1 and not os.path.isfile(bias_slow)) \
or (fitsutils.get_par(img, "ADCSPEED")==2 and not os.path.isfile(bias_fast)) ):
logger.warn( "Master bias not found! Tryting to copy from reference folder...")
copy_ref_calib(mydir, "Bias")
if ( (fitsutils.get_par(img, "ADCSPEED")==0.1 and not os.path.isfile(bias_slow)) \
or (fitsutils.get_par(img, "ADCSPEED")==2 and not os.path.isfile(bias_fast)) ):
logger.error( "Bias not found in reference folder")
return
#Clean first
if (os.path.isfile(debiased)):
os.remove(debiased)
#Debias
if (fitsutils.get_par(img, "ADCSPEED")==2):
iraf.imarith(img, "-", bias_fast, debiased)
fitsutils.update_par(debiased, "BIASFILE", bias_fast)
fitsutils.update_par(debiased, "RDNOISE", 20.)
else:
iraf.imarith(img, "-", bias_slow, debiased)
fitsutils.update_par(debiased, "BIASFILE", bias_slow)
fitsutils.update_par(debiased, "RDNOISE", 4.)
#Set negative counts to zero
hdu = fits.open(debiased)
header = hdu[0].header
hdu[0].data[hdu[0].data<0] = 0
hdu.writeto(debiased, clobber=True)
#Slicing the image for flats
slice_names = slice_rc(debiased)
print "Creating sliced files, ", slice_names
#Remove un-sliced image
os.remove(debiased)
# DE-flat each filter and store under object name
for i, debiased_f in enumerate(slice_names):
b = fitsutils.get_par(debiased_f, 'filter')
deflatted = os.path.join(os.path.dirname(image), target_dir, imname + "_f_b_" + astro + objectname + "_%s.fits"%b)
#Flat to be used for that filter
flat = os.path.join(flatdir, "Flat_%s_%s_norm.fits"%(channel, b))
if (not os.path.isfile(flat)):
logger.warn( "Master flat not found in %s"% flat)
copy_ref_calib(mydir, "Flat_%s_%s_norm"%(channel, b))
continue
else:
logger.info( "Using flat %s"%flat)
#Cleans the deflatted file if exists
if (os.path.isfile(deflatted)):
os.remove(deflatted)
if (os.path.isfile(debiased_f) and os.path.isfile(flat)):
logger.info( "Storing de-flatted %s as %s"%(debiased_f, deflatted))
time.sleep(1)
iraf.imarith(debiased_f, "/", flat, deflatted)
else:
logger.error( "SOMETHING IS WRONG. Error when dividing %s by the flat field %s!"%(debiased_f, flat))
#Removes the de-biased file
os.remove(debiased_f)
logger.info( "Updating header with original filename and flat field used.")
fitsutils.update_par(deflatted, "ORIGFILE", os.path.basename(image))
fitsutils.update_par(deflatted, "FLATFILE", flat)
slice_names[i] = deflatted
#Moving files to the target directory
for image in slice_names:
bkg = get_median_bkg(image)
fitsutils.update_par(image, "SKYBKG", bkg)
#shutil.move(name, newname)
#Compute the zeropoints
for image in slice_names:
zeropoint.calibrate_zeropoint(image)
return slice_names
def start_listening_loop():
'''
Start accepting connections from pylos.
'''
#Set address
ip = _host
port = _port
#Log into a file
FORMAT = '%(asctime)-15s %(levelname)s [%(name)s] %(message)s'
root_dir = _logpath
now = datetime.datetime.utcnow()
timestamp = datetime.datetime.isoformat(now)
timestamp = timestamp.split("T")[0]
logging.basicConfig(format=FORMAT,
filename=os.path.join(
root_dir, "listener_{0}.log".format(timestamp)),
level=logging.INFO)
logger = logging.getLogger('listener')
#bind socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.bind((ip, port))
except socket.error as e:
#If there is already a listener there, it will generate an error.
#In this case we just return.
return
s.listen(10)
logger.info("Starting a new listener. Hello World!")
#create continous while loop to listen for request
#Exit the loop at 11:00AM, as the new day will start.
while True:
connection, caddress = s.accept()
time.sleep(1)
cmd = "touch %s" % _alivefile
subprocess.call(cmd, shell=True)
while True:
subprocess.call(cmd, shell=True)
data = connection.recv(2048)
logger.info("Incoming command: %s " % data)
now = datetime.datetime.utcnow()
timestamp = datetime.datetime.isoformat(now)
timestamp = timestamp.split("T")[0]
logging.basicConfig(format=FORMAT,
filename=os.path.join(
root_dir,
"listener_{0}.log".format(timestamp)),
level=logging.INFO)
logger = logging.getLogger('listener')
command = data.split(",")[0]
if "FOCUSIFU" in command:
logger.info("Finding the best focus for the IFU.")
lfiles = data.split(",")[1:]
for i, lf in enumerate(lfiles):
lf = lf.replace("raw", "phot")
lfiles[i] = lf
#Wait until the image is available.
while (not os.path.isfile(lfiles[-1])):
time.sleep(0.5)
focus, sigma = sextractor.get_focus_ifu(lfiles, plot=True)
connection.sendall("%.2f,%.2f\n" % (focus, sigma))
logger.info("Selected focus ifu: %.2f,%.2f\n" % (focus, sigma))
elif "FOCUS" in command:
logger.info("Finding the best focus for RC.")
lfiles = data.split(",")[1:]
for i, lf in enumerate(lfiles):
lf = lf.replace("raw", "phot")
lfiles[i] = lf
#Wait until the image is available.
while (not os.path.isfile(lfiles[-1])):
time.sleep(0.5)
focus, sigma = sextractor.get_focus(lfiles, plot=True)
connection.sendall("%.2f,%.2f\n" % (focus, sigma))
logger.info("Selected focus: %.2f,%.2f\n" % (focus, sigma))
elif "SAO" in command:
try:
logger.info("Looking for a nice SAO star.")
name, ra, dec = sao.get_sao()
connection.sendall("%d,%s,%s,%s\n" % (0, name, ra, dec))
logger.info("Found star. Returning: %d,%s,%s,%s\n" %
(0, name, ra, dec))
except Exception as e:
logger.error(str(sys.exc_info()[0]))
logger.error(e)
connection.sendall(
"%d,%s,%s,%s\n" %
(-1, "null",
(datetime.datetime.utcnow()[4] + 3) * 15, 40))
elif "STATS" in command:
logger.info("Finding the statistics for the image.")
lfile = data.split(",")[1]
lfile = lfile.strip()
lfile = lfile.replace("raw", "phot")
while (not os.path.isfile(lfile)):
time.sleep(0.5)
try:
nsources, fwhm, ellipticity, bkg = sextractor.get_image_pars(
lfile)
connection.sendall("%d,%d,%d,%.3f\n" %
(0, nsources, fwhm, ellipticity, bkg))
except Exception as e:
logger.error(str(sys.exc_info()[0]))
logger.error(e)
connection.sendall("%d,%d,%d,%.3f\n" % (-1, 0, 0, 0, 0))
elif "FWHM" in command:
logger.info("Finding the FWHM for the last 3 images.")
mydir = timestamp.replace("-", "")
statsfile = os.path.join("%s/%s/stats" % (_photpath, mydir),
"stats_%s.log" % mydir)
if (os.path.isfile(statsfile)):
stats = np.genfromtxt(statsfile, dtype=None, delimiter=",")
if len(stats) > 3:
connection.sendall("%d,%.2f\n" %
(0, np.median(stats["f4"][-4:])))
logger.info("Sent (%d,%.2f)" %
(0, np.median(stats["f4"][-4:])))
else:
connection.sendall("%d,%.2f\n" %
(0, np.median(stats["f4"])))
logger.info("Sent (%d,%.2f)" %
(0, np.median(stats["f4"])))
else:
logger.error("Stats file %s does not exists!" % statsfile)
connection.sendall("%d,%.2f\n" % (-1, 0))
elif "OFFSET" in command:
try:
isABstr = data.split(",")[1]
isAB = (isABstr == "AB")
#astrometry = int(data.split(",")[2])
astrometry = 0
image = data.split(",")[2].rstrip()
logger.info(
"Get Offsets AB=%s for image %s. Astrometry active=%s"
% (isAB, image, astrometry))
#if(astrometry==0):
astrofile = os.path.basename(image)
date = astrofile.split("_")[0].replace("rc", "")
astrofile = astrofile.replace("rc", "a_rc").replace(
".new", ".fits")
endpath = "%s/%s/%s" % (_photpath, date, astrofile)
if (not os.path.isdir(os.path.dirname(endpath))):
os.makedirs(os.path.dirname(endpath))
os.system(
'scp %s [email protected]:%s %s'
% (_rsa, image, endpath))
astro = False
#Only run astrometry if image is unavailable.
if (not os.path.isfile(endpath)):
astro = True
logger.error(
"Astrometry resolved image %s could not be copied into %s. Setting astrometry to True."
% (image, endpath))
endpath = os.path.join(os.path.dirname(endpath),
os.path.basename(image))
res = recenter_ifu.main(endpath,
isAB,
astro=astro,
plot=True)
retcode = res[0]
offsets = res
logger.info("OFFSETS %s Return code %d" %
(offsets, retcode))
if (isAB):
connection.sendall("%d,%s,%s,%s,%s\n" %
(retcode, offsets[1], offsets[2],
offsets[3], offsets[4]))
else:
connection.sendall("%d,%s,%s\n" %
(retcode, offsets[1], offsets[2]))
except Exception as e:
logger.error("Error occurred when processing command " +
str(data))
logger.error(str(sys.exc_info()[0]))
logger.error(e)
connection.sendall("%d,%s,%s\n" % (-1, 0, 0))
elif "PING" in command:
try:
logger.info("Received a PING command. Responding...")
connection.sendall("PONG\n")
logger.info("PONG sent.\n")
except Exception as e:
logger.error(str(sys.exc_info()[0]))
logger.error(e)
else:
logger.error("Unknown command: " % command)
break
#Exit and restart the listener if the program is running for more than 12 hours and
#it is later than 10AM, so we don't disrupt the night scheduler.
'''if (datetime.datetime.utcnow()-now).total_seconds() > 12*3600. and (datetime.datetime.utcnow()).hour>17:
