#open and read the files
tbl = "catalog.cat"
gal,field,ra,dec = np.loadtxt (tbl, dtype=str, usecols=(0,1,2,3), unpack=True)
galf = gal.astype(float)
galaxy = galf.astype(int)
table = open("cutted_gal.txt","w")
table.write("#galaxy \t field \t ra \t dec \n")
for ii in range(len(gal)):
ra_center = ra[ii].astype(float)
dec_center = dec[ii].astype(float)
if field[ii] == "GOODS-S":
input_img = "hlsp_candels_hst_wfc3_gs-tot_f160w_v1.0_drz.fits"
montage.mSubimage(input_img, str(galaxy[ii])+"_cut.fits", ra_center, dec_center, size)
table.write(str(galaxy[ii])+"\t"+str(field[ii])+"\t"+str(ra_center)+"\t"+str(dec_center)+"\n")
elif field[ii] == "GOODS-N":
#input_img = "hlsp_candels_hst_wfc3_gn13_f160w_v0.5_drz.fits"
input_img = "goodsn_all_wfc3_ir_f160w_060mas_v1.0_drz.fits"
montage.mSubimage(input_img, str(galaxy[ii])+"_cut.fits", ra_center, dec_center, size)
table.write(str(galaxy[ii])+"\t"+str(field[ii])+"\t"+str(ra_center)+"\t"+str(dec_center)+"\n")
elif field[ii] == "COSMOS":
input_img = "hlsp_candels_hst_wfc3_cos-tot_f160w_v1.0_drz.fits"
montage.mSubimage(input_img, str(galaxy[ii])+"_cut.fits", ra_center, dec_center, size)
table.write(str(galaxy[ii])+"\t"+str(field[ii])+"\t"+str(ra_center)+"\t"+str(dec_center)+"\n")
elif field[ii] == "UDS":
input_img = "hlsp_candels_hst_wfc3_uds-tot_f160w_v1.0_drz.fits"
montage.mSubimage(input_img, str(galaxy[ii])+"_cut.fits", ra_center, dec_center, size)
table.write(str(galaxy[ii])+"\t"+str(field[ii])+"\t"+str(ra_center)+"\t"+str(dec_center)+"\n")
elif field[ii] == "AEGIS":
if (DEBUG): print ("Retrieving data from SDSS SAS server for "+ band +"band")
for i in data :
out = "frame-"+str(band)+"-"+str(i[0]).zfill(6)+"-"+str(i[1])+"-"+str(i[2]).zfill(4)
os.system("wget http://data.sdss3.org/sas/dr10/boss/photoObj/frames/301/"+str(i[0])+"/"+ str(i[1]) +"/"+out+".fits.bz2")
os.system("bunzip2 "+out+".fits.bz2")
# print (os.getcwd())
os.chdir("../")
if (DEBUG) : print("Creating mosaic for " +" "+ band + " band.")
montage.mImgtbl("raw","images.tbl")
montage.mHdr(str(ra)+" "+str(dec),radius,out+".hdr")
if (DEBUG): print ("Reprojecting images")
#Sometimes you can't find the files and result in images.tbl => empty doc
#need to put data file inside raw AND unzip it so that Montage detect that it is a fit file
os.chdir("raw")
montage.mProjExec("../images.tbl","../"+out+".hdr","../projected", "../stats.tbl")
os.chdir("..")
montage.mImgtbl("projected","pimages.tbl")
#mAdd coadds the reprojected images using the FITS header template and mImgtbl list.
os.chdir("projected")
montage.mAdd("../pimages.tbl","../"+out+".hdr","SDSS_"+out+".fits")
montage.mSubimage("SDSS_"+out+".fits","SDSS_"+ele+"_"+str(trunc(ra))+"_"+str(trunc(dec))+".fits",ra,dec,2*margin) # mSubImage takes xsize which should be twice the margin (margin measures center to edge of image)
shutil.move("SDSS_"+ele+"_"+str(trunc(ra))+"_"+str(trunc(dec))+".fits",os.getcwd()[:-11] )#if change to :-11 then move out of u,g,r,i,z directory, may be more convenient for mJPEG
if (DEBUG) : print ("Completed Mosaic for " + band)
os.chdir("../..")
# Superimposing R,G,B image mosaics into TIFF using STIFF
os.system("stiff "+" SDSS_i_"+str(trunc(ra))+"_"+str(trunc(dec))+ ".fits "+ " SDSS_r_"+str(trunc(ra))+"_"+str(trunc(dec))+ ".fits "+" SDSS_g_"+str(trunc(ra))+"_"+str(trunc(dec))+ ".fits "+ " -c stiff.conf " +" -OUTFILE_NAME "+str(trunc(ra))+"_"+str(trunc(dec))+"_COLORSAT_5_MAX_MAN_3 -COLOUR_SAT 5 -MAX_TYPE MANUAL -MAX_LEVEL 3")
# for b in bands:
# os.system("rm -r "+b+"/")
#we want to keep the fit files, but for testing purposes Python will throw file-already-exist error , if we dont delete them.
#os.system("rm -r " + "SDSS_frame-"+b+"-"+str(run).zfill(6)+"-"+str(camcol)+"-"+str(field).zfill(4)+ ".fits" )
if (DEBUG) : print ("Completed Mosaic")
def mosaic_band(self,band,ra,dec,margin,radius,pgc):#,clean=True):
'''
Input: source info param
Create a mosaic fit file for the specified band.
Return: String filename of resulting mosaic
'''
print ("------------------mosaic_band----------------------")
DEBUG = True
# output = open("rc3_galaxies_outside_SDSS_footprint.txt",'a') # 'a' for append #'w')
# unclean = open("rc3_galaxies_unclean","a")
# filename = "{},{}".format(str(ra),str(dec))
filename = str(ra)+str(dec)
#print (margin/radius)
if (DEBUG) : print ("Querying data that lies inside margin")
#result = sqlcl.query( "SELECT distinct run,camcol,field FROM PhotoObj WHERE ra between {0}-{1} and {0}+{1}and dec between {2}-{3} and {2}+{3}".format(str(ra),str(margin),str(dec),str(margin))).readlines()
result = sqlcl.query( "SELECT distinct run,camcol,field FROM PhotoObj WHERE ra between "+str(ra)+"-"+str(margin)+" and " +str(ra)+"+"+str(margin)+"and dec between "+str(dec)+"-"+str(margin)+" and "+ str(dec)+"+"+str(margin)).readlines()
clean_result = sqlcl.query( "SELECT distinct run,camcol,field FROM PhotoObj WHERE CLEAN =1 and ra between "+str(ra)+"-"+str(margin)+" and " +str(ra)+"+"+str(margin)+"and dec between "+str(dec)+"-"+str(margin)+" and "+ str(dec)+"+"+str(margin)).readlines()
# clean_result = sqlcl.query( "SELECT distinct run,camcol,field FROM PhotoObj WHERE CLEAN =1 and ra between {0}-{1} and {0}+{1}and dec between {2}-{3} and {2}+{3}".format(str(ra),str(margin),str(dec),str(margin))) .readlines()
clean = True
print (result)
print (clean_result)
if (result[0][5:]=="<html>"):
print("strange error from SQL server")
return -1
if (result[1]=='error_message\n' or clean_result[1]=='error_message\n'):
#Case where doing more than 60 queries in 1 minute
time.sleep(60)
#results are messed up, need to re-query
result = sqlcl.query( "SELECT distinct run,camcol,field FROM PhotoObj WHERE ra between "+str(ra)+"-"+str(margin)+" and " +str(ra)+"+"+str(margin)+"and dec between "+str(dec)+"-"+str(margin)+" and "+ str(dec)+"+"+str(margin)).readlines()
clean_result = sqlcl.query( "SELECT distinct run,camcol,field FROM PhotoObj WHERE CLEAN =1 and ra between "+str(ra)+"-"+str(margin)+" and " +str(ra)+"+"+str(margin)+"and dec between "+str(dec)+"-"+str(margin)+" and "+ str(dec)+"+"+str(margin)).readlines()
if (len(result)!=len(clean_result) and band=='u'):
#only print this once in the u band. If it is unclean in u band (ex. cosmic ray, bright star..etc) then it must be unclean in the other bands too.
print ("Data contain unclean images")
clean=False
unclean.write(str(ra)+" "+str(dec)+" "+str(radius)+" "+pgc)
# unclean.write("{} {} {} {} \n".format(str(ra),str(dec),str(radius),pgc))
data =[]
count =0
for i in result:
if count>1:
list =i.split(',')
list[2]= list[2][:-1]
data.append(list)
count += 1
print (data)
if (len(data)==0 and band=='r'): #you will only evounter non-footprint galaxy inint run , because after that we just take the footprint gaalxy already mosaiced (init) from rfits
if (DEBUG): print ('The given ra, dec of this galaxy does not lie in the SDSS footprint. Onto the next galaxy!')#Exit Program.'
output.write(str(ra)+ " "+ str(dec)+" "+str(radius)+"\n")
# output.write("{} {} {} {} \n".format(str(ra),str(dec),str(radius),pgc))
output.write(str(ra)+" "+str(dec)+" "+str(radius)+" "+pgc)
#sys.exit()
return -1 #special value reserved for not in SDSS footprint galaxies
else :
if (DEBUG):
print ( "Complete Query. These data lies within margin: ")
print (data)
# os.mkdir(filename)
# os.chdir(filename)
#if (os.path.exists(band)):
#os.system("rm -r "+band)
os.mkdir(band)
os.chdir(band)
os.mkdir ("raw")
os.mkdir ("projected")
os.chdir("raw")
if (DEBUG): print ("Retrieving data from SDSS SAS server for "+ band +"band")
for i in data :
out = "frame-"+str(band)+"-"+str(i[0]).zfill(6)+"-"+str(i[1])+"-"+str(i[2]).zfill(4)
os.system("wget http://mirror.sdss3.org/sas/dr10/boss/photoObj/frames/301/"+str(i[0])+"/"+str(i[1])+"/"+out+".fits.bz2")
os.system("bunzip2 "+out+".fits.bz2")
os.chdir("../")
if (DEBUG) : print("Creating mosaic for "+band+" band.")
outfile_r="SDSS_"+band+"_"+str(ra)+"_"+str(dec)+"r.fits"
outfile="SDSS_"+band+"_"+str(ra)+"_"+str(dec)+".fits"
if (len(data)==1):
#With header info, len of processed result list is 1 if there is only 1 field lying in the margin, simply do mSubImage without mosaicing
#This patch should not be necessary but the program is aparently not mosaicing for the case where there is only one field.
print ("Only one field in region of interest")
os.chdir("raw")
montage.mSubimage(out+".fits",outfile,ra,dec,2*margin) # mSubImage takes xsize which should be twice the margin (margin measures center to edge of image)
#os.chdir("../..")
hdulist = pyfits.open(outfile)
shutil.move(outfile,"../..")
os.chdir("../..")
else:
montage.mImgtbl("raw","images.tbl")
montage.mHdr(str(ra)+" "+str(dec),margin,out+".hdr")
if (DEBUG): print ("Reprojecting images")
os.chdir("raw")
montage.mProjExec("../images.tbl","../"+out+".hdr","../projected", "../stats.tbl")
os.chdir("..")
montage.mImgtbl("projected","pimages.tbl")
os.chdir("projected")
montage.mAdd("../pimages.tbl","../"+out+".hdr","SDSS_"+out+".fits")
# outfile_r="SDSS_{}_{}_{}r.fits".format(band,str(ra),str(dec))
#outfile_r="SDSS_"+band+"_"+str(ra)+"_"+str(dec)+"r.fits"
montage.mSubimage("SDSS_"+out+".fits",outfile_r,ra,dec,2*margin) # mSubImage takes xsize which should be twice the margin (margin measures center to edge of image)
shutil.move(outfile_r,os.getcwd()[:-11] )#if change to :-11 then move out of u,g,r,i,z directory, may be more convenient for mJPEG
if (DEBUG) : print ("Completed Mosaic for " + band)
os.chdir("../..")
hdulist = pyfits.open(outfile_r)
hdulist[0].header['RA']=ra
hdulist[0].header['DEC']=dec
hdulist[0].header['RADIUS']=radius
hdulist[0].header['PGC']=pgc
hdulist[0].header['NED']=("http://ned.ipac.caltech.edu/cgi-bin/objsearch?objname="+ str(hdulist[0].header['PGC'])+"&extend=no&hconst=73&omegam=0.27&omegav=0.73&corr_z=1&out_csys=Equatorial&out_equinox=J2000.0&obj_sort=RA+or+Longitude&of=pre_text&zv_breaker=30000.0&list_limit=5&img_stamp=YES")
hdulist[0].header['CLEAN']=clean
hdulist[0].header['MARGIN']=margin
#if (os.path.exists(outfile)):
#os.system("rm "+ outfile)
hdulist.writeto(outfile)
if (os.path.exists(outfile_r)):
os.system("rm "+outfile_r)
#print("Deleting")
os.system("rm -r "+band+"/")
print ("Completed Mosaic")
return outfile
#if band4_present==1:
# f.write('<td>5.5 GHz</td>')
#if band5_present==1:
# f.write('<td>3.6 micron image</td>')
#if band6_present==1:
# f.write('<td>4.5 micron image</td>')
#f.write('</tr>')
#==================to be continued in the loop=========================
try:
montage.mSubimage(input_fits_image_band1,
'./NGP6D1_cutout.fits',
ra=200.74129,
dec=33.403903,
xsize=0.01666666666)
except:
print '==problem in ks image=='
#==============================do cutoff==========================================================
#for counter_cutoff in range(len(X_WORLD_K)):
# try:
# montage.mSubimage(input_fits_image_band1, './output/cutoff_250_'+str(NUMBER_K[counter_cutoff])+'_'+str(counter_cutoff)+'.fits', ra=X_WORLD_K[counter_cutoff], dec=Y_WORLD_K[counter_cutoff], xsize=0.05)
# except:
# print '==problem in ks image=='
#============================update cutoff images using aplpy=====================================================
nsa_parent_name = datadir + 'nsa/ivar/%s-parent-%s.fits.gz' % (iauname, pid)
nsa_pimage_name = datadir + 'nsa/pimages/%s-pimage.fits.gz' % (iauname)
# output files
out_image_name = datadir + 'images/%s_cutout.fits' % (iauname)
out_ivar_name = datadir + 'ivar/%s_ivar.fits' % (iauname)
# tempfiles
out_image_hdr = datadir + 'hdr/%s_cutout.hdr' % (iauname)
out_nsa_diff_name = datadir + 'diff/%s.fits' % (iauname)
out_nsa_diff_re_name = datadir + 'diffre/%s.fits' % (iauname)
out_nsa_pimage_re_name = datadir + 'pimagere/%s_nsa_pimage_re.fits' % (iauname)
log.info("Process %s", iauname)
if not os.path.isfile(ivar_name): save_ivar(filter, run, camcol, field)
log.info("Cutout field image: %s", out_image_name)
montage.mSubimage(field_name, out_image_name, ra, dec, size)
log.info("Making header file: %s", out_image_hdr)
hdr = montage.mGetHdr(out_image_name, out_image_hdr)
# reproject everything to cutout field image
log.info("Cutout ivar image: %s", out_ivar_name)
montage.reproject(ivar_name, out_ivar_name, header=out_image_hdr, exact_size=True,
silent_cleanup=montage_silent)
log.info("Save parent - child: %s", out_nsa_diff_name)
hdu_child = fits.open(nsa_image_name)
hdu_parent = fits.open(nsa_parent_name)
fits.writeto(out_nsa_diff_name, data=hdu_parent[child_ext*2].data - hdu_child[child_ext].data,
header=hdu_child[2].header, clobber=True)
log.info("Reprojecting diff: %s", out_nsa_diff_re_name)
montage.reproject(out_nsa_diff_name, out_nsa_diff_re_name, header=out_image_hdr, exact_size=True,
def coadd_rgb(name_red=None,
name_green=None,
name_blue=None,
out=None,
pmin=0.02,
pmax=0.98,
ra0=None,
dec0=None,
sr0=None,
north=False,
match=True):
if ((not name_red or not posixpath.isfile(name_red))
and (not name_green or not posixpath.isfile(name_green))
and (not name_blue or not posixpath.isfile(name_blue))):
print "Not enough frames to make three-color image!"
return
if not out:
out = "rgb.jpg"
fd, cube = tempfile.mkstemp(prefix='cube', suffix='.fits')
os.close(fd)
print "Co-adding three-color image: red=%s green=%s blue=%s" % (
name_red, name_green, name_blue)
if match:
matched_red = blind_match(name_red)
matched_green = blind_match(name_green)
matched_blue = blind_match(name_blue)
else:
matched_red = make_temp_file(name_red)
matched_green = make_temp_file(name_green)
matched_blue = make_temp_file(name_blue)
fix_fits_crop(matched_red)
fix_fits_crop(matched_green)
fix_fits_crop(matched_blue)
if ra0 or dec0 or sr0:
import montage_wrapper as montage
montage.mSubimage(matched_red, matched_red, ra0, dec0, sr0)
montage.mSubimage(matched_green, matched_green, ra0, dec0, sr0)
montage.mSubimage(matched_blue, matched_blue, ra0, dec0, sr0)
if match and (not matched_red or not matched_green or not matched_blue):
print "Can't match input frames"
return None
make_rgb_cube([matched_red, matched_green, matched_blue],
cube,
north=north)
make_rgb_image(cube,
out,
embed_avm_tags=False,
make_nans_transparent=True,
pmin_r=pmin * 100,
pmin_g=pmin * 100,
pmin_b=pmin * 100,
pmax_r=pmax * 100,
pmax_g=pmax * 100,
pmax_b=pmax * 100)
os.unlink(cube)
if matched_red:
os.unlink(matched_red)
if matched_green:
os.unlink(matched_green)
if matched_blue:
os.unlink(matched_blue)
if posixpath.isfile(out):
print "Successfully created RGB image %s" % out
else:
out = None
return out
shutil.move(out3+".fits",band+"/raw/" )
os.chdir(band)
if (DEBUG) : print("Creating mosaic for " +" "+ band + " band.")
montage.mImgtbl("raw","images.tbl")
montage.mHdr(str(ra)+" "+str(dec),radius,out+".hdr")
if (DEBUG): print ("Reprojecting images")
#Sometimes you can't find the files and result in images.tbl => empty doc
#need to put data file inside raw AND unzip it so that Montage detect that it is a fit file
os.chdir("raw")
montage.mProjExec("../images.tbl","../"+out+".hdr","../projected", "../stats.tbl")
os.chdir("..")
montage.mImgtbl("projected","pimages.tbl")
#mAdd coadds the reprojected images using the FITS header template and mImgtbl list.
os.chdir("projected")
montage.mAdd("../pimages.tbl","../"+out+".hdr","SDSS_"+out+".fits")
montage.mSubimage("SDSS_"+out+".fits","SDSS_"+out+".fits",ra,dec,0.05)
shutil.move("SDSS_"+out+".fits",os.getcwd()[:-11] )#if change to :-11 then move out of u,g,r,i,z directory, may be more convenient for mJPEG
if (DEBUG) : print ("Completed Mosaic for " + band)
os.chdir("../..")
# Superimposing R,G,B image mosaics into JPEG
# os.system("mJPEG "+ " -t 20 \ "+
# " -red "+ "SDSS_frame-"+"i"+"-"+str(run).zfill(6)+"-"+str(camcol)+"-"+str(field).zfill(4)+ ".fits "+ " 1s max gaussian-log \
# -green SDSS_frame-"+"r"+"-"+str(run).zfill(6)+"-"+str(camcol)+"-"+str(field).zfill(4)+".fits "+" 1s max gaussian-log \
# -blue SDSS_frame-"+"g"+"-"+str(run).zfill(6)+"-"+str(camcol)+"-"+str(field).zfill(4)+".fits "+ " 2s max gaussian-log \
# -out "+str(floor(ra))+"_"+str(floor(dec))+"_tcolor_10_min1s_max_final.jpg")
# Using STIFF
os.system("stiff "+" SDSS_frame-"+"i"+"-"+str(run).zfill(6)+"-"+str(camcol)+"-"+str(field).zfill(4)+ ".fits "+ " SDSS_frame-"+"r"+"-"+str(run).zfill(6)+"-"+str(camcol)+"-"+str(field).zfill(4)+".fits "+ " SDSS_frame-"+"g"+"-"+str(run).zfill(6)+"-"+str(camcol)+"-"+str(field).zfill(4)+".fits "+ " -c stiff.conf " +" -OUTFILE_NAME "+str(floor(ra))+"_"+str(floor(dec))+" -COLOUR_SAT 1")
for b in bands:
os.system("rm -r "+b+"/")
#we want to keep the fit files, but for testing purposes Python will throw file-already-exist error , if we dont delete them.
#os.system("rm -r " + "SDSS_frame-"+b+"-"+str(run).zfill(6)+"-"+str(camcol)+"-"+str(field).zfill(4)+ ".fits" )
gal, field, ra, dec = np.loadtxt(tbl,
dtype=str,
usecols=(0, 1, 2, 3),
unpack=True)
galf = gal.astype(float)
galaxy = galf.astype(int)
table = open("cutted_gal.txt", "w")
table.write("#galaxy \t field \t ra \t dec \n")
for ii in range(len(gal)):
ra_center = ra[ii].astype(float)
dec_center = dec[ii].astype(float)
if field[ii] == "GOODS-S":
input_img = "hlsp_candels_hst_wfc3_gs-tot_f160w_v1.0_drz.fits"
montage.mSubimage(input_img,
str(galaxy[ii]) + "_cut.fits", ra_center, dec_center,
size)
table.write(
str(galaxy[ii]) + "\t" + str(field[ii]) + "\t" + str(ra_center) +
"\t" + str(dec_center) + "\n")
elif field[ii] == "GOODS-N":
#input_img = "hlsp_candels_hst_wfc3_gn13_f160w_v0.5_drz.fits"
input_img = "goodsn_all_wfc3_ir_f160w_060mas_v1.0_drz.fits"
montage.mSubimage(input_img,
str(galaxy[ii]) + "_cut.fits", ra_center, dec_center,
size)
table.write(
str(galaxy[ii]) + "\t" + str(field[ii]) + "\t" + str(ra_center) +
"\t" + str(dec_center) + "\n")
elif field[ii] == "COSMOS":
input_img = "hlsp_candels_hst_wfc3_cos-tot_f160w_v1.0_drz.fits"
def mosaic_band(self,band,ra,dec,margin,radius,pgc,survey,remove_bkgrd=False):
'''
Input: source info param
Create a mosaic fit file for the specified band.
Return: String filename of resulting mosaic
'''
print ("------------------mosaic_band----------------------")
print ("Now mosaic_band on {}".format(pgc))
output = open("../rc3_galaxies_outside_{}_footprint".format(survey.name),'a') # 'a' for append #'w')
unclean = open("../rc3_galaxies_unclean_{}".format(survey.name),"a")
filename = str(ra)+str(dec)
if (DEBUG) : print ("Querying data that lies inside margin")
print (ra,dec,margin)
result = survey.data_server.surveyFieldConverter(float(ra),float(dec),float(margin))
clean_result = survey.data_server.surveyFieldConverter(float(ra),float(dec),float(margin),True)
clean = True
if(DEBUG):print ("result: "+str(result))
if(DEBUG):print ("clean_result: "+str(clean_result))
if (len(result)!=len(clean_result)and band=='u'):
# Only print this once in the u band.
# Assume that if it is unclean in u band (ex. cosmic ray, bright star..etc) then it must be unclean in the other bands too.
print ("Data contain unclean images")
clean=False
unclean.write("{} {} {} {} \n".format(self.rc3_ra,self.rc3_dec,self.rc3_radius,self.pgc))
if (len(result)==0):
if (DEBUG): print ('The given ra, dec of this galaxy does not lie in the survey footprint. Onto the next galaxy!')#Exit Program.'
output.write("{} {} {} {} \n".format(str(ra),str(dec),str(radius),str(pgc)))
return -1 #special value reserved for not in survey footprint galaxies
else :
if (DEBUG):
print ( "Complete Query. These data lies within margin: ")
print (result)
os.mkdir(band)
os.chdir(band)
os.mkdir ("rawdir")
os.mkdir ("projdir")
if (remove_bkgrd):
os.mkdir ("diffdir")
os.mkdir ("corrdir")
# os.mkdir("final")
if (DEBUG): print ("Retrieving data from server for "+ band +"band")
os.chdir("rawdir")
out=""
# Raw Imaging Data naming
for i in result :
if (survey.data_server.name=='Gator'):
survey.data_server.getData(band,ra,dec,margin,survey)
out = i # 2MASS designation
print out
elif (survey.data_server.name=='SkyServer'):
survey.data_server.getData(band,str(i[0]), str(i[1]),str(i[2]))
# run-camcol-field
out = "frame-"+str(band)+"-"+str(i[0]).zfill(6)+"-"+str(i[1])+"-"+str(i[2]).zfill(4)
elif (survey.data_server.name=='DSSServer'):
survey.data_server.getData(band,ra,dec,margin)
# Patch for when we can not pass in th pgc number in getData of dssServer class, we rename the file here to conform with RC3's filename expectation for the imaging data
raw_data = glob.glob("raw_*.fits")
print (raw_data)
for i in raw_data:
os.rename(i,"DSS_{}_{}.fits".format(band, self.pgc))
out = "raw_{}_{}".format(band,self.pgc)
print ("dss_out: "+out)
else:
raise TypeError("Missing implementation for data retrieval")
os.chdir("../")
if (DEBUG) : print("Creating mosaic for "+band+" band.")
outfile_r = "{}_{}_{}r.fits".format(survey.name,band,self.pgc)
outfile = "{}_{}_{}.fits".format(survey.name,band,self.pgc)
if (len(result)==1):
#With header info, len of processed result list is 1 if there is only 1 field lying in the margin, simply do mSubImage without mosaicing
print ("Only one field in region of interest")
os.chdir("rawdir")
if (DEBUG):print ("m:{}".format(margin))
try:
if (DEBUG):print ("2m:{}".format(2*margin))
if (DEBUG):print ([outfile_r,outfile,ra,dec,2*margin])
montage.mSubimage(outfile,outfile,ra,dec,2*margin) # mSubImage takes xsize which should be twice the margin (margin measures center to edge of image)
except(montage.status.MontageError):
print ("montage_wrapper.status.MontageError: mSubimage: Region outside image.")
try :#give it one last chance
if (DEBUG):print ("lastchancem:{}".format(margin))
montage.mSubimage(out+".fits",outfile,ra,dec,margin)
except(montage.status.MontageError):
print("Doesn't work after trying half the margin, just keep the raw FITS file")
if (DEBUG):print (out+".fits")
if (DEBUG):print (outfile)
shutil.move(out+".fits","../..")
os.chdir("../../")
os.rename(out+".fits",outfile)
os.system("rm -r {}".format(survey.best_band))
return outfile
if (DEBUG):print (os.getcwd())
os.chdir("../../") #Get out of directory for that galaxy and move on
os.system("rm -r {}".format(survey.best_band))
if (DEBUG):print(os.getcwd())
failed_msubimage = open ("failed_msubimage","a")
failed_msubimage.write("{} {} {} {} \n".format(str(ra),str(dec),str(radius),str(pgc)))
return -1 # masking with special value reserved for not in survey footprint galaxies
hdulist = pyfits.open(outfile)
if (os.path.exists("../../"+outfile)):
os.system("rm ../../"+outfile)
shutil.move(outfile,"../..")
os.chdir("../..")
else:
imgtbl="images-rawdir.tbl"
hdr="template.hdr"
montage.mImgtbl("rawdir",imgtbl)
# montage.mHdr(str(ra)+" "+str(dec),margin,out+".hdr")
montage.mMakeHdr(imgtbl,hdr)
if (DEBUG): print ("Reprojecting images")
# os.chdir("rawdir")
if (DEBUG):print(os.getcwd())
montage.mProjExec(imgtbl,hdr,"projdir", "stats.tbl",raw_dir="rawdir")#, mpi=enable_mpi,debug=True)
if os.listdir("projdir") == []:
print "Projection Failed. No projected images produced. Skip to the next galaxy"
os.chdir("../") #Get out of directory for that galaxy and move on
os.system("rm -r {}".format(survey.best_band))
failed_projection = open ("failed_projection","a")
failed_projection.write("{} {} {} {} \n".format(str(ra),str(dec),str(radius),str(pgc)))
return -1 # masking with special value reserved for not in survey footprint galaxies
if (remove_bkgrd):
if (DEBUG): print "Calling the bash script containing Montage routines to rectify the background"
if os.getcwd()[-4:-2]==str(pgc):
os.system("bash ../../mosaic.sh")
else:
os.system("bash ../mosaic.sh")
print "mSubimage"
montage.mSubimage("mosaic.fits" ,"mosaic.fits",ra,dec,2*margin) # mSubImage takes xsize which should be twice the margin (margin measures center to edge of image)
shutil.move("mosaic.fits","../{}".format(outfile_r))#if change to :-11 then move out of u,g,r,i,z directory, may be more convenient for mJPEG
if (DEBUG) : print ("Completed Mosaic for " + band)
else:
montage.mImgtbl("projdir","pimages.tbl")
os.chdir("projdir")
montage.mAdd("../pimages.tbl","../"+hdr,"{}_{}.fits".format(survey.name,out))#, mpi=enable_mpi)
montage.mSubimage("{}_{}.fits".format(survey.name,out),outfile_r,ra,dec,2*margin) # mSubImage takes xsize which should be twice the margin (margin measures center to edge of image)
shutil.move(outfile_r,"../../{}".format(outfile_r) )#if change to :-11 then move out of u,g,r,i,z directory, may be more convenient for mJPEG
if (DEBUG) : print ("Completed Mosaic for " + band)
os.chdir("..")
os.chdir("../")
hdulist = pyfits.open(outfile_r)
hdulist[0].header['RA']=float(ra)
hdulist[0].header['DEC']=float(dec)
hdulist[0].header['RADIUS']=radius
if (DEBUG):print ("Finished mosaic_band on {}".format(pgc))
hdulist[0].header['PGC']=pgc
hdulist[0].header['NED']=("http://ned.ipac.caltech.edu/cgi-bin/objsearch?objname="+ str(hdulist[0].header['PGC'])+"&extend=no&hconst=73&omegam=0.27&omegav=0.73&corr_z=1&out_csys=Equatorial&out_equinox=J2000.0&obj_sort=RA+or+Longitude&of=pre_text&zv_breaker=30000.0&list_limit=5&img_stamp=YES")
hdulist[0].header['CLEAN']=clean
hdulist[0].header['MARGIN']=margin
if (os.path.exists(outfile)):
os.system("rm "+ outfile)
hdulist.writeto(outfile)
if (os.path.exists(outfile_r)):
os.system("rm "+outfile_r)
os.system("rm -r {}".format(band))
return outfile
f.write('<td>3.6 micron image</td>')
if band6_present == 1:
f.write('<td>4.5 micron image</td>')
f.write('</tr>')
#==================to be continued in the loop=========================
#==============================do cutoff==========================================================
for counter_cutoff in range(len(X_WORLD_K)):
try:
montage.mSubimage(input_fits_image_band1,
'./output/cutoff_ks_' +
str(NUMBER_K[counter_cutoff]) + '.fits',
ra=X_WORLD_K[counter_cutoff],
dec=Y_WORLD_K[counter_cutoff],
xsize=0.0023)
except:
print '==problem in ks image=='
try:
montage.mSubimage(input_fits_image_band2,
'./output/cutoff_j_' +
str(NUMBER_K[counter_cutoff]) + '.fits',
ra=X_WORLD_K[counter_cutoff],
dec=Y_WORLD_K[counter_cutoff],
xsize=0.0023)
except:
print '==problem in j image=='
try:
montage.mSubimage(input_fits_image_band3,
def process_images(gal, filter, sizeopt=30., datadir='data', outdir='pipeline', dryrun=False):
"""
Process galaxy images for bdfitter
Prints the following info in a line:
name
desired cutout size in arcsec
desired cutout size in pixels
fraction actuall area / desired area of cutout
TODO: complete docstring
gal : dict-like
must have these keys
IAUNAME, RUN, CAMCOL, FIELD, ...
filter : str
one of u, g, r, i, z
sizeopt : str or float
if string, it is the key for the desired cutout size in degrees
if float, it is the cutoff surface brightness (mag/arcsec^2) using given Sersic model
specified by these keys
SERSIC_N : sersic index
SERSIC_MU : surface brightness at Re
SERSIC_TH50 : Re in arcsec
"""
# Determine the size of the cutout image
if type(sizeopt) is str:
# size is explicitly specified in degrees
cutout_size_deg = gal[sizeopt]
elif type(sizeopt) is float:
# use given Sersic best-fit to calculate the radius at which
# the surface brightness (mag/arcsec^2) falls below *sizeopt*
n = gal['SERSIC_N'] # sersic index
mu_e = gal['SERSIC_MU'] # surface brightness at Re in mag/arcsec^2
Re_pix = gal['SERSIC_TH50'] / 0.396 # Re in pixels
x = (1. + log(10)/2.5/bn(n) * (sizeopt - mu_e))**n
cutout_size_deg = 2*Re_pix * x * 0.396 / 3600.
# Check if the cutout will large enough
rectField = [0, 0, 2048, 1489]
ce = r_[gal['XPOS'], gal['YPOS']] # center pixel position in field image
halfwidth = cutout_size_deg*3600/0.396/2 # in pixels
rectCutout = [ce[0]-halfwidth, ce[1]-halfwidth, ce[0]+halfwidth, ce[1]+halfwidth]
# overlapping fraction of desired cutout
frac_overlap = (min(rectCutout[2], 2048) - max(rectCutout[0],0)) * \
(min(rectCutout[3],1489) - max(rectCutout[1],0)) / (4*halfwidth**2)
print gal['IAUNAME'], '%8.2f %8.2f %8.4f' % (cutout_size_deg*3600, 2*halfwidth, frac_overlap)
if not dryrun:
# Start processing
# save psf
save_psf(gal, filter, outdir+'/psf', datadir=datadir)
# SDSS field image
fn_field = datadir+'/sdss_field/'+get_framename(gal['RUN'], gal['CAMCOL'], gal['FIELD'], filter)
# NSA mask image
fn_pimage = datadir + '/nsa/pimages/'+'%s-pimage.fits.gz' % (gal['IAUNAME'])
# cutout field image and open it
fn_cutout = outdir+'/cutout/%s_cutout.fits'%(gal['IAUNAME'])
montage.mSubimage(fn_field, fn_cutout, gal['RA_1'], gal['DEC_1'], cutout_size_deg)
hdu_cutout = fits.open(fn_cutout)
# make and cutout ivar image
fn_field_ivar = save_ivar(filter, gal['RUN'], gal['CAMCOL'], gal['FIELD'], 'data/')
fn_cutout_ivar = outdir+'/ivar/%s.fits' % (gal['IAUNAME'])
montage.mSubimage(fn_field_ivar, fn_cutout_ivar, gal['RA_1'], gal['DEC_1'], cutout_size_deg)
# resample pimage in cutout image coordinates using nearest neighbor method
# to preserve child id's exactly
hdu_pimage = fits.open(fn_pimage)
wcs_pimage = wcs.WCS(hdu_pimage[0].header, fix=False)
x, y = meshgrid(arange(wcs_pimage.naxis1), arange(wcs_pimage.naxis2))
wx, wy = wcs_pimage.all_pix2sky(x, y, 0)
val = hdu_pimage[0].data
wcs_cutout = wcs.WCS(hdu_cutout[0].header, fix=False)
x1, y1 = meshgrid(arange(wcs_cutout.naxis1), arange(wcs_cutout.naxis2))
wx1, wy1 = wcs_cutout.all_pix2sky(x1, y1, 0)
pim = interpolate.griddata((wx.ravel(), wy.ravel()), val.ravel(), (wx1, wy1), method='nearest')
# To prevent masks on the edge to extend unnecesarily due to nearest interpolation
# reset values outside of pim to -1
xout = (wx1<wx.min()) | (wx1>wx.max())
yout = (wy1<wy.min()) | (wy1>wy.max())
pim[xout | yout] = -1
# find out child ID of the galaxy
pix_x, pix_y = wcs_pimage.wcs_world2pix(gal['RA_1'], gal['DEC_1'], 0)#, ra_dec_order=True)
gal_id = hdu_pimage[0].data[rint(pix_y), rint(pix_y)]
# make a mask image with 0 for pixels that should be masked, 1 for others
# to be multiplied to cutout image
mask = where((pim != gal_id) & (pim != -1), 0., 1.)
# write mask image
hdu_mask = fits.HDUList([fits.PrimaryHDU(data=mask)])
hdu_mask[0].header.update(wcs_cutout.to_header()) # put WCS header keywords
hdu_mask.writeto(outdir+'/mask/%s_mask.fits' % (gal['IAUNAME']), clobber=True)
# figure for visual check
figure(figsize=(10,5))
subplots_adjust(left=0.08, right=0.95)
suptitle(' '.join([gal['IAUNAME'], fn_field.split('/')[-1]]), fontsize=10)
subplot(121)
imshow(hdu_cutout[0].data, norm=mpl.colors.LogNorm(), origin='lower', aspect='auto')
subplot(122)
# make a color map of fixed colors for mask image
cmap = mpl.colors.ListedColormap(['black', 'white'])
norm = mpl.colors.BoundaryNorm([0., .5, 1.], cmap.N)
imshow(mask, interpolation='nearest', origin='lower', aspect='auto', cmap=cmap, norm=norm)
savefig(outdir+'/fig/%s.png' % (gal['IAUNAME']), dpi=80)
close()
# save masked image
hdu_cutout[0].data *= mask
hdu_cutout.writeto(outdir+'/images/%s.fits' % (gal['IAUNAME']), clobber=True)
####################################################################
i = 0
cropList = []
while i < len(NAXISList):
#print NAXISList[i]
currentFile = str(NAXISList[i])
#These are for Herschel SPIRE 250 um!
# if i == 4 or i == 9 or i == 17:
# outputFile = currentFile.strip('_NAXIS_2.fits')+'_crop_2.fits'
# if i == 8:
# outputFile = currentFile.strip('_NAXIS_2.fits')+'_crop_3.fits'
# else:
# outputFile = currentFile.strip('_NAXIS_2.fits')+'_crop.fits'
outputFile = currentFile.strip('_NAXIS_2.fits') + '_crop.fits'
montage.mSubimage(currentFile, outputFile, ra_center_list[i],
dec_center_list[i], 0.2)
cropList.append(outputFile)
i += 1
i = 0
#finalFile = open('Final_sources.txt','w')
#finalFile.write('SCUBA Herschel\n')
#850 & 160 um
finalFile = open('Final_sources_160.txt', 'w')
finalFile.write('SCUBA Herschel 160um\n')
while i < len(cropList):
scubaNow = SCUBAList[i]
herNow = cropList[i]
finalFile.write(scubaNow + ' ' + herNow + '\n')
#if band5_present==1:
# f.write('<td>3.6 micron image</td>')
#if band6_present==1:
# f.write('<td>4.5 micron image</td>')
#f.write('</tr>')
#==================to be continued in the loop=========================
try:
montage.mSubimage(input_fits_image_band1, './output/H12-00_source30_cutout.fits', ra=176.64708, dec=-0.18040577, xsize=0.01)
except:
print '==problem in ks image=='
#==============================do cutoff==========================================================
#for counter_cutoff in range(len(X_WORLD_K)):
# try:
# montage.mSubimage(input_fits_image_band1, './output/cutoff_250_'+str(NUMBER_K[counter_cutoff])+'_'+str(counter_cutoff)+'.fits', ra=X_WORLD_K[counter_cutoff], dec=Y_WORLD_K[counter_cutoff], xsize=0.05)
# except:
# print '==problem in ks image=='
if band6_present == 1:
f.write('<td>4.5 micron image</td>')
f.write('</tr>')
#==================to be continued in the loop=========================
#==============================do cutoff==========================================================
for counter_cutoff in range(len(X_WORLD_K)):
try:
montage.mSubimage(input_fits_image_band1,
'./output/cutoff_250_' +
str(NUMBER_K[counter_cutoff]) + '_' +
str(counter_cutoff) + '.fits',
ra=X_WORLD_K[counter_cutoff],
dec=Y_WORLD_K[counter_cutoff],
xsize=0.05)
except:
print '==problem in ks image=='
try:
montage.mSubimage(input_fits_image_band2,
'./output/cutoff_350_' +
str(NUMBER_K[counter_cutoff]) + '_' +
str(counter_cutoff) + '.fits',
ra=X_WORLD_K[counter_cutoff],
dec=Y_WORLD_K[counter_cutoff],
xsize=0.05)
except:
print '==problem in j image=='
