Beispiel #1
0
def reproject_images(template_header,
                     input_dir,
                     reprojected_dir,
                     imtype,
                     whole=False,
                     exact=True,
                     img_list=None):

    input_table = os.path.join(input_dir, imtype + '_input.tbl')
    montage.mImgtbl(input_dir, input_table, corners=True, img_list=img_list)

    # Create reprojection directory, reproject, and get image metadata
    stats_table = os.path.join(reprojected_dir,
                               imtype + '_mProjExec_stats.log')
    montage.mProjExec(input_table,
                      template_header,
                      reprojected_dir,
                      stats_table,
                      raw_dir=input_dir,
                      whole=whole,
                      exact=exact)

    reprojected_table = os.path.join(reprojected_dir,
                                     imtype + '_reprojected.tbl')
    montage.mImgtbl(reprojected_dir, reprojected_table, corners=True)
def reproject_images(template_header, int_images, rrhr_images, flag_images, input_dir, reprojected_dir, whole=True, exact=True):

    # MASK IMAGES
    for i in range(len(int_images)):
        image_infile = int_images[i]
        wt_infile = rrhr_images[i]
        flg_infile = flag_images[i]

        image_outfile = os.path.join(input_dir, os.path.basename(image_infile).replace('.fits', '_masked.fits'))
        wt_outfile = os.path.join(input_dir, os.path.basename(wt_infile).replace('.fits', '_masked.fits'))

        #mask_galex_edges(image_infile, flg_infile, outfile=image_outfile)
        #mask_galex_edges(wt_infile, flg_infile, outfile=wt_outfile)
        mask_galex(image_infile, wt_infile, flg_infile, out_intfile=image_outfile, out_wtfile=wt_outfile)


    # REPROJECT IMAGES
    input_table = os.path.join(input_dir, 'input.tbl')
    montage.mImgtbl(input_dir, input_table, corners=True)

    # Create reprojection directory, reproject, and get image metadata
    #whole = True #if background_match else False
    stats_table = os.path.join(reprojected_dir, 'mProjExec_stats.log')

    montage.mProjExec(input_table, template_header, reprojected_dir, stats_table, raw_dir=input_dir, whole=whole, exact=exact)

    reprojected_table = os.path.join(reprojected_dir, 'reprojected.tbl')
    montage.mImgtbl(reprojected_dir, reprojected_table, corners=True)
Beispiel #3
0
def reproject_images(template_header,
                     input_dir,
                     reproj_dir,
                     imtype,
                     whole=True,
                     exact=True,
                     corners=True,
                     img_list=None):
    """
    Reproject input images to a new WCS as given by a template header

    Parameters
    ----------
    template_header : ascii file
        ASCII file containing the WCS to which you want to reproject. This is what Montage requires.
    input_dir : str
        Path to directory containing input data
    reproj_imtype_dir : 
        Path to new directory for storing reprojected data
    imtype : str
        The type of image you are reprojecting; one of [int, rrhr]
    whole : bool, optional
        Montage argument: Force reprojection of whole images, even if they exceed the area of the FITS 
        header template (Default: True)
    exact : bool, optional
        Montage argument: Flag indicating output image should exactly match the FITS header template, 
        and not crop off blank pixels (Default: True)
    corners : bool, optional
        Montage argument: Adds 8 columns for the RA and Dec of the image corners to the output metadata table 
        (Default: True)
    img_list : list of strs, optional 
        Montage argument: only process files with names specified in table img_list, ignoring any other files
        in the directory. (Default: None)
    """

    # get image metadata from input images
    input_table = os.path.join(input_dir, imtype + '_input.tbl')
    montage.mImgtbl(input_dir, input_table, corners=corners, img_list=img_list)

    # reproject images
    stats_table = os.path.join(reproj_dir, imtype + '_mProjExec_stats.log')
    montage.mProjExec(input_table,
                      template_header,
                      reproj_dir,
                      stats_table,
                      raw_dir=input_dir,
                      whole=whole,
                      exact=exact)

    # get new image metadata with new header information
    reprojected_table = os.path.join(reproj_dir, imtype + '_reprojected.tbl')
    montage.mImgtbl(reproj_dir, reprojected_table, corners=corners)
Beispiel #4
0
def mosaic(input_files, mosaic_file, work_dir, ext=0, background_match=False,
           cdelt=None, density=False, equinox=None, header=None,
           level_only=False, north_aligned=False, postprocess=None,
           preprocess=None, system=None, weights_file=None):
    """Make a mosiac.

    High-level wrapper around several Montage operations similar to
    `montage_wrapper.mosaic`. The main differences are 1) added support for
    preprocessing the input images before reprojection and postprocessing
    the final image after mosaicking, 2) options for using images in total
    flux units instead of flux density (as assumed by Montage), 3) more of
    the `montage_wrapper.mMakeHdr` keywords available for header creation,
    and 4) the `whole` keyword for `montage_wrapper.mProjExec` is
    automatically set to True when `background_match` is True. The latter
    is important since backround matching behaves unreliably otherwise.

    Parameters
    ----------
    input_files : list or string
        List of paths to the input images. This may also be the path to a
        directory containing all input images, in which case `input_files`
        will automatically be set to a list of all files in the directory
        ending with ".fits".
    mosaic_file : str
        Path to the output mosaic file. The final mosaic always has the
        same units as the `input_files` images.
    work_dir : str
        Path to the working directory for all intermediate files produced
        by Montage. The directory has the following structure::

          work_dir/
            input/
              Contains either symlinks to `input_files` or new files
              depending on the `preprocess` and `density` keywords.
              Assuming the `density` keyword has been set correctly, these
              images will always be in flux density units.
            reprojected/
              The reprojected images.
            differences/
              Difference calculations for background matching (only if
              `background_match` is True).
            corrected/
              Background-matched images (only if `background_match` is
              True).
            output/
              The intermediate mosiac used to produce the final mosaic
              file, depending on the `density` and `postprocess` keywords.

    background_match : bool, optional
        If True, match the background levels of the reprojected images
        before mosaicking. Automatically sets ``whole = True`` in
        `montage_wrapper.mProjExec`. Default is False.
    cdelt : float, optional
        See `header` and `montage_wrapper.mMakeHdr`. Default is None.
    density : bool, optional
        If True, the input images are in flux density units (i.e., signal
        per unit pixel area). If False (default), the input images are
        assumed to be in units of total flux, and are automatically scaled
        to flux density before reprojection.
    equinox : str, optional
        See `header` and `montage_wrapper.mMakeHdr`. Default is None.
    header : str, optional
        Path to the template header file describing the output mosaic.
        Default is None, in which case a template header is created
        automatically using `montage_wrapper.mMakeHdr` and the `cdelt`,
        `equinox`, `north_aligned`, and `system` keyword arguments.
    level_only : bool, optional
        See `montage_wrapper.mBgModel`. Ignored if `background_match` is
        False. Default is False.
    north_aligned : bool, optional
        See `header` and `montage_wrapper.mMakeHdr`. Default is None.
    postprocess, preprocess : function, optional
        Functions for processing the raw input images before the input
        density images are created (`preprocess`) and after the final
        mosaic is created (`postprocess`). The function arguments should be
        the image data array and the image header
        (`astropy.io.fits.Header`), and the return values should be the
        same. Default is None.
    system : str, optional
        See `header` and `montage_wrapper.mMakeHdr`. Default is None.
    weights_file : str, optional
        Path to output pixel weights file. Pixel weights are derived from
        the final mosaic area file. Weights are normalized to 1, and
        represent coverage of the mosaic area by the input images. Unlike
        Montage area files, regions where the input images overlap are not
        considered. Default is None.

    Returns
    -------
    None

    """
    # Get list of files if input_files is a directory name
    if isinstance(input_files, basestring):
        dirname = os.path.dirname(input_files)
        input_files = [os.path.join(dirname, basename)
                       for basename in os.listdir(dirname)
                       if os.path.splitext(basename)[1] == '.fits']

    # Create working directory
    try:
        os.makedirs(work_dir)
    except OSError:
        shutil.rmtree(work_dir)
        os.makedirs(work_dir)


    # Create input directory, populate it, and get image metadata
    input_dir = os.path.join(work_dir, 'input')
    os.mkdir(input_dir)

    if preprocess or not density or ext>0:
        # Create new input files
        for input_file in input_files:
            data, hdr = astropy.io.fits.getdata(input_file, header=True,
                                                ext=ext)
            if preprocess:
                data, hdr = preprocess(data, hdr)

            if not density:
                # Convert total flux into flux density
                dx, dy = wcs.calc_pixscale(hdr, ref='crpix').arcsec
                pixarea = dx * dy  # arcsec2
                data /= pixarea

            # Write
            basename = os.path.basename(input_file)
            basename = '_density'.join(os.path.splitext(basename))
            new_input_file = os.path.join(input_dir, basename)
            hdu = astropy.io.fits.PrimaryHDU(data=data, header=hdr)
            hdu.writeto(new_input_file, output_verify='ignore')

    else:
        # Symlink existing files
        for input_file in input_files:
            basename = os.path.basename(input_file)
            new_input_file = os.path.join(input_dir, basename)
            os.symlink(input_file, new_input_file)

    input_table = os.path.join(input_dir, 'input.tbl')
    montage.mImgtbl(input_dir, input_table, corners=True)

    # Template header
    if header is None:
        template_header = os.path.join(work_dir, 'template.hdr')
        montage.mMakeHdr(input_table, template_header, cdelt=cdelt,
                         equinox=equinox, north_aligned=north_aligned,
                         system=system)
    else:
        template_header = header

    # Create reprojection directory, reproject, and get image metadata
    proj_dir = os.path.join(work_dir, 'reprojected')
    os.makedirs(proj_dir)
    whole = True if background_match else False
    stats_table = os.path.join(proj_dir, 'mProjExec_stats.log')

    montage.mProjExec(input_table, template_header, proj_dir, stats_table,
                      raw_dir=input_dir, whole=whole)

    reprojected_table = os.path.join(proj_dir, 'reprojected.tbl')
    montage.mImgtbl(proj_dir, reprojected_table, corners=True)

    # Background matching
    if background_match:
        diff_dir = os.path.join(work_dir, 'differences')
        os.makedirs(diff_dir)

        # Find overlaps
        diffs_table = os.path.join(diff_dir, 'differences.tbl')
        montage.mOverlaps(reprojected_table, diffs_table)

        # Calculate differences between overlapping images
        montage.mDiffExec(diffs_table, template_header, diff_dir,
                          proj_dir=proj_dir)

        # Find best-fit plane coefficients
        fits_table = os.path.join(diff_dir, 'fits.tbl')
        montage.mFitExec(diffs_table, fits_table, diff_dir)

        # Calculate corrections
        corr_dir = os.path.join(work_dir, 'corrected')
        os.makedirs(corr_dir)
        corrections_table = os.path.join(corr_dir, 'corrections.tbl')
        montage.mBgModel(reprojected_table, fits_table, corrections_table,
                         level_only=level_only)

        # Apply corrections
        montage.mBgExec(reprojected_table, corrections_table, corr_dir,
                        proj_dir=proj_dir)

        img_dir = corr_dir

    else:
        img_dir = proj_dir


    # Make mosaic
    output_dir = os.path.join(work_dir, 'output')
    os.makedirs(output_dir)

    out_image = os.path.join(output_dir, 'mosaic.fits')
    montage.mAdd(reprojected_table, template_header, out_image,
                 img_dir=img_dir, exact=True)


    # Pixel areas and weights
    if weights_file or not density:
        area_file = '_area'.join(os.path.splitext(out_image))
        area, hdr = astropy.io.fits.getdata(area_file, header=True)  # steradians
        area *= (180/np.pi*3600)**2  # arcsec2
        dx, dy = wcs.calc_pixscale(hdr, ref='crpix').arcsec
        pixarea = dx * dy  # arcsec2
        area = np.clip(area, 0, pixarea)  # Don't care about overlaps
        if weights_file:
            weights = area / pixarea  # Normalize to 1
            hdu = astropy.io.fits.PrimaryHDU(weights, header=hdr)
            try:
                hdu.writeto(weights_file)
            except IOError:
                os.remove(weights_file)
                hdu.writeto(weights_file)


    # Write final mosaic
    dirname = os.path.dirname(mosaic_file)
    try:
        os.makedirs(dirname)
    except OSError:
        pass

    if postprocess or not density:
        # Create new file
        data, hdr = astropy.io.fits.getdata(out_image, header=True)

        if not density:
            # Convert flux density into total flux
            data *= pixarea

        if postprocess:
            data, hdr = postprocess(data, hdr)

        # Write
        hdu = astropy.io.fits.PrimaryHDU(data, header=hdr)
        try:
            hdu.writeto(mosaic_file)
        except IOError:
            os.remove(mosaic_file)
            hdu.writeto(mosaic_file)

    else:
        # Move existing file
        os.rename(out_image, mosaic_file)

    return
Beispiel #5
0
def _montage_test():
    # create density images

    input_dir = os.path.dirname(density_files[0])

    # image metadata
    meta1_file = os.path.join(input_dir, 'meta1.tbl')
    montage.mImgtbl(input_dir, meta1_file, corners=True)

    # make header
    #lon, lat = [], []
    #for density_file in density_files:
    #    data, hdr = astropy.io.fits.getdata(density_file, header=True)
    #    wcs = astropy.wcs.WCS(hdr)
    #    x1, y1 = 0.5, 0.5
    #    y2, x2 = data.shape
    #    x2, y2 = x2 + 0.5, y2 + 0.5
    #    x, y = [x1, x2, x2, x1], [y1, y1, y2, y2]
    #    ln, lt = wcs.wcs_pix2world(x, y, 1)
    #    lon += list(ln)
    #    lat += list(lt)
    #lon1, lon2 = np.min(lon), np.max(lon)
    #lat1, lat2 = np.min(lat), np.max(lat)
    hdr_file = os.path.join(os.path.dirname(input_dir), 'test.hdr')
    montage.mMakeHdr(meta1_file, hdr_file)

    # reproject
    proj_dir = os.path.dirname(proj_files[0])
    safe_mkdir(proj_dir)
    stats_file = os.path.join(proj_dir, 'stats.tbl')
    montage.mProjExec(meta1_file, hdr_file, proj_dir, stats_file,
                      raw_dir=input_dir, exact=True)

    # image metadata
    meta2_file = os.path.join(proj_dir, 'meta2.tbl')
    montage.mImgtbl(proj_dir, meta2_file, corners=True)

    # Background modeling
    diff_dir = os.path.join(os.path.dirname(proj_dir), 'difference')
    safe_mkdir(diff_dir)
    diff_file = os.path.join(diff_dir, 'diffs.tbl')
    montage.mOverlaps(meta2_file, diff_file)
    montage.mDiffExec(diff_file, hdr_file, diff_dir, proj_dir)
    fits_file = os.path.join(diff_dir, 'fits.tbl')
    montage.mFitExec(diff_file, fits_file, diff_dir)

    # Background matching
    corr_dir = os.path.join(os.path.dirname(proj_dir), 'correct')
    safe_mkdir(corr_dir)
    corr_file = os.path.join(corr_dir, 'corrections.tbl')
    montage.mBgModel(meta2_file, fits_file, corr_file, level_only=False)
    montage.mBgExec(meta2_file, corr_file, corr_dir, proj_dir=proj_dir)

    # Native mosaic
    projadd_file = config.path('{:s}.reproject.add'.format(kind))
    projadd_dir, filename = os.path.split(projadd_file)
    filename, ext = os.path.splitext(filename)
    filename = '{0:s}_native{1:s}'.format(filename, ext)
    projaddnative_file = os.path.join(projadd_dir, filename)
    safe_mkdir(projadd_dir)
    montage.mAdd(meta2_file, hdr_file, projaddnative_file, img_dir=corr_dir, exact=True)

    # Reproject to final header
    header_file = config.path('{:s}.hdr'.format(kind))
    montage.mProject(projaddnative_file, projadd_file, header_file)

    # Postprocess
    data, hdr = astropy.io.fits.getdata(projaddnative_file, header=True)
    x1, x2 = 900, 1900
    y1, y2 = 3000, 4500
    val = np.mean(data[y1:y2,x1:x2])

    data, hdr = astropy.io.fits.getdata(projadd_file, header=True)
    data = data - val
    areaadd_file = config.path('{:s}.area.add'.format(kind))
    area = astropy.io.fits.getdata(areaadd_file) * (180/np.pi*3600)**2 # arcsec2
    data = data * area

    add_file = config.path('{:s}.add'.format(kind))
    dirname = os.path.dirname(add_file)
    safe_mkdir(dirname)
    if os.path.exists(add_file):
        os.remove(add_file)
    hdu = astropy.io.fits.PrimaryHDU(data, header=hdr)
    hdu.writeto(add_file)
    def mosaic_band(self,band,ra,dec,margin,radius,pgc):#,clean=True):
        '''
        Input: source info param
        Create a mosaic fit file for the specified band.
        Return: String filename of resulting mosaic
        '''
        print ("------------------mosaic_band----------------------")
        DEBUG = True
        # output = open("rc3_galaxies_outside_SDSS_footprint.txt",'a') # 'a' for append #'w')
        # unclean = open("rc3_galaxies_unclean","a")
        # filename = "{},{}".format(str(ra),str(dec))
        filename = str(ra)+str(dec)
        #print (margin/radius)
        if (DEBUG) : print ("Querying data that lies inside margin")
        #result = sqlcl.query( "SELECT distinct run,camcol,field FROM PhotoObj WHERE  ra between {0}-{1} and  {0}+{1}and dec between {2}-{3} and  {2}+{3}".format(str(ra),str(margin),str(dec),str(margin))).readlines()
        result = sqlcl.query( "SELECT distinct run,camcol,field FROM PhotoObj WHERE  ra between "+str(ra)+"-"+str(margin)+" and " +str(ra)+"+"+str(margin)+"and dec between "+str(dec)+"-"+str(margin)+" and "+ str(dec)+"+"+str(margin)).readlines()
        clean_result = sqlcl.query( "SELECT distinct run,camcol,field FROM PhotoObj WHERE  CLEAN =1 and ra between "+str(ra)+"-"+str(margin)+" and " +str(ra)+"+"+str(margin)+"and dec between "+str(dec)+"-"+str(margin)+" and "+ str(dec)+"+"+str(margin)).readlines()
        # clean_result = sqlcl.query( "SELECT distinct run,camcol,field FROM PhotoObj WHERE  CLEAN =1 and ra between {0}-{1} and  {0}+{1}and dec between {2}-{3} and  {2}+{3}".format(str(ra),str(margin),str(dec),str(margin))) .readlines()
        clean = True
        print (result)
        print (clean_result)
        if (result[0][5:]=="<html>"):
            print("strange error from SQL server")
            return -1
        if (result[1]=='error_message\n' or clean_result[1]=='error_message\n'):
    	    #Case where doing more than 60 queries in 1 minute
            time.sleep(60)
            #results are messed up, need to re-query
            result = sqlcl.query( "SELECT distinct run,camcol,field FROM PhotoObj WHERE  ra between "+str(ra)+"-"+str(margin)+" and " +str(ra)+"+"+str(margin)+"and dec between "+str(dec)+"-"+str(margin)+" and "+ str(dec)+"+"+str(margin)).readlines()
            clean_result = sqlcl.query( "SELECT distinct run,camcol,field FROM PhotoObj WHERE  CLEAN =1 and ra between "+str(ra)+"-"+str(margin)+" and " +str(ra)+"+"+str(margin)+"and dec between "+str(dec)+"-"+str(margin)+" and "+ str(dec)+"+"+str(margin)).readlines()
        if (len(result)!=len(clean_result) and band=='u'):
            #only print this once in the u band. If it is unclean in u band (ex. cosmic ray, bright star..etc) then it must be unclean in the other bands too.
            print ("Data contain unclean images")
            clean=False
            unclean.write(str(ra)+"     "+str(dec)+"     "+str(radius)+"     "+pgc)
            # unclean.write("{}     {}     {}     {} \n".format(str(ra),str(dec),str(radius),pgc))    
        data =[]
        count =0
        for i in result:
            if count>1:
                list =i.split(',')
                list[2]= list[2][:-1]
                data.append(list)
            count += 1 
        print (data)
        if (len(data)==0 and band=='r'): #you will only evounter non-footprint galaxy inint run , because after that we just take the footprint gaalxy already mosaiced (init) from rfits
            if (DEBUG): print ('The given ra, dec of this galaxy does not lie in the SDSS footprint. Onto the next galaxy!')#Exit Program.'
            output.write(str(ra)+ "     "+ str(dec)+"     "+str(radius)+"\n")
            # output.write("{}     {}     {}     {} \n".format(str(ra),str(dec),str(radius),pgc))
            output.write(str(ra)+"     "+str(dec)+"     "+str(radius)+"     "+pgc)
            #sys.exit()
            return -1 #special value reserved for not in SDSS footprint galaxies
        else :
            if (DEBUG): 
                print ( "Complete Query. These data lies within margin: ")
                print (data)
        # os.mkdir(filename)
        # os.chdir(filename)
        #if (os.path.exists(band)):
	    #os.system("rm -r "+band)
        os.mkdir(band)
        os.chdir(band)
        os.mkdir ("raw")
        os.mkdir ("projected")
        os.chdir("raw")
        if (DEBUG): print ("Retrieving data from SDSS SAS server for "+ band +"band")
        for i in data :  
            out = "frame-"+str(band)+"-"+str(i[0]).zfill(6)+"-"+str(i[1])+"-"+str(i[2]).zfill(4)
            os.system("wget http://mirror.sdss3.org/sas/dr10/boss/photoObj/frames/301/"+str(i[0])+"/"+str(i[1])+"/"+out+".fits.bz2")
            os.system("bunzip2 "+out+".fits.bz2")
        os.chdir("../")
        if (DEBUG) : print("Creating mosaic for "+band+" band.")
        outfile_r="SDSS_"+band+"_"+str(ra)+"_"+str(dec)+"r.fits"
        outfile="SDSS_"+band+"_"+str(ra)+"_"+str(dec)+".fits"
        if (len(data)==1):
    	    #With header info, len of processed result list is 1 if there is only 1 field lying in the margin, simply do mSubImage without mosaicing
    	    #This patch should not be necessary but the program is aparently not mosaicing for the case where there is only one field.
            print ("Only one field in region of interest")
            os.chdir("raw")
            montage.mSubimage(out+".fits",outfile,ra,dec,2*margin) # mSubImage takes xsize which should be twice the margin (margin measures center to edge of image)
            #os.chdir("../..")
            hdulist = pyfits.open(outfile)
            shutil.move(outfile,"../..")
            os.chdir("../..")
        else:
            montage.mImgtbl("raw","images.tbl")
            montage.mHdr(str(ra)+" "+str(dec),margin,out+".hdr")
            if (DEBUG): print ("Reprojecting images")
            os.chdir("raw")
            montage.mProjExec("../images.tbl","../"+out+".hdr","../projected", "../stats.tbl") 
            os.chdir("..")
            montage.mImgtbl("projected","pimages.tbl")
            os.chdir("projected")
            montage.mAdd("../pimages.tbl","../"+out+".hdr","SDSS_"+out+".fits")
            # outfile_r="SDSS_{}_{}_{}r.fits".format(band,str(ra),str(dec))
            #outfile_r="SDSS_"+band+"_"+str(ra)+"_"+str(dec)+"r.fits"
            montage.mSubimage("SDSS_"+out+".fits",outfile_r,ra,dec,2*margin) # mSubImage takes xsize which should be twice the margin (margin measures center to edge of image)
            shutil.move(outfile_r,os.getcwd()[:-11] )#if change to :-11 then move out of u,g,r,i,z directory, may be more convenient for mJPEG
            if (DEBUG) : print ("Completed Mosaic for " + band)
            os.chdir("../..")
            hdulist = pyfits.open(outfile_r)
        hdulist[0].header['RA']=ra
        hdulist[0].header['DEC']=dec
        hdulist[0].header['RADIUS']=radius
        hdulist[0].header['PGC']=pgc
        hdulist[0].header['NED']=("http://ned.ipac.caltech.edu/cgi-bin/objsearch?objname="+ str(hdulist[0].header['PGC'])+"&extend=no&hconst=73&omegam=0.27&omegav=0.73&corr_z=1&out_csys=Equatorial&out_equinox=J2000.0&obj_sort=RA+or+Longitude&of=pre_text&zv_breaker=30000.0&list_limit=5&img_stamp=YES")
        hdulist[0].header['CLEAN']=clean
        hdulist[0].header['MARGIN']=margin
        
        #if (os.path.exists(outfile)):
            #os.system("rm "+ outfile)
        hdulist.writeto(outfile)
        if (os.path.exists(outfile_r)):
            os.system("rm "+outfile_r)
        #print("Deleting")
        os.system("rm -r "+band+"/")
        print ("Completed Mosaic")
        return outfile 
    os.chdir("raw")
    if (DEBUG): print ("Retrieving data from SDSS SAS server for "+ band +"band")
    for i in data :  
        out = "frame-"+str(band)+"-"+str(i[0]).zfill(6)+"-"+str(i[1])+"-"+str(i[2]).zfill(4)
        os.system("wget http://data.sdss3.org/sas/dr10/boss/photoObj/frames/301/"+str(i[0])+"/"+ str(i[1]) +"/"+out+".fits.bz2")
        os.system("bunzip2 "+out+".fits.bz2")
    # print (os.getcwd())
    os.chdir("../")
    if (DEBUG) : print("Creating mosaic for " +" "+ band + " band.")
    montage.mImgtbl("raw","images.tbl")
    montage.mHdr(str(ra)+" "+str(dec),radius,out+".hdr")
    if (DEBUG): print ("Reprojecting images")
	#Sometimes you can't find the files and result in images.tbl => empty doc
	#need to put data file inside raw AND unzip it so that Montage detect that it is a fit file
    os.chdir("raw")
    montage.mProjExec("../images.tbl","../"+out+".hdr","../projected", "../stats.tbl") 
    os.chdir("..")
    montage.mImgtbl("projected","pimages.tbl")
	#mAdd coadds the reprojected images using the FITS header template and mImgtbl list.
    os.chdir("projected")
    montage.mAdd("../pimages.tbl","../"+out+".hdr","SDSS_"+out+".fits")
    montage.mSubimage("SDSS_"+out+".fits","SDSS_"+ele+"_"+str(trunc(ra))+"_"+str(trunc(dec))+".fits",ra,dec,2*margin) # mSubImage takes xsize which should be twice the margin (margin measures center to edge of image)
    shutil.move("SDSS_"+ele+"_"+str(trunc(ra))+"_"+str(trunc(dec))+".fits",os.getcwd()[:-11] )#if change to :-11 then move out of u,g,r,i,z directory, may be more convenient for mJPEG
    if (DEBUG) : print ("Completed Mosaic for " + band)
    os.chdir("../..")
# Superimposing R,G,B image mosaics into TIFF using STIFF
os.system("stiff "+" SDSS_i_"+str(trunc(ra))+"_"+str(trunc(dec))+ ".fits "+ " SDSS_r_"+str(trunc(ra))+"_"+str(trunc(dec))+ ".fits "+" SDSS_g_"+str(trunc(ra))+"_"+str(trunc(dec))+ ".fits "+ "  -c stiff.conf  " +"  -OUTFILE_NAME  "+str(trunc(ra))+"_"+str(trunc(dec))+"_COLORSAT_5_MAX_MAN_3  -COLOUR_SAT  5 -MAX_TYPE MANUAL -MAX_LEVEL 3")
# for b in bands:
# 	os.system("rm -r "+b+"/")
	#we want to keep the fit files, but for testing purposes Python will throw file-already-exist error , if we dont delete them.
	#os.system("rm -r " + "SDSS_frame-"+b+"-"+str(run).zfill(6)+"-"+str(camcol)+"-"+str(field).zfill(4)+ ".fits" )
Beispiel #8
0
			hdr_out.close()
			os.mkdir(bands)
			os.mkdir(bands+'/raw')
			coor = ascii.read(list_dir+'list_'+bands+'.txt')
				
			for icoor in range(len(coor)):
				if (l_0+1.5 > coor['l'][icoor]/10.0 and l_0-1.5 < coor['l'][icoor]/10.0):
					shutil.copy(data_dir+coor['Name'][icoor], 
						bands+'/raw/'+coor['Name'][icoor])
			
			os.chdir(bands)
			os.mkdir('projected')
			os.mkdir('diffdir')
			os.mkdir('corrdir')
			mt.mImgtbl('raw','rimages.tbl')
			mt.mProjExec('rimages.tbl', '../'+bands+'.hdr', 'projected', 'stats.tbl',
				raw_dir='raw')
			mt.mImgtbl('projected', 'pimages.tbl')
			len_dir = len(os.listdir('projected'))
			if len_dir < 3 :
				mt.mAdd('pimages.tbl', '../'+bands+'.hdr', 
					'../'+sour_name+'_'+bands+'.fits', 
					img_dir='projected')
			else:
				mt.mOverlaps('pimages.tbl', 'diffs.tbl')
				mt.mDiffExec('diffs.tbl',  '../'+bands+'.hdr', 'diffdir', 
					proj_dir = 'projected')
				mt.mFitExec('diffs.tbl', 'fits.tbl', 'diffdir')
				if ((len(os.listdir("diffdir")) > 1 and 
					os.path.getsize("diffdir/"+os.listdir("diffdir")[1]) < 10000) or 
					(len(os.listdir("diffdir")) < 1)) :
					listPro = os.listdir('projected')
Beispiel #9
0
    def mosaic_band(self,band,ra,dec,margin,radius,pgc,survey,remove_bkgrd=False):
        '''
        Input: source info param
        Create a mosaic fit file for the specified band.
        Return: String filename of resulting mosaic
        '''
        print ("------------------mosaic_band----------------------")
        print ("Now mosaic_band on {}".format(pgc))
        output = open("../rc3_galaxies_outside_{}_footprint".format(survey.name),'a') # 'a' for append #'w')
        unclean = open("../rc3_galaxies_unclean_{}".format(survey.name),"a")
        filename = str(ra)+str(dec)
        if (DEBUG) : print ("Querying data that lies inside margin")
        print (ra,dec,margin)
        result = survey.data_server.surveyFieldConverter(float(ra),float(dec),float(margin))
        clean_result = survey.data_server.surveyFieldConverter(float(ra),float(dec),float(margin),True)
        clean = True
        if(DEBUG):print ("result: "+str(result))
        if(DEBUG):print ("clean_result: "+str(clean_result))
        
        if (len(result)!=len(clean_result)and band=='u'):
            # Only print this once in the u band. 
            # Assume that if it is unclean in u band (ex. cosmic ray, bright star..etc) then it must be unclean in the other bands too.
            print ("Data contain unclean images")
            clean=False
            unclean.write("{}     {}     {}     {} \n".format(self.rc3_ra,self.rc3_dec,self.rc3_radius,self.pgc))

        if (len(result)==0):             
            if (DEBUG): print ('The given ra, dec of this galaxy does not lie in the survey footprint. Onto the next galaxy!')#Exit Program.'
            output.write("{}     {}     {}     {} \n".format(str(ra),str(dec),str(radius),str(pgc)))
            return -1 #special value reserved for not in survey footprint galaxies
        else :
            if (DEBUG): 
                print ( "Complete Query. These data lies within margin: ")
                print (result)

        os.mkdir(band)
        os.chdir(band)
        os.mkdir ("rawdir")
        os.mkdir ("projdir")
        if (remove_bkgrd):
            os.mkdir ("diffdir")
            os.mkdir ("corrdir")
            # os.mkdir("final")

        if (DEBUG): print ("Retrieving data from server for "+ band +"band")
        os.chdir("rawdir")
        out=""
        # Raw Imaging Data naming
        for i in result :  
            if (survey.data_server.name=='Gator'):
                survey.data_server.getData(band,ra,dec,margin,survey)
                out = i # 2MASS designation
                print out
            elif (survey.data_server.name=='SkyServer'):
                survey.data_server.getData(band,str(i[0]), str(i[1]),str(i[2]))
                # run-camcol-field
                out = "frame-"+str(band)+"-"+str(i[0]).zfill(6)+"-"+str(i[1])+"-"+str(i[2]).zfill(4)
            elif (survey.data_server.name=='DSSServer'):
                survey.data_server.getData(band,ra,dec,margin)
                # Patch for when we can not pass in th pgc number in getData of dssServer class, we rename the file here to conform with RC3's filename expectation for the imaging data
                raw_data = glob.glob("raw_*.fits")
                print (raw_data)
                for i in raw_data:
                    os.rename(i,"DSS_{}_{}.fits".format(band, self.pgc))    
                out = "raw_{}_{}".format(band,self.pgc)
                print ("dss_out: "+out)
            else:
                raise TypeError("Missing implementation for data retrieval")
        os.chdir("../")
        if (DEBUG) : print("Creating mosaic for "+band+" band.")
        outfile_r = "{}_{}_{}r.fits".format(survey.name,band,self.pgc)
        outfile = "{}_{}_{}.fits".format(survey.name,band,self.pgc)
        if (len(result)==1):
            #With header info, len of processed result list is 1 if there is only 1 field lying in the margin, simply do mSubImage without mosaicing
            print ("Only one field in region of interest")
            os.chdir("rawdir")
            if (DEBUG):print ("m:{}".format(margin))
            try:
                if (DEBUG):print ("2m:{}".format(2*margin))
                if (DEBUG):print ([outfile_r,outfile,ra,dec,2*margin])
                montage.mSubimage(outfile,outfile,ra,dec,2*margin) # mSubImage takes xsize which should be twice the margin (margin measures center to edge of image)
            except(montage.status.MontageError):
                print ("montage_wrapper.status.MontageError: mSubimage: Region outside image.")
                try :#give it one last chance
                    if (DEBUG):print ("lastchancem:{}".format(margin))
                    montage.mSubimage(out+".fits",outfile,ra,dec,margin)
                except(montage.status.MontageError):
                    print("Doesn't work after trying half the margin, just keep the raw FITS file")
                    if (DEBUG):print (out+".fits")
                    if (DEBUG):print (outfile)
                    shutil.move(out+".fits","../..")
                    os.chdir("../../")
                    os.rename(out+".fits",outfile)
                    os.system("rm -r {}".format(survey.best_band))
                    return outfile
                if (DEBUG):print (os.getcwd())
                os.chdir("../../") #Get out of directory for that galaxy and move on
                os.system("rm -r {}".format(survey.best_band))
                if (DEBUG):print(os.getcwd())
                failed_msubimage = open ("failed_msubimage","a")
                failed_msubimage.write("{}     {}     {}     {} \n".format(str(ra),str(dec),str(radius),str(pgc)))
                return -1 # masking with special value reserved for not in survey footprint galaxies
        
            hdulist = pyfits.open(outfile)
            if (os.path.exists("../../"+outfile)):
                os.system("rm ../../"+outfile)
            shutil.move(outfile,"../..")
            os.chdir("../..")
        else:
            imgtbl="images-rawdir.tbl"
            hdr="template.hdr"
            montage.mImgtbl("rawdir",imgtbl)
            # montage.mHdr(str(ra)+" "+str(dec),margin,out+".hdr")
            montage.mMakeHdr(imgtbl,hdr)
            if (DEBUG): print ("Reprojecting images")
            # os.chdir("rawdir")
            if (DEBUG):print(os.getcwd())
            montage.mProjExec(imgtbl,hdr,"projdir", "stats.tbl",raw_dir="rawdir")#, mpi=enable_mpi,debug=True) 
            if os.listdir("projdir") == []: 
                print "Projection Failed. No projected images produced. Skip to the next galaxy" 
                os.chdir("../") #Get out of directory for that galaxy and move on
                os.system("rm -r {}".format(survey.best_band))
                failed_projection = open ("failed_projection","a")
                failed_projection.write("{}     {}     {}     {} \n".format(str(ra),str(dec),str(radius),str(pgc)))
                return -1 # masking with special value reserved for not in survey footprint galaxies
            if (remove_bkgrd): 
                if (DEBUG): print "Calling the bash script containing Montage routines to rectify the background" 
                if os.getcwd()[-4:-2]==str(pgc):
                    os.system("bash ../../mosaic.sh")
                else:
                    os.system("bash ../mosaic.sh")
                print "mSubimage"
                montage.mSubimage("mosaic.fits" ,"mosaic.fits",ra,dec,2*margin) # mSubImage takes xsize which should be twice the margin (margin measures center to edge of image)
                shutil.move("mosaic.fits","../{}".format(outfile_r))#if change to :-11 then move out of u,g,r,i,z directory, may be more convenient for mJPEG
                if (DEBUG) : print ("Completed Mosaic for " + band)
            else:
                montage.mImgtbl("projdir","pimages.tbl")
                os.chdir("projdir")
                montage.mAdd("../pimages.tbl","../"+hdr,"{}_{}.fits".format(survey.name,out))#, mpi=enable_mpi)
                montage.mSubimage("{}_{}.fits".format(survey.name,out),outfile_r,ra,dec,2*margin) # mSubImage takes xsize which should be twice the margin (margin measures center to edge of image)
                shutil.move(outfile_r,"../../{}".format(outfile_r) )#if change to :-11 then move out of u,g,r,i,z directory, may be more convenient for mJPEG
                if (DEBUG) : print ("Completed Mosaic for " + band)
                os.chdir("..")
            os.chdir("../")
            hdulist = pyfits.open(outfile_r)

        hdulist[0].header['RA']=float(ra)
        hdulist[0].header['DEC']=float(dec)
        hdulist[0].header['RADIUS']=radius
        if (DEBUG):print ("Finished mosaic_band on {}".format(pgc))
        hdulist[0].header['PGC']=pgc
        hdulist[0].header['NED']=("http://ned.ipac.caltech.edu/cgi-bin/objsearch?objname="+ str(hdulist[0].header['PGC'])+"&extend=no&hconst=73&omegam=0.27&omegav=0.73&corr_z=1&out_csys=Equatorial&out_equinox=J2000.0&obj_sort=RA+or+Longitude&of=pre_text&zv_breaker=30000.0&list_limit=5&img_stamp=YES")
        hdulist[0].header['CLEAN']=clean
        hdulist[0].header['MARGIN']=margin

        if (os.path.exists(outfile)):
            os.system("rm "+ outfile)
        hdulist.writeto(outfile)
        if (os.path.exists(outfile_r)):
            os.system("rm "+outfile_r)
        os.system("rm -r {}".format(band))
        return outfile 
def re_project(orig_dir,
               Name_f0,
               proj_dir,
               Name_f1,
               para,
               exact=True,
               clear_files=True):
    '''
    To re-project fits files using montage_wrapper.

    Inputs
        orig_dir: directory of fits to be re-projected, should omit 
                  unnecessary '/'
        Name_f0: list of fits names before re-projection
        proj_dir: directory of re-projected fits, DO NOT LET proj_dir=orig_dir!
        Name_f1: list of fits names after re-projection, ordered like Name_f0
        para: list of parameters defining the framework of re-projected image,
              which includes:

                NAXIS1: integer
                NAXIS2: integer
                projection: 3-letter string, e.g. 'TAN'
                CRVAL1
                CRVAL2
                CRPIX1
                CRPIX2
                CD1_1
                CD1_2
                CD2_1
                CD2_2

        exact: whether the output shape exactly match the FITS header
        clear_files: whether to delete intermediate files
    
    Outputs
        re-projected fits in proj_dir

    Caveats
        1. This code is still in test.
        2. Only tested for 2D fits with one hdu.
        3. It's quite often that "exact=True" doesn't work, try to use mAdd. 
            Sometimes mAdd gives fits with zero file sizes. Maybe mAdd can't 
            handle single fits.
    '''

    # create a folder containing the original fits
    raw_dir = orig_dir + '/fits_orig'
    os.mkdir(raw_dir)
    for name_f in Name_f0:
        shutil.copy('%s/%s' % (orig_dir, name_f), raw_dir)

    # make images_table
    images_table = raw_dir + '/images_table.txt'
    mw.mImgtbl(raw_dir, images_table)

    # create header file
    (NAXIS1, NAXIS2, projection, CRVAL1, CRVAL2, CRPIX1, CRPIX2, CD1_1, CD1_2,
     CD2_1, CD2_2) = para
    f = open(raw_dir + '/header.txt', 'w')
    f.write('SIMPLE  = T\n')
    f.write('BITPIX  = -64\n')
    f.write('BUNIT  = none\n')
    f.write('NAXIS   = 2\n')
    f.write('NAXIS1  = %d\n' % NAXIS1)
    f.write('NAXIS2  = %d\n' % NAXIS2)
    f.write("CTYPE1  = 'RA---%s'\n" % projection)
    f.write("CTYPE2  = 'DEC--%s'\n" % projection)
    f.write('CRPIX1  = %d\n' % CRPIX1)
    f.write('CRPIX2  = %d\n' % CRPIX2)
    f.write('CRVAL1  = %f\n' % CRVAL1)
    f.write('CRVAL2  = %f\n' % CRVAL2)
    f.write('CD1_1   = %f\n' % CD1_1)
    f.write('CD1_2   = %f\n' % CD1_2)
    f.write('CD2_1   = %f\n' % CD2_1)
    f.write('CD2_2   = %f\n' % CD2_2)
    f.write('HISTORY =  By Yue Cao\n')
    f.write('END')
    f.close()

    # re-project
    stats_table = raw_dir + '/stats_table.txt'
    mw.mProjExec(images_table=images_table,
                 template_header=raw_dir + '/header.txt',
                 raw_dir=raw_dir,
                 proj_dir=proj_dir,
                 stats_table=stats_table,
                 exact=exact)

    # delete intermediate files
    if clear_files:
        shutil.rmtree(raw_dir)
        for name_f in Name_f0:
            n_f = name_f.split('.')[0]
            os.remove('%s/hdu0_%s_area.fits' % (proj_dir, n_f))

    # rename the re-projected fits
    for i in range(len(Name_f0)):
        os.rename('%s/hdu0_%s' % (proj_dir, Name_f0[i]),
                  '%s/%s' % (proj_dir, Name_f1[i]))

# def re_project(orig_dir,proj_dir,header,exact=True,clear_files=True):
    '''