def build_hsc_sample(sql_file,
out_dir,
tmp_dir,
filters='i',
cutout_size=10.0, # in arcsec
data_release='pdr2',
rerun='pdr2_wide',
nproc=2):
"""
This function runs an sql query to extract a catalog, then
proceeds to download cutouts in requested bands for all
catalog entries.
"""
# Creates output directory if doesn't already exists
if not os.path.exists(out_dir):
os.makedirs(out_dir)
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
# Create archive instance and login
archive = hsc.Hsc(dr=data_release, rerun=rerun)
catalog_filename = os.path.join(out_dir, 'catalog.fits')
if os.path.isfile(catalog_filename):
print("Readding hsc catalog from %s"%catalog_filename)
catalog = Table.read(catalog_filename)
else:
# Query the database
catalog = archive.sql_query(sql_file,
from_file=True,
verbose=True)
print("Saving hsc catalog to %s"%catalog_filename)
catalog.write(catalog_filename)
# Query corresponding postage stamps
cutouts_filename = task.hsc_bulk_cutout(catalog,
cutout_size=cutout_size* u.Unit('arcsec'),
filters=filters,
archive=archive,
nproc=nproc,
tmp_dir=tmp_dir,
output_dir=out_dir)
# Rename the cutout file
shutil.move(cutouts_filename, os.path.join(out_dir, 'cutouts.hdf'))
#!/usr/bin/env python
import os, sys, time
import numpy as np
import astropy.units as u
from astropy import wcs
from astropy.io import fits
from astropy.coordinates import SkyCoord
from astropy.utils.data import download_file, clear_download_cache
from unagi import config
from unagi import hsc
from unagi import plotting
from unagi.task import hsc_cutout
pdr2 = hsc.Hsc(dr='pdr2', rerun='pdr2_wide')
cutout_width = 525 # pixels
arcsec_per_pixel = 0.168 # arcsec/pixel
s_ang = cutout_width / 2 * arcsec_per_pixel * u.arcsec # arcsec
# Catalogue
catalogue = '/Users/cbottrell/Project/HSC-Subaru/Catalogues/HSC-TF_all_2019-07-16.txt'
output_dir = '/Volumes/Project_Data/HSC_Subaru/Input/'
# filename prefix
prefix = '{}_Cutout-{}x{}'
cat_data = np.loadtxt(catalogue, delimiter=',', dtype='str')
objIDs = cat_data[:, 0].astype(int)
ras = cat_data[:, 2].astype(float)
decs = cat_data[:, 3].astype(float)
zs = cat_data[:, 1].astype(float)
def download_hsc_large(ra, dec, band, size=0.7*u.deg, radius=0.5*u.deg, verbose=True,
output_dir='./', output_name='HSC', overwrite=True):
'''Download HSC patches and stitch them together using ``swarp``. Hence ``swarp`` must be installed!
``swarp`` resamples the image, but doesn't correct background.
Parameters:
ra (float): RA of the object.
dec (float): DEC of the object.
band (string): such as 'r' or 'g'.
size (``astropy.units`` object): size of cutout, it should be comparable to ``radius``.
radius (``astropy.units`` object): bricks whose distances to the object are
nearer than this radius will be download.
output_dir (str): directory of output files.
output_name (str): prefix of output images. The suffix `.fits` will be appended automatically.
overwrite (bool): overwrite files or not.
Return:
None
'''
import urllib
from astropy.coordinates import SkyCoord
from .utils import save_to_fits
import subprocess
## Setup ``unagi``
try:
from unagi import hsc
except:
raise ImportError('`unagi` (https://github.com/dr-guangtou/unagi) must be installed to download HSC data!')
pdr2 = hsc.Hsc(dr='pdr2', rerun='pdr2_wide')
## Download survey-summary
URL = 'https://github.com/AstroJacobLi/slug/raw/master/demo/HSC_tracts_patches_pdr2_wide.fits'
if os.path.isfile('_survey_summary_hsc.fits'):
os.remove('_survey_summary_hsc.fits')
urllib.request.urlretrieve(URL, filename='_survey_summary_hsc.fits', data=None)
# Find nearby bricks
patch_cat = Table.read('_survey_summary_hsc.fits', format='fits')
patch_sky = SkyCoord(ra=np.array(patch_cat['ra_cen']),
dec=np.array(patch_cat['dec_cen']), unit='deg')
object_coord = SkyCoord(ra, dec, unit='deg')
flag = patch_sky.separation(object_coord) <= radius
to_download = patch_cat[flag]
distance = patch_sky.separation(object_coord)[flag]
to_download.add_column(Column(data=distance.value, name='distance'))
to_download.sort('distance')
print('# You have {} patches to be downloaded.'.format(len(to_download)))
filenameset = []
for obj in to_download:
file = 'calexp_{0}_{1}_{2}.fits'.format(obj['tract'], obj['patch'].replace(',', '_'), band)
filenameset.append(os.path.join(output_dir, file))
if not os.path.isfile(file):
pdr2.download_patch(obj['tract'], obj['patch'], filt='HSC-{}'.format(band.upper()), output_file=file);
hdu = fits.open(os.path.join(output_dir, file))
img = hdu[1].data
hdr = hdu[1].header
mask = hdu[2].data
mask_hdr = hdu[2].header
hdr['XTENSION'] = 'IMAGE'
mask_hdr['XTENSION'] = 'IMAGE'
hdu.close()
save_to_fits(img, os.path.join(output_dir, file), header=hdr);
save_to_fits(img, os.path.join(output_dir, file), header=hdr);
# Calculating image size in pixels
imgsize = int(size.to(u.arcsec).value / HSC_pixel_scale)
# Configure ``swarp``
with open("config_swarp.sh","w+") as f:
# check if swarp is installed
f.write('for cmd in SWarp; do\n')
f.write('\t hasCmd=$(which ${cmd} 2>/dev/null)\n')
f.write('\t if [[ -z "${hasCmd}" ]]; then\n')
f.write('\t\t echo "This script requires ${cmd}, which is not in your \$PATH." \n')
f.write('\t\t exit 1 \n')
f.write('\t fi \n done \n\n')
# Write ``default.swarp``.
f.write('/bin/rm -f default.swarp \n')
f.write('cat > default.swarp <<EOT \n')
f.write('IMAGEOUT_NAME \t\t {}.fits # Output filename\n'.format(os.path.join(output_dir, '_'.join([output_name, band]))))
f.write('WEIGHTOUT_NAME \t\t {}_weights.fits # Output weight-map filename\n\n'.format(os.path.join(output_dir, '_'.join([output_name, band]))))
f.write('HEADER_ONLY N # Only a header as an output file (Y/N)?\nHEADER_SUFFIX .head # Filename extension for additional headers\n\n')
f.write('#------------------------------- Input Weights --------------------------------\n\nWEIGHT_TYPE NONE # BACKGROUND,MAP_RMS,MAP_VARIANCE\n # or MAP_WEIGHT\nWEIGHT_SUFFIX weight.fits # Suffix to use for weight-maps\nWEIGHT_IMAGE # Weightmap filename if suffix not used\n # (all or for each weight-map)\n\n')
f.write('#------------------------------- Co-addition ----------------------------------\n\nCOMBINE Y # Combine resampled images (Y/N)?\nCOMBINE_TYPE MEDIAN # MEDIAN,AVERAGE,MIN,MAX,WEIGHTED,CHI2\n # or SUM\n\n')
f.write('#-------------------------------- Astrometry ----------------------------------\n\nCELESTIAL_TYPE NATIVE # NATIVE, PIXEL, EQUATORIAL,\n # GALACTIC,ECLIPTIC, or SUPERGALACTIC\nPROJECTION_TYPE TAN # Any WCS projection code or NONE\nPROJECTION_ERR 0.001 # Maximum projection error (in output\n # pixels), or 0 for no approximation\nCENTER_TYPE MANUAL # MANUAL, ALL or MOST\n')
f.write('CENTER {0}, {1} # Image Center\n'.format(ra, dec))
f.write('PIXELSCALE_TYPE MANUAL # MANUAL,FIT,MIN,MAX or MEDIAN\n')
f.write('PIXEL_SCALE {} # Pixel scale\n'.format(HSC_pixel_scale))
f.write('IMAGE_SIZE {0},{1} # scale = 0.262 arcsec/pixel\n\n'.format(imgsize, imgsize))
f.write('#-------------------------------- Resampling ----------------------------------\n\nRESAMPLE Y # Resample input images (Y/N)?\nRESAMPLE_DIR . # Directory path for resampled images\nRESAMPLE_SUFFIX .resamp.fits # filename extension for resampled images\n\nRESAMPLING_TYPE LANCZOS3 # NEAREST,BILINEAR,LANCZOS2,LANCZOS3\n # or LANCZOS4 (1 per axis)\nOVERSAMPLING 0 # Oversampling in each dimension\n # (0 = automatic)\nINTERPOLATE N # Interpolate bad input pixels (Y/N)?\n # (all or for each image)\n\nFSCALASTRO_TYPE FIXED # NONE,FIXED, or VARIABLE\nFSCALE_KEYWORD FLXSCALE # FITS keyword for the multiplicative\n # factor applied to each input image\nFSCALE_DEFAULT 1.0 # Default FSCALE value if not in header\n\nGAIN_KEYWORD GAIN # FITS keyword for effect. gain (e-/ADU)\nGAIN_DEFAULT 0.0 # Default gain if no FITS keyword found\n\n')
f.write('#--------------------------- Background subtraction ---------------------------\n\nSUBTRACT_BACK N # Subtraction sky background (Y/N)?\n # (all or for each image)\n\nBACK_TYPE AUTO # AUTO or MANUAL\n # (all or for each image)\nBACK_DEFAULT 0.0 # Default background value in MANUAL\n # (all or for each image)\nBACK_SIZE 128 # Background mesh size (pixels)\n # (all or for each image)\nBACK_FILTERSIZE 3 # Background map filter range (meshes)\n # (all or for each image)\n\n')
f.write('#------------------------------ Memory management -----------------------------\n\nVMEM_DIR . # Directory path for swap files\nVMEM_MAX 2047 # Maximum amount of virtual memory (MB)\nMEM_MAX 2048 # Maximum amount of usable RAM (MB)\nCOMBINE_BUFSIZE 1024 # Buffer size for combine (MB)\n\n')
f.write('#------------------------------ Miscellaneous ---------------------------------\n\nDELETE_TMPFILES Y # Delete temporary resampled FITS files\n # (Y/N)?\nCOPY_KEYWORDS OBJECT # List of FITS keywords to propagate\n # from the input to the output headers\nWRITE_FILEINFO Y # Write information about each input\n # file in the output image header?\nWRITE_XML N # Write XML file (Y/N)?\nXML_NAME swarp.xml # Filename for XML output\nVERBOSE_TYPE QUIET # QUIET,NORMAL or FULL\n\nNTHREADS 0 # Number of simultaneous threads for\n # the SMP version of SWarp\n # 0 = automatic \n')
f.write('EOT\n')
f.write('SWarp ' + ' '.join(filenameset) + '\n\n')
#f.write('rm ' + os.path.join(output_dir, '_*'))
f.close()
filename = '{}.fits'.format(os.path.join(output_dir, '_'.join([output_name, band])))
if os.path.isfile(filename):
if not overwrite:
raise FileExistsError('# ' + filename + ' already exists!')
else:
os.system('/bin/bash config_swarp.sh')
else:
os.system('/bin/bash config_swarp.sh')
print('# The image is save as ' + filename)
def download_highres(lowres_dir, high_res='hsc', band='g', overwrite=False):
""" Download high resolution image which overlaps with the given low resolution image.
This could be **TIME CONSUMING**! Typically one frame could be 500M to 2G.
You also need to install ``unagi`` (https://github.com/dr-guangtou/unagi) to download HSC images.
Parameters:
lowres_dir (string): the directory of input low-resolution image.
high_res (string): name of high-resolution survey, now support 'HSC' and 'CFHT'.
band (string): name of filter, typically 'g' or 'r'.
overwrite (bool): it will overwrite the image if `overwrite=True`.
Returns:
None
"""
from astropy.coordinates import SkyCoord
import astropy.units as u
from astropy import wcs
hdu = fits.open(lowres_dir)
img = hdu[0].data
header = hdu[0].header
w = wcs.WCS(header)
hdu.close()
# Calculate the (diagnal) size of image in degrees
c1 = SkyCoord(float(w.wcs_pix2world(0, 0, 0)[0]),
float(w.wcs_pix2world(0, 0, 0)[1]),
frame='icrs', unit='deg')
c2 = SkyCoord(float(w.wcs_pix2world(img.shape[1], img.shape[0], 0)[0]),
float(w.wcs_pix2world(img.shape[1], img.shape[0], 0)[1]),
frame='icrs', unit='deg')
c_cen = SkyCoord(float(w.wcs_pix2world(img.shape[1]//2, img.shape[0]//2, 0)[0]),
float(w.wcs_pix2world(img.shape[1]//2, img.shape[0]//2, 0)[1]),
frame='icrs', unit='deg')
print('# The center of input image is: ', c_cen.ra, c_cen.dec)
radius = c1.separation(c2).to(u.degree)
print('# The diagnal size of input low-resolution image is ' + str(radius))
return
if high_res.lower() == 'hsc':
print(radius)
if radius / 1.414 > 2116 * u.arcsec:
raise ValueError('# Input image size is too large for HSC! Try other methods!')
try:
from unagi import hsc
from unagi.task import hsc_cutout, hsc_tricolor, hsc_check_coverage
except:
raise ImportError('You should install `unagi` https://github.com/dr-guangtou/unagi to download HSC images! ')
# Setup HSC server
pdr2 = hsc.Hsc(dr='pdr2', rerun='pdr2_wide')
#pdr2 = hsc.Hsc(dr='dr2', rerun='s18a_wide')
# Check if it is within the footprint
cover = hsc_check_coverage(c_cen, archive=pdr2, verbose=False, return_filter=True)
if len(cover) == 0:
raise ValueError('# This region is NOT in the footprint of HSC PDR2!')
# Angular size (must with unit!)
else:
if os.path.isfile('hsc_cutout_' + band):
if not overwrite:
raise FileExistsError('# hsc_cutout_' + band + 'already exists!')
else:
cutout = hsc_cutout(c_cen,
cutout_size=radius / np.sqrt(2) / 2,
filters=band,
archive=pdr2,
use_saved=False,
mask=False,
variance=False,
output_dir='./',
prefix='hsc_cutout',
verbose=True,
save_output=True)
cutout.close()
elif high_res.lower() == 'cfht':
get_megapipe_catalog(c_cen.ra.value, c_cen.dec.value, radius / 2)
download_cfht_megapipe(img, header, band=band.upper(), output_dir='./', overwrite=overwrite)
else:
raise ValueError('# This survey is not supported yet. Please use "HSC" or "CFHT"!')
return