def bg_model(reprojected_dir,
bg_model_dir,
diff_dir,
corr_dir,
template_header,
im_type='intbgsub',
level_only=True):
"""
Model the background for the mosaiced image
Parameters
----------
reprojected_dir : str
Path to temp directory containing reprojected images
bg_model_dir : str
Path to directory inside gal_dir to hold the background modeling information
diff_dir : str
Path to directory inside bg_model_dir to hold the difference images
corr_dir : str
Path to directory inside bg_model_dir to hold the background corrected images
template_header : ascii file
Path to file containing the WCS to which we want to reproject our images
im_type : str
Type of image used (Default: intbgsub)
level_only : bool, optional
Montage argument: Adjust background levels only, don't try to fit the slope (Default: True)
"""
# FIND OVERLAPS
diff_dir = os.path.join(diff_dir, im_type)
os.makedirs(diff_dir)
reprojected_table = os.path.join(reprojected_dir,
im_type + '_reprojected.tbl')
diffs_table = os.path.join(diff_dir, 'differences.tbl')
montage.mOverlaps(reprojected_table, diffs_table)
# CALCULATE DIFFERENCES BETWEEN OVERLAPPING IMAGES
montage.mDiffExec(diffs_table,
template_header,
diff_dir,
proj_dir=reprojected_dir)
# BEST-FIT PLANE COEFFICIENTS
fits_table = os.path.join(diff_dir, 'fits.tbl')
montage.mFitExec(diffs_table, fits_table, diff_dir)
# CALCULATE CORRECTIONS
corr_dir = os.path.join(corr_dir, im_type)
os.makedirs(corr_dir)
corrections_table = os.path.join(corr_dir, 'corrections.tbl')
montage.mBgModel(reprojected_table,
fits_table,
corrections_table,
level_only=level_only)
# APPLY CORRECTIONS
montage.mBgExec(reprojected_table,
corrections_table,
corr_dir,
proj_dir=reprojected_dir)
def bg_model(gal_dir, reprojected_dir, template_header, level_only=False):
bg_model_dir = os.path.join(gal_dir, 'background_model')
os.makedirs(bg_model_dir)
# FIND OVERLAPS
diff_dir = os.path.join(bg_model_dir, 'differences')
os.makedirs(diff_dir)
reprojected_table = os.path.join(reprojected_dir, 'int_reprojected.tbl')
diffs_table = os.path.join(diff_dir, 'differences.tbl')
montage.mOverlaps(reprojected_table, diffs_table)
# CALCULATE DIFFERENCES BETWEEN OVERLAPPING IMAGES
montage.mDiffExec(diffs_table,
template_header,
diff_dir,
proj_dir=reprojected_dir)
# BEST-FIT PLANE COEFFICIENTS
fits_table = os.path.join(diff_dir, 'fits.tbl')
montage.mFitExec(diffs_table, fits_table, diff_dir)
# CALCULATE CORRECTIONS
corr_dir = os.path.join(bg_model_dir, 'corrected')
os.makedirs(corr_dir)
corrections_table = os.path.join(corr_dir, 'corrections.tbl')
montage.mBgModel(reprojected_table,
fits_table,
corrections_table,
level_only=level_only)
# APPLY CORRECTIONS
montage.mBgExec(reprojected_table,
corrections_table,
corr_dir,
proj_dir=reprojected_dir)
return corr_dir
def mosaic(input_files, mosaic_file, work_dir, ext=0, background_match=False,
cdelt=None, density=False, equinox=None, header=None,
level_only=False, north_aligned=False, postprocess=None,
preprocess=None, system=None, weights_file=None):
"""Make a mosiac.
High-level wrapper around several Montage operations similar to
`montage_wrapper.mosaic`. The main differences are 1) added support for
preprocessing the input images before reprojection and postprocessing
the final image after mosaicking, 2) options for using images in total
flux units instead of flux density (as assumed by Montage), 3) more of
the `montage_wrapper.mMakeHdr` keywords available for header creation,
and 4) the `whole` keyword for `montage_wrapper.mProjExec` is
automatically set to True when `background_match` is True. The latter
is important since backround matching behaves unreliably otherwise.
Parameters
----------
input_files : list or string
List of paths to the input images. This may also be the path to a
directory containing all input images, in which case `input_files`
will automatically be set to a list of all files in the directory
ending with ".fits".
mosaic_file : str
Path to the output mosaic file. The final mosaic always has the
same units as the `input_files` images.
work_dir : str
Path to the working directory for all intermediate files produced
by Montage. The directory has the following structure::
work_dir/
input/
Contains either symlinks to `input_files` or new files
depending on the `preprocess` and `density` keywords.
Assuming the `density` keyword has been set correctly, these
images will always be in flux density units.
reprojected/
The reprojected images.
differences/
Difference calculations for background matching (only if
`background_match` is True).
corrected/
Background-matched images (only if `background_match` is
True).
output/
The intermediate mosiac used to produce the final mosaic
file, depending on the `density` and `postprocess` keywords.
background_match : bool, optional
If True, match the background levels of the reprojected images
before mosaicking. Automatically sets ``whole = True`` in
`montage_wrapper.mProjExec`. Default is False.
cdelt : float, optional
See `header` and `montage_wrapper.mMakeHdr`. Default is None.
density : bool, optional
If True, the input images are in flux density units (i.e., signal
per unit pixel area). If False (default), the input images are
assumed to be in units of total flux, and are automatically scaled
to flux density before reprojection.
equinox : str, optional
See `header` and `montage_wrapper.mMakeHdr`. Default is None.
header : str, optional
Path to the template header file describing the output mosaic.
Default is None, in which case a template header is created
automatically using `montage_wrapper.mMakeHdr` and the `cdelt`,
`equinox`, `north_aligned`, and `system` keyword arguments.
level_only : bool, optional
See `montage_wrapper.mBgModel`. Ignored if `background_match` is
False. Default is False.
north_aligned : bool, optional
See `header` and `montage_wrapper.mMakeHdr`. Default is None.
postprocess, preprocess : function, optional
Functions for processing the raw input images before the input
density images are created (`preprocess`) and after the final
mosaic is created (`postprocess`). The function arguments should be
the image data array and the image header
(`astropy.io.fits.Header`), and the return values should be the
same. Default is None.
system : str, optional
See `header` and `montage_wrapper.mMakeHdr`. Default is None.
weights_file : str, optional
Path to output pixel weights file. Pixel weights are derived from
the final mosaic area file. Weights are normalized to 1, and
represent coverage of the mosaic area by the input images. Unlike
Montage area files, regions where the input images overlap are not
considered. Default is None.
Returns
-------
None
"""
# Get list of files if input_files is a directory name
if isinstance(input_files, basestring):
dirname = os.path.dirname(input_files)
input_files = [os.path.join(dirname, basename)
for basename in os.listdir(dirname)
if os.path.splitext(basename)[1] == '.fits']
# Create working directory
try:
os.makedirs(work_dir)
except OSError:
shutil.rmtree(work_dir)
os.makedirs(work_dir)
# Create input directory, populate it, and get image metadata
input_dir = os.path.join(work_dir, 'input')
os.mkdir(input_dir)
if preprocess or not density or ext>0:
# Create new input files
for input_file in input_files:
data, hdr = astropy.io.fits.getdata(input_file, header=True,
ext=ext)
if preprocess:
data, hdr = preprocess(data, hdr)
if not density:
# Convert total flux into flux density
dx, dy = wcs.calc_pixscale(hdr, ref='crpix').arcsec
pixarea = dx * dy # arcsec2
data /= pixarea
# Write
basename = os.path.basename(input_file)
basename = '_density'.join(os.path.splitext(basename))
new_input_file = os.path.join(input_dir, basename)
hdu = astropy.io.fits.PrimaryHDU(data=data, header=hdr)
hdu.writeto(new_input_file, output_verify='ignore')
else:
# Symlink existing files
for input_file in input_files:
basename = os.path.basename(input_file)
new_input_file = os.path.join(input_dir, basename)
os.symlink(input_file, new_input_file)
input_table = os.path.join(input_dir, 'input.tbl')
montage.mImgtbl(input_dir, input_table, corners=True)
# Template header
if header is None:
template_header = os.path.join(work_dir, 'template.hdr')
montage.mMakeHdr(input_table, template_header, cdelt=cdelt,
equinox=equinox, north_aligned=north_aligned,
system=system)
else:
template_header = header
# Create reprojection directory, reproject, and get image metadata
proj_dir = os.path.join(work_dir, 'reprojected')
os.makedirs(proj_dir)
whole = True if background_match else False
stats_table = os.path.join(proj_dir, 'mProjExec_stats.log')
montage.mProjExec(input_table, template_header, proj_dir, stats_table,
raw_dir=input_dir, whole=whole)
reprojected_table = os.path.join(proj_dir, 'reprojected.tbl')
montage.mImgtbl(proj_dir, reprojected_table, corners=True)
# Background matching
if background_match:
diff_dir = os.path.join(work_dir, 'differences')
os.makedirs(diff_dir)
# Find overlaps
diffs_table = os.path.join(diff_dir, 'differences.tbl')
montage.mOverlaps(reprojected_table, diffs_table)
# Calculate differences between overlapping images
montage.mDiffExec(diffs_table, template_header, diff_dir,
proj_dir=proj_dir)
# Find best-fit plane coefficients
fits_table = os.path.join(diff_dir, 'fits.tbl')
montage.mFitExec(diffs_table, fits_table, diff_dir)
# Calculate corrections
corr_dir = os.path.join(work_dir, 'corrected')
os.makedirs(corr_dir)
corrections_table = os.path.join(corr_dir, 'corrections.tbl')
montage.mBgModel(reprojected_table, fits_table, corrections_table,
level_only=level_only)
# Apply corrections
montage.mBgExec(reprojected_table, corrections_table, corr_dir,
proj_dir=proj_dir)
img_dir = corr_dir
else:
img_dir = proj_dir
# Make mosaic
output_dir = os.path.join(work_dir, 'output')
os.makedirs(output_dir)
out_image = os.path.join(output_dir, 'mosaic.fits')
montage.mAdd(reprojected_table, template_header, out_image,
img_dir=img_dir, exact=True)
# Pixel areas and weights
if weights_file or not density:
area_file = '_area'.join(os.path.splitext(out_image))
area, hdr = astropy.io.fits.getdata(area_file, header=True) # steradians
area *= (180/np.pi*3600)**2 # arcsec2
dx, dy = wcs.calc_pixscale(hdr, ref='crpix').arcsec
pixarea = dx * dy # arcsec2
area = np.clip(area, 0, pixarea) # Don't care about overlaps
if weights_file:
weights = area / pixarea # Normalize to 1
hdu = astropy.io.fits.PrimaryHDU(weights, header=hdr)
try:
hdu.writeto(weights_file)
except IOError:
os.remove(weights_file)
hdu.writeto(weights_file)
# Write final mosaic
dirname = os.path.dirname(mosaic_file)
try:
os.makedirs(dirname)
except OSError:
pass
if postprocess or not density:
# Create new file
data, hdr = astropy.io.fits.getdata(out_image, header=True)
if not density:
# Convert flux density into total flux
data *= pixarea
if postprocess:
data, hdr = postprocess(data, hdr)
# Write
hdu = astropy.io.fits.PrimaryHDU(data, header=hdr)
try:
hdu.writeto(mosaic_file)
except IOError:
os.remove(mosaic_file)
hdu.writeto(mosaic_file)
else:
# Move existing file
os.rename(out_image, mosaic_file)
return
def _montage_test():
# create density images
input_dir = os.path.dirname(density_files[0])
# image metadata
meta1_file = os.path.join(input_dir, 'meta1.tbl')
montage.mImgtbl(input_dir, meta1_file, corners=True)
# make header
#lon, lat = [], []
#for density_file in density_files:
# data, hdr = astropy.io.fits.getdata(density_file, header=True)
# wcs = astropy.wcs.WCS(hdr)
# x1, y1 = 0.5, 0.5
# y2, x2 = data.shape
# x2, y2 = x2 + 0.5, y2 + 0.5
# x, y = [x1, x2, x2, x1], [y1, y1, y2, y2]
# ln, lt = wcs.wcs_pix2world(x, y, 1)
# lon += list(ln)
# lat += list(lt)
#lon1, lon2 = np.min(lon), np.max(lon)
#lat1, lat2 = np.min(lat), np.max(lat)
hdr_file = os.path.join(os.path.dirname(input_dir), 'test.hdr')
montage.mMakeHdr(meta1_file, hdr_file)
# reproject
proj_dir = os.path.dirname(proj_files[0])
safe_mkdir(proj_dir)
stats_file = os.path.join(proj_dir, 'stats.tbl')
montage.mProjExec(meta1_file, hdr_file, proj_dir, stats_file,
raw_dir=input_dir, exact=True)
# image metadata
meta2_file = os.path.join(proj_dir, 'meta2.tbl')
montage.mImgtbl(proj_dir, meta2_file, corners=True)
# Background modeling
diff_dir = os.path.join(os.path.dirname(proj_dir), 'difference')
safe_mkdir(diff_dir)
diff_file = os.path.join(diff_dir, 'diffs.tbl')
montage.mOverlaps(meta2_file, diff_file)
montage.mDiffExec(diff_file, hdr_file, diff_dir, proj_dir)
fits_file = os.path.join(diff_dir, 'fits.tbl')
montage.mFitExec(diff_file, fits_file, diff_dir)
# Background matching
corr_dir = os.path.join(os.path.dirname(proj_dir), 'correct')
safe_mkdir(corr_dir)
corr_file = os.path.join(corr_dir, 'corrections.tbl')
montage.mBgModel(meta2_file, fits_file, corr_file, level_only=False)
montage.mBgExec(meta2_file, corr_file, corr_dir, proj_dir=proj_dir)
# Native mosaic
projadd_file = config.path('{:s}.reproject.add'.format(kind))
projadd_dir, filename = os.path.split(projadd_file)
filename, ext = os.path.splitext(filename)
filename = '{0:s}_native{1:s}'.format(filename, ext)
projaddnative_file = os.path.join(projadd_dir, filename)
safe_mkdir(projadd_dir)
montage.mAdd(meta2_file, hdr_file, projaddnative_file, img_dir=corr_dir, exact=True)
# Reproject to final header
header_file = config.path('{:s}.hdr'.format(kind))
montage.mProject(projaddnative_file, projadd_file, header_file)
# Postprocess
data, hdr = astropy.io.fits.getdata(projaddnative_file, header=True)
x1, x2 = 900, 1900
y1, y2 = 3000, 4500
val = np.mean(data[y1:y2,x1:x2])
data, hdr = astropy.io.fits.getdata(projadd_file, header=True)
data = data - val
areaadd_file = config.path('{:s}.area.add'.format(kind))
area = astropy.io.fits.getdata(areaadd_file) * (180/np.pi*3600)**2 # arcsec2
data = data * area
add_file = config.path('{:s}.add'.format(kind))
dirname = os.path.dirname(add_file)
safe_mkdir(dirname)
if os.path.exists(add_file):
os.remove(add_file)
hdu = astropy.io.fits.PrimaryHDU(data, header=hdr)
hdu.writeto(add_file)
def Run(ra,
dec,
width,
name=None,
out_dir=None,
temp_dir=None,
replace=False,
flux=True,
thumbnails=False,
gzip=True,
montage_path=None,
swarp_path=None):
"""
Function to generate standardised cutouts of Herschel observations.
Arguments
ra: {float, sequence of float}
A sequence of right ascension values, in decimal degrees, of the targets to be processed. Alternatively,
if you're only interested in one target, a single RA value can be given here.
dec: {float, sequence of float}
A sequence of declination values, in decimal degrees, of the targets to be processed. Alternatively, if
you're only interested in one target, a single Dec value can be given here.
width: {float, sequence of float}
A sequence giving the desired width of the cutout square for each target, in decimal degrees.
Alternatively, if you're only interested in one target, a single width value can be given here.
Keyword arguments
name: {str, sequence of str}, optional
A sequence giving the name of each target; if you're only interested in one target, a
single name can be given here. If not provided, a name is constructed automatrically from the target
coordinates, according to the IAU catalogue convention.
out_dir: str, optional
A string giving the path to the directory where the output FITS files will be placed. If not provided,
files will simply be written to the current working directory.
temp_dir: str, optional
A string giving the path to be used as a temporary working directory by Herschel_Button. If not provided,
a temporary directory will be created inside the output directory.
replace: bool, optional
If False, Herschel_Button will search the output directory for any pre-existing output FITS files from
previous runs of the function, and will not bother repeat creating these maps (making it easy to resume
processing a large number of targets from an interruption. If True, Herschel_Button will produce maps for
all input targets, regardless of whether maps for these targets already exist in the output directory.
flux: bool, optional
If True, output maps will be in flux density units of Jy/pix. If false, output maps will be in surface
brightness units of MJy/sr.
thumbnails: bool, optional
If True, JPG thumbnail images of the generated maps will also be proced and placed in out_dir.
montage_path: str, optional
Path to directory that contains the Montage commands (mProject, etc); useful if this directory is not in $PATH
swarp_path: str: optional
Path to directory that contains the SWarp command; useful if this directory is not in $PATH
"""
# Handle Montage and SWarp paths, if kwargs provided
if montage_path != None:
os.environ['PATH'] += ':' + montage_path
if swarp_path != None:
os.environ['PATH'] += ':' + swarp_path
import montage_wrapper
# Make sure input values are in list format, and sort out variable names for rest of function
if not hasattr(ra, '__iter__'):
ra = [ra]
ra_list = np.array(ra)
del (ra)
if not hasattr(dec, '__iter__'):
dec = [dec]
dec_list = np.array(dec)
del (dec)
# Check that ra and declists all have same lengths
if np.std([float(len(ra_list)), float(len(dec_list))]) > 0:
raise Exception(
'Input sequences of ra and dec all need to be the same length')
# If single width provided, but multiple coordinates, create width array of same value repeated required number of times
if not hasattr(width, '__iter__'):
if len(ra_list) > 1:
width_list = [width] * len(ra_list)
# Else, if only one RA and one width given, stick width value into list, too
elif len(ra_list) == 1:
width_list = [width]
width_list = np.array(width_list)
del (width)
# If no names provided, use coordinates to generate standardised names as per IAU catalogue convention
if not hasattr(name, '__iter__'):
if (name == None):
name = []
for i in range(len(ra_list)):
coord = astropy.coordinates.SkyCoord(
str(ra_list[i]) + 'd ' + str(dec_list[i]) + 'd')
name_coord = re.sub('[hmsdms. ]', ' ',
coord.to_string('hmsdms'))
name_coord = name_coord.split(' ')
name_coord[3] = name_coord[3][:min(2, len(name_coord[3]))]
name_coord[8] = name_coord[8][:min(2, len(name_coord[8]))]
name_coord = 'J' + ''.join(name_coord)
name.append(
re.sub('[hmsdms. ]', ' ', coord.to_string('hmsdms')))
# If only one name provided, stick it into an array
name_list = np.array([name])
# If a sequence of names is provided, make sure it's in array format (and stop single names becoming zero-dim array)
else:
name_list = np.array(copy.deepcopy(name))
if name_list.shape == ():
name_list = np.array([name_list.tolist()])
del (name)
# Do final check that all input sequences are the right length
if np.std([
float(ra_list.size),
float(dec_list.size),
float(width_list.size),
float(name_list.size)
]) > 0:
raise Exception(
'Input sequences of ra, dec, with, and name all need to be the same length'
)
# If no outout directory specified, set to current working directory
if out_dir == None:
out_dir = os.getcwd()
# Check that output directory exists
if not os.path.exists(out_dir):
raise Exception('Specified output directory does not exist')
# Create temporary directory
if temp_dir == None:
temp_dir = os.path.join(out_dir, 'Temp')
# Check that temp directory exists, if it does, warn user that contents may be overwritten
if os.path.exists(temp_dir):
print(
'Specificed temporary directory already exists; note that any existing contents may be overwritten'
)
# Else, if temp directory doesn't already exist, create it
else:
os.mkdir(temp_dir)
# State band information
bands_dict = {
'70': {
'band': '70',
'instrument': 'PACS',
'wavelength': '70um',
'filter': 'PHOTBLUE',
'pix_size': 2,
'hdr_inst_card_kwrd': 'CAMERA',
'hdr_inst_card_entry': 'PHOTBLUE',
'hdr_blueband_kwrd': 'blue1',
'hdr_err_ext_name': 'stDev'
},
'100': {
'band': '100',
'instrument': 'PACS',
'wavelength': '100um',
'filter': 'PHOTGREEN',
'pix_size': 3,
'hdr_inst_card_kwrd': 'CAMERA',
'hdr_inst_card_entry': 'PHOTBLUE',
'hdr_blueband_kwrd': 'blue2',
'hdr_err_ext_name': 'stDev'
},
'160': {
'band': '160',
'instrument': 'PACS',
'wavelength': '160um',
'filter': 'PHOTRED',
'pix_size': 4,
'hdr_inst_card_kwrd': 'CAMERA',
'hdr_inst_card_entry': 'PHOTRED',
'hdr_blueband_kwrd': False,
'hdr_err_ext_name': 'stDev'
},
'250': {
'band': '250',
'instrument': 'SPIRE',
'wavelength': '250um',
'filter': 'PSW',
'pix_size': 6,
'hdr_inst_card_kwrd': 'DETECTOR',
'hdr_inst_card_entry': 'PSW',
'hdr_blueband_kwrd': False,
'hdr_err_ext_name': 'error'
},
'350': {
'band': '350',
'instrument': 'SPIRE',
'wavelength': '350um',
'filter': 'PMW',
'pix_size': 8,
'hdr_inst_card_kwrd': 'DETECTOR',
'hdr_inst_card_entry': 'PMW',
'hdr_blueband_kwrd': False,
'hdr_err_ext_name': 'error'
},
'500': {
'band': '500',
'instrument': 'SPIRE',
'wavelength': '500um',
'filter': 'PLW',
'pix_size': 12,
'hdr_inst_card_kwrd': 'DETECTOR',
'hdr_inst_card_entry': 'PLW',
'hdr_blueband_kwrd': False,
'hdr_err_ext_name': 'error'
}
}
# State map mode prefixes we care about
req_obs_modes = [
'SpirePhotoLargeScan', 'SpirePhotoSmallScan', 'PacsPhoto',
'SpirePacsParallel'
]
# Record time taken
time_list = [time.time()]
# Loop over each target
for i in np.random.permutation(range(name_list.shape[0])):
name = name_list[i].replace(' ', '_')
ra = ra_list[i]
dec = dec_list[i]
width = width_list[i]
# If we're not repeating already-processed targets, check if this target has already been completed
if not replace:
bands_done = 0
for band in bands_dict.keys():
if os.path.exists(
os.path.join(
out_dir, name + '_Herschel_' +
bands_dict[band]['wavelength'] + '.fits.gz')):
bands_done += 1
# Also check for null files, indicated data not available for a givne band
elif os.path.exists(
os.path.join(
out_dir, '.' + name + '_Herschel_' +
bands_dict[band]['wavelength'] + '.null')):
bands_done += 1
# If this source has already been processed in all bands, skip it
if bands_done == len(bands_dict.keys()):
print(
'Herschel data for ' + name +
' already processed (if available); continuing to next target'
)
time_list.append(time.time())
continue
print('Processing Herschel data for target ' + name)
# Create field processing dirctories (deleting any prior)
gal_dir = os.path.join(temp_dir, str(name)) + '/'
if os.path.exists(gal_dir):
ChrisFuncs.RemoveCrawl(gal_dir)
if not os.path.exists(os.path.join(gal_dir, 'Raw')):
os.makedirs(os.path.join(gal_dir, 'Raw'))
os.chdir(os.path.join(gal_dir, 'Raw'))
# Create band-specific directories
for band in bands_dict.keys():
if not os.path.exists(os.path.join(gal_dir, 'Raw', band)):
os.makedirs(os.path.join(gal_dir, 'Raw', band))
# Perform query, with error handling
print('Querying HSA')
query_success = False
query_fail_count = 0
while query_success == False:
if query_fail_count >= 10:
raise Exception(
'HSA query failing consistently; maybe HSA is down, or something else has gone wrong'
)
try:
query_url = 'http://archives.esac.esa.int/hsa/aio/jsp/siap.jsp?POS=' + str(
ra) + ',' + str(dec) + '&SIZE=' + str(
width) + '&INTERSECT=OVERLAPS'
query_filename = os.path.join(temp_dir, name,
str(name) + '.vot')
if os.path.exists(query_filename):
os.remove(query_filename)
urllib.request.urlretrieve(query_url, query_filename)
query_success = True
except:
print('HSA query failed; reattempting')
query_fail_count += 1
time.sleep(60)
if not os.path.exists(query_filename):
query_success = False
# Read query result VOTable
query_output = astropy.io.votable.parse_single_table(query_filename)
query_table = query_output.array
# Check if query returned any results; if not, create null file, and continue to next target
if len(query_table) == 0:
print('No Herschel coverage for ' + name +
'; continuing to next target')
os.system('touch ' +
os.path.join(temp_dir, '.' + name + '_Herschel_' + band +
'.null'))
continue
# Record which urls correspond to data in the desired modes (dealing with awkwardness for if there is only 1 entry, or silly massive files)
hsa_urls = []
if query_table.size == 1:
if query_table['OBS_MODE'] in req_obs_modes:
hsa_urls.append(query_table['DATA_ACCESS'])
else:
for j in range(0, query_table.size):
if query_table['OBS_MODE'][j].decode('utf-8') in req_obs_modes:
hsa_urls.append(
query_table['DATA_LINK'][j].decode('utf-8'))
# In parallel, download and extract files
os.chdir(os.path.join(gal_dir, 'Raw'))
dl_pool = mp.Pool(processes=20)
for j in range(0, len(hsa_urls)):
data_url = hsa_urls[j]
data_filename = os.path.join(gal_dir, 'Raw',
name + '_' + str(j) + '_HSA.fits')
#dl_pool.apply_async( Herschel_Download, args=(data_url, data_filename,) )
Herschel_Download(data_url, data_filename)
dl_pool.close()
dl_pool.join()
# Loop over bands, and downloaded files (skipping folders), for sorting files into separate folders
for band in bands_dict.keys():
prev_hdr_filenames = []
for listfile in os.listdir(os.path.join(gal_dir, 'Raw')):
if '.tmp' in listfile:
os.remove(os.path.join(gal_dir, 'Raw', listfile))
continue
if '.fits' not in listfile:
continue
# Determine what band this is
try:
list_hdr = astropy.io.fits.getheader(os.path.join(
gal_dir, 'Raw', listfile),
ext=0)
except:
pdb.set_trace()
if list_hdr['INSTRUME'] == bands_dict[band]['instrument']:
if list_hdr[bands_dict[band]
['hdr_inst_card_kwrd']] == bands_dict[band][
'hdr_inst_card_entry']:
# Handle the fact that 70um and 100um are hard to tell apart in headers
if bands_dict[band]['hdr_blueband_kwrd'] != False:
if bands_dict[band][
'hdr_blueband_kwrd'] not in list_hdr[
'BLUEBAND']:
continue
# Skip dud PACS calibration(?) maps
if list_hdr['OBSERVER'][-4:].lower() == 'pacs':
os.remove(os.path.join(gal_dir, 'Raw', listfile))
continue
# Check that we havne't already grabbed a duplicate of this map; if not, move it to band-specific directory
if 'FILENAME' in list_hdr.keys():
if list_hdr['FILENAME'] in prev_hdr_filenames:
os.remove(
os.path.join(gal_dir, 'Raw', listfile))
continue
else:
prev_hdr_filenames.append(list_hdr['FILENAME'])
shutil.copy2(os.path.join(gal_dir, 'Raw', listfile),
os.path.join(gal_dir, 'Raw', band))
os.remove(os.path.join(gal_dir, 'Raw', listfile))
# Loop over PACS bands and files to delete dud PACS calibration(?) maps
for band in bands_dict.keys():
if bands_dict[band]['instrument'] == 'PACS':
for listfile in os.listdir(os.path.join(gal_dir, 'Raw', band)):
if astropy.io.fits.getheader(
os.path.join(gal_dir, 'Raw', band, listfile),
ext=0)['OBSERVER'][-4:].lower() == 'pacs':
os.remove(os.path.join(gal_dir, 'Raw', band, listfile))
# Loop over each band's files, to save image map to separate FITS files
for band in bands_dict.keys():
for listfile in os.listdir(os.path.join(gal_dir, 'Raw', band)):
print('Extracting components from ' + band + ' um map ' +
listfile)
if '.tmp' in listfile:
pdb.set_trace()
# Check map has error and coverage data; open if so, skip forward if not
with astropy.io.fits.open(
os.path.join(gal_dir, 'Raw', band,
listfile)) as listfile_hdulist:
if len(listfile_hdulist) < 4:
print('Some FITS extensions missing from ' + band +
' um map ' + listfile + '; skipping')
continue
img_map, img_header = astropy.io.fits.getdata(os.path.join(
gal_dir, 'Raw', band, listfile),
header=True,
extname='image')
# Record which image pixels are zeros, and convert to NaNs
where_zero = np.where(img_map == 0)
img_map[where_zero] = np.NaN
astropy.io.fits.writeto(os.path.join(
gal_dir, 'Raw', band,
listfile.replace('.fits', '_Img.fits')),
img_map,
header=img_header)
# Now save coverage and error maps to separate files, with zeros similarly converted to NaNs
cov_map, cov_header = astropy.io.fits.getdata(
os.path.join(gal_dir, 'Raw', band, listfile),
header=True,
extname='coverage')
cov_map[where_zero] = np.NaN
astropy.io.fits.writeto(os.path.join(
gal_dir, 'Raw', band,
listfile.replace('.fits', '_Cov.fits')),
cov_map,
header=cov_header)
err_map, err_header = astropy.io.fits.getdata(
os.path.join(gal_dir, 'Raw', band, listfile),
header=True,
extname=bands_dict[band]['hdr_err_ext_name'])
err_map[where_zero] = np.NaN
astropy.io.fits.writeto(os.path.join(
gal_dir, 'Raw', band,
listfile.replace('.fits', '_Error.fits')),
err_map,
header=err_header)
# Loop over each band for coaddition
for band in bands_dict.keys():
if not os.path.exists(os.path.join(gal_dir, 'Raw', band)):
continue
if len(os.path.join(gal_dir, 'Raw', band)) == 0:
continue
print('Commencing processing of ' + name + '_Herschel_' + band)
# Create processing directories
os.chdir(os.path.join(gal_dir, 'Raw', band))
os.mkdir(os.path.join(gal_dir, 'Raw', band, 'Img_Maps'))
os.mkdir(os.path.join(gal_dir, 'Raw', band, 'Cov_Maps'))
os.mkdir(os.path.join(gal_dir, 'Raw', band, 'Err_Maps'))
os.mkdir(os.path.join(gal_dir, 'Raw', band, 'Exp_Maps'))
os.mkdir(os.path.join(gal_dir, 'Raw', band, 'Wgt_Temp'))
os.mkdir(os.path.join(gal_dir, 'Raw', band, 'Pff_Temp'))
os.mkdir(os.path.join(gal_dir, 'Raw', band, 'Backsub_Temp'))
os.mkdir(os.path.join(gal_dir, 'Raw', band, 'SWarp_Temp'))
# Create Montage FITS header
location_string = str(ra) + ' ' + str(dec)
pix_size = bands_dict[band]['pix_size']
montage_wrapper.mHdr(location_string,
width,
os.path.join(gal_dir, 'Raw', band,
str(name) + '.hdr'),
pix_size=pix_size)
# Use Montage wrapper to reproject all fits files to common projection, skipping if none acually overlap
print('Performing reprojections for ' + name + '_Herschel_' +
band + ' maps')
target_files = []
proj_fail = 0
[
target_files.append(target_file) for target_file in os.listdir(
os.path.join(gal_dir, 'Raw', band))
if '.fits' in target_file
]
for target_file in target_files:
try:
montage_wrapper.reproject(
os.path.join(
os.path.join(gal_dir, 'Raw', band, target_file)),
os.path.join(
os.path.join(gal_dir, 'Raw', band, target_file)),
header=os.path.join(gal_dir, 'Raw', band,
str(name) + '.hdr'),
exact_size=True)
except:
os.remove(
os.path.join(
os.path.join(gal_dir, 'Raw', band, target_file)))
proj_fail += 1
if proj_fail == len(target_files):
print('No Herschel coverage for ' + name + ' at ' + band)
os.system('touch ' + os.path.join(
temp_dir, '.' + name + '_Herschel_' + band + '.null'))
continue
# Move reprojcted maps to relevant locations
for listfile in os.listdir(os.path.join(gal_dir, 'Raw', band)):
if '_Img.fits' in os.path.join(gal_dir, 'Raw', band, listfile):
shutil.move(os.path.join(gal_dir, 'Raw', band, listfile),
os.path.join(gal_dir, 'Raw', band, 'Img_Maps'))
elif '_Cov.fits' in os.path.join(gal_dir, 'Raw', band,
listfile):
shutil.move(os.path.join(gal_dir, 'Raw', band, listfile),
os.path.join(gal_dir, 'Raw', band, 'Cov_Maps'))
elif '_Error.fits' in os.path.join(gal_dir, 'Raw', band,
listfile):
shutil.move(os.path.join(gal_dir, 'Raw', band, listfile),
os.path.join(gal_dir, 'Raw', band, 'Err_Maps'))
# If only one image file, proceed straight to co-adding; otherwise, commence background-matching
mosaic_count = 0
for listfile in os.listdir(
os.path.join(gal_dir, 'Raw', band, 'Img_Maps')):
if '_Img.fits' in listfile:
mosaic_count += 1
if mosaic_count == 1:
for listfile in os.listdir(
os.path.join(gal_dir, 'Raw', band, 'Img_Maps')):
if '.fits' in listfile:
shutil.move(
os.path.join(gal_dir, 'Raw', band, 'Img_Maps',
listfile),
os.path.join(gal_dir, 'Raw', band, 'SWarp_Temp'))
mBgExec_uberfail = False
if mosaic_count > 1:
# Use Montage wrapper to determine appropriate corrections for background matching
print('Determining background corrections for ' + name +
'_Herschel_' + band + ' maps')
os.chdir(os.path.join(gal_dir, 'Raw', band, 'Img_Maps'))
montage_wrapper.mImgtbl(
os.path.join(gal_dir, 'Raw', band, 'Img_Maps'),
os.path.join(gal_dir, 'Raw', band, 'Img_Maps',
band + '_Image_Metadata_Table.dat'),
corners=True)
montage_wrapper.mOverlaps(
os.path.join(gal_dir, 'Raw', band, 'Img_Maps',
band + '_Image_Metadata_Table.dat'),
os.path.join(gal_dir, 'Raw', band, 'Img_Maps',
band + '_Image_Diffs_Table.dat'))
montage_wrapper.mDiffExec(
os.path.join(gal_dir, 'Raw', band, 'Img_Maps',
band + '_Image_Diffs_Table.dat'),
os.path.join(gal_dir, 'Raw', band,
str(name) + '.hdr'),
os.path.join(gal_dir, 'Raw', band, 'Pff_Temp'),
no_area=True,
proj_dir=os.path.join(gal_dir, 'Raw', band, 'Img_Maps'))
montage_wrapper.mFitExec(
os.path.join(gal_dir, 'Raw', band, 'Img_Maps',
band + '_Image_Diffs_Table.dat'),
os.path.join(gal_dir, 'Raw', band, 'Img_Maps',
band + '_Image_Fitting_Table.dat'),
os.path.join(gal_dir, 'Raw', band, 'Pff_Temp'))
montage_wrapper.mBgModel(
os.path.join(gal_dir, 'Raw', band, 'Img_Maps',
band + '_Image_Metadata_Table.dat'),
os.path.join(gal_dir, 'Raw', band, 'Img_Maps',
band + '_Image_Fitting_Table.dat'),
os.path.join(gal_dir, 'Raw', band, 'Img_Maps',
band + '_Image_Corrections_Table.dat'),
level_only=True,
n_iter=16384)
# Apply background corrections using Montage subprocess, with timeout handling
print('Applying background corrections to ' + name +
'_Herschel_' + band + ' maps')
mBgExec_fail_count = 0
mBgExec_success = False
mBgExec_uberfail = False
while mBgExec_success == False:
# Attempt background-matching
mBgExec_sp = subprocess.Popen([
'mBgExec', '-n', '-p',
os.path.join(gal_dir, 'Raw', band, 'Img_Maps'),
os.path.join(gal_dir, 'Raw', band, 'Img_Maps',
band + '_Image_Metadata_Table.dat'),
os.path.join(gal_dir, 'Raw', band, 'Img_Maps',
band + '_Image_Corrections_Table.dat'),
os.path.join(gal_dir, 'Raw', band, 'SWarp_Temp')
],
preexec_fn=os.setsid,
stdout=subprocess.PIPE)
mBgExec_fail = False
seconds = 0
minutes_max = 45
while mBgExec_fail == False:
time.sleep(1)
mBgExec_stdout = mBgExec_sp.stdout.readline().decode()
if mBgExec_sp.poll() == None:
seconds += 1
if 'Table has no data records' in mBgExec_stdout:
mBgExec_fail = True
mBgExec_fail_count += 1
break
if seconds >= (60 * minutes_max):
mBgExec_fail = True
mBgExec_fail_count += 1
break
if mBgExec_sp.poll() != None:
mBgExec_success = True
break
# Handle timeouts and other failures
if mBgExec_fail_count > 1:
print('Background matching with Montage has failed ' +
str(mBgExec_fail_count) +
' time(s); reattempting')
if mBgExec_fail == True and mBgExec_success == False and mBgExec_fail_count >= 5:
mBgExec_uberfail = True
print(
'Background matching with Montage has failed 5 times; proceeding directly to co-additon'
)
try:
os.killpg(os.getpgid(mBgExec_sp.pid), 15)
except:
'Background matching subprocess appears to have imploded; no task to kill'
break
if mBgExec_uberfail:
raise Exception(
'Background matching with Montage has failed utterly')
"""for listfile in os.listdir(os.path.join(gal_dir,'Raw',band,'Img_Maps')):
if '_HSA_Img.fits' in listfile:
shutil.move(listfile, os.path.join(gal_dir,'Raw',band,'SWarp_Temp'))"""
# Create weight maps, and copy to SWarp directory
for listfile in os.listdir(
os.path.join(gal_dir, 'Raw', band, 'Cov_Maps')):
if '.fits' in listfile:
shutil.copy2(
os.path.join(gal_dir, 'Raw', band, 'Cov_Maps',
listfile),
os.path.join(gal_dir, 'Raw', band, 'SWarp_Temp'))
wgt_image, wgt_header = astropy.io.fits.getdata(
os.path.join(gal_dir, 'Raw', band, 'Cov_Maps',
listfile),
header=True)
wgt_image = wgt_image**0.5
astropy.io.fits.writeto(os.path.join(
gal_dir, 'Raw', band, 'SWarp_Temp',
listfile.replace('_Cov.fits', '_Wgt.fits')),
wgt_image,
header=wgt_header)
# Sort out daft filename differences between image maps and error maps
for listfile in os.listdir(
os.path.join(gal_dir, 'Raw', band, 'SWarp_Temp')):
os.rename(
os.path.join(gal_dir, 'Raw', band, 'SWarp_Temp', listfile),
os.path.join(gal_dir, 'Raw', band, 'SWarp_Temp',
listfile.replace('_Img.fits', '.fits')))
# Perform least-squares plane fitting to match image levels
ChrisFuncs.Coadd.LevelFITS(os.path.join(gal_dir, 'Raw', band,
'SWarp_Temp'),
'Img.fits',
convfile_dir=False)
# Use SWarp to co-add images weighted by their coverage maps
print('Co-adding ' + name + '_Herschel_' + band + ' maps')
os.chdir(os.path.join(gal_dir, 'Raw', band, 'SWarp_Temp'))
os.system(
'swarp *HSA.fits -IMAGEOUT_NAME ' + name + '_Herschel_' +
band +
'_SWarp.fits -WEIGHT_SUFFIX _Wgt.fits -WEIGHT_TYPE MAP_RMS -COMBINE_TYPE WEIGHTED -COMBINE_BUFSIZE 2048 -GAIN_KEYWORD DIESPIZERDIE -RESCALE_WEIGHTS N -SUBTRACT_BACK N -RESAMPLE N -VMEM_MAX 4095 -MEM_MAX 4096 -WEIGHT_TYPE MAP_WEIGHT -NTHREADS 4 -VERBOSE_TYPE QUIET'
)
Herschel_SWarp_NaN(name + '_Herschel_' + band + '_SWarp.fits')
# Check that the final maps provides actual coverage of the point in question
coadd_image, coadd_header = astropy.io.fits.getdata(os.path.join(
gal_dir, 'Raw', band, 'SWarp_Temp',
name + '_Herschel_' + band + '_SWarp.fits'),
header=True)
coadd_wcs = astropy.wcs.WCS(coadd_header)
coords_xy = np.round(
coadd_wcs.all_world2pix(np.array([[ra, dec]]), 0)).astype(int)
coord_i, coord_j = coords_xy[0, 1], coords_xy[0, 0]
if np.isnan(
np.nanmax(coadd_image[coord_i - 2:coord_i + 2 + 1,
coord_j - 2:coord_j + 2 + 2])):
print('No Herschel coverage for ' + name + ' at ' + band)
os.system('touch ' + os.path.join(
temp_dir, '.' + name + '_Herschel_' + band + '.null'))
continue
# Re-project finalised image map using Montage
montage_wrapper.reproject(
os.path.join(gal_dir, 'Raw', band, 'SWarp_Temp',
name + '_Herschel_' + band + '_SWarp.fits'),
os.path.join(gal_dir, name + '_Herschel_' + band + '.fits'),
header=os.path.join(gal_dir, 'Raw', band,
str(name) + '.hdr'),
exact_size=True)
# Compress finalised FITS file
os.chdir(gal_dir)
if gzip:
os.system('gzip ' +
os.path.join(gal_dir, name + '_Herschel_' + band +
'.fits'))
print('Completed processing ' + name + '_Herschel_' + band +
' image map')
# Turn error maps into exposure time maps
for listfile in os.listdir(
os.path.join(gal_dir, 'Raw', band, 'Err_Maps')):
if '_Error.fits' in listfile:
err_image, err_header = astropy.io.fits.getdata(
os.path.join(gal_dir, 'Raw', band, 'Err_Maps',
listfile),
header=True)
err_image = err_image**-2.0
astropy.io.fits.writeto(os.path.join(
gal_dir, 'Raw', band, 'Exp_Maps',
listfile.replace('_Error.fits', '_Exp.fits')),
err_image,
header=err_header)
# Use Montage to add exposure time images
print('Processing ' + name + '_Herschel_' + band +
' uncertainty map')
target_files = []
[
target_files.append(dir_file) for dir_file in os.listdir(
os.path.join(gal_dir, 'Raw', band, 'Exp_Maps'))
if '_Exp.fits' in dir_file
]
for i in range(0, len(target_files)):
exp_image, exp_header = astropy.io.fits.getdata(os.path.join(
gal_dir, 'Raw', band, 'Exp_Maps', target_files[i]),
header=True)
if i == 0:
add_image = np.zeros(
[exp_image.shape[0], exp_image.shape[1]])
add_header = exp_header.copy()
exp_good = np.where(np.isnan(exp_image) == False)
add_image[exp_good] += exp_image[exp_good]
add_hdu = astropy.io.fits.PrimaryHDU(data=add_image,
header=add_header)
add_hdulist = astropy.io.fits.HDUList([add_hdu])
astropy.io.fits.writeto(os.path.join(
gal_dir, 'Raw', band, 'Exp_Maps',
name + '_Herschel_' + band + '_Exp_Add.fits'),
add_image,
header=add_header,
clobber=True)
# Re-project final exposure map using Montage
montage_wrapper.reproject(
os.path.join(gal_dir, 'Raw', band, 'Exp_Maps',
name + '_Herschel_' + band + '_Exp_Add.fits'),
os.path.join(gal_dir, 'Raw', band, 'Exp_Maps',
name + '_Herschel_' + band + '_Exp.fits'),
header=os.path.join(gal_dir, 'Raw', band,
str(name) + '.hdr'),
exact_size=True)
# Convert final exposure time map into error map
err_image, err_header = astropy.io.fits.getdata(os.path.join(
gal_dir, 'Raw', band, 'Exp_Maps',
name + '_Herschel_' + band + '_Exp.fits'),
header=True)
err_image[np.where(err_image < 0)] = np.NaN
err_image = err_image**-0.5
err_image[np.where(err_image == np.inf)] = np.NaN
astropy.io.fits.writeto(os.path.join(
gal_dir, name + '_Herschel_' + band + '_Error.fits'),
err_image,
header=err_header,
clobber=True)
# Compress finalised exposure time map
os.chdir(out_dir)
if gzip:
os.system('gzip ' + os.path.join(
gal_dir, name + '_Herschel_' + band + '_Error.fits'))
print('Completed processing ' + name + '_Herschel_' + band +
' uncertainty map')
# In parallel, generate final standardised maps for each band
pool = mp.Pool(processes=9)
for key in bands_dict.keys():
band_dict = bands_dict[key]
#pool.apply_async( Herschel_Generator, args=(name, ra, dec, temp_dir, out_dir, band_dict, flux, thumbnails, gzip=gzip,) )
Herschel_Generator(name,
ra,
dec,
temp_dir,
out_dir,
band_dict,
flux,
thumbnails,
gzip=gzip)
pool.close()
pool.join()
# Clean memory, and return timings (if more than one target being processed)
gc.collect()
time_list.append(time.time())
time_est = ChrisFuncs.TimeEst(time_list, len(name_list))
if len(name) > 1:
print(
'Estimated time until Herschel data completed for all targets: '
+ time_est)
# Tidy up (best as we can)
gc.collect()
try:
shutil.rmtree(temp_dir)
except:
ChrisFuncs.RemoveCrawl(temp_dir)
print(
'Unable to fully tidy up temporary directory; probably due to NFS locks on network drive'
)
# Report completion
print('Total time elapsed: ' + str((time.time() - time_list[0]) / 3600.0) +
' hours')
print('All available Herschel imagery acquired for all targets')
os.chdir(bands)
os.mkdir('projected')
os.mkdir('diffdir')
os.mkdir('corrdir')
mt.mImgtbl('raw','rimages.tbl')
mt.mProjExec('rimages.tbl', '../'+bands+'.hdr', 'projected', 'stats.tbl',
raw_dir='raw')
mt.mImgtbl('projected', 'pimages.tbl')
len_dir = len(os.listdir('projected'))
if len_dir < 3 :
mt.mAdd('pimages.tbl', '../'+bands+'.hdr',
'../'+sour_name+'_'+bands+'.fits',
img_dir='projected')
else:
mt.mOverlaps('pimages.tbl', 'diffs.tbl')
mt.mDiffExec('diffs.tbl', '../'+bands+'.hdr', 'diffdir',
proj_dir = 'projected')
mt.mFitExec('diffs.tbl', 'fits.tbl', 'diffdir')
if ((len(os.listdir("diffdir")) > 1 and
os.path.getsize("diffdir/"+os.listdir("diffdir")[1]) < 10000) or
(len(os.listdir("diffdir")) < 1)) :
listPro = os.listdir('projected')
listPro = np.array(listPro)
fileSize = np.array(range(len(listPro)))
for ifile in range(len(listPro)):
fileSize[ifile] = os.path.getsize('projected/'+listPro[ifile])
proTable = Table([listPro, fileSize], names = ["Name", "size"])
proTable.sort(["size", "Name"])
shutil.copy('projected/'+proTable['Name'][-2],
'../'+sour_name+'_'+bands+'.fits')