def ned_query(ra, dec):
# getting skycoordinate object
co = coordinates.SkyCoord(ra=ra,
dec=dec,
unit=(u.deg, u.deg),
frame='icrs')
search_radius = 0.01 * u.deg
# this search radius is smaller than Fermi LAT resolution (please check)
# get table with all objects inside the search radius
result_table = Ned.query_region(co,
radius=search_radius,
equinox='J2000.0')
result_table = Ned.query_object("NGC 224")
print('names of the columns are ', result_table.colnames)
print('to get an impression, here the whole table:')
print(result_table)
# get all the object names to get speific data
object_names = result_table['Object Name']
print('identified objects = ', object_names)
# get table with positions of the first object as an example:
position_table = Ned.get_table(object_names[0], table='positions')
# position table is something you normally don't need but might be interesting
# This should always work'
# print('position table: ', position_table)
spectra = Ned.get_spectra("3c 273")
return result_table, spectra
def query_size(gal):
""" Search for largest size estimate on NED. """
try:
result_table = Ned.query_object(gal)
except:
return 0 * u.arcsec
ndiameters = result_table["Diameter Points"]
if ndiameters > 0:
diam = Ned.get_table(gal, table='diameters')
try:
units = [_ for _ in diam["Major Axis Unit"].data]
except:
units = [_.decode() for _ in diam["Major Axis Unit"].data]
idx = [
i for i, un in enumerate(units)
if un in ["arcmin", "arcsec", "degree"]
]
if len(idx) > 0:
diam = diam[idx]
sizes = [
d["Major Axis"] * u.Unit(d["Major Axis Unit"]) for d in diam
]
sizes = np.array([s.to("arcsec").value for s in sizes])
return np.nanmax(sizes) * u.arcsec
return 0 * u.arcsec
def make_sed(ned_name='OJ 287',
savefig='/home/sean/Downloads/images/oj287-sed.png'):
'''Plot the SED from NED data.'''
photometry = ned.get_table(ned_name, table='photometry')
frequency = photometry['Frequency'] # hertz
flux = photometry['Flux Density'] # jansky
nu_f_nu = np.array(frequency) * np.array(flux)
do_plotting(ned_name, frequency, nu_f_nu, savefig)
def ned_query(ra, dec):
# getting skycoordinate object
co = coordinates.SkyCoord(ra=ra,
dec=dec,
unit=(u.deg, u.deg),
frame='icrs')
search_radius = 0.01 * u.deg
# this search radius is smaller than Fermi LAT resolution (please check)
# get table with all objects inside the search radius
result_table = Ned.query_region(co,
radius=search_radius,
equinox='J2000.0')
result_table = Ned.query_object("NGC 224")
print('names of the columns are ', result_table.colnames)
print('to get an impression, here the whole table:')
print(result_table)
# get all the object names to get speific data
object_names = result_table['Object Name']
print('identified objects = ', object_names)
# get table with positions of the first object as an example:
position_table = Ned.get_table(object_names[0], table='positions')
# position table is something you normally don't need but might be interesting
# This should always work'
# print('position table: ', position_table)
# now the redshift is for many sources not available,will give you an error
# therefore we will use try and exept
for specific_object in object_names:
try:
redshift_table = Ned.get_table(specific_object, table='redshifts')
print('redshift found four ', specific_object)
print(redshift_table)
except:
print('no redshift for ', specific_object)
return result_table
def get_photometry(self):
"""
:return: Returns photometry data as astropy.table.Table` object.:
Available dict.keys() are:
['No.', 'Object Name', 'RA(deg)', 'DEC(deg)', 'Type', 'Velocity', 'Redshift',
'Redshift Flag', 'Magnitude and Filter', 'Distance (arcmin)', 'References', 'Notes',
'Photometry Points', 'Positions', 'Redshift Points', 'Diameter Points', 'Associations']
"""
return Ned.get_table(self.name)
def get_diameters(self):
"""
:return ` Returns data as astropy.table.Table` object.:
Available dict.keys() are:
['No.', 'Frequency targeted', 'Refcode', 'Major Axis', 'Major Axis Flag',
'Major Axis Unit', 'Minor Axis', 'Minor Axis Flag', 'Minor Axis Unit', 'Axis Ratio',
'Axis Ratio Flag', 'Major Axis Uncertainty', 'Ellipticity', 'Eccentricity',
'Position Angle', 'Equinox', 'Reference Level', 'NED Frequency', 'NED Major Axis',
'NED Major Axis Uncertainty', 'NED Axis Ratio', 'NED Ellipticity', 'NED Eccentricity',
'NED cos-1_axis_ratio', 'NED Position Angle', 'NED Minor Axis', 'Minor Axis Uncertainty',
'NED Minor Axis Uncertainty', 'Axis Ratio Uncertainty', 'NED Axis Ratio Uncertainty',
'Ellipticity Uncertainty', 'NED Ellipticity Uncertainty', 'Eccentricity Uncertainty',
'NED Eccentricity Uncertainty', 'Position Angle Uncertainty',
'NED Position Angle Uncertainty', 'Significance', 'Frequency', 'Frequency Unit',
'Frequency Mode', 'Detector Type', 'Fitting Technique', 'Features', 'Measured Quantity',
'Measurement Qualifiers', 'Targeted RA', 'Targeted DEC', 'Targeted Equinox',
'NED Qualifiers', 'NED Comment']
"""
return Ned.get_table(self.name, table="diameters")
def redshift_type(line, RA, DEC, un):
"""
Determine which type of redshift a source has associated with it. We do this
by doing a targeted search on each source and checking its redshift
measurements. If there is no such table, or if the error on the redshift
measurement is large, then the measurement is either missing or there are
very large errors (ie the redshift is photometric). Return the line if the
values are good
Input:
line: line in original NED batch search
Return
return line of the table if it contains good spectroscopic redshift;
else, return None
"""
try:
result_table = Ned.get_table(line["Object Name"], table="redshifts")
except:
return None
if any(result_table["Published Redshift Uncertainty"] < un):
return line[RA, DEC, "Redshift"]
else:
return None
def _ateam_flux(self):
"""
"""
fluxes = {}
for key in self.sources.keys():
if not NED_ENABLED:
warnings.warn(('Astroquery not loaded. '
'Sky model for {} is taken by default '
'at 38 MHz'.format(key)))
fluxes[key] = default[key]
continue
try:
spectrum = Ned.get_table(key if key != 'tau a' else 'crab',
table='photometry')
except:
raise Exception('Unable to find {} on NED'.format(key))
spectrum = spectrum[spectrum['Frequency'] < 2.e9]
from scipy.interpolate import interp1d
interp_spec = interp1d(spectrum['Frequency'] * 1e-6,
spectrum['Flux Density'])
fluxes[key] = interp_spec(self.frequency) * 1
return fluxes
def planck_button(
galaxies,
frequencies="all",
radius=None,
filepath=None,
download_data=True,
jy_conversion=True,
verbose=False,
):
"""Obtain Planck cutout, given a galaxy name.
Using a galaxy name and radius, uses SkyView to download a Planck
cutout.
Args:
galaxies (str or list): Names of galaxies to create mosaics for.
Resolved by NED.
frequencies (str or list, optional): Any combination of '030',
'044', '070', '100', '143', '217', '353', '545', '857'. If you
want everything, select 'all'. Defaults to 'all'.
radius (astropy.units.Quantity, optional): Radius around the
galaxy to search for observations. Defaults to None, where
it will query Ned to get size.
filepath (str, optional): Path to save the working and output
files to. If not specified, saves to current working
directory.
download_data (bool, optional): If True, will download data from
SkyView. Defaults to True.
jy_conversion (bool, optional): Convert the downloaded file from
raw units to Jy/pix. Defaults to True.
verbose (bool, optional): Print out messages during the process.
Useful mainly for debugging purposes or large images.
Defaults to False.
"""
if isinstance(galaxies, str):
galaxies = [galaxies]
if frequencies == "all":
frequencies = ['030','044','070','100','143','217','353','545','857']
if isinstance(frequencies, str):
frequencies = [frequencies]
if filepath is not None:
os.chdir(filepath)
if radius is not None:
original_radius = radius.copy()
else:
original_radius = None
steps = []
if download_data:
steps.append(1)
if jy_conversion:
steps.append(2)
for galaxy in galaxies:
if verbose:
print('Beginning '+galaxy)
if radius is None:
try:
size_query = Ned.get_table(galaxy,table='diameters')
radius = 1.2*np.max(size_query['NED Major Axis'])/2*u.arcsec
radius = radius.to(u.deg)
except:
raise Warning(galaxy+' not resolved by Ned, using 0.2deg radius.')
radius = 0.2*u.degree
if not os.path.exists(galaxy):
os.mkdir(galaxy)
for planck_freq in frequencies:
# Generate cutout size that will approximately Nyquist
# sample the map
fwhm = {'030':0.54,
'044':0.45,
'070':0.22,
'100':0.16,
'143':0.12,
'217':0.08,
'353':0.08,
'545':0.08,
'857':0.08}[planck_freq]
pixel_size = fwhm/3
n_pix = int(2*radius.value/pixel_size)
pixels = str(n_pix)+','+str(n_pix)
if verbose:
print('Beginning Planck '+planck_freq)
if not os.path.exists(galaxy + "/Planck/"):
os.mkdir(galaxy + "/Planck/")
if not os.path.exists(galaxy + "/Planck/" + planck_freq):
os.mkdir(galaxy + "/Planck/" + planck_freq)
if not os.path.exists(galaxy + "/Planck/" + planck_freq+'/outputs'):
os.mkdir(galaxy + "/Planck/" + planck_freq+'/outputs')
if 1 in steps:
if verbose:
print("Downloading data")
# Use SkyView to query and download a cutout around the
# galaxy.
planck_hdu = SkyView.get_images(position=galaxy,
survey='Planck '+planck_freq+' I',
radius=2*radius,
pixels=pixels,
cache=False,
)
planck_hdu[0].writeto(galaxy + "/Planck/" + planck_freq+'/outputs/'+galaxy+'.fits',
overwrite=True)
if 2 in steps:
if verbose:
print('Converting to Jy')
# Convert to Jy.
convert_to_jy(galaxy + "/Planck/" + planck_freq + "/outputs/"+galaxy+".fits",
planck_freq,
hdu_out=galaxy + "/Planck/"+galaxy+"_" + planck_freq+".fits")
if original_radius is None:
radius = None
else:
radius = original_radius.copy()
def sdss_button(
galaxies,
filters="all",
radius=None,
filepath=None,
download_data=True,
create_mosaic=True,
jy_conversion=True,
verbose=False,
**kwargs
):
"""Create an SDSS mosaic, given a galaxy name.
Using a galaxy name and radius, queries around that object,
downloads available SDSS data and mosaics into a final product.
Args:
galaxies (str or list): Names of galaxies to create mosaics for.
Resolved by NED.
filters (str or list, optional): Any combination of 'u', 'g',
'r', 'i', or 'z'. If you want everything, select 'all'.
Defaults to 'all'.
radius (astropy.units.Quantity, optional): Radius around the
galaxy to search for observations. Defaults to None, where
it will query Ned to get size.
filepath (str, optional): Path to save the working and output
files to. If not specified, saves to current working
directory.
download_data (bool, optional): If True, will download data using
MontagePy. Defaults to True.
create_mosaic (bool, optional): Switching this to True will
mosaic data as appropriate. Defaults to True.
jy_conversion (bool, optional): Convert the mosaicked file from
raw units to Jy/pix. Defaults to True.
verbose (bool, optional): Print out messages during the process.
Useful mainly for debugging purposes or large images.
Defaults to False.
"""
if isinstance(galaxies, str):
galaxies = [galaxies]
if filters == "all":
filters = ["u", "g", "r", "i", "z"]
if isinstance(filters, str):
filters = [filters]
if filepath is not None:
os.chdir(filepath)
if radius is not None:
original_radius = radius.copy()
else:
original_radius = None
steps = []
if download_data:
steps.append(1)
if create_mosaic:
steps.append(2)
if jy_conversion:
steps.append(3)
# Read in the list of SDSS Primary Fields
run, _, camcol, field = np.loadtxt(
os.path.dirname(os.path.realpath(__file__))
+'/SDSS_DR12_Primary_Fields.dat',
unpack=True)
for galaxy in galaxies:
if verbose:
print('Beginning '+galaxy)
if radius is None:
try:
size_query = Ned.get_table(galaxy,table='diameters')
radius = 1.2*np.max(size_query['NED Major Axis'])/2*u.arcsec
radius = radius.to(u.deg)
except:
raise Warning(galaxy+' not resolved by Ned, using 0.2deg radius.')
radius = 0.2*u.degree
if not os.path.exists(galaxy):
os.mkdir(galaxy)
for sdss_filter in filters:
if verbose:
print('Beginning SDSS_'+sdss_filter)
if not os.path.exists(galaxy + "/SDSS/"):
os.mkdir(galaxy + "/SDSS/")
if not os.path.exists(galaxy + "/SDSS/" + sdss_filter):
os.mkdir(galaxy + "/SDSS/" + sdss_filter)
if not os.path.exists(galaxy + "/SDSS/" + sdss_filter+'/raw'):
os.mkdir(galaxy + "/SDSS/" + sdss_filter+'/raw')
if not os.path.exists(galaxy + "/SDSS/" + sdss_filter+'/outputs'):
os.mkdir(galaxy + "/SDSS/" + sdss_filter+'/outputs')
if 1 in steps:
if verbose:
print("Downloading data")
# Montage uses its size as the length of the square, since
# we want a radius use twice that.
mArchiveDownload(
"SDSS " + sdss_filter,
galaxy,
2 * radius.value,
galaxy + "/SDSS/" + sdss_filter+'/raw',
)
# Filter out any frames that aren't primary
sdss_files = glob.glob(galaxy + "/SDSS/" + sdss_filter+'/raw/*.fits')
for sdss_file in sdss_files:
sdss_file_strip = sdss_file.split('/')[-1].split('.fits')[0]
sdss_file_split = sdss_file_strip.split('-')
primary_idx = np.where( (run == int(sdss_file_split[2])) &
(camcol == int(sdss_file_split[3])) &
(field == int(sdss_file_split[4])))
if len(primary_idx[0]) == 0:
os.remove(sdss_file)
if 2 in steps:
# Mosaic all these files together.
if verbose:
print("Beginning mosaic")
mHdr(
galaxy,
2 * radius.value,
2 * radius.value,
galaxy + "/SDSS/"+sdss_filter+"/outputs/header.hdr",
resolution=0.4,
)
tools.mosaic(
galaxy + "/SDSS/" + sdss_filter+'/raw',
header=galaxy + "/SDSS/"+sdss_filter+"/outputs/header.hdr",
verbose=verbose,
**kwargs
)
os.rename("mosaic/mosaic.fits",
galaxy + "/SDSS/" + sdss_filter + "/outputs/"+galaxy+".fits")
# Clear out the mosaic folder.
shutil.rmtree("mosaic/", ignore_errors=True)
if 3 in steps:
if verbose:
print('Converting to Jy')
# Convert to Jy.
convert_to_jy(galaxy + "/SDSS/" + sdss_filter + "/outputs/"+galaxy+".fits",
sdss_filter,
hdu_out=galaxy + "/SDSS/"+galaxy+"_" + sdss_filter+".fits")
if original_radius is None:
radius = None
else:
radius = original_radius.copy()
def galex_button(
galaxies,
filters="both",
radius=None,
filepath=None,
download_data=True,
create_mosaic=True,
jy_conversion=True,
verbose=False,
):
"""Create a GALEX mosaic, given a galaxy name.
Using a galaxy name and radius, queries around that object,
downloads available GALEX data and mosaics into a final product.
Because GALEX images are in counts/s and the integrations may be
different lengths, we convert back to a raw count, add the frames
and convert back to counts/s at the end. This effectively weights
the frame by exposure time.
Args:
galaxies (str or list): Names of galaxies to create mosaics for.
Resolved by NED.
filters (str, optional): One of 'FUV', 'NUV', or 'both'. Selects
which GALEX filters to create a mosaic for. Defaults to
'both'.
radius (astropy.units.Quantity, optional): Radius around the
galaxy to search for observations. Defaults to None, where
it will query Ned to get size.
filepath (str, optional): Path to save the working and output
files to. If not specified, saves to current working
directory.
download_data (bool, optional): If True, will download data from
MAST. Defaults to True.
create_mosaic (bool, optional): Switching this to True will
mosaic data as appropriate. Defaults to True.
jy_conversion (bool, optional): Convert the mosaicked file from
raw units to Jy/pix. Defaults to True.
verbose (bool, optional): Print out messages during the process.
Useful mainly for debugging purposes or large images.
Defaults to False.
"""
if isinstance(galaxies, str):
galaxies = [galaxies]
if filters == "both":
filters = ["NUV", "FUV"]
else:
filters = [filters]
if filepath is not None:
os.chdir(filepath)
if radius is not None:
original_radius = radius.copy()
else:
original_radius = None
steps = []
if download_data:
steps.append(1)
if create_mosaic:
steps.append(2)
if jy_conversion:
steps.append(3)
for galaxy in galaxies:
if verbose:
print('Beginning ' + galaxy)
if radius is None:
try:
size_query = Ned.get_table(galaxy, table='diameters')
radius = 1.2 * np.max(
size_query['NED Major Axis']) / 2 * u.arcsec
radius = radius.to(u.deg)
except:
raise Warning(galaxy +
' not resolved by Ned, using 0.2deg radius.')
radius = 0.2 * u.degree
if not os.path.exists(galaxy):
os.mkdir(galaxy)
obs_table = Observations.query_criteria(objectname=galaxy,
radius=radius,
obs_type='all',
obs_collection='GALEX')
# Ignore any calibration observations.
obs_table = obs_table[obs_table['intentType'] == 'science']
for galex_filter in filters:
if verbose:
print('Beginning GALEX ' + galex_filter)
if 1 in steps:
# Pull out available data, and download it
query_results = np.where(
obs_table["filters"] == galex_filter)[0]
# If there isn't any GALEX coverage, just skip
if len(query_results) == 0:
print(galaxy + " missing!")
continue
# We only want to print out download messages if
# verbose is True, so redirect otherwise.
if not verbose:
sys.stdout = open(os.devnull, 'w')
dataProductsByID = Observations.get_product_list(
obs_table[query_results])
Observations.download_products(
dataProductsByID,
download_dir="galex_temp/" + galaxy,
mrp_only=True,
)
# And set back to the original for printing.
if not verbose:
sys.stdout = sys.__stdout__
if not os.path.exists(galaxy + "/GALEX/"):
os.mkdir(galaxy + "/GALEX/")
if not os.path.exists(galaxy + "/GALEX/" + galex_filter):
os.mkdir(galaxy + "/GALEX/" + galex_filter)
if not os.path.exists(galaxy + "/GALEX/" + galex_filter +
'/raw'):
os.mkdir(galaxy + "/GALEX/" + galex_filter + '/raw')
if not os.path.exists(galaxy + "/GALEX/" + galex_filter +
'/data'):
os.mkdir(galaxy + "/GALEX/" + galex_filter + '/data')
if not os.path.exists(galaxy + "/GALEX/" + galex_filter +
'/reprojected'):
os.mkdir(galaxy + "/GALEX/" + galex_filter +
'/reprojected')
if not os.path.exists(galaxy + "/GALEX/" + galex_filter +
'/weight'):
os.mkdir(galaxy + "/GALEX/" + galex_filter + '/weight')
if not os.path.exists(galaxy + "/GALEX/" + galex_filter +
'/outputs'):
os.mkdir(galaxy + "/GALEX/" + galex_filter + '/outputs')
ext_name = {"NUV": "nd", "FUV": "fd"}[galex_filter]
# Pull out the relevant filter files files (either *nd*
# or *fd*), extract and move to base folder.
matches = []
for root, _, filenames in os.walk("galex_temp/" + galaxy):
for filename in fnmatch.filter(
filenames, "*" + ext_name + "-int.fits.gz"):
matches.append(os.path.join(root, filename))
for match in matches:
with gzip.open(match, "rb") as f_in:
with open(match[:-3], "wb") as f_out:
shutil.copyfileobj(f_in, f_out)
filename = match[:-3].split('/')
os.rename(
match[:-3],
galaxy + "/GALEX/" + galex_filter + "/raw/" +
filename[-1],
)
# Clean up any temporary files.
shutil.rmtree("galex_temp/" + galaxy, ignore_errors=True)
mHdr(
galaxy,
2 * radius.value,
2 * radius.value,
galaxy + "/GALEX/" + galex_filter + "/outputs/header.hdr",
resolution=1.5,
)
if 2 in steps:
# Read in these files and set anything more than 35
# arcmin out to NaN.
if verbose:
print('Performing initial weighted reprojections')
galex_files = glob.glob(galaxy + "/GALEX/" + galex_filter +
"/raw/*")
for galex_file in galex_files:
hdu = fits.open(galex_file)[0]
i = np.linspace(-hdu.data.shape[0] / 2,
hdu.data.shape[0] / 2, hdu.data.shape[0])
j = np.linspace(-hdu.data.shape[1] / 2,
hdu.data.shape[1] / 2, hdu.data.shape[1])
iv, jv = np.meshgrid(i, j)
r = iv**2 + jv**2
hdu.data[r >= 1400**2] = np.nan
hdu.writeto(galex_file.replace('/raw/', '/data/'),
overwrite=True)
# Also create a weight map (sqrt EXPTIME).
exp_time = np.ones(
hdu.data.shape) * hdu.header['EXPTIME']**0.5
exp_time[np.isnan(hdu.data) == True] = np.nan
fits.writeto(galex_file.replace('/raw/', '/weight/'),
exp_time,
hdu.header,
overwrite=True)
# And reproject each map separately using this weighting.
mProject(galex_file.replace('/raw/', '/data/'),
galex_file.replace('/raw/', '/reprojected/'),
galaxy + "/GALEX/" + galex_filter +
"/outputs/header.hdr",
weight_file=galex_file.replace(
'/raw/', '/weight/'))
# And mosaic!
# Montage uses its size as the length of the square,
# since we want a radius use twice that.
mHdr(
galaxy,
2 * radius.value,
2 * radius.value,
galaxy + "/GALEX/" + galex_filter + "/outputs/header.hdr",
resolution=1.5,
)
tools.mosaic(
galaxy + "/GALEX/" + galex_filter + '/reprojected',
header=galaxy + "/GALEX/" + galex_filter +
"/outputs/header.hdr",
verbose=verbose,
reproject=False,
haveAreas=True,
)
os.rename(
"mosaic/mosaic.fits", galaxy + "/GALEX/" + galex_filter +
"/outputs/" + galaxy + '.fits')
shutil.rmtree("mosaic/", ignore_errors=True)
if 3 in steps:
if verbose:
print('Converting to Jy')
# Convert to Jy.
convert_to_jy(galaxy + "/GALEX/" + galex_filter + '/outputs/' +
galaxy + ".fits",
galex_filter,
hdu_out=galaxy + "/GALEX/" + galaxy + '_' +
galex_filter + ".fits")
if original_radius is None:
radius = None
else:
radius = original_radius.copy()
def hst_button(
galaxies,
skymethod='globalmin+match',
instruments="ACS/WFC",
prop_ids=None,
filters=None,
radius=None,
filepath=None,
download_data=True,
correct_astrometry=True,
create_mosaic=True,
jy_conversion=True,
verbose=False,
log_filename='hst.log',
):
"""Create a HST mosaic, given a galaxy name.
Using a galaxy name and radius, queries around that object,
downloads available HST data and mosaics into a final product. It
will create separate mosaics for each proposal ID, and the file structure
will look like ``/galaxy/HST/proposal_id/galaxy_instrument_filter_proposal_id.fits``.
N.B. I must confess to not being well-versed with HST data, so if
anyone can help improve this please let me know.
This data button uses a number of tools included in the drizzlepac
Python package. This includes alignimages/tweakreg and astrodrizzle,
which correct astrometry and are specifically tailored for the setup
of HST data. This means that 1) creating mosaics with this will likely
take a long time and 2) you will need a beefy computer (especially with
regards to hard drive space).
Args:
galaxies (str or list): Names of galaxies to create mosaics for.
Resolved by NED.
skymethod (str, optional): Method used for AstroDrizzle's background
matching step. In general, this can be left untouched but for
mosaics with little overlap, it may be worth playing around
with this. For instance, I've had some luck when there isn't
much overlap between exposures using 'globalmin'. Options are
'localmin', 'globalmin+match', 'globalmin', and 'match'. Defaults
to 'globalmin+match'.
instruments (str or list, optional): Instrument to download data
for. Can be any combination of 'ACS/WFC', 'WFC3/IR',
'WFC3/UVIS', 'WFPC2/PC', or 'WFPC2/WFC'. If you want all
available data for all these instruments, select 'all', but
this is not recommended! Defaults to 'ACS/WFC'.
prop_ids (str or list, optional): Proposal IDs to download data for.
Defaults to None, which will pull out all proposal IDs for each
instrument.
filters (str or list, optional): Filters to download data for.
The script will look for each filter, for each instrument.
Defaults to None, which will pull out all applicable filters
for each instrument, for each proposal ID.
radius (astropy.units.Quantity, optional): Radius around the
galaxy to search for observations. Defaults to None, where
it will query Ned to get size.
filepath (str, optional): Path to save the working and output
files to. If not specified, saves to current working
directory.
download_data (bool, optional): If True, will download data from
MAST. Defaults to True.
correct_astrometry (bool, optional): If True, will perform astrometric
corrections to the downloaded data using alignimages. Defaults
to True.
create_mosaic (bool, optional): Switching this to True will
mosaic the data using astrodrizzle as appropriate. Defaults
to True.
jy_conversion (bool, optional): Convert the mosaicked file from
raw units to Jy/pix. Defaults to True.
verbose (bool, optional): Can be used to suppress most of the
output messages produced during the process. Mainly useful
for debugging. Defaults to False.
log_filename (str, optional): Will produce a stripped down log
of what data the code is reducing. By default, will save to
galaxy/hst.log.
"""
if isinstance(galaxies, str):
galaxies = [galaxies]
if isinstance(instruments,str):
instruments = [instruments]
if instruments == 'all':
instruments = ['ACS/WFC',
'WFC3/IR','WFC3/UVIS',
'WFPC2/PC','WFPC2/WFC']
if isinstance(filters,str):
filters = [filters]
if isinstance(prop_ids,str):
prop_ids = [prop_ids]
if filepath is not None:
os.chdir(filepath)
orig_dir = os.getcwd()
if radius is not None:
original_radius = radius.copy()
else:
original_radius = None
steps = []
if download_data:
steps.append(1)
if correct_astrometry:
steps.append(2)
if create_mosaic:
steps.append(3)
if jy_conversion:
steps.append(4)
# Set up folders for various corrections
os.environ['CRDS_SERVER_URL'] = 'https://hst-crds.stsci.edu'
os.environ['CRDS_PATH'] = orig_dir+'/reference_files'
os.environ['iref'] = orig_dir+'/reference_files/references/hst/wfc3/'
os.environ['jref'] = orig_dir+'/reference_files/references/hst/acs/'
os.environ['uref'] = orig_dir+'/reference_files/references/hst/wfpc2/'
# For large proposals, astrodrizzle can run into file open
# issues so raise the max file open amount.
_, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE,(hard,hard))
# Change the temp directory -- if this gets filled up it can cause
# problems.
orig_tmpdir = os.environ['TMPDIR']
if not os.path.exists('tmp'):
os.mkdir('tmp')
os.environ['TMPDIR'] = orig_dir+'/tmp'
for galaxy in galaxies:
if not os.path.exists(galaxy):
os.mkdir(galaxy)
if not os.path.exists(galaxy+'/HST'):
os.mkdir(galaxy+'/HST')
if not verbose:
# Various packages used here put out a lot of messages. Silence info messages.
loggers = [logging.getLogger(name) for name in logging.root.manager.loggerDict]
for logger in loggers:
logger.setLevel(logging.ERROR)
# Even if verbose is not True, still print out some useful messages to the
# console.
hst_logger = logging.getLogger('data_buttons')
handler = logging.FileHandler(galaxy+'/'+log_filename,mode='w')
hst_logger.addHandler(handler)
hst_logger.addHandler(logging.StreamHandler())
hst_logger.setLevel(logging.INFO)
hst_logger.info('Beginning '+galaxy)
hst_logger.info(' ')
hst_logger.info(' ')
if radius is None:
try:
size_query = Ned.get_table(galaxy,table='diameters')
radius = np.max(size_query['NED Major Axis'])/2*u.arcsec
radius = radius.to(u.deg)
except:
hst_logger.warning(galaxy+' not resolved by Ned, using 0.2deg radius.')
radius = 0.2*u.degree
obs_table = Observations.query_criteria(objectname=galaxy,
radius=radius,
obs_type='all',
obs_collection='HST')
# Ignore any calibration observations.
obs_table = obs_table[obs_table['intentType'] == 'science']
for instrument in instruments:
# Pixel sizes for final mosaics selected to match the HLA.
pix_size = {'ACS/HRC':0.025,
'ACS/SBC':0.03,
'ACS/WFC':0.05,
'NICMOS/NIC1':0.025,
'NICMOS/NIC2':0.05,
'NICMOS/NIC3':0.1,
'WFC3/IR':0.09,
'WFC3/UVIS':0.04,
'WFPC2/PC':0.05,
'WFPC2/WFC':0.1}[instrument]
# Bits to consider good for drizzling.
bits = {'ACS/HRC':256,
'ACS/SBC':256,
'ACS/WFC':256,
'NICMOS/NIC1':0,
'NICMOS/NIC2':0,
'NICMOS/NIC3':0,
'WFC3/IR':768,
'WFC3/UVIS':256,
'WFPC2/PC':'8,1024',
'WFPC2/WFC':'8,1024'}[instrument]
# Filename extension, in order of preference.
suffixes = {'ACS/WFC':['FLC','FLT'],
'WFC3/IR':['FLT'],
'WFC3/UVIS':['FLC','FLT'],
'WFPC2/PC':[['C0M','C1M']],
'WFPC2/WFC':[['C0M','C1M']],
}[instrument]
# The instruments often have / in the name, so account for
# this in making folders and files.
hst_logger.info('Beginning '+instrument)
if not os.path.exists(galaxy+'/HST/'+instrument.replace('/','_')):
os.mkdir(galaxy+'/HST/'+instrument.replace('/','_'))
reset_filters = False
instrument_table = obs_table[obs_table['instrument_name'] == instrument]
reset_prop_ids = False
if not prop_ids:
prop_ids = list(np.unique(instrument_table['proposal_id']))
reset_prop_ids = True
hst_logger.info('Available proposal IDs: '+','.join(prop_ids))
hst_logger.info(' ')
for prop_id in prop_ids:
hst_logger.info('Proposal ID: '+str(prop_id))
prop_table = instrument_table[instrument_table['proposal_id'] == prop_id]
if not filters:
filters = list(np.unique(prop_table['filters']))
reset_filters = True
hst_logger.info('Available filters: '+','.join(filters))
for hst_filter in filters:
# If we have a highly illegal filter, just skip.
# TODO: This needs to be sorted for some fringe
# cases, probably.
if not hst_filter[0] == 'F':
continue
hst_logger.info('Filter: '+str(hst_filter))
# Pull out available data and download.
filter_table = prop_table[prop_table['filters'] == hst_filter]
if len(filter_table) == 0:
hst_logger.warning('No available data to download. Skipping...')
continue
data_products_id = Observations.get_product_list(filter_table)
for suffix in suffixes:
download_table = Observations.filter_products(data_products_id,
productSubGroupDescription=suffix,
mrp_only=False)
if len(download_table) > 0:
break
if isinstance(suffix,list):
filename_exts = [ext.lower() for ext in suffix]
else:
filename_exts = [suffix.lower()]
hst_logger.info(instrument+'/'+prop_id+'/'+hst_filter)
if not os.path.exists(galaxy+
'/HST/'+
instrument.replace('/','_')+
'/'+
hst_filter):
os.mkdir(galaxy+
'/HST/'+
instrument.replace('/','_')+
'/'+
hst_filter)
if not os.path.exists(galaxy+'/HST/'+prop_id):
os.mkdir(galaxy+'/HST/'+prop_id)
full_filepath = (galaxy+
'/HST/'+
instrument.replace('/','_')+
'/'+
hst_filter+
'/'
+prop_id)
if not os.path.exists(full_filepath):
os.mkdir(full_filepath)
if 1 in steps:
# Download files
download_mast(download_table,
download_dir="hst_temp/" + galaxy)
if not os.path.exists(full_filepath+'/raw'):
os.mkdir(full_filepath+'/raw')
if not os.path.exists(full_filepath+'/outputs'):
os.mkdir(full_filepath+'/outputs')
# Pull out the relevant files, and move to base folder.
for filename_ext in filename_exts:
matches = []
for root, _, filenames in os.walk("hst_temp/" + galaxy):
for filename in fnmatch.filter(
filenames, "*_"+filename_ext+".fits"
):
matches.append(os.path.join(root, filename))
for match in matches:
filename = match.split('/')
os.rename(match,full_filepath+'/raw/'+filename[-1])
# Clean up any temporary files.
shutil.rmtree("hst_temp/" + galaxy, ignore_errors=True)
filename_ext = filename_exts[0]
hst_files = glob.glob(full_filepath+'/raw/*_'+filename_ext+'.fits')
if 2 in steps:
# First, update the WCS information in case it's
# required.
for filename_ext in filename_exts:
hst_files = glob.glob(full_filepath+'/raw/*_'+filename_ext+'.fits')
crds.assign_bestrefs(hst_files,
sync_references=True)
# For WFPC2, the CRDS doesn't download everything
# needed. Download the GEIS data files and
# rerun the bestrefs assignment.
if 'WFPC2' in instrument:
geis_hdrs = glob.glob(os.environ['uref']+'/*h')
for geis_hdr in geis_hdrs:
geis_data = geis_hdr[:-1]+'d'
if not os.path.exists(geis_data):
geis_data = geis_data.split('/')[-1]
print(geis_data)
print(os.environ['uref'])
wget.download(
os.environ['CRDS_SERVER_URL']+'/unchecked_get/references/hst/'+geis_data,
out=os.environ['uref'])
crds.assign_bestrefs(hst_files,sync_references=True)
for hst_file in hst_files:
stwcs.updatewcs.updatewcs(hst_file,
use_db=False)
os.chdir(full_filepath+'/raw')
filename_ext = filename_exts[0]
hst_files = glob.glob('*_'+filename_ext+'.fits')
# Normalize all files.
photeq.photeq(', '.join(hst_files),readonly=False)
os.rename('photeq.log','../outputs/photeq.log')
if 'WFPC' in instrument:
# Using tweakreg, align each frame to GAIA.
gaia_table = Gaia.query_object_async(coordinate=galaxy,
radius=2*radius)
ras = gaia_table['ra']
decs = gaia_table['dec']
source_table = Table([ras,decs])
source_table.write('gaia.cat',
format='ascii.fast_commented_header')
tweakreg.TweakReg(hst_files,
imagefindcfg={'threshold':5,'conv_width':3},
refcat='gaia.cat',
#expand_refcat=True,
enforce_user_order=False,
shiftfile=True,
outshifts='shifts.txt',
searchrad=10,
minobj=5,
separation=0,
updatehdr=True,
reusename=True,
wcsname='TWEAK',
interactive=False,
fitgeometry='general',
clean=True,
see2dplot=False
)
# Update the c1m files to use the TWEAK
# wcs
for hst_file in hst_files:
dq_file = hst_file.replace('c0','c1')
tweakback.tweakback(hst_file,
dq_file,
newname='TWEAK')
plot_files = glob.glob('*.png')
for plot_file in plot_files:
os.remove(plot_file)
cat_files = glob.glob('*.coo')
for cat_file in cat_files:
os.remove(cat_file)
os.rename('shifts_wcs.fits','../outputs/shifts_wcs.fits')
os.rename('tweakreg.log','../outputs/tweakreg.log')
os.rename('shifts.txt','../outputs/shifts.txt')
elif 'ACS' in instrument or 'WFC3' in instrument:
# Correct astrometry using alignimages. First,
# correct each frame separately.
pool = mp.Pool(mp.cpu_count())
suitable_hst_files = pool.map(astrometric_correction,
hst_files)
pool.close()
suitable_hst_files = [x for x in suitable_hst_files
if x is not None]
if len(suitable_hst_files) == 0:
hst_logger.warning('Failure with astrometry corrections. Skipping')
os.chdir(orig_dir)
continue
# Now, align every suitable frame simultaneously.
output_table = astrometric_correction(suitable_hst_files)
with open('../outputs/astrometry.pkl','wb') as table_file:
pickle.dump(output_table,table_file)
else:
raise Exception('Unknown instrument!')
os.chdir(orig_dir)
os.chdir(full_filepath)
if 3 in steps:
os.chdir('raw')
if 'WFPC2' in instrument:
hst_files = glob.glob('*_c0m.fits')
wcskey = 'TWEAK'
elif 'ACS' in instrument or 'WFC3' in instrument:
with open('../outputs/astrometry.pkl','rb') as table_file:
output_table = pickle.load(table_file)
# We only want fits where an acceptable astrometric
# solution has been found.
suitable_fits = np.where(output_table['fit_qual'] < 5)
# (output_table['fit_qual'] >= 1)
hst_files = list(output_table[suitable_fits]['imageName'])
if len(output_table[suitable_fits]) == 0:
hst_logger.warning('Failure with astrometry corrections. Skipping')
os.chdir(orig_dir)
continue
wcskey = ' '
else:
raise Exception('Unknown instrument!')
# Following Dalcanton+ (2012), group exposures into
# long (>50s) and short (<=50s), and process for cosmic
# rays separately
exp_times = []
for hst_file in hst_files:
hdu = fits.open(hst_file)[0]
exp_time = hdu.header['EXPTIME']
exp_times.append(exp_time)
for exp_group in ['short','long']:
hst_files_group = []
for i in range(len(exp_times)):
if exp_times[i] > 50 and exp_group == 'long':
hst_files_group.append(hst_files[i])
elif exp_times[i] <= 50 and exp_group == 'short':
hst_files_group.append(hst_files[i])
if len(hst_files_group) == len(hst_files):
exp_group = ''
if len(hst_files_group) == 0:
continue
if len(exp_group) > 0:
output_name = '../outputs/'+galaxy+'_'+exp_group
drizzle_log_name = '../outputs/astrodrizzle_'+exp_group+'.log'
else:
output_name = '../outputs/'+galaxy
drizzle_log_name = '../outputs/astrodrizzle.log'
# Perform the mosaicking. Generally, use iminmed.
# However, sometimes iminmed will fail so
# for the other instruments we'll use imedian as
# a fallback.
combine_types = ['iminmed','imedian']
if 'WFPC2' in instrument:
combine_nhigh = 1
else:
combine_nhigh = 0
for combine_type in combine_types:
try:
astrodrizzle.AstroDrizzle(
input=hst_files_group,
output=output_name,
preserve=False,
clean=True,
combine_type=combine_type,
combine_nhigh=combine_nhigh,
skymethod=skymethod,
sky_bits=bits,
driz_sep_bits=bits,
driz_sep_fillval=99999,
combine_hthresh=90000,
final_scale=pix_size,
final_bits=bits,
final_fillval=0,
wcskey=wcskey,
final_rot=0,
)
break
except ValueError:
pass
# Move the AstroDrizzle log.
os.rename('astrodrizzle.log',
drizzle_log_name)
# Move back to the original directory.
os.chdir(orig_dir)
if 4 in steps:
mosaic_outputs = glob.glob(full_filepath+'/outputs/*_sci.fits')
for mosaic_output in mosaic_outputs:
# Replace any fillvals with NaNs.
hdu = fits.open(mosaic_output)[0]
hdu.data[hdu.data == 0] = np.nan
fits.writeto(mosaic_output,
hdu.data,hdu.header,
overwrite=True)
if '_long_' in mosaic_output.split('/')[-1]:
new_filename = (galaxy+
'/HST/'
+prop_id
+'/'
+galaxy
+'_'
+instrument.replace('/','_')
+'_'
+hst_filter
+'_'
+prop_id
+'_long.fits')
elif '_short_' in mosaic_output.split('/')[-1]:
new_filename = (galaxy+
'/HST/'
+prop_id
+'/'
+galaxy
+'_'
+instrument.replace('/','_')
+'_'
+hst_filter
+'_'
+prop_id
+'_short.fits')
else:
new_filename = (galaxy+
'/HST/'
+prop_id
+'/'
+galaxy
+'_'
+instrument.replace('/','_')
+'_'
+hst_filter
+'_'
+prop_id
+'.fits')
convert_to_jy(mosaic_output,
new_filename)
if reset_filters:
filters = None
hst_logger.info(' ')
if reset_prop_ids:
prop_ids = None
hst_logger.info(' ')
if original_radius is None:
radius = None
else:
radius = original_radius.copy()
# Clear out the tmp folder and reset to the original.
shutil.rmtree('tmp/', ignore_errors=True)
os.environ['TMPDIR'] = orig_tmpdir
def Area_GC_R_N(Gname):
"""
Gname: str- The name of the galaxy in the form NGC #
Data: array- a table of data containg the coordinates of each object
This fucntion takes the inputs for the name of the galaxy and returns a histrogram that plots the number of objects per bin by the
area enclosed by the cirlce centered on the center of the galaxy that inculdes the objects in each bin in
square degrees divided by the visible area of the galaxy in square degrees.
This function plots the visible Major axis of the galaxy area enclosed by a circle that uses the Major axis as
the diameter of the cirlce divided by itself to give 1 on histogram.
This function uses astroquary to get data directly from NED
#THIS IS THE CURRENT RUNNING VERSION OF THIS CODE
"""
import math
from astropy.io import ascii
import matplotlib.pyplot as plt
system('pwd')
#system('cd ~/Desktop/SQL_Standard_File/')
#import os
dir = os.path.dirname(__file__)
#filename= os.path.join(dir, '~','Desktop','SQL_Standard_File',)
#filepath=os.path.abspath("~/Desktop/SQL_Standard_File")
#print "Filepath =",filepath
#path= os.path.join(dir,'~','Desktop','SQL_Standard_File',)
#path=os.path.realpath('~/Desktop/SQL_Standard_File/SQL_Sandard_File.csv')
path = os.path.realpath('../SQL_Standard_File/SQL_Sandard_File.csv')
print "Path=", path
#os.chdir(path)
#os.chdir('~/Desktop/SQL_Standard_File/')
#system('cd ~/Desktop/Big_Object_Regions/')
#system('cd ../SQL_Standard_File/')
system('pwd')
#system('ls')
#data = ascii.read("SQL_Sandard_File.csv") #data:-astropy.table.table.Table, data, The data from the SQL_Standard_File
#data = ascii.read(filename) #data:-astropy.table.table.Table, data, The data from the SQL_Standard_File
#data = ascii.read(filepath) #data:-astropy.table.table.Table, data, The data from the SQL_Standard_File
data = ascii.read(
path
) #data:-astropy.table.table.Table, data, The data from the SQL_Standard_File
#data2=open("SQL_Sandard_File.csv","r")
#print data2
#system('cd ~/Desktop/galaxies/out')
RA_A = data[
'sourceRA'] #RA_A:-astropy.table.column.Column, Right_Ascension_Array, The array containing all Right Ascensions in the SQL Standard File
#print type(RA_A)
RA_L = list(
RA_A
) #RA_L:-list, Right_Ascension_List, The list containing all Right Ascensions in the SQL Standard File
#print RA_L
Dec_A = data[
'sourceDec'] #Dec_A:-astropy.table.column.Column, Declination_Array, The array containing all Declinations in the SQL Standard File
Dec_L = list(
Dec_A
) #Dec_L:-List, Declination_List, The list containing all Declinations in the SQL Standard File
#print Dec_L
#Obs_ID_A=data["obsid"] #Obs_ID_A:-astropy.table.column.Column, Observation_Idenification_Array, The array containing all Observation IDs in the SQL_Standard_File (not indexable)
#print type(Obs_ID_A)
#Obs_ID_L=list(Obs_ID_A) #Obs_ID_L:-List, Observation_Idenification_List, The list containing all Observation IDs in the SQL_Standard_File (So it is indexable)
#print "Obs_ID_L ", Obs_ID_L
#print type(Obs_ID_L)
#print Obs_ID_A
#FGname_A=data["foundName"]
#FGname_L=list(FGname_A)
#print FGname_A
QGname_A = data[
"queriedName"] #QGname_A:-Obs_ID_A:-astropy.table.column.Column, Query_Galaxy_Name_Array, The array containing all Query Galaxy Names in the SQL_Standard_File (not indexable)
QGname_L = list(
QGname_A
) #QGname_L:-List, Query_Galaxy_Name_Array, The list containing all Query Galaxy Names in the SQL_Standard_File (So it is indexable)
#print type(QGname_A)
#print QGname_A
Matching_Index_List = [
] #Matching_Index_List:-List, Matching_Index_List, The list of all indexes (ref. QGname_L) that corresepond to the input Galaxy Name, All arrays are of equal lenth, and "ith" value of an array is the correseponding value for any other arrays "ith" value, so for example Obs_ID_L[228]=794 and the Galaxy in the Observation is QGname_L[228]="NGC 891", Note both lists have the same index
for i in range(0, len(QGname_L)): # i:-int, i, the "ith" index of QGname_L
#print "i ", i
QGname = QGname_L[
i] #QGname:-string, Query_Galaxy_Name, The current test Galaxy Name, if this Galaxy name equals the input Galaxy Name (Gname) then this Matching_Index, i (ref. QGname_L) will be appended to the Matching_Index_List
#QGname_Reduced=QGname.replace(" ", "")
#print "QGname ", QGname
#print "QGname_Reduced ", QGname_Reduced
if (
Gname == QGname
): #Checks to see if the current test Galaxy Name is the same as the input Galaxy Name, if so it appends the current index (ref. QGname_L) to the Matching_Index_List
#print "i ", i
Matching_Index_List.append(
i
) #Appends the current index (ref. QGname_L) to the Matching_Index_List
RA_Match_L = [
] #RA_Match_L:-List, Right_Ascension_Match_List, The list of all source RA's for the input Galaxy Name in decimal degrees
Dec_Match_L = [
] #Dec_Match_L:-List, Declination_Match_List, The list of all source Dec's for the input Galaxy Name in decimal degrees
for Cur_Matching_Index in Matching_Index_List: #Cur_Matching_Index:-int, Current_Matching_Index, The current index (ref. QGname_L) in the list of matching indexes for the current input Galaxy Name (Matching_Index_List)
Cur_Match_RA = RA_L[
Cur_Matching_Index] #Cur_Match_RA:-numpy.float64, Current_Match_Right_Ascension, The RA of the current source in decimal degrees
#print type(Cur_Match_RA)
Cur_Match_Dec = Dec_L[
Cur_Matching_Index] #Cur_Match_Dec:-numpy.float64, Current_Match_Declination, The Dec of the current source in decimal degrees
RA_Match_L.append(
Cur_Match_RA
) #RA_Match_L:-list, Right_Ascension_Match_List, The list of all source RA's for the input Galaxy Name in decimal degrees
Dec_Match_L.append(
Cur_Match_Dec
) #Dec_Match_L:-list, Declination_Match_List, The list of all source Dec's for the input Galaxy Name in decimal degrees
#print RA_Match_L
#print len(RA_Match_L)
#print Dec_Match_L
#print len(Dec_Match_L)
#decA=Data['dec']
#raA=Data['ra']
#Maj=Maj/3600
#S_Maj=Maj/2
#area_T=((S_Maj)**2)*math.pi
G_Data = Ned.query_object(
Gname
) #G_Data:-astropy.table.table.Table, Galaxy_Data, The Galaxy Data Table queried from NED
#print type(G_Data)
Dia_Table = Ned.get_table(
Gname, table='diameters'
) #Dia_Table:-astropy.table.table.Table, Diameter_Table, The Data table queried from NED that contains the infomation about the Major Axis of the input Galaxy Name
#print type(Dia_Table)
#print G_Data
#print Dia_Table
#print Dia_Table.colnames
#print Dia_Table.meta
#print Dia_Table.columns
Dia_Table_Feq = Dia_Table[
'Frequency targeted'] #Dia_Table_Feq:-astropy.table.column.MaskedColumn, Diameter_Table_Fequency, The Array containing all named frequencies of light that are being used for the Major Axis Measurement
#print Dia_Table['NED Frequency']
#print Dia_Table_Feq
#print type(Dia_Table_Feq)
Dia_Table_Feq_L = list(
Dia_Table_Feq
) #Dia_Table_Feq_L:-List, Diameter_Table_Fequency_List, The list containing all named frequencies of light that are being used for the Major Axis Measurement
#print Dia_Table_Feq_L
Dia_Table_Num = Dia_Table[
'No.'] #Dia_Table_Num:-astropy.table.column.MaskedColumn, Diameter_Table_Number, The number Ned assigns to
#print Dia_Table_Num
#print type(Dia_Table_Num)
Dia_Table_Num_L = list(Dia_Table_Num)
#print Dia_Table_Num_L
for i in range(
0,
len(Dia_Table_Feq_L) - 1
): #There is a bug here with index matching, The matched index isn't that same index for the major axis
Cur_Feq = Dia_Table_Feq_L[i]
#print Cur_Feq
if (Cur_Feq == "RC3 D_25, R_25 (blue)"):
Match_inx = i
Match_Feq = Dia_Table_Feq_L[Match_inx]
Match_Num = Dia_Table_Num_L[Match_inx]
#Match_Num
#print "Match_Feq ", Match_Feq
#print "Match_inx ", Match_inx
#print "Match_Num ", Match_Num
#Dia_Table_Maj=Dia_Table['Major Axis']
Dia_Table_Maj = Dia_Table['NED Major Axis']
#print Dia_Table_Maj
Dia_Table_Maj_L = list(Dia_Table_Maj)
#print Dia_Table_Maj_L
Dia_Table_Maj_Units = Dia_Table['Major Axis Unit']
#print Dia_Table_Maj_Units
Dia_Table_Maj_Units_L = list(Dia_Table_Maj_Units)
#print Dia_Table_Maj_Units_L
#print "i ", i
D25_Maj = Dia_Table_Maj_L[Match_inx]
#print "D25_Maj ", D25_Maj
D25_Units = Dia_Table_Maj_Units[Match_inx]
#print "D25_Units ", D25_Units
#print type(Dia_Table)
#print Dia_Table.info()
#Dia_Table_2=Dia_Table[6]
#print Dia_Table_2
#Maj=Dia_Table_2[18]
#print "Maj, ! ! !", Maj
D25_S_Maj = D25_Maj / 2.0
D25_S_Maj_Deg = D25_S_Maj / 3600.0
area_T = ((D25_S_Maj_Deg)**2) * math.pi
raGC = float(G_Data['RA(deg)'])
decGC = float(G_Data['DEC(deg)'])
#area_A=[((((((decGC-dec)**2)+((raGC-ra)**2)))*(math.pi))/area_T) for dec,ra in zip(decA,raA)]
#area_A=[((((((decGC-dec)**2)+((raGC-ra)**2)))*(math.pi))/area_T) for dec,ra in zip(Dec_Match_L,RA_Match_L)] #REAL ONE
#disA=[math.sqrt(((decGC-dec)**2)+((raGC-ra)**2)) for dec,ra in zip(dec_A,raA)] #REAL ONE?
disA = [
math.sqrt(((decGC - dec)**2) + ((raGC - ra)**2))
for dec, ra in zip(Dec_Match_L, RA_Match_L)
] #REAL ONE
#print area_A
#area_max=max(area_A)
#print area_max
#plt.vlines(0.00001,0,10,color='red')
#plt.vlines(1,0,10,color='red')
#plt.hist(area_A)
Hist_A = plt.hist(disA)
Bin_Hight_A = Hist_A[0]
Bin_Hight_Max = max(Bin_Hight_A)
#print Bin_Hight_Max
plt.vlines(D25_S_Maj_Deg, 0, Bin_Hight_Max,
color='red') #Plots red line at D25
print D25_S_Maj_Deg
#Hist_Max=max(area_A)
#print "Hist_Max ", Hist_Max
plt.plot()
#plt.savefig('Test2.png')
path_2 = os.path.realpath(
'../Master_Code/Master_Output/'
) #Goes to Master_Output folder, which will hold all the data calculated for the galaxies including the histogram pictures
print "Path_2=", path_2
"""
path_Hist=path_2+'/Histograms/'
directory_Hist=os.path.dirname(path_Hist)
if not os.path.exists(directory_Hist):
os.makedirs(directory_Hist)
print "path_Hist=",path_Hist
os.chdir(path_Hist)
"""
#system('mkdir '+Gname) #Creates Current Galaxy's Folder, Folder Named after Galaxy, Note: will have to remove space from "NGC #" to change to "NGC_#", I Don't know if this works
Gname_L = Gname.split(" ")
print "Gname_L: ", Gname_L
if (len(Gname_L) > 1):
Gname_Modifed = Gname_L[0] + "_" + Gname_L[
1] #Adds underscore to remove space from "NGC #" to change to "NGC_#" if there is a space in the name
else:
Gname_Modifed = Gname # Does nothing if the galaxy name has no space, ie. NGC#, For example NGC253 instead of NGC 253 or NGC_253
print Gname_Modifed
path_3 = path_2 + '/' + Gname_Modifed + '/'
directory = os.path.dirname(path_3)
if not os.path.exists(directory):
os.makedirs(directory)
os.chdir(path_3) #Goes to Current Galaxies Folder
path_Hist = path_3 + '/Histograms/'
directory_Hist = os.path.dirname(path_Hist)
if not os.path.exists(directory_Hist):
os.makedirs(directory_Hist)
print "path_Hist=", path_Hist
os.chdir(path_Hist)
plt.savefig(Gname_Modifed + '_Ang.png') #Saves angluar histogram figure
#system('pwd')
path_4 = os.path.realpath('../../../../Histogram_Code/')
print "Path_4=", path_4
os.chdir(
path_4
) #Goes back to where this code (the histogram code) is being run, ie. Desktop/GitHub
plt.close()
if len(result_table) > 0:
has_photometry = has_photometry + 1
print has_photometry
'''
# sne w1mag w1sigmn w2mag w2sigm w3mag w3sigm w4mag w4sigm freq1, freq2, freq3, ...... freqn
hasboth = 0
for i, unique_host in enumerate(unique_hosts):
print i
isWise = 0
isUV = 0
try:
photometry = Ned.get_table(unique_host, table="photometry")
freq = photometry["Frequency"]
isUV = np.where(freq > 1280000000000000.0)
if isUV > 0:
freq_u = np.unique(freq)
mag = photometry["Photometry Measurement"]
sigs = photometry["Uncertainty"]
signew = np.zeros((len(sigs), 1))
neddata = np.zeros((len(freq_u), 3))
neddata[:, 0] = freq_u
for k, sig in enumerate(sigs[:-2]):
sigval = sig[3:]
signew[k] = 0
if len(sigval) > 0:
signew[k] = float(sig[3:])
__author__ = 'Jakub Wojtanek, [email protected]' from astroquery.ned import Ned import unittest result_table = Ned.query_object("NGC 6720") result_table2 = Ned.get_table("NGC 6720", table='diameters') for k in result_table: print k print result_table.keys() print result_table2.keys() print result_table['RA(deg)'] print result_table['DEC(deg)'] print result_table['Magnitude and Filter'] print result_table['Distance (arcmin)'] print result_table['Diameter Points'] print result_table2['Major Axis'] print result_table2['Minor Axis']
def herschel_button(
galaxies,
filters="all",
radius=None,
filepath=None,
download_data=True,
create_mosaic=True,
jy_conversion=True,
verbose=False,
):
"""Create an Herschel mosaic, given a galaxy name.
Using a galaxy name and radius, queries around that object,
downloads available Herschel data from the HSA and mosaics.
Args:
galaxies (str or list): Names of galaxies to create mosaics for.
filters (str or list, optional): Any combination of 'PACS70',
'PACS100', 'PACS160', 'SPIRE250', 'SPIRE350', 'SPIRE500'. If
you want everything, select 'all'. Defaults to 'all'.
radius (astropy.units.Quantity, optional): Radius around the
galaxy to search for observations. Defaults to None, where
it will query Ned to get size.
filepath (str, optional): Path to save the working and output
files to. If not specified, saves to current working
directory.
download_data (bool, optional): If True, will download data using
Astroquery. Defaults to True.
create_mosaic (bool, optional): Switching this to True will
mosaic data as appropriate. Defaults to True.
jy_conversion (bool, optional): Convert the mosaicked file from
raw units to Jy/pix. Defaults to True.
verbose (bool, optional): Print out messages during the process.
Useful mainly for debugging purposes or large images.
Defaults to False.
"""
if isinstance(galaxies, str):
galaxies = [galaxies]
if filters == "all":
filters = [
'PACS70', 'PACS100', 'PACS160', 'SPIRE250', 'SPIRE350', 'SPIRE500'
]
if isinstance(filters, str):
filters = [filters]
if filepath is not None:
os.chdir(filepath)
if radius is not None:
original_radius = radius.copy()
else:
original_radius = None
steps = []
if download_data:
steps.append(1)
if create_mosaic:
steps.append(2)
if jy_conversion:
steps.append(3)
if not os.path.exists('Herschel_temp'):
os.mkdir('Herschel_temp')
for galaxy in galaxies:
if verbose:
print('Beginning ' + galaxy)
if radius is None:
try:
size_query = Ned.get_table(galaxy, table='diameters')
radius = 1.2 * np.max(
size_query['NED Major Axis']) / 2 * u.arcsec
radius = radius.to(u.deg)
except:
raise Warning(galaxy +
' not resolved by Ned, using 0.2deg radius.')
radius = 0.2 * u.degree
if not os.path.exists(galaxy):
os.mkdir(galaxy)
if not os.path.exists(galaxy + "/Herschel/"):
os.mkdir(galaxy + "/Herschel/")
for herschel_filter in filters:
pixel_scale = {
'PACS70': 2,
'PACS100': 3,
'PACS160': 4,
'SPIRE250': 6,
'SPIRE350': 8,
'SPIRE500': 12
}[herschel_filter]
if not os.path.exists(galaxy + "/Herschel/" + herschel_filter):
os.mkdir(galaxy + "/Herschel/" + herschel_filter)
if not os.path.exists(galaxy + "/Herschel/" + herschel_filter +
"/raw"):
os.mkdir(galaxy + "/Herschel/" + herschel_filter + "/raw")
if not os.path.exists(galaxy + "/Herschel/" + herschel_filter +
"/data"):
os.mkdir(galaxy + "/Herschel/" + herschel_filter + "/data")
if not os.path.exists(galaxy + "/Herschel/" + herschel_filter +
"/reprojected"):
os.mkdir(galaxy + "/Herschel/" + herschel_filter +
"/reprojected")
if not os.path.exists(galaxy + "/Herschel/" + herschel_filter +
"/weight"):
os.mkdir(galaxy + "/Herschel/" + herschel_filter + "/weight")
if not os.path.exists(galaxy + "/Herschel/" + herschel_filter +
"/outputs"):
os.mkdir(galaxy + "/Herschel/" + herschel_filter + "/outputs")
if 1 in steps:
if verbose:
print('Downloading available data')
# Download available Herschel images for the object.
images = ESASky.get_images(galaxy,
radius=0.4 * u.degree,
missions=['Herschel'],
download_dir='Herschel_temp/' +
galaxy)
images = images['HERSCHEL']
# Pull out available data for each waveband and save.
herschel_key = {
'PACS70': '70',
'PACS100': '100',
'PACS160': '160',
'SPIRE250': '250',
'SPIRE350': '350',
'SPIRE500': '500'
}[herschel_filter]
i = 0
for image in images:
for key in image.keys():
if key == herschel_key:
image[key].writeto(galaxy + "/Herschel/" +
herschel_filter + "/raw/" +
herschel_filter + "_" + str(i) +
".fits",
overwrite=True)
print(image[key][0].header['INSTMODE'])
i += 1
# We now want to pull out the data and the coverage map for
# each image.
if verbose:
print('Beginning initial weighted reprojection')
mHdr(
galaxy,
2 * radius.value,
2 * radius.value,
galaxy + "/Herschel/" + herschel_filter +
"/outputs/header.hdr",
resolution=pixel_scale,
)
herschel_files = glob.glob(galaxy + "/Herschel/" +
herschel_filter + "/raw/*.fits")
for herschel_file in herschel_files:
hdu_data = fits.open(herschel_file)[1]
hdu_data.data[hdu_data.data == 0] = np.nan
orig_pix_scale = np.abs(hdu_data.header['CDELT1'])
# Coverage is proportional to the exposure time so
# use sqrt of that as the weight.
hdu_weight = fits.open(herschel_file)[2]
hdu_weight.data[hdu_weight.data == 0] = np.nan
hdu_weight.data = hdu_weight.data**0.5
fits.writeto(herschel_file.replace('/raw/', '/data/'),
hdu_data.data,
hdu_data.header,
overwrite=True)
fits.writeto(herschel_file.replace('/raw/', '/weight/'),
hdu_weight.data,
hdu_weight.header,
overwrite=True)
# Perform an initial reprojection, weighting by the
# weight map
mProject(
herschel_file.replace('raw', 'data'),
herschel_file.replace('raw', 'reprojected'),
galaxy + "/Herschel/" + herschel_filter +
"/outputs/header.hdr",
weight_file=herschel_file.replace('raw', 'weight'),
)
try:
hdu_reproj = fits.open(
herschel_file.replace('raw', 'reprojected'))[0]
# If PACS, account for pixel scale
if 'PACS' in herschel_filter:
new_pix_scale = np.abs(hdu_reproj.header['CDELT1'])
hdu_reproj.data *= new_pix_scale**2 / orig_pix_scale**2
hdu_reproj.writeto(herschel_file.replace(
'raw', 'reprojected'),
overwrite=True)
except FileNotFoundError:
pass
if 2 in steps:
# Mosaic everything together.
if verbose:
print("Beginning mosaics")
mHdr(
galaxy,
2 * radius.value,
2 * radius.value,
galaxy + "/Herschel/" + herschel_filter +
"/outputs/header.hdr",
resolution=pixel_scale,
)
tools.mosaic(
galaxy + "/Herschel/" + herschel_filter + '/reprojected',
header=galaxy + "/Herschel/" + herschel_filter +
"/outputs/header.hdr",
verbose=verbose,
reproject=False,
haveAreas=True,
)
os.rename(
"mosaic/mosaic.fits", galaxy + "/Herschel/" +
herschel_filter + "/outputs/" + galaxy + '.fits')
shutil.rmtree("mosaic/", ignore_errors=True)
if 3 in steps:
# Convert to Jy.
if verbose:
print('Converting to Jy')
# For PACS, we already have this in Jy so
if 'PACS' in herschel_filter:
shutil.copy(
galaxy + "/Herschel/" + herschel_filter + "/outputs/" +
galaxy + '.fits', galaxy + "/Herschel/" + galaxy +
'_' + herschel_filter + '.fits')
# For SPIRE, this is in MJy/sr
if 'SPIRE' in herschel_filter:
convert_to_jy(
galaxy + "/Herschel/" + herschel_filter + "/outputs/" +
galaxy + '.fits', galaxy + "/Herschel/" + galaxy +
'_' + herschel_filter + '.fits')
shutil.rmtree('Herschel_temp/' + galaxy, ignore_errors=True)
if original_radius is None:
radius = None
else:
radius = original_radius.copy()
def D25_Query(ObsID):
##Gname=Gname_Query(ObsID)
#Evt2_File_H_L=File_Query_Code_5.File_Query(Gname,"evt2")
#print "Evt2_File_H_L : ", Evt2_File_H_L
#Evt2_File_L=Evt2_File_H_L[0]
#Obs_ID=Evt2_File_L[0]
Obs_ID=int(ObsID)
#Evt2filepath=Evt2_File_Query(ObsID)
#print "Obs_ID : ", Obs_ID
#path_Source_Flux_Table=os.path.realpath('../SQL_Standard_File/Source_Flux_Table.csv') #Reltive Path
path_Source_Flux_Table=os.path.realpath('/Volumes/xray/anthony/Research_Git/SQL_Standard_File/Source_Flux_Table.csv') #Absolute Path
#print "path_Source_Flux_Table : ", path_Source_Flux_Table
Source_Flux_Table=ascii.read(path_Source_Flux_Table)
#print "Source_Flux_Table : \n", Source_Flux_Table
Obs_ID_A=Source_Flux_Table["OBSID"]
Obs_ID_L=list(Obs_ID_A)
#print "Obs_ID_L : ", Obs_ID_L
Obs_ID_Inx=Obs_ID_L.index(Obs_ID)
#print "Obs_ID_Inx : ", Obs_ID_Inx
Resolved_Name_A=Source_Flux_Table["resolvedObject"]
#print "Resolved_Name_A : ", Resolved_Name_A
Resolved_Name_L=list(Resolved_Name_A)
#print "Resolved_Name_L : ", Resolved_Name_L
Resolved_Name=Resolved_Name_L[Obs_ID_Inx]
#print "Resolved_Name : ", Resolved_Name
#Dia_Table = Ned.get_table(Gname, table='diameters') #Dia_Table:-astropy.table.table.Table, Diameter_Table, The Data table queried from NED that contains the infomation about the Major Axis of the input Galaxy Name
Dia_Table = Ned.get_table(Resolved_Name, table='diameters') #Dia_Table:-astropy.table.table.Table, Diameter_Table, The Data table queried from NED that contains the infomation about the Major Axis of the input Galaxy Name
#print type(Dia_Table)
#print G_Data
#print Dia_Table
#print Dia_Table.colnames
#print Dia_Table.meta
#print Dia_Table.columns
Dia_Table_Feq=Dia_Table['Frequency targeted'] #Dia_Table_Feq:-astropy.table.column.MaskedColumn, Diameter_Table_Fequency, The Array containing all named frequencies of light that are being used for the Major Axis Measurement
#print Dia_Table['NED Frequency']
#print Dia_Table_Feq
#print type(Dia_Table_Feq)
Dia_Table_Feq_L=list(Dia_Table_Feq) #Dia_Table_Feq_L:-List, Diameter_Table_Fequency_List, The list containing all named frequencies of light that are being used for the Major Axis Measurement
#print Dia_Table_Feq_L
Dia_Table_Num=Dia_Table['No.'] #Dia_Table_Num:-astropy.table.column.MaskedColumn, Diameter_Table_Number, The number Ned assigns to
#print Dia_Table_Num
#print type(Dia_Table_Num)
Dia_Table_Num_L=list(Dia_Table_Num)
#print Dia_Table_Num_L
for i in range(0,len(Dia_Table_Feq_L)-1): #There is a bug here with index matching, The matched index isn't that same index for the major axis
Cur_Feq=Dia_Table_Feq_L[i]
#print Cur_Feq
if(Cur_Feq=="RC3 D_25, R_25 (blue)"):
Match_inx=i
Match_Feq=Dia_Table_Feq_L[Match_inx]
Match_Num=Dia_Table_Num_L[Match_inx]
#Match_Num
#print "Match_Feq ", Match_Feq
#print "Match_inx ", Match_inx
#print "Match_Num ", Match_Num
#Dia_Table_Maj=Dia_Table['Major Axis']
Dia_Table_Maj=Dia_Table['NED Major Axis']
#print Dia_Table_Maj
Dia_Table_Maj_L=list(Dia_Table_Maj)
#print Dia_Table_Maj_L
Dia_Table_Maj_Units=Dia_Table['Major Axis Unit']
#print Dia_Table_Maj_Units
Dia_Table_Maj_Units_L=list(Dia_Table_Maj_Units)
#print Dia_Table_Maj_Units_L
#print "i ", i
D25_Maj=Dia_Table_Maj_L[Match_inx]
#print "D25_Maj ", D25_Maj
D25_Units=Dia_Table_Maj_Units[Match_inx]
#print "D25_Units ", D25_Units
#print type(Dia_Table)
#print Dia_Table.info()
#Dia_Table_2=Dia_Table[6]
#print Dia_Table_2
#Maj=Dia_Table_2[18]
#print "Maj, ! ! !", Maj
D25_S_Maj=D25_Maj/2.0
D25_S_Maj_Deg=D25_S_Maj/3600.0
D25_S_Maj_Arcmin=D25_S_Maj_Deg*60.0
#return D25_S_Maj_Deg
return D25_S_Maj_Arcmin
def spitzer_button(
galaxies,
filters="all",
radius=None,
filepath=None,
download_data=True,
create_mosaic=True,
jy_conversion=True,
verbose=False,
):
"""Create an Spitzer mosaic, given a galaxy name.
Using a galaxy name and radius, queries around that object,
downloads available Spitzer data from the SHA and mosaics.
Args:
galaxies (str or list): Names of galaxies to create mosaics for.
filters (str or list, optional): Any combination of 'IRAC1', 'IRAC2',
'IRAC3', 'IRAC4', 'MIPS24', 'MIPS70', or 'MIPS160'. If you
want everything, select 'all'. Defaults to 'all'.
radius (astropy.units.Quantity, optional): Radius around the
galaxy to search for observations. Defaults to None, where
it will query Ned to get size.
filepath (str, optional): Path to save the working and output
files to. If not specified, saves to current working
directory.
download_data (bool, optional): If True, will download data using
Astroquery. Defaults to True.
create_mosaic (bool, optional): Switching this to True will
mosaic data as appropriate. Defaults to True.
jy_conversion (bool, optional): Convert the mosaicked file from
raw units to Jy/pix. Defaults to True.
verbose (bool, optional): Print out messages during the process.
Useful mainly for debugging purposes or large images.
Defaults to False.
"""
base_url = 'https://sha.ipac.caltech.edu/applications/Spitzer/SHA/servlet/DataService'
if isinstance(galaxies, str):
galaxies = [galaxies]
if filters == "all":
filters = [
'IRAC1', 'IRAC2', 'IRAC3', 'IRAC4', 'MIPS24', 'MIPS70', 'MIPS160'
]
if isinstance(filters, str):
filters = [filters]
if filepath is not None:
os.chdir(filepath)
if radius is not None:
original_radius = radius.copy()
else:
original_radius = None
steps = []
if download_data:
steps.append(1)
if create_mosaic:
steps.append(2)
if jy_conversion:
steps.append(3)
if not os.path.exists('Spitzer_temp'):
os.mkdir('Spitzer_temp')
for galaxy in galaxies:
if verbose:
print('Beginning ' + galaxy)
if radius is None:
try:
size_query = Ned.get_table(galaxy, table='diameters')
radius = 1.2 * np.max(
size_query['NED Major Axis']) / 2 * u.arcsec
radius = radius.to(u.deg)
except:
raise Warning(galaxy +
' not resolved by Ned, using 0.2deg radius.')
radius = 0.2 * u.degree
if not os.path.exists(galaxy):
os.mkdir(galaxy)
if not os.path.exists(galaxy + "/Spitzer/"):
os.mkdir(galaxy + "/Spitzer/")
# Download VOTable covering the user-specified area. Resolve
# position of galaxy
simbad_query = Simbad.query_object(galaxy)
ra = simbad_query[0]['RA']
dec = simbad_query[0]['DEC']
# Convert to decimal degrees
c = SkyCoord(ra=ra, dec=dec, unit=(u.hourangle, u.deg))
diam_deg = str(2 * radius.to(u.degree).value)
query_url = base_url + '?RA=' + str(c.ra.value) + '&DEC=' + str(
c.dec.value) + '&SIZE=' + str(diam_deg)
query_url += '&VERB=3&DATASET=ivo%3A%2F%2Firsa.csv%2Fspitzer.level2'
if os.path.exists(galaxy + '/Spitzer/table.xml'):
os.remove(galaxy + '/Spitzer/table.xml')
wget.download(query_url, out=galaxy + '/Spitzer/table.xml')
votable = ascii.read(galaxy + '/Spitzer/table.xml')
for spitzer_filter in filters:
# Name of wavelengths for the table
table_wl = {
'IRAC1': 'IRAC 3.6um',
'IRAC2': 'IRAC 4.5um',
'IRAC3': 'IRAC 5.8um',
'IRAC4': 'IRAC 8.0um',
'MIPS24': 'MIPS 24um',
'MIPS70': 'MIPS 70um',
'MIPS160': 'MIPS 160um'
}[spitzer_filter]
# Output mosaic pixel scales in arcsec
pixel_scale = {
'IRAC1': 0.6,
'IRAC2': 0.6,
'IRAC3': 0.6,
'IRAC4': 0.6,
'MIPS24': 2.45,
'MIPS70': 4,
'MIPS160': 8
}[spitzer_filter]
if verbose:
print('Beginning ' + spitzer_filter)
if not os.path.exists(galaxy + "/Spitzer/" + spitzer_filter):
os.mkdir(galaxy + "/Spitzer/" + spitzer_filter)
if not os.path.exists(galaxy + "/Spitzer/" + spitzer_filter +
'/raw'):
os.mkdir(galaxy + "/Spitzer/" + spitzer_filter + '/raw')
if not os.path.exists(galaxy + "/Spitzer/" + spitzer_filter +
'/data'):
os.mkdir(galaxy + "/Spitzer/" + spitzer_filter + '/data')
if not os.path.exists(galaxy + "/Spitzer/" + spitzer_filter +
'/weight'):
os.mkdir(galaxy + "/Spitzer/" + spitzer_filter + '/weight')
if not os.path.exists(galaxy + "/Spitzer/" + spitzer_filter +
'/outputs'):
os.mkdir(galaxy + "/Spitzer/" + spitzer_filter + '/outputs')
if 1 in steps:
if verbose:
print("Downloading data")
if not os.path.exists('Spitzer_temp/' + galaxy):
os.mkdir('Spitzer_temp/' + galaxy)
# Match up the wavelength column of the table to the
# correct name.
for row in votable:
if row['wavelength'] == table_wl:
# Download files along with ancillary stuff.
dl_file = row['accessWithAnc1Url']
# Sometimes we get a NONE so in that case, skip.
if dl_file == 'NONE':
continue
wget.download(dl_file,
out='Spitzer_temp/' + galaxy + '/')
# Unzip all these files
spitzer_files = glob.glob('Spitzer_temp/' + galaxy + '/*.zip')
for spitzer_file in spitzer_files:
os.system('unzip ' + spitzer_file + ' -d ' +
'Spitzer_temp/' + galaxy)
# Move any maic and munc files to raw
extensions = ['maic', 'munc']
for extension in extensions:
for root, _, filenames in os.walk("Spitzer_temp/" +
galaxy):
for filename in fnmatch.filter(
filenames, "*" + extension + ".fits"):
match = os.path.join(root, filename)
filename = match.split('/')
os.rename(
match,
galaxy + "/Spitzer/" + spitzer_filter +
"/raw/" + filename[-1],
)
shutil.rmtree('Spitzer_temp/' + galaxy, ignore_errors=True)
spitzer_files = glob.glob(galaxy + "/Spitzer/" +
spitzer_filter + '/raw/*maic.fits')
# Reproject, taking into account the uncertainties to weight
# the maps initially
if verbose:
print('Performing inital round of weighted reprojection')
mHdr(
galaxy,
2 * radius.value,
2 * radius.value,
galaxy + "/Spitzer/" + spitzer_filter +
"/outputs/header.hdr",
resolution=pixel_scale,
)
for spitzer_file in spitzer_files:
# Do a little fiddling to find the name of the
# uncertainty file.
unc_name = spitzer_file.split('/')[-1]
unc_name = '_'.join(unc_name.split('_')[2:5])
unc_file = glob.glob(galaxy + "/Spitzer/" +
spitzer_filter + '/raw/*' + unc_name +
'*munc.fits')
unc_file = unc_file[0]
# Reproject each image to final pixel scale, weighting
# by inverse uncertainty.
hdu_exp = fits.open(unc_file)[0]
hdu_exp.data = hdu_exp.data**-1
hdu_exp.writeto(unc_file.replace('raw', 'weight'),
overwrite=True)
mProject(spitzer_file,
spitzer_file.replace('raw', 'data'),
galaxy + "/Spitzer/" + spitzer_filter +
"/outputs/header.hdr",
weight_file=unc_file.replace('raw', 'weight'))
if 2 in steps:
# Mosaic all these files together.
if verbose:
print("Beginning mosaics")
mHdr(
galaxy,
2 * radius.value,
2 * radius.value,
galaxy + "/Spitzer/" + spitzer_filter +
"/outputs/header.hdr",
resolution=pixel_scale,
)
tools.mosaic(
galaxy + "/Spitzer/" + spitzer_filter + '/data',
header=galaxy + "/Spitzer/" + spitzer_filter +
"/outputs/header.hdr",
verbose=verbose,
reproject=False,
haveAreas=True,
)
os.rename(
"mosaic/mosaic.fits", galaxy + "/Spitzer/" +
spitzer_filter + "/outputs/" + galaxy + '.fits')
shutil.rmtree("mosaic/", ignore_errors=True)
if 3 in steps:
if verbose:
print('Converting to Jy')
# Convert to Jy.
convert_to_jy(galaxy + "/Spitzer/" + spitzer_filter +
"/outputs/" + galaxy + ".fits",
hdu_out=galaxy + "/Spitzer/" + galaxy + "_" +
spitzer_filter + ".fits")
if original_radius is None:
radius = None
else:
radius = original_radius.copy()
TeVCat_Flag += [all_fermi[1].data['TeVCat_Flag'][ii]]
alt_name_gamma += [all_fermi[1].data['ASSOC_FGL'][ii]]
print(all_fermi[1].data[ii][0], all_fermi[1].data[ii][65])
try:
result = Ned.query_object(all_fermi[1].data[ii][65])
redshift_temp = result['Redshift'][0]
redshift_flag_temp = result['Redshift Flag'][0]
except:
redshift_temp = '--'
redshift_flag_temp = ''
redshift += [redshift_temp]
redshift_flag += [redshift_flag_temp]
try:
result1 = Ned.get_table(all_fermi[1].data[ii][65])
# optical_fluxes = result1[['Observed Passband', 'Frequency', 'Photometry Measurement', 'Uncertainty', 'Units', 'Flux Density', 'Lower limit of uncertainty']][np.logical_and(result1['Frequency'] < 1e16, result1['Frequency'] > 1e14)]
optical_fluxes = result1['Flux Density'][np.logical_and(
result1['Frequency'] < 1e16, result1['Frequency'] > 1e14)]
optical_fluxes = np.array(optical_fluxes)
num_optical_flux_temp = len(optical_fluxes)
median_optical_flux_temp = np.nanmedian(optical_fluxes) * 1.e3
except:
median_optical_flux_temp = '--'
num_optical_flux_temp = '--'
median_optical_flux += [median_optical_flux_temp]
num_optical_flux += [num_optical_flux_temp]
# print(all_fermi[1].data[ii][0])
time.sleep(1)
def Background_Finder_3(
gname, evtfname, objLfname, R
): #Need to apply energy filter (0.3kev to 10kev) to the counts, This may allow the code to treat back illuminated chips and front illuminated chips the same, if not then the code must be modifed to consider both cases
"""
gname:-str, Galaxy Name, The name of the galaxy in the form NGC #, For Example 'NGC 3077'
evtfname:-str, Event File Name, The name of the event file of the observation, For Example 'acisf02076_repro_evt2.fits'
objLfname:-str, Object List File Name, The name of the object list file which is a list of circluar regions around the X-ray objects. For Example 'ngc3077_ObsID-2076_Source_List_R_Mod_2.txt'
n:-int, Number of objects, The number of objects in the observation
R:-float(?) or int, Radius, The radius of the circle used to find the background in pixels
Returns: BG_Ratio:-float, Background Ratio, The background ratio in number of counts per pixel
or "None" if a region without an object in it cannot be found
"""
Obj_L = [
] #Obj_L:-List, Object_List, The list of all object string shapes in the observation
Obj_B = True #Obj_B:-bool, Object Boolean, A Boolean statement in regards to if there is no X-ray objects in the area being used to find the background
List_Done_Bool = False #List_Done_Bool:-bool, List_Done_Boolean, A Boolean statement in regards to if 3 background measurments were found in the observation
#BG_Circle_Overlap_Bool=False
BG_R = R # Note: Physical Radius might not be equal to the Pixel Radius
Num_BG_Pix = math.pi * (
(BG_R)**2
) #Num_BG_Pix:-float or int, the number of pixels in the background test region
print Num_BG_Pix
CCD_L = [
] # Note: I don't even know if I need this, It's only defined here and never used again I think
Obj_Shape = "" # Note: I don't even know if I need this, It's only defined here and never used again I think
#system('pwd')
#system('ls')
#system('cd ~/Desktop/Big_Object_Regions/')
#os.chdir('~/Desktop/Big_Object_Regions/')
dir = os.path.dirname(__file__)
#filename= os.path.join(dir, '~','Desktop','SQL_Standard_File',)
#filepath=os.path.abspath("~/Desktop/SQL_Standard_File")
#print "Filepath =",filepath
#path= os.path.join(dir,'~','Desktop','SQL_Standard_File',)
#path=os.path.realpath('~/Desktop/SQL_Standard_File/SQL_Sandard_File.csv')
path = os.path.realpath('../Big_Object_Regions/')
print "Path=", path
#system('pwd')
os.chdir(path)
#system('ls')
#os.chdir("~")
#os.system("cd ~")
#Objfile=open("Desktop/Big_Object_Regions/"+str(objLfname),"r") #Objfile:-file, Objectfile, a file containing the regions of the X-ray objects in the observation as strings regions
Objfile = open(str(objLfname), "r")
#print type(Objfile)
path2 = os.path.realpath(
'../Background_Finder/'
) #Changes PWD back to this code's PWD in Desktop/Background_Finder, this may be Changed later to go to the location of the Evt2 file that will be used in the DMCOORDS, the location will be given by File_Query_Code
os.chdir(path2)
Objstring = Objfile.read(
) #Objstring:-str, Objstring, the all X-ray object regions all in one big string with each object "\n" seperated
#print Objstring
#print type(Objstring)
G_Data = Ned.query_object(
gname
) #G_Data:-astropy.table.table.Table, Galaxy_Data, The queryed data of the galaxy from NED in the form of a astropy table
#print G_Data
#print type(G_Data)
raGC = float(
G_Data['RA(deg)']
) #raGC:-float, Right Ascension of Galatic Center, The right ascension of the galatic center of the current galaxy in degrees.
decGC = float(
G_Data['DEC(deg)']
) #decGC:-float, Declination of Galatic Center, The declination of the galatic center of the current galaxy in degrees.
"""
Dia_A= Ned.get_table(gname,table='diameters') #Dia_A:-astropy.table.table.Table, Diameter_Array, The astropy table that contains the diameter info for the galaxy, which is referred to as an array
#print type(Dia_A)
#print Dia_A
Dia_A2=Dia_A[6] #Dia_A2:-astropy.table.row.Row, Diameter Array 2, The diameter subarray using RC3 D_0 (blue) standard for the diameter, contians the galaxy diameter infomation as an astropy row
#print type(Dia_A2)
#print Dia_A2
Maj=Dia_A2[18] #Maj:-numpy.float64, Major axis, The major axis of the galaxy in arcseconds
#print type(Maj)
#print Maj
#Maj=Dia_A2[18]
Min=Dia_A2[25] #Min:-numpy.float64, Minor axis, The minor axis of the galaxy in arcseconds
#print type(Min)
#print Min
S_Maj=Maj/2 #S_Maj:-numpy.float64, Semi_Major axis, The semi major axis of the galaxy in acrseconds
"""
G_Data = Ned.query_object(gname)
Dia_Table = Ned.get_table(gname, table='diameters')
#print G_Data
#print Dia_Table
#print Dia_Table.colnames
#print Dia_Table.meta
#print Dia_Table.columns
Dia_Table_Feq = Dia_Table['Frequency targeted']
#print Dia_Table['NED Frequency']
#print Dia_Table_Feq
Dia_Table_Feq_L = list(Dia_Table_Feq)
#print Dia_Table_Feq_L
Dia_Table_Num = Dia_Table['No.']
#print Dia_Table_Num
Dia_Table_Num_L = list(Dia_Table_Num)
#print Dia_Table_Num_L
for i in range(
0,
len(Dia_Table_Feq_L) - 1
): #There is a bug here with index matching, The matched index isn't that same index for the major axis
Cur_Feq = Dia_Table_Feq_L[i]
#print Cur_Feq
if (Cur_Feq == "RC3 D_25, R_25 (blue)"):
Match_inx = i
Match_Feq = Dia_Table_Feq_L[Match_inx]
Match_Num = Dia_Table_Num_L[Match_inx]
#Match_Num
#print "Match_Feq ", Match_Feq
#print "Match_inx ", Match_inx
#print "Match_Num ", Match_Num
#Dia_Table_Maj=Dia_Table['Major Axis']
Dia_Table_Maj = Dia_Table['NED Major Axis']
#print Dia_Table_Maj
Dia_Table_Maj_L = list(Dia_Table_Maj)
#print Dia_Table_Maj_L
Dia_Table_Maj_Units = Dia_Table['Major Axis Unit']
#print Dia_Table_Maj_Units
Dia_Table_Maj_Units_L = list(Dia_Table_Maj_Units)
#print Dia_Table_Maj_Units_L
#print "i ", i
D25_Maj = Dia_Table_Maj_L[Match_inx]
#print "D25_Maj ", D25_Maj
D25_Units = Dia_Table_Maj_Units[Match_inx]
#print "D25_Units ", D25_Units
#print type(Dia_Table)
#print Dia_Table.info()
#Dia_Table_2=Dia_Table[6]
#print Dia_Table_2
#Maj=Dia_Table_2[18]
#print "Maj, ! ! !", Maj
D25_S_Maj = D25_Maj / 2.0
#D25_S_Maj_Deg=D25_S_Maj/3600.0
dmcoords(
infile=str(evtfname),
ra=str(raGC),
dec=str(decGC),
option='cel',
verbose=0,
celfmt='deg'
) # Runs the dmcoords CIAO tool, which converts coordinates like CHIP_ID to SKY, the tool is now being used to convert the RA and Dec of the GC to SKY coodinates in pixels (?)
X_Phys = dmcoords.x #X_Phys:-float, X_Physical, The sky plane X pixel coordinate in units of pixels of the galatic center
Y_Phys = dmcoords.y #Y_Phys:-float, Y_Physical, The sky plane Y pixel coordinate in units of pixels of the galatic center
Chip_ID = dmcoords.chip_id #Chip_ID:-int, Chip_ID, The Chip ID number the GC is on
print Chip_ID
print "GC X is ", X_Phys
print "GC Y is ", Y_Phys
#R_Phys=S_Maj*2.03252032520325 #R_Phys:-numpy.float64, Radius_Physical, The radius of the galaxy in pixels, the converstion factor is 2.03252032520325pix/arcsec
R_Phys = D25_S_Maj * 2.03252032520325 #R_Phys:-numpy.float64, Radius_Physical, The radius of the galaxy in pixels, the converstion factor is 2.03252032520325pix/arcsec
#D25_S_Maj
#print type(R_Phys)
print "Radius of Galaxy is ", R_Phys
Gal_V_Shape = 'circle(' + str(X_Phys) + ',' + str(Y_Phys) + ',' + str(
R_Phys) + ')' # This might not be used at all in this code
Objstring_L = Objstring.split("\n")
del Objstring_L[len(Objstring_L) - 1]
#print "n ", n
#print "Objstring_L ", Objstring_L
#print "len(Objstring_L) ", len(Objstring_L)
for Cur_Obj in Objstring_L:
Obj_L.append(Cur_Obj)
"""
for i in range(0,n):
Cur_Obj= Objstring.split("\n")[i] #Cur_Obj:-str, Current Object, The current X-ray object region string that is being added to the Object List
Obj_L.append(Cur_Obj) #Obj_L:-List, Object List, list of the string regions of all the X-ray objects that are in the observation
"""
Header_String = dmlist(infile=str(evtfname), opt="header")
#print Header_String
Header_String_Reduced = Header_String.split("DETNAM")[1]
#print Header_String_Reduced
Header_String_Reduced_2 = Header_String_Reduced.split("String")[0]
#print Header_String_Reduced_2
Header_String_Reduced_3 = Header_String_Reduced_2.replace(' ', '')
print Header_String_Reduced_3
#dmkeypar(infile=str(evtfname), keyword="DETNAM")
#pget(paramfile, paramname)
#Chip_ID_String=pget(toolname="dmkeypar", parameter="value")
#Chip_ID_String=pget("dmkeypar","value") #Chip_ID_String:-str, Chip_Idenifcation_String, Runs the pget tool to get the string containing what CCDs are used in the FOV1.fits file from the parameter file asscoiated with the dmkeypar tool and sets it equal to the Chip_ID_String (This) variable
Chip_ID_String = Header_String_Reduced_3 #Chip_ID_String:-str, Chip_Idenifcation_String, Runs the pget tool to get the string containing what CCDs are used in the FOV1.fits file from the parameter file asscoiated with the dmkeypar tool and sets it equal to the Chip_ID_String (This) variable
#Chip_ID_String=pget(toolname="dmkeypar", p_value="value")
print "Chip_ID_String ", Chip_ID_String
Chip_ID_String_L = Chip_ID_String.split(
'-'
) #Chip_ID_String_L:-List, Chip_Idenifcation_String_List, The resulting list from spliting the Chip_ID_String on "_", This list contains 2 elements, the first element is the string "ACIS" and the second element is the string segment in the form (Example) "356789" where each number in the list is its own CCD ID
#print "Chip_ID_String_L ", Chip_ID_String_L
Chip_ID_String_Reduced = Chip_ID_String_L[
1] #Chip_ID_String_Reduced:-str, Chip_Idenifcation_String_Reduced, the string segment in the form (Example) "356789" where each number in the list is its own CCD ID
print "Chip_ID_String_Reduced ", Chip_ID_String_Reduced
Chip_ID_L = [
] #Chip_ID_L:-List, Chip_Idenifcation_List, The list of all the int CCD IDs in FOV1.fits file
for Cur_Chip_ID_Str in Chip_ID_String_Reduced: #Cur_Chip_ID_Str:-str, Current_Chip_Idenifcation_Str, The string vaule of the current string CCD ID in the Chip_ID_String_Reduced string, for example "3"
Cur_Chip_ID = int(
Cur_Chip_ID_Str
) #Cur_Chip_ID:-int, Current_Chip_Idenifcation, The current chip ID number as an int, for example 3
Chip_ID_L.append(
Cur_Chip_ID
) #Appends The current chip ID number as an int to Chip_Idenifcation_List
print "Chip_ID_L ", Chip_ID_L
#Step_L=[500,250,100,50,25,10,5,1]
Step_L = [500, 250, 100]
Background_L = []
BG_Circle_Info_L = []
#BG_Circle_Overlap_Bool=False
if (len(Background_L) <= 3):
for Step in Step_L:
#print "Step ", Step
for Chip_ID_Test in Chip_ID_L:
#print "Chip_ID_Test ", Chip_ID_Test
for c in range(
0 + BG_R, 1025 - BG_R, Step
): # c is "x" #Check Bounds #The Bounds for CHIP coordinates are (1,1024)(both included), ie range(1,1025), So if this is correct (I am not 100% sure about these CHIP bounds), "for c in range(0+BG_R,1025-BG_R):" should instead be "for c in range(1+BG_R,1025-BG_R):"
for v in range(
0 + BG_R, 1025 - BG_R, Step
): # v is "y" #Check Bounds, should instead be "for v in range(1+BG_R,1025-BG_R):"(?)
BG_Circle_Overlap_Bool = False
Obj_B = True #Obj_B:-bool, Object Boolean, A Boolean statement in regards to if there is no X-ray objects in the area being used to find the background
#print " " # Puts a space between objects
BG_X = c #BG_X:-int, BackGround circle_X, The x coordinate of the backgound circle in Chip coordinates, Note: This should probably be a float along with all numerical imputs to this function
#print type(BG_X)
BG_Y = v #BG_Y:-int, BackGround circle_Y, The y coordinate of the backgound circle in Chip coordinates, Note: This should probably be a float along with all numerical imputs to this function
#print "Chip x is ",c
#print "Chip y is ",v
#dmcoords(infile=str(evtfname),chipx=BG_X, chipy=BG_Y, chip_id=Chip_ID, option='chip', verbose=0) # Runs the dmcoords CIAO tool, which converts coordinates like CHIP_ID to SKY, The tool is now being used to convert the Background Circle center from CHIP to SKY coordinates (?)
dmcoords(
infile=str(evtfname),
chipx=BG_X,
chipy=BG_Y,
chip_id=Chip_ID_Test,
option='chip',
verbose=0
) # Runs the dmcoords CIAO tool, which converts coordinates like CHIP_ID to SKY, The tool is now being used to convert the Background Circle center from CHIP to SKY coordinates (?)
BG_X_Pix = dmcoords.x #BG_X_Pix:-float, BackGround circle_X_Pixels, The x of the center of the background circle in SKY coordinates in pixels
BG_Y_Pix = dmcoords.y #BG_Y_Pix:-float, BackGround circle_Y_Pixels, The y of the center of the background circle in SKY coordinates in pixels
#print "Background X is ", BG_X_Pix
#print "Background Y is ", BG_Y_Pix
#print "Background R is ", BG_R
Dis_GC = math.sqrt(
((BG_X_Pix - X_Phys)**2) + ((BG_Y_Pix - Y_Phys)**2)
) #Dis_GC:-float, Distance_Galatic_Center, The distance from the background circle to the galatic center in pixels
#print "BG_Circle_Info_L ", BG_Circle_Info_L
if (
len(BG_Circle_Info_L) > 0
): #Need to stop testing against background cirlces only on the current chip and instead on all chips
for BG_Circle_Info_Old in BG_Circle_Info_L:
BG_X_Pix_Old = BG_Circle_Info_Old[0]
BG_Y_Pix_Old = BG_Circle_Info_Old[1]
BG_R_Pix_Old = BG_Circle_Info_Old[2]
Dis_BG_to_BG = math.sqrt((
(BG_X_Pix - BG_X_Pix_Old)**2) + (
(BG_Y_Pix - BG_Y_Pix_Old)**2))
BG_Total_Reach = Dis_BG_to_BG - BG_R - BG_R_Pix_Old
if (BG_Total_Reach <= 0):
BG_Circle_Overlap_Bool = True
#print type(Dis_GC)
#print "Distance to GC is ", Dis_GC
#print "R_Phys is ", R_Phys
#print "The GC Test is ", Dis_GC-R_Phys-BG_R
if (
(Dis_GC - R_Phys - BG_R) > 0
): # Makes sure that the backgound cirlce does not intersect with the radius of the galaxy, ie this functions disregards all X-ray objects in the visible extent of the galaxy
for Obj_S in Obj_L: #String split X, Y and the R out
Cur_X = Obj_S.split(
","
)[0] #Cur_X:-str, Current_X, The unreduced X-ray object string in the form "circle(5330.96623132" with the X coordinate in it in pixels
Cur_X_R = Cur_X.split(
'('
)[1] #Cur_X_R:-str, Current_X_Reduced, The reduced X-ray object string in the form "5330.96623132" which is the X coordinate in pixels
Cur_Y = Obj_S.split(
","
)[1] #Cur_Y:-str, Current_Y_Reduced, The reduced X-ray object string in the form "5333.51369932" which is the Y coordinate in pixels
Cur_R = Obj_S.split(
","
)[2] #Cur_R:-str, Current_Radius, The unreduced X-ray object string in the form "233.272357724)" with the radius in it in pixels
Cur_R_R = Cur_R.split(
')'
)[0] #Cur_R_R:-str, Current_Radius_Reduced, The reduced X-ray object string in the form "233.272357724" which is the radius in pixels
Cur_X_N = float(
Cur_X_R
) #Cur_X_N:-float, Current_X_Number, The current coordinate of the X-ray object region's X coordinate in pixels
Cur_Y_N = float(
Cur_Y
) #Cur_Y_N:-float, Current_Y_Number, The current coordinate of the X-ray object region's Y coordinate in pixels
Cur_R_N = float(
Cur_R_R
) #Cur_R_N:-float, Current_Radius_Number, The current coordinate of the X-ray object region's radius in pixels
Dis_Obj = math.sqrt(
((BG_X_Pix - Cur_X_N)**2) +
((BG_Y_Pix - Cur_Y_N)**2)
) #Dis_Obj:-float, Distance_Object, The distance from the backgound cirlce to the current object
#print "Distance to Object is ", Dis_Obj
#print "Cur_R_N is ", Cur_R_N
#print "BG_R is ", BG_R
#print "The Obj Test is ", Dis_Obj-Cur_R_N-BG_R
if (
(Dis_Obj - Cur_R_N - BG_R) <= 0
): # Checks to see if the backgound circle contian or is touching an object
Obj_B = False #Obj_B:-bool, Object_Boolean, A boolean that is false if the background cirlce is intersecting with an object region
#print "Obj_B ", Obj_B
#print "BG_Circle_Overlap_Bool ", BG_Circle_Overlap_Bool
if (
(Obj_B == True)
and (BG_Circle_Overlap_Bool == False)
): #Makes sure that the background circle is not intersecting with any objects or any other other background circle
#print "Background Found ! ! !"
#Dm_Out=dmlist(infile=str(evtfname)+"[sky=circle("+str(BG_X_Pix)+","+str(BG_Y_Pix)+","+str(BG_R)+")]", opt='counts', outfile="", verbose=2) #Dm_Out:-ciao_contrib.runtool.CIAOPrintableString,Dmlist_Out,Uses the Dmlist CIAO tool to find the amount of counts in the background cirlce, Note: mlist "acis_evt2.fits[sky=rotbox(4148,4044,8,22,44.5)]" counts #Need to apply energy filter (0.3kev to 10kev) to the counts, This may allow the code to treat back illuminated chips and front illuminated chips the same, if not then the code must be modifed to consider both cases
Dm_Out = dmlist(
infile=str(evtfname) + "[sky=circle(" +
str(BG_X_Pix) + "," + str(BG_Y_Pix) + "," +
str(BG_R) + "),energy=300:10000]",
opt='counts',
outfile="",
verbose=2
) #Dm_Out:-ciao_contrib.runtool.CIAOPrintableString,Dmlist_Out,Uses the Dmlist CIAO tool to find the amount of counts in the background cirlce, Note: mlist "acis_evt2.fits[sky=rotbox(4148,4044,8,22,44.5)]" counts #Energy filter (0.3kev to 10kev) has been applied to the counts, This may allow the code to treat back illuminated chips and front illuminated chips the same, if not then the code must be modifed to consider both cases
#print Dm_Out
#print type(Dm_Out)
Num_Counts_S = Dm_Out.split(
'\n'
)[9] #Num_Counts_S:-str, Number_of_Counts_String, The number of counts in the background cirlce as a string
#print Num_Counts_S
#print type(Num_Counts_S)
Num_Counts = float(
Num_Counts_S
) #Num_Counts:-float, Number_of_Counts, The number of counts as a float
BG_Ratio = Num_Counts / Num_BG_Pix #BG_Ratio:-float, Background_Ratio, The background of the observation
#return BG_Ratio #Returns the background of the observation
Background_L.append(BG_Ratio)
Cur_BG_Circle_Info = [
BG_X_Pix, BG_Y_Pix, BG_R, Chip_ID_Test
]
BG_Circle_Info_L.append(Cur_BG_Circle_Info)
if (len(Background_L) == 3):
print "List Done ! ! ! ! ! ! ! ! !"
List_Done_Bool = True
print "Background_L ", Background_L
print "BG_Circle_Info_L Final", BG_Circle_Info_L
if (List_Done_Bool == True):
BG_Ratio_Avg = np.average(Background_L)
return BG_Ratio_Avg
#print "List_Done_Bool ", List_Done_Bool
#print "List_Done_Bool==False ", str(List_Done_Bool==False)
if (List_Done_Bool == False):
#print "List_Done_Bool==False ", str(List_Done_Bool==False)
return "None_Found" # returns the string "None" if there is no place to put the background cirlce without intersecting the visible extent of the galaxy or an X-ray object
def two_mass_button(
galaxies,
filters="all",
radius=None,
filepath=None,
download_data=True,
create_mosaic=True,
jy_conversion=True,
verbose=False,
**kwargs
):
"""Create a 2MASS mosaic, given a galaxy name.
Using a galaxy name and radius, queries around that object,
downloads available 2MASS data and mosaics into a final product.
Args:
galaxies (str or list): Names of galaxies to create mosaics for.
Resolved by NED.
filters (str or list, optional): Any combination of 'J', 'H',
and 'K'. If you want everything, select 'all'. Defaults
to 'all'.
radius (astropy.units.Quantity, optional): Radius around the
galaxy to search for observations. Defaults to None, where
it will query Ned to get size.
filepath (str, optional): Path to save the working and output
files to. If not specified, saves to current working
directory.
download_data (bool, optional): If True, will download data using
MontagePy. Defaults to True.
create_mosaic (bool, optional): Switching this to True will
mosaic data as appropriate. Defaults to True.
jy_conversion (bool, optional): Convert the mosaicked file from
raw units to Jy/pix. Defaults to True.
verbose (bool, optional): Print out messages during the process.
Useful mainly for debugging purposes or large images.
Defaults to False.
"""
if isinstance(galaxies, str):
galaxies = [galaxies]
if filters == "all":
filters = ['J', 'H', 'K']
if isinstance(filters, str):
filters = [filters]
if filepath is not None:
os.chdir(filepath)
if radius is not None:
original_radius = radius.copy()
else:
original_radius = None
steps = []
if download_data:
steps.append(1)
if create_mosaic:
steps.append(2)
if jy_conversion:
steps.append(3)
for galaxy in galaxies:
if verbose:
print('Beginning '+galaxy)
if radius is None:
try:
size_query = Ned.get_table(galaxy,table='diameters')
radius = 1.2*np.max(size_query['NED Major Axis'])/2*u.arcsec
radius = radius.to(u.deg)
except:
raise Warning(galaxy+' not resolved by Ned, using 0.2deg radius.')
radius = 0.2*u.degree
if not os.path.exists(galaxy):
os.mkdir(galaxy)
for two_mass_filter in filters:
if verbose:
print('Beginning 2MASS '+two_mass_filter)
if not os.path.exists(galaxy + "/2MASS"):
os.mkdir(galaxy + "/2MASS")
if not os.path.exists(galaxy + "/2MASS/" + two_mass_filter):
os.mkdir(galaxy + "/2MASS/" + two_mass_filter)
if not os.path.exists(galaxy + "/2MASS/" + two_mass_filter+"/raw"):
os.mkdir(galaxy + "/2MASS/" + two_mass_filter+"/raw")
if not os.path.exists(galaxy + "/2MASS/" + two_mass_filter+"/data"):
os.mkdir(galaxy + "/2MASS/" + two_mass_filter+"/data")
if not os.path.exists(galaxy + "/2MASS/" + two_mass_filter+"/outputs"):
os.mkdir(galaxy + "/2MASS/" + two_mass_filter+"/outputs")
if 1 in steps:
if verbose:
print("Downloading data")
# Montage uses its size as the length of the square,
# since we want a radius use twice that.
mArchiveDownload(
"2MASS " + two_mass_filter,
galaxy,
2 * radius.value,
galaxy + "/2MASS/" + two_mass_filter+"/raw",
)
# We need to now convert these into magnitudes.
two_mass_files = glob.glob(galaxy+'/2MASS/'+ two_mass_filter+'/raw/*')
for two_mass_file in two_mass_files:
hdu = fits.open(two_mass_file)[0]
magzp = hdu.header['MAGZP']
hdu.data = magzp - 2.5*np.log10(hdu.data)
fits.writeto(two_mass_file.replace('/raw/','/data/'),
hdu.data,hdu.header,
overwrite=True)
if 2 in steps:
# Mosaic all these files together.
if verbose:
print("Beginning mosaic")
mHdr(
galaxy,
2 * radius.value,
2 * radius.value,
galaxy + "/2MASS/"+ two_mass_filter+"/outputs/header.hdr",
resolution=1,
)
tools.mosaic(
galaxy + "/2MASS/" + two_mass_filter+"/data",
header=galaxy + "/2MASS/"+ two_mass_filter+"/outputs/header.hdr",
verbose=verbose,
**kwargs
)
os.rename("mosaic/mosaic.fits",
galaxy + "/2MASS/"+ two_mass_filter+"/outputs/"+galaxy+ ".fits")
# Clear out the mosaic folder.
shutil.rmtree("mosaic/", ignore_errors=True)
if 3 in steps:
if verbose:
print('Converting to Jy')
# Convert to Jy.
convert_to_jy(galaxy + "/2MASS/"+ two_mass_filter+"/outputs/"+galaxy+ ".fits",
two_mass_filter,
galaxy + "/2MASS/"+galaxy+"_"+ two_mass_filter+".fits")
if original_radius is None:
radius = None
else:
radius = original_radius.copy()
def buildTable():
"""
Allows users to create a fully customizable HTML table containing information about a list of input galaxies.
"""
input_file = typer.prompt(
'Enter the path to the csv/txt file containing the targets (format "target,ra,dec"), where ra, dec are the ICRS coordinates of target in degrees in J2000 equinox'
)
output_name = typer.prompt('Enter the name of the output table')
# get the basic columns
columns = [
'NAME', 'RA', 'DEC', 'REDSHIFT', 'REFERENCE FOR REDSHIFT', 'NED LINK'
]
# add the columns for images/plots/other files
add_spectra = (typer.prompt(
'Add spectra plots to the table?(y/n)|(default: n)').lower() == 'y')
create_spectra = False
if add_spectra:
create_spectra = (typer.prompt(
'Create spectra plots now? Type "n" if you already have them (y/n)|(default: n)'
).lower() == 'y')
if not create_spectra:
spectra_dir = typer.prompt(
'Enter directory with spectra plot files. File names must be in format (target).png/jpg'
)
columns.append('SPECTRA')
add_pahfitplot = (typer.prompt(
'Add pahfit plots and tables to the table?(y/n)|(default: n)').lower()
== 'y')
create_pahfit = False
if add_pahfitplot:
create_pahfit = (typer.prompt(
'Create pahfit plots and tables now? Type "n" if you already have them (y/n)|(default: n)'
).lower() == 'y')
if not create_pahfit:
pahfitplot_dir = typer.prompt(
'Enter directory with pahfit plot files. File names must be in format (target).png/jpg'
)
pahfittable_dir = typer.prompt(
'Enter directory with pahfit output table files. File names must be in format (target)_output.ipac or (target).ipac'
)
columns.extend(['PAHFIT PLOT', 'PAHFIT TABLE'])
add_rgb = (typer.prompt(
'Add RGB images to the table(y/n)|(default: n)').lower() == 'y')
create_rgb = False
if add_rgb:
create_rgb = (typer.prompt(
'Create rgb images now using SDSS? Type "n" if you already have the images(y/n)|(default: n)'
).lower() == 'y')
if not create_rgb:
rgb_dir = typer.prompt(
'Enter directory with the RGB images. File names musst be in format (target).png/jpg'
)
columns.append('RGB IMAGE')
add_visibility = (typer.prompt(
'Add visibility plots to the table?(y/n)|(default: n)').lower() == 'y')
create_visibility = False
if add_visibility:
create_visibility = (typer.prompt(
'Create visibility plots now? Type "n" if you already have the plots(y/n)|(default: n)'
).lower() == 'y')
if not create_visibility:
visibility_dir = typer.prompt(
'Enter directory with the visibility plots. File names must be in format (target).png/jpg:'
)
columns.append('VISIBILITY PLOT')
# add other images and files the use might have
other_image_dirs = []
add_other_images = True
while add_other_images:
add_other_images = (typer.prompt(
'Add any other images/plots to the table?(y/n)|(default: n)').
lower() == 'y')
if add_other_images:
other_image_dirs.append(
typer.prompt(
'Enter directory with the other images. File names must be in format (target).png/jpg:'
))
columns.append(
typer.prompt(
'Enter the name of the column for these images/plots').
upper())
other_files = []
add_other_files = True
while add_other_files:
add_other_files = (typer.prompt(
'Add links to any other files?(y/n)|(default: n)').lower() == 'y')
if add_other_files:
format = typer.prompt('Enter the format of these files')
dir = typer.prompt(
'Enter directory with the other images. File names must be in format (target).(format):'
)
other_files.append({'format': format, 'dir': dir})
# add all optional non-file columns
add_eso = (typer.prompt(
'Add ESO links to the table?(y/n)|(default: n)').lower() == 'y')
add_alma = (typer.prompt(
'Add ALMA links s to the table?(y/n)|(default: n)').lower() == 'y')
add_manga = (typer.prompt(
'Add MANGA links to the table?(y/n)|(default: n)').lower() == 'y')
add_gemini = (typer.prompt(
'Add GEMINI links to the table?(y/n)|(default: n)').lower() == 'y')
add_guide_stars = (typer.prompt(
'Add guide star info to the table?(y/n)|(default: n)').lower() == 'y')
if add_eso:
columns.extend(['ESO INSTRUMENTS', 'ESO LINKS'])
if add_alma:
columns.append('ALMA LINK')
if add_manga:
columns.extend(['MaNGA ID', 'MaNGA PLATE IFUDesign', 'MaNGA LINK'])
if add_gemini:
columns.extend(['GEMINI INSTRUMENTS', 'GEMINI LINK'])
if add_guide_stars:
columns.extend(['HIGH STREHL', 'LOW STREHL'])
# set up eso_tap_obs
if add_eso:
ESO_TAP_OBS = "http://archive.eso.org/tap_obs"
tapobs = tap.TAPService(ESO_TAP_OBS)
if add_manga:
drpall = fits.open(
r'C:\Users\bpart\OneDrive\Desktop\astro\MASSIVE\data\manga-drpall.fits'
)
# set up Vizier so all the necessary columns can be obtained for the guide stars, including flags that signify galaxies
if add_guide_stars:
v = Vizier(columns=[
'RAJ2000', 'DEJ2000', 'pmRA', 'pmDE', 'UCAC4', 'LEDA', 'rmag',
'f_rmag', 'g', 'c', 'of'
])
# create spizter spectra plots
if create_spectra:
spectra_dir = getSpectra()
# create pahfit plots and tables
if create_pahfit:
pahfitplot_dir, pahfittable_dir = getPahfit()
# create rgb images
if create_rgb:
rgb_dir = getSDSS_RGB(input_file)
# create visibility plots if user chose that option
if create_visibility:
visibility_dir = getVisibility(input_file)
dataFile = open(input_file, 'r')
dataFile = csv.reader(dataFile)
output_name = 'index' if output_name == '' else output_name
with open(f'{output_name}.csv', 'w') as outfile:
csv_writer = csv.writer(outfile)
csv_writer.writerow(columns)
for line in dataFile:
name = line[0]
ra = line[1]
dec = line[2]
typer.secho('Building row for ' +
typer.style(name, fg=typer.colors.MAGENTA))
# get redshift from NED
coord = SkyCoord(ra, dec, unit='deg')
resTable = Ned.query_region(coord, radius=0.016 * u.deg)
resTable = resTable.to_pandas()
redshift = ''
refCode, reflink = '', ''
for i, row in resTable.iterrows():
if row['Redshift'] > 0:
nName = row['Object Name']
redshift = row['Redshift']
# have to use name to query NED's redshift table to get reference
try:
redTable = Ned.get_table(
nName, table='redshifts').to_pandas()
for j, r in redTable.iterrows():
if r['Published Redshift'] == redshift:
refCode = r['Refcode']
reflink = f'https://ui.adsabs.harvard.edu/abs/{refCode}/abstract'
except:
refCode = "No refcode found"
break
# get nedLink using coordinates
nedLink = f'https://ned.ipac.caltech.edu/conesearch?search_type=Near%20Position%20Search&coordinates={ra}d%20{dec}d&radius=1'
# prepare row for required columns
row = [
name, ra, dec, redshift, f"<a href='{reflink}'>{refCode}</>",
f"<a href='{nedLink}'>NED Link</>"
]
# prepare rows for other columns
# Add spectra plot
if add_spectra:
row.append(getImagePath(spectra_dir, name))
# Add PAHFIT plot and PAHFIT table, accounting for possible variations in naming conventions
if add_pahfitplot:
row.append(getImagePath(f'{pahfitplot_dir}', name))
if os.path.exists(f'{pahfittable_dir}/{name}.ipac'):
row.append(
f"<a href='{pahfittable_dir}/{name}.ipac'>PAHFIT Table</>"
)
else:
row.append(
f"<a href='{pahfittable_dir}/{name}_output.ipac'>PAHFIT Table</>"
)
# Add visibility plot
if add_visibility:
row.append(getImagePath(visibility_dir, name))
# Add RGB plots from SDSS
if add_rgb:
row.append(getImagePath(rgb_dir, name))
# Add other plots/images
for d in other_image_dirs:
row.append(getImagePath(d, name))
# ESO
# tried using astroquery.eso but it would not work. I believe this is the best way to do it as of 04/2021
if add_eso:
query = f"""SELECT *
FROM ivoa.ObsCore
WHERE intersects(s_region, circle('', {ra}, {dec}, 0.02))=1"""
res = tapobs.search(query=query, maxrec=1000)
# get all unique instruments
esoInstruments = set(res['instrument_name'])
esoInstruments = list(esoInstruments)
esoInstruments = " ".join(i for i in esoInstruments).replace(
",", "-")
esoLink, esoText = '', ''
# get the ESO link using coordinates
if len(esoInstruments) > 1:
esoLink = f'https://archive.eso.org/scienceportal/home?pos={ra}%2C{dec}&r=0.02'
esoText = 'ESO Link'
row.append(esoInstruments)
row.append(f"<a href='{esoLink}'>{esoText}</>")
# ALMA
# there is a chance of astroquery.alma not working. Try installing the latest version of astroquery directly from the GitHub source in this case
if add_alma:
almaLink = ''
almaText = ''
try:
res = Alma.query_region(coord, 0.02 * u.deg)
if len(res) > 0:
almaLink = f'https://almascience.eso.org/asax/?result_view=observation&raDec={ra}%20{dec}'
almaText = "ALMA"
except:
pass
row.append(f"<a href='{almaLink}'>{almaText}</>")
# MaNGA
# the drpall file version 2_4_3 is required to obtain MaNGA data from release 16
if add_manga:
tbdata = drpall[1].data
i = np.where(
np.sqrt((tbdata['objra'] - float(ra))**2 +
(tbdata['objdec'] - float(dec))**2) <= 0.02)
mid, plate, ifudsgn, mlink, mtext = "", "", "", "", ""
if len(i[0]) > 0:
# the mid, plate, and ifudsgn are used to locate manga data
# TODO provide more info on this
mid = tbdata['mangaid'][i][0]
plate = tbdata['plate'][i][0]
ifudsgn = tbdata['ifudsgn'][i][0]
mlink = f'https://data.sdss.org/sas/dr16/manga/spectro/redux/v2_4_3/{plate}/stack/'
mtext = "MaNGA Link"
row.append(mid)
row.append(ifudsgn)
row.append(f'<a href="{mlink}">{mtext}</>')
# GEMINI
if add_gemini:
gData = Observations.query_region(coordinates=SkyCoord(
ra, dec, unit='deg'),
radius=0.02 * u.deg)
gInstruments = set()
gText, gLink = '', ''
if len(gData) > 0:
gInstruments = set(gData['instrument'])
gText = "Gemini Link"
gLink = f'https://archive.gemini.edu/searchform/NotFail/ra={ra}/not_site_monitoring/notengineering/dec={dec}/cols=CTOWEQ'
gInstruments = list(gInstruments)
gInstruments = " ".join(i for i in gInstruments).replace(
",", "-")
row.append(gInstruments)
row.append(f'<a href="{gLink}">{gText}</>')
# Get guide stars using Vizier from the I/322A catalogue
# Refer here for more information https://www.gemini.edu/instrumentation/altair
if add_guide_stars:
ucac4 = v.query_region(SkyCoord(ra, dec, unit='deg'),
radius=25 * u.arcsec,
catalog='I/322A')
highList, lowList = [], []
if ucac4:
ucac4 = ucac4[0]
for star in ucac4:
dist = abs(
(np.float32(star['RAJ2000']) - np.float32(ra))**2 +
abs(np.float32(star['DEJ2000']) -
np.float32(dec))**2)**0.5 * 3600
magn = star['rmag']
# Objects with LEDA > 0 are galaxies and will be disregarded.
if (3 < dist < 15 and 8.5 < magn < 15
) and star['LEDA'] == star['g'] == star['c'] == 0:
highList.append(
(star['UCAC4'], round(dist, 2), round(magn, 2),
star['of'], star['f_rmag']))
elif (15 < magn < 18.5) and (
3 < dist
) and star['LEDA'] == star['g'] == star['c'] == 0:
lowList.append(
(star['UCAC4'], round(dist, 2), round(magn, 2),
star['of'], star['f_rmag']))
# prepare outputs for High and Low Strehl columns using data obtained from catalogue
highList = " ".join((
f'{str(h[0])} (rmag={str(h[2])}; f_rmag={str(h[4])}; distance={str(h[1])}"; of={str(h[3])})'
) for h in highList)
lowList = " ".join((
f'{str(h[0])} (rmag={str(h[2])}; f_rmag={str(h[4])}; distance={str(h[1])}"; of={str(h[3])})'
) for h in lowList)
row.append(highList)
row.append(lowList)
# avoid adding empty strings to output as this can cause problems when converting to HTML
for i in range(len(row)):
if row[i] == "":
row[i] = " "
csv_writer.writerow(row)
# create HTMl table from the csv table using pandas. The resulting table will look pretty ugly and as such it is recommended to add some basic styling.
data = pd.read_csv(f'{output_name}.csv')
file = open(f'{output_name}.html', 'w')
file.writelines(data.to_html(escape=False, index=False))
