def checkNED(self):
table = Ned.query_region(self.coord,
radius=self.ned_radius * u.arcmin, equinox='J2000.0')
# Pick only galaxies
mask = table['Type'] == b'G'
return(table[mask])
def get_z_ned(coord=None, RA=None, DEC=None):
'''query NED and return redshift of nearest galaxy or None if failed'''
if coord is None:
coord = SkyCoord('{} {}'.format(RA, DEC), unit=(u.hourangle, u.deg))
# query ned
result_table = Ned.query_region(coord, radius=1 * u.arcmin)
r = result_table.to_pandas()
# select only galaxy with a redshift
cand = r[r['Type'].isin([b'G', b'GPair', b'GTrpl', b'GGroup'])
& r['Redshift'].notnull()]
if len(cand) == 0:
return None
elif len(cand) == 1:
return cand['Redshift'].item()
else:
# rank by separation and use the closest
# NB: columns names apparently changed in NED query results in March 2019, hence the change
#cand.loc[:,'sep'] = coord.separation(SkyCoord(ra = cand.loc[:,'RA(deg)'], dec = cand.loc[:,'DEC(deg)'], unit = (u.deg, u.deg)))
cand.loc[:, 'sep'] = coord.separation(
SkyCoord(ra=cand.loc[:, 'RA'],
dec=cand.loc[:, 'DEC'],
unit=(u.deg, u.deg)))
return cand.sort_values(by='sep').iloc[0].loc['Redshift'].item()
def ned_query(ra, dec):
# getting skycoordinate object
co = coordinates.SkyCoord(ra=ra,
dec=dec,
unit=(u.deg, u.deg),
frame='icrs')
search_radius = 0.01 * u.deg
# this search radius is smaller than Fermi LAT resolution (please check)
# get table with all objects inside the search radius
result_table = Ned.query_region(co,
radius=search_radius,
equinox='J2000.0')
result_table = Ned.query_object("NGC 224")
print('names of the columns are ', result_table.colnames)
print('to get an impression, here the whole table:')
print(result_table)
# get all the object names to get speific data
object_names = result_table['Object Name']
print('identified objects = ', object_names)
# get table with positions of the first object as an example:
position_table = Ned.get_table(object_names[0], table='positions')
# position table is something you normally don't need but might be interesting
# This should always work'
# print('position table: ', position_table)
spectra = Ned.get_spectra("3c 273")
return result_table, spectra
def query_loc(ra, dec, radius=1., z=None, verb=0):
"""
Query NED based on sky coordninates; return closest match with similar z
Inputs:
ra, dec (float): sky coords in HHhMMmSS.Ss str format
radius (float, optional): query radius in arcmin, default=1
verb (int or bool, optional): verbose output? Default: False
Outputs:
ned_table: astropy table of query results
"""
coord = SkyCoord(ra, dec)
# Astroquery search by location
if verb:
print('\tsending query...')
ned_results = Ned.query_region(coord, radius=radius * u.arcmin)
if verb:
print('\tcomplete')
# Sort results by separation from target coords
ned_sorted = ned_results[np.argsort(ned_results['Separation'])]
z_sorted = ned_sorted[ned_sorted['Redshift'].mask != True]
# Choose closest result
ned_table = ned_sorted[0]
# If provided a z, search for result with similar z value
if z:
for object in z_sorted:
if np.abs(object['Redshift'] - z) / z < 0.1:
ned_table = object
break
sleep(1)
if verb:
print(ned_table)
return ned_table
def nearby2starlist(target_list, radius=45, equinox=2000):
'''
query targets in a list and search nearby objects
print in Keck's starlist format
Parameters:
----
target_list (list) : a list of strings include the targets you want to search
radius (float) : the radius (in arcmin) you would like to search in nearby
Returns:
----
starlist (str) : Keck style starlist for nearby objects
'''
starlist = []
for target_name in target_list:
region = Ned.query_region(target_name, radius=radius * u.arcsec)
all_ras = region.as_array().data["RA"]
all_decs = region.as_array().data["DEC"]
all_obj_names = region.as_array().data["Object Name"]
# exclude the target
ind = ( region.as_array().data["Separation"] != 0 )
target_ra = all_ras[~ind][0]
target_dec = all_decs[~ind][0]
target_obj_name = all_obj_names[~ind][0]
all_ras = all_ras[ind]
all_decs = all_decs[ind]
all_obj_names = all_obj_names[ind]
# convert deg to hh:mm:ss
all_ras_hrs = [ deg2RA(deg) for deg in all_ras ]
all_dec_segs = [ deg2deg_arcmin_arcsec(deg) for deg in all_decs ]
all_raoffsets = deg2arcsec(all_ras - target_ra) # in arcsec
all_decoffsets = deg2arcsec(all_decs - target_dec) # in arcsec
# str methods to combine all info to a starlist
this_starlist = ["# science target {} with nearby offset stars".format(target_name)]
for ra_hrs, dec_segs, obj_name, raoffset, decoffset in zip(all_ras_hrs, all_dec_segs, all_obj_names, all_raoffsets, all_decoffsets):
row_str = "{:<30} {:02d} {:02d} {:05.2f} {:02d} {:02d} {:05.2f} {:.1f} raoffset={:05.2f} decoffset={:05.2f}".format(
obj_name.decode("UTF-8"), ra_hrs[0], ra_hrs[1], ra_hrs[2], dec_segs[0], dec_segs[1], dec_segs[2],
equinox, raoffset, decoffset
)
this_starlist.append(row_str)
starlist.append("\n".join(this_starlist))
print("\n".join(this_starlist))
print("\n")
return starlist
def getned(ra, dec, radius=0.1):
try:
co = coordinates.SkyCoord(ra=ra,
dec=dec,
unit=(u.deg, u.deg),
frame='fk4')
result_table = Ned.query_region(co, radius=radius * u.deg)
except:
result_table = None
return result_table
def sourcesearch_ned(ra, dec, radius):#radius in unit of arcmin
co = coordinates.SkyCoord(ra=ra, dec=dec,unit=(u.deg, u.deg), frame='fk4', equinox = 'J2000.000')
result_table = Ned.query_region(co, radius=radius * u.arcmin, equinox='J2000.000')
gals = result_table[result_table['Type']==b'G']
cans = gals[np.invert(gals['Redshift'].mask)]
if len(cans)>0:
cans['d'] = 0.
cans['norm_d'] = 999.
for source in cans:
d = sep(ra, dec, source['RA'], source['DEC'])
source['d'] = d
return cans
def query_region_NED(ra, dec, radius):
pos = SkyCoord(ra=ra, dec=dec, unit=(u.deg, u.deg))
try:
# Query NED
results = Ned.query_region(pos, radius=radius*u.arcmin)
# Convert results to pandas dataframe
results_df = results.to_pandas()
except:
# Return blank entry if no match found
results_df = pd.DataFrame([], columns=['Object Name',
'Type',
'Magnitude and Filter',
'RA',
'Dec',
'Redshift',
'Separation'])
# Format strings to UTF-8
for col in results_df:
if col in ['Object Name', 'Type', 'Magnitude and Filter']:
results_df[col] = results_df[col].str.decode('utf-8')
# Define columns
cols = ['ned_name', 'ned_host_ra', 'ned_host_dec', 'ned_type',
'ned_mag_filter', 'ned_host_redshift', 'ned_offset (arcmin)']
# Rename columns
results_df = results_df.rename(columns={'Object Name':'ned_name',
'Type':'ned_type',
'Magnitude and Filter':'ned_mag_filter',
'RA':'ned_host_ra',
'DEC':'ned_host_dec',
'Redshift':'ned_host_redshift',
'Separation':'ned_offset (arcmin)'})
results_df = results_df[cols]
# Calculate luminosity distance from host redshift, Mpc
results_df['luminosity_distance (Mpc)'] = cosmo.luminosity_distance(results_df['ned_host_redshift']).value
# Keep nearest galaxy
results_df = results_df[
results_df['ned_offset (arcmin)']==min(results_df['ned_offset (arcmin)'])
].reset_index(drop=True)
return results_df.to_dict(orient='records')[0]
def query_ned_astroquery(ra_deg: float,
dec_deg: float,
searchradius_arcsec: float = 0.5):
"""
Function to obtain NED crossmatches via astroquery
"""
c = SkyCoord(ra_deg, dec_deg, unit=u.deg, frame="icrs")
r = searchradius_arcsec * u.arcsecond
try:
return Ned.query_region(c, radius=r)
except RemoteServiceError:
return None
def query_ned_for_redshifts(name_dictionary, coordinate_dictionary):
'''Query NED for redshifts based on target names'''
print("\n\n =========== FINDING REDSHIFTS ========\n")
# Instantiate an empty dictionary for redshifts
redshift_dictionary = {}
continued_failures = []
target_names = name_dictionary.values()
for name in target_names:
try:
z = Ned.query_object(name)["Redshift"][0]
redshift_dictionary["{}".format(name)] = z
print("The NED redshift for {} is {}.".format(name, z))
except:
print(
"Cannot resolve redshift using NAME {}, trying coordinate search."
.format(name))
z = Ned.query_region(coordinate_dictionary[name],
radius=20 * u.arcsec,
equinox='J2000.0')["Redshift"][0]
# If this fails, it will silently set Z to a numpy.ma MaskedConstant (looks like '--')
if np.ma.is_masked(z) is True:
continued_failures.append(name)
print(
"Still cannot find a redshift for {}, skipping it.".format(
name))
elif z < 1.0: # none of these sources are high redshift, this is a dumb sanity check:
redshift_dictionary["{}".format(name)] = z
print("{} is at RA={}, Dec={}. NED finds a redshift of {}.".
format(name, coordinate_dictionary[name].ra,
coordinate_dictionary[name].dec, z))
if len(continued_failures) > 0:
print(
"You need to manually fix these, which still cannot be resolved: ",
continued_failures)
print("In the meantime, they'll be skipped by the movie maker.")
elif len(continued_failures) == 0:
print("It SEEMS like all redshifts have successfully been found, ")
print(
"Here are your (hopefully) successful redshift identifications - check these!"
)
print(" NAME = Z")
for name, z in redshift_dictionary.items():
print(" {} = {}".format(name, z))
return redshift_dictionary
def ned_query(ra, dec):
# getting skycoordinate object
co = coordinates.SkyCoord(ra=ra,
dec=dec,
unit=(u.deg, u.deg),
frame='icrs')
search_radius = 0.01 * u.deg
# this search radius is smaller than Fermi LAT resolution (please check)
# get table with all objects inside the search radius
result_table = Ned.query_region(co,
radius=search_radius,
equinox='J2000.0')
result_table = Ned.query_object("NGC 224")
print('names of the columns are ', result_table.colnames)
print('to get an impression, here the whole table:')
print(result_table)
# get all the object names to get speific data
object_names = result_table['Object Name']
print('identified objects = ', object_names)
# get table with positions of the first object as an example:
position_table = Ned.get_table(object_names[0], table='positions')
# position table is something you normally don't need but might be interesting
# This should always work'
# print('position table: ', position_table)
# now the redshift is for many sources not available,will give you an error
# therefore we will use try and exept
for specific_object in object_names:
try:
redshift_table = Ned.get_table(specific_object, table='redshifts')
print('redshift found four ', specific_object)
print(redshift_table)
except:
print('no redshift for ', specific_object)
return result_table
def Ned_query_dist(df, H_0=69.6):
distances = []
for index in df.index:
coordinate = SkyCoord(ra=df['RA'][index] * u.degree,
dec=df['Dec'][index] * u.degree)
result = Ned.query_region(coordinate, radius=20 * u.arcsec)
typeInds = np.where(result['Type'] != b'G')
result.remove_rows(typeInds)
if len(result) > 0:
if np.where(~np.isnan(result['Velocity']))[0].size > 0:
index2 = np.where(~np.isnan(result['Velocity']))[0]
velocity = result['Velocity'][index2[0]]
distance = velocity / H_0
distances.append(distance)
else:
distances.append(np.nan)
else:
distances.append(np.nan)
df['distance'] = distances
return df
def xmatch_ned(ra, dec, r=3):
"""
Cross-match with the NED catalog with given cone of search.
Parameters:
----------
ra: float
ra in degree
dec: float
dec in degree
r: float
searching radius in arcsec
Return:
----------
integer indicating which class of object
"""
# create SkyCoord object for the (ra,dec)
coord = SkyCoord(ra=ra, dec=dec, unit=(u.degree, u.degree), frame='icrs')
# define searching radius in arcsec
radius = u.Quantity(r, u.arcsec)
# cross-match using astroquery
try:
tab = Ned.query_region(coord, radius)
try:
# pick the closest one within the search cone
tab.sort('Separation')
obj_type = [str(d['Type'])[2:-1] for d in tab]
# return the object class
return classification(obj_type[0])
except:
# return NaN if no object within the search cone
return np.nan
except:
print("Error occurs during NED query!")
def search_near_objs(folder_path, hand_events, scd_pry_events,
first_radius_search, second_radius_search,
dist_from_the_milkyway, lst_of_scd_pry_obj_type):
for filename in os.listdir(folder_path + 'info'):
if filename.endswith(".txt"):
file_path = folder_path + 'info/' + filename
with open(file_path) as json_file:
obj = json.load(json_file)
if obj.get('comment', '0') == '0':
obj['comments'] = ''
co = coordinates.SkyCoord(ra=obj['data']['reply']['radeg'],
dec=obj['data']['reply']['decdeg'],
unit=(u.deg, u.deg),
frame='fk5')
near_the_milky_way(
co, obj,
dist_from_the_milkyway) # if yes add it to the comments
near_a_big_galaxy(
obj, co, folder_path
) # do def near_a_big_galaxy(co) if yes add to a diffrent list.
near_objs = Ned.query_region(co,
radius=first_radius_search * u.deg,
equinox='J2000.0')
if len(near_objs) == 0:
hand_events.append(obj)
f = open(
folder_path + 'hand_eve/' +
str(obj['data']['received_data']['objname']) + '.txt',
'w+')
json.dump(obj, f)
os.remove(file_path)
else:
min = near_objs[0][9]
closest_obj = near_objs[0]
for line in near_objs:
if line[9] <= min:
min = line[9]
closest_obj = line
# find if the closest obj is from a certian type
if closest_obj[4] in lst_of_scd_pry_obj_type:
scd_pry_events.append(closest_obj)
f = open(
folder_path + 'scd_pry_eve/' +
str(obj['data']['received_data']['objname']) + '.txt',
'w+')
json.dump(obj, f)
if closest_obj[9] > second_radius_search:
obj['comments'] = obj[
'comments'] + ',the closest obj far from ' + str(
second_radius_search)
obj['closest_obj'] = []
obj['closest_obj'].append({
'name': closest_obj[1],
'type': closest_obj[4],
'dist': closest_obj[9],
'diameter': {
'value': closest_obj[15],
'unit': 'Diameter points'
}
})
if closest_obj[
6] is not ma.masked: # there is info about the redshift
insert_data_from_redshift(obj, closest_obj)
os.remove(file_path)
new = str(obj)
f = open(
folder_path + 'found_near_obj/' +
str(obj['data']['received_data']['objname']) + '.txt',
'w+')
json.dump(new, f)
continue
else:
continue
def sdss(input_file, output_dir, filters, output_format, pmin_r, pmax_r,
pmin_g, pmax_g, pmin_b, pmax_b):
"""
Generates RGB images using SDSS data and APLPY. Information for parameters can be found in inputHandler(). I separated them up to allow easier access from makeTable.py
"""
warnings.filterwarnings("ignore")
# get parameters and inputs into a suitable format
df = pd.read_csv(input_file, header=None)
output_format = output_format.lower()
filters = filters.split(',')
if len(filters) != 3:
raise Exception('Error. Please provide 3 filters separated by commas')
try:
os.mkdir(output_dir)
except:
typer.echo(
f'A dir with the name {typer.style(output_dir, bold=True, fg=typer.colors.RED)} already exists'
)
for i, line in df.iterrows():
try:
name = line[0]
ra = line[1]
dec = line[2]
styled_name = typer.style(name, fg=typer.colors.MAGENTA, bold=True)
typer.echo(f'Plotting: ' + styled_name)
pos = coords.SkyCoord(ra, dec, unit='deg')
# query SDSS for each target and get the images
try:
xid = Table(SDSS.query_region(pos, spectro=False)[0])
except:
raise Exception(f'No images found on SDSS for target {name}')
im = SDSS.get_images(matches=xid, band=filters)
# raise exception if no images are found
if len(im) == 0:
raise Exception(f'No images found on SDSS for target {name}')
# Obtain the PrimaryHDU from the HDUList for each band
r, g, b = im[0][0], im[1][0], im[2][0]
# save the fits files so they can be combined into a single rgb cube
r.writeto('r.fits', overwrite=True)
g.writeto('g.fits', overwrite=True)
b.writeto('b.fits', overwrite=True)
aplpy.make_rgb_cube(['r.fits', 'g.fits', 'b.fits'],
f'image.fits',
north=True)
aplpy.make_rgb_image(f'image.fits',
fr'{output_dir}/{name}.{output_format}',
pmin_r=pmin_r,
pmax_r=pmax_r,
pmin_g=pmin_g,
pmax_g=pmax_g,
pmin_b=pmin_b,
pmax_b=pmax_b)
image = aplpy.FITSFigure(fr'{output_dir}/{name}.{output_format}')
image.show_rgb()
# add labels for filters used
image.add_label(0.08,
0.15,
"FILTERS:",
relative=True,
color='white')
filter_wavelengths = {
'z': 913,
'i': 763,
'r': 623,
'g': 477,
'u': 354
}
bandColours = ['red', 'green', 'cyan']
for i in range(3):
w = filter_wavelengths[filters[i]]
c = bandColours[i]
image.add_label(0.08,
0.11 - 0.03 * i,
f"{filters[i]} ({w}nm)",
relative=True,
color=c)
# add arrows pointing to object
image.show_arrows(
x=[ra + 0.01, ra - 0.01, ra + 0.01, ra - 0.01],
y=[dec + 0.01, dec - 0.01, dec - 0.01, dec + 0.01],
dx=[-0.007, 0.007, -0.007, 0.007],
dy=[-0.007, 0.007, 0.007, -0.007],
color='red')
# add object name as label
image.add_label(ra, dec + 0.02, name, color='red')
# get redshift for scalebar
resTable = Ned.query_region(pos, radius=0.01 * u.deg)
resTable = resTable.to_pandas()
redshift = 0
for i, row in resTable.iterrows():
if row['Redshift'] > 0:
redshift = row['Redshift']
break
kpcArcmin = cosmo.kpc_proper_per_arcmin(float(redshift)).value
length = round(kpcArcmin * 2, 1)
# add scalebar
image.add_scalebar(1 / 30,
label=f'120" | {length}kpc | r = {redshift}',
color='red')
# save to output dir in the appropriate format
image.save(fr'{output_dir}/{name}.{output_format}')
typer.echo('Finished plotting: ' + styled_name)
except Exception as e:
typer.echo('Failed for: ' + styled_name)
print(e)
cleanup()
def make_catalog(
region_name,
cloud_name,
distance,
good_cores_array,
additional_cores_array,
cross_matched_core_indices,
cross_matched_proto_indices,
alpha_BE,
getsources_core_catalog,
R_deconv,
FWHM_mean,
Masses,
Masses_err,
Temps,
Temps_err,
not_accepted_counter,
CSAR_catalog='/mnt/scratch-lustre/jkeown/DS9_regions/L1157/CSAR/CEPl1157_CSAR.dat',
high_res_coldens_image='/mnt/scratch-lustre/jkeown/Getsources/Prepare/Images/cep1157/080615/cep1157_255_mu.image.resamp.fits',
SED_figure_directory='/mnt/scratch-lustre/jkeown/DS9_regions/HGBS_pipeline/L1157/L1157_core_SED/',
Dunham_YSOs_file='Dunham_YSOs.dat'):
# These are the values in each column of "good_cores_array"
#NO, XCO_P, YCO_P, WCS_ACOOR, WCS_DCOOR, SIG_GLOB, FG, GOOD, SIG_MONO01, FM01, FXP_BEST01, FXP_ERRO01, FXT_BEST01, FXT_ERRO01, AFWH01, BFWH01, THEP01, SIG_MONO02, FM02, FXP_BEST02, FXP_ERRO02, FXT_BEST02, FXT_ERRO02, AFWH02, BFWH02, THEP02, SIG_MONO03, FM03, FXP_BEST03, FXP_ERRO03, FXT_BEST03, FXT_ERRO03, AFWH03, BFWH03, THEP03, SIG_MONO04, FM04, FXP_BEST04, FXP_ERRO04, FXT_BEST04, FXT_ERRO04, AFWH04, BFWH04, THEP04, SIG_MONO05, FM05, FXP_BEST05, FXP_ERRO05, FXT_BEST05, FXT_ERRO05, AFWH05, BFWH05, THEP05, SIG_MONO06, FM06, FXP_BEST06, FXP_ERRO06, FXT_BEST06, FXT_ERRO06, AFWH06, BFWH06, THEP06, SIG_MONO07, FM07, FXP_BEST07, FXP_ERRO07, FXT_BEST07, FXT_ERRO07, AFWH07, BFWH07, THEP07
# These are the values in each column of the "additional" "cores_array" and "protostar_array"
# NO XCO_P YCO_P PEAK_SRC01 PEAK_BGF01 CONV_SRC01 CONV_BGF01 PEAK_SRC02 PEAK_BGF02 CONV_SRC02 CONV_BGF02 PEAK_SRC03 PEAK_BGF03 CONV_SRC03 CONV_BGF03 PEAK_SRC04 PEAK_BGF04 CONV_SRC04 CONV_BGF04 PEAK_SRC05 PEAK_BGF05 CONV_SRC05 CONV_BGF05 PEAK_SRC06 PEAK_BGF06 CONV_SRC06 CONV_BGF06 PEAK_SRC07 PEAK_BGF07 CONV_SRC07 CONV_BGF07
# Make a column of 1's identifying protostellar cores
protostellar_catalog_column = numpy.empty(len(good_cores_array[:, 0]),
dtype='object')
if len(cross_matched_core_indices) > 0:
protostellar_catalog_column[numpy.array(
cross_matched_core_indices)] = '1'
protostellar_catalog_column = numpy.array(protostellar_catalog_column,
dtype='S12')
# Make a column indicating core type: starless, prestellar, or protostellar
core_type_column = numpy.where(alpha_BE <= 5.0, "prestellar", "starless")
core_type_column = numpy.array(core_type_column, dtype='S12')
core_type_column2 = numpy.where(protostellar_catalog_column == '1',
"protostellar", core_type_column)
# Make S_peak/S_background column at each wavelength
S_peak_bg_070 = additional_cores_array[:, 3] / additional_cores_array[:, 4]
S_peak_bg_160 = additional_cores_array[:, 7] / additional_cores_array[:, 8]
S_peak_bg_165 = additional_cores_array[:, 11] / additional_cores_array[:,
12]
S_peak_bg_250 = additional_cores_array[:, 15] / additional_cores_array[:,
16]
S_peak_bg_255 = additional_cores_array[:, 19] / additional_cores_array[:,
20]
S_peak_bg_350 = additional_cores_array[:, 23] / additional_cores_array[:,
24]
S_peak_bg_500 = additional_cores_array[:, 27] / additional_cores_array[:,
28]
# Make S_conv column at each wavelength (convert MJy/str to Jy/beam, then to H2/cm**2)
# Prepareobs scales down the column density image by a factor of 1e20
# The final units in the catalog will be off by a factor of 1e20
S_conv_070 = numpy.array(additional_cores_array[:, 5] * (10**6) *
((numpy.pi / 180.0 / 3600.0)**2) * 1.13309 *
(36.3**2))
S_conv_160 = numpy.array(additional_cores_array[:, 9] * (10**6) *
((numpy.pi / 180.0 / 3600.0)**2) * 1.13309 *
(36.3**2))
S_conv_165 = numpy.array(additional_cores_array[:, 13] * (10**6) *
((numpy.pi / 180.0 / 3600.0)**2) * 1.13309 *
(36.3**2))
S_conv_250 = numpy.array(additional_cores_array[:, 17] * (10**6) *
((numpy.pi / 180.0 / 3600.0)**2) * 1.13309 *
(36.3**2))
S_conv_255 = numpy.array(additional_cores_array[:, 21] * (10**6) *
((numpy.pi / 180.0 / 3600.0)**2) * 1.13309 *
(36.3**2))
S_conv_350 = numpy.array(additional_cores_array[:, 25] * (10**6) *
((numpy.pi / 180.0 / 3600.0)**2) * 1.13309 *
(36.3**2))
S_conv_500 = numpy.array(additional_cores_array[:, 29] * (10**6) *
((numpy.pi / 180.0 / 3600.0)**2) * 1.13309 *
(36.3**2))
N_H2_bg = numpy.array(additional_cores_array[:, 20])
# Define a function that produces a Flux/beam given wavelength, Temp, and ColDense
# We will input wavelength then find T and M using least squares minimization below
def col_dense(wavelength, T, N_H2):
#wavelength input in microns, Temp in Kelvin, N_H2 in cm**-2
#returns S_v in units of Jy/beam
wavelength_mm = numpy.array(wavelength) * 10.**-3.
exponent = 1.439 * (wavelength_mm**-1) * ((T / 10.)**-1)
aaa = ((2.02 * 10**20) * (numpy.exp(exponent) - 1.0))**-1.0
bbb = (0.1 * ((numpy.array(wavelength) / 300.)**-2.0)) / 0.01
ccc = (36.3 / 10.)**2.
ddd = wavelength_mm**-3.
return N_H2 * aaa * bbb * ccc * ddd * (10**-3)
guess = [10.0, 1.0 * 10.**21.]
N_H2_peak = []
counter = 0
for S_160, S_250, S_350, S_500 in zip(S_conv_160, S_conv_250, S_conv_350,
S_conv_500):
#print 'Fitting S_peak for Core ' + str(counter) + ' of ' + str(int(len(good_cores_array[:,0])))
wavelengths = [160., 250., 350., 500.]
fluxes = [S_160, S_250, S_350, S_500]
flux_err = [S_160 * 0.2, S_250 * 0.1, S_350 * 0.1, S_500 * 0.1]
try:
popt, pcov = curve_fit(col_dense,
wavelengths,
fluxes,
p0=guess,
sigma=flux_err)
except RuntimeError:
popt = [-9999., -9999.]
N_H2_peak.append(popt[1])
counter += 1
# Calculate the FWHM_mean at 500 microns
AFWH07 = good_cores_array[:, 68]
BFWH07 = good_cores_array[:, 69]
A = numpy.float64(((((AFWH07) / 60.) / 60.) * numpy.pi) / 180.) #radians
A1 = numpy.float64(numpy.tan(A / 2.) * 2. * distance * (3.086e18)) #cm
B = numpy.float64(((((BFWH07) / 60.) / 60.) * numpy.pi) / 180.) #radians
B1 = numpy.float64(numpy.tan(B / 2.) * 2. * distance * (3.086e18)) #cm
FWHM_mean_500 = mstats.gmean([A1, B1])
Vol_dense_peak = (((4.0 * numpy.log(2.0)) / numpy.pi)**
0.5) * (numpy.array(N_H2_peak) / FWHM_mean_500)
# Import CSAR-matched core indices
print "Cross-matching getsources and CSAR Catalogs:"
CSAR_matched_cores_indices = CSAR_core_cross_match.cross_match_CSAR(
getsources_core_catalog, CSAR_catalog, high_res_coldens_image)
# Get a cloumn of 1's identifying CSAR cross-matched cores
CSAR_catalog_column = numpy.zeros(len(good_cores_array[:, 0]), dtype='int')
CSAR_catalog_column[numpy.array(CSAR_matched_cores_indices)] += 1
# Make a column indicating the number of significant Herschel bands
N_SED = []
for line in good_cores_array:
counter = 0
if line[8] > 5 and line[12] > 0:
counter += 1
# Statement below uses the 160micron map, not the temp-corrected map
if line[17] > 5 and line[21] > 0:
counter += 1
if line[35] > 5 and line[39] > 0:
counter += 1
if line[53] > 5 and line[57] > 0:
counter += 1
if line[62] > 5 and line[66] > 0:
counter += 1
N_SED.append(counter)
# Convert the decimal degrees coordinates of getsources into hh:mm:ss and dd:mm:ss
RA_array = []
Dec_array = []
HGBS_name_array = []
for line in good_cores_array:
RA = astropy.coordinates.Angle(line[3], u.degree)
DEC = astropy.coordinates.Angle(line[4], u.degree)
RA_hours = str('{:.0f}'.format(round(RA.hms[0], 2)).zfill(2))
RA_minutes = str('{:.0f}'.format(round(RA.hms[1], 2)).zfill(2))
RA_seconds = str('{:.2f}'.format(round(RA.hms[2], 2)).zfill(5))
if DEC.hms[0] > 0:
DEC_degs = str('{:.0f}'.format(round(DEC.dms[0], 2)).zfill(2))
DEC_minutes = str('{:.0f}'.format(round(DEC.dms[1], 2)).zfill(2))
DEC_seconds = str('{:.2f}'.format(round(DEC.dms[2], 2)).zfill(5))
name_sign = '+'
HGBS_name = RA_hours + RA_minutes + RA_seconds[
0:4] + name_sign + DEC_degs + DEC_minutes + DEC_seconds[0:2]
else:
DEC_degs = str('{:.0f}'.format(round(DEC.dms[0], 2)).zfill(3))
DEC_minutes = str('{:.0f}'.format(round(DEC.dms[0] * -1,
2)).zfill(2))
DEC_seconds = str('{:.2f}'.format(round(DEC.dms[2] * -1,
2)).zfill(5))
HGBS_name = RA_hours + RA_minutes + RA_seconds[
0:4] + DEC_degs + DEC_minutes + DEC_seconds[0:2]
RA_array.append(RA_hours + ':' + RA_minutes + ':' + RA_seconds)
Dec_array.append(DEC_degs + ':' + DEC_minutes + ':' + DEC_seconds)
HGBS_name_array.append("HGBS_J" + HGBS_name)
core_number = numpy.arange(len(good_cores_array[:, 0])) + 1
catalog_array_70_160 = good_cores_array[:, 8:26]
catalog_array_70_160 = numpy.delete(catalog_array_70_160, (1, 10), 1)
catalog_array_250 = good_cores_array[:, 35:44]
catalog_array_250 = numpy.delete(catalog_array_250, 1, 1)
catalog_array_coldense = good_cores_array[:, 44:53]
catalog_array_coldense = numpy.delete(catalog_array_coldense, 1, 1)
catalog_array_350_500 = good_cores_array[:, 53:71]
catalog_array_350_500 = numpy.delete(catalog_array_350_500, (1, 10), 1)
Cloud, Name, Av, alpha, T_bol, L_bol, alphaPrime, TbolPrime, LbolPrime, likelyAGB, Dunham_RA, Dunham_DEC, Class = numpy.loadtxt(
Dunham_YSOs_file,
delimiter=',',
unpack=True,
dtype=[('Cloud', 'S30'), ('Name', 'S40'), ('Av', float),
('alpha', float), ('T_bol', float), ('L_bol', float),
('alphaPrime', float), ('TbolPrime', float),
('LbolPrime', float), ('likelyAGB', 'S1'), ('Dunham_RA', float),
('Dunham_DEC', float), ('Class', 'S10')])
Dunham_indices = numpy.where(Cloud == cloud_name)
Spitzer_YSOs_RA = Dunham_RA[Dunham_indices]
Spitzer_YSOs_DEC = Dunham_DEC[Dunham_indices]
Spitzer_YSOs_Name = Name[Dunham_indices]
potential_matches = []
YSO_matches = []
count = 0
for line in good_cores_array:
match_counter = 0
YSO_index = 0
for RA, DEC in zip(Spitzer_YSOs_RA, Spitzer_YSOs_DEC):
distance = ((line[3] - RA)**2 + (line[4] - DEC)**2)**0.5
if distance < 6.0 / 3600. and match_counter == 0:
# matched_counter prevents counting indices twice
# if two YSO candidates fall within getsources ellipse
potential_matches.append(count)
match_counter += 1
YSO_matches.append(YSO_index)
YSO_index += 1
count += 1
Spitzer_column = numpy.zeros(len(good_cores_array[:, 0]), dtype='S40')
Spitzer_column[numpy.arange(0, len(good_cores_array[:, 0]))] = 'None'
if len(potential_matches) > 0:
Spitzer_column[numpy.array(potential_matches)] = Spitzer_YSOs_Name[
numpy.array(YSO_matches)]
# Cross-match cores with SIMBAD catalog
print "Cross-matching SIMBAD catalog:"
RA, Dec = numpy.loadtxt(SED_figure_directory + region_name +
'_SIMBAD_RA_DEC.dat',
unpack=True)
Simbad.ROW_LIMIT = 1
results = []
for i, j in zip(RA, Dec):
result_table = Simbad.query_region(astropy.coordinates.SkyCoord(
ra=i, dec=j, unit=(u.deg, u.deg)),
radius=6. * u.arcsec)
if result_table != None:
results.append(result_table['MAIN_ID'][0].replace(" ", "_"))
else:
results.append('None')
# Cross-match cores with NED catalog
print "Cross-matching NED catalog:"
Ned.ROW_LIMIT = 1
results2 = []
for i, j in zip(RA, Dec):
result_table_value = 'Yes'
try:
result_table = Ned.query_region(astropy.coordinates.SkyCoord(
ra=i, dec=j, unit=(u.deg, u.deg)),
radius=6. * u.arcsec)
except astroquery.exceptions.RemoteServiceError:
result_table_value = None
if result_table_value != None:
results2.append(result_table['Object Name'][0].replace(" ", "_"))
else:
results2.append('None')
zipped_array = zip(
core_number, HGBS_name_array, RA_array, Dec_array,
catalog_array_70_160[:, 0], catalog_array_70_160[:, 1],
catalog_array_70_160[:, 2], S_peak_bg_070, S_conv_070,
catalog_array_70_160[:, 3], catalog_array_70_160[:, 4],
catalog_array_70_160[:, 5], catalog_array_70_160[:, 6],
catalog_array_70_160[:, 7], catalog_array_70_160[:, 8],
catalog_array_70_160[:, 9], catalog_array_70_160[:, 10], S_peak_bg_160,
S_conv_160, catalog_array_70_160[:, 11], catalog_array_70_160[:, 12],
catalog_array_70_160[:, 13], catalog_array_70_160[:, 14],
catalog_array_70_160[:, 15], catalog_array_250[:, 0],
catalog_array_250[:,
1], catalog_array_250[:,
2], S_peak_bg_250, S_conv_250,
catalog_array_250[:, 3], catalog_array_250[:, 4], catalog_array_250[:,
5],
catalog_array_250[:,
6], catalog_array_250[:,
7], catalog_array_350_500[:,
0],
catalog_array_350_500[:, 1], catalog_array_350_500[:,
2], S_peak_bg_350,
S_conv_350, catalog_array_350_500[:, 3], catalog_array_350_500[:, 4],
catalog_array_350_500[:, 5], catalog_array_350_500[:, 6],
catalog_array_350_500[:, 7], catalog_array_350_500[:, 8],
catalog_array_350_500[:, 9], catalog_array_350_500[:, 10],
S_peak_bg_500, catalog_array_350_500[:, 11], catalog_array_350_500[:,
12],
catalog_array_350_500[:, 13], catalog_array_350_500[:, 14],
catalog_array_350_500[:, 15], catalog_array_coldense[:, 0],
additional_cores_array[:, 19], S_peak_bg_255, S_conv_255, N_H2_bg,
catalog_array_coldense[:, 5], catalog_array_coldense[:, 6],
catalog_array_coldense[:, 7], N_SED, CSAR_catalog_column,
core_type_column2, results, results2, Spitzer_column)
catalog1 = numpy.array(zipped_array,
dtype=[('core_number', int),
('HGBS_name_array', 'S30'),
('RA_array', 'S16'), ('Dec_array', 'S16'),
('catalog_array_70_160_1', float),
('catalog_array_70_160_2', float),
('catalog_array_70_160_3', float),
('S_peak_bg_070', float),
('S_conv_070', float),
('catalog_array_70_160_4', float),
('catalog_array_70_160_5', float),
('catalog_array_70_160_6', float),
('catalog_array_70_160_7', float),
('catalog_array_70_160_8', float),
('catalog_array_70_160_9', float),
('catalog_array_70_160_10', float),
('catalog_array_70_160_11', float),
('S_peak_bg_160', float),
('S_conv_160', float),
('catalog_array_70_160_12', float),
('catalog_array_70_160_13', float),
('catalog_array_70_160_14', float),
('catalog_array_70_160_15', float),
('catalog_array_70_160_16', float),
('catalog_array_250_1', float),
('catalog_array_250_2', float),
('catalog_array_250_3', float),
('S_peak_bg_250', float),
('S_conv_250', float),
('catalog_array_250_4', float),
('catalog_array_250_5', float),
('catalog_array_250_6', float),
('catalog_array_250_7', float),
('catalog_array_250_8', float),
('catalog_array_350_500_1', float),
('catalog_array_350_500_2', float),
('catalog_array_350_500_3', float),
('S_peak_bg_350', float),
('S_conv_350', float),
('catalog_array_350_500_4', float),
('catalog_array_350_500_5', float),
('catalog_array_350_500_6', float),
('catalog_array_350_500_7', float),
('catalog_array_350_500_8', float),
('catalog_array_350_500_9', float),
('catalog_array_350_500_10', float),
('catalog_array_350_500_11', float),
('S_peak_bg_500', float),
('catalog_array_350_500_12', float),
('catalog_array_350_500_13', float),
('catalog_array_350_500_14', float),
('catalog_array_350_500_15', float),
('catalog_array_350_500_16', float),
('catalog_array_coldense_1', float),
('catalog_array_coldense_2', float),
('S_peak_bg_255', float),
('S_conv_255', float),
('additional_cores_array_28', float),
('catalog_array_coldense_6', float),
('catalog_array_coldense_7', float),
('catalog_array_coldense_8', float),
('N_SED', int), ('CSAR_catalog_column', int),
('core_type_column', 'S16'),
('SIMBAD_column', 'S60'),
('NED_column', 'S60'),
('Spitzer_column', 'S40')])
header1 = 'core_number, core_name, RA_hms, DEC_dms, sig_070, peak_flux_070, peak_flux_err_070, peak_flux_over_bg_070, peak_070_conv_500, total_flux_070, total_flux_err_070, AFWHM_070, BFWHM_070, PA_070, sig_160, peak_flux_160, peak_flux_err_160, peak_flux_over_bg_160, peak_160_conv_500, total_flux_160, total_flux_err_160, AFWHM_160, BFWHM_160, PA_160, sig_250, peak_flux_250, peak_flux_err_250, peak_flux_over_bg_250, peak_250_conv_500, total_flux_250, total_flux_err_250, AFWHM_250, BFWHM_250, PA_250, sig_350, peak_flux_350, peak_flux_err_350, peak_flux_over_bg_350, peak_350_conv_500, total_flux_350, total_flux_err_350, AFWHM_350, BFWHM_350, PA_350, sig_500, peak_flux_500, peak_flux_err_500, peak_flux_over_bg_500, total_flux_500, total_flux_err_500, AFWHM_500, BFWHM_500, PA_500, sig_coldens, peak_flux_coldens, peak_flux_over_bg_coldens, peak_coldens_conv_500, peak_bg_coldens, AFWHM_coldens, BFWHM_coldens, PA_coldens, N_SED, CSAR, core_type, SIMBAD_match, NED_match, Spitzer_match'
numpy.savetxt(
SED_figure_directory + region_name + '_core_catalog1.dat',
catalog1,
fmt=
"%i %s %s %s %3.1f %1.2e %1.1e %3.3f %1.2e %1.2e %1.1e %3.1f %3.1f %3.1f %3.1f %1.2e %1.1e %3.3f %1.2e %1.2e %1.1e %3.1f %3.1f %3.1f %3.1f %1.2e %1.1e %3.3f %1.2e %1.2e %1.1e %3.1f %3.1f %3.1f %3.1f %1.2e %1.1e %3.3f %1.2e %1.2e %1.1e %3.1f %3.1f %3.1f %3.1f %1.2e %1.1e %3.3f %1.2e %1.1e %3.1f %3.1f %3.1f %3.1f %3.1f %3.1f %3.1f %3.1f %3.1f %3.1f %3.1f %i %i %s %s %s %s",
header=header1)
mu = 2.8 # mean molecular weight
mass_H = 1.67372e-24 # (grams) mass of neutral Hydrogen atom
solar_mass = 1.989e33 # (grams)
mass_H_solar_masses = mass_H / solar_mass
parsec = 3.086e18 # cm
R_deconv_cm = numpy.array(R_deconv) * parsec
FWHM_mean_cm = numpy.array(FWHM_mean) * parsec
N_H2_avg_1 = (numpy.array(Masses) /
(numpy.pi *
(R_deconv_cm**2.))) * (1 / (mu * mass_H_solar_masses))
N_H2_avg_2 = (numpy.array(Masses) /
(numpy.pi *
(FWHM_mean_cm**2.))) * (1 / (mu * mass_H_solar_masses))
avg_Volume_dens_1 = (numpy.array(Masses) /
(numpy.pi * (4. / 3.) *
(R_deconv_cm**3.))) * (1 /
(mu * mass_H_solar_masses))
avg_Volume_dens_2 = (numpy.array(Masses) /
(numpy.pi * (4. / 3.) *
(FWHM_mean_cm**3.))) * (1 /
(mu * mass_H_solar_masses))
catalog2 = numpy.array(zip(core_number, HGBS_name_array, RA_array,
Dec_array, R_deconv, FWHM_mean, Masses,
Masses_err, Temps, Temps_err, N_H2_peak,
N_H2_avg_1, N_H2_avg_2, Vol_dense_peak,
avg_Volume_dens_1, avg_Volume_dens_2, alpha_BE,
core_type_column2, not_accepted_counter),
dtype=[('core_number', int),
('HGBS_name_array', 'S30'),
('RA_array', 'S16'), ('Dec_array', 'S16'),
('R_deconv', float), ('FWHM_mean', float),
('Masses', float), ('Masses_err', float),
('Temps', float), ('Temps_err', float),
('N_H2_peak', float), ('N_H2_avg_1', float),
('N_H2_avg_2', float),
('Vol_dense_peak', float),
('avg_Volume_dens1', float),
('avg_Volume_dens_2', float),
('alpha_BE', float),
('core_type_column2', 'S16'),
('not_accepted_counter', 'S16')])
header2 = 'core_number, core_name, RA_hms, DEC_dms, R_deconv, FWHM_mean, Mass, Mass_err, Temp_dust, Temp_dust_err, N_H2_peak, N_H2_avg_1, N_H2_avg_2, Vol_dense_peak, avg_Volume_dens_1, avg_Volume_dens_2, alpha_BE, core_type_column2, not_accepted_counter'
numpy.savetxt(
SED_figure_directory + region_name + '_core_catalog2.dat',
catalog2,
fmt=
"%i %s %s %s %1.1e %1.1e %1.3f %1.2f %2.1f %2.1f %1.2e %1.2e %1.2e %1.2e %1.2e %1.2e %2.1f %s %s",
header=header2)
def query_object(outfile,
database=None,
querytype="both",
inname=None,
inra=None,
indec=None,
verbose=False,
sdss_dr=15,
sdss_onlyprim=True,
conrad=3,
sdsstable=None,
overwrite=False):
"""
Query an online database for an object either by name or coordinates
(or both) and write the results into an output file. Mostly this is a
subroutine for the query_and_download routine which does the query for a
list of objects/coordinates in a parallelized way. (called by query_and_download)
"""
if not overwrite and os.path.isfile(outfile):
if verbose:
print("File exists! Skipping to next...")
return (0)
# --- first try to find a match by name
if querytype == "both" or querytype == "name":
if verbose:
print(" - Searching by name ...")
try:
if database == "NED":
res = Ned.query_object(inname)
elif database == "SIMBAD":
time.sleep(1)
res = cSimbad.query_object(inname)
# nname +=1
_ = res.colnames
if len(res) == 0:
namefail = True
else:
namefail = False
if verbose:
if namefail:
print(" - object not found by name")
else:
print(" - Found by name: ", res)
except:
namefail = True
if verbose:
print(" - object not found by name")
if querytype == "name":
if verbose:
print("Object not found: ", inname, inra, indec)
return (-1)
# --- if coordinate search or by name fails, try by coordinates:
if querytype == "coordinates" or (querytype == "both" and namefail):
if verbose:
print(" - Searching by coordinates ...")
try:
coord = coordinates.SkyCoord(ra=inra,
dec=indec,
unit=(u.deg, u.deg),
frame='fk5')
if verbose:
print(coord)
if database == "NED":
res = Ned.query_region(coord,
radius=conrad * u.arcsec,
equinox='J2000')
if len(res) == 0:
if verbose:
print(" - Object not found: ", inname, inra, indec)
return (-1)
elif database == "SIMBAD":
time.sleep(1)
res = cSimbad.query_region(coord, radius=conrad * u.arcsec)
print(" query done..")
elif database == "SDSS":
# --- query region only looks for coordinates in the photo
# catalog which for some reason is null for many
# galaxies. Therefore, we have to query by sql
sqlquery = ("SELECT TOP 50 s.ra, s.dec, s.specobjid, s.z, \
s.zErr, s.zWarning, s.sciencePrimary, \
s.class, s.subclass, p.type FROM .." + sdsstable +
" as\
s LEFT JOIN ..PhotoObj AS p ON \
s.bestObjID=p.objID JOIN \
dbo.fGetNearby" + sdsstable + "Eq(" + str(inra) +
"," + str(indec) + "," + str(conrad / 60.0) + ") \
AS b ON b.SpecobjID = S.SpecobjID")
res = SDSS.query_sql(sqlquery, data_release=sdss_dr)
# res = None
if verbose:
print(" - Found by coordinates: ", res)
# return(res)
res = res[np.where(
np.array(res['sciencePrimary']) == 1)[0]]
if verbose:
print(" - Found by coordinates: ", res)
except:
# --- if both fail, object is not in database apparently
# print("ERROR: No object found neither by name nor by coordinates!")
if verbose:
print(" - Object not found: ", inname, inra, indec)
return (-1)
# f.write(res)
if res is not None:
if len(res) > 0:
# return(res)
res.write(outfile,
delimiter=',',
format='ascii',
fill_values=[(ascii.masked, '')],
overwrite=True)
return (1)
def runQueriesWithLines(self,
restFreqs,
redshiftRange=(0, 1000),
lineNames=[],
public=False,
science=False,
**kwargs):
"""Run queries for spectral lines.
Parameters
----------
restFreqs : sequence of floats
The spectral line rest frequencies to search the query results for.
redshiftRange : sequence of floats, optional
A two-element sequence defining the lower and upper limits of the
object redshifts (in that order) to be searched for. The restFreqs
will be shifted using this range to only find observations that
have spectral coverage in that redshift range. Default is to search
0 <= z <= 1000 (i.e. all redshifts).
lineNames : sequence of strs, optional
A sequence of strings containing names for each spectral line to
be searched for that will be used as column names in the results
table. This must be the same length as restFreqs. Default is to
name lines like "Line0", "Line1", "Line2", etc.
public : bool
Return only publicly available datasets?
science : bool
Return only data marked as "science" in the archive?
kwargs : dict
Keywords that are accepted by the ALMA archive system. You can look
these up by examining the forms at http://almascience.org/aq.
Passed to `astroquery.alma.Alma.query`. "frequency" cannot be
specified here since it is used to limit the query to frequencies
that could contain the lines in the specified redshift range. If
archiveSearch was initialized with the `targets` argument then
"source_name_resolver" and "ra_dec" also cannot be used here.
Matching against NED to find source redshifts is attempted first with
the ALMA archive coordinates, searching in NED with a search radius of
30 arcseconds and only keeping results with type G (galaxy). If more
or less than one NED result matches the positional search then a search
is attempted based on a sanitized version of the ALMA archive source
name. If there is no match to name then the ALMA observation is placed
in the queryResultsNoNED dictionary.
"""
if 'frequency' in kwargs:
msg = '"frequency" cannot be passed to runQueriesWithLines'
raise ValueError(msg)
restFreqs = np.array(restFreqs)
lineNames = np.array(lineNames)
if (len(lineNames) != len(restFreqs) and len(lineNames) != 0):
msg = 'length mismatch between ' \
+ '"restFreqs" ({:})'.format(len(restFreqs)) \
+ ' and "lineNames" ({:})'.format(len(lineNames))
raise ValueError(msg)
if len(lineNames) == 0:
lineNames = ['Line{:}'.format(i) for i in range(len(restFreqs))]
lineNames = np.array(lineNames)
inds = restFreqs.argsort()
restFreqs = restFreqs[inds]
lineNames = lineNames[inds]
redshiftRange = np.array(redshiftRange)
redshiftRange.sort()
# define frequency range from lines and redshifts
lowFreq = self._observedFreq(restFreqs[0], redshiftRange[1])
highFreq = self._observedFreq(restFreqs[-1], redshiftRange[0])
freqLimits = '{:} .. {:}'.format(lowFreq, highFreq)
self.runQueries(public=public,
science=science,
frequency=freqLimits,
**kwargs)
for target in self.targets:
if len(self.queryResults[target]) > 0: # targets with ALMA results
currTable = self.queryResults[target]
# sanitize ALMA source names
safeNames = currTable['target_name']
safeNames = np.char.replace(safeNames, b' ', b'')
safeNames = np.char.replace(safeNames, b'_', b'')
safeNames = np.char.upper(safeNames)
currTable['ALMA sanitized source name'] = safeNames
# query NED for object redshifts
nedResult = list()
noNEDinds = list()
searchCoords = SkyCoord(ra=currTable['s_ra'],
dec=currTable['s_dec'],
unit=(u.deg, u.deg),
frame='icrs')
pBar = trange(len(currTable),
desc='NED cross matching',
unit=' source')
for i in pBar:
# coordinate search
try:
nedSearch = Ned.query_region(searchCoords[i],
radius=30 * u.arcsec,
equinox='J2000.0')
except Exception:
pass
# only want galaxies
typeInds = np.where(nedSearch['Type'] != b'G')
nedSearch.remove_rows(typeInds)
# try name search when not just one coordinate match
if len(nedSearch) != 1:
try:
nedSearch = Ned.query_object(
currTable['ALMA sanitized source name'][i])
except Exception:
pass
if len(nedSearch) != 1:
noNEDinds.append(i)
else:
# next line prevents vstack warnings
nedSearch.meta = None
nedResult.append(nedSearch)
if len(nedResult) > 0:
nedResult = vstack(nedResult, join_type='exact')
else:
msg = 'No NED results for {:} returned. nedResult = {:}'
raise ValueError(msg.format(target, nedResult))
# store away rows without a single NED match
self.queryResultsNoNED[target] = currTable[noNEDinds]
currTable.remove_rows(noNEDinds)
# store away rows without redshift in NED
noZinds = nedResult['Redshift'].mask.nonzero()
nedResult.remove_rows(noZinds)
self.queryResultsNoNEDz[target] = currTable[noZinds]
currTable.remove_rows(noZinds)
# remove rows where redshift not in range
outOfRangeZInds = list()
for i, row in enumerate(nedResult):
if (redshiftRange[0] > row['Redshift']
or redshiftRange[1] < row['Redshift']):
outOfRangeZInds.append(i)
nedResult.remove_rows(outOfRangeZInds)
currTable.remove_rows(outOfRangeZInds)
# rectify this naming difference between NED and ALMA
nedResult.rename_column('DEC', 'Dec')
nedResult.keep_columns(
['Object Name', 'RA', 'Dec', 'Redshift'])
ALMAnedResults = hstack([currTable, nedResult],
join_type='exact')
# tidy up column names
ALMAnedResults.rename_column('target_name', 'ALMA source name')
ALMAnedResults.rename_column('s_ra', 'ALMA RA')
ALMAnedResults.rename_column('s_dec', 'ALMA Dec')
ALMAnedResults.rename_column('Object Name', 'NED source name')
ALMAnedResults.rename_column('RA', 'NED RA')
ALMAnedResults.rename_column('Dec', 'NED Dec')
ALMAnedResults.rename_column('Redshift', 'NED Redshift')
# mark flags if spw is on line (initialized to False)
lineObserved = np.zeros((len(ALMAnedResults), len(restFreqs)),
dtype=bool)
for i, row in enumerate(ALMAnedResults):
obsFreqs = self._observedFreq(restFreqs,
row['NED Redshift'])
for j in range(len(obsFreqs)):
for spwRange in row['Frequency ranges']:
if not lineObserved[i, j]:
if spwRange[0] <= obsFreqs[j] <= spwRange[1]:
lineObserved[i, j] = True
else:
break
for i in range(len(restFreqs)):
ALMAnedResults[lineNames[i]] = lineObserved[:, i]
# remove rows which have no lines covered
lineCount = np.array(ALMAnedResults[lineNames[0]], dtype=int)
for i in range(1, len(restFreqs)):
lineCount += np.array(ALMAnedResults[lineNames[i]],
dtype=int)
noLinesInds = np.where(lineCount == 0)
ALMAnedResults.remove_rows(noLinesInds)
self.queryResults[target] = ALMAnedResults
def query_region(self,
objectname,
match_tol=1.0,
obj_radius=1.0,
bycoord=False):
'''
Fetch remote data from NED and SIMBAD matching coordinates and build table.
'''
# Create custom query objects.
customSimbad = Simbad()
customNed = Ned()
# Log SIMBAD votable (changeable) fields.
logging.debug("SIMBAD votable fields")
logging.debug(customSimbad.get_votable_fields())
customSimbad.remove_votable_fields('coordinates')
# customSimbad.add_votable_fields("otype(3)", "ra(d)", "dec(d)")
customSimbad.add_votable_fields("otype", "ra(d)", "dec(d)")
# Download object data from both SIMBAD and NED.
logging.info(
"Querying SIMBAD and NED for region {}".format(objectname))
if bycoord:
try:
objectcoords = SkyCoord(objectname)
except (ValueError, u.UnitsError):
logging.info("Invalid coordinates.")
return
else:
# Resolve the object name into sky coordinate using NED
# ensures that NED and SIMBAD searches are using the same position
sesame_database.set('ned')
try:
objectcoords = get_icrs_coordinates(objectname)
except NameResolveError:
logging.info("Name resolution failed.")
return
logging.info("Name resolved to coordinates {}".format(objectcoords))
# SIMBAD
logging.info("SIMBAD is currently being queried...")
try:
with warnings.catch_warnings(
): # suppress warnings generated by SIMBAD query
warnings.simplefilter("ignore")
simbad_table = customSimbad.query_region(objectcoords,
radius=obj_radius *
u.arcmin)
# workaround. If SIMBAD query finds nothing, returns None but we want a zero-length table
if type(simbad_table) is not Table:
logging.debug("No SIMBAD objects")
simbad_table = Table(data=None,
names=DataController.simbad_table_cols,
dtype=DataController.simbad_table_dtypes,
masked=True)
logging.info("SUCCESS: SIMBAD Data retrieved.")
except Timeout:
logging.debug("SIMBAD timeout error")
return
# NED
logging.info("NED is currently being queried...")
for attempt in range(
3): # sometimes NED times out, so try a couple of times
Ned.TIMEOUT = (attempt + 1) * DataController.ned_timeout_default
try:
ned_table = Ned.query_region(objectcoords,
radius=obj_radius * u.arcmin)
logging.info("SUCCESS: NED Data retrieved.")
except RequestException:
logging.debug("NED problem, retrying")
else: # if attempt successful break out of loop, no need to try again
break
else: # closes for loop: executes only if all attempts fail
logging.debug("NED query failed")
return
# Save some query stats.
self.stats.query_name = objectname
self.stats.sim_count = len(simbad_table)
self.stats.ned_count = len(ned_table)
# process tables
ned_table = self.reformat_table(
ned_table,
keepcolsifpresent=[
'Object Name',
# cover NED changing names of cols
'RA(deg)',
'RA',
'DEC(deg)',
'DEC',
'Type'
],
old_name='Object Name',
new_name='Name_N',
old_type='Type',
new_type='Type_N')
logging.info("Reformatting tables.")
simbad_table = self.reformat_table(
simbad_table,
keepcolsifpresent=["MAIN_ID", "RA_d", "DEC_d", "OTYPE"],
old_name='MAIN_ID',
new_name='Name_S',
old_type='OTYPE',
new_type='Type_S')
logging.info("Building sky coordinates.")
# Build SkyCoord from appropriate ned and simbad col's with matching units
ned_coo = SkyCoord(ra=ned_table['RA(deg)'], dec=ned_table['DEC(deg)'])
sim_coo = SkyCoord(ra=simbad_table['RA_d'], dec=simbad_table['DEC_d'])
logging.info("Finding object matches.")
# Find object matches
if len(ned_coo) > 0 and len(sim_coo) > 0:
matched_ned, matched_sim, ned_only, sim_only = self.symmetric_match_sky_coords_v2(
ned_coo, sim_coo, match_tol * u.arcsec)
else:
matched_ned = []
matched_sim = []
ned_only = []
sim_only = []
logging.debug("")
logging.debug("Matched NED rows:")
logging.debug(ned_table[matched_ned])
logging.debug("Matched SIMBAD rows:")
logging.debug(simbad_table[matched_sim])
logging.debug("")
self.stats.overlap_count = len(matched_ned)
# Explore results
logging.debug("Matched NED:")
logging.debug(matched_ned)
logging.debug("Matched SIMBAD")
logging.debug(matched_sim)
logging.debug("NED ONLY")
logging.debug(ned_only)
logging.debug("SIMBAD ONLY")
logging.debug(sim_only)
# Generate the matched table and save the result.
logging.info("Building combined table.")
matched_table = hstack(
[ned_table[matched_ned], simbad_table[matched_sim]],
join_type='outer',
metadata_conflicts='silent') # Hide the metadata warning.
self.combined_table = matched_table
def test2():
result_table = Ned.query_region("3c 273", radius=0.05*u.deg)
print(result_table)
def test3():
result_table = Ned.query_region(coord.FK4(ra=56.38, dec=38.43,
unit=(u.deg, u.deg)), radius=0.1 * u.deg, equinox='B1950.0')
print(result_table)
def getNEDInfo(df):
df.reset_index(inplace=True, drop=True)
df['NED_name'] = ""
df['NED_type'] = ""
df["NED_vel"] = np.nan
df["NED_redshift"] = np.nan
df["NED_mag"] = np.nan
ra = df["raMean"]
dec = df["decMean"]
# setup lists for ra and dec in hr format, names of NED-identified object, and
# separation between host in PS1 and host in NED
ra_hms = []
dec_dms = []
names = []
sep = []
missingCounter = 0
for index, row in df.iterrows():
tempRA = ra[index]
tempDEC = dec[index]
# create a sky coordinate to query NED
c = SkyCoord(ra=tempRA * u.degree,
dec=tempDEC * u.degree,
frame='icrs')
# execute query
result_table = []
tempName = ""
tempType = ""
tempRed = np.nan
tempVel = np.nan
tempMag = np.nan
try:
result_table = Ned.query_region(c,
radius=(0.00055555) * u.deg,
equinox='J2000.0')
#print(result_table)
if len(result_table) > 0:
missingCounter = 0
except:
missingCounter += 1
#print(c)
if len(result_table) > 0:
result_table = result_table[result_table['Separation'] == np.min(
result_table['Separation'])]
result_table = result_table[result_table['Type'] != b'SN']
result_table = result_table[result_table['Type'] != b'MCld']
result_gal = result_table[result_table['Type'] == b'G']
if len(result_gal) > 0:
result_table = result_gal
if len(result_table) > 0:
result_table = result_table[
result_table['Photometry Points'] == np.nanmax(
result_table['Photometry Points'])]
result_table = result_table[result_table['References'] == np.
nanmax(result_table['References'])]
#return result_table
# NED Info is presented as:
# No. ObjectName RA DEC Type Velocity Redshift Redshift Flag Magnitude and Filter Separation References Notes Photometry Points Positions Redshift Points Diameter Points Associations
#Split NED info up - specifically, we want to pull the type, velocity, redshift, mag
tempNED = str(np.array(result_table)[0]).split(",")
if len(tempNED) > 2:
#print("Found one!")
tempName = tempNED[1].strip().strip("b").strip("'")
if len(tempNED) > 20:
seps = [
float(tempNED[9].strip()),
float(tempNED[25].strip())
]
if np.argmin(seps):
tempNED = tempNED[16:]
tempType = tempNED[4].strip().strip("b").strip("''")
tempVel = tempNED[5].strip()
tempRed = tempNED[6].strip()
tempMag = tempNED[8].strip().strip("b").strip("''").strip(
">").strip("<")
if tempName:
df.loc[index, 'NED_name'] = tempName
if tempType:
df.loc[index, 'NED_type'] = tempType
if tempVel:
df.loc[index, 'NED_vel'] = float(tempVel)
if tempRed:
df.loc[index, 'NED_redshift'] = float(tempRed)
if tempMag:
tempMag = re.findall(r"[-+]?\d*\.\d+|\d+", tempMag)[0]
df.loc[index, 'NED_mag'] = float(tempMag)
if missingCounter > 5000:
print("Locked out of NED, will have to try again later...")
return df
return df
h['CALC_COO'] = h['CALC_COO'].replace('h', ':')
h['CALC_COO'] = h['CALC_COO'].replace('d', ':')
h['CALC_COO'] = h['CALC_COO'].replace('m', ':')
h['CALC_COO'] = h['CALC_COO'].replace('s', '')
s = 'Object Name: {OBJECT:s}\n' \
'Target Coords: {RA:s} {DEC:s} \n' \
'Coords Found: {CALC_COO:s}'.format(**hdu.header)
ax2 = fig.add_subplot(gs[1])
ax2.text(0.05, 0.5, s, va='center', ha='left')
ax2.set_xticks([])
ax2.set_yticks([])
r = 3 * u.arcmin
region = Ned.query_region(coordinates=c, radius=r)
region = region.to_pandas()
print(region)
# for obj in region:
# obj_coord = SkyCoord(ra=obj['RA(deg)'] * u.deg, dec=obj['DEC(deg)'] * u.deg)
# x, y = obj_coord.to_pixel(wcs)
# ax1.scatter(x, y, edgecolor='None', facecolor='red', marker='o')
#
# plt.tight_layout(pad=1.3, w_pad=2)
# plt.show()
def ProcessTNSEmails(self,post=True,posturl=None,db=None):
body = ""
html = ""
tns_objs = []
radius = 5 # arcminutes
########################################################
# Get All Email
########################################################
mail = imaplib.IMAP4_SSL('imap.gmail.com', 993) #, ssl_context=ctx
## NOTE: This is not the way to do this. You will want to implement an industry-standard login step ##
mail.login(self.login, self.password)
mail.select('TNS', readonly=False)
retcode, msg_ids_bytes = mail.search(None, '(UNSEEN)')
msg_ids = msg_ids_bytes[0].decode("utf-8").split(" ")
try:
if retcode != "OK" or msg_ids[0] == "":
raise ValueError("No messages")
except ValueError as err:
print("%s. Exiting..." % err.args)
mail.close()
mail.logout()
del mail
print("Process done.")
return
for i in range(len(msg_ids)):
########################################################
# Iterate Over Email
########################################################
typ, data = mail.fetch(msg_ids[i],'(RFC822)')
msg = email.message_from_bytes(data[0][1])
# Mark messages as "Unseen"
# result, wdata = mail.store(msg_ids[i], '-FLAGS', '\Seen')
if msg.is_multipart():
for part in msg.walk():
ctype = part.get_content_type()
cdispo = str(part.get('Content-Disposition'))
# skip any text/plain (txt) attachments
if ctype == 'text/plain' and 'attachment' not in cdispo:
body = part.get_payload(decode=True) # decode
break
# not multipart - i.e. plain text, no attachments, keeping fingers crossed
else:
body = msg.get_payload(decode=True)
objs = re.findall(reg_obj,body)
print(objs)
ras = re.findall(reg_ra,body)
print(ras)
decs = re.findall(reg_dec,body)
print(decs)
try:
########################################################
# For Item in Email, Get TNS
########################################################
for j in range(len(objs)):
print("Object: %s\nRA: %s\nDEC: %s" % (objs[j].decode('utf-8'),
ras[j].decode('utf-8'),
decs[j].decode('utf-8')))
# Get TNS page
int_name=""
evt_type=""
z=""
host_name=""
host_redshift = ""
ned_url = ""
tns_url = "https://wis-tns.weizmann.ac.il/object/" + objs[j].decode("utf-8")
print(tns_url)
tstart = time.time()
try:
response = requests.get(tns_url,timeout=20)
html = response.content
except:
print('trying again')
response = requests.get(tns_url,timeout=20)
html = response.content
soup = BeautifulSoup(html, "lxml")
# Get Internal Name, Type, Disc. Date, Disc. Mag, Redshift, Host Name, Host Redshift, NED URL
int_name = soup.find('td', attrs={'class':'cell-internal_name'}).text
evt_type = soup.find('div', attrs={'class':'field-type'}).find('div').find('b').text
evt_type = evt_type #.replace(' ','')
disc_date = soup.find('div', attrs={'class':'field field-discoverydate'}).find('div').find('b').text
disc_mag = soup.find('div', attrs={'class':'field field-discoverymag'}).find('div').find('b').text
try: source_group = soup.find('div', attrs={'class':'field field-source_group_name'}).find('div').find('b').text
except AttributeError: source_group = "Unknown"
try: disc_filter = soup.find('td', attrs={'cell':'cell-filter_name'}).text
except AttributeError: disc_filter = "Unknown"
if '-' in disc_filter:
disc_instrument = disc_filter.split('-')[1]
else: disc_instrument = 'Unknown'
# lets pull the photometry
nondetectmaglim = None
nondetectdate = None
nondetectfilt = None
tmag,tmagerr,tflux,tfluxerr,tfilt,tinst,tobsdate = \
np.array([]),np.array([]),np.array([]),np.array([]),\
np.array([]),np.array([]),np.array([])
try:
tables = soup.find_all('table',attrs={'class':'photometry-results-table'})
for table in tables:
data = []
table_body = table.find('tbody')
header = table.find('thead')
headcols = header.find_all('th')
header = np.array([ele.text.strip() for ele in headcols])
#header.append([ele for ele in headcols if ele])
rows = table_body.find_all('tr')
for row in rows:
cols = row.find_all('td')
data.append([ele.text.strip() for ele in cols])
for datarow in data:
datarow = np.array(datarow)
if photkeydict['unit'] in header:
if 'mag' in datarow[header == photkeydict['unit']][0].lower():
if photkeydict['magflux'] in header:
tmag = np.append(tmag,datarow[header == photkeydict['magflux']])
tflux = np.append(tflux,'')
else:
tmag = np.append(tmag,'')
tflux = np.append(tflux,'')
if photkeydict['magfluxerr'] in header:
tmagerr = np.append(tmagerr,datarow[header == photkeydict['magfluxerr']])
tfluxerr = np.append(tfluxerr,'')
else:
tmagerr = np.append(tmagerr,None)
tfluxerr= np.append(tfluxerr,'')
elif 'flux' in datarow[header == photkeydict['unit']][0].lower():
if photkeydict['magflux'] in header:
tflux = np.append(tflux,datarow[header == photkeydict['magflux']])
tmag = np.append(tmag,'')
else:
tflux = np.append(tflux,'')
tmag = np.append(tmag,'')
if photkeydict['magfluxerr'] in header:
tfluxerr = np.append(tfluxerr,datarow[header == photkeydict['magfluxerr']])
tmagerr = np.append(tmagerr,'')
else:
tfluxerr = np.append(tfluxerr,None)
tmagerr = np.append(tmagerr,'')
if photkeydict['filter'] in header:
tfilt = np.append(tfilt,datarow[header == photkeydict['filter']])
if photkeydict['inst'] in header:
tinst = np.append(tinst,datarow[header == photkeydict['inst']])
if photkeydict['obsdate'] in header:
tobsdate = np.append(tobsdate,datarow[header == photkeydict['obsdate']])
if photkeydict['remarks'] in header and photkeydict['maglim'] in header:
if 'last' in datarow[header == photkeydict['remarks']][0].lower() and \
'non' in datarow[header == photkeydict['remarks']][0].lower() and \
'detection' in datarow[header == photkeydict['remarks']][0].lower():
nondetectmaglim = datarow[header == photkeydict['maglim']][0]
nondetectdate = datarow[header == photkeydict['obsdate']][0]
nondetectfilt = datarow[header == photkeydict['filter']][0]
# set the discovery flag
disc_flag = np.zeros(len(tmag))
iMagsExist = np.where(tmag != '')[0]
if len(iMagsExist) == 1: disc_flag[np.where(tmag != '')] = 1
elif len(iMagsExist) > 1:
mjd = np.zeros(len(iMagsExist))
for d in range(len(mjd)):
mjd[d] = date_to_mjd(tobsdate[d])
iMinMJD = np.where(mjd == np.min(mjd))[0]
if len(iMinMJD) > 1: iMinMJD = [iMinMJD[0]]
for im,iim in zip(iMagsExist,range(len(iMagsExist))):
if len(iMinMJD) and iim == iMinMJD[0]:
disc_flag[im] = 1
except:
print('Error : couldn\'t get photometry!!!')
z = soup.find('div', attrs={'class':'field-redshift'}).find('div').find('b').text
hn_div = soup.find('div', attrs={'class':'field-hostname'})
if hn_div is not None:
host_name = hn_div.find('div').find('b').text
z_div = soup.find('div', attrs={'class':'field-host_redshift'})
if z_div is not None:
host_redshift = z_div.find('div').find('b').text
ned_url = soup.find('div', attrs={'class':'additional-links clearfix'}).find('a')['href']
# Get photometry records
table = soup.findAll('table', attrs={'class':'photometry-results-table'})
prs = []
for k in range(len(table)):
table_body = table[k].find('tbody')
rows = table_body.find_all('tr')
print(type(rows))
for l in range(len(rows)):
prs.append(phot_row(rows[l]))
########################################################
# For Item in Email, Get NED
########################################################
ra_j = ras[j].decode("utf-8")
dec_j = decs[j].decode("utf-8")
co = coordinates.SkyCoord(ra=ra_j, dec=dec_j, unit=(u.hour, u.deg), frame='fk4', equinox='J2000.0')
dust_table_l = IrsaDust.get_query_table(co)
ebv = dust_table_l['ext SandF mean'][0]
ned_region_table = None
gal_candidates = 0
radius = 5
while (radius < 11 and gal_candidates < 21):
try:
print("Radius: %s" % radius)
ned_region_table = Ned.query_region(co, radius=radius*u.arcmin, equinox='J2000.0')
gal_candidates = len(ned_region_table)
radius += 1
print("Result length: %s" % gal_candidates)
except Exception as e:
radius += 1
print("NED exception: %s" % e.args)
galaxy_names = []
galaxy_zs = []
galaxy_seps = []
galaxies_with_z = []
galaxy_ras = []
galaxy_decs = []
galaxy_mags = []
if ned_region_table is not None:
print("NED Matches: %s" % len(ned_region_table))
galaxy_candidates = np.asarray([entry.decode("utf-8") for entry in ned_region_table["Type"]])
galaxies_indices = np.where(galaxy_candidates == 'G')
galaxies = ned_region_table[galaxies_indices]
print("Galaxy Candidates: %s" % len(galaxies))
# Get Galaxy name, z, separation for each galaxy with z
for l in range(len(galaxies)):
if isinstance(galaxies[l]["Redshift"], float):
galaxies_with_z.append(galaxies[l])
galaxy_names.append(galaxies[l]["Object Name"])
galaxy_zs.append(galaxies[l]["Redshift"])
galaxy_seps.append(galaxies[l]["Distance (arcmin)"])
galaxy_ras.append(galaxies[l]["RA(deg)"])
galaxy_decs.append(galaxies[l]["DEC(deg)"])
galaxy_mags.append(galaxies[l]["Magnitude and Filter"])
print("Galaxies with z: %s" % len(galaxies_with_z))
# Get Dust in LoS for each galaxy with z
if len(galaxies_with_z) > 0:
for l in range(len(galaxies_with_z)):
co_l = coordinates.SkyCoord(ra=galaxies_with_z[l]["RA(deg)"],
dec=galaxies_with_z[l]["DEC(deg)"],
unit=(u.deg, u.deg), frame='fk4', equinox='J2000.0')
else:
print("No NED Galaxy hosts with z")
tns_objs.append(tns_obj(name = objs[j].decode("utf-8"),
tns_url = tns_url,
internal_name = int_name,
event_type = evt_type,
ra = ras[j].decode("utf-8"),
dec = decs[j].decode("utf-8"),
ebv = ebv,
z = z,
tns_host = host_name,
tns_host_z = host_redshift,
ned_nearest_host = galaxy_names,
ned_nearest_z = galaxy_zs,
ned_nearest_sep = galaxy_seps,
discovery_date = disc_date,
phot_rows = prs,
disc_mag = disc_mag
))
if post:
snid = objs[j].decode("utf-8")
# if source_group doesn't exist, we need to add it
groupid = db.get_ID_from_DB('observationgroups',source_group)
if not groupid:
groupid = db.get_ID_from_DB('observationgroups','Unknown')#db.post_object_to_DB('observationgroup',{'name':source_group})
# get the status
statusid = db.get_ID_from_DB('transientstatuses','New')
if not statusid: raise RuntimeError('Error : not all statuses are defined')
# put in the hosts
hostcoords = ''; hosturl = ''; ned_mag = ''
for z,name,ra,dec,sep,mag in zip(galaxy_zs,galaxy_names,galaxy_ras,galaxy_decs,galaxy_seps,galaxy_mags):
if sep == np.min(galaxy_seps):
hostdict = {'name':name,'ra':ra,'dec':dec,'redshift':z}
hostoutput = db.post_object_to_DB('host',hostdict,return_full=True)
hosturl = hostoutput['url']
ned_mag = mag
hostcoords += 'ra=%.7f, dec=%.7f\n'%(ra,dec)
# put in the spec type
eventid = db.get_ID_from_DB('transientclasses',evt_type)
if not eventid:
eventid = db.get_ID_from_DB('transientclasses','Unknown')#db.post_object_to_DB('transientclasses',{'name':evt_type})
# first check if already exists
dbid = db.get_ID_from_DB('transients',snid)
k2id = db.get_ID_from_DB('internalsurveys','K2')
# then POST or PUT, depending
# put in main transient
sc = SkyCoord(ras[j].decode("utf-8"),decs[j].decode("utf-8"),FK5,unit=(u.hourangle,u.deg))
db.options.best_spec_classapi = db.options.transientclassesapi
newobjdict = {'name':objs[j].decode("utf-8"),
'ra':sc.ra.deg,
'dec':sc.dec.deg,
#'status':statusid,
'obs_group':groupid,
'host':hosturl,
'candidate_hosts':hostcoords,
'best_spec_class':eventid,
'TNS_spec_class':evt_type,
'mw_ebv':ebv,
'disc_date':disc_date.replace(' ','T'),
'tags':[]}
if nondetectdate: newobjdict['non_detect_date'] = nondetectdate.replace(' ','T')
if nondetectmaglim: newobjdict['non_detect_limit'] = nondetectmaglim
if nondetectfilt:
nondetectid = db.get_ID_from_DB('photometricbands',nondetectfilt)
if nondetectid:
newobjdict['non_detect_filter'] = nondetectid
if dbid:
# if the status is ignore, we're going to promote this to new
status_getid = db.get_key_from_object(dbid,'status')
statusname = db.get_key_from_object(status_getid,'name')
if statusname == 'Ignore':
newobjdict['status'] = statusid
transientid = db.patch_object_to_DB('transient',newobjdict,dbid)
else:
newobjdict['status'] = statusid
transientid = db.post_object_to_DB('transient',newobjdict)
# only add in host info and photometry if galaxy wasn't already in the database
# (avoids duplicates)
if not dbid:
# the photometry table probably won't exist, so add this in
# phot table needs an instrument, which needs a telescope, which needs an observatory
for ins in np.unique(tinst):
instrumentid = db.get_ID_from_DB('instruments',ins)
if not instrumentid:
instrumentid = db.get_ID_from_DB('instruments','Unknown')
if not instrumentid:
observatoryid = db.post_object_to_DB(
'observatory',{'name':'Unknown','tz_name':0,'utc_offset':0})
teldict= {'name':'Unknown',
'observatory':observatoryid,
'longitude':0,
'latitude':0,
'elevation':0}
telid = db.post_object_to_DB('telescope',teldict)
instrumentid = db.post_object_to_DB(
'instrument',{'name':'Unknown','telescope':telid})
phottabledict = {'transient':transientid,
'obs_group':groupid,
'instrument':instrumentid}
phottableid = db.post_object_to_DB('photometry',phottabledict)
for f in np.unique(tfilt):
bandid = db.get_ID_from_DB('photometricbands',f)
if not bandid:
bandid = db.post_object_to_DB('band',{'name':f,'instrument':instrumentid})
# put in the photometry
for m,me,f,fe,od,df in zip(tmag[(f == tfilt) & (ins == tinst)],
tmagerr[(f == tfilt) & (ins == tinst)],
tflux[(f == tfilt) & (ins == tinst)],
tfluxerr[(f == tfilt) & (ins == tinst)],
tobsdate[(f == tfilt) & (ins == tinst)],
disc_flag[(f == tfilt) & (ins == tinst)]):
if not m and not me and not f and not fe: continue
# TODO: compare od to disc_date.replace(' ','T')
# if they're close or equal? Set discovery flag
photdatadict = {'obs_date':od.replace(' ','T'),
'band':bandid,
'photometry':phottableid}
if m: photdatadict['mag'] = m
if me: photdatadict['mag_err'] = me
if f: photdatadict['flux'] = f
if fe: photdatadict['flux_err'] = fe
if df: photdatadict['discovery_point'] = 1
photdataid = db.post_object_to_DB('photdata',photdatadict)
# put in the galaxy photometry
if ned_mag:
try:
unknowninstid = db.get_ID_from_DB('instruments','Unknown')
unknowngroupid = db.get_ID_from_DB('observationgroups','NED')
if not unknowngroupid:
unknowngroupid = db.get_ID_from_DB('observationgroups','Unknown')
unknownbandid = db.get_ID_from_DB('photometricbands','Unknown')
hostphottabledict = {'host':hosturl,
'obs_group':unknowngroupid,
'instrument':unknowninstid}
hostphottableid = db.post_object_to_DB('hostphotometry',hostphottabledict)
# put in the photometry
hostphotdatadict = {'obs_date':disc_date.replace(' ','T'),#'2000-01-01 00:00:00',
'mag':ned_mag.decode('utf-8')[:-1],
'band':unknownbandid,
'photometry':hostphottableid}
hostphotdataid = db.post_object_to_DB('hostphotdata',hostphotdatadict)
except:
print('getting host mag failed')
# Mark messages as "Seen"
result, wdata = mail.store(msg_ids[i], '+FLAGS', '\\Seen')
except: # ValueError as err:
for j in range(len(objs)):
print('Something went wrong!!! Sticking to basic info only')
print("Object: %s\nRA: %s\nDEC: %s" % (objs[j].decode('utf-8'),
ras[j].decode('utf-8'),
decs[j].decode('utf-8')))
snid = objs[j].decode("utf-8")
# if source_group doesn't exist, we need to add it
source_group = "Unknown"
groupid = db.get_ID_from_DB('observationgroups',source_group)
if not groupid:
groupid = db.get_ID_from_DB('observationgroups','Unknown')#db.post_object_to_DB('observationgroup',{'name':source_group})
# get the status
statusid = db.get_ID_from_DB('transientstatuses','New')
if not statusid: raise RuntimeError('Error : not all statuses are defined')
dbid = db.get_ID_from_DB('transients',snid)
k2id = db.get_ID_from_DB('internalsurveys','K2')
# then POST or PUT, depending
# put in main transient
sc = SkyCoord(ras[j].decode("utf-8"),decs[j].decode("utf-8"),FK5,unit=(u.hourangle,u.deg))
db.options.best_spec_classapi = db.options.transientclassesapi
newobjdict = {'name':objs[j].decode("utf-8"),
'ra':sc.ra.deg,
'dec':sc.dec.deg,
'status':statusid,
'obs_group':groupid,
'tags':[]}
if dbid:
transientid = db.put_object_to_DB('transient',newobjdict,dbid)
else:
transientid = db.post_object_to_DB('transient',newobjdict)
#WriteOutput(tns_objs)
print("Process done.")
def make_catalog(region_name, cloud_name, distance, good_cores_array, additional_cores_array, cross_matched_core_indices, cross_matched_proto_indices, alpha_BE, getsources_core_catalog, R_deconv, FWHM_mean, Masses, Masses_err, Temps, Temps_err, not_accepted_counter, CSAR_catalog = '/mnt/scratch-lustre/jkeown/DS9_regions/L1157/CSAR/CEPl1157_CSAR.dat', high_res_coldens_image = '/mnt/scratch-lustre/jkeown/Getsources/Prepare/Images/cep1157/080615/cep1157_255_mu.image.resamp.fits', SED_figure_directory = '/mnt/scratch-lustre/jkeown/DS9_regions/HGBS_pipeline/L1157/L1157_core_SED/', Dunham_YSOs_file = 'Dunham_YSOs.dat'):
# These are the values in each column of "good_cores_array"
#NO, XCO_P, YCO_P, WCS_ACOOR, WCS_DCOOR, SIG_GLOB, FG, GOOD, SIG_MONO01, FM01, FXP_BEST01, FXP_ERRO01, FXT_BEST01, FXT_ERRO01, AFWH01, BFWH01, THEP01, SIG_MONO02, FM02, FXP_BEST02, FXP_ERRO02, FXT_BEST02, FXT_ERRO02, AFWH02, BFWH02, THEP02, SIG_MONO03, FM03, FXP_BEST03, FXP_ERRO03, FXT_BEST03, FXT_ERRO03, AFWH03, BFWH03, THEP03, SIG_MONO04, FM04, FXP_BEST04, FXP_ERRO04, FXT_BEST04, FXT_ERRO04, AFWH04, BFWH04, THEP04, SIG_MONO05, FM05, FXP_BEST05, FXP_ERRO05, FXT_BEST05, FXT_ERRO05, AFWH05, BFWH05, THEP05, SIG_MONO06, FM06, FXP_BEST06, FXP_ERRO06, FXT_BEST06, FXT_ERRO06, AFWH06, BFWH06, THEP06, SIG_MONO07, FM07, FXP_BEST07, FXP_ERRO07, FXT_BEST07, FXT_ERRO07, AFWH07, BFWH07, THEP07
# These are the values in each column of the "additional" "cores_array" and "protostar_array"
# NO XCO_P YCO_P PEAK_SRC01 PEAK_BGF01 CONV_SRC01 CONV_BGF01 PEAK_SRC02 PEAK_BGF02 CONV_SRC02 CONV_BGF02 PEAK_SRC03 PEAK_BGF03 CONV_SRC03 CONV_BGF03 PEAK_SRC04 PEAK_BGF04 CONV_SRC04 CONV_BGF04 PEAK_SRC05 PEAK_BGF05 CONV_SRC05 CONV_BGF05 PEAK_SRC06 PEAK_BGF06 CONV_SRC06 CONV_BGF06 PEAK_SRC07 PEAK_BGF07 CONV_SRC07 CONV_BGF07
# Make a column of 1's identifying protostellar cores
protostellar_catalog_column = numpy.empty(len(good_cores_array[:,0]), dtype='object')
if len(cross_matched_core_indices)>0:
protostellar_catalog_column[numpy.array(cross_matched_core_indices)]='1'
protostellar_catalog_column = numpy.array(protostellar_catalog_column, dtype='S12')
# Make a column indicating core type: starless, prestellar, or protostellar
core_type_column = numpy.where(alpha_BE<=5.0, "prestellar", "starless")
core_type_column = numpy.array(core_type_column, dtype='S12')
core_type_column2 = numpy.where(protostellar_catalog_column=='1', "protostellar", core_type_column)
# Make S_peak/S_background column at each wavelength
S_peak_bg_070 = additional_cores_array[:,3]/additional_cores_array[:,4]
S_peak_bg_160 = additional_cores_array[:,7]/additional_cores_array[:,8]
S_peak_bg_165 = additional_cores_array[:,11]/additional_cores_array[:,12]
S_peak_bg_250 = additional_cores_array[:,15]/additional_cores_array[:,16]
S_peak_bg_255 = additional_cores_array[:,19]/additional_cores_array[:,20]
S_peak_bg_350 = additional_cores_array[:,23]/additional_cores_array[:,24]
S_peak_bg_500 = additional_cores_array[:,27]/additional_cores_array[:,28]
# Make S_conv column at each wavelength (convert MJy/str to Jy/beam, then to H2/cm**2)
# Prepareobs scales down the column density image by a factor of 1e20
# The final units in the catalog will be off by a factor of 1e20
S_conv_070 = numpy.array(additional_cores_array[:,5]*(10**6)*((numpy.pi/180.0/3600.0)**2)*1.13309*(36.3**2))
S_conv_160 = numpy.array(additional_cores_array[:,9]*(10**6)*((numpy.pi/180.0/3600.0)**2)*1.13309*(36.3**2))
S_conv_165 = numpy.array(additional_cores_array[:,13]*(10**6)*((numpy.pi/180.0/3600.0)**2)*1.13309*(36.3**2))
S_conv_250 = numpy.array(additional_cores_array[:,17]*(10**6)*((numpy.pi/180.0/3600.0)**2)*1.13309*(36.3**2))
S_conv_255 = numpy.array(additional_cores_array[:,21]*(10**6)*((numpy.pi/180.0/3600.0)**2)*1.13309*(36.3**2))
S_conv_350 = numpy.array(additional_cores_array[:,25]*(10**6)*((numpy.pi/180.0/3600.0)**2)*1.13309*(36.3**2))
S_conv_500 = numpy.array(additional_cores_array[:,29]*(10**6)*((numpy.pi/180.0/3600.0)**2)*1.13309*(36.3**2))
N_H2_bg = numpy.array(additional_cores_array[:,20])
# Define a function that produces a Flux/beam given wavelength, Temp, and ColDense
# We will input wavelength then find T and M using least squares minimization below
def col_dense(wavelength, T, N_H2):
#wavelength input in microns, Temp in Kelvin, N_H2 in cm**-2
#returns S_v in units of Jy/beam
wavelength_mm = numpy.array(wavelength)*10.**-3.
exponent = 1.439*(wavelength_mm**-1)*((T/10.)**-1)
aaa = ((2.02*10**20)*(numpy.exp(exponent)-1.0))**-1.0
bbb = (0.1*((numpy.array(wavelength)/300.)**-2.0))/0.01
ccc = (36.3/10.)**2.
ddd = wavelength_mm**-3.
return N_H2*aaa*bbb*ccc*ddd*(10**-3)
guess = [10.0, 1.0*10.**21.]
N_H2_peak = []
counter = 0
for S_160, S_250, S_350, S_500 in zip(S_conv_160, S_conv_250, S_conv_350, S_conv_500):
#print 'Fitting S_peak for Core ' + str(counter) + ' of ' + str(int(len(good_cores_array[:,0])))
wavelengths = [160.,250.,350.,500.]
fluxes = [S_160, S_250, S_350, S_500]
flux_err = [S_160*0.2, S_250*0.1, S_350*0.1, S_500*0.1]
try:
popt,pcov = curve_fit(col_dense, wavelengths, fluxes, p0=guess, sigma=flux_err)
except RuntimeError:
popt = [-9999., -9999.]
N_H2_peak.append(popt[1])
counter+=1
# Calculate the FWHM_mean at 500 microns
AFWH07 = good_cores_array[:,68]
BFWH07 = good_cores_array[:,69]
A = numpy.float64(((((AFWH07)/60.)/60.)*numpy.pi)/180.) #radians
A1 = numpy.float64(numpy.tan(A/2.)*2.*distance*(3.086e18)) #cm
B = numpy.float64(((((BFWH07)/60.)/60.)*numpy.pi)/180.) #radians
B1 = numpy.float64(numpy.tan(B/2.)*2.*distance*(3.086e18)) #cm
FWHM_mean_500 = mstats.gmean([A1,B1])
Vol_dense_peak = (((4.0*numpy.log(2.0))/numpy.pi)**0.5)*(numpy.array(N_H2_peak)/FWHM_mean_500)
# Import CSAR-matched core indices
print "Cross-matching getsources and CSAR Catalogs:"
CSAR_matched_cores_indices = CSAR_core_cross_match.cross_match_CSAR(getsources_core_catalog, CSAR_catalog, high_res_coldens_image)
# Get a cloumn of 1's identifying CSAR cross-matched cores
CSAR_catalog_column = numpy.zeros(len(good_cores_array[:,0]), dtype='int')
CSAR_catalog_column[numpy.array(CSAR_matched_cores_indices)]+=1
# Make a column indicating the number of significant Herschel bands
N_SED = []
for line in good_cores_array:
counter = 0
if line[8]>5 and line[12]>0:
counter+=1
# Statement below uses the 160micron map, not the temp-corrected map
if line[17]>5 and line[21]>0:
counter+=1
if line[35]>5 and line[39]>0:
counter+=1
if line[53]>5 and line[57]>0:
counter+=1
if line[62]>5 and line[66]>0:
counter+=1
N_SED.append(counter)
# Convert the decimal degrees coordinates of getsources into hh:mm:ss and dd:mm:ss
RA_array = []
Dec_array = []
HGBS_name_array = []
for line in good_cores_array:
RA = astropy.coordinates.Angle(line[3], u.degree)
DEC = astropy.coordinates.Angle(line[4], u.degree)
RA_hours = str('{:.0f}'.format(round(RA.hms[0],2)).zfill(2))
RA_minutes = str('{:.0f}'.format(round(RA.hms[1],2)).zfill(2))
RA_seconds = str('{:.2f}'.format(round(RA.hms[2],2)).zfill(5))
if DEC.hms[0] > 0:
DEC_degs = str('{:.0f}'.format(round(DEC.dms[0],2)).zfill(2))
DEC_minutes = str('{:.0f}'.format(round(DEC.dms[1],2)).zfill(2))
DEC_seconds = str('{:.2f}'.format(round(DEC.dms[2],2)).zfill(5))
name_sign = '+'
HGBS_name = RA_hours+RA_minutes+RA_seconds[0:4]+name_sign+DEC_degs+DEC_minutes+DEC_seconds[0:2]
else:
DEC_degs = str('{:.0f}'.format(round(DEC.dms[0],2)).zfill(3))
DEC_minutes = str('{:.0f}'.format(round(DEC.dms[0]*-1,2)).zfill(2))
DEC_seconds = str('{:.2f}'.format(round(DEC.dms[2]*-1,2)).zfill(5))
HGBS_name = RA_hours+RA_minutes+RA_seconds[0:4]+DEC_degs+DEC_minutes+DEC_seconds[0:2]
RA_array.append(RA_hours + ':' + RA_minutes + ':' + RA_seconds)
Dec_array.append(DEC_degs + ':' + DEC_minutes + ':' + DEC_seconds)
HGBS_name_array.append("HGBS_J"+HGBS_name)
core_number = numpy.arange(len(good_cores_array[:,0]))+1
catalog_array_70_160 = good_cores_array[:,8:26]
catalog_array_70_160 = numpy.delete(catalog_array_70_160, (1,10), 1)
catalog_array_250 = good_cores_array[:,35:44]
catalog_array_250 = numpy.delete(catalog_array_250, 1, 1)
catalog_array_coldense = good_cores_array[:,44:53]
catalog_array_coldense = numpy.delete(catalog_array_coldense, 1, 1)
catalog_array_350_500 = good_cores_array[:,53:71]
catalog_array_350_500 = numpy.delete(catalog_array_350_500, (1,10), 1)
Cloud,Name,Av,alpha,T_bol,L_bol,alphaPrime,TbolPrime,LbolPrime,likelyAGB,Dunham_RA,Dunham_DEC,Class = numpy.loadtxt(Dunham_YSOs_file, delimiter=',', unpack=True, dtype=[('Cloud','S30'),('Name','S40'), ('Av',float),('alpha',float), ('T_bol',float),('L_bol',float), ('alphaPrime',float),('TbolPrime',float), ('LbolPrime',float),('likelyAGB','S1'), ('Dunham_RA',float),('Dunham_DEC',float),('Class','S10') ])
Dunham_indices = numpy.where(Cloud==cloud_name)
Spitzer_YSOs_RA = Dunham_RA[Dunham_indices]
Spitzer_YSOs_DEC = Dunham_DEC[Dunham_indices]
Spitzer_YSOs_Name = Name[Dunham_indices]
potential_matches = []
YSO_matches = []
count = 0
for line in good_cores_array:
match_counter=0
YSO_index = 0
for RA,DEC in zip(Spitzer_YSOs_RA, Spitzer_YSOs_DEC):
distance = ((line[3]-RA)**2 + (line[4]-DEC)**2)**0.5
if distance < 6.0/3600. and match_counter==0:
# matched_counter prevents counting indices twice
# if two YSO candidates fall within getsources ellipse
potential_matches.append(count)
match_counter+=1
YSO_matches.append(YSO_index)
YSO_index+=1
count += 1
Spitzer_column = numpy.zeros(len(good_cores_array[:,0]), dtype='S40')
Spitzer_column[numpy.arange(0,len(good_cores_array[:,0]))] = 'None'
if len(potential_matches)>0:
Spitzer_column[numpy.array(potential_matches)] = Spitzer_YSOs_Name[numpy.array(YSO_matches)]
# Cross-match cores with SIMBAD catalog
print "Cross-matching SIMBAD catalog:"
RA, Dec = numpy.loadtxt(SED_figure_directory + region_name +'_SIMBAD_RA_DEC.dat', unpack=True)
Simbad.ROW_LIMIT = 1
results = []
for i,j in zip(RA,Dec):
result_table = Simbad.query_region(astropy.coordinates.SkyCoord(ra=i, dec=j, unit=(u.deg, u.deg)), radius=6. * u.arcsec)
if result_table != None:
results.append(result_table['MAIN_ID'][0].replace(" ", "_"))
else:
results.append('None')
# Cross-match cores with NED catalog
print "Cross-matching NED catalog:"
Ned.ROW_LIMIT = 1
results2 = []
for i,j in zip(RA,Dec):
result_table_value='Yes'
try:
result_table = Ned.query_region(astropy.coordinates.SkyCoord(ra=i, dec=j, unit=(u.deg, u.deg)), radius=6. * u.arcsec)
except astroquery.exceptions.RemoteServiceError:
result_table_value=None
if result_table_value != None:
results2.append(result_table['Object Name'][0].replace(" ", "_"))
else:
results2.append('None')
zipped_array = zip(core_number, HGBS_name_array, RA_array, Dec_array, catalog_array_70_160[:,0], catalog_array_70_160[:,1], catalog_array_70_160[:,2], S_peak_bg_070, S_conv_070, catalog_array_70_160[:,3], catalog_array_70_160[:,4], catalog_array_70_160[:,5], catalog_array_70_160[:,6], catalog_array_70_160[:,7], catalog_array_70_160[:,8], catalog_array_70_160[:,9], catalog_array_70_160[:,10], S_peak_bg_160, S_conv_160, catalog_array_70_160[:,11], catalog_array_70_160[:,12], catalog_array_70_160[:,13], catalog_array_70_160[:,14], catalog_array_70_160[:,15], catalog_array_250[:,0], catalog_array_250[:,1], catalog_array_250[:,2], S_peak_bg_250, S_conv_250, catalog_array_250[:,3], catalog_array_250[:,4], catalog_array_250[:,5], catalog_array_250[:,6], catalog_array_250[:,7], catalog_array_350_500[:,0], catalog_array_350_500[:,1], catalog_array_350_500[:,2], S_peak_bg_350, S_conv_350, catalog_array_350_500[:,3], catalog_array_350_500[:,4], catalog_array_350_500[:,5], catalog_array_350_500[:,6], catalog_array_350_500[:,7], catalog_array_350_500[:,8], catalog_array_350_500[:,9], catalog_array_350_500[:,10], S_peak_bg_500, catalog_array_350_500[:,11], catalog_array_350_500[:,12], catalog_array_350_500[:,13], catalog_array_350_500[:,14], catalog_array_350_500[:,15], catalog_array_coldense[:,0], additional_cores_array[:,19], S_peak_bg_255, S_conv_255, N_H2_bg, catalog_array_coldense[:,5], catalog_array_coldense[:,6], catalog_array_coldense[:,7], N_SED, CSAR_catalog_column, core_type_column2, results, results2, Spitzer_column)
catalog1 = numpy.array(zipped_array, dtype=[('core_number',int),('HGBS_name_array','S30'),('RA_array','S16'),('Dec_array','S16'),('catalog_array_70_160_1',float),('catalog_array_70_160_2',float), ('catalog_array_70_160_3',float),('S_peak_bg_070',float),('S_conv_070',float), ('catalog_array_70_160_4',float),('catalog_array_70_160_5',float),('catalog_array_70_160_6',float), ('catalog_array_70_160_7',float),('catalog_array_70_160_8',float), ('catalog_array_70_160_9',float),('catalog_array_70_160_10',float), ('catalog_array_70_160_11',float),('S_peak_bg_160',float),('S_conv_160',float),('catalog_array_70_160_12',float),('catalog_array_70_160_13',float),('catalog_array_70_160_14',float), ('catalog_array_70_160_15',float),('catalog_array_70_160_16',float), ('catalog_array_250_1',float),('catalog_array_250_2',float), ('catalog_array_250_3',float),('S_peak_bg_250',float),('S_conv_250',float),('catalog_array_250_4',float),('catalog_array_250_5',float),('catalog_array_250_6',float), ('catalog_array_250_7',float),('catalog_array_250_8',float),('catalog_array_350_500_1',float),('catalog_array_350_500_2',float), ('catalog_array_350_500_3',float),('S_peak_bg_350',float),('S_conv_350',float),('catalog_array_350_500_4',float),('catalog_array_350_500_5',float),('catalog_array_350_500_6',float), ('catalog_array_350_500_7',float),('catalog_array_350_500_8',float), ('catalog_array_350_500_9',float),('catalog_array_350_500_10',float), ('catalog_array_350_500_11',float),('S_peak_bg_500',float),('catalog_array_350_500_12',float),('catalog_array_350_500_13',float),('catalog_array_350_500_14',float), ('catalog_array_350_500_15',float),('catalog_array_350_500_16',float), ('catalog_array_coldense_1',float),('catalog_array_coldense_2',float),('S_peak_bg_255',float),('S_conv_255',float),('additional_cores_array_28',float),('catalog_array_coldense_6',float), ('catalog_array_coldense_7',float),('catalog_array_coldense_8',float),('N_SED',int),('CSAR_catalog_column',int),('core_type_column','S16'), ('SIMBAD_column','S60'), ('NED_column','S60'), ('Spitzer_column','S40')])
header1 = 'core_number, core_name, RA_hms, DEC_dms, sig_070, peak_flux_070, peak_flux_err_070, peak_flux_over_bg_070, peak_070_conv_500, total_flux_070, total_flux_err_070, AFWHM_070, BFWHM_070, PA_070, sig_160, peak_flux_160, peak_flux_err_160, peak_flux_over_bg_160, peak_160_conv_500, total_flux_160, total_flux_err_160, AFWHM_160, BFWHM_160, PA_160, sig_250, peak_flux_250, peak_flux_err_250, peak_flux_over_bg_250, peak_250_conv_500, total_flux_250, total_flux_err_250, AFWHM_250, BFWHM_250, PA_250, sig_350, peak_flux_350, peak_flux_err_350, peak_flux_over_bg_350, peak_350_conv_500, total_flux_350, total_flux_err_350, AFWHM_350, BFWHM_350, PA_350, sig_500, peak_flux_500, peak_flux_err_500, peak_flux_over_bg_500, total_flux_500, total_flux_err_500, AFWHM_500, BFWHM_500, PA_500, sig_coldens, peak_flux_coldens, peak_flux_over_bg_coldens, peak_coldens_conv_500, peak_bg_coldens, AFWHM_coldens, BFWHM_coldens, PA_coldens, N_SED, CSAR, core_type, SIMBAD_match, NED_match, Spitzer_match'
numpy.savetxt(SED_figure_directory + region_name + '_core_catalog1.dat', catalog1, fmt="%i %s %s %s %3.1f %1.2e %1.1e %3.3f %1.2e %1.2e %1.1e %3.1f %3.1f %3.1f %3.1f %1.2e %1.1e %3.3f %1.2e %1.2e %1.1e %3.1f %3.1f %3.1f %3.1f %1.2e %1.1e %3.3f %1.2e %1.2e %1.1e %3.1f %3.1f %3.1f %3.1f %1.2e %1.1e %3.3f %1.2e %1.2e %1.1e %3.1f %3.1f %3.1f %3.1f %1.2e %1.1e %3.3f %1.2e %1.1e %3.1f %3.1f %3.1f %3.1f %3.1f %3.1f %3.1f %3.1f %3.1f %3.1f %3.1f %i %i %s %s %s %s", header=header1)
mu = 2.8 # mean molecular weight
mass_H = 1.67372e-24 # (grams) mass of neutral Hydrogen atom
solar_mass = 1.989e33 # (grams)
mass_H_solar_masses = mass_H / solar_mass
parsec = 3.086e18 # cm
R_deconv_cm = numpy.array(R_deconv)*parsec
FWHM_mean_cm = numpy.array(FWHM_mean)*parsec
N_H2_avg_1 = (numpy.array(Masses)/(numpy.pi*(R_deconv_cm**2.))) * (1/(mu*mass_H_solar_masses))
N_H2_avg_2 = (numpy.array(Masses)/(numpy.pi*(FWHM_mean_cm**2.))) * (1/(mu*mass_H_solar_masses))
avg_Volume_dens_1 = (numpy.array(Masses)/(numpy.pi*(4./3.)*(R_deconv_cm**3.))) * (1/(mu*mass_H_solar_masses))
avg_Volume_dens_2 = (numpy.array(Masses)/(numpy.pi*(4./3.)*(FWHM_mean_cm**3.))) * (1/(mu*mass_H_solar_masses))
catalog2 = numpy.array(zip(core_number, HGBS_name_array, RA_array, Dec_array, R_deconv, FWHM_mean, Masses, Masses_err, Temps, Temps_err, N_H2_peak, N_H2_avg_1, N_H2_avg_2, Vol_dense_peak, avg_Volume_dens_1, avg_Volume_dens_2, alpha_BE, core_type_column2, not_accepted_counter), dtype=[('core_number',int),('HGBS_name_array','S30'),('RA_array','S16'),('Dec_array','S16'),('R_deconv',float),('FWHM_mean',float),('Masses',float), ('Masses_err',float),('Temps',float),('Temps_err',float),('N_H2_peak',float),('N_H2_avg_1',float),('N_H2_avg_2',float),('Vol_dense_peak',float),('avg_Volume_dens1',float),('avg_Volume_dens_2',float),('alpha_BE',float),('core_type_column2','S16'),('not_accepted_counter','S16')])
header2 = 'core_number, core_name, RA_hms, DEC_dms, R_deconv, FWHM_mean, Mass, Mass_err, Temp_dust, Temp_dust_err, N_H2_peak, N_H2_avg_1, N_H2_avg_2, Vol_dense_peak, avg_Volume_dens_1, avg_Volume_dens_2, alpha_BE, core_type_column2, not_accepted_counter'
numpy.savetxt(SED_figure_directory + region_name +'_core_catalog2.dat', catalog2, fmt="%i %s %s %s %1.1e %1.1e %1.3f %1.2f %2.1f %2.1f %1.2e %1.2e %1.2e %1.2e %1.2e %1.2e %2.1f %s %s", header=header2)
def core_mass_fits(region_name='Aquila',
T_Dust=15.0,
T_Dust_err=3.0,
distance=260.,
Dunham_YSOs_file='Dunham_YSOs.dat'):
getsources_core_catalog = '/mnt/scratch-lustre/jkeown/GS_Extractions/GS-Extractions/' + region_name + '/' + region_name + '.sw.final.reliable.ok.cat'
SED_figure_directory = '/mnt/scratch-lustre/jkeown/JCMT_GBS_Jybeam/JCMT_pipeline/Figures/' + region_name + '/'
# These are the values in each column of "cores_array" and "protostar_array"
#NO, XCO_P, YCO_P, WCS_ACOOR, WCS_DCOOR, SIG_GLOB, FG, GOOD, SIG_MONO01, FM01, FXP_BEST01, FXP_ERRO01, FXT_BEST01, FXT_ERRO01, AFWH01, BFWH01, THEP01, SIG_MONO02, FM02, FXP_BEST02, FXP_ERRO02, FXT_BEST02, FXT_ERRO02, AFWH02, BFWH02, THEP02
# These are the values in each column of the "additional" "cores_array" and "protostar_array"
# NO XCO_P YCO_P PEAK_SRC01 PEAK_BGF01 CONV_SRC01 CONV_BGF01 PEAK_SRC02 PEAK_BGF02 CONV_SRC02 CONV_BGF02
# Import the raw getsources core catalog (includes "bad" sources that don't pass HGBS selection criteria)
cores_array1 = numpy.loadtxt(getsources_core_catalog, comments='!')
# Find the indices of the "good" cores that pass HGBS selection criteria
good_core_indices = good_cores_getsources.get_good_cores(region_name)
# Create new array of only the "good" cores to be used for our analysis
cores_array = cores_array1[numpy.array(good_core_indices)]
if region_name == 'PipeE1':
cores_array = numpy.array(cores_array1)
print cores_array
z = numpy.zeros(len(cores_array))
cores_array = numpy.stack((cores_array, z))
print cores_array
# Calculate the deconvolved core radii
AFWH05 = cores_array[:, 23]
BFWH05 = cores_array[:, 24]
A = numpy.float64(((((AFWH05) / 60.) / 60.) * numpy.pi) / 180.) #radians
A1 = numpy.float64(numpy.tan(A / 2.) * 2. * distance) #pc
B = numpy.float64(((((BFWH05) / 60.) / 60.) * numpy.pi) / 180.) #radians
B1 = numpy.float64(numpy.tan(B / 2.) * 2. * distance) #pc
FWHM_mean = mstats.gmean([A1, B1])
HPBW = numpy.float64(((((14.1) / 60.) / 60.) * numpy.pi) / 180.) #radians
HPBW1 = numpy.float64(numpy.tan(HPBW / 2.) * 2. * distance) #pc
R_deconv = ((FWHM_mean**2.0) - (HPBW1**2.0))**0.5 #pc
R_deconv = numpy.where(((FWHM_mean**2.0) - (HPBW1**2.0)) <= 0., FWHM_mean,
R_deconv)
resolved = numpy.where(((FWHM_mean**2.0) - (HPBW1**2.0)) <= 0., 0, 1)
# Calculate the Bonnor-Ebert masses of each core based on their R_deconvolved
c_s = 0.2 #km/s
G = 4.302 * 10**-3 #pc/M_solar (km/s)^2
M_BE = (2.4 * R_deconv * (c_s**2.)) / G #M_solar
#M_BE = numpy.where(((FWHM_mean**2.0) - (HPBW1**2.0))<0, 9999., M_BE)
# Define a function that produces a Flux given wavelength, Temp, and Mass
# We will input wavelength then find T and M using least squares minimization below
def core_mass(wavelength, T, M):
#wavelength input in microns, Temp in Kelvin, Mass in M_solar
#returns S_v (i.e., Flux) in units of Jy
D = distance #parsecs to cloud
wavelength_mm = numpy.array(wavelength) * 10.**-3.
exponent = 1.439 * (wavelength_mm**-1) * ((T / 10.)**-1)
aaa = (0.12 * (numpy.exp(exponent) - 1.0))**-1.0
bbb = (0.1 * ((numpy.array(wavelength) / 300.)**-2.0)) / 0.01
ccc = (D / 100.)**-2
ddd = wavelength_mm**-3.
return M * aaa * bbb * ccc * ddd
# Define another function that calculates Mass directly from wavelength, Temp, and Flux
# This will be used to find the Mass of cores that don't have reliable least-squares fits
def core_mass_from_flux(wavelength, T, S_v):
#wavelength input in microns, Temp in Kelvin, Mass in M_solar
#returns S_v (i.e., Flux) in units of Jy
D = distance #parsecs to cloud
wavelength_mm = wavelength * 10.**-3.
exponent = 1.439 * (wavelength_mm**-1) * ((T / 10.)**-1)
aaa = 0.12 * (numpy.exp(exponent) - 1.0)
bbb = ((0.1 * ((wavelength / 300.)**-2.0)) / 0.01)**-1.0
ccc = (D / 100.)**2.0
ddd = wavelength_mm**3.0
return S_v * aaa * bbb * ccc * ddd
# Define another function that calculates Mass uncertainty due to temp
def core_mass_err_dT(wavelength, T, S_v, dT):
#wavelength input in microns, Temp in Kelvin, Mass in M_solar
#returns S_v (i.e., Flux) in units of Jy
D = distance #parsecs to cloud
wavelength_mm = wavelength * 10.**-3.
exponent = 1.439 * (wavelength_mm**-1) * ((T / 10.)**-1)
aaa = 0.12 * (numpy.exp(exponent))
bbb = ((0.1 * ((wavelength / 300.)**-2.0)) / 0.01)**-1.0
ccc = (D / 100.)**2.0
ddd = wavelength_mm**3.0
eee = 1.439 * 10 * (wavelength_mm**-1) * (T**-2.)
return S_v * aaa * bbb * ccc * ddd * dT * eee
# Define another function that calculates Mass uncertainty due to flux
def core_mass_err_dS_v(wavelength, T, S_v, dS_v):
#wavelength input in microns, Temp in Kelvin, Mass in M_solar
#returns S_v (i.e., Flux) in units of Jy
D = distance #parsecs to cloud
wavelength_mm = wavelength * 10.**-3.
exponent = 1.439 * (wavelength_mm**-1) * ((T / 10.)**-1)
aaa = 0.12 * (numpy.exp(exponent) - 1.0)
bbb = ((0.1 * ((wavelength / 300.)**-2.0)) / 0.01)**-1.0
ccc = (D / 100.)**2.0
ddd = wavelength_mm**3.0
return aaa * bbb * ccc * ddd * dS_v
# Create some empty arrays to which we will append accepted values
Masses = []
Temps = []
Masses_err = []
Temps_err = []
counter = 0
# Loop through all the "good" cores
for NO, XCO_P, YCO_P, WCS_ACOOR, WCS_DCOOR, SIG_GLOB, FG, GOOD, SIG_MONO01, FM01, FXP_BEST01, FXP_ERRO01, FXT_BEST01, FXT_ERRO01, AFWH01, BFWH01, THEP01, SIG_MONO02, FM02, FXP_BEST02, FXP_ERRO02, FXT_BEST02, FXT_ERRO02, AFWH02, BFWH02, THEP02 in cores_array:
#flux_err_run1 = [FXT_BEST01/SIG_MONO01, FXT_BEST02/SIG_MONO02] ## Should these be FXP_BEST?
# Find the longest significant wavelength and corresponding flux
wave = 850.
flux_fit = FXT_BEST02 # Fluxes are in mJy, so multiple by 10^3 to get Jy
flux_fit_err = FXT_ERRO02 # Fluxes are in mJy, so multiple by 10^3 to get Jy
# Find the mass corresponding to that flux measurement
# ***This uses the median of the best-fit Temps from the cores with
# reliable SED fits (i.e., those that pass the test above)
Mass_fit = core_mass_from_flux(wave, T_Dust, flux_fit)
# Can add more uncertainties (e.g., calibration, etc.) below
Mass_error = (
core_mass_err_dT(wave, T_Dust, flux_fit, T_Dust_err) +
core_mass_err_dS_v(wave, T_Dust, flux_fit, flux_fit_err)**2.0)**0.5
# Store the Mass with uncertainties
# Need to perform a more in-depth error analysis
Masses.append(Mass_fit)
Masses_err.append(Mass_error)
Temps.append(T_Dust)
Temps_err.append(T_Dust_err)
#Replace nans if they exist in the M_BE array
if len(cores_array) > 1:
where_are_nans = numpy.isnan(M_BE)
M_BE[where_are_nans] = 9999
# Calculate the alpha_BE ratio to determine prestellar cores
alpha_BE = numpy.array(M_BE) / numpy.array(Masses)
#alpha_BE = numpy.where(((FWHM_mean**2.0) - (HPBW1**2.0))<0, 9999., alpha_BE)
# Create an array indicating a core as candidate(1)/robust(2) prestellar
candidate_array = numpy.where(alpha_BE <= 5.0, 1, 0)
#candidate_array = numpy.where(alpha_BE<=alpha_factor, 1, 0)
robust_candidate_array = numpy.where(alpha_BE <= 2.0, 2, candidate_array)
# Remove protostars from the alpha_BE array and find the indices of the remaining
# candidate/robust prestellar cores
robust_prestellar_indices = numpy.where(alpha_BE <= 2.0)
candidate_prestellar_indices = numpy.where(alpha_BE <= 5.0)
Masses2 = numpy.array(Masses)
# Find the final list of prestellar candidate/robust Masses with protostars removed
prestellar_candidates = Masses2[numpy.array(
candidate_prestellar_indices[0])]
prestellar_robust = Masses2[numpy.array(robust_prestellar_indices[0])]
print 'prestellar candidates: ' + str(len(prestellar_candidates))
print 'robust prestellar candidates: ' + str(len(prestellar_robust))
# Plot Mass versus Radius and save the figure
fig = plt.figure()
plt.scatter(R_deconv, Masses, label='starless')
if len(Masses) > 1:
plt.scatter(R_deconv[numpy.array(candidate_prestellar_indices[0])],
prestellar_candidates,
color='red',
label='candidate')
plt.scatter(R_deconv[numpy.array(robust_prestellar_indices[0])],
prestellar_robust,
color='green',
label='robust')
plt.yscale('log')
plt.xscale('log')
#plt.legend()
plt.title(region_name + ' Cores')
plt.ylabel("Mass, M (M$_\odot$)")
plt.xlabel("Deconvolved FWHM size, R (pc)")
#plt.xlim([10**-3, 2*10**-1])
#plt.ylim([10**-3, 10**2])
fig.savefig(SED_figure_directory + 'mass_vs_radius_' + region_name +
'_Feb2018.png')
Cloud, Name, Av, alpha, T_bol, L_bol, alphaPrime, TbolPrime, LbolPrime, likelyAGB, Dunham_RA, Dunham_DEC, Class = numpy.loadtxt(
Dunham_YSOs_file,
delimiter=',',
unpack=True,
dtype=[('Cloud', 'S30'), ('Name', 'S40'), ('Av', float),
('alpha', float), ('T_bol', float), ('L_bol', float),
('alphaPrime', float), ('TbolPrime', float),
('LbolPrime', float), ('likelyAGB', 'S1'), ('Dunham_RA', float),
('Dunham_DEC', float), ('Class', 'S10')])
#Dunham_indices = numpy.where(Cloud==cloud_name)
Spitzer_YSOs_RA = Dunham_RA
Spitzer_YSOs_DEC = Dunham_DEC
Spitzer_YSOs_Name = Name
potential_matches = []
YSO_matches = []
count = 0
for line in cores_array:
match_counter = 0
YSO_index = 0
for RA, DEC in zip(Spitzer_YSOs_RA, Spitzer_YSOs_DEC):
distance = ((line[3] - RA)**2 + (line[4] - DEC)**2)**0.5
if distance < 6.0 / 3600. and match_counter == 0:
# matched_counter prevents counting indices twice
# if two YSO candidates fall within getsources ellipse
potential_matches.append(count)
match_counter += 1
YSO_matches.append(YSO_index)
YSO_index += 1
count += 1
Spitzer_column = numpy.zeros(len(cores_array[:, 0]), dtype='S50')
Spitzer_column[numpy.arange(0, len(cores_array[:, 0]))] = 'None'
if len(potential_matches) > 0:
Spitzer_column[numpy.array(potential_matches)] = Spitzer_YSOs_Name[
numpy.array(YSO_matches)]
# Save a text file with RA and Dec of the "good" cores
# This is needed for the SIMBAD cross-match
numpy.savetxt(SED_figure_directory + region_name + '_SIMBAD_RA_DEC.dat',
zip(cores_array[:, 3], cores_array[:, 4]))
# Cross-match cores with SIMBAD catalog
print "Cross-matching SIMBAD catalog:"
RA, Dec = numpy.loadtxt(SED_figure_directory + region_name +
'_SIMBAD_RA_DEC.dat',
unpack=True)
Simbad.ROW_LIMIT = 1
results = []
if len(cores_array) == 1:
result_table = Simbad.query_region(astropy.coordinates.SkyCoord(
ra=RA, dec=Dec, unit=(u.deg, u.deg)),
radius=6. * u.arcsec)
if result_table != None:
results.append(result_table['MAIN_ID'][0].replace(" ", "_"))
else:
results.append('None')
# Cross-match cores with NED catalog
print "Cross-matching NED catalog:"
Ned.ROW_LIMIT = 1
results2 = []
result_table_value = 'Yes'
try:
result_table = Ned.query_region(astropy.coordinates.SkyCoord(
ra=RA, dec=Dec, unit=(u.deg, u.deg)),
radius=6. * u.arcsec)
except astroquery.exceptions.RemoteServiceError:
result_table_value = None
if result_table_value != None:
results2.append(result_table['Object Name'][0].replace(" ", "_"))
else:
results2.append('None')
else:
for i, j in zip(RA, Dec):
result_table = Simbad.query_region(astropy.coordinates.SkyCoord(
ra=i, dec=j, unit=(u.deg, u.deg)),
radius=6. * u.arcsec)
if result_table != None:
results.append(result_table['MAIN_ID'][0].replace(" ", "_"))
else:
results.append('None')
# Cross-match cores with NED catalog
print "Cross-matching NED catalog:"
Ned.ROW_LIMIT = 1
results2 = []
for i, j in zip(RA, Dec):
result_table_value = 'Yes'
try:
result_table = Ned.query_region(astropy.coordinates.SkyCoord(
ra=i, dec=j, unit=(u.deg, u.deg)),
radius=6. * u.arcsec)
except astroquery.exceptions.RemoteServiceError:
result_table_value = None
if result_table_value != None:
results2.append(result_table['Object Name'][0].replace(
" ", "_"))
else:
results2.append('None')
running_number = numpy.arange(len(cores_array[:, 0])) + 1
header1 = 'running_NO, getsources_NO, XCO_P, YCO_P, WCS_ACOOR, WCS_DCOOR, SIG_GLOB, FG, GOOD, SIG_MONO01, FM01, FXP_BEST01, FXP_ERRO01, FXT_BEST01, FXT_ERRO01, AFWH01, BFWH01, THEP01, SIG_MONO02, FM02, FXP_BEST02, FXP_ERRO02, FXT_BEST02, FXT_ERRO02, AFWH02, BFWH02, THEP02, R_deconv, R_fwhm_mean, resolved, Mass, M_err, Temp, T_err, alpha_BE, SIMBAD_match, NED_match, Spitzer_match'
# Append the Radius, Mass, Temperature, alpha_BE, etc. arrays as columns
# onto the "good cores" array and save as a .dat file
numpy.savetxt(
SED_figure_directory + region_name + '_good_sources_Feb2018.dat',
numpy.column_stack((running_number, cores_array, numpy.array(R_deconv),
numpy.array(FWHM_mean), numpy.array(resolved),
numpy.array(Masses), numpy.array(Masses_err),
numpy.array(Temps), numpy.array(Temps_err),
numpy.array(alpha_BE), numpy.array(results),
numpy.array(results2), Spitzer_column)),
fmt='%s',
header=header1)
def GetAndUploadAllData(self,objs,ras,decs,doNED=True):
TransientUploadDict = {}
assert len(ras) == len(decs)
if type(ras[0]) == float:
scall = SkyCoord(ras,decs,frame="fk5",unit=u.deg)
else:
scall = SkyCoord(ras,decs,frame="fk5",unit=(u.hourangle,u.deg))
ebvall,nedtables = [],[]
ebvtstart = time.time()
if doNED:
for sc in scall:
dust_table_l = IrsaDust.get_query_table(sc)
ebvall += [dust_table_l['ext SandF mean'][0]]
try:
ned_region_table = Ned.query_region(sc, radius=self.nedradius*u.arcmin, equinox='J2000.0')
except:
ned_region_table = None
nedtables += [ned_region_table]
print('E(B-V)/NED time: %.1f seconds'%(time.time()-ebvtstart))
tstart = time.time()
TNSData = []
json_data = []
for j in range(len(objs)):
TNSGetSingle = [("objname",objs[j]),
("photometry","1"),
("spectra","1")]
response=get(self.tnsapi, TNSGetSingle, self.tnsapikey)
json_data += [format_to_json(response.text)]
print(time.time()-tstart)
print('getting TNS content takes %.1f seconds'%(time.time()-tstart))
for j,jd in zip(range(len(objs)),json_data):
tallstart = time.time()
obj = objs[j]
iobj = np.where(obj == np.array(objs))[0]
if len(iobj) > 1: iobj = int(iobj[0])
else: iobj = int(iobj)
if doNED: sc,ebv,nedtable = scall[iobj],ebvall[iobj],nedtables[iobj]
else: sc = scall[iobj]; ebv = None; nedtable = None
print("Object: %s\nRA: %s\nDEC: %s" % (obj,ras[iobj],decs[iobj]))
########################################################
# For Item in Email, Get NED
########################################################
if type(jd['data']['reply']['name']) == str:
jd = jd['data']['reply']
else:
jd = None
transientdict = self.getTNSData(jd,obj,sc,ebv)
try:
photdict = self.getZTFPhotometry(sc)
except: photdict = None
try:
if jd:
photdict,nondetectdate,nondetectmaglim,nondetectfilt,nondetectins = \
self.getTNSPhotometry(jd,PhotUploadAll=photdict)
specdict = self.getTNSSpectra(jd,sc)
transientdict['transientphotometry'] = photdict
transientdict['transientspectra'] = specdict
if nondetectdate: transientdict['non_detect_date'] = nondetectdate
if nondetectmaglim: transientdict['non_detect_limit'] = nondetectmaglim
if nondetectfilt: transientdict['non_detect_band'] = nondetectfilt
if nondetectfilt: transientdict['non_detect_instrument'] = nondetectins
except: pass
try:
if doNED:
hostdict,hostcoords = self.getNEDData(jd,sc,nedtable)
transientdict['host'] = hostdict
transientdict['candidate_hosts'] = hostcoords
except: pass
TransientUploadDict[obj] = transientdict
TransientUploadDict['noupdatestatus'] = self.noupdatestatus
self.UploadTransients(TransientUploadDict)
return(len(TransientUploadDict))
def buildTable():
"""
Allows users to create a fully customizable HTML table containing information about a list of input galaxies.
"""
input_file = typer.prompt(
'Enter the path to the csv/txt file containing the targets (format "target,ra,dec"), where ra, dec are the ICRS coordinates of target in degrees in J2000 equinox'
)
output_name = typer.prompt('Enter the name of the output table')
# get the basic columns
columns = [
'NAME', 'RA', 'DEC', 'REDSHIFT', 'REFERENCE FOR REDSHIFT', 'NED LINK'
]
# add the columns for images/plots/other files
add_spectra = (typer.prompt(
'Add spectra plots to the table?(y/n)|(default: n)').lower() == 'y')
create_spectra = False
if add_spectra:
create_spectra = (typer.prompt(
'Create spectra plots now? Type "n" if you already have them (y/n)|(default: n)'
).lower() == 'y')
if not create_spectra:
spectra_dir = typer.prompt(
'Enter directory with spectra plot files. File names must be in format (target).png/jpg'
)
columns.append('SPECTRA')
add_pahfitplot = (typer.prompt(
'Add pahfit plots and tables to the table?(y/n)|(default: n)').lower()
== 'y')
create_pahfit = False
if add_pahfitplot:
create_pahfit = (typer.prompt(
'Create pahfit plots and tables now? Type "n" if you already have them (y/n)|(default: n)'
).lower() == 'y')
if not create_pahfit:
pahfitplot_dir = typer.prompt(
'Enter directory with pahfit plot files. File names must be in format (target).png/jpg'
)
pahfittable_dir = typer.prompt(
'Enter directory with pahfit output table files. File names must be in format (target)_output.ipac or (target).ipac'
)
columns.extend(['PAHFIT PLOT', 'PAHFIT TABLE'])
add_rgb = (typer.prompt(
'Add RGB images to the table(y/n)|(default: n)').lower() == 'y')
create_rgb = False
if add_rgb:
create_rgb = (typer.prompt(
'Create rgb images now using SDSS? Type "n" if you already have the images(y/n)|(default: n)'
).lower() == 'y')
if not create_rgb:
rgb_dir = typer.prompt(
'Enter directory with the RGB images. File names musst be in format (target).png/jpg'
)
columns.append('RGB IMAGE')
add_visibility = (typer.prompt(
'Add visibility plots to the table?(y/n)|(default: n)').lower() == 'y')
create_visibility = False
if add_visibility:
create_visibility = (typer.prompt(
'Create visibility plots now? Type "n" if you already have the plots(y/n)|(default: n)'
).lower() == 'y')
if not create_visibility:
visibility_dir = typer.prompt(
'Enter directory with the visibility plots. File names must be in format (target).png/jpg:'
)
columns.append('VISIBILITY PLOT')
# add other images and files the use might have
other_image_dirs = []
add_other_images = True
while add_other_images:
add_other_images = (typer.prompt(
'Add any other images/plots to the table?(y/n)|(default: n)').
lower() == 'y')
if add_other_images:
other_image_dirs.append(
typer.prompt(
'Enter directory with the other images. File names must be in format (target).png/jpg:'
))
columns.append(
typer.prompt(
'Enter the name of the column for these images/plots').
upper())
other_files = []
add_other_files = True
while add_other_files:
add_other_files = (typer.prompt(
'Add links to any other files?(y/n)|(default: n)').lower() == 'y')
if add_other_files:
format = typer.prompt('Enter the format of these files')
dir = typer.prompt(
'Enter directory with the other images. File names must be in format (target).(format):'
)
other_files.append({'format': format, 'dir': dir})
# add all optional non-file columns
add_eso = (typer.prompt(
'Add ESO links to the table?(y/n)|(default: n)').lower() == 'y')
add_alma = (typer.prompt(
'Add ALMA links s to the table?(y/n)|(default: n)').lower() == 'y')
add_manga = (typer.prompt(
'Add MANGA links to the table?(y/n)|(default: n)').lower() == 'y')
add_gemini = (typer.prompt(
'Add GEMINI links to the table?(y/n)|(default: n)').lower() == 'y')
add_guide_stars = (typer.prompt(
'Add guide star info to the table?(y/n)|(default: n)').lower() == 'y')
if add_eso:
columns.extend(['ESO INSTRUMENTS', 'ESO LINKS'])
if add_alma:
columns.append('ALMA LINK')
if add_manga:
columns.extend(['MaNGA ID', 'MaNGA PLATE IFUDesign', 'MaNGA LINK'])
if add_gemini:
columns.extend(['GEMINI INSTRUMENTS', 'GEMINI LINK'])
if add_guide_stars:
columns.extend(['HIGH STREHL', 'LOW STREHL'])
# set up eso_tap_obs
if add_eso:
ESO_TAP_OBS = "http://archive.eso.org/tap_obs"
tapobs = tap.TAPService(ESO_TAP_OBS)
if add_manga:
drpall = fits.open(
r'C:\Users\bpart\OneDrive\Desktop\astro\MASSIVE\data\manga-drpall.fits'
)
# set up Vizier so all the necessary columns can be obtained for the guide stars, including flags that signify galaxies
if add_guide_stars:
v = Vizier(columns=[
'RAJ2000', 'DEJ2000', 'pmRA', 'pmDE', 'UCAC4', 'LEDA', 'rmag',
'f_rmag', 'g', 'c', 'of'
])
# create spizter spectra plots
if create_spectra:
spectra_dir = getSpectra()
# create pahfit plots and tables
if create_pahfit:
pahfitplot_dir, pahfittable_dir = getPahfit()
# create rgb images
if create_rgb:
rgb_dir = getSDSS_RGB(input_file)
# create visibility plots if user chose that option
if create_visibility:
visibility_dir = getVisibility(input_file)
dataFile = open(input_file, 'r')
dataFile = csv.reader(dataFile)
output_name = 'index' if output_name == '' else output_name
with open(f'{output_name}.csv', 'w') as outfile:
csv_writer = csv.writer(outfile)
csv_writer.writerow(columns)
for line in dataFile:
name = line[0]
ra = line[1]
dec = line[2]
typer.secho('Building row for ' +
typer.style(name, fg=typer.colors.MAGENTA))
# get redshift from NED
coord = SkyCoord(ra, dec, unit='deg')
resTable = Ned.query_region(coord, radius=0.016 * u.deg)
resTable = resTable.to_pandas()
redshift = ''
refCode, reflink = '', ''
for i, row in resTable.iterrows():
if row['Redshift'] > 0:
nName = row['Object Name']
redshift = row['Redshift']
# have to use name to query NED's redshift table to get reference
try:
redTable = Ned.get_table(
nName, table='redshifts').to_pandas()
for j, r in redTable.iterrows():
if r['Published Redshift'] == redshift:
refCode = r['Refcode']
reflink = f'https://ui.adsabs.harvard.edu/abs/{refCode}/abstract'
except:
refCode = "No refcode found"
break
# get nedLink using coordinates
nedLink = f'https://ned.ipac.caltech.edu/conesearch?search_type=Near%20Position%20Search&coordinates={ra}d%20{dec}d&radius=1'
# prepare row for required columns
row = [
name, ra, dec, redshift, f"<a href='{reflink}'>{refCode}</>",
f"<a href='{nedLink}'>NED Link</>"
]
# prepare rows for other columns
# Add spectra plot
if add_spectra:
row.append(getImagePath(spectra_dir, name))
# Add PAHFIT plot and PAHFIT table, accounting for possible variations in naming conventions
if add_pahfitplot:
row.append(getImagePath(f'{pahfitplot_dir}', name))
if os.path.exists(f'{pahfittable_dir}/{name}.ipac'):
row.append(
f"<a href='{pahfittable_dir}/{name}.ipac'>PAHFIT Table</>"
)
else:
row.append(
f"<a href='{pahfittable_dir}/{name}_output.ipac'>PAHFIT Table</>"
)
# Add visibility plot
if add_visibility:
row.append(getImagePath(visibility_dir, name))
# Add RGB plots from SDSS
if add_rgb:
row.append(getImagePath(rgb_dir, name))
# Add other plots/images
for d in other_image_dirs:
row.append(getImagePath(d, name))
# ESO
# tried using astroquery.eso but it would not work. I believe this is the best way to do it as of 04/2021
if add_eso:
query = f"""SELECT *
FROM ivoa.ObsCore
WHERE intersects(s_region, circle('', {ra}, {dec}, 0.02))=1"""
res = tapobs.search(query=query, maxrec=1000)
# get all unique instruments
esoInstruments = set(res['instrument_name'])
esoInstruments = list(esoInstruments)
esoInstruments = " ".join(i for i in esoInstruments).replace(
",", "-")
esoLink, esoText = '', ''
# get the ESO link using coordinates
if len(esoInstruments) > 1:
esoLink = f'https://archive.eso.org/scienceportal/home?pos={ra}%2C{dec}&r=0.02'
esoText = 'ESO Link'
row.append(esoInstruments)
row.append(f"<a href='{esoLink}'>{esoText}</>")
# ALMA
# there is a chance of astroquery.alma not working. Try installing the latest version of astroquery directly from the GitHub source in this case
if add_alma:
almaLink = ''
almaText = ''
try:
res = Alma.query_region(coord, 0.02 * u.deg)
if len(res) > 0:
almaLink = f'https://almascience.eso.org/asax/?result_view=observation&raDec={ra}%20{dec}'
almaText = "ALMA"
except:
pass
row.append(f"<a href='{almaLink}'>{almaText}</>")
# MaNGA
# the drpall file version 2_4_3 is required to obtain MaNGA data from release 16
if add_manga:
tbdata = drpall[1].data
i = np.where(
np.sqrt((tbdata['objra'] - float(ra))**2 +
(tbdata['objdec'] - float(dec))**2) <= 0.02)
mid, plate, ifudsgn, mlink, mtext = "", "", "", "", ""
if len(i[0]) > 0:
# the mid, plate, and ifudsgn are used to locate manga data
# TODO provide more info on this
mid = tbdata['mangaid'][i][0]
plate = tbdata['plate'][i][0]
ifudsgn = tbdata['ifudsgn'][i][0]
mlink = f'https://data.sdss.org/sas/dr16/manga/spectro/redux/v2_4_3/{plate}/stack/'
mtext = "MaNGA Link"
row.append(mid)
row.append(ifudsgn)
row.append(f'<a href="{mlink}">{mtext}</>')
# GEMINI
if add_gemini:
gData = Observations.query_region(coordinates=SkyCoord(
ra, dec, unit='deg'),
radius=0.02 * u.deg)
gInstruments = set()
gText, gLink = '', ''
if len(gData) > 0:
gInstruments = set(gData['instrument'])
gText = "Gemini Link"
gLink = f'https://archive.gemini.edu/searchform/NotFail/ra={ra}/not_site_monitoring/notengineering/dec={dec}/cols=CTOWEQ'
gInstruments = list(gInstruments)
gInstruments = " ".join(i for i in gInstruments).replace(
",", "-")
row.append(gInstruments)
row.append(f'<a href="{gLink}">{gText}</>')
# Get guide stars using Vizier from the I/322A catalogue
# Refer here for more information https://www.gemini.edu/instrumentation/altair
if add_guide_stars:
ucac4 = v.query_region(SkyCoord(ra, dec, unit='deg'),
radius=25 * u.arcsec,
catalog='I/322A')
highList, lowList = [], []
if ucac4:
ucac4 = ucac4[0]
for star in ucac4:
dist = abs(
(np.float32(star['RAJ2000']) - np.float32(ra))**2 +
abs(np.float32(star['DEJ2000']) -
np.float32(dec))**2)**0.5 * 3600
magn = star['rmag']
# Objects with LEDA > 0 are galaxies and will be disregarded.
if (3 < dist < 15 and 8.5 < magn < 15
) and star['LEDA'] == star['g'] == star['c'] == 0:
highList.append(
(star['UCAC4'], round(dist, 2), round(magn, 2),
star['of'], star['f_rmag']))
elif (15 < magn < 18.5) and (
3 < dist
) and star['LEDA'] == star['g'] == star['c'] == 0:
lowList.append(
(star['UCAC4'], round(dist, 2), round(magn, 2),
star['of'], star['f_rmag']))
# prepare outputs for High and Low Strehl columns using data obtained from catalogue
highList = " ".join((
f'{str(h[0])} (rmag={str(h[2])}; f_rmag={str(h[4])}; distance={str(h[1])}"; of={str(h[3])})'
) for h in highList)
lowList = " ".join((
f'{str(h[0])} (rmag={str(h[2])}; f_rmag={str(h[4])}; distance={str(h[1])}"; of={str(h[3])})'
) for h in lowList)
row.append(highList)
row.append(lowList)
# avoid adding empty strings to output as this can cause problems when converting to HTML
for i in range(len(row)):
if row[i] == "":
row[i] = " "
csv_writer.writerow(row)
# create HTMl table from the csv table using pandas. The resulting table will look pretty ugly and as such it is recommended to add some basic styling.
data = pd.read_csv(f'{output_name}.csv')
file = open(f'{output_name}.html', 'w')
file.writelines(data.to_html(escape=False, index=False))
