def apokasc_load(combine=True):
"""
NAME:
apokasc_load
PURPOSE:
load apokasc result (Precise surface gravity measurement)
INPUT:
combine (boolean): True to combine gold snd basic standard
OUTPUT:
HISTORY:
2017-Dec-23 - Written - Henry Leung (University of Toronto)
"""
catalog_list = Vizier.find_catalogs('apokasc')
Vizier.ROW_LIMIT = 99999
catalogs_gold = Vizier.get_catalogs(catalog_list.keys())[1]
catalogs_basic = Vizier.get_catalogs(catalog_list.keys())[2]
gold_ra = catalogs_gold['_RA']
gold_dec = catalogs_gold['_DE']
gold_logg = catalogs_gold['log_g_']
basic_ra = catalogs_basic['_RA']
basic_dec = catalogs_basic['_DE']
basic_logg = catalogs_basic['log.g2']
if combine is True:
ra = np.append(np.array(gold_ra), np.array(basic_ra))
dec = np.append(np.array(gold_dec), np.array(basic_dec))
logg = np.append(np.array(gold_logg), np.array(basic_logg))
return ra, dec, logg
else:
return gold_ra, gold_dec, gold_logg, basic_ra, basic_dec, basic_logg
def load_apokasc(combine=True):
"""
Load APOKASC asteroseismic surface gravity measurement
:param combine: True to combine gold and basic standard, otherwise only get gold standard log(g)
:type combine: boolean
:return: numpy array of ra, dec, array
:rtype: ndarrays
:History:
| 2017-Dec-23 - Written - Henry Leung (University of Toronto)
"""
catalog_list = Vizier.find_catalogs('apokasc')
Vizier.ROW_LIMIT = 99999
catalogs_gold = Vizier.get_catalogs(catalog_list.keys())[1]
catalogs_basic = Vizier.get_catalogs(catalog_list.keys())[2]
gold_ra = catalogs_gold['_RA']
gold_dec = catalogs_gold['_DE']
gold_logg = catalogs_gold['log_g_']
basic_ra = catalogs_basic['_RA']
basic_dec = catalogs_basic['_DE']
basic_logg = catalogs_basic['log.g2']
if combine is True:
ra = np.append(np.array(gold_ra), np.array(basic_ra))
dec = np.append(np.array(gold_dec), np.array(basic_dec))
logg = np.append(np.array(gold_logg), np.array(basic_logg))
return ra, dec, logg
else:
return gold_ra, gold_dec, gold_logg, basic_ra, basic_dec, basic_logg
def get_tess_limb_darkening_guesses(teff, logg):
'''
Given Teff and log(g), query the Claret+2017 limb darkening coefficient
grid. Return the nearest match.
TODO: interpolate instead of doing the nearest match. Nearest match is
good to maybe only ~200 K and ~0.3 in log(g).
'''
if not vizier_dependency:
raise ImportError('This function requires astroquery.'
'Try: `conda install -c astropy astroquery`')
# Assume solar metallicity.
metallicity = 0
# Get the Claret quadratic priors for TESS bandpass. The table below is
# good from Teff = 1500 - 12000K, logg = 2.5 to 6. We choose values
# computed with the "r method", see
# http://vizier.u-strasbg.fr/viz-bin/VizieR-n?-source=METAnot&catid=36000030¬id=1&-out=text
if not 2300 < teff < 12000:
if teff < 15000:
LOGWARNING(
'using 12000K atmosphere LD coeffs even tho teff={}'.format(
teff))
else:
LOGERROR('got teff error')
if not 2.5 < logg < 6:
if teff < 15000:
# Rough guess: assume star is B6V, Pecaut & Mamajek (2013) table.
_Mstar = 4 * units.Msun
_Rstar = 2.9 * units.Rsun
logg = np.log10((const.G * _Mstar / _Rstar**2).cgs.value)
else:
LOGERROR('got logg error')
Vizier.ROW_LIMIT = -1
catalog_list = Vizier.find_catalogs('J/A+A/600/A30')
catalogs = Vizier.get_catalogs(catalog_list.keys())
t = catalogs[1]
sel = (t['Type'] == 'r')
df = t[sel].to_pandas()
# Each Teff has 8 tabulated logg values. First, find the best teff match.
foo = df.iloc[(df['Teff'] - teff).abs().argsort()[:8]]
# Then, among those best 8, get the best logg match.
bar = foo.iloc[(foo['logg'] - logg).abs().argsort()].iloc[0]
# TODO: should probably determine these coefficients by INTERPOLATING.
# (especially in cases when you're FIXING them, rather than letting them
# float).
LOGWARNING('skipping interpolation for Claret coefficients.')
LOGWARNING('data logg={:.3f}, teff={:.1f}'.format(logg, teff))
LOGWARNING('Claret logg={:.3f}, teff={:.1f}'.format(
bar['logg'], bar['Teff']))
u_linear = bar['aLSM']
u_quad = bar['bLSM']
return float(u_linear), float(u_quad)
def find_author_vizier(cat):
""" Search Vizier catalog with astroquery and find the reference
in the description.
Parameters:
-----------
`cat`: {str}
Name of the catalog (e.g.: "I/337/gaia").
Returns:
--------
`cat_ref`: {str}
Name of the original publication of the catalog (e.g.: "Gaia Collaboration, 2016").
"""
vizier_catalog = cat.split('/')[:-1]
vizier_name = vizier_catalog[0]
for j in range(len(vizier_catalog)-1):
vizier_name += '/%s' % vizier_catalog[j + 1]
catalog_list = Vizier.find_catalogs(vizier_name)
cat_desc = catalog_list[vizier_name].description
i2 = -1
while True:
i = i2
i2 = cat_desc.find("(", i2 + 1)
if i2 == -1:
break
cat_ref = cat_desc[cat_desc.find("(", i)+1: cat_desc.find(")", i)]
return cat_ref
def download_furlan_radius_correction_table():
# Furlan et al 2017
catalog_list = Vizier.find_catalogs('J/AJ/153/71')
print({k: v.description for k, v in catalog_list.items()})
Vizier.ROW_LIMIT = -1
f17_tablelist = Vizier.get_catalogs(catalog_list.keys())
companiontab = f17_tablelist[-1]
df = companiontab.to_pandas()
for colname in ['KOI', 'Nc', 'Nobs']:
df[colname] = df[colname].astype(int)
# Assume planet orbits primary; this is lower bound on contamination.
# Therefore it's weaker than assuming the planet orbits the next-brightest
# companion. However, it's what Petigura+ 2018 did. (Probably b/c the converse
# would throw out too many).
df = df[df['Orbit'] == b'primary']
outname = '../data/Furlan_2017_table9.csv'
df.to_csv(outname, index=False)
print('saved to {:s}'.format(outname))
def get_vizier_table_as_dataframe(vizier_search_str, srccolumns, dstcolumns,
table_num=0, verbose=1,
whichcataloglist='default'):
"""
Download a table from Vizier (specified by `vizier_search_str`). Get
columns specified by the |-separated string, `columns`. Write the
resulting table to a CSV file, `outcsvpath`. Sometimes, the table number
on Vizier (`table_num`) must be specified. In the context of CDIPS catalog
construction, this is mostly for making "source|cluster|age" output.
Args:
vizier_search_str (str): specifies Vizier search.
srccolumns (str): columns in the Vizier catalog to get e.g.,
"RA|DE|age"
dstcolumns (str): columns to rename them to, e.g., "ra|dec|log10age".
whichcataloglist (str): "default", else a string to specify the
catalog, e.g., ""J/A+A/640/A1""
"""
Vizier.ROW_LIMIT = -1
catalog_list = Vizier.find_catalogs(vizier_search_str)
if whichcataloglist == 'default':
catalogs = Vizier.get_catalogs(catalog_list.keys())
else:
catalogs = Vizier.get_catalogs(whichcataloglist)
tab = catalogs[table_num]
if verbose:
print(f'initial number of members: {len(tab)}')
df = tab.to_pandas()
# rename columns to homogeneous format. e.g.,
# df = df.rename(columns={"Gaia": "source_id", "Group": "cluster"})
df = df.rename(
columns={
k:v for k,v in zip(srccolumns.split("|"), dstcolumns.split("|"))
}
)
if 'source_id' in dstcolumns.split("|"):
# strip "Gaia DR2 6083332579305755520" style format if present
try:
if np.any(df.source_id.str.contains('Gaia DR2 ')):
df['source_id'] = df.source_id.str.replace('Gaia DR2 ', '')
except AttributeError:
pass
# if NaNs are present, omit them. (e.g., from EsplinLuhman19)
if np.any(pd.isnull(df.source_id)):
N = len(df[pd.isnull(df.source_id)])
print(f'WRN! Found {N} null source_id values of {len(df)}. Omitting.')
df = df[~pd.isnull(df.source_id)]
df['source_id'] = df['source_id'].astype('int64')
sdf = df[dstcolumns.split("|")]
return sdf
def make_Rizzuto15_GaiaDR2_crossmatch():
'''
Aaron Rizzuto et al (2015) picked out ~400 candidate USco members, then
surveyed them for Li absorption. Led to 237 spectroscopically confirmed
members.
see
http://vizier.cfa.harvard.edu/viz-bin/VizieR?-source=J/MNRAS/448/2737
'''
Vizier.ROW_LIMIT = -1
catalog_list = Vizier.find_catalogs('J/MNRAS/448/2737')
catalogs = Vizier.get_catalogs(catalog_list.keys())
cands = catalogs[0]
usco_pms = catalogs[1] # pre-MS stars in USco
usco_disk = catalogs[2] # members of USco w/ circumstellar disk
c = SkyCoord([ra.replace(' ',':')
for ra in list(map(str,usco_pms['RAJ2000']))],
[de.replace(' ',':')
for de in list(map(str,usco_pms['DEJ2000']))],
unit=(u.hourangle, u.deg))
usco_pms['RA'] = c.ra.value
usco_pms['DEC'] = c.dec.value
usco_pms.remove_column('RAJ2000')
usco_pms.remove_column('DEJ2000')
foo = usco_pms.to_pandas()
outname = '../data/cluster_data/moving_groups/Rizzuto_15_table_2_USco_PMS.csv'
foo.to_csv(outname,index=False)
print('saved {:s}'.format(outname))
c = SkyCoord([ra.replace(' ',':')
for ra in list(map(str,usco_disk['RAJ2000']))],
[de.replace(' ',':')
for de in list(map(str,usco_disk['DEJ2000']))],
unit=(u.hourangle, u.deg))
usco_disk['RA'] = c.ra.value
usco_disk['DEC'] = c.dec.value
usco_disk.remove_column('RAJ2000')
usco_disk.remove_column('DEJ2000')
foo = usco_disk.to_pandas()
outname = os.path.join(datadir, 'Rizzuto_15_table_3_USco_hosts_disk.csv')
foo.to_csv(outname,index=False)
print('saved {:s}'.format(outname))
print(
''' I then uploaded these lists to MAST, and used their spatial
cross-matching with a 3 arcsecond cap, following
https://archive.stsci.edu/tess/tutorials/upload_list.html
This crossmatch is the output that I then saved to
'../results/Rizzuto_15_table_2_USco_PMS_GaiaDR2_crossmatched_2arcsec_MAST.csv'
'../results/Rizzuto_15_table_3_USco_disks_GaiaDR2_crossmatched_2arcsec_MAST.csv'
'''
)
def make_Casagrande_11_GaiaDR2_crossmatch():
'''
Casagrande et al (2011) re-analyzed the Geneva-Copenhagen survey, and got a
kinematically unbiased sample of solar neighborhood stars with kinematics,
metallicites, and ages.
If Casagrande's reported max likelihood ages (from Padova isochrones) are
below 1 Gyr, then that's interesting enough to look into.
NOTE that this age cut introduces serious biases into the stellar mass
distribution -- see Casagrande+2011, Figure 14.
http://vizier.cfa.harvard.edu/viz-bin/VizieR?-source=J/A+A/530/A138
'''
vizier_search_str = 'J/A+A/530/A138'
table_num = 0
ra_str = 'RAJ2000'
dec_str = 'DEJ2000'
outname = os.path.join(datadir,
'Casagrande_2011_table_1_GCS_ages_lt_1Gyr.csv')
Vizier.ROW_LIMIT = -1
catalog_list = Vizier.find_catalogs(vizier_search_str)
catalogs = Vizier.get_catalogs(catalog_list.keys())
tab = catalogs[0]
# ageMLP: max-likelihood padova isochrone ages
# ageMLB: max-likelihood BASTI isochrone ages. not queried.
sel = (tab['ageMLP'] > 0)
sel &= (tab['ageMLP'] < 1)
coords = SkyCoord(ra=tab[ra_str], dec=tab[dec_str], frame='icrs',
unit=(u.hourangle, u.deg))
# MAST uploads need these two column names
tab['RA'] = coords.ra.value
tab['DEC'] = coords.dec.value
tab.remove_column('RAJ2000')
tab.remove_column('DEJ2000')
foo = tab[sel].to_pandas()
foo.to_csv(outname,index=False)
print('saved {:s}'.format(outname))
print(
''' I then uploaded these lists to MAST, and used their spatial
cross-matching with a 3 arcsecond cap, following
https://archive.stsci.edu/tess/tutorials/upload_list.html
This crossmatch is the output that I then saved to
{:s}
'''.format(outname.replace('data','results').replace('.csv','_GaiaDR2_3arcsec_crossmatch_MAST.csv'))
)
def get_k13_index():
#
# the ~3784 row table
#
Vizier.ROW_LIMIT = -1
catalog_list = Vizier.find_catalogs('J/A+A/558/A53')
catalogs = Vizier.get_catalogs(catalog_list.keys())
k13_index = catalogs[1].to_pandas()
for c in k13_index.columns:
if c != 'N':
k13_index[c] = k13_index[c].str.decode('utf-8')
return k13_index
def search_vizier_for_sources(ra, dec, radius, catalog, row_limit=-1):
"""Function to perform online query of the 2MASS catalogue and return
a catalogue of known objects within the field of view
Inputs:
:param str ra: RA J2000 in sexigesimal format
:param str dec: Dec J2000 in sexigesimal format
:param float radius: Search radius in arcmin
:param str catalog: Catalog to search. Options include:
['2MASS', 'VPHAS+']
"""
print('Seaching '+catalog+' catalog in Vizier at pointing '+ra+', '+dec+\
' with radius '+str(radius)+' arcmin' )
supported_catalogs = { '2MASS': ['2MASS',
['_RAJ2000', '_DEJ2000', 'Jmag', 'e_Jmag', \
'Hmag', 'e_Hmag','Kmag', 'e_Kmag'],
{'Jmag':'<20'}],
'VPHAS+': ['II/341',
['sourceID', '_RAJ2000', '_DEJ2000', 'gmag', 'e_gmag', 'rmag', 'e_rmag', 'imag', 'e_imag', 'clean'],
{}],
'PS1': ['II/349',
['PrimaryID', 'RAJ2000', 'DEJ2000', 'Vmag', 'e_Vmag', 'Imag', 'e_Imag', 'clean'],
{}]
}
(cat_id, cat_col_list, cat_filters) = supported_catalogs[catalog]
v = Vizier(columns=cat_col_list,\
column_filters=cat_filters)
v.ROW_LIMIT = row_limit
c = coordinates.SkyCoord(ra + ' ' + dec,
frame='icrs',
unit=(units.hourangle, units.deg))
r = radius * units.arcminute
catalog_list = Vizier.find_catalogs(cat_id)
result = v.query_region(c, radius=r, catalog=[cat_id])
if len(result) == 1:
result = result[0]
return result
def search_catalog(self, term, obsolete):
json_result = '{"catalogs":[]}'
try:
catalog_list = Vizier.find_catalogs(term, include_obsolete=obsolete)
if catalog_list:
catalogs = CatalogCollection(catalog_list)
json_result = dumper(catalogs)
self.logger.info("Result from Vizier catalog search for '{0}' \n {1}".format(term, json_result))
else:
self.logger.info("Nothing found for '{0}'".format(term))
except:
self.logger.error("Error in SimbadProvider", exc_info=True)
raise ProviderException("error", "something")
return json_result
def get_k13_param_table():
#
# the ~3000 row table with determined parameters
#
cols = [
'map', 'cmd', 'stars', 'Name', 'MWSC', 'Type', 'RAJ2000', 'DEJ2000',
'r0', 'r1', 'r2', 'pmRA', 'pmDE', 'RV', 'e_RV', 'o_RV', 'd', 'E_B-V_',
'logt', 'N1sr2', 'rc', 'rt', 'k', 'SType', '__Fe_H_', 'Simbad'
]
v = Vizier(columns=cols)
v.ROW_LIMIT = -1
catalog_list = v.find_catalogs('J/A+A/558/A53')
catalogs = v.get_catalogs(catalog_list.keys())
k13 = catalogs[0].to_pandas()
k13['Name'] = k13['Name'].str.decode('utf-8')
return k13
def get_soubiran_19_rv_table():
cols = [
'ID', 'ID2', 'RA_ICRS', 'DE_ICRS', 'dmode', 'Nmemb', 'Nsele', 'RV',
'e_RV', 's_RV', 'X', 'e_X', 'Y', 'e_Y', 'Z', 'e_Z', 'U', 'e_U', 'V',
'e_V', 'W', 'e_W', 'Vr', 'e_Vr', 'Vphi', 'e_Vphi', 'Vz', 'e_Vz',
'Simbad'
]
v = Vizier(columns=cols)
v.ROW_LIMIT = -1
catalog_list = v.find_catalogs('J/A+A/619/A155')
catalogs = v.get_catalogs(catalog_list.keys())
df = catalogs[0].to_pandas()
df['ID'] = df['ID'].str.decode('utf-8')
return df
def CantatGaudin2019_velaOB2_to_csv():
# https://ui.adsabs.harvard.edu/abs/2019A%26A...626A..17C/abstract
Vizier.ROW_LIMIT = -1
catalog_list = Vizier.find_catalogs('J/A+A/626/A17')
catalogs = Vizier.get_catalogs(catalog_list.keys())
tab = catalogs[0]
df = tab.to_pandas()
outdf = df[['Source','Pop']]
outdf = outdf.rename(columns={"Source": "source_id", "Pop": "cluster"})
# Quoting Tristan, discussing Fig3 from the paper:
# """
# the clumps that are labeled are known "open clusters". The diffuse
# stellar distribution in the middle and around was called the Vela OB2
# association, supposed to be in front of the clusters or maybe in
# between, and everyone thought they were unrelated objects, different
# age, different history. In the figure, the colour code indicates stars
# that have the same age and velocity.
# My conclusion is that there is no object that can be called "Vela OB2"
# association. There are clusters, and a diffuse distribution of stars around
# each cluster. And the sum of all those fluffy distributions is "the
# association". Aggregates of young stars are so sub-structured that it
# doesn't even make sense to label all the clumps, and it can even be
# difficult to guess which clump observers referred to when they looked at
# those regions in the 19th century. There are ongoing debates over whether
# NGC 1746 exists, for instance. Which are the stars that were originally
# classified as NGC 1746, and does it even matter?
# """
outdf['cluster'] = np.core.defchararray.add(
np.repeat('cg19velaOB2_pop', len(outdf)),
nparr(outdf['cluster']).astype(str)
)
outpath = os.path.join(clusterdatadir, 'moving_groups',
'CantatGaudin2019_velaOB2_MATCH.csv')
outdf.to_csv(outpath, index=False)
print('made {}'.format(outpath))
def download_catalog(self):
"""
Downloads the flare catalog using Vizier.
The flare catalog is named 'Guenther_2020_flare_catalog.txt'.
The star catalog is named 'Guenther_2020_star_catalog.txt'.
Attributes
----------
flare_table : astropy.table.Table
Flare catalog that was downloaded.
"""
Vizier.ROW_LIMIT = -1
catalog_list = Vizier.find_catalogs('TESS flares sectors')
catalogs = Vizier.get_catalogs(catalog_list.keys())
self.flare_table = catalogs[1]
self.flare_table.rename_column('_tab2_5', 'tpeak')
self.flare_table.write(os.path.join(self.fn_dir,
self.flare_catalog_name),
format='csv')
return
from astroquery.vizier import Vizier
import numpy as np
import scipy.optimize as op
catalog_list = Vizier.find_catalogs('New white dwarf SDSS DR12')
print({k:v.description for k,v in catalog_list.items()})
Vizier.ROW_LIMIT = -1
catalogs = Vizier.get_catalogs(list(catalog_list))
# Our model has a temperature effect, then absorption features on top of the model
# Constants
h_planck = 4.136e-15 # in eV*s
c_light = 2.998e10 # in cgs
k_boltz = 8.617e-5 # in eV/K
# Balmer line centers (in cgs):
lambda_balmer = np.array([6563, 4861, 4341, 4102, 3970, 3889])
window_dict = {}
# Lorentz distribution:
def lorentz(C_coeff, x_center, gamma, x):
return -C_coeff / (np.pi * gamma * (1.0 + ((x-x_center)/gamma)**2))
def make_Oh17_GaiaDR2_crossmatch(
namestr='Oh_2017_clustering_GaiaDR2_crossmatched',
outdir=datadir, homedir='/home/luke/'):
'''
Semyeong Oh et al (2017) discovered 10.6k stars with separations <10pc that
are in likely comoving pairs / groups.
see
http://vizier.cfa.harvard.edu/viz-bin/VizieR?-source=J/AJ/153/257
'''
# Download Oh's tables of stars, pairs, and groups.
Vizier.ROW_LIMIT = -1
catalog_list = Vizier.find_catalogs('J/AJ/153/257')
catalogs = Vizier.get_catalogs(catalog_list.keys())
stars = catalogs[0]
pairs = catalogs[1]
groups = catalogs[2]
# IMPOSE GROUP SIZE >= 3.
stars = stars[stars['Size'] >= 3]
# use the gaia dr1 ids, to match the dr1_neighbourhood table.
outfile = os.path.join(outdir,'gotmatches_{}.xml.gz'.format(namestr))
xmltouploadpath = os.path.join(outdir,'toupload_{}.xml'.format(namestr))
if os.path.exists(outfile):
os.remove(outfile)
if not os.path.exists(outfile):
t = Table()
t['name'] = arr(stars['Star']).astype(str)
# note "Group" is a bad thing to name a column b/c it is a SQL
# keyword...
t['groupname'] = arr(stars['Group']).astype(str)
t['ra'] = arr(stars['RAJ2000'])*u.deg
t['dec'] = arr(stars['DEJ2000'])*u.deg
t['gaia'] = arr(stars['Gaia']).astype(int)
t['gmag'] = arr(stars['Gmag'])*u.mag
t['size'] = arr(stars['Size']).astype(int)
votable = from_table(t)
writeto(votable, xmltouploadpath)
print('made {}'.format(xmltouploadpath))
Gaia.login(credentials_file=os.path.join(homedir, '.gaia_credentials'))
# https://gea.esac.esa.int/archive/documentation/GDR2/...
# Gaia_archive/chap_datamodel/sec_dm_auxiliary_tables/...
# ssec_dm_dr1_neighbourhood.html
jobstr = (
'''
SELECT TOP {ncut:d} u.name, u.gaia, u.ra, u.dec,
u.groupname as assoc, u.size, u.gmag,
n.dr2_source_id as source_id, n.angular_distance as dist, n.rank,
n.magnitude_difference
FROM tap_upload.foobar as u, gaiadr2.dr1_neighbourhood AS n
WHERE u.gaia = n.dr1_source_id
'''
)
maxncut = int(5*len(stars)) # to avoid query timeout
query = jobstr.format(ncut=maxncut)
if not os.path.exists(outfile):
# might do async if this times out. but it doesn't.
j = Gaia.launch_job(query=query,
upload_resource=xmltouploadpath,
upload_table_name="foobar", verbose=True,
dump_to_file=True, output_file=outfile)
Gaia.logout()
vot = parse(outfile)
tab = vot.get_first_table().to_table()
if maxncut - len(tab) < 10:
errmsg = 'ERROR! too many matches'
raise AssertionError(errmsg)
# if nonzero and finite proper motion, require Gaia pm match to sign
# of stated Gagne PMs.
df = tab.to_pandas()
print('\n'+42*'-')
print('{} stars in original Oh table (size>=3)'.format(len(stars)))
print('{} stars in gaia nbhd match'.format(len(df)))
# we want DR2 mags. "gmag" is the uploaded DR1 mag.
df['gaia_gmag'] = df['gmag'] + df['magnitude_difference']
df.drop(['gmag'], axis=1, inplace=True)
_, inv, cnts = np.unique(df['name'], return_inverse=True,
return_counts=True)
df['n_in_nbhd'] = cnts[inv]
df['name'] = df['name'].str.decode('utf-8')
df['assoc'] = df['assoc'].str.decode('utf-8')
df = df.sort_values(['name','dist'])
df = df.drop_duplicates(subset='name', keep='first')
df['gaia'] = df['gaia'].astype('int64')
df = df.rename(index=str, columns={"gaia":"gaia_dr1_source_id"})
df['size'] = df['size'].astype('int')
df['source_id'] = df['source_id'].astype('int64')
print('{} stars after above cuts + chosing nearest nbhr by spatial sep'.
format(len(df)))
outpath = os.path.join(outdir,'MATCHED_{}.csv'.format(namestr))
df.to_csv(outpath, index=False)
print('made {}'.format(outpath))
print(79*'=')
from astroquery.vizier import Vizier
from astroquery.simbad import Simbad
import requests
from bs4 import BeautifulSoup
import numpy as np
from astropy.table import Table, vstack
from astropy.io import ascii
from astropy import units as u
from astropy.coordinates import SkyCoord, get_constellation
import pandas as pd
import glob, os
print('Downloading constellation_borders.csv...')
# This loads all the catalogs by the keyword "constellation"
catalogue_list = Vizier.find_catalogs("Constellation")
Vizier.ROW_LIMIT = -1
# Following table contains the relevant data
catalog = Vizier.get_catalogs("VI/49")[1]
catalog.remove_columns(['cst', 'type'])
coords = SkyCoord(catalog['RAJ2000'], catalog['DEJ2000'], unit="deg")
const = coords.get_constellation()
const = ['Bootes' if x == 'Boötes' else x
for x in const] # fixing for the unicode problem
catalog.add_column(const, name="Constellation", index=2)
catalog.write("constellation_borders.csv", format="csv", overwrite="True")
print('Done - Constellation borders\n')
print('Downloading messier_objects.csv...')
#Download magnitdue data from astropixels/SEDS
from astroquery.vizier import Vizier
import regions
from astropy import wcs
from astropy import coordinates as coord
from astropy.io import fits
from astropy.table import Table
from files import files
from paths import catalog_path
Vizier.find_catalogs('Kalcheva')
Vizier.ROW_LIMIT = 1e9
cats = Vizier.get_catalogs('J/A+A/615/A103')
cat = cats[0]
coords = coord.SkyCoord(cat['RAJ2000'], cat['DEJ2000'], frame='fk5')
reglist = regions.io.read_ds9('cutout_regions.reg')
full_table = Table.read(f'{catalog_path}/concatenated_catalog.ipac',
format='ascii.ipac')
ttl_uchii = 0
ttl_compact = 0
for regname, fn in files.items():
reg = [x for x in reglist if x.meta['text'] == regname][0]
match_cornish = reg.contains(coords, wcs.WCS(fits.getheader(fn)))
ncornish = match_cornish.sum()
def make_vizier_GaiaDR2_crossmatch(vizier_search_str, ra_str, dec_str,
pmra_str, pmdec_str, name_str, assoc_name,
table_num=0, outname='', maxsep=10,
outdir=datadir,
homedir='/home/luke/' ):
'''
Spatially crossmatch catalog of <~100,000 members w/ coords and PMs against
Gaia DR2. This assumes that the catalog is on vizier.
make_Kraus14_GaiaDR2_crossmatch is an example of a call.
'''
Vizier.ROW_LIMIT = -1
catalog_list = Vizier.find_catalogs(vizier_search_str)
catalogs = Vizier.get_catalogs(catalog_list.keys())
tab = catalogs[table_num]
print(42*'-')
print('{}'.format(outname))
print('initial number of members: {}'.format(len(tab)))
# gaia xmatch need these two column names
RA = tab[ra_str]
dec = tab[dec_str]
pm_RA = tab[pmra_str]
pm_dec = tab[pmdec_str]
name = tab[name_str]
assoc = np.repeat(assoc_name, len(RA))
assert tab[ra_str].unit == u.deg
assert tab[pmra_str].unit == u.mas/u.year
xmatchoutpath = outname.replace('.csv','_MATCHES_GaiaDR2.csv')
outfile = outname.replace('.csv','_GOTMATCHES_GaiaDR2.xml')
xmltouploadpath = outname.replace('.csv','_TOUPLOAD_GaiaDR2.xml')
# do the spatial crossmatch...
if os.path.exists(outfile):
os.remove(outfile)
if not os.path.exists(outfile):
_ = make_votable_given_cols(name, assoc, RA, dec, pm_RA, pm_dec,
outpath=xmltouploadpath)
Gaia.login(credentials_file=os.path.join(homedir, '.gaia_credentials'))
# separated less than 10 arcsec.
jobstr = (
'''
SELECT TOP {ncut:d} u.name, u.assoc, u.ra, u.dec, u.pm_ra, u.pm_dec,
g.source_id, DISTANCE(
POINT('ICRS', u.ra, u.dec),
POINT('ICRS', g.ra,g.dec)) AS dist,
g.phot_g_mean_mag as gaia_gmag,
g.pmra AS gaia_pmra,
g.pmdec AS gaia_pmdec
FROM tap_upload.foobar as u, gaiadr2.gaia_source AS g
WHERE 1=CONTAINS(
POINT('ICRS', u.ra, u.dec),
CIRCLE('ICRS', g.ra, g.dec, {sep:.8f})
)
'''
)
maxncut = int(5*len(name)) # to avoid query timeout
maxsep = (maxsep*u.arcsec).to(u.deg).value
query = jobstr.format(sep=maxsep, ncut=maxncut)
if not os.path.exists(outfile):
# might do async if this times out. but it doesn't.
j = Gaia.launch_job(query=query,
upload_resource=xmltouploadpath,
upload_table_name="foobar", verbose=True,
dump_to_file=True, output_file=outfile)
Gaia.logout()
vot = parse(outfile)
tab = vot.get_first_table().to_table()
if maxncut - len(tab) < 10:
errmsg = 'ERROR! too many matches'
raise AssertionError(errmsg)
print('number of members after gaia 10 arcsec search: {}'.format(len(tab)))
# if nonzero and finite proper motion, require Gaia pm match to sign
# of stated PMs.
df = tab.to_pandas()
print('\n'+42*'-')
sel = (df['gaia_gmag'] < 18)
print('{} stars in sep < 10 arcsec, G<18, xmatch'.format(len(df[sel])))
sel &= (
( (df['pm_ra'] != 0 ) & (df['pm_dec'] != 0 ) &
( np.sign(df['pm_ra']) == np.sign(df['gaia_pmra']) ) &
( np.sign(df['pm_dec']) == np.sign(df['gaia_pmdec']) )
)
|
(
(df['pm_ra'] == 0 ) & (df['pm_dec'] == 0 )
)
)
df = df[sel]
print('{} stars in sep < 10 as xmatch, G<18, after pm cut (xor zero pm)'.
format(len(df)))
# make multiplicity column. then sort by name, then by distance. then drop
# name duplicates, keeping the first (you have nearest neighbor saved!)
_, inv, cnts = np.unique(df['name'], return_inverse=True,
return_counts=True)
df['n_in_nbhd'] = cnts[inv]
df['name'] = df['name'].str.decode('utf-8')
df['assoc'] = df['assoc'].str.decode('utf-8')
df = df.sort_values(['name','dist'])
df = df.drop_duplicates(subset='name', keep='first')
df['source_id'] = df['source_id'].astype('int64')
print('{} stars after above cuts + chosing nearest nbhr by spatial sep'.
format(len(df)))
df.to_csv(xmatchoutpath, index=False)
print('made {}'.format(xmatchoutpath))
print(79*'=')
Store the paths and names of all the input clusters stored in the
input folder.
'''
cl_files = [join('input/', f) for f in listdir('input/') if
isfile(join('input/', f))]
# Remove readme file it is still there.
try:
cl_files.remove('input/README.md')
except ValueError:
pass
# Remove any '_query.dat' file.
rm_idx = []
for i, _ in enumerate(cl_files):
if _.endswith('_query.dat'):
rm_idx.append(i)
cl_files = np.delete(cl_files, rm_idx).tolist()
return cl_files
if __name__ == '__main__':
# To see available catalogs:
if False:
from astroquery.vizier import Vizier
# catalog_list = Vizier.find_catalogs('Pan-STARRS')
catalog_list = Vizier.find_catalogs('ALLWISE')
catalogs = Vizier.get_catalogs(catalog_list.keys())
print(catalogs)
main()
error_tot = np.sqrt(error.dot(error.T)) * 1e3
if error_tot > 100:
warnings.warn("Satellite position off by more than 100 m")
velocity_stk_eci = np.array([1.031261, 0.146421, 7.510402])
error = np.array([position_ecef.velocity.d_x.value,position_ecef.velocity.d_y.value,position_ecef.velocity.d_z.value]) - \
velocity_stk_eci
error_tot = np.sqrt(error.dot(error.T)) * 1e3
if error_tot > 500:
warnings.warn("Satellite velocity off by more than 500 m/s")
#Find look vector for tangent point of MATS (92 km in orbit plane)
#Find orbit plane for two positions
#test star gazing
Vizier.ROW_LIMIT = 50
catalog_list = Vizier.find_catalogs('UCAC4')
catalogs = Vizier.get_catalogs(catalog_list.keys())
v = Vizier(columns=['_RAJ2000', '_DEJ2000', 'Vmag'],
column_filters={"Vmag": "<10"})
result = v.query_region(coord.SkyCoord(ra=299.590,
dec=35.201,
unit=(u.deg, u.deg),
frame='icrs'),
width="60m",
height="60m",
catalog=["I/322A"])
from astroquery.vizier import Vizier
catalog_list = Vizier.find_catalogs('J/ApJS/204/5')
Vizier.ROW_LIMIT = -1
cata = Vizier.get_catalogs(catalog_list.keys())[0]
print len(cata)
import matplotlib.pyplot as plt
import matplotlib.lines as mlines
from matplotlib.patches import Circle
import mpl_toolkits.mplot3d.art3d as art3d
from astroquery.vizier import Vizier
from spaceutils import show_maximized_plot, kiloparsec_to_lightyear
# =====================================================================================================
catalog_list = Vizier.find_catalogs('VII/202')
Vizier.ROW_LIMIT = 1000000
catalogs = Vizier.get_catalogs(catalog_list.keys())
data = catalogs[0]
# =====================================================================================================
data.sort('Name')
data["X"] = kiloparsec_to_lightyear(data["X"])
data["Y"] = kiloparsec_to_lightyear(data["Y"])
data["Z"] = kiloparsec_to_lightyear(data["Z"])
data["Rsun"] = kiloparsec_to_lightyear(data["Rsun"])
SUN_TO_CENTER_DISTANCE = 27200
MILKY_WAY_RADIUS = 110000 / 2
def get_k13_name_match(task):
"""
call in for loop, or in map, once you have cdips_df['cluster'] type column
args:
originalname : name from arbitrary reference of cluster
target_ra : in decimal deg
target_dec
reference: source of cluster membership
k13: dataframe of Kharchenko+2013 result
returns:
(matchname, have_name_match, have_mwsc_id_match, is_known_asterism,
not_in_k13, why_not_in_k13)
"""
originalname, target_ra, target_dec, reference, k13 = task
cluster = originalname
print(42*'#')
print(cluster)
have_name_match = False
have_mwsc_id_match = False
is_known_asterism = False
how_match = np.nan
why_not_in_k13 = ''
if pd.isnull(originalname):
return (np.nan, how_match, have_name_match, have_mwsc_id_match,
is_known_asterism, True, 'no_cluster_name')
if reference in ['Zari_2018_PMS','Zari_2018_UMS']:
not_in_k13 = True
is_zari_ums = False
if 'Zari_2018_UMS' in reference:
is_zari_ums = True
is_zari_pms = False
if 'Zari_2018_PMS' in reference:
is_zari_pms = True
if is_zari_pms:
why_not_in_k13 = 'is_zari_pms'
elif is_zari_ums:
why_not_in_k13 = 'is_zari_ums'
return (np.nan, how_match, have_name_match, have_mwsc_id_match,
is_known_asterism, not_in_k13, why_not_in_k13)
special_references = ['Velez_2018_scoOB2', 'Kounkel_2018_Ori',
'CantatGaudin_2019_velaOB2']
for r in special_references:
if r in reference:
# None of these works have meaningful matches in Kharchenko+2013.
not_in_k13 = True
why_not_in_k13 = 'gaia_supersedes'
return (np.nan, how_match, have_name_match, have_mwsc_id_match,
is_known_asterism, not_in_k13, why_not_in_k13)
#
# lowest name match priority: whatever KC2019 said!
#
if reference in ['Kounkel_2019'] and reference.startswith('kc19group_'):
# Don't attempt Kharchenko matches for the new KC2019 groups.
not_in_k13 = True
why_not_in_k13 = 'kc19_gaia_supersedes'
return (np.nan, how_match, have_name_match, have_mwsc_id_match,
is_known_asterism, not_in_k13, why_not_in_k13)
is_gaia_member = False
if 'Gulliver' in cluster and 'CantatGaudin_2018' in reference:
is_gaia_member = True
not_in_k13 = True
why_not_in_k13 = 'is_gaia_member'
return (np.nan, how_match, have_name_match, have_mwsc_id_match,
is_known_asterism, not_in_k13, why_not_in_k13)
# Hyades: I checked in a distance sort that it's not in K13. Probably b/c
# it's very extended.
# RSG7,8: Soubiran+2018 note "RSG 7 and RSG 8, two OCs recently found by
# Röser et al. (2016), are confirmed as two separate OCs very close in
# space and motion." So they were found after K13.
known_missing_clusters = ['Hyades', 'RSG_7', 'RSG_8', 'RSG_1', "RSG_5"]
for c in cluster.split(','):
if c in known_missing_clusters:
not_in_k13 = True
why_not_in_k13 = 'known missing from K13'
how_match = 'manual_check'
return (np.nan, how_match, have_name_match, have_mwsc_id_match,
is_known_asterism, not_in_k13, why_not_in_k13)
#
# special formatting cases:
# * Pozzo 1 is Vel OB2
# * Majaess50 is 10' from FSR0721, from D. Majaess (2012).
# * Majaess58 is 6' from FSR0775, from D. Majaess (2012).
# * Ditto for a few from CG2018...
# * "Collinder 34" is a sub-cluster of the larger Collinder 33 nebula, not
# appreciably different from SAI-24 either. (Saurin+ 2015,
# https://arxiv.org/pdf/1502.00373.pdf)
# * Some catalogs, eg., Cantat-Gaudin's, replaced "vdBergh-Hagen" with "BH"
# * Coma Star Cluster in Coma Berenices is Melotte 111
#
if 'Pozzo_1' in cluster:
cluster = 'Vel_OB2'
how_match = 'manual_override'
if 'Majaess 50' in cluster:
cluster = 'FSR_0721'
how_match = 'manual_override'
if 'Majaess 58' in cluster:
cluster = 'FSR_0775'
how_match = 'manual_override'
if 'Alessi_Teutsch_5' in cluster:
# underscores mess up the duplicate name list
cluster = "ASCC 118"
how_match = 'manual_override'
if 'Collinder 34' in cluster:
cluster = 'Collinder_33'
how_match = 'manual_override'
if 'BH' in cluster:
cluster = cluster.replace("BH","vdBergh-Hagen")
if 'ComaBer' in cluster:
cluster = cluster.replace('ComaBer','Melotte_111')
how_match = 'manual_override'
if 'CBER' in cluster:
cluster = cluster.replace('CBER','Melotte_111')
how_match = 'manual_override'
if 'PLE' in cluster:
cluster = cluster.replace('PLE','Melotte_22')
how_match = 'manual_override'
if 'ScoOB2' in cluster:
cluster = cluster.replace('ScoOB2','Sco_OB2')
how_match = 'string_match'
if 'USCO' in cluster:
cluster = cluster.replace('USCO','Sco_OB2')
how_match = 'string_match'
if "BDSB" in cluster:
if ' ' not in cluster and "_" not in cluster:
cluster = cluster.replace("BDSB","BDSB_")
how_match = 'string_match'
if "Casado Alessi 1" in cluster:
cluster = "Alessi_1"
how_match = 'manual_override'
if "Coll140" in cluster:
cluster = "Collinder_140"
how_match = 'manual_override'
if "alphaPer" in cluster:
cluster = "Melotte_20"
how_match = 'manual_override'
if "Trump10" in cluster:
cluster = "Trumpler_10"
how_match = 'manual_override'
if "Trump10" in cluster:
cluster = cluster.replace("Trump10","Trumpler_10")
how_match = 'manual_override'
if "Trump02" in cluster:
cluster = cluster.replace("Trump02","Trumpler_2")
how_match = 'manual_override'
if "PL8" in cluster:
cluster = cluster.replace("PL8","Platais_8")
how_match = 'manual_override'
if cluster == "NGC2451":
# Gaia Collaboration 2018 table 1a forgot the "A" (nb. from their
# paper, it's definitely the one they considered).
cluster = "NGC_2451A"
how_match = 'manual_override'
# types like: 'Aveni_Hunter_42,nan,Aveni_Hunter_42,nan' need to be recast
# to e.g., 'Aveni-Hunter_42,nan,Aveni-Hunter_42,nan'
for m in re.finditer('[a-zA-Z]+_[a-zA-Z]+_[0-9]+', cluster):
cluster = cluster.replace(
m.group(0),
m.group(0).split('_')[0]+"-"+'_'.join(m.group(0).split('_')[1:])
)
# types like: "Aveni Hunter 1,nan,Aveni Hunter 1,nan", or 'Alessi Teutsch
# 10' need to follow the same "Alessi-Teutsch_10" syntax
for m in re.finditer('[a-zA-Z]+\ [a-zA-Z]+\ [0-9]+', cluster):
cluster = cluster.replace(
m.group(0),
m.group(0).split(' ')[0]+"-"+'_'.join(m.group(0).split(' ')[1:])
)
# reformat ESO clusters like "ESO 129 32" (D14) to "ESO_129-32" (K13 format)
for m in re.finditer('ESO\ [0-9]+\ [0-9]+', cluster):
cluster = cluster.replace(
m.group(0),
"ESO_"+'-'.join(m.group(0).split(' ')[1:])
)
# reformat ESO clusters like "ESO_129_32" (CG18) to "ESO_129-32" (K13 format)
for m in re.finditer('ESO_[0-9]+_[0-9]+', cluster):
cluster = cluster.replace(
m.group(0),
"ESO_"+'-'.join(m.group(0).split('_')[1:])
)
#
# initial normal match: try matching against replacing spaces with
# underscores
#
clustersplt = cluster.split(',')
trystrs = []
for c in clustersplt:
trystrs.append(c)
trystrs.append(c.replace(' ','_'))
for trystr in trystrs:
if trystr in nparr(k13['Name']):
have_name_match = True
name_match = trystr
how_match = 'string_match'
not_in_k13 = False
why_not_in_k13 = ''
break
#
# try if SIMBAD's name matcher has anything.
#
if not have_name_match:
for c in clustersplt:
try:
res = Simbad.query_objectids(c)
except requests.exceptions.ConnectionError:
time.sleep(15)
res = Simbad.query_objectids(c)
try:
resdf = res.to_pandas()
except AttributeError:
print('{}: simbad no matches'.format(c))
continue
resdf['ID'] = resdf['ID'].str.decode('utf-8')
smatches = nparr(resdf['ID'])
# some names have format 'Name M 42'
clean_smatches = [s.lstrip('NAME ') for s in smatches]
# some names have format '[KPS2012] MWSC 0531'
for ix, s in enumerate(clean_smatches):
strm = re.search("\[.*\]\ ", s)
if strm is not None:
clean_smatches[ix] = s.lstrip(strm.group())
# some names have variable length whitespace... e.g., 'NGC 2224'
# first set of attempts: everything in clean matches (irrespective if
# MWSC number exists)
trystrs = []
for _c in clean_smatches:
trystrs.append(_c)
trystrs.append(_c.replace(' ','_'))
for trystr in trystrs:
if trystr in nparr(k13['Name']):
have_name_match = True
how_match = 'SIMBAD_name_match'
name_match = trystr
break
# only then: check if you have MWSC identifier.
inds = ['MWSC' in _c for _c in clean_smatches]
mwsc_match = nparr(clean_smatches)[inds]
if len(mwsc_match) > 1:
pass
if len(mwsc_match) == 1:
have_mwsc_id_match = True
mwsc_id_match = mwsc_match[0].replace('MWSC ','')
if len(mwsc_match) == 0:
pass
if have_mwsc_id_match:
break
#
# if you got mwsc id above, use the index table to convert to a name
#
if have_mwsc_id_match and not have_name_match:
Vizier.ROW_LIMIT = -1
catalog_list = Vizier.find_catalogs('J/A+A/558/A53')
catalogs = Vizier.get_catalogs(catalog_list.keys())
t = catalogs[1]
df_index = t.to_pandas()
del t
# columns are MWSC,Name,flag,Type,n_Type,Src,SType,N
for c in df_index.columns:
if c != 'N':
df_index[c] = df_index[c].str.decode('utf-8')
mapd = {
"=": "cluster parameters are determined",
":": ("possibly this is a cluster, but parameters are not "
"determined"),
"-": "this is not a cluster",
"&": "duplicated/coincides with other cluster"
}
_k13 = df_index[df_index['MWSC'] == mwsc_id_match]
have_name_match=True
how_match = 'SIMBAD_MWSCID_match'
name_match = _k13['Name'].iloc[0]
flag = str(_k13['flag'].iloc[0])
if flag in [":","-","&"]:
is_in_index = True
why_not_in_k13 = "K13index: "+mapd[flag]
else:
pass
#
# check against the larger kharchenko index, which includes the clusters
# that they found dubious, or didn't report parameters for.
#
is_in_index = False
if not have_name_match and not have_mwsc_id_match:
Vizier.ROW_LIMIT = -1
catalog_list = Vizier.find_catalogs('J/A+A/558/A53')
catalogs = Vizier.get_catalogs(catalog_list.keys())
t = catalogs[1]
df_index = t.to_pandas()
del t
# columns are MWSC,Name,flag,Type,n_Type,Src,SType,N
for c in df_index.columns:
if c != 'N':
df_index[c] = df_index[c].str.decode('utf-8')
mapd = {
"=": "cluster parameters are determined",
":": ("possibly this is a cluster, but parameters are not "
"determined"),
"-": "this is not a cluster",
"&": "duplicated/coincides with other cluster"
}
clustersplt = cluster.split(',')
trystrs = []
for c in clustersplt:
trystrs.append(c)
trystrs.append(c.replace(' ','_'))
for trystr in trystrs:
if trystr in nparr(df_index['Name']):
have_name_match=True
how_match = 'K13_index_table_string_match'
name_match = trystr
is_in_index = True
_k13 = df_index.loc[df_index['Name'] == name_match]
flag = str(_k13['flag'].iloc[0])
if flag not in [":","-","&"]:
raise NotImplementedError('why do u match here, not '
'earlier?')
else:
why_not_in_k13 = "K13index: "+mapd[flag]
break
#
# try searching K13 within circles of 5,10 arcminutes of the quoted
# position. if more than 1 match, omit (to not get false name matches). if
# only 1 match, use the name. (this introduces some false matches,
# probably...)
#
if not have_name_match and not have_mwsc_id_match:
ra,dec = target_ra, target_dec
c = SkyCoord(ra, dec, unit=(u.deg,u.deg), frame='icrs')
k13_c = SkyCoord(nparr(k13['RAJ2000']), nparr(k13['DEJ2000']),
frame='icrs', unit=(u.deg,u.deg))
seps = k13_c.separation(c)
CUTOFFS = [5*u.arcmin, 10*u.arcmin]
for CUTOFF in CUTOFFS:
cseps = seps < CUTOFF
if len(cseps[cseps]) == 1:
have_name_match=True
how_match = 'K13_spatial_match_lt_{}arcmin'.format(CUTOFF.value)
name_match = k13.loc[cseps, 'Name'].iloc[0]
elif len(cseps[cseps]) > 1:
print('got too many matches within {} arcminutes!'.
format(CUTOFF))
pass
else:
pass
if have_name_match:
break
#
# if after all this, no match, check double cluster names list, and recall
# this function
#
if not have_name_match and not have_mwsc_id_match:
ddf = pd.read_csv(
os.path.join(clusterdatadir, 'double_names_WEBDA_20190606.csv'),
sep=';')
for c in clustersplt:
if len(ddf[ddf['cluster_name'] == c.replace('_',' ')]) == 1:
adopted_name = (
ddf[ddf['cluster_name'] ==
c.replace('_',' ')].iloc[0]['adopted_name']
)
return get_k13_name_match((adopted_name, target_ra, target_dec,
reference, k13))
#
# Check against known asterisms
#
is_known_asterism = False
for c in clustersplt:
# Baumgardt 1998.
if c in ['Collinder 399', 'Upgren 1', 'NGC 1252', 'Melotte 227',
'NGC 1746']:
is_known_asterism = True
break
if 'NGC' in c:
getfile = '/nfs/phn12/ar0/H/PROJ/lbouma/cdips/data/cluster_data/Sulentic_1973_NGC_known_asterisms.vot'
vot = parse(getfile)
tab = vot.get_first_table().to_table()
ddf = tab.to_pandas()
del tab
ngc_asterisms = nparr(ddf['NGC'])
if c.startswith('NGC '):
c = c.lstrip('NGC ')
elif c.startswith('NGC_'):
c = c.lstrip('NGC_')
elif c.startswith('NGC'):
c = c.lstrip('NGC')
# https://en.wikipedia.org/wiki/NGC_2451
if c.endswith('A'):
c = c.rstrip('A')
if c.endswith('B'):
c = c.rstrip('B')
try:
if int(c) in ngc_asterisms:
is_known_asterism = True
break
except ValueError:
# e.g., "NGC 2467-east", or similiar bs
pass
# NGC 1252 was also by Baumgardt 1998.
# identified by Kos+2018, MNRAS 480 5242-5259 as asterisms
try:
if int(c) in [1252, 6994, 7772, 7826]:
is_known_asterism = True
break
except ValueError:
# e.g., "NGC 2467-east"
pass
is_gagne_mg = False
if 'Gagne' in reference:
is_gagne_mg = True
why_not_in_k13 = 'is_gagne_mg'
is_oh_mg = False
if 'Oh' in reference:
is_oh_mg = True
why_not_in_k13 = 'is_oh_mg'
is_rizzuto_mg = False
if 'Rizzuto' in reference:
is_rizzuto_mg = True
why_not_in_k13 = 'is_rizzuto_mg'
is_bell32ori_mg = False
if 'Bell_2017_32Ori' in reference:
is_bell32ori_mg = True
why_not_in_k13 = 'is_bell_mg'
is_kraus_mg = False
if 'Kraus' in reference:
is_kraus_mg = True
why_not_in_k13 = 'is_kraus_mg'
not_in_k13 = False
if (
(is_gagne_mg or is_oh_mg or is_rizzuto_mg or is_bell32ori_mg or
is_kraus_mg or is_in_index)
and
not have_name_match
):
not_in_k13 = True
name_match = np.nan
#
# finally, if we failed to get matches above, (e.g., for some of the IR
# Majaess clusters), skip
#
for c in cluster.split(','):
if "Majaess" in c:
not_in_k13 = True
why_not_in_k13 = 'Majaess IR cluster match missing in K13'
how_match = 'majaess_flag'
return (np.nan, how_match, have_name_match, have_mwsc_id_match,
is_known_asterism, not_in_k13, why_not_in_k13)
if have_name_match:
return (name_match, how_match, have_name_match, have_mwsc_id_match,
is_known_asterism, not_in_k13, why_not_in_k13)
#
# lowest name match priority: whatever KC2019 said (even if there are
# others)!
#
if 'Kounkel_2019' in reference:
not_in_k13 = True
why_not_in_k13 = 'kc19_gaia_lastmatch'
return (np.nan, how_match, have_name_match, have_mwsc_id_match,
is_known_asterism, not_in_k13, why_not_in_k13)
try:
return (name_match, how_match, have_name_match, have_mwsc_id_match,
is_known_asterism, not_in_k13, why_not_in_k13)
except UnboundLocalError:
print('got UnboundLocalError for {} ({})'.format(
repr(cluster), repr(reference)))
import IPython; IPython.embed()
def construct_unique_cluster_name_column(cdips_cat_vnum=0.4):
"""
We already have a catalog with the following columns:
source_id;cluster;reference;ext_catalog_name;ra;dec;pmra;pmdec;parallax;
phot_g_mean_mag;phot_bp_mean_mag;phot_rp_mean_mag;
We want to supplement it with:
unique_cluster_name;k13_name_match;how_match;not_in_k13;
comment;logt;e_logt;logt_provenance
----------
The need for a "unique_cluster_name" is mainly in order to (a) connect with
other literature, and (b) know roughly how many "unique clusters" exist and
have been searched. The concept of "unique cluster" in substuctured regions
like Orion or Sco-OB2 is not defined, so whatever we arrive at for the
unique name should not be taken as gospel.
For "comment", we want any relevant information about the cluster (e.g., if
its existence is dubious, if it overlaps with other clusters, etc.)
For "logt", "e_logt", and "logt_provenance", we want both the K13 age if
it's available, and the KC19 age, if that's available.
----------
"""
cdips_df = ccl.get_cdips_catalog(ver=cdips_cat_vnum)
k13 = get_k13_df()
sourceid = nparr(cdips_df['source_id'])
clusterarr = nparr(cdips_df['cluster'])
ras = nparr(cdips_df['ra'])
decs = nparr(cdips_df['dec'])
referencearr = nparr(cdips_df['reference'])
#
# Attempt to get Kharchenko+2013 matches from a mix of cluster names and
# star positions (i.e., Appendix B of CDIPS-I).
#
inds = ~pd.isnull(clusterarr)
sourceid = sourceid[inds]
clusterarr = clusterarr[inds]
ras = ras[inds]
decs = decs[inds]
referencearr = referencearr[inds]
uarr, inds = np.unique(clusterarr, return_index=True)
sourceid = sourceid[inds]
clusterarr = clusterarr[inds]
ras = ras[inds]
decs = decs[inds]
referencearr = referencearr[inds]
namematchpath = (
'/nfs/phtess1/ar1/TESS/PROJ/lbouma/OC_MG_FINAL_v{}_with_K13_name_match.csv'.
format(cdips_cat_vnum)
)
if not os.path.exists(namematchpath):
res = list(map(get_k13_name_match,
zip(clusterarr, ras, decs, referencearr, repeat(k13))))
resdf = pd.DataFrame(res,
columns=['k13_name_match', 'how_match', 'have_name_match',
'have_mwsc_id_match', 'is_known_asterism', 'not_in_k13',
'why_not_in_k13']
)
resdf['source_id'] = sourceid
mdf = resdf.merge(cdips_df, how='left', on='source_id')
mdf.to_csv(namematchpath, index=False, sep=';')
print('made {}'.format(namematchpath))
else:
mdf = pd.read_csv(namematchpath, sep=';')
uniqpath = (
'/nfs/phtess1/ar1/TESS/PROJ/lbouma/OC_MG_FINAL_v{}_uniq.csv'.
format(cdips_cat_vnum)
)
if not os.path.exists(uniqpath):
mdf['unique_cluster_name'] = list(map(
get_unique_cluster_name, zip(mdf.iterrows()))
)
cols = ['unique_cluster_name', 'k13_name_match', 'how_match',
'have_name_match', 'have_mwsc_id_match', 'is_known_asterism',
'not_in_k13', 'why_not_in_k13', 'source_id', 'cluster', 'dec',
'dist', 'ext_catalog_name', 'parallax', 'phot_bp_mean_mag',
'phot_g_mean_mag', 'phot_rp_mean_mag', 'pmdec', 'pmra', 'ra',
'reference']
mdf[cols].to_csv(uniqpath, index=False, sep=';')
print('made {}'.format(uniqpath))
else:
mdf = pd.read_csv(uniqpath, sep=';')
#
# Merge the unique cluster names against the whole cdips dataframe, using
# mdf as a lookup table. This is slightly wrong, b/c some spatial matches
# would give different results than using this string match. However they
# are a small subset, this is computationally cheaper, and assigning a
# unique name has inherent limitations.
#
subdf = mdf[['unique_cluster_name', 'k13_name_match', 'how_match',
'have_name_match', 'have_mwsc_id_match', 'is_known_asterism',
'not_in_k13', 'why_not_in_k13', 'cluster']]
print('beginning big merge...')
fdf = cdips_df.merge(subdf, on='cluster', how='left')
assert len(fdf) == len(cdips_df)
print(42*'#')
print('# unique cluster names: {}'.format(
len(np.unique(
fdf[~pd.isnull(fdf['unique_cluster_name'])]['unique_cluster_name'])
))
)
print('fraction with k13 name match: {:.5f}'.format(
len(fdf[~pd.isnull(fdf['k13_name_match'])])/len(fdf)
))
print('fraction with any unique name: {:.5f}'.format(
len(fdf[~pd.isnull(fdf['unique_cluster_name'])])/len(fdf)
))
#
# Merge fdf k13_name_match against K13 index, and get updated comments.
# Join "why_not_in_k13" with "Source object type" "SType" from K13 index.
# This gives the "comments" column.
#
Vizier.ROW_LIMIT = -1
catalog_list = Vizier.find_catalogs('J/A+A/558/A53')
catalogs = Vizier.get_catalogs(catalog_list.keys())
k13_index = catalogs[1].to_pandas()
for c in k13_index.columns:
if c != 'N':
k13_index[c] = k13_index[c].str.decode('utf-8')
k13_df = k13
styped = {
"ass":"stellar association",
"ast":"Dias: possible asterism/dust hole/star cloud",
"dub":"Dias: dubious, objects considered doubtful by the DSS images inspection",
"emb":"embedded open cluster/cluster associated with nebulosity",
"glo":"globular cluster/possible globular cluster",
"irc":"infrared cluster",
"irg":"infrared stellar group",
"mog":"Dias: possible moving group",
"non":"Dias: non-existent NGC/ objects not found in DSS images inspection",
"rem":"Possible cluster remnant",
"var":"clusters with variable extinction"
}
k13_index['STypeComment'] = k13_index['SType'].map(styped)
# in order to do the desired name join, must remove duplicates from k13
# index. since we only care about the comments, keep "last" works from
# inspection.
ids = k13_index['Name']
print('removing duplicates from K13 index...\n{}'.format(repr(
k13_index[ids.isin(ids[ids.duplicated()])].sort_values(by='Name'))))
k13_index = k13_index.drop_duplicates(subset=['Name'], keep='last')
_df = fdf.merge(k13_index, left_on='k13_name_match', right_on='Name',
how='left')
assert len(_df) == len(fdf)
comment = _df['why_not_in_k13']
stypecomment = _df['STypeComment']
_df['comment'] = comment.map(str) + ". " + stypecomment.map(str)
_df['comment'] = _df['comment'].map(lambda x: x.replace('nan',''))
_df['comment'] = _df['comment'].map(lambda x: x.lstrip('. ').rstrip('. '))
# unique comments include:
# array(['',
# 'Dias: dubious, objects considered doubtful by the DSS images inspection',
# 'Dias: non-existent NGC/ objects not found in DSS images inspection',
# 'Dias: possible asterism/dust hole/star cloud',
# 'K13index: duplicated/coincides with other cluster',
# 'K13index: duplicated/coincides with other cluster. Dias: dubious, objects considered doubtful by the DSS images inspection',
# 'K13index: duplicated/coincides with other cluster. Dias: non-existent NGC/ objects not found in DSS images inspection',
# 'K13index: duplicated/coincides with other cluster. clusters with variable extinction',
# 'K13index: duplicated/coincides with other cluster. embedded open cluster/cluster associated with nebulosity',
# 'K13index: duplicated/coincides with other cluster. infrared cluster',
# 'K13index: duplicated/coincides with other cluster. stellar association',
# 'K13index: possibly this is a cluster, but parameters are not determined',
# 'K13index: possibly this is a cluster, but parameters are not determined. Dias: dubious, objects considered doubtful by the DSS images inspection',
# 'K13index: possibly this is a cluster, but parameters are not determined. Dias: non-existent NGC/ objects not found in DSS images inspection',
# 'K13index: possibly this is a cluster, but parameters are not determined. infrared cluster',
# 'K13index: possibly this is a cluster, but parameters are not determined. stellar association',
# 'K13index: this is not a cluster',
# 'K13index: this is not a cluster. Dias: dubious, objects considered doubtful by the DSS images inspection',
# 'K13index: this is not a cluster. Dias: non-existent NGC/ objects not found in DSS images inspection',
# 'K13index: this is not a cluster. Dias: possible asterism/dust hole/star cloud',
# 'K13index: this is not a cluster. Possible cluster remt',
# 'K13index: this is not a cluster. infrared cluster',
# 'Majaess IR cluster match missing in K13', 'Possible cluster remt',
# 'clusters with variable extinction',
# 'embedded open cluster/cluster associated with nebulosity',
# 'infrared cluster', 'is_bell_mg', 'is_gagne_mg',
# 'is_gagne_mg. infrared cluster',
# 'is_gagne_mg. stellar association', 'is_gaia_member',
# 'is_kraus_mg', 'is_oh_mg', 'is_oh_mg. stellar association',
# 'is_rizzuto_mg. stellar association', 'known missing from K13',
# 'stellar association']
fdf['comment'] = _df['comment']
del _df
#
# Assign ages. If a Kharchenko+2013 name match was found, report that age.
# In addition, if the star is in the Kounkel & Covey 2019 table, report
# their age as well.
#
_df = fdf.merge(k13_df,
left_on='k13_name_match',
right_on='Name',
how='left')
assert len(_df) == len(fdf)
k13_logt = np.round(nparr(_df['logt']),2).astype(str)
k13_e_logt = np.round(nparr(_df['e_logt']),2).astype(str)
k13_logt_prov = np.repeat('',len(fdf)).astype('>U20')
k13_logt_prov[k13_logt != 'nan'] = 'Kharchenko2013'
# kounkel & covey 2019 age matches are only for the groups that were not
# already known. (otherwise, use the kharchenko age, which is assumed to
# exist). do it by searching the cluster name. extract the groupid from the
# name.
kc19_df1 = pd.read_csv(os.path.join(clusterdatadir,'KC19_string_table1.csv'))
kc19_df2 = pd.read_csv(os.path.join(clusterdatadir,'KC19_string_table2.csv'))
kc19_sdf2 = kc19_df2[['group_id','name','age']]
kc19_mdf = kc19_df1.merge(kc19_sdf2, on='group_id', how='left')
# if the source_id is in the Kounkel & Covey table, then also report the
# KC19 age.
kc19_merge_cdips_df = fdf.merge(kc19_mdf, how='left', on='source_id')
assert len(kc19_merge_cdips_df) == len(fdf)
kc19_logt = np.round(nparr(kc19_merge_cdips_df['age']),2).astype(str)
kc19_e_logt = np.repeat('',len(fdf)).astype('>U10')
kc19_e_logt[kc19_logt != 'nan'] = '0.15' # Kounkel&Covey 2019 abstract precision
kc19_logt_prov = np.repeat('',len(fdf)).astype('>U20')
kc19_logt_prov[kc19_logt != 'nan'] = 'Kounkel_2019'
#
# Concatenate and separate by ",". Then remove all "nans". NOTE: if you add
# extra age provenances (i.e., more than K13 and KC19), this nan-stripping
# will require adjustments.
#
logt = list(map(','.join, zip(k13_logt, kc19_logt)))
e_logt = list(map(','.join, zip(k13_e_logt, kc19_e_logt)))
logt_prov = list(map(','.join, zip(k13_logt_prov, kc19_logt_prov)))
logt = [_.lstrip('nan,').rstrip(',nan') for _ in logt]
e_logt = [_.lstrip('nan,').rstrip(',nan') for _ in e_logt]
logt_prov = [_.lstrip(',').rstrip(',') for _ in logt_prov]
fdf['logt'] = logt
fdf['e_logt'] = e_logt
fdf['logt_provenance'] = logt_prov
# reformat for table to publish
scols = ['source_id', 'cluster', 'reference', 'ext_catalog_name', 'ra',
'dec', 'pmra', 'pmdec', 'parallax', 'phot_g_mean_mag',
'phot_bp_mean_mag', 'phot_rp_mean_mag', 'k13_name_match',
'unique_cluster_name', 'how_match', 'not_in_k13', 'comment',
'logt', 'e_logt', 'logt_provenance']
_df = fdf[scols]
pubpath = (
'/nfs/phtess1/ar1/TESS/PROJ/lbouma/OC_MG_FINAL_v{}_publishable.csv'.
format(cdips_cat_vnum)
)
_df.to_csv(pubpath, index=False, sep=';')
print('made {}'.format(pubpath))
import IPython; IPython.embed()
import numpy as np
from astroquery.vizier import Vizier
from astropy.coordinates import SkyCoord
import astropy.units as u
from astropy import table
import numpy as np
usnob_cat = Vizier.find_catalogs('USNO-B1.0')
def get_usnob1_cat(ra, dec, blim):
ra_u = ra*u.degree
dec_u = dec*u.degree
coords = SkyCoord(ra_u, dec_u, frame='icrs') #Should this be ICRS or FK5
#Only Class 0 (stars) - unable to implement this at the current time. Need to understand
#USNO-B1 s/g classification
v = Vizier(columns=['USNO-B1.0', '_RAJ2000', '_DEJ2000',
'B1mag', 'R1mag', 'B2mag', 'R2mag',
'pmRA', 'pmDE', 'Imag', 'B1s/g', '_r'],
row_limit=500000,
column_filters={"B2mag":">6", 'B2mag':'<{}'.format(blim)}) #B2mag fainter than 6, brighter than blim
new_table_list = v.query_region(coords,
radius=900*u.arcsecond, #Search 900 arcseconds
catalog = 'I/284')
if len(new_table_list) ==0:
return None
else:
new_table = new_table_list[0]
#Get the 5000 closest
new_table.sort('_r')
if len(new_table) > 5000:
new_table.remove_rows(np.arange(5001, len(new_table)))
import numpy as np
from astroquery.vizier import Vizier
from astropy.coordinates import SkyCoord
import astropy.units as u
from astropy import table
import astropy
usnob_cat = Vizier.find_catalogs('USNO-B1.0')
def get_usnob1_cat(ra, dec, blim,radius=15.0*u.arcmin):
"""
Assume that the ra,dec are given as float parameters in units of deg
if (type(ra) == float) & (type(dec) == float):
ra_u = ra*u.degree
dec_u = dec*u.degree
elif (type(ra) == np.float64) & (type(dec) == np.float64):
ra_u = ra*u.degree
dec_u = dec*u.degree
elif hasattr(ra,'to'):
ra_u = ra.to(u.deg)
dec_u = dec.to(u.deg)
else:
raise IOError(f"ra,dec not in correct format\n")
"""
ra_u = ra * u.deg
dec_u = dec * u.deg
coords = SkyCoord(ra_u, dec_u, frame='icrs') #Should this be ICRS or FK5
#Only Class 0 (stars) - unable to implement this at the current time. Need to understand
#USNO-B1 s/g classification
from astroquery.vizier import Vizier
from astropy import coordinates as coord
from astropy.coordinates import Angle
from astropy import units as u
catalog_list = Vizier.find_catalogs('Kang W51')
print({k: v.description for k, v in catalog_list.items()})
print(catalog_list)
catalogs = Vizier.get_catalogs(catalog_list.keys())
print(catalogs)
fovsec = 10 # field size in arcsec
fov = fovsec / 3600. # field size in deg
table = Vizier(columns=['all']).query_region(coord.SkyCoord(ra=str(r),
dec=str(d),
unit=(u.deg,
u.deg),
frame='icrs'),
Angle(fov, "deg"),
catalog=[link])[0]
import math
import time
import tarfile
import shutil
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
#import montage_wrapper as mt
from astropy.io import ascii
from astropy.table import Table, Column, vstack
from astropy import units as u
import astropy.coordinates as coords
from astroquery.vizier import Vizier
Vizier.ROW_LIMIT = -1
catalog_list = Vizier.find_catalogs('Csengeri 2014')
catalogs = Vizier.get_catalogs(catalog_list.keys())
atlasgalList = catalogs['J/A+A/565/A75/table1']
print len(atlasgalList)
atlasgalList.rename_column("RAJ2000", "ra")
atlasgalList.rename_column("DEJ2000", "dec")
coor = coords.SkyCoord(atlasgalList['ra'],
atlasgalList['dec'],
frame='icrs',
unit={u.deg, u.deg})
l = coor.galactic.l.deg
b = coor.galactic.b.deg
l = Column(l, name="l")
]
for j, target in enumerate(targets):
print '\n------------------' + target + '------------------\n'
results = Vizier.query_object(target)
for i, table in enumerate(results):
for head in possible_heads:
if head in table.keys():
string = results.keys()[i].encode('ascii', 'ignore')
split_str = string.split('/')
search_str = '/'.join(split_str[:-1])
cat = Vizier.find_catalogs(search_str)
cat_dict = ({k: v.description for k, v in cat.items()})
#print({k: v.description for k, v in cat.items()})
#print results[results.keys()[i]]
f = open(target + '.txt', 'a+')
f.write(string + '\n')
#f.write(cat_dict.keys()[0].encode('ascii', 'ignore')+'\n')
f.write(
cat_dict[cat_dict.keys()[0]].encode('ascii', 'ignore') +
'\n')
f.write(
tabulate(results[results.keys()[i]], headers=table.keys()))
f.write('\n\n\n')
f.close()
def bright_psource(self,
nbins): #should also have preprocessed data as input
#### SCOOP THE DATA FROM VIZIER CATALOG #####
'''
This method selects, from the GLEAM catalogue, sources
brighter than 100 mJy
'''
Vizier.ROW_LIMIT = -1
catalog_list = Vizier.find_catalogs('GLEAM')
catalogs = Vizier.get_catalogs(catalog_list.keys())
'''
--------------------------------------------------------------------------------
FileName Lrecl Records Explanations
--------------------------------------------------------------------------------
ReadMe 80 . This file
table1.dat 39 28 GLEAM first year observing parameters
gleamegc.dat 3155 307455 GLEAM EGC catalog, version 2
GLEAM_EGC_v2.fits 2880 137887 FITS version of the catalog
--------------------------------------------------------------------------------
'''
#We will extract version 2 catalogue
tt = catalogs['VIII/100/gleamegc']
#List all the keys
#Details of the keys are available here: http://cdsarc.u-strasbg.fr/ftp/cats/VIII/100/ReadMe
#And in more details here: https://heasarc.gsfc.nasa.gov/W3Browse/all/gleamegcat.html
src_name = tt['GLEAM'] #Source ID
RA = tt['RAJ2000'] #RA
DEC = tt['DEJ2000'] #DEC
flux = tt['Fpwide'] #Peak flux in wide (170-231MHz) image
###########UNIT CONVERSIONS ################################
#convert to equatorial coords (only DEC is changed)
DEC_co_lat = 90 - DEC
#convert to radians
RA_rads = RA * (np.pi / 180.)
DEC_rads = DEC_co_lat * (np.pi / 180.)
##########INITIAL CUT: Pick out only the sources bright enough to be seen at center of beam
psource_flux = []
psource_phi = []
psource_theta = []
for i in range(len(flux)):
if flux[i] >= 0.100:
psource_flux.append(flux[i])
psource_phi.append(RA_rads[i])
psource_theta.append(DEC_rads[i])
#convert to array for ease of use
psource_flux = np.asarray(psource_flux)
psource_phi = np.asarray(psource_phi)
psource_theta = np.asarray(psource_theta)
###### ORGANIZING THE DATA BY DIST TO CENTRE OF BEAM ############
co_lat = np.pi / 2. - self.latitude
#DEC distance to centre of the beam, may need to do actual distance...
dist_from_centre = np.abs(psource_theta - co_lat)
data = np.stack(
[psource_flux, psource_phi, psource_theta, dist_from_centre],
axis=1) # check axis stack
psource_data = data[np.argsort(
data[:, 3]
)] #sort by distance from centre of beam (becuase that way you do the brightest possiblesources first = less computing time)
########## COMPUTE PBEAM #########################
phis = (2. * np.pi * self.times) + self.position[0, 1]
primary = np.zeros((self.Nt, psource_data.shape[0]))
for i in range(self.Nt): #compute the elements of pbeam
primary[i] = np.exp(
-((psource_data[:, 1] - phis[i])**2 +
(psource_data[:, 2] - co_lat)**2) /
float(self.beam_width**2),
dtype="float64") # 2D gaussian beam (N_position,2)
#this primary beam should now be in order of closest to furthest
###########PICK OUT ALL THE BRIGHT BOISS ###################
bin_index = np.int(len(primary[1]) / nbins)
psource_final = []
for i in range(nbins):
#find max pbeam of all time
lower_bound = i * bin_index
upper_bound = (i + 1) * bin_index
maxes = []
for j in range(self.Nt):
maxes.append(max(primary[j, lower_bound:upper_bound]))
maxi = max(
maxes) #This is now the max pbeam you use to check the fluxes
for k in range(bin_index):
if psource_data[(
(i + 1) * k), 0] * maxi >= 0.100: #find the bright guys
psource_final.append(psource_data[(
(i + 1) * k), :3]) #append bright guys to final list
else:
continue
self.psource_final = np.asarray(psource_final)
print(self.psource_final)
self.compute_omega()
wavelength_fid = 1.525 #meters
temp_conv = (sc.c**2) / (2 * sc.k * ((self.freq_fid * 1e6)**2))
# temp_conv = (wavelength_fid**2)/(2*sc.k*self.omega_pix)
print(temp_conv)
self.psource_final[:, 0] = self.psource_final[:, 0] * temp_conv
'''
DATA STRUCTUE
--------------------------------------------------------
flux(K) phi(rads) theta(rads) dist_from_cent(rads)
--------------------------------------------------------
'''
print(self.psource_final)
# ################### CONVERT TO 3D ##########################
# z_coord = np.zeros((self.psource_final.shape[0],1))
# self.psource_final = np.concatenate((self.psource_final, z_coord), axis = 1)
# for i in range(self.psource_final.shape[0]):
# self.psource_final[i,1] = np.sin(self.psource_final[i,1])*np.cos(self.psource_final[i,2])#X
# self.psource_final[i,2] = np.sin(self.psource_final[i,1])*np.sin(self.psource_final[i,2])#Y
# self.psource_final[i,3] = np.cos(self.psource_final[i,1])#Z
return self.psource_final #IN KELVIN
# the luminosity is just not right...
# (this is not included in the paper because I'm worried about a factor of
# at least 4 pi being left out...)
sfr2cm = ((cont2cm*u.K).to(u.Jy, u.brightness_temperature(gbbeamarea,
h2co22freq)) *
#gb_ppbeam / gb_pixelsize_deg**2 * u.sr *
#(4*np.pi*distance**2) *
distance**2 * # there is a 4 pi missing here... but it may come from elsewhere
sfrperergshz).to(u.M_sun/u.yr)
sfr2cmd = (sfr2cm / pixel_area).to(u.M_sun/u.yr/u.kpc**2)
ok2cm = cont2cm > 0.1
from astroquery.vizier import Vizier
catalog_list = Vizier.find_catalogs('Kang W51')
ysos = Vizier(row_limit=1e6).query_region(coordinates.SkyCoord.from_name('W51'),
radius=1*u.deg,
catalog=catalog_list.keys())
cl1lt10 = ((ysos[0]['Cl1'] == 'I') |
(ysos[0]['Cl1'] == 'F') |
(ysos[0]['Cl2'] == 'I') |
(ysos[0]['Cl2'] == 'F')) & (ysos[0]['Mstar'] < 10)
cl1gt10 = ((ysos[0]['Cl1'] == 'I') |
(ysos[0]['Cl1'] == 'F') |
(ysos[0]['Cl2'] == 'I') |
(ysos[0]['Cl2'] == 'F')) & (ysos[0]['Mstar'] >= 10)
cl1lt10coords = coordinates.SkyCoord(ysos[0][cl1lt10]['_RAJ2000'],
ysos[0][cl1lt10]['_DEJ2000'],
frame='fk5').galactic
cl1gt10coords = coordinates.SkyCoord(ysos[0][cl1gt10]['_RAJ2000'],
from astroquery.vizier import Vizier
import requests
from bs4 import BeautifulSoup
from astroquery.simbad import Simbad
import numpy as np
from astropy.table import Table, vstack
from astropy.io import ascii
from astropy import units as u
from astropy.coordinates import SkyCoord, get_constellation
import pandas as pd
import kaggle
import glob, os
print('Downloading constellation_borders.csv...')
catalogue_list = Vizier.find_catalogs("Constellation")#This loads all the catalogs by the keyword "constellation"
Vizier.ROW_LIMIT = -1
catalog = Vizier.get_catalogs("VI/49")[1]#This selects the 2nd of three tables that were found by the key-word search. This table contains relevant data
catalog.remove_columns(['cst','type'])
coords = SkyCoord(catalog['RAJ2000'],catalog['DEJ2000'],unit="deg")
const = coords.get_constellation()
catalog.add_column(const,name="Constellation", index = 2)
catalog.write("constellation_borders.csv",format="csv",overwrite="True")
print('\n')
print('Done')
print('\n')
print('Downloading Messier Catalogue...')
#the next three lines are to simplify the output table
Simbad.reset_votable_fields()
Simbad.remove_votable_fields('coordinates')
def bright_psources(self,nbins): #should also be preprocessed
#### SCOOP THE DATA FROM VIZIER CATALOG #####
'''
This method selects, from the GLEAM catalogue, sources
brighter than 100 mJy
'''
Vizier.ROW_LIMIT = -1
catalog_list = Vizier.find_catalogs('GLEAM')
catalogs = Vizier.get_catalogs(catalog_list.keys())
'''
--------------------------------------------------------------------------------
FileName Lrecl Records Explanations
--------------------------------------------------------------------------------
ReadMe 80 . This file
table1.dat 39 28 GLEAM first year observing parameters
gleamegc.dat 3155 307455 GLEAM EGC catalog, version 2
GLEAM_EGC_v2.fits 2880 137887 FITS version of the catalog
--------------------------------------------------------------------------------
'''
#We will extract version 2 catalogue
tt = catalogs['VIII/100/gleamegc']
#List all the keys
#Details of the keys are available here: http://cdsarc.u-strasbg.fr/ftp/cats/VIII/100/ReadMe
#And in more details here: https://heasarc.gsfc.nasa.gov/W3Browse/all/gleamegcat.html
src_name = tt['GLEAM'] #Source ID
RA = tt['RAJ2000'] #RA
DEC = tt['DEJ2000'] #DEC
flux = tt['Fpwide'] #Peak flux in wide (170-231MHz) image
###########UNIT CONVERSIONS ################################
#convert to equatorial coords (only DEC is changed)
DEC_co_lat = 90-DEC
#convert to radians
RA_rads = RA*(np.pi/180.)
DEC_rads = DEC_co_lat*(np.pi/180.)
##########INITIAL CUT: Pick out only the sources bright enough to be seen at center of beam
psource_flux = []
psource_phi = []
psource_theta = []
for i in range(len(flux)):
if flux[i] >= 0.100:
psource_flux.append(flux[i])
psource_phi.append(RA_rads[i])
psource_theta.append(DEC_rads[i])
#convert to array for ease of use
psource_flux = np.asarray(psource_flux)
psource_phi = np.asarray(psource_phi)
psource_theta = np.asarray(psource_theta)
###### ORGANIZING THE DATA BY DIST TO CENTRE OF BEAM ############ I think that this is unecessary
co_lat = np.pi / 2. - self.latitude
#DEC distance to centre of the beam, may need to do actual distance...
dist_from_centre = np.abs(psource_theta-co_lat)
data = np.stack([psource_flux,psource_phi,psource_theta,dist_from_centre], axis = 1) # check axis stack
psource_data = data[np.argsort(data[:, 3])] #sort by distance from centre of beam (becuase that way you do the brightest possiblesources first = less computing time)
########## COMPUTE PBEAM #########################
phis = (2. * np.pi * self.times) + self.position[0,1]
primary = np.zeros((self.Nt, psource_data.shape[0])) #Nt * n_sources
for i in range(self.Nt): #compute the elements of pbeam
primary[i] = np.exp(-((psource_data[:,1]-phis[i])**2 +(psource_data[:,2]-co_lat)**2)/float(self.beam_width**2), dtype = "float64")# 2D gaussian beam (N_position,2)
#this primary beam should now be in order of closest to furthest
########### PICK OUT ALL THE BRIGHT BOISS ###################
# ## REDO THIS SECTION
# bin_index = np.int(len(primary[1])/nbins)
# psource_final = []
# psource_pbeam = []
# for i in range(nbins):
# #find max pbeam of all time
# lower_bound = i*bin_index
# upper_bound = (i+1)*bin_index
# maxes = []
# for j in range(self.Nt):
# maxes.append(max(primary[j,lower_bound:upper_bound]))
# maxi = max(maxes) #This is now the max pbeam you use to check the fluxes
# for k in range(bin_index):
# if psource_data[((i+1)*k),0]* maxi >= 0.100: #find the bright guys
# psource_final.append(psource_data[((i+1)*k),:3]) #append bright guys' to final list
# psource_pbeam.append(primary[:,((i+1)*k)]) # also make a list with that guys' primary beams
# else:
# continue
psource_final = []
psource_pbeam = []
#this will replace the binned selection technique
#here we go through each source and check if it is bright enough to be seen even for 1 time, if so, append to list
for i in range(psource_data.shape[0]):
max_primary = max(primary[:,i]) #find the max primary value of all time for that source
if psource_data[i,0]*max_primary >= 0.100: #if max primary doesn't dampen flux below threshold
psource_final.append(psource_data[i,:3]) #append bright guys' to final list
psource_pbeam.append(primary[:,i]) # also make a list with that guys' primary beams
else:
continue
self.psource_final = np.asarray(psource_final)
self.psource_pbeam = np.asarray(psource_pbeam)
self.psource_pbeam = self.psource_pbeam.T
