def Pbdot_Gal(pf):
try:
name = pf.PSRJ
except:
name = pf.PSR
ra = RA(pf.RAJ[0])
dec = Dec(pf.DECJ[0])
#pos = coord.FK5Coordinates(str(ra) +' '+ str(dec))
pos = coord(str(ra), str(dec), unit=(u.deg, u.hourangle) )
l = pos.galactic.l.radian
b = pos.galactic.b.radian
Pb = float(pf.PB[0] * secperday)
#d = AU * 180 / PI * 3600 / pf.PX[0] * 1000
d = float(1/pf.PX[0])
pf.DIST = d
z_kpc = float(d)*(abs(sin(b)))#/kpc must be positive
a_z = ((2.27)*z_kpc + (3.68)*(1 - exp((-4.31)*z_kpc)))*(1.e-9) #cm s^-2
#print 'a_z:', a_z
A_z = -1 * a_z *abs(sin(b))/c
pf.A_z = A_z
R0 = (8.34) #* kpc # Reid et al. 2014
beta = float(d/R0) * cos(b) - cos(l)
Omega0 = (240. * 1.e5) #240+/-8 km/s; Reid et al 2014
#print b,l, cos(b), cos(l), beta
A_x = -1/c * (cos(b)) * (Omega0**2/R0/kpc) * (cos(l) + beta/(sin(l)**2 + beta**2))
pf.A_x = A_x
#print 'Ax, Az: ',A_x, A_z
fac1 = float(pf.PX[1]/pf.PX[0])
fac2 = 8/240 #Omega_0
fac3 = 0.16/8.34 #R0 Reid et al 2014
val = float(Pb*(A_z + A_x))
err1 = A_x * fac1
err2 = A_z * sqrt(fac1**2 + fac2**2 + fac3**2)
err = sqrt(err1**2 + err2**2) * float(Pb)
return val, err
def radecs_to_lb(ras, decs):
"""
Transformation between lists of ras, decs, to ls, bs. Assumes ra, dec in degrees
Conforms to astropy 0.4.3
taken from https://github.com/seclark/FITSHandling/commit/f04a6e54c6624741e4f3077ba8ba96af620871ac
for lb masks
"""
obj = coord(ras, decs, unit="deg", frame="icrs")
obj = obj.galactic
ls = obj.l.degree
bs = obj.b.degree
return ls, bs
def do_ppmxl(catalog):
task_str = catalog.get_current_task_str()
keys = list(catalog.entries.keys())
warnings.filterwarnings("ignore")
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
try:
name = catalog.add_entry(oname)
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
if (FASTSTARS.RA not in catalog.entries[name] or
FASTSTARS.DEC not in catalog.entries[name]):
continue
else:
radec= str(catalog.entries[name][FASTSTARS.RA][0]['value'])+str(catalog.entries[name][FASTSTARS.DEC][0]['value'])
c=coord(radec,unit=(un.hourangle, un.deg),frame='icrs')
cnttry = 0
foundstar = False
while foundstar == False and cnttry < 1:
try:
cnttry += 1
time.sleep(0.1)
result = Irsa.query_region(c,catalog='ppmxl',radius='0d0m10s')
except TypeError:
#print(radec,cnttry)
continue
if len(result) > 1:
foundstar = True
if foundstar == True:
source = (catalog.entries[name]
.add_source(name='The PPMXL Catalog',
bibcode="2010AJ....139.2440R",
url="https://irsa.ipac.caltech.edu/Missions/ppmxl.html",
secondary=True))
catalog.entries[name].add_quantity(FASTSTARS.PROPER_MOTION_RA, str(result['pmra'][0]*degperyrtomasperyear), source, e_value=str(result['e_pmra'][0]*degperyrtomasperyear), u_value='mas/yr')
catalog.entries[name].add_quantity(FASTSTARS.PROPER_MOTION_DEC, str(result['pmde'][0]*degperyrtomasperyear), source, e_value=str(result['e_pmde'][0]*degperyrtomasperyear), u_value='mas/yr')
catalog.journal_entries()
return
def do_ptss_meta(catalog):
"""Import metadata from PTSS webpage."""
task_str = catalog.get_current_task_str()
years = list(range(2015, datetime.today().year + 1))
for year in years:
jsontxt = None
while jsontxt is None:
try:
jsontxt = catalog.load_url(
'http://www.cneost.org/ptss/fetchlist.php?vip=sn&gdate=' +
str(year),
os.path.join(catalog.get_current_task_repo(),
'PTSS/catalog-' + str(year) + '.json'),
json_sort='name',
timeout=5)
except Exception:
pass
meta = json.loads(jsontxt)
for met in pbar(meta, task_str + ' - ' + str(year)):
oldname = met['name']
name, source = catalog.new_entry(
oldname,
srcname='PMO & Tsinghua Supernova Survey (PTSS)',
url='http://www.cneost.org/ptss/index.php')
coo = coord(met['ra'], met['dec'], unit=(un.deg, un.deg))
catalog.entries[name].add_quantity(
SUPERNOVA.RA, coo.ra.to_string(unit=un.hour, sep=':'), source)
catalog.entries[name].add_quantity(
SUPERNOVA.DEC, coo.dec.to_string(unit=un.degree, sep=':'),
source)
if met['filter'] is not None:
mjd = str(astrotime(met['obsdate'], format='isot').mjd)
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.MAGNITUDE: str(met['mag']),
PHOTOMETRY.E_MAGNITUDE: str(met['magerr']),
PHOTOMETRY.BAND: met['filter'].replace('sdss-', ''),
PHOTOMETRY.SOURCE: source
}
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
return
def load_cats():
"""Reload the catalog dictionaries."""
logger.info('Loading catalog...')
for cat in apidata._CATS:
apidata._catalogs[cat] = json.load(open(
os.path.join(apidata._AC_PATH, apidata._CATS[cat][0], 'output',
apidata._CATS[cat][1]), 'r'),
object_pairs_hook=OrderedDict)
# Add some API-specific fields to each catalog.
for i, x in enumerate(apidata._catalogs[cat]):
apidata._catalogs[cat][i]['catalog'] = cat
apidata._catalogs[cat] = OrderedDict(
sorted(dict(
zip([x['name'] for x in apidata._catalogs[cat]],
apidata._catalogs[cat])).items(),
key=lambda s: (s[0].upper(), s[0])))
if cat not in apidata._extras:
apidata._extras[cat] = OrderedDict()
logger.info('Creating alias dictionary and position arrays...')
apidata._rdnames = []
apidata._ras = []
apidata._decs = []
apidata._all = []
# Load object apidata._catalogs.
for cat in apidata._CATS:
apidata._cat_keys[cat] = set()
for event in apidata._catalogs[cat]:
add_event(cat, event, convert_coords=False)
apidata._all = list(sorted(set(apidata._all), key=lambda s:
(s.upper(), s)))
apidata._coo = coord(apidata._ras,
apidata._decs,
unit=(un.hourangle, un.deg))
del (apidata._ras, apidata._decs)
# Re-append events that were added later.
for cat in apidata._extras:
for event in apidata._extras[cat]:
if event not in apidata._catalogs[cat]:
apidata._catalogs[cat][event] = apidata._extras[cat][event]
add_event(cat, event)
def do_ptss_meta(catalog):
"""Import metadata from PTSS webpage."""
task_str = catalog.get_current_task_str()
years = list(range(2015, datetime.today().year + 1))
for year in years:
jsontxt = None
while jsontxt is None:
try:
jsontxt = catalog.load_url(
'http://www.cneost.org/ptss/fetchlist.php?vip=sn&gdate=' +
str(year),
os.path.join(catalog.get_current_task_repo(),
'PTSS/catalog-' + str(year) + '.json'),
json_sort='name', timeout=5)
except Exception:
pass
meta = json.loads(jsontxt)
for met in pbar(meta, task_str + ' - ' + str(year)):
oldname = met['name']
name, source = catalog.new_entry(
oldname, srcname='PMO & Tsinghua Supernova Survey (PTSS)',
url='http://www.cneost.org/ptss/index.php')
coo = coord(met['ra'], met['dec'], unit=(un.deg, un.deg))
catalog.entries[name].add_quantity(
SUPERNOVA.RA, coo.ra.to_string(unit=un.hour, sep=':'), source)
catalog.entries[name].add_quantity(
SUPERNOVA.DEC, coo.dec.to_string(unit=un.degree, sep=':'),
source)
if met['filter'] is not None:
mjd = str(astrotime(met['obsdate'], format='isot').mjd)
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.MAGNITUDE: str(met['mag']),
PHOTOMETRY.E_MAGNITUDE: str(met['magerr']),
PHOTOMETRY.BAND: met['filter'].replace('sdss-', ''),
PHOTOMETRY.SOURCE: source
}
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
return
def do_dust(catalog):
task_str = catalog.get_current_task_str()
# Set preferred names, calculate some columns based on imported data,
# sanitize some fields
keys = list(catalog.entries.keys())
check_dustmaps(catalog.get_current_task_repo())
sfd = SFDQuery()
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
try:
name = catalog.add_entry(oname)
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
if (FASTSTARS.RA not in catalog.entries[name]
or FASTSTARS.DEC not in catalog.entries[name]):
continue
else:
Mname = name
Mradec = str(
catalog.entries[name][FASTSTARS.RA][0]['value']) + str(
catalog.entries[name][FASTSTARS.DEC][0]['value'])
Mdist = '-1'
c = coord(Mradec, unit=(un.hourangle, un.deg), frame='icrs')
reddening = sfd(c)
source = catalog.entries[name].add_source(
bibcode='1998ApJ...500..525S')
catalog.entries[name].add_quantity(FASTSTARS.EBV,
str(reddening),
source,
upperlimit=True,
derived=True)
catalog.journal_entries()
return
def lbs_to_radecs(ls, bs, remin=False):
"""
Transformation between lists of ls, bs, to ras, decs. Assumes all in degrees
Conforms to astropy 0.4.3
taken from https://github.com/seclark/FITSHandling/commit/f04a6e54c6624741e4f3077ba8ba96af620871ac
"""
assert len(ls) == len(bs)
obj = coord(ls, bs, unit="deg", frame="galactic")
obj = obj.icrs
ras = obj.ra.degree
decs = obj.dec.degree
if remin:
ra_min = np.argmin(ras)
ras = np.hstack((ras[ra_min:], ras[:ra_min]))
decs = np.hstack((decs[ra_min:], decs[:ra_min]))
return ras, decs
return ras, decs
def add_event(cat, event, convert_coords=True):
"""Add event to global arrays."""
apidata._all.append(event)
apidata._cat_keys[cat].update(list(apidata._catalogs[cat][event].keys()))
levent = apidata._catalogs[cat].get(event, {})
laliases = levent.get('alias', [])
lev = event.lower()
laliases = list(
set([
lev,
replace_multiple(lev, ['sn', 'at']) if lev.startswith(
('sn', 'at')) else lev
] + [x['value'].lower() for x in laliases] + [
replace_multiple(x['value'].lower(), ['sn', 'at'])
for x in laliases if x['value'].lower().startswith(('sn', 'at'))
]))
for alias in laliases:
apidata._aliases.setdefault(alias.lower().replace(' ', ''),
[]).append([cat, event, alias])
apidata._all_aliases.add(alias)
lra = levent.get('ra')
ldec = levent.get('dec')
if lra is None and ldec is None:
return
lra = lra[0].get('value')
ldec = ldec[0].get('value')
if lra is None or ldec is None:
return
if not raregex.match(lra) or not decregex.match(ldec):
return
apidata._rdnames.append(event)
if convert_coords:
apidata._coo = coord_concat(
(apidata._coo, coord(lra, ldec, unit=(un.hourangle, un.deg))))
else:
apidata._ras.append(lra)
apidata._decs.append(ldec)
def do_cleanup(catalog):
"""Task to cleanup catalog before final write."""
task_str = catalog.get_current_task_str()
# Set preferred names, calculate some columns based on imported data,
# sanitize some fields
keys = catalog.entries.copy().keys()
cleanupcnt = 0
for oname in pbar(keys, task_str):
name = catalog.add_entry(oname)
# Set the preferred name, switching to that name if name changed.
name = catalog.entries[name].set_preferred_name()
aliases = catalog.entries[name].get_aliases()
catalog.entries[name].set_first_max_light()
if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['MLS', 'SSS', 'CSS', 'GRB ']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix) and
is_number(alias.replace(prefix, '')[:2])):
discoverdate = ('/'.join([
'20' + alias.replace(prefix, '')[:2],
alias.replace(prefix, '')[2:4],
alias.replace(prefix, '')[4:6]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]:
break
if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]:
prefixes = [
'ASASSN-', 'PS1-', 'PS1', 'PS', 'iPTF', 'PTF', 'SCP-', 'SNLS-',
'SPIRITS', 'LSQ', 'DES', 'SNHiTS', 'Gaia', 'GND', 'GNW', 'GSD',
'GSW', 'EGS', 'COS', 'OGLE', 'HST'
]
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix) and
is_number(alias.replace(prefix, '')[:2]) and
is_number(alias.replace(prefix, '')[:1])):
discoverdate = '20' + alias.replace(prefix, '')[:2]
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]:
break
if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['SNF']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix) and
is_number(alias.replace(prefix, '')[:4])):
discoverdate = ('/'.join([
alias.replace(prefix, '')[:4],
alias.replace(prefix, '')[4:6],
alias.replace(prefix, '')[6:8]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]:
break
if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['PTFS', 'SNSDF']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix) and
is_number(alias.replace(prefix, '')[:2])):
discoverdate = ('/'.join([
'20' + alias.replace(prefix, '')[:2],
alias.replace(prefix, '')[2:4]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]:
break
if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['AT', 'SN', 'OGLE-', 'SM ', 'KSN-']
for alias in aliases:
for prefix in prefixes:
if alias.startswith(prefix):
year = re.findall(r'\d+', alias)
if len(year) == 1:
year = year[0]
else:
continue
if alias.replace(prefix, '').index(year) != 0:
continue
if (year and is_number(year) and '.' not in year and
len(year) <= 4):
discoverdate = year
if catalog.args.verbose:
tprint('Added discoverdate from name [' +
alias + ']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]:
break
if (TIDALDISRUPTION.RA not in catalog.entries[name] or
TIDALDISRUPTION.DEC not in catalog.entries[name]):
prefixes = [
'PSN J', 'MASJ', 'CSS', 'SSS', 'MASTER OT J', 'HST J', 'TCP J',
'MACS J', '2MASS J', 'EQ J', 'CRTS J', 'SMT J'
]
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix) and
is_number(alias.replace(prefix, '')[:6])):
noprefix = alias.split(':')[-1].replace(
prefix, '').replace('.', '')
decsign = '+' if '+' in noprefix else '-'
noprefix = noprefix.replace('+', '|').replace('-', '|')
nops = noprefix.split('|')
if len(nops) < 2:
continue
rastr = nops[0]
decstr = nops[1]
ra = ':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + \
('.' + rastr[6:] if len(rastr) > 6 else '')
dec = (decsign + ':'.join(
[decstr[:2], decstr[2:4], decstr[4:6]]) +
('.' + decstr[6:] if len(decstr) > 6 else ''))
if catalog.args.verbose:
tprint('Added ra/dec from name: ' + ra + ' ' + dec)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.RA, ra, source, derived=True)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DEC, dec, source, derived=True)
break
if TIDALDISRUPTION.RA in catalog.entries[name]:
break
no_host = (TIDALDISRUPTION.HOST not in catalog.entries[name] or
not any([
x[QUANTITY.VALUE] == 'Milky Way'
for x in catalog.entries[name][TIDALDISRUPTION.HOST]
]))
if (TIDALDISRUPTION.RA in catalog.entries[name] and
TIDALDISRUPTION.DEC in catalog.entries[name] and no_host):
from astroquery.irsa_dust import IrsaDust
if name not in catalog.extinctions_dict:
try:
ra_dec = (catalog.entries[name][TIDALDISRUPTION.RA][0][
QUANTITY.VALUE] + " " + catalog.entries[name][
TIDALDISRUPTION.DEC][0][QUANTITY.VALUE])
result = IrsaDust.get_query_table(ra_dec, section='ebv')
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
warnings.warn("Coordinate lookup for " + name +
" failed in IRSA.")
else:
ebv = result['ext SandF mean'][0]
ebverr = result['ext SandF std'][0]
catalog.extinctions_dict[name] = [ebv, ebverr]
if name in catalog.extinctions_dict:
sources = uniq_cdl([
catalog.entries[name].add_self_source(),
catalog.entries[name]
.add_source(bibcode='2011ApJ...737..103S')
])
(catalog.entries[name].add_quantity(
TIDALDISRUPTION.EBV,
str(catalog.extinctions_dict[name][0]),
sources,
e_value=str(catalog.extinctions_dict[name][1]),
derived=True))
if ((TIDALDISRUPTION.HOST in catalog.entries[name] and
(TIDALDISRUPTION.HOST_RA not in catalog.entries[name] or
TIDALDISRUPTION.HOST_DEC not in catalog.entries[name]))):
for host in catalog.entries[name][TIDALDISRUPTION.HOST]:
alias = host[QUANTITY.VALUE]
if ' J' in alias and is_number(alias.split(' J')[-1][:6]):
noprefix = alias.split(' J')[-1].split(':')[-1].replace(
'.', '')
decsign = '+' if '+' in noprefix else '-'
noprefix = noprefix.replace('+', '|').replace('-', '|')
nops = noprefix.split('|')
if len(nops) < 2:
continue
rastr = nops[0]
decstr = nops[1]
hostra = (':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) +
('.' + rastr[6:] if len(rastr) > 6 else ''))
hostdec = decsign + ':'.join([
decstr[:2], decstr[2:4], decstr[4:6]
]) + ('.' + decstr[6:] if len(decstr) > 6 else '')
if catalog.args.verbose:
tprint('Added hostra/hostdec from name: ' + hostra +
' ' + hostdec)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.HOST_RA, hostra, source, derived=True)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.HOST_DEC,
hostdec,
source,
derived=True)
break
if TIDALDISRUPTION.HOST_RA in catalog.entries[name]:
break
if (TIDALDISRUPTION.REDSHIFT not in catalog.entries[name] and
TIDALDISRUPTION.VELOCITY in catalog.entries[name]):
# Find the "best" velocity to use for this
bestsig = 0
for hv in catalog.entries[name][TIDALDISRUPTION.VELOCITY]:
sig = get_sig_digits(hv[QUANTITY.VALUE])
if sig > bestsig:
besthv = hv[QUANTITY.VALUE]
bestsrc = hv['source']
bestsig = sig
if bestsig > 0 and is_number(besthv):
voc = float(besthv) * 1.e5 / CLIGHT
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
(catalog.entries[name].add_quantity(
TIDALDISRUPTION.REDSHIFT,
pretty_num(
sqrt((1. + voc) / (1. - voc)) - 1., sig=bestsig),
sources,
kind='heliocentric',
derived=True))
if (TIDALDISRUPTION.REDSHIFT not in catalog.entries[name] and
len(catalog.nedd_dict) > 0 and
TIDALDISRUPTION.HOST in catalog.entries[name]):
reference = "NED-D"
refurl = "http://ned.ipac.caltech.edu/Library/Distances/"
for host in catalog.entries[name][TIDALDISRUPTION.HOST]:
if host[QUANTITY.VALUE] in catalog.nedd_dict:
source = catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
secondarysource = catalog.entries[name].add_source(
name=reference, url=refurl, secondary=True)
meddist = statistics.median(catalog.nedd_dict[host[
QUANTITY.VALUE]])
redz = z_at_value(cosmo.comoving_distance,
float(meddist) * un.Mpc)
redshift = pretty_num(
redz, sig=get_sig_digits(str(meddist)))
catalog.entries[name].add_quantity(
TIDALDISRUPTION.REDSHIFT,
redshift,
uniq_cdl([source, secondarysource]),
kind='host',
derived=True)
if (TIDALDISRUPTION.MAX_ABS_MAG not in catalog.entries[name] and
TIDALDISRUPTION.MAX_APP_MAG in catalog.entries[name] and
TIDALDISRUPTION.LUM_DIST in catalog.entries[name]):
# Find the "best" distance to use for this
bestsig = 0
for ld in catalog.entries[name][TIDALDISRUPTION.LUM_DIST]:
sig = get_sig_digits(ld[QUANTITY.VALUE])
if sig > bestsig:
bestld = ld[QUANTITY.VALUE]
bestsrc = ld['source']
bestsig = sig
if bestsig > 0 and is_number(bestld) and float(bestld) > 0.:
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
bestldz = z_at_value(cosmo.luminosity_distance,
float(bestld) * un.Mpc)
pnum = (float(catalog.entries[name][
TIDALDISRUPTION.MAX_APP_MAG][0][QUANTITY.VALUE]) - 5.0 *
(log10(float(bestld) * 1.0e6) - 1.0
) + 2.5 * log10(1.0 + bestldz))
pnum = pretty_num(pnum, sig=bestsig)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.MAX_ABS_MAG, pnum, sources, derived=True)
if TIDALDISRUPTION.REDSHIFT in catalog.entries[name]:
# Find the "best" redshift to use for this
bestz, bestkind, bestsig, bestsrc = catalog.entries[
name].get_best_redshift()
if bestsig > 0:
try:
bestz = float(bestz)
except Exception:
print(catalog.entries[name])
raise
if TIDALDISRUPTION.VELOCITY not in catalog.entries[name]:
source = catalog.entries[name].add_self_source()
# FIX: what's happening here?!
pnum = CLIGHT / KM * \
((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.)
pnum = pretty_num(pnum, sig=bestsig)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.VELOCITY,
pnum,
source,
kind=PREF_KINDS[bestkind],
derived=True)
if bestz > 0.:
from astropy.cosmology import Planck15 as cosmo
if TIDALDISRUPTION.LUM_DIST not in catalog.entries[name]:
dl = cosmo.luminosity_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name]
.add_source(bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
TIDALDISRUPTION.LUM_DIST,
pretty_num(
dl.value, sig=bestsig),
sources,
kind=PREF_KINDS[bestkind],
derived=True)
if (TIDALDISRUPTION.MAX_ABS_MAG not in
catalog.entries[name] and
TIDALDISRUPTION.MAX_APP_MAG in
catalog.entries[name]):
source = catalog.entries[name].add_self_source()
pnum = pretty_num(
float(catalog.entries[name][
TIDALDISRUPTION.MAX_APP_MAG][0][
QUANTITY.VALUE]) - 5.0 *
(log10(dl.to('pc').value) - 1.0
) + 2.5 * log10(1.0 + bestz),
sig=bestsig + 1)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.MAX_ABS_MAG,
pnum,
sources,
derived=True)
if TIDALDISRUPTION.COMOVING_DIST not in catalog.entries[
name]:
cd = cosmo.comoving_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name]
.add_source(bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
TIDALDISRUPTION.COMOVING_DIST,
pretty_num(
cd.value, sig=bestsig),
sources,
derived=True)
if all([
x in catalog.entries[name]
for x in [
TIDALDISRUPTION.RA, TIDALDISRUPTION.DEC,
TIDALDISRUPTION.HOST_RA, TIDALDISRUPTION.HOST_DEC
]
]):
# For now just using first coordinates that appear in entry
try:
c1 = coord(
ra=catalog.entries[name][TIDALDISRUPTION.RA][0][
QUANTITY.VALUE],
dec=catalog.entries[name][TIDALDISRUPTION.DEC][0][
QUANTITY.VALUE],
unit=(un.hourangle, un.deg))
c2 = coord(
ra=catalog.entries[name][TIDALDISRUPTION.HOST_RA][0][
QUANTITY.VALUE],
dec=catalog.entries[name][TIDALDISRUPTION.HOST_DEC][0][
QUANTITY.VALUE],
unit=(un.hourangle, un.deg))
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
pass
else:
sources = uniq_cdl(
[catalog.entries[name].add_self_source()] + catalog.
entries[name][TIDALDISRUPTION.RA][0]['source'].split(',') +
catalog.entries[name][TIDALDISRUPTION.DEC][0]['source'].
split(',') + catalog.entries[name][TIDALDISRUPTION.HOST_RA]
[0]['source'].split(',') + catalog.entries[name][
TIDALDISRUPTION.HOST_DEC][0]['source'].split(','))
if 'hostoffsetang' not in catalog.entries[name]:
hosa = Decimal(
hypot(c1.ra.degree - c2.ra.degree, c1.dec.degree -
c2.dec.degree))
hosa = pretty_num(hosa * Decimal(3600.))
catalog.entries[name].add_quantity(
TIDALDISRUPTION.HOST_OFFSET_ANG,
hosa,
sources,
derived=True,
u_value='arcseconds')
if (TIDALDISRUPTION.COMOVING_DIST in catalog.entries[name] and
TIDALDISRUPTION.REDSHIFT in catalog.entries[name] and
TIDALDISRUPTION.HOST_OFFSET_DIST not in
catalog.entries[name]):
offsetsig = get_sig_digits(catalog.entries[name][
TIDALDISRUPTION.HOST_OFFSET_ANG][0][QUANTITY.VALUE])
sources = uniq_cdl(
sources.split(',') + (catalog.entries[name][
TIDALDISRUPTION.COMOVING_DIST][0]['source']).
split(',') + (catalog.entries[name][
TIDALDISRUPTION.REDSHIFT][0]['source']).split(','))
(catalog.entries[name].add_quantity(
TIDALDISRUPTION.HOST_OFFSET_DIST,
pretty_num(
float(catalog.entries[name][
TIDALDISRUPTION.HOST_OFFSET_ANG][0][
QUANTITY.VALUE]) / 3600. * (pi / 180.) *
float(catalog.entries[name][
TIDALDISRUPTION.COMOVING_DIST][0][
QUANTITY.VALUE]) * 1000. /
(1.0 + float(catalog.entries[name][
TIDALDISRUPTION.REDSHIFT][0][QUANTITY.VALUE])),
sig=offsetsig),
sources))
catalog.entries[name].sanitize()
catalog.journal_entries(bury=True, final=True, gz=True)
cleanupcnt = cleanupcnt + 1
if catalog.args.travis and cleanupcnt % 1000 == 0:
break
catalog.save_caches()
return
newitem = OrderedDict()
if 'redshift' in item and 'host' not in item and 'ra' in item and 'dec' in item and item['ra'] and item['dec']:
#if 'ra' in item and 'dec' in item and item['ra'] and item['dec']:
newitem['name'] = item['name']
newitem['alias'] = [x['value'] for x in item['alias']]
newitem['ra'] = item['ra'][0]['value']
newitem['dec'] = item['dec'][0]['value']
if 'redshift' in item:
newitem['redshift'] = item['redshift'][0]['value']
# Temporary fix for David's typo
if newitem['dec'].count('.') == 2:
newitem['dec'] = newitem['dec'][:newitem['dec'].rfind('.')]
newcatalog.append(newitem)
coo = coord([x['ra'] for x in newcatalog], [x['dec'] for x in newcatalog], unit = (un.hourangle, un.deg))
for ci, co in enumerate(tqdm(coo)):
customSimbad = Simbad()
customSimbad.add_votable_fields('otype', 'z_value')
regstr = 'region(ICRS, ' + co.to_string('hmsdms') + ', 1m)'
print(regstr)
result_table = customSimbad.query_criteria(regstr, otype='Galaxy')
if result_table:
print(newcatalog[ci])
if 'redshift' in newcatalog[ci]:
print(newcatalog[ci]['redshift'])
print(result_table)
# Convert to array since that's what datatables expects
thistype = ct.replace('?', '').replace('*', '')
if thistype.upper() in nonsnetypes:
continue
elif thistype in ('Other', 'not Ia', 'SN', 'unconf', 'Radio',
'CC', 'CCSN', 'Candidate', 'nIa'):
sntypes.append('Unknown')
break
else:
sntypes.append(thistype)
break
else:
sntypes.append('Unknown')
tprint(thisevent['name'])
try:
c = coord(ra=thisevent['ra'][0]['value'], dec=thisevent[
'dec'][0]['value'], unit=(un.hourangle, un.deg))
except (KeyboardInterrupt, SystemExit):
raise
except:
warnings.warn('Mangled coordinate, skipping')
continue
else:
snnames.append(thisevent['name'])
rarad = c.ra.radian - pi
decrad = c.dec.radian
snhx = 2.0**1.5 * cos(decrad) * sin(rarad / 2.0) / \
sqrt(1.0 + cos(decrad) * cos(rarad / 2.0))
snhy = sqrt(2.0) * sin(decrad) / \
sqrt(1.0 + cos(decrad) * cos(rarad / 2.0))
snras.append(c.ra.deg)
sndecs.append(c.dec.deg)
if not is_number(newitem['ra'].split(':')[0]):
continue
newitem['dec'] = item['dec'][0]['value']
if not is_number(newitem['dec'].split(':')[0]):
continue
newitem['raerr'] = float(item['ra'][0].get('e_value', 0))
newitem['decerr'] = float(item['dec'][0].get('e_value', 0))
# Temporary fix for David's typo
if newitem['dec'].count('.') == 2:
newitem['dec'] = newitem['dec'][:newitem['dec'].rfind('.')]
if 'distinctfrom' in item:
newitem['distinctfrom'] = [x['value']
for x in item['distinctfrom']]
newcatalog.append(newitem)
coo = coord([x['ra'] for x in newcatalog],
[x['dec'] for x in newcatalog], unit=(un.hourangle, un.deg))
response = urllib.request.urlopen('http://www.astronomy.swin.edu.au/pulsar/frbcat/table.php?format=text&sep=comma')
frbtxt = response.read().decode('utf-8').splitlines()
with open(utmost, 'r') as f:
utmosttable = f.read().splitlines()
frbcatalog = []
frbnames = []
utmostcoords = {}
for row in utmosttable:
if row[0] == '#':
frbname = row[1:].strip()
utmostcoords[frbname] = []
def do_dust(catalog):
task_str = catalog.get_current_task_str()
# Set preferred names, calculate some columns based on imported data,
# sanitize some fields
keys = list(catalog.entries.keys())
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
try:
name = catalog.add_entry(oname)
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
if (FASTSTARS.RA not in catalog.entries[name]
or FASTSTARS.DEC not in catalog.entries[name]):
continue
elif (FASTSTARS.LUM_DIST not in catalog.entries[name]):
Mname = name
Mradec = str(
catalog.entries[name][FASTSTARS.RA][0]['value']) + str(
catalog.entries[name][FASTSTARS.DEC][0]['value'])
Mdist = '-1'
c = coord(Mradec, unit=(un.hourangle, un.deg), frame='icrs')
reddening = sfd(c)
source = catalog.entries[name].add_source(
bibcode='1998ApJ...500..525S')
catalog.entries[name].add_quantity(FASTSTARS.EBV,
str(reddening),
source,
upperlimit=True,
derived=True)
else:
Mname = name
Mradec = str(
catalog.entries[name][FASTSTARS.RA][0]['value']) + str(
catalog.entries[name][FASTSTARS.DEC][0]['value'])
Mdist = str(catalog.entries[name][FASTSTARS.LUM_DIST][0]['value'])
Mdist_u = str(
catalog.entries[name][FASTSTARS.LUM_DIST][0]['u_value'])
if Mdist_u == 'kpc':
c = coord(Mradec,
unit=(un.hourangle, un.deg),
distance=float(Mdist) * un.kpc,
frame='icrs')
elif Mdist_u == 'pc':
c = coord(Mradec,
unit=(un.hourangle, un.deg),
distance=float(Mdist) * un.pc,
frame='icrs')
else:
catalog.log.warning(
'"{}" has a distance but not an attached unit.'.format(
oname))
continue
reddening = bayestar(c, mode='median')
# If that reddening is a nan (ie. the star lies outside PanSTARRS) switch to SFD
if np.isnan(reddening) == True:
c = coord(Mradec, unit=(un.hourangle, un.deg), frame='icrs')
reddening = sfd(c)
source = catalog.entries[name].add_source(
bibcode='1998ApJ...500..525S')
catalog.entries[name].add_quantity(FASTSTARS.EBV,
str(reddening),
source,
upperlimit=True,
derived=True)
else:
source = catalog.entries[name].add_source(
bibcode='2018arXiv180103555G')
catalog.entries[name].add_quantity(FASTSTARS.EBV,
str(reddening),
source,
derived=True)
catalog.journal_entries()
return
continue
newitem['dec'] = item['dec'][0]['value']
if not is_number(newitem['dec'].split(':')[0]):
continue
newitem['raerr'] = float(item['ra'][0].get('e_value', 0))
newitem['decerr'] = float(item['dec'][0].get('e_value', 0))
# Temporary fix for David's typo
if newitem['dec'].count('.') == 2:
newitem['dec'] = newitem['dec'][:newitem['dec'].rfind('.')]
if 'distinctfrom' in item:
newitem['distinctfrom'] = [
x['value'] for x in item['distinctfrom']
]
newcatalog.append(newitem)
coo = coord([x['ra'] for x in newcatalog], [x['dec'] for x in newcatalog],
unit=(un.hourangle, un.deg))
response = urllib.request.urlopen(
'http://www.astronomy.swin.edu.au/pulsar/frbcat/table.php?format=text&sep=comma'
)
frbtxt = response.read().decode('utf-8').splitlines()
with open(utmost, 'r') as f:
utmosttable = f.read().splitlines()
frbcatalog = []
frbnames = []
utmostcoords = {}
for row in utmosttable:
if row[0] == '#':
def do_sdss_photo(catalog):
task_str = catalog.get_current_task_str()
D25 = Decimal('2.5')
# fits_path = os.path.join(catalog.get_current_task_repo(),
# 'SDSS/SDSS_allCandidates+BOSS_HEAD.FITS')
#
# hdulist = fits.open(fits_path)
# print(hdulist[1].columns)
# for ri, row in enumerate(hdulist[1].data['SNID']):
# print([[tag, hdulist[1].data[tag][ri]] for tag in hdulist[1].data])
# print(hdulist[1].data['SNID'][ri], hdulist[1].data['IAUC'][ri],
# hdulist[1].data['REDSHIFT_HELIO'][ri])
#
# # print(hdulist[1].data['MJD'])
# hdulist.close()
# return
# Load up metadata first
with open(
os.path.join(catalog.get_current_task_repo(),
'SDSS/sdsssn_master.dat2'), 'r') as f:
rows = list(csv.reader(f.read().splitlines()[1:], delimiter=' '))
ignored_cids = []
columns = {
SUPERNOVA.RA: 1,
SUPERNOVA.DEC: 2,
SUPERNOVA.ALIAS: 4,
SUPERNOVA.CLAIMED_TYPE: 5,
SUPERNOVA.REDSHIFT: 11,
SUPERNOVA.MAX_DATE: 21,
SUPERNOVA.HOST_RA: 99,
SUPERNOVA.HOST_DEC: 100
}
colnums = {v: k for k, v in columns.items()}
rows = [[x.replace('\\N', '') for x in y] for y in rows]
co = [[x[0], x[99], x[100]] for x in rows if x[99] and x[100]]
coo = coord([x[1] for x in co], [x[2] for x in co], unit="deg")
coo = [
''.join([y[:9] for y in x.split()])
for x in coo.to_string('hmsdms', sep='')
]
hostdict = dict(
zip([x[0] for x in co], ['SDSS J' + x[1:] for x in coo]))
for ri, row in enumerate(pbar(rows, task_str + ": metadata")):
name = ''
# Check if type is non-SNe first
ct = row[columns[SUPERNOVA.CLAIMED_TYPE]]
al = row[columns[SUPERNOVA.ALIAS]]
if ct in ['AGN', 'Variable', 'Unknown'] and not al:
catalog.log.info('`{}` is not a SN, not '
'adding.'.format(row[0]))
ignored_cids.append(row[0])
continue
# Add entry
(name, source) = catalog.new_entry(
'SDSS-II SN ' + row[0],
bibcode='2014arXiv1401.3317S',
url='http://data.sdss3.org/sas/dr10/boss/papers/supernova/')
# Add host name
if row[0] in hostdict:
catalog.entries[name].add_quantity(SUPERNOVA.HOST,
hostdict[row[0]], source)
# Add other metadata
for cn in colnums:
key = colnums[cn]
if not key:
continue
ic = int(cn)
val = row[ic]
if not val:
continue
kwargs = {}
if key == SUPERNOVA.ALIAS:
val = 'SN' + val
elif key in [
SUPERNOVA.RA, SUPERNOVA.DEC, SUPERNOVA.HOST_RA,
SUPERNOVA.HOST_DEC
]:
kwargs = {QUANTITY.U_VALUE: 'floatdegrees'}
if key in [SUPERNOVA.RA, SUPERNOVA.HOST_RA]:
fval = float(val)
if fval < 0.0:
val = str(Decimal(360) + Decimal(fval))
elif key == SUPERNOVA.CLAIMED_TYPE:
val = val.lstrip('pz').replace('SN', '')
elif key == SUPERNOVA.REDSHIFT:
kwargs[QUANTITY.KIND] = 'spectroscopic'
if float(val) < -1.0:
continue
if float(row[ic + 1]) > 0.0:
kwargs[QUANTITY.E_VALUE] = row[ic + 1]
elif key == SUPERNOVA.MAX_DATE:
dt = astrotime(float(val), format='mjd').datetime
val = make_date_string(dt.year, dt.month, dt.day)
catalog.entries[name].add_quantity(key, val, source, **kwargs)
with open(
os.path.join(catalog.get_current_task_repo(),
'SDSS/2010ApJ...708..661D.txt'), 'r') as sdss_file:
bibcodes2010 = sdss_file.read().split('\n')
sdssbands = ['u', 'g', 'r', 'i', 'z']
file_names = (list(
glob(os.path.join(catalog.get_current_task_repo(), 'SDSS/sum/*.sum')))
+ list(
glob(
os.path.join(catalog.get_current_task_repo(),
'SDSS/SMP_Data/*.dat'))))
skipphoto = ['SDSS-II SN 15557']
for fi, fname in enumerate(pbar_strings(file_names, task_str)):
tsvin = csv.reader(open(fname, 'r'),
delimiter=' ',
skipinitialspace=True)
basename = os.path.basename(fname)
hasred = True
rst = 19
if '.dat' in fname:
bibcode = '2014arXiv1401.3317S'
hasred = False
rst = 4
elif basename in bibcodes2010:
bibcode = '2010ApJ...708..661D'
else:
bibcode = '2008AJ....136.2306H'
skip_entry = False
for rr, row in enumerate(tsvin):
if skip_entry:
break
if rr == 0:
# Ignore non-SNe objects and those not in metadata table above
if row[3] in ignored_cids:
skip_entry = True
continue
# Ignore IAU names from file headers as they are unreliable
oname = 'SDSS-II SN ' + row[3]
(name, source) = catalog.new_entry(oname, bibcode=bibcode)
catalog.entries[name].add_quantity(SUPERNOVA.RA,
row[-4],
source,
u_value='floatdegrees')
catalog.entries[name].add_quantity(SUPERNOVA.DEC,
row[-2],
source,
u_value='floatdegrees')
if hasred and rr == 1:
error = row[4] if float(row[4]) >= 0.0 else ''
val = row[2]
if float(val) < -1.0:
continue
(catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT,
val,
source,
e_value=error,
kind='heliocentric'))
if rr >= rst:
# Skip bad measurements
if int(row[0]) > 1024:
continue
if oname in skipphoto:
break
mjd = row[1]
band = sdssbands[int(row[2])] + "'"
magnitude = row[3]
e_mag = row[4]
fluxd = row[7]
e_fluxd = row[8]
telescope = 'SDSS'
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.TELESCOPE: telescope,
PHOTOMETRY.BAND: band,
PHOTOMETRY.MAGNITUDE: magnitude,
PHOTOMETRY.E_MAGNITUDE: e_mag,
PHOTOMETRY.FLUX_DENSITY: fluxd,
PHOTOMETRY.E_FLUX_DENSITY: e_fluxd,
PHOTOMETRY.U_FLUX_DENSITY: 'μJy',
PHOTOMETRY.SOURCE: source,
PHOTOMETRY.BAND_SET: 'SDSS',
PHOTOMETRY.SYSTEM: 'SDSS'
}
if float(fluxd) > 0.0:
photodict[PHOTOMETRY.ZERO_POINT] = str(
D25 * Decimal(fluxd).log10() + Decimal(magnitude))
ul_sigma = 3.0
if int(row[0]) & 32 or float(
fluxd) < ul_sigma * float(e_fluxd):
photodict[PHOTOMETRY.UPPER_LIMIT] = True
photodict[PHOTOMETRY.UPPER_LIMIT_SIGMA] = str(ul_sigma)
catalog.entries[name].add_photometry(**photodict)
if catalog.args.travis and fi >= catalog.TRAVIS_QUERY_LIMIT:
break
if not fi % 1000:
catalog.journal_entries()
catalog.journal_entries()
return
def do_mergeradec(catalog):
"""Merge stars based on close ra/dec prior to cleanup."""
task_str = catalog.get_current_task_str()
# Set preferred names, calculate some columns based on imported data,
# sanitize some fields
keys = list(catalog.entries.keys())
Mnames, Mradec = [], []
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
try:
name = catalog.add_entry(oname)
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
# Set the preferred name, switching to that name if name changed.
#name = catalog.entries[name].set_preferred_name()
#aliases = catalog.entries[name].get_aliases()
if (FASTSTARS.RA not in catalog.entries[name]
or FASTSTARS.DEC not in catalog.entries[name]):
continue
else:
Mnames.append(name)
Mradec.append(
str(catalog.entries[name][FASTSTARS.RA][0]['value']) +
str(catalog.entries[name][FASTSTARS.DEC][0]['value']))
if len(Mradec) > 1:
# Only can merge if we have at least two stars!
c = coord(Mradec, unit=(un.hourangle, un.deg))
### Construct tree to look for duplicates
# Convert to pixel space
pixx = np.cos(c.dec.rad) * np.cos(c.ra.rad)
pixy = np.cos(c.dec.rad) * np.sin(c.ra.rad)
pixz = np.sin(c.dec.rad)
pix = np.vstack([pixx, pixy, pixz]).T
# Construct tree
pixtree = cKDTree(pix)
# Query pairs
pixarcsec = np.tan(5. * np.pi / 180. / 3600.)
pairs = pixtree.query_pairs(pixarcsec)
arrpairs = np.array(list(pairs))
# Attempt to merge pairs
for i, j in arrpairs:
try:
catalog.copy_entry_to_entry(catalog.entries[Mnames[i]],
catalog.entries[Mnames[j]])
del catalog.entries[Mnames[i]]
print("`{}` and `{}` merged".format(Mnames[i], Mnames[j]))
except KeyError:
print("`{}` and `{}` pair already broken".format(
Mnames[i], Mnames[j]))
catalog.save_caches()
return
def do_simbad(catalog):
task_str = catalog.get_current_task_str()
keys = list(catalog.entries.keys())
customSimbad = Simbad()
customSimbad.ROW_LIMIT = -1
customSimbad.TIMEOUT = 120
customSimbad.add_votable_fields('otype', 'sptype', 'sp_bibcode', 'id')
warnings.filterwarnings("ignore")
Mnames, Mradec = [], []
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
try:
name = catalog.add_entry(oname)
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
if (FASTSTARS.RA not in catalog.entries[name]
or FASTSTARS.DEC not in catalog.entries[name]):
continue
else:
Mnames.append(name)
radec = str(catalog.entries[name][FASTSTARS.RA][0]['value']) + str(
catalog.entries[name][FASTSTARS.DEC][0]['value'])
c = coord(radec, unit=(un.hourangle, un.deg), frame='icrs')
cnttry = 0
foundstar = False
while foundstar == False and cnttry < 10:
try:
cnttry += 1
time.sleep(0.1)
result = customSimbad.query_region(c, radius='0d0m5s')
aliases = re.sub(r'b\'(.*)\'', r'\1',
str(result['ID'].tolist()[0])).split(',')
except TypeError:
#print(radec,cnttry)
continue
foundstar = True
if foundstar == True:
source = (catalog.entries[name].add_source(
name='SIMBAD astronomical database',
bibcode="2000A&AS..143....9W",
url="http://simbad.u-strasbg.fr/",
secondary=True))
for alias in aliases:
ali = single_spaces(
re.sub(r'\[[^)]*\]', '', alias).strip())
if is_number(ali.replace(' ', '')):
continue
if ali[:4] == "HVS ":
continue
ali = name_clean(ali)
catalog.entries[name].add_quantity(FASTSTARS.ALIAS, ali,
source)
catalog.journal_entries()
return
def do_cleanup(catalog):
"""Cleanup catalog after importing all data."""
task_str = catalog.get_current_task_str()
# Set preferred names, calculate some columns based on imported data,
# sanitize some fields
keys = list(catalog.entries.keys())
cleanupcnt = 0
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
try:
name = catalog.add_entry(oname)
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
# Set the preferred name, switching to that name if name changed.
name = catalog.entries[name].set_preferred_name()
aliases = catalog.entries[name].get_aliases()
catalog.entries[name].purge_bandless_photometry()
catalog.entries[name].set_first_max_light()
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['MLS', 'SSS', 'CSS', 'GRB ']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:2])):
discoverdate = ('/'.join([
'20' + alias.replace(prefix, '')[:2],
alias.replace(prefix, '')[2:4],
alias.replace(prefix, '')[4:6]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = [
'ASASSN-', 'PS1-', 'PS1', 'PS', 'iPTF', 'PTF', 'SCP-', 'SNLS-',
'SPIRITS', 'LSQ', 'DES', 'SNHiTS', 'Gaia', 'GND', 'GNW', 'GSD',
'GSW', 'EGS', 'COS', 'OGLE', 'HST'
]
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:2])
and is_number(alias.replace(prefix, '')[:1])):
discoverdate = '20' + alias.replace(prefix, '')[:2]
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['SNF']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:4])):
discoverdate = ('/'.join([
alias.replace(prefix, '')[:4],
alias.replace(prefix, '')[4:6],
alias.replace(prefix, '')[6:8]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['PTFS', 'SNSDF']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:2])):
discoverdate = ('/'.join([
'20' + alias.replace(prefix, '')[:2],
alias.replace(prefix, '')[2:4]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['AT', 'SN', 'OGLE-', 'SM ', 'KSN']
for alias in aliases:
for prefix in prefixes:
if alias.startswith(prefix):
year = re.findall(r'\d+', alias)
if len(year) == 1:
year = year[0]
else:
continue
if alias.replace(prefix, '').index(year) != 0:
continue
if (year and is_number(year) and '.' not in year
and len(year) <= 4):
discoverdate = year
if catalog.args.verbose:
tprint('Added discoverdate from name [' +
alias + ']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if (SUPERNOVA.RA not in catalog.entries[name]
or SUPERNOVA.DEC not in catalog.entries[name]):
prefixes = [
'PSN J', 'MASJ', 'CSS', 'SSS', 'MASTER OT J', 'HST J', 'TCP J',
'MACS J', '2MASS J', 'EQ J', 'CRTS J', 'SMT J'
]
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:6])):
noprefix = alias.split(':')[-1].replace(prefix,
'').replace(
'.', '')
decsign = '+' if '+' in noprefix else '-'
noprefix = noprefix.replace('+', '|').replace('-', '|')
nops = noprefix.split('|')
if len(nops) < 2:
continue
rastr = nops[0]
decstr = nops[1]
ra = ':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + \
('.' + rastr[6:] if len(rastr) > 6 else '')
dec = (
decsign +
':'.join([decstr[:2], decstr[2:4], decstr[4:6]]) +
('.' + decstr[6:] if len(decstr) > 6 else ''))
if catalog.args.verbose:
tprint('Added ra/dec from name: ' + ra + ' ' + dec)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(SUPERNOVA.RA,
ra,
source,
derived=True)
catalog.entries[name].add_quantity(SUPERNOVA.DEC,
dec,
source,
derived=True)
break
if SUPERNOVA.RA in catalog.entries[name]:
break
no_host = (SUPERNOVA.HOST not in catalog.entries[name] or not any([
x[QUANTITY.VALUE] == 'Milky Way'
for x in catalog.entries[name][SUPERNOVA.HOST]
]))
if (SUPERNOVA.RA in catalog.entries[name]
and SUPERNOVA.DEC in catalog.entries[name] and no_host):
from astroquery.irsa_dust import IrsaDust
if name not in catalog.extinctions_dict:
try:
ra_dec = catalog.entries[name][
SUPERNOVA.RA][0][QUANTITY.VALUE] + \
" " + \
catalog.entries[name][SUPERNOVA.DEC][0][QUANTITY.VALUE]
result = IrsaDust.get_query_table(ra_dec, section='ebv')
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
warnings.warn("Coordinate lookup for " + name +
" failed in IRSA.")
else:
ebv = result['ext SandF mean'][0]
ebverr = result['ext SandF std'][0]
catalog.extinctions_dict[name] = [ebv, ebverr]
if name in catalog.extinctions_dict:
sources = uniq_cdl([
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2011ApJ...737..103S')
])
(catalog.entries[name].add_quantity(
SUPERNOVA.EBV,
str(catalog.extinctions_dict[name][0]),
sources,
e_value=str(catalog.extinctions_dict[name][1]),
derived=True))
if ((SUPERNOVA.HOST in catalog.entries[name]
and (SUPERNOVA.HOST_RA not in catalog.entries[name]
or SUPERNOVA.HOST_DEC not in catalog.entries[name]))):
for host in catalog.entries[name][SUPERNOVA.HOST]:
alias = host[QUANTITY.VALUE]
if ' J' in alias and is_number(alias.split(' J')[-1][:6]):
noprefix = alias.split(' J')[-1].split(':')[-1].replace(
'.', '')
decsign = '+' if '+' in noprefix else '-'
noprefix = noprefix.replace('+', '|').replace('-', '|')
nops = noprefix.split('|')
if len(nops) < 2:
continue
rastr = nops[0]
decstr = nops[1]
hostra = (':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) +
('.' + rastr[6:] if len(rastr) > 6 else ''))
hostdec = decsign + ':'.join([
decstr[:2], decstr[2:4], decstr[4:6]
]) + ('.' + decstr[6:] if len(decstr) > 6 else '')
if catalog.args.verbose:
tprint('Added hostra/hostdec from name: ' + hostra +
' ' + hostdec)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(SUPERNOVA.HOST_RA,
hostra,
source,
derived=True)
catalog.entries[name].add_quantity(SUPERNOVA.HOST_DEC,
hostdec,
source,
derived=True)
break
if SUPERNOVA.HOST_RA in catalog.entries[name]:
break
if (SUPERNOVA.REDSHIFT not in catalog.entries[name]
and SUPERNOVA.VELOCITY in catalog.entries[name]):
# Find the "best" velocity to use for this
bestsig = 0
for hv in catalog.entries[name][SUPERNOVA.VELOCITY]:
sig = get_sig_digits(hv[QUANTITY.VALUE])
if sig > bestsig:
besthv = hv[QUANTITY.VALUE]
bestsrc = hv['source']
bestsig = sig
if bestsig > 0 and is_number(besthv):
voc = float(besthv) * 1.e5 / CLIGHT
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
(catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT,
pretty_num(sqrt((1. + voc) / (1. - voc)) - 1.,
sig=bestsig),
sources,
kind='heliocentric',
derived=True))
if (SUPERNOVA.REDSHIFT not in catalog.entries[name]
and len(catalog.nedd_dict) > 0
and SUPERNOVA.HOST in catalog.entries[name]):
reference = "NED-D"
refurl = "http://ned.ipac.caltech.edu/Library/Distances/"
refbib = "1991ASSL..171...89H"
for host in catalog.entries[name][SUPERNOVA.HOST]:
if host[QUANTITY.VALUE] in catalog.nedd_dict:
source = catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
secondarysource = catalog.entries[name].add_source(
name=reference,
url=refurl,
bibcode=refbib,
secondary=True)
meddist = statistics.median(
catalog.nedd_dict[host[QUANTITY.VALUE]])
redz = z_at_value(cosmo.comoving_distance,
float(meddist) * un.Mpc)
redshift = pretty_num(redz,
sig=get_sig_digits(str(meddist)))
catalog.entries[name].add_quantity(
[SUPERNOVA.REDSHIFT, SUPERNOVA.HOST_REDSHIFT],
redshift,
uniq_cdl([source, secondarysource]),
kind='host',
derived=True)
if (SUPERNOVA.MAX_ABS_MAG not in catalog.entries[name]
and SUPERNOVA.MAX_APP_MAG in catalog.entries[name]
and SUPERNOVA.LUM_DIST in catalog.entries[name]):
# Find the "best" distance to use for this
bestsig = 0
for ld in catalog.entries[name][SUPERNOVA.LUM_DIST]:
sig = get_sig_digits(ld[QUANTITY.VALUE])
if sig > bestsig:
bestld = ld[QUANTITY.VALUE]
bestsrc = ld[QUANTITY.SOURCE]
bestsig = sig
if bestsig > 0 and is_number(bestld) and float(bestld) > 0.:
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
bestldz = z_at_value(cosmo.luminosity_distance,
float(bestld) * un.Mpc)
pnum = (float(catalog.entries[name][SUPERNOVA.MAX_APP_MAG][0][
QUANTITY.VALUE]) - 5.0 *
(log10(float(bestld) * 1.0e6) - 1.0) +
2.5 * log10(1.0 + bestldz))
pnum = pretty_num(pnum, sig=bestsig + 1)
catalog.entries[name].add_quantity(SUPERNOVA.MAX_ABS_MAG,
pnum,
sources,
derived=True)
if (SUPERNOVA.MAX_VISUAL_ABS_MAG not in catalog.entries[name]
and SUPERNOVA.MAX_VISUAL_APP_MAG in catalog.entries[name]
and SUPERNOVA.LUM_DIST in catalog.entries[name]):
# Find the "best" distance to use for this
bestsig = 0
for ld in catalog.entries[name][SUPERNOVA.LUM_DIST]:
sig = get_sig_digits(ld[QUANTITY.VALUE])
if sig > bestsig:
bestld = ld[QUANTITY.VALUE]
bestsrc = ld[QUANTITY.SOURCE]
bestsig = sig
if bestsig > 0 and is_number(bestld) and float(bestld) > 0.:
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
# FIX: what's happening here?!
pnum = (float(catalog.entries[name][
SUPERNOVA.MAX_VISUAL_APP_MAG][0][QUANTITY.VALUE]) - 5.0 *
(log10(float(bestld) * 1.0e6) - 1.0))
pnum = pretty_num(pnum, sig=bestsig + 1)
catalog.entries[name].add_quantity(
SUPERNOVA.MAX_VISUAL_ABS_MAG, pnum, sources, derived=True)
if SUPERNOVA.REDSHIFT in catalog.entries[name]:
# Find the "best" redshift to use for this
bestz, bestkind, bestsig, bestsrc = catalog.entries[
name].get_best_redshift()
if bestsig > 0:
try:
bestz = float(bestz)
except Exception:
print(catalog.entries[name])
raise
if SUPERNOVA.VELOCITY not in catalog.entries[name]:
source = catalog.entries[name].add_self_source()
# FIX: what's happening here?!
pnum = CLIGHT / KM * \
((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.)
pnum = pretty_num(pnum, sig=bestsig)
catalog.entries[name].add_quantity(
SUPERNOVA.VELOCITY,
pnum,
source,
kind=(SUPERNOVA.VELOCITY.kind_preference[bestkind]
if bestkind else ''))
if bestz > 0.:
if SUPERNOVA.LUM_DIST not in catalog.entries[name]:
dl = cosmo.luminosity_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
SUPERNOVA.LUM_DIST,
pretty_num(dl.value, sig=bestsig + 1),
sources,
kind=(SUPERNOVA.LUM_DIST.kind_preference[bestkind]
if bestkind else ''),
derived=True)
if (SUPERNOVA.MAX_ABS_MAG not in catalog.entries[name]
and SUPERNOVA.MAX_APP_MAG
in catalog.entries[name]):
source = catalog.entries[name].add_self_source()
pnum = pretty_num(
float(catalog.entries[name][
SUPERNOVA.MAX_APP_MAG][0][QUANTITY.VALUE])
- 5.0 * (log10(dl.to('pc').value) - 1.0) +
2.5 * log10(1.0 + bestz),
sig=bestsig + 1)
catalog.entries[name].add_quantity(
SUPERNOVA.MAX_ABS_MAG,
pnum,
sources,
derived=True)
if (SUPERNOVA.MAX_VISUAL_ABS_MAG
not in catalog.entries[name]
and SUPERNOVA.MAX_VISUAL_APP_MAG
in catalog.entries[name]):
source = catalog.entries[name].add_self_source()
pnum = pretty_num(float(catalog.entries[name][
SUPERNOVA.MAX_VISUAL_APP_MAG][0][
QUANTITY.VALUE]) - 5.0 *
(log10(dl.to('pc').value) - 1.0),
sig=bestsig + 1)
catalog.entries[name].add_quantity(
SUPERNOVA.MAX_VISUAL_ABS_MAG,
pnum,
sources,
derived=True)
if SUPERNOVA.COMOVING_DIST not in catalog.entries[name]:
cd = cosmo.comoving_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
SUPERNOVA.COMOVING_DIST,
pretty_num(cd.value, sig=bestsig),
sources,
derived=True)
if SUPERNOVA.HOST_REDSHIFT in catalog.entries[name]:
# Find the "best" redshift to use for this
bestz, bestkind, bestsig, bestsrc = catalog.entries[
name].get_best_redshift(SUPERNOVA.HOST_REDSHIFT)
if bestsig > 0:
try:
bestz = float(bestz)
except Exception:
print(catalog.entries[name])
raise
if SUPERNOVA.HOST_VELOCITY not in catalog.entries[name]:
source = catalog.entries[name].add_self_source()
# FIX: what's happening here?!
pnum = CLIGHT / KM * \
((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.)
pnum = pretty_num(pnum, sig=bestsig)
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_VELOCITY,
pnum,
source,
kind=(SUPERNOVA.HOST_VELOCITY.kind_preference[bestkind]
if bestkind else ''))
if bestz > 0.:
if SUPERNOVA.HOST_LUM_DIST not in catalog.entries[name]:
dl = cosmo.luminosity_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_LUM_DIST,
pretty_num(dl.value, sig=bestsig + 1),
sources,
kind=(SUPERNOVA.HOST_LUM_DIST.
kind_preference[bestkind]
if bestkind else ''),
derived=True)
if SUPERNOVA.HOST_COMOVING_DIST not in catalog.entries[
name]:
cd = cosmo.comoving_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_COMOVING_DIST,
pretty_num(cd.value, sig=bestsig),
sources,
derived=True)
if all([
x in catalog.entries[name] for x in [
SUPERNOVA.RA, SUPERNOVA.DEC, SUPERNOVA.HOST_RA,
SUPERNOVA.HOST_DEC
]
]):
# For now just using first coordinates that appear in entry
try:
c1 = coord(
ra=catalog.entries[name][SUPERNOVA.RA][0][QUANTITY.VALUE],
dec=catalog.entries[name][SUPERNOVA.DEC][0][
QUANTITY.VALUE],
unit=(un.hourangle, un.deg))
c2 = coord(ra=catalog.entries[name][SUPERNOVA.HOST_RA][0][
QUANTITY.VALUE],
dec=catalog.entries[name][SUPERNOVA.HOST_DEC][0][
QUANTITY.VALUE],
unit=(un.hourangle, un.deg))
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
pass
else:
sources = uniq_cdl([catalog.entries[name].add_self_source()] +
catalog.entries[name][SUPERNOVA.RA][0][
QUANTITY.SOURCE].split(',') +
catalog.entries[name][SUPERNOVA.DEC][0][
QUANTITY.SOURCE].split(',') +
catalog.entries[name][SUPERNOVA.HOST_RA][0][
QUANTITY.SOURCE].split(',') +
catalog.entries[name][SUPERNOVA.HOST_DEC][0]
[QUANTITY.SOURCE].split(','))
if SUPERNOVA.HOST_OFFSET_ANG not in catalog.entries[name]:
hosa = Decimal(c1.separation(c2).arcsecond)
hosa = pretty_num(hosa)
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_OFFSET_ANG,
hosa,
sources,
derived=True,
u_value='arcseconds')
if (SUPERNOVA.COMOVING_DIST in catalog.entries[name]
and SUPERNOVA.REDSHIFT in catalog.entries[name]
and SUPERNOVA.HOST_OFFSET_DIST
not in catalog.entries[name]):
offsetsig = get_sig_digits(catalog.entries[name][
SUPERNOVA.HOST_OFFSET_ANG][0][QUANTITY.VALUE])
sources = uniq_cdl(
sources.split(',') +
(catalog.entries[name][SUPERNOVA.COMOVING_DIST][0][
QUANTITY.SOURCE]).split(',') +
(catalog.entries[name][SUPERNOVA.REDSHIFT][0][
QUANTITY.SOURCE]).split(','))
(catalog.entries[name].add_quantity(
SUPERNOVA.HOST_OFFSET_DIST,
pretty_num(
float(catalog.entries[name][
SUPERNOVA.HOST_OFFSET_ANG][0][QUANTITY.VALUE])
/ 3600. * (pi / 180.) *
float(catalog.entries[name][
SUPERNOVA.COMOVING_DIST][0][QUANTITY.VALUE]) *
1000. / (1.0 + float(catalog.entries[name][
SUPERNOVA.REDSHIFT][0][QUANTITY.VALUE])),
sig=offsetsig), sources))
catalog.entries[name].sanitize()
catalog.journal_entries(bury=True, final=True, gz=True)
cleanupcnt = cleanupcnt + 1
if catalog.args.travis and cleanupcnt % 1000 == 0:
break
catalog.save_caches()
return
def do_sdss_photo(catalog):
task_str = catalog.get_current_task_str()
# Load up metadata first
with open(os.path.join(catalog.get_current_task_repo(),
'SDSS/sdsssn_master.dat2'), 'r') as f:
rows = list(csv.reader(f.read().splitlines()[1:], delimiter=' '))
ignored_cids = []
columns = {
SUPERNOVA.RA: 1,
SUPERNOVA.DEC: 2,
SUPERNOVA.ALIAS: 4,
SUPERNOVA.CLAIMED_TYPE: 5,
SUPERNOVA.REDSHIFT: 11,
SUPERNOVA.MAX_DATE: 21,
SUPERNOVA.HOST_RA: 99,
SUPERNOVA.HOST_DEC: 100
}
colnums = {v: k for k, v in columns.items()}
rows = [[x.replace('\\N', '') for x in y] for y in rows]
co = [[x[0], x[99], x[100]] for x in rows if x[99] and x[100]]
coo = coord([x[1] for x in co], [x[2] for x in co], unit="deg")
coo = [''.join([y[:9] for y in x.split()]) for x in
coo.to_string('hmsdms', sep='')]
hostdict = dict(zip([x[0] for x in co],
['SDSS J' + x[1:] for x in coo]))
for ri, row in enumerate(pbar(rows, task_str + ": metadata")):
name = ''
# Check if type is non-SNe first
ct = row[columns[SUPERNOVA.CLAIMED_TYPE]]
al = row[columns[SUPERNOVA.ALIAS]]
if ct in ['AGN', 'Variable'] and not al:
catalog.log.info('`{}` is not a SN, not '
'adding.'.format(row[0]))
ignored_cids.append(row[0])
continue
# Add entry
(name, source) = catalog.new_entry(
'SDSS-II SN ' + row[0], bibcode='2014arXiv1401.3317S',
url='http://data.sdss3.org/sas/dr10/boss/papers/supernova/')
# Add host name
if row[0] in hostdict:
catalog.entries[name].add_quantity(SUPERNOVA.HOST,
hostdict[row[0]], source)
# Add other metadata
for cn in colnums:
key = colnums[cn]
if not key:
continue
ic = int(cn)
val = row[ic]
if not val:
continue
kwargs = {}
if key == SUPERNOVA.ALIAS:
val = 'SN' + val
elif key in [SUPERNOVA.RA, SUPERNOVA.DEC, SUPERNOVA.HOST_RA,
SUPERNOVA.HOST_DEC]:
kwargs = {QUANTITY.U_VALUE: 'floatdegrees'}
if key in [SUPERNOVA.RA, SUPERNOVA.HOST_RA]:
fval = float(val)
if fval < 0.0:
val = str(Decimal(360) + Decimal(fval))
elif key == SUPERNOVA.CLAIMED_TYPE:
val = val.lstrip('pz').replace('SN', '')
elif key == SUPERNOVA.REDSHIFT:
kwargs[QUANTITY.KIND] = 'spectroscopic'
if float(row[ic + 1]) > 0.0:
kwargs[QUANTITY.E_VALUE] = row[ic + 1]
elif key == SUPERNOVA.MAX_DATE:
dt = astrotime(float(val), format='mjd').datetime
val = make_date_string(dt.year, dt.month, dt.day)
catalog.entries[name].add_quantity(key, val, source, **kwargs)
with open(os.path.join(catalog.get_current_task_repo(),
'SDSS/2010ApJ...708..661D.txt'), 'r') as sdss_file:
bibcodes2010 = sdss_file.read().split('\n')
sdssbands = ['u', 'g', 'r', 'i', 'z']
file_names = (list(glob(os.path.join(catalog
.get_current_task_repo(),
'SDSS/sum/*.sum'))) +
list(glob(os.path.join(catalog
.get_current_task_repo(),
'SDSS/SMP_Data/*.dat'))))
for fi, fname in enumerate(pbar_strings(file_names, task_str)):
tsvin = csv.reader(open(fname, 'r'), delimiter=' ',
skipinitialspace=True)
basename = os.path.basename(fname)
hasred = True
rst = 19
if '.dat' in fname:
bibcode = '2014arXiv1401.3317S'
hasred = False
rst = 4
elif basename in bibcodes2010:
bibcode = '2010ApJ...708..661D'
else:
bibcode = '2008AJ....136.2306H'
skip_entry = False
for rr, row in enumerate(tsvin):
if skip_entry:
break
if rr == 0:
# Ignore non-SNe objects and those not in metadata table above
if row[3] in ignored_cids:
skip_entry = True
continue
# Ignore IAU names from Sako 2014 as they are unreliable
if row[5] == 'RA:' or bibcode == '2014arXiv1401.3317S':
name = 'SDSS-II SN ' + row[3]
else:
name = 'SN' + row[5]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(bibcode=bibcode)
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, 'SDSS-II SN ' + row[3], source)
if row[5] != 'RA:' and bibcode == '2014arXiv1401.3317S':
year = re.findall(r'\d+', name)[0]
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, year, source)
catalog.entries[name].add_quantity(
SUPERNOVA.RA, row[-4], source, u_value='floatdegrees')
catalog.entries[name].add_quantity(
SUPERNOVA.DEC, row[-2], source, u_value='floatdegrees')
if hasred and rr == 1:
error = row[4] if float(row[4]) >= 0.0 else ''
(catalog.entries[name]
.add_quantity(SUPERNOVA.REDSHIFT, row[2], source,
e_value=error,
kind='heliocentric'))
if rr >= rst:
# Skip bad measurements
if int(row[0]) > 1024:
continue
mjd = row[1]
band = sdssbands[int(row[2])]
magnitude = row[3]
e_mag = row[4]
telescope = 'SDSS'
(catalog.entries[name]
.add_photometry(time=mjd, telescope=telescope,
band=band, magnitude=magnitude,
e_magnitude=e_mag, source=source,
system='SDSS'))
if not fi % 1000:
catalog.journal_entries()
catalog.journal_entries()
return
def do_gaiaviavizier(catalog):
# Disable warnings
warnings.filterwarnings("ignore")
catalog.log.warning(
'Some warnings are thrown by the Gaia module which do not affect the result of the Gaia queries.'
)
task_str = catalog.get_current_task_str()
keys = list(catalog.entries.keys())
##### Gaia parallax offset
gaiaparallaxoffset = -0.029
datafile = os.path.join(catalog.get_current_task_repo(), 'AUXILIARY',
'lindegren_error_inflation.csv')
lei_data = pd.read_csv(datafile, header=None)
lei_G = lei_data[0].values # Gaia magnitude
lei_EIR = lei_data[1].values # ratio of external to internal error
lei_EIR_interp = interp1d(lei_G, lei_EIR)
def add_lindegren_inflation(AST_ERRORS, GMAG):
_EIR_factor = lei_EIR_interp(float(GMAG))
if isinstance(AST_ERRORS, list):
return [str(float(AE) * _EIR_factor) for AE in AST_ERRORS]
else:
return str(float(AST_ERRORS) * _EIR_factor)
##### Boubert et al. (2019) RVS error
with open(
'./astrocats/faststars/input/faststars-external/AUXILIARY/boubert_2019_gaiarvscuts.p',
'rb') as fp:
b19_valid_rvs = pickle.load(fp)['source_id']
_caught_by_cut = []
cntgphot = 0
cntgast = 0
for oname in pbar(keys, task_str):
try:
name = catalog.add_entry(oname)
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
### Check if has DR1 or DR2 ID
gaiaid = ''
try:
for alias in catalog.entries[name][FASTSTARS.ALIAS]:
if alias['value'][:8] == 'Gaia DR2' and len(
alias['value']) > len(gaiaid):
gaiaid = alias['value']
elif alias['value'][:8] == 'Gaia DR1' and len(
alias['value']) > len(gaiaid):
gaiaid = alias['value']
except KeyError:
print(name)
hasgaiaid = True if len(gaiaid) > 0 else False
if (FASTSTARS.RA not in catalog.entries[name] or FASTSTARS.DEC
not in catalog.entries[name]) and not hasgaiaid:
continue
else:
if hasgaiaid:
result = silentgaiaobjectquery(gaiaid,
radius=un.Quantity(
10.0, un.arcsecond),
catalog='I/345/gaia2')
if len(result) < 1:
catalog.log.warning(
'"{}" should really have had a cross-match.'.format(
name))
print(gaiaid)
else:
Mradec = str(
catalog.entries[name][FASTSTARS.RA][0]['value']) + str(
catalog.entries[name][FASTSTARS.DEC][0]['value'])
#print(name,Mradec)
c = coord(Mradec, unit=(un.hourangle, un.deg), frame='icrs')
queryradius = 0.
if (FASTSTARS.PROPER_MOTION_RA in catalog.entries[name] and
FASTSTARS.PROPER_MOTION_DEC in catalog.entries[name]):
queryradius = 1e-3 * 3. * 10. * np.sqrt(
float(catalog.entries[name][FASTSTARS.PROPER_MOTION_RA]
[0]['value'])**2. +
float(catalog.entries[name][
FASTSTARS.PROPER_MOTION_DEC][0]['value'])**2.)
queryradius = max(queryradius, 3.)
if name in []:
result = silentgaiaregionquery(c,
radius=un.Quantity(
30.0, un.arcsecond),
catalog='I/345/gaia2')
else:
result = silentgaiaregionquery(c,
radius=un.Quantity(
queryradius,
un.arcsecond),
catalog='I/345/gaia2')
if len(result) > 0:
if len(result[0]) > 1:
indexclosest = np.argmin(result[0]['_r'])
result = result[0][indexclosest]
else:
result = result[0][0]
# Utility function for accessing the Vizier table's columns
def gtab(parstring):
return str(result[parstring])
cntgphot += 1
source = catalog.entries[name].add_source(
bibcode='2016A&A...595A...2G')
catalog.entries[name].add_photometry(
time=57023,
u_time='MJD',
telescope='Gaia',
band='G',
magnitude=gtab('Gmag'),
e_magnitude=gtab('e_Gmag'),
source=source)
catalog.entries[name].add_photometry(
time=57023,
u_time='MJD',
telescope='Gaia',
band='GBP',
magnitude=gtab('BPmag'),
e_magnitude=gtab('e_BPmag'),
source=source)
catalog.entries[name].add_photometry(
time=57023,
u_time='MJD',
telescope='Gaia',
band='GRP',
magnitude=gtab('RPmag'),
e_magnitude=gtab('e_RPmag'),
source=source)
if not hasgaiaid:
catalog.entries[name].add_quantity(FASTSTARS.ALIAS,
gtab('DR2Name'),
source=source)
gra, gde = coord(ra=float(gtab('RA_ICRS')) * un.deg,
dec=float(gtab('DE_ICRS')) * un.deg,
frame='icrs').to_string('hmsdms',
sep=':').split()
if gtab('Plx') == '--':
catalog.entries[name].add_quantity(
FASTSTARS.RA,
gra,
source,
e_value=add_lindegren_inflation(
gtab('e_RA_ICRS'), gtab('Gmag')),
u_e_value='mas',
correlations=[{
CORRELATION.VALUE: gtab('RADEcor'),
CORRELATION.QUANTITY: FASTSTARS.DEC,
CORRELATION.KIND: 'Pearson'
}])
catalog.entries[name].add_quantity(
FASTSTARS.DEC,
gde,
source,
e_value=add_lindegren_inflation(
gtab('e_DE_ICRS'), gtab('Gmag')),
u_e_value='mas',
correlations=[{
CORRELATION.VALUE: gtab('RADEcor'),
CORRELATION.QUANTITY: FASTSTARS.RA,
CORRELATION.KIND: 'Pearson'
}])
else:
#print(gtab('Plx'))
cntgast += 1
#catalog.log.warning(
# '"{}" has Gaia astrometry.'.format(name))
ast_keys = [
FASTSTARS.RA, FASTSTARS.DEC, FASTSTARS.PARALLAX,
FASTSTARS.PROPER_MOTION_RA, FASTSTARS.PROPER_MOTION_DEC
]
ast_values = [
gra, gde,
str(float(gtab('Plx')) - gaiaparallaxoffset),
gtab('pmRA'),
gtab('pmDE')
]
ast_errors = add_lindegren_inflation([
gtab('e_RA_ICRS'),
gtab('e_DE_ICRS'),
gtab('e_Plx'),
gtab('e_pmRA'),
gtab('e_pmDE')
], gtab('Gmag'))
ast_corr = [[
'1.0',
gtab('RADEcor'),
gtab('RAPlxcor'),
gtab('RApmRAcor'),
gtab('RApmDEcor')
],
[
gtab('RADEcor'), '1.0',
gtab('DEPlxcor'),
gtab('DEpmRAcor'),
gtab('DEpmDEcor')
],
[
gtab('RAPlxcor'),
gtab('DEPlxcor'), '1.0',
gtab('PlxpmRAcor'),
gtab('PlxpmDEcor')
],
[
gtab('RApmRAcor'),
gtab('DEpmRAcor'),
gtab('PlxpmRAcor'), '1.0',
gtab('pmRApmDEcor')
],
[
gtab('RApmDEcor'),
gtab('DEpmDEcor'),
gtab('PlxpmDEcor'),
gtab('pmRApmDEcor'), '1.0'
]]
ast_names = [
'ra', 'dec', 'parallax', 'propermotionra',
'propermotiondec'
]
ast_units = ['hms', 'dms', 'mas', 'mas/yr', 'mas/yr']
ast_error_units = ['mas', 'mas', 'mas', 'mas/yr', 'mas/yr']
#print(ast_values[0])
n_ast = len(ast_keys) # number of columns
for i in range(n_ast):
corr_dict = [{
CORRELATION.VALUE:
ast_corr[i][(i + j) % n_ast],
CORRELATION.QUANTITY:
ast_names[(i + j) % n_ast],
CORRELATION.KIND:
'Pearson'
} for j in list(range(1, n_ast))]
catalog.entries[name].add_quantity(
ast_keys[i],
ast_values[i],
source,
e_value=ast_errors[i],
u_value=ast_units[i],
e_u_value=ast_error_units[i],
correlations=corr_dict)
if gtab('RV') != '--':
if int(gtab('Source')) in b19_valid_rvs:
catalog.entries[name].add_quantity(
FASTSTARS.VELOCITY,
gtab('RV'),
source,
e_value=gtab('e_RV'),
u_value='km/s')
else:
#print(int(gtab('Source')))
_caught_by_cut.append(int(gtab('Source')))
##### This has been moved to boundprobability.py
# Convert parallax to distance
#if (FASTSTARS.LUM_DIST in catalog.entries[name]):
# catalog.log.warning(
# '"{}" has distance from photmetric distance prior.'.format(name))
# distance, distance_error = parallax_to_distance(name,result['Plx'][0],result['e_Plx'][0],float(catalog.entries[name][FASTSTARS.LUM_DIST][0]['value']),float(catalog.entries[name][FASTSTARS.LUM_DIST][0]['e_value']))
# catalog.entries[name].add_quantity(FASTSTARS.LUM_DIST, str(distance), e_value=str(distance_error), u_value='kpc', source=source, derived=True)
#else:
# catalog.log.warning(
# '"{}" has distance from Astraatmadja & Bailer-Jones (2016) prior.'.format(name))
# distance, distance_error = parallax_to_distance(name,result['Plx'][0],result['e_Plx'][0])
# catalog.entries[name].add_quantity(FASTSTARS.LUM_DIST, str(distance), e_value=str(distance_error), u_value='kpc', source=source, derived=True)
_box = {'source_id': _caught_by_cut}
with open('./boubert_2019_gaiarvscuts_caught.p', "wb") as f:
pickle.dump(_box, f, protocol=3)
catalog.log.warning(
'"{}" have Gaia photometry and "{}" have Gaia astrometry.'.format(
cntgphot, cntgast))
catalog.journal_entries()
# Reactivate warnings
warnings.filterwarnings("default")
return
# 2018arXiv180410607M
datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII',
'marchetti2018.txt')
data = read(datafile)
for row in pbar(data, task_str):
oname = 'Gaia DR2 '+str(row['source_id']).strip()[:6]
name, source = catalog.new_entry(oname, bibcode='2018arXiv180410607M')
lname = 'Gaia DR2 '+str(row['source_id']).strip()
catalog.entries[name].add_quantity(FASTSTARS.ALIAS, lname, source=source)
if (FASTSTARS.DISCOVERER not in catalog.entries[name]):
catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'T. Marchetti, E. M. Rossi, A. G. A. Brown', source)
catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2018), source)
#sourcegaia = catalog.entries[name].add_source(bibcode='2016A&A...595A...2G')
ra, dec = coord(ra=float(row['ra'])*u.deg,dec=float(row['dec'])*u.deg,frame='icrs').to_string(
'hmsdms', sep=':').split()
catalog.entries[name].add_quantity(
FASTSTARS.RA, ra, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.DEC, dec, source=source)
# 2018arXiv180503194H
datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII','hattori2018.csv')
data = read(datafile)
for row in pbar(data, task_str):
oname = str(row['ID']).strip(' ')
name, source = catalog.new_entry(oname, bibcode='2018arXiv180503194H')
lgname = 'Gaia DR2 '+str(row['GaiaDR2']).strip(' ')
sgname = 'Gaia DR2 '+str(row['GaiaDR2']).strip(' ')[:6]
catalog.entries[name].add_quantity(FASTSTARS.ALIAS, lgname, source=source)
if not is_number(newitem['ra'].split(':')[0]):
continue
newitem['dec'] = item['dec'][0]['value']
if not is_number(newitem['dec'].split(':')[0]):
continue
newitem['raerr'] = float(item['ra'][0].get('e_value', 0))
newitem['decerr'] = float(item['dec'][0].get('e_value', 0))
# Temporary fix for David's typo
if newitem['dec'].count('.') == 2:
newitem['dec'] = newitem['dec'][:newitem['dec'].rfind('.')]
if 'distinctfrom' in item:
newitem['distinctfrom'] = [x['value']
for x in item['distinctfrom']]
newcatalog.append(newitem)
coo = coord([x['ra'] for x in newcatalog],
[x['dec'] for x in newcatalog], unit=(un.hourangle, un.deg))
path = download_file(
'http://cxc.harvard.edu/csc2/preliminary/preliminary_detlist.fits',
'astrocats/supernovae/output/cache')
cxctable = fits.getdata(path, 1)
cxccatalog = []
for ri, row in enumerate(tqdm(cxctable)):
if ri > 1000:
break
name = row[0]
if row[2] != 'TRUE' or row[34] != 'POINT' or row[5] == 'T':
continue
cxcdict = {
'name': name,
def do_boundprobability(catalog):
task_str = catalog.get_current_task_str()
keys = list(catalog.entries.keys())
if vescmethod == 'galpy':
from galpy.potential import MWPotential2014, vesc
elif vescmethod == 'williams2017':
datafile = os.path.join(catalog.get_current_task_repo(),
'spherical_powerlaw.dat')
chain = np.genfromtxt(datafile)
VESCSOLAR = chain[:, 5]
ALPHA = chain[:, 6]
def vescsolar_sampler(NAME, NSAMP):
# Set random number generator based on name
seed = NAME
def craft_seed(SEED):
nSEED = len(SEED)
# The modulo ensures we don't overrun.
indSEED = range(2 * (nSEED + 3))
return np.prod(
np.array([ord(SEED[i % nSEED])
for i in indSEED])) % 4294967296
np.random.seed(seed=craft_seed(seed))
choice_chain = np.random.choice(range(len(VESCSOLAR)), NSAMP)
return VESCSOLAR[choice_chain], ALPHA[choice_chain]
else:
catalog.log.warning(
'vescmethod was not properly defined, please check.')
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
try:
name = catalog.add_entry(oname)
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
if (FASTSTARS.RA not in catalog.entries[name]
or FASTSTARS.DEC not in catalog.entries[name]
or (FASTSTARS.LUM_DIST not in catalog.entries[name]
and FASTSTARS.PARALLAX not in catalog.entries[name])):
# Don't need to worry about not having any velocities. Can't be in the catalogue if it doesn't have velocities!
continue
else:
# Obtain highest precision position
#best_position_i = best_parameter(catalog.entries[name][FASTSTARS.RA])
Mradec = str(
catalog.entries[name][FASTSTARS.RA][0]['value']) + str(
catalog.entries[name][FASTSTARS.DEC][0]['value'])
c = coord(Mradec, unit=(un.hourangle, un.deg), frame='icrs')
# Obtain galactic coordinates
ra = c.ra.rad
dec = c.dec.rad
l = c.galactic.l.rad
b = c.galactic.b.rad
# Storage for means, errors and correlation
kine_n = 4
kine_values = np.zeros(kine_n)
kine_errors = 9999.9 * np.ones(kine_n)
kine_corr = np.diag(np.ones(kine_n))
kine_L = 1.35
kine_k = 3.0
# Do we have distance prior length scale?
if FASTSTARS.DISTANCE_PRIOR_LENGTH_SCALE in catalog.entries[name]:
kine_L = float(catalog.entries[name][
FASTSTARS.DISTANCE_PRIOR_LENGTH_SCALE][0]['value'])
# Flags for having proper motions and radial velocities
haveparallax = False
havepropermotions = False
havevelocities = False
# Do we have a parallax?
if FASTSTARS.PARALLAX in catalog.entries[name]:
# Yes.
haveparallax = True
kine_values[0] = float(
catalog.entries[name][FASTSTARS.PARALLAX][0]['value'])
kine_errors[0] = float(
catalog.entries[name][FASTSTARS.PARALLAX][0]['e_value'])
# Is the parallax correlated with the proper motions?
if QUANTITY.CORRELATIONS in catalog.entries[name][
FASTSTARS.PARALLAX]:
# Yes. Loop through correlations.
parallax_corr = catalog.entries[name][
FASTSTARS.PARALLAX][0]['correlations']
n_corr = len(parallax_corr)
for i in range(n_corr):
if parallax_corr[i]['quantity'] == 'propermotionra':
kine_corr[0, 1] = kine_corr[1, 0] = float(
parallax_corr[i]['value'])
if parallax_corr[i]['quantity'] == 'propermotiondec':
kine_corr[0, 2] = kine_corr[2, 0] = float(
parallax_corr[i]['value'])
# Do we have proper motions?
if (FASTSTARS.PROPER_MOTION_RA in catalog.entries[name]
and FASTSTARS.PROPER_MOTION_DEC in catalog.entries[name]):
# Yes.
havepropermotions = True
best_propermotion_i = best_parameter(
catalog.entries[name][FASTSTARS.PROPER_MOTION_RA])[0]
kine_values[1] = float(catalog.entries[name][
FASTSTARS.PROPER_MOTION_RA][best_propermotion_i]['value'])
kine_values[2] = float(catalog.entries[name][
FASTSTARS.PROPER_MOTION_DEC][best_propermotion_i]['value'])
# Do the proper motions have errors?
if (QUANTITY.E_VALUE in catalog.entries[name][
FASTSTARS.PROPER_MOTION_RA][best_propermotion_i]
and QUANTITY.E_VALUE in catalog.entries[name][
FASTSTARS.PROPER_MOTION_DEC][best_propermotion_i]):
kine_errors[1] = float(
catalog.entries[name][FASTSTARS.PROPER_MOTION_RA]
[best_propermotion_i]['e_value'])
kine_errors[2] = float(
catalog.entries[name][FASTSTARS.PROPER_MOTION_DEC]
[best_propermotion_i]['e_value'])
else:
kine_errors[1] = errorifmissing * kine_values[1]
kine_errors[2] = errorifmissing * kine_values[2]
# Are the proper motions correlated?
if (QUANTITY.CORRELATIONS in catalog.entries[name][
FASTSTARS.PROPER_MOTION_RA][best_propermotion_i]
and QUANTITY.CORRELATIONS in catalog.entries[name][
FASTSTARS.PROPER_MOTION_DEC][best_propermotion_i]):
# Yes. Loop through correlations.
propermotionra_corr = catalog.entries[name][
FASTSTARS.
PROPER_MOTION_RA][best_propermotion_i]['correlations']
n_corr = len(propermotionra_corr)
for i in range(n_corr):
if propermotionra_corr[i][
'quantity'] == 'propermotiondec':
kine_corr[1, 2] = kine_corr[2, 1] = float(
propermotionra_corr[i]['value'])
else:
# No. Reset correlations with parallax.
kine_corr[0, 1] = kine_corr[1, 0] = kine_corr[
0, 2] = kine_corr[2, 0] = 0.0
# Do we have a radial velocity?
if FASTSTARS.VELOCITY in catalog.entries[name]:
# Yes.
havevelocities = True
# What is the best radial velocity?
best_velocity_i, best_velocity_err = best_parameter(
catalog.entries[name][FASTSTARS.VELOCITY])
kine_values[3] = float(catalog.entries[name][
FASTSTARS.VELOCITY][best_velocity_i]['value'])
# Does the radial velocity have an error?
#if QUANTITY.E_VALUE in catalog.entries[name][FASTSTARS.VELOCITY][best_velocity_i]:
#kine_errors[3] = float(catalog.entries[name][FASTSTARS.VELOCITY][best_velocity_i]['e_value'])
if best_velocity_err < 9999.0:
kine_errors[3] = best_velocity_err
else:
kine_errors[3] = errorifmissing * kine_values[3]
# Do we have a previous photometric distance?
if FASTSTARS.LUM_DIST in catalog.entries[name]:
# Yes.
# What is the best photometric distance?
best_lumdist_i, best_lumdist_err = best_parameter(
catalog.entries[name][FASTSTARS.LUM_DIST])
kine_mu = float(catalog.entries[name][FASTSTARS.LUM_DIST]
[best_lumdist_i]['value'])
# Does the photometric distance have an error?
#if QUANTITY.E_VALUE in catalog.entries[name][FASTSTARS.LUM_DIST][best_lumdist_i]:
# kine_sigma = float(catalog.entries[name][FASTSTARS.LUM_DIST][best_lumdist_i]['e_value'])
if best_lumdist_err < 9999.0:
kine_sigma = best_lumdist_err
else:
kine_sigma = errorifmissing * kine_mu
kine_L = kine_sigma**2 / kine_mu
kine_k = kine_mu / kine_L
# Sample
kine_samples = kinematic_sampler(name,
kine_values,
kine_errors,
kine_corr,
KINE_L=kine_L,
KINE_K=kine_k)
kine_samples_corrected = np.copy(kine_samples)
kine_samples_corrected[:, 1] *= k * kine_samples[:, 0]
kine_samples_corrected[:, 2] *= k * kine_samples[:, 0]
kine_samples_n = kine_samples.shape[0]
# Calculate solar reflex correction
cosl = cos(l)
sinl = sin(l)
cosb = cos(b)
sinb = sin(b)
cosdec = cos(dec)
sindec = sin(dec)
cosra = cos(ra)
sinra = sin(ra)
A = np.array([[cosra * cosdec, -sinra, -cosra * sindec],
[sinra * cosdec, cosra, -sinra * sindec],
[sindec, 0., cosdec]])
B = np.dot(T, A) # T*A
#invB = np.linalg.inv(B)
kine_samples_solar = solarmotion_sampler(name, kine_samples_n)
kine_vhel = kine_samples_solar[:, 2:] # samples in UVW_solar
kine_vhel[:, 1] += kine_samples_solar[:,
1] # add the vdisk component
kine_D = np.einsum('ei,ni->ne', B.T, kine_vhel)
# If had no proper motions, then assume had exactly solar reflex
if havepropermotions == False:
kine_samples_corrected[:, 1] = -1.0 * kine_D[:, 1]
kine_samples_corrected[:, 2] = -1.0 * kine_D[:, 2]
# If had no radial velocity, then assume had exactly solar reflex
if havevelocities == False:
kine_samples_corrected[:, 3] = -1.0 * kine_D[:, 0]
kine_samples_vhel = kine_vhel + np.einsum(
'ij,nj->ni', B,
np.vstack([
kine_samples_corrected[:, 3], kine_samples_corrected[:, 1],
kine_samples_corrected[:, 2]
]).T)
kine_vgrf = np.sqrt(kine_samples_vhel[:, 0]**2 +
kine_samples_vhel[:, 1]**2 +
kine_samples_vhel[:, 2]**2)
# Calculate escape velocity
kine_galrad = np.sqrt(kine_samples_solar[:, 0]**2 +
(kine_samples_corrected[:, 0])**2 -
2. * kine_samples_solar[:, 0] *
kine_samples_corrected[:, 0] * cosb * cosl)
if vescmethod == 'galpy':
kine_vesc = vesc(MWPotential2014, kine_galrad *
un.kpc) * kine_samples_solar[:, 1]
elif vescmethod == 'williams2017':
kine_vesc_solar, kine_alpha = vescsolar_sampler(
name, kine_samples_n)
kine_vesc = kine_vesc_solar * np.power(
kine_galrad / kine_samples_solar[:, 0], -kine_alpha / 2.0)
# Store samples for each star
if False:
if FASTSTARS.DISCOVERER in catalog.entries[
name] and FASTSTARS.DISCOVER_DATE in catalog.entries[
name]:
_discoverer = catalog.entries[name][
FASTSTARS.DISCOVERER][0]['value']
_discover_date = catalog.entries[name][
FASTSTARS.DISCOVER_DATE][0]['value']
if ('Bromley' in _discoverer or 'Scholz'
in _discoverer) and _discover_date == '2018':
kine_samples_output = np.copy(kine_samples_corrected)
kine_samples_output[:, 1] /= (k * kine_samples[:, 0])
kine_samples_output[:, 2] /= (k * kine_samples[:, 0])
np.savez_compressed(
'/Users/douglasboubert/Documents/Science/GaiaDR2/LiteratureRV/data/'
+ name + '.npz',
radec=Mradec,
kine_samples_output=kine_samples_output,
kine_samples_solar=kine_samples_solar,
kine_vgrf=kine_vgrf)
#np.savez_compressed('/data/dpb33/GaiaHypervelocity/WhiteDwarfs/mainsequence/samples/'+name+'.npz',radec=Mradec,kine_samples_output=kine_samples_output,kine_samples_solar=kine_samples_solar,kine_vgrf=kine_vgrf)
# Bound probability
kine_bound_n = np.where(kine_vesc > kine_vgrf)[0].shape[0]
kine_betaa = kine_bound_n + 0.5
kine_betab = kine_samples_n - kine_bound_n + 0.5
kine_bound_percentiles = [
beta.ppf(confidenceintervalcoverage / 2.0, kine_betaa,
kine_betab),
beta.ppf(0.5, kine_betaa, kine_betab),
beta.ppf(1.0 - confidenceintervalcoverage / 2.0, kine_betaa,
kine_betab)
]
# Store all outcomes.
source = catalog.entries[name].add_self_source()
boundprobability_upperlimit = (havepropermotions == False
or havevelocities == False)
catalog.entries[name].add_quantity(
FASTSTARS.BOUND_PROBABILITY,
str(kine_bound_percentiles[1]),
e_lower_value=str(kine_bound_percentiles[1] -
kine_bound_percentiles[0]),
e_upper_value=str(kine_bound_percentiles[2] -
kine_bound_percentiles[1]),
upperlimit=boundprobability_upperlimit,
source=source,
derived=True)
catalog.entries[name].add_quantity(
FASTSTARS.VELOCITY,
str(kine_vesc.mean()),
e_value=str(kine_vesc.std()),
u_value='km/s',
source=source,
derived=True,
kind=['galactocentric', 'escape'])
catalog.entries[name].add_quantity(
FASTSTARS.VELOCITY,
str(kine_vgrf.mean()),
e_value=str(kine_vgrf.std()),
u_value='km/s',
lowerlimit=boundprobability_upperlimit,
source=source,
derived=True,
kind=['galactocentric', 'total'])
if haveparallax == True:
catalog.entries[name].add_quantity(
FASTSTARS.LUM_DIST,
str(kine_samples_corrected[:, 0].mean()),
e_value=str(kine_samples_corrected[:, 0].std()),
u_value='kpc',
source=source,
derived=True)
catalog.journal_entries()
return
def retrieve_objects(self,
catalog_name,
event_name=None,
quantity_name=None,
attribute_name=None,
full=False):
"""Retrieve data, first trying catalog file then event files."""
event = None
use_full = full
search_all = False
ename = event_name
qname = quantity_name
aname = attribute_name
req_vals = request.get_json()
if not req_vals:
req_vals = request.values
# Load event/quantity/attribute if provided by request.
event_req = req_vals.get('event')
quantity_req = req_vals.get('quantity')
attribute_req = req_vals.get('attribute')
if ename is None and event_req is not None:
if not isinstance(event_req, string_types):
ename = '+'.join(listify(event_req))
else:
ename = event_req
if qname is None and quantity_req is not None:
if not isinstance(quantity_req, string_types):
qname = '+'.join(listify(quantity_req))
else:
qname = quantity_req
if aname is None and attribute_req is not None:
if not isinstance(attribute_req, string_types):
aname = '+'.join(listify(attribute_req))
else:
aname = attribute_req
if ename is None:
return msg('no_root_data')
# Options
if not use_full:
rfull = req_vals.get('full')
if rfull is not None:
return self.retrieve_objects(catalog_name,
event_name=ename,
quantity_name=qname,
attribute_name=aname,
full=True)
fmt = req_vals.get('format')
fmt = fmt.lower() if fmt is not None else fmt
fmt = None if fmt == 'json' else fmt
ra = req_vals.get('ra')
dec = req_vals.get('dec')
radius = req_vals.get('radius')
width = req_vals.get('width')
height = req_vals.get('height')
complete = req_vals.get('complete')
first = req_vals.get('first')
closest = req_vals.get('closest')
sortby = req_vals.get('sortby')
sortby = sortby.lower() if sortby is not None else sortby
include_keys = list(sorted(set(req_vals.keys()) - self._SPECIAL_ATTR))
includes = OrderedDict()
iincludes = OrderedDict()
for key in include_keys:
val = req_vals.get(key, '')
if not is_number(val) and val != '':
val = '^' + val + '$'
try:
includes[key] = re.compile(val)
iincludes[key] = re.compile(val, re.IGNORECASE)
except Exception:
return msg('invalid_regex', [req_vals.get(key, ''), key])
excludes = OrderedDict([('realization', '')])
if first is None:
item = req_vals.get('item')
try:
item = int(item)
except Exception:
item = None
else:
item = 0
if radius is not None:
try:
radius = float(radius)
except Exception:
radius = 0.0
if radius >= self._ANGLE_LIMIT:
return msg('radius_limited', self._ANGLE_LIMIT / 3600.)
if width is not None:
try:
width = float(width)
except Exception:
width = 0.0
if width >= self._ANGLE_LIMIT:
return msg('width_limited', self._ANGLE_LIMIT / 3600.)
if height is not None:
try:
height = float(height)
except Exception:
height = 0.0
if height >= self._ANGLE_LIMIT:
return msg('height_limited', self._ANGLE_LIMIT / 3600.)
if ename and ename.lower() in ['catalog', 'all']:
if ra is not None and dec is not None:
try:
ldec = str(dec).lower().strip(' .')
if is_number(ldec) or decregex.match(ldec):
sra = str(ra)
lra = sra.lower().replace('h', '').strip(' .')
if raregex.match(lra) or (is_number(lra)
and 'h' in sra):
lcoo = coord(lra,
ldec,
unit=(un.hourangle, un.deg))
elif is_number(lra):
lcoo = coord(lra, ldec, unit=(un.deg, un.deg))
else:
raise Exception
else:
raise Exception
except Exception:
return msg('bad_coordinates')
if (width is not None and height is not None and width > 0.0
and height > 0.0):
idxcat = np.where((
abs(lcoo.ra - apidata._coo.ra) <= width * un.arcsecond)
& (abs(lcoo.dec - apidata._coo.dec) <=
height * un.arcsecond))[0]
elif width is not None and width > 0.0:
idxcat = np.where(
abs(lcoo.ra - apidata._coo.ra) <= width *
un.arcsecond)[0]
elif height is not None and height > 0.0:
idxcat = np.where(
abs(lcoo.dec - apidata._coo.dec) <= height *
un.arcsecond)[0]
else:
if radius is None or radius == 0.0:
radius = 1.0
idxcat = np.where(
lcoo.separation(apidata._coo) <= radius *
un.arcsecond)[0]
if len(idxcat):
ename_arr = [
apidata._rdnames[i].replace('+', '$PLUS$')
for i in idxcat
]
else:
return msg('no_objects')
elif catalog_name in apidata._catalogs:
ename_arr = [
i.replace('+', '$PLUS$')
for i in apidata._catalogs[catalog_name]
]
search_all = True
else:
ename_arr = [
a for b in [[
i.replace('+', '$PLUS$')
for i in apidata._catalogs[cat]
] for cat in apidata._catalogs] for a in b
]
search_all = True
ename = '+'.join(list(sorted(set(ename_arr))))
if qname is None:
# Short circuit to full if keyword is present.
if full:
return self.retrieve_objects(catalog_name,
event_name=ename,
full=True)
search_all = True
if catalog_name not in apidata._CATS:
qname = '+'.join(
list(
set(
sorted([
a for b in [
apidata._cat_keys[x]
for x in apidata._cat_keys
] for a in b
]))))
else:
qname = '+'.join(
list(sorted(set(apidata._cat_keys[catalog_name]))))
# if fmt is not None and qname is None:
# return Response((
# 'Error: \'{}\' format only supported if quantity '
# 'is specified.').format(
# fmt), mimetype='text/plain')
# Events
event_names = [] if ename is None else ename.split('+')
# Check for + in names
nevent_names = []
joined = False
for ni, name in enumerate(event_names):
if joined:
joined = False
continue
if ni < len(event_names) - 1:
jname = '+'.join(event_names[ni:ni + 2])
if jname.lower().replace(' ', '') in apidata._aliases:
nevent_names.append(jname)
joined = True
continue
nevent_names.append(name)
event_names = nevent_names
event_names = [x.replace('$PLUS$', '+') for x in event_names]
if not len(event_names):
search_all = True
event_names = apidata._all
# Quantities
quantity_names = [] if qname is None else qname.split('+')
# Attributes. Always append source.
attribute_names = [] if aname is None else aname.split('+')
if use_full and len(event_names) > self._FULL_LIMIT:
return msg('max_events', self._FULL_LIMIT)
if fmt is not None and any(
[n in attribute_names for n in self._NO_CSV]):
return msg('no_delimited')
if len(event_names) > self._EXPENSIVE_LIMIT:
for quantity in quantity_names:
for exp in self._EXPENSIVE:
if any([e in attribute_names for e in self._EXPENSIVE]):
return msg('too_expensive')
edict = OrderedDict()
fcatalogs = OrderedDict()
sources = OrderedDict()
new_event_names = []
for event in event_names:
skip_entry = False
my_cat, my_event = None, None
alopts = apidata._aliases.get(event.lower().replace(' ', ''), [])
for opt in alopts:
if opt[0] == catalog_name:
my_cat, my_event, my_alias = tuple(opt)
break
if not my_cat:
for opt in alopts:
if opt[0] != catalog_name:
my_cat, my_event, my_alias = tuple(opt)
break
if not my_cat:
if len(event_names) == 1:
return msg('event_not_found', event)
continue
if full:
fcatalogs.update(
json.load(open(
os.path.join(apidata._AC_PATH,
apidata._CATS[my_cat][0], 'output',
'json', get_filename(my_event)), 'r'),
object_pairs_hook=OrderedDict))
sources[my_event] = [
x.get('bibcode', x.get('arxivid', x.get('name')))
for x in fcatalogs[my_event].get('sources')
]
if qname is None:
if full:
edict[event] = fcatalogs.get(my_event, {})
else:
edict[event] = apidata._catalogs.get(my_cat,
{}).get(my_event, {})
else:
# Check if user passed quantity or attribute names to filter
# by.
qdict = OrderedDict()
if full:
my_event_dict = fcatalogs.get(my_event, {})
else:
my_event_dict = apidata._catalogs.get(my_cat, {}).get(
my_event, {})
if aname is None:
for incl in iincludes:
incll = incl.lower()
if incll not in my_event_dict or (
iincludes[incl].pattern != '' and not any([
bool(iincludes[incl].match(
x.get('value', ''
) if isinstance(x, dict) else x))
for x in my_event_dict.get(incll, [{}])
])):
skip_entry = True
break
if not skip_entry:
for quantity in quantity_names:
my_quantity = listify(my_event_dict.get(quantity, {}))
closest_locs = []
if closest is not None:
closest_locs = list(
sorted(
list(
set([
np.argmin([
abs(
np.mean([
float(y)
for y in listify(
x.get(i))
]) -
float(includes[i].pattern))
for x in my_quantity
]) for i in includes
if len(my_quantity) and is_number(
includes[i].pattern) and all([
is_number(x.get(i, ''))
for x in my_quantity
])
]))))
if aname is None and quantity in my_event_dict:
qdict[quantity] = [
x for xi, x in enumerate(my_quantity)
if not len(closest_locs) or xi in closest_locs
]
if item is not None:
try:
qdict[quantity] = qdict[quantity][item]
except Exception:
pass
else:
qdict[quantity] = self.get_attributes(
attribute_names,
my_quantity,
complete=complete,
full=use_full,
item=item,
includes=includes,
iincludes=iincludes,
excludes=excludes,
closest_locs=closest_locs,
sources=np.array(sources.get(my_event, [])))
if not search_all and not qdict.get(quantity):
use_full = True
break
if not full and use_full:
new_event_names = list(event_names)
break
if qdict:
edict[event] = qdict
if not full and use_full:
break
if not (skip_entry and (full or search_all)) and event in edict:
new_event_names.append(event)
event_names = new_event_names
ename = '+'.join([i.replace('+', '$PLUS$') for i in event_names])
if not full and use_full:
return self.retrieve_objects(catalog_name,
event_name=ename,
quantity_name=qname,
attribute_name=aname,
full=True)
if fmt is not None:
return self.get_event_dsv(edict, event_names, quantity_names,
attribute_names, fmt, sortby)
return edict
def do_ascii(catalog):
"""Process ASCII files extracted from datatables of published works."""
task_str = catalog.get_current_task_str()
# catalog.journal_entries()
# return
#def holding():
# 2007ApJ...660..311B
datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII',
'ApJ_660_311_table1.csv')
data = read(datafile, format='csv')
for row in pbar(data, task_str):
oname = 'SDSS'+str(row['SDSS'])
name, source = catalog.new_entry(oname, bibcode='2007ApJ...660..311B')
if (FASTSTARS.DISCOVERER not in catalog.entries[name]):
catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'Warren R. Brown, Margaret J. Geller, Scott J. Kenyon, Michael J. Kurtz, Benjamin C. Bromley', source)
catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2007), source)
gallon = float(str(row['Glon']))
gallat = float(str(row['Glat']))
radec = oname.strip('SDSSJ')
radec = radec[0:2]+' '+radec[2:4]+' '+radec[4:9]+' '+radec[9:12]+' '+radec[12:14]+' '+radec[14:18]
ra, dec = coord(radec,
unit=(u.hourangle, u.deg)).to_string(
'hmsdms', sep=':').split()
catalog.entries[name].add_quantity(
FASTSTARS.RA, ra, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.DEC, dec, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.VELOCITY, str(row['Vhelio']), source=source)
galrad_MS = float(str(row['Ra']))
galrad_BHB = float(str(row['Rb']))
dhel_MS = rgc_to_dhel(galrad_MS,gallon,gallat)
dhel_BHB = rgc_to_dhel(galrad_BHB,gallon,gallat)
catalog.entries[name].add_quantity(
FASTSTARS.LUM_DIST, str(dhel_MS), u_value='kpc', source=source,derived=True)
catalog.entries[name].add_quantity(
FASTSTARS.LUM_DIST, str(dhel_BHB), u_value='kpc', source=source,derived=True)
catalog.entries[name].add_quantity(
FASTSTARS.SPECTRAL_TYPE, str(row['Sp']), source=source)
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, "d", source=source)
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, "bhb", source=source)
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, "g", source=source)
catalog.journal_entries()
# 2009ApJ...690.1639B
datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII',
'apj292642t1_ascii.csv')
data = read(datafile, format='csv')
for row in pbar(data, task_str):
oname = row['Catalog']
name, source = catalog.new_entry(oname, bibcode='2009ApJ...690.1639B')
if (FASTSTARS.DISCOVERER not in catalog.entries[name]):
catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'Warren R. Brown, Margaret J. Geller, Scott J. Kenyon', source)
catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2009), source)
gallon = float(str(row['Glon']))
gallat = float(str(row['Glat']))
if (oname!='US708') & (oname!='HE0437-5439'):
radec = oname.strip('SDSSJ')
radec = radec[0:2]+' '+radec[2:4]+' '+radec[4:9]+' '+radec[9:12]+' '+radec[12:14]+' '+radec[14:18]
ra, dec = coord(radec,
unit=(u.hourangle, u.deg)).to_string(
'hmsdms', sep=':').split()
catalog.entries[name].add_quantity(
FASTSTARS.RA, ra, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.DEC, dec, source=source)
if str(row['e_Vhel'])!='NA':
catalog.entries[name].add_quantity(
FASTSTARS.VELOCITY, str(row['Vhel']), e_value=str(row['e_Vhel']), source=source)
else:
catalog.entries[name].add_quantity(
FASTSTARS.VELOCITY, str(row['Vhel']), source=source)
galrad = float(str(row['RGC']))
dhel = rgc_to_dhel(galrad,gallon,gallat)
catalog.entries[name].add_quantity(
FASTSTARS.LUM_DIST, str(dhel), u_value='kpc', source=source,derived=True)
sptype = str(row['Type']).split('/')
for SPTYPE in sptype:
if SPTYPE == "sdO":
catalog.entries[name].add_quantity(
FASTSTARS.SPECTRAL_TYPE, "O", source=source)
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, "sd", source=source)
elif SPTYPE == "BHB":
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, "bhb", source=source)
else:
catalog.entries[name].add_quantity(
FASTSTARS.SPECTRAL_TYPE, SPTYPE, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, "d", source=source)
if str(row['ID'])[:3]=='HVS':
catalog.entries[name].add_quantity(
FASTSTARS.ALIAS, row['ID'], source=source)
catalog.journal_entries()
# 2012ApJ...744L..24L
datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII',
'apjl415156t1t2.csv')
data = read(datafile, format='csv')
for row in pbar(data, task_str):
oname = str(row['Catalog'])
name, source = catalog.new_entry(oname, bibcode='2012ApJ...744L..24L')
if (FASTSTARS.DISCOVERER not in catalog.entries[name]):
catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'Yinbi Li, Ali Luo, Gang Zhao, Youjun Lu, Juanjuan Ren, Fang Zuo', source)
catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2012), source)
catalog.entries[name].add_quantity(
FASTSTARS.ALIAS, row['ID'], source=source)
radec = oname.strip('SDSSJ')
radec = radec[0:2]+' '+radec[2:4]+' '+radec[4:9]+' '+radec[9:12]+' '+radec[12:14]+' '+radec[14:18]
ra, dec = coord(radec,
unit=(u.hourangle, u.deg)).to_string(
'hmsdms', sep=':').split()
catalog.entries[name].add_quantity(
FASTSTARS.RA, ra, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.DEC, dec, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.VELOCITY, str(row['Vhel']), e_value=str(row['e_Vhel']), source=source)
catalog.entries[name].add_quantity(
FASTSTARS.LUM_DIST, str(row['Dhel']), e_value=str(row['e_Dhel']), u_value='kpc', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.PROPER_MOTION_RA, str(row['pmra']), e_value=str(row['e_pmra']), u_value='mas/yr', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.PROPER_MOTION_DEC, str(row['pmdec']), e_value=str(row['e_pmdec']), u_value='mas/yr', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.SPECTRAL_TYPE, "F", source=source)
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, "d", source=source)
catalog.journal_entries()
# 2012ApJ...751...55B
datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII',
'apj427101t1_ascii.csv')
data = read(datafile, format='csv')
for row in pbar(data, task_str):
oname = str(row['Catalog']).replace(' ','')
name, source = catalog.new_entry(oname, bibcode='2012ApJ...751...55B')
if (FASTSTARS.DISCOVERER not in catalog.entries[name]):
catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'Warren R. Brown, Margaret J. Geller, Scott J. Kenyon', source)
catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2012), source)
if name == 'SDSSJ112359.47+751807.73':
sourcewarren = catalog.entries[name].add_source(name='Warren R. Brown (private comm.)',secondary=True)
catalog.entries[name].add_quantity(FASTSTARS.SPECTRAL_TYPE, 'B9', source=sourcewarren)
gallon = float(str(row['Glon']))
gallat = float(str(row['Glat']))
if (oname!='US708') & (oname!='HE0437-5439'):
radec = oname.strip('SDSSJ')
radec = radec[0:2]+' '+radec[2:4]+' '+radec[4:9]+' '+radec[9:12]+' '+radec[12:14]+' '+radec[14:]
ra, dec = coord(radec,
unit=(u.hourangle, u.deg)).to_string(
'hmsdms', sep=':').split()
catalog.entries[name].add_quantity(
FASTSTARS.RA, ra, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.DEC, dec, source=source)
if str(row['e_Vhel'])!='NA':
catalog.entries[name].add_quantity(
FASTSTARS.VELOCITY, str(row['Vhel']), e_value=str(row['e_Vhel']), source=source)
else:
catalog.entries[name].add_quantity(
FASTSTARS.VELOCITY, str(row['Vhel']), source=source)
galrad = float(str(row['RGC']))
dhel = rgc_to_dhel(galrad,gallon,gallat)
catalog.entries[name].add_quantity(
FASTSTARS.LUM_DIST, str(dhel), u_value='kpc', source=source, derived=True)
sptype = str(row['Type']).split('/')
for SPTYPE in sptype:
if SPTYPE != "NA":
catalog.entries[name].add_quantity(
FASTSTARS.SPECTRAL_TYPE, SPTYPE, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, "d", source=source)
if str(row['ID'])[:3]=='HVS':
catalog.entries[name].add_quantity(
FASTSTARS.ALIAS, row['ID'], source=source)
catalog.journal_entries()
# 2014ApJ...780....7P
datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII',
'apj485719t1_ascii.csv')
data = read(datafile, format='csv')
for row in pbar(data, task_str):
oname = 'SDSS'+str(row['Catalog']).replace(' ','')
name, source = catalog.new_entry(oname, bibcode='2014ApJ...780....7P')
if (FASTSTARS.DISCOVERER not in catalog.entries[name]):
catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'Lauren E. Palladino, Katharine J. Schlesinger, Kelly Holley-Bockelmann, Carlos Allende Prieto, Timothy C. Beers, Young Sun Lee, Donald P. Schneider', source)
catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2014), source)
catalog.entries[name].add_quantity(
FASTSTARS.ALIAS, 'Pal'+str(row['Pal']), source=source)
radec = oname.strip('SDSSJ')
radec = radec[0:2]+' '+radec[2:4]+' '+radec[4:9]+' '+radec[9:12]+' '+radec[12:14]+' '+radec[14:18]
ra, dec = coord(radec,
unit=(u.hourangle, u.deg)).to_string(
'hmsdms', sep=':').split()
catalog.entries[name].add_quantity(
FASTSTARS.RA, ra, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.DEC, dec, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.VELOCITY, str(row['Vhel']), source=source)
catalog.entries[name].add_quantity(
FASTSTARS.LUM_DIST, str(row['Dhel']), u_value='kpc', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.SPECTRAL_TYPE, "G", source=source)
catalog.entries[name].add_quantity(
FASTSTARS.SPECTRAL_TYPE, "K", source=source)
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, "d", source=source)
catalog.journal_entries()
# 2014ApJ...787...89B
datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII',
'apj494602t1_ascii.csv')
data = read(datafile, format='csv')
for row in pbar(data, task_str):
oname = str(row['Catalog']).replace(' ','')
name, source = catalog.new_entry(oname, bibcode='2014ApJ...787...89B')
if (FASTSTARS.DISCOVERER not in catalog.entries[name]):
catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'Warren R. Brown, Margaret J. Geller, Scott J. Kenyon', source)
catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2014), source)
radec = oname.strip('SDSSJ')
radec = radec[0:2]+' '+radec[2:4]+' '+radec[4:9]+' '+radec[9:12]+' '+radec[12:14]+' '+radec[14:19]
ra, dec = coord(radec,
unit=(u.hourangle, u.deg)).to_string(
'hmsdms', sep=':').split()
catalog.entries[name].add_quantity(
FASTSTARS.RA, ra, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.DEC, dec, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.VELOCITY, str(row['Vhel']), e_value=str(row['e_Vhel']), source=source)
if FASTSTARS.SPECTRAL_TYPE not in catalog.entries[name]:
sourceboubert = catalog.entries[name].add_source(bibcode='2018arXiv180410179B',secondary=True)
catalog.entries[name].add_quantity(FASTSTARS.SPECTRAL_TYPE, 'A', source=sourceboubert)
catalog.entries[name].add_quantity(FASTSTARS.SPECTRAL_TYPE, 'B', source=sourceboubert)
galrad = float(str(row['RGC']))
errgalrad = float(str(row['e_RGC']))
dhel = rgc_to_dhel(galrad,gallon,gallat)
dhel_lo = rgc_to_dhel(galrad-errgalrad,gallon,gallat)
dhel_hi = rgc_to_dhel(galrad+errgalrad,gallon,gallat)
catalog.entries[name].add_quantity(
FASTSTARS.LUM_DIST, str(dhel), e_lower_value=dhel-dhel_lo, e_upper_value=dhel_hi-dhel, u_value='kpc', source=source, derived=True)
if str(row['ID'])!='pBHVS':
catalog.entries[name].add_quantity(
FASTSTARS.ALIAS, 'HVS'+str(row['ID']), source=source)
if str(row['ID'])=='22':
catalog.entries[name].add_quantity(
FASTSTARS.SPECTRAL_TYPE, "B", source=source)
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, "d", source=source)
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, "bhb", source=source)
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, "g", source=source)
elif str(row['ID'])=='23':
catalog.entries[name].add_quantity(
FASTSTARS.SPECTRAL_TYPE, "B", source=source)
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, "d", source=source)
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, "bhb", source=source)
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, "g", source=source)
elif str(row['ID'])=='24':
catalog.entries[name].add_quantity(
FASTSTARS.SPECTRAL_TYPE, "B", source=source)
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, "d", source=source)
catalog.journal_entries()
# 2014EAS....67..255Z
datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII',
'1501.07824.csv')
data = read(datafile, format='csv')
for row in pbar(data, task_str):
oname = str(row['ID'])
name, source = catalog.new_entry(oname, bibcode='2014EAS....67..255Z')
if (FASTSTARS.DISCOVERER not in catalog.entries[name]):
catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER, 'Y. Q. Zhang, M. C. Smith, J. L. Carlin', source)
catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2014), source)
radeg = str(row['ra'])
decdeg = str(row['dec'])
ra, dec = coord(ra=radeg*u.deg, dec=decdeg*u.deg).to_string(
'hmsdms', sep=':').split()
catalog.entries[name].add_quantity(
FASTSTARS.RA, ra, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.DEC, dec, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.VELOCITY, str(row['Vhel']), e_value=str(row['e_Vhel']), source=source)
catalog.entries[name].add_quantity(
FASTSTARS.PROPER_MOTION_RA, str(row['pmra']), e_value=str(row['e_pmra']), u_value='mas/yr', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.PROPER_MOTION_DEC, str(row['pmdec']), e_value=str(row['e_pmdec']), u_value='mas/yr', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.LUM_DIST, str(row['Dhel']), u_value='kpc', source=source) # This distance may have some metallicity dependent uncertainty?
catalog.entries[name].add_quantity(
FASTSTARS.SPECTRAL_TYPE, 'K', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.SPECTRAL_TYPE, 'M', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, 'd', source=source)
catalog.journal_entries()
# 2014ApJ...789L...2Z
datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII',
'apjl496832t1t2t3_ascii.csv')
data = read(datafile, format='csv')
for row in pbar(data, task_str):
oname = str(row['ID'])
name, source = catalog.new_entry(oname, bibcode='2014ApJ...789L...2Z')
if (FASTSTARS.DISCOVERER not in catalog.entries[name]):
catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'Jing Zhong, Li Chen, Chao Liu, Richard de Grijs, Jinliang Hou, Shiyin Shen, Zhengyi Shao, Jing Li, Ali Luo, Jianrong Shi, Haotong Zhang, Ming Yang, Licai Deng, Ge Jin, Yong Zhang, Yonghui Hou, Zhenchao Zhang', source)
catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2014), source)
catalog.entries[name].add_quantity(
FASTSTARS.ALIAS, str(row['Catalog']), source=source)
radec = str(row['Catalog']).strip('J')
radec = radec[0:2]+' '+radec[2:4]+' '+radec[4:9]+' '+radec[9:12]+' '+radec[12:14]+' '+radec[14:18]
ra, dec = coord(radec,
unit=(u.hourangle, u.deg)).to_string(
'hmsdms', sep=':').split()
catalog.entries[name].add_quantity(
FASTSTARS.RA, ra, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.DEC, dec, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.VELOCITY, str(row['Vrb']), source=source)
catalog.entries[name].add_quantity(
FASTSTARS.PROPER_MOTION_RA, str(row['pmra']), e_value=str(row['e_pmra']), u_value='mas/yr', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.PROPER_MOTION_DEC, str(row['pmdec']), e_value=str(row['e_pmdec']), u_value='mas/yr', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.LUM_DIST, str(row['Dhel']), u_value='kpc', source=source) # This distance may have some metallicity dependent uncertainty?
catalog.entries[name].add_quantity(
FASTSTARS.SPECTRAL_TYPE, str(row['Type']), source=source)
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, 'd', source=source)
catalog.journal_entries()
# 2014ApJ...794..146T
datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII',
'ApJ794146.csv')
data = read(datafile, format='csv')
for row in pbar(data, task_str):
oname = str(row['Catalog']).replace(' ','')
name, source = catalog.new_entry(oname, bibcode='2014ApJ...794..146T')
if (FASTSTARS.DISCOVERER not in catalog.entries[name]):
catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'Christopher A. Theissen, Andrew A. West', source)
catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2014), source)
radec = oname.strip('SDSSJ')
radec = radec[0:2]+' '+radec[2:4]+' '+radec[4:9]+' '+radec[9:12]+' '+radec[12:14]+' '+radec[14:18]
ra, dec = coord(radec,
unit=(u.hourangle, u.deg)).to_string(
'hmsdms', sep=':').split()
catalog.entries[name].add_quantity(
FASTSTARS.RA, ra, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.DEC, dec, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.VELOCITY, str(row['Vhel']), e_value=str(row['e_Vhel']), source=source)
catalog.entries[name].add_quantity(
FASTSTARS.PROPER_MOTION_RA, str(row['pmra']), e_value=str(row['e_pmra']), u_value='mas/yr', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.PROPER_MOTION_DEC, str(row['pmdec']), e_value=str(row['e_pmdec']), u_value='mas/yr', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.LUM_DIST, str(row['Dhel']), e_value=str(row['e_Dhel']), u_value='kpc', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.SPECTRAL_TYPE, str(row['Type']), source=source)
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, 'd', source=source)
catalog.journal_entries()
# 2014ApJ...794..145S
datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII',
'apj501503t8_ascii.csv')
data = read(datafile, format='csv')
for row in pbar(data, task_str):
oname = str(row['Catalog'])
oname = oname[:4]+'J'+oname[4:]
name, source = catalog.new_entry(oname, bibcode='2014ApJ...794..145S')
if (FASTSTARS.DISCOVERER not in catalog.entries[name]):
catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'Antonia S. Savcheva, Andrew A. West, John J. Bochanski', source)
catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2014), source)
radec = oname.strip('SDSSJ')
radec = radec[0:2]+' '+radec[2:4]+' '+radec[4:8]+' '+radec[8:11]+' '+radec[11:13]+' '+radec[13:17]
ra, dec = coord(radec,
unit=(u.hourangle, u.deg)).to_string(
'hmsdms', sep=':').split()
catalog.entries[name].add_quantity(
FASTSTARS.RA, ra, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.DEC, dec, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.VELOCITY, str(row['Vhel']), source=source)
catalog.entries[name].add_quantity(
FASTSTARS.PROPER_MOTION_RA, str(row['pmra']), e_value=str(row['e_pmra']), u_value='mas/yr', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.PROPER_MOTION_DEC, str(row['pmdec']), e_value=str(row['e_pmdec']), u_value='mas/yr', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.LUM_DIST, str(row['Dhel']), e_value=str(row['e_Dhel']), u_value='kpc', source=source)
sptype = str(row['Type'])
catalog.entries[name].add_quantity(
FASTSTARS.SPECTRAL_TYPE, sptype[2:], source=source)
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, "sd", source=source)
catalog.journal_entries()
# 2015MNRAS.447.2046H
datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII',
'hawkins2015.csv')
data = read(datafile, format='csv')
for row in pbar(data, task_str):
oname = str(row['Catalog']).strip(' ')
oname = 'RAVE'+oname
name, source = catalog.new_entry(oname, bibcode='2015MNRAS.447.2046H')
if (FASTSTARS.DISCOVERER not in catalog.entries[name]):
catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'K. Hawkins, G. Kordopatis, G. Gilmore, T. Masseron, R. F. G. Wyse, G. Ruchti, O. Bienaymé, J. Bland-Hawthorn, C. Boeche, K. Freeman, B. K. Gibson, E. K. Grebel, A. Helmi, A. Kunder, U. Munari, J. F. Navarro, Q. A. Parker, W. A. Reid, R. D. Scholz, G. Seabroke, A. Siebert, M. Steinmetz, F. Watson, T. Zwitter', source)
catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2015), source)
radec = oname.strip('RAVEJ')
radec = radec[0:2]+' '+radec[2:4]+' '+radec[4:8]+' '+radec[8:11]+' '+radec[11:13]+' '+radec[13:15]
ra, dec = coord(radec,
unit=(u.hourangle, u.deg)).to_string(
'hmsdms', sep=':').split()
catalog.entries[name].add_quantity(
FASTSTARS.RA, ra, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.DEC, dec, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.VELOCITY, str(row['Vhel']).strip(' '), e_value=str(row['e_Vhel']).strip(' '), source=source)
catalog.entries[name].add_quantity(
FASTSTARS.PROPER_MOTION_RA, str(row['pmra']).strip(' '), e_value=str(row['e_pmra']).strip(' '), u_value='mas/yr', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.PROPER_MOTION_DEC, str(row['pmdec']).strip(' '), e_value=str(row['e_pmdec']).strip(' '), u_value='mas/yr', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.LUM_DIST, str(float(row['Dhel'])/1e3).strip(' '), e_value=str(float(row['e_Dhel'])/1e3).strip(' '), u_value='kpc', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, "g", source=source)
catalog.journal_entries()
# 2015A&A...576L..14Z
datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII',
'zeigerer.csv')
data = read(datafile, format='csv')
for row in pbar(data, task_str):
oname = 'Pal'+str(row['Pal'])
name, source = catalog.new_entry(oname, bibcode='2015A&A...576L..14Z')
if (FASTSTARS.DISCOVERER not in catalog.entries[name]):
catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'E. Ziegerer, M. Volkert, U. Heber, A. Irrgang, B. T. Gänsicke, S. Geier', source)
catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2015), source)
catalog.entries[name].add_quantity(
FASTSTARS.PROPER_MOTION_RA, str(row['pmra']), e_value=str(row['e_pmra']), u_value='mas/yr', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.PROPER_MOTION_DEC, str(row['pmdec']), e_value=str(row['e_pmdec']), u_value='mas/yr', source=source)
catalog.journal_entries()
# 2015ApJ...804...49B
datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII',
'apj510826t1_ascii.csv')
data = read(datafile, format='csv')
for row in pbar(data, task_str):
oname = str(row['Catalog']).strip(' ')
if oname[1]==':':
# Add the leading 0
oname = '0'+oname
oname = 'SDSSJ'+oname
name, source = catalog.new_entry(oname.replace(':',''), bibcode='2015ApJ...804...49B')
if (FASTSTARS.DISCOVERER not in catalog.entries[name]):
catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'Warren R. Brown, Jay Anderson, Oleg Y. Gnedin, Howard E. Bond, Margaret J. Geller, Scott J. Kenyon', source)
catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2015), source)
catalog.entries[name].add_quantity(
FASTSTARS.ALIAS, str(row['ID']), source=source)
catalog.entries[name].add_quantity(
FASTSTARS.PROPER_MOTION_RA, str(row['pmra']).strip(' '), e_value=str(row['e_pmra']).strip(' '), u_value='mas/yr', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.PROPER_MOTION_DEC, str(row['pmdec']).strip(' '), e_value=str(row['e_pmdec']).strip(' '), u_value='mas/yr', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.LUM_DIST, str(row['Dhel']).strip(' '), e_value=str(row['e_Dhel']).strip(' '), u_value='kpc', source=source)
if str(row['newspec']) == 'y':
radec = oname.strip('SDSSJ')
ra, dec = radec[:11], radec[11:]
catalog.entries[name].add_quantity(
FASTSTARS.RA, ra, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.DEC, dec, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.VELOCITY, str(row['Vhel']), e_value=str(row['e_Vhel']), source=source)
#catalog.entries[name].add_quantity(
# FASTSTARS.LUM_DIST, str(row['Dhel']), e_value=str(row['e_Dhel']), u_value='kpc', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.SPECTRAL_TYPE, str(row['Type']), source=source)
catalog.journal_entries()
# 2015RAA....15.1364L
datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII',
'li2015.csv')
data = read(datafile, format='csv')
for row in pbar(data, task_str):
oname = str(row['Catalog'])
oname = 'LAMOST'+oname
name, source = catalog.new_entry(oname, bibcode='2015RAA....15.1364L')
if (FASTSTARS.DISCOVERER not in catalog.entries[name]):
catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'Yin-Bi Li, A.-Li Luo, Gang Zhao, You-Jun Lu, Peng Wei, Bing Du, Xiang Li, Yong-Heng Zhao, Zhan-Wen Han, Bo Wang, Yue Wu, Yong Zhang, Yong-Hui Hou, Yue-Fei Wang, Ming Yang', source)
catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2015), source)
radec = oname.strip('LAMOSTJ')
radec = radec[0:2]+' '+radec[2:4]+' '+radec[4:9]+' '+radec[9:12]+' '+radec[12:14]+' '+radec[14:]
ra, dec = coord(radec,
unit=(u.hourangle, u.deg)).to_string(
'hmsdms', sep=':').split()
catalog.entries[name].add_quantity(
FASTSTARS.ALIAS, 'Li'+str(row['ID']), source=source)
catalog.entries[name].add_quantity(
FASTSTARS.RA, ra, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.DEC, dec, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.VELOCITY, str(row['Vhel']), e_value=str(row['e_Vhel']), source=source)
catalog.entries[name].add_quantity(
FASTSTARS.PROPER_MOTION_RA, str(row['pmra']), e_value=str(row['e_pmra']), u_value='mas/yr', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.PROPER_MOTION_DEC, str(row['pmdec']), e_value=str(row['e_pmdec']), u_value='mas/yr', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.LUM_DIST, str(row['Dhel']), e_value=str(row['e_Dhel']), u_value='kpc', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.SPECTRAL_TYPE, 'F', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.SPECTRAL_TYPE, 'G', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.SPECTRAL_TYPE, 'K', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, 'd', source=source)
catalog.journal_entries()
# 2015AJ....150...77V
datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII',
'vickers2015final.csv')
data = read(datafile, format='csv')
for row in pbar(data, task_str):
oname = str(row['Catalog'])
oname = 'SDSS'+oname
name, source = catalog.new_entry(oname, bibcode='2015AJ....150...77V')
if (FASTSTARS.DISCOVERER not in catalog.entries[name]):
catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'John J.Vickers, Martin C. Smith, Eva K. Grebel', source)
catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2015), source)
radec = oname.strip('SDSSJ')
radec = radec[0:2]+' '+radec[2:4]+' '+radec[4:9]+' '+radec[9:12]+' '+radec[12:14]+' '+radec[14:]
ra, dec = coord(radec,
unit=(u.hourangle, u.deg)).to_string(
'hmsdms', sep=':').split()
catalog.entries[name].add_quantity(
FASTSTARS.RA, ra, source=source)
catalog.entries[name].add_quantity(
FASTSTARS.DEC, dec, source=source)
if str(row['distance']) != '':
catalog.entries[name].add_quantity(
FASTSTARS.LUM_DIST, str(row['distance']), u_value='kpc', source=source))
catalog.entries[name].add_quantity(
FASTSTARS.SPECTRAL_TYPE, 'F', source=source)
catalog.entries[name].add_quantity(
FASTSTARS.SPECTRAL_TYPE, 'G', source=source)
continue
newitem['dec'] = item['dec'][0]['value']
if not is_number(newitem['dec'].split(':')[0]):
continue
newitem['raerr'] = float(item['ra'][0].get('e_value', 0))
newitem['decerr'] = float(item['dec'][0].get('e_value', 0))
# Temporary fix for David's typo
if newitem['dec'].count('.') == 2:
newitem['dec'] = newitem['dec'][:newitem['dec'].rfind('.')]
if 'distinctfrom' in item:
newitem['distinctfrom'] = [
x['value'] for x in item['distinctfrom']
]
newcatalog.append(newitem)
coo = coord([x['ra'] for x in newcatalog], [x['dec'] for x in newcatalog],
unit=(un.hourangle, un.deg))
path = download_file(
'http://cxc.harvard.edu/csc2/preliminary/preliminary_detlist.fits',
'astrocats/supernovae/output/cache')
cxctable = fits.getdata(path, 1)
cxccatalog = []
for ri, row in enumerate(tqdm(cxctable)):
if ri > 1000:
break
name = row[0]
if row[2] != 'TRUE' or row[34] != 'POINT' or row[5] == 'T':
continue
cxcdict = {
'name': name,
def do_mast_spectra(catalog):
"""Import HST spectra from MAST."""
task_str = catalog.get_current_task_str()
masturl = 'https://mast.stsci.edu'
mastref = 'MAST'
fureps = {'erg/cm2/s/A': 'erg/s/cm^2/Angstrom'}
histfile = os.path.join(
catalog.get_current_task_repo(), 'MAST', 'history.json')
if os.path.exists(histfile):
histdict = json.load(open(histfile, 'r'))
else:
histdict = OrderedDict()
# objectOfInterest = 'ASASSN-14li'
#
# resolverRequest = {'service': 'Mast.Name.Lookup',
# 'params': {'input': objectOfInterest,
# 'format': 'json'},
# }
#
# headers, resolvedObjectString = mastQuery(resolverRequest)
#
# resolvedObject = json.loads(resolvedObjectString)
#
# pp.pprint(resolvedObject)
# objRa = resolvedObject['resolvedCoordinate'][0]['ra']
# objDec = resolvedObject['resolvedCoordinate'][0]['decl']
objs = []
for entry in catalog.entries:
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[entry]:
dd = catalog.entries[entry][
TIDALDISRUPTION.DISCOVER_DATE][0][QUANTITY.VALUE]
try:
dt = datetime.strptime(dd, '%Y/%m/%d')
if dt < datetime(1997, 1, 1):
continue
except Exception:
pass
if (TIDALDISRUPTION.RA in catalog.entries[entry] and
TIDALDISRUPTION.DEC in catalog.entries[entry]):
objs.append([
entry,
catalog.entries[entry][
TIDALDISRUPTION.RA][0][QUANTITY.VALUE],
catalog.entries[entry][
TIDALDISRUPTION.DEC][0][QUANTITY.VALUE]])
if not len(objs):
return
objs = np.array(objs).T
coords = coord(objs[1], objs[2], unit=(un.hourangle, un.deg))
for ci, co in enumerate(pbar(coords, desc=task_str)):
entry = objs[0][ci]
use_cache = False
if entry in histdict:
if dt < datetime(datetime.now().year - 5, 1, 1):
use_cache = True
if (dt >= datetime(datetime.now().year - 5, 1, 1) and
time.time() - histdict[entry][0] < 30. * 86400.):
use_cache = True
if use_cache:
spectra = histdict[entry][1]
else:
objRa, objDec = tuple(co.to_string().split())
mastRequest = {'service': 'Mast.Caom.Cone',
'params': {'ra': objRa,
'dec': objDec,
'radius': 0.008},
'format': 'json',
'pagesize': 1000,
'page': 1,
'removenullcolumns': True,
'removecache': True}
try:
headers, mastDataString = mastQuery(mastRequest)
except Exception:
print('`mastQuery` failed for `{}`, skipping.'.format(entry))
continue
mastData = json.loads(mastDataString)
# print(mastData.keys())
# print("Query status:", mastData['status'])
mastDataTable = Table()
if 'fields' not in mastData:
print('`fields` not found for `{}`'.format(entry))
continue
for col, atype in [(
x['name'], x['type']) for x in mastData['fields']]:
if atype == "string":
atype = "str"
if atype == "boolean":
atype = "bool"
mastDataTable[col] = np.array(
[x.get(col, None) for x in mastData['data']], dtype=atype)
# print(mastDataTable.columns)
spectra = [
x for x in mastDataTable if
x['dataproduct_type'] == 'spectrum' and x[
'obs_collection'] == 'HST']
spectra = [{
'target_classification': x['target_classification'],
'obsid': x['obsid'],
't_min': x['t_min'],
't_max': x['t_max'],
'instrument_name': x['instrument_name'],
'proposal_pi': x['proposal_pi'],
} for x in spectra]
mjd = ''
instrument = ''
observer = ''
for si in range(len(spectra)):
spec = spectra[si]
# if all([x not in spec['target_classification'].upper()
# for x in ['SUPERNOVA', 'UNIDENTIFIED']]):
# continue
obsid = spec['obsid']
mjd = str(Decimal('0.5') * (
Decimal(str(spec['t_min'])) + Decimal(str(spec['t_max']))))
instrument = spec['instrument_name']
if ('STIS' not in instrument.upper() and
'COS' not in instrument.upper()):
continue
observer = spec['proposal_pi']
if use_cache:
scienceProducts = spec.get('sciProds', [])
else:
productRequest = {'service': 'Mast.Caom.Products',
'params': {'obsid': obsid},
'format': 'json',
'pagesize': 100,
'page': 1}
try:
headers, obsProductsString = mastQuery(productRequest)
except Exception:
print(
'`mastQuery` failed for `{}`, skipping.'.format(obsid))
continue
obsProducts = json.loads(obsProductsString)
if 'fields' not in obsProducts:
print('`fields` not found for `{}`'.format(obsid))
print(obsProducts)
continue
sciProdArr = [x for x in obsProducts['data']
if x.get("productType", None) == 'SCIENCE']
scienceProducts = OrderedDict()
for col, atype in [
(x['name'], x['type']) for x in obsProducts['fields']]:
if atype == "string":
atype = "str"
if atype == "boolean":
atype = "bool"
if atype == "int":
# array may contain nan values, and they do not exist
# in numpy integer arrays.
atype = "float"
scienceProducts[col] = [
x.get(col, None) for x in sciProdArr]
spectra[si]['sciProds'] = scienceProducts
# print("Number of science products:", len(scienceProducts))
# print(scienceProducts)
search_str = '_x1d.fits'
summed = False
if any(['_x1dsum.fits' in x for x in scienceProducts[
'productFilename']]):
search_str = '_x1dsum.fits'
summed = True
if any(['_sx1.fits' in x for x in scienceProducts[
'productFilename']]):
search_str = '_sx1.fits'
summed = True
for ri in range(len(scienceProducts['productFilename'])):
if search_str not in scienceProducts['productFilename'][ri]:
continue
filename = str(obsid) + "_" + \
scienceProducts['productFilename'][ri]
datafile = os.path.join(
catalog.get_current_task_repo(), 'MAST', filename)
if not os.path.exists(datafile):
# link is url, so can just dl
if "http" in scienceProducts['dataURI'][ri]:
urlretrieve(scienceProducts['dataURI'][ri], datafile)
else: # link is uri, need to go through direct dl request
server = 'mast.stsci.edu'
conn = httplib.HTTPSConnection(server)
conn.request(
"GET", "/api/v0/download/file/" +
scienceProducts['dataURI'][ri].lstrip('mast:'))
resp = conn.getresponse()
fileContent = resp.read()
with open(datafile, 'wb') as FLE:
FLE.write(fileContent)
conn.close()
try:
hdulist = fits.open(datafile)
except Exception:
print(
"Couldn't read `{}`, maybe private.".format(filename))
os.remove(datafile)
continue
for oi, obj in enumerate(hdulist[0].header):
if any(x in ['.', '/'] for x in obj):
del (hdulist[0].header[oi])
hdulist[0].verify('silentfix')
hdrkeys = list(hdulist[0].header.keys())
# print(hdrkeys)
name = entry
if not name:
name = hdulist[0].header['OBJECT']
name, source = catalog.new_entry(
name, srcname=mastref, url=masturl, secondary=True)
sources = [source]
if 'OBSERVER' in hdrkeys:
sources.append(
catalog.entries[name].add_source(
name=hdulist[0].header['OBSERVER']))
if observer:
source = catalog.entries[name].add_source(name=observer)
sources.append(source)
source = ','.join(sources)
if summed:
wcol = 3
fcol = 4
else:
wcol = 2
fcol = 3
try:
waves = [str(x) for x in list(hdulist[1].data)[0][wcol]]
fluxes = [str(x) for x in list(hdulist[1].data)[0][fcol]]
except Exception:
print('Failed to find waves/fluxes for `{}`.'.format(filename))
continue
if 'BUNIT' in hdrkeys:
fluxunit = hdulist[0].header['BUNIT']
if fluxunit in fureps:
fluxunit = fureps[fluxunit]
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.WAVELENGTHS: waves,
SPECTRUM.TIME: mjd,
SPECTRUM.U_TIME: 'MJD',
SPECTRUM.FLUXES: fluxes,
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.FILENAME: filename,
SPECTRUM.SOURCE: source
}
if 'TELESCOP' in hdrkeys:
specdict[SPECTRUM.TELESCOPE] = hdulist[0].header[
'TELESCOP']
if not instrument and 'INSTRUME' in hdrkeys:
instrument = hdulist[0].header['INSTRUME']
if instrument:
specdict[SPECTRUM.INSTRUMENT] = instrument
if 'SITENAME' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header[
'SITENAME']
elif 'OBSERVAT' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header[
'OBSERVAT']
if 'OBSERVER' in hdrkeys:
specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER']
catalog.entries[name].add_spectrum(**specdict)
if not use_cache and (ci % 100 == 0 or ci == len(coords) - 1):
histdict[entry] = [time.time(), spectra]
json.dump(histdict, open(histfile, 'w'),
indent='\t', separators=(',', ':'))
catalog.journal_entries()
return
if thistype.upper() in nontypes:
continue
elif thistype in ('Other', 'not Ia', 'SN', 'unconf', 'Radio',
'CC', 'CCSN', 'Candidate', 'nIa'):
evtypes.append('Unknown')
break
else:
evtypes.append(thistype)
break
else:
evtypes.append('Unknown')
tprint(thisevent['name'])
try:
c = coord(ra=thisevent['ra'][0]['value'],
dec=thisevent['dec'][0]['value'],
unit=(un.hourangle, un.deg))
except (KeyboardInterrupt, SystemExit):
raise
except:
warnings.warn('Mangled coordinate, skipping')
continue
else:
evnames.append(thisevent['name'])
rarad = c.ra.radian - pi
decrad = c.dec.radian
snhx = 2.0**1.5 * cos(decrad) * sin(rarad / 2.0) / \
sqrt(1.0 + cos(decrad) * cos(rarad / 2.0))
snhy = sqrt(2.0) * sin(decrad) / \
sqrt(1.0 + cos(decrad) * cos(rarad / 2.0))
evras.append(c.ra.deg)
thistype = thistype[0]
if thistype not in spectral_ordering:
continue
levtypes.append(thistype)
if not len(levtypes):
evtype = untype
elif moduletype == 'spectraltype':
evtype = [x for x in spectral_ordering if x in levtypes][0]
else:
evtype = levtypes[0]
else:
evtype = untype
try:
c = coord(ra=thisevent['ra'][0]['value'], dec=thisevent[
'dec'][0]['value'], unit=(un.hourangle, un.deg))
except (KeyboardInterrupt, SystemExit):
raise
except:
warnings.warn('Mangled coordinate, skipping {}.'.format(thisevent['name']))
continue
else:
evtypes.append(evtype)
evnames.append(thisevent['name'])
rarad = c.ra.radian - pi
decrad = c.dec.radian
snhx = 2.0**1.5 * cos(decrad) * sin(rarad / 2.0) / \
sqrt(1.0 + cos(decrad) * cos(rarad / 2.0))
snhy = sqrt(2.0) * sin(decrad) / \
sqrt(1.0 + cos(decrad) * cos(rarad / 2.0))
rastr = str(c.ra.deg)
if newsources:
cts.append(quantum['value'])
ctsources.append({'idtype':','.join([x['idtype'] for x in newsources]), 'id':','.join([x['id'] for x in newsources])})
edit = True if os.path.isfile('../sne-internal/' + get_event_filename(item['name']) + '.json') else False
if ras and decs and item['name'] and item['name'] not in ['SN2002fz']:
oralen = len(ras)
odeclen = len(decs)
if len(ras) > len(decs):
decs = decs + [decs[0] for x in range(len(ras) - len(decs))]
elif len(ras) < len(decs):
ras = ras + [ras[0] for x in range(len(decs) - len(ras))]
try:
coo = coord(ras, decs, unit = (un.hourangle, un.deg))
except:
warnings.warn('Mangled coordinate, setting to 0')
radegs = []
decdegs = []
else:
radegs = coo.ra.deg[:oralen]
decdegs = coo.dec.deg[:odeclen]
ras = ras[:oralen]
decs = decs[:odeclen]
if len(ras) != len(radegs):
tqdm.write('Mangled R.A. for ' + item['name'])
conflicts.append(OrderedDict([('name', item['name']), ('alias', item['alias']), ('edit',edit),
('quantity', 'ra'), ('difference', '?'), ('values', ras), ('sources', rasources)]))
def do_sdss(catalog):
task_str = catalog.get_current_task_str()
keys = list(catalog.entries.keys())
Mnames, Mradec = [], []
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
try:
name = catalog.add_entry(oname)
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
if (FASTSTARS.RA not in catalog.entries[name]
or FASTSTARS.DEC not in catalog.entries[name]):
continue
else:
Mnames.append(name)
Mradec.append(
str(catalog.entries[name][FASTSTARS.RA][0]['value']) +
str(catalog.entries[name][FASTSTARS.DEC][0]['value']))
c = coord(Mradec, unit=(un.hourangle, un.deg), frame='icrs')
# We must step through the data to query it, >100 is too many
maxstep = 100 # stepsize
Nentries = len(Mradec)
roundindex = np.zeros(
0
) # the stepping round that this star's data was acquired in, needed to connect the obj_id in the query to the name of the star
for i in range(int(Nentries / maxstep) + 1):
result_tmp = SDSS.query_crossid(
c[maxstep * i:min(maxstep * (i + 1), Nentries)],
timeout=200.,
photoobj_fields=[
'u', 'err_u', 'g', 'err_g', 'r', 'err_r', 'i', 'err_i', 'z',
'err_z', 'MJD'
])
roundindex = np.concatenate(
[roundindex, i * np.ones(len(result_tmp['obj_id']))])
if i == 0:
result = result_tmp
else:
result = vstack([result, result_tmp])
flagsuccess = result['obj_id']
listfilter = ['u', 'g', 'r', 'i', 'z']
for i in range(len(flagsuccess)):
Mi = int(flagsuccess[i].strip('obj_')) + maxstep * int(roundindex[i])
name = Mnames[Mi]
source = catalog.entries[name].add_source(
bibcode='2015ApJS..219...12A')
for j in range(5):
catalog.entries[name].add_photometry(
time=str(result[i]['MJD']),
u_time='MJD',
telescope='SDSS',
band=listfilter[j],
magnitude=str(result[i][listfilter[j]]),
e_magnitude=str(result[i]['err_' + listfilter[j]]),
source=source)
catalog.journal_entries()
return
def do_gaia(catalog):
# Disable warnings
warnings.filterwarnings("ignore")
catalog.log.warning(
'Some warnings are thrown by the Gaia module which do not affect the result of the Gaia queries.'
)
task_str = catalog.get_current_task_str()
keys = list(catalog.entries.keys())
cntgphot = 0
cntgast = 0
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
try:
name = catalog.add_entry(oname)
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
if (FASTSTARS.RA not in catalog.entries[name]
or FASTSTARS.DEC not in catalog.entries[name]):
continue
else:
Mradec = str(
catalog.entries[name][FASTSTARS.RA][0]['value']) + str(
catalog.entries[name][FASTSTARS.DEC][0]['value'])
print(Mradec)
c = coord(Mradec, unit=(un.hourangle, un.deg), frame='icrs')
result = silentgaiaquery(c, radius=un.Quantity(2.0, un.arcsecond))
if len(result['dist']) > 0:
cntgphot += 1
source = catalog.entries[name].add_source(
bibcode='2016A&A...595A...2G')
catalog.entries[name].add_photometry(
time=57023,
u_time='MJD',
telescope='Gaia',
band='G',
magnitude=str(result['phot_g_mean_mag'][0]),
source=source)
if result['parallax'][0] != '--':
cntgast += 1
catalog.log.warning(
'"{}" has Gaia astrometry.'.format(name))
catalog.entries[name].add_quantity(
FASTSTARS.PROPER_MOTION_RA,
str(result['pmra'][0]),
source,
e_value=str(result['pmra_error'][0]),
u_value='mas/yr')
catalog.entries[name].add_quantity(
FASTSTARS.PROPER_MOTION_DEC,
str(result['pmdec'][0]),
source,
e_value=str(result['pmdec_error'][0]),
u_value='mas/yr')
catalog.entries[name].add_quantity(
FASTSTARS.PARALLAX,
str(result['parallax'][0]),
source,
e_value=str(result['parallax_error'][0]),
u_value='mas')
# Convert parallax to distance
if (FASTSTARS.LUM_DIST in catalog.entries[name]):
catalog.log.warning(
'"{}" has distance from photmetric distance prior.'
.format(name))
distance, distance_error = parallax_to_distance(
result['parallax'][0], result['parallax_error'][0],
float(catalog.entries[name][FASTSTARS.LUM_DIST][0]
['value']),
float(catalog.entries[name][FASTSTARS.LUM_DIST][0]
['e_value']))
catalog.entries[name].add_quantity(
FASTSTARS.LUM_DIST,
str(distance),
e_value=str(distance_error),
u_value='kpc',
source=source,
derived=True)
else:
catalog.log.warning(
'"{}" has distance from Astraatmadja & Bailer-Jones (2016) prior.'
.format(name))
distance, distance_error = parallax_to_distance(
result['parallax'][0], result['parallax_error'][0])
catalog.entries[name].add_quantity(
FASTSTARS.LUM_DIST,
str(distance),
e_value=str(distance_error),
u_value='kpc',
source=source,
derived=True)
catalog.log.warning(
'"{}" have Gaia photometry and "{}" have Gaia astrometry.'.format(
cntgphot, cntgast))
catalog.journal_entries()
# Reactivate warnings
warnings.filterwarnings("default")
return
def do_mast_spectra(catalog):
"""Import HST spectra from MAST."""
task_str = catalog.get_current_task_str()
masturl = 'https://mast.stsci.edu'
mastref = 'MAST'
fureps = {'erg/cm2/s/A': 'erg/s/cm^2/Angstrom'}
histfile = os.path.join(catalog.get_current_task_repo(), 'MAST',
'history.json')
if os.path.exists(histfile):
histdict = json.load(open(histfile, 'r'))
else:
histdict = OrderedDict()
# objectOfInterest = 'ASASSN-14li'
#
# resolverRequest = {'service': 'Mast.Name.Lookup',
# 'params': {'input': objectOfInterest,
# 'format': 'json'},
# }
#
# headers, resolvedObjectString = mastQuery(resolverRequest)
#
# resolvedObject = json.loads(resolvedObjectString)
#
# pp.pprint(resolvedObject)
# objRa = resolvedObject['resolvedCoordinate'][0]['ra']
# objDec = resolvedObject['resolvedCoordinate'][0]['decl']
objs = []
for entry in catalog.entries:
if SUPERNOVA.DISCOVER_DATE in catalog.entries[entry]:
dd = catalog.entries[entry][SUPERNOVA.DISCOVER_DATE][0][
QUANTITY.VALUE]
try:
dt = datetime.strptime(dd, '%Y/%m/%d')
if dt < datetime(1997, 1, 1):
continue
except Exception:
pass
if (SUPERNOVA.RA in catalog.entries[entry]
and SUPERNOVA.DEC in catalog.entries[entry]):
objs.append([
entry, catalog.entries[entry][SUPERNOVA.RA][0][QUANTITY.VALUE],
catalog.entries[entry][SUPERNOVA.DEC][0][QUANTITY.VALUE]
])
if not len(objs):
return
objs = np.array(objs).T
coords = coord(objs[1], objs[2], unit=(un.hourangle, un.deg))
for ci, co in enumerate(pbar(coords, desc=task_str)):
entry = objs[0][ci]
use_cache = False
if entry in histdict:
if dt < datetime(datetime.now().year - 5, 1, 1):
use_cache = True
if (dt >= datetime(datetime.now().year - 5, 1, 1)
and time.time() - histdict[entry][0] < 30. * 86400.):
use_cache = True
if use_cache:
spectra = histdict[entry][1]
else:
objRa, objDec = tuple(co.to_string().split())
mastRequest = {
'service': 'Mast.Caom.Cone',
'params': {
'ra': objRa,
'dec': objDec,
'radius': 0.008
},
'format': 'json',
'pagesize': 1000,
'page': 1,
'removenullcolumns': True,
'removecache': True
}
try:
headers, mastDataString = mastQuery(mastRequest)
except Exception:
print('`mastQuery` failed for `{}`, request failed, skipping.'.
format(entry))
continue
try:
mastData = json.loads(mastDataString)
except Exception:
print(
'`mastQuery` failed for `{}`, couldn\'t decode JSON, skipping.'
.format(entry))
continue
# print(mastData.keys())
# print("Query status:", mastData['status'])
mastDataTable = Table()
if 'fields' not in mastData:
print('`fields` not found for `{}`'.format(entry))
continue
for col, atype in [(x['name'], x['type'])
for x in mastData['fields']]:
if atype == "string":
atype = "str"
if atype == "boolean":
atype = "bool"
mastDataTable[col] = np.array(
[x.get(col, None) for x in mastData['data']], dtype=atype)
# print(mastDataTable.columns)
spectra = [
x for x in mastDataTable if x['dataproduct_type'] == 'spectrum'
and x['obs_collection'] == 'HST'
]
spectra = [{
'target_classification': x['target_classification'],
'obsid': x['obsid'] if 'obsid' in x else None,
't_min': x['t_min'],
't_max': x['t_max'],
'instrument_name': x['instrument_name'],
'proposal_pi': x['proposal_pi'],
} for x in spectra]
mjd = ''
instrument = ''
observer = ''
for si in range(len(spectra)):
spec = spectra[si]
if all([
x not in spec['target_classification'].upper()
for x in ['SUPERNOVA', 'UNIDENTIFIED']
]):
continue
obsid = spec['obsid']
if obsid is None:
continue
mjd = str(
Decimal('0.5') *
(Decimal(str(spec['t_min'])) + Decimal(str(spec['t_max']))))
instrument = spec['instrument_name']
if ('STIS' not in instrument.upper()
and 'COS' not in instrument.upper()):
continue
observer = spec['proposal_pi']
if use_cache:
scienceProducts = spec.get('sciProds', [])
else:
productRequest = {
'service': 'Mast.Caom.Products',
'params': {
'obsid': obsid
},
'format': 'json',
'pagesize': 100,
'page': 1
}
try:
headers, obsProductsString = mastQuery(productRequest)
except Exception:
print(
'`mastQuery` failed for `{}`, skipping.'.format(obsid))
continue
obsProducts = json.loads(obsProductsString)
if 'fields' not in obsProducts:
print('`fields` not found for `{}`'.format(obsid))
print(obsProducts)
continue
sciProdArr = [
x for x in obsProducts['data']
if x.get("productType", None) == 'SCIENCE'
]
scienceProducts = OrderedDict()
for col, atype in [(x['name'], x['type'])
for x in obsProducts['fields']]:
if atype == "string":
atype = "str"
if atype == "boolean":
atype = "bool"
if atype == "int":
# array may contain nan values, and they do not exist
# in numpy integer arrays.
atype = "float"
scienceProducts[col] = [
x.get(col, None) for x in sciProdArr
]
spectra[si]['sciProds'] = scienceProducts
# print("Number of science products:", len(scienceProducts))
# print(scienceProducts)
search_str = '_x1d.fits'
summed = False
if any([
'_x1dsum.fits' in x
for x in scienceProducts['productFilename']
]):
search_str = '_x1dsum.fits'
summed = True
if any(
['_sx1.fits' in x
for x in scienceProducts['productFilename']]):
search_str = '_sx1.fits'
summed = True
for ri in range(len(scienceProducts['productFilename'])):
if search_str not in scienceProducts['productFilename'][ri]:
continue
filename = str(obsid) + "_" + \
scienceProducts['productFilename'][ri]
datafile = os.path.join(catalog.get_current_task_repo(),
'MAST', filename)
if not os.path.exists(datafile):
# link is url, so can just dl
if "http" in scienceProducts['dataURI'][ri]:
urlretrieve(scienceProducts['dataURI'][ri], datafile)
else: # link is uri, need to go through direct dl request
server = 'mast.stsci.edu'
conn = httplib.HTTPSConnection(server)
conn.request(
"GET", "/api/v0/download/file/" +
scienceProducts['dataURI'][ri].lstrip('mast:'))
resp = conn.getresponse()
fileContent = resp.read()
with open(datafile, 'wb') as FLE:
FLE.write(fileContent)
conn.close()
try:
hdulist = fits.open(datafile)
except Exception:
print(
"Couldn't read `{}`, maybe private.".format(filename))
os.remove(datafile)
continue
for oi, obj in enumerate(hdulist[0].header):
if any(x in ['.', '/'] for x in obj):
del (hdulist[0].header[oi])
hdulist[0].verify('silentfix')
hdrkeys = list(hdulist[0].header.keys())
# print(hdrkeys)
name = entry
if not name:
name = hdulist[0].header['OBJECT']
name, source = catalog.new_entry(name,
srcname=mastref,
url=masturl,
secondary=True)
sources = [source]
if 'OBSERVER' in hdrkeys:
sources.append(catalog.entries[name].add_source(
name=hdulist[0].header['OBSERVER']))
if observer:
source = catalog.entries[name].add_source(name=observer)
sources.append(source)
source = ','.join(sources)
if summed:
wcol = 3
fcol = 4
else:
wcol = 2
fcol = 3
try:
waves = [str(x) for x in list(hdulist[1].data)[0][wcol]]
fluxes = [str(x) for x in list(hdulist[1].data)[0][fcol]]
except Exception:
print('Failed to find waves/fluxes for `{}`.'.format(
filename))
continue
if 'BUNIT' in hdrkeys:
fluxunit = hdulist[0].header['BUNIT']
if fluxunit in fureps:
fluxunit = fureps[fluxunit]
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.WAVELENGTHS: waves,
SPECTRUM.TIME: mjd,
SPECTRUM.U_TIME: 'MJD',
SPECTRUM.FLUXES: fluxes,
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.FILENAME: filename,
SPECTRUM.SOURCE: source
}
if 'TELESCOP' in hdrkeys:
specdict[
SPECTRUM.TELESCOPE] = hdulist[0].header['TELESCOP']
if not instrument and 'INSTRUME' in hdrkeys:
instrument = hdulist[0].header['INSTRUME']
if instrument:
specdict[SPECTRUM.INSTRUMENT] = instrument
if 'SITENAME' in hdrkeys:
specdict[
SPECTRUM.OBSERVATORY] = hdulist[0].header['SITENAME']
elif 'OBSERVAT' in hdrkeys:
specdict[
SPECTRUM.OBSERVATORY] = hdulist[0].header['OBSERVAT']
if 'OBSERVER' in hdrkeys:
specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER']
catalog.entries[name].add_spectrum(**specdict)
if not use_cache and (ci % 100 == 0 or ci == len(coords) - 1):
histdict[entry] = [time.time(), spectra]
json.dump(histdict,
open(histfile, 'w'),
indent='\t',
separators=(',', ':'))
catalog.journal_entries()
return
if ('redshift' in item and 'host' not in item and 'ra' in item
and 'dec' in item and item['ra'] and item['dec']):
# if 'ra' in item and 'dec' in item and item['ra'] and item['dec']:
newitem['name'] = item['name']
newitem['alias'] = [x['value'] for x in item['alias']]
newitem['ra'] = item['ra'][0]['value']
newitem['dec'] = item['dec'][0]['value']
if 'redshift' in item:
newitem['redshift'] = item['redshift'][0]['value']
# Temporary fix for David's typo
if newitem['dec'].count('.') == 2:
newitem['dec'] = newitem['dec'][:newitem['dec'].rfind('.')]
newcatalog.append(newitem)
coo = coord([x['ra'] for x in newcatalog], [x['dec'] for x in newcatalog],
unit=(un.hourangle, un.deg))
for ci, co in enumerate(tqdm(coo)):
customSimbad = Simbad()
customSimbad.add_votable_fields('otype', 'z_value')
regstr = 'region(ICRS, ' + co.to_string('hmsdms') + ', 1m)'
print(regstr)
result_table = customSimbad.query_criteria(regstr, otype='Galaxy')
if result_table:
print(newcatalog[ci])
if 'redshift' in newcatalog[ci]:
print(newcatalog[ci]['redshift'])
print(result_table)
# Convert to array since that's what datatables expects
import gPhoton.gAperture
import numpy as np
from astropy import units as un
from astropy.coordinates import SkyCoord as coord
from astropy.time import Time as astrotime
with open('all-bright-sne.csv', 'r') as f:
rows = list(csv.reader(f, delimiter=',', quotechar='"'))
datas = []
for ri, row in enumerate(rows[1:]):
ddate = row[1].split(',')[0]
ra = row[2].split(',')[0]
dec = row[3].split(',')[0]
if not ddate or not ra or not dec:
continue
coo = coord(ra=ra, dec=dec, unit=(un.hourangle, un.deg))
dra, ddec = coo.ra.deg, coo.dec.deg
gtime = astrotime(ddate.replace('/', '-')).unix - 315964800.
tmin = gtime - 86400. * 365.0
tmax = gtime + 86400. * 365.0
try:
data = gPhoton.gAperture(band='NUV',
skypos=[dra, ddec],
radius=0.03,
annulus=[0.03, 0.04],
trange=[tmin, tmax],
stepsz=600.)
except:
data = None
if data is not None:
for key in data:
