def do_snls_photo(catalog):
task_str = catalog.get_current_task_str()
snls_path = os.path.join(catalog.get_current_task_repo(), 'SNLS-ugriz.dat')
data = list(csv.reader(open(snls_path, 'r'), delimiter=' ',
quotechar='"', skipinitialspace=True))
for row in pbar(data, task_str):
flux = row[3]
err = row[4]
# Being extra strict here with the flux constraint, see note below.
if float(flux) < 3.0 * float(err):
continue
name = 'SNLS-' + row[0]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2010A&A...523A...7G')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
band = row[1]
mjd = row[2]
sig = get_sig_digits(flux.split('E')[0]) + 1
# Conversion comes from SNLS-Readme
# NOTE: Datafiles avail for download suggest diff zeropoints than 30,
# need to inquire.
magnitude = pretty_num(30.0 - 2.5 * log10(float(flux)), sig=sig)
e_mag = pretty_num(
2.5 * log10(1.0 + float(err) / float(flux)), sig=sig)
# e_mag = pretty_num(2.5*(log10(float(flux) + float(err)) -
# log10(float(flux))), sig=sig)
catalog.entries[name].add_photometry(
time=mjd, band=band, magnitude=magnitude, e_magnitude=e_mag,
counts=flux, e_counts=err, source=source)
catalog.journal_entries()
return
def get_best_redshift(self):
bestsig = -1
bestkind = 10
for z in self['redshift']:
kind = PREF_KINDS.index(z['kind'] if 'kind' in z else '')
sig = get_sig_digits(z[QUANTITY.VALUE])
if sig > bestsig and kind <= bestkind:
bestz = z[QUANTITY.VALUE]
bestkind = kind
bestsig = sig
bestsrc = z['source']
return bestz, bestkind, bestsig, bestsrc
def set_pd_mag_from_flux_density(photodict,
fd='',
efd='',
lefd='',
uefd='',
sig=DEFAULT_UL_SIGMA):
"""Set photometry dictionary from a flux density measurement.
`fd` is assumed to be in microjanskys.
"""
with localcontext() as ctx:
if lefd == '' or uefd == '':
lefd = efd
uefd = efd
prec = max(
get_sig_digits(str(fd), strip_zeroes=False),
get_sig_digits(str(lefd), strip_zeroes=False),
get_sig_digits(str(uefd), strip_zeroes=False)) + 1
ctx.prec = prec
dlefd = Decimal(str(lefd))
duefd = Decimal(str(uefd))
if fd != '':
dfd = Decimal(str(fd))
dsig = Decimal(str(sig))
if fd == '' or float(fd) < DEFAULT_UL_SIGMA * float(uefd):
photodict[PHOTOMETRY.UPPER_LIMIT] = True
photodict[PHOTOMETRY.UPPER_LIMIT_SIGMA] = str(sig)
photodict[PHOTOMETRY.MAGNITUDE] = str(Decimal('23.9') - D25 * (
dsig * duefd).log10())
if fd:
photodict[PHOTOMETRY.E_UPPER_MAGNITUDE] = str(D25 * (
(dfd + duefd).log10() - dfd.log10()))
else:
photodict[PHOTOMETRY.MAGNITUDE] = str(Decimal('23.9') - D25 *
dfd.log10())
photodict[PHOTOMETRY.E_UPPER_MAGNITUDE] = str(D25 * (
(dfd + duefd).log10() - dfd.log10()))
photodict[PHOTOMETRY.E_LOWER_MAGNITUDE] = str(D25 * (
dfd.log10() - (dfd - dlefd).log10()))
def set_pd_mag_from_counts(photodict,
c='',
ec='',
lec='',
uec='',
zp=DEFAULT_ZP,
sig=DEFAULT_UL_SIGMA):
"""Set photometry dictionary from a counts measurement."""
with localcontext() as ctx:
if lec == '' or uec == '':
lec = ec
uec = ec
prec = max(get_sig_digits(str(c), strip_zeroes=False),
get_sig_digits(str(lec), strip_zeroes=False),
get_sig_digits(str(uec), strip_zeroes=False)) + 1
ctx.prec = prec
dlec = Decimal(str(lec))
duec = Decimal(str(uec))
if c != '':
dc = Decimal(str(c))
dzp = Decimal(str(zp))
dsig = Decimal(str(sig))
photodict[PHOTOMETRY.ZERO_POINT] = str(zp)
if c == '' or float(c) < DEFAULT_UL_SIGMA * float(uec):
photodict[PHOTOMETRY.UPPER_LIMIT] = True
photodict[PHOTOMETRY.UPPER_LIMIT_SIGMA] = str(sig)
photodict[PHOTOMETRY.MAGNITUDE] = str(dzp -
(D25 *
(dsig * duec).log10()))
dnec = Decimal('10.0')**(
(dzp - Decimal(photodict[PHOTOMETRY.MAGNITUDE])) / D25)
photodict[PHOTOMETRY.E_UPPER_MAGNITUDE] = str(
D25 * ((dnec + duec).log10() - dnec.log10()))
else:
photodict[PHOTOMETRY.MAGNITUDE] = str(dzp - D25 * dc.log10())
photodict[PHOTOMETRY.E_UPPER_MAGNITUDE] = str(
D25 * ((dc + duec).log10() - dc.log10()))
photodict[PHOTOMETRY.E_LOWER_MAGNITUDE] = str(
D25 * (dc.log10() - (dc - dlec).log10()))
def set_pd_mag_from_counts(photodict,
c,
ec='',
lec='',
uec='',
zp=DEFAULT_ZP,
sig=DEFAULT_UL_SIGMA):
with localcontext() as ctx:
if lec == '' or uec == '':
lec = ec
uec = ec
prec = max(
get_sig_digits(str(c)),
get_sig_digits(str(lec)), get_sig_digits(str(uec)))
ctx.prec = prec
dlec = Decimal(str(lec))
duec = Decimal(str(uec))
dc = Decimal(str(c))
dzp = Decimal(str(zp))
dsig = Decimal(str(sig))
photodict[PHOTOMETRY.ZERO_POINT] = str(zp)
if float(c) < DEFAULT_UL_SIGMA * float(uec):
photodict[PHOTOMETRY.UPPER_LIMIT] = True
photodict[PHOTOMETRY.UPPER_LIMIT_SIGMA] = str(sig)
photodict[PHOTOMETRY.MAGNITUDE] = str(dzp - (D25 * (dsig * duec
).log10()))
dnec = Decimal('10.0')**(
(dzp - Decimal(photodict[PHOTOMETRY.MAGNITUDE])) / D25)
photodict[PHOTOMETRY.E_UPPER_MAGNITUDE] = str(D25 * (
(dnec + duec).log10() - dnec.log10()))
else:
photodict[PHOTOMETRY.MAGNITUDE] = str(dzp - D25 * dc.log10())
photodict[PHOTOMETRY.E_UPPER_MAGNITUDE] = str(D25 * (
(dc + duec).log10() - dc.log10()))
photodict[PHOTOMETRY.E_LOWER_MAGNITUDE] = str(D25 * (
dc.log10() - (dc - dlec).log10()))
def set_pd_mag_from_counts(photodict,
c,
ec='',
lec='',
uec='',
zp=DEFAULT_ZP,
sig=DEFAULT_UL_SIGMA):
with localcontext() as ctx:
if lec == '' or uec == '':
lec = ec
uec = ec
prec = max(get_sig_digits(str(c)), get_sig_digits(str(lec)),
get_sig_digits(str(uec)))
ctx.prec = prec
dlec = Decimal(str(lec))
duec = Decimal(str(uec))
dc = Decimal(str(c))
dzp = Decimal(str(zp))
dsig = Decimal(str(sig))
photodict[PHOTOMETRY.ZERO_POINT] = str(zp)
if float(c) < DEFAULT_UL_SIGMA * float(uec):
photodict[PHOTOMETRY.UPPER_LIMIT] = True
photodict[PHOTOMETRY.UPPER_LIMIT_SIGMA] = str(sig)
photodict[PHOTOMETRY.MAGNITUDE] = str(dzp -
(D25 *
(dsig * duec).log10()))
dnec = Decimal('10.0')**(
(dzp - Decimal(photodict[PHOTOMETRY.MAGNITUDE])) / D25)
photodict[PHOTOMETRY.E_UPPER_MAGNITUDE] = str(
D25 * ((dnec + duec).log10() - dnec.log10()))
else:
photodict[PHOTOMETRY.MAGNITUDE] = str(dzp - D25 * dc.log10())
photodict[PHOTOMETRY.E_UPPER_MAGNITUDE] = str(
D25 * ((dc + duec).log10() - dc.log10()))
photodict[PHOTOMETRY.E_LOWER_MAGNITUDE] = str(
D25 * (dc.log10() - (dc - dlec).log10()))
def get_best_redshift(self, key=SUPERNOVA.REDSHIFT):
bestsig = -1
bestkind = None
for z in self[key]:
try:
kind = min([
z.kind_preference.index(x)
for x in listify(z.get(QUANTITY.KIND, []))
])
except:
kind = None
sig = get_sig_digits(z[QUANTITY.VALUE])
if (sig > bestsig and
((kind is None and bestkind is None) or kind <= bestkind)):
bestz = z[QUANTITY.VALUE]
bestkind = kind
bestsig = sig
bestsrc = z[QUANTITY.SOURCE]
return bestz, bestkind, bestsig, bestsrc
def get_best_redshift(self, key=SUPERNOVA.REDSHIFT):
bestsig = -1
bestkind = None
for z in self[key]:
try:
kind = min([
z.kind_preference.index(x)
for x in listify(z.get(QUANTITY.KIND, []))
])
except:
kind = None
sig = get_sig_digits(z[QUANTITY.VALUE])
if (sig > bestsig and ((kind is None and bestkind is None) or
kind <= bestkind)):
bestz = z[QUANTITY.VALUE]
bestkind = kind
bestsig = sig
bestsrc = z[QUANTITY.SOURCE]
return bestz, bestkind, bestsig, bestsrc
def do_snls_spectra(catalog):
"""
"""
task_str = catalog.get_current_task_str()
result = Vizier.get_catalogs('J/A+A/507/85/table1')
table = result[list(result.keys())[0]]
table.convert_bytestring_to_unicode()
datedict = {}
for row in table:
datedict['SNLS-' + row['SN']] = str(astrotime(row['Date']).mjd)
oldname = ''
file_names = glob(os.path.join(catalog.get_current_task_repo(), 'SNLS/*'))
for fi, fname in enumerate(pbar_strings(file_names, task_str)):
filename = os.path.basename(fname)
fileparts = filename.split('_')
name = 'SNLS-' + fileparts[1]
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2009A&A...507...85B')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE,
'20' + fileparts[1][:2], source)
f = open(fname, 'r')
data = csv.reader(f, delimiter=' ', skipinitialspace=True)
specdata = []
for r, row in enumerate(data):
if row[0] == '@TELESCOPE':
telescope = row[1].strip()
elif row[0] == '@REDSHIFT':
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT,
row[1].strip(), source)
if r < 14:
continue
specdata.append(list(filter(None, [x.strip(' \t') for x in row])))
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[1]
fluxes = [
pretty_num(float(x) * 1.e-16, sig=get_sig_digits(x))
for x in specdata[2]
]
# FIX: this isnt being used
errors = [
pretty_num(float(x) * 1.e-16, sig=get_sig_digits(x))
for x in specdata[3]
]
fluxunit = 'erg/s/cm^2/Angstrom'
specdict = {
SPECTRUM.WAVELENGTHS: wavelengths,
SPECTRUM.FLUXES: fluxes,
SPECTRUM.ERRORS: errors,
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.U_ERRORS: fluxunit,
SPECTRUM.TELESCOPE: telescope,
SPECTRUM.FILENAME: filename,
SPECTRUM.SOURCE: source
}
if name in datedict:
specdict[SPECTRUM.TIME] = datedict[name]
specdict[SPECTRUM.U_TIME] = 'MJD'
catalog.entries[name].add_spectrum(**specdict)
if catalog.args.travis and fi >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
return
def do_nedd(catalog):
task_str = catalog.get_current_task_str()
nedd_path = os.path.join(
catalog.get_current_task_repo(), 'NED26.05.1-D-12.1.0-20160501.csv')
f = open(nedd_path, 'r')
data = sorted(list(csv.reader(f, delimiter=',', quotechar='"'))[
13:], key=lambda x: (x[9], x[3]))
reference = "NED-D"
refurl = "http://ned.ipac.caltech.edu/Library/Distances/"
nedd_dict = OrderedDict()
olddistname = ''
for r, row in enumerate(pbar(data, task_str)):
if r <= 12:
continue
distname = row[3]
name = name_clean(distname)
# distmod = row[4]
# moderr = row[5]
dist = row[6]
bibcode = unescape(row[8])
snname = name_clean(row[9])
redshift = row[10]
cleanhost = ''
if name != snname and (name + ' HOST' != snname):
cleanhost = host_clean(distname)
if cleanhost.endswith(' HOST'):
cleanhost = ''
if not is_number(dist):
print(dist)
if dist:
nedd_dict.setdefault(cleanhost, []).append(Decimal(dist))
if snname and 'HOST' not in snname:
snname, secondarysource = catalog.new_entry(
snname, srcname=reference, url=refurl, secondary=True)
if bibcode:
source = catalog.entries[snname].add_source(bibcode=bibcode)
sources = uniq_cdl([source, secondarysource])
else:
sources = secondarysource
if name == snname:
if redshift:
catalog.entries[snname].add_quantity(
'redshift', redshift, sources)
if dist:
catalog.entries[snname].add_quantity(
'comovingdist', dist, sources)
if not redshift:
try:
zatval = z_at_value(cosmo.comoving_distance,
float(dist) * un.Mpc, zmax=5.0)
sigd = get_sig_digits(str(dist))
redshift = pretty_num(zatval, sig=sigd)
except (KeyboardInterrupt, SystemExit):
raise
except:
pass
else:
cosmosource = catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
combsources = uniq_cdl(sources.split(',') +
[cosmosource])
catalog.entries[snname].add_quantity('redshift',
redshift,
combsources)
if cleanhost:
catalog.entries[snname].add_quantity(
'host', cleanhost, sources)
if catalog.args.update and olddistname != distname:
catalog.journal_entries()
olddistname = distname
catalog.journal_entries()
f.close()
return
def do_cleanup(catalog):
"""Task to cleanup catalog before final write."""
task_str = catalog.get_current_task_str()
# Set preferred names, calculate some columns based on imported data,
# sanitize some fields
keys = catalog.entries.copy().keys()
cleanupcnt = 0
for oname in pbar(keys, task_str):
name = catalog.add_entry(oname)
# Set the preferred name, switching to that name if name changed.
name = catalog.entries[name].set_preferred_name()
aliases = catalog.entries[name].get_aliases()
catalog.entries[name].set_first_max_light()
if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['MLS', 'SSS', 'CSS', 'GRB ']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix) and
is_number(alias.replace(prefix, '')[:2])):
discoverdate = ('/'.join([
'20' + alias.replace(prefix, '')[:2],
alias.replace(prefix, '')[2:4],
alias.replace(prefix, '')[4:6]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]:
break
if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]:
prefixes = [
'ASASSN-', 'PS1-', 'PS1', 'PS', 'iPTF', 'PTF', 'SCP-', 'SNLS-',
'SPIRITS', 'LSQ', 'DES', 'SNHiTS', 'Gaia', 'GND', 'GNW', 'GSD',
'GSW', 'EGS', 'COS', 'OGLE', 'HST'
]
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix) and
is_number(alias.replace(prefix, '')[:2]) and
is_number(alias.replace(prefix, '')[:1])):
discoverdate = '20' + alias.replace(prefix, '')[:2]
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]:
break
if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['SNF']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix) and
is_number(alias.replace(prefix, '')[:4])):
discoverdate = ('/'.join([
alias.replace(prefix, '')[:4],
alias.replace(prefix, '')[4:6],
alias.replace(prefix, '')[6:8]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]:
break
if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['PTFS', 'SNSDF']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix) and
is_number(alias.replace(prefix, '')[:2])):
discoverdate = ('/'.join([
'20' + alias.replace(prefix, '')[:2],
alias.replace(prefix, '')[2:4]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]:
break
if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['AT', 'SN', 'OGLE-', 'SM ', 'KSN-']
for alias in aliases:
for prefix in prefixes:
if alias.startswith(prefix):
year = re.findall(r'\d+', alias)
if len(year) == 1:
year = year[0]
else:
continue
if alias.replace(prefix, '').index(year) != 0:
continue
if (year and is_number(year) and '.' not in year and
len(year) <= 4):
discoverdate = year
if catalog.args.verbose:
tprint('Added discoverdate from name [' +
alias + ']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]:
break
if (TIDALDISRUPTION.RA not in catalog.entries[name] or
TIDALDISRUPTION.DEC not in catalog.entries[name]):
prefixes = [
'PSN J', 'MASJ', 'CSS', 'SSS', 'MASTER OT J', 'HST J', 'TCP J',
'MACS J', '2MASS J', 'EQ J', 'CRTS J', 'SMT J'
]
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix) and
is_number(alias.replace(prefix, '')[:6])):
noprefix = alias.split(':')[-1].replace(
prefix, '').replace('.', '')
decsign = '+' if '+' in noprefix else '-'
noprefix = noprefix.replace('+', '|').replace('-', '|')
nops = noprefix.split('|')
if len(nops) < 2:
continue
rastr = nops[0]
decstr = nops[1]
ra = ':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + \
('.' + rastr[6:] if len(rastr) > 6 else '')
dec = (decsign + ':'.join(
[decstr[:2], decstr[2:4], decstr[4:6]]) +
('.' + decstr[6:] if len(decstr) > 6 else ''))
if catalog.args.verbose:
tprint('Added ra/dec from name: ' + ra + ' ' + dec)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.RA, ra, source, derived=True)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DEC, dec, source, derived=True)
break
if TIDALDISRUPTION.RA in catalog.entries[name]:
break
no_host = (TIDALDISRUPTION.HOST not in catalog.entries[name] or
not any([
x[QUANTITY.VALUE] == 'Milky Way'
for x in catalog.entries[name][TIDALDISRUPTION.HOST]
]))
if (TIDALDISRUPTION.RA in catalog.entries[name] and
TIDALDISRUPTION.DEC in catalog.entries[name] and no_host):
from astroquery.irsa_dust import IrsaDust
if name not in catalog.extinctions_dict:
try:
ra_dec = (catalog.entries[name][TIDALDISRUPTION.RA][0][
QUANTITY.VALUE] + " " + catalog.entries[name][
TIDALDISRUPTION.DEC][0][QUANTITY.VALUE])
result = IrsaDust.get_query_table(ra_dec, section='ebv')
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
warnings.warn("Coordinate lookup for " + name +
" failed in IRSA.")
else:
ebv = result['ext SandF mean'][0]
ebverr = result['ext SandF std'][0]
catalog.extinctions_dict[name] = [ebv, ebverr]
if name in catalog.extinctions_dict:
sources = uniq_cdl([
catalog.entries[name].add_self_source(),
catalog.entries[name]
.add_source(bibcode='2011ApJ...737..103S')
])
(catalog.entries[name].add_quantity(
TIDALDISRUPTION.EBV,
str(catalog.extinctions_dict[name][0]),
sources,
e_value=str(catalog.extinctions_dict[name][1]),
derived=True))
if ((TIDALDISRUPTION.HOST in catalog.entries[name] and
(TIDALDISRUPTION.HOST_RA not in catalog.entries[name] or
TIDALDISRUPTION.HOST_DEC not in catalog.entries[name]))):
for host in catalog.entries[name][TIDALDISRUPTION.HOST]:
alias = host[QUANTITY.VALUE]
if ' J' in alias and is_number(alias.split(' J')[-1][:6]):
noprefix = alias.split(' J')[-1].split(':')[-1].replace(
'.', '')
decsign = '+' if '+' in noprefix else '-'
noprefix = noprefix.replace('+', '|').replace('-', '|')
nops = noprefix.split('|')
if len(nops) < 2:
continue
rastr = nops[0]
decstr = nops[1]
hostra = (':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) +
('.' + rastr[6:] if len(rastr) > 6 else ''))
hostdec = decsign + ':'.join([
decstr[:2], decstr[2:4], decstr[4:6]
]) + ('.' + decstr[6:] if len(decstr) > 6 else '')
if catalog.args.verbose:
tprint('Added hostra/hostdec from name: ' + hostra +
' ' + hostdec)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.HOST_RA, hostra, source, derived=True)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.HOST_DEC,
hostdec,
source,
derived=True)
break
if TIDALDISRUPTION.HOST_RA in catalog.entries[name]:
break
if (TIDALDISRUPTION.REDSHIFT not in catalog.entries[name] and
TIDALDISRUPTION.VELOCITY in catalog.entries[name]):
# Find the "best" velocity to use for this
bestsig = 0
for hv in catalog.entries[name][TIDALDISRUPTION.VELOCITY]:
sig = get_sig_digits(hv[QUANTITY.VALUE])
if sig > bestsig:
besthv = hv[QUANTITY.VALUE]
bestsrc = hv['source']
bestsig = sig
if bestsig > 0 and is_number(besthv):
voc = float(besthv) * 1.e5 / CLIGHT
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
(catalog.entries[name].add_quantity(
TIDALDISRUPTION.REDSHIFT,
pretty_num(
sqrt((1. + voc) / (1. - voc)) - 1., sig=bestsig),
sources,
kind='heliocentric',
derived=True))
if (TIDALDISRUPTION.REDSHIFT not in catalog.entries[name] and
len(catalog.nedd_dict) > 0 and
TIDALDISRUPTION.HOST in catalog.entries[name]):
reference = "NED-D"
refurl = "http://ned.ipac.caltech.edu/Library/Distances/"
for host in catalog.entries[name][TIDALDISRUPTION.HOST]:
if host[QUANTITY.VALUE] in catalog.nedd_dict:
source = catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
secondarysource = catalog.entries[name].add_source(
name=reference, url=refurl, secondary=True)
meddist = statistics.median(catalog.nedd_dict[host[
QUANTITY.VALUE]])
redz = z_at_value(cosmo.comoving_distance,
float(meddist) * un.Mpc)
redshift = pretty_num(
redz, sig=get_sig_digits(str(meddist)))
catalog.entries[name].add_quantity(
TIDALDISRUPTION.REDSHIFT,
redshift,
uniq_cdl([source, secondarysource]),
kind='host',
derived=True)
if (TIDALDISRUPTION.MAX_ABS_MAG not in catalog.entries[name] and
TIDALDISRUPTION.MAX_APP_MAG in catalog.entries[name] and
TIDALDISRUPTION.LUM_DIST in catalog.entries[name]):
# Find the "best" distance to use for this
bestsig = 0
for ld in catalog.entries[name][TIDALDISRUPTION.LUM_DIST]:
sig = get_sig_digits(ld[QUANTITY.VALUE])
if sig > bestsig:
bestld = ld[QUANTITY.VALUE]
bestsrc = ld['source']
bestsig = sig
if bestsig > 0 and is_number(bestld) and float(bestld) > 0.:
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
bestldz = z_at_value(cosmo.luminosity_distance,
float(bestld) * un.Mpc)
pnum = (float(catalog.entries[name][
TIDALDISRUPTION.MAX_APP_MAG][0][QUANTITY.VALUE]) - 5.0 *
(log10(float(bestld) * 1.0e6) - 1.0
) + 2.5 * log10(1.0 + bestldz))
pnum = pretty_num(pnum, sig=bestsig)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.MAX_ABS_MAG, pnum, sources, derived=True)
if TIDALDISRUPTION.REDSHIFT in catalog.entries[name]:
# Find the "best" redshift to use for this
bestz, bestkind, bestsig, bestsrc = catalog.entries[
name].get_best_redshift()
if bestsig > 0:
try:
bestz = float(bestz)
except Exception:
print(catalog.entries[name])
raise
if TIDALDISRUPTION.VELOCITY not in catalog.entries[name]:
source = catalog.entries[name].add_self_source()
# FIX: what's happening here?!
pnum = CLIGHT / KM * \
((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.)
pnum = pretty_num(pnum, sig=bestsig)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.VELOCITY,
pnum,
source,
kind=PREF_KINDS[bestkind],
derived=True)
if bestz > 0.:
from astropy.cosmology import Planck15 as cosmo
if TIDALDISRUPTION.LUM_DIST not in catalog.entries[name]:
dl = cosmo.luminosity_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name]
.add_source(bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
TIDALDISRUPTION.LUM_DIST,
pretty_num(
dl.value, sig=bestsig),
sources,
kind=PREF_KINDS[bestkind],
derived=True)
if (TIDALDISRUPTION.MAX_ABS_MAG not in
catalog.entries[name] and
TIDALDISRUPTION.MAX_APP_MAG in
catalog.entries[name]):
source = catalog.entries[name].add_self_source()
pnum = pretty_num(
float(catalog.entries[name][
TIDALDISRUPTION.MAX_APP_MAG][0][
QUANTITY.VALUE]) - 5.0 *
(log10(dl.to('pc').value) - 1.0
) + 2.5 * log10(1.0 + bestz),
sig=bestsig + 1)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.MAX_ABS_MAG,
pnum,
sources,
derived=True)
if TIDALDISRUPTION.COMOVING_DIST not in catalog.entries[
name]:
cd = cosmo.comoving_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name]
.add_source(bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
TIDALDISRUPTION.COMOVING_DIST,
pretty_num(
cd.value, sig=bestsig),
sources,
derived=True)
if all([
x in catalog.entries[name]
for x in [
TIDALDISRUPTION.RA, TIDALDISRUPTION.DEC,
TIDALDISRUPTION.HOST_RA, TIDALDISRUPTION.HOST_DEC
]
]):
# For now just using first coordinates that appear in entry
try:
c1 = coord(
ra=catalog.entries[name][TIDALDISRUPTION.RA][0][
QUANTITY.VALUE],
dec=catalog.entries[name][TIDALDISRUPTION.DEC][0][
QUANTITY.VALUE],
unit=(un.hourangle, un.deg))
c2 = coord(
ra=catalog.entries[name][TIDALDISRUPTION.HOST_RA][0][
QUANTITY.VALUE],
dec=catalog.entries[name][TIDALDISRUPTION.HOST_DEC][0][
QUANTITY.VALUE],
unit=(un.hourangle, un.deg))
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
pass
else:
sources = uniq_cdl(
[catalog.entries[name].add_self_source()] + catalog.
entries[name][TIDALDISRUPTION.RA][0]['source'].split(',') +
catalog.entries[name][TIDALDISRUPTION.DEC][0]['source'].
split(',') + catalog.entries[name][TIDALDISRUPTION.HOST_RA]
[0]['source'].split(',') + catalog.entries[name][
TIDALDISRUPTION.HOST_DEC][0]['source'].split(','))
if 'hostoffsetang' not in catalog.entries[name]:
hosa = Decimal(
hypot(c1.ra.degree - c2.ra.degree, c1.dec.degree -
c2.dec.degree))
hosa = pretty_num(hosa * Decimal(3600.))
catalog.entries[name].add_quantity(
TIDALDISRUPTION.HOST_OFFSET_ANG,
hosa,
sources,
derived=True,
u_value='arcseconds')
if (TIDALDISRUPTION.COMOVING_DIST in catalog.entries[name] and
TIDALDISRUPTION.REDSHIFT in catalog.entries[name] and
TIDALDISRUPTION.HOST_OFFSET_DIST not in
catalog.entries[name]):
offsetsig = get_sig_digits(catalog.entries[name][
TIDALDISRUPTION.HOST_OFFSET_ANG][0][QUANTITY.VALUE])
sources = uniq_cdl(
sources.split(',') + (catalog.entries[name][
TIDALDISRUPTION.COMOVING_DIST][0]['source']).
split(',') + (catalog.entries[name][
TIDALDISRUPTION.REDSHIFT][0]['source']).split(','))
(catalog.entries[name].add_quantity(
TIDALDISRUPTION.HOST_OFFSET_DIST,
pretty_num(
float(catalog.entries[name][
TIDALDISRUPTION.HOST_OFFSET_ANG][0][
QUANTITY.VALUE]) / 3600. * (pi / 180.) *
float(catalog.entries[name][
TIDALDISRUPTION.COMOVING_DIST][0][
QUANTITY.VALUE]) * 1000. /
(1.0 + float(catalog.entries[name][
TIDALDISRUPTION.REDSHIFT][0][QUANTITY.VALUE])),
sig=offsetsig),
sources))
catalog.entries[name].sanitize()
catalog.journal_entries(bury=True, final=True, gz=True)
cleanupcnt = cleanupcnt + 1
if catalog.args.travis and cleanupcnt % 1000 == 0:
break
catalog.save_caches()
return
def do_snls_spectra(catalog):
"""
"""
task_str = catalog.get_current_task_str()
result = Vizier.get_catalogs('J/A+A/507/85/table1')
table = result[list(result.keys())[0]]
table.convert_bytestring_to_unicode(python3_only=True)
datedict = {}
for row in table:
datedict['SNLS-' + row['SN']] = str(astrotime(row['Date']).mjd)
oldname = ''
file_names = glob(os.path.join(catalog.get_current_task_repo(), 'SNLS/*'))
for fi, fname in enumerate(pbar_strings(file_names, task_str)):
filename = os.path.basename(fname)
fileparts = filename.split('_')
name = 'SNLS-' + fileparts[1]
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2009A&A...507...85B')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, '20' + fileparts[1][:2], source)
f = open(fname, 'r')
data = csv.reader(f, delimiter=' ', skipinitialspace=True)
specdata = []
for r, row in enumerate(data):
if row[0] == '@TELESCOPE':
telescope = row[1].strip()
elif row[0] == '@REDSHIFT':
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, row[1].strip(), source)
if r < 14:
continue
specdata.append(list(filter(None, [x.strip(' \t') for x in row])))
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[1]
fluxes = [pretty_num(float(x) * 1.e-16, sig=get_sig_digits(x))
for x in specdata[2]]
# FIX: this isnt being used
# errors = [pretty_num(float(x)*1.e-16, sig=get_sig_digits(x)) for x in
# specdata[3]]
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom', u_fluxes='erg/s/cm^2/Angstrom',
wavelengths=wavelengths,
fluxes=fluxes, u_time='MJD' if name in datedict else '',
time=datedict[name] if name in datedict else '',
telescope=telescope, source=source,
filename=filename)
if catalog.args.travis and fi >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
return
def radec_clean(svalue, quantity, unit=''):
if unit == 'floatdegrees':
if not is_number(svalue):
return (svalue, unit)
deg = float('%g' % Decimal(svalue))
sig = get_sig_digits(svalue)
if 'ra' in quantity:
flhours = deg / 360.0 * 24.0
hours = floor(flhours)
minutes = floor((flhours - hours) * 60.0)
seconds = (flhours * 60.0 - (hours * 60.0 + minutes)) * 60.0
hours = 0 if hours < 1.e-6 else hours
minutes = 0 if minutes < 1.e-6 else minutes
seconds = 0.0 if seconds < 1.e-6 else seconds
if seconds > 60.0:
raise(ValueError('Invalid seconds value for ' + quantity))
svalue = str(hours).zfill(2) + ':' + str(minutes).zfill(2) + \
':' + zpad(pretty_num(seconds, sig=sig - 1))
elif 'dec' in quantity:
fldeg = abs(deg)
degree = floor(fldeg)
minutes = floor((fldeg - degree) * 60.0)
seconds = (fldeg * 60.0 - (degree * 60.0 + minutes)) * 60.0
if seconds > 60.0:
raise(ValueError('Invalid seconds value for ' + quantity))
svalue = (('+' if deg >= 0.0 else '-') +
str(degree).strip('+-').zfill(2) + ':' +
str(minutes).zfill(2) + ':' +
zpad(pretty_num(seconds, sig=sig - 1)))
elif unit == 'nospace' and 'ra' in quantity:
svalue = svalue[:2] + ':' + svalue[2:4] + \
((':' + zpad(svalue[4:])) if len(svalue) > 4 else '')
elif unit == 'nospace' and 'dec' in quantity:
if svalue.startswith(('+', '-')):
svalue = svalue[:3] + ':' + svalue[3:5] + \
((':' + zpad(svalue[5:])) if len(svalue) > 5 else '')
else:
svalue = '+' + svalue[:2] + ':' + svalue[2:4] + \
((':' + zpad(svalue[4:])) if len(svalue) > 4 else '')
else:
svalue = svalue.replace(' ', ':')
if 'dec' in quantity:
valuesplit = svalue.split(':')
svalue = (('-' if valuesplit[0].startswith('-') else '+') +
valuesplit[0].strip('+-').zfill(2) +
(':' + valuesplit[1].zfill(2) if
len(valuesplit) > 1 else '') +
(':' + zpad(valuesplit[2]) if
len(valuesplit) > 2 else ''))
if 'ra' in quantity:
sunit = 'hours'
elif 'dec' in quantity:
sunit = 'degrees'
# Correct case of arcseconds = 60.0.
valuesplit = svalue.split(':')
if len(valuesplit) == 3 and valuesplit[-1] in ["60.0", "60.", "60"]:
svalue = valuesplit[0] + ':' + str(Decimal(valuesplit[1]) +
Decimal(1.0)) + ':' + "00.0"
# Strip trailing dots.
svalue = svalue.rstrip('.')
return (svalue, sunit)
def add_quantity(self,
quantities,
value,
source,
forcereplacebetter=False,
**kwargs):
success = super(Supernova, self).add_quantity(quantities, value,
source, **kwargs)
if not success:
return
for quantity in listify(quantities):
my_quantity_list = self.get(quantity, [])
if ((forcereplacebetter or quantity.replace_better)
and len(my_quantity_list) > 1):
# The quantity that was just added should be last in the list
added_quantity = my_quantity_list.pop()
newquantities = []
isworse = True
if quantity in [self._KEYS.DISCOVER_DATE, self._KEYS.MAX_DATE]:
for ct in my_quantity_list:
ctsplit = ct[QUANTITY.VALUE].split('/')
svsplit = added_quantity[QUANTITY.VALUE].split('/')
if len(ctsplit) < len(svsplit):
isworse = False
continue
elif len(ctsplit) < len(svsplit) and len(svsplit) == 3:
val_one = max(
2, get_sig_digits(ctsplit[-1].lstrip('0')))
val_two = max(
2, get_sig_digits(svsplit[-1].lstrip('0')))
if val_one < val_two:
isworse = False
continue
newquantities.append(ct)
else:
if type(quantity) != Key:
isworse = False
elif quantity.type == KEY_TYPES.NUMERIC:
newsig = get_sig_digits(added_quantity[QUANTITY.VALUE])
for ct in my_quantity_list:
addct = False
checke = False
if (len(quantity.kind_preference) > 0 and
not set(listify(ct.get(
QUANTITY.KIND, []))).isdisjoint(
quantity.kind_preference)
and not set(
listify(
added_quantity.get(
QUANTITY.KIND,
[]))).isdisjoint(
quantity.kind_preference)):
aqi = min([
quantity.kind_preference.index(x) for x in
listify(added_quantity[QUANTITY.KIND])
])
qqi = min([
quantity.kind_preference.index(x)
for x in listify(ct[QUANTITY.KIND])
])
if aqi > qqi:
addct = True
if aqi == qqi:
checke = True
if aqi <= qqi:
isworse = False
else:
checke = True
if checke and QUANTITY.E_VALUE in ct:
if QUANTITY.E_VALUE in added_quantity:
if (float(added_quantity[QUANTITY.E_VALUE])
>= float(ct[QUANTITY.E_VALUE])):
addct = True
if (float(added_quantity[QUANTITY.E_VALUE])
<= float(ct[QUANTITY.E_VALUE])):
isworse = False
else:
if (checke and QUANTITY.E_VALUE
in added_quantity):
isworse = False
else:
oldsig = get_sig_digits(ct[QUANTITY.VALUE])
if oldsig >= newsig:
addct = True
if newsig >= oldsig:
isworse = False
if addct:
newquantities.append(ct)
elif quantity.type == KEY_TYPES.STRING:
for ct in my_quantity_list:
addct = False
if (len(quantity.kind_preference) > 0 and
not set(listify(ct.get(
QUANTITY.KIND, []))).isdisjoint(
quantity.kind_preference)
and not set(
listify(
added_quantity.get(
QUANTITY.KIND,
[]))).isdisjoint(
quantity.kind_preference)):
aqi = min([
quantity.kind_preference.index(x) for x in
listify(added_quantity[QUANTITY.KIND])
])
qqi = min([
quantity.kind_preference.index(x)
for x in listify(ct[QUANTITY.KIND])
])
if aqi >= qqi:
addct = True
if aqi <= qqi:
isworse = False
else:
addct = True
isworse = False
if addct:
newquantities.append(ct)
if isworse:
self._log.info("Removing quantity '{}' with value '{}' "
"and kind '{}' determined to be worse than "
"existing alternative values.".format(
quantity,
added_quantity[QUANTITY.VALUE],
added_quantity.get(QUANTITY.KIND, '')))
else:
newquantities.append(added_quantity)
if len(newquantities) > 0:
self[quantity] = newquantities
# As all SN####xx designations for 2016+ have corresponding AT
# designations, add the AT alias when the SN alias is added.
if quantity == self._KEYS.ALIAS:
for alias in self.get(quantity, []):
cleaned_value = alias[QUANTITY.VALUE]
if (cleaned_value.startswith('SN')
and is_integer(cleaned_value[2:6])
and int(cleaned_value[2:6]) >= 2016):
success = super(Supernova, self).add_quantity(
SUPERNOVA.ALIAS, 'AT' + cleaned_value[2:], source,
**kwargs)
return True
def radec_clean(svalue, quantity, unit=''):
"""Clean R.A. and Dec."""
svalue = svalue.strip()
if unit == 'floatdegrees':
if not is_number(svalue):
return (svalue, unit)
deg = float('%g' % Decimal(svalue))
sig = get_sig_digits(svalue)
if 'ra' in quantity:
flhours = deg / 360.0 * 24.0
hours = floor(flhours)
minutes = floor((flhours - hours) * 60.0)
seconds = (flhours * 60.0 - (hours * 60.0 + minutes)) * 60.0
hours = 0 if hours < 1.e-6 else hours
minutes = 0 if minutes < 1.e-6 else minutes
seconds = 0.0 if seconds < 1.e-6 else seconds
if seconds > 60.0:
raise (ValueError('Invalid seconds value for ' + quantity))
svalue = str(hours).zfill(2) + ':' + str(minutes).zfill(2) + \
':' + zpad(pretty_num(seconds, sig=sig - 1))
elif 'dec' in quantity:
fldeg = abs(deg)
degree = floor(fldeg)
minutes = floor((fldeg - degree) * 60.0)
seconds = (fldeg * 60.0 - (degree * 60.0 + minutes)) * 60.0
minutes = 0 if minutes < 1.e-6 else minutes
seconds = 0.0 if seconds < 1.e-6 else seconds
if seconds > 60.0:
raise (ValueError('Invalid seconds value for ' + quantity))
svalue = (('+' if deg >= 0.0 else '-') +
str(degree).strip('+-').zfill(2) + ':' +
str(minutes).zfill(2) + ':' +
zpad(pretty_num(seconds, sig=sig - 1)))
elif unit == 'nospace' and 'ra' in quantity:
svalue = svalue[:2] + ':' + svalue[2:4] + \
((':' + zpad(svalue[4:])) if len(svalue) > 4 else '')
elif unit == 'nospace' and 'dec' in quantity:
if svalue.startswith(('+', '-')):
svalue = svalue[:3] + ':' + svalue[3:5] + \
((':' + zpad(svalue[5:])) if len(svalue) > 5 else '')
else:
svalue = '+' + svalue[:2] + ':' + svalue[2:4] + \
((':' + zpad(svalue[4:])) if len(svalue) > 4 else '')
else:
svalue = svalue.replace(' ', ':')
if 'dec' in quantity:
valuesplit = svalue.split(':')
svalue = (
('-' if valuesplit[0].startswith('-') else '+') +
valuesplit[0].strip('+-').zfill(2) +
(':' + valuesplit[1].zfill(2) if len(valuesplit) > 1 else '') +
(':' + zpad(valuesplit[2]) if len(valuesplit) > 2 else ''))
if 'ra' in quantity:
sunit = 'hours'
elif 'dec' in quantity:
sunit = 'degrees'
# Correct case of arcseconds = 60.0.
valuesplit = svalue.split(':')
if len(valuesplit) == 3 and valuesplit[-1] in ["60.0", "60.", "60"]:
svalue = valuesplit[0] + ':' + str(
Decimal(valuesplit[1]) + Decimal(1.0)) + ':' + "00.0"
# Strip trailing dots.
svalue = svalue.rstrip('.')
return (svalue, sunit)
def do_donated_spectra(catalog):
"""Import donated spectra."""
task_str = catalog.get_current_task_str()
fpath = os.path.join(catalog.get_current_task_repo(), 'donations')
with open(os.path.join(fpath, 'meta.json'), 'r') as f:
metadict = json.loads(f.read())
donationscnt = 0
oldname = ''
for fname in pbar(list(metadict.keys()), task_str):
name = metadict[fname]['name']
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
sec_bibc = metadict[fname]['bibcode']
name, source = catalog.new_entry(name, bibcode=sec_bibc)
date = metadict[fname].get('date', '')
year, month, day = date.split('/')
sig = get_sig_digits(day) + 5
day_fmt = str(floor(float(day))).zfill(2)
time = astrotime(year + '-' + month + '-' + day_fmt).mjd
time = time + float(day) - floor(float(day))
time = pretty_num(time, sig=sig)
with open(os.path.join(fpath, fname), 'r') as f:
specdata = list(
csv.reader(
f, delimiter=' ', skipinitialspace=True))
specdata = list(filter(None, specdata))
newspec = []
oldval = ''
for row in specdata:
if row[0][0] == '#':
continue
if row[1] == oldval:
continue
newspec.append(row)
oldval = row[1]
specdata = newspec
haserrors = len(specdata[0]) == 3 and specdata[0][2] and specdata[0][
2] != 'NaN'
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.U_TIME: 'MJD',
SPECTRUM.TIME: time,
SPECTRUM.WAVELENGTHS: wavelengths,
SPECTRUM.FLUXES: fluxes,
SPECTRUM.ERRORS: errors,
SPECTRUM.SOURCE: source,
SPECTRUM.FILENAME: fname
}
if 'instrument' in metadict[fname]:
specdict[SPECTRUM.INSTRUMENT] = metadict[fname]['instrument']
if 'telescope' in metadict[fname]:
specdict[SPECTRUM.TELESCOPE] = metadict[fname]['telescope']
if 'yunit' in metadict[fname]:
specdict[SPECTRUM.U_FLUXES] = metadict[fname]['yunit']
specdict[SPECTRUM.U_ERRORS] = metadict[fname]['yunit']
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict[SPECTRUM.U_FLUXES] = fluxunit
specdict[SPECTRUM.U_ERRORS] = fluxunit
catalog.entries[name].add_spectrum(**specdict)
donationscnt = donationscnt + 1
if (catalog.args.travis and
donationscnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
def add_quantity(self,
quantities,
value,
source,
forcereplacebetter=False,
**kwargs):
"""Add `Quantity` to `Supernova`."""
success = super(Supernova, self).add_quantity(
quantities, value, source, **kwargs)
if not success:
return
for quantity in listify(quantities):
my_quantity_list = self.get(quantity, [])
if ((forcereplacebetter or quantity.replace_better) and
len(my_quantity_list) > 1):
# The quantity that was just added should be last in the list
added_quantity = my_quantity_list.pop()
newquantities = []
isworse = True
if quantity in [self._KEYS.DISCOVER_DATE, self._KEYS.MAX_DATE]:
for ct in my_quantity_list:
ctsplit = ct[QUANTITY.VALUE].split('/')
svsplit = added_quantity[QUANTITY.VALUE].split('/')
if len(ctsplit) < len(svsplit):
isworse = False
continue
elif len(ctsplit) < len(svsplit) and len(svsplit) == 3:
val_one = max(
2, get_sig_digits(ctsplit[-1].lstrip('0')))
val_two = max(
2, get_sig_digits(svsplit[-1].lstrip('0')))
if val_one < val_two:
isworse = False
continue
newquantities.append(ct)
else:
if type(quantity) != Key:
isworse = False
elif quantity.type == KEY_TYPES.NUMERIC:
newsig = get_sig_digits(added_quantity[QUANTITY.VALUE])
for ct in my_quantity_list:
addct = False
checke = False
if (len(quantity.kind_preference) > 0 and not set(
listify(ct.get(QUANTITY.KIND, [])))
.isdisjoint(quantity.kind_preference) and
not set(
listify(
added_quantity.get(QUANTITY.KIND,
[])))
.isdisjoint(quantity.kind_preference)):
aqi = min([
quantity.kind_preference.index(x)
for x in listify(added_quantity[
QUANTITY.KIND])
])
qqi = min([
quantity.kind_preference.index(x)
for x in listify(ct[QUANTITY.KIND])
])
if aqi > qqi:
addct = True
if aqi == qqi:
checke = True
if aqi <= qqi:
isworse = False
else:
checke = True
if checke and QUANTITY.E_VALUE in ct:
if QUANTITY.E_VALUE in added_quantity:
if (float(added_quantity[QUANTITY.E_VALUE])
>= float(ct[QUANTITY.E_VALUE])):
addct = True
if (float(added_quantity[QUANTITY.E_VALUE])
<= float(ct[QUANTITY.E_VALUE])):
isworse = False
else:
if (checke and
QUANTITY.E_VALUE in added_quantity):
isworse = False
else:
oldsig = get_sig_digits(ct[QUANTITY.VALUE])
if oldsig >= newsig:
addct = True
if newsig >= oldsig:
isworse = False
if addct:
newquantities.append(ct)
elif quantity.type == KEY_TYPES.STRING:
for ct in my_quantity_list:
addct = False
if (len(quantity.kind_preference) > 0 and not set(
listify(ct.get(QUANTITY.KIND, [])))
.isdisjoint(quantity.kind_preference) and
not set(
listify(
added_quantity.get(QUANTITY.KIND,
[])))
.isdisjoint(quantity.kind_preference)):
aqi = min([
quantity.kind_preference.index(x)
for x in listify(added_quantity[
QUANTITY.KIND])
])
qqi = min([
quantity.kind_preference.index(x)
for x in listify(ct[QUANTITY.KIND])
])
if aqi >= qqi:
addct = True
if aqi <= qqi:
isworse = False
else:
addct = True
isworse = False
if addct:
newquantities.append(ct)
if isworse:
self._log.info("Removing quantity '{}' with value '{}' "
"and kind '{}' determined to be worse than "
"existing alternative values.".format(
quantity, added_quantity[
QUANTITY.VALUE],
added_quantity.get(QUANTITY.KIND, '')))
else:
newquantities.append(added_quantity)
if len(newquantities) > 0:
self[quantity] = newquantities
# As all SN####xx designations for 2016+ have corresponding AT
# designations, add the AT alias when the SN alias is added.
if quantity == self._KEYS.ALIAS:
for alias in self.get(quantity, []):
cleaned_value = alias[QUANTITY.VALUE]
if (cleaned_value.startswith('SN') and
is_integer(cleaned_value[2:6]) and
int(cleaned_value[2:6]) >= 2016):
success = super(Supernova, self).add_quantity(
SUPERNOVA.ALIAS, 'AT' + cleaned_value[2:], source,
**kwargs)
return True
def do_ucb_spectra(catalog):
task_str = catalog.get_current_task_str()
sec_reference = 'UCB Filippenko Group\'s Supernova Database (SNDB)'
sec_refurl = 'http://heracles.astro.berkeley.edu/sndb/info'
sec_refbib = '2012MNRAS.425.1789S'
ucbspectracnt = 0
jsontxt = catalog.load_url(
'http://heracles.astro.berkeley.edu/sndb/download?id=allpubspec',
os.path.join(catalog.get_current_task_repo(), 'UCB/allpubspec.json'),
json_sort='SpecID')
if not jsontxt:
return
spectra = json.loads(jsontxt)
spectra = sorted(spectra, key=lambda kk: kk['SpecID'])
oldname = ''
for spectrum in pbar(spectra, task_str):
name = spectrum['ObjName']
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
sec_source = catalog.entries[name].add_source(name=sec_reference,
url=sec_refurl,
bibcode=sec_refbib,
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, sec_source)
sources = [sec_source]
if spectrum['Reference']:
sources += [
catalog.entries[name].add_source(bibcode=spectrum['Reference'])
]
sources = uniq_cdl(sources)
if spectrum['Type'] and spectrum['Type'].strip() != 'NoMatch':
for ct in spectrum['Type'].strip().split(','):
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE,
ct.replace('-norm', '').strip(), sources)
if spectrum['DiscDate']:
ddate = spectrum['DiscDate'].replace('-', '/')
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, ddate,
sources)
if spectrum['HostName']:
host = urllib.parse.unquote(spectrum['HostName']).replace('*', '')
catalog.entries[name].add_quantity(SUPERNOVA.HOST, host, sources)
if spectrum['UT_Date']:
epoch = str(spectrum['UT_Date'])
year = epoch[:4]
month = epoch[4:6]
day = epoch[6:]
sig = get_sig_digits(day) + 5
mjd = astrotime(year + '-' + month + '-' +
str(floor(float(day))).zfill(2)).mjd
mjd = pretty_num(mjd + float(day) - floor(float(day)), sig=sig)
filename = spectrum['Filename'] if spectrum['Filename'] else ''
instrument = spectrum['Instrument'] if spectrum['Instrument'] else ''
reducer = spectrum['Reducer'] if spectrum['Reducer'] else ''
observer = spectrum['Observer'] if spectrum['Observer'] else ''
snr = str(spectrum['SNR']) if spectrum['SNR'] else ''
if not filename:
raise ValueError('Filename not found for SNDB spectrum!')
if not spectrum['SpecID']:
raise ValueError('ID not found for SNDB spectrum!')
filepath = os.path.join(catalog.get_current_task_repo(),
'UCB/') + filename
spectxt = catalog.load_url(
'http://heracles.astro.berkeley.edu/sndb/download?id=ds:' +
str(spectrum['SpecID']),
filepath,
archived_mode=True)
specdata = list(
csv.reader(spectxt.splitlines(),
delimiter=' ',
skipinitialspace=True))
newspecdata = []
for row in specdata:
if not row or not row[0] or row[0][0] == '#':
continue
else:
newspecdata.append(row)
specdata = newspecdata
haserrors = len(
specdata[0]) == 3 and specdata[0][2] and specdata[0][2] != 'NaN'
specdata = [list(ii) for ii in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
if not list(filter(None, errors)):
errors = ''
units = 'Uncalibrated'
catalog.entries[name].add_spectrum(u_wavelengths='Angstrom',
u_fluxes=units,
u_time='MJD',
time=mjd,
wavelengths=wavelengths,
filename=filename,
fluxes=fluxes,
errors=errors,
u_errors=units,
instrument=instrument,
source=sources,
snr=snr,
observer=observer,
reducer=reducer,
deredshifted=('-noz' in filename))
ucbspectracnt = ucbspectracnt + 1
if catalog.args.travis and ucbspectracnt >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
return
def add_quantity(self, quantity, value, source, forcereplacebetter=False,
**kwargs):
success = super().add_quantity(quantity, value, source, **kwargs)
if not success:
return
my_quantity_list = self.get(quantity, [])
if (forcereplacebetter or quantity in REPR_BETTER_QUANTITY) and \
len(my_quantity_list) > 1:
# The quantity that was just added should be last in the list
added_quantity = my_quantity_list.pop()
newquantities = []
isworse = True
if quantity in [self._KEYS.DISCOVER_DATE, self._KEYS.MAX_DATE]:
for ct in my_quantity_list:
ctsplit = ct[QUANTITY.VALUE].split('/')
svsplit = added_quantity[QUANTITY.VALUE].split('/')
if len(ctsplit) < len(svsplit):
isworse = False
continue
elif len(ctsplit) < len(svsplit) and len(svsplit) == 3:
val_one = max(2, get_sig_digits(
ctsplit[-1].lstrip('0')))
val_two = max(2, get_sig_digits(
svsplit[-1].lstrip('0')))
if val_one < val_two:
isworse = False
continue
newquantities.append(ct)
else:
newsig = get_sig_digits(added_quantity[QUANTITY.VALUE])
for ct in my_quantity_list:
if QUANTITY.E_VALUE in ct:
if QUANTITY.E_VALUE in added_quantity:
if (float(added_quantity[QUANTITY.E_VALUE]) <=
float(ct[QUANTITY.E_VALUE])):
isworse = False
continue
newquantities.append(ct)
else:
if QUANTITY.E_VALUE in added_quantity:
isworse = False
continue
oldsig = get_sig_digits(ct[QUANTITY.VALUE])
if oldsig >= newsig:
newquantities.append(ct)
if newsig >= oldsig:
isworse = False
if not isworse:
newquantities.append(added_quantity)
self[quantity] = newquantities
# As all SN####xx designations have corresponding AT designations, add
# the AT alias when the SN alias is added.
if quantity == self._KEYS.ALIAS:
cleaned_quantity = quantity.strip()
if (cleaned_quantity.startswith('SN') and
is_integer(cleaned_quantity[2:6]) and
int(cleaned_quantity[2:6]) >= 2016):
success = super().add_quantity(
'AT' + cleaned_quantity[2:], value, source, **kwargs)
return True
def do_donated_spectra(catalog):
"""Import donated spectra."""
task_str = catalog.get_current_task_str()
fpath = os.path.join(catalog.get_current_task_repo(), 'donations')
with open(os.path.join(fpath, 'meta.json'), 'r') as f:
metadict = json.loads(f.read())
donationscnt = 0
oldname = ''
for fname in pbar(metadict, task_str):
name = metadict[fname]['name']
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
sec_bibc = metadict[fname]['bibcode']
name, source = catalog.new_entry(name, bibcode=sec_bibc)
date = metadict[fname].get('date', '')
year, month, day = date.split('/')
sig = get_sig_digits(day) + 5
day_fmt = str(floor(float(day))).zfill(2)
time = astrotime(year + '-' + month + '-' + day_fmt).mjd
time = time + float(day) - floor(float(day))
time = pretty_num(time, sig=sig)
with open(os.path.join(fpath, fname), 'r') as f:
specdata = list(csv.reader(f, delimiter=' ',
skipinitialspace=True))
specdata = list(filter(None, specdata))
newspec = []
oldval = ''
for row in specdata:
if row[0][0] == '#':
continue
if row[1] == oldval:
continue
newspec.append(row)
oldval = row[1]
specdata = newspec
haserrors = len(
specdata[0]) == 3 and specdata[0][2] and specdata[0][2] != 'NaN'
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.U_TIME: 'MJD',
SPECTRUM.TIME: time,
SPECTRUM.WAVELENGTHS: wavelengths,
SPECTRUM.FLUXES: fluxes,
SPECTRUM.ERRORS: errors,
SPECTRUM.SOURCE: source,
SPECTRUM.FILENAME: fname
}
if 'instrument' in metadict[fname]:
specdict[SPECTRUM.INSTRUMENT] = metadict[fname]['instrument']
if 'telescope' in metadict[fname]:
specdict[SPECTRUM.TELESCOPE] = metadict[fname]['telescope']
if 'yunit' in metadict[fname]:
specdict[SPECTRUM.U_FLUXES] = metadict[fname]['yunit']
specdict[SPECTRUM.U_ERRORS] = metadict[fname]['yunit']
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict[SPECTRUM.U_FLUXES] = fluxunit
specdict[SPECTRUM.U_ERRORS] = fluxunit
catalog.entries[name].add_spectrum(**specdict)
donationscnt = donationscnt + 1
if (catalog.args.travis
and donationscnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
def do_suspect_spectra(catalog):
task_str = catalog.get_current_task_str()
with open(os.path.join(catalog.get_current_task_repo(),
'Suspect/sources.json'), 'r') as f:
sourcedict = json.loads(f.read())
with open(os.path.join(catalog.get_current_task_repo(),
'Suspect/filename-changes.txt'), 'r') as f:
rows = f.readlines()
changedict = {}
for row in rows:
if not row.strip() or row[0] == "#":
continue
items = row.strip().split(' ')
changedict[items[1]] = items[0]
suspectcnt = 0
folders = next(os.walk(os.path.join(
catalog.get_current_task_repo(), 'Suspect')))[1]
for folder in pbar(folders, task_str):
eventfolders = next(os.walk(os.path.join(
catalog.get_current_task_repo(), 'Suspect/') + folder))[1]
oldname = ''
for eventfolder in pbar(eventfolders, task_str):
name = eventfolder
if is_number(name[:4]):
name = 'SN' + name
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
sec_ref = 'SUSPECT'
sec_refurl = 'https://www.nhn.ou.edu/~suspect/'
sec_bibc = '2001AAS...199.8408R'
sec_source = catalog.entries[name].add_source(
name=sec_ref, url=sec_refurl, bibcode=sec_bibc,
secondary=True)
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, name, sec_source)
fpath = os.path.join(catalog.get_current_task_repo(),
'Suspect', folder, eventfolder)
eventspectra = next(os.walk(fpath))[2]
for spectrum in eventspectra:
sources = [sec_source]
bibcode = ''
if spectrum in changedict:
specalias = changedict[spectrum]
else:
specalias = spectrum
if specalias in sourcedict:
bibcode = sourcedict[specalias]
elif name in sourcedict:
bibcode = sourcedict[name]
if bibcode:
source = catalog.entries[name].add_source(
bibcode=unescape(bibcode))
sources += [source]
sources = uniq_cdl(sources)
date = spectrum.split('_')[1]
year = date[:4]
month = date[4:6]
day = date[6:]
sig = get_sig_digits(day) + 5
day_fmt = str(floor(float(day))).zfill(2)
time = astrotime(year + '-' + month + '-' + day_fmt).mjd
time = time + float(day) - floor(float(day))
time = pretty_num(time, sig=sig)
fpath = os.path.join(catalog.get_current_task_repo(),
'Suspect',
folder,
eventfolder, spectrum)
with open(fpath, 'r') as f:
specdata = list(csv.reader(
f, delimiter=' ', skipinitialspace=True))
specdata = list(filter(None, specdata))
newspec = []
oldval = ''
for row in specdata:
if row[1] == oldval:
continue
newspec.append(row)
oldval = row[1]
specdata = newspec
haserrors = len(specdata[0]) == 3 and specdata[
0][2] and specdata[0][2] != 'NaN'
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom', u_fluxes='Uncalibrated',
u_time='MJD',
time=time,
wavelengths=wavelengths, fluxes=fluxes, errors=errors,
u_errors='Uncalibrated',
source=sources, filename=spectrum)
suspectcnt = suspectcnt + 1
if (catalog.args.travis and
suspectcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
def do_cleanup(catalog):
"""Cleanup catalog after importing all data."""
task_str = catalog.get_current_task_str()
# Set preferred names, calculate some columns based on imported data,
# sanitize some fields
keys = list(catalog.entries.keys())
cleanupcnt = 0
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
try:
name = catalog.add_entry(oname)
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
# Set the preferred name, switching to that name if name changed.
name = catalog.entries[name].set_preferred_name()
aliases = catalog.entries[name].get_aliases()
catalog.entries[name].purge_bandless_photometry()
catalog.entries[name].set_first_max_light()
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['MLS', 'SSS', 'CSS', 'GRB ']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:2])):
discoverdate = ('/'.join([
'20' + alias.replace(prefix, '')[:2],
alias.replace(prefix, '')[2:4],
alias.replace(prefix, '')[4:6]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = [
'ASASSN-', 'PS1-', 'PS1', 'PS', 'iPTF', 'PTF', 'SCP-', 'SNLS-',
'SPIRITS', 'LSQ', 'DES', 'SNHiTS', 'Gaia', 'GND', 'GNW', 'GSD',
'GSW', 'EGS', 'COS', 'OGLE', 'HST'
]
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:2])
and is_number(alias.replace(prefix, '')[:1])):
discoverdate = '20' + alias.replace(prefix, '')[:2]
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['SNF']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:4])):
discoverdate = ('/'.join([
alias.replace(prefix, '')[:4],
alias.replace(prefix, '')[4:6],
alias.replace(prefix, '')[6:8]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['PTFS', 'SNSDF']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:2])):
discoverdate = ('/'.join([
'20' + alias.replace(prefix, '')[:2],
alias.replace(prefix, '')[2:4]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['AT', 'SN', 'OGLE-', 'SM ', 'KSN']
for alias in aliases:
for prefix in prefixes:
if alias.startswith(prefix):
year = re.findall(r'\d+', alias)
if len(year) == 1:
year = year[0]
else:
continue
if alias.replace(prefix, '').index(year) != 0:
continue
if (year and is_number(year) and '.' not in year
and len(year) <= 4):
discoverdate = year
if catalog.args.verbose:
tprint('Added discoverdate from name [' +
alias + ']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if (SUPERNOVA.RA not in catalog.entries[name]
or SUPERNOVA.DEC not in catalog.entries[name]):
prefixes = [
'PSN J', 'MASJ', 'CSS', 'SSS', 'MASTER OT J', 'HST J', 'TCP J',
'MACS J', '2MASS J', 'EQ J', 'CRTS J', 'SMT J'
]
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:6])):
noprefix = alias.split(':')[-1].replace(prefix,
'').replace(
'.', '')
decsign = '+' if '+' in noprefix else '-'
noprefix = noprefix.replace('+', '|').replace('-', '|')
nops = noprefix.split('|')
if len(nops) < 2:
continue
rastr = nops[0]
decstr = nops[1]
ra = ':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + \
('.' + rastr[6:] if len(rastr) > 6 else '')
dec = (
decsign +
':'.join([decstr[:2], decstr[2:4], decstr[4:6]]) +
('.' + decstr[6:] if len(decstr) > 6 else ''))
if catalog.args.verbose:
tprint('Added ra/dec from name: ' + ra + ' ' + dec)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(SUPERNOVA.RA,
ra,
source,
derived=True)
catalog.entries[name].add_quantity(SUPERNOVA.DEC,
dec,
source,
derived=True)
break
if SUPERNOVA.RA in catalog.entries[name]:
break
no_host = (SUPERNOVA.HOST not in catalog.entries[name] or not any([
x[QUANTITY.VALUE] == 'Milky Way'
for x in catalog.entries[name][SUPERNOVA.HOST]
]))
if (SUPERNOVA.RA in catalog.entries[name]
and SUPERNOVA.DEC in catalog.entries[name] and no_host):
from astroquery.irsa_dust import IrsaDust
if name not in catalog.extinctions_dict:
try:
ra_dec = catalog.entries[name][
SUPERNOVA.RA][0][QUANTITY.VALUE] + \
" " + \
catalog.entries[name][SUPERNOVA.DEC][0][QUANTITY.VALUE]
result = IrsaDust.get_query_table(ra_dec, section='ebv')
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
warnings.warn("Coordinate lookup for " + name +
" failed in IRSA.")
else:
ebv = result['ext SandF mean'][0]
ebverr = result['ext SandF std'][0]
catalog.extinctions_dict[name] = [ebv, ebverr]
if name in catalog.extinctions_dict:
sources = uniq_cdl([
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2011ApJ...737..103S')
])
(catalog.entries[name].add_quantity(
SUPERNOVA.EBV,
str(catalog.extinctions_dict[name][0]),
sources,
e_value=str(catalog.extinctions_dict[name][1]),
derived=True))
if ((SUPERNOVA.HOST in catalog.entries[name]
and (SUPERNOVA.HOST_RA not in catalog.entries[name]
or SUPERNOVA.HOST_DEC not in catalog.entries[name]))):
for host in catalog.entries[name][SUPERNOVA.HOST]:
alias = host[QUANTITY.VALUE]
if ' J' in alias and is_number(alias.split(' J')[-1][:6]):
noprefix = alias.split(' J')[-1].split(':')[-1].replace(
'.', '')
decsign = '+' if '+' in noprefix else '-'
noprefix = noprefix.replace('+', '|').replace('-', '|')
nops = noprefix.split('|')
if len(nops) < 2:
continue
rastr = nops[0]
decstr = nops[1]
hostra = (':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) +
('.' + rastr[6:] if len(rastr) > 6 else ''))
hostdec = decsign + ':'.join([
decstr[:2], decstr[2:4], decstr[4:6]
]) + ('.' + decstr[6:] if len(decstr) > 6 else '')
if catalog.args.verbose:
tprint('Added hostra/hostdec from name: ' + hostra +
' ' + hostdec)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(SUPERNOVA.HOST_RA,
hostra,
source,
derived=True)
catalog.entries[name].add_quantity(SUPERNOVA.HOST_DEC,
hostdec,
source,
derived=True)
break
if SUPERNOVA.HOST_RA in catalog.entries[name]:
break
if (SUPERNOVA.REDSHIFT not in catalog.entries[name]
and SUPERNOVA.VELOCITY in catalog.entries[name]):
# Find the "best" velocity to use for this
bestsig = 0
for hv in catalog.entries[name][SUPERNOVA.VELOCITY]:
sig = get_sig_digits(hv[QUANTITY.VALUE])
if sig > bestsig:
besthv = hv[QUANTITY.VALUE]
bestsrc = hv['source']
bestsig = sig
if bestsig > 0 and is_number(besthv):
voc = float(besthv) * 1.e5 / CLIGHT
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
(catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT,
pretty_num(sqrt((1. + voc) / (1. - voc)) - 1.,
sig=bestsig),
sources,
kind='heliocentric',
derived=True))
if (SUPERNOVA.REDSHIFT not in catalog.entries[name]
and len(catalog.nedd_dict) > 0
and SUPERNOVA.HOST in catalog.entries[name]):
reference = "NED-D"
refurl = "http://ned.ipac.caltech.edu/Library/Distances/"
refbib = "1991ASSL..171...89H"
for host in catalog.entries[name][SUPERNOVA.HOST]:
if host[QUANTITY.VALUE] in catalog.nedd_dict:
source = catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
secondarysource = catalog.entries[name].add_source(
name=reference,
url=refurl,
bibcode=refbib,
secondary=True)
meddist = statistics.median(
catalog.nedd_dict[host[QUANTITY.VALUE]])
redz = z_at_value(cosmo.comoving_distance,
float(meddist) * un.Mpc)
redshift = pretty_num(redz,
sig=get_sig_digits(str(meddist)))
catalog.entries[name].add_quantity(
[SUPERNOVA.REDSHIFT, SUPERNOVA.HOST_REDSHIFT],
redshift,
uniq_cdl([source, secondarysource]),
kind='host',
derived=True)
if (SUPERNOVA.MAX_ABS_MAG not in catalog.entries[name]
and SUPERNOVA.MAX_APP_MAG in catalog.entries[name]
and SUPERNOVA.LUM_DIST in catalog.entries[name]):
# Find the "best" distance to use for this
bestsig = 0
for ld in catalog.entries[name][SUPERNOVA.LUM_DIST]:
sig = get_sig_digits(ld[QUANTITY.VALUE])
if sig > bestsig:
bestld = ld[QUANTITY.VALUE]
bestsrc = ld[QUANTITY.SOURCE]
bestsig = sig
if bestsig > 0 and is_number(bestld) and float(bestld) > 0.:
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
bestldz = z_at_value(cosmo.luminosity_distance,
float(bestld) * un.Mpc)
pnum = (float(catalog.entries[name][SUPERNOVA.MAX_APP_MAG][0][
QUANTITY.VALUE]) - 5.0 *
(log10(float(bestld) * 1.0e6) - 1.0) +
2.5 * log10(1.0 + bestldz))
pnum = pretty_num(pnum, sig=bestsig + 1)
catalog.entries[name].add_quantity(SUPERNOVA.MAX_ABS_MAG,
pnum,
sources,
derived=True)
if (SUPERNOVA.MAX_VISUAL_ABS_MAG not in catalog.entries[name]
and SUPERNOVA.MAX_VISUAL_APP_MAG in catalog.entries[name]
and SUPERNOVA.LUM_DIST in catalog.entries[name]):
# Find the "best" distance to use for this
bestsig = 0
for ld in catalog.entries[name][SUPERNOVA.LUM_DIST]:
sig = get_sig_digits(ld[QUANTITY.VALUE])
if sig > bestsig:
bestld = ld[QUANTITY.VALUE]
bestsrc = ld[QUANTITY.SOURCE]
bestsig = sig
if bestsig > 0 and is_number(bestld) and float(bestld) > 0.:
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
# FIX: what's happening here?!
pnum = (float(catalog.entries[name][
SUPERNOVA.MAX_VISUAL_APP_MAG][0][QUANTITY.VALUE]) - 5.0 *
(log10(float(bestld) * 1.0e6) - 1.0))
pnum = pretty_num(pnum, sig=bestsig + 1)
catalog.entries[name].add_quantity(
SUPERNOVA.MAX_VISUAL_ABS_MAG, pnum, sources, derived=True)
if SUPERNOVA.REDSHIFT in catalog.entries[name]:
# Find the "best" redshift to use for this
bestz, bestkind, bestsig, bestsrc = catalog.entries[
name].get_best_redshift()
if bestsig > 0:
try:
bestz = float(bestz)
except Exception:
print(catalog.entries[name])
raise
if SUPERNOVA.VELOCITY not in catalog.entries[name]:
source = catalog.entries[name].add_self_source()
# FIX: what's happening here?!
pnum = CLIGHT / KM * \
((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.)
pnum = pretty_num(pnum, sig=bestsig)
catalog.entries[name].add_quantity(
SUPERNOVA.VELOCITY,
pnum,
source,
kind=(SUPERNOVA.VELOCITY.kind_preference[bestkind]
if bestkind else ''))
if bestz > 0.:
if SUPERNOVA.LUM_DIST not in catalog.entries[name]:
dl = cosmo.luminosity_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
SUPERNOVA.LUM_DIST,
pretty_num(dl.value, sig=bestsig + 1),
sources,
kind=(SUPERNOVA.LUM_DIST.kind_preference[bestkind]
if bestkind else ''),
derived=True)
if (SUPERNOVA.MAX_ABS_MAG not in catalog.entries[name]
and SUPERNOVA.MAX_APP_MAG
in catalog.entries[name]):
source = catalog.entries[name].add_self_source()
pnum = pretty_num(
float(catalog.entries[name][
SUPERNOVA.MAX_APP_MAG][0][QUANTITY.VALUE])
- 5.0 * (log10(dl.to('pc').value) - 1.0) +
2.5 * log10(1.0 + bestz),
sig=bestsig + 1)
catalog.entries[name].add_quantity(
SUPERNOVA.MAX_ABS_MAG,
pnum,
sources,
derived=True)
if (SUPERNOVA.MAX_VISUAL_ABS_MAG
not in catalog.entries[name]
and SUPERNOVA.MAX_VISUAL_APP_MAG
in catalog.entries[name]):
source = catalog.entries[name].add_self_source()
pnum = pretty_num(float(catalog.entries[name][
SUPERNOVA.MAX_VISUAL_APP_MAG][0][
QUANTITY.VALUE]) - 5.0 *
(log10(dl.to('pc').value) - 1.0),
sig=bestsig + 1)
catalog.entries[name].add_quantity(
SUPERNOVA.MAX_VISUAL_ABS_MAG,
pnum,
sources,
derived=True)
if SUPERNOVA.COMOVING_DIST not in catalog.entries[name]:
cd = cosmo.comoving_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
SUPERNOVA.COMOVING_DIST,
pretty_num(cd.value, sig=bestsig),
sources,
derived=True)
if SUPERNOVA.HOST_REDSHIFT in catalog.entries[name]:
# Find the "best" redshift to use for this
bestz, bestkind, bestsig, bestsrc = catalog.entries[
name].get_best_redshift(SUPERNOVA.HOST_REDSHIFT)
if bestsig > 0:
try:
bestz = float(bestz)
except Exception:
print(catalog.entries[name])
raise
if SUPERNOVA.HOST_VELOCITY not in catalog.entries[name]:
source = catalog.entries[name].add_self_source()
# FIX: what's happening here?!
pnum = CLIGHT / KM * \
((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.)
pnum = pretty_num(pnum, sig=bestsig)
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_VELOCITY,
pnum,
source,
kind=(SUPERNOVA.HOST_VELOCITY.kind_preference[bestkind]
if bestkind else ''))
if bestz > 0.:
if SUPERNOVA.HOST_LUM_DIST not in catalog.entries[name]:
dl = cosmo.luminosity_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_LUM_DIST,
pretty_num(dl.value, sig=bestsig + 1),
sources,
kind=(SUPERNOVA.HOST_LUM_DIST.
kind_preference[bestkind]
if bestkind else ''),
derived=True)
if SUPERNOVA.HOST_COMOVING_DIST not in catalog.entries[
name]:
cd = cosmo.comoving_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_COMOVING_DIST,
pretty_num(cd.value, sig=bestsig),
sources,
derived=True)
if all([
x in catalog.entries[name] for x in [
SUPERNOVA.RA, SUPERNOVA.DEC, SUPERNOVA.HOST_RA,
SUPERNOVA.HOST_DEC
]
]):
# For now just using first coordinates that appear in entry
try:
c1 = coord(
ra=catalog.entries[name][SUPERNOVA.RA][0][QUANTITY.VALUE],
dec=catalog.entries[name][SUPERNOVA.DEC][0][
QUANTITY.VALUE],
unit=(un.hourangle, un.deg))
c2 = coord(ra=catalog.entries[name][SUPERNOVA.HOST_RA][0][
QUANTITY.VALUE],
dec=catalog.entries[name][SUPERNOVA.HOST_DEC][0][
QUANTITY.VALUE],
unit=(un.hourangle, un.deg))
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
pass
else:
sources = uniq_cdl([catalog.entries[name].add_self_source()] +
catalog.entries[name][SUPERNOVA.RA][0][
QUANTITY.SOURCE].split(',') +
catalog.entries[name][SUPERNOVA.DEC][0][
QUANTITY.SOURCE].split(',') +
catalog.entries[name][SUPERNOVA.HOST_RA][0][
QUANTITY.SOURCE].split(',') +
catalog.entries[name][SUPERNOVA.HOST_DEC][0]
[QUANTITY.SOURCE].split(','))
if SUPERNOVA.HOST_OFFSET_ANG not in catalog.entries[name]:
hosa = Decimal(c1.separation(c2).arcsecond)
hosa = pretty_num(hosa)
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_OFFSET_ANG,
hosa,
sources,
derived=True,
u_value='arcseconds')
if (SUPERNOVA.COMOVING_DIST in catalog.entries[name]
and SUPERNOVA.REDSHIFT in catalog.entries[name]
and SUPERNOVA.HOST_OFFSET_DIST
not in catalog.entries[name]):
offsetsig = get_sig_digits(catalog.entries[name][
SUPERNOVA.HOST_OFFSET_ANG][0][QUANTITY.VALUE])
sources = uniq_cdl(
sources.split(',') +
(catalog.entries[name][SUPERNOVA.COMOVING_DIST][0][
QUANTITY.SOURCE]).split(',') +
(catalog.entries[name][SUPERNOVA.REDSHIFT][0][
QUANTITY.SOURCE]).split(','))
(catalog.entries[name].add_quantity(
SUPERNOVA.HOST_OFFSET_DIST,
pretty_num(
float(catalog.entries[name][
SUPERNOVA.HOST_OFFSET_ANG][0][QUANTITY.VALUE])
/ 3600. * (pi / 180.) *
float(catalog.entries[name][
SUPERNOVA.COMOVING_DIST][0][QUANTITY.VALUE]) *
1000. / (1.0 + float(catalog.entries[name][
SUPERNOVA.REDSHIFT][0][QUANTITY.VALUE])),
sig=offsetsig), sources))
catalog.entries[name].sanitize()
catalog.journal_entries(bury=True, final=True, gz=True)
cleanupcnt = cleanupcnt + 1
if catalog.args.travis and cleanupcnt % 1000 == 0:
break
catalog.save_caches()
return
def do_cleanup(catalog):
"""Cleanup catalog after importing all data."""
task_str = catalog.get_current_task_str()
# Set preferred names, calculate some columns based on imported data,
# sanitize some fields
keys = list(catalog.entries.keys())
cleanupcnt = 0
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
try:
name = catalog.add_entry(oname)
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
# Set the preferred name, switching to that name if name changed.
name = catalog.entries[name].set_preferred_name()
aliases = catalog.entries[name].get_aliases()
catalog.entries[name].set_first_max_light()
# Clean discoverer field
if FASTSTARS.DISCOVERER in catalog.entries[name]:
if len(catalog.entries[name][FASTSTARS.DISCOVERER]) > 1:
POSSIBLEDISCOVERER = [
catalog.entries[name][FASTSTARS.DISCOVERER][i]['value']
for i in range(
len(catalog.entries[name][FASTSTARS.DISCOVERER]))
]
POSSIBLEDISCOVERER_DATE = [
int(DATE['value'])
for DATE in catalog.entries[name][FASTSTARS.DISCOVER_DATE]
]
POSSIBLEDISCOVERER_DATE_SOURCES = [
DATE['source']
for DATE in catalog.entries[name][FASTSTARS.DISCOVER_DATE]
]
EARLIESTSOURCE = POSSIBLEDISCOVERER_DATE_SOURCES[np.argmin(
POSSIBLEDISCOVERER_DATE)]
EARLIESTDISCOVER_DATE = catalog.entries[name][
FASTSTARS.DISCOVER_DATE][np.argmin(
POSSIBLEDISCOVERER_DATE)]
# Deal with case where a star was 'discovered' multiple times in one year
if ',' in EARLIESTSOURCE:
EARLIESTSOURCE = EARLIESTSOURCE.split(',')[0]
for DISCOVERER in catalog.entries[name][FASTSTARS.DISCOVERER]:
for DISCOVERERSOURCE in DISCOVERER['source'].split(','):
if DISCOVERERSOURCE == EARLIESTSOURCE:
EARLIESTDISCOVERER = DISCOVERER
for DISCOVERER in catalog.entries[name][FASTSTARS.DISCOVERER]:
for DISCOVERERSOURCE in DISCOVERER['source'].split(','):
if DISCOVERERSOURCE == EARLIESTSOURCE:
EARLIESTDISCOVERER = DISCOVERER
catalog.entries[name][FASTSTARS.DISCOVERER] = [
EARLIESTDISCOVERER
]
catalog.entries[name][FASTSTARS.DISCOVER_DATE] = [
EARLIESTDISCOVER_DATE
]
# Convert all distances to kpc.
if FASTSTARS.LUM_DIST in catalog.entries[name]:
for li, ld in enumerate(catalog.entries[name][FASTSTARS.LUM_DIST]):
if ld.get('u_value') != 'kpc':
if ld.get('u_value') == 'pc':
catalog.entries[name][
FASTSTARS.LUM_DIST][li]['value'] = str(
Decimal(catalog.entries[name][
FASTSTARS.LUM_DIST][li]['value']) *
Decimal('0.001'))
elif ld.get('u_value') == 'Mpc':
catalog.entries[name][
FASTSTARS.LUM_DIST][li]['value'] = str(
Decimal(catalog.entries[name][
FASTSTARS.LUM_DIST][li]['value']) *
Decimal('1000'))
else:
raise ValueError('unknown distance unit')
catalog.entries[name][
FASTSTARS.LUM_DIST][li]['u_value'] = 'kpc'
if (FASTSTARS.RA not in catalog.entries[name]
or FASTSTARS.DEC not in catalog.entries[name]):
prefixes = ['SDSS']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:6])):
noprefix = alias.split(':')[-1].replace(prefix,
'').replace(
'.', '')
decsign = '+' if '+' in noprefix else '-'
noprefix = noprefix.replace('+', '|').replace('-', '|')
nops = noprefix.split('|')
if len(nops) < 2:
continue
rastr = nops[0]
decstr = nops[1]
ra = ':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + \
('.' + rastr[6:] if len(rastr) > 6 else '')
dec = (
decsign +
':'.join([decstr[:2], decstr[2:4], decstr[4:6]]) +
('.' + decstr[6:] if len(decstr) > 6 else ''))
if catalog.args.verbose:
tprint('Added ra/dec from name: ' + ra + ' ' + dec)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(FASTSTARS.RA,
ra,
source,
derived=True)
catalog.entries[name].add_quantity(FASTSTARS.DEC,
dec,
source,
derived=True)
break
if FASTSTARS.RA in catalog.entries[name]:
break
if (FASTSTARS.MAX_ABS_MAG not in catalog.entries[name]
and FASTSTARS.MAX_APP_MAG in catalog.entries[name]
and FASTSTARS.LUM_DIST in catalog.entries[name]):
# Find the "best" distance to use for this
bestsig = 0
for ld in catalog.entries[name][FASTSTARS.LUM_DIST]:
sig = get_sig_digits(ld[QUANTITY.VALUE])
if sig > bestsig:
bestld = ld[QUANTITY.VALUE]
bestsrc = ld[QUANTITY.SOURCE]
bestsig = sig
if bestsig > 0 and is_number(bestld) and float(bestld) > 0.:
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
bestldz = z_at_value(cosmo.luminosity_distance,
float(bestld) * un.Mpc)
pnum = (float(catalog.entries[name][FASTSTARS.MAX_APP_MAG][0][
QUANTITY.VALUE]) - 5.0 *
(log10(float(bestld) * 1.0e6) - 1.0) +
2.5 * log10(1.0 + bestldz))
pnum = pretty_num(pnum, sig=bestsig + 1)
catalog.entries[name].add_quantity(FASTSTARS.MAX_ABS_MAG,
pnum,
sources,
derived=True)
catalog.entries[name].sanitize()
catalog.journal_entries(bury=True, final=True, gz=True)
cleanupcnt = cleanupcnt + 1
if catalog.args.travis and cleanupcnt % 1000 == 0:
break
catalog.save_caches()
return
def do_ucb_spectra(catalog):
task_str = catalog.get_current_task_str()
sec_reference = 'UCB Filippenko Group\'s Supernova Database (SNDB)'
sec_refurl = 'http://heracles.astro.berkeley.edu/sndb/info'
sec_refbib = '2012MNRAS.425.1789S'
ucbspectracnt = 0
jsontxt = catalog.load_url(
'http://heracles.astro.berkeley.edu/sndb/download?id=allpubspec',
os.path.join(catalog.get_current_task_repo(), 'UCB/allpubspec.json'),
json_sort='SpecID')
if not jsontxt:
return
spectra = json.loads(jsontxt)
spectra = sorted(spectra, key=lambda kk: kk['SpecID'])
oldname = ''
for spectrum in pbar(spectra, task_str):
name = spectrum['ObjName']
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
sec_source = catalog.entries[name].add_source(
name=sec_reference,
url=sec_refurl,
bibcode=sec_refbib,
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, sec_source)
sources = [sec_source]
if spectrum['Reference']:
sources += [catalog.entries[name]
.add_source(bibcode=spectrum['Reference'])]
sources = uniq_cdl(sources)
if spectrum['Type'] and spectrum['Type'].strip() != 'NoMatch':
for ct in spectrum['Type'].strip().split(','):
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, ct.replace('-norm', '').strip(),
sources)
if spectrum['DiscDate']:
ddate = spectrum['DiscDate'].replace('-', '/')
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, ddate,
sources)
if spectrum['HostName']:
host = urllib.parse.unquote(spectrum['HostName']).replace('*', '')
catalog.entries[name].add_quantity(SUPERNOVA.HOST, host, sources)
if spectrum['UT_Date']:
epoch = str(spectrum['UT_Date'])
year = epoch[:4]
month = epoch[4:6]
day = epoch[6:]
sig = get_sig_digits(day) + 5
mjd = astrotime(year + '-' + month + '-' + str(floor(float(
day))).zfill(2)).mjd
mjd = pretty_num(mjd + float(day) - floor(float(day)), sig=sig)
filename = spectrum['Filename'] if spectrum['Filename'] else ''
instrument = spectrum['Instrument'] if spectrum['Instrument'] else ''
reducer = spectrum['Reducer'] if spectrum['Reducer'] else ''
observer = spectrum['Observer'] if spectrum['Observer'] else ''
snr = str(spectrum['SNR']) if spectrum['SNR'] else ''
if not filename:
raise ValueError('Filename not found for SNDB spectrum!')
if not spectrum['SpecID']:
raise ValueError('ID not found for SNDB spectrum!')
filepath = os.path.join(catalog.get_current_task_repo(),
'UCB/') + filename
spectxt = catalog.load_url(
'http://heracles.astro.berkeley.edu/sndb/download?id=ds:' +
str(spectrum['SpecID']),
filepath,
archived_mode=True)
specdata = list(
csv.reader(
spectxt.splitlines(), delimiter=' ', skipinitialspace=True))
newspecdata = []
for row in specdata:
if row[0][0] == '#':
continue
else:
newspecdata.append(row)
specdata = newspecdata
haserrors = len(specdata[0]) == 3 and specdata[0][2] and specdata[0][
2] != 'NaN'
specdata = [list(ii) for ii in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
if not list(filter(None, errors)):
errors = ''
units = 'Uncalibrated'
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom',
u_fluxes=units,
u_time='MJD',
time=mjd,
wavelengths=wavelengths,
filename=filename,
fluxes=fluxes,
errors=errors,
u_errors=units,
instrument=instrument,
source=sources,
snr=snr,
observer=observer,
reducer=reducer,
deredshifted=('-noz' in filename))
ucbspectracnt = ucbspectracnt + 1
if catalog.args.travis and ucbspectracnt >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
return
def do_nedd(catalog):
task_str = catalog.get_current_task_str()
nedd_path = os.path.join(catalog.get_current_task_repo(),
'NED26.10.1-D-13.1.0-20160930.csv')
f = open(nedd_path, 'r')
data = sorted(list(csv.reader(f, delimiter=',', quotechar='"'))[13:],
key=lambda x: (x[9], x[3]))
reference = "NED-D v" + nedd_path.split('-')[-2]
refurl = "http://ned.ipac.caltech.edu/Library/Distances/"
nedbib = "1991ASSL..171...89H"
olddistname = ''
loopcnt = 0
for r, row in enumerate(pbar(data, task_str)):
if r <= 12:
continue
distname = row[3]
name = name_clean(distname)
# distmod = row[4]
# moderr = row[5]
dist = row[6]
bibcode = unescape(row[8])
snname = name_clean(row[9])
redshift = row[10]
cleanhost = ''
if name != snname and (name + ' HOST' != snname):
cleanhost = host_clean(distname)
if cleanhost.endswith(' HOST'):
cleanhost = ''
if not is_number(dist):
print(dist)
if dist:
catalog.nedd_dict.setdefault(cleanhost,
[]).append(Decimal(dist))
if snname and 'HOST' not in snname:
snname, secondarysource = catalog.new_entry(snname,
srcname=reference,
bibcode=nedbib,
url=refurl,
secondary=True)
if bibcode:
source = catalog.entries[snname].add_source(bibcode=bibcode)
sources = uniq_cdl([source, secondarysource])
else:
sources = secondarysource
if name == snname:
if redshift:
catalog.entries[snname].add_quantity(
SUPERNOVA.REDSHIFT, redshift, sources)
if dist:
catalog.entries[snname].add_quantity(
SUPERNOVA.COMOVING_DIST, dist, sources)
if not redshift:
try:
zatval = z_at_value(cosmo.comoving_distance,
float(dist) * un.Mpc,
zmax=5.0)
sigd = get_sig_digits(str(dist))
redshift = pretty_num(zatval, sig=sigd)
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
pass
else:
cosmosource = catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
combsources = uniq_cdl(
sources.split(',') + [cosmosource])
catalog.entries[snname].add_quantity(
SUPERNOVA.REDSHIFT,
redshift,
combsources,
derived=True)
if cleanhost:
catalog.entries[snname].add_quantity(SUPERNOVA.HOST, cleanhost,
sources)
if catalog.args.update and olddistname != distname:
catalog.journal_entries()
olddistname = distname
loopcnt = loopcnt + 1
if catalog.args.travis and loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0:
break
catalog.journal_entries()
f.close()
return
def add_quantity(self,
quantities,
value,
source,
forcereplacebetter=False,
**kwargs):
"""Add `Quantity` to `FastStars`."""
success = super(FastStars, self).add_quantity(
quantities, value, source, **kwargs)
if not success:
return
for quantity in listify(quantities):
my_quantity_list = self.get(quantity, [])
if ((forcereplacebetter or quantity.replace_better) and
len(my_quantity_list) > 1):
# The quantity that was just added should be last in the list
added_quantity = my_quantity_list.pop()
newquantities = []
isworse = True
if type(quantity) != Key:
isworse = False
elif quantity.type == KEY_TYPES.NUMERIC:
newsig = get_sig_digits(added_quantity[QUANTITY.VALUE])
for ct in my_quantity_list:
addct = False
checke = False
if (len(quantity.kind_preference) > 0 and not set(
listify(ct.get(QUANTITY.KIND, [])))
.isdisjoint(quantity.kind_preference) and
not set(
listify(
added_quantity.get(QUANTITY.KIND,
[])))
.isdisjoint(quantity.kind_preference)):
aqi = min([
quantity.kind_preference.index(x)
for x in listify(added_quantity[
QUANTITY.KIND])
])
qqi = min([
quantity.kind_preference.index(x)
for x in listify(ct[QUANTITY.KIND])
])
if aqi > qqi:
addct = True
if aqi == qqi:
checke = True
if aqi <= qqi:
isworse = False
else:
checke = True
if checke and QUANTITY.CORRELATIONS in ct:
if QUANTITY.CORRELATIONS in added_quantity:
if len(ct[QUANTITY.CORRELATIONS]) > len(added_quantity[QUANTITY.CORRELATIONS]):
addct = True
checke = False
else:
addct = True
checke = False
if checke and QUANTITY.E_VALUE in ct:
if QUANTITY.E_VALUE in added_quantity:
if (float(added_quantity[QUANTITY.E_VALUE])
>= float(ct[QUANTITY.E_VALUE])):
addct = True
if (float(added_quantity[QUANTITY.E_VALUE])
<= float(ct[QUANTITY.E_VALUE])):
isworse = False
if QUANTITY.E_LOWER_VALUE in added_quantity and QUANTITY.E_UPPER_VALUE in added_quantity:
if (0.5*float(added_quantity[QUANTITY.E_LOWER_VALUE])+0.5*float(added_quantity[QUANTITY.E_UPPER_VALUE])
>= float(ct[QUANTITY.E_VALUE])):
addct = True
if (0.5*float(added_quantity[QUANTITY.E_LOWER_VALUE])+0.5*float(added_quantity[QUANTITY.E_UPPER_VALUE])
<= float(ct[QUANTITY.E_VALUE])):
isworse = False
elif checke and QUANTITY.E_LOWER_VALUE in ct and QUANTITY.E_UPPER_VALUE in ct:
if QUANTITY.E_VALUE in added_quantity:
if (float(added_quantity[QUANTITY.E_VALUE])
>= 0.5*float(ct[QUANTITY.E_LOWER_VALUE])+0.5*float(ct[QUANTITY.E_UPPER_VALUE])):
addct = True
if (float(added_quantity[QUANTITY.E_VALUE])
<= 0.5*float(ct[QUANTITY.E_LOWER_VALUE])+0.5*float(ct[QUANTITY.E_UPPER_VALUE])):
isworse = False
if QUANTITY.E_LOWER_VALUE in added_quantity and QUANTITY.E_UPPER_VALUE in added_quantity:
if (0.5*float(added_quantity[QUANTITY.E_LOWER_VALUE])+0.5*float(added_quantity[QUANTITY.E_UPPER_VALUE])
>= 0.5*float(ct[QUANTITY.E_LOWER_VALUE])+0.5*float(ct[QUANTITY.E_UPPER_VALUE])):
addct = True
if (0.5*float(added_quantity[QUANTITY.E_LOWER_VALUE])+0.5*float(added_quantity[QUANTITY.E_UPPER_VALUE])
<= 0.5*float(ct[QUANTITY.E_LOWER_VALUE])+0.5*float(ct[QUANTITY.E_UPPER_VALUE])):
isworse = False
else:
if (checke and
((QUANTITY.E_VALUE in added_quantity) or (QUANTITY.E_LOWER_VALUE in added_quantity and QUANTITY.E_UPPER_VALUE in added_quantity))):
isworse = False
elif (checke and QUANTITY.CORRELATIONS in added_quantity):
isworse = False
else:
oldsig = get_sig_digits(ct[QUANTITY.VALUE])
if oldsig >= newsig:
addct = True
if newsig >= oldsig:
isworse = False
if addct:
newquantities.append(ct)
elif quantity.type == KEY_TYPES.STRING:
for ct in my_quantity_list:
addct = False
if (len(quantity.kind_preference) > 0 and not set(
listify(ct.get(QUANTITY.KIND, [])))
.isdisjoint(quantity.kind_preference) and
not set(
listify(
added_quantity.get(QUANTITY.KIND,
[])))
.isdisjoint(quantity.kind_preference)):
aqi = min([
quantity.kind_preference.index(x)
for x in listify(added_quantity[
QUANTITY.KIND])
])
qqi = min([
quantity.kind_preference.index(x)
for x in listify(ct[QUANTITY.KIND])
])
if aqi >= qqi:
addct = True
if aqi <= qqi:
isworse = False
else:
addct = True
isworse = False
if addct:
newquantities.append(ct)
if isworse:
self._log.info("Removing quantity '{}' with value '{}' "
"and kind '{}' determined to be worse than "
"existing alternative values.".format(
quantity, added_quantity[
QUANTITY.VALUE],
added_quantity.get(QUANTITY.KIND, '')))
else:
newquantities.append(added_quantity)
if len(newquantities) > 0:
self[quantity] = newquantities
return True
def do_nedd(catalog):
task_str = catalog.get_current_task_str()
nedd_path = os.path.join(
catalog.get_current_task_repo(), 'NED26.10.1-D-13.1.0-20160930.csv')
f = open(nedd_path, 'r')
data = sorted(list(csv.reader(f, delimiter=',', quotechar='"'))[
13:], key=lambda x: (x[9], x[3]))
reference = "NED-D v" + nedd_path.split('-')[-2]
refurl = "http://ned.ipac.caltech.edu/Library/Distances/"
nedbib = "1991ASSL..171...89H"
olddistname = ''
loopcnt = 0
for r, row in enumerate(pbar(data, task_str)):
if r <= 12:
continue
distname = row[3]
name = name_clean(distname)
# distmod = row[4]
# moderr = row[5]
dist = row[6]
bibcode = unescape(row[8])
snname = name_clean(row[9])
redshift = row[10]
cleanhost = ''
if name != snname and (name + ' HOST' != snname):
cleanhost = host_clean(distname)
if cleanhost.endswith(' HOST') or cleanhost.startswith('SN'):
cleanhost = ''
if not is_number(dist):
print(dist)
if dist and cleanhost:
catalog.nedd_dict.setdefault(
cleanhost, []).append(Decimal(dist))
if snname and 'HOST' not in snname:
snname, secondarysource = catalog.new_entry(
snname, srcname=reference, bibcode=nedbib, url=refurl,
secondary=True)
if bibcode:
source = catalog.entries[snname].add_source(bibcode=bibcode)
sources = uniq_cdl([source, secondarysource])
else:
sources = secondarysource
if name == snname:
if redshift:
catalog.entries[snname].add_quantity(
SUPERNOVA.REDSHIFT, redshift, sources)
if dist:
catalog.entries[snname].add_quantity(
SUPERNOVA.COMOVING_DIST, dist, sources)
if not redshift:
try:
zatval = z_at_value(cosmo.comoving_distance,
float(dist) * un.Mpc, zmax=5.0)
sigd = get_sig_digits(str(dist))
redshift = pretty_num(zatval, sig=sigd)
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
pass
else:
cosmosource = catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
combsources = uniq_cdl(sources.split(',') +
[cosmosource])
catalog.entries[snname].add_quantity(
SUPERNOVA.REDSHIFT, redshift, combsources,
derived=True)
if cleanhost:
catalog.entries[snname].add_quantity(
SUPERNOVA.HOST, cleanhost, sources)
if catalog.args.update and olddistname != distname:
catalog.journal_entries()
olddistname = distname
loopcnt = loopcnt + 1
if catalog.args.travis and loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0:
break
catalog.journal_entries()
f.close()
return
def do_suspect_spectra(catalog):
task_str = catalog.get_current_task_str()
with open(
os.path.join(catalog.get_current_task_repo(),
'Suspect/sources.json'), 'r') as f:
sourcedict = json.loads(f.read())
with open(
os.path.join(catalog.get_current_task_repo(),
'Suspect/filename-changes.txt'), 'r') as f:
rows = f.readlines()
changedict = {}
for row in rows:
if not row.strip() or row[0] == "#":
continue
items = row.strip().split(' ')
changedict[items[1]] = items[0]
suspectcnt = 0
folders = next(
os.walk(os.path.join(catalog.get_current_task_repo(), 'Suspect')))[1]
for folder in pbar(folders, task_str):
eventfolders = next(
os.walk(
os.path.join(catalog.get_current_task_repo(), 'Suspect/') +
folder))[1]
oldname = ''
for eventfolder in pbar(eventfolders, task_str):
name = eventfolder
if is_number(name[:4]):
name = 'SN' + name
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
sec_ref = 'SUSPECT'
sec_refurl = 'https://www.nhn.ou.edu/~suspect/'
sec_bibc = '2001AAS...199.8408R'
sec_source = catalog.entries[name].add_source(name=sec_ref,
url=sec_refurl,
bibcode=sec_bibc,
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name,
sec_source)
fpath = os.path.join(catalog.get_current_task_repo(), 'Suspect',
folder, eventfolder)
eventspectra = next(os.walk(fpath))[2]
for spectrum in eventspectra:
sources = [sec_source]
bibcode = ''
if spectrum in changedict:
specalias = changedict[spectrum]
else:
specalias = spectrum
if specalias in sourcedict:
bibcode = sourcedict[specalias]
elif name in sourcedict:
bibcode = sourcedict[name]
if bibcode:
source = catalog.entries[name].add_source(
bibcode=unescape(bibcode))
sources += [source]
sources = uniq_cdl(sources)
date = spectrum.split('_')[1]
year = date[:4]
month = date[4:6]
day = date[6:]
sig = get_sig_digits(day) + 5
day_fmt = str(floor(float(day))).zfill(2)
time = astrotime(year + '-' + month + '-' + day_fmt).mjd
time = time + float(day) - floor(float(day))
time = pretty_num(time, sig=sig)
fpath = os.path.join(catalog.get_current_task_repo(),
'Suspect', folder, eventfolder, spectrum)
with open(fpath, 'r') as f:
specdata = list(
csv.reader(f, delimiter=' ', skipinitialspace=True))
specdata = list(filter(None, specdata))
newspec = []
oldval = ''
for row in specdata:
if row[1] == oldval:
continue
newspec.append(row)
oldval = row[1]
specdata = newspec
haserrors = len(
specdata[0]
) == 3 and specdata[0][2] and specdata[0][2] != 'NaN'
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
catalog.entries[name].add_spectrum(u_wavelengths='Angstrom',
u_fluxes='Uncalibrated',
u_time='MJD',
time=time,
wavelengths=wavelengths,
fluxes=fluxes,
errors=errors,
u_errors='Uncalibrated',
source=sources,
filename=spectrum)
suspectcnt = suspectcnt + 1
if (catalog.args.travis
and suspectcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
def add_quantity(self,
quantities,
value,
source,
forcereplacebetter=False,
**kwargs):
success = super().add_quantity(quantities, value, source, **kwargs)
if not success:
return
for quantity in listify(quantities):
my_quantity_list = self.get(quantity, [])
if ((forcereplacebetter or quantity.replace_better) and
len(my_quantity_list) > 1):
# The quantity that was just added should be last in the list
added_quantity = my_quantity_list.pop()
newquantities = []
isworse = True
if quantity in [self._KEYS.DISCOVER_DATE, self._KEYS.MAX_DATE]:
for ct in my_quantity_list:
ctsplit = ct[QUANTITY.VALUE].split('/')
svsplit = added_quantity[QUANTITY.VALUE].split('/')
if len(ctsplit) < len(svsplit):
isworse = False
continue
elif len(ctsplit) < len(svsplit) and len(svsplit) == 3:
val_one = max(
2, get_sig_digits(ctsplit[-1].lstrip('0')))
val_two = max(
2, get_sig_digits(svsplit[-1].lstrip('0')))
if val_one < val_two:
isworse = False
continue
newquantities.append(ct)
else:
if type(quantity) != Key:
isworse = False
elif quantity.type == KEY_TYPES.NUMERIC:
newsig = get_sig_digits(added_quantity[QUANTITY.VALUE])
for ct in my_quantity_list:
addct = False
checke = False
if (len(quantity.kind_preference) > 0 and not set(
listify(ct.get(QUANTITY.KIND, [])))
.isdisjoint(quantity.kind_preference) and
not set(
listify(
added_quantity.get(QUANTITY.KIND,
[])))
.isdisjoint(quantity.kind_preference)):
aqi = min([
quantity.kind_preference.index(x)
for x in listify(added_quantity[
QUANTITY.KIND])
])
qqi = min([
quantity.kind_preference.index(x)
for x in listify(ct[QUANTITY.KIND])
])
if aqi > qqi:
addct = True
if aqi == qqi:
checke = True
if aqi <= qqi:
isworse = False
else:
checke = True
if checke and QUANTITY.E_VALUE in ct:
if QUANTITY.E_VALUE in added_quantity:
if (float(added_quantity[QUANTITY.E_VALUE])
>= float(ct[QUANTITY.E_VALUE])):
addct = True
if (float(added_quantity[QUANTITY.E_VALUE])
<= float(ct[QUANTITY.E_VALUE])):
isworse = False
else:
if (checke and
QUANTITY.E_VALUE in added_quantity):
isworse = False
else:
oldsig = get_sig_digits(ct[QUANTITY.VALUE])
if oldsig >= newsig:
addct = True
if newsig >= oldsig:
isworse = False
if addct:
newquantities.append(ct)
elif quantity.type == KEY_TYPES.STRING:
for ct in my_quantity_list:
addct = False
if (len(quantity.kind_preference) > 0 and not set(
listify(ct.get(QUANTITY.KIND, [])))
.isdisjoint(quantity.kind_preference) and
not set(
listify(
added_quantity.get(QUANTITY.KIND,
[])))
.isdisjoint(quantity.kind_preference)):
aqi = min([
quantity.kind_preference.index(x)
for x in listify(added_quantity[
QUANTITY.KIND])
])
qqi = min([
quantity.kind_preference.index(x)
for x in listify(ct[QUANTITY.KIND])
])
if aqi >= qqi:
addct = True
if aqi <= qqi:
isworse = False
else:
isworse = False
if addct:
newquantities.append(ct)
if isworse:
self._log.info("Removing quantity '{}' with value '{}' "
"and kind '{}' determined to be worse than "
"existing alternative values.".format(
quantity, added_quantity[
QUANTITY.VALUE],
added_quantity.get(QUANTITY.KIND, '')))
else:
newquantities.append(added_quantity)
if len(newquantities) > 0:
self[quantity] = newquantities
return
