def do_snls_photo(catalog):
task_str = catalog.get_current_task_str()
snls_path = os.path.join(catalog.get_current_task_repo(), 'SNLS-ugriz.dat')
data = list(csv.reader(open(snls_path, 'r'), delimiter=' ',
quotechar='"', skipinitialspace=True))
for row in pbar(data, task_str):
flux = row[3]
err = row[4]
# Being extra strict here with the flux constraint, see note below.
if float(flux) < 3.0 * float(err):
continue
name = 'SNLS-' + row[0]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2010A&A...523A...7G')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
band = row[1]
mjd = row[2]
sig = get_sig_digits(flux.split('E')[0]) + 1
# Conversion comes from SNLS-Readme
# NOTE: Datafiles avail for download suggest diff zeropoints than 30,
# need to inquire.
magnitude = pretty_num(30.0 - 2.5 * log10(float(flux)), sig=sig)
e_mag = pretty_num(
2.5 * log10(1.0 + float(err) / float(flux)), sig=sig)
# e_mag = pretty_num(2.5*(log10(float(flux) + float(err)) -
# log10(float(flux))), sig=sig)
catalog.entries[name].add_photometry(
time=mjd, band=band, magnitude=magnitude, e_magnitude=e_mag,
counts=flux, e_counts=err, source=source)
catalog.journal_entries()
return
def set_first_max_light(self):
if ENTRY.MAX_APP_MAG not in self:
mldt, mlmag, mlband, mlsource = self._get_max_light()
if mldt or mlmag or mlband:
source = self.add_self_source()
uniq_src = uniq_cdl([source] + mlsource.split(','))
if mldt:
max_date = make_date_string(mldt.year, mldt.month, mldt.day)
self.add_quantity(ENTRY.MAX_DATE, max_date, uniq_src,
derived=True)
if mlmag:
mlmag = pretty_num(mlmag)
self.add_quantity(ENTRY.MAX_APP_MAG, mlmag, uniq_src,
derived=True)
if mlband:
self.add_quantity(ENTRY.MAX_BAND, mlband, uniq_src,
derived=True)
if (self._KEYS.DISCOVER_DATE not in self or
max([len(x[QUANTITY.VALUE].split('/')) for x in
self[self._KEYS.DISCOVER_DATE]]) < 3):
fldt, flsource = self._get_first_light()
if fldt:
source = self.add_self_source()
disc_date = make_date_string(fldt.year, fldt.month, fldt.day)
self.add_quantity(
self._KEYS.DISCOVER_DATE, disc_date,
uniq_cdl([source] + flsource.split(',')),
derived=True)
if self._KEYS.DISCOVER_DATE not in self and self._KEYS.SPECTRA in self:
minspecmjd = float("+inf")
for spectrum in self[self._KEYS.SPECTRA]:
if 'time' in spectrum and 'u_time' in spectrum:
if spectrum['u_time'] == 'MJD':
mjd = float(spectrum['time'])
elif spectrum['u_time'] == 'JD':
mjd = float(jd_to_mjd(Decimal(spectrum['time'])))
else:
continue
if mjd < minspecmjd:
minspecmjd = mjd
minspecsource = spectrum['source']
if minspecmjd < float("+inf"):
fldt = astrotime(minspecmjd, format='mjd').datetime
source = self.add_self_source()
disc_date = make_date_string(fldt.year, fldt.month, fldt.day)
self.add_quantity(
self._KEYS.DISCOVER_DATE, disc_date,
uniq_cdl([source] + minspecsource.split(',')),
derived=True)
return
def set_first_max_light(self):
if FASTSTARS.MAX_APP_MAG not in self:
# Get the maximum amongst all bands
mldt, mlmag, mlband, mlsource = self._get_max_light()
if mldt or mlmag or mlband:
source = self.add_self_source()
uniq_src = uniq_cdl([source] + mlsource.split(','))
if mldt:
max_date = make_date_string(mldt.year, mldt.month, mldt.day)
self.add_quantity(
FASTSTARS.MAX_DATE, max_date, uniq_src, derived=True)
if mlmag:
mlmag = pretty_num(mlmag)
self.add_quantity(
FASTSTARS.MAX_APP_MAG, mlmag, uniq_src, derived=True)
if mlband:
self.add_quantity(
FASTSTARS.MAX_BAND, mlband, uniq_src, derived=True)
return
def do_donated_spectra(catalog):
"""Import donated spectra."""
task_str = catalog.get_current_task_str()
fpath = os.path.join(catalog.get_current_task_repo(), 'donations')
with open(os.path.join(fpath, 'meta.json'), 'r') as f:
metadict = json.loads(f.read())
donationscnt = 0
oldname = ''
for fname in pbar(list(metadict.keys()), task_str):
name = metadict[fname]['name']
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
sec_bibc = metadict[fname]['bibcode']
name, source = catalog.new_entry(name, bibcode=sec_bibc)
date = metadict[fname].get('date', '')
year, month, day = date.split('/')
sig = get_sig_digits(day) + 5
day_fmt = str(floor(float(day))).zfill(2)
time = astrotime(year + '-' + month + '-' + day_fmt).mjd
time = time + float(day) - floor(float(day))
time = pretty_num(time, sig=sig)
with open(os.path.join(fpath, fname), 'r') as f:
specdata = list(
csv.reader(
f, delimiter=' ', skipinitialspace=True))
specdata = list(filter(None, specdata))
newspec = []
oldval = ''
for row in specdata:
if row[0][0] == '#':
continue
if row[1] == oldval:
continue
newspec.append(row)
oldval = row[1]
specdata = newspec
haserrors = len(specdata[0]) == 3 and specdata[0][2] and specdata[0][
2] != 'NaN'
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.U_TIME: 'MJD',
SPECTRUM.TIME: time,
SPECTRUM.WAVELENGTHS: wavelengths,
SPECTRUM.FLUXES: fluxes,
SPECTRUM.ERRORS: errors,
SPECTRUM.SOURCE: source,
SPECTRUM.FILENAME: fname
}
if 'instrument' in metadict[fname]:
specdict[SPECTRUM.INSTRUMENT] = metadict[fname]['instrument']
if 'telescope' in metadict[fname]:
specdict[SPECTRUM.TELESCOPE] = metadict[fname]['telescope']
if 'yunit' in metadict[fname]:
specdict[SPECTRUM.U_FLUXES] = metadict[fname]['yunit']
specdict[SPECTRUM.U_ERRORS] = metadict[fname]['yunit']
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict[SPECTRUM.U_FLUXES] = fluxunit
specdict[SPECTRUM.U_ERRORS] = fluxunit
catalog.entries[name].add_spectrum(**specdict)
donationscnt = donationscnt + 1
if (catalog.args.travis and
donationscnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
def radec_clean(svalue, quantity, unit=''):
"""Clean R.A. and Dec."""
svalue = svalue.strip()
if unit == 'floatdegrees':
if not is_number(svalue):
return (svalue, unit)
deg = float('%g' % Decimal(svalue))
sig = get_sig_digits(svalue)
if 'ra' in quantity:
flhours = deg / 360.0 * 24.0
hours = floor(flhours)
minutes = floor((flhours - hours) * 60.0)
seconds = (flhours * 60.0 - (hours * 60.0 + minutes)) * 60.0
hours = 0 if hours < 1.e-6 else hours
minutes = 0 if minutes < 1.e-6 else minutes
seconds = 0.0 if seconds < 1.e-6 else seconds
if seconds > 60.0:
raise (ValueError('Invalid seconds value for ' + quantity))
svalue = str(hours).zfill(2) + ':' + str(minutes).zfill(2) + \
':' + zpad(pretty_num(seconds, sig=sig - 1))
elif 'dec' in quantity:
fldeg = abs(deg)
degree = floor(fldeg)
minutes = floor((fldeg - degree) * 60.0)
seconds = (fldeg * 60.0 - (degree * 60.0 + minutes)) * 60.0
minutes = 0 if minutes < 1.e-6 else minutes
seconds = 0.0 if seconds < 1.e-6 else seconds
if seconds > 60.0:
raise (ValueError('Invalid seconds value for ' + quantity))
svalue = (('+' if deg >= 0.0 else '-') +
str(degree).strip('+-').zfill(2) + ':' +
str(minutes).zfill(2) + ':' +
zpad(pretty_num(seconds, sig=sig - 1)))
elif unit == 'nospace' and 'ra' in quantity:
svalue = svalue[:2] + ':' + svalue[2:4] + \
((':' + zpad(svalue[4:])) if len(svalue) > 4 else '')
elif unit == 'nospace' and 'dec' in quantity:
if svalue.startswith(('+', '-')):
svalue = svalue[:3] + ':' + svalue[3:5] + \
((':' + zpad(svalue[5:])) if len(svalue) > 5 else '')
else:
svalue = '+' + svalue[:2] + ':' + svalue[2:4] + \
((':' + zpad(svalue[4:])) if len(svalue) > 4 else '')
else:
svalue = svalue.replace(' ', ':')
if 'dec' in quantity:
valuesplit = svalue.split(':')
svalue = (
('-' if valuesplit[0].startswith('-') else '+') +
valuesplit[0].strip('+-').zfill(2) +
(':' + valuesplit[1].zfill(2) if len(valuesplit) > 1 else '') +
(':' + zpad(valuesplit[2]) if len(valuesplit) > 2 else ''))
if 'ra' in quantity:
sunit = 'hours'
elif 'dec' in quantity:
sunit = 'degrees'
# Correct case of arcseconds = 60.0.
valuesplit = svalue.split(':')
if len(valuesplit) == 3 and valuesplit[-1] in ["60.0", "60.", "60"]:
svalue = valuesplit[0] + ':' + str(
Decimal(valuesplit[1]) + Decimal(1.0)) + ':' + "00.0"
# Strip trailing dots.
svalue = svalue.rstrip('.')
return (svalue, sunit)
def do_ucb_spectra(catalog):
task_str = catalog.get_current_task_str()
sec_reference = 'UCB Filippenko Group\'s Supernova Database (SNDB)'
sec_refurl = 'http://heracles.astro.berkeley.edu/sndb/info'
sec_refbib = '2012MNRAS.425.1789S'
ucbspectracnt = 0
jsontxt = catalog.load_url(
'http://heracles.astro.berkeley.edu/sndb/download?id=allpubspec',
os.path.join(catalog.get_current_task_repo(), 'UCB/allpubspec.json'),
json_sort='SpecID')
if not jsontxt:
return
spectra = json.loads(jsontxt)
spectra = sorted(spectra, key=lambda kk: kk['SpecID'])
oldname = ''
for spectrum in pbar(spectra, task_str):
name = spectrum['ObjName']
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
sec_source = catalog.entries[name].add_source(
name=sec_reference,
url=sec_refurl,
bibcode=sec_refbib,
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, sec_source)
sources = [sec_source]
if spectrum['Reference']:
sources += [catalog.entries[name]
.add_source(bibcode=spectrum['Reference'])]
sources = uniq_cdl(sources)
if spectrum['Type'] and spectrum['Type'].strip() != 'NoMatch':
for ct in spectrum['Type'].strip().split(','):
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, ct.replace('-norm', '').strip(),
sources)
if spectrum['DiscDate']:
ddate = spectrum['DiscDate'].replace('-', '/')
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, ddate,
sources)
if spectrum['HostName']:
host = urllib.parse.unquote(spectrum['HostName']).replace('*', '')
catalog.entries[name].add_quantity(SUPERNOVA.HOST, host, sources)
if spectrum['UT_Date']:
epoch = str(spectrum['UT_Date'])
year = epoch[:4]
month = epoch[4:6]
day = epoch[6:]
sig = get_sig_digits(day) + 5
mjd = astrotime(year + '-' + month + '-' + str(floor(float(
day))).zfill(2)).mjd
mjd = pretty_num(mjd + float(day) - floor(float(day)), sig=sig)
filename = spectrum['Filename'] if spectrum['Filename'] else ''
instrument = spectrum['Instrument'] if spectrum['Instrument'] else ''
reducer = spectrum['Reducer'] if spectrum['Reducer'] else ''
observer = spectrum['Observer'] if spectrum['Observer'] else ''
snr = str(spectrum['SNR']) if spectrum['SNR'] else ''
if not filename:
raise ValueError('Filename not found for SNDB spectrum!')
if not spectrum['SpecID']:
raise ValueError('ID not found for SNDB spectrum!')
filepath = os.path.join(catalog.get_current_task_repo(),
'UCB/') + filename
spectxt = catalog.load_url(
'http://heracles.astro.berkeley.edu/sndb/download?id=ds:' +
str(spectrum['SpecID']),
filepath,
archived_mode=True)
specdata = list(
csv.reader(
spectxt.splitlines(), delimiter=' ', skipinitialspace=True))
newspecdata = []
for row in specdata:
if row[0][0] == '#':
continue
else:
newspecdata.append(row)
specdata = newspecdata
haserrors = len(specdata[0]) == 3 and specdata[0][2] and specdata[0][
2] != 'NaN'
specdata = [list(ii) for ii in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
if not list(filter(None, errors)):
errors = ''
units = 'Uncalibrated'
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom',
u_fluxes=units,
u_time='MJD',
time=mjd,
wavelengths=wavelengths,
filename=filename,
fluxes=fluxes,
errors=errors,
u_errors=units,
instrument=instrument,
source=sources,
snr=snr,
observer=observer,
reducer=reducer,
deredshifted=('-noz' in filename))
ucbspectracnt = ucbspectracnt + 1
if catalog.args.travis and ucbspectracnt >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
return
def do_suspect_spectra(catalog):
task_str = catalog.get_current_task_str()
with open(os.path.join(catalog.get_current_task_repo(),
'Suspect/sources.json'), 'r') as f:
sourcedict = json.loads(f.read())
with open(os.path.join(catalog.get_current_task_repo(),
'Suspect/filename-changes.txt'), 'r') as f:
rows = f.readlines()
changedict = {}
for row in rows:
if not row.strip() or row[0] == "#":
continue
items = row.strip().split(' ')
changedict[items[1]] = items[0]
suspectcnt = 0
folders = next(os.walk(os.path.join(
catalog.get_current_task_repo(), 'Suspect')))[1]
for folder in pbar(folders, task_str):
eventfolders = next(os.walk(os.path.join(
catalog.get_current_task_repo(), 'Suspect/') + folder))[1]
oldname = ''
for eventfolder in pbar(eventfolders, task_str):
name = eventfolder
if is_number(name[:4]):
name = 'SN' + name
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
sec_ref = 'SUSPECT'
sec_refurl = 'https://www.nhn.ou.edu/~suspect/'
sec_bibc = '2001AAS...199.8408R'
sec_source = catalog.entries[name].add_source(
name=sec_ref, url=sec_refurl, bibcode=sec_bibc,
secondary=True)
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, name, sec_source)
fpath = os.path.join(catalog.get_current_task_repo(),
'Suspect', folder, eventfolder)
eventspectra = next(os.walk(fpath))[2]
for spectrum in eventspectra:
sources = [sec_source]
bibcode = ''
if spectrum in changedict:
specalias = changedict[spectrum]
else:
specalias = spectrum
if specalias in sourcedict:
bibcode = sourcedict[specalias]
elif name in sourcedict:
bibcode = sourcedict[name]
if bibcode:
source = catalog.entries[name].add_source(
bibcode=unescape(bibcode))
sources += [source]
sources = uniq_cdl(sources)
date = spectrum.split('_')[1]
year = date[:4]
month = date[4:6]
day = date[6:]
sig = get_sig_digits(day) + 5
day_fmt = str(floor(float(day))).zfill(2)
time = astrotime(year + '-' + month + '-' + day_fmt).mjd
time = time + float(day) - floor(float(day))
time = pretty_num(time, sig=sig)
fpath = os.path.join(catalog.get_current_task_repo(),
'Suspect',
folder,
eventfolder, spectrum)
with open(fpath, 'r') as f:
specdata = list(csv.reader(
f, delimiter=' ', skipinitialspace=True))
specdata = list(filter(None, specdata))
newspec = []
oldval = ''
for row in specdata:
if row[1] == oldval:
continue
newspec.append(row)
oldval = row[1]
specdata = newspec
haserrors = len(specdata[0]) == 3 and specdata[
0][2] and specdata[0][2] != 'NaN'
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom', u_fluxes='Uncalibrated',
u_time='MJD',
time=time,
wavelengths=wavelengths, fluxes=fluxes, errors=errors,
u_errors='Uncalibrated',
source=sources, filename=spectrum)
suspectcnt = suspectcnt + 1
if (catalog.args.travis and
suspectcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
def set_first_max_light(self):
if SUPERNOVA.MAX_APP_MAG not in self:
# Get the maximum amongst all bands
mldt, mlmag, mlband, mlsource = self._get_max_light()
if mldt or mlmag or mlband:
source = self.add_self_source()
uniq_src = uniq_cdl([source] + mlsource.split(','))
if mldt:
max_date = make_date_string(mldt.year, mldt.month, mldt.day)
self.add_quantity(SUPERNOVA.MAX_DATE,
max_date,
uniq_src,
derived=True)
if mlmag:
mlmag = pretty_num(mlmag)
self.add_quantity(SUPERNOVA.MAX_APP_MAG,
mlmag,
uniq_src,
derived=True)
if mlband:
self.add_quantity(SUPERNOVA.MAX_BAND,
mlband,
uniq_src,
derived=True)
if SUPERNOVA.MAX_VISUAL_APP_MAG not in self:
# Get the "visual" maximum
mldt, mlmag, mlband, mlsource = self._get_max_light(visual=True)
if mldt or mlmag or mlband:
source = self.add_self_source()
uniq_src = uniq_cdl([source] + mlsource.split(','))
if mldt:
max_date = make_date_string(mldt.year, mldt.month, mldt.day)
self.add_quantity(SUPERNOVA.MAX_VISUAL_DATE,
max_date,
uniq_src,
derived=True)
if mlmag:
mlmag = pretty_num(mlmag)
self.add_quantity(SUPERNOVA.MAX_VISUAL_APP_MAG,
mlmag,
uniq_src,
derived=True)
if mlband:
self.add_quantity(SUPERNOVA.MAX_VISUAL_BAND,
mlband,
uniq_src,
derived=True)
if (self._KEYS.DISCOVER_DATE not in self or max([
len(x[QUANTITY.VALUE].split('/'))
for x in self[self._KEYS.DISCOVER_DATE]
]) < 3):
fldt, flsource = self._get_first_light()
if fldt:
source = self.add_self_source()
disc_date = make_date_string(fldt.year, fldt.month, fldt.day)
self.add_quantity(self._KEYS.DISCOVER_DATE,
disc_date,
uniq_cdl([source] + flsource.split(',')),
derived=True)
if self._KEYS.DISCOVER_DATE not in self and self._KEYS.SPECTRA in self:
minspecmjd = float("+inf")
for spectrum in self[self._KEYS.SPECTRA]:
if 'time' in spectrum and 'u_time' in spectrum:
if spectrum['u_time'] == 'MJD':
mjd = float(spectrum['time'])
elif spectrum['u_time'] == 'JD':
mjd = float(jd_to_mjd(Decimal(spectrum['time'])))
else:
continue
if mjd < minspecmjd:
minspecmjd = mjd
minspecsource = spectrum['source']
if minspecmjd < float("+inf"):
fldt = astrotime(minspecmjd, format='mjd').datetime
source = self.add_self_source()
disc_date = make_date_string(fldt.year, fldt.month, fldt.day)
self.add_quantity(self._KEYS.DISCOVER_DATE,
disc_date,
uniq_cdl([source] +
minspecsource.split(',')),
derived=True)
return
def do_cleanup(catalog):
"""Task to cleanup catalog before final write."""
task_str = catalog.get_current_task_str()
# Set preferred names, calculate some columns based on imported data,
# sanitize some fields
keys = catalog.entries.copy().keys()
cleanupcnt = 0
for oname in pbar(keys, task_str):
name = catalog.add_entry(oname)
# Set the preferred name, switching to that name if name changed.
name = catalog.entries[name].set_preferred_name()
aliases = catalog.entries[name].get_aliases()
catalog.entries[name].set_first_max_light()
if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['MLS', 'SSS', 'CSS', 'GRB ']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix) and
is_number(alias.replace(prefix, '')[:2])):
discoverdate = ('/'.join([
'20' + alias.replace(prefix, '')[:2],
alias.replace(prefix, '')[2:4],
alias.replace(prefix, '')[4:6]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]:
break
if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]:
prefixes = [
'ASASSN-', 'PS1-', 'PS1', 'PS', 'iPTF', 'PTF', 'SCP-', 'SNLS-',
'SPIRITS', 'LSQ', 'DES', 'SNHiTS', 'Gaia', 'GND', 'GNW', 'GSD',
'GSW', 'EGS', 'COS', 'OGLE', 'HST'
]
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix) and
is_number(alias.replace(prefix, '')[:2]) and
is_number(alias.replace(prefix, '')[:1])):
discoverdate = '20' + alias.replace(prefix, '')[:2]
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]:
break
if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['SNF']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix) and
is_number(alias.replace(prefix, '')[:4])):
discoverdate = ('/'.join([
alias.replace(prefix, '')[:4],
alias.replace(prefix, '')[4:6],
alias.replace(prefix, '')[6:8]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]:
break
if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['PTFS', 'SNSDF']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix) and
is_number(alias.replace(prefix, '')[:2])):
discoverdate = ('/'.join([
'20' + alias.replace(prefix, '')[:2],
alias.replace(prefix, '')[2:4]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]:
break
if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['AT', 'SN', 'OGLE-', 'SM ', 'KSN-']
for alias in aliases:
for prefix in prefixes:
if alias.startswith(prefix):
year = re.findall(r'\d+', alias)
if len(year) == 1:
year = year[0]
else:
continue
if alias.replace(prefix, '').index(year) != 0:
continue
if (year and is_number(year) and '.' not in year and
len(year) <= 4):
discoverdate = year
if catalog.args.verbose:
tprint('Added discoverdate from name [' +
alias + ']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]:
break
if (TIDALDISRUPTION.RA not in catalog.entries[name] or
TIDALDISRUPTION.DEC not in catalog.entries[name]):
prefixes = [
'PSN J', 'MASJ', 'CSS', 'SSS', 'MASTER OT J', 'HST J', 'TCP J',
'MACS J', '2MASS J', 'EQ J', 'CRTS J', 'SMT J'
]
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix) and
is_number(alias.replace(prefix, '')[:6])):
noprefix = alias.split(':')[-1].replace(
prefix, '').replace('.', '')
decsign = '+' if '+' in noprefix else '-'
noprefix = noprefix.replace('+', '|').replace('-', '|')
nops = noprefix.split('|')
if len(nops) < 2:
continue
rastr = nops[0]
decstr = nops[1]
ra = ':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + \
('.' + rastr[6:] if len(rastr) > 6 else '')
dec = (decsign + ':'.join(
[decstr[:2], decstr[2:4], decstr[4:6]]) +
('.' + decstr[6:] if len(decstr) > 6 else ''))
if catalog.args.verbose:
tprint('Added ra/dec from name: ' + ra + ' ' + dec)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.RA, ra, source, derived=True)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DEC, dec, source, derived=True)
break
if TIDALDISRUPTION.RA in catalog.entries[name]:
break
no_host = (TIDALDISRUPTION.HOST not in catalog.entries[name] or
not any([
x[QUANTITY.VALUE] == 'Milky Way'
for x in catalog.entries[name][TIDALDISRUPTION.HOST]
]))
if (TIDALDISRUPTION.RA in catalog.entries[name] and
TIDALDISRUPTION.DEC in catalog.entries[name] and no_host):
from astroquery.irsa_dust import IrsaDust
if name not in catalog.extinctions_dict:
try:
ra_dec = (catalog.entries[name][TIDALDISRUPTION.RA][0][
QUANTITY.VALUE] + " " + catalog.entries[name][
TIDALDISRUPTION.DEC][0][QUANTITY.VALUE])
result = IrsaDust.get_query_table(ra_dec, section='ebv')
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
warnings.warn("Coordinate lookup for " + name +
" failed in IRSA.")
else:
ebv = result['ext SandF mean'][0]
ebverr = result['ext SandF std'][0]
catalog.extinctions_dict[name] = [ebv, ebverr]
if name in catalog.extinctions_dict:
sources = uniq_cdl([
catalog.entries[name].add_self_source(),
catalog.entries[name]
.add_source(bibcode='2011ApJ...737..103S')
])
(catalog.entries[name].add_quantity(
TIDALDISRUPTION.EBV,
str(catalog.extinctions_dict[name][0]),
sources,
e_value=str(catalog.extinctions_dict[name][1]),
derived=True))
if ((TIDALDISRUPTION.HOST in catalog.entries[name] and
(TIDALDISRUPTION.HOST_RA not in catalog.entries[name] or
TIDALDISRUPTION.HOST_DEC not in catalog.entries[name]))):
for host in catalog.entries[name][TIDALDISRUPTION.HOST]:
alias = host[QUANTITY.VALUE]
if ' J' in alias and is_number(alias.split(' J')[-1][:6]):
noprefix = alias.split(' J')[-1].split(':')[-1].replace(
'.', '')
decsign = '+' if '+' in noprefix else '-'
noprefix = noprefix.replace('+', '|').replace('-', '|')
nops = noprefix.split('|')
if len(nops) < 2:
continue
rastr = nops[0]
decstr = nops[1]
hostra = (':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) +
('.' + rastr[6:] if len(rastr) > 6 else ''))
hostdec = decsign + ':'.join([
decstr[:2], decstr[2:4], decstr[4:6]
]) + ('.' + decstr[6:] if len(decstr) > 6 else '')
if catalog.args.verbose:
tprint('Added hostra/hostdec from name: ' + hostra +
' ' + hostdec)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.HOST_RA, hostra, source, derived=True)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.HOST_DEC,
hostdec,
source,
derived=True)
break
if TIDALDISRUPTION.HOST_RA in catalog.entries[name]:
break
if (TIDALDISRUPTION.REDSHIFT not in catalog.entries[name] and
TIDALDISRUPTION.VELOCITY in catalog.entries[name]):
# Find the "best" velocity to use for this
bestsig = 0
for hv in catalog.entries[name][TIDALDISRUPTION.VELOCITY]:
sig = get_sig_digits(hv[QUANTITY.VALUE])
if sig > bestsig:
besthv = hv[QUANTITY.VALUE]
bestsrc = hv['source']
bestsig = sig
if bestsig > 0 and is_number(besthv):
voc = float(besthv) * 1.e5 / CLIGHT
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
(catalog.entries[name].add_quantity(
TIDALDISRUPTION.REDSHIFT,
pretty_num(
sqrt((1. + voc) / (1. - voc)) - 1., sig=bestsig),
sources,
kind='heliocentric',
derived=True))
if (TIDALDISRUPTION.REDSHIFT not in catalog.entries[name] and
len(catalog.nedd_dict) > 0 and
TIDALDISRUPTION.HOST in catalog.entries[name]):
reference = "NED-D"
refurl = "http://ned.ipac.caltech.edu/Library/Distances/"
for host in catalog.entries[name][TIDALDISRUPTION.HOST]:
if host[QUANTITY.VALUE] in catalog.nedd_dict:
source = catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
secondarysource = catalog.entries[name].add_source(
name=reference, url=refurl, secondary=True)
meddist = statistics.median(catalog.nedd_dict[host[
QUANTITY.VALUE]])
redz = z_at_value(cosmo.comoving_distance,
float(meddist) * un.Mpc)
redshift = pretty_num(
redz, sig=get_sig_digits(str(meddist)))
catalog.entries[name].add_quantity(
TIDALDISRUPTION.REDSHIFT,
redshift,
uniq_cdl([source, secondarysource]),
kind='host',
derived=True)
if (TIDALDISRUPTION.MAX_ABS_MAG not in catalog.entries[name] and
TIDALDISRUPTION.MAX_APP_MAG in catalog.entries[name] and
TIDALDISRUPTION.LUM_DIST in catalog.entries[name]):
# Find the "best" distance to use for this
bestsig = 0
for ld in catalog.entries[name][TIDALDISRUPTION.LUM_DIST]:
sig = get_sig_digits(ld[QUANTITY.VALUE])
if sig > bestsig:
bestld = ld[QUANTITY.VALUE]
bestsrc = ld['source']
bestsig = sig
if bestsig > 0 and is_number(bestld) and float(bestld) > 0.:
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
bestldz = z_at_value(cosmo.luminosity_distance,
float(bestld) * un.Mpc)
pnum = (float(catalog.entries[name][
TIDALDISRUPTION.MAX_APP_MAG][0][QUANTITY.VALUE]) - 5.0 *
(log10(float(bestld) * 1.0e6) - 1.0
) + 2.5 * log10(1.0 + bestldz))
pnum = pretty_num(pnum, sig=bestsig)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.MAX_ABS_MAG, pnum, sources, derived=True)
if TIDALDISRUPTION.REDSHIFT in catalog.entries[name]:
# Find the "best" redshift to use for this
bestz, bestkind, bestsig, bestsrc = catalog.entries[
name].get_best_redshift()
if bestsig > 0:
try:
bestz = float(bestz)
except Exception:
print(catalog.entries[name])
raise
if TIDALDISRUPTION.VELOCITY not in catalog.entries[name]:
source = catalog.entries[name].add_self_source()
# FIX: what's happening here?!
pnum = CLIGHT / KM * \
((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.)
pnum = pretty_num(pnum, sig=bestsig)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.VELOCITY,
pnum,
source,
kind=PREF_KINDS[bestkind],
derived=True)
if bestz > 0.:
from astropy.cosmology import Planck15 as cosmo
if TIDALDISRUPTION.LUM_DIST not in catalog.entries[name]:
dl = cosmo.luminosity_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name]
.add_source(bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
TIDALDISRUPTION.LUM_DIST,
pretty_num(
dl.value, sig=bestsig),
sources,
kind=PREF_KINDS[bestkind],
derived=True)
if (TIDALDISRUPTION.MAX_ABS_MAG not in
catalog.entries[name] and
TIDALDISRUPTION.MAX_APP_MAG in
catalog.entries[name]):
source = catalog.entries[name].add_self_source()
pnum = pretty_num(
float(catalog.entries[name][
TIDALDISRUPTION.MAX_APP_MAG][0][
QUANTITY.VALUE]) - 5.0 *
(log10(dl.to('pc').value) - 1.0
) + 2.5 * log10(1.0 + bestz),
sig=bestsig + 1)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.MAX_ABS_MAG,
pnum,
sources,
derived=True)
if TIDALDISRUPTION.COMOVING_DIST not in catalog.entries[
name]:
cd = cosmo.comoving_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name]
.add_source(bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
TIDALDISRUPTION.COMOVING_DIST,
pretty_num(
cd.value, sig=bestsig),
sources,
derived=True)
if all([
x in catalog.entries[name]
for x in [
TIDALDISRUPTION.RA, TIDALDISRUPTION.DEC,
TIDALDISRUPTION.HOST_RA, TIDALDISRUPTION.HOST_DEC
]
]):
# For now just using first coordinates that appear in entry
try:
c1 = coord(
ra=catalog.entries[name][TIDALDISRUPTION.RA][0][
QUANTITY.VALUE],
dec=catalog.entries[name][TIDALDISRUPTION.DEC][0][
QUANTITY.VALUE],
unit=(un.hourangle, un.deg))
c2 = coord(
ra=catalog.entries[name][TIDALDISRUPTION.HOST_RA][0][
QUANTITY.VALUE],
dec=catalog.entries[name][TIDALDISRUPTION.HOST_DEC][0][
QUANTITY.VALUE],
unit=(un.hourangle, un.deg))
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
pass
else:
sources = uniq_cdl(
[catalog.entries[name].add_self_source()] + catalog.
entries[name][TIDALDISRUPTION.RA][0]['source'].split(',') +
catalog.entries[name][TIDALDISRUPTION.DEC][0]['source'].
split(',') + catalog.entries[name][TIDALDISRUPTION.HOST_RA]
[0]['source'].split(',') + catalog.entries[name][
TIDALDISRUPTION.HOST_DEC][0]['source'].split(','))
if 'hostoffsetang' not in catalog.entries[name]:
hosa = Decimal(
hypot(c1.ra.degree - c2.ra.degree, c1.dec.degree -
c2.dec.degree))
hosa = pretty_num(hosa * Decimal(3600.))
catalog.entries[name].add_quantity(
TIDALDISRUPTION.HOST_OFFSET_ANG,
hosa,
sources,
derived=True,
u_value='arcseconds')
if (TIDALDISRUPTION.COMOVING_DIST in catalog.entries[name] and
TIDALDISRUPTION.REDSHIFT in catalog.entries[name] and
TIDALDISRUPTION.HOST_OFFSET_DIST not in
catalog.entries[name]):
offsetsig = get_sig_digits(catalog.entries[name][
TIDALDISRUPTION.HOST_OFFSET_ANG][0][QUANTITY.VALUE])
sources = uniq_cdl(
sources.split(',') + (catalog.entries[name][
TIDALDISRUPTION.COMOVING_DIST][0]['source']).
split(',') + (catalog.entries[name][
TIDALDISRUPTION.REDSHIFT][0]['source']).split(','))
(catalog.entries[name].add_quantity(
TIDALDISRUPTION.HOST_OFFSET_DIST,
pretty_num(
float(catalog.entries[name][
TIDALDISRUPTION.HOST_OFFSET_ANG][0][
QUANTITY.VALUE]) / 3600. * (pi / 180.) *
float(catalog.entries[name][
TIDALDISRUPTION.COMOVING_DIST][0][
QUANTITY.VALUE]) * 1000. /
(1.0 + float(catalog.entries[name][
TIDALDISRUPTION.REDSHIFT][0][QUANTITY.VALUE])),
sig=offsetsig),
sources))
catalog.entries[name].sanitize()
catalog.journal_entries(bury=True, final=True, gz=True)
cleanupcnt = cleanupcnt + 1
if catalog.args.travis and cleanupcnt % 1000 == 0:
break
catalog.save_caches()
return
def do_snls_spectra(catalog):
"""
"""
task_str = catalog.get_current_task_str()
result = Vizier.get_catalogs('J/A+A/507/85/table1')
table = result[list(result.keys())[0]]
table.convert_bytestring_to_unicode()
datedict = {}
for row in table:
datedict['SNLS-' + row['SN']] = str(astrotime(row['Date']).mjd)
oldname = ''
file_names = glob(os.path.join(catalog.get_current_task_repo(), 'SNLS/*'))
for fi, fname in enumerate(pbar_strings(file_names, task_str)):
filename = os.path.basename(fname)
fileparts = filename.split('_')
name = 'SNLS-' + fileparts[1]
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2009A&A...507...85B')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE,
'20' + fileparts[1][:2], source)
f = open(fname, 'r')
data = csv.reader(f, delimiter=' ', skipinitialspace=True)
specdata = []
for r, row in enumerate(data):
if row[0] == '@TELESCOPE':
telescope = row[1].strip()
elif row[0] == '@REDSHIFT':
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT,
row[1].strip(), source)
if r < 14:
continue
specdata.append(list(filter(None, [x.strip(' \t') for x in row])))
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[1]
fluxes = [
pretty_num(float(x) * 1.e-16, sig=get_sig_digits(x))
for x in specdata[2]
]
# FIX: this isnt being used
errors = [
pretty_num(float(x) * 1.e-16, sig=get_sig_digits(x))
for x in specdata[3]
]
fluxunit = 'erg/s/cm^2/Angstrom'
specdict = {
SPECTRUM.WAVELENGTHS: wavelengths,
SPECTRUM.FLUXES: fluxes,
SPECTRUM.ERRORS: errors,
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.U_ERRORS: fluxunit,
SPECTRUM.TELESCOPE: telescope,
SPECTRUM.FILENAME: filename,
SPECTRUM.SOURCE: source
}
if name in datedict:
specdict[SPECTRUM.TIME] = datedict[name]
specdict[SPECTRUM.U_TIME] = 'MJD'
catalog.entries[name].add_spectrum(**specdict)
if catalog.args.travis and fi >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
return
def do_tns(catalog):
from datetime import timedelta
session = requests.Session()
task_str = catalog.get_current_task_str()
tns_url = 'https://wis-tns.weizmann.ac.il/'
search_url = tns_url + \
'search?&num_page=1&format=html&sort=desc&order=id&format=csv&page=0'
csvtxt = catalog.load_cached_url(
search_url, os.path.join(catalog.get_current_task_repo(),
'TNS/index.csv'))
if not csvtxt:
return
maxid = csvtxt.splitlines()[1].split(',')[0].strip('"')
maxpages = ceil(int(maxid) / 1000.)
for page in pbar(range(maxpages), task_str):
fname = os.path.join(catalog.get_current_task_repo(), 'TNS/page-') + \
str(page).zfill(2) + '.csv'
if (catalog.current_task.load_archive(catalog.args) and
os.path.isfile(fname) and page < 7):
with open(fname, 'r') as tns_file:
csvtxt = tns_file.read()
else:
with open(fname, 'w') as tns_file:
session = requests.Session()
ses_url = (tns_url + 'search?&num_page=1000&format=html&edit'
'[type]=&edit[objname]=&edit[id]=&sort=asc&order=id'
'&display[redshift]=1'
'&display[hostname]=1&display[host_redshift]=1'
'&display[source_group_name]=1'
'&display[programs_name]=1'
'&display[internal_name]=1'
'&display[isTNS_AT]=1'
'&display[public]=1'
'&display[end_pop_period]=0'
'&display[spectra_count]=1'
'&display[discoverymag]=1&display[discmagfilter]=1'
'&display[discoverydate]=1&display[discoverer]=1'
'&display[sources]=1'
'&display[bibcode]=1&format=csv&page=' + str(page))
try:
response = session.get(ses_url)
csvtxt = response.text
except:
if os.path.isfile(fname):
with open(fname, 'r') as tns_file:
csvtxt = tns_file.read()
else:
continue
else:
tns_file.write(csvtxt)
tsvin = list(csv.reader(csvtxt.splitlines(), delimiter=','))
for ri, row in enumerate(pbar(tsvin, task_str, leave=False)):
if ri == 0:
continue
if row[4] and 'SN' not in row[4]:
continue
name = row[1].replace(' ', '')
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
name='Transient Name Server', url=tns_url)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
if row[2] and row[2] != '00:00:00.00':
catalog.entries[name].add_quantity(
SUPERNOVA.RA, row[2], source)
if row[3] and row[3] != '+00:00:00.00':
catalog.entries[name].add_quantity(
SUPERNOVA.DEC, row[3], source)
if row[4]:
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, row[4].replace('SN', '').strip(),
source)
if row[5]:
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, row[5], source, kind='spectroscopic')
if row[6]:
catalog.entries[name].add_quantity(
SUPERNOVA.HOST, row[6], source)
if row[7]:
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, row[7], source, kind='host')
if row[8]:
catalog.entries[name].add_quantity(
'discoverer', row[8], source)
# Currently, all events listing all possible observers. TNS bug?
# if row[9]:
# observers = row[9].split(',')
# for observer in observers:
# catalog.entries[name].add_quantity('observer',
# observer.strip(),
# source)
if row[10]:
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, row[10], source)
if row[8] and row[14] and row[15] and row[16]:
survey = row[8]
magnitude = row[14]
band = row[15].split('-')[0]
mjd = astrotime(row[16]).mjd
(catalog.entries[name]
.add_photometry(time=mjd, magnitude=magnitude,
band=band, survey=survey, source=source))
if row[16]:
date = row[16].split()[0].replace('-', '/')
if date != '0000/00/00':
date = date.replace('/00', '')
time = row[16].split()[1]
if time != '00:00:00':
ts = time.split(':')
dt = timedelta(hours=int(ts[0]), minutes=int(
ts[1]), seconds=int(ts[2]))
date += pretty_num(dt.total_seconds() /
(24 * 60 * 60), sig=6).lstrip('0')
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, date, source)
if catalog.args.update:
catalog.journal_entries()
catalog.journal_entries()
return
def do_suspect_spectra(catalog):
task_str = catalog.get_current_task_str()
with open(
os.path.join(catalog.get_current_task_repo(),
'Suspect/sources.json'), 'r') as f:
sourcedict = json.loads(f.read())
with open(
os.path.join(catalog.get_current_task_repo(),
'Suspect/filename-changes.txt'), 'r') as f:
rows = f.readlines()
changedict = {}
for row in rows:
if not row.strip() or row[0] == "#":
continue
items = row.strip().split(' ')
changedict[items[1]] = items[0]
suspectcnt = 0
folders = next(
os.walk(os.path.join(catalog.get_current_task_repo(), 'Suspect')))[1]
for folder in pbar(folders, task_str):
eventfolders = next(
os.walk(
os.path.join(catalog.get_current_task_repo(), 'Suspect/') +
folder))[1]
oldname = ''
for eventfolder in pbar(eventfolders, task_str):
name = eventfolder
if is_number(name[:4]):
name = 'SN' + name
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
sec_ref = 'SUSPECT'
sec_refurl = 'https://www.nhn.ou.edu/~suspect/'
sec_bibc = '2001AAS...199.8408R'
sec_source = catalog.entries[name].add_source(name=sec_ref,
url=sec_refurl,
bibcode=sec_bibc,
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name,
sec_source)
fpath = os.path.join(catalog.get_current_task_repo(), 'Suspect',
folder, eventfolder)
eventspectra = next(os.walk(fpath))[2]
for spectrum in eventspectra:
sources = [sec_source]
bibcode = ''
if spectrum in changedict:
specalias = changedict[spectrum]
else:
specalias = spectrum
if specalias in sourcedict:
bibcode = sourcedict[specalias]
elif name in sourcedict:
bibcode = sourcedict[name]
if bibcode:
source = catalog.entries[name].add_source(
bibcode=unescape(bibcode))
sources += [source]
sources = uniq_cdl(sources)
date = spectrum.split('_')[1]
year = date[:4]
month = date[4:6]
day = date[6:]
sig = get_sig_digits(day) + 5
day_fmt = str(floor(float(day))).zfill(2)
time = astrotime(year + '-' + month + '-' + day_fmt).mjd
time = time + float(day) - floor(float(day))
time = pretty_num(time, sig=sig)
fpath = os.path.join(catalog.get_current_task_repo(),
'Suspect', folder, eventfolder, spectrum)
with open(fpath, 'r') as f:
specdata = list(
csv.reader(f, delimiter=' ', skipinitialspace=True))
specdata = list(filter(None, specdata))
newspec = []
oldval = ''
for row in specdata:
if row[1] == oldval:
continue
newspec.append(row)
oldval = row[1]
specdata = newspec
haserrors = len(
specdata[0]
) == 3 and specdata[0][2] and specdata[0][2] != 'NaN'
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
catalog.entries[name].add_spectrum(u_wavelengths='Angstrom',
u_fluxes='Uncalibrated',
u_time='MJD',
time=time,
wavelengths=wavelengths,
fluxes=fluxes,
errors=errors,
u_errors='Uncalibrated',
source=sources,
filename=spectrum)
suspectcnt = suspectcnt + 1
if (catalog.args.travis
and suspectcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
def do_nedd(catalog):
task_str = catalog.get_current_task_str()
nedd_path = os.path.join(
catalog.get_current_task_repo(), 'NED26.10.1-D-13.1.0-20160930.csv')
f = open(nedd_path, 'r')
data = sorted(list(csv.reader(f, delimiter=',', quotechar='"'))[
13:], key=lambda x: (x[9], x[3]))
reference = "NED-D v" + nedd_path.split('-')[-2]
refurl = "http://ned.ipac.caltech.edu/Library/Distances/"
nedbib = "1991ASSL..171...89H"
olddistname = ''
loopcnt = 0
for r, row in enumerate(pbar(data, task_str)):
if r <= 12:
continue
distname = row[3]
name = name_clean(distname)
# distmod = row[4]
# moderr = row[5]
dist = row[6]
bibcode = unescape(row[8])
snname = name_clean(row[9])
redshift = row[10]
cleanhost = ''
if name != snname and (name + ' HOST' != snname):
cleanhost = host_clean(distname)
if cleanhost.endswith(' HOST') or cleanhost.startswith('SN'):
cleanhost = ''
if not is_number(dist):
print(dist)
if dist and cleanhost:
catalog.nedd_dict.setdefault(
cleanhost, []).append(Decimal(dist))
if snname and 'HOST' not in snname:
snname, secondarysource = catalog.new_entry(
snname, srcname=reference, bibcode=nedbib, url=refurl,
secondary=True)
if bibcode:
source = catalog.entries[snname].add_source(bibcode=bibcode)
sources = uniq_cdl([source, secondarysource])
else:
sources = secondarysource
if name == snname:
if redshift:
catalog.entries[snname].add_quantity(
SUPERNOVA.REDSHIFT, redshift, sources)
if dist:
catalog.entries[snname].add_quantity(
SUPERNOVA.COMOVING_DIST, dist, sources)
if not redshift:
try:
zatval = z_at_value(cosmo.comoving_distance,
float(dist) * un.Mpc, zmax=5.0)
sigd = get_sig_digits(str(dist))
redshift = pretty_num(zatval, sig=sigd)
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
pass
else:
cosmosource = catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
combsources = uniq_cdl(sources.split(',') +
[cosmosource])
catalog.entries[snname].add_quantity(
SUPERNOVA.REDSHIFT, redshift, combsources,
derived=True)
if cleanhost:
catalog.entries[snname].add_quantity(
SUPERNOVA.HOST, cleanhost, sources)
if catalog.args.update and olddistname != distname:
catalog.journal_entries()
olddistname = distname
loopcnt = loopcnt + 1
if catalog.args.travis and loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0:
break
catalog.journal_entries()
f.close()
return
def do_sdss_spectra(catalog):
"""Import spectra from LAMOST."""
task_str = catalog.get_current_task_str()
# Set preferred names, calculate some columns based on imported data,
# sanitize some fields
keys = list(catalog.entries.keys())
fureps = {
'erg/cm2/s/A': 'erg/s/cm^2/Angstrom',
'1E-17 erg/cm^2/s/Ang': 'erg/s/cm^2/Angstrom'
}
c_kms = con.c.cgs.value / 1.0e5
cntsdss = 0
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
if (FASTSTARS.RA not in catalog.entries[oname]
or FASTSTARS.DEC not in catalog.entries[oname]):
continue
else:
xid = SDSS.query_region(coord.SkyCoord(
ra=catalog.entries[oname][FASTSTARS.RA][0]['value'],
dec=catalog.entries[oname][FASTSTARS.DEC][0]['value'],
unit=(un.hourangle, un.deg),
frame='icrs'),
spectro=True)
# xid = SDSS.query_region(coord.SkyCoord(
# ra='14:34:06.17',
# dec='+56:30:47.24',
# unit=(un.hourangle, un.deg), frame='icrs'), spectro=True)
if xid is None:
continue
while len(xid) > 1:
notstar = xid['z'].argmax()
xid.remove_row(notstar)
#print(xid)
# star = None
# for row in tab:
# if (row['objType'] == 'Star' and
# row['Class'].lower() in ['star', 'unknown']):
# star = row
# break
# if not star:
# continue
try:
name, source = catalog.new_entry(oname,
bibcode='2015ApJS..219...12A',
srcname='SDSS',
url='http://www.sdss.org/')
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
ffile = ('spec-' + str(xid['specobjid'][0]) + '.fits.gz')
# furl = 'http://dr3.lamost.org/sas/fits/' + vname + '/' + ffile
datafile = os.path.join(catalog.get_current_task_repo(), 'SDSS',
ffile)
if not os.path.exists(datafile):
# Download spectra
try:
sp = SDSS.get_spectra(matches=xid)[0]
except urllib.error.HTTPError:
catalog.log.warning(
'"{}" threw an HTTP 404, must be error upstream. Will likely go away on the next run.'
.format(oname))
continue
# Identify star
# assert sp[2].data['class'][0]=='STAR'
# Write spectra
# print(catalog.entries[oname][FASTSTARS.RA][0]['value'],catalog.entries[oname][FASTSTARS.DEC][0]['value'])
sp.writeto(datafile, overwrite=True)
# open(datafile, 'wb').write(fr.content)
# Open spectra
hdulist = fits.open(datafile)
# sp contains a list of fits datafiles, identify main one
i_primary = 0
i_coadd = 1
i_specobj = 2
assert hdulist[i_primary].name == 'PRIMARY'
assert hdulist[i_coadd].name == 'COADD'
assert (hdulist[i_specobj].name == 'SPECOBJ'
or hdulist[i_specobj].name == 'SPALL')
# xid = SDSS.query_region(coord.SkyCoord(
# ra='12:11:50.27',
# dec='+14:37:16.2',
# unit=(un.hourangle, un.deg), frame='icrs'), spectro=True)
# from SPECOBJ
# print('.'+hdulist[i_specobj].data['ELODIE_SPTYPE'][0]+'.')
if hdulist[i_specobj].data['ELODIE_SPTYPE'][0] != 'unknown':
ST, SCfull = hdulist[i_specobj].data['ELODIE_SPTYPE'][
0][:2], hdulist[i_specobj].data['ELODIE_SPTYPE'][0][2:]
if len(SCfull) > 0:
if 'IV' in SCfull:
SC = 'sg'
elif 'III' in SCfull:
SC = 'g'
elif 'V' in SCfull:
SC = 'd'
elif 'I' in SCfull:
SC = 'Sg'
else:
SC = False
if SC is not False:
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, SC, source=source)
catalog.entries[name].add_quantity(FASTSTARS.SPECTRAL_TYPE,
ST,
source=source)
if hdulist[i_specobj].data['Z'][0] != 0.0:
catalog.entries[name].add_quantity(
FASTSTARS.REDSHIFT,
str(hdulist[i_specobj].data['Z'][0]),
e_value=str(hdulist[i_specobj].data['Z_ERR'][0]),
source=source)
catalog.entries[name].add_quantity(
FASTSTARS.VELOCITY,
pretty_num(float(hdulist[i_specobj].data['Z'][0]) * c_kms,
sig=5),
e_value=pretty_num(float(
hdulist[i_specobj].data['Z_ERR'][0] * c_kms),
sig=5),
source=source)
for oi, obj in enumerate(hdulist[0].header):
if any(x in ['.', '/'] for x in obj):
del (hdulist[0].header[oi])
hdulist[0].verify('silentfix')
hdrkeys = list(hdulist[0].header.keys())
# print(hdrkeys)
# for key in hdulist[0].header.keys():
# print(key, hdulist[0].header[key])
if hdulist[0].header['SIMPLE']:
if 'JD' in hdrkeys:
mjd = str(jd_to_mjd(Decimal(str(hdulist[0].header['JD']))))
elif 'MJD' in hdrkeys:
mjd = str(hdulist[0].header['MJD'])
elif 'DATE-OBS' in hdrkeys:
if 'T' in hdulist[0].header['DATE-OBS']:
dateobs = hdulist[0].header['DATE-OBS'].strip()
elif 'UTC-OBS' in hdrkeys:
dateobs = hdulist[0].header['DATE-OBS'].strip(
) + 'T' + hdulist[0].header['UTC-OBS'].strip()
mjd = str(astrotime(dateobs, format='isot').mjd)
else:
raise ValueError("Couldn't find JD/MJD for spectrum.")
if hdulist[i_coadd].header['NAXIS'] == 2:
waves = [
str(x)
for x in list(10**hdulist[i_coadd].data['loglam'])
]
fluxes = [
str(x) for x in list(hdulist[i_coadd].data['flux'])
]
else:
print('Warning: Skipping FITS spectrum `{}`.'.format(
datafile))
continue
else:
raise ValueError('Non-simple FITS import not yet supported.')
if 'BUNIT' in hdrkeys:
fluxunit = hdulist[0].header['BUNIT']
if fluxunit in fureps:
fluxunit = fureps[fluxunit]
if fluxunit[:3] == '1E-17':
fluxes = [
str(x * 1e-17)
for x in list(hdulist[i_coadd].data['flux'])
]
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.WAVELENGTHS: waves,
SPECTRUM.TIME: mjd,
SPECTRUM.U_TIME: 'MJD',
SPECTRUM.FLUXES: fluxes,
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.FILENAME: ffile,
SPECTRUM.SOURCE: source
}
if 'TELESCOP' in hdrkeys:
specdict[SPECTRUM.TELESCOPE] = hdulist[0].header['TELESCOP']
if 'INSTRUME' in hdrkeys:
specdict[SPECTRUM.INSTRUMENT] = hdulist[0].header['INSTRUME']
if 'SITENAME' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['SITENAME']
elif 'OBSERVAT' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['OBSERVAT']
if 'OBSERVER' in hdrkeys:
specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER']
if 'AIRMASS' in hdrkeys:
specdict[SPECTRUM.AIRMASS] = hdulist[0].header['AIRMASS']
catalog.entries[name].add_spectrum(**specdict)
cntsdss += 1
hdulist.close()
catalog.journal_entries()
print('`{}` have SDSS spectra.'.format(cntsdss))
return
def do_snf_specta(catalog):
task_str = catalog.get_current_task_str()
bibcodes = {
'SN2005gj': '2006ApJ...650..510A',
'SN2006D': '2007ApJ...654L..53T',
'SN2007if': '2010ApJ...713.1073S',
'SN2011fe': '2013A&A...554A..27P'
}
oldname = ''
snfcnt = 0
eventfolders = next(
os.walk(os.path.join(catalog.get_current_task_repo(), 'SNFactory')))[1]
for eventfolder in pbar(eventfolders, task_str):
oname = eventfolder
name = catalog.get_preferred_name(oname)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
sec_reference = 'Nearby Supernova Factory'
sec_refurl = 'http://snfactory.lbl.gov/'
sec_bibcode = '2002SPIE.4836...61A'
sec_source = catalog.entries[name].add_source(name=sec_reference,
url=sec_refurl,
bibcode=sec_bibcode,
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oname, sec_source)
bibcode = bibcodes[oname]
source = catalog.entries[name].add_source(bibcode=bibcode)
sources = uniq_cdl([source, sec_source])
use_path = os.path.join(catalog.get_current_task_repo(), 'SNFactory',
eventfolder, '*.dat')
eventspectra = glob(use_path)
for spectrum in pbar(eventspectra, task_str):
filename = os.path.basename(spectrum)
with open(spectrum) as spec_file:
specdata = list(
csv.reader(spec_file, delimiter=' ',
skipinitialspace=True))
specdata = list(filter(None, specdata))
newspec = []
time = ''
telescope = ''
instrument = ''
observer = ''
observatory = ''
if 'Keck_20060202_R' in spectrum:
time = '53768.23469'
elif 'Spectrum05_276' in spectrum:
time = pretty_num(astrotime('2005-10-03').mjd, sig=5)
elif 'Spectrum05_329' in spectrum:
time = pretty_num(astrotime('2005-11-25').mjd, sig=5)
elif 'Spectrum05_336' in spectrum:
time = pretty_num(astrotime('2005-12-02').mjd, sig=5)
for row in specdata:
if row[0][0] == '#':
joinrow = (' '.join(row)).split('=')
if len(joinrow) < 2:
continue
field = joinrow[0].strip('# ')
value = joinrow[1].split('/')[0].strip('\' ')
if not time:
if field == 'JD':
time = str(jd_to_mjd(Decimal(value)))
elif field == 'MJD':
time = value
elif field == 'MJD-OBS':
time = value
if field == 'OBSERVER':
observer = value.capitalize()
if field == 'OBSERVAT':
observatory = value.capitalize()
if field == 'TELESCOP':
telescope = value.capitalize()
if field == 'INSTRUME':
instrument = value.capitalize()
else:
newspec.append(row)
if not time:
raise ValueError('Time missing from spectrum.')
specdata = newspec
haserrors = len(
specdata[0]
) == 3 and specdata[0][2] and specdata[0][2] != 'NaN'
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
unit_err = ('Variance'
if oldname == 'SN2011fe' else 'erg/s/cm^2/Angstrom')
unit_flx = 'erg/s/cm^2/Angstrom'
catalog.entries[name].add_spectrum(u_wavelengths='Angstrom',
u_fluxes=unit_flx,
u_time='MJD',
time=time,
wavelengths=wavelengths,
fluxes=fluxes,
errors=errors,
observer=observer,
observatory=observatory,
telescope=telescope,
instrument=instrument,
u_errors=unit_err,
source=sources,
filename=filename)
snfcnt = snfcnt + 1
if (catalog.args.travis
and snfcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
def do_ucb_spectra(catalog):
task_str = catalog.get_current_task_str()
sec_reference = 'UCB Filippenko Group\'s Supernova Database (SNDB)'
sec_refurl = 'http://heracles.astro.berkeley.edu/sndb/info'
sec_refbib = '2012MNRAS.425.1789S'
ucbspectracnt = 0
jsontxt = catalog.load_url(
'http://heracles.astro.berkeley.edu/sndb/download?id=allpubspec',
os.path.join(catalog.get_current_task_repo(), 'UCB/allpubspec.json'),
json_sort='SpecID')
if not jsontxt:
return
spectra = json.loads(jsontxt)
spectra = sorted(spectra, key=lambda kk: kk['SpecID'])
oldname = ''
for spectrum in pbar(spectra, task_str):
name = spectrum['ObjName']
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
sec_source = catalog.entries[name].add_source(name=sec_reference,
url=sec_refurl,
bibcode=sec_refbib,
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, sec_source)
sources = [sec_source]
if spectrum['Reference']:
sources += [
catalog.entries[name].add_source(bibcode=spectrum['Reference'])
]
sources = uniq_cdl(sources)
if spectrum['Type'] and spectrum['Type'].strip() != 'NoMatch':
for ct in spectrum['Type'].strip().split(','):
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE,
ct.replace('-norm', '').strip(), sources)
if spectrum['DiscDate']:
ddate = spectrum['DiscDate'].replace('-', '/')
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, ddate,
sources)
if spectrum['HostName']:
host = urllib.parse.unquote(spectrum['HostName']).replace('*', '')
catalog.entries[name].add_quantity(SUPERNOVA.HOST, host, sources)
if spectrum['UT_Date']:
epoch = str(spectrum['UT_Date'])
year = epoch[:4]
month = epoch[4:6]
day = epoch[6:]
sig = get_sig_digits(day) + 5
mjd = astrotime(year + '-' + month + '-' +
str(floor(float(day))).zfill(2)).mjd
mjd = pretty_num(mjd + float(day) - floor(float(day)), sig=sig)
filename = spectrum['Filename'] if spectrum['Filename'] else ''
instrument = spectrum['Instrument'] if spectrum['Instrument'] else ''
reducer = spectrum['Reducer'] if spectrum['Reducer'] else ''
observer = spectrum['Observer'] if spectrum['Observer'] else ''
snr = str(spectrum['SNR']) if spectrum['SNR'] else ''
if not filename:
raise ValueError('Filename not found for SNDB spectrum!')
if not spectrum['SpecID']:
raise ValueError('ID not found for SNDB spectrum!')
filepath = os.path.join(catalog.get_current_task_repo(),
'UCB/') + filename
spectxt = catalog.load_url(
'http://heracles.astro.berkeley.edu/sndb/download?id=ds:' +
str(spectrum['SpecID']),
filepath,
archived_mode=True)
specdata = list(
csv.reader(spectxt.splitlines(),
delimiter=' ',
skipinitialspace=True))
newspecdata = []
for row in specdata:
if not row or not row[0] or row[0][0] == '#':
continue
else:
newspecdata.append(row)
specdata = newspecdata
haserrors = len(
specdata[0]) == 3 and specdata[0][2] and specdata[0][2] != 'NaN'
specdata = [list(ii) for ii in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
if not list(filter(None, errors)):
errors = ''
units = 'Uncalibrated'
catalog.entries[name].add_spectrum(u_wavelengths='Angstrom',
u_fluxes=units,
u_time='MJD',
time=mjd,
wavelengths=wavelengths,
filename=filename,
fluxes=fluxes,
errors=errors,
u_errors=units,
instrument=instrument,
source=sources,
snr=snr,
observer=observer,
reducer=reducer,
deredshifted=('-noz' in filename))
ucbspectracnt = ucbspectracnt + 1
if catalog.args.travis and ucbspectracnt >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
return
hosts[hn]['photocount'] += len(item['photometry'])
if 'spectra' in item:
hosts[hn]['spectracount'] += len(item['spectra'])
curtime = time.time()
centrate = 100.0 * 365.25 * 24.0 * 60.0 * 60.0
for hn in hosts:
finitedates = sorted(
[x for x in hosts[hn]['eventdates'] if x != float("inf")])
if len(finitedates) >= 2:
datediff = curtime - finitedates[0]
lamb = float(len(finitedates)) / (curtime - finitedates[0]) * centrate
hosts[hn]['rate'] = (
pretty_num(lamb, sig=3) + ',' +
pretty_num(lamb / sqrt(float(len(finitedates))), sig=3))
else:
hosts[hn]['rate'] = ''
hosts[hn]['events'] = [
x
for (y, x) in sorted(zip(hosts[hn]['eventdates'], hosts[hn]['events']),
key=lambda ev: ev[0])
]
del hosts[hn]['eventdates']
# Convert to array since that's what datatables expects
hosts = list(hosts.values())
jsonstring = json.dumps(hosts,
indent='\t',
def do_nedd(catalog):
task_str = catalog.get_current_task_str()
nedd_path = os.path.join(catalog.get_current_task_repo(),
'NED26.10.1-D-13.1.0-20160930.csv')
f = open(nedd_path, 'r')
data = sorted(list(csv.reader(f, delimiter=',', quotechar='"'))[13:],
key=lambda x: (x[9], x[3]))
reference = "NED-D v" + nedd_path.split('-')[-2]
refurl = "http://ned.ipac.caltech.edu/Library/Distances/"
nedbib = "1991ASSL..171...89H"
olddistname = ''
loopcnt = 0
for r, row in enumerate(pbar(data, task_str)):
if r <= 12:
continue
distname = row[3]
name = name_clean(distname)
# distmod = row[4]
# moderr = row[5]
dist = row[6]
bibcode = unescape(row[8])
snname = name_clean(row[9])
redshift = row[10]
cleanhost = ''
if name != snname and (name + ' HOST' != snname):
cleanhost = host_clean(distname)
if cleanhost.endswith(' HOST'):
cleanhost = ''
if not is_number(dist):
print(dist)
if dist:
catalog.nedd_dict.setdefault(cleanhost,
[]).append(Decimal(dist))
if snname and 'HOST' not in snname:
snname, secondarysource = catalog.new_entry(snname,
srcname=reference,
bibcode=nedbib,
url=refurl,
secondary=True)
if bibcode:
source = catalog.entries[snname].add_source(bibcode=bibcode)
sources = uniq_cdl([source, secondarysource])
else:
sources = secondarysource
if name == snname:
if redshift:
catalog.entries[snname].add_quantity(
SUPERNOVA.REDSHIFT, redshift, sources)
if dist:
catalog.entries[snname].add_quantity(
SUPERNOVA.COMOVING_DIST, dist, sources)
if not redshift:
try:
zatval = z_at_value(cosmo.comoving_distance,
float(dist) * un.Mpc,
zmax=5.0)
sigd = get_sig_digits(str(dist))
redshift = pretty_num(zatval, sig=sigd)
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
pass
else:
cosmosource = catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
combsources = uniq_cdl(
sources.split(',') + [cosmosource])
catalog.entries[snname].add_quantity(
SUPERNOVA.REDSHIFT,
redshift,
combsources,
derived=True)
if cleanhost:
catalog.entries[snname].add_quantity(SUPERNOVA.HOST, cleanhost,
sources)
if catalog.args.update and olddistname != distname:
catalog.journal_entries()
olddistname = distname
loopcnt = loopcnt + 1
if catalog.args.travis and loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0:
break
catalog.journal_entries()
f.close()
return
def do_snls_spectra(catalog):
"""
"""
task_str = catalog.get_current_task_str()
result = Vizier.get_catalogs('J/A+A/507/85/table1')
table = result[list(result.keys())[0]]
table.convert_bytestring_to_unicode(python3_only=True)
datedict = {}
for row in table:
datedict['SNLS-' + row['SN']] = str(astrotime(row['Date']).mjd)
oldname = ''
file_names = glob(os.path.join(catalog.get_current_task_repo(), 'SNLS/*'))
for fi, fname in enumerate(pbar_strings(file_names, task_str)):
filename = os.path.basename(fname)
fileparts = filename.split('_')
name = 'SNLS-' + fileparts[1]
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2009A&A...507...85B')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, '20' + fileparts[1][:2], source)
f = open(fname, 'r')
data = csv.reader(f, delimiter=' ', skipinitialspace=True)
specdata = []
for r, row in enumerate(data):
if row[0] == '@TELESCOPE':
telescope = row[1].strip()
elif row[0] == '@REDSHIFT':
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, row[1].strip(), source)
if r < 14:
continue
specdata.append(list(filter(None, [x.strip(' \t') for x in row])))
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[1]
fluxes = [pretty_num(float(x) * 1.e-16, sig=get_sig_digits(x))
for x in specdata[2]]
# FIX: this isnt being used
# errors = [pretty_num(float(x)*1.e-16, sig=get_sig_digits(x)) for x in
# specdata[3]]
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom', u_fluxes='erg/s/cm^2/Angstrom',
wavelengths=wavelengths,
fluxes=fluxes, u_time='MJD' if name in datedict else '',
time=datedict[name] if name in datedict else '',
telescope=telescope, source=source,
filename=filename)
if catalog.args.travis and fi >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
return
def do_nedd(catalog):
task_str = catalog.get_current_task_str()
nedd_path = os.path.join(
catalog.get_current_task_repo(), 'NED26.05.1-D-12.1.0-20160501.csv')
f = open(nedd_path, 'r')
data = sorted(list(csv.reader(f, delimiter=',', quotechar='"'))[
13:], key=lambda x: (x[9], x[3]))
reference = "NED-D"
refurl = "http://ned.ipac.caltech.edu/Library/Distances/"
nedd_dict = OrderedDict()
olddistname = ''
for r, row in enumerate(pbar(data, task_str)):
if r <= 12:
continue
distname = row[3]
name = name_clean(distname)
# distmod = row[4]
# moderr = row[5]
dist = row[6]
bibcode = unescape(row[8])
snname = name_clean(row[9])
redshift = row[10]
cleanhost = ''
if name != snname and (name + ' HOST' != snname):
cleanhost = host_clean(distname)
if cleanhost.endswith(' HOST'):
cleanhost = ''
if not is_number(dist):
print(dist)
if dist:
nedd_dict.setdefault(cleanhost, []).append(Decimal(dist))
if snname and 'HOST' not in snname:
snname, secondarysource = catalog.new_entry(
snname, srcname=reference, url=refurl, secondary=True)
if bibcode:
source = catalog.entries[snname].add_source(bibcode=bibcode)
sources = uniq_cdl([source, secondarysource])
else:
sources = secondarysource
if name == snname:
if redshift:
catalog.entries[snname].add_quantity(
'redshift', redshift, sources)
if dist:
catalog.entries[snname].add_quantity(
'comovingdist', dist, sources)
if not redshift:
try:
zatval = z_at_value(cosmo.comoving_distance,
float(dist) * un.Mpc, zmax=5.0)
sigd = get_sig_digits(str(dist))
redshift = pretty_num(zatval, sig=sigd)
except (KeyboardInterrupt, SystemExit):
raise
except:
pass
else:
cosmosource = catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
combsources = uniq_cdl(sources.split(',') +
[cosmosource])
catalog.entries[snname].add_quantity('redshift',
redshift,
combsources)
if cleanhost:
catalog.entries[snname].add_quantity(
'host', cleanhost, sources)
if catalog.args.update and olddistname != distname:
catalog.journal_entries()
olddistname = distname
catalog.journal_entries()
f.close()
return
def do_lamost(catalog):
"""Import spectra from LAMOST."""
task_str = catalog.get_current_task_str()
# Set preferred names, calculate some columns based on imported data,
# sanitize some fields
keys = list(catalog.entries.keys())
viz = Vizier(columns=["**"])
fureps = {'erg/cm2/s/A': 'erg/s/cm^2/Angstrom'}
c_kms = con.c.cgs.value / 1.0e5
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
if (FASTSTARS.RA not in catalog.entries[oname]
or FASTSTARS.DEC not in catalog.entries[oname]):
continue
else:
result = viz.query_region(coord.SkyCoord(
ra=catalog.entries[oname][FASTSTARS.RA][0]['value'],
dec=catalog.entries[oname][FASTSTARS.DEC][0]['value'],
unit=(un.hourangle, un.deg),
frame='icrs'),
width="2s",
catalog="V/149/dr2")
if not result.keys():
continue
tab = result['V/149/dr2']
star = None
for row in tab:
if (row['objType'] == 'Star'
and row['Class'].lower() in ['star', 'unknown']):
star = row
break
if not star:
continue
try:
name, source = catalog.new_entry(oname,
bibcode='2016yCat.5149....0L',
srcname='LAMOST',
url='http://dr3.lamost.org/')
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
if row['SubClass'] is not 'Non':
#catalog.entries[name].add_quantity(
# FASTSTARS.SPECTRAL_TYPE, row['SubClass'], source=source)
ST, SCfull = row['SubClass'][:2], row['SubClass'][2:]
if len(SCfull) > 0:
if 'IV' in SCfull:
SC = 'sg'
elif 'III' in SCfull:
SC = 'g'
elif 'V' in SCfull:
SC = 'd'
elif 'I' in SCfull:
SC = 'Sg'
catalog.entries[name].add_quantity(FASTSTARS.STELLAR_CLASS,
SC,
source=source)
catalog.entries[name].add_quantity(FASTSTARS.SPECTRAL_TYPE,
ST,
source=source)
if row['z'] and is_number(row['z']):
catalog.entries[name].add_quantity(FASTSTARS.REDSHIFT,
str(row['z']),
e_value=str(row['e_z']),
source=source)
catalog.entries[name].add_quantity(
FASTSTARS.VELOCITY,
pretty_num(float(row['z']) * c_kms, sig=5),
e_value=pretty_num(float(row['e_z'] * c_kms), sig=5),
source=source)
mag_types = list(row['magType'].replace('psf_', ''))
nmt = []
nmi = 0
for mt in mag_types:
if is_number(mt):
nmt[nmi - 1] += mt
else:
nmt += mt
nmi += 1
mag_types = [
x.upper() if x in ['b', 'v', 'j', 'h'] else x for x in nmt
]
for mi, mt in enumerate(mag_types):
snrf = 'snr' + mt.lower()
if snrf in row.columns and float(row[snrf]) < 3:
continue
photodict = {
PHOTOMETRY.TIME: str(row['MJD']),
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.BAND: mt,
PHOTOMETRY.TELESCOPE: 'LAMOST',
PHOTOMETRY.MAGNITUDE: str(row['mag' + str(mi + 1)]),
PHOTOMETRY.SOURCE: source
}
if snrf in row.columns:
photodict[PHOTOMETRY.E_MAGNITUDE] = str(
Decimal('2.5') *
(Decimal('1') +
Decimal('1') / Decimal(str(row[snrf]))).log10())[:5]
catalog.entries[name].add_photometry(**photodict)
vname = row['PlanId']
ffile = ('spec-' + row['LMJD'] + '-' + vname + '_sp' +
row['spId'] + '-' + row['FiberId'] + '.fits.gz')
furl = 'http://dr3.lamost.org/sas/fits/' + vname + '/' + ffile
datafile = os.path.join(catalog.get_current_task_repo(), 'LAMOST',
ffile)
if not os.path.exists(datafile):
fr = requests.get(furl)
open(datafile, 'wb').write(fr.content)
hdulist = fits.open(datafile)
for oi, obj in enumerate(hdulist[0].header):
if any(x in ['.', '/'] for x in obj):
del (hdulist[0].header[oi])
hdulist[0].verify('silentfix')
hdrkeys = list(hdulist[0].header.keys())
# print(hdrkeys)
# for key in hdulist[0].header.keys():
# print(key, hdulist[0].header[key])
if hdulist[0].header['SIMPLE']:
if 'JD' in hdrkeys:
mjd = str(jd_to_mjd(Decimal(str(hdulist[0].header['JD']))))
elif 'MJD' in hdrkeys:
mjd = str(hdulist[0].header['MJD'])
elif 'DATE-OBS' in hdrkeys:
if 'T' in hdulist[0].header['DATE-OBS']:
dateobs = hdulist[0].header['DATE-OBS'].strip()
elif 'UTC-OBS' in hdrkeys:
dateobs = hdulist[0].header['DATE-OBS'].strip(
) + 'T' + hdulist[0].header['UTC-OBS'].strip()
mjd = str(astrotime(dateobs, format='isot').mjd)
else:
raise ValueError("Couldn't find JD/MJD for spectrum.")
if hdulist[0].header['NAXIS'] == 2:
waves = [str(x) for x in list(hdulist[0].data)[2]]
fluxes = [str(x) for x in list(hdulist[0].data)[0]]
else:
print('Warning: Skipping FITS spectrum `{}`.'.format(
datafile))
continue
else:
raise ValueError('Non-simple FITS import not yet supported.')
if 'BUNIT' in hdrkeys:
fluxunit = hdulist[0].header['BUNIT']
if fluxunit in fureps:
fluxunit = fureps[fluxunit]
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.WAVELENGTHS: waves,
SPECTRUM.TIME: mjd,
SPECTRUM.U_TIME: 'MJD',
SPECTRUM.FLUXES: fluxes,
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.FILENAME: ffile,
SPECTRUM.SOURCE: source
}
if 'TELESCOP' in hdrkeys:
specdict[SPECTRUM.TELESCOPE] = hdulist[0].header['TELESCOP']
if 'INSTRUME' in hdrkeys:
specdict[SPECTRUM.INSTRUMENT] = hdulist[0].header['INSTRUME']
if 'SITENAME' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['SITENAME']
elif 'OBSERVAT' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['OBSERVAT']
if 'OBSERVER' in hdrkeys:
specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER']
if 'AIRMASS' in hdrkeys:
specdict[SPECTRUM.AIRMASS] = hdulist[0].header['AIRMASS']
catalog.entries[name].add_spectrum(**specdict)
hdulist.close()
catalog.journal_entries()
return
def do_tns(catalog):
"""Load TNS metadata."""
session = requests.Session()
task_str = catalog.get_current_task_str()
tns_url = 'https://wis-tns.weizmann.ac.il/'
search_url = tns_url + \
'search?&num_page=1&format=html&sort=desc&order=id&format=csv&page=0'
csvtxt = catalog.load_url(search_url,
os.path.join(catalog.get_current_task_repo(),
'TNS', 'index.csv'))
if not csvtxt:
return
maxid = csvtxt.splitlines()[1].split(',')[0].strip('"')
maxpages = ceil(int(maxid) / 1000.)
for page in pbar(range(maxpages), task_str):
fname = os.path.join(
catalog.get_current_task_repo(), 'TNS',
'page-') + str(page).zfill(2) + '.csv'
if (catalog.current_task.load_archive(catalog.args) and
os.path.isfile(fname) and page < 7):
with open(fname, 'r') as tns_file:
csvtxt = tns_file.read()
else:
session = requests.Session()
ses_url = (tns_url + 'search?&num_page=1000&format=html&edit'
'[type]=&edit[objname]=&edit[id]=&sort=asc&order=id'
'&display[redshift]=1'
'&display[hostname]=1&display[host_redshift]=1'
'&display[source_group_name]=1'
'&display[programs_name]=1'
'&display[internal_name]=1'
'&display[isTNS_AT]=1'
'&display[public]=1'
'&display[end_pop_period]=0'
'&display[spectra_count]=1'
'&display[discoverymag]=1&display[discmagfilter]=1'
'&display[discoverydate]=1&display[discoverer]=1'
'&display[sources]=1'
'&display[bibcode]=1&format=csv&page=' + str(page))
try:
response = session.get(ses_url, timeout=40)
csvtxt = response.text
except Exception as e:
catalog.log.warning(
'Could not download TNS page #{}.'.format(str(page)))
if os.path.isfile(fname):
with open(fname, 'r') as tns_file:
csvtxt = tns_file.read()
else:
continue
else:
with open(fname, 'w') as tns_file:
tns_file.write(csvtxt)
tsvin = list(csv.reader([x.replace('[data validation error, please contact site admin]',
'""') for x in csvtxt.splitlines()], delimiter=','))
for ri, row in enumerate(pbar(tsvin, task_str, leave=False)):
if ri == 0:
continue
if row[4] and 'SN' not in row[4]:
continue
name = row[1].replace(' ', '')
if len(name) < 5:
continue
name, source = catalog.new_entry(
name, srcname='Transient Name Server', url=tns_url)
if row[2] and row[2] != '00:00:00.00':
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[2],
source)
if row[3] and row[3] != '+00:00:00.00':
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[3],
source)
if row[4]:
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, row[4].replace('SN', '').strip(),
source)
if row[5]:
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, row[5], source, kind='spectroscopic')
if row[6]:
catalog.entries[name].add_quantity(SUPERNOVA.HOST, row[6],
source)
if row[7]:
catalog.entries[name].add_quantity(
[SUPERNOVA.REDSHIFT, SUPERNOVA.HOST_REDSHIFT],
row[7],
source,
kind='host')
if row[8]:
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVERER,
row[8], source)
# Currently, all events listing all possible observers. TNS bug?
# if row[9]:
# observers = row[9].split(',')
# for observer in observers:
# catalog.entries[name].add_quantity('observer',
# observer.strip(),
# source)
if row[11]:
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[11],
source)
if row[19]:
date = row[19].split()[0].replace('-', '/')
if 'clear' in row[19].lower():
print(row)
if date != '0000/00/00' and 'clear' not in row[19].lower():
date = date.replace('/00', '')
dsplit = row[19].split()
if len(dsplit) >= 2:
t = dsplit[1]
if t != '00:00:00':
ts = t.split(':')
dt = timedelta(
hours=int(ts[0]),
minutes=int(ts[1]),
seconds=int(ts[2]))
date += pretty_num(
dt.total_seconds() / (24 * 60 * 60),
sig=6).lstrip('0')
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE,
date, source)
if catalog.args.travis and ri >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
catalog.journal_entries()
if 'photometry' in item:
hosts[hn]['photocount'] += len(item['photometry'])
if 'spectra' in item:
hosts[hn]['spectracount'] += len(item['spectra'])
curtime = time.time()
centrate = 100.0 * 365.25 * 24.0 * 60.0 * 60.0
for hn in hosts:
finitedates = sorted(
[x for x in hosts[hn]['eventdates'] if x != float("inf")])
if len(finitedates) >= 2:
datediff = curtime - finitedates[0]
lamb = float(len(finitedates)) / (curtime - finitedates[0]) * centrate
hosts[hn]['rate'] = (pretty_num(lamb, sig=3) + ',' +
pretty_num(lamb / sqrt(float(len(finitedates))), sig=3))
else:
hosts[hn]['rate'] = ''
hosts[hn]['events'] = [x for (y, x) in sorted(
zip(hosts[hn]['eventdates'], hosts[hn]['events']), key=lambda ev: ev[0])]
del hosts[hn]['eventdates']
# Convert to array since that's what datatables expects
hosts = list(hosts.values())
jsonstring = json.dumps(
hosts, indent='\t', separators=(',', ':'), ensure_ascii=False)
with open(outdir + 'hosts.json', 'w') as f:
f.write(jsonstring)
def do_cleanup(catalog):
"""Cleanup catalog after importing all data."""
task_str = catalog.get_current_task_str()
# Set preferred names, calculate some columns based on imported data,
# sanitize some fields
keys = list(catalog.entries.keys())
cleanupcnt = 0
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
try:
name = catalog.add_entry(oname)
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
# Set the preferred name, switching to that name if name changed.
name = catalog.entries[name].set_preferred_name()
aliases = catalog.entries[name].get_aliases()
catalog.entries[name].purge_bandless_photometry()
catalog.entries[name].set_first_max_light()
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['MLS', 'SSS', 'CSS', 'GRB ']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:2])):
discoverdate = ('/'.join([
'20' + alias.replace(prefix, '')[:2],
alias.replace(prefix, '')[2:4],
alias.replace(prefix, '')[4:6]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = [
'ASASSN-', 'PS1-', 'PS1', 'PS', 'iPTF', 'PTF', 'SCP-', 'SNLS-',
'SPIRITS', 'LSQ', 'DES', 'SNHiTS', 'Gaia', 'GND', 'GNW', 'GSD',
'GSW', 'EGS', 'COS', 'OGLE', 'HST'
]
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:2])
and is_number(alias.replace(prefix, '')[:1])):
discoverdate = '20' + alias.replace(prefix, '')[:2]
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['SNF']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:4])):
discoverdate = ('/'.join([
alias.replace(prefix, '')[:4],
alias.replace(prefix, '')[4:6],
alias.replace(prefix, '')[6:8]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['PTFS', 'SNSDF']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:2])):
discoverdate = ('/'.join([
'20' + alias.replace(prefix, '')[:2],
alias.replace(prefix, '')[2:4]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['AT', 'SN', 'OGLE-', 'SM ', 'KSN']
for alias in aliases:
for prefix in prefixes:
if alias.startswith(prefix):
year = re.findall(r'\d+', alias)
if len(year) == 1:
year = year[0]
else:
continue
if alias.replace(prefix, '').index(year) != 0:
continue
if (year and is_number(year) and '.' not in year
and len(year) <= 4):
discoverdate = year
if catalog.args.verbose:
tprint('Added discoverdate from name [' +
alias + ']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if (SUPERNOVA.RA not in catalog.entries[name]
or SUPERNOVA.DEC not in catalog.entries[name]):
prefixes = [
'PSN J', 'MASJ', 'CSS', 'SSS', 'MASTER OT J', 'HST J', 'TCP J',
'MACS J', '2MASS J', 'EQ J', 'CRTS J', 'SMT J'
]
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:6])):
noprefix = alias.split(':')[-1].replace(prefix,
'').replace(
'.', '')
decsign = '+' if '+' in noprefix else '-'
noprefix = noprefix.replace('+', '|').replace('-', '|')
nops = noprefix.split('|')
if len(nops) < 2:
continue
rastr = nops[0]
decstr = nops[1]
ra = ':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + \
('.' + rastr[6:] if len(rastr) > 6 else '')
dec = (
decsign +
':'.join([decstr[:2], decstr[2:4], decstr[4:6]]) +
('.' + decstr[6:] if len(decstr) > 6 else ''))
if catalog.args.verbose:
tprint('Added ra/dec from name: ' + ra + ' ' + dec)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(SUPERNOVA.RA,
ra,
source,
derived=True)
catalog.entries[name].add_quantity(SUPERNOVA.DEC,
dec,
source,
derived=True)
break
if SUPERNOVA.RA in catalog.entries[name]:
break
no_host = (SUPERNOVA.HOST not in catalog.entries[name] or not any([
x[QUANTITY.VALUE] == 'Milky Way'
for x in catalog.entries[name][SUPERNOVA.HOST]
]))
if (SUPERNOVA.RA in catalog.entries[name]
and SUPERNOVA.DEC in catalog.entries[name] and no_host):
from astroquery.irsa_dust import IrsaDust
if name not in catalog.extinctions_dict:
try:
ra_dec = catalog.entries[name][
SUPERNOVA.RA][0][QUANTITY.VALUE] + \
" " + \
catalog.entries[name][SUPERNOVA.DEC][0][QUANTITY.VALUE]
result = IrsaDust.get_query_table(ra_dec, section='ebv')
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
warnings.warn("Coordinate lookup for " + name +
" failed in IRSA.")
else:
ebv = result['ext SandF mean'][0]
ebverr = result['ext SandF std'][0]
catalog.extinctions_dict[name] = [ebv, ebverr]
if name in catalog.extinctions_dict:
sources = uniq_cdl([
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2011ApJ...737..103S')
])
(catalog.entries[name].add_quantity(
SUPERNOVA.EBV,
str(catalog.extinctions_dict[name][0]),
sources,
e_value=str(catalog.extinctions_dict[name][1]),
derived=True))
if ((SUPERNOVA.HOST in catalog.entries[name]
and (SUPERNOVA.HOST_RA not in catalog.entries[name]
or SUPERNOVA.HOST_DEC not in catalog.entries[name]))):
for host in catalog.entries[name][SUPERNOVA.HOST]:
alias = host[QUANTITY.VALUE]
if ' J' in alias and is_number(alias.split(' J')[-1][:6]):
noprefix = alias.split(' J')[-1].split(':')[-1].replace(
'.', '')
decsign = '+' if '+' in noprefix else '-'
noprefix = noprefix.replace('+', '|').replace('-', '|')
nops = noprefix.split('|')
if len(nops) < 2:
continue
rastr = nops[0]
decstr = nops[1]
hostra = (':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) +
('.' + rastr[6:] if len(rastr) > 6 else ''))
hostdec = decsign + ':'.join([
decstr[:2], decstr[2:4], decstr[4:6]
]) + ('.' + decstr[6:] if len(decstr) > 6 else '')
if catalog.args.verbose:
tprint('Added hostra/hostdec from name: ' + hostra +
' ' + hostdec)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(SUPERNOVA.HOST_RA,
hostra,
source,
derived=True)
catalog.entries[name].add_quantity(SUPERNOVA.HOST_DEC,
hostdec,
source,
derived=True)
break
if SUPERNOVA.HOST_RA in catalog.entries[name]:
break
if (SUPERNOVA.REDSHIFT not in catalog.entries[name]
and SUPERNOVA.VELOCITY in catalog.entries[name]):
# Find the "best" velocity to use for this
bestsig = 0
for hv in catalog.entries[name][SUPERNOVA.VELOCITY]:
sig = get_sig_digits(hv[QUANTITY.VALUE])
if sig > bestsig:
besthv = hv[QUANTITY.VALUE]
bestsrc = hv['source']
bestsig = sig
if bestsig > 0 and is_number(besthv):
voc = float(besthv) * 1.e5 / CLIGHT
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
(catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT,
pretty_num(sqrt((1. + voc) / (1. - voc)) - 1.,
sig=bestsig),
sources,
kind='heliocentric',
derived=True))
if (SUPERNOVA.REDSHIFT not in catalog.entries[name]
and len(catalog.nedd_dict) > 0
and SUPERNOVA.HOST in catalog.entries[name]):
reference = "NED-D"
refurl = "http://ned.ipac.caltech.edu/Library/Distances/"
refbib = "1991ASSL..171...89H"
for host in catalog.entries[name][SUPERNOVA.HOST]:
if host[QUANTITY.VALUE] in catalog.nedd_dict:
source = catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
secondarysource = catalog.entries[name].add_source(
name=reference,
url=refurl,
bibcode=refbib,
secondary=True)
meddist = statistics.median(
catalog.nedd_dict[host[QUANTITY.VALUE]])
redz = z_at_value(cosmo.comoving_distance,
float(meddist) * un.Mpc)
redshift = pretty_num(redz,
sig=get_sig_digits(str(meddist)))
catalog.entries[name].add_quantity(
[SUPERNOVA.REDSHIFT, SUPERNOVA.HOST_REDSHIFT],
redshift,
uniq_cdl([source, secondarysource]),
kind='host',
derived=True)
if (SUPERNOVA.MAX_ABS_MAG not in catalog.entries[name]
and SUPERNOVA.MAX_APP_MAG in catalog.entries[name]
and SUPERNOVA.LUM_DIST in catalog.entries[name]):
# Find the "best" distance to use for this
bestsig = 0
for ld in catalog.entries[name][SUPERNOVA.LUM_DIST]:
sig = get_sig_digits(ld[QUANTITY.VALUE])
if sig > bestsig:
bestld = ld[QUANTITY.VALUE]
bestsrc = ld[QUANTITY.SOURCE]
bestsig = sig
if bestsig > 0 and is_number(bestld) and float(bestld) > 0.:
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
bestldz = z_at_value(cosmo.luminosity_distance,
float(bestld) * un.Mpc)
pnum = (float(catalog.entries[name][SUPERNOVA.MAX_APP_MAG][0][
QUANTITY.VALUE]) - 5.0 *
(log10(float(bestld) * 1.0e6) - 1.0) +
2.5 * log10(1.0 + bestldz))
pnum = pretty_num(pnum, sig=bestsig + 1)
catalog.entries[name].add_quantity(SUPERNOVA.MAX_ABS_MAG,
pnum,
sources,
derived=True)
if (SUPERNOVA.MAX_VISUAL_ABS_MAG not in catalog.entries[name]
and SUPERNOVA.MAX_VISUAL_APP_MAG in catalog.entries[name]
and SUPERNOVA.LUM_DIST in catalog.entries[name]):
# Find the "best" distance to use for this
bestsig = 0
for ld in catalog.entries[name][SUPERNOVA.LUM_DIST]:
sig = get_sig_digits(ld[QUANTITY.VALUE])
if sig > bestsig:
bestld = ld[QUANTITY.VALUE]
bestsrc = ld[QUANTITY.SOURCE]
bestsig = sig
if bestsig > 0 and is_number(bestld) and float(bestld) > 0.:
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
# FIX: what's happening here?!
pnum = (float(catalog.entries[name][
SUPERNOVA.MAX_VISUAL_APP_MAG][0][QUANTITY.VALUE]) - 5.0 *
(log10(float(bestld) * 1.0e6) - 1.0))
pnum = pretty_num(pnum, sig=bestsig + 1)
catalog.entries[name].add_quantity(
SUPERNOVA.MAX_VISUAL_ABS_MAG, pnum, sources, derived=True)
if SUPERNOVA.REDSHIFT in catalog.entries[name]:
# Find the "best" redshift to use for this
bestz, bestkind, bestsig, bestsrc = catalog.entries[
name].get_best_redshift()
if bestsig > 0:
try:
bestz = float(bestz)
except Exception:
print(catalog.entries[name])
raise
if SUPERNOVA.VELOCITY not in catalog.entries[name]:
source = catalog.entries[name].add_self_source()
# FIX: what's happening here?!
pnum = CLIGHT / KM * \
((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.)
pnum = pretty_num(pnum, sig=bestsig)
catalog.entries[name].add_quantity(
SUPERNOVA.VELOCITY,
pnum,
source,
kind=(SUPERNOVA.VELOCITY.kind_preference[bestkind]
if bestkind else ''))
if bestz > 0.:
if SUPERNOVA.LUM_DIST not in catalog.entries[name]:
dl = cosmo.luminosity_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
SUPERNOVA.LUM_DIST,
pretty_num(dl.value, sig=bestsig + 1),
sources,
kind=(SUPERNOVA.LUM_DIST.kind_preference[bestkind]
if bestkind else ''),
derived=True)
if (SUPERNOVA.MAX_ABS_MAG not in catalog.entries[name]
and SUPERNOVA.MAX_APP_MAG
in catalog.entries[name]):
source = catalog.entries[name].add_self_source()
pnum = pretty_num(
float(catalog.entries[name][
SUPERNOVA.MAX_APP_MAG][0][QUANTITY.VALUE])
- 5.0 * (log10(dl.to('pc').value) - 1.0) +
2.5 * log10(1.0 + bestz),
sig=bestsig + 1)
catalog.entries[name].add_quantity(
SUPERNOVA.MAX_ABS_MAG,
pnum,
sources,
derived=True)
if (SUPERNOVA.MAX_VISUAL_ABS_MAG
not in catalog.entries[name]
and SUPERNOVA.MAX_VISUAL_APP_MAG
in catalog.entries[name]):
source = catalog.entries[name].add_self_source()
pnum = pretty_num(float(catalog.entries[name][
SUPERNOVA.MAX_VISUAL_APP_MAG][0][
QUANTITY.VALUE]) - 5.0 *
(log10(dl.to('pc').value) - 1.0),
sig=bestsig + 1)
catalog.entries[name].add_quantity(
SUPERNOVA.MAX_VISUAL_ABS_MAG,
pnum,
sources,
derived=True)
if SUPERNOVA.COMOVING_DIST not in catalog.entries[name]:
cd = cosmo.comoving_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
SUPERNOVA.COMOVING_DIST,
pretty_num(cd.value, sig=bestsig),
sources,
derived=True)
if SUPERNOVA.HOST_REDSHIFT in catalog.entries[name]:
# Find the "best" redshift to use for this
bestz, bestkind, bestsig, bestsrc = catalog.entries[
name].get_best_redshift(SUPERNOVA.HOST_REDSHIFT)
if bestsig > 0:
try:
bestz = float(bestz)
except Exception:
print(catalog.entries[name])
raise
if SUPERNOVA.HOST_VELOCITY not in catalog.entries[name]:
source = catalog.entries[name].add_self_source()
# FIX: what's happening here?!
pnum = CLIGHT / KM * \
((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.)
pnum = pretty_num(pnum, sig=bestsig)
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_VELOCITY,
pnum,
source,
kind=(SUPERNOVA.HOST_VELOCITY.kind_preference[bestkind]
if bestkind else ''))
if bestz > 0.:
if SUPERNOVA.HOST_LUM_DIST not in catalog.entries[name]:
dl = cosmo.luminosity_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_LUM_DIST,
pretty_num(dl.value, sig=bestsig + 1),
sources,
kind=(SUPERNOVA.HOST_LUM_DIST.
kind_preference[bestkind]
if bestkind else ''),
derived=True)
if SUPERNOVA.HOST_COMOVING_DIST not in catalog.entries[
name]:
cd = cosmo.comoving_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_COMOVING_DIST,
pretty_num(cd.value, sig=bestsig),
sources,
derived=True)
if all([
x in catalog.entries[name] for x in [
SUPERNOVA.RA, SUPERNOVA.DEC, SUPERNOVA.HOST_RA,
SUPERNOVA.HOST_DEC
]
]):
# For now just using first coordinates that appear in entry
try:
c1 = coord(
ra=catalog.entries[name][SUPERNOVA.RA][0][QUANTITY.VALUE],
dec=catalog.entries[name][SUPERNOVA.DEC][0][
QUANTITY.VALUE],
unit=(un.hourangle, un.deg))
c2 = coord(ra=catalog.entries[name][SUPERNOVA.HOST_RA][0][
QUANTITY.VALUE],
dec=catalog.entries[name][SUPERNOVA.HOST_DEC][0][
QUANTITY.VALUE],
unit=(un.hourangle, un.deg))
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
pass
else:
sources = uniq_cdl([catalog.entries[name].add_self_source()] +
catalog.entries[name][SUPERNOVA.RA][0][
QUANTITY.SOURCE].split(',') +
catalog.entries[name][SUPERNOVA.DEC][0][
QUANTITY.SOURCE].split(',') +
catalog.entries[name][SUPERNOVA.HOST_RA][0][
QUANTITY.SOURCE].split(',') +
catalog.entries[name][SUPERNOVA.HOST_DEC][0]
[QUANTITY.SOURCE].split(','))
if SUPERNOVA.HOST_OFFSET_ANG not in catalog.entries[name]:
hosa = Decimal(c1.separation(c2).arcsecond)
hosa = pretty_num(hosa)
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_OFFSET_ANG,
hosa,
sources,
derived=True,
u_value='arcseconds')
if (SUPERNOVA.COMOVING_DIST in catalog.entries[name]
and SUPERNOVA.REDSHIFT in catalog.entries[name]
and SUPERNOVA.HOST_OFFSET_DIST
not in catalog.entries[name]):
offsetsig = get_sig_digits(catalog.entries[name][
SUPERNOVA.HOST_OFFSET_ANG][0][QUANTITY.VALUE])
sources = uniq_cdl(
sources.split(',') +
(catalog.entries[name][SUPERNOVA.COMOVING_DIST][0][
QUANTITY.SOURCE]).split(',') +
(catalog.entries[name][SUPERNOVA.REDSHIFT][0][
QUANTITY.SOURCE]).split(','))
(catalog.entries[name].add_quantity(
SUPERNOVA.HOST_OFFSET_DIST,
pretty_num(
float(catalog.entries[name][
SUPERNOVA.HOST_OFFSET_ANG][0][QUANTITY.VALUE])
/ 3600. * (pi / 180.) *
float(catalog.entries[name][
SUPERNOVA.COMOVING_DIST][0][QUANTITY.VALUE]) *
1000. / (1.0 + float(catalog.entries[name][
SUPERNOVA.REDSHIFT][0][QUANTITY.VALUE])),
sig=offsetsig), sources))
catalog.entries[name].sanitize()
catalog.journal_entries(bury=True, final=True, gz=True)
cleanupcnt = cleanupcnt + 1
if catalog.args.travis and cleanupcnt % 1000 == 0:
break
catalog.save_caches()
return
def do_donated_spectra(catalog):
"""Import donated spectra."""
task_str = catalog.get_current_task_str()
fpath = os.path.join(catalog.get_current_task_repo(), 'donations')
with open(os.path.join(fpath, 'meta.json'), 'r') as f:
metadict = json.loads(f.read())
donationscnt = 0
oldname = ''
for fname in pbar(metadict, task_str):
name = metadict[fname]['name']
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
sec_bibc = metadict[fname]['bibcode']
name, source = catalog.new_entry(name, bibcode=sec_bibc)
date = metadict[fname].get('date', '')
year, month, day = date.split('/')
sig = get_sig_digits(day) + 5
day_fmt = str(floor(float(day))).zfill(2)
time = astrotime(year + '-' + month + '-' + day_fmt).mjd
time = time + float(day) - floor(float(day))
time = pretty_num(time, sig=sig)
with open(os.path.join(fpath, fname), 'r') as f:
specdata = list(csv.reader(f, delimiter=' ',
skipinitialspace=True))
specdata = list(filter(None, specdata))
newspec = []
oldval = ''
for row in specdata:
if row[0][0] == '#':
continue
if row[1] == oldval:
continue
newspec.append(row)
oldval = row[1]
specdata = newspec
haserrors = len(
specdata[0]) == 3 and specdata[0][2] and specdata[0][2] != 'NaN'
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.U_TIME: 'MJD',
SPECTRUM.TIME: time,
SPECTRUM.WAVELENGTHS: wavelengths,
SPECTRUM.FLUXES: fluxes,
SPECTRUM.ERRORS: errors,
SPECTRUM.SOURCE: source,
SPECTRUM.FILENAME: fname
}
if 'instrument' in metadict[fname]:
specdict[SPECTRUM.INSTRUMENT] = metadict[fname]['instrument']
if 'telescope' in metadict[fname]:
specdict[SPECTRUM.TELESCOPE] = metadict[fname]['telescope']
if 'yunit' in metadict[fname]:
specdict[SPECTRUM.U_FLUXES] = metadict[fname]['yunit']
specdict[SPECTRUM.U_ERRORS] = metadict[fname]['yunit']
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict[SPECTRUM.U_FLUXES] = fluxunit
specdict[SPECTRUM.U_ERRORS] = fluxunit
catalog.entries[name].add_spectrum(**specdict)
donationscnt = donationscnt + 1
if (catalog.args.travis
and donationscnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
def radec_clean(svalue, quantity, unit=''):
if unit == 'floatdegrees':
if not is_number(svalue):
return (svalue, unit)
deg = float('%g' % Decimal(svalue))
sig = get_sig_digits(svalue)
if 'ra' in quantity:
flhours = deg / 360.0 * 24.0
hours = floor(flhours)
minutes = floor((flhours - hours) * 60.0)
seconds = (flhours * 60.0 - (hours * 60.0 + minutes)) * 60.0
hours = 0 if hours < 1.e-6 else hours
minutes = 0 if minutes < 1.e-6 else minutes
seconds = 0.0 if seconds < 1.e-6 else seconds
if seconds > 60.0:
raise(ValueError('Invalid seconds value for ' + quantity))
svalue = str(hours).zfill(2) + ':' + str(minutes).zfill(2) + \
':' + zpad(pretty_num(seconds, sig=sig - 1))
elif 'dec' in quantity:
fldeg = abs(deg)
degree = floor(fldeg)
minutes = floor((fldeg - degree) * 60.0)
seconds = (fldeg * 60.0 - (degree * 60.0 + minutes)) * 60.0
if seconds > 60.0:
raise(ValueError('Invalid seconds value for ' + quantity))
svalue = (('+' if deg >= 0.0 else '-') +
str(degree).strip('+-').zfill(2) + ':' +
str(minutes).zfill(2) + ':' +
zpad(pretty_num(seconds, sig=sig - 1)))
elif unit == 'nospace' and 'ra' in quantity:
svalue = svalue[:2] + ':' + svalue[2:4] + \
((':' + zpad(svalue[4:])) if len(svalue) > 4 else '')
elif unit == 'nospace' and 'dec' in quantity:
if svalue.startswith(('+', '-')):
svalue = svalue[:3] + ':' + svalue[3:5] + \
((':' + zpad(svalue[5:])) if len(svalue) > 5 else '')
else:
svalue = '+' + svalue[:2] + ':' + svalue[2:4] + \
((':' + zpad(svalue[4:])) if len(svalue) > 4 else '')
else:
svalue = svalue.replace(' ', ':')
if 'dec' in quantity:
valuesplit = svalue.split(':')
svalue = (('-' if valuesplit[0].startswith('-') else '+') +
valuesplit[0].strip('+-').zfill(2) +
(':' + valuesplit[1].zfill(2) if
len(valuesplit) > 1 else '') +
(':' + zpad(valuesplit[2]) if
len(valuesplit) > 2 else ''))
if 'ra' in quantity:
sunit = 'hours'
elif 'dec' in quantity:
sunit = 'degrees'
# Correct case of arcseconds = 60.0.
valuesplit = svalue.split(':')
if len(valuesplit) == 3 and valuesplit[-1] in ["60.0", "60.", "60"]:
svalue = valuesplit[0] + ':' + str(Decimal(valuesplit[1]) +
Decimal(1.0)) + ':' + "00.0"
# Strip trailing dots.
svalue = svalue.rstrip('.')
return (svalue, sunit)
def do_tns(catalog):
"""Load TNS metadata."""
session = requests.Session()
task_str = catalog.get_current_task_str()
tns_url = 'https://www.wis-tns.org/'
search_url = tns_url + \
'search?&num_page=1&format=html&sort=desc&order=id&format=csv&page=0'
csvtxt = catalog.load_url(search_url,
os.path.join(catalog.get_current_task_repo(),
'TNS', 'index.csv'))
if not csvtxt:
return
maxid = csvtxt.splitlines()[1].split(',')[0].strip('"')
maxpages = ceil(int(maxid) / 1000.)
for page in pbar(range(maxpages), task_str):
fname = os.path.join(
catalog.get_current_task_repo(), 'TNS',
'page-') + str(page).zfill(2) + '.csv'
if (catalog.current_task.load_archive(catalog.args) and
os.path.isfile(fname) and page < 7):
with open(fname, 'r') as tns_file:
csvtxt = tns_file.read()
else:
session = requests.Session()
ses_url = (tns_url + 'search?&num_page=1000&format=html&edit'
'[type]=&edit[objname]=&edit[id]=&sort=asc&order=id'
'&display[redshift]=1'
'&display[hostname]=1&display[host_redshift]=1'
'&display[source_group_name]=1'
'&display[programs_name]=1'
'&display[internal_name]=1'
'&display[isTNS_AT]=1'
'&display[public]=1'
'&display[end_pop_period]=0'
'&display[spectra_count]=1'
'&display[discoverymag]=1&display[discmagfilter]=1'
'&display[discoverydate]=1&display[discoverer]=1'
'&display[sources]=1'
'&display[bibcode]=1&format=csv&page=' + str(page))
try:
response = session.get(ses_url, timeout=40)
csvtxt = response.text
except Exception as e:
catalog.log.warning(
'Could not download TNS page #{}.'.format(str(page)))
if os.path.isfile(fname):
with open(fname, 'r') as tns_file:
csvtxt = tns_file.read()
else:
continue
else:
with open(fname, 'w') as tns_file:
tns_file.write(csvtxt)
tsvin = list(csv.reader([x.replace('[data validation error, please contact site admin]',
'""') for x in csvtxt.splitlines()], delimiter=','))
for ri, row in enumerate(pbar(tsvin, task_str, leave=False)):
if ri == 0:
continue
if row[4] and 'SN' not in row[4]:
continue
name = row[1].replace(' ', '')
if len(name) < 5:
continue
name, source = catalog.new_entry(
name, srcname='Transient Name Server', url=tns_url)
if row[2] and row[2] != '00:00:00.00':
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[2],
source)
if row[3] and row[3] != '+00:00:00.00':
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[3],
source)
if row[4]:
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, row[4].replace('SN', '').strip(),
source)
if row[5]:
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, row[5], source, kind='spectroscopic')
if row[6]:
catalog.entries[name].add_quantity(SUPERNOVA.HOST, row[6],
source)
if row[7]:
catalog.entries[name].add_quantity(
[SUPERNOVA.REDSHIFT, SUPERNOVA.HOST_REDSHIFT],
row[7],
source,
kind='host')
if row[8]:
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVERER,
row[8], source)
# Currently, all events listing all possible observers. TNS bug?
# if row[9]:
# observers = row[9].split(',')
# for observer in observers:
# catalog.entries[name].add_quantity('observer',
# observer.strip(),
# source)
if row[12]:
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[12],
source)
if row[20]:
date = row[20].split()[0].replace('-', '/')
if 'clear' in row[20].lower():
print(row)
if date != '0000/00/00' and 'clear' not in row[20].lower():
date = date.replace('/00', '')
dsplit = row[20].split()
if len(dsplit) >= 2:
t = dsplit[1]
if t != '00:00:00':
ts = t.split(':')
dt = timedelta(
hours=int(ts[0]),
minutes=int(ts[1]),
seconds=float(ts[2]))
date += pretty_num(
dt.total_seconds() / (24 * 60 * 60),
sig=6).lstrip('0')
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE,
date, source)
if catalog.args.travis and ri >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
catalog.journal_entries()
def do_snf_specta(catalog):
task_str = catalog.get_current_task_str()
bibcodes = {'SN2005gj': '2006ApJ...650..510A',
'SN2006D': '2007ApJ...654L..53T',
'SN2007if': '2010ApJ...713.1073S',
'SN2011fe': '2013A&A...554A..27P'}
oldname = ''
snfcnt = 0
eventfolders = next(os.walk(os.path.join(
catalog.get_current_task_repo(), 'SNFactory')))[1]
for eventfolder in pbar(eventfolders, task_str):
oname = eventfolder
name = catalog.get_preferred_name(oname)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
sec_reference = 'Nearby Supernova Factory'
sec_refurl = 'http://snfactory.lbl.gov/'
sec_bibcode = '2002SPIE.4836...61A'
sec_source = catalog.entries[name].add_source(
name=sec_reference, url=sec_refurl, bibcode=sec_bibcode,
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oname, sec_source)
bibcode = bibcodes[oname]
source = catalog.entries[name].add_source(bibcode=bibcode)
sources = uniq_cdl([source, sec_source])
use_path = os.path.join(
catalog.get_current_task_repo(), 'SNFactory', eventfolder, '*.dat')
eventspectra = glob(use_path)
for spectrum in pbar(eventspectra, task_str):
filename = os.path.basename(spectrum)
with open(spectrum) as spec_file:
specdata = list(csv.reader(
spec_file, delimiter=' ', skipinitialspace=True))
specdata = list(filter(None, specdata))
newspec = []
time = ''
telescope = ''
instrument = ''
observer = ''
observatory = ''
if 'Keck_20060202_R' in spectrum:
time = '53768.23469'
elif 'Spectrum05_276' in spectrum:
time = pretty_num(astrotime('2005-10-03').mjd, sig=5)
elif 'Spectrum05_329' in spectrum:
time = pretty_num(astrotime('2005-11-25').mjd, sig=5)
elif 'Spectrum05_336' in spectrum:
time = pretty_num(astrotime('2005-12-02').mjd, sig=5)
for row in specdata:
if row[0][0] == '#':
joinrow = (' '.join(row)).split('=')
if len(joinrow) < 2:
continue
field = joinrow[0].strip('# ')
value = joinrow[1].split('/')[0].strip('\' ')
if not time:
if field == 'JD':
time = str(jd_to_mjd(Decimal(value)))
elif field == 'MJD':
time = value
elif field == 'MJD-OBS':
time = value
if field == 'OBSERVER':
observer = value.capitalize()
if field == 'OBSERVAT':
observatory = value.capitalize()
if field == 'TELESCOP':
telescope = value.capitalize()
if field == 'INSTRUME':
instrument = value.capitalize()
else:
newspec.append(row)
if not time:
raise ValueError('Time missing from spectrum.')
specdata = newspec
haserrors = len(specdata[0]) == 3 and specdata[
0][2] and specdata[0][2] != 'NaN'
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
unit_err = ('Variance' if oldname == 'SN2011fe' else
'erg/s/cm^2/Angstrom')
unit_flx = 'erg/s/cm^2/Angstrom'
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom', u_fluxes=unit_flx, u_time='MJD',
time=time,
wavelengths=wavelengths, fluxes=fluxes, errors=errors,
observer=observer, observatory=observatory,
telescope=telescope, instrument=instrument, u_errors=unit_err,
source=sources, filename=filename)
snfcnt = snfcnt + 1
if (catalog.args.travis and
snfcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
def do_cleanup(catalog):
"""Cleanup catalog after importing all data."""
task_str = catalog.get_current_task_str()
# Set preferred names, calculate some columns based on imported data,
# sanitize some fields
keys = list(catalog.entries.keys())
cleanupcnt = 0
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
try:
name = catalog.add_entry(oname)
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
# Set the preferred name, switching to that name if name changed.
name = catalog.entries[name].set_preferred_name()
aliases = catalog.entries[name].get_aliases()
catalog.entries[name].set_first_max_light()
# Clean discoverer field
if FASTSTARS.DISCOVERER in catalog.entries[name]:
if len(catalog.entries[name][FASTSTARS.DISCOVERER]) > 1:
POSSIBLEDISCOVERER = [
catalog.entries[name][FASTSTARS.DISCOVERER][i]['value']
for i in range(
len(catalog.entries[name][FASTSTARS.DISCOVERER]))
]
POSSIBLEDISCOVERER_DATE = [
int(DATE['value'])
for DATE in catalog.entries[name][FASTSTARS.DISCOVER_DATE]
]
POSSIBLEDISCOVERER_DATE_SOURCES = [
DATE['source']
for DATE in catalog.entries[name][FASTSTARS.DISCOVER_DATE]
]
EARLIESTSOURCE = POSSIBLEDISCOVERER_DATE_SOURCES[np.argmin(
POSSIBLEDISCOVERER_DATE)]
EARLIESTDISCOVER_DATE = catalog.entries[name][
FASTSTARS.DISCOVER_DATE][np.argmin(
POSSIBLEDISCOVERER_DATE)]
# Deal with case where a star was 'discovered' multiple times in one year
if ',' in EARLIESTSOURCE:
EARLIESTSOURCE = EARLIESTSOURCE.split(',')[0]
for DISCOVERER in catalog.entries[name][FASTSTARS.DISCOVERER]:
for DISCOVERERSOURCE in DISCOVERER['source'].split(','):
if DISCOVERERSOURCE == EARLIESTSOURCE:
EARLIESTDISCOVERER = DISCOVERER
for DISCOVERER in catalog.entries[name][FASTSTARS.DISCOVERER]:
for DISCOVERERSOURCE in DISCOVERER['source'].split(','):
if DISCOVERERSOURCE == EARLIESTSOURCE:
EARLIESTDISCOVERER = DISCOVERER
catalog.entries[name][FASTSTARS.DISCOVERER] = [
EARLIESTDISCOVERER
]
catalog.entries[name][FASTSTARS.DISCOVER_DATE] = [
EARLIESTDISCOVER_DATE
]
# Convert all distances to kpc.
if FASTSTARS.LUM_DIST in catalog.entries[name]:
for li, ld in enumerate(catalog.entries[name][FASTSTARS.LUM_DIST]):
if ld.get('u_value') != 'kpc':
if ld.get('u_value') == 'pc':
catalog.entries[name][
FASTSTARS.LUM_DIST][li]['value'] = str(
Decimal(catalog.entries[name][
FASTSTARS.LUM_DIST][li]['value']) *
Decimal('0.001'))
elif ld.get('u_value') == 'Mpc':
catalog.entries[name][
FASTSTARS.LUM_DIST][li]['value'] = str(
Decimal(catalog.entries[name][
FASTSTARS.LUM_DIST][li]['value']) *
Decimal('1000'))
else:
raise ValueError('unknown distance unit')
catalog.entries[name][
FASTSTARS.LUM_DIST][li]['u_value'] = 'kpc'
if (FASTSTARS.RA not in catalog.entries[name]
or FASTSTARS.DEC not in catalog.entries[name]):
prefixes = ['SDSS']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:6])):
noprefix = alias.split(':')[-1].replace(prefix,
'').replace(
'.', '')
decsign = '+' if '+' in noprefix else '-'
noprefix = noprefix.replace('+', '|').replace('-', '|')
nops = noprefix.split('|')
if len(nops) < 2:
continue
rastr = nops[0]
decstr = nops[1]
ra = ':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + \
('.' + rastr[6:] if len(rastr) > 6 else '')
dec = (
decsign +
':'.join([decstr[:2], decstr[2:4], decstr[4:6]]) +
('.' + decstr[6:] if len(decstr) > 6 else ''))
if catalog.args.verbose:
tprint('Added ra/dec from name: ' + ra + ' ' + dec)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(FASTSTARS.RA,
ra,
source,
derived=True)
catalog.entries[name].add_quantity(FASTSTARS.DEC,
dec,
source,
derived=True)
break
if FASTSTARS.RA in catalog.entries[name]:
break
if (FASTSTARS.MAX_ABS_MAG not in catalog.entries[name]
and FASTSTARS.MAX_APP_MAG in catalog.entries[name]
and FASTSTARS.LUM_DIST in catalog.entries[name]):
# Find the "best" distance to use for this
bestsig = 0
for ld in catalog.entries[name][FASTSTARS.LUM_DIST]:
sig = get_sig_digits(ld[QUANTITY.VALUE])
if sig > bestsig:
bestld = ld[QUANTITY.VALUE]
bestsrc = ld[QUANTITY.SOURCE]
bestsig = sig
if bestsig > 0 and is_number(bestld) and float(bestld) > 0.:
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
bestldz = z_at_value(cosmo.luminosity_distance,
float(bestld) * un.Mpc)
pnum = (float(catalog.entries[name][FASTSTARS.MAX_APP_MAG][0][
QUANTITY.VALUE]) - 5.0 *
(log10(float(bestld) * 1.0e6) - 1.0) +
2.5 * log10(1.0 + bestldz))
pnum = pretty_num(pnum, sig=bestsig + 1)
catalog.entries[name].add_quantity(FASTSTARS.MAX_ABS_MAG,
pnum,
sources,
derived=True)
catalog.entries[name].sanitize()
catalog.journal_entries(bury=True, final=True, gz=True)
cleanupcnt = cleanupcnt + 1
if catalog.args.travis and cleanupcnt % 1000 == 0:
break
catalog.save_caches()
return
