def do_itep(catalog):
task_str = catalog.get_current_task_str()
itepbadsources = ['2004ApJ...602..571B']
needsbib = []
with open(os.path.join(catalog.get_current_task_repo(),
'itep-refs.txt'), 'r') as refs_file:
refrep = refs_file.read().splitlines()
refrepf = dict(list(zip(refrep[1::2], refrep[::2])))
fname = os.path.join(catalog.get_current_task_repo(),
'itep-lc-cat-28dec2015.txt')
tsvin = list(csv.reader(open(fname, 'r'),
delimiter='|', skipinitialspace=True))
curname = ''
for rr, row in enumerate(pbar(tsvin, task_str)):
if rr <= 1 or len(row) < 7:
continue
oldname = 'SN' + row[0].strip()
mjd = str(jd_to_mjd(Decimal(row[1].strip())))
band = row[2].strip()
magnitude = row[3].strip()
e_magnitude = row[4].strip()
reference = row[6].strip().strip(',')
if curname != oldname:
curname = oldname
name = catalog.add_entry(oldname)
sec_reference = ('Sternberg Astronomical Institute '
'Supernova Light Curve Catalogue')
sec_refurl = 'http://dau.itep.ru/sn/node/72'
sec_source = catalog.entries[name].add_source(
name=sec_reference, url=sec_refurl, secondary=True)
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, oldname, sec_source)
year = re.findall(r'\d+', name)[0]
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, year, sec_source)
if reference in refrepf:
bibcode = unescape(refrepf[reference])
source = catalog.entries[name].add_source(bibcode=bibcode)
else:
needsbib.append(reference)
source = catalog.entries[name].add_source(
name=reference) if reference else ''
if bibcode not in itepbadsources:
catalog.entries[name].add_photometry(time=mjd, band=band,
magnitude=magnitude,
e_magnitude=e_magnitude,
source=sec_source + ',' +
source)
# Write out references that could use aa bibcode
needsbib = list(OrderedDict.fromkeys(needsbib))
with open('../itep-needsbib.txt', 'w') as bib_file:
bib_file.writelines(['%ss\n' % ii for ii in needsbib])
catalog.journal_entries()
return
def do_csp_spectra(catalog):
"""Import CSP spectra."""
oldname = ''
task_str = catalog.get_current_task_str()
file_names = glob(os.path.join(catalog.get_current_task_repo(), 'CSP/*'))
for fi, fname in enumerate(pbar_strings(file_names, task_str)):
filename = os.path.basename(fname)
sfile = filename.split('.')
if sfile[1] == 'txt':
continue
sfile = sfile[0]
fileparts = sfile.split('_')
name = 'SN20' + fileparts[0][2:]
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
telescope = fileparts[-2]
instrument = fileparts[-1]
source = catalog.entries[name].add_source(
bibcode='2013ApJ...773...53F')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
data = csv.reader(open(fname, 'r'),
delimiter=' ',
skipinitialspace=True)
specdata = []
for r, row in enumerate(data):
if row[0] == '#JDate_of_observation:':
jd = row[1].strip()
time = str(jd_to_mjd(Decimal(jd)))
elif row[0] == '#Redshift:':
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT,
row[1].strip(), source)
if r < 7:
continue
specdata.append(list(filter(None, [x.strip(' ') for x in row])))
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
catalog.entries[name].add_spectrum(u_wavelengths='Angstrom',
u_fluxes='erg/s/cm^2/Angstrom',
u_time='MJD',
time=time,
wavelengths=wavelengths,
fluxes=fluxes,
telescope=telescope,
instrument=instrument,
source=source,
deredshifted=True,
filename=filename)
if catalog.args.travis and fi >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
return
def set_first_max_light(self):
if ENTRY.MAX_APP_MAG not in self:
mldt, mlmag, mlband, mlsource = self._get_max_light()
if mldt or mlmag or mlband:
source = self.add_self_source()
uniq_src = uniq_cdl([source] + mlsource.split(','))
if mldt:
max_date = make_date_string(mldt.year, mldt.month, mldt.day)
self.add_quantity(ENTRY.MAX_DATE, max_date, uniq_src,
derived=True)
if mlmag:
mlmag = pretty_num(mlmag)
self.add_quantity(ENTRY.MAX_APP_MAG, mlmag, uniq_src,
derived=True)
if mlband:
self.add_quantity(ENTRY.MAX_BAND, mlband, uniq_src,
derived=True)
if (self._KEYS.DISCOVER_DATE not in self or
max([len(x[QUANTITY.VALUE].split('/')) for x in
self[self._KEYS.DISCOVER_DATE]]) < 3):
fldt, flsource = self._get_first_light()
if fldt:
source = self.add_self_source()
disc_date = make_date_string(fldt.year, fldt.month, fldt.day)
self.add_quantity(
self._KEYS.DISCOVER_DATE, disc_date,
uniq_cdl([source] + flsource.split(',')),
derived=True)
if self._KEYS.DISCOVER_DATE not in self and self._KEYS.SPECTRA in self:
minspecmjd = float("+inf")
for spectrum in self[self._KEYS.SPECTRA]:
if 'time' in spectrum and 'u_time' in spectrum:
if spectrum['u_time'] == 'MJD':
mjd = float(spectrum['time'])
elif spectrum['u_time'] == 'JD':
mjd = float(jd_to_mjd(Decimal(spectrum['time'])))
else:
continue
if mjd < minspecmjd:
minspecmjd = mjd
minspecsource = spectrum['source']
if minspecmjd < float("+inf"):
fldt = astrotime(minspecmjd, format='mjd').datetime
source = self.add_self_source()
disc_date = make_date_string(fldt.year, fldt.month, fldt.day)
self.add_quantity(
self._KEYS.DISCOVER_DATE, disc_date,
uniq_cdl([source] + minspecsource.split(',')),
derived=True)
return
def do_csp_spectra(catalog):
"""Import CSP spectra."""
oldname = ''
task_str = catalog.get_current_task_str()
file_names = glob(os.path.join(catalog.get_current_task_repo(), 'CSP/*'))
for fi, fname in enumerate(pbar_strings(file_names, task_str)):
filename = os.path.basename(fname)
sfile = filename.split('.')
if sfile[1] == 'txt':
continue
sfile = sfile[0]
fileparts = sfile.split('_')
name = 'SN20' + fileparts[0][2:]
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
telescope = fileparts[-2]
instrument = fileparts[-1]
source = catalog.entries[name].add_source(
bibcode='2013ApJ...773...53F')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
data = csv.reader(open(fname, 'r'), delimiter=' ',
skipinitialspace=True)
specdata = []
for r, row in enumerate(data):
if row[0] == '#JDate_of_observation:':
jd = row[1].strip()
time = str(jd_to_mjd(Decimal(jd)))
elif row[0] == '#Redshift:':
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT,
row[1].strip(),
source)
if r < 7:
continue
specdata.append(list(filter(None, [x.strip(' ') for x in row])))
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom', u_fluxes='erg/s/cm^2/Angstrom',
u_time='MJD',
time=time, wavelengths=wavelengths, fluxes=fluxes,
telescope=telescope, instrument=instrument,
source=source, deredshifted=True, filename=filename)
if catalog.args.travis and fi >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
return
def do_asas_atels(catalog):
"""Import LCs exposed in ASASSN Atels."""
import gzip
try:
with gzip.open('/root/better-atel/atels.json.gz', 'rb') as f:
ateljson = json.load(f)
except Exception:
print('ATel data unavailable, skipping ASAS ATel task.')
return
for entry in ateljson:
if ('asas-sn.osu.edu/light_curve' in entry['body'] and
'Supernovae' in entry['subjects']):
matches = re.findall(r'<a\s+[^>]*?href="([^"]*)".*?>(.*?)<\/a>',
entry['body'], re.DOTALL)
lcurl = ''
objname = ''
for match in matches:
if 'asas-sn.osu.edu/light_curve' in match[0]:
lcurl = match[0]
objname = re.findall(
r'\bASASSN-[0-9][0-9].*?\b', match[1])
if len(objname):
objname = objname[0]
if objname and lcurl:
name, source = catalog.new_entry(
objname, srcname='ASAS-SN Sky Patrol',
bibcode='2017arXiv170607060K',
url='https://asas-sn.osu.edu')
csv = catalog.load_url(lcurl + '.csv', os.path.join(
catalog.get_current_task_repo(), os.path.join(
'ASASSN', objname + '.csv')))
data = read(csv, format='csv')
for row in data:
mag = str(row['mag'])
if float(mag.strip('>')) > 50.0:
continue
photodict = {
PHOTOMETRY.TIME: str(jd_to_mjd(
Decimal(str(row['HJD'])))),
PHOTOMETRY.MAGNITUDE: mag.strip('>'),
PHOTOMETRY.SURVEY: 'ASASSN',
PHOTOMETRY.SOURCE: source
}
if '>' in mag:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
else:
photodict[PHOTOMETRY.E_MAGNITUDE] = str(row['mag_err'])
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
return
def do_asas_atels(catalog):
"""Import LCs exposed in ASASSN Atels."""
import gzip
try:
with gzip.open('/root/better-atel/atels.json.gz', 'rb') as f:
ateljson = json.load(f)
except Exception:
print('ATel data unavailable, skipping ASAS ATel task.')
return
for entry in ateljson:
if ('asas-sn.osu.edu/light_curve' in entry['body']
and 'Supernovae' in entry['subjects']):
matches = re.findall(r'<a\s+[^>]*?href="([^"]*)".*?>(.*?)<\/a>',
entry['body'], re.DOTALL)
lcurl = ''
objname = ''
for match in matches:
if 'asas-sn.osu.edu/light_curve' in match[0]:
lcurl = match[0]
objname = re.findall(r'\bASASSN-[0-9][0-9].*?\b', match[1])
if len(objname):
objname = objname[0]
if objname and lcurl:
name, source = catalog.new_entry(objname,
srcname='ASAS-SN Sky Patrol',
bibcode='2017arXiv170607060K',
url='https://asas-sn.osu.edu')
csv = catalog.load_url(
lcurl + '.csv',
os.path.join(catalog.get_current_task_repo(),
os.path.join('ASASSN', objname + '.csv')))
data = read(csv, format='csv')
for row in data:
mag = str(row['mag'])
if float(mag.strip('>')) > 50.0:
continue
photodict = {
PHOTOMETRY.TIME:
str(jd_to_mjd(Decimal(str(row['HJD'])))),
PHOTOMETRY.MAGNITUDE: mag.strip('>'),
PHOTOMETRY.SURVEY: 'ASASSN',
PHOTOMETRY.SOURCE: source
}
if '>' in mag:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
else:
photodict[PHOTOMETRY.E_MAGNITUDE] = str(row['mag_err'])
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
return
def do_donated_photo(catalog):
"""Import donated photometry."""
task_str = catalog.get_current_task_str()
# Private donations here #
if not catalog.args.travis:
pass
# End private donations #
# Ponder 05-12-17 donation
with open(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Ponder-05-12-17', 'meta.json'), 'r') as f:
metadict = json.loads(f.read())
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Ponder-05-12-17', '*.dat'))
for path in file_names:
with open(path, 'r') as f:
tsvin = list(csv.reader(f, delimiter=' ', skipinitialspace=True))
oname = path.split('/')[-1].split('.')[0]
name, source = catalog.new_entry(oname,
bibcode=metadict[oname]['bibcode'])
for row in pbar(tsvin, task_str + ': Ponder ' + oname):
if row[0][0] == '#' or not is_number(row[-1]):
continue
mjd = row[1]
bandinst = row[2].split('_')
band = bandinst[0]
inst = ''
if len(bandinst) > 1:
inst = bandinst[1]
mag = row[3]
uerr = row[4]
lerr = row[5]
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.BAND: band,
PHOTOMETRY.MAGNITUDE: mag,
PHOTOMETRY.E_LOWER_MAGNITUDE: lerr,
PHOTOMETRY.E_UPPER_MAGNITUDE: uerr,
PHOTOMETRY.SOURCE: source
}
if inst:
photodict[PHOTOMETRY.INSTRUMENT] = inst
catalog.entries[name].add_photometry(**photodict)
# Benetti 03-08-17 donation
path = os.path.join(catalog.get_current_task_repo(), 'Donations',
'Benetti-03-08-17', '1999E.dat')
with open(path, 'r') as f:
tsvin = list(csv.reader(f, delimiter=' ', skipinitialspace=True))
name, source = catalog.new_entry('SN1999E',
bibcode='2003MNRAS.340..191R')
bands = None
for row in tsvin:
if not row or row[0][0] == '#':
continue
if not bands:
bands = row[2:-2]
continue
mjd = row[1]
tel = row[-1] if 'IAUC' not in row[-1] else None
for bi, band in enumerate(bands):
mag = row[2 + 2 * bi]
if mag == '9999':
continue
err = row[2 + 2 * bi + 1]
limit = row[6] == 'True'
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.TELESCOPE: tel,
PHOTOMETRY.BAND: band,
PHOTOMETRY.MAGNITUDE: mag,
PHOTOMETRY.SOURCE: source
}
if err != '.00':
photodict[PHOTOMETRY.E_MAGNITUDE] = str(Decimal(err))
if tel:
photodict[PHOTOMETRY.TELESCOPE] = tel
catalog.entries[name].add_photometry(**photodict)
# Nicholl 01-29-17 donation
with open(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-01-29-17', 'meta.json'), 'r') as f:
metadict = json.loads(f.read())
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-01-29-17', '*.txt'))
for path in file_names:
data = read(path, format='cds')
oname = path.split('/')[-1].split('_')[0]
name, source = catalog.new_entry(oname,
bibcode=metadict[oname]['bibcode'])
for row in pbar(data, task_str + ': Nicholl ' + oname):
photodict = {
PHOTOMETRY.TIME: str(row['MJD']),
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.MAGNITUDE: str(row['mag']),
PHOTOMETRY.BAND: row['Filter'],
PHOTOMETRY.SOURCE: source
}
if 'system' in metadict[oname]:
photodict[PHOTOMETRY.SYSTEM] = metadict[oname]['system']
if 'l_mag' in row.columns and row['l_mag'] == '>':
photodict[PHOTOMETRY.UPPER_LIMIT] = True
elif 'e_mag' in row.columns:
photodict[PHOTOMETRY.E_MAGNITUDE] = str(row['e_mag'])
if 'Telescope' in row.columns:
photodict[PHOTOMETRY.TELESCOPE] = row['Telescope']
catalog.entries[name].add_photometry(**photodict)
# Arcavi 2016gkg donation
path = os.path.join(catalog.get_current_task_repo(), 'Donations',
'Arcavi-01-24-17', 'SN2016gkg.txt')
with open(path, 'r') as f:
tsvin = list(csv.reader(f, delimiter=' ', skipinitialspace=True))
name, source = catalog.new_entry('SN2016gkg',
bibcode='2016arXiv161106451A')
for row in tsvin:
if row[0][0] == '#':
continue
mjd = str(jd_to_mjd(Decimal(row[0])))
tel = row[1]
band = row[3]
mag = row[4]
err = row[5]
limit = row[6] == 'True'
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.TELESCOPE: tel,
PHOTOMETRY.BAND: band,
PHOTOMETRY.MAGNITUDE: mag,
PHOTOMETRY.SOURCE: source
}
if limit:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
else:
photodict[PHOTOMETRY.E_MAGNITUDE] = err
catalog.entries[name].add_photometry(**photodict)
# Nicholl Gaia16apd donation
path = os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-01-20-17', 'gaia16apd_phot.txt')
data = read(path, format='cds')
name, source = catalog.new_entry('Gaia16apd',
bibcode='2017ApJ...835L...8N')
for row in pbar(data, task_str + ': Nicholl Gaia16apd'):
photodict = {
PHOTOMETRY.TIME: str(row['MJD']),
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.MAGNITUDE: str(row['mag']),
PHOTOMETRY.BAND: row['Filter'],
PHOTOMETRY.TELESCOPE: row['Telescope'],
PHOTOMETRY.SOURCE: source
}
if row['l_mag'] == '>':
photodict[PHOTOMETRY.UPPER_LIMIT] = True
else:
photodict[PHOTOMETRY.E_MAGNITUDE] = str(row['e_mag'])
catalog.entries[name].add_photometry(**photodict)
# Kuncarayakti-01-09-17
datafile = os.path.join(catalog.get_current_task_repo(), 'Donations',
'Kuncarayakti-01-09-17', 'SN1978K.dat')
inpname = os.path.basename(datafile).split('.')[0]
with open(datafile, 'r') as f:
tsvin = csv.reader(f, delimiter=' ', skipinitialspace=True)
host = False
for ri, row in enumerate(tsvin):
if ri == 0:
continue
if row[0][0] == '#':
rsplit = [x.strip('# ') for x in ' '.join(row).split(',')]
bc = rsplit[0]
tel, ins = '', ''
if len(rsplit) > 1:
tel = rsplit[1]
if len(rsplit) > 2:
ins = rsplit[2]
continue
(name, source) = catalog.new_entry(inpname, bibcode=bc)
mag = row[4]
err = row[5]
mjd = str(astrotime('-'.join(row[:3]), format='iso').mjd)
photodict = {
PHOTOMETRY.BAND: row[3],
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.MAGNITUDE: mag.strip('>s'),
PHOTOMETRY.SOURCE: source
}
if is_number(err):
photodict[PHOTOMETRY.E_MAGNITUDE] = err
if tel:
photodict[PHOTOMETRY.TELESCOPE] = tel
if ins:
photodict[PHOTOMETRY.INSTRUMENT] = ins
if '>' in mag:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
if 's' in mag:
photodict[PHOTOMETRY.SYNTHETIC] = True
catalog.entries[name].add_photometry(**photodict)
# Nugent 01-09-17 donation
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nugent-01-09-17', '*.dat'))
for datafile in pbar_strings(file_names, task_str + ': Nugent-01-09-17'):
inpname = os.path.basename(datafile).split('.')[0]
(name, source) = catalog.new_entry(inpname,
bibcode='2006ApJ...645..841N')
with open(datafile, 'r') as f:
tsvin = csv.reader(f, delimiter=' ', skipinitialspace=True)
host = False
for urow in tsvin:
row = list(filter(None, urow))
counts = row[2]
e_counts = row[3]
zp = row[4]
photodict = {
PHOTOMETRY.BAND: row[1],
PHOTOMETRY.TIME: row[0],
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.COUNT_RATE: counts,
PHOTOMETRY.E_COUNT_RATE: e_counts,
PHOTOMETRY.ZERO_POINT: zp,
PHOTOMETRY.TELESCOPE: 'CFHT',
PHOTOMETRY.SURVEY: 'SNLS',
PHOTOMETRY.SOURCE: source
}
set_pd_mag_from_counts(photodict, counts, ec=e_counts, zp=zp)
catalog.entries[name].add_photometry(**photodict)
# Inserra 09-04-16 donation
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Inserra-09-04-16', '*.txt'))
for datafile in pbar_strings(file_names, task_str + ': Inserra-09-04-16'):
inpname = os.path.basename(datafile).split('.')[0]
(name, source) = catalog.new_entry(inpname,
bibcode='2013ApJ...770..128I')
with open(datafile, 'r') as f:
tsvin = csv.reader(f, delimiter=' ', skipinitialspace=True)
host = False
for row in tsvin:
if row[0][0] == '#':
if row[0] == '#Host':
host = True
continue
host = False
bands = row[3:-1]
continue
for bi, ba in enumerate(bands):
mag = row[5 + 2 * bi]
if not is_number(mag):
continue
system = 'AB'
if ba in ['U', 'B', 'V', 'R', 'I', 'J', 'H', 'K']:
system = 'Vega'
photodict = {
PHOTOMETRY.TIME: row[3],
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.BAND: ba,
PHOTOMETRY.MAGNITUDE: mag.strip('< '),
PHOTOMETRY.SOURCE: source,
PHOTOMETRY.SYSTEM: system
}
if 'ATel' not in row[-1]:
photodict[PHOTOMETRY.TELESCOPE] = row[-1]
if host:
photodict[PHOTOMETRY.HOST] = True
if '<' in mag:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
e_mag = row[5 + 2 * bi + 1].strip('() ')
if is_number(e_mag):
photodict[PHOTOMETRY.E_MAGNITUDE] = e_mag
catalog.entries[name].add_photometry(**photodict)
# Nicholl 04-01-16 donation
with open(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-04-01-16', 'bibcodes.json'), 'r') as f:
bcs = json.loads(f.read())
kcorrected = ['SN2011ke', 'SN2011kf', 'SN2012il', 'PTF10hgi', 'PTF11rks']
ignorephoto = ['PTF10hgi', 'PTF11rks', 'SN2011ke', 'SN2011kf', 'SN2012il']
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-04-01-16/*.txt'))
for datafile in pbar_strings(file_names, task_str + ': Nicholl-04-01-16'):
inpname = os.path.basename(datafile).split('_')[0]
isk = inpname in kcorrected
name = catalog.add_entry(inpname)
bibcode = ''
for bc in bcs:
if inpname in bcs[bc]:
bibcode = bc
if not bibcode:
raise ValueError('Bibcode not found!')
source = catalog.entries[name].add_source(bibcode=bibcode)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, inpname, source)
if inpname in ignorephoto:
continue
with open(datafile, 'r') as f:
tsvin = csv.reader(f, delimiter='\t', skipinitialspace=True)
rtelescope = ''
for r, rrow in enumerate(tsvin):
row = list(filter(None, rrow))
if not row:
continue
if row[0] == '#MJD':
bands = [x for x in row[1:] if x and 'err' not in x]
elif row[0][0] == '#' and len(row[0]) > 1:
rtelescope = row[0][1:]
if row[0][0] == '#':
continue
mjd = row[0]
if not is_number(mjd):
continue
for v, val in enumerate(row[1::2]):
upperlimit = ''
mag = val.strip('>')
emag = row[2 * v + 2]
if '>' in val or (is_number(emag) and float(emag) == 0.0):
upperlimit = True
if (not is_number(mag) or isnan(float(mag))
or float(mag) > 90.0):
continue
band = bands[v]
instrument = ''
survey = ''
system = ''
telescope = rtelescope
if telescope == 'LSQ':
instrument = 'QUEST'
elif telescope == 'PS1':
instrument = 'GPC'
elif telescope == 'NTT':
instrument = 'EFOSC'
elif telescope == 'GROND':
instrument = 'GROND'
telescope = 'MPI/ESO 2.2m'
else:
if band == 'NUV':
instrument = 'GALEX'
telescope = 'GALEX'
elif band in ['u', 'g', 'r', 'i', 'z']:
if inpname.startswith('PS1'):
instrument = 'GPC'
telescope = 'PS1'
survey = 'Pan-STARRS'
elif inpname.startswith('PTF'):
telescope = 'P60'
survey = 'PTF'
elif band.upper() in ['UVW2', 'UVW1', 'UVM2']:
instrument = 'UVOT'
telescope = 'Swift'
if inpname in ['PTF12dam']:
system = 'AB'
if inpname in ['SCP-06F6']:
system = 'Vega'
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.BAND: band,
PHOTOMETRY.MAGNITUDE: mag,
PHOTOMETRY.UPPER_LIMIT: upperlimit,
PHOTOMETRY.SOURCE: source
}
if instrument:
photodict[PHOTOMETRY.INSTRUMENT] = instrument
if telescope:
photodict[PHOTOMETRY.TELESCOPE] = telescope
if survey:
photodict[PHOTOMETRY.SURVEY] = survey
if system:
photodict[PHOTOMETRY.SYSTEM] = system
if (is_number(emag) and not isnan(float(emag))
and float(emag) > 0.0):
photodict[PHOTOMETRY.E_MAGNITUDE] = emag
if isk:
photodict[PHOTOMETRY.KCORRECTED] = True
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
# Maggi 04-11-16 donation (MC SNRs)
with open(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Maggi-04-11-16', 'LMCSNRs_OpenSNe.csv')) as f:
tsvin = csv.reader(f, delimiter=',')
for row in pbar(list(tsvin), task_str + ': Maggi-04-11-16/LMCSNRs'):
name = 'MCSNR ' + row[0]
name = catalog.add_entry(name)
ra = row[2]
dec = row[3]
source = (catalog.entries[name].add_source(
bibcode='2016A&A...585A.162M'))
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS,
'LMCSNR J' + rep_chars(ra, ' :.') + rep_chars(dec, ' :.'),
source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
if row[1] != 'noname':
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[1],
source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[2], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[3], source)
catalog.entries[name].add_quantity(SUPERNOVA.HOST, 'LMC', source)
if row[4] == '1':
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE,
'Ia', source)
elif row[4] == '2':
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE,
'CC', source)
with open(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Maggi-04-11-16', 'SMCSNRs_OpenSNe.csv')) as f:
tsvin = csv.reader(f, delimiter=',')
for row in pbar(list(tsvin), task_str + ': Maggi-04-11-16/SMCSNRs'):
name = 'MCSNR ' + row[0]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(name='Pierre Maggi')
ra = row[3]
dec = row[4]
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, 'SMCSNR J' + ra.replace(':', '')[:6] +
dec.replace(':', '')[:7], source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[1], source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[2], source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[3], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[4], source)
catalog.entries[name].add_quantity(SUPERNOVA.HOST, 'SMC', source)
catalog.journal_entries()
# Galbany 04-18-16 donation
folders = next(
os.walk(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Galbany-04-18-16/')))[1]
bibcode = '2016AJ....151...33G'
for folder in folders:
infofiles = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Galbany-04-18-16/') + folder + '/*.info')
photfiles = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Galbany-04-18-16/') + folder + '/*.out*')
zhel = ''
zcmb = ''
zerr = ''
for path in infofiles:
with open(path, 'r') as f:
lines = f.read().splitlines()
for line in lines:
splitline = line.split(':')
field = splitline[0].strip().lower()
value = splitline[1].strip()
if field == 'name':
name = value[:6].upper()
name += (value[6].upper()
if len(value) == 7 else value[6:])
name = catalog.add_entry(name)
source = (catalog.entries[name].add_source(
bibcode=bibcode))
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, name, source)
elif field == 'type':
claimedtype = value.replace('SN', '')
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, claimedtype, source)
elif field == 'zhel':
zhel = value
elif field == 'redshift_error':
zerr = value
elif field == 'zcmb':
zcmb = value
elif field == 'ra':
catalog.entries[name].add_quantity(
SUPERNOVA.RA,
value,
source,
u_value='floatdegrees')
elif field == 'dec':
catalog.entries[name].add_quantity(
SUPERNOVA.DEC,
value,
source,
u_value='floatdegrees')
elif field == 'host':
value = value.replace('- ', '-').replace('G ', 'G')
catalog.entries[name].add_quantity(
SUPERNOVA.HOST, value, source)
elif field == 'e(b-v)_mw':
catalog.entries[name].add_quantity(
SUPERNOVA.EBV, value, source)
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT,
zhel,
source,
e_value=zerr,
kind='heliocentric')
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT,
zcmb,
source,
e_value=zerr,
kind='cmb')
for path in photfiles:
with open(path, 'r') as f:
band = ''
lines = f.read().splitlines()
for li, line in enumerate(lines):
if li in [0, 2, 3]:
continue
if li == 1:
band = line.split(':')[-1].strip()
else:
cols = list(filter(None, line.split()))
if not cols:
continue
catalog.entries[name].add_photometry(
time=cols[0],
u_time='MJD',
magnitude=cols[1],
e_magnitude=cols[2],
band=band,
system=cols[3],
telescope=cols[4],
source=source)
catalog.journal_entries()
# Nicholl 05-03-16
files = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-05-03-16', '*.txt'))
name = catalog.add_entry('SN2015bn')
for fi in pbar(files, task_str + ': Nicholl-05-03-16'):
if 'late' in fi:
bc = '2016ApJ...828L..18N'
else:
bc = '2016ApJ...826...39N'
source = catalog.entries[name].add_source(bibcode=bc)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, 'PS15ae', source)
telescope = os.path.basename(fi).split('_')[1]
with open(fi, 'r') as f:
lines = f.read().splitlines()
for li, line in enumerate(lines):
if not line or (line[0] == '#' and li != 0):
continue
cols = list(filter(None, line.split()))
if not cols:
continue
if li == 0:
bands = cols[1:]
continue
mjd = cols[0]
for ci, col in enumerate(cols[1::2]):
if not is_number(col) or np.isnan(float(col)):
continue
band = bands[ci]
band_set = ''
system = 'Vega'
if bands[ci] in ["u'", "g'", "r'", "i'", "z'"]:
band_set = 'SDSS'
system = 'SDSS'
elif telescope == 'ASASSN':
band_set = 'ASASSN'
system = 'Vega'
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.MAGNITUDE: col,
PHOTOMETRY.BAND: bands[ci],
PHOTOMETRY.SOURCE: source,
PHOTOMETRY.TELESCOPE: telescope,
PHOTOMETRY.SYSTEM: system
}
if band_set:
photodict[PHOTOMETRY.BAND_SET] = band_set
emag = cols[2 * ci + 2]
if is_number(emag):
photodict[PHOTOMETRY.E_MAGNITUDE] = emag
else:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
if telescope == 'Swift':
photodict[PHOTOMETRY.INSTRUMENT] = 'UVOT'
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
return
def do_snf_specta(catalog):
task_str = catalog.get_current_task_str()
bibcodes = {
'SN2005gj': '2006ApJ...650..510A',
'SN2006D': '2007ApJ...654L..53T',
'SN2007if': '2010ApJ...713.1073S',
'SN2011fe': '2013A&A...554A..27P'
}
oldname = ''
snfcnt = 0
eventfolders = next(
os.walk(os.path.join(catalog.get_current_task_repo(), 'SNFactory')))[1]
for eventfolder in pbar(eventfolders, task_str):
oname = eventfolder
name = catalog.get_preferred_name(oname)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
sec_reference = 'Nearby Supernova Factory'
sec_refurl = 'http://snfactory.lbl.gov/'
sec_bibcode = '2002SPIE.4836...61A'
sec_source = catalog.entries[name].add_source(name=sec_reference,
url=sec_refurl,
bibcode=sec_bibcode,
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oname, sec_source)
bibcode = bibcodes[oname]
source = catalog.entries[name].add_source(bibcode=bibcode)
sources = uniq_cdl([source, sec_source])
use_path = os.path.join(catalog.get_current_task_repo(), 'SNFactory',
eventfolder, '*.dat')
eventspectra = glob(use_path)
for spectrum in pbar(eventspectra, task_str):
filename = os.path.basename(spectrum)
with open(spectrum) as spec_file:
specdata = list(
csv.reader(spec_file, delimiter=' ',
skipinitialspace=True))
specdata = list(filter(None, specdata))
newspec = []
time = ''
telescope = ''
instrument = ''
observer = ''
observatory = ''
if 'Keck_20060202_R' in spectrum:
time = '53768.23469'
elif 'Spectrum05_276' in spectrum:
time = pretty_num(astrotime('2005-10-03').mjd, sig=5)
elif 'Spectrum05_329' in spectrum:
time = pretty_num(astrotime('2005-11-25').mjd, sig=5)
elif 'Spectrum05_336' in spectrum:
time = pretty_num(astrotime('2005-12-02').mjd, sig=5)
for row in specdata:
if row[0][0] == '#':
joinrow = (' '.join(row)).split('=')
if len(joinrow) < 2:
continue
field = joinrow[0].strip('# ')
value = joinrow[1].split('/')[0].strip('\' ')
if not time:
if field == 'JD':
time = str(jd_to_mjd(Decimal(value)))
elif field == 'MJD':
time = value
elif field == 'MJD-OBS':
time = value
if field == 'OBSERVER':
observer = value.capitalize()
if field == 'OBSERVAT':
observatory = value.capitalize()
if field == 'TELESCOP':
telescope = value.capitalize()
if field == 'INSTRUME':
instrument = value.capitalize()
else:
newspec.append(row)
if not time:
raise ValueError('Time missing from spectrum.')
specdata = newspec
haserrors = len(
specdata[0]
) == 3 and specdata[0][2] and specdata[0][2] != 'NaN'
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
unit_err = ('Variance'
if oldname == 'SN2011fe' else 'erg/s/cm^2/Angstrom')
unit_flx = 'erg/s/cm^2/Angstrom'
catalog.entries[name].add_spectrum(u_wavelengths='Angstrom',
u_fluxes=unit_flx,
u_time='MJD',
time=time,
wavelengths=wavelengths,
fluxes=fluxes,
errors=errors,
observer=observer,
observatory=observatory,
telescope=telescope,
instrument=instrument,
u_errors=unit_err,
source=sources,
filename=filename)
snfcnt = snfcnt + 1
if (catalog.args.travis
and snfcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
def do_external_fits_spectra(catalog):
fpath = catalog.get_current_task_repo()
with open(os.path.join(fpath, 'fits', 'meta.json'), 'r') as f:
metadict = json.loads(f.read())
fureps = {'erg/cm2/s/A': 'erg/s/cm^2/Angstrom'}
task_str = catalog.get_current_task_str()
path_pattern = os.path.join(catalog.get_current_task_repo(), 'fits',
'*.fits')
files = glob(path_pattern)
for datafile in files:
filename = datafile.split('/')[-1]
if filename == 'meta.json':
continue
hdulist = fits.open(datafile)
for oi, obj in enumerate(hdulist[0].header):
if any(x in ['.', '/'] for x in obj):
del (hdulist[0].header[oi])
hdulist[0].verify('silentfix')
hdrkeys = list(hdulist[0].header.keys())
# print(hdrkeys)
name = ''
if filename in metadict:
if 'name' in metadict[filename]:
name = metadict[filename]['name']
if not name:
name = hdulist[0].header['OBJECT']
if 'bibcode' in metadict[filename]:
name, source = catalog.new_entry(
name, bibcode=metadict[filename]['bibcode'])
elif 'donator' in metadict[filename]:
name, source = catalog.new_entry(
name, srcname=metadict[filename]['donator'])
else:
if 'OBSERVER' in hdrkeys:
name, source = catalog.new_entry(
name, srcname=hdulist[0].header['OBSERVER'])
else:
name = catalog.add_entry(name)
source = catalog.entries[name].add_self_source()
# for key in hdulist[0].header.keys():
# print(key, hdulist[0].header[key])
if hdulist[0].header['SIMPLE']:
if 'JD' in hdrkeys:
mjd = str(jd_to_mjd(Decimal(str(hdulist[0].header['JD']))))
elif 'MJD' in hdrkeys:
mjd = str(hdulist[0].header['MJD'])
elif 'DATE-OBS' in hdrkeys:
if 'T' in hdulist[0].header['DATE-OBS']:
dateobs = hdulist[0].header['DATE-OBS'].strip()
elif 'UTC-OBS' in hdrkeys:
dateobs = hdulist[0].header['DATE-OBS'].strip(
) + 'T' + hdulist[0].header['UTC-OBS'].strip()
mjd = str(astrotime(dateobs, format='isot').mjd)
else:
raise ValueError("Couldn't find JD/MJD for spectrum.")
w0 = hdulist[0].header['CRVAL1']
if hdulist[0].header['NAXIS'] == 1:
wd = hdulist[0].header['CDELT1']
fluxes = [str(x) for x in list(hdulist[0].data)]
errors = False
elif hdulist[0].header['NAXIS'] == 2:
wd = hdulist[0].header['CD1_1']
fluxes = [str(x) for x in list(hdulist[0].data)[0]]
errors = False
elif hdulist[0].header['NAXIS'] == 3:
wd = hdulist[0].header['CD1_1']
fluxes = [str(x) for x in list(hdulist[0].data)[0][0]]
errors = [str(x) for x in list(hdulist[0].data)[3][0]]
else:
print('Warning: Skipping FITS spectrum `{}`.'.format(filename))
continue
waves = [str(w0 + wd * x) for x in range(0, len(fluxes))]
else:
raise ValueError('Non-simple FITS import not yet supported.')
if 'BUNIT' in hdrkeys:
fluxunit = hdulist[0].header['BUNIT']
if fluxunit in fureps:
fluxunit = fureps[fluxunit]
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.WAVELENGTHS: waves,
SPECTRUM.TIME: mjd,
SPECTRUM.U_TIME: 'MJD',
SPECTRUM.FLUXES: fluxes,
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.FILENAME: filename,
SPECTRUM.SOURCE: source
}
if 'TELESCOP' in hdrkeys:
specdict[SPECTRUM.TELESCOPE] = hdulist[0].header['TELESCOP']
if 'INSTRUME' in hdrkeys:
specdict[SPECTRUM.INSTRUMENT] = hdulist[0].header['INSTRUME']
if 'AIRMASS' in hdrkeys:
specdict[SPECTRUM.AIRMASS] = hdulist[0].header['AIRMASS']
if errors:
specdict[SPECTRUM.ERRORS] = errors
specdict[SPECTRUM.U_ERRORS] = fluxunit
if 'SITENAME' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['SITENAME']
elif 'OBSERVAT' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['OBSERVAT']
if 'OBSERVER' in hdrkeys:
specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER']
catalog.entries[name].add_spectrum(**specdict)
hdulist.close()
catalog.journal_entries()
return
def do_ogle(catalog):
task_str = catalog.get_current_task_str()
basenames = [
'transients', 'transients/2015', 'transients/2014b', 'transients/2014',
'transients/2013', 'transients/2012'
]
oglenames = []
ogleupdate = [True, False, False, False, False]
for b, bn in enumerate(pbar(basenames, task_str)):
if catalog.args.update and not ogleupdate[b]:
continue
filepath = os.path.join(catalog.get_current_task_repo(), 'OGLE-')
filepath += bn.replace('/', '-') + '-transients.html'
htmltxt = catalog.load_url(
'http://ogle.astrouw.edu.pl/ogle4/' + bn + '/transients.html',
filepath)
if not htmltxt:
continue
soup = BeautifulSoup(htmltxt, 'html5lib')
links = soup.findAll('a')
breaks = soup.findAll('br')
datalinks = []
datafnames = []
for a in links:
if a.has_attr('href'):
if '.dat' in a['href']:
datalinks.append('http://ogle.astrouw.edu.pl/ogle4/' + bn +
'/' + a['href'])
datafnames.append(
bn.replace('/', '-') + '-' + a['href'].replace('/',
'-'))
ec = -1
reference = 'OGLE-IV Transient Detection System'
refurl = 'http://ogle.astrouw.edu.pl/ogle4/transients/transients.html'
for bi, br in enumerate(pbar(breaks, task_str)):
sibling = br.nextSibling
if 'Ra,Dec=' in sibling:
line = sibling.replace('\n', '').split('Ra,Dec=')
name = line[0].strip()
ec += 1
if 'NOVA' in name or 'dupl' in name:
continue
if name in oglenames:
continue
oglenames.append(name)
name = catalog.add_entry(name)
mySibling = sibling.nextSibling
atelref = ''
claimedtype = ''
while 'Ra,Dec=' not in mySibling:
if isinstance(mySibling, NavigableString):
if not claimedtype and 'class=' in str(mySibling):
claimedtype = re.sub(r'\([^)]*\)', '',
str(mySibling).split('=')[-1])
claimedtype = claimedtype.replace('SN', '').strip()
if claimedtype == '-':
claimedtype = ''
if isinstance(mySibling, Tag):
atela = mySibling
if (atela and atela.has_attr('href') and
'astronomerstelegram' in atela['href']):
atelref = atela.contents[0].strip()
atelurl = atela['href']
mySibling = mySibling.nextSibling
if mySibling is None:
break
# nextSibling = sibling.nextSibling
# if ((isinstance(nextSibling, Tag) and
# nextSibling.has_attr('alt') and
# nextSibling.contents[0].strip() != 'NED')):
# radec = nextSibling.contents[0].strip().split()
# else:
# radec = line[-1].split()
# ra = radec[0]
# dec = radec[1]
fname = os.path.join(catalog.get_current_task_repo(),
'OGLE/') + datafnames[ec]
csvtxt = catalog.load_url(datalinks[ec], fname)
lcdat = csvtxt.splitlines()
sources = [
catalog.entries[name].add_source(
name=reference, url=refurl)
]
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name,
sources[0])
if atelref and atelref != 'ATel#----':
sources.append(catalog.entries[name].add_source(
name=atelref, url=atelurl))
sources = uniq_cdl(sources)
if name.startswith('OGLE'):
if name[4] == '-':
if is_number(name[5:9]):
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, name[5:9], sources)
else:
if is_number(name[4:6]):
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, '20' + name[4:6],
sources)
# RA and Dec from OGLE pages currently not reliable
# catalog.entries[name].add_quantity(SUPERNOVA.RA, ra, sources)
# catalog.entries[name].add_quantity(SUPERNOVA.DEC, dec,
# sources)
if claimedtype and claimedtype != '-':
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE,
claimedtype, sources)
elif ('SN' not in name and
SUPERNOVA.CLAIMED_TYPE not in catalog.entries[name]):
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE,
'Candidate', sources)
for row in lcdat:
row = row.split()
mjd = str(jd_to_mjd(Decimal(row[0])))
magnitude = row[1]
if float(magnitude) > 90.0:
continue
e_mag = row[2]
upperlimit = False
if e_mag == '-1' or float(e_mag) > 10.0:
e_mag = ''
upperlimit = True
catalog.entries[name].add_photometry(
time=mjd,
u_time='MJD',
band='I',
magnitude=magnitude,
e_magnitude=e_mag,
system='Vega',
source=sources,
upperlimit=upperlimit)
if catalog.args.update:
catalog.journal_entries()
if catalog.args.travis and bi >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
return
def do_itep(catalog):
task_str = catalog.get_current_task_str()
itepignoresources = ['2004ApJ...602..571B', '2013NewA...20...30M']
itepignorephot = ['SN2006gy']
needsbib = []
with open(os.path.join(catalog.get_current_task_repo(), 'itep-refs.txt'),
'r') as refs_file:
refrep = refs_file.read().splitlines()
refrepf = dict(list(zip(refrep[1::2], refrep[::2])))
fname = os.path.join(catalog.get_current_task_repo(),
'itep-lc-cat-28dec2015.txt')
tsvin = list(
csv.reader(open(fname, 'r'), delimiter='|', skipinitialspace=True))
curname = ''
for rr, row in enumerate(pbar(tsvin, task_str)):
if rr <= 1 or len(row) < 7:
continue
oldname = 'SN' + row[0].strip()
mjd = str(jd_to_mjd(Decimal(row[1].strip())))
band = row[2].strip()
magnitude = row[3].strip()
e_magnitude = row[4].strip()
reference = row[6].strip().strip(',')
if curname != oldname:
curname = oldname
name = catalog.add_entry(oldname)
sec_reference = ('Sternberg Astronomical Institute '
'Supernova Light Curve Catalogue')
sec_refurl = 'http://dau.itep.ru/sn/node/72'
sec_source = catalog.entries[name].add_source(name=sec_reference,
url=sec_refurl,
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oldname,
sec_source)
year = re.findall(r'\d+', name)[0]
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, year,
sec_source)
if reference in refrepf:
bibcode = unescape(refrepf[reference])
source = catalog.entries[name].add_source(bibcode=bibcode)
else:
needsbib.append(reference)
source = catalog.entries[name].add_source(
name=reference) if reference else ''
if oldname in itepignorephot or bibcode in itepignoresources:
continue
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.MAGNITUDE: magnitude,
PHOTOMETRY.SOURCE: uniq_cdl([sec_source, source])
}
if e_magnitude:
photodict[PHOTOMETRY.E_MAGNITUDE] = e_magnitude
if band.endswith('_SDSS'):
photodict[PHOTOMETRY.BAND_SET] = 'SDSS'
photodict[PHOTOMETRY.SYSTEM] = 'SDSS'
band = band.replace('_SDSS', "'")
photodict[PHOTOMETRY.BAND] = band
catalog.entries[name].add_photometry(**photodict)
if catalog.args.travis and rr >= catalog.TRAVIS_QUERY_LIMIT:
break
# Write out references that could use aa bibcode
needsbib = list(OrderedDict.fromkeys(needsbib))
with open('../itep-needsbib.txt', 'w') as bib_file:
bib_file.writelines(['%ss\n' % ii for ii in needsbib])
catalog.journal_entries()
return
def do_tns_photo(catalog):
"""Load TNS photometry."""
task_str = catalog.get_current_task_str()
tns_url = 'https://wis-tns.weizmann.ac.il/'
try:
with open('tns.key', 'r') as f:
tnskey = f.read().splitlines()[0]
except Exception:
catalog.log.warning('TNS API key not found, make sure a file named '
'`tns.key` containing the key is placed the '
'astrocats directory.')
tnskey = ''
bandreps = {'Clear': 'C'}
fails = 0
for name in pbar(list(catalog.entries.keys()), task_str):
if name not in catalog.entries:
continue
aliases = catalog.entries[name].get_aliases()
oname = ''
for alias in aliases:
if (alias.startswith(('SN', 'AT')) and is_integer(alias[2:6]) and
int(alias[2:6]) >= 2016) and alias[6:].isalpha():
oname = alias
break
if not oname:
continue
reqname = oname[2:]
jsonpath = os.path.join(catalog.get_current_task_repo(), 'TNS',
reqname + '.json')
download_json = True
if os.path.isfile(jsonpath):
with open(jsonpath, 'r') as f:
objdict = json.load(f)
if ('discoverydate' in objdict and
(datetime.now() - datetime.strptime(objdict['discoverydate'],
'%Y-%m-%d %H:%M:%S')
).days > 90):
download_json = False
if download_json:
data = urllib.parse.urlencode({
'api_key': tnskey,
'data': json.dumps({
'objname': reqname,
'photometry': '1'
})
}).encode('ascii')
req = urllib.request.Request(
'https://wis-tns.weizmann.ac.il/api/get/object', data=data)
trys = 0
objdict = None
while trys < 3 and not objdict:
try:
objdict = json.loads(
urllib.request.urlopen(req, timeout=30).read().decode('ascii'))[
'data']['reply']
except KeyboardInterrupt:
raise
except Exception:
catalog.log.warning('API request failed for `{}`.'.format(
name))
time.sleep(5)
trys = trys + 1
if (not objdict or 'objname' not in objdict or
not isinstance(objdict['objname'], str)):
fails = fails + 1
catalog.log.warning('Object `{}` not found!'.format(name))
if fails >= 5:
break
continue
# Cache object here
with open(jsonpath, 'w') as f:
json.dump(sortOD(objdict), f, indent='\t',
separators=(',', ':'), ensure_ascii=False,
sort_keys=True)
if 'photometry' not in objdict:
continue
photoarr = objdict['photometry']
name, source = catalog.new_entry(
oname, srcname='Transient Name Server', url=tns_url)
for photo in photoarr:
if 'mag' not in photo['flux_unit']['name'].lower():
catalog.log.warning('Unknown flux unit `{}`.'.format(photo[
'flux_unit']['name']))
continue
if not photo['jd']:
continue
if not photo['flux'] and not photo['limflux']:
continue
mag = photo['flux'] if photo['flux'] else photo['limflux']
photodict = {
PHOTOMETRY.TIME: str(jd_to_mjd(Decimal(str(photo['jd'])))),
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.MAGNITUDE: mag,
PHOTOMETRY.SOURCE: source
}
if photo.get('fluxerr', ''):
photodict[PHOTOMETRY.E_MAGNITUDE] = photo['fluxerr']
if not photo['flux']:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
band = photo['filters']['name']
if band:
if band in bandreps:
band = bandreps[band]
photodict[PHOTOMETRY.BAND] = band
if photo.get('observer', ''):
photodict[PHOTOMETRY.OBSERVER] = photo['observer']
if 'source_group' in photo:
survey = photo['source_group']['group_name']
if survey:
photodict[PHOTOMETRY.SURVEY] = survey
if 'telescope' in photo:
telescope = photo['telescope']['name']
if telescope and telescope != 'Other':
photodict[PHOTOMETRY.TELESCOPE] = telescope
if 'instrument' in photo:
instrument = photo['instrument']['name']
if instrument and instrument != 'Other':
photodict[PHOTOMETRY.INSTRUMENT] = instrument
system = ''
if 'Vega' in photo['flux_unit']['name']:
system = 'Vega'
elif 'ab' in photo['flux_unit']['name']:
system = 'AB'
if system:
photodict[PHOTOMETRY.SYSTEM] = system
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
return
def do_csp_fits_spectra(catalog):
from astropy.io import fits
fpath = catalog.get_current_task_repo()
fureps = {'erg/cm2/s/A': 'erg/s/cm^2/Angstrom'}
task_str = catalog.get_current_task_str()
dirs = [x[0] for x in os.walk(os.path.join(fpath, 'Gutierrez_et_al_2017'))]
files = []
for dir in dirs:
files.extend(glob(os.path.join(dir, '*.fits')))
for datafile in pbar(files, task_str):
filename = datafile.split('/')[-1]
hdulist = fits.open(datafile)
for oi, obj in enumerate(hdulist[0].header):
if any(x in ['.', '/'] for x in obj):
del (hdulist[0].header[oi])
try:
hdulist[0].verify('silentfix')
except Exception as e:
print(e)
hdrkeys = list(hdulist[0].header.keys())
# print(hdrkeys)
name = datafile.split('/')[-2]
if name[2] in '6789':
name = 'SN19' + name[2:]
elif name != 'SN210':
name = 'SN20' + name[2:]
name, source = catalog.new_entry(name, bibcode='2017ApJ...850...89G')
# for key in hdulist[0].header.keys():
# print(key, hdulist[0].header[key])
mjd = None
if hdulist[0].header['SIMPLE']:
if 'JD' in hdrkeys:
mjd = str(jd_to_mjd(Decimal(str(hdulist[0].header['JD']))))
elif 'MJD' in hdrkeys:
mjd = str(hdulist[0].header['MJD'])
elif 'DATE-OBS' in hdrkeys or 'DATE' in hdrkeys:
dkey = 'DATE-OBS' if 'DATE-OBS' in hdrkeys else 'DATE'
dval = hdulist[0].header[dkey]
if is_number(dval):
dkey = 'DATE' if dkey == 'DATE-OBS' else 'DATE-OBS'
dval = hdulist[0].header[dkey]
dateobs = None
if 'T' in dval:
dateobs = dval.strip()
elif 'UTC-OBS' in hdrkeys:
dateobs = dval.strip(
) + 'T' + hdulist[0].header['UTC-OBS'].strip()
if dateobs is not None:
mjd = str(astrotime(dateobs, format='isot').mjd)
# print(hdulist[0].header)
if 'CRVAL1' in hdulist[0].header:
w0 = hdulist[0].header['CRVAL1']
elif hdulist[0].header['CTYPE1'] == 'MULTISPE':
w0 = float(
hdulist[0].header['WAT2_001'].split('"')[-1].split()[3])
else:
raise ValueError('Unsupported spectrum format.')
if hdulist[0].header['NAXIS'] == 1:
wd = hdulist[0].header['CDELT1']
fluxes = [str(x) for x in list(hdulist[0].data)]
errors = False
elif hdulist[0].header['NAXIS'] == 3:
wd = hdulist[0].header['CD1_1']
fluxes = [str(x) for x in list(hdulist[0].data)[0][0]]
errors = [str(x) for x in list(hdulist[0].data)[-1][0]]
else:
print('Warning: Skipping FITS spectrum `{}`.'.format(filename))
continue
waves = [str(w0 + wd * x) for x in range(0, len(fluxes))]
else:
raise ValueError('Non-simple FITS import not yet supported.')
if 'BUNIT' in hdrkeys:
fluxunit = hdulist[0].header['BUNIT']
if fluxunit in fureps:
fluxunit = fureps[fluxunit]
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.WAVELENGTHS: waves,
SPECTRUM.FLUXES: fluxes,
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.FILENAME: filename,
SPECTRUM.SOURCE: source
}
if mjd is not None:
specdict[SPECTRUM.TIME] = mjd
specdict[SPECTRUM.U_TIME] = 'MJD'
if 'TELESCOP' in hdrkeys:
specdict[SPECTRUM.TELESCOPE] = hdulist[0].header['TELESCOP']
if 'INSTRUME' in hdrkeys:
specdict[SPECTRUM.INSTRUMENT] = hdulist[0].header['INSTRUME']
if 'AIRMASS' in hdrkeys:
specdict[SPECTRUM.AIRMASS] = hdulist[0].header['AIRMASS']
if errors:
specdict[SPECTRUM.ERRORS] = errors
specdict[SPECTRUM.U_ERRORS] = fluxunit
if 'SITENAME' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['SITENAME']
elif 'OBSERVAT' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['OBSERVAT']
if 'OBSERVER' in hdrkeys:
specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER']
catalog.entries[name].add_spectrum(**specdict)
hdulist.close()
catalog.journal_entries()
return
def do_suspect_photo(catalog):
task_str = catalog.get_current_task_str()
with open(
os.path.join(catalog.get_current_task_repo(),
'suspectreferences.csv'), 'r') as f:
tsvin = csv.reader(f, delimiter=',', skipinitialspace=True)
suspectrefdict = {}
for row in tsvin:
suspectrefdict[row[0]] = row[1]
file_names = list(
sorted(
glob(
os.path.join(catalog.get_current_task_repo(),
'SUSPECT/*.html'))))
for datafile in pbar_strings(file_names, task_str):
basename = os.path.basename(datafile)
basesplit = basename.split('-')
oldname = basesplit[1]
name = catalog.add_entry(oldname)
if name.startswith('SN') and is_number(name[2:]):
name = name + 'A'
band = basesplit[3].split('.')[0]
ei = int(basesplit[2])
bandlink = 'file://' + os.path.abspath(datafile)
bandresp = urllib.request.urlopen(bandlink)
bandsoup = BeautifulSoup(bandresp, 'html5lib')
bandtable = bandsoup.find('table')
names = bandsoup.body.findAll(text=re.compile('Name'))
reference = ''
for link in bandsoup.body.findAll('a'):
if 'adsabs' in link['href']:
reference = str(link).replace('"', "'")
bibcode = unescape(suspectrefdict[reference])
source = catalog.entries[name].add_source(bibcode=bibcode)
sec_ref = 'SUSPECT'
sec_refurl = 'https://www.nhn.ou.edu/~suspect/'
sec_source = catalog.entries[name].add_source(name=sec_ref,
url=sec_refurl,
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oldname,
sec_source)
if ei == 1:
year = re.findall(r'\d+', name)[0]
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, year,
sec_source)
catalog.entries[name].add_quantity(SUPERNOVA.HOST,
names[1].split(':')[1].strip(),
sec_source)
redshifts = bandsoup.body.findAll(text=re.compile('Redshift'))
if redshifts:
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT,
redshifts[0].split(':')[1].strip(),
sec_source,
kind='heliocentric')
# hvels = bandsoup.body.findAll(text=re.compile('Heliocentric
# Velocity'))
# if hvels:
# vel = hvels[0].split(':')[1].strip().split(' ')[0]
# catalog.entries[name].add_quantity(SUPERNOVA.VELOCITY, vel,
# sec_source,
# kind='heliocentric')
types = bandsoup.body.findAll(text=re.compile('Type'))
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE,
types[0].split(':')[1].strip().split(' ')[0], sec_source)
for r, row in enumerate(bandtable.findAll('tr')):
if r == 0:
continue
col = row.findAll('td')
mjd = str(jd_to_mjd(Decimal(col[0].contents[0])))
mag = col[3].contents[0]
if mag.isspace():
mag = ''
else:
mag = str(mag)
e_magnitude = col[4].contents[0]
if e_magnitude.isspace():
e_magnitude = ''
else:
e_magnitude = str(e_magnitude)
catalog.entries[name].add_photometry(time=mjd,
u_time='MJD',
band=band,
magnitude=mag,
e_magnitude=e_magnitude,
source=sec_source + ',' +
source)
catalog.journal_entries()
return
def generate_event_list(self, event_list):
"""Generate a list of events and/or convert events to JSON format."""
prt = self._printer
cidict = OrderedDict()
intro_shown = False
new_event_list = []
previous_file = None
for event in event_list:
rsource = {SOURCE.NAME: self._DEFAULT_SOURCE}
use_self_source = None
new_events = []
toffset = Decimal('0')
if ('.' in event and os.path.isfile(event) and
not event.endswith('.json')):
if not intro_shown:
prt.message('converter_info')
intro_shown = True
prt.message('converting_to_json', [event])
with open(event, 'r') as f:
ftxt = f.read()
# Try a couple of table formats from astropy.
table = None
try:
table = read(ftxt, Reader=Cds, guess=False)
except Exception:
pass
else:
prt.message('convert_cds')
flines = [table.colnames] + [
list(x) for x in np.array(table).tolist()]
for i in range(len(flines)):
flines[i] = [str(x) for x in flines[i]]
try:
table = read(ftxt, Reader=Latex, guess=False)
except Exception:
pass
else:
prt.message('convert_latex')
flines = [table.colnames] + [
list(x) for x in np.array(table).tolist()]
if table is None:
# Count to try and determine delimiter.
delims = [' ', '\t', ',', ';', '|', '&']
delimnames = [
'Space: ` `', 'Tab: `\t`', 'Comma: `,`',
'Semi-colon: `;`', 'Bar: `|`', 'Ampersand: `&`']
delim = None
delimcounts = [ftxt.count(x) for x in delims]
maxdelimcount = max(delimcounts)
delim = delims[delimcounts.index(maxdelimcount)]
# If two delimiter options are close in count, ask user.
for i, x in enumerate(delimcounts):
if x > 0.5 * maxdelimcount and delims[i] != delim:
delim = None
if delim is None:
odelims = list(np.array(delimnames)[
np.array(delimcounts) > 0])
delim = delims[prt.prompt(
'delim', kind='option', options=odelims) - 1]
ad = list(delims)
ad.remove(delim)
ad = ''.join(ad)
fsplit = ftxt.splitlines()
fsplit = [
x.replace('$', '').replace('\\pm', delim)
.replace('±', delim).replace('(', delim + '(')
.strip(ad + '()# ').replace('′', "'")
for x in fsplit]
flines = []
for fs in fsplit:
flines.append(list(
csv.reader([fs], delimiter=delim))[0])
flines = [[
x.strip(ad + '#$()\\')
for x in y] for y in flines]
# Find band columns if they exist and insert error columns
# if they don't exist.
for fi, fl in enumerate(list(flines)):
flcopy = list(fl)
offset = 0
if not any([is_number(x) for x in fl]):
for fci, fc in enumerate(fl):
if (fc in self._band_names and
(fci == len(fl) - 1 or
fl[fci + 1] not in self._emagstrs)):
flcopy.insert(fci + 1 + offset, 'e mag')
offset += 1
flines[fi] = flcopy
# Find the most frequent column count. These are probably
# the tables we wish to read.
flens = [len(x) for x in flines]
ncols = Counter(flens).most_common(1)[0][0]
newlines = []
potential_name = None
for fi, fl in enumerate(flines):
if (len(fl) and flens[fi] == 1 and
fi < len(flines) - 1 and
flens[fi + 1] == ncols and not len(newlines)):
potential_name = fl[0]
if flens[fi] == ncols:
if potential_name is not None and any(
[is_number(x) for x in fl]):
newlines.append([potential_name] + list(fl))
else:
newlines.append(list(fl))
flines = newlines
for fi, fl in enumerate(flines):
if len(fl) == ncols and potential_name is not None:
if not any([is_number(x) for x in fl]):
flines[fi] = ['name'] + list(fl)
# If none of the rows contain numeric data, the file
# is likely a list of transient names.
if (len(flines) and
(not any(any([is_number(x) or x == '' for x in y])
for y in flines) or
len(flines) == 1)):
new_events = [
it for s in flines for it in s]
# If last row is numeric, then likely this is a file with
# transient data.
elif (len(flines) > 1 and
any([is_number(x) for x in flines[-1]])):
# Check that each row has the same number of columns.
if len(set([len(x) for x in flines])) > 1:
print(set([len(x) for x in flines]))
raise ValueError(
'Number of columns in each row not '
'consistent!')
if len(cidict) and len(new_event_list):
msg = ('is_file_same' if
previous_file else 'is_event_same')
reps = [previous_file] if previous_file else [''.join(
new_event_list[-1].split('.')[:-1])]
text = prt.text(msg, reps)
is_same = prt.prompt(text, message=False,
kind='bool')
if not is_same:
cidict = OrderedDict()
# If the first row has no numbers it is likely a header.
if not len(cidict):
self.assign_columns(cidict, flines)
perms = 1
for key in cidict:
if isinstance(cidict[key], list) and not isinstance(
cidict[key], string_types):
if cidict[key][0] != 'j':
perms = len(cidict[key])
# Get event name (if single event) or list of names from
# table.
event_names = []
if ENTRY.NAME in cidict:
for fi, fl in enumerate(flines):
flines[fi][cidict[ENTRY.NAME]] = name_clean(
fl[cidict[ENTRY.NAME]])
event_names = list(sorted(set([
x[cidict[ENTRY.NAME]] for x in flines[
self._first_data:]])))
new_events = [x + '.json' for x in event_names]
else:
new_event_name = '.'.join(event.split(
'.')[:-1]).split('/')[-1]
text = prt.message(
'is_event_name', [new_event_name], prt=False)
is_name = prt.prompt(text, message=False,
kind='bool', default='y')
if not is_name:
new_event_name = ''
while new_event_name.strip() == '':
new_event_name = prt.prompt(
'enter_name', kind='string')
event_names.append(new_event_name)
new_events = [new_event_name + '.json']
# Create a new event, populate the photometry, and dump
# to a JSON file in the run directory.
entries = OrderedDict([(x, Entry(name=x))
for x in event_names])
# Clean up the data a bit now that we know the column
# identities.
# Strip common prefixes/suffixes from band names
if PHOTOMETRY.BAND in cidict:
bi = cidict[PHOTOMETRY.BAND]
for d in [True, False]:
if not isinstance(bi, (int, np.integer)):
break
strip_cols = []
lens = [len(x[bi])
for x in flines[self._first_data:]]
llen = min(lens)
ra = range(llen) if d else range(-1, -llen - 1, -1)
for li in ra:
letter = None
for row in list(flines[self._first_data:]):
if letter is None:
letter = row[bi][li]
elif row[bi][li] != letter:
letter = None
break
if letter is not None:
strip_cols.append(li)
else:
break
if len(strip_cols) == llen:
break
for ri in range(len(flines[self._first_data:])):
flines[self._first_data + ri][bi] = ''.join(
[c for i, c in enumerate(flines[
self._first_data + ri][bi])
if (i if d else i - len(flines[
self._first_data + ri][bi])) not in
strip_cols])
if (PHOTOMETRY.TIME in cidict and
(not isinstance(cidict[PHOTOMETRY.TIME], list) or
len(cidict[PHOTOMETRY.TIME]) <= 2)):
bi = cidict[PHOTOMETRY.TIME]
if isinstance(bi, list) and not isinstance(
bi, string_types) and isinstance(
bi[0], string_types) and bi[0] == 'jd':
bi = bi[-1]
mmtimes = [float(x[bi])
for x in flines[self._first_data:]]
mintime, maxtime = min(mmtimes), max(mmtimes)
if mintime < 10000:
while True:
try:
response = prt.prompt(
'small_time_offset', kind='string')
if response is not None:
toffset = Decimal(response)
break
except Exception:
pass
elif maxtime > 60000 and cidict[
PHOTOMETRY.TIME][0] != 'jd':
isjd = prt.prompt(
'large_time_offset',
kind='bool', default='y')
if isjd:
toffset = Decimal('-2400000.5')
for row in flines[self._first_data:]:
photodict = {}
rname = (row[cidict[ENTRY.NAME]]
if ENTRY.NAME in cidict else event_names[0])
for pi in range(perms):
sources = set()
for key in cidict:
if key in self._bool_keys:
rval = row[cidict[key]]
if rval in self._FALSE_VALS:
rval = False
elif rval in self._TRUE_VALS:
rval = True
if type(rval) != 'bool':
try:
rval = bool(rval)
except Exception:
pass
if type(rval) != 'bool':
try:
rval = bool(float(rval))
except Exception:
rval = True
if not rval:
continue
row[cidict[key]] = rval
elif key == 'reference':
if (isinstance(cidict[key],
string_types) and
len(cidict[key]) == 19):
new_src = entries[rname].add_source(
bibcode=cidict[key])
sources.update(new_src)
row[
cidict[key]] = new_src
elif key == ENTRY.NAME:
continue
elif (isinstance(key, Key) and
key.type == KEY_TYPES.TIME and
isinstance(cidict[key], list) and not
isinstance(cidict[key],
string_types)):
tval = np.array(row)[np.array(cidict[key][
1:], dtype=int)]
if cidict[key][0] == 'j':
date = '-'.join([x.zfill(2) for x in
tval])
date = self._month_rep.sub(
lambda x: self._MONTH_IDS[
x.group()], date)
photodict[key] = str(
astrotime(date, format='isot').mjd)
elif cidict[key][0] == 'jd':
photodict[key] = str(
jd_to_mjd(Decimal(tval[-1])))
continue
val = cidict[key]
if (isinstance(val, list) and not
isinstance(val, string_types)):
val = val[pi]
if isinstance(val, string_types):
if val != '':
photodict[key] = val
else:
photodict[key] = row[val]
else:
if isinstance(val, string_types):
if val != '':
photodict[key] = val
else:
photodict[key] = row[val]
if self._data_type == 2:
if self._zp:
photodict[PHOTOMETRY.ZERO_POINT] = self._zp
else:
photodict[PHOTOMETRY.ZERO_POINT] = (
row[cidict[PHOTOMETRY.ZERO_POINT][pi]]
if isinstance(cidict[
PHOTOMETRY.ZERO_POINT], list) else
row[cidict[PHOTOMETRY.ZERO_POINT]])
zpp = photodict[PHOTOMETRY.ZERO_POINT]
cc = (
row[cidict[PHOTOMETRY.COUNT_RATE][pi]] if
isinstance(cidict[
PHOTOMETRY.COUNT_RATE], list) else
row[cidict[PHOTOMETRY.COUNT_RATE]])
ecc = (
row[cidict[PHOTOMETRY.E_COUNT_RATE][pi]] if
isinstance(cidict[
PHOTOMETRY.E_COUNT_RATE], list) else
row[cidict[PHOTOMETRY.E_COUNT_RATE]])
if '<' in cc:
set_pd_mag_from_counts(
photodict, ec=cc.strip('<'), zp=zpp)
else:
set_pd_mag_from_counts(
photodict, c=cc, ec=ecc, zp=zpp)
elif self._data_type == 3:
photodict[
PHOTOMETRY.U_FLUX_DENSITY] = self._ufd
if PHOTOMETRY.U_FLUX_DENSITY in cidict:
photodict[PHOTOMETRY.U_FLUX_DENSITY] = (
row[cidict[
PHOTOMETRY.U_FLUX_DENSITY][pi]]
if isinstance(cidict[
PHOTOMETRY.
U_FLUX_DENSITY], list) else
row[cidict[PHOTOMETRY.U_FLUX_DENSITY]])
if photodict[
PHOTOMETRY.U_FLUX_DENSITY] == '':
photodict[
PHOTOMETRY.U_FLUX_DENSITY] = 'µJy'
fd = (
row[cidict[PHOTOMETRY.FLUX_DENSITY][pi]] if
isinstance(cidict[
PHOTOMETRY.FLUX_DENSITY], list) else
row[cidict[PHOTOMETRY.FLUX_DENSITY]])
efd = (
row[cidict[
PHOTOMETRY.E_FLUX_DENSITY][pi]] if
isinstance(cidict[
PHOTOMETRY.E_FLUX_DENSITY], list) else
row[cidict[PHOTOMETRY.E_FLUX_DENSITY]])
mult = Decimal('1')
ufd = photodict[PHOTOMETRY.U_FLUX_DENSITY]
if ufd.lower() in [
'mjy', 'millijy', 'millijansky']:
mult = Decimal('1e3')
elif ufd.lower() in ['jy', 'jansky']:
mult = Decimal('1e6')
if '<' in fd:
set_pd_mag_from_flux_density(
photodict, efd=str(
Decimal(fd.strip('<')) * mult))
else:
set_pd_mag_from_flux_density(
photodict, fd=Decimal(fd) * mult,
efd=Decimal(efd) * mult)
if not len(sources):
if use_self_source is None:
sopts = [
('Bibcode', 'b'), ('Last name', 'l')]
if self._require_source:
sel_str = 'must_select_source'
else:
sel_str = 'select_source'
text = prt.text(sel_str)
skind = prt.prompt(
text, kind='option',
options=sopts, default='b',
none_string=(
None if self._require_source else
'Neither, tag MOSFiT as source'))
if skind == 'b':
rsource = {}
bibcode = ''
while len(bibcode) != 19:
bibcode = prt.prompt(
'bibcode',
kind='string',
allow_blank=False
)
bibcode = bibcode.strip()
if (re.search(
'[0-9]{4}..........[\.0-9]{4}'
'[A-Za-z]', bibcode)
is None):
bibcode = ''
rsource[
SOURCE.BIBCODE] = bibcode
use_self_source = False
elif skind == 'l':
rsource = {}
last_name = prt.prompt(
'last_name', kind='string'
)
rsource[
SOURCE.NAME] = (
last_name.strip().title() +
' et al., in preparation')
use_self_source = False
elif skind == 'n':
use_self_source = True
photodict[
PHOTOMETRY.SOURCE] = entries[
rname].add_source(**rsource)
if any([x in photodict.get(
PHOTOMETRY.MAGNITUDE, '')
for x in ['<', '>']]):
photodict[PHOTOMETRY.UPPER_LIMIT] = True
photodict[
PHOTOMETRY.MAGNITUDE] = photodict[
PHOTOMETRY.MAGNITUDE].strip('<>')
if '<' in photodict.get(PHOTOMETRY.COUNT_RATE, ''):
photodict[PHOTOMETRY.UPPER_LIMIT] = True
photodict[
PHOTOMETRY.COUNT_RATE] = photodict[
PHOTOMETRY.COUNT_RATE].strip('<')
if PHOTOMETRY.E_COUNT_RATE in photodict:
del(photodict[PHOTOMETRY.E_COUNT_RATE])
if '<' in photodict.get(
PHOTOMETRY.FLUX_DENSITY, ''):
photodict[PHOTOMETRY.UPPER_LIMIT] = True
photodict[
PHOTOMETRY.FLUX_DENSITY] = photodict[
PHOTOMETRY.FLUX_DENSITY].strip('<')
if PHOTOMETRY.E_FLUX_DENSITY in photodict:
del(photodict[PHOTOMETRY.E_FLUX_DENSITY])
# Apply offset time if set.
if (PHOTOMETRY.TIME in photodict and
toffset != Decimal('0')):
photodict[PHOTOMETRY.TIME] = str(
Decimal(photodict[PHOTOMETRY.TIME]) +
toffset)
# Skip entries for which key values are not
# expected type.
if not all([
is_number(photodict.get(x, ''))
for x in photodict.keys() if
(PHOTOMETRY.get_key_by_name(x).type ==
KEY_TYPES.NUMERIC)]):
continue
# Skip placeholder values.
if float(photodict.get(
PHOTOMETRY.MAGNITUDE, 0.0)) > 50.0:
continue
# Add system if specified by user.
if (self._system is not None and
PHOTOMETRY.SYSTEM not in photodict):
photodict[PHOTOMETRY.SYSTEM] = self._system
# Remove keys not in the `PHOTOMETRY` class.
for key in list(photodict.keys()):
if key not in PHOTOMETRY.vals():
del(photodict[key])
# Add the photometry.
entries[rname].add_photometry(
**photodict)
merge_with_existing = None
for ei, entry in enumerate(entries):
entries[entry].sanitize()
if os.path.isfile(new_events[ei]):
if merge_with_existing is None:
merge_with_existing = prt.prompt(
'merge_with_existing', default='y')
if merge_with_existing:
existing = Entry.init_from_file(
catalog=None,
name=event_names[ei],
path=new_events[ei],
merge=False,
pop_schema=False,
ignore_keys=[ENTRY.MODELS],
compare_to_existing=False)
Catalog().copy_entry_to_entry(
existing, entries[entry])
oentry = entries[entry]._ordered(entries[entry])
entabbed_json_dump(
{entry: oentry}, open(new_events[ei], 'w'),
separators=(',', ':'))
self._converted.extend([
[event_names[x], new_events[x]]
for x in range(len(event_names))])
new_event_list.extend(new_events)
previous_file = event
else:
new_event_list.append(event)
return new_event_list
def do_lamost(catalog):
"""Import spectra from LAMOST."""
task_str = catalog.get_current_task_str()
# Set preferred names, calculate some columns based on imported data,
# sanitize some fields
keys = list(catalog.entries.keys())
viz = Vizier(columns=["**"])
fureps = {'erg/cm2/s/A': 'erg/s/cm^2/Angstrom'}
c_kms = con.c.cgs.value / 1.0e5
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
if (FASTSTARS.RA not in catalog.entries[oname]
or FASTSTARS.DEC not in catalog.entries[oname]):
continue
else:
result = viz.query_region(coord.SkyCoord(
ra=catalog.entries[oname][FASTSTARS.RA][0]['value'],
dec=catalog.entries[oname][FASTSTARS.DEC][0]['value'],
unit=(un.hourangle, un.deg),
frame='icrs'),
width="2s",
catalog="V/149/dr2")
if not result.keys():
continue
tab = result['V/149/dr2']
star = None
for row in tab:
if (row['objType'] == 'Star'
and row['Class'].lower() in ['star', 'unknown']):
star = row
break
if not star:
continue
try:
name, source = catalog.new_entry(oname,
bibcode='2016yCat.5149....0L',
srcname='LAMOST',
url='http://dr3.lamost.org/')
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
if row['SubClass'] is not 'Non':
#catalog.entries[name].add_quantity(
# FASTSTARS.SPECTRAL_TYPE, row['SubClass'], source=source)
ST, SCfull = row['SubClass'][:2], row['SubClass'][2:]
if len(SCfull) > 0:
if 'IV' in SCfull:
SC = 'sg'
elif 'III' in SCfull:
SC = 'g'
elif 'V' in SCfull:
SC = 'd'
elif 'I' in SCfull:
SC = 'Sg'
catalog.entries[name].add_quantity(FASTSTARS.STELLAR_CLASS,
SC,
source=source)
catalog.entries[name].add_quantity(FASTSTARS.SPECTRAL_TYPE,
ST,
source=source)
if row['z'] and is_number(row['z']):
catalog.entries[name].add_quantity(FASTSTARS.REDSHIFT,
str(row['z']),
e_value=str(row['e_z']),
source=source)
catalog.entries[name].add_quantity(
FASTSTARS.VELOCITY,
pretty_num(float(row['z']) * c_kms, sig=5),
e_value=pretty_num(float(row['e_z'] * c_kms), sig=5),
source=source)
mag_types = list(row['magType'].replace('psf_', ''))
nmt = []
nmi = 0
for mt in mag_types:
if is_number(mt):
nmt[nmi - 1] += mt
else:
nmt += mt
nmi += 1
mag_types = [
x.upper() if x in ['b', 'v', 'j', 'h'] else x for x in nmt
]
for mi, mt in enumerate(mag_types):
snrf = 'snr' + mt.lower()
if snrf in row.columns and float(row[snrf]) < 3:
continue
photodict = {
PHOTOMETRY.TIME: str(row['MJD']),
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.BAND: mt,
PHOTOMETRY.TELESCOPE: 'LAMOST',
PHOTOMETRY.MAGNITUDE: str(row['mag' + str(mi + 1)]),
PHOTOMETRY.SOURCE: source
}
if snrf in row.columns:
photodict[PHOTOMETRY.E_MAGNITUDE] = str(
Decimal('2.5') *
(Decimal('1') +
Decimal('1') / Decimal(str(row[snrf]))).log10())[:5]
catalog.entries[name].add_photometry(**photodict)
vname = row['PlanId']
ffile = ('spec-' + row['LMJD'] + '-' + vname + '_sp' +
row['spId'] + '-' + row['FiberId'] + '.fits.gz')
furl = 'http://dr3.lamost.org/sas/fits/' + vname + '/' + ffile
datafile = os.path.join(catalog.get_current_task_repo(), 'LAMOST',
ffile)
if not os.path.exists(datafile):
fr = requests.get(furl)
open(datafile, 'wb').write(fr.content)
hdulist = fits.open(datafile)
for oi, obj in enumerate(hdulist[0].header):
if any(x in ['.', '/'] for x in obj):
del (hdulist[0].header[oi])
hdulist[0].verify('silentfix')
hdrkeys = list(hdulist[0].header.keys())
# print(hdrkeys)
# for key in hdulist[0].header.keys():
# print(key, hdulist[0].header[key])
if hdulist[0].header['SIMPLE']:
if 'JD' in hdrkeys:
mjd = str(jd_to_mjd(Decimal(str(hdulist[0].header['JD']))))
elif 'MJD' in hdrkeys:
mjd = str(hdulist[0].header['MJD'])
elif 'DATE-OBS' in hdrkeys:
if 'T' in hdulist[0].header['DATE-OBS']:
dateobs = hdulist[0].header['DATE-OBS'].strip()
elif 'UTC-OBS' in hdrkeys:
dateobs = hdulist[0].header['DATE-OBS'].strip(
) + 'T' + hdulist[0].header['UTC-OBS'].strip()
mjd = str(astrotime(dateobs, format='isot').mjd)
else:
raise ValueError("Couldn't find JD/MJD for spectrum.")
if hdulist[0].header['NAXIS'] == 2:
waves = [str(x) for x in list(hdulist[0].data)[2]]
fluxes = [str(x) for x in list(hdulist[0].data)[0]]
else:
print('Warning: Skipping FITS spectrum `{}`.'.format(
datafile))
continue
else:
raise ValueError('Non-simple FITS import not yet supported.')
if 'BUNIT' in hdrkeys:
fluxunit = hdulist[0].header['BUNIT']
if fluxunit in fureps:
fluxunit = fureps[fluxunit]
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.WAVELENGTHS: waves,
SPECTRUM.TIME: mjd,
SPECTRUM.U_TIME: 'MJD',
SPECTRUM.FLUXES: fluxes,
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.FILENAME: ffile,
SPECTRUM.SOURCE: source
}
if 'TELESCOP' in hdrkeys:
specdict[SPECTRUM.TELESCOPE] = hdulist[0].header['TELESCOP']
if 'INSTRUME' in hdrkeys:
specdict[SPECTRUM.INSTRUMENT] = hdulist[0].header['INSTRUME']
if 'SITENAME' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['SITENAME']
elif 'OBSERVAT' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['OBSERVAT']
if 'OBSERVER' in hdrkeys:
specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER']
if 'AIRMASS' in hdrkeys:
specdict[SPECTRUM.AIRMASS] = hdulist[0].header['AIRMASS']
catalog.entries[name].add_spectrum(**specdict)
hdulist.close()
catalog.journal_entries()
return
def do_gaia(catalog):
"""Import from the GAIA alerts page."""
task_str = catalog.get_current_task_str()
fname = os.path.join(catalog.get_current_task_repo(), 'GAIA/alerts.csv')
csvtxt = catalog.load_url('http://gsaweb.ast.cam.ac.uk/alerts/alerts.csv',
fname)
if not csvtxt:
return
tsvin = list(
csv.reader(csvtxt.splitlines(), delimiter=',', skipinitialspace=True))
reference = 'Gaia Photometric Science Alerts'
refurl = 'http://gsaweb.ast.cam.ac.uk/alerts/alertsindex'
loopcnt = 0
for ri, row in enumerate(pbar(tsvin, task_str)):
if ri == 0 or not row:
continue
name = catalog.add_entry(row[0])
source = catalog.entries[name].add_source(name=reference, url=refurl)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
year = '20' + re.findall(r'\d+', row[0])[0]
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, year,
source)
catalog.entries[name].add_quantity(SUPERNOVA.RA,
row[2],
source,
u_value='floatdegrees')
catalog.entries[name].add_quantity(SUPERNOVA.DEC,
row[3],
source,
u_value='floatdegrees')
if row[7] and row[7] != 'unknown':
type = row[7].replace('SNe', '').replace('SN', '').strip()
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, type,
source)
elif any([
xx in row[9].upper()
for xx in ['SN CANDIATE', 'CANDIDATE SN', 'HOSTLESS SN']
]):
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE,
'Candidate', source)
if ('aka' in row[9].replace('gakaxy', 'galaxy').lower()
and 'AKARI' not in row[9]):
commentsplit = (row[9].replace('_', ' ').replace(
'MLS ', 'MLS').replace('CSS ', 'CSS').replace(
'SN iPTF', 'iPTF').replace('SN ',
'SN').replace('AT ', 'AT'))
commentsplit = commentsplit.split()
for csi, cs in enumerate(commentsplit):
if 'aka' in cs.lower() and csi < len(commentsplit) - 1:
alias = commentsplit[csi + 1].strip('(),:.').replace(
'PSNJ', 'PSN J')
if alias[:6] == 'ASASSN' and alias[6] != '-':
alias = 'ASASSN-' + alias[6:]
if alias.lower() != 'master':
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, alias, source)
break
fname = os.path.join(catalog.get_current_task_repo(),
'GAIA/') + row[0] + '.csv'
csvtxt = catalog.load_url(
'http://gsaweb.ast.cam.ac.uk/alerts/alert/' + row[0] +
'/lightcurve.csv', fname)
if csvtxt:
tsvin2 = csv.reader(csvtxt.splitlines())
for ri2, row2 in enumerate(tsvin2):
if ri2 <= 1 or not row2:
continue
mjd = str(jd_to_mjd(Decimal(row2[1].strip())))
magnitude = row2[2].strip()
if magnitude == 'null':
continue
e_mag = 0.
telescope = 'GAIA'
band = 'G'
catalog.entries[name].add_photometry(time=mjd,
u_time='MJD',
telescope=telescope,
band=band,
magnitude=magnitude,
e_magnitude=e_mag,
source=source)
if catalog.args.update:
catalog.journal_entries()
loopcnt = loopcnt + 1
if catalog.args.travis and loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0:
break
catalog.journal_entries()
return
def do_gaia(catalog):
"""Import from the GAIA alerts page."""
task_str = catalog.get_current_task_str()
fname = os.path.join(catalog.get_current_task_repo(), 'GAIA/alerts.csv')
csvtxt = catalog.load_url('http://gsaweb.ast.cam.ac.uk/alerts/alerts.csv',
fname)
if not csvtxt:
return
tsvin = list(
csv.reader(
csvtxt.splitlines(), delimiter=',', skipinitialspace=True))
reference = 'Gaia Photometric Science Alerts'
refurl = 'http://gsaweb.ast.cam.ac.uk/alerts/alertsindex'
loopcnt = 0
for ri, row in enumerate(pbar(tsvin, task_str)):
if ri == 0 or not row:
continue
name = catalog.add_entry(row[0])
source = catalog.entries[name].add_source(name=reference, url=refurl)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
year = '20' + re.findall(r'\d+', row[0])[0]
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, year,
source)
catalog.entries[name].add_quantity(
SUPERNOVA.RA, row[2], source, u_value='floatdegrees')
catalog.entries[name].add_quantity(
SUPERNOVA.DEC, row[3], source, u_value='floatdegrees')
if row[7] and row[7] != 'unknown':
type = row[7].replace('SNe', '').replace('SN', '').strip()
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, type,
source)
elif any([
xx in row[9].upper()
for xx in ['SN CANDIATE', 'CANDIDATE SN', 'HOSTLESS SN']
]):
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE,
'Candidate', source)
if ('aka' in row[9].replace('gakaxy', 'galaxy').lower() and
'AKARI' not in row[9]):
commentsplit = (row[9].replace('_', ' ').replace('MLS ', 'MLS')
.replace('CSS ', 'CSS').replace('SN iPTF', 'iPTF')
.replace('SN ', 'SN').replace('AT ', 'AT'))
commentsplit = commentsplit.split()
for csi, cs in enumerate(commentsplit):
if 'aka' in cs.lower() and csi < len(commentsplit) - 1:
alias = commentsplit[csi + 1].strip('(),:.').replace(
'PSNJ', 'PSN J')
if alias[:6] == 'ASASSN' and alias[6] != '-':
alias = 'ASASSN-' + alias[6:]
if alias.lower() != 'master':
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS,
alias, source)
break
fname = os.path.join(catalog.get_current_task_repo(),
'GAIA/') + row[0] + '.csv'
csvtxt = catalog.load_url('http://gsaweb.ast.cam.ac.uk/alerts/alert/' +
row[0] + '/lightcurve.csv', fname)
tsvin2 = csv.reader(csvtxt.splitlines())
for ri2, row2 in enumerate(tsvin2):
if ri2 <= 1 or not row2:
continue
mjd = str(jd_to_mjd(Decimal(row2[1].strip())))
magnitude = row2[2].strip()
if magnitude == 'null':
continue
e_mag = 0.
telescope = 'GAIA'
band = 'G'
catalog.entries[name].add_photometry(
time=mjd,
u_time='MJD',
telescope=telescope,
band=band,
magnitude=magnitude,
e_magnitude=e_mag,
source=source)
if catalog.args.update:
catalog.journal_entries()
loopcnt = loopcnt + 1
if catalog.args.travis and loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0:
break
catalog.journal_entries()
return
def do_snax(catalog):
"""Import from the SNaX X-ray database."""
task_str = catalog.get_current_task_str()
dlurl = 'http://kronos.uchicago.edu/snax/export.php?exportType=TSV&exportFields=standard&objid=&name=&typeid=&type=&galaxyid=&galaxy=&fluxMin=&fluxMax=&fluxEnergyLMin=&fluxEnergyLMax=&fluxEnergyHMin=&fluxEnergyHMax=&lumMin=&lumMax=&instrumentid=&instrument=&ageMin=&ageMax=&dateMin=&dateMax=&sortA=dateExploded' # noqa: E501
file_path = os.path.join(catalog.get_current_task_repo(), 'SNaX.TSV')
tsv = catalog.load_url(dlurl, file_path)
# csvtxt = catalog.load_url(
# 'http://www.grbcatalog.org/'
# 'download_data?cut_0_min=5&cut_0=BAT%20T90'
# '&cut_0_max=100000&num_cuts=1&no_date_cut=True',
# file_path)
data = [x.split('\t') for x in tsv.split('\n')]
for r, row in enumerate(pbar(data, task_str)):
if r == 0 or not row[0]:
continue
(name,
source) = catalog.new_entry(row[0],
srcname='SNaX',
url='http://kronos.uchicago.edu/snax/',
secondary=True)
sources = [source]
bibcode = row[-6].strip()
if len(bibcode) != 19:
continue
expsrc = uniq_cdl(
sources +
[catalog.entries[name].add_source(bibcode=row[-6].strip())])
coosrc = uniq_cdl(
sources +
[catalog.entries[name].add_source(bibcode=row[-5].strip())])
dissrc = uniq_cdl(
sources +
[catalog.entries[name].add_source(bibcode=row[-4].strip())])
flxsrc = uniq_cdl(sources + [
catalog.entries[name].add_source(
bibcode=row[-3].strip()), catalog.entries[name].add_source(
bibcode=row[-2].strip())
])
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, row[1],
source)
date = astrotime(float(row[2]), format='jd').datetime
catalog.entries[name].add_quantity(
SUPERNOVA.EXPLOSION_DATE,
make_date_string(date.year, date.month, date.day), expsrc)
catalog.entries[name].add_quantity(SUPERNOVA.RA,
' '.join(row[3].split()[:3]),
coosrc)
catalog.entries[name].add_quantity(SUPERNOVA.DEC,
' '.join(row[3].split()[3:]),
coosrc)
catalog.entries[name].add_quantity(SUPERNOVA.LUM_DIST, row[4], dissrc)
catalog.entries[name].add_quantity(SUPERNOVA.HOST, row[5], source)
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT,
row[6],
source,
e_value=row[7] if (row[7] and float(row[7]) != 0.0) else '')
photodict = {
PHOTOMETRY.TIME: jd_to_mjd(Decimal(row[8])),
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.ENERGY: row[15:17],
PHOTOMETRY.U_ENERGY: 'keV',
PHOTOMETRY.FLUX: str(Decimal('1.0e-13') * Decimal(row[11])),
PHOTOMETRY.U_FLUX: 'ergs/s/cm^2',
PHOTOMETRY.E_LOWER_FLUX:
str(Decimal('1.0e-13') * Decimal(row[13])),
PHOTOMETRY.E_UPPER_FLUX:
str(Decimal('1.0e-13') * Decimal(row[14])),
PHOTOMETRY.INSTRUMENT: row[9],
PHOTOMETRY.SOURCE: flxsrc
}
if row[12] == '1':
photodict[PHOTOMETRY.UPPER_LIMIT] = True
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
return
def do_sdss_spectra(catalog):
"""Import spectra from LAMOST."""
task_str = catalog.get_current_task_str()
# Set preferred names, calculate some columns based on imported data,
# sanitize some fields
keys = list(catalog.entries.keys())
fureps = {
'erg/cm2/s/A': 'erg/s/cm^2/Angstrom',
'1E-17 erg/cm^2/s/Ang': 'erg/s/cm^2/Angstrom'
}
c_kms = con.c.cgs.value / 1.0e5
cntsdss = 0
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
if (FASTSTARS.RA not in catalog.entries[oname]
or FASTSTARS.DEC not in catalog.entries[oname]):
continue
else:
xid = SDSS.query_region(coord.SkyCoord(
ra=catalog.entries[oname][FASTSTARS.RA][0]['value'],
dec=catalog.entries[oname][FASTSTARS.DEC][0]['value'],
unit=(un.hourangle, un.deg),
frame='icrs'),
spectro=True)
# xid = SDSS.query_region(coord.SkyCoord(
# ra='14:34:06.17',
# dec='+56:30:47.24',
# unit=(un.hourangle, un.deg), frame='icrs'), spectro=True)
if xid is None:
continue
while len(xid) > 1:
notstar = xid['z'].argmax()
xid.remove_row(notstar)
#print(xid)
# star = None
# for row in tab:
# if (row['objType'] == 'Star' and
# row['Class'].lower() in ['star', 'unknown']):
# star = row
# break
# if not star:
# continue
try:
name, source = catalog.new_entry(oname,
bibcode='2015ApJS..219...12A',
srcname='SDSS',
url='http://www.sdss.org/')
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
ffile = ('spec-' + str(xid['specobjid'][0]) + '.fits.gz')
# furl = 'http://dr3.lamost.org/sas/fits/' + vname + '/' + ffile
datafile = os.path.join(catalog.get_current_task_repo(), 'SDSS',
ffile)
if not os.path.exists(datafile):
# Download spectra
try:
sp = SDSS.get_spectra(matches=xid)[0]
except urllib.error.HTTPError:
catalog.log.warning(
'"{}" threw an HTTP 404, must be error upstream. Will likely go away on the next run.'
.format(oname))
continue
# Identify star
# assert sp[2].data['class'][0]=='STAR'
# Write spectra
# print(catalog.entries[oname][FASTSTARS.RA][0]['value'],catalog.entries[oname][FASTSTARS.DEC][0]['value'])
sp.writeto(datafile, overwrite=True)
# open(datafile, 'wb').write(fr.content)
# Open spectra
hdulist = fits.open(datafile)
# sp contains a list of fits datafiles, identify main one
i_primary = 0
i_coadd = 1
i_specobj = 2
assert hdulist[i_primary].name == 'PRIMARY'
assert hdulist[i_coadd].name == 'COADD'
assert (hdulist[i_specobj].name == 'SPECOBJ'
or hdulist[i_specobj].name == 'SPALL')
# xid = SDSS.query_region(coord.SkyCoord(
# ra='12:11:50.27',
# dec='+14:37:16.2',
# unit=(un.hourangle, un.deg), frame='icrs'), spectro=True)
# from SPECOBJ
# print('.'+hdulist[i_specobj].data['ELODIE_SPTYPE'][0]+'.')
if hdulist[i_specobj].data['ELODIE_SPTYPE'][0] != 'unknown':
ST, SCfull = hdulist[i_specobj].data['ELODIE_SPTYPE'][
0][:2], hdulist[i_specobj].data['ELODIE_SPTYPE'][0][2:]
if len(SCfull) > 0:
if 'IV' in SCfull:
SC = 'sg'
elif 'III' in SCfull:
SC = 'g'
elif 'V' in SCfull:
SC = 'd'
elif 'I' in SCfull:
SC = 'Sg'
else:
SC = False
if SC is not False:
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, SC, source=source)
catalog.entries[name].add_quantity(FASTSTARS.SPECTRAL_TYPE,
ST,
source=source)
if hdulist[i_specobj].data['Z'][0] != 0.0:
catalog.entries[name].add_quantity(
FASTSTARS.REDSHIFT,
str(hdulist[i_specobj].data['Z'][0]),
e_value=str(hdulist[i_specobj].data['Z_ERR'][0]),
source=source)
catalog.entries[name].add_quantity(
FASTSTARS.VELOCITY,
pretty_num(float(hdulist[i_specobj].data['Z'][0]) * c_kms,
sig=5),
e_value=pretty_num(float(
hdulist[i_specobj].data['Z_ERR'][0] * c_kms),
sig=5),
source=source)
for oi, obj in enumerate(hdulist[0].header):
if any(x in ['.', '/'] for x in obj):
del (hdulist[0].header[oi])
hdulist[0].verify('silentfix')
hdrkeys = list(hdulist[0].header.keys())
# print(hdrkeys)
# for key in hdulist[0].header.keys():
# print(key, hdulist[0].header[key])
if hdulist[0].header['SIMPLE']:
if 'JD' in hdrkeys:
mjd = str(jd_to_mjd(Decimal(str(hdulist[0].header['JD']))))
elif 'MJD' in hdrkeys:
mjd = str(hdulist[0].header['MJD'])
elif 'DATE-OBS' in hdrkeys:
if 'T' in hdulist[0].header['DATE-OBS']:
dateobs = hdulist[0].header['DATE-OBS'].strip()
elif 'UTC-OBS' in hdrkeys:
dateobs = hdulist[0].header['DATE-OBS'].strip(
) + 'T' + hdulist[0].header['UTC-OBS'].strip()
mjd = str(astrotime(dateobs, format='isot').mjd)
else:
raise ValueError("Couldn't find JD/MJD for spectrum.")
if hdulist[i_coadd].header['NAXIS'] == 2:
waves = [
str(x)
for x in list(10**hdulist[i_coadd].data['loglam'])
]
fluxes = [
str(x) for x in list(hdulist[i_coadd].data['flux'])
]
else:
print('Warning: Skipping FITS spectrum `{}`.'.format(
datafile))
continue
else:
raise ValueError('Non-simple FITS import not yet supported.')
if 'BUNIT' in hdrkeys:
fluxunit = hdulist[0].header['BUNIT']
if fluxunit in fureps:
fluxunit = fureps[fluxunit]
if fluxunit[:3] == '1E-17':
fluxes = [
str(x * 1e-17)
for x in list(hdulist[i_coadd].data['flux'])
]
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.WAVELENGTHS: waves,
SPECTRUM.TIME: mjd,
SPECTRUM.U_TIME: 'MJD',
SPECTRUM.FLUXES: fluxes,
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.FILENAME: ffile,
SPECTRUM.SOURCE: source
}
if 'TELESCOP' in hdrkeys:
specdict[SPECTRUM.TELESCOPE] = hdulist[0].header['TELESCOP']
if 'INSTRUME' in hdrkeys:
specdict[SPECTRUM.INSTRUMENT] = hdulist[0].header['INSTRUME']
if 'SITENAME' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['SITENAME']
elif 'OBSERVAT' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['OBSERVAT']
if 'OBSERVER' in hdrkeys:
specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER']
if 'AIRMASS' in hdrkeys:
specdict[SPECTRUM.AIRMASS] = hdulist[0].header['AIRMASS']
catalog.entries[name].add_spectrum(**specdict)
cntsdss += 1
hdulist.close()
catalog.journal_entries()
print('`{}` have SDSS spectra.'.format(cntsdss))
return
def do_ascii(catalog):
"""Process ASCII files that were extracted from datatables appearing in
published works.
"""
task_str = catalog.get_current_task_str()
# 2006ApJ...645..841N
file_path = os.path.join(
catalog.get_current_task_repo(), '2006ApJ...645..841N-table3.csv')
tsvin = list(csv.reader(open(file_path, 'r'), delimiter=','))
for ri, row in enumerate(pbar(tsvin, task_str)):
name = 'SNLS-' + row[0]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2006ApJ...645..841N')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, row[1], source, kind='spectroscopic')
astrot = astrotime(float(row[4]) + 2450000., format='jd').datetime
date_str = make_date_string(astrot.year, astrot.month, astrot.day)
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, date_str, source)
catalog.journal_entries()
# Anderson 2014
file_names = list(
glob(os.path.join(
catalog.get_current_task_repo(), 'SNII_anderson2014/*.dat')))
for datafile in pbar_strings(file_names, task_str):
basename = os.path.basename(datafile)
if not is_number(basename[:2]):
continue
if basename == '0210_V.dat':
name = 'SN0210'
else:
name = ('SN20' if int(basename[:2]) <
50 else 'SN19') + basename.split('_')[0]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2014ApJ...786...67A')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
if name in ['SN1999ca', 'SN2003dq', 'SN2008aw']:
system = 'Swope'
else:
system = 'Landolt'
with open(datafile, 'r') as ff:
tsvin = csv.reader(ff, delimiter=' ', skipinitialspace=True)
for row in tsvin:
if not row[0]:
continue
time = str(jd_to_mjd(Decimal(row[0])))
catalog.entries[name].add_photometry(
time=time, band='V',
magnitude=row[1], e_magnitude=row[2],
system=system, source=source)
catalog.journal_entries()
# stromlo
stromlobands = ['B', 'V', 'R', 'I', 'VM', 'RM']
file_path = os.path.join(
catalog.get_current_task_repo(), 'J_A+A_415_863-1/photometry.csv')
tsvin = list(csv.reader(open(file_path, 'r'), delimiter=','))
for row in pbar(tsvin, task_str):
name = row[0]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2004A&A...415..863G')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
mjd = str(jd_to_mjd(Decimal(row[1])))
for ri, ci in enumerate(range(2, len(row), 3)):
if not row[ci]:
continue
band = stromlobands[ri]
upperlimit = True if (not row[ci + 1] and row[ci + 2]) else False
e_upper_magnitude = str(
abs(Decimal(row[ci + 1]))) if row[ci + 1] else ''
e_lower_magnitude = str(
abs(Decimal(row[ci + 2]))) if row[ci + 2] else ''
teles = 'MSSSO 1.3m' if band in ['VM', 'RM'] else 'CTIO'
instr = 'MaCHO' if band in ['VM', 'RM'] else ''
catalog.entries[name].add_photometry(
time=mjd, band=band, magnitude=row[ci],
e_upper_magnitude=e_upper_magnitude,
e_lower_magnitude=e_lower_magnitude,
upperlimit=upperlimit, telescope=teles,
instrument=instr, source=source)
catalog.journal_entries()
# 2015MNRAS.449..451W
file_path = os.path.join(
catalog.get_current_task_repo(), '2015MNRAS.449..451W.dat')
data = list(csv.reader(open(file_path, 'r'), delimiter='\t',
quotechar='"', skipinitialspace=True))
for rr, row in enumerate(pbar(data, task_str)):
if rr == 0:
continue
namesplit = row[0].split('/')
name = namesplit[-1]
if name.startswith('SN'):
name = name.replace(' ', '')
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2015MNRAS.449..451W')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
if len(namesplit) > 1:
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, namesplit[0], source)
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, row[1], source)
catalog.entries[name].add_photometry(
time=row[2], band=row[4], magnitude=row[3], source=source)
catalog.journal_entries()
# 2016MNRAS.459.1039T
file_path = os.path.join(
catalog.get_current_task_repo(), '2016MNRAS.459.1039T.tsv')
data = list(csv.reader(open(file_path, 'r'), delimiter='\t',
quotechar='"', skipinitialspace=True))
name = catalog.add_entry('LSQ13zm')
source = catalog.entries[name].add_source(bibcode='2016MNRAS.459.1039T')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
for rr, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
bands = [xx.replace('(err)', '') for xx in row[3:-1]]
continue
mjd = row[1]
mags = [re.sub(r'\([^)]*\)', '', xx) for xx in row[3:-1]]
upps = [True if '>' in xx else '' for xx in mags]
mags = [xx.replace('>', '') for xx in mags]
errs = [xx[xx.find('(') + 1:xx.find(')')]
if '(' in xx else '' for xx in row[3:-1]]
for mi, mag in enumerate(mags):
if not is_number(mag):
continue
catalog.entries[name].add_photometry(
time=mjd, band=bands[mi], magnitude=mag, e_magnitude=errs[mi],
instrument=row[-1], upperlimit=upps[mi], source=source)
catalog.journal_entries()
# 2015ApJ...804...28G
file_path = os.path.join(
catalog.get_current_task_repo(), '2015ApJ...804...28G.tsv')
data = list(csv.reader(open(file_path, 'r'), delimiter='\t',
quotechar='"', skipinitialspace=True))
name = catalog.add_entry('PS1-13arp')
source = catalog.entries[name].add_source(bibcode='2015ApJ...804...28G')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
for rr, row in enumerate(pbar(data, task_str)):
if rr == 0:
continue
mjd = row[1]
mag = row[3]
upp = True if '<' in mag else ''
mag = mag.replace('<', '')
err = row[4] if is_number(row[4]) else ''
ins = row[5]
catalog.entries[name].add_photometry(
time=mjd, band=row[0], magnitude=mag, e_magnitude=err,
instrument=ins, upperlimit=upp, source=source)
catalog.journal_entries()
# 2016ApJ...819...35A
file_path = os.path.join(
catalog.get_current_task_repo(), '2016ApJ...819...35A.tsv')
data = list(csv.reader(open(file_path, 'r'), delimiter='\t',
quotechar='"', skipinitialspace=True))
for rr, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
continue
name = catalog.add_entry(row[0])
source = catalog.entries[name].add_source(
bibcode='2016ApJ...819...35A')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[1], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[2], source)
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, row[3], source)
disc_date = datetime.strptime(row[4], '%Y %b %d').isoformat()
disc_date = disc_date.split('T')[0].replace('-', '/')
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, disc_date, source)
catalog.journal_entries()
# 2014ApJ...784..105W
file_path = os.path.join(
catalog.get_current_task_repo(), '2014ApJ...784..105W.tsv')
data = list(csv.reader(open(file_path, 'r'), delimiter='\t',
quotechar='"', skipinitialspace=True))
for rr, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
continue
name = catalog.add_entry(row[0])
source = catalog.entries[name].add_source(
bibcode='2014ApJ...784..105W')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
mjd = row[1]
band = row[2]
mag = row[3]
err = row[4]
catalog.entries[name].add_photometry(
time=mjd, band=row[2], magnitude=mag, e_magnitude=err,
instrument='WHIRC', telescope='WIYN 3.5 m', observatory='NOAO',
system='WHIRC', source=source)
catalog.journal_entries()
# 2012MNRAS.425.1007B
file_path = os.path.join(
catalog.get_current_task_repo(), '2012MNRAS.425.1007B.tsv')
data = list(csv.reader(open(file_path, 'r'), delimiter='\t',
quotechar='"', skipinitialspace=True))
for rr, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
bands = row[2:]
continue
name = catalog.add_entry(row[0])
source = catalog.entries[name].add_source(
bibcode='2012MNRAS.425.1007B')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
mjd = row[1]
mags = [xx.split('±')[0].strip() for xx in row[2:]]
errs = [xx.split('±')[1].strip()
if '±' in xx else '' for xx in row[2:]]
if row[0] == 'PTF09dlc':
ins = 'HAWK-I'
tel = 'VLT 8.1m'
obs = 'ESO'
else:
ins = 'NIRI'
tel = 'Gemini North 8.2m'
obs = 'Gemini'
for mi, mag in enumerate(mags):
if not is_number(mag):
continue
catalog.entries[name].add_photometry(
time=mjd, band=bands[mi], magnitude=mag, e_magnitude=errs[mi],
instrument=ins, telescope=tel, observatory=obs,
system='Natural', source=source)
catalog.journal_entries()
# 2014ApJ...783...28G
file_path = os.path.join(
catalog.get_current_task_repo(), 'apj490105t2_ascii.txt')
with open(file_path, 'r') as f:
data = list(csv.reader(f, delimiter='\t',
quotechar='"', skipinitialspace=True))
for r, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
continue
name, source = catalog.new_entry(
row[0], bibcode='2014ApJ...783...28G')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[1], source)
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, '20' + row[0][3:5], source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[2], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[3], source)
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, row[13] if is_number(row[13]) else
row[10], source)
catalog.journal_entries()
# 2005ApJ...634.1190H
file_path = os.path.join(
catalog.get_current_task_repo(), '2005ApJ...634.1190H.tsv')
with open(file_path, 'r') as f:
data = list(csv.reader(f, delimiter='\t',
quotechar='"', skipinitialspace=True))
for r, row in enumerate(pbar(data, task_str)):
name, source = catalog.new_entry(
'SNLS-' + row[0], bibcode='2005ApJ...634.1190H')
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, '20' + row[0][:2], source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[1], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[2], source)
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, row[5].replace('?', ''), source,
e_value=row[6], kind='host')
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, row[7].replace('SN', '').strip(':* '),
source)
catalog.journal_entries()
# 2014MNRAS.444.2133S
file_path = os.path.join(
catalog.get_current_task_repo(), '2014MNRAS.444.2133S.tsv')
with open(file_path, 'r') as f:
data = list(csv.reader(f, delimiter='\t',
quotechar='"', skipinitialspace=True))
for r, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
continue
name = row[0]
if is_number(name[:4]):
name = 'SN' + name
name, source = catalog.new_entry(
name, bibcode='2014MNRAS.444.2133S')
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[1], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[2], source)
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, row[3],
source,
kind='host')
catalog.journal_entries()
# 2009MNRAS.398.1041B
file_path = os.path.join(
catalog.get_current_task_repo(), '2009MNRAS.398.1041B.tsv')
with open(file_path, 'r') as f:
data = list(csv.reader(f, delimiter='\t',
quotechar='"', skipinitialspace=True))
for r, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
bands = row[2:-1]
continue
name, source = catalog.new_entry(
'SN2008S', bibcode='2009MNRAS.398.1041B')
mjd = str(jd_to_mjd(Decimal(row[0])))
mags = [x.split('±')[0].strip() for x in row[2:]]
upps = [('<' in x.split('±')[0]) for x in row[2:]]
errs = [x.split('±')[1].strip()
if '±' in x else '' for x in row[2:]]
instrument = row[-1]
for mi, mag in enumerate(mags):
if not is_number(mag):
continue
catalog.entries[name].add_photometry(
time=mjd, band=bands[mi],
magnitude=mag, e_magnitude=errs[mi],
instrument=instrument, source=source)
catalog.journal_entries()
# 2010arXiv1007.0011P
file_path = os.path.join(
catalog.get_current_task_repo(), '2010arXiv1007.0011P.tsv')
with open(file_path, 'r') as f:
data = list(csv.reader(f, delimiter='\t',
quotechar='"', skipinitialspace=True))
for r, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
bands = row[1:]
continue
name, source = catalog.new_entry(
'SN2008S', bibcode='2010arXiv1007.0011P')
mjd = row[0]
mags = [x.split('±')[0].strip() for x in row[1:]]
errs = [x.split('±')[1].strip()
if '±' in x else '' for x in row[1:]]
for mi, mag in enumerate(mags):
if not is_number(mag):
continue
catalog.entries[name].add_photometry(
time=mjd, band=bands[mi],
magnitude=mag, e_magnitude=errs[mi],
instrument='LBT', source=source)
catalog.journal_entries()
# 2000ApJ...533..320G
file_path = os.path.join(
catalog.get_current_task_repo(), '2000ApJ...533..320G.tsv')
with open(file_path, 'r') as f:
data = list(csv.reader(f, delimiter='\t',
quotechar='"', skipinitialspace=True))
name, source = catalog.new_entry(
'SN1997cy', bibcode='2000ApJ...533..320G')
for r, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
bands = row[1:-1]
continue
mjd = str(jd_to_mjd(Decimal(row[0])))
mags = row[1:len(bands)]
for mi, mag in enumerate(mags):
if not is_number(mag):
continue
catalog.entries[name].add_photometry(
time=mjd, band=bands[mi],
magnitude=mag,
observatory='Mount Stromlo', telescope='MSSSO',
source=source, kcorrected=True)
catalog.journal_entries()
return
def do_external_fits_spectra(catalog):
fpath = catalog.get_current_task_repo()
with open(os.path.join(fpath, 'meta.json'), 'r') as f:
metadict = json.loads(f.read())
fureps = {'erg/cm2/s/A': 'erg/s/cm^2/Angstrom'}
task_str = catalog.get_current_task_str()
path_pattern = os.path.join(catalog.get_current_task_repo(), '*.fits')
files = glob(path_pattern)
for datafile in files:
filename = datafile.split('/')[-1]
if filename == 'meta.json':
continue
hdulist = fits.open(datafile)
for oi, obj in enumerate(hdulist[0].header):
if any(x in ['.', '/'] for x in obj):
del (hdulist[0].header[oi])
hdulist[0].verify('silentfix')
hdrkeys = list(hdulist[0].header.keys())
# print(hdrkeys)
name = ''
if filename in metadict:
if 'name' in metadict[filename]:
name = metadict[filename]['name']
if not name:
name = hdulist[0].header['OBJECT']
if 'bibcode' in metadict[filename]:
name, source = catalog.new_entry(
name, bibcode=metadict[filename]['bibcode'])
elif 'donator' in metadict[filename]:
name, source = catalog.new_entry(
name, srcname=metadict[filename]['donator'])
else:
if 'OBSERVER' in hdrkeys:
name, source = catalog.new_entry(
name, srcname=hdulist[0].header['OBSERVER'])
else:
name = catalog.add_entry(name)
source = catalog.entries[name].add_self_source()
# for key in hdulist[0].header.keys():
# print(key, hdulist[0].header[key])
if hdulist[0].header['SIMPLE']:
if 'JD' in hdrkeys:
mjd = str(jd_to_mjd(Decimal(str(hdulist[0].header['JD']))))
elif 'MJD' in hdrkeys:
mjd = str(hdulist[0].header['MJD'])
elif 'DATE-OBS' in hdrkeys:
if 'T' in hdulist[0].header['DATE-OBS']:
dateobs = hdulist[0].header['DATE-OBS'].strip()
elif 'UTC-OBS' in hdrkeys:
dateobs = hdulist[0].header['DATE-OBS'].strip(
) + 'T' + hdulist[0].header['UTC-OBS'].strip()
mjd = str(astrotime(dateobs, format='isot').mjd)
else:
raise ValueError("Couldn't find JD/MJD for spectrum.")
w0 = hdulist[0].header['CRVAL1']
if hdulist[0].header['NAXIS'] == 1:
wd = hdulist[0].header['CDELT1']
fluxes = [str(x) for x in list(hdulist[0].data)]
errors = False
elif hdulist[0].header['NAXIS'] == 3:
wd = hdulist[0].header['CD1_1']
fluxes = [str(x) for x in list(hdulist[0].data)[0][0]]
errors = [str(x) for x in list(hdulist[0].data)[3][0]]
else:
print('Warning: Skipping FITS spectrum `{}`.'.format(filename))
continue
waves = [str(w0 + wd * x) for x in range(0, len(fluxes))]
else:
raise ValueError('Non-simple FITS import not yet supported.')
airmass = hdulist[0].header['AIRMASS']
if 'BUNIT' in hdrkeys:
fluxunit = hdulist[0].header['BUNIT']
if fluxunit in fureps:
fluxunit = fureps[fluxunit]
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.WAVELENGTHS: waves,
SPECTRUM.TIME: mjd,
SPECTRUM.U_TIME: 'MJD',
SPECTRUM.FLUXES: fluxes,
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.AIRMASS: airmass,
SPECTRUM.FILENAME: filename,
SPECTRUM.SOURCE: source
}
if 'TELESCOP' in hdrkeys:
specdict[SPECTRUM.TELESCOPE] = hdulist[0].header['TELESCOP']
if 'INSTRUME' in hdrkeys:
specdict[SPECTRUM.INSTRUMENT] = hdulist[0].header['INSTRUME']
if errors:
specdict[SPECTRUM.ERRORS] = errors
specdict[SPECTRUM.U_ERRORS] = fluxunit
if 'SITENAME' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['SITENAME']
elif 'OBSERVAT' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['OBSERVAT']
if 'OBSERVER' in hdrkeys:
specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER']
catalog.entries[name].add_spectrum(**specdict)
hdulist.close()
catalog.journal_entries()
return
def do_gaia(catalog):
task_str = catalog.get_current_task_str()
fname = os.path.join(catalog.get_current_task_repo(), 'GAIA/alerts.csv')
csvtxt = catalog.load_cached_url(
'http://gsaweb.ast.cam.ac.uk/alerts/alerts.csv', fname)
if not csvtxt:
return
tsvin = list(csv.reader(csvtxt.splitlines(),
delimiter=',', skipinitialspace=True))
reference = 'Gaia Photometric Science Alerts'
refurl = 'http://gsaweb.ast.cam.ac.uk/alerts/alertsindex'
for ri, row in enumerate(pbar(tsvin, task_str)):
if ri == 0 or not row:
continue
name = catalog.add_entry(row[0])
source = catalog.entries[name].add_source(
name=reference, url=refurl)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
year = '20' + re.findall(r'\d+', row[0])[0]
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, year, source)
catalog.entries[name].add_quantity(
SUPERNOVA.RA, row[2], source, u_value='floatdegrees')
catalog.entries[name].add_quantity(
SUPERNOVA.DEC, row[3], source, u_value='floatdegrees')
if row[7] and row[7] != 'unknown':
type = row[7].replace('SNe', '').replace('SN', '').strip()
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, type, source)
elif any([xx in row[9].upper() for xx in
['SN CANDIATE', 'CANDIDATE SN', 'HOSTLESS SN']]):
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, 'Candidate', source)
if ('aka' in row[9].replace('gakaxy', 'galaxy').lower() and
'AKARI' not in row[9]):
commentsplit = (row[9]
.replace('_', ' ')
.replace('MLS ', 'MLS')
.replace('CSS ', 'CSS')
.replace('SN iPTF', 'iPTF')
.replace('SN ', 'SN')
.replace('AT ', 'AT'))
commentsplit = commentsplit.split()
for csi, cs in enumerate(commentsplit):
if 'aka' in cs.lower() and csi < len(commentsplit) - 1:
alias = commentsplit[
csi + 1].strip('(),:.').replace('PSNJ', 'PSN J')
if alias[:6] == 'ASASSN' and alias[6] != '-':
alias = 'ASASSN-' + alias[6:]
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, alias, source)
break
fname = os.path.join(catalog.get_current_task_repo(),
'GAIA/') + row[0] + '.csv'
if (catalog.current_task.load_archive(catalog.args) and
os.path.isfile(fname)):
with open(fname, 'r') as ff:
csvtxt = ff.read()
else:
response = urllib.request.urlopen('http://gsaweb.ast.cam.ac.uk/'
'alerts/alert/' +
row[0] + '/lightcurve.csv')
with open(fname, 'w') as ff:
csvtxt = response.read().decode('utf-8')
ff.write(csvtxt)
tsvin2 = csv.reader(csvtxt.splitlines())
for ri2, row2 in enumerate(tsvin2):
if ri2 <= 1 or not row2:
continue
mjd = str(jd_to_mjd(Decimal(row2[1].strip())))
magnitude = row2[2].strip()
if magnitude == 'null':
continue
e_mag = 0.
telescope = 'GAIA'
band = 'G'
catalog.entries[name].add_photometry(
time=mjd, telescope=telescope, band=band, magnitude=magnitude,
e_magnitude=e_mag, source=source)
if catalog.args.update:
catalog.journal_entries()
catalog.journal_entries()
return
def do_suspect_photo(catalog):
task_str = catalog.get_current_task_str()
with open(os.path.join(catalog.get_current_task_repo(),
'suspectreferences.csv'), 'r') as f:
tsvin = csv.reader(f, delimiter=',', skipinitialspace=True)
suspectrefdict = {}
for row in tsvin:
suspectrefdict[row[0]] = row[1]
file_names = list(sorted(glob(os.path.join(
catalog.get_current_task_repo(), 'SUSPECT/*.html'))))
for datafile in pbar_strings(file_names, task_str):
basename = os.path.basename(datafile)
basesplit = basename.split('-')
oldname = basesplit[1]
name = catalog.add_entry(oldname)
if name.startswith('SN') and is_number(name[2:]):
name = name + 'A'
band = basesplit[3].split('.')[0]
ei = int(basesplit[2])
bandlink = 'file://' + os.path.abspath(datafile)
bandresp = urllib.request.urlopen(bandlink)
bandsoup = BeautifulSoup(bandresp, 'html5lib')
bandtable = bandsoup.find('table')
names = bandsoup.body.findAll(text=re.compile('Name'))
reference = ''
for link in bandsoup.body.findAll('a'):
if 'adsabs' in link['href']:
reference = str(link).replace('"', "'")
bibcode = unescape(suspectrefdict[reference])
source = catalog.entries[name].add_source(bibcode=bibcode)
sec_ref = 'SUSPECT'
sec_refurl = 'https://www.nhn.ou.edu/~suspect/'
sec_source = catalog.entries[name].add_source(
name=sec_ref, url=sec_refurl, secondary=True)
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, oldname, sec_source)
if ei == 1:
year = re.findall(r'\d+', name)[0]
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, year, sec_source)
catalog.entries[name].add_quantity(
SUPERNOVA.HOST, names[1].split(':')[1].strip(), sec_source)
redshifts = bandsoup.body.findAll(text=re.compile('Redshift'))
if redshifts:
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, redshifts[0].split(':')[1].strip(),
sec_source, kind='heliocentric')
# hvels = bandsoup.body.findAll(text=re.compile('Heliocentric
# Velocity'))
# if hvels:
# vel = hvels[0].split(':')[1].strip().split(' ')[0]
# catalog.entries[name].add_quantity(SUPERNOVA.VELOCITY, vel,
# sec_source,
# kind='heliocentric')
types = bandsoup.body.findAll(text=re.compile('Type'))
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, types[0].split(
':')[1].strip().split(' ')[0],
sec_source)
for r, row in enumerate(bandtable.findAll('tr')):
if r == 0:
continue
col = row.findAll('td')
mjd = str(jd_to_mjd(Decimal(col[0].contents[0])))
mag = col[3].contents[0]
if mag.isspace():
mag = ''
else:
mag = str(mag)
e_magnitude = col[4].contents[0]
if e_magnitude.isspace():
e_magnitude = ''
else:
e_magnitude = str(e_magnitude)
catalog.entries[name].add_photometry(
time=mjd, band=band, magnitude=mag, e_magnitude=e_magnitude,
source=sec_source + ',' + source)
catalog.journal_entries()
return
def do_tns_spectra(catalog):
"""Load TNS spectra."""
requests.packages.urllib3.disable_warnings()
task_str = catalog.get_current_task_str()
tns_url = 'https://wis-tns.weizmann.ac.il/'
try:
with open('tns.key', 'r') as f:
tnskey = f.read().splitlines()[0]
except Exception:
catalog.log.warning('TNS API key not found, make sure a file named '
'`tns.key` containing the key is placed the '
'astrocats directory.')
tnskey = ''
fails = 0
for name in pbar(list(catalog.entries.keys()), task_str):
if name not in catalog.entries:
continue
aliases = catalog.entries[name].get_aliases()
oname = ''
for alias in aliases:
if (alias.startswith(('SN', 'AT')) and is_integer(alias[2:6]) and
int(alias[2:6]) >= 2016) and alias[6:].isalpha():
oname = alias
break
if not oname:
continue
reqname = oname[2:]
jsonpath = os.path.join(catalog.get_current_task_repo(), 'TNS', 'meta',
reqname + '.json')
download_json = True
if os.path.isfile(jsonpath):
with open(jsonpath, 'r') as f:
objdict = json.load(f)
if ('discoverydate' in objdict and
(datetime.now() - datetime.strptime(objdict['discoverydate'],
'%Y-%m-%d %H:%M:%S')
).days > 90):
download_json = False
if download_json:
data = urllib.parse.urlencode({
'api_key': tnskey,
'data': json.dumps({
'objname': reqname,
'spectra': '1'
})
}).encode('ascii')
req = urllib.request.Request(
'https://wis-tns.weizmann.ac.il/api/get/object', data=data)
trys = 0
objdict = None
while trys < 3 and not objdict:
try:
objdict = json.loads(
urllib.request.urlopen(req, timeout=30).read().decode('ascii'))[
'data']['reply']
except KeyboardInterrupt:
raise
except Exception:
catalog.log.warning('API request failed for `{}`.'.format(
name))
time.sleep(5)
trys = trys + 1
if (not objdict or 'objname' not in objdict or
not isinstance(objdict['objname'], str)):
fails = fails + 1
catalog.log.warning('Object `{}` not found!'.format(name))
if fails >= 5:
break
continue
# Cache object here
with open(jsonpath, 'w') as f:
json.dump(sortOD(objdict), f, indent='\t',
separators=(',', ':'), ensure_ascii=False,
sort_keys=True)
if 'spectra' not in objdict:
continue
specarr = objdict['spectra']
name, source = catalog.new_entry(
oname, srcname='Transient Name Server', url=tns_url)
for spectrum in specarr:
spectrumdict = {
PHOTOMETRY.SOURCE: source
}
if 'jd' in spectrum:
spectrumdict[SPECTRUM.TIME] = str(
jd_to_mjd(Decimal(str(spectrum['jd']))))
spectrumdict[SPECTRUM.U_TIME] = 'MJD'
if spectrum.get('observer', ''):
spectrumdict[SPECTRUM.OBSERVER] = spectrum['observer']
if spectrum.get('reducer', ''):
spectrumdict[SPECTRUM.OBSERVER] = spectrum['observer']
if 'source_group' in spectrum:
survey = spectrum['source_group']['name']
if survey:
spectrumdict[SPECTRUM.SURVEY] = survey
if 'telescope' in spectrum:
telescope = spectrum['telescope']['name']
if telescope and telescope != 'Other':
spectrumdict[SPECTRUM.TELESCOPE] = telescope
if 'instrument' in spectrum:
instrument = spectrum['instrument']['name']
if instrument and instrument != 'Other':
spectrumdict[SPECTRUM.INSTRUMENT] = instrument
if 'asciifile' in spectrum:
fname = urllib.parse.unquote(
spectrum['asciifile'].split('/')[-1])
spectxt = catalog.load_url(
spectrum['asciifile'],
os.path.join(
catalog.get_current_task_repo(), 'TNS', 'spectra',
fname), archived_mode=True)
data = [x.split() for x in spectxt.splitlines()]
skipspec = False
newdata = []
oldval = ''
for row in data:
if row and '#' not in row[0]:
if (len(row) >= 2 and is_number(row[0]) and
is_number(row[1]) and row[1] != oldval):
newdata.append(row)
oldval = row[1]
if skipspec or not newdata:
warnings.warn('Skipped adding spectrum file ' + fname)
continue
data = [list(i) for i in zip(*newdata)]
wavelengths = data[0]
fluxes = data[1]
errors = ''
if len(data) == 3:
errors = data[1]
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
spectrumdict.update({
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.ERRORS: errors,
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.U_ERRORS: fluxunit if errors else '',
SPECTRUM.WAVELENGTHS: wavelengths,
SPECTRUM.FLUXES: fluxes
})
catalog.entries[name].add_spectrum(**spectrumdict)
catalog.journal_entries()
return
def set_first_max_light(self):
if SUPERNOVA.MAX_APP_MAG not in self:
# Get the maximum amongst all bands
mldt, mlmag, mlband, mlsource = self._get_max_light()
if mldt or mlmag or mlband:
source = self.add_self_source()
uniq_src = uniq_cdl([source] + mlsource.split(','))
if mldt:
max_date = make_date_string(mldt.year, mldt.month, mldt.day)
self.add_quantity(SUPERNOVA.MAX_DATE,
max_date,
uniq_src,
derived=True)
if mlmag:
mlmag = pretty_num(mlmag)
self.add_quantity(SUPERNOVA.MAX_APP_MAG,
mlmag,
uniq_src,
derived=True)
if mlband:
self.add_quantity(SUPERNOVA.MAX_BAND,
mlband,
uniq_src,
derived=True)
if SUPERNOVA.MAX_VISUAL_APP_MAG not in self:
# Get the "visual" maximum
mldt, mlmag, mlband, mlsource = self._get_max_light(visual=True)
if mldt or mlmag or mlband:
source = self.add_self_source()
uniq_src = uniq_cdl([source] + mlsource.split(','))
if mldt:
max_date = make_date_string(mldt.year, mldt.month, mldt.day)
self.add_quantity(SUPERNOVA.MAX_VISUAL_DATE,
max_date,
uniq_src,
derived=True)
if mlmag:
mlmag = pretty_num(mlmag)
self.add_quantity(SUPERNOVA.MAX_VISUAL_APP_MAG,
mlmag,
uniq_src,
derived=True)
if mlband:
self.add_quantity(SUPERNOVA.MAX_VISUAL_BAND,
mlband,
uniq_src,
derived=True)
if (self._KEYS.DISCOVER_DATE not in self or max([
len(x[QUANTITY.VALUE].split('/'))
for x in self[self._KEYS.DISCOVER_DATE]
]) < 3):
fldt, flsource = self._get_first_light()
if fldt:
source = self.add_self_source()
disc_date = make_date_string(fldt.year, fldt.month, fldt.day)
self.add_quantity(self._KEYS.DISCOVER_DATE,
disc_date,
uniq_cdl([source] + flsource.split(',')),
derived=True)
if self._KEYS.DISCOVER_DATE not in self and self._KEYS.SPECTRA in self:
minspecmjd = float("+inf")
for spectrum in self[self._KEYS.SPECTRA]:
if 'time' in spectrum and 'u_time' in spectrum:
if spectrum['u_time'] == 'MJD':
mjd = float(spectrum['time'])
elif spectrum['u_time'] == 'JD':
mjd = float(jd_to_mjd(Decimal(spectrum['time'])))
else:
continue
if mjd < minspecmjd:
minspecmjd = mjd
minspecsource = spectrum['source']
if minspecmjd < float("+inf"):
fldt = astrotime(minspecmjd, format='mjd').datetime
source = self.add_self_source()
disc_date = make_date_string(fldt.year, fldt.month, fldt.day)
self.add_quantity(self._KEYS.DISCOVER_DATE,
disc_date,
uniq_cdl([source] +
minspecsource.split(',')),
derived=True)
return
def clean_internal(self, data):
"""Clean input data from the 'Supernovae/input/internal' repository.
FIX: instead of making changes in place to `dirty_event`, should a new
event be created, values filled, then returned??
FIX: currently will fail if no bibcode and no url
"""
self._log.debug("clean_internal(): {}".format(self.name()))
bibcodes = []
# Remove 'names' when 'bibcodes' are given
for ss, source in enumerate(data.get(self._KEYS.SOURCES, [])):
if SOURCE.BIBCODE in source:
bibcodes.append(source[SOURCE.BIBCODE])
# If there are no existing sources, add OSC as one
if len(bibcodes) == 0:
self.add_self_source()
bibcodes = [self.catalog.OSC_BIBCODE]
# Clean some legacy fields
alias_key = 'aliases'
if alias_key in data:
# Remove the entry in the data
aliases = data.pop(alias_key)
# Make sure this is a list
if not isinstance(aliases, list):
raise ValueError("{}: aliases not a list '{}'".format(
self.name(), aliases))
# Add OSC source entry
source = self.add_self_source()
for alias in aliases:
self.add_quantity(self._KEYS.ALIAS, alias, source)
dist_key = 'distinctfrom'
if dist_key in data:
distincts = data.pop(dist_key)
if ((isinstance(distincts, list) and
isinstance(distincts[0], str))):
source = self.add_self_source()
for df in distincts:
self.add_quantity(dist_key, df, source)
# Go through all remaining keys in 'dirty' event, and make sure
# everything is a quantity with a source (OSC if no other)
for key in data.keys():
# The following line should be used to replace the above once keys
# returns the superclass keys too
if self._KEYS.get_key_by_name(key).no_source:
pass
elif key == self._KEYS.PHOTOMETRY:
for p, photo in enumerate(data[self._KEYS.PHOTOMETRY]):
if photo[PHOTOMETRY.U_TIME] == 'JD':
data[self._KEYS.PHOTOMETRY][p][
PHOTOMETRY.U_TIME] = 'MJD'
data[self._KEYS.PHOTOMETRY][p][
PHOTOMETRY.TIME] = str(
jd_to_mjd(Decimal(photo['time'])))
if QUANTITY.SOURCE not in photo:
source = self.add_source(bibcode=bibcodes[0])
data[self._KEYS.PHOTOMETRY][p][
QUANTITY.SOURCE] = source
else:
for qi, quantity in enumerate(data[key]):
if QUANTITY.SOURCE not in quantity:
source = self.add_source(bibcode=bibcodes[0])
data[key][qi][QUANTITY.SOURCE] = source
return data
def do_snax(catalog):
task_str = catalog.get_current_task_str()
file_path = os.path.join(catalog.get_current_task_repo(), 'SNaX.TSV')
# csvtxt = catalog.load_url(
# 'http://www.grbcatalog.org/'
# 'download_data?cut_0_min=5&cut_0=BAT%20T90'
# '&cut_0_max=100000&num_cuts=1&no_date_cut=True',
# file_path)
data = list(
csv.reader(
open(file_path, 'r'),
delimiter='\t',
quotechar='"',
skipinitialspace=True))
for r, row in enumerate(pbar(data, task_str)):
if r == 0:
continue
(name, source) = catalog.new_entry(
row[0], srcname='SNaX', url='http://kronos.uchicago.edu/snax/')
sources = [source]
expsrc = uniq_cdl(sources + [
catalog.entries[name].add_source(bibcode=row[-6].strip())
])
coosrc = uniq_cdl(sources + [
catalog.entries[name].add_source(bibcode=row[-5].strip())
])
dissrc = uniq_cdl(sources + [
catalog.entries[name].add_source(bibcode=row[-4].strip())
])
flxsrc = uniq_cdl(sources + [
catalog.entries[name].add_source(bibcode=row[-3].strip()),
catalog.entries[name].add_source(bibcode=row[-2].strip())
])
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, row[1],
source)
date = astrotime(float(row[2]), format='jd').datetime
catalog.entries[name].add_quantity(
SUPERNOVA.EXPLOSION_DATE,
make_date_string(date.year, date.month, date.day), expsrc)
catalog.entries[name].add_quantity(
SUPERNOVA.RA, ' '.join(row[3].split()[:3]), coosrc)
catalog.entries[name].add_quantity(
SUPERNOVA.DEC, ' '.join(row[3].split()[:3]), coosrc)
catalog.entries[name].add_quantity(SUPERNOVA.LUM_DIST, row[4], dissrc)
catalog.entries[name].add_quantity(SUPERNOVA.HOST, row[5], source)
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT,
row[6],
source,
e_value=row[7] if (row[7] and float(row[7]) != 0.0) else '')
photodict = {
PHOTOMETRY.TIME: jd_to_mjd(Decimal(row[8])),
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.ENERGY: row[15:17],
PHOTOMETRY.U_ENERGY: 'keV',
PHOTOMETRY.FLUX: str(Decimal('1.0e-13') * Decimal(row[11])),
PHOTOMETRY.U_FLUX: 'ergs/s/cm^2',
PHOTOMETRY.E_LOWER_FLUX:
str(Decimal('1.0e-13') * Decimal(row[13])),
PHOTOMETRY.E_UPPER_FLUX:
str(Decimal('1.0e-13') * Decimal(row[14])),
PHOTOMETRY.INSTRUMENT: row[9],
PHOTOMETRY.SOURCE: flxsrc
}
if row[12] == '1':
photodict[PHOTOMETRY.UPPER_LIMIT] = True
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
return
def clean_internal(self, data):
"""Clean input data from the 'Supernovae/input/internal' repository.
FIX: instead of making changes in place to `dirty_event`, should a new
event be created, values filled, then returned??
FIX: currently will fail if no bibcode and no url
"""
self._log.debug("clean_internal(): {}".format(self.name()))
def_source_dict = {}
# Find source that will be used as default
sources = data.get(self._KEYS.SOURCES, [])
if sources:
def_source_dict = sources[0]
allow_alias = False
if SOURCE.ALIAS in def_source_dict:
del (def_source_dict[SOURCE.ALIAS])
else:
# If there are no existing sources, add OSC as one
self.add_self_source()
sources = self.get(self._KEYS.SOURCES, [])
def_source_dict = sources[0]
allow_alias = True
# Clean some legacy fields
alias_key = 'aliases'
if alias_key in data:
# Remove the entry in the data
aliases = data.pop(alias_key)
# Make sure this is a list
if not isinstance(aliases, list):
raise ValueError("{}: aliases not a list '{}'".format(
self.name(), aliases))
# Add OSC source entry
source = self.add_self_source()
for alias in aliases:
self.add_quantity(self._KEYS.ALIAS, alias, source)
dist_key = 'distinctfrom'
if dist_key in data:
distincts = data.pop(dist_key)
if ((isinstance(distincts, list)
and isinstance(distincts[0], string_types))):
source = self.add_self_source()
for df in distincts:
self.add_quantity(self._KEYS.DISTINCT_FROM, df, source)
else:
data[dist_key] = list(distincts)
# Go through all remaining keys in 'dirty' event, and make sure
# everything is a quantity with a source (OSC if no other)
for key in data.keys():
# The following line should be used to replace the above once keys
# returns the superclass keys too
if self._KEYS.get_key_by_name(key).no_source:
pass
elif key == self._KEYS.PHOTOMETRY:
for p, photo in enumerate(data[self._KEYS.PHOTOMETRY]):
if photo[PHOTOMETRY.U_TIME] == 'JD':
data[self._KEYS.PHOTOMETRY][p][
PHOTOMETRY.U_TIME] = 'MJD'
data[self._KEYS.PHOTOMETRY][p][PHOTOMETRY.TIME] = str(
jd_to_mjd(Decimal(photo['time'])))
if QUANTITY.SOURCE not in photo:
if not def_source_dict:
raise ValueError("No sources found, can't add "
"photometry.")
source = self.add_source(allow_alias=allow_alias,
**def_source_dict)
data[self._KEYS.PHOTOMETRY][p][
QUANTITY.SOURCE] = source
else:
for qi, quantity in enumerate(data[key]):
if QUANTITY.SOURCE not in quantity:
if not def_source_dict:
raise ValueError("No sources found, can't add "
"quantity.")
source = self.add_source(allow_alias=allow_alias,
**def_source_dict)
data[key][qi][QUANTITY.SOURCE] = source
return data
def do_tns_photo(catalog):
"""Load TNS photometry."""
task_str = catalog.get_current_task_str()
tns_url = 'https://www.wis-tns.org/'
try:
with open('tns.key', 'r') as f:
tnskey = f.read().splitlines()[0]
except Exception:
catalog.log.warning('TNS API key not found, make sure a file named '
'`tns.key` containing the key is placed the '
'astrocats directory.')
tnskey = ''
bandreps = {'Clear': 'C'}
fails = 0
for name in pbar(list(catalog.entries.keys()), task_str):
if name not in catalog.entries:
continue
aliases = catalog.entries[name].get_aliases()
oname = ''
for alias in aliases:
if (alias.startswith(('SN', 'AT')) and is_integer(alias[2:6]) and
int(alias[2:6]) >= 2016) and alias[6:].isalpha():
oname = alias
break
if not oname:
continue
reqname = oname[2:]
jsonpath = os.path.join(catalog.get_current_task_repo(), 'TNS',
reqname + '.json')
download_json = True
if os.path.isfile(jsonpath):
with open(jsonpath, 'r') as f:
objdict = json.load(f)
if 'discoverydate' in objdict:
discoverydate = objdict['discoverydate']
if '.' not in discoverydate:
discoverydate += '.0'
try:
if (datetime.now() - datetime.strptime(
discoverydate, '%Y-%m-%d %H:%M:%S.%f')).days > 90:
download_json = False
except ValueError:
download_json = False
if download_json:
data = urllib.parse.urlencode({
'api_key': tnskey,
'data': json.dumps({
'objname': reqname,
'photometry': '1'
})
}).encode('ascii')
req = urllib.request.Request(
'https://www.wis-tns.org/api/get/object', data=data)
trys = 0
objdict = None
while trys < 3 and not objdict:
try:
objdict = json.loads(
urllib.request.urlopen(req, timeout=30).read().decode('ascii'))[
'data']['reply']
except KeyboardInterrupt:
raise
except Exception:
catalog.log.warning('API request failed for `{}`.'.format(
name))
time.sleep(5)
trys = trys + 1
if (not objdict or 'objname' not in objdict or
not isinstance(objdict['objname'], str)):
fails = fails + 1
catalog.log.warning('Object `{}` not found!'.format(name))
if fails >= 5:
break
continue
# Cache object here
with open(jsonpath, 'w') as f:
json.dump(sortOD(objdict), f, indent='\t',
separators=(',', ':'), ensure_ascii=False,
sort_keys=True)
if 'photometry' not in objdict:
continue
photoarr = objdict['photometry']
name, source = catalog.new_entry(
oname, srcname='Transient Name Server', url=tns_url)
for photo in photoarr:
if 'mag' not in photo['flux_unit']['name'].lower():
catalog.log.warning('Unknown flux unit `{}`.'.format(photo[
'flux_unit']['name']))
continue
if not photo['jd']:
continue
if not photo['flux'] and not photo['limflux']:
continue
mag = photo['flux'] if photo['flux'] else photo['limflux']
photodict = {
PHOTOMETRY.TIME: str(jd_to_mjd(Decimal(str(photo['jd'])))),
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.MAGNITUDE: mag,
PHOTOMETRY.SOURCE: source
}
if photo.get('fluxerr', ''):
photodict[PHOTOMETRY.E_MAGNITUDE] = photo['fluxerr']
if not photo['flux']:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
band = photo['filters']['name']
if band:
if band in bandreps:
band = bandreps[band]
photodict[PHOTOMETRY.BAND] = band
if photo.get('observer', ''):
photodict[PHOTOMETRY.OBSERVER] = photo['observer']
if 'source_group' in photo:
survey = photo['source_group']['group_name']
if survey:
photodict[PHOTOMETRY.SURVEY] = survey
if 'telescope' in photo:
telescope = photo['telescope']['name']
if telescope and telescope != 'Other':
photodict[PHOTOMETRY.TELESCOPE] = telescope
if 'instrument' in photo:
instrument = photo['instrument']['name']
if instrument and instrument != 'Other':
photodict[PHOTOMETRY.INSTRUMENT] = instrument
system = ''
if 'Vega' in photo['flux_unit']['name']:
system = 'Vega'
elif 'ab' in photo['flux_unit']['name']:
system = 'AB'
if system:
photodict[PHOTOMETRY.SYSTEM] = system
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
return
def do_cfa_photo(catalog):
"""Import photometry from the CfA archive."""
from html import unescape
import re
task_str = catalog.get_current_task_str()
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'cfa-input/*.dat'))
for fname in pbar_strings(file_names, task_str):
f = open(fname, 'r')
tsvin = csv.reader(f, delimiter=' ', skipinitialspace=True)
csv_data = []
for r, row in enumerate(tsvin):
new = []
for item in row:
new.extend(item.split('\t'))
csv_data.append(new)
for r, row in enumerate(csv_data):
for c, col in enumerate(row):
csv_data[r][c] = col.strip()
csv_data[r] = [_f for _f in csv_data[r] if _f]
eventname = os.path.basename(os.path.splitext(fname)[0])
eventparts = eventname.split('_')
name = clean_snname(eventparts[0])
name = catalog.add_entry(name)
secondaryname = 'CfA Supernova Archive'
secondaryurl = 'https://www.cfa.harvard.edu/supernova/SNarchive.html'
secondarysource = catalog.entries[name].add_source(
name=secondaryname,
url=secondaryurl,
secondary=True,
acknowledgment=ACKN_CFA)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name,
secondarysource)
year = re.findall(r'\d+', name)[0]
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, year,
secondarysource)
eventbands = list(eventparts[1])
tu = 'MJD'
jdoffset = Decimal(0.)
for rc, row in enumerate(csv_data):
if len(row) > 0 and row[0][0] == "#":
if len(row[0]) > 2 and row[0][:3] == '#JD':
tu = 'JD'
rowparts = row[0].split('-')
jdoffset = Decimal(rowparts[1])
elif len(row[0]) > 6 and row[0][:7] == '#Julian':
tu = 'JD'
jdoffset = Decimal(0.)
elif len(row) > 1 and row[1].lower() == 'photometry':
for ci, col in enumerate(row[2:]):
if col[0] == "(":
refstr = ' '.join(row[2 + ci:])
refstr = refstr.replace('(', '').replace(')', '')
bibcode = unescape(refstr)
source = catalog.entries[name].add_source(
bibcode=bibcode)
elif len(row) > 1 and row[1] == 'HJD':
tu = 'HJD'
continue
elif len(row) > 0:
mjd = row[0]
for v, val in enumerate(row):
if v == 0:
if tu == 'JD':
mjd = str(jd_to_mjd(Decimal(val) + jdoffset))
tuout = 'MJD'
elif tu == 'HJD':
mjd = str(jd_to_mjd(Decimal(val)))
tuout = 'MJD'
else:
mjd = val
tuout = tu
elif v % 2 != 0:
if float(row[v]) < 90.0:
src = secondarysource + ',' + source
photodict = {
PHOTOMETRY.U_TIME: tuout,
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.BAND_SET: 'Standard',
PHOTOMETRY.BAND: eventbands[(v - 1) // 2],
PHOTOMETRY.MAGNITUDE: row[v],
PHOTOMETRY.E_MAGNITUDE: row[v + 1],
PHOTOMETRY.SOURCE: src
}
catalog.entries[name].add_photometry(**photodict)
f.close()
# Hicken 2012
with open(
os.path.join(catalog.get_current_task_repo(),
'hicken-2012-standard.dat'), 'r') as infile:
tsvin = list(csv.reader(infile, delimiter='|', skipinitialspace=True))
for r, row in enumerate(pbar(tsvin, task_str)):
if r <= 47:
continue
if row[0][:2] != 'sn':
name = 'SN' + row[0].strip()
else:
name = row[0].strip()
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2012ApJS..200...12H')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, 'Ia',
source)
photodict = {
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.TIME: row[2].strip(),
PHOTOMETRY.BAND: row[1].strip(),
PHOTOMETRY.BAND_SET: 'Standard',
PHOTOMETRY.MAGNITUDE: row[6].strip(),
PHOTOMETRY.E_MAGNITUDE: row[7].strip(),
PHOTOMETRY.SOURCE: source
}
catalog.entries[name].add_photometry(**photodict)
# Bianco 2014
with open(
os.path.join(catalog.get_current_task_repo(),
'bianco-2014-standard.dat'), 'r') as infile:
tsvin = list(csv.reader(infile, delimiter=' ', skipinitialspace=True))
for row in pbar(tsvin, task_str):
name = 'SN' + row[0]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2014ApJS..213...19B')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
photodict = {
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.TIME: row[2],
PHOTOMETRY.BAND: row[1],
PHOTOMETRY.MAGNITUDE: row[3],
PHOTOMETRY.E_MAGNITUDE: row[4],
PHOTOMETRY.TELESCOPE: row[5],
PHOTOMETRY.BAND_SET: 'Standard',
PHOTOMETRY.SOURCE: source
}
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
return
def do_tns_spectra(catalog, tns_url='https://www.wis-tns.org/', directory='TNS'):
"""Load TNS spectra."""
requests.packages.urllib3.disable_warnings()
task_str = catalog.get_current_task_str()
try:
with open('tns.key', 'r') as f:
tnskey = f.read().splitlines()[0]
except Exception:
catalog.log.warning('TNS API key not found, make sure a file named '
'`tns.key` containing the key is placed the '
'astrocats directory.')
tnskey = ''
fails = 0
for name in pbar(list(catalog.entries.keys()), task_str):
if name not in catalog.entries:
continue
aliases = catalog.entries[name].get_aliases()
oname = ''
for alias in aliases:
if (alias.startswith(('SN', 'AT')) and is_integer(alias[2:6]) and
int(alias[2:6]) >= 2016) and alias[6:].isalpha():
oname = alias
break
if not oname:
continue
reqname = oname[2:]
jsonpath = os.path.join(catalog.get_current_task_repo(), directory, 'meta',
reqname + '.json')
download_json = True
if os.path.isfile(jsonpath):
with open(jsonpath, 'r') as f:
objdict = json.load(f)
if 'discoverydate' in objdict:
discoverydate = objdict['discoverydate']
if '.' not in discoverydate:
discoverydate += '.0'
try:
if (datetime.now() - datetime.strptime(
discoverydate, '%Y-%m-%d %H:%M:%S.%f')).days > 90:
download_json = False
except ValueError:
download_json = False
if download_json:
data = urllib.parse.urlencode({
'api_key': tnskey,
'data': json.dumps({
'objname': reqname,
'spectra': '1'
})
}).encode('ascii')
req = urllib.request.Request(
tns_url + 'api/get/object', data=data)
trys = 0
objdict = None
while trys < 3 and not objdict:
try:
objdict = json.loads(
urllib.request.urlopen(req, timeout=30).read().decode('ascii'))[
'data']['reply']
except KeyboardInterrupt:
raise
except Exception:
catalog.log.warning('API request failed for `{}`.'.format(
name))
time.sleep(5)
trys = trys + 1
if (not objdict or 'objname' not in objdict or
not isinstance(objdict['objname'], str)):
fails = fails + 1
catalog.log.warning('Object `{}` not found!'.format(name))
if fails >= 5:
break
continue
# Cache object here
with open(jsonpath, 'w') as f:
json.dump(sortOD(objdict), f, indent='\t',
separators=(',', ':'), ensure_ascii=False,
sort_keys=True)
if 'spectra' not in objdict:
continue
specarr = objdict['spectra']
name, source = catalog.new_entry(
oname, srcname='Transient Name Server', url=tns_url)
for spectrum in specarr:
spectrumdict = {
PHOTOMETRY.SOURCE: source
}
if 'jd' in spectrum:
spectrumdict[SPECTRUM.TIME] = str(
jd_to_mjd(Decimal(str(spectrum['jd']))))
spectrumdict[SPECTRUM.U_TIME] = 'MJD'
if spectrum.get('observer', ''):
spectrumdict[SPECTRUM.OBSERVER] = spectrum['observer']
if spectrum.get('reducer', ''):
spectrumdict[SPECTRUM.OBSERVER] = spectrum['observer']
if 'source_group' in spectrum:
survey = spectrum['source_group']['name']
if survey:
spectrumdict[SPECTRUM.SURVEY] = survey
if 'telescope' in spectrum:
telescope = spectrum['telescope']['name']
if telescope and telescope != 'Other':
spectrumdict[SPECTRUM.TELESCOPE] = telescope
if 'instrument' in spectrum:
instrument = spectrum['instrument']['name']
if instrument and instrument != 'Other':
spectrumdict[SPECTRUM.INSTRUMENT] = instrument
if 'asciifile' in spectrum:
fname = urllib.parse.unquote(
spectrum['asciifile'].split('/')[-1])
spectxt = catalog.load_url(
spectrum['asciifile'],
os.path.join(
catalog.get_current_task_repo(), directory, 'spectra',
fname), archived_mode=True)
data = [x.split() for x in spectxt.splitlines()]
skipspec = False
newdata = []
oldval = ''
for row in data:
if row and '#' not in row[0]:
if (len(row) >= 2 and is_number(row[0]) and
is_number(row[1]) and row[1] != oldval):
newdata.append(row)
oldval = row[1]
if skipspec or not newdata:
warnings.warn('Skipped adding spectrum file ' + fname)
continue
data = [list(i) for i in zip(*newdata)]
wavelengths = data[0]
fluxes = data[1]
errors = ''
if len(data) == 3:
errors = data[1]
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
spectrumdict.update({
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.ERRORS: errors,
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.U_ERRORS: fluxunit if errors else '',
SPECTRUM.WAVELENGTHS: wavelengths,
SPECTRUM.FLUXES: fluxes
})
catalog.entries[name].add_spectrum(**spectrumdict)
catalog.journal_entries()
return
def do_csp_fits_spectra(catalog):
from astropy.io import fits
fpath = catalog.get_current_task_repo()
fureps = {'erg/cm2/s/A': 'erg/s/cm^2/Angstrom'}
task_str = catalog.get_current_task_str()
dirs = [x[0] for x in os.walk(
os.path.join(fpath, 'Gutierrez_et_al_2017'))]
files = []
for dir in dirs:
files.extend(glob(os.path.join(dir, '*.fits')))
for datafile in pbar(files, task_str):
filename = datafile.split('/')[-1]
hdulist = fits.open(datafile)
for oi, obj in enumerate(hdulist[0].header):
if any(x in ['.', '/'] for x in obj):
del (hdulist[0].header[oi])
try:
hdulist[0].verify('silentfix')
except Exception as e:
print(e)
hdrkeys = list(hdulist[0].header.keys())
# print(hdrkeys)
name = datafile.split('/')[-2]
if name[2] in '6789':
name = 'SN19' + name[2:]
elif name != 'SN210':
name = 'SN20' + name[2:]
name, source = catalog.new_entry(name, bibcode='2017ApJ...850...89G')
# for key in hdulist[0].header.keys():
# print(key, hdulist[0].header[key])
mjd = None
if hdulist[0].header['SIMPLE']:
if 'JD' in hdrkeys:
mjd = str(jd_to_mjd(Decimal(str(hdulist[0].header['JD']))))
elif 'MJD' in hdrkeys:
mjd = str(hdulist[0].header['MJD'])
elif 'DATE-OBS' in hdrkeys or 'DATE' in hdrkeys:
dkey = 'DATE-OBS' if 'DATE-OBS' in hdrkeys else 'DATE'
dval = hdulist[0].header[dkey]
if is_number(dval):
dkey = 'DATE' if dkey == 'DATE-OBS' else 'DATE-OBS'
dval = hdulist[0].header[dkey]
dateobs = None
if 'T' in dval:
dateobs = dval.strip()
elif 'UTC-OBS' in hdrkeys:
dateobs = dval.strip(
) + 'T' + hdulist[0].header['UTC-OBS'].strip()
if dateobs is not None:
mjd = str(astrotime(dateobs, format='isot').mjd)
# print(hdulist[0].header)
if 'CRVAL1' in hdulist[0].header:
w0 = hdulist[0].header['CRVAL1']
elif hdulist[0].header['CTYPE1'] == 'MULTISPE':
w0 = float(hdulist[0].header['WAT2_001'].split(
'"')[-1].split()[3])
else:
raise ValueError('Unsupported spectrum format.')
if hdulist[0].header['NAXIS'] == 1:
wd = hdulist[0].header['CDELT1']
fluxes = [str(x) for x in list(hdulist[0].data)]
errors = False
elif hdulist[0].header['NAXIS'] == 3:
wd = hdulist[0].header['CD1_1']
fluxes = [str(x) for x in list(hdulist[0].data)[0][0]]
errors = [str(x) for x in list(hdulist[0].data)[-1][0]]
else:
print('Warning: Skipping FITS spectrum `{}`.'.format(filename))
continue
waves = [str(w0 + wd * x) for x in range(0, len(fluxes))]
else:
raise ValueError('Non-simple FITS import not yet supported.')
if 'BUNIT' in hdrkeys:
fluxunit = hdulist[0].header['BUNIT']
if fluxunit in fureps:
fluxunit = fureps[fluxunit]
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.WAVELENGTHS: waves,
SPECTRUM.FLUXES: fluxes,
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.FILENAME: filename,
SPECTRUM.SOURCE: source
}
if mjd is not None:
specdict[SPECTRUM.TIME] = mjd
specdict[SPECTRUM.U_TIME] = 'MJD'
if 'TELESCOP' in hdrkeys:
specdict[SPECTRUM.TELESCOPE] = hdulist[0].header['TELESCOP']
if 'INSTRUME' in hdrkeys:
specdict[SPECTRUM.INSTRUMENT] = hdulist[0].header['INSTRUME']
if 'AIRMASS' in hdrkeys:
specdict[SPECTRUM.AIRMASS] = hdulist[0].header['AIRMASS']
if errors:
specdict[SPECTRUM.ERRORS] = errors
specdict[SPECTRUM.U_ERRORS] = fluxunit
if 'SITENAME' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['SITENAME']
elif 'OBSERVAT' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['OBSERVAT']
if 'OBSERVER' in hdrkeys:
specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER']
catalog.entries[name].add_spectrum(**specdict)
hdulist.close()
catalog.journal_entries()
return
def clean_internal(self, data):
"""Clean input data from the 'Supernovae/input/internal' repository.
FIX: instead of making changes in place to `dirty_event`, should a new
event be created, values filled, then returned??
FIX: currently will fail if no bibcode and no url
"""
self._log.debug("clean_internal(): {}".format(self.name()))
def_source_dict = {}
# Find source that will be used as default
sources = data.get(self._KEYS.SOURCES, [])
if sources:
def_source_dict = sources[0]
allow_alias = False
if SOURCE.ALIAS in def_source_dict:
del (def_source_dict[SOURCE.ALIAS])
else:
# If there are no existing sources, add OSC as one
self.add_self_source()
sources = self.get(self._KEYS.SOURCES, [])
def_source_dict = sources[0]
allow_alias = True
# Clean some legacy fields
alias_key = 'aliases'
if alias_key in data:
# Remove the entry in the data
aliases = data.pop(alias_key)
# Make sure this is a list
if not isinstance(aliases, list):
raise ValueError("{}: aliases not a list '{}'".format(
self.name(), aliases))
# Add OSC source entry
source = self.add_self_source()
for alias in aliases:
self.add_quantity(self._KEYS.ALIAS, alias, source)
dist_key = 'distinctfrom'
if dist_key in data:
distincts = data.pop(dist_key)
if ((isinstance(distincts, list) and
isinstance(distincts[0], string_types))):
source = self.add_self_source()
for df in distincts:
self.add_quantity(self._KEYS.DISTINCT_FROM, df, source)
else:
data[dist_key] = list(distincts)
# Go through all remaining keys in 'dirty' event, and make sure
# everything is a quantity with a source (OSC if no other)
for key in data.keys():
# The following line should be used to replace the above once keys
# returns the superclass keys too
if self._KEYS.get_key_by_name(key).no_source:
pass
elif key == self._KEYS.PHOTOMETRY:
for p, photo in enumerate(data[self._KEYS.PHOTOMETRY]):
if photo.get(PHOTOMETRY.U_TIME) == 'JD':
data[self._KEYS.PHOTOMETRY][p][
PHOTOMETRY.U_TIME] = 'MJD'
data[self._KEYS.PHOTOMETRY][p][PHOTOMETRY.TIME] = str(
jd_to_mjd(Decimal(photo['time'])))
if QUANTITY.SOURCE not in photo:
if not def_source_dict:
raise ValueError("No sources found, can't add "
"photometry.")
source = self.add_source(
allow_alias=allow_alias, **def_source_dict)
data[self._KEYS.PHOTOMETRY][p][
QUANTITY.SOURCE] = source
else:
for qi, quantity in enumerate(data[key]):
if QUANTITY.SOURCE not in quantity:
if not def_source_dict:
raise ValueError("No sources found, can't add "
"quantity.")
source = self.add_source(
allow_alias=allow_alias, **def_source_dict)
data[key][qi][QUANTITY.SOURCE] = source
return data
def do_snf_specta(catalog):
task_str = catalog.get_current_task_str()
bibcodes = {'SN2005gj': '2006ApJ...650..510A',
'SN2006D': '2007ApJ...654L..53T',
'SN2007if': '2010ApJ...713.1073S',
'SN2011fe': '2013A&A...554A..27P'}
oldname = ''
snfcnt = 0
eventfolders = next(os.walk(os.path.join(
catalog.get_current_task_repo(), 'SNFactory')))[1]
for eventfolder in pbar(eventfolders, task_str):
oname = eventfolder
name = catalog.get_preferred_name(oname)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
sec_reference = 'Nearby Supernova Factory'
sec_refurl = 'http://snfactory.lbl.gov/'
sec_bibcode = '2002SPIE.4836...61A'
sec_source = catalog.entries[name].add_source(
name=sec_reference, url=sec_refurl, bibcode=sec_bibcode,
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oname, sec_source)
bibcode = bibcodes[oname]
source = catalog.entries[name].add_source(bibcode=bibcode)
sources = uniq_cdl([source, sec_source])
use_path = os.path.join(
catalog.get_current_task_repo(), 'SNFactory', eventfolder, '*.dat')
eventspectra = glob(use_path)
for spectrum in pbar(eventspectra, task_str):
filename = os.path.basename(spectrum)
with open(spectrum) as spec_file:
specdata = list(csv.reader(
spec_file, delimiter=' ', skipinitialspace=True))
specdata = list(filter(None, specdata))
newspec = []
time = ''
telescope = ''
instrument = ''
observer = ''
observatory = ''
if 'Keck_20060202_R' in spectrum:
time = '53768.23469'
elif 'Spectrum05_276' in spectrum:
time = pretty_num(astrotime('2005-10-03').mjd, sig=5)
elif 'Spectrum05_329' in spectrum:
time = pretty_num(astrotime('2005-11-25').mjd, sig=5)
elif 'Spectrum05_336' in spectrum:
time = pretty_num(astrotime('2005-12-02').mjd, sig=5)
for row in specdata:
if row[0][0] == '#':
joinrow = (' '.join(row)).split('=')
if len(joinrow) < 2:
continue
field = joinrow[0].strip('# ')
value = joinrow[1].split('/')[0].strip('\' ')
if not time:
if field == 'JD':
time = str(jd_to_mjd(Decimal(value)))
elif field == 'MJD':
time = value
elif field == 'MJD-OBS':
time = value
if field == 'OBSERVER':
observer = value.capitalize()
if field == 'OBSERVAT':
observatory = value.capitalize()
if field == 'TELESCOP':
telescope = value.capitalize()
if field == 'INSTRUME':
instrument = value.capitalize()
else:
newspec.append(row)
if not time:
raise ValueError('Time missing from spectrum.')
specdata = newspec
haserrors = len(specdata[0]) == 3 and specdata[
0][2] and specdata[0][2] != 'NaN'
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
unit_err = ('Variance' if oldname == 'SN2011fe' else
'erg/s/cm^2/Angstrom')
unit_flx = 'erg/s/cm^2/Angstrom'
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom', u_fluxes=unit_flx, u_time='MJD',
time=time,
wavelengths=wavelengths, fluxes=fluxes, errors=errors,
observer=observer, observatory=observatory,
telescope=telescope, instrument=instrument, u_errors=unit_err,
source=sources, filename=filename)
snfcnt = snfcnt + 1
if (catalog.args.travis and
snfcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
def do_donated_photo(catalog):
"""Import donated photometry."""
task_str = catalog.get_current_task_str()
# Private donations here #
if not catalog.args.travis:
pass
# End private donations #
# Ponder 05-12-17 donation
with open(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Ponder-05-12-17', 'meta.json'), 'r') as f:
metadict = json.loads(f.read())
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Ponder-05-12-17', '*.dat'))
for path in file_names:
with open(path, 'r') as f:
tsvin = list(csv.reader(f, delimiter=' ', skipinitialspace=True))
oname = path.split('/')[-1].split('.')[0]
name, source = catalog.new_entry(
oname, bibcode=metadict[oname]['bibcode'])
for row in pbar(tsvin, task_str + ': Ponder ' + oname):
if row[0][0] == '#' or not is_number(row[-1]):
continue
mjd = row[1]
bandinst = row[2].split('_')
band = bandinst[0]
inst = ''
if len(bandinst) > 1:
inst = bandinst[1]
mag = row[3]
uerr = row[4]
lerr = row[5]
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.BAND: band,
PHOTOMETRY.MAGNITUDE: mag,
PHOTOMETRY.E_LOWER_MAGNITUDE: lerr,
PHOTOMETRY.E_UPPER_MAGNITUDE: uerr,
PHOTOMETRY.SOURCE: source
}
if inst:
photodict[PHOTOMETRY.INSTRUMENT] = inst
catalog.entries[name].add_photometry(**photodict)
# Benetti 03-08-17 donation
path = os.path.join(catalog.get_current_task_repo(), 'Donations',
'Benetti-03-08-17', '1999E.dat')
with open(path, 'r') as f:
tsvin = list(csv.reader(f, delimiter=' ', skipinitialspace=True))
name, source = catalog.new_entry(
'SN1999E', bibcode='2003MNRAS.340..191R')
bands = None
for row in tsvin:
if not row or row[0][0] == '#':
continue
if not bands:
bands = row[2:-2]
continue
mjd = row[1]
tel = row[-1] if 'IAUC' not in row[-1] else None
for bi, band in enumerate(bands):
mag = row[2 + 2 * bi]
if mag == '9999':
continue
err = row[2 + 2 * bi + 1]
limit = row[6] == 'True'
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.TELESCOPE: tel,
PHOTOMETRY.BAND: band,
PHOTOMETRY.MAGNITUDE: mag,
PHOTOMETRY.SOURCE: source
}
if err != '.00':
photodict[PHOTOMETRY.E_MAGNITUDE] = str(Decimal(err))
if tel:
photodict[PHOTOMETRY.TELESCOPE] = tel
catalog.entries[name].add_photometry(**photodict)
# Nicholl 01-29-17 donation
with open(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-01-29-17', 'meta.json'), 'r') as f:
metadict = json.loads(f.read())
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-01-29-17', '*.txt'))
for path in file_names:
data = read(path, format='cds')
oname = path.split('/')[-1].split('_')[0]
name, source = catalog.new_entry(
oname, bibcode=metadict[oname]['bibcode'])
for row in pbar(data, task_str + ': Nicholl ' + oname):
photodict = {
PHOTOMETRY.TIME: str(row['MJD']),
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.MAGNITUDE: str(row['mag']),
PHOTOMETRY.BAND: row['Filter'],
PHOTOMETRY.SOURCE: source
}
if 'system' in metadict[oname]:
photodict[PHOTOMETRY.SYSTEM] = metadict[oname]['system']
if 'l_mag' in row.columns and row['l_mag'] == '>':
photodict[PHOTOMETRY.UPPER_LIMIT] = True
elif 'e_mag' in row.columns:
photodict[PHOTOMETRY.E_MAGNITUDE] = str(row['e_mag'])
if 'Telescope' in row.columns:
photodict[PHOTOMETRY.TELESCOPE] = row['Telescope']
catalog.entries[name].add_photometry(**photodict)
# Arcavi 2016gkg donation
path = os.path.join(catalog.get_current_task_repo(), 'Donations',
'Arcavi-01-24-17', 'SN2016gkg.txt')
with open(path, 'r') as f:
tsvin = list(csv.reader(f, delimiter=' ', skipinitialspace=True))
name, source = catalog.new_entry(
'SN2016gkg', bibcode='2016arXiv161106451A')
for row in tsvin:
if row[0][0] == '#':
continue
mjd = str(jd_to_mjd(Decimal(row[0])))
tel = row[1]
band = row[3]
mag = row[4]
err = row[5]
limit = row[6] == 'True'
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.TELESCOPE: tel,
PHOTOMETRY.BAND: band,
PHOTOMETRY.MAGNITUDE: mag,
PHOTOMETRY.SOURCE: source
}
if limit:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
else:
photodict[PHOTOMETRY.E_MAGNITUDE] = err
catalog.entries[name].add_photometry(**photodict)
# Nicholl Gaia16apd donation
path = os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-01-20-17', 'gaia16apd_phot.txt')
data = read(path, format='cds')
name, source = catalog.new_entry(
'Gaia16apd', bibcode='2017ApJ...835L...8N')
for row in pbar(data, task_str + ': Nicholl Gaia16apd'):
photodict = {
PHOTOMETRY.TIME: str(row['MJD']),
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.MAGNITUDE: str(row['mag']),
PHOTOMETRY.BAND: row['Filter'],
PHOTOMETRY.TELESCOPE: row['Telescope'],
PHOTOMETRY.SOURCE: source
}
if row['l_mag'] == '>':
photodict[PHOTOMETRY.UPPER_LIMIT] = True
else:
photodict[PHOTOMETRY.E_MAGNITUDE] = str(row['e_mag'])
catalog.entries[name].add_photometry(**photodict)
# Kuncarayakti-01-09-17
datafile = os.path.join(catalog.get_current_task_repo(), 'Donations',
'Kuncarayakti-01-09-17', 'SN1978K.dat')
inpname = os.path.basename(datafile).split('.')[0]
with open(datafile, 'r') as f:
tsvin = csv.reader(f, delimiter=' ', skipinitialspace=True)
host = False
for ri, row in enumerate(tsvin):
if ri == 0:
continue
if row[0][0] == '#':
rsplit = [x.strip('# ') for x in ' '.join(row).split(',')]
bc = rsplit[0]
tel, ins = '', ''
if len(rsplit) > 1:
tel = rsplit[1]
if len(rsplit) > 2:
ins = rsplit[2]
continue
(name, source) = catalog.new_entry(inpname, bibcode=bc)
mag = row[4]
err = row[5]
mjd = str(astrotime('-'.join(row[:3]), format='iso').mjd)
photodict = {
PHOTOMETRY.BAND: row[3],
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.MAGNITUDE: mag.strip('>s'),
PHOTOMETRY.SOURCE: source
}
if is_number(err):
photodict[PHOTOMETRY.E_MAGNITUDE] = err
if tel:
photodict[PHOTOMETRY.TELESCOPE] = tel
if ins:
photodict[PHOTOMETRY.INSTRUMENT] = ins
if '>' in mag:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
if 's' in mag:
photodict[PHOTOMETRY.SYNTHETIC] = True
catalog.entries[name].add_photometry(**photodict)
# Nugent 01-09-17 donation
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nugent-01-09-17', '*.dat'))
for datafile in pbar_strings(file_names, task_str + ': Nugent-01-09-17'):
inpname = os.path.basename(datafile).split('.')[0]
(name, source) = catalog.new_entry(
inpname, bibcode='2006ApJ...645..841N')
with open(datafile, 'r') as f:
tsvin = csv.reader(f, delimiter=' ', skipinitialspace=True)
host = False
for urow in tsvin:
row = list(filter(None, urow))
counts = row[2]
e_counts = row[3]
zp = row[4]
photodict = {
PHOTOMETRY.BAND: row[1],
PHOTOMETRY.TIME: row[0],
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.COUNT_RATE: counts,
PHOTOMETRY.E_COUNT_RATE: e_counts,
PHOTOMETRY.ZERO_POINT: zp,
PHOTOMETRY.TELESCOPE: 'CFHT',
PHOTOMETRY.SURVEY: 'SNLS',
PHOTOMETRY.SOURCE: source
}
set_pd_mag_from_counts(photodict, counts, ec=e_counts, zp=zp,
sig=5.0)
catalog.entries[name].add_photometry(**photodict)
# Inserra 09-04-16 donation
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Inserra-09-04-16', '*.txt'))
for datafile in pbar_strings(file_names, task_str + ': Inserra-09-04-16'):
inpname = os.path.basename(datafile).split('.')[0]
(name, source) = catalog.new_entry(
inpname, bibcode='2013ApJ...770..128I')
with open(datafile, 'r') as f:
tsvin = csv.reader(f, delimiter=' ', skipinitialspace=True)
host = False
for row in tsvin:
if row[0][0] == '#':
if row[0] == '#Host':
host = True
continue
host = False
bands = row[3:-1]
continue
for bi, ba in enumerate(bands):
mag = row[5 + 2 * bi]
if not is_number(mag):
continue
system = 'AB'
if ba in ['U', 'B', 'V', 'R', 'I', 'J', 'H', 'K']:
system = 'Vega'
photodict = {
PHOTOMETRY.TIME: row[3],
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.BAND: ba,
PHOTOMETRY.MAGNITUDE: mag.strip('< '),
PHOTOMETRY.SOURCE: source,
PHOTOMETRY.SYSTEM: system
}
if 'ATel' not in row[-1]:
photodict[PHOTOMETRY.TELESCOPE] = row[-1]
if host:
photodict[PHOTOMETRY.HOST] = True
if '<' in mag:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
e_mag = row[5 + 2 * bi + 1].strip('() ')
if is_number(e_mag):
photodict[PHOTOMETRY.E_MAGNITUDE] = e_mag
catalog.entries[name].add_photometry(**photodict)
# Nicholl 04-01-16 donation
with open(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-04-01-16', 'bibcodes.json'), 'r') as f:
bcs = json.loads(f.read())
kcorrected = ['SN2011ke', 'SN2011kf', 'SN2012il', 'PTF10hgi', 'PTF11rks']
ignorephoto = ['PTF10hgi', 'PTF11rks', 'SN2011ke', 'SN2011kf', 'SN2012il']
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-04-01-16/*.txt'))
for datafile in pbar_strings(file_names, task_str + ': Nicholl-04-01-16'):
inpname = os.path.basename(datafile).split('_')[0]
isk = inpname in kcorrected
name = catalog.add_entry(inpname)
bibcode = ''
for bc in bcs:
if inpname in bcs[bc]:
bibcode = bc
if not bibcode:
raise ValueError('Bibcode not found!')
source = catalog.entries[name].add_source(bibcode=bibcode)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, inpname, source)
if inpname in ignorephoto:
continue
with open(datafile, 'r') as f:
tsvin = csv.reader(f, delimiter='\t', skipinitialspace=True)
rtelescope = ''
for r, rrow in enumerate(tsvin):
row = list(filter(None, rrow))
if not row:
continue
if row[0] == '#MJD':
bands = [x for x in row[1:] if x and 'err' not in x]
elif row[0][0] == '#' and len(row[0]) > 1:
rtelescope = row[0][1:]
if row[0][0] == '#':
continue
mjd = row[0]
if not is_number(mjd):
continue
for v, val in enumerate(row[1::2]):
upperlimit = ''
mag = val.strip('>')
emag = row[2 * v + 2]
if '>' in val or (is_number(emag) and float(emag) == 0.0):
upperlimit = True
if (not is_number(mag) or isnan(float(mag)) or
float(mag) > 90.0):
continue
band = bands[v]
instrument = ''
survey = ''
system = ''
telescope = rtelescope
if telescope == 'LSQ':
instrument = 'QUEST'
elif telescope == 'PS1':
instrument = 'GPC'
elif telescope == 'NTT':
instrument = 'EFOSC'
elif telescope == 'GROND':
instrument = 'GROND'
telescope = 'MPI/ESO 2.2m'
else:
if band == 'NUV':
instrument = 'GALEX'
telescope = 'GALEX'
elif band in ['u', 'g', 'r', 'i', 'z']:
if inpname.startswith('PS1'):
instrument = 'GPC'
telescope = 'PS1'
survey = 'Pan-STARRS'
elif inpname.startswith('PTF'):
telescope = 'P60'
survey = 'PTF'
elif band.upper() in ['UVW2', 'UVW1', 'UVM2']:
instrument = 'UVOT'
telescope = 'Swift'
if inpname in ['PTF12dam']:
system = 'AB'
if inpname in ['SCP-06F6']:
system = 'Vega'
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.BAND: band,
PHOTOMETRY.MAGNITUDE: mag,
PHOTOMETRY.UPPER_LIMIT: upperlimit,
PHOTOMETRY.SOURCE: source
}
if instrument:
photodict[PHOTOMETRY.INSTRUMENT] = instrument
if telescope:
photodict[PHOTOMETRY.TELESCOPE] = telescope
if survey:
photodict[PHOTOMETRY.SURVEY] = survey
if system:
photodict[PHOTOMETRY.SYSTEM] = system
if (is_number(emag) and
not isnan(float(emag)) and float(emag) > 0.0):
photodict[PHOTOMETRY.E_MAGNITUDE] = emag
if isk:
photodict[PHOTOMETRY.KCORRECTED] = True
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
# Maggi 04-11-16 donation (MC SNRs)
with open(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Maggi-04-11-16', 'LMCSNRs_OpenSNe.csv')) as f:
tsvin = csv.reader(f, delimiter=',')
for row in pbar(list(tsvin), task_str + ': Maggi-04-11-16/LMCSNRs'):
name = 'MCSNR ' + row[0]
name = catalog.add_entry(name)
ra = row[2]
dec = row[3]
source = (catalog.entries[name]
.add_source(bibcode='2016A&A...585A.162M'))
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS,
'LMCSNR J' + rep_chars(ra, ' :.') + rep_chars(dec, ' :.'),
source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
if row[1] != 'noname':
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[1],
source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[2], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[3], source)
catalog.entries[name].add_quantity(SUPERNOVA.HOST, 'LMC', source)
if row[4] == '1':
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE,
'Ia', source)
elif row[4] == '2':
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE,
'CC', source)
with open(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Maggi-04-11-16', 'SMCSNRs_OpenSNe.csv')) as f:
tsvin = csv.reader(f, delimiter=',')
for row in pbar(list(tsvin), task_str + ': Maggi-04-11-16/SMCSNRs'):
name = 'MCSNR ' + row[0]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(name='Pierre Maggi')
ra = row[3]
dec = row[4]
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, 'SMCSNR J' + ra.replace(
':', '')[:6] + dec.replace(':', '')[:7], source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[1], source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[2], source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[3], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[4], source)
catalog.entries[name].add_quantity(SUPERNOVA.HOST, 'SMC', source)
catalog.journal_entries()
# Galbany 04-18-16 donation
folders = next(
os.walk(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Galbany-04-18-16/')))[1]
bibcode = '2016AJ....151...33G'
for folder in folders:
infofiles = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Galbany-04-18-16/') + folder + '/*.info')
photfiles = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Galbany-04-18-16/') + folder + '/*.out*')
zhel = ''
zcmb = ''
zerr = ''
for path in infofiles:
with open(path, 'r') as f:
lines = f.read().splitlines()
for line in lines:
splitline = line.split(':')
field = splitline[0].strip().lower()
value = splitline[1].strip()
if field == 'name':
name = value[:6].upper()
name += (value[6].upper()
if len(value) == 7 else value[6:])
name = catalog.add_entry(name)
source = (catalog.entries[name]
.add_source(bibcode=bibcode))
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS,
name, source)
elif field == 'type':
claimedtype = value.replace('SN', '')
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, claimedtype, source)
elif field == 'zhel':
zhel = value
elif field == 'redshift_error':
zerr = value
elif field == 'zcmb':
zcmb = value
elif field == 'ra':
catalog.entries[name].add_quantity(
SUPERNOVA.RA,
value,
source,
u_value='floatdegrees')
elif field == 'dec':
catalog.entries[name].add_quantity(
SUPERNOVA.DEC,
value,
source,
u_value='floatdegrees')
elif field == 'host':
value = value.replace('- ', '-').replace('G ', 'G')
catalog.entries[name].add_quantity(SUPERNOVA.HOST,
value, source)
elif field == 'e(b-v)_mw':
catalog.entries[name].add_quantity(SUPERNOVA.EBV,
value, source)
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT,
zhel,
source,
e_value=zerr,
kind='heliocentric')
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, zcmb, source, e_value=zerr, kind='cmb')
for path in photfiles:
with open(path, 'r') as f:
band = ''
lines = f.read().splitlines()
for li, line in enumerate(lines):
if li in [0, 2, 3]:
continue
if li == 1:
band = line.split(':')[-1].strip()
else:
cols = list(filter(None, line.split()))
if not cols:
continue
catalog.entries[name].add_photometry(
time=cols[0],
u_time='MJD',
magnitude=cols[1],
e_magnitude=cols[2],
band=band,
system=cols[3],
telescope=cols[4],
source=source)
catalog.journal_entries()
# Nicholl 05-03-16
files = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-05-03-16', '*.txt'))
name = catalog.add_entry('SN2015bn')
for fi in pbar(files, task_str + ': Nicholl-05-03-16'):
if 'late' in fi:
bc = '2016ApJ...828L..18N'
else:
bc = '2016ApJ...826...39N'
source = catalog.entries[name].add_source(bibcode=bc)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, 'PS15ae', source)
telescope = os.path.basename(fi).split('_')[1]
with open(fi, 'r') as f:
lines = f.read().splitlines()
for li, line in enumerate(lines):
if not line or (line[0] == '#' and li != 0):
continue
cols = list(filter(None, line.split()))
if not cols:
continue
if li == 0:
bands = cols[1:]
continue
mjd = cols[0]
for ci, col in enumerate(cols[1::2]):
if not is_number(col) or np.isnan(float(col)):
continue
band = bands[ci]
band_set = ''
system = 'Vega'
if bands[ci] in ["u'", "g'", "r'", "i'", "z'"]:
band_set = 'SDSS'
system = 'SDSS'
elif telescope == 'ASASSN':
band_set = 'ASASSN'
system = 'Vega'
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.MAGNITUDE: col,
PHOTOMETRY.BAND: bands[ci],
PHOTOMETRY.SOURCE: source,
PHOTOMETRY.TELESCOPE: telescope,
PHOTOMETRY.SYSTEM: system
}
if band_set:
photodict[PHOTOMETRY.BAND_SET] = band_set
emag = cols[2 * ci + 2]
if is_number(emag):
photodict[PHOTOMETRY.E_MAGNITUDE] = emag
else:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
if telescope == 'Swift':
photodict[PHOTOMETRY.INSTRUMENT] = 'UVOT'
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
return
def do_ogle(catalog):
task_str = catalog.get_current_task_str()
basenames = ['transients', 'transients/2014b', 'transients/2014',
'transients/2013', 'transients/2012']
oglenames = []
ogleupdate = [True, False, False, False, False]
for b, bn in enumerate(pbar(basenames, task_str)):
if catalog.args.update and not ogleupdate[b]:
continue
filepath = os.path.join(catalog.get_current_task_repo(), 'OGLE-')
filepath += bn.replace('/', '-') + '-transients.html'
htmltxt = catalog.load_cached_url(
'http://ogle.astrouw.edu.pl/ogle4/' + bn +
'/transients.html', filepath)
if not htmltxt:
continue
soup = BeautifulSoup(htmltxt, 'html5lib')
links = soup.findAll('a')
breaks = soup.findAll('br')
datalinks = []
datafnames = []
for a in links:
if a.has_attr('href'):
if '.dat' in a['href']:
datalinks.append(
'http://ogle.astrouw.edu.pl/ogle4/' + bn + '/' +
a['href'])
datafnames.append(bn.replace('/', '-') +
'-' + a['href'].replace('/', '-'))
ec = -1
reference = 'OGLE-IV Transient Detection System'
refurl = 'http://ogle.astrouw.edu.pl/ogle4/transients/transients.html'
for br in pbar(breaks, task_str):
sibling = br.nextSibling
if 'Ra,Dec=' in sibling:
line = sibling.replace('\n', '').split('Ra,Dec=')
name = line[0].strip()
ec += 1
if 'NOVA' in name or 'dupl' in name:
continue
if name in oglenames:
continue
oglenames.append(name)
name = catalog.add_entry(name)
mySibling = sibling.nextSibling
atelref = ''
claimedtype = ''
while 'Ra,Dec=' not in mySibling:
if isinstance(mySibling, NavigableString):
if 'Phot.class=' in str(mySibling):
claimedtype = re.sub(
r'\([^)]*\)', '',
str(mySibling).split('=')[-1])
claimedtype = claimedtype.replace('SN', '').strip()
if isinstance(mySibling, Tag):
atela = mySibling
if (atela and atela.has_attr('href') and
'astronomerstelegram' in atela['href']):
atelref = atela.contents[0].strip()
atelurl = atela['href']
mySibling = mySibling.nextSibling
if mySibling is None:
break
# nextSibling = sibling.nextSibling
# if ((isinstance(nextSibling, Tag) and
# nextSibling.has_attr('alt') and
# nextSibling.contents[0].strip() != 'NED')):
# radec = nextSibling.contents[0].strip().split()
# else:
# radec = line[-1].split()
# ra = radec[0]
# dec = radec[1]
fname = os.path.join(catalog.get_current_task_repo(),
'OGLE/') + datafnames[ec]
if (catalog.current_task.load_archive(catalog.args) and
os.path.isfile(fname)):
with open(fname, 'r') as f:
csvtxt = f.read()
else:
response = urllib.request.urlopen(datalinks[ec])
with open(fname, 'w') as f:
csvtxt = response.read().decode('utf-8')
f.write(csvtxt)
lcdat = csvtxt.splitlines()
sources = [catalog.entries[name].add_source(
name=reference, url=refurl)]
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, name, sources[0])
if atelref and atelref != 'ATel#----':
sources.append(catalog.entries[name].add_source(
name=atelref, url=atelurl))
sources = uniq_cdl(sources)
if name.startswith('OGLE'):
if name[4] == '-':
if is_number(name[5:9]):
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, name[5:9], sources)
else:
if is_number(name[4:6]):
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, '20' + name[4:6],
sources)
# RA and Dec from OGLE pages currently not reliable
# catalog.entries[name].add_quantity(SUPERNOVA.RA, ra, sources)
# catalog.entries[name].add_quantity(SUPERNOVA.DEC, dec,
# sources)
if claimedtype and claimedtype != '-':
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, claimedtype, sources)
elif ('SN' not in name and SUPERNOVA.CLAIMED_TYPE not in
catalog.entries[name]):
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, 'Candidate', sources)
for row in lcdat:
row = row.split()
mjd = str(jd_to_mjd(Decimal(row[0])))
magnitude = row[1]
if float(magnitude) > 90.0:
continue
e_mag = row[2]
upperlimit = False
if e_mag == '-1' or float(e_mag) > 10.0:
e_mag = ''
upperlimit = True
catalog.entries[name].add_photometry(
time=mjd, band='I', magnitude=magnitude,
e_magnitude=e_mag,
system='Vega', source=sources, upperlimit=upperlimit)
if catalog.args.update:
catalog.journal_entries()
catalog.journal_entries()
return
def do_snax(catalog):
"""Import from the SNaX X-ray database."""
task_str = catalog.get_current_task_str()
dlurl = 'http://kronos.uchicago.edu/snax/export.php?exportType=TSV&exportFields=standard&objid=&name=&typeid=&type=&galaxyid=&galaxy=&fluxMin=&fluxMax=&fluxEnergyLMin=&fluxEnergyLMax=&fluxEnergyHMin=&fluxEnergyHMax=&lumMin=&lumMax=&instrumentid=&instrument=&ageMin=&ageMax=&dateMin=&dateMax=&sortA=dateExploded' # noqa: E501
file_path = os.path.join(catalog.get_current_task_repo(), 'SNaX.TSV')
tsv = catalog.load_url(dlurl, file_path)
# csvtxt = catalog.load_url(
# 'http://www.grbcatalog.org/'
# 'download_data?cut_0_min=5&cut_0=BAT%20T90'
# '&cut_0_max=100000&num_cuts=1&no_date_cut=True',
# file_path)
data = [x.split('\t') for x in tsv.split('\n')]
for r, row in enumerate(pbar(data, task_str)):
if r == 0 or not row[0]:
continue
(name, source) = catalog.new_entry(
row[0], srcname='SNaX', url='http://kronos.uchicago.edu/snax/',
secondary=True)
sources = [source]
expsrc = uniq_cdl(sources + [
catalog.entries[name].add_source(bibcode=row[-6].strip())
])
coosrc = uniq_cdl(sources + [
catalog.entries[name].add_source(bibcode=row[-5].strip())
])
dissrc = uniq_cdl(sources + [
catalog.entries[name].add_source(bibcode=row[-4].strip())
])
flxsrc = uniq_cdl(sources + [
catalog.entries[name].add_source(bibcode=row[-3].strip()),
catalog.entries[name].add_source(bibcode=row[-2].strip())
])
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, row[1],
source)
date = astrotime(float(row[2]), format='jd').datetime
catalog.entries[name].add_quantity(
SUPERNOVA.EXPLOSION_DATE,
make_date_string(date.year, date.month, date.day), expsrc)
catalog.entries[name].add_quantity(
SUPERNOVA.RA, ' '.join(row[3].split()[:3]), coosrc)
catalog.entries[name].add_quantity(
SUPERNOVA.DEC, ' '.join(row[3].split()[3:]), coosrc)
catalog.entries[name].add_quantity(SUPERNOVA.LUM_DIST, row[4], dissrc)
catalog.entries[name].add_quantity(SUPERNOVA.HOST, row[5], source)
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT,
row[6],
source,
e_value=row[7] if (row[7] and float(row[7]) != 0.0) else '')
photodict = {
PHOTOMETRY.TIME: jd_to_mjd(Decimal(row[8])),
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.ENERGY: row[15:17],
PHOTOMETRY.U_ENERGY: 'keV',
PHOTOMETRY.FLUX: str(Decimal('1.0e-13') * Decimal(row[11])),
PHOTOMETRY.U_FLUX: 'ergs/s/cm^2',
PHOTOMETRY.E_LOWER_FLUX:
str(Decimal('1.0e-13') * Decimal(row[13])),
PHOTOMETRY.E_UPPER_FLUX:
str(Decimal('1.0e-13') * Decimal(row[14])),
PHOTOMETRY.INSTRUMENT: row[9],
PHOTOMETRY.SOURCE: flxsrc
}
if row[12] == '1':
photodict[PHOTOMETRY.UPPER_LIMIT] = True
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
return
