def _get_first_light(self):
if self._KEYS.PHOTOMETRY not in self:
return None, None
eventphoto = [(Decimal(x[PHOTOMETRY.TIME])
if not isinstance(x[PHOTOMETRY.TIME], list) else
Decimal(min(float(y) for y in x[PHOTOMETRY.TIME])),
x[PHOTOMETRY.SOURCE])
for x in self[self._KEYS.PHOTOMETRY]
if PHOTOMETRY.UPPER_LIMIT not in x and PHOTOMETRY.TIME in
x and x.get(PHOTOMETRY.U_TIME, '') == 'MJD' and
PHOTOMETRY.INCLUDES_HOST not in x]
# Use photometry that includes host if no other photometry available.
if not eventphoto:
eventphoto = [
(Decimal(x[PHOTOMETRY.TIME])
if not isinstance(x[PHOTOMETRY.TIME], list) else
Decimal(min(float(y) for y in x[PHOTOMETRY.TIME])),
x[PHOTOMETRY.SOURCE]) for x in self[self._KEYS.PHOTOMETRY]
if PHOTOMETRY.UPPER_LIMIT not in x and PHOTOMETRY.TIME in x and
x.get(PHOTOMETRY.U_TIME, '') == 'MJD'
]
if not eventphoto:
return None, None
flmjd = min([x[0] for x in eventphoto])
flindex = [x[0] for x in eventphoto].index(flmjd)
flmjd = astrotime(float(flmjd), format='mjd').datetime
flsource = eventphoto[flindex][1]
return flmjd, flsource
def do_ps_mds(catalog):
task_str = catalog.get_current_task_str()
with open(
os.path.join(catalog.get_current_task_repo(),
'MDS/apj506838t1_mrt.txt')) as f:
for ri, row in enumerate(pbar(f.read().splitlines(), task_str)):
if ri < 35:
continue
cols = [x.strip() for x in row.split(',')]
name = catalog.add_entry(cols[0])
source = catalog.entries[name].add_source(
bibcode='2015ApJ...799..208S')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, cols[2], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, cols[3], source)
astrot = astrotime(float(cols[4]), format='mjd').datetime
ddate = make_date_string(astrot.year, astrot.month, astrot.day)
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, ddate,
source)
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, cols[5], source, kind='spectroscopic')
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, 'II P',
source)
catalog.journal_entries()
return
def _get_first_light(self):
if self._KEYS.PHOTOMETRY not in self:
return None, None
eventphoto = [
(Decimal(x[PHOTOMETRY.TIME])
if not isinstance(x[PHOTOMETRY.TIME], list) else Decimal(
min(float(y)
for y in x[PHOTOMETRY.TIME])), x[PHOTOMETRY.SOURCE])
for x in self[self._KEYS.PHOTOMETRY]
if PHOTOMETRY.UPPER_LIMIT not in x and PHOTOMETRY.TIME in x
and x.get(PHOTOMETRY.U_TIME, '') == 'MJD'
and PHOTOMETRY.INCLUDES_HOST not in x
]
# Use photometry that includes host if no other photometry available.
if not eventphoto:
eventphoto = [
(Decimal(x[PHOTOMETRY.TIME])
if not isinstance(x[PHOTOMETRY.TIME], list) else Decimal(
min(float(y)
for y in x[PHOTOMETRY.TIME])), x[PHOTOMETRY.SOURCE])
for x in self[self._KEYS.PHOTOMETRY]
if PHOTOMETRY.UPPER_LIMIT not in x and PHOTOMETRY.TIME in x
and x.get(PHOTOMETRY.U_TIME, '') == 'MJD'
]
if not eventphoto:
return None, None
flmjd = min([x[0] for x in eventphoto])
flindex = [x[0] for x in eventphoto].index(flmjd)
flmjd = astrotime(float(flmjd), format='mjd').datetime
flsource = eventphoto[flindex][1]
return flmjd, flsource
def _get_max_light(self):
if self._KEYS.PHOTOMETRY not in self:
return (None, None, None, None)
# FIX: THIS
eventphoto = [(x['u_time'], x['time'],
Decimal(x['magnitude']), x[
'band'] if 'band' in x else '',
x['source']) for x in self[self._KEYS.PHOTOMETRY] if
('magnitude' in x and 'time' in x and 'u_time' in x and
'upperlimit' not in x)]
if not eventphoto:
return None, None, None, None
mlmag = None
for mb in MAX_BANDS:
leventphoto = [x for x in eventphoto if x[3] in mb]
if leventphoto:
mlmag = min([x[2] for x in leventphoto])
eventphoto = leventphoto
break
if not mlmag:
mlmag = min([x[2] for x in eventphoto])
mlindex = [x[2] for x in eventphoto].index(mlmag)
mlband = eventphoto[mlindex][3]
mlsource = eventphoto[mlindex][4]
if eventphoto[mlindex][0] == 'MJD':
mlmjd = float(eventphoto[mlindex][1])
mlmjd = astrotime(mlmjd, format='mjd').datetime
return mlmjd, mlmag, mlband, mlsource
else:
return None, mlmag, mlband, mlsource
def __init__(self, parent, **kwargs):
"""Initialize."""
self._REQ_KEY_SETS = [[PHOTOMETRY.SOURCE, PHOTOMETRY.MODEL],
[PHOTOMETRY.TIME, PHOTOMETRY.HOST],
[
PHOTOMETRY.MAGNITUDE, PHOTOMETRY.FLUX,
PHOTOMETRY.UNABSORBED_FLUX,
PHOTOMETRY.FLUX_DENSITY,
PHOTOMETRY.COUNT_RATE, PHOTOMETRY.LUMINOSITY
]]
# Note: `_check()` is called at end of `super().__init__`
super(Photometry, self).__init__(parent, **kwargs)
# If `BAND` is given, but any of `bandmetaf_keys` is not, try to infer
if self._KEYS.BAND in self:
sband = self[self._KEYS.BAND]
bandmetaf_keys = [
self._KEYS.INSTRUMENT, self._KEYS.TELESCOPE, self._KEYS.SYSTEM
]
for bmf in bandmetaf_keys:
if bmf not in self:
temp = bandmetaf(sband, bmf)
if temp is not None:
self[bmf] = temp
# Convert dates to MJD
timestrs = [str(x) for x in listify(self.get(self._KEYS.TIME, ''))]
for ti, timestr in enumerate(timestrs):
if (any(x in timestr for x in ['-', '/'])
and not timestr.startswith('-')):
timestrs[ti] = timestr.replace('/', '-')
try:
timestrs[ti] = str(
astrotime(timestrs[ti], format='isot').mjd)
except Exception:
raise CatDictError('Unable to convert date to MJD.')
elif timestr: # Make sure time is string
timestrs[ti] = timestr
if len(timestrs) > 0 and timestrs[0] != '':
self[self._KEYS.
TIME] = timestrs if len(timestrs) > 1 else timestrs[0]
# Time unit is necessary for maximum time determination
if self._KEYS.U_TIME not in self and self._KEYS.TIME in self:
self._log.info('`{}` not found in photometry, assuming '
' MJD.'.format(self._KEYS.U_TIME))
self[self._KEYS.U_TIME] = 'MJD'
if (self._KEYS.U_COUNT_RATE not in self
and self._KEYS.COUNT_RATE in self):
self._log.info('`{}` not found in photometry, assuming '
' s^-1.'.format(self._KEYS.U_COUNT_RATE))
self[self._KEYS.U_COUNT_RATE] = 's^-1'
return
def get_first_light(name):
if 'photometry' not in events[name]:
return None
eventtime = [Decimal(x['time']) for x in events[name]['photometry'] if 'upperlimit' not in x and 'timeunit' in x and x['timeunit'] == 'MJD']
if not eventtime:
return None
flindex = eventtime.index(min(eventtime))
flmjd = float(eventtime[flindex])
return astrotime(flmjd, format='mjd').datetime
def print(self):
print("ident = " + str(self.ident) + ", time = " + str(self.time) +
" = " +
str(astrotime(self.time, format="jd", scale='utc').isot) +
", x = " + str(self.x) + ", y = " + str(self.y) + ", z = " +
str(self.z) + ", Vx = " + str(self.Vx) + ", Vy = " +
str(self.Vy) + ", Vz = " + str(self.Vz) + ", mass = " +
str(self.mass) + ", radius = " + str(self.radius) +
", color1 = " + str(self.color1) + ", color2 = " +
str(self.color2) + ", color3 = " + str(self.color3))
def display(self):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glMatrixMode(GL_MODELVIEW)
glLoadIdentity()
gluLookAt(
init.eyeRho * math.sin(init.eyePhi) * math.sin(init.eyeTheta),
init.eyeRho * math.cos(init.eyePhi),
init.eyeRho * math.sin(init.eyePhi) * math.cos(init.eyeTheta),
init.look[0], init.look[1], init.look[2], 0, init.upY, 0)
light_sun_position = [
planet_system.system[0].x, planet_system.system[0].y,
planet_system.system[0].z
]
glLightfv(GL_LIGHT0, GL_POSITION, light_sun_position)
self.glut_print(10, 10, GLUT_BITMAP_9_BY_15,
astrotime(self.system[0].time, format='jd').iso, 1.0,
1.0, 1.0, 1.0)
self.if_collision()
if init.display:
for body in self.system:
glPushMatrix()
glTranslated(init.SCALE * body.x, init.SCALE * body.y,
init.SCALE * body.z)
#glRotatef((-1)*init.eyeRho * math.sin(init.eyePhi) * math.sin(init.eyeTheta), 1, 0, 0)
#glRotatef((-1)*init.eyeRho * math.cos(init.eyePhi), 0, 1, 0)
#glRotatef((-1)*init.eyeRho * math.sin(init.eyePhi) * math.sin(init.eyeTheta), 0, 0, 1)
body.display()
glPopMatrix()
glDisable(GL_LIGHTING)
glLineWidth(1)
glColor3f(body.color1 / 255, body.color2 / 255,
body.color3 / 255)
if init.orbit == True:
glBegin(GL_LINE_STRIP)
dis_coord = body.coord
if init.short_orbit:
if len(dis_coord) > ORBIT_LENGTH:
dis_coord = dis_coord[(-1) * ORBIT_LENGTH:]
for point in dis_coord:
glVertex3f(init.SCALE * point[0],
init.SCALE * point[1],
init.SCALE * point[2])
glEnd()
self.glut_print3(init.SCALE * body.x, init.SCALE * body.y,
init.SCALE * body.z, GLUT_BITMAP_9_BY_15,
body.ident, body.color1 / 255,
body.color2 / 255, body.color3 / 255, 1.0)
glEnable(GL_LIGHTING)
glutSwapBuffers()
def set_first_max_light(self):
if ENTRY.MAX_APP_MAG not in self:
mldt, mlmag, mlband, mlsource = self._get_max_light()
if mldt or mlmag or mlband:
source = self.add_self_source()
uniq_src = uniq_cdl([source] + mlsource.split(','))
if mldt:
max_date = make_date_string(mldt.year, mldt.month, mldt.day)
self.add_quantity(ENTRY.MAX_DATE, max_date, uniq_src,
derived=True)
if mlmag:
mlmag = pretty_num(mlmag)
self.add_quantity(ENTRY.MAX_APP_MAG, mlmag, uniq_src,
derived=True)
if mlband:
self.add_quantity(ENTRY.MAX_BAND, mlband, uniq_src,
derived=True)
if (self._KEYS.DISCOVER_DATE not in self or
max([len(x[QUANTITY.VALUE].split('/')) for x in
self[self._KEYS.DISCOVER_DATE]]) < 3):
fldt, flsource = self._get_first_light()
if fldt:
source = self.add_self_source()
disc_date = make_date_string(fldt.year, fldt.month, fldt.day)
self.add_quantity(
self._KEYS.DISCOVER_DATE, disc_date,
uniq_cdl([source] + flsource.split(',')),
derived=True)
if self._KEYS.DISCOVER_DATE not in self and self._KEYS.SPECTRA in self:
minspecmjd = float("+inf")
for spectrum in self[self._KEYS.SPECTRA]:
if 'time' in spectrum and 'u_time' in spectrum:
if spectrum['u_time'] == 'MJD':
mjd = float(spectrum['time'])
elif spectrum['u_time'] == 'JD':
mjd = float(jd_to_mjd(Decimal(spectrum['time'])))
else:
continue
if mjd < minspecmjd:
minspecmjd = mjd
minspecsource = spectrum['source']
if minspecmjd < float("+inf"):
fldt = astrotime(minspecmjd, format='mjd').datetime
source = self.add_self_source()
disc_date = make_date_string(fldt.year, fldt.month, fldt.day)
self.add_quantity(
self._KEYS.DISCOVER_DATE, disc_date,
uniq_cdl([source] + minspecsource.split(',')),
derived=True)
return
def _get_max_light(self, visual=False):
if self._KEYS.PHOTOMETRY not in self:
return (None, None, None, None)
eventphoto = [
(x[PHOTOMETRY.U_TIME], Decimal(x[PHOTOMETRY.TIME])
if not isinstance(x[PHOTOMETRY.TIME], list) else Decimal(
np.mean(float(y) for y in x[PHOTOMETRY.TIME])),
Decimal(x[PHOTOMETRY.MAGNITUDE]), x.get(PHOTOMETRY.BAND,
''), x[PHOTOMETRY.SOURCE])
for x in self[self._KEYS.PHOTOMETRY]
if (PHOTOMETRY.MAGNITUDE in x and PHOTOMETRY.TIME in x and x.get(
PHOTOMETRY.U_TIME, '') == 'MJD' and PHOTOMETRY.UPPER_LIMIT
not in x and not x.get(PHOTOMETRY.INCLUDES_HOST, False))
]
# Use photometry that includes host if no other photometry available.
if not eventphoto:
eventphoto = [
(x[PHOTOMETRY.U_TIME], Decimal(x[PHOTOMETRY.TIME])
if not isinstance(x[PHOTOMETRY.TIME], list) else Decimal(
np.mean(float(y) for y in x[PHOTOMETRY.TIME])),
Decimal(x[PHOTOMETRY.MAGNITUDE]), x.get(PHOTOMETRY.BAND, ''),
x[PHOTOMETRY.SOURCE]) for x in self[self._KEYS.PHOTOMETRY]
if (PHOTOMETRY.MAGNITUDE in x and PHOTOMETRY.TIME in x
and x.get(PHOTOMETRY.U_TIME, '') == 'MJD'
and not x.get(PHOTOMETRY.INCLUDES_HOST, False))
]
if not eventphoto:
return None, None, None, None
mlmag = None
if visual:
for mb in MAX_VISUAL_BANDS:
leventphoto = [x for x in eventphoto if x[3] in mb]
if leventphoto:
mlmag = min([x[2] for x in leventphoto])
eventphoto = leventphoto
break
if not mlmag:
mlmag = min([x[2] for x in eventphoto])
mlindex = [x[2] for x in eventphoto].index(mlmag)
mlband = eventphoto[mlindex][3]
mlsource = eventphoto[mlindex][4]
if eventphoto[mlindex][0] == 'MJD':
mlmjd = float(eventphoto[mlindex][1])
mlmjd = astrotime(mlmjd, format='mjd').datetime
return mlmjd, mlmag, mlband, mlsource
else:
return None, mlmag, mlband, mlsource
def _get_max_light(self, visual=False):
if self._KEYS.PHOTOMETRY not in self:
return (None, None, None, None)
eventphoto = [
(x[PHOTOMETRY.U_TIME], Decimal(x[PHOTOMETRY.TIME])
if not isinstance(x[PHOTOMETRY.TIME], list) else
Decimal(np.mean([float(y) for y in x[PHOTOMETRY.TIME]])),
Decimal(x[PHOTOMETRY.MAGNITUDE]), x.get(PHOTOMETRY.BAND, ''),
x[PHOTOMETRY.SOURCE]) for x in self[self._KEYS.PHOTOMETRY]
if (PHOTOMETRY.MAGNITUDE in x and PHOTOMETRY.TIME in x and x.get(
PHOTOMETRY.U_TIME, '') == 'MJD' and PHOTOMETRY.UPPER_LIMIT
not in x and not x.get(PHOTOMETRY.INCLUDES_HOST, False))
]
# Use photometry that includes host if no other photometry available.
if not eventphoto:
eventphoto = [
(x[PHOTOMETRY.U_TIME], Decimal(x[PHOTOMETRY.TIME])
if not isinstance(x[PHOTOMETRY.TIME], list) else
Decimal(np.mean([float(y) for y in x[PHOTOMETRY.TIME]])),
Decimal(x[PHOTOMETRY.MAGNITUDE]), x.get(PHOTOMETRY.BAND, ''),
x[PHOTOMETRY.SOURCE]) for x in self[self._KEYS.PHOTOMETRY]
if (PHOTOMETRY.MAGNITUDE in x and PHOTOMETRY.TIME in x and
x.get(PHOTOMETRY.U_TIME, '') == 'MJD' and not x.get(
PHOTOMETRY.INCLUDES_HOST, False))
]
if not eventphoto:
return None, None, None, None
mlmag = None
if visual:
for mb in MAX_VISUAL_BANDS:
leventphoto = [x for x in eventphoto if x[3] in mb]
if leventphoto:
mlmag = min([x[2] for x in leventphoto])
eventphoto = leventphoto
break
if not mlmag:
mlmag = min([x[2] for x in eventphoto])
mlindex = [x[2] for x in eventphoto].index(mlmag)
mlband = eventphoto[mlindex][3]
mlsource = eventphoto[mlindex][4]
if eventphoto[mlindex][0] == 'MJD':
mlmjd = float(eventphoto[mlindex][1])
mlmjd = astrotime(mlmjd, format='mjd').datetime
return mlmjd, mlmag, mlband, mlsource
else:
return None, mlmag, mlband, mlsource
def get_max_light(name):
if 'photometry' not in events[name]:
return (None, None)
eventphoto = [(x['timeunit'], x['time'], Decimal(x['magnitude'])) for x in events[name]['photometry'] if 'magnitude' in x]
if not eventphoto:
return (None, None)
mlmag = min([x[2] for x in eventphoto])
mlindex = [x[2] for x in eventphoto].index(mlmag)
if eventphoto[mlindex][0] == 'MJD':
mlmjd = float(eventphoto[mlindex][1])
return (astrotime(mlmjd, format='mjd').datetime, mlmag)
else:
return (None, mlmag)
def do_ptss_meta(catalog):
"""Import metadata from PTSS webpage."""
task_str = catalog.get_current_task_str()
years = list(range(2015, datetime.today().year + 1))
for year in years:
jsontxt = None
while jsontxt is None:
try:
jsontxt = catalog.load_url(
'http://www.cneost.org/ptss/fetchlist.php?vip=sn&gdate=' +
str(year),
os.path.join(catalog.get_current_task_repo(),
'PTSS/catalog-' + str(year) + '.json'),
json_sort='name',
timeout=5)
except Exception:
pass
meta = json.loads(jsontxt)
for met in pbar(meta, task_str + ' - ' + str(year)):
oldname = met['name']
name, source = catalog.new_entry(
oldname,
srcname='PMO & Tsinghua Supernova Survey (PTSS)',
url='http://www.cneost.org/ptss/index.php')
coo = coord(met['ra'], met['dec'], unit=(un.deg, un.deg))
catalog.entries[name].add_quantity(
SUPERNOVA.RA, coo.ra.to_string(unit=un.hour, sep=':'), source)
catalog.entries[name].add_quantity(
SUPERNOVA.DEC, coo.dec.to_string(unit=un.degree, sep=':'),
source)
if met['filter'] is not None:
mjd = str(astrotime(met['obsdate'], format='isot').mjd)
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.MAGNITUDE: str(met['mag']),
PHOTOMETRY.E_MAGNITUDE: str(met['magerr']),
PHOTOMETRY.BAND: met['filter'].replace('sdss-', ''),
PHOTOMETRY.SOURCE: source
}
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
return
def _get_first_light(self):
if self._KEYS.PHOTOMETRY not in self:
return None, None
# FIX THIS
eventphoto = [(Decimal(x['time']) if isinstance(x['time'], str) else
Decimal(min(float(y) for y in x['time'])),
x['source']) for x in self[self._KEYS.PHOTOMETRY] if
'upperlimit' not in x and
'time' in x and 'u_time' in x and x['u_time'] == 'MJD']
if not eventphoto:
return None, None
flmjd = min([x[0] for x in eventphoto])
flindex = [x[0] for x in eventphoto].index(flmjd)
flmjd = astrotime(float(flmjd), format='mjd').datetime
flsource = eventphoto[flindex][1]
return flmjd, flsource
def do_ptss_meta(catalog):
"""Import metadata from PTSS webpage."""
task_str = catalog.get_current_task_str()
years = list(range(2015, datetime.today().year + 1))
for year in years:
jsontxt = None
while jsontxt is None:
try:
jsontxt = catalog.load_url(
'http://www.cneost.org/ptss/fetchlist.php?vip=sn&gdate=' +
str(year),
os.path.join(catalog.get_current_task_repo(),
'PTSS/catalog-' + str(year) + '.json'),
json_sort='name', timeout=5)
except Exception:
pass
meta = json.loads(jsontxt)
for met in pbar(meta, task_str + ' - ' + str(year)):
oldname = met['name']
name, source = catalog.new_entry(
oldname, srcname='PMO & Tsinghua Supernova Survey (PTSS)',
url='http://www.cneost.org/ptss/index.php')
coo = coord(met['ra'], met['dec'], unit=(un.deg, un.deg))
catalog.entries[name].add_quantity(
SUPERNOVA.RA, coo.ra.to_string(unit=un.hour, sep=':'), source)
catalog.entries[name].add_quantity(
SUPERNOVA.DEC, coo.dec.to_string(unit=un.degree, sep=':'),
source)
if met['filter'] is not None:
mjd = str(astrotime(met['obsdate'], format='isot').mjd)
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.MAGNITUDE: str(met['mag']),
PHOTOMETRY.E_MAGNITUDE: str(met['magerr']),
PHOTOMETRY.BAND: met['filter'].replace('sdss-', ''),
PHOTOMETRY.SOURCE: source
}
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
return
def do_sdss_spectra(catalog):
"""Import spectra from LAMOST."""
task_str = catalog.get_current_task_str()
# Set preferred names, calculate some columns based on imported data,
# sanitize some fields
keys = list(catalog.entries.keys())
fureps = {
'erg/cm2/s/A': 'erg/s/cm^2/Angstrom',
'1E-17 erg/cm^2/s/Ang': 'erg/s/cm^2/Angstrom'
}
c_kms = con.c.cgs.value / 1.0e5
cntsdss = 0
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
if (FASTSTARS.RA not in catalog.entries[oname]
or FASTSTARS.DEC not in catalog.entries[oname]):
continue
else:
xid = SDSS.query_region(coord.SkyCoord(
ra=catalog.entries[oname][FASTSTARS.RA][0]['value'],
dec=catalog.entries[oname][FASTSTARS.DEC][0]['value'],
unit=(un.hourangle, un.deg),
frame='icrs'),
spectro=True)
# xid = SDSS.query_region(coord.SkyCoord(
# ra='14:34:06.17',
# dec='+56:30:47.24',
# unit=(un.hourangle, un.deg), frame='icrs'), spectro=True)
if xid is None:
continue
while len(xid) > 1:
notstar = xid['z'].argmax()
xid.remove_row(notstar)
#print(xid)
# star = None
# for row in tab:
# if (row['objType'] == 'Star' and
# row['Class'].lower() in ['star', 'unknown']):
# star = row
# break
# if not star:
# continue
try:
name, source = catalog.new_entry(oname,
bibcode='2015ApJS..219...12A',
srcname='SDSS',
url='http://www.sdss.org/')
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
ffile = ('spec-' + str(xid['specobjid'][0]) + '.fits.gz')
# furl = 'http://dr3.lamost.org/sas/fits/' + vname + '/' + ffile
datafile = os.path.join(catalog.get_current_task_repo(), 'SDSS',
ffile)
if not os.path.exists(datafile):
# Download spectra
try:
sp = SDSS.get_spectra(matches=xid)[0]
except urllib.error.HTTPError:
catalog.log.warning(
'"{}" threw an HTTP 404, must be error upstream. Will likely go away on the next run.'
.format(oname))
continue
# Identify star
# assert sp[2].data['class'][0]=='STAR'
# Write spectra
# print(catalog.entries[oname][FASTSTARS.RA][0]['value'],catalog.entries[oname][FASTSTARS.DEC][0]['value'])
sp.writeto(datafile, overwrite=True)
# open(datafile, 'wb').write(fr.content)
# Open spectra
hdulist = fits.open(datafile)
# sp contains a list of fits datafiles, identify main one
i_primary = 0
i_coadd = 1
i_specobj = 2
assert hdulist[i_primary].name == 'PRIMARY'
assert hdulist[i_coadd].name == 'COADD'
assert (hdulist[i_specobj].name == 'SPECOBJ'
or hdulist[i_specobj].name == 'SPALL')
# xid = SDSS.query_region(coord.SkyCoord(
# ra='12:11:50.27',
# dec='+14:37:16.2',
# unit=(un.hourangle, un.deg), frame='icrs'), spectro=True)
# from SPECOBJ
# print('.'+hdulist[i_specobj].data['ELODIE_SPTYPE'][0]+'.')
if hdulist[i_specobj].data['ELODIE_SPTYPE'][0] != 'unknown':
ST, SCfull = hdulist[i_specobj].data['ELODIE_SPTYPE'][
0][:2], hdulist[i_specobj].data['ELODIE_SPTYPE'][0][2:]
if len(SCfull) > 0:
if 'IV' in SCfull:
SC = 'sg'
elif 'III' in SCfull:
SC = 'g'
elif 'V' in SCfull:
SC = 'd'
elif 'I' in SCfull:
SC = 'Sg'
else:
SC = False
if SC is not False:
catalog.entries[name].add_quantity(
FASTSTARS.STELLAR_CLASS, SC, source=source)
catalog.entries[name].add_quantity(FASTSTARS.SPECTRAL_TYPE,
ST,
source=source)
if hdulist[i_specobj].data['Z'][0] != 0.0:
catalog.entries[name].add_quantity(
FASTSTARS.REDSHIFT,
str(hdulist[i_specobj].data['Z'][0]),
e_value=str(hdulist[i_specobj].data['Z_ERR'][0]),
source=source)
catalog.entries[name].add_quantity(
FASTSTARS.VELOCITY,
pretty_num(float(hdulist[i_specobj].data['Z'][0]) * c_kms,
sig=5),
e_value=pretty_num(float(
hdulist[i_specobj].data['Z_ERR'][0] * c_kms),
sig=5),
source=source)
for oi, obj in enumerate(hdulist[0].header):
if any(x in ['.', '/'] for x in obj):
del (hdulist[0].header[oi])
hdulist[0].verify('silentfix')
hdrkeys = list(hdulist[0].header.keys())
# print(hdrkeys)
# for key in hdulist[0].header.keys():
# print(key, hdulist[0].header[key])
if hdulist[0].header['SIMPLE']:
if 'JD' in hdrkeys:
mjd = str(jd_to_mjd(Decimal(str(hdulist[0].header['JD']))))
elif 'MJD' in hdrkeys:
mjd = str(hdulist[0].header['MJD'])
elif 'DATE-OBS' in hdrkeys:
if 'T' in hdulist[0].header['DATE-OBS']:
dateobs = hdulist[0].header['DATE-OBS'].strip()
elif 'UTC-OBS' in hdrkeys:
dateobs = hdulist[0].header['DATE-OBS'].strip(
) + 'T' + hdulist[0].header['UTC-OBS'].strip()
mjd = str(astrotime(dateobs, format='isot').mjd)
else:
raise ValueError("Couldn't find JD/MJD for spectrum.")
if hdulist[i_coadd].header['NAXIS'] == 2:
waves = [
str(x)
for x in list(10**hdulist[i_coadd].data['loglam'])
]
fluxes = [
str(x) for x in list(hdulist[i_coadd].data['flux'])
]
else:
print('Warning: Skipping FITS spectrum `{}`.'.format(
datafile))
continue
else:
raise ValueError('Non-simple FITS import not yet supported.')
if 'BUNIT' in hdrkeys:
fluxunit = hdulist[0].header['BUNIT']
if fluxunit in fureps:
fluxunit = fureps[fluxunit]
if fluxunit[:3] == '1E-17':
fluxes = [
str(x * 1e-17)
for x in list(hdulist[i_coadd].data['flux'])
]
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.WAVELENGTHS: waves,
SPECTRUM.TIME: mjd,
SPECTRUM.U_TIME: 'MJD',
SPECTRUM.FLUXES: fluxes,
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.FILENAME: ffile,
SPECTRUM.SOURCE: source
}
if 'TELESCOP' in hdrkeys:
specdict[SPECTRUM.TELESCOPE] = hdulist[0].header['TELESCOP']
if 'INSTRUME' in hdrkeys:
specdict[SPECTRUM.INSTRUMENT] = hdulist[0].header['INSTRUME']
if 'SITENAME' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['SITENAME']
elif 'OBSERVAT' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['OBSERVAT']
if 'OBSERVER' in hdrkeys:
specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER']
if 'AIRMASS' in hdrkeys:
specdict[SPECTRUM.AIRMASS] = hdulist[0].header['AIRMASS']
catalog.entries[name].add_spectrum(**specdict)
cntsdss += 1
hdulist.close()
catalog.journal_entries()
print('`{}` have SDSS spectra.'.format(cntsdss))
return
def do_suspect_spectra(catalog):
task_str = catalog.get_current_task_str()
with open(os.path.join(catalog.get_current_task_repo(),
'Suspect/sources.json'), 'r') as f:
sourcedict = json.loads(f.read())
with open(os.path.join(catalog.get_current_task_repo(),
'Suspect/filename-changes.txt'), 'r') as f:
rows = f.readlines()
changedict = {}
for row in rows:
if not row.strip() or row[0] == "#":
continue
items = row.strip().split(' ')
changedict[items[1]] = items[0]
suspectcnt = 0
folders = next(os.walk(os.path.join(
catalog.get_current_task_repo(), 'Suspect')))[1]
for folder in pbar(folders, task_str):
eventfolders = next(os.walk(os.path.join(
catalog.get_current_task_repo(), 'Suspect/') + folder))[1]
oldname = ''
for eventfolder in pbar(eventfolders, task_str):
name = eventfolder
if is_number(name[:4]):
name = 'SN' + name
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
sec_ref = 'SUSPECT'
sec_refurl = 'https://www.nhn.ou.edu/~suspect/'
sec_bibc = '2001AAS...199.8408R'
sec_source = catalog.entries[name].add_source(
name=sec_ref, url=sec_refurl, bibcode=sec_bibc,
secondary=True)
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, name, sec_source)
fpath = os.path.join(catalog.get_current_task_repo(),
'Suspect', folder, eventfolder)
eventspectra = next(os.walk(fpath))[2]
for spectrum in eventspectra:
sources = [sec_source]
bibcode = ''
if spectrum in changedict:
specalias = changedict[spectrum]
else:
specalias = spectrum
if specalias in sourcedict:
bibcode = sourcedict[specalias]
elif name in sourcedict:
bibcode = sourcedict[name]
if bibcode:
source = catalog.entries[name].add_source(
bibcode=unescape(bibcode))
sources += [source]
sources = uniq_cdl(sources)
date = spectrum.split('_')[1]
year = date[:4]
month = date[4:6]
day = date[6:]
sig = get_sig_digits(day) + 5
day_fmt = str(floor(float(day))).zfill(2)
time = astrotime(year + '-' + month + '-' + day_fmt).mjd
time = time + float(day) - floor(float(day))
time = pretty_num(time, sig=sig)
fpath = os.path.join(catalog.get_current_task_repo(),
'Suspect',
folder,
eventfolder, spectrum)
with open(fpath, 'r') as f:
specdata = list(csv.reader(
f, delimiter=' ', skipinitialspace=True))
specdata = list(filter(None, specdata))
newspec = []
oldval = ''
for row in specdata:
if row[1] == oldval:
continue
newspec.append(row)
oldval = row[1]
specdata = newspec
haserrors = len(specdata[0]) == 3 and specdata[
0][2] and specdata[0][2] != 'NaN'
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom', u_fluxes='Uncalibrated',
u_time='MJD',
time=time,
wavelengths=wavelengths, fluxes=fluxes, errors=errors,
u_errors='Uncalibrated',
source=sources, filename=spectrum)
suspectcnt = suspectcnt + 1
if (catalog.args.travis and
suspectcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
def do_psst(catalog):
task_str = catalog.get_current_task_str()
# 2016MNRAS.462.4094S
file_path = os.path.join(catalog.get_current_task_repo(), 'ASCII',
'2016MNRAS.462.4094S-tab1.tsv')
with open(file_path, 'r') as f:
data = list(
csv.reader(
f, delimiter='\t', quotechar='"', skipinitialspace=True))
for r, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
continue
(name, source) = catalog.new_entry(
row[0], bibcode='2016MNRAS.462.4094S')
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, row[3].replace('SN', '').strip('() '),
source)
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT,
row[5].strip('() '),
source,
kind='spectroscopic')
file_path = os.path.join(catalog.get_current_task_repo(), 'ASCII',
'2016MNRAS.462.4094S-tab2.tsv')
with open(file_path, 'r') as f:
data = list(
csv.reader(
f, delimiter='\t', quotechar='"', skipinitialspace=True))
for r, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
continue
(name, source) = catalog.new_entry(
row[0], bibcode='2016MNRAS.462.4094S')
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[1], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[2], source)
mldt = astrotime(float(row[4]), format='mjd').datetime
discoverdate = make_date_string(mldt.year, mldt.month, mldt.day)
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE,
discoverdate, source)
catalog.journal_entries()
# 2016ApJ...827L..40S
file_path = os.path.join(catalog.get_current_task_repo(), 'ASCII',
'2016ApJ...827L..40S.tsv')
with open(file_path, 'r') as f:
data = list(
csv.reader(
f, delimiter='\t', quotechar='"', skipinitialspace=True))
for r, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
continue
(name, source) = catalog.new_entry(
row[0], bibcode='2016ApJ...827L..40S')
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[1], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[2], source)
mldt = astrotime(float(row[3]), format='mjd').datetime
discoverdate = make_date_string(mldt.year, mldt.month, mldt.day)
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE,
discoverdate, source)
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, row[6],
source)
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, row[7], source, kind='spectroscopic')
for alias in [x.strip() for x in row[8].split(',')]:
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, alias,
source)
catalog.journal_entries()
return
def do_ascii(catalog):
"""Process ASCII files that were extracted from datatables appearing in
published works.
"""
task_str = catalog.get_current_task_str()
# 2006ApJ...645..841N
file_path = os.path.join(
catalog.get_current_task_repo(), '2006ApJ...645..841N-table3.csv')
tsvin = list(csv.reader(open(file_path, 'r'), delimiter=','))
for ri, row in enumerate(pbar(tsvin, task_str)):
name = 'SNLS-' + row[0]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2006ApJ...645..841N')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, row[1], source, kind='spectroscopic')
astrot = astrotime(float(row[4]) + 2450000., format='jd').datetime
date_str = make_date_string(astrot.year, astrot.month, astrot.day)
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, date_str, source)
catalog.journal_entries()
# Anderson 2014
file_names = list(
glob(os.path.join(
catalog.get_current_task_repo(), 'SNII_anderson2014/*.dat')))
for datafile in pbar_strings(file_names, task_str):
basename = os.path.basename(datafile)
if not is_number(basename[:2]):
continue
if basename == '0210_V.dat':
name = 'SN0210'
else:
name = ('SN20' if int(basename[:2]) <
50 else 'SN19') + basename.split('_')[0]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2014ApJ...786...67A')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
if name in ['SN1999ca', 'SN2003dq', 'SN2008aw']:
system = 'Swope'
else:
system = 'Landolt'
with open(datafile, 'r') as ff:
tsvin = csv.reader(ff, delimiter=' ', skipinitialspace=True)
for row in tsvin:
if not row[0]:
continue
time = str(jd_to_mjd(Decimal(row[0])))
catalog.entries[name].add_photometry(
time=time, band='V',
magnitude=row[1], e_magnitude=row[2],
system=system, source=source)
catalog.journal_entries()
# stromlo
stromlobands = ['B', 'V', 'R', 'I', 'VM', 'RM']
file_path = os.path.join(
catalog.get_current_task_repo(), 'J_A+A_415_863-1/photometry.csv')
tsvin = list(csv.reader(open(file_path, 'r'), delimiter=','))
for row in pbar(tsvin, task_str):
name = row[0]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2004A&A...415..863G')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
mjd = str(jd_to_mjd(Decimal(row[1])))
for ri, ci in enumerate(range(2, len(row), 3)):
if not row[ci]:
continue
band = stromlobands[ri]
upperlimit = True if (not row[ci + 1] and row[ci + 2]) else False
e_upper_magnitude = str(
abs(Decimal(row[ci + 1]))) if row[ci + 1] else ''
e_lower_magnitude = str(
abs(Decimal(row[ci + 2]))) if row[ci + 2] else ''
teles = 'MSSSO 1.3m' if band in ['VM', 'RM'] else 'CTIO'
instr = 'MaCHO' if band in ['VM', 'RM'] else ''
catalog.entries[name].add_photometry(
time=mjd, band=band, magnitude=row[ci],
e_upper_magnitude=e_upper_magnitude,
e_lower_magnitude=e_lower_magnitude,
upperlimit=upperlimit, telescope=teles,
instrument=instr, source=source)
catalog.journal_entries()
# 2015MNRAS.449..451W
file_path = os.path.join(
catalog.get_current_task_repo(), '2015MNRAS.449..451W.dat')
data = list(csv.reader(open(file_path, 'r'), delimiter='\t',
quotechar='"', skipinitialspace=True))
for rr, row in enumerate(pbar(data, task_str)):
if rr == 0:
continue
namesplit = row[0].split('/')
name = namesplit[-1]
if name.startswith('SN'):
name = name.replace(' ', '')
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2015MNRAS.449..451W')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
if len(namesplit) > 1:
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, namesplit[0], source)
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, row[1], source)
catalog.entries[name].add_photometry(
time=row[2], band=row[4], magnitude=row[3], source=source)
catalog.journal_entries()
# 2016MNRAS.459.1039T
file_path = os.path.join(
catalog.get_current_task_repo(), '2016MNRAS.459.1039T.tsv')
data = list(csv.reader(open(file_path, 'r'), delimiter='\t',
quotechar='"', skipinitialspace=True))
name = catalog.add_entry('LSQ13zm')
source = catalog.entries[name].add_source(bibcode='2016MNRAS.459.1039T')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
for rr, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
bands = [xx.replace('(err)', '') for xx in row[3:-1]]
continue
mjd = row[1]
mags = [re.sub(r'\([^)]*\)', '', xx) for xx in row[3:-1]]
upps = [True if '>' in xx else '' for xx in mags]
mags = [xx.replace('>', '') for xx in mags]
errs = [xx[xx.find('(') + 1:xx.find(')')]
if '(' in xx else '' for xx in row[3:-1]]
for mi, mag in enumerate(mags):
if not is_number(mag):
continue
catalog.entries[name].add_photometry(
time=mjd, band=bands[mi], magnitude=mag, e_magnitude=errs[mi],
instrument=row[-1], upperlimit=upps[mi], source=source)
catalog.journal_entries()
# 2015ApJ...804...28G
file_path = os.path.join(
catalog.get_current_task_repo(), '2015ApJ...804...28G.tsv')
data = list(csv.reader(open(file_path, 'r'), delimiter='\t',
quotechar='"', skipinitialspace=True))
name = catalog.add_entry('PS1-13arp')
source = catalog.entries[name].add_source(bibcode='2015ApJ...804...28G')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
for rr, row in enumerate(pbar(data, task_str)):
if rr == 0:
continue
mjd = row[1]
mag = row[3]
upp = True if '<' in mag else ''
mag = mag.replace('<', '')
err = row[4] if is_number(row[4]) else ''
ins = row[5]
catalog.entries[name].add_photometry(
time=mjd, band=row[0], magnitude=mag, e_magnitude=err,
instrument=ins, upperlimit=upp, source=source)
catalog.journal_entries()
# 2016ApJ...819...35A
file_path = os.path.join(
catalog.get_current_task_repo(), '2016ApJ...819...35A.tsv')
data = list(csv.reader(open(file_path, 'r'), delimiter='\t',
quotechar='"', skipinitialspace=True))
for rr, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
continue
name = catalog.add_entry(row[0])
source = catalog.entries[name].add_source(
bibcode='2016ApJ...819...35A')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[1], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[2], source)
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, row[3], source)
disc_date = datetime.strptime(row[4], '%Y %b %d').isoformat()
disc_date = disc_date.split('T')[0].replace('-', '/')
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, disc_date, source)
catalog.journal_entries()
# 2014ApJ...784..105W
file_path = os.path.join(
catalog.get_current_task_repo(), '2014ApJ...784..105W.tsv')
data = list(csv.reader(open(file_path, 'r'), delimiter='\t',
quotechar='"', skipinitialspace=True))
for rr, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
continue
name = catalog.add_entry(row[0])
source = catalog.entries[name].add_source(
bibcode='2014ApJ...784..105W')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
mjd = row[1]
band = row[2]
mag = row[3]
err = row[4]
catalog.entries[name].add_photometry(
time=mjd, band=row[2], magnitude=mag, e_magnitude=err,
instrument='WHIRC', telescope='WIYN 3.5 m', observatory='NOAO',
system='WHIRC', source=source)
catalog.journal_entries()
# 2012MNRAS.425.1007B
file_path = os.path.join(
catalog.get_current_task_repo(), '2012MNRAS.425.1007B.tsv')
data = list(csv.reader(open(file_path, 'r'), delimiter='\t',
quotechar='"', skipinitialspace=True))
for rr, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
bands = row[2:]
continue
name = catalog.add_entry(row[0])
source = catalog.entries[name].add_source(
bibcode='2012MNRAS.425.1007B')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
mjd = row[1]
mags = [xx.split('±')[0].strip() for xx in row[2:]]
errs = [xx.split('±')[1].strip()
if '±' in xx else '' for xx in row[2:]]
if row[0] == 'PTF09dlc':
ins = 'HAWK-I'
tel = 'VLT 8.1m'
obs = 'ESO'
else:
ins = 'NIRI'
tel = 'Gemini North 8.2m'
obs = 'Gemini'
for mi, mag in enumerate(mags):
if not is_number(mag):
continue
catalog.entries[name].add_photometry(
time=mjd, band=bands[mi], magnitude=mag, e_magnitude=errs[mi],
instrument=ins, telescope=tel, observatory=obs,
system='Natural', source=source)
catalog.journal_entries()
# 2014ApJ...783...28G
file_path = os.path.join(
catalog.get_current_task_repo(), 'apj490105t2_ascii.txt')
with open(file_path, 'r') as f:
data = list(csv.reader(f, delimiter='\t',
quotechar='"', skipinitialspace=True))
for r, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
continue
name, source = catalog.new_entry(
row[0], bibcode='2014ApJ...783...28G')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[1], source)
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, '20' + row[0][3:5], source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[2], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[3], source)
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, row[13] if is_number(row[13]) else
row[10], source)
catalog.journal_entries()
# 2005ApJ...634.1190H
file_path = os.path.join(
catalog.get_current_task_repo(), '2005ApJ...634.1190H.tsv')
with open(file_path, 'r') as f:
data = list(csv.reader(f, delimiter='\t',
quotechar='"', skipinitialspace=True))
for r, row in enumerate(pbar(data, task_str)):
name, source = catalog.new_entry(
'SNLS-' + row[0], bibcode='2005ApJ...634.1190H')
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, '20' + row[0][:2], source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[1], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[2], source)
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, row[5].replace('?', ''), source,
e_value=row[6], kind='host')
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, row[7].replace('SN', '').strip(':* '),
source)
catalog.journal_entries()
# 2014MNRAS.444.2133S
file_path = os.path.join(
catalog.get_current_task_repo(), '2014MNRAS.444.2133S.tsv')
with open(file_path, 'r') as f:
data = list(csv.reader(f, delimiter='\t',
quotechar='"', skipinitialspace=True))
for r, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
continue
name = row[0]
if is_number(name[:4]):
name = 'SN' + name
name, source = catalog.new_entry(
name, bibcode='2014MNRAS.444.2133S')
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[1], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[2], source)
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, row[3],
source,
kind='host')
catalog.journal_entries()
# 2009MNRAS.398.1041B
file_path = os.path.join(
catalog.get_current_task_repo(), '2009MNRAS.398.1041B.tsv')
with open(file_path, 'r') as f:
data = list(csv.reader(f, delimiter='\t',
quotechar='"', skipinitialspace=True))
for r, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
bands = row[2:-1]
continue
name, source = catalog.new_entry(
'SN2008S', bibcode='2009MNRAS.398.1041B')
mjd = str(jd_to_mjd(Decimal(row[0])))
mags = [x.split('±')[0].strip() for x in row[2:]]
upps = [('<' in x.split('±')[0]) for x in row[2:]]
errs = [x.split('±')[1].strip()
if '±' in x else '' for x in row[2:]]
instrument = row[-1]
for mi, mag in enumerate(mags):
if not is_number(mag):
continue
catalog.entries[name].add_photometry(
time=mjd, band=bands[mi],
magnitude=mag, e_magnitude=errs[mi],
instrument=instrument, source=source)
catalog.journal_entries()
# 2010arXiv1007.0011P
file_path = os.path.join(
catalog.get_current_task_repo(), '2010arXiv1007.0011P.tsv')
with open(file_path, 'r') as f:
data = list(csv.reader(f, delimiter='\t',
quotechar='"', skipinitialspace=True))
for r, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
bands = row[1:]
continue
name, source = catalog.new_entry(
'SN2008S', bibcode='2010arXiv1007.0011P')
mjd = row[0]
mags = [x.split('±')[0].strip() for x in row[1:]]
errs = [x.split('±')[1].strip()
if '±' in x else '' for x in row[1:]]
for mi, mag in enumerate(mags):
if not is_number(mag):
continue
catalog.entries[name].add_photometry(
time=mjd, band=bands[mi],
magnitude=mag, e_magnitude=errs[mi],
instrument='LBT', source=source)
catalog.journal_entries()
# 2000ApJ...533..320G
file_path = os.path.join(
catalog.get_current_task_repo(), '2000ApJ...533..320G.tsv')
with open(file_path, 'r') as f:
data = list(csv.reader(f, delimiter='\t',
quotechar='"', skipinitialspace=True))
name, source = catalog.new_entry(
'SN1997cy', bibcode='2000ApJ...533..320G')
for r, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
bands = row[1:-1]
continue
mjd = str(jd_to_mjd(Decimal(row[0])))
mags = row[1:len(bands)]
for mi, mag in enumerate(mags):
if not is_number(mag):
continue
catalog.entries[name].add_photometry(
time=mjd, band=bands[mi],
magnitude=mag,
observatory='Mount Stromlo', telescope='MSSSO',
source=source, kcorrected=True)
catalog.journal_entries()
return
def do_wiserep_spectra(catalog):
#if not catalog.args.travis:
# from ..input.WISeWEBSpider.wisewebspider import spider
# try:
# spider(update=True, daysago=7, path="/../../sne-external-WISEREP/")
# except:
# catalog.log.warning(
# 'Spider errored, continuing without letting it complete.')
task_str = catalog.get_current_task_str()
secondaryreference = 'WISeREP'
secondaryrefurl = 'http://wiserep.weizmann.ac.il/'
secondarybibcode = '2012PASP..124..668Y'
wiserepcnt = 0
# These are known to be in error on the WISeREP page, either fix or ignore
# them.
wiserepbibcorrectdict = {
'2000AJ....120..367G]': '2000AJ....120..367G',
'Harutyunyan et al. 2008': '2008A&A...488..383H',
'0609268': '2007AJ....133...58K',
'2006ApJ...636...400Q': '2006ApJ...636..400Q',
'2011ApJ...741...76': '2011ApJ...741...76C',
'2016PASP...128...961': '2016PASP..128...961',
'2002AJ....1124..417H': '2002AJ....1124.417H',
'2013ApJ…774…58D': '2013ApJ...774...58D',
'2011Sci.333..856S': '2011Sci...333..856S',
'2014MNRAS.438,368': '2014MNRAS.438..368T',
'2012MNRAS.420.1135': '2012MNRAS.420.1135S',
'2012Sci..337..942D': '2012Sci...337..942D',
'stt1839': '2013MNRAS.436.3614S',
'arXiv:1605.03136': '2016MNRAS.460.3447T',
'10.1093/mnras/stt1839': '2013MNRAS.436.3614S'
}
file_names = list(glob(os.path.join(catalog.get_current_task_repo(), '*')))
for folder in pbar_strings(file_names, task_str):
if '.txt' in folder or '.json' in folder:
continue
name = os.path.basename(folder).strip()
if name.startswith('sn'):
name = 'SN' + name[2:]
if (name.startswith(('CSS', 'SSS', 'MLS')) and ':' not in name):
name = name.replace('-', ':', 1)
if name.startswith('MASTERJ'):
name = name.replace('MASTERJ', 'MASTER OT J')
if name.startswith('PSNJ'):
name = name.replace('PSNJ', 'PSN J')
name = catalog.add_entry(name)
secondarysource = catalog.entries[name].add_source(
name=secondaryreference,
url=secondaryrefurl,
bibcode=secondarybibcode,
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name,
secondarysource)
readme_path = os.path.join(folder, 'README.json')
if not os.path.exists(readme_path):
catalog.log.warning(
'Metadata file not found for event "{}"'.format(name))
continue
with open(readme_path, 'r') as f:
fileinfo = json.loads(f.read())
files = list(
set(glob(folder + '/*')) - set(glob(folder + '/README.json')))
for fname in pbar(files, task_str):
specfile = os.path.basename(fname)
if specfile not in fileinfo:
catalog.log.warning(
'Metadata not found for "{}"'.format(fname))
continue
claimedtype = fileinfo[specfile]["Type"]
instrument = fileinfo[specfile]["Instrument"]
epoch = fileinfo[specfile]["Obs. Date"]
observer = fileinfo[specfile]["Observer"]
reducer = fileinfo[specfile]["Reducer"]
bibcode = fileinfo[specfile]["Bibcode"]
redshift = fileinfo[specfile]["Redshift"]
survey = fileinfo[specfile]["Program"]
reduction = fileinfo[specfile]["Reduction Status"]
if bibcode:
newbibcode = bibcode
if bibcode in wiserepbibcorrectdict:
newbibcode = wiserepbibcorrectdict[bibcode]
if newbibcode and len(newbibcode) == 19:
source = catalog.entries[name].add_source(
bibcode=unescape(newbibcode))
else:
bibname = unescape(bibcode)
source = catalog.entries[name].add_source(name=bibname)
catalog.log.warning('Bibcode "{}" is invalid, using as '
'`{}` instead'.format(
bibname, SOURCE.NAME))
sources = uniq_cdl([source, secondarysource])
else:
sources = secondarysource
if claimedtype not in ['Other']:
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE,
claimedtype,
secondarysource)
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, redshift,
secondarysource)
with open(fname, 'r') as f:
data = [x.split() for x in f]
skipspec = False
newdata = []
oldval = ''
for row in data:
if row and '#' not in row[0]:
if (len(row) >= 2 and is_number(row[0])
and is_number(row[1]) and row[1] != oldval):
newdata.append(row)
oldval = row[1]
if skipspec or not newdata:
warnings.warn('Skipped adding spectrum file ' + specfile)
continue
data = [list(i) for i in zip(*newdata)]
wavelengths = data[0]
fluxes = data[1]
errors = ''
if len(data) == 3:
errors = data[1]
time = str(astrotime(epoch).mjd)
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom',
errors=errors,
u_fluxes=fluxunit,
u_errors=fluxunit if errors else '',
wavelengths=wavelengths,
fluxes=fluxes,
u_time='MJD',
time=time,
instrument=instrument,
source=sources,
observer=observer,
reducer=reducer,
reduction=reduction,
filename=specfile,
survey=survey,
redshift=redshift)
catalog.journal_entries()
wiserepcnt = wiserepcnt + 1
if (catalog.args.travis
and wiserepcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
return
def do_suspect_spectra(catalog):
task_str = catalog.get_current_task_str()
with open(
os.path.join(catalog.get_current_task_repo(),
'Suspect/sources.json'), 'r') as f:
sourcedict = json.loads(f.read())
with open(
os.path.join(catalog.get_current_task_repo(),
'Suspect/filename-changes.txt'), 'r') as f:
rows = f.readlines()
changedict = {}
for row in rows:
if not row.strip() or row[0] == "#":
continue
items = row.strip().split(' ')
changedict[items[1]] = items[0]
suspectcnt = 0
folders = next(
os.walk(os.path.join(catalog.get_current_task_repo(), 'Suspect')))[1]
for folder in pbar(folders, task_str):
eventfolders = next(
os.walk(
os.path.join(catalog.get_current_task_repo(), 'Suspect/') +
folder))[1]
oldname = ''
for eventfolder in pbar(eventfolders, task_str):
name = eventfolder
if is_number(name[:4]):
name = 'SN' + name
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
sec_ref = 'SUSPECT'
sec_refurl = 'https://www.nhn.ou.edu/~suspect/'
sec_bibc = '2001AAS...199.8408R'
sec_source = catalog.entries[name].add_source(name=sec_ref,
url=sec_refurl,
bibcode=sec_bibc,
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name,
sec_source)
fpath = os.path.join(catalog.get_current_task_repo(), 'Suspect',
folder, eventfolder)
eventspectra = next(os.walk(fpath))[2]
for spectrum in eventspectra:
sources = [sec_source]
bibcode = ''
if spectrum in changedict:
specalias = changedict[spectrum]
else:
specalias = spectrum
if specalias in sourcedict:
bibcode = sourcedict[specalias]
elif name in sourcedict:
bibcode = sourcedict[name]
if bibcode:
source = catalog.entries[name].add_source(
bibcode=unescape(bibcode))
sources += [source]
sources = uniq_cdl(sources)
date = spectrum.split('_')[1]
year = date[:4]
month = date[4:6]
day = date[6:]
sig = get_sig_digits(day) + 5
day_fmt = str(floor(float(day))).zfill(2)
time = astrotime(year + '-' + month + '-' + day_fmt).mjd
time = time + float(day) - floor(float(day))
time = pretty_num(time, sig=sig)
fpath = os.path.join(catalog.get_current_task_repo(),
'Suspect', folder, eventfolder, spectrum)
with open(fpath, 'r') as f:
specdata = list(
csv.reader(f, delimiter=' ', skipinitialspace=True))
specdata = list(filter(None, specdata))
newspec = []
oldval = ''
for row in specdata:
if row[1] == oldval:
continue
newspec.append(row)
oldval = row[1]
specdata = newspec
haserrors = len(
specdata[0]
) == 3 and specdata[0][2] and specdata[0][2] != 'NaN'
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
catalog.entries[name].add_spectrum(u_wavelengths='Angstrom',
u_fluxes='Uncalibrated',
u_time='MJD',
time=time,
wavelengths=wavelengths,
fluxes=fluxes,
errors=errors,
u_errors='Uncalibrated',
source=sources,
filename=spectrum)
suspectcnt = suspectcnt + 1
if (catalog.args.travis
and suspectcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
def do_rochester(catalog):
"""Import data from the Latest Supernovae page."""
rochestermirrors = [
'http://www.rochesterastronomy.org/',
'http://www.supernova.thistlethwaites.com/'
]
rochesterpaths = [
'snimages/snredshiftall.html', 'sn2017/snredshift.html',
'snimages/snredboneyard.html', 'snimages/snredboneyard-old.html'
]
rochesterupdate = [False, True, True, False]
task_str = catalog.get_current_task_str()
baddates = ['2440587', '2440587.292', '0001/01/01']
for pp, path in enumerate(pbar(rochesterpaths, task_str)):
if catalog.args.update and not rochesterupdate[pp]:
continue
if 'snredboneyard.html' in path:
cns = {
'name': 0,
'host': 1,
'ra': 2,
'dec': 3,
'type': 7,
'z': 8,
'mmag': 9,
'max': 10,
'disc': 11,
'ref': 12,
'dver': 13,
'aka': 14
}
else:
cns = {
'name': 0,
'type': 1,
'host': 2,
'ra': 3,
'dec': 4,
'disc': 6,
'max': 7,
'mmag': 8,
'z': 11,
'zh': 12,
'ref': 13,
'dver': 14,
'aka': 15
}
filepath = (
os.path.join(catalog.get_current_task_repo(), 'rochester/') +
path.replace('/', '-'))
for mirror in rochestermirrors:
html = catalog.load_url(
mirror + path, filepath, fail=(mirror != rochestermirrors[-1]))
if html:
break
if not html:
continue
soup = BeautifulSoup(html, 'html5lib')
rows = soup.findAll('tr')
sec_ref = 'Latest Supernovae'
sec_refurl = ('http://www.rochesterastronomy.org/'
'snimages/snredshiftall.html')
loopcnt = 0
for rr, row in enumerate(pbar(rows, task_str)):
if rr == 0:
continue
cols = row.findAll('td')
if not len(cols):
continue
name = ''
if cols[cns['aka']].contents:
for rawaka in str(cols[cns['aka']].contents[0]).split(','):
aka = rawaka.strip()
if is_number(aka.strip('?')):
aka = 'SN' + aka.strip('?') + 'A'
oldname = aka
name = catalog.add_entry(aka)
elif len(aka) == 4 and is_number(aka[:4]):
aka = 'SN' + aka
oldname = aka
name = catalog.add_entry(aka)
ra = str(cols[cns['ra']].contents[0]).strip()
dec = str(cols[cns['dec']].contents[0]).strip()
sn = re.sub('<[^<]+?>', '',
str(cols[cns['name']].contents[0])).strip()
if is_number(sn.strip('?')):
sn = 'SN' + sn.strip('?') + 'A'
elif len(sn) == 4 and is_number(sn[:4]):
sn = 'SN' + sn
if not name:
if not sn or sn in ['Transient']:
continue
if sn[:8] == 'MASTER J':
sn = sn.replace('MASTER J', 'MASTER OT J').replace(
'SNHunt', 'SNhunt')
if 'POSSIBLE' in sn.upper() and ra and dec:
sn = 'PSN J' + ra.replace(':', '').replace('.', '')
sn += dec.replace(':', '').replace('.', '')
oldname = sn
name = catalog.add_entry(sn)
sec_source = catalog.entries[name].add_source(
name=sec_ref, url=sec_refurl, secondary=True)
sources = []
if 'ref' in cns:
reftag = reference = cols[cns['ref']].findAll('a')
if len(reftag):
reference = reftag[0].contents[0].strip()
refurl = reftag[0]['href'].strip()
sources.append(catalog.entries[name].add_source(
name=reference, url=refurl))
sources.append(sec_source)
sources = uniq_cdl(list(filter(None, sources)))
catalog.entries[name].add_quantity(TIDALDISRUPTION.ALIAS, oldname,
sources)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.ALIAS, sn, sources)
if cols[cns['aka']].contents:
for rawaka in str(cols[cns['aka']].contents[0]).split(','):
aka = rawaka.strip()
if aka == 'SNR G1.9+0.3':
aka = 'G001.9+00.3'
if aka[:4] == 'PS1 ':
aka = 'PS1-' + aka[4:]
if aka[:8] == 'MASTER J':
aka = aka.replace('MASTER J', 'MASTER OT J').replace(
'SNHunt', 'SNhunt')
if 'POSSIBLE' in aka.upper() and ra and dec:
aka = 'PSN J' + ra.replace(':', '').replace('.', '')
aka += dec.replace(':', '').replace('.', '')
catalog.entries[name].add_quantity(
TIDALDISRUPTION.ALIAS, aka, sources)
if str(cols[cns['type']].contents[0]).strip() != 'unk':
ctype = str(cols[cns['type']].contents[0]).strip(' :,')
catalog.entries[name].add_quantity(
TIDALDISRUPTION.CLAIMED_TYPE, ctype, sources)
if (len(cols[cns['host']].contents) > 0 and
str(cols[cns['host']].contents[0]).strip() != 'anonymous'):
catalog.entries[name].add_quantity(
TIDALDISRUPTION.HOST,
str(cols[cns['host']].contents[0]).strip(), sources)
catalog.entries[name].add_quantity(TIDALDISRUPTION.RA, ra, sources)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DEC, dec, sources)
discstr = str(cols[cns['disc']].contents[0]).strip()
if discstr and discstr not in baddates:
if '/' not in discstr:
astrot = astrotime(float(discstr), format='jd').datetime
ddate = make_date_string(astrot.year, astrot.month,
astrot.day)
else:
ddate = discstr
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE, ddate, sources)
maxstr = str(cols[cns.get('max', '')].contents[0]).strip()
if maxstr and maxstr not in baddates:
try:
if '/' not in maxstr:
astrot = astrotime(float(maxstr), format='jd')
else:
astrot = astrotime(
maxstr.replace('/', '-'), format='iso')
except Exception:
catalog.log.info(
'Max date conversion failed for `{}`.'.format(maxstr))
if ((float(str(
cols[cns['mmag']].contents[0])
.strip()) <= 90.0 and not any(
'GRB' in xx for xx in
catalog.entries[name].get_aliases()))):
mag = str(cols[cns['mmag']].contents[0]).strip()
catalog.entries[name].add_photometry(
time=str(astrot.mjd),
u_time='MJD',
magnitude=mag,
source=sources)
if 'z' in cns and cols[cns['z']].contents[0] != 'n/a':
catalog.entries[name].add_quantity(
TIDALDISRUPTION.REDSHIFT,
str(cols[cns['z']].contents[0]).strip(), sources)
if 'zh' in cns:
zhost = str(cols[cns['zh']].contents[0]).strip()
if is_number(zhost):
catalog.entries[name].add_quantity(
TIDALDISRUPTION.REDSHIFT, zhost, sources)
if 'dver' in cns:
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVERER,
str(cols[cns['dver']].contents[0]).strip(), sources)
if catalog.args.update:
catalog.journal_entries()
loopcnt = loopcnt + 1
if (catalog.args.travis and
loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
if not catalog.args.update:
vsnetfiles = ['latestsne.dat']
for vsnetfile in vsnetfiles:
file_name = os.path.join(catalog.get_current_task_repo(),
"" + vsnetfile)
with open(file_name, 'r', encoding='latin1') as csv_file:
tsvin = csv.reader(
csv_file, delimiter=' ', skipinitialspace=True)
loopcnt = 0
for rr, row in enumerate(tsvin):
if (not row or row[0] in ['Transient'] or
row[0][:4] in ['http', 'www.'] or len(row) < 3):
continue
name = row[0].strip()
if name[:4].isdigit():
name = 'SN' + name
if name.startswith('PSNJ'):
name = 'PSN J' + name[4:]
if name.startswith('MASTEROTJ'):
name = name.replace('MASTEROTJ', 'MASTER OT J')
name = catalog.add_entry(name)
sec_source = catalog.entries[name].add_source(
name=sec_ref, url=sec_refurl, secondary=True)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.ALIAS, name, sec_source)
if not is_number(row[1]):
continue
year = row[1][:4]
month = row[1][4:6]
day = row[1][6:]
if '.' not in day:
day = day[:2] + '.' + day[2:]
mjd = astrotime(year + '-' + month + '-' + str(
floor(float(day))).zfill(2)).mjd
mjd += float(day) - floor(float(day))
magnitude = row[2].rstrip(ascii_letters)
if not is_number(magnitude):
continue
if magnitude.isdigit():
if int(magnitude) > 100:
magnitude = magnitude[:2] + '.' + magnitude[2:]
if float(str(cols[8].contents[0]).strip()) >= 90.0:
continue
if len(row) >= 4:
if is_number(row[3]):
e_magnitude = row[3]
refind = 4
else:
e_magnitude = ''
refind = 3
if refind >= len(row):
sources = sec_source
else:
reference = ' '.join(row[refind:])
source = catalog.entries[name].add_source(
name=reference)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.ALIAS, name, sec_source)
sources = uniq_cdl([source, sec_source])
else:
sources = sec_source
band = row[2].lstrip('1234567890.')
catalog.entries[name].add_photometry(
time=mjd,
u_time='MJD',
band=band,
magnitude=magnitude,
e_magnitude=e_magnitude,
source=sources)
if (catalog.args.travis and
loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
if ('photometry' not in thisevent or 'maxdate' not in thisevent
or 'maxabsmag' not in thisevent or 'maxappmag' not in thisevent
or 'claimedtype' not in thisevent
or len(thisevent['maxdate'][0]['value'].split('/')) < 3
or 'discoverdate' not in thisevent):
continue
foundtype = False
for ct in thisevent['claimedtype']:
if ct['value'] == averagetype:
foundtype = True
break
if not foundtype:
continue
maxdate = astrotime(thisevent['maxdate'][0]['value'].replace('/',
'-')).mjd
discoverdate = astrotime(thisevent['discoverdate'][0]['value'].replace(
'/', '-')).mjd
if maxdate == discoverdate:
continue
distmod = float(thisevent['maxappmag'][0]['value']) - \
float(thisevent['maxabsmag'][0]['value'])
tprint(thisevent['name'])
prange = list(
range(len([x for x in thisevent['photometry'] if photo_cut(x)
]))) if 'photometry' in thisevent else []
def do_donated_spectra(catalog):
"""Import donated spectra."""
task_str = catalog.get_current_task_str()
fpath = os.path.join(catalog.get_current_task_repo(), 'donations')
with open(os.path.join(fpath, 'meta.json'), 'r') as f:
metadict = json.loads(f.read())
donationscnt = 0
oldname = ''
for fname in pbar(metadict, task_str):
name = metadict[fname]['name']
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
sec_bibc = metadict[fname]['bibcode']
name, source = catalog.new_entry(name, bibcode=sec_bibc)
date = metadict[fname].get('date', '')
year, month, day = date.split('/')
sig = get_sig_digits(day) + 5
day_fmt = str(floor(float(day))).zfill(2)
time = astrotime(year + '-' + month + '-' + day_fmt).mjd
time = time + float(day) - floor(float(day))
time = pretty_num(time, sig=sig)
with open(os.path.join(fpath, fname), 'r') as f:
specdata = list(csv.reader(f, delimiter=' ',
skipinitialspace=True))
specdata = list(filter(None, specdata))
newspec = []
oldval = ''
for row in specdata:
if row[0][0] == '#':
continue
if row[1] == oldval:
continue
newspec.append(row)
oldval = row[1]
specdata = newspec
haserrors = len(
specdata[0]) == 3 and specdata[0][2] and specdata[0][2] != 'NaN'
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.U_TIME: 'MJD',
SPECTRUM.TIME: time,
SPECTRUM.WAVELENGTHS: wavelengths,
SPECTRUM.FLUXES: fluxes,
SPECTRUM.ERRORS: errors,
SPECTRUM.SOURCE: source,
SPECTRUM.FILENAME: fname
}
if 'instrument' in metadict[fname]:
specdict[SPECTRUM.INSTRUMENT] = metadict[fname]['instrument']
if 'telescope' in metadict[fname]:
specdict[SPECTRUM.TELESCOPE] = metadict[fname]['telescope']
if 'yunit' in metadict[fname]:
specdict[SPECTRUM.U_FLUXES] = metadict[fname]['yunit']
specdict[SPECTRUM.U_ERRORS] = metadict[fname]['yunit']
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict[SPECTRUM.U_FLUXES] = fluxunit
specdict[SPECTRUM.U_ERRORS] = fluxunit
catalog.entries[name].add_spectrum(**specdict)
donationscnt = donationscnt + 1
if (catalog.args.travis
and donationscnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
def do_donated_photo(catalog):
"""Import donated photometry."""
task_str = catalog.get_current_task_str()
# Private donations here #
if not catalog.args.travis:
pass
# End private donations #
# Ponder 05-12-17 donation
with open(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Ponder-05-12-17', 'meta.json'), 'r') as f:
metadict = json.loads(f.read())
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Ponder-05-12-17', '*.dat'))
for path in file_names:
with open(path, 'r') as f:
tsvin = list(csv.reader(f, delimiter=' ', skipinitialspace=True))
oname = path.split('/')[-1].split('.')[0]
name, source = catalog.new_entry(oname,
bibcode=metadict[oname]['bibcode'])
for row in pbar(tsvin, task_str + ': Ponder ' + oname):
if row[0][0] == '#' or not is_number(row[-1]):
continue
mjd = row[1]
bandinst = row[2].split('_')
band = bandinst[0]
inst = ''
if len(bandinst) > 1:
inst = bandinst[1]
mag = row[3]
uerr = row[4]
lerr = row[5]
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.BAND: band,
PHOTOMETRY.MAGNITUDE: mag,
PHOTOMETRY.E_LOWER_MAGNITUDE: lerr,
PHOTOMETRY.E_UPPER_MAGNITUDE: uerr,
PHOTOMETRY.SOURCE: source
}
if inst:
photodict[PHOTOMETRY.INSTRUMENT] = inst
catalog.entries[name].add_photometry(**photodict)
# Benetti 03-08-17 donation
path = os.path.join(catalog.get_current_task_repo(), 'Donations',
'Benetti-03-08-17', '1999E.dat')
with open(path, 'r') as f:
tsvin = list(csv.reader(f, delimiter=' ', skipinitialspace=True))
name, source = catalog.new_entry('SN1999E',
bibcode='2003MNRAS.340..191R')
bands = None
for row in tsvin:
if not row or row[0][0] == '#':
continue
if not bands:
bands = row[2:-2]
continue
mjd = row[1]
tel = row[-1] if 'IAUC' not in row[-1] else None
for bi, band in enumerate(bands):
mag = row[2 + 2 * bi]
if mag == '9999':
continue
err = row[2 + 2 * bi + 1]
limit = row[6] == 'True'
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.TELESCOPE: tel,
PHOTOMETRY.BAND: band,
PHOTOMETRY.MAGNITUDE: mag,
PHOTOMETRY.SOURCE: source
}
if err != '.00':
photodict[PHOTOMETRY.E_MAGNITUDE] = str(Decimal(err))
if tel:
photodict[PHOTOMETRY.TELESCOPE] = tel
catalog.entries[name].add_photometry(**photodict)
# Nicholl 01-29-17 donation
with open(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-01-29-17', 'meta.json'), 'r') as f:
metadict = json.loads(f.read())
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-01-29-17', '*.txt'))
for path in file_names:
data = read(path, format='cds')
oname = path.split('/')[-1].split('_')[0]
name, source = catalog.new_entry(oname,
bibcode=metadict[oname]['bibcode'])
for row in pbar(data, task_str + ': Nicholl ' + oname):
photodict = {
PHOTOMETRY.TIME: str(row['MJD']),
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.MAGNITUDE: str(row['mag']),
PHOTOMETRY.BAND: row['Filter'],
PHOTOMETRY.SOURCE: source
}
if 'system' in metadict[oname]:
photodict[PHOTOMETRY.SYSTEM] = metadict[oname]['system']
if 'l_mag' in row.columns and row['l_mag'] == '>':
photodict[PHOTOMETRY.UPPER_LIMIT] = True
elif 'e_mag' in row.columns:
photodict[PHOTOMETRY.E_MAGNITUDE] = str(row['e_mag'])
if 'Telescope' in row.columns:
photodict[PHOTOMETRY.TELESCOPE] = row['Telescope']
catalog.entries[name].add_photometry(**photodict)
# Arcavi 2016gkg donation
path = os.path.join(catalog.get_current_task_repo(), 'Donations',
'Arcavi-01-24-17', 'SN2016gkg.txt')
with open(path, 'r') as f:
tsvin = list(csv.reader(f, delimiter=' ', skipinitialspace=True))
name, source = catalog.new_entry('SN2016gkg',
bibcode='2016arXiv161106451A')
for row in tsvin:
if row[0][0] == '#':
continue
mjd = str(jd_to_mjd(Decimal(row[0])))
tel = row[1]
band = row[3]
mag = row[4]
err = row[5]
limit = row[6] == 'True'
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.TELESCOPE: tel,
PHOTOMETRY.BAND: band,
PHOTOMETRY.MAGNITUDE: mag,
PHOTOMETRY.SOURCE: source
}
if limit:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
else:
photodict[PHOTOMETRY.E_MAGNITUDE] = err
catalog.entries[name].add_photometry(**photodict)
# Nicholl Gaia16apd donation
path = os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-01-20-17', 'gaia16apd_phot.txt')
data = read(path, format='cds')
name, source = catalog.new_entry('Gaia16apd',
bibcode='2017ApJ...835L...8N')
for row in pbar(data, task_str + ': Nicholl Gaia16apd'):
photodict = {
PHOTOMETRY.TIME: str(row['MJD']),
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.MAGNITUDE: str(row['mag']),
PHOTOMETRY.BAND: row['Filter'],
PHOTOMETRY.TELESCOPE: row['Telescope'],
PHOTOMETRY.SOURCE: source
}
if row['l_mag'] == '>':
photodict[PHOTOMETRY.UPPER_LIMIT] = True
else:
photodict[PHOTOMETRY.E_MAGNITUDE] = str(row['e_mag'])
catalog.entries[name].add_photometry(**photodict)
# Kuncarayakti-01-09-17
datafile = os.path.join(catalog.get_current_task_repo(), 'Donations',
'Kuncarayakti-01-09-17', 'SN1978K.dat')
inpname = os.path.basename(datafile).split('.')[0]
with open(datafile, 'r') as f:
tsvin = csv.reader(f, delimiter=' ', skipinitialspace=True)
host = False
for ri, row in enumerate(tsvin):
if ri == 0:
continue
if row[0][0] == '#':
rsplit = [x.strip('# ') for x in ' '.join(row).split(',')]
bc = rsplit[0]
tel, ins = '', ''
if len(rsplit) > 1:
tel = rsplit[1]
if len(rsplit) > 2:
ins = rsplit[2]
continue
(name, source) = catalog.new_entry(inpname, bibcode=bc)
mag = row[4]
err = row[5]
mjd = str(astrotime('-'.join(row[:3]), format='iso').mjd)
photodict = {
PHOTOMETRY.BAND: row[3],
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.MAGNITUDE: mag.strip('>s'),
PHOTOMETRY.SOURCE: source
}
if is_number(err):
photodict[PHOTOMETRY.E_MAGNITUDE] = err
if tel:
photodict[PHOTOMETRY.TELESCOPE] = tel
if ins:
photodict[PHOTOMETRY.INSTRUMENT] = ins
if '>' in mag:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
if 's' in mag:
photodict[PHOTOMETRY.SYNTHETIC] = True
catalog.entries[name].add_photometry(**photodict)
# Nugent 01-09-17 donation
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nugent-01-09-17', '*.dat'))
for datafile in pbar_strings(file_names, task_str + ': Nugent-01-09-17'):
inpname = os.path.basename(datafile).split('.')[0]
(name, source) = catalog.new_entry(inpname,
bibcode='2006ApJ...645..841N')
with open(datafile, 'r') as f:
tsvin = csv.reader(f, delimiter=' ', skipinitialspace=True)
host = False
for urow in tsvin:
row = list(filter(None, urow))
counts = row[2]
e_counts = row[3]
zp = row[4]
photodict = {
PHOTOMETRY.BAND: row[1],
PHOTOMETRY.TIME: row[0],
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.COUNT_RATE: counts,
PHOTOMETRY.E_COUNT_RATE: e_counts,
PHOTOMETRY.ZERO_POINT: zp,
PHOTOMETRY.TELESCOPE: 'CFHT',
PHOTOMETRY.SURVEY: 'SNLS',
PHOTOMETRY.SOURCE: source
}
set_pd_mag_from_counts(photodict, counts, ec=e_counts, zp=zp)
catalog.entries[name].add_photometry(**photodict)
# Inserra 09-04-16 donation
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Inserra-09-04-16', '*.txt'))
for datafile in pbar_strings(file_names, task_str + ': Inserra-09-04-16'):
inpname = os.path.basename(datafile).split('.')[0]
(name, source) = catalog.new_entry(inpname,
bibcode='2013ApJ...770..128I')
with open(datafile, 'r') as f:
tsvin = csv.reader(f, delimiter=' ', skipinitialspace=True)
host = False
for row in tsvin:
if row[0][0] == '#':
if row[0] == '#Host':
host = True
continue
host = False
bands = row[3:-1]
continue
for bi, ba in enumerate(bands):
mag = row[5 + 2 * bi]
if not is_number(mag):
continue
system = 'AB'
if ba in ['U', 'B', 'V', 'R', 'I', 'J', 'H', 'K']:
system = 'Vega'
photodict = {
PHOTOMETRY.TIME: row[3],
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.BAND: ba,
PHOTOMETRY.MAGNITUDE: mag.strip('< '),
PHOTOMETRY.SOURCE: source,
PHOTOMETRY.SYSTEM: system
}
if 'ATel' not in row[-1]:
photodict[PHOTOMETRY.TELESCOPE] = row[-1]
if host:
photodict[PHOTOMETRY.HOST] = True
if '<' in mag:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
e_mag = row[5 + 2 * bi + 1].strip('() ')
if is_number(e_mag):
photodict[PHOTOMETRY.E_MAGNITUDE] = e_mag
catalog.entries[name].add_photometry(**photodict)
# Nicholl 04-01-16 donation
with open(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-04-01-16', 'bibcodes.json'), 'r') as f:
bcs = json.loads(f.read())
kcorrected = ['SN2011ke', 'SN2011kf', 'SN2012il', 'PTF10hgi', 'PTF11rks']
ignorephoto = ['PTF10hgi', 'PTF11rks', 'SN2011ke', 'SN2011kf', 'SN2012il']
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-04-01-16/*.txt'))
for datafile in pbar_strings(file_names, task_str + ': Nicholl-04-01-16'):
inpname = os.path.basename(datafile).split('_')[0]
isk = inpname in kcorrected
name = catalog.add_entry(inpname)
bibcode = ''
for bc in bcs:
if inpname in bcs[bc]:
bibcode = bc
if not bibcode:
raise ValueError('Bibcode not found!')
source = catalog.entries[name].add_source(bibcode=bibcode)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, inpname, source)
if inpname in ignorephoto:
continue
with open(datafile, 'r') as f:
tsvin = csv.reader(f, delimiter='\t', skipinitialspace=True)
rtelescope = ''
for r, rrow in enumerate(tsvin):
row = list(filter(None, rrow))
if not row:
continue
if row[0] == '#MJD':
bands = [x for x in row[1:] if x and 'err' not in x]
elif row[0][0] == '#' and len(row[0]) > 1:
rtelescope = row[0][1:]
if row[0][0] == '#':
continue
mjd = row[0]
if not is_number(mjd):
continue
for v, val in enumerate(row[1::2]):
upperlimit = ''
mag = val.strip('>')
emag = row[2 * v + 2]
if '>' in val or (is_number(emag) and float(emag) == 0.0):
upperlimit = True
if (not is_number(mag) or isnan(float(mag))
or float(mag) > 90.0):
continue
band = bands[v]
instrument = ''
survey = ''
system = ''
telescope = rtelescope
if telescope == 'LSQ':
instrument = 'QUEST'
elif telescope == 'PS1':
instrument = 'GPC'
elif telescope == 'NTT':
instrument = 'EFOSC'
elif telescope == 'GROND':
instrument = 'GROND'
telescope = 'MPI/ESO 2.2m'
else:
if band == 'NUV':
instrument = 'GALEX'
telescope = 'GALEX'
elif band in ['u', 'g', 'r', 'i', 'z']:
if inpname.startswith('PS1'):
instrument = 'GPC'
telescope = 'PS1'
survey = 'Pan-STARRS'
elif inpname.startswith('PTF'):
telescope = 'P60'
survey = 'PTF'
elif band.upper() in ['UVW2', 'UVW1', 'UVM2']:
instrument = 'UVOT'
telescope = 'Swift'
if inpname in ['PTF12dam']:
system = 'AB'
if inpname in ['SCP-06F6']:
system = 'Vega'
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.BAND: band,
PHOTOMETRY.MAGNITUDE: mag,
PHOTOMETRY.UPPER_LIMIT: upperlimit,
PHOTOMETRY.SOURCE: source
}
if instrument:
photodict[PHOTOMETRY.INSTRUMENT] = instrument
if telescope:
photodict[PHOTOMETRY.TELESCOPE] = telescope
if survey:
photodict[PHOTOMETRY.SURVEY] = survey
if system:
photodict[PHOTOMETRY.SYSTEM] = system
if (is_number(emag) and not isnan(float(emag))
and float(emag) > 0.0):
photodict[PHOTOMETRY.E_MAGNITUDE] = emag
if isk:
photodict[PHOTOMETRY.KCORRECTED] = True
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
# Maggi 04-11-16 donation (MC SNRs)
with open(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Maggi-04-11-16', 'LMCSNRs_OpenSNe.csv')) as f:
tsvin = csv.reader(f, delimiter=',')
for row in pbar(list(tsvin), task_str + ': Maggi-04-11-16/LMCSNRs'):
name = 'MCSNR ' + row[0]
name = catalog.add_entry(name)
ra = row[2]
dec = row[3]
source = (catalog.entries[name].add_source(
bibcode='2016A&A...585A.162M'))
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS,
'LMCSNR J' + rep_chars(ra, ' :.') + rep_chars(dec, ' :.'),
source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
if row[1] != 'noname':
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[1],
source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[2], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[3], source)
catalog.entries[name].add_quantity(SUPERNOVA.HOST, 'LMC', source)
if row[4] == '1':
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE,
'Ia', source)
elif row[4] == '2':
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE,
'CC', source)
with open(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Maggi-04-11-16', 'SMCSNRs_OpenSNe.csv')) as f:
tsvin = csv.reader(f, delimiter=',')
for row in pbar(list(tsvin), task_str + ': Maggi-04-11-16/SMCSNRs'):
name = 'MCSNR ' + row[0]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(name='Pierre Maggi')
ra = row[3]
dec = row[4]
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, 'SMCSNR J' + ra.replace(':', '')[:6] +
dec.replace(':', '')[:7], source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[1], source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[2], source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[3], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[4], source)
catalog.entries[name].add_quantity(SUPERNOVA.HOST, 'SMC', source)
catalog.journal_entries()
# Galbany 04-18-16 donation
folders = next(
os.walk(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Galbany-04-18-16/')))[1]
bibcode = '2016AJ....151...33G'
for folder in folders:
infofiles = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Galbany-04-18-16/') + folder + '/*.info')
photfiles = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Galbany-04-18-16/') + folder + '/*.out*')
zhel = ''
zcmb = ''
zerr = ''
for path in infofiles:
with open(path, 'r') as f:
lines = f.read().splitlines()
for line in lines:
splitline = line.split(':')
field = splitline[0].strip().lower()
value = splitline[1].strip()
if field == 'name':
name = value[:6].upper()
name += (value[6].upper()
if len(value) == 7 else value[6:])
name = catalog.add_entry(name)
source = (catalog.entries[name].add_source(
bibcode=bibcode))
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, name, source)
elif field == 'type':
claimedtype = value.replace('SN', '')
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, claimedtype, source)
elif field == 'zhel':
zhel = value
elif field == 'redshift_error':
zerr = value
elif field == 'zcmb':
zcmb = value
elif field == 'ra':
catalog.entries[name].add_quantity(
SUPERNOVA.RA,
value,
source,
u_value='floatdegrees')
elif field == 'dec':
catalog.entries[name].add_quantity(
SUPERNOVA.DEC,
value,
source,
u_value='floatdegrees')
elif field == 'host':
value = value.replace('- ', '-').replace('G ', 'G')
catalog.entries[name].add_quantity(
SUPERNOVA.HOST, value, source)
elif field == 'e(b-v)_mw':
catalog.entries[name].add_quantity(
SUPERNOVA.EBV, value, source)
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT,
zhel,
source,
e_value=zerr,
kind='heliocentric')
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT,
zcmb,
source,
e_value=zerr,
kind='cmb')
for path in photfiles:
with open(path, 'r') as f:
band = ''
lines = f.read().splitlines()
for li, line in enumerate(lines):
if li in [0, 2, 3]:
continue
if li == 1:
band = line.split(':')[-1].strip()
else:
cols = list(filter(None, line.split()))
if not cols:
continue
catalog.entries[name].add_photometry(
time=cols[0],
u_time='MJD',
magnitude=cols[1],
e_magnitude=cols[2],
band=band,
system=cols[3],
telescope=cols[4],
source=source)
catalog.journal_entries()
# Nicholl 05-03-16
files = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-05-03-16', '*.txt'))
name = catalog.add_entry('SN2015bn')
for fi in pbar(files, task_str + ': Nicholl-05-03-16'):
if 'late' in fi:
bc = '2016ApJ...828L..18N'
else:
bc = '2016ApJ...826...39N'
source = catalog.entries[name].add_source(bibcode=bc)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, 'PS15ae', source)
telescope = os.path.basename(fi).split('_')[1]
with open(fi, 'r') as f:
lines = f.read().splitlines()
for li, line in enumerate(lines):
if not line or (line[0] == '#' and li != 0):
continue
cols = list(filter(None, line.split()))
if not cols:
continue
if li == 0:
bands = cols[1:]
continue
mjd = cols[0]
for ci, col in enumerate(cols[1::2]):
if not is_number(col) or np.isnan(float(col)):
continue
band = bands[ci]
band_set = ''
system = 'Vega'
if bands[ci] in ["u'", "g'", "r'", "i'", "z'"]:
band_set = 'SDSS'
system = 'SDSS'
elif telescope == 'ASASSN':
band_set = 'ASASSN'
system = 'Vega'
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.MAGNITUDE: col,
PHOTOMETRY.BAND: bands[ci],
PHOTOMETRY.SOURCE: source,
PHOTOMETRY.TELESCOPE: telescope,
PHOTOMETRY.SYSTEM: system
}
if band_set:
photodict[PHOTOMETRY.BAND_SET] = band_set
emag = cols[2 * ci + 2]
if is_number(emag):
photodict[PHOTOMETRY.E_MAGNITUDE] = emag
else:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
if telescope == 'Swift':
photodict[PHOTOMETRY.INSTRUMENT] = 'UVOT'
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
return
def do_external(catalog):
task_str = catalog.get_current_task_str()
oldbanddict = {
"Pg": {
"instrument": "Pan-STARRS1",
"band": "g"
},
"Pr": {
"instrument": "Pan-STARRS1",
"band": "r"
},
"Pi": {
"instrument": "Pan-STARRS1",
"band": "i"
},
"Pz": {
"instrument": "Pan-STARRS1",
"band": "z"
},
"Mu": {
"instrument": "MegaCam",
"band": "u"
},
"Mg": {
"instrument": "MegaCam",
"band": "g"
},
"Mr": {
"instrument": "MegaCam",
"band": "r"
},
"Mi": {
"instrument": "MegaCam",
"band": "i"
},
"Mz": {
"instrument": "MegaCam",
"band": "z"
},
"Su": {
"instrument": "SDSS",
"band": "u"
},
"Sg": {
"instrument": "SDSS",
"band": "g"
},
"Sr": {
"instrument": "SDSS",
"band": "r"
},
"Si": {
"instrument": "SDSS",
"band": "i"
},
"Sz": {
"instrument": "SDSS",
"band": "z"
},
"bU": {
"instrument": "Bessel",
"band": "U"
},
"bB": {
"instrument": "Bessel",
"band": "B"
},
"bV": {
"instrument": "Bessel",
"band": "V"
},
"bR": {
"instrument": "Bessel",
"band": "R"
},
"bI": {
"instrument": "Bessel",
"band": "I"
},
"4g": {
"instrument": "PTF 48-Inch",
"band": "g"
},
"4r": {
"instrument": "PTF 48-Inch",
"band": "r"
},
"6g": {
"instrument": "PTF 60-Inch",
"band": "g"
},
"6r": {
"instrument": "PTF 60-Inch",
"band": "r"
},
"6i": {
"instrument": "PTF 60-Inch",
"band": "i"
},
"Uu": {
"instrument": "UVOT",
"band": "U"
},
"Ub": {
"instrument": "UVOT",
"band": "B"
},
"Uv": {
"instrument": "UVOT",
"band": "V"
},
"Um": {
"instrument": "UVOT",
"band": "M2"
},
"U1": {
"instrument": "UVOT",
"band": "W1"
},
"U2": {
"instrument": "UVOT",
"band": "W2"
},
"GN": {
"instrument": "GALEX",
"band": "NUV"
},
"GF": {
"instrument": "GALEX",
"band": "FUV"
},
"CR": {
"instrument": "Clear",
"band": "r"
},
"RO": {
"instrument": "ROTSE"
},
"X1": {
"instrument": "Chandra"
},
"X2": {
"instrument": "XRT"
},
"Xs": {
"instrument": "XRT",
"band": "soft"
},
"Xm": {
"instrument": "XRT",
"band": "hard"
},
"XM": {
"instrument": "XMM"
}
}
path_pattern = os.path.join(catalog.get_current_task_repo(),
'old-tdefit/*.dat')
for datafile in pbar_strings(glob(path_pattern), task_str):
f = open(datafile, 'r')
tsvin = csv.reader(f, delimiter='\t', skipinitialspace=True)
sources = []
yrsmjdoffset = 0.
for row in tsvin:
if row[0] == 'name':
name = re.sub('<[^<]+?>', '', row[1].split(',')[0].strip())
name = catalog.add_entry(name)
elif row[0] == 'citations':
citarr = row[1].split(',')
for cite in citarr:
if '*' in cite:
bibcode = urllib.parse.unquote(
cite.split('/')[-2].split("'")[0])
sources.append(catalog.entries[name].add_source(
bibcode=bibcode))
elif row[0] == 'nhcorr':
hostnhcorr = True if row[1] == 'T' else False
elif row[0] == 'restframe':
restframe = True if row[1] == 'T' else False
elif row[0] == 'yrsmjdoffset':
yrsmjdoffset = float(row[1])
if row[0] == 'redshift':
redshift = float(row[1].split(',')[0].strip(' *'))
source = ','.join(sources)
if not source:
source = catalog.entries[name].add_self_source()
f.seek(0)
for row in tsvin:
if not row or len(row) < 2 or not row[1]:
continue
if row[0] == 'redshift':
for rs in [x.strip() for x in row[1].split(',')]:
catalog.entries[name].add_quantity(
TIDALDISRUPTION.REDSHIFT, rs.strip(' *'), source)
elif row[0] == 'host':
hostname = re.sub('<[^<]+?>', '', row[1])
catalog.entries[name].add_quantity(
TIDALDISRUPTION.HOST, hostname, source)
elif row[0] == 'claimedtype' and row[1] != 'TDE':
cts = row[1].split(',')
for ct in cts:
ctype = ct.strip()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.CLAIMED_TYPE, ctype, source)
elif row[0] == 'citations':
catalog.entries[name].add_quantity(
Key('citations', KEY_TYPES.STRING), row[1], source)
elif row[0] == 'notes':
catalog.entries[name].add_quantity(
Key('notes', KEY_TYPES.STRING), row[1], source)
elif row[0] == 'nh':
catalog.entries[name].add_quantity(
Key('nhhost', KEY_TYPES.STRING), row[1], source)
elif row[0] == 'photometry':
timeunit = row[1]
if timeunit == 'yrs':
timeunit = 'MJD'
if restframe:
# Currently presume only the time, not the flux, has
# been affected by redshifting.
time = str(yrsmjdoffset + float(row[2]) * 365.25 * (
1.0 + redshift))
else:
time = str(yrsmjdoffset + float(row[2]) * 365.25)
lrestframe = False
else:
time = row[2]
if timeunit == 'floatyr':
timeunit = 'MJD'
time = str(astrotime(float(time), format='jyear').mjd)
lrestframe = restframe
instrument = ''
iband = row[3]
if iband in oldbanddict:
if 'band' in oldbanddict[iband]:
band = oldbanddict[iband]['band']
if 'instrument' in oldbanddict[iband]:
instrument = oldbanddict[iband]['instrument']
else:
band = iband
upperlimit = True if row[6] == '1' else False
if 'X' in iband:
counts = Decimal(10.0)**Decimal(row[4])
photodict = {
PHOTOMETRY.TIME: time,
PHOTOMETRY.U_TIME: timeunit,
PHOTOMETRY.BAND: band,
PHOTOMETRY.COUNT_RATE: counts,
PHOTOMETRY.UPPER_LIMIT: upperlimit,
PHOTOMETRY.REST_FRAME: lrestframe,
PHOTOMETRY.HOST_NH_CORR: hostnhcorr,
PHOTOMETRY.INSTRUMENT: instrument,
PHOTOMETRY.SOURCE: source
}
# Old TDEFit stored counts in log
if float(row[5]) != 0.0:
photodict[PHOTOMETRY.E_COUNT_RATE] = str(
(Decimal(10.0)**(Decimal(row[4]) + Decimal(row[5]))
- Decimal(10.0)**Decimal(row[4])))
else:
magnitude = row[4]
photodict = {
PHOTOMETRY.TIME: time,
PHOTOMETRY.U_TIME: timeunit,
PHOTOMETRY.BAND: band,
PHOTOMETRY.MAGNITUDE: magnitude,
PHOTOMETRY.UPPER_LIMIT: upperlimit,
PHOTOMETRY.REST_FRAME: lrestframe,
PHOTOMETRY.HOST_NH_CORR: hostnhcorr,
PHOTOMETRY.INSTRUMENT: instrument,
PHOTOMETRY.SOURCE: source
}
if float(row[5]) != 0.0:
photodict[PHOTOMETRY.E_MAGNITUDE] = row[5]
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
return
def do_batse(catalog):
task_str = catalog.get_current_task_str()
file_path = os.path.join(catalog.get_current_task_repo(),
'BATSE/basic_table.txt')
csvtxt = catalog.load_url(
'http://gammaray.nsstc.nasa.gov/batse/grb/catalog/current/tables/'
'basic_table.txt', file_path)
if not csvtxt:
return
data = list(
csv.reader(csvtxt.splitlines(),
delimiter=' ',
quotechar='"',
skipinitialspace=True))
file_path = os.path.join(catalog.get_current_task_repo(),
'BATSE/duration_table.txt')
csvtxt = catalog.load_url(
'http://gammaray.nsstc.nasa.gov/batse/grb/catalog/current/tables/'
'duration_table.txt', file_path)
if not csvtxt:
return
data2 = list(
csv.reader(csvtxt.splitlines(),
delimiter=' ',
quotechar='"',
skipinitialspace=True))
t90s = {}
for row in data2:
# Add one sigma to quoted T90 to compare to
t90s[row[0]] = float(row[-3]) + float(row[-2])
prev_oname = ''
grb_letter = 'A'
for r, row in enumerate(pbar(data, task_str)):
if row[0].startswith('#'):
continue
oname = 'GRB ' + row[2]
if oname.replace('-', '') == prev_oname:
grb_letter = chr(ord(grb_letter) + 1)
else:
grb_letter = 'A'
prev_oname = oname.replace('-', '')
if oname.endswith('-'):
oname = oname.replace('-', grb_letter)
if row[-1] == 'Y':
continue
if row[0] not in t90s or t90s[row[0]] < 3.0:
continue
(name, source) = catalog.new_entry(
oname,
srcname='BATSE Catalog',
bibcode='1999ApJS..122..465P',
url='http://gammaray.nsstc.nasa.gov/batse/grb/catalog/')
jd = Decimal(2440000.5) + Decimal(row[3])
astrot = astrotime(float(jd), format='jd').datetime
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
make_date_string(astrot.year, astrot.month, astrot.day), source)
pos_err = str(Decimal(row[9]) * Decimal(3600))
catalog.entries[name].add_quantity(SUPERNOVA.RA,
row[5],
source,
u_value='floatdegrees',
e_value=pos_err)
catalog.entries[name].add_quantity(SUPERNOVA.DEC,
row[6],
source,
u_value='floatdegrees',
e_value=pos_err)
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, 'LGRB',
source)
catalog.journal_entries()
return
def do_essence_spectra(catalog):
task_str = catalog.get_current_task_str()
insdict = {
"lris": "LRIS",
"esi": "ESI",
"deimos": "DEIMOS",
"gmos": "GMOS",
"fors1": "FORS1",
"bluechannel": "Blue Channel",
"ldss2": "LDSS-2",
"ldss3": "LDSS-3",
"imacs": "IMACS",
"fast": "FAST"
}
teldict = {
"lris": "Keck",
"esi": "Keck",
"deimos": "Keck",
"gmos": "Gemini",
"fors1": "VLT",
"bluechannel": "MMT",
"ldss2": "Magellan Clay & Baade",
"ldss3": "Magellan Clay & Baade",
"imacs": "Magellan Clay & Baade",
"fast": "FLWO 1.5m"
}
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'ESSENCE', '*'))
oldname = ''
for fi, fname in enumerate(pbar_strings(file_names, task_str)):
filename = os.path.basename(fname)
fileparts = filename.split('_')
name = 'ESSENCE ' + fileparts[0]
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
if is_number(fileparts[1]):
doffset = 1
else:
if fileparts[1] != 'comb':
continue
doffset = 2
dstr = fileparts[doffset]
mjd = str(
astrotime(
datetime.datetime(
year=int(dstr[:4]),
month=int(dstr[4:6]),
day=int(dstr[6:8])) + datetime.timedelta(days=float(dstr[
8:]))).mjd)
instrument = fileparts[-1].split('.')[0]
telescope = teldict.get(instrument, '')
instrument = insdict.get(instrument, '')
with open(fname, 'r') as f:
data = csv.reader(f, delimiter=' ', skipinitialspace=True)
data = [list(i) for i in zip(*data)]
wavelengths = data[0]
fluxes = [str(Decimal('1.0e-15') * Decimal(x)) for x in data[1]]
name, source = catalog.new_entry(name, bibcode='2016ApJS..224....3N')
specdict = {
SPECTRUM.TIME: mjd,
SPECTRUM.U_TIME: 'MJD',
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.WAVELENGTHS: wavelengths,
SPECTRUM.FLUXES: fluxes,
SPECTRUM.U_FLUXES: 'erg/s/cm^2/Angstrom',
SPECTRUM.FILENAME: filename,
SPECTRUM.SOURCE: source
}
if instrument:
specdict[SPECTRUM.INSTRUMENT] = instrument
if telescope:
specdict[SPECTRUM.TELESCOPE] = telescope
catalog.entries[name].add_spectrum(**specdict)
if catalog.args.travis and fi >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
return
def create_goodrun_dict(runlist, filter_runs):
"""
Create a dict of lists. Each entry in each list is one run.
Parameters
----------
runlist : str
Path to a valid good run runlist snapshot from [1]_ in JSON format.
Must have keys 'latest_snapshot' and 'runs'.
filter_runs : function
Filter function to remove unwanted runs from the goodrun list.
Called as `filter_runs(run)`. Function must operate on a single
dictionary `run`, with keys:
['good_i3', 'good_it', 'good_tstart', 'good_tstop', 'run',
'reason_i3', 'reason_it', 'source_tstart', 'source_tstop',
'snapshot', 'sha']
Returns
-------
goodrun_dict : dict
Dictionary with run attributes as keys. The values are stored in
lists in each key. One list item is one run.
Notes
-----
.. [1] https://live.icecube.wisc.edu/snapshots/
"""
with open(runlist, 'r') as jsonFile:
goodruns = json.load(jsonFile)
if not all([k in goodruns.keys() for k in ["latest_snapshot", "runs"]]):
raise ValueError("Runlist misses 'latest_snapshot' or 'runs'")
# This is a list of dicts (one dict per run)
goodrun_list = goodruns["runs"]
# Filter to remove unwanted runs
if filter_runs is None:
def filter_runs(runs):
return True
goodrun_list = list(filter(filter_runs, goodrun_list))
# Convert the run list of dicts to a dict of arrays for easier handling
goodrun_dict = dict(zip(goodrun_list[0].keys(),
zip(*[r.values() for r in goodrun_list])))
for k in goodrun_dict.keys():
goodrun_dict[k] = np.array(goodrun_dict[k])
# Add times to MJD floats
goodrun_dict["good_start_mjd"] = astrotime(
goodrun_dict["good_tstart"]).mjd
goodrun_dict["good_stop_mjd"] = astrotime(
goodrun_dict["good_tstop"]).mjd
# Add runtimes in MJD days
goodrun_dict["runtime_days"] = (goodrun_dict["good_stop_mjd"] -
goodrun_dict["good_start_mjd"])
livetime = np.sum(goodrun_dict["runtime_days"])
return goodrun_dict, livetime
def do_snls_spectra(catalog):
"""
"""
task_str = catalog.get_current_task_str()
result = Vizier.get_catalogs('J/A+A/507/85/table1')
table = result[list(result.keys())[0]]
table.convert_bytestring_to_unicode(python3_only=True)
datedict = {}
for row in table:
datedict['SNLS-' + row['SN']] = str(astrotime(row['Date']).mjd)
oldname = ''
file_names = glob(os.path.join(catalog.get_current_task_repo(), 'SNLS/*'))
for fi, fname in enumerate(pbar_strings(file_names, task_str)):
filename = os.path.basename(fname)
fileparts = filename.split('_')
name = 'SNLS-' + fileparts[1]
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2009A&A...507...85B')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, '20' + fileparts[1][:2], source)
f = open(fname, 'r')
data = csv.reader(f, delimiter=' ', skipinitialspace=True)
specdata = []
for r, row in enumerate(data):
if row[0] == '@TELESCOPE':
telescope = row[1].strip()
elif row[0] == '@REDSHIFT':
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, row[1].strip(), source)
if r < 14:
continue
specdata.append(list(filter(None, [x.strip(' \t') for x in row])))
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[1]
fluxes = [pretty_num(float(x) * 1.e-16, sig=get_sig_digits(x))
for x in specdata[2]]
# FIX: this isnt being used
# errors = [pretty_num(float(x)*1.e-16, sig=get_sig_digits(x)) for x in
# specdata[3]]
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom', u_fluxes='erg/s/cm^2/Angstrom',
wavelengths=wavelengths,
fluxes=fluxes, u_time='MJD' if name in datedict else '',
time=datedict[name] if name in datedict else '',
telescope=telescope, source=source,
filename=filename)
if catalog.args.travis and fi >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
return
def do_tns(catalog):
from datetime import timedelta
session = requests.Session()
task_str = catalog.get_current_task_str()
tns_url = 'https://wis-tns.weizmann.ac.il/'
search_url = tns_url + \
'search?&num_page=1&format=html&sort=desc&order=id&format=csv&page=0'
csvtxt = catalog.load_cached_url(
search_url, os.path.join(catalog.get_current_task_repo(),
'TNS/index.csv'))
if not csvtxt:
return
maxid = csvtxt.splitlines()[1].split(',')[0].strip('"')
maxpages = ceil(int(maxid) / 1000.)
for page in pbar(range(maxpages), task_str):
fname = os.path.join(catalog.get_current_task_repo(), 'TNS/page-') + \
str(page).zfill(2) + '.csv'
if (catalog.current_task.load_archive(catalog.args) and
os.path.isfile(fname) and page < 7):
with open(fname, 'r') as tns_file:
csvtxt = tns_file.read()
else:
with open(fname, 'w') as tns_file:
session = requests.Session()
ses_url = (tns_url + 'search?&num_page=1000&format=html&edit'
'[type]=&edit[objname]=&edit[id]=&sort=asc&order=id'
'&display[redshift]=1'
'&display[hostname]=1&display[host_redshift]=1'
'&display[source_group_name]=1'
'&display[programs_name]=1'
'&display[internal_name]=1'
'&display[isTNS_AT]=1'
'&display[public]=1'
'&display[end_pop_period]=0'
'&display[spectra_count]=1'
'&display[discoverymag]=1&display[discmagfilter]=1'
'&display[discoverydate]=1&display[discoverer]=1'
'&display[sources]=1'
'&display[bibcode]=1&format=csv&page=' + str(page))
try:
response = session.get(ses_url)
csvtxt = response.text
except:
if os.path.isfile(fname):
with open(fname, 'r') as tns_file:
csvtxt = tns_file.read()
else:
continue
else:
tns_file.write(csvtxt)
tsvin = list(csv.reader(csvtxt.splitlines(), delimiter=','))
for ri, row in enumerate(pbar(tsvin, task_str, leave=False)):
if ri == 0:
continue
if row[4] and 'SN' not in row[4]:
continue
name = row[1].replace(' ', '')
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
name='Transient Name Server', url=tns_url)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
if row[2] and row[2] != '00:00:00.00':
catalog.entries[name].add_quantity(
SUPERNOVA.RA, row[2], source)
if row[3] and row[3] != '+00:00:00.00':
catalog.entries[name].add_quantity(
SUPERNOVA.DEC, row[3], source)
if row[4]:
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, row[4].replace('SN', '').strip(),
source)
if row[5]:
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, row[5], source, kind='spectroscopic')
if row[6]:
catalog.entries[name].add_quantity(
SUPERNOVA.HOST, row[6], source)
if row[7]:
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, row[7], source, kind='host')
if row[8]:
catalog.entries[name].add_quantity(
'discoverer', row[8], source)
# Currently, all events listing all possible observers. TNS bug?
# if row[9]:
# observers = row[9].split(',')
# for observer in observers:
# catalog.entries[name].add_quantity('observer',
# observer.strip(),
# source)
if row[10]:
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, row[10], source)
if row[8] and row[14] and row[15] and row[16]:
survey = row[8]
magnitude = row[14]
band = row[15].split('-')[0]
mjd = astrotime(row[16]).mjd
(catalog.entries[name]
.add_photometry(time=mjd, magnitude=magnitude,
band=band, survey=survey, source=source))
if row[16]:
date = row[16].split()[0].replace('-', '/')
if date != '0000/00/00':
date = date.replace('/00', '')
time = row[16].split()[1]
if time != '00:00:00':
ts = time.split(':')
dt = timedelta(hours=int(ts[0]), minutes=int(
ts[1]), seconds=int(ts[2]))
date += pretty_num(dt.total_seconds() /
(24 * 60 * 60), sig=6).lstrip('0')
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, date, source)
if catalog.args.update:
catalog.journal_entries()
catalog.journal_entries()
return
def do_rochester(catalog):
"""Import data from latest supernova page."""
rochestermirrors = [
'http://www.rochesterastronomy.org/',
'http://www.supernova.thistlethwaites.com/'
]
rochesterpaths = [
'snimages/snredshiftall.html', 'sn2020/snredshift.html',
'snimages/snredboneyard.html', 'snimages/snredboneyard-old.html'
]
rochesterupdate = [False, True, True, False]
task_str = catalog.get_current_task_str()
baddates = ['2440587', '2440587.292', '0001/01/01']
for pp, path in enumerate(pbar(rochesterpaths, task_str)):
if catalog.args.update and not rochesterupdate[pp]:
continue
if 'snredboneyard.html' in path:
cns = {
'name': 0,
'host': 1,
'ra': 2,
'dec': 3,
'type': 7,
'z': 8,
'mmag': 9,
'max': 10,
'disc': 11,
'ref': 12,
'dver': 13,
'aka': 14
}
else:
cns = {
'name': 0,
'type': 1,
'host': 2,
'ra': 3,
'dec': 4,
'disc': 6,
'max': 7,
'mmag': 8,
'z': 11,
'zh': 12,
'ref': 13,
'dver': 14,
'aka': 15
}
filepath = (
os.path.join(catalog.get_current_task_repo(), 'rochester/') +
path.replace('/', '-'))
for mirror in rochestermirrors:
html = catalog.load_url(mirror + path,
filepath,
fail=(mirror != rochestermirrors[-1]))
if html:
break
if not html:
continue
soup = BeautifulSoup(html, 'html5lib')
rows = soup.findAll('tr')
sec_ref = 'Latest Supernovae'
sec_refurl = ('http://www.rochesterastronomy.org/'
'snimages/snredshiftall.html')
loopcnt = 0
for rr, row in enumerate(pbar(rows, task_str)):
if rr == 0:
continue
cols = row.findAll('td')
if not len(cols):
continue
name = ''
if cols[cns['aka']].contents:
for rawaka in str(cols[cns['aka']].contents[0]).split(','):
aka = rawaka.strip()
if is_number(aka.strip('?')):
aka = 'SN' + aka.strip('?') + 'A'
oldname = aka
name = catalog.add_entry(aka)
elif len(aka) == 4 and is_number(aka[:4]):
aka = 'SN' + aka
oldname = aka
name = catalog.add_entry(aka)
sn = re.sub('<[^<]+?>', '',
str(cols[cns['name']].contents[0])).strip()
if is_number(sn.strip('?')):
sn = 'SN' + sn.strip('?') + 'A'
elif len(sn) == 4 and is_number(sn[:4]):
sn = 'SN' + sn
if not name:
if not sn or sn in ['Transient']:
continue
ra = str(cols[cns['ra']].contents[0]).strip().replace(':.', '.')
dec = str(cols[cns['dec']].contents[0]).strip().replace(':.', '.')
if not name:
if sn[:8] == 'MASTER J':
sn = sn.replace('MASTER J',
'MASTER OT J').replace('SNHunt', 'SNhunt')
if 'POSSIBLE' in sn.upper() and ra and dec:
sn = 'PSN J' + ra.replace(':', '').replace('.', '')
sn += dec.replace(':', '').replace('.', '')
oldname = sn
name = catalog.add_entry(sn)
sec_source = catalog.entries[name].add_source(name=sec_ref,
url=sec_refurl,
secondary=True)
sources = []
if 'ref' in cns:
reftag = reference = cols[cns['ref']].findAll('a')
if len(reftag) and len(reftag[0].contents):
reference = reftag[0].contents[0].strip()
refurl = reftag[0]['href'].strip()
sources.append(catalog.entries[name].add_source(
name=reference, url=refurl))
sources.append(sec_source)
sources = uniq_cdl(list(filter(None, sources)))
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oldname,
sources)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, sn, sources)
if cols[cns['aka']].contents:
for rawaka in str(cols[cns['aka']].contents[0]).split(','):
aka = rawaka.strip()
if aka == 'SNR G1.9+0.3':
aka = 'G001.9+00.3'
if aka[:4] == 'PS1 ':
aka = 'PS1-' + aka[4:]
if aka[:8] == 'MASTER J':
aka = aka.replace('MASTER J', 'MASTER OT J').replace(
'SNHunt', 'SNhunt')
if 'POSSIBLE' in aka.upper() and ra and dec:
aka = 'PSN J' + ra.replace(':', '').replace('.', '')
aka += dec.replace(':', '').replace('.', '')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, aka,
sources)
if (len(cols[cns['type']].contents) > 0
and str(cols[cns['type']].contents[0]).strip() != 'unk'):
type = str(cols[cns['type']].contents[0]).strip(' :,')
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE,
type, sources)
if (len(cols[cns['host']].contents) > 0 and
str(cols[cns['host']].contents[0]).strip() != 'anonymous'):
catalog.entries[name].add_quantity(
SUPERNOVA.HOST,
str(cols[cns['host']].contents[0]).strip(), sources)
catalog.entries[name].add_quantity(SUPERNOVA.RA, ra, sources)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, dec, sources)
discstr = str(cols[cns['disc']].contents[0]).strip()
if discstr and discstr not in baddates:
if '/' not in discstr:
astrot = astrotime(float(discstr), format='jd').datetime
ddate = make_date_string(astrot.year, astrot.month,
astrot.day)
else:
ddate = discstr
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE,
ddate, sources)
maxstr = str(cols[cns.get('max', '')].contents[0]).strip()
if maxstr and maxstr not in baddates:
try:
if '/' not in maxstr:
astrot = astrotime(float(maxstr), format='jd')
else:
astrot = astrotime(maxstr.replace('/', '-'),
format='iso')
except:
catalog.log.info(
'Max date conversion failed for `{}`.'.format(maxstr))
if ((float(str(cols[cns['mmag']].contents[0]).strip()) <= 90.0
and
not any('GRB' in xx
for xx in catalog.entries[name].get_aliases()))):
mag = str(cols[cns['mmag']].contents[0]).strip()
catalog.entries[name].add_photometry(time=str(astrot.mjd),
u_time='MJD',
magnitude=mag,
source=sources)
if 'z' in cns and cols[cns['z']].contents[0] != 'n/a':
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT,
str(cols[cns['z']].contents[0]).strip(), sources)
if 'zh' in cns:
zhost = str(cols[cns['zh']].contents[0]).strip()
if is_number(zhost):
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT,
zhost, sources)
if 'dver' in cns:
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVERER,
str(cols[cns['dver']].contents[0]).strip(), sources)
if catalog.args.update:
catalog.journal_entries()
loopcnt = loopcnt + 1
if (catalog.args.travis
and loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
if not catalog.args.update:
vsnetfiles = ['latestsne.dat']
for vsnetfile in vsnetfiles:
file_name = os.path.join(catalog.get_current_task_repo(),
"" + vsnetfile)
with open(file_name, 'r', encoding='latin1') as csv_file:
tsvin = csv.reader(csv_file,
delimiter=' ',
skipinitialspace=True)
loopcnt = 0
for rr, row in enumerate(tsvin):
if (not row or row[0] in ['Transient']
or row[0][:4] in ['http', 'www.'] or len(row) < 3):
continue
name = row[0].strip()
if name[:4].isdigit():
name = 'SN' + name
if name.startswith('PSNJ'):
name = 'PSN J' + name[4:]
if name.startswith('MASTEROTJ'):
name = name.replace('MASTEROTJ', 'MASTER OT J')
name = catalog.add_entry(name)
sec_source = catalog.entries[name].add_source(
name=sec_ref, url=sec_refurl, secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name,
sec_source)
if not is_number(row[1]):
continue
year = row[1][:4]
month = row[1][4:6]
day = row[1][6:]
if '.' not in day:
day = day[:2] + '.' + day[2:]
mjd = astrotime(year + '-' + month + '-' +
str(floor(float(day))).zfill(2)).mjd
mjd += float(day) - floor(float(day))
magnitude = row[2].rstrip(ascii_letters)
if not is_number(magnitude):
continue
if magnitude.isdigit():
if int(magnitude) > 100:
magnitude = magnitude[:2] + '.' + magnitude[2:]
if float(str(cols[8].contents[0]).strip()) >= 90.0:
continue
if len(row) >= 4:
if is_number(row[3]):
e_magnitude = row[3]
refind = 4
else:
e_magnitude = ''
refind = 3
if refind >= len(row):
sources = sec_source
else:
reference = ' '.join(row[refind:])
source = catalog.entries[name].add_source(
name=reference)
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, name, sec_source)
sources = uniq_cdl([source, sec_source])
else:
sources = sec_source
band = row[2].lstrip('1234567890.')
catalog.entries[name].add_photometry(
time=mjd,
u_time='MJD',
band=band,
magnitude=magnitude,
e_magnitude=e_magnitude,
source=sources)
if (catalog.args.travis
and loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
else:
hosts[hn]['redshift'] = item['redshift'][ind]['value']
if not hosts[hn]['hostra'] and 'hostra' in item:
hosts[hn]['hostra'] = item['hostra'][0]['value']
if not hosts[hn]['hostdec'] and 'hostdec' in item:
hosts[hn]['hostdec'] = item['hostdec'][0]['value']
if 'discoverdate' in item and item['discoverdate']:
datestr = item['discoverdate'][0]['value'].replace('/', '-')
if datestr.count('-') == 1:
datestr += '-01'
elif datestr.count('-') == 0:
datestr += '-01-01'
try:
hosts[hn]['eventdates'].append(astrotime(datestr, format = 'isot').unix)
except:
hosts[hn]['eventdates'].append(float("inf"))
else:
hosts[hn]['eventdates'].append(float("inf"))
if 'claimedtype' in item:
cts = []
for ct in item['claimedtype']:
sct = ct['value'].strip('?')
if sct:
cts.append(sct)
hosts[hn]['types'] = list(set(hosts[hn]['types']).union(cts))
if 'photometry' in item:
hosts[hn]['photocount'] += len(item['photometry'])
def do_batse(catalog):
task_str = catalog.get_current_task_str()
file_path = os.path.join(catalog.get_current_task_repo(),
'BATSE/basic_table.txt')
csvtxt = catalog.load_url(
'http://gammaray.nsstc.nasa.gov/batse/grb/catalog/current/tables/'
'basic_table.txt', file_path)
if not csvtxt:
return
data = list(
csv.reader(
csvtxt.splitlines(),
delimiter=' ',
quotechar='"',
skipinitialspace=True))
file_path = os.path.join(catalog.get_current_task_repo(),
'BATSE/duration_table.txt')
csvtxt = catalog.load_url(
'http://gammaray.nsstc.nasa.gov/batse/grb/catalog/current/tables/'
'duration_table.txt', file_path)
if not csvtxt:
return
data2 = list(
csv.reader(
csvtxt.splitlines(),
delimiter=' ',
quotechar='"',
skipinitialspace=True))
t90s = {}
for row in data2:
# Add one sigma to quoted T90 to compare to
t90s[row[0]] = float(row[-3]) + float(row[-2])
prev_oname = ''
grb_letter = 'A'
for r, row in enumerate(pbar(data, task_str)):
if row[0].startswith('#'):
continue
oname = 'GRB ' + row[2]
if oname.replace('-', '') == prev_oname:
grb_letter = chr(ord(grb_letter) + 1)
else:
grb_letter = 'A'
prev_oname = oname.replace('-', '')
if oname.endswith('-'):
oname = oname.replace('-', grb_letter)
if row[-1] == 'Y':
continue
if row[0] not in t90s or t90s[row[0]] < 3.0:
continue
(name, source) = catalog.new_entry(
oname,
srcname='BATSE Catalog',
bibcode='1999ApJS..122..465P',
url='http://gammaray.nsstc.nasa.gov/batse/grb/catalog/')
jd = Decimal(2440000.5) + Decimal(row[3])
astrot = astrotime(float(jd), format='jd').datetime
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
make_date_string(astrot.year, astrot.month, astrot.day), source)
pos_err = str(Decimal(row[9]) * Decimal(3600))
catalog.entries[name].add_quantity(
SUPERNOVA.RA,
row[5],
source,
u_value='floatdegrees',
e_value=pos_err)
catalog.entries[name].add_quantity(
SUPERNOVA.DEC,
row[6],
source,
u_value='floatdegrees',
e_value=pos_err)
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, 'LGRB',
source)
catalog.journal_entries()
return
('types', []), ('photocount', 0),
('spectracount', 0),
('metacount', 0)])
biblio[bc]['events'].append(item['name'])
if 'discoverdate' in item and item['discoverdate']:
datestr = item['discoverdate'][0]['value'].replace(
'/', '-')
if datestr.count('-') == 1:
datestr += '-01'
elif datestr.count('-') == 0:
datestr += '-01-01'
try:
biblio[bc]['eventdates'].append(
astrotime(datestr, format='isot').unix)
except:
biblio[bc]['eventdates'].append(float("inf"))
else:
biblio[bc]['eventdates'].append(float("inf"))
if 'claimedtype' in item:
cts = []
for ct in item['claimedtype']:
cts.append(ct['value'].strip('?'))
biblio[bc]['types'] = list(
set(biblio[bc]['types']).union(cts))
if 'photometry' in item:
bcalias = source['alias']
lc = 0
def generate_event_list(self, event_list):
"""Generate a list of events and/or convert events to JSON format."""
prt = self._printer
cidict = OrderedDict()
intro_shown = False
new_event_list = []
previous_file = None
for event in event_list:
rsource = {SOURCE.NAME: self._DEFAULT_SOURCE}
use_self_source = None
new_events = []
toffset = Decimal('0')
if ('.' in event and os.path.isfile(event) and
not event.endswith('.json')):
if not intro_shown:
prt.message('converter_info')
intro_shown = True
prt.message('converting_to_json', [event])
with open(event, 'r') as f:
ftxt = f.read()
# Try a couple of table formats from astropy.
table = None
try:
table = read(ftxt, Reader=Cds, guess=False)
except Exception:
pass
else:
prt.message('convert_cds')
flines = [table.colnames] + [
list(x) for x in np.array(table).tolist()]
for i in range(len(flines)):
flines[i] = [str(x) for x in flines[i]]
try:
table = read(ftxt, Reader=Latex, guess=False)
except Exception:
pass
else:
prt.message('convert_latex')
flines = [table.colnames] + [
list(x) for x in np.array(table).tolist()]
if table is None:
# Count to try and determine delimiter.
delims = [' ', '\t', ',', ';', '|', '&']
delimnames = [
'Space: ` `', 'Tab: `\t`', 'Comma: `,`',
'Semi-colon: `;`', 'Bar: `|`', 'Ampersand: `&`']
delim = None
delimcounts = [ftxt.count(x) for x in delims]
maxdelimcount = max(delimcounts)
delim = delims[delimcounts.index(maxdelimcount)]
# If two delimiter options are close in count, ask user.
for i, x in enumerate(delimcounts):
if x > 0.5 * maxdelimcount and delims[i] != delim:
delim = None
if delim is None:
odelims = list(np.array(delimnames)[
np.array(delimcounts) > 0])
delim = delims[prt.prompt(
'delim', kind='option', options=odelims) - 1]
ad = list(delims)
ad.remove(delim)
ad = ''.join(ad)
fsplit = ftxt.splitlines()
fsplit = [
x.replace('$', '').replace('\\pm', delim)
.replace('±', delim).replace('(', delim + '(')
.strip(ad + '()# ').replace('′', "'")
for x in fsplit]
flines = []
for fs in fsplit:
flines.append(list(
csv.reader([fs], delimiter=delim))[0])
flines = [[
x.strip(ad + '#$()\\')
for x in y] for y in flines]
# Find band columns if they exist and insert error columns
# if they don't exist.
for fi, fl in enumerate(list(flines)):
flcopy = list(fl)
offset = 0
if not any([is_number(x) for x in fl]):
for fci, fc in enumerate(fl):
if (fc in self._band_names and
(fci == len(fl) - 1 or
fl[fci + 1] not in self._emagstrs)):
flcopy.insert(fci + 1 + offset, 'e mag')
offset += 1
flines[fi] = flcopy
# Find the most frequent column count. These are probably
# the tables we wish to read.
flens = [len(x) for x in flines]
ncols = Counter(flens).most_common(1)[0][0]
newlines = []
potential_name = None
for fi, fl in enumerate(flines):
if (len(fl) and flens[fi] == 1 and
fi < len(flines) - 1 and
flens[fi + 1] == ncols and not len(newlines)):
potential_name = fl[0]
if flens[fi] == ncols:
if potential_name is not None and any(
[is_number(x) for x in fl]):
newlines.append([potential_name] + list(fl))
else:
newlines.append(list(fl))
flines = newlines
for fi, fl in enumerate(flines):
if len(fl) == ncols and potential_name is not None:
if not any([is_number(x) for x in fl]):
flines[fi] = ['name'] + list(fl)
# If none of the rows contain numeric data, the file
# is likely a list of transient names.
if (len(flines) and
(not any(any([is_number(x) or x == '' for x in y])
for y in flines) or
len(flines) == 1)):
new_events = [
it for s in flines for it in s]
# If last row is numeric, then likely this is a file with
# transient data.
elif (len(flines) > 1 and
any([is_number(x) for x in flines[-1]])):
# Check that each row has the same number of columns.
if len(set([len(x) for x in flines])) > 1:
print(set([len(x) for x in flines]))
raise ValueError(
'Number of columns in each row not '
'consistent!')
if len(cidict) and len(new_event_list):
msg = ('is_file_same' if
previous_file else 'is_event_same')
reps = [previous_file] if previous_file else [''.join(
new_event_list[-1].split('.')[:-1])]
text = prt.text(msg, reps)
is_same = prt.prompt(text, message=False,
kind='bool')
if not is_same:
cidict = OrderedDict()
# If the first row has no numbers it is likely a header.
if not len(cidict):
self.assign_columns(cidict, flines)
perms = 1
for key in cidict:
if isinstance(cidict[key], list) and not isinstance(
cidict[key], string_types):
if cidict[key][0] != 'j':
perms = len(cidict[key])
# Get event name (if single event) or list of names from
# table.
event_names = []
if ENTRY.NAME in cidict:
for fi, fl in enumerate(flines):
flines[fi][cidict[ENTRY.NAME]] = name_clean(
fl[cidict[ENTRY.NAME]])
event_names = list(sorted(set([
x[cidict[ENTRY.NAME]] for x in flines[
self._first_data:]])))
new_events = [x + '.json' for x in event_names]
else:
new_event_name = '.'.join(event.split(
'.')[:-1]).split('/')[-1]
text = prt.message(
'is_event_name', [new_event_name], prt=False)
is_name = prt.prompt(text, message=False,
kind='bool', default='y')
if not is_name:
new_event_name = ''
while new_event_name.strip() == '':
new_event_name = prt.prompt(
'enter_name', kind='string')
event_names.append(new_event_name)
new_events = [new_event_name + '.json']
# Create a new event, populate the photometry, and dump
# to a JSON file in the run directory.
entries = OrderedDict([(x, Entry(name=x))
for x in event_names])
# Clean up the data a bit now that we know the column
# identities.
# Strip common prefixes/suffixes from band names
if PHOTOMETRY.BAND in cidict:
bi = cidict[PHOTOMETRY.BAND]
for d in [True, False]:
if not isinstance(bi, (int, np.integer)):
break
strip_cols = []
lens = [len(x[bi])
for x in flines[self._first_data:]]
llen = min(lens)
ra = range(llen) if d else range(-1, -llen - 1, -1)
for li in ra:
letter = None
for row in list(flines[self._first_data:]):
if letter is None:
letter = row[bi][li]
elif row[bi][li] != letter:
letter = None
break
if letter is not None:
strip_cols.append(li)
else:
break
if len(strip_cols) == llen:
break
for ri in range(len(flines[self._first_data:])):
flines[self._first_data + ri][bi] = ''.join(
[c for i, c in enumerate(flines[
self._first_data + ri][bi])
if (i if d else i - len(flines[
self._first_data + ri][bi])) not in
strip_cols])
if (PHOTOMETRY.TIME in cidict and
(not isinstance(cidict[PHOTOMETRY.TIME], list) or
len(cidict[PHOTOMETRY.TIME]) <= 2)):
bi = cidict[PHOTOMETRY.TIME]
if isinstance(bi, list) and not isinstance(
bi, string_types) and isinstance(
bi[0], string_types) and bi[0] == 'jd':
bi = bi[-1]
mmtimes = [float(x[bi])
for x in flines[self._first_data:]]
mintime, maxtime = min(mmtimes), max(mmtimes)
if mintime < 10000:
while True:
try:
response = prt.prompt(
'small_time_offset', kind='string')
if response is not None:
toffset = Decimal(response)
break
except Exception:
pass
elif maxtime > 60000 and cidict[
PHOTOMETRY.TIME][0] != 'jd':
isjd = prt.prompt(
'large_time_offset',
kind='bool', default='y')
if isjd:
toffset = Decimal('-2400000.5')
for row in flines[self._first_data:]:
photodict = {}
rname = (row[cidict[ENTRY.NAME]]
if ENTRY.NAME in cidict else event_names[0])
for pi in range(perms):
sources = set()
for key in cidict:
if key in self._bool_keys:
rval = row[cidict[key]]
if rval in self._FALSE_VALS:
rval = False
elif rval in self._TRUE_VALS:
rval = True
if type(rval) != 'bool':
try:
rval = bool(rval)
except Exception:
pass
if type(rval) != 'bool':
try:
rval = bool(float(rval))
except Exception:
rval = True
if not rval:
continue
row[cidict[key]] = rval
elif key == 'reference':
if (isinstance(cidict[key],
string_types) and
len(cidict[key]) == 19):
new_src = entries[rname].add_source(
bibcode=cidict[key])
sources.update(new_src)
row[
cidict[key]] = new_src
elif key == ENTRY.NAME:
continue
elif (isinstance(key, Key) and
key.type == KEY_TYPES.TIME and
isinstance(cidict[key], list) and not
isinstance(cidict[key],
string_types)):
tval = np.array(row)[np.array(cidict[key][
1:], dtype=int)]
if cidict[key][0] == 'j':
date = '-'.join([x.zfill(2) for x in
tval])
date = self._month_rep.sub(
lambda x: self._MONTH_IDS[
x.group()], date)
photodict[key] = str(
astrotime(date, format='isot').mjd)
elif cidict[key][0] == 'jd':
photodict[key] = str(
jd_to_mjd(Decimal(tval[-1])))
continue
val = cidict[key]
if (isinstance(val, list) and not
isinstance(val, string_types)):
val = val[pi]
if isinstance(val, string_types):
if val != '':
photodict[key] = val
else:
photodict[key] = row[val]
else:
if isinstance(val, string_types):
if val != '':
photodict[key] = val
else:
photodict[key] = row[val]
if self._data_type == 2:
if self._zp:
photodict[PHOTOMETRY.ZERO_POINT] = self._zp
else:
photodict[PHOTOMETRY.ZERO_POINT] = (
row[cidict[PHOTOMETRY.ZERO_POINT][pi]]
if isinstance(cidict[
PHOTOMETRY.ZERO_POINT], list) else
row[cidict[PHOTOMETRY.ZERO_POINT]])
zpp = photodict[PHOTOMETRY.ZERO_POINT]
cc = (
row[cidict[PHOTOMETRY.COUNT_RATE][pi]] if
isinstance(cidict[
PHOTOMETRY.COUNT_RATE], list) else
row[cidict[PHOTOMETRY.COUNT_RATE]])
ecc = (
row[cidict[PHOTOMETRY.E_COUNT_RATE][pi]] if
isinstance(cidict[
PHOTOMETRY.E_COUNT_RATE], list) else
row[cidict[PHOTOMETRY.E_COUNT_RATE]])
if '<' in cc:
set_pd_mag_from_counts(
photodict, ec=cc.strip('<'), zp=zpp)
else:
set_pd_mag_from_counts(
photodict, c=cc, ec=ecc, zp=zpp)
elif self._data_type == 3:
photodict[
PHOTOMETRY.U_FLUX_DENSITY] = self._ufd
if PHOTOMETRY.U_FLUX_DENSITY in cidict:
photodict[PHOTOMETRY.U_FLUX_DENSITY] = (
row[cidict[
PHOTOMETRY.U_FLUX_DENSITY][pi]]
if isinstance(cidict[
PHOTOMETRY.
U_FLUX_DENSITY], list) else
row[cidict[PHOTOMETRY.U_FLUX_DENSITY]])
if photodict[
PHOTOMETRY.U_FLUX_DENSITY] == '':
photodict[
PHOTOMETRY.U_FLUX_DENSITY] = 'µJy'
fd = (
row[cidict[PHOTOMETRY.FLUX_DENSITY][pi]] if
isinstance(cidict[
PHOTOMETRY.FLUX_DENSITY], list) else
row[cidict[PHOTOMETRY.FLUX_DENSITY]])
efd = (
row[cidict[
PHOTOMETRY.E_FLUX_DENSITY][pi]] if
isinstance(cidict[
PHOTOMETRY.E_FLUX_DENSITY], list) else
row[cidict[PHOTOMETRY.E_FLUX_DENSITY]])
mult = Decimal('1')
ufd = photodict[PHOTOMETRY.U_FLUX_DENSITY]
if ufd.lower() in [
'mjy', 'millijy', 'millijansky']:
mult = Decimal('1e3')
elif ufd.lower() in ['jy', 'jansky']:
mult = Decimal('1e6')
if '<' in fd:
set_pd_mag_from_flux_density(
photodict, efd=str(
Decimal(fd.strip('<')) * mult))
else:
set_pd_mag_from_flux_density(
photodict, fd=Decimal(fd) * mult,
efd=Decimal(efd) * mult)
if not len(sources):
if use_self_source is None:
sopts = [
('Bibcode', 'b'), ('Last name', 'l')]
if self._require_source:
sel_str = 'must_select_source'
else:
sel_str = 'select_source'
text = prt.text(sel_str)
skind = prt.prompt(
text, kind='option',
options=sopts, default='b',
none_string=(
None if self._require_source else
'Neither, tag MOSFiT as source'))
if skind == 'b':
rsource = {}
bibcode = ''
while len(bibcode) != 19:
bibcode = prt.prompt(
'bibcode',
kind='string',
allow_blank=False
)
bibcode = bibcode.strip()
if (re.search(
'[0-9]{4}..........[\.0-9]{4}'
'[A-Za-z]', bibcode)
is None):
bibcode = ''
rsource[
SOURCE.BIBCODE] = bibcode
use_self_source = False
elif skind == 'l':
rsource = {}
last_name = prt.prompt(
'last_name', kind='string'
)
rsource[
SOURCE.NAME] = (
last_name.strip().title() +
' et al., in preparation')
use_self_source = False
elif skind == 'n':
use_self_source = True
photodict[
PHOTOMETRY.SOURCE] = entries[
rname].add_source(**rsource)
if any([x in photodict.get(
PHOTOMETRY.MAGNITUDE, '')
for x in ['<', '>']]):
photodict[PHOTOMETRY.UPPER_LIMIT] = True
photodict[
PHOTOMETRY.MAGNITUDE] = photodict[
PHOTOMETRY.MAGNITUDE].strip('<>')
if '<' in photodict.get(PHOTOMETRY.COUNT_RATE, ''):
photodict[PHOTOMETRY.UPPER_LIMIT] = True
photodict[
PHOTOMETRY.COUNT_RATE] = photodict[
PHOTOMETRY.COUNT_RATE].strip('<')
if PHOTOMETRY.E_COUNT_RATE in photodict:
del(photodict[PHOTOMETRY.E_COUNT_RATE])
if '<' in photodict.get(
PHOTOMETRY.FLUX_DENSITY, ''):
photodict[PHOTOMETRY.UPPER_LIMIT] = True
photodict[
PHOTOMETRY.FLUX_DENSITY] = photodict[
PHOTOMETRY.FLUX_DENSITY].strip('<')
if PHOTOMETRY.E_FLUX_DENSITY in photodict:
del(photodict[PHOTOMETRY.E_FLUX_DENSITY])
# Apply offset time if set.
if (PHOTOMETRY.TIME in photodict and
toffset != Decimal('0')):
photodict[PHOTOMETRY.TIME] = str(
Decimal(photodict[PHOTOMETRY.TIME]) +
toffset)
# Skip entries for which key values are not
# expected type.
if not all([
is_number(photodict.get(x, ''))
for x in photodict.keys() if
(PHOTOMETRY.get_key_by_name(x).type ==
KEY_TYPES.NUMERIC)]):
continue
# Skip placeholder values.
if float(photodict.get(
PHOTOMETRY.MAGNITUDE, 0.0)) > 50.0:
continue
# Add system if specified by user.
if (self._system is not None and
PHOTOMETRY.SYSTEM not in photodict):
photodict[PHOTOMETRY.SYSTEM] = self._system
# Remove keys not in the `PHOTOMETRY` class.
for key in list(photodict.keys()):
if key not in PHOTOMETRY.vals():
del(photodict[key])
# Add the photometry.
entries[rname].add_photometry(
**photodict)
merge_with_existing = None
for ei, entry in enumerate(entries):
entries[entry].sanitize()
if os.path.isfile(new_events[ei]):
if merge_with_existing is None:
merge_with_existing = prt.prompt(
'merge_with_existing', default='y')
if merge_with_existing:
existing = Entry.init_from_file(
catalog=None,
name=event_names[ei],
path=new_events[ei],
merge=False,
pop_schema=False,
ignore_keys=[ENTRY.MODELS],
compare_to_existing=False)
Catalog().copy_entry_to_entry(
existing, entries[entry])
oentry = entries[entry]._ordered(entries[entry])
entabbed_json_dump(
{entry: oentry}, open(new_events[ei], 'w'),
separators=(',', ':'))
self._converted.extend([
[event_names[x], new_events[x]]
for x in range(len(event_names))])
new_event_list.extend(new_events)
previous_file = event
else:
new_event_list.append(event)
return new_event_list
def do_snf_specta(catalog):
task_str = catalog.get_current_task_str()
bibcodes = {
'SN2005gj': '2006ApJ...650..510A',
'SN2006D': '2007ApJ...654L..53T',
'SN2007if': '2010ApJ...713.1073S',
'SN2011fe': '2013A&A...554A..27P'
}
oldname = ''
snfcnt = 0
eventfolders = next(
os.walk(os.path.join(catalog.get_current_task_repo(), 'SNFactory')))[1]
for eventfolder in pbar(eventfolders, task_str):
oname = eventfolder
name = catalog.get_preferred_name(oname)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
sec_reference = 'Nearby Supernova Factory'
sec_refurl = 'http://snfactory.lbl.gov/'
sec_bibcode = '2002SPIE.4836...61A'
sec_source = catalog.entries[name].add_source(name=sec_reference,
url=sec_refurl,
bibcode=sec_bibcode,
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oname, sec_source)
bibcode = bibcodes[oname]
source = catalog.entries[name].add_source(bibcode=bibcode)
sources = uniq_cdl([source, sec_source])
use_path = os.path.join(catalog.get_current_task_repo(), 'SNFactory',
eventfolder, '*.dat')
eventspectra = glob(use_path)
for spectrum in pbar(eventspectra, task_str):
filename = os.path.basename(spectrum)
with open(spectrum) as spec_file:
specdata = list(
csv.reader(spec_file, delimiter=' ',
skipinitialspace=True))
specdata = list(filter(None, specdata))
newspec = []
time = ''
telescope = ''
instrument = ''
observer = ''
observatory = ''
if 'Keck_20060202_R' in spectrum:
time = '53768.23469'
elif 'Spectrum05_276' in spectrum:
time = pretty_num(astrotime('2005-10-03').mjd, sig=5)
elif 'Spectrum05_329' in spectrum:
time = pretty_num(astrotime('2005-11-25').mjd, sig=5)
elif 'Spectrum05_336' in spectrum:
time = pretty_num(astrotime('2005-12-02').mjd, sig=5)
for row in specdata:
if row[0][0] == '#':
joinrow = (' '.join(row)).split('=')
if len(joinrow) < 2:
continue
field = joinrow[0].strip('# ')
value = joinrow[1].split('/')[0].strip('\' ')
if not time:
if field == 'JD':
time = str(jd_to_mjd(Decimal(value)))
elif field == 'MJD':
time = value
elif field == 'MJD-OBS':
time = value
if field == 'OBSERVER':
observer = value.capitalize()
if field == 'OBSERVAT':
observatory = value.capitalize()
if field == 'TELESCOP':
telescope = value.capitalize()
if field == 'INSTRUME':
instrument = value.capitalize()
else:
newspec.append(row)
if not time:
raise ValueError('Time missing from spectrum.')
specdata = newspec
haserrors = len(
specdata[0]
) == 3 and specdata[0][2] and specdata[0][2] != 'NaN'
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
unit_err = ('Variance'
if oldname == 'SN2011fe' else 'erg/s/cm^2/Angstrom')
unit_flx = 'erg/s/cm^2/Angstrom'
catalog.entries[name].add_spectrum(u_wavelengths='Angstrom',
u_fluxes=unit_flx,
u_time='MJD',
time=time,
wavelengths=wavelengths,
fluxes=fluxes,
errors=errors,
observer=observer,
observatory=observatory,
telescope=telescope,
instrument=instrument,
u_errors=unit_err,
source=sources,
filename=filename)
snfcnt = snfcnt + 1
if (catalog.args.travis
and snfcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
def do_snls_spectra(catalog):
"""
"""
task_str = catalog.get_current_task_str()
result = Vizier.get_catalogs('J/A+A/507/85/table1')
table = result[list(result.keys())[0]]
table.convert_bytestring_to_unicode()
datedict = {}
for row in table:
datedict['SNLS-' + row['SN']] = str(astrotime(row['Date']).mjd)
oldname = ''
file_names = glob(os.path.join(catalog.get_current_task_repo(), 'SNLS/*'))
for fi, fname in enumerate(pbar_strings(file_names, task_str)):
filename = os.path.basename(fname)
fileparts = filename.split('_')
name = 'SNLS-' + fileparts[1]
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2009A&A...507...85B')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE,
'20' + fileparts[1][:2], source)
f = open(fname, 'r')
data = csv.reader(f, delimiter=' ', skipinitialspace=True)
specdata = []
for r, row in enumerate(data):
if row[0] == '@TELESCOPE':
telescope = row[1].strip()
elif row[0] == '@REDSHIFT':
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT,
row[1].strip(), source)
if r < 14:
continue
specdata.append(list(filter(None, [x.strip(' \t') for x in row])))
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[1]
fluxes = [
pretty_num(float(x) * 1.e-16, sig=get_sig_digits(x))
for x in specdata[2]
]
# FIX: this isnt being used
errors = [
pretty_num(float(x) * 1.e-16, sig=get_sig_digits(x))
for x in specdata[3]
]
fluxunit = 'erg/s/cm^2/Angstrom'
specdict = {
SPECTRUM.WAVELENGTHS: wavelengths,
SPECTRUM.FLUXES: fluxes,
SPECTRUM.ERRORS: errors,
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.U_ERRORS: fluxunit,
SPECTRUM.TELESCOPE: telescope,
SPECTRUM.FILENAME: filename,
SPECTRUM.SOURCE: source
}
if name in datedict:
specdict[SPECTRUM.TIME] = datedict[name]
specdict[SPECTRUM.U_TIME] = 'MJD'
catalog.entries[name].add_spectrum(**specdict)
if catalog.args.travis and fi >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
return
allauthors = allauthors[0].author
else:
allauthors = []
biblio[bc] = OrderedDict([('authors', authors), ('allauthors', allauthors), ('bibcode', bc), ('events', []), ('eventdates', []),
('types', []), ('photocount', 0), ('spectracount', 0), ('metacount', 0)])
biblio[bc]['events'].append(item['name'])
if 'discoverdate' in item and item['discoverdate']:
datestr = item['discoverdate'][0]['value'].replace('/', '-')
if datestr.count('-') == 1:
datestr += '-01'
elif datestr.count('-') == 0:
datestr += '-01-01'
try:
biblio[bc]['eventdates'].append(astrotime(datestr, format = 'isot').unix)
except:
biblio[bc]['eventdates'].append(float("inf"))
else:
biblio[bc]['eventdates'].append(float("inf"))
if 'claimedtype' in item:
cts = []
for ct in item['claimedtype']:
cts.append(ct['value'].strip('?'))
biblio[bc]['types'] = list(set(biblio[bc]['types']).union(cts))
if 'photometry' in item:
bcalias = source['alias']
lc = 0
for photo in item['photometry']:
def do_csp_fits_spectra(catalog):
from astropy.io import fits
fpath = catalog.get_current_task_repo()
fureps = {'erg/cm2/s/A': 'erg/s/cm^2/Angstrom'}
task_str = catalog.get_current_task_str()
dirs = [x[0] for x in os.walk(os.path.join(fpath, 'Gutierrez_et_al_2017'))]
files = []
for dir in dirs:
files.extend(glob(os.path.join(dir, '*.fits')))
for datafile in pbar(files, task_str):
filename = datafile.split('/')[-1]
hdulist = fits.open(datafile)
for oi, obj in enumerate(hdulist[0].header):
if any(x in ['.', '/'] for x in obj):
del (hdulist[0].header[oi])
try:
hdulist[0].verify('silentfix')
except Exception as e:
print(e)
hdrkeys = list(hdulist[0].header.keys())
# print(hdrkeys)
name = datafile.split('/')[-2]
if name[2] in '6789':
name = 'SN19' + name[2:]
elif name != 'SN210':
name = 'SN20' + name[2:]
name, source = catalog.new_entry(name, bibcode='2017ApJ...850...89G')
# for key in hdulist[0].header.keys():
# print(key, hdulist[0].header[key])
mjd = None
if hdulist[0].header['SIMPLE']:
if 'JD' in hdrkeys:
mjd = str(jd_to_mjd(Decimal(str(hdulist[0].header['JD']))))
elif 'MJD' in hdrkeys:
mjd = str(hdulist[0].header['MJD'])
elif 'DATE-OBS' in hdrkeys or 'DATE' in hdrkeys:
dkey = 'DATE-OBS' if 'DATE-OBS' in hdrkeys else 'DATE'
dval = hdulist[0].header[dkey]
if is_number(dval):
dkey = 'DATE' if dkey == 'DATE-OBS' else 'DATE-OBS'
dval = hdulist[0].header[dkey]
dateobs = None
if 'T' in dval:
dateobs = dval.strip()
elif 'UTC-OBS' in hdrkeys:
dateobs = dval.strip(
) + 'T' + hdulist[0].header['UTC-OBS'].strip()
if dateobs is not None:
mjd = str(astrotime(dateobs, format='isot').mjd)
# print(hdulist[0].header)
if 'CRVAL1' in hdulist[0].header:
w0 = hdulist[0].header['CRVAL1']
elif hdulist[0].header['CTYPE1'] == 'MULTISPE':
w0 = float(
hdulist[0].header['WAT2_001'].split('"')[-1].split()[3])
else:
raise ValueError('Unsupported spectrum format.')
if hdulist[0].header['NAXIS'] == 1:
wd = hdulist[0].header['CDELT1']
fluxes = [str(x) for x in list(hdulist[0].data)]
errors = False
elif hdulist[0].header['NAXIS'] == 3:
wd = hdulist[0].header['CD1_1']
fluxes = [str(x) for x in list(hdulist[0].data)[0][0]]
errors = [str(x) for x in list(hdulist[0].data)[-1][0]]
else:
print('Warning: Skipping FITS spectrum `{}`.'.format(filename))
continue
waves = [str(w0 + wd * x) for x in range(0, len(fluxes))]
else:
raise ValueError('Non-simple FITS import not yet supported.')
if 'BUNIT' in hdrkeys:
fluxunit = hdulist[0].header['BUNIT']
if fluxunit in fureps:
fluxunit = fureps[fluxunit]
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.WAVELENGTHS: waves,
SPECTRUM.FLUXES: fluxes,
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.FILENAME: filename,
SPECTRUM.SOURCE: source
}
if mjd is not None:
specdict[SPECTRUM.TIME] = mjd
specdict[SPECTRUM.U_TIME] = 'MJD'
if 'TELESCOP' in hdrkeys:
specdict[SPECTRUM.TELESCOPE] = hdulist[0].header['TELESCOP']
if 'INSTRUME' in hdrkeys:
specdict[SPECTRUM.INSTRUMENT] = hdulist[0].header['INSTRUME']
if 'AIRMASS' in hdrkeys:
specdict[SPECTRUM.AIRMASS] = hdulist[0].header['AIRMASS']
if errors:
specdict[SPECTRUM.ERRORS] = errors
specdict[SPECTRUM.U_ERRORS] = fluxunit
if 'SITENAME' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['SITENAME']
elif 'OBSERVAT' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['OBSERVAT']
if 'OBSERVER' in hdrkeys:
specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER']
catalog.entries[name].add_spectrum(**specdict)
hdulist.close()
catalog.journal_entries()
return
def do_wiserep_spectra(catalog):
task_str = catalog.get_current_task_str()
secondaryreference = 'WISeREP'
secondaryrefurl = 'http://wiserep.weizmann.ac.il/'
secondarybibcode = '2012PASP..124..668Y'
wiserepcnt = 0
# These are known to be in error on the WISeREP page, either fix or ignore
# them.
wiserepbibcorrectdict = {'2000AJ....120..367G]': '2000AJ....120..367G',
'Harutyunyan et al. 2008': '2008A&A...488..383H',
'0609268': '2007AJ....133...58K',
'2006ApJ...636...400Q': '2006ApJ...636..400Q',
'2011ApJ...741...76': '2011ApJ...741...76C',
'2016PASP...128...961': '2016PASP..128...961',
'2002AJ....1124..417H': '2002AJ....1124.417H',
'2013ApJ…774…58D': '2013ApJ...774...58D',
'2011Sci.333..856S': '2011Sci...333..856S',
'2014MNRAS.438,368': '2014MNRAS.438..368T',
'2012MNRAS.420.1135': '2012MNRAS.420.1135S',
'2012Sci..337..942D': '2012Sci...337..942D',
'stt1839': '2013MNRAS.436.3614S',
'arXiv:1605.03136': '2016arXiv160503136T',
'10.1093/mnras/stt1839': '2013MNRAS.436.3614S'}
file_names = list(
glob(os.path.join(
catalog.get_current_task_repo(), '*')))
for folder in pbar_strings(file_names, task_str):
if '.txt' in folder:
continue
name = os.path.basename(folder).strip()
if name.startswith('sn'):
name = 'SN' + name[2:]
if (name.startswith(('CSS', 'SSS', 'MLS')) and
':' not in name):
name = name.replace('-', ':', 1)
if name.startswith('MASTERJ'):
name = name.replace('MASTERJ', 'MASTER OT J')
if name.startswith('PSNJ'):
name = name.replace('PSNJ', 'PSN J')
name = catalog.add_entry(name)
secondarysource = catalog.entries[name].add_source(
name=secondaryreference,
url=secondaryrefurl,
bibcode=secondarybibcode, secondary=True)
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, name, secondarysource)
with open(os.path.join(folder, 'README.json'), 'r') as f:
fileinfo = json.loads(f.read())
files = list(set(glob(folder + '/*')) -
set(glob(folder + '/README.json')))
for fname in pbar(files, task_str):
specfile = os.path.basename(fname)
claimedtype = fileinfo[specfile]["Type"]
instrument = fileinfo[specfile]["Instrument"]
epoch = fileinfo[specfile]["Obs. Date"]
observer = fileinfo[specfile]["Observer"]
reducer = fileinfo[specfile]["Reducer"]
bibcode = fileinfo[specfile]["Bibcode"]
redshift = fileinfo[specfile]["Redshift"]
survey = fileinfo[specfile]["Program"]
reduction = fileinfo[specfile]["Reduction Status"]
if bibcode:
newbibcode = bibcode
if bibcode in wiserepbibcorrectdict:
newbibcode = wiserepbibcorrectdict[bibcode]
if newbibcode and len(newbibcode) == 19:
source = catalog.entries[name].add_source(
bibcode=unescape(newbibcode))
else:
bibname = unescape(bibcode)
source = catalog.entries[name].add_source(
name=bibname)
catalog.log.warning('Bibcode "{}" is invalid, using as '
'`{}` instead'.format(bibname,
SOURCE.NAME))
sources = uniq_cdl([source, secondarysource])
else:
sources = secondarysource
if claimedtype not in ['Other']:
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, claimedtype,
secondarysource)
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, redshift, secondarysource)
with open(fname, 'r') as f:
data = [x.split() for x in f]
skipspec = False
newdata = []
oldval = ''
for row in data:
if row and '#' not in row[0]:
if (len(row) >= 2 and
is_number(row[0]) and
is_number(row[1]) and
row[1] != oldval):
newdata.append(row)
oldval = row[1]
if skipspec or not newdata:
warnings.warn(
'Skipped adding spectrum file ' +
specfile)
continue
data = [list(i) for i in zip(*newdata)]
wavelengths = data[0]
fluxes = data[1]
errors = ''
if len(data) == 3:
errors = data[1]
time = str(astrotime(epoch).mjd)
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom',
errors=errors,
u_fluxes=fluxunit,
u_errors=fluxunit if errors else '',
wavelengths=wavelengths,
fluxes=fluxes,
u_time='MJD', time=time,
instrument=instrument, source=sources,
observer=observer, reducer=reducer, reduction=reduction,
filename=specfile, survey=survey, redshift=redshift)
catalog.journal_entries()
wiserepcnt = wiserepcnt + 1
if (catalog.args.travis and
wiserepcnt %
catalog.TRAVIS_QUERY_LIMIT == 0):
break
return
def do_snax(catalog):
"""Import from the SNaX X-ray database."""
task_str = catalog.get_current_task_str()
dlurl = 'http://kronos.uchicago.edu/snax/export.php?exportType=TSV&exportFields=standard&objid=&name=&typeid=&type=&galaxyid=&galaxy=&fluxMin=&fluxMax=&fluxEnergyLMin=&fluxEnergyLMax=&fluxEnergyHMin=&fluxEnergyHMax=&lumMin=&lumMax=&instrumentid=&instrument=&ageMin=&ageMax=&dateMin=&dateMax=&sortA=dateExploded' # noqa: E501
file_path = os.path.join(catalog.get_current_task_repo(), 'SNaX.TSV')
tsv = catalog.load_url(dlurl, file_path)
# csvtxt = catalog.load_url(
# 'http://www.grbcatalog.org/'
# 'download_data?cut_0_min=5&cut_0=BAT%20T90'
# '&cut_0_max=100000&num_cuts=1&no_date_cut=True',
# file_path)
data = [x.split('\t') for x in tsv.split('\n')]
for r, row in enumerate(pbar(data, task_str)):
if r == 0 or not row[0]:
continue
(name,
source) = catalog.new_entry(row[0],
srcname='SNaX',
url='http://kronos.uchicago.edu/snax/',
secondary=True)
sources = [source]
bibcode = row[-6].strip()
if len(bibcode) != 19:
continue
expsrc = uniq_cdl(
sources +
[catalog.entries[name].add_source(bibcode=row[-6].strip())])
coosrc = uniq_cdl(
sources +
[catalog.entries[name].add_source(bibcode=row[-5].strip())])
dissrc = uniq_cdl(
sources +
[catalog.entries[name].add_source(bibcode=row[-4].strip())])
flxsrc = uniq_cdl(sources + [
catalog.entries[name].add_source(
bibcode=row[-3].strip()), catalog.entries[name].add_source(
bibcode=row[-2].strip())
])
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, row[1],
source)
date = astrotime(float(row[2]), format='jd').datetime
catalog.entries[name].add_quantity(
SUPERNOVA.EXPLOSION_DATE,
make_date_string(date.year, date.month, date.day), expsrc)
catalog.entries[name].add_quantity(SUPERNOVA.RA,
' '.join(row[3].split()[:3]),
coosrc)
catalog.entries[name].add_quantity(SUPERNOVA.DEC,
' '.join(row[3].split()[3:]),
coosrc)
catalog.entries[name].add_quantity(SUPERNOVA.LUM_DIST, row[4], dissrc)
catalog.entries[name].add_quantity(SUPERNOVA.HOST, row[5], source)
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT,
row[6],
source,
e_value=row[7] if (row[7] and float(row[7]) != 0.0) else '')
photodict = {
PHOTOMETRY.TIME: jd_to_mjd(Decimal(row[8])),
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.ENERGY: row[15:17],
PHOTOMETRY.U_ENERGY: 'keV',
PHOTOMETRY.FLUX: str(Decimal('1.0e-13') * Decimal(row[11])),
PHOTOMETRY.U_FLUX: 'ergs/s/cm^2',
PHOTOMETRY.E_LOWER_FLUX:
str(Decimal('1.0e-13') * Decimal(row[13])),
PHOTOMETRY.E_UPPER_FLUX:
str(Decimal('1.0e-13') * Decimal(row[14])),
PHOTOMETRY.INSTRUMENT: row[9],
PHOTOMETRY.SOURCE: flxsrc
}
if row[12] == '1':
photodict[PHOTOMETRY.UPPER_LIMIT] = True
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
return
def do_ucb_spectra(catalog):
task_str = catalog.get_current_task_str()
sec_reference = 'UCB Filippenko Group\'s Supernova Database (SNDB)'
sec_refurl = 'http://heracles.astro.berkeley.edu/sndb/info'
sec_refbib = '2012MNRAS.425.1789S'
ucbspectracnt = 0
jsontxt = catalog.load_url(
'http://heracles.astro.berkeley.edu/sndb/download?id=allpubspec',
os.path.join(catalog.get_current_task_repo(), 'UCB/allpubspec.json'),
json_sort='SpecID')
if not jsontxt:
return
spectra = json.loads(jsontxt)
spectra = sorted(spectra, key=lambda kk: kk['SpecID'])
oldname = ''
for spectrum in pbar(spectra, task_str):
name = spectrum['ObjName']
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
sec_source = catalog.entries[name].add_source(
name=sec_reference,
url=sec_refurl,
bibcode=sec_refbib,
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, sec_source)
sources = [sec_source]
if spectrum['Reference']:
sources += [catalog.entries[name]
.add_source(bibcode=spectrum['Reference'])]
sources = uniq_cdl(sources)
if spectrum['Type'] and spectrum['Type'].strip() != 'NoMatch':
for ct in spectrum['Type'].strip().split(','):
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, ct.replace('-norm', '').strip(),
sources)
if spectrum['DiscDate']:
ddate = spectrum['DiscDate'].replace('-', '/')
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, ddate,
sources)
if spectrum['HostName']:
host = urllib.parse.unquote(spectrum['HostName']).replace('*', '')
catalog.entries[name].add_quantity(SUPERNOVA.HOST, host, sources)
if spectrum['UT_Date']:
epoch = str(spectrum['UT_Date'])
year = epoch[:4]
month = epoch[4:6]
day = epoch[6:]
sig = get_sig_digits(day) + 5
mjd = astrotime(year + '-' + month + '-' + str(floor(float(
day))).zfill(2)).mjd
mjd = pretty_num(mjd + float(day) - floor(float(day)), sig=sig)
filename = spectrum['Filename'] if spectrum['Filename'] else ''
instrument = spectrum['Instrument'] if spectrum['Instrument'] else ''
reducer = spectrum['Reducer'] if spectrum['Reducer'] else ''
observer = spectrum['Observer'] if spectrum['Observer'] else ''
snr = str(spectrum['SNR']) if spectrum['SNR'] else ''
if not filename:
raise ValueError('Filename not found for SNDB spectrum!')
if not spectrum['SpecID']:
raise ValueError('ID not found for SNDB spectrum!')
filepath = os.path.join(catalog.get_current_task_repo(),
'UCB/') + filename
spectxt = catalog.load_url(
'http://heracles.astro.berkeley.edu/sndb/download?id=ds:' +
str(spectrum['SpecID']),
filepath,
archived_mode=True)
specdata = list(
csv.reader(
spectxt.splitlines(), delimiter=' ', skipinitialspace=True))
newspecdata = []
for row in specdata:
if row[0][0] == '#':
continue
else:
newspecdata.append(row)
specdata = newspecdata
haserrors = len(specdata[0]) == 3 and specdata[0][2] and specdata[0][
2] != 'NaN'
specdata = [list(ii) for ii in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
if not list(filter(None, errors)):
errors = ''
units = 'Uncalibrated'
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom',
u_fluxes=units,
u_time='MJD',
time=mjd,
wavelengths=wavelengths,
filename=filename,
fluxes=fluxes,
errors=errors,
u_errors=units,
instrument=instrument,
source=sources,
snr=snr,
observer=observer,
reducer=reducer,
deredshifted=('-noz' in filename))
ucbspectracnt = ucbspectracnt + 1
if catalog.args.travis and ucbspectracnt >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
return
def set_first_max_light(self):
if SUPERNOVA.MAX_APP_MAG not in self:
# Get the maximum amongst all bands
mldt, mlmag, mlband, mlsource = self._get_max_light()
if mldt or mlmag or mlband:
source = self.add_self_source()
uniq_src = uniq_cdl([source] + mlsource.split(','))
if mldt:
max_date = make_date_string(mldt.year, mldt.month, mldt.day)
self.add_quantity(SUPERNOVA.MAX_DATE,
max_date,
uniq_src,
derived=True)
if mlmag:
mlmag = pretty_num(mlmag)
self.add_quantity(SUPERNOVA.MAX_APP_MAG,
mlmag,
uniq_src,
derived=True)
if mlband:
self.add_quantity(SUPERNOVA.MAX_BAND,
mlband,
uniq_src,
derived=True)
if SUPERNOVA.MAX_VISUAL_APP_MAG not in self:
# Get the "visual" maximum
mldt, mlmag, mlband, mlsource = self._get_max_light(visual=True)
if mldt or mlmag or mlband:
source = self.add_self_source()
uniq_src = uniq_cdl([source] + mlsource.split(','))
if mldt:
max_date = make_date_string(mldt.year, mldt.month, mldt.day)
self.add_quantity(SUPERNOVA.MAX_VISUAL_DATE,
max_date,
uniq_src,
derived=True)
if mlmag:
mlmag = pretty_num(mlmag)
self.add_quantity(SUPERNOVA.MAX_VISUAL_APP_MAG,
mlmag,
uniq_src,
derived=True)
if mlband:
self.add_quantity(SUPERNOVA.MAX_VISUAL_BAND,
mlband,
uniq_src,
derived=True)
if (self._KEYS.DISCOVER_DATE not in self or max([
len(x[QUANTITY.VALUE].split('/'))
for x in self[self._KEYS.DISCOVER_DATE]
]) < 3):
fldt, flsource = self._get_first_light()
if fldt:
source = self.add_self_source()
disc_date = make_date_string(fldt.year, fldt.month, fldt.day)
self.add_quantity(self._KEYS.DISCOVER_DATE,
disc_date,
uniq_cdl([source] + flsource.split(',')),
derived=True)
if self._KEYS.DISCOVER_DATE not in self and self._KEYS.SPECTRA in self:
minspecmjd = float("+inf")
for spectrum in self[self._KEYS.SPECTRA]:
if 'time' in spectrum and 'u_time' in spectrum:
if spectrum['u_time'] == 'MJD':
mjd = float(spectrum['time'])
elif spectrum['u_time'] == 'JD':
mjd = float(jd_to_mjd(Decimal(spectrum['time'])))
else:
continue
if mjd < minspecmjd:
minspecmjd = mjd
minspecsource = spectrum['source']
if minspecmjd < float("+inf"):
fldt = astrotime(minspecmjd, format='mjd').datetime
source = self.add_self_source()
disc_date = make_date_string(fldt.year, fldt.month, fldt.day)
self.add_quantity(self._KEYS.DISCOVER_DATE,
disc_date,
uniq_cdl([source] +
minspecsource.split(',')),
derived=True)
return
def callback():
nds = []
new = namefield.value
newnames = []
for name in [x.strip() for x in new.split(',')]:
finalname = ''
if name:
newname = name.upper()
choices = []
for ui, un in enumerate(unames):
if un == newname:
finalname = names[ui]
break
if newname in un:
choices.append(names[ui])
if not finalname:
finalname = process.extractOne(name, choices)[0]
if not finalname:
finalname = 'SN1987A'
newnames.append(finalname)
namefield.value = ', '.join(newnames)
new = bandfield.value
newbands = []
for band in [x.strip() for x in new.split(',')]:
finalband = ''
if band:
newband = band.upper()
choices = []
for ui, un in enumerate(bands):
if un == newband:
finalband = bands[ui]
break
if newband.upper() in un.upper():
choices.append(bands[ui])
if not finalband:
extract = process.extractOne(band, choices)
if extract:
finalband = extract[0]
else:
finalband = band
if not finalband:
finalband = 'V'
newbands.append(finalband)
bandfield.value = ', '.join(newbands)
for name in newnames:
with open('/root/astrocats/astrocats/supernovae/output/json/' +
name.replace('/', '_') + '.json') as f:
event = json.loads(f.read())
event = event[list(event.keys())[0]]
for band in newbands:
print(band)
new_data = {}
new_data['x'] = []
new_data['y'] = []
new_data['band'] = band
new_data['maxmjd'] = astrotime(event['maxdate'][0]['value'].replace('/', '-')).mjd if 'maxdate' in event else ''
new_data['stretch'] = (1.0 / (
1.0 + float(event['redshift'][0]['value']))) if (
'redshift' in event) else 1.0
new_data['distancemod'] = ((float(event['maxappmag'][0]['value']) -
float(event['maxabsmag'][0]['value']))) if ('maxabsmag' in event and 'maxappmag' in event) else 0.0
for photo in event['photometry']:
if photo.get('band', '') != band:
continue
if 'time' not in photo or 'magnitude' not in photo:
continue
new_data['x'].append(float(photo['time']))
new_data['y'].append(float(photo['magnitude']))
nds.append(new_data)
maxmjds = []
if len(newnames) > 1:
for ndi, nd in enumerate(nds):
maxmjds.append(nd['maxmjd'])
if maxmjds[ndi]:
continue
if not len(nd['x']):
continue
maxmjds[ndi] = nd['x'][nd['y'].index(min(nd['y']))]
for ndi, nd in enumerate(nds):
if not len(nd['x']):
continue
nds[ndi]['x'] = [x - maxmjds[ndi] for x in nd['x']]
for ndi, nd in enumerate(nds):
if not len(nd['x']) or not len(nd['y']):
continue
nds[ndi]['y'] = [x - nd['distancemod'] for x in nd['y']]
nds[ndi]['x'] = [x*nd['stretch'] for x in nd['x']]
nxs = []
nys = []
for nd in nds:
nxs.extend(nd['x'])
nys.extend(nd['y'])
if len(nxs) and len(nys):
x_buf = 0.1*(max(nxs) - min(nxs))
y_buf = 0.1*(max(nys) - min(nys))
p.x_range.start = min(nxs) - x_buf
p.x_range.end = max(nxs) + x_buf
p.y_range.start = max(nys) + y_buf
p.y_range.end = min(nys) - y_buf
mld = {}
mldf = {}
for nd in nds:
mld.setdefault('xs',[]).append(nd['x'] if len(nd['x']) else [0.0])
mld.setdefault('ys',[]).append(nd['y'] if len(nd['y']) else [0.0])
mld.setdefault('colors',[]).append(plotting.bandcolorf(nd['band']) if len(nd['x']) else 'white')
mld.setdefault('lws',[]).append(2)
mldf.setdefault('xs',[]).extend(nd['x'] if len(nd['x']) else [])
mldf.setdefault('ys',[]).extend(nd['y'] if len(nd['y']) else [])
mldf.setdefault('colors',[]).extend([plotting.bandcolorf(nd['band']) for x in range(len(nd['x']))])
mlobs.data_source.data = mld
circobs.data_source.data = mldf
def do_cfa_spectra(catalog):
"""Import spectra from the CfA archive."""
task_str = catalog.get_current_task_str()
# II spectra
oldname = ''
file_names = next(
os.walk(os.path.join(catalog.get_current_task_repo(), 'CfA_SNII')))[1]
for ni, name in enumerate(pbar_strings(file_names, task_str)):
fullpath = os.path.join(catalog.get_current_task_repo(),
'CfA_SNII/') + name
origname = name
if name.startswith('sn') and is_number(name[2:6]):
name = 'SN' + name[2:]
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
reference = 'CfA Supernova Archive'
refurl = 'https://www.cfa.harvard.edu/supernova/SNarchive.html'
source = catalog.entries[name].add_source(
name=reference,
url=refurl,
secondary=True,
acknowledgment=ACKN_CFA)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
for fi, fname in enumerate(
sorted(
glob(fullpath + '/*'), key=lambda s: s.lower())):
filename = os.path.basename(fname)
fileparts = filename.split('-')
if origname.startswith('sn') and is_number(origname[2:6]):
year = fileparts[1][:4]
month = fileparts[1][4:6]
day = fileparts[1][6:]
instrument = fileparts[2].split('.')[0]
else:
year = fileparts[2][:4]
month = fileparts[2][4:6]
day = fileparts[2][6:]
instrument = fileparts[3].split('.')[0]
time = str(
astrotime(year + '-' + month + '-' + str(floor(float(day)))
.zfill(2)).mjd + float(day) - floor(float(day)))
f = open(fname, 'r')
data = csv.reader(f, delimiter=' ', skipinitialspace=True)
data = [list(i) for i in zip(*data)]
wavelengths = data[0]
fluxes = data[1]
errors = data[2]
sources = uniq_cdl([
source,
(catalog.entries[name]
.add_source(bibcode='2017arXiv170601030H'))
])
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom',
u_fluxes='erg/s/cm^2/Angstrom',
filename=filename,
wavelengths=wavelengths,
fluxes=fluxes,
u_time='MJD' if time else '',
time=time,
instrument=instrument,
u_errors='ergs/s/cm^2/Angstrom',
errors=errors,
source=sources,
dereddened=False,
deredshifted=False)
if catalog.args.travis and ni >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
# Ia spectra
oldname = ''
file_names = next(
os.walk(os.path.join(catalog.get_current_task_repo(), 'CfA_SNIa')))[1]
for ni, name in enumerate(pbar_strings(file_names, task_str)):
fullpath = os.path.join(catalog.get_current_task_repo(),
'CfA_SNIa/') + name
origname = name
if name.startswith('sn') and is_number(name[2:6]):
name = 'SN' + name[2:]
if name.startswith('snf') and is_number(name[3:7]):
name = 'SNF' + name[3:]
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
reference = 'CfA Supernova Archive'
refurl = 'https://www.cfa.harvard.edu/supernova/SNarchive.html'
source = catalog.entries[name].add_source(
name=reference,
url=refurl,
secondary=True,
acknowledgment=ACKN_CFA)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
for fi, fname in enumerate(
sorted(
glob(fullpath + '/*'), key=lambda s: s.lower())):
filename = os.path.basename(fname)
fileparts = filename.split('-')
if origname.startswith('sn') and is_number(origname[2:6]):
year = fileparts[1][:4]
month = fileparts[1][4:6]
day = fileparts[1][6:]
instrument = fileparts[2].split('.')[0]
else:
year = fileparts[2][:4]
month = fileparts[2][4:6]
day = fileparts[2][6:]
instrument = fileparts[3].split('.')[0]
time = str(
astrotime(year + '-' + month + '-' + str(floor(float(day)))
.zfill(2)).mjd + float(day) - floor(float(day)))
f = open(fname, 'r')
data = csv.reader(f, delimiter=' ', skipinitialspace=True)
data = [list(i) for i in zip(*data)]
wavelengths = data[0]
fluxes = data[1]
errors = data[2]
sources = uniq_cdl([
source, (catalog.entries[name]
.add_source(bibcode='2012AJ....143..126B')),
(catalog.entries[name]
.add_source(bibcode='2008AJ....135.1598M'))
])
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom',
u_fluxes='erg/s/cm^2/Angstrom',
filename=filename,
wavelengths=wavelengths,
fluxes=fluxes,
u_time='MJD' if time else '',
time=time,
instrument=instrument,
u_errors='ergs/s/cm^2/Angstrom',
errors=errors,
source=sources,
dereddened=False,
deredshifted=False)
if catalog.args.travis and ni >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
# Ibc spectra
oldname = ''
file_names = next(
os.walk(os.path.join(catalog.get_current_task_repo(), 'CfA_SNIbc')))[1]
for ni, name in enumerate(pbar(file_names, task_str)):
fullpath = os.path.join(catalog.get_current_task_repo(),
'CfA_SNIbc/') + name
if name.startswith('sn') and is_number(name[2:6]):
name = 'SN' + name[2:]
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
reference = 'CfA Supernova Archive'
refurl = 'https://www.cfa.harvard.edu/supernova/SNarchive.html'
source = catalog.entries[name].add_source(
name=reference,
url=refurl,
secondary=True,
acknowledgment=ACKN_CFA)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
for fi, fname in enumerate(
sorted(
glob(fullpath + '/*'), key=lambda s: s.lower())):
filename = os.path.basename(fname)
fileparts = filename.split('-')
instrument = ''
year = fileparts[1][:4]
month = fileparts[1][4:6]
day = fileparts[1][6:].split('.')[0]
if len(fileparts) > 2:
instrument = fileparts[-1].split('.')[0]
time = str(
astrotime(year + '-' + month + '-' + str(floor(float(day)))
.zfill(2)).mjd + float(day) - floor(float(day)))
f = open(fname, 'r')
data = csv.reader(f, delimiter=' ', skipinitialspace=True)
data = [list(i) for i in zip(*data)]
wavelengths = data[0]
fluxes = data[1]
sources = uniq_cdl([
source, catalog.entries[name]
.add_source(bibcode='2014AJ....147...99M')
])
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom',
u_fluxes='erg/s/cm^2/Angstrom',
wavelengths=wavelengths,
filename=filename,
fluxes=fluxes,
u_time='MJD' if time else '',
time=time,
instrument=instrument,
source=sources,
dereddened=False,
deredshifted=False)
if catalog.args.travis and ni >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
# Other spectra
oldname = ''
file_names = next(
os.walk(os.path.join(catalog.get_current_task_repo(), 'CfA_Extra')))[1]
for ni, name in enumerate(pbar_strings(file_names, task_str)):
fullpath = os.path.join(catalog.get_current_task_repo(),
'CfA_Extra/') + name
if name.startswith('sn') and is_number(name[2:6]):
name = 'SN' + name[2:]
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
reference = 'CfA Supernova Archive'
refurl = 'https://www.cfa.harvard.edu/supernova/SNarchive.html'
source = catalog.entries[name].add_source(
name=reference,
url=refurl,
secondary=True,
acknowledgment=ACKN_CFA)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
for fi, fname in enumerate(
sorted(
glob(fullpath + '/*'), key=lambda s: s.lower())):
if not os.path.isfile(fname):
continue
filename = os.path.basename(fname)
if ((not filename.startswith('sn') or
not filename.endswith('flm') or any(
x in filename
for x in ['-interp', '-z', '-dered', '-obj', '-gal']))):
continue
fileparts = filename.split('.')[0].split('-')
instrument = ''
time = ''
if len(fileparts) > 1:
year = fileparts[1][:4]
month = fileparts[1][4:6]
day = fileparts[1][6:]
if is_number(year) and is_number(month) and is_number(day):
if len(fileparts) > 2:
instrument = fileparts[-1]
time = str(
astrotime(year + '-' + month + '-' + str(
floor(float(day))).zfill(2)).mjd + float(day) -
floor(float(day)))
f = open(fname, 'r')
data = csv.reader(f, delimiter=' ', skipinitialspace=True)
data = [list(i) for i in zip(*data)]
wavelengths = data[0]
fluxes = [str(Decimal(x) * Decimal(1.0e-15)) for x in data[1]]
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom',
u_fluxes='erg/s/cm^2/Angstrom',
wavelengths=wavelengths,
filename=filename,
fluxes=fluxes,
u_time='MJD' if time else '',
time=time,
instrument=instrument,
source=source,
dereddened=False,
deredshifted=False)
if catalog.args.travis and ni >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
return
def do_lamost(catalog):
"""Import spectra from LAMOST."""
task_str = catalog.get_current_task_str()
# Set preferred names, calculate some columns based on imported data,
# sanitize some fields
keys = list(catalog.entries.keys())
viz = Vizier(columns=["**"])
fureps = {'erg/cm2/s/A': 'erg/s/cm^2/Angstrom'}
c_kms = con.c.cgs.value / 1.0e5
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
if (FASTSTARS.RA not in catalog.entries[oname]
or FASTSTARS.DEC not in catalog.entries[oname]):
continue
else:
result = viz.query_region(coord.SkyCoord(
ra=catalog.entries[oname][FASTSTARS.RA][0]['value'],
dec=catalog.entries[oname][FASTSTARS.DEC][0]['value'],
unit=(un.hourangle, un.deg),
frame='icrs'),
width="2s",
catalog="V/149/dr2")
if not result.keys():
continue
tab = result['V/149/dr2']
star = None
for row in tab:
if (row['objType'] == 'Star'
and row['Class'].lower() in ['star', 'unknown']):
star = row
break
if not star:
continue
try:
name, source = catalog.new_entry(oname,
bibcode='2016yCat.5149....0L',
srcname='LAMOST',
url='http://dr3.lamost.org/')
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
if row['SubClass'] is not 'Non':
#catalog.entries[name].add_quantity(
# FASTSTARS.SPECTRAL_TYPE, row['SubClass'], source=source)
ST, SCfull = row['SubClass'][:2], row['SubClass'][2:]
if len(SCfull) > 0:
if 'IV' in SCfull:
SC = 'sg'
elif 'III' in SCfull:
SC = 'g'
elif 'V' in SCfull:
SC = 'd'
elif 'I' in SCfull:
SC = 'Sg'
catalog.entries[name].add_quantity(FASTSTARS.STELLAR_CLASS,
SC,
source=source)
catalog.entries[name].add_quantity(FASTSTARS.SPECTRAL_TYPE,
ST,
source=source)
if row['z'] and is_number(row['z']):
catalog.entries[name].add_quantity(FASTSTARS.REDSHIFT,
str(row['z']),
e_value=str(row['e_z']),
source=source)
catalog.entries[name].add_quantity(
FASTSTARS.VELOCITY,
pretty_num(float(row['z']) * c_kms, sig=5),
e_value=pretty_num(float(row['e_z'] * c_kms), sig=5),
source=source)
mag_types = list(row['magType'].replace('psf_', ''))
nmt = []
nmi = 0
for mt in mag_types:
if is_number(mt):
nmt[nmi - 1] += mt
else:
nmt += mt
nmi += 1
mag_types = [
x.upper() if x in ['b', 'v', 'j', 'h'] else x for x in nmt
]
for mi, mt in enumerate(mag_types):
snrf = 'snr' + mt.lower()
if snrf in row.columns and float(row[snrf]) < 3:
continue
photodict = {
PHOTOMETRY.TIME: str(row['MJD']),
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.BAND: mt,
PHOTOMETRY.TELESCOPE: 'LAMOST',
PHOTOMETRY.MAGNITUDE: str(row['mag' + str(mi + 1)]),
PHOTOMETRY.SOURCE: source
}
if snrf in row.columns:
photodict[PHOTOMETRY.E_MAGNITUDE] = str(
Decimal('2.5') *
(Decimal('1') +
Decimal('1') / Decimal(str(row[snrf]))).log10())[:5]
catalog.entries[name].add_photometry(**photodict)
vname = row['PlanId']
ffile = ('spec-' + row['LMJD'] + '-' + vname + '_sp' +
row['spId'] + '-' + row['FiberId'] + '.fits.gz')
furl = 'http://dr3.lamost.org/sas/fits/' + vname + '/' + ffile
datafile = os.path.join(catalog.get_current_task_repo(), 'LAMOST',
ffile)
if not os.path.exists(datafile):
fr = requests.get(furl)
open(datafile, 'wb').write(fr.content)
hdulist = fits.open(datafile)
for oi, obj in enumerate(hdulist[0].header):
if any(x in ['.', '/'] for x in obj):
del (hdulist[0].header[oi])
hdulist[0].verify('silentfix')
hdrkeys = list(hdulist[0].header.keys())
# print(hdrkeys)
# for key in hdulist[0].header.keys():
# print(key, hdulist[0].header[key])
if hdulist[0].header['SIMPLE']:
if 'JD' in hdrkeys:
mjd = str(jd_to_mjd(Decimal(str(hdulist[0].header['JD']))))
elif 'MJD' in hdrkeys:
mjd = str(hdulist[0].header['MJD'])
elif 'DATE-OBS' in hdrkeys:
if 'T' in hdulist[0].header['DATE-OBS']:
dateobs = hdulist[0].header['DATE-OBS'].strip()
elif 'UTC-OBS' in hdrkeys:
dateobs = hdulist[0].header['DATE-OBS'].strip(
) + 'T' + hdulist[0].header['UTC-OBS'].strip()
mjd = str(astrotime(dateobs, format='isot').mjd)
else:
raise ValueError("Couldn't find JD/MJD for spectrum.")
if hdulist[0].header['NAXIS'] == 2:
waves = [str(x) for x in list(hdulist[0].data)[2]]
fluxes = [str(x) for x in list(hdulist[0].data)[0]]
else:
print('Warning: Skipping FITS spectrum `{}`.'.format(
datafile))
continue
else:
raise ValueError('Non-simple FITS import not yet supported.')
if 'BUNIT' in hdrkeys:
fluxunit = hdulist[0].header['BUNIT']
if fluxunit in fureps:
fluxunit = fureps[fluxunit]
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.WAVELENGTHS: waves,
SPECTRUM.TIME: mjd,
SPECTRUM.U_TIME: 'MJD',
SPECTRUM.FLUXES: fluxes,
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.FILENAME: ffile,
SPECTRUM.SOURCE: source
}
if 'TELESCOP' in hdrkeys:
specdict[SPECTRUM.TELESCOPE] = hdulist[0].header['TELESCOP']
if 'INSTRUME' in hdrkeys:
specdict[SPECTRUM.INSTRUMENT] = hdulist[0].header['INSTRUME']
if 'SITENAME' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['SITENAME']
elif 'OBSERVAT' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['OBSERVAT']
if 'OBSERVER' in hdrkeys:
specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER']
if 'AIRMASS' in hdrkeys:
specdict[SPECTRUM.AIRMASS] = hdulist[0].header['AIRMASS']
catalog.entries[name].add_spectrum(**specdict)
hdulist.close()
catalog.journal_entries()
return
def set_data(self,
all_data,
req_key_values={},
subtract_minimum_keys=[],
smooth_times=-1,
extrapolate_time=0.0,
limit_fitting_mjds=False,
exclude_bands=[],
exclude_instruments=[],
exclude_systems=[],
exclude_sources=[],
exclude_kinds=[],
time_unit=None,
time_list=[],
band_list=[],
band_telescopes=[],
band_systems=[],
band_instruments=[],
band_modes=[],
band_bandsets=[]):
"""Set transient data."""
prt = self._printer
self._all_data = all_data
self._data = OrderedDict()
if not self._all_data:
return
name = list(self._all_data.keys())[0]
self._data['name'] = name
numeric_keys = set()
ex_kinds = [
self._EX_REPS.get(x.lower(), x.lower()) for x in exclude_kinds
]
# Construct some source dictionaries for exclusion rules
src_dict = OrderedDict()
sources = self._all_data[name].get('sources', [])
for src in sources:
if SOURCE.BIBCODE in src:
src_dict[src[SOURCE.ALIAS]] = src[SOURCE.BIBCODE]
if SOURCE.ARXIVID in src:
src_dict[src[SOURCE.ALIAS]] = src[SOURCE.ARXIVID]
if SOURCE.NAME in src:
src_dict[src[SOURCE.ALIAS]] = src[SOURCE.NAME]
for key in self._keys:
subkeys = self._keys[key]
req_subkeys = [
x for x in subkeys if not isinstance(subkeys, dict)
or 'required' in listify(subkeys[x])
]
num_subkeys = [
x for x in subkeys if 'numeric' in listify(subkeys[x])
]
boo_subkeys = [
x for x in subkeys if 'boolean' in listify(subkeys[x])
]
exc_subkeys = [
x for x in subkeys if 'exclude' in listify(subkeys[x])
]
if (key not in self._all_data[name]
and not self._model.is_parameter_fixed_by_user(key)):
if subkeys.get('value', None) == 'recommended':
self._unset_recommended_keys.add(key)
continue
subdata = self._all_data[name].get(key)
if subdata is None:
continue
# Only include data that contains all subkeys
for entry in subdata:
if any([x not in entry for x in req_subkeys]):
continue
if any([x in entry for x in exc_subkeys]):
continue
if any([
x in entry and
((isinstance(entry[x], list) and any([
not is_number(y) or np.isnan(float(y))
for y in entry[x]
])) or not isinstance(entry[x], list) and
(not is_number(entry[x]) or np.isnan(float(entry[x]))))
for x in num_subkeys
]):
continue
skip_key = False
if 'frequency' not in entry:
for qkey in req_key_values:
if qkey in entry and entry[qkey] != '':
if entry[qkey] not in req_key_values[qkey]:
skip_key = True
break
if key == 'photometry':
if ('fluxdensity' in entry and 'magnitude' not in entry
and 'countrate' not in entry):
self._kinds_needed.add('radio')
if ('radio' in ex_kinds or
(not len(ex_kinds) or 'none' not in ex_kinds) and
'radio' not in self._model._kinds_supported):
continue
if (('countrate' in entry or 'unabsorbedflux' in entry
or 'flux' in entry) and 'magnitude' not in entry
and 'fluxdensity' not in entry):
self._kinds_needed.add('x-ray')
if ('x-ray' in ex_kinds or
(not len(ex_kinds) or 'none' not in ex_kinds) and
'x-ray' not in self._model._kinds_supported):
continue
if 'magnitude' in entry:
# For now, magnitudes are not excludable.
self._kinds_needed |= set(
['infrared', 'optical', 'ultraviolet'])
skip_entry = False
for x in subkeys:
if limit_fitting_mjds is not False and x == 'time':
val = np.mean([
float(x) for x in listify(entry.get(x, None))
])
if (val < limit_fitting_mjds[0]
or val > limit_fitting_mjds[1]):
skip_entry = True
break
if exclude_bands is not False and x == 'band':
if (entry.get(x, '') in exclude_bands and
(not exclude_instruments or entry.get(
'instrument', '') in exclude_instruments)
and (not exclude_systems or entry.get(
'system', '') in exclude_systems)):
skip_entry = True
break
if (exclude_instruments is not False
and x == 'instrument'):
if (entry.get(x, '') in exclude_instruments and
(not exclude_bands
or entry.get('band', '') in exclude_bands)
and (not exclude_systems or entry.get(
'system', '') in exclude_systems)):
skip_entry = True
break
if (exclude_systems is not False and x == 'system'):
if (entry.get(x, '') in exclude_systems and
(not exclude_bands
or entry.get('band', '') in exclude_bands) and
(not exclude_instruments or entry.get(
'instrument', '') in exclude_instruments)):
skip_entry = True
break
if (exclude_sources is not False and x == 'source'):
val = entry.get(x, '')
if (any(
[x in exclude_sources for x in val.split(',')])
or any([
src_dict.get(x, '') in exclude_sources
for x in val.split(',')
])):
skip_entry = True
break
if skip_entry:
continue
if ((('magnitude' in entry) !=
('band' in entry)) or ((('fluxdensity' in entry) !=
('frequency' in entry)) and
('magnitude' not in entry))
or (('countrate' in entry) and
('magnitude' not in entry) and
('instrument' not in entry))):
continue
for x in subkeys:
falseval = (False if x in boo_subkeys else
None if x in num_subkeys else '')
if x == 'value':
if not skip_key:
self._data[key] = entry.get(x, falseval)
else:
plural = self._model.plural(x)
val = entry.get(x, falseval)
if x in num_subkeys:
val = None if val is None else np.mean(
[float(x) for x in listify(val)])
if not skip_key:
self._data.setdefault(plural, []).append(val)
if x in num_subkeys:
numeric_keys.add(plural)
else:
self._data.setdefault('unmatched_' + plural,
[]).append(val)
if 'times' not in self._data or not any([
x in self._data
for x in ['magnitudes', 'frequencies', 'countrates']
]):
prt.message('no_fittable_data', [name])
return False
for key in list(self._data.keys()):
if isinstance(self._data[key], list):
self._data[key] = np.array(self._data[key])
if key not in numeric_keys:
continue
num_values = [
x for x in self._data[key] if isinstance(x, float)
]
if len(num_values):
self._data['min_' + key] = min(num_values)
self._data['max_' + key] = max(num_values)
else:
if is_number(self._data[key]):
self._data[key] = float(self._data[key])
self._data_determined_parameters.append(key)
if any(x in self._data
for x in ['magnitudes', 'countrates', 'fluxdensities']):
# Add a list of tags for each observation to indicate what unit
# observation is provided in.
self._data['measures'] = [
((['magnitude'] if x else []) + (['countrate'] if y else []) +
(['fluxdensity'] if x else []))
for x, y, z in zip(*(self._data['magnitudes'],
self._data['countrates'],
self._data['fluxdensities']))
]
if 'times' in self._data and (smooth_times >= 0 or time_list):
# Build an observation array out of the real data first.
obs = list(
zip(*(self._data[x] for x in self._OBS_KEYS if x != 'times')))
# Append extra observations if requested.
if len(band_list):
b_teles = band_telescopes if len(band_telescopes) == len(
band_list) else (
[band_telescopes[0] for x in band_list]
if len(band_telescopes) else ['' for x in band_list])
b_systs = band_systems if len(band_systems) == len(
band_list) else ([band_systems[0]
for x in band_list] if len(band_systems)
else ['' for x in band_list])
b_modes = band_modes if len(band_modes) == len(
band_list) else ([band_modes[0]
for x in band_list] if len(band_modes)
else ['' for x in band_list])
b_insts = band_instruments if len(band_instruments) == len(
band_list) else (
[band_instruments[0] for x in band_list]
if len(band_instruments) else ['' for x in band_list])
b_bsets = band_bandsets if len(band_bandsets) == len(
band_list) else ([band_bandsets[0]
for x in band_list] if len(band_bandsets)
else ['' for x in band_list])
b_freqs = [None for x in band_list]
b_u_freqs = ['' for x in band_list]
b_zerops = [None for x in band_list]
b_measures = [[] for x in band_list]
obs.extend(
list(
zip(*(b_teles, b_systs, b_modes, b_insts, b_bsets,
band_list, b_freqs, b_u_freqs, b_zerops,
b_measures))))
# Prune extra observations if they are duplicitous to existing.
uniqueobs = []
for o in obs:
to = tuple(o)
if to not in uniqueobs:
uniqueobs.append(to)
# Preprend times to real observations list.
minet, maxet = (extrapolate_time, extrapolate_time) if isinstance(
extrapolate_time, (float, int)) else (
(tuple(extrapolate_time) if len(extrapolate_time) == 2 else
(extrapolate_time[0], extrapolate_time[0])))
mint, maxt = (min(self._data['times']) - minet,
max(self._data['times']) + maxet)
if time_unit is None:
alltimes = time_list + [x for x in self._data['times']]
elif time_unit == 'mjd':
alltimes = [x - min(self._data['times']) for x in time_list
] + [x for x in self._data['times']]
elif time_unit == 'phase':
if 'maxdate' not in self._data:
raise (prt.message('no_maxdate', name))
max_mjd = astrotime(self._data['maxdate'].replace('/',
'-')).mjd
alltimes = [x + max_mjd for x in time_list
] + [x for x in self._data['times']]
else:
raise ('Unknown `time_unit`.')
if smooth_times >= 0:
alltimes += list(np.linspace(mint, maxt, max(smooth_times, 2)))
alltimes = list(sorted(set(alltimes)))
# Create additional fake observations.
currobslist = list(zip(*(self._data[x] for x in self._OBS_KEYS)))
obslist = []
for ti, t in enumerate(alltimes):
new_per = np.round(100.0 * float(ti) / len(alltimes), 1)
prt.message('construct_obs_array', [new_per], inline=True)
for o in uniqueobs:
newobs = tuple([t] + list(o))
if newobs not in currobslist:
obslist.append(newobs)
obslist.sort()
# Save these fake observations under keys with `extra_` prefix.
if obslist:
for x, y in zip(self._OBS_KEYS, zip(*obslist)):
self._data['extra_' + x] = y
for qkey in subtract_minimum_keys:
if 'upperlimits' in self._data:
new_vals = np.array(self._data[qkey])[np.array(
self._data['upperlimits']) != True] # noqa E712
if new_vals.size:
self._data['min_' + qkey] = min(new_vals)
self._data['max_' + qkey] = max(new_vals)
minv = self._data['min_' + qkey]
self._data[qkey] = [x - minv for x in self._data[qkey]]
if 'extra_' + qkey in self._data:
self._data['extra_' + qkey] = [
x - minv for x in self._data['extra_' + qkey]
]
return True
def do_csp_fits_spectra(catalog):
from astropy.io import fits
fpath = catalog.get_current_task_repo()
fureps = {'erg/cm2/s/A': 'erg/s/cm^2/Angstrom'}
task_str = catalog.get_current_task_str()
dirs = [x[0] for x in os.walk(
os.path.join(fpath, 'Gutierrez_et_al_2017'))]
files = []
for dir in dirs:
files.extend(glob(os.path.join(dir, '*.fits')))
for datafile in pbar(files, task_str):
filename = datafile.split('/')[-1]
hdulist = fits.open(datafile)
for oi, obj in enumerate(hdulist[0].header):
if any(x in ['.', '/'] for x in obj):
del (hdulist[0].header[oi])
try:
hdulist[0].verify('silentfix')
except Exception as e:
print(e)
hdrkeys = list(hdulist[0].header.keys())
# print(hdrkeys)
name = datafile.split('/')[-2]
if name[2] in '6789':
name = 'SN19' + name[2:]
elif name != 'SN210':
name = 'SN20' + name[2:]
name, source = catalog.new_entry(name, bibcode='2017ApJ...850...89G')
# for key in hdulist[0].header.keys():
# print(key, hdulist[0].header[key])
mjd = None
if hdulist[0].header['SIMPLE']:
if 'JD' in hdrkeys:
mjd = str(jd_to_mjd(Decimal(str(hdulist[0].header['JD']))))
elif 'MJD' in hdrkeys:
mjd = str(hdulist[0].header['MJD'])
elif 'DATE-OBS' in hdrkeys or 'DATE' in hdrkeys:
dkey = 'DATE-OBS' if 'DATE-OBS' in hdrkeys else 'DATE'
dval = hdulist[0].header[dkey]
if is_number(dval):
dkey = 'DATE' if dkey == 'DATE-OBS' else 'DATE-OBS'
dval = hdulist[0].header[dkey]
dateobs = None
if 'T' in dval:
dateobs = dval.strip()
elif 'UTC-OBS' in hdrkeys:
dateobs = dval.strip(
) + 'T' + hdulist[0].header['UTC-OBS'].strip()
if dateobs is not None:
mjd = str(astrotime(dateobs, format='isot').mjd)
# print(hdulist[0].header)
if 'CRVAL1' in hdulist[0].header:
w0 = hdulist[0].header['CRVAL1']
elif hdulist[0].header['CTYPE1'] == 'MULTISPE':
w0 = float(hdulist[0].header['WAT2_001'].split(
'"')[-1].split()[3])
else:
raise ValueError('Unsupported spectrum format.')
if hdulist[0].header['NAXIS'] == 1:
wd = hdulist[0].header['CDELT1']
fluxes = [str(x) for x in list(hdulist[0].data)]
errors = False
elif hdulist[0].header['NAXIS'] == 3:
wd = hdulist[0].header['CD1_1']
fluxes = [str(x) for x in list(hdulist[0].data)[0][0]]
errors = [str(x) for x in list(hdulist[0].data)[-1][0]]
else:
print('Warning: Skipping FITS spectrum `{}`.'.format(filename))
continue
waves = [str(w0 + wd * x) for x in range(0, len(fluxes))]
else:
raise ValueError('Non-simple FITS import not yet supported.')
if 'BUNIT' in hdrkeys:
fluxunit = hdulist[0].header['BUNIT']
if fluxunit in fureps:
fluxunit = fureps[fluxunit]
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.WAVELENGTHS: waves,
SPECTRUM.FLUXES: fluxes,
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.FILENAME: filename,
SPECTRUM.SOURCE: source
}
if mjd is not None:
specdict[SPECTRUM.TIME] = mjd
specdict[SPECTRUM.U_TIME] = 'MJD'
if 'TELESCOP' in hdrkeys:
specdict[SPECTRUM.TELESCOPE] = hdulist[0].header['TELESCOP']
if 'INSTRUME' in hdrkeys:
specdict[SPECTRUM.INSTRUMENT] = hdulist[0].header['INSTRUME']
if 'AIRMASS' in hdrkeys:
specdict[SPECTRUM.AIRMASS] = hdulist[0].header['AIRMASS']
if errors:
specdict[SPECTRUM.ERRORS] = errors
specdict[SPECTRUM.U_ERRORS] = fluxunit
if 'SITENAME' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['SITENAME']
elif 'OBSERVAT' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['OBSERVAT']
if 'OBSERVER' in hdrkeys:
specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER']
catalog.entries[name].add_spectrum(**specdict)
hdulist.close()
catalog.journal_entries()
return
('types', []), ('photocount', 0),
('spectracount', 0),
('metacount', 0)])
biblio[bc]['events'].append(item['name'])
if 'discoverdate' in item and item['discoverdate']:
datestr = item['discoverdate'][
0]['value'].replace('/', '-')
if datestr.count('-') == 1:
datestr += '-01'
elif datestr.count('-') == 0:
datestr += '-01-01'
try:
biblio[bc]['eventdates'].append(
astrotime(datestr, format='isot').unix)
except:
biblio[bc]['eventdates'].append(float("inf"))
else:
biblio[bc]['eventdates'].append(float("inf"))
if 'claimedtype' in item:
cts = []
for ct in item['claimedtype']:
cts.append(ct['value'].strip('?'))
biblio[bc]['types'] = list(
set(biblio[bc]['types']).union(cts))
if 'photometry' in item:
bcalias = source['alias']
lc = 0
def do_snf_specta(catalog):
task_str = catalog.get_current_task_str()
bibcodes = {'SN2005gj': '2006ApJ...650..510A',
'SN2006D': '2007ApJ...654L..53T',
'SN2007if': '2010ApJ...713.1073S',
'SN2011fe': '2013A&A...554A..27P'}
oldname = ''
snfcnt = 0
eventfolders = next(os.walk(os.path.join(
catalog.get_current_task_repo(), 'SNFactory')))[1]
for eventfolder in pbar(eventfolders, task_str):
oname = eventfolder
name = catalog.get_preferred_name(oname)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
sec_reference = 'Nearby Supernova Factory'
sec_refurl = 'http://snfactory.lbl.gov/'
sec_bibcode = '2002SPIE.4836...61A'
sec_source = catalog.entries[name].add_source(
name=sec_reference, url=sec_refurl, bibcode=sec_bibcode,
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oname, sec_source)
bibcode = bibcodes[oname]
source = catalog.entries[name].add_source(bibcode=bibcode)
sources = uniq_cdl([source, sec_source])
use_path = os.path.join(
catalog.get_current_task_repo(), 'SNFactory', eventfolder, '*.dat')
eventspectra = glob(use_path)
for spectrum in pbar(eventspectra, task_str):
filename = os.path.basename(spectrum)
with open(spectrum) as spec_file:
specdata = list(csv.reader(
spec_file, delimiter=' ', skipinitialspace=True))
specdata = list(filter(None, specdata))
newspec = []
time = ''
telescope = ''
instrument = ''
observer = ''
observatory = ''
if 'Keck_20060202_R' in spectrum:
time = '53768.23469'
elif 'Spectrum05_276' in spectrum:
time = pretty_num(astrotime('2005-10-03').mjd, sig=5)
elif 'Spectrum05_329' in spectrum:
time = pretty_num(astrotime('2005-11-25').mjd, sig=5)
elif 'Spectrum05_336' in spectrum:
time = pretty_num(astrotime('2005-12-02').mjd, sig=5)
for row in specdata:
if row[0][0] == '#':
joinrow = (' '.join(row)).split('=')
if len(joinrow) < 2:
continue
field = joinrow[0].strip('# ')
value = joinrow[1].split('/')[0].strip('\' ')
if not time:
if field == 'JD':
time = str(jd_to_mjd(Decimal(value)))
elif field == 'MJD':
time = value
elif field == 'MJD-OBS':
time = value
if field == 'OBSERVER':
observer = value.capitalize()
if field == 'OBSERVAT':
observatory = value.capitalize()
if field == 'TELESCOP':
telescope = value.capitalize()
if field == 'INSTRUME':
instrument = value.capitalize()
else:
newspec.append(row)
if not time:
raise ValueError('Time missing from spectrum.')
specdata = newspec
haserrors = len(specdata[0]) == 3 and specdata[
0][2] and specdata[0][2] != 'NaN'
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
unit_err = ('Variance' if oldname == 'SN2011fe' else
'erg/s/cm^2/Angstrom')
unit_flx = 'erg/s/cm^2/Angstrom'
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom', u_fluxes=unit_flx, u_time='MJD',
time=time,
wavelengths=wavelengths, fluxes=fluxes, errors=errors,
observer=observer, observatory=observatory,
telescope=telescope, instrument=instrument, u_errors=unit_err,
source=sources, filename=filename)
snfcnt = snfcnt + 1
if (catalog.args.travis and
snfcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
def main():
"""Run MOSFiT."""
prt = Printer(wrap_length=100,
quiet=False,
language='en',
exit_on_prompt=False)
parser = get_parser(only='language')
args, remaining = parser.parse_known_args()
if args.language == 'en':
loc = locale.getlocale()
if loc[0]:
args.language = loc[0].split('_')[0]
if args.language != 'en':
try:
from googletrans.constants import LANGUAGES
except Exception:
raise RuntimeError('`--language` requires `googletrans` package, '
'install with `pip install googletrans`.')
if args.language == 'select' or args.language not in LANGUAGES:
languages = list(
sorted([
LANGUAGES[x].title().replace('_', ' ') + ' (' + x + ')'
for x in LANGUAGES
]))
sel = prt.prompt('Select a language:',
kind='select',
options=languages,
message=False)
args.language = sel.split('(')[-1].strip(')')
prt = Printer(language=args.language)
language = args.language
parser = get_parser(printer=prt)
args = parser.parse_args()
args.language = language
prt = Printer(wrap_length=100,
quiet=args.quiet,
language=args.language,
exit_on_prompt=args.exit_on_prompt)
if args.version:
print('MOSFiT v{}'.format(__version__))
return
dir_path = os.path.dirname(os.path.realpath(__file__))
if args.speak:
speak('Mosfit', args.speak)
args.start_time = time.time()
if args.limiting_magnitude == []:
args.limiting_magnitude = 20.0
args.return_fits = False
if (isinstance(args.extrapolate_time, list)
and len(args.extrapolate_time) == 0):
args.extrapolate_time = 100.0
if len(args.band_list) and args.smooth_times == -1:
prt.message('enabling_s')
args.smooth_times = 0
args.method = 'nester' if args.method.lower() in [
'nest', 'nested', 'nested_sampler', 'nester'
] else 'ensembler'
if is_master():
if args.method == 'nester':
unused_args = [[args.burn, '-b'], [args.post_burn, '-p'],
[args.frack_step, '-f'], [args.num_temps, '-T'],
[args.run_until_uncorrelated, '-U'],
[args.draw_above_likelihood, '-d'],
[args.gibbs, '-g'], [args.save_full_chain, '-c'],
[args.maximum_memory, '-M']]
for ua in unused_args:
if ua[0] is not None:
prt.message('argument_not_used',
reps=[ua[1], '-D nester'],
warning=True)
if args.method == 'nester':
if args.run_until_converged and args.iterations >= 0:
raise ValueError(prt.text('R_i_mutually_exclusive'))
if args.walker_paths is not None:
raise ValueError(prt.text('w_nester_mutually_exclusive'))
if args.generative:
if args.iterations > 0:
prt.message('generative_supercedes', warning=True)
args.iterations = 0
no_events = False
if args.iterations == -1:
if len(args.events) == 0:
no_events = True
args.iterations = 0
else:
args.iterations = 5000
if len(args.date_list):
if no_events:
prt.message('no_dates_gen', warning=True)
else:
args.time_list = [
str(astrotime(x.replace('/', '-')).mjd) for x in args.date_list
]
args.time_unit = 'mjd'
if len(args.mjd_list):
if no_events:
prt.message('no_dates_gen', warning=True)
else:
args.time_list = args.mjd_list
args.time_unit = 'mjd'
if len(args.jd_list):
if no_events:
prt.message('no_dates_gen', warning=True)
else:
args.time_list = [
str(astrotime(float(x), format='jd').mjd) for x in args.jd_list
]
args.time_unit = 'mjd'
if len(args.phase_list):
if no_events:
prt.message('no_dates_gen', warning=True)
else:
args.time_list = args.phase_list
args.time_unit = 'phase'
if len(args.time_list):
if any([any([y in x]) for y in ['-', '/'] for x in args.time_list]):
try:
args.time_list = [
astrotime(x.replace('/', '-')).mjd for x in args.time_list
]
except ValueError:
if len(args.time_list) == 1 and isinstance(
args.time_list[0], string_types):
args.time_list = args.time_list[0].split()
args.time_list = [float(x) for x in args.time_list]
args.time_unit = 'phase'
else:
if any(['+' in x for x in args.time_list]):
args.time_unit = 'phase'
args.time_list = [float(x) for x in args.time_list]
if min(args.time_list) > 2400000:
prt.message('assuming_jd')
args.time_list = [x - 2400000.5 for x in args.time_list]
args.time_unit = 'mjd'
elif min(args.time_list) > 50000:
prt.message('assuming_mjd')
args.time_unit = 'mjd'
args.time_unit = None
if args.burn is None and args.post_burn is None:
args.burn = int(np.floor(args.iterations / 2))
if args.frack_step == 0:
args.fracking = False
if (args.run_until_uncorrelated is not None and args.run_until_converged):
raise ValueError(
'`-R` and `-U` options are incompatible, please use one or the '
'other.')
if args.run_until_uncorrelated is not None:
args.convergence_type = 'acor'
args.convergence_criteria = args.run_until_uncorrelated
elif args.run_until_converged:
if args.method == 'ensembler':
args.convergence_type = 'psrf'
args.convergence_criteria = (1.1
if args.run_until_converged is True
else args.run_until_converged)
else:
args.convergence_type = 'dlogz'
if args.method == 'nester':
args.convergence_criteria = (0.02 if args.run_until_converged is True
else args.run_until_converged)
if is_master():
# Get hash of ourselves
mosfit_hash = get_mosfit_hash()
# Print our amazing ASCII logo.
if not args.quiet:
with codecs.open(os.path.join(dir_path, 'logo.txt'), 'r',
'utf-8') as f:
logo = f.read()
firstline = logo.split('\n')[0]
# if isinstance(firstline, bytes):
# firstline = firstline.decode('utf-8')
width = len(normalize('NFC', firstline))
prt.prt(logo, colorify=True)
prt.message(
'byline',
reps=[__version__, mosfit_hash, __author__, __contributors__],
center=True,
colorify=True,
width=width,
wrapped=False)
# Get/set upload token
upload_token = ''
get_token_from_user = False
if args.set_upload_token:
if args.set_upload_token is not True:
upload_token = args.set_upload_token
get_token_from_user = True
upload_token_path = os.path.join(dir_path, 'cache', 'dropbox.token')
# Perform a few checks on upload before running (to keep size
# manageable)
if args.upload and not args.test and args.smooth_times > 100:
response = prt.prompt('ul_warning_smooth')
if response:
args.upload = False
else:
sys.exit()
if (args.upload and not args.test and args.num_walkers is not None
and args.num_walkers < 100):
response = prt.prompt('ul_warning_few_walkers')
if response:
args.upload = False
else:
sys.exit()
if (args.upload and not args.test and args.num_walkers
and args.num_walkers * args.num_temps > 500):
response = prt.prompt('ul_warning_too_many_walkers')
if response:
args.upload = False
else:
sys.exit()
if args.upload:
if not os.path.isfile(upload_token_path):
get_token_from_user = True
else:
with open(upload_token_path, 'r') as f:
upload_token = f.read().splitlines()
if len(upload_token) != 1:
get_token_from_user = True
elif len(upload_token[0]) != 64:
get_token_from_user = True
else:
upload_token = upload_token[0]
if get_token_from_user:
if args.test:
upload_token = ('1234567890abcdefghijklmnopqrstuvwxyz'
'1234567890abcdefghijklmnopqr')
while len(upload_token) != 64:
prt.message('no_ul_token', ['https://sne.space/mosfit/'],
wrapped=True)
upload_token = prt.prompt('paste_token', kind='string')
if len(upload_token) != 64:
prt.prt(
'Error: Token must be exactly 64 characters in '
'length.',
wrapped=True)
continue
break
with open_atomic(upload_token_path, 'w') as f:
f.write(upload_token)
if args.upload:
prt.prt("Upload flag set, will upload results after completion.",
wrapped=True)
prt.prt("Dropbox token: " + upload_token, wrapped=True)
args.upload_token = upload_token
if no_events:
prt.message('iterations_0', wrapped=True)
# Create the user directory structure, if it doesn't already exist.
if args.copy:
prt.message('copying')
fc = False
if args.force_copy:
fc = prt.prompt('force_copy')
if not os.path.exists('jupyter'):
os.mkdir(os.path.join('jupyter'))
if not os.path.isfile(os.path.join('jupyter',
'mosfit.ipynb')) or fc:
shutil.copy(
os.path.join(dir_path, 'jupyter', 'mosfit.ipynb'),
os.path.join(os.getcwd(), 'jupyter', 'mosfit.ipynb'))
if not os.path.exists('modules'):
os.mkdir(os.path.join('modules'))
module_dirs = next(os.walk(os.path.join(dir_path, 'modules')))[1]
for mdir in module_dirs:
if mdir.startswith('__'):
continue
full_mdir = os.path.join(dir_path, 'modules', mdir)
copy_path = os.path.join(full_mdir, '.copy')
to_copy = []
if os.path.isfile(copy_path):
to_copy = list(
filter(None,
open(copy_path, 'r').read().split()))
mdir_path = os.path.join('modules', mdir)
if not os.path.exists(mdir_path):
os.mkdir(mdir_path)
for tc in to_copy:
tc_path = os.path.join(full_mdir, tc)
if os.path.isfile(tc_path):
shutil.copy(tc_path, os.path.join(mdir_path, tc))
elif os.path.isdir(tc_path) and not os.path.exists(
os.path.join(mdir_path, tc)):
os.mkdir(os.path.join(mdir_path, tc))
readme_path = os.path.join(mdir_path, 'README')
if not os.path.exists(readme_path):
txt = prt.message('readme-modules', [
os.path.join(dir_path, 'modules', 'mdir'),
os.path.join(dir_path, 'modules')
],
prt=False)
open(readme_path, 'w').write(txt)
if not os.path.exists('models'):
os.mkdir(os.path.join('models'))
model_dirs = next(os.walk(os.path.join(dir_path, 'models')))[1]
for mdir in model_dirs:
if mdir.startswith('__'):
continue
mdir_path = os.path.join('models', mdir)
if not os.path.exists(mdir_path):
os.mkdir(mdir_path)
model_files = next(
os.walk(os.path.join(dir_path, 'models', mdir)))[2]
readme_path = os.path.join(mdir_path, 'README')
if not os.path.exists(readme_path):
txt = prt.message('readme-models', [
os.path.join(dir_path, 'models', mdir),
os.path.join(dir_path, 'models')
],
prt=False)
with open(readme_path, 'w') as f:
f.write(txt)
for mfil in model_files:
if 'parameters.json' not in mfil:
continue
fil_path = os.path.join(mdir_path, mfil)
if os.path.isfile(fil_path) and not fc:
continue
shutil.copy(os.path.join(dir_path, 'models', mdir, mfil),
os.path.join(fil_path))
# Set some default values that we checked above.
if args.frack_step == 0:
args.fracking = False
elif args.frack_step is None:
args.frack_step = 50
if args.burn is None and args.post_burn is None:
args.burn = int(np.floor(args.iterations / 2))
if args.draw_above_likelihood is None:
args.draw_above_likelihood = False
if args.maximum_memory is None:
args.maximum_memory = np.inf
if args.gibbs is None:
args.gibbs = False
if args.save_full_chain is None:
args.save_full_chain = False
if args.num_temps is None:
args.num_temps = 1
if args.walker_paths is None:
args.walker_paths = []
# Then, fit the listed events with the listed models.
fitargs = vars(args)
Fitter(**fitargs).fit_events(**fitargs)
def do_superfit_spectra(catalog):
superfit_url = 'http://www.dahowell.com/superfit.html'
task_str = catalog.get_current_task_str()
sfdirs = list(
glob(os.path.join(catalog.get_current_task_repo(), 'superfit/*')))
for sfdir in pbar(sfdirs, task_str):
sffiles = sorted(glob(sfdir + '/*.dat'))
lastname = ''
oldname = ''
for sffile in pbar(sffiles, task_str):
basename = os.path.basename(sffile)
name = basename.split('.')[0]
if name.startswith('sn'):
name = 'SN' + name[2:]
if len(name) == 7:
name = name[:6] + name[6].upper()
elif name.startswith('ptf'):
name = 'PTF' + name[3:]
if 'theory' in name:
continue
if catalog.entry_exists(name):
prefname = catalog.get_preferred_name(name)
if ('spectra' in catalog.entries[prefname]
and lastname != prefname):
continue
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
epoch = basename.split('.')[1]
mldt, mlmag, mlband, mlsource = \
catalog.entries[name]._get_max_light()
if mldt:
if epoch == 'max':
epoff = Decimal(0.0)
elif epoch[0] == 'p':
epoff = Decimal(epoch[1:])
else:
epoff = -Decimal(epoch[1:])
else:
epoff = ''
source = catalog.entries[name].add_source(name='Superfit',
url=superfit_url,
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oldname,
source)
with open(sffile) as ff:
rows = ff.read().splitlines()
specdata = []
for row in rows:
if row.strip():
specdata.append(
list(filter(None, re.split('\t+|\s+', row,
maxsplit=0))))
specdata = [[xx.replace('D', 'E') for xx in list(ii)]
for ii in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
if epoff != '':
mlmjd = astrotime('-'.join(
[str(mldt.year),
str(mldt.month),
str(mldt.day)])).mjd
mlmjd = str(Decimal(mlmjd) + epoff)
else:
mlmjd = ''
catalog.entries[name].add_spectrum(u_wavelengths='Angstrom',
u_fluxes='Uncalibrated',
u_time='MJD' if mlmjd else '',
time=mlmjd,
wavelengths=wavelengths,
fluxes=fluxes,
source=source,
filename=basename)
lastname = name
catalog.journal_entries()
return
thisevent = thisevent[list(thisevent.keys())[0]]
if ('photometry' not in thisevent or 'maxdate' not in thisevent or 'maxabsmag' not in thisevent or
'maxappmag' not in thisevent or 'claimedtype' not in thisevent or
len(thisevent['maxdate'][0]['value'].split('/')) < 3 or 'discoverdate' not in thisevent):
continue
foundtype = False
for ct in thisevent['claimedtype']:
if ct['value'] == averagetype:
foundtype = True
break
if not foundtype:
continue
maxdate = astrotime(thisevent['maxdate'][0][
'value'].replace('/', '-')).mjd
discoverdate = astrotime(thisevent['discoverdate'][0][
'value'].replace('/', '-')).mjd
if maxdate == discoverdate:
continue
distmod = float(thisevent['maxappmag'][0]['value']) - \
float(thisevent['maxabsmag'][0]['value'])
tprint(thisevent['name'])
prange = list(range(len([x for x in thisevent['photometry'] if photo_cut(
x)]))) if 'photometry' in thisevent else []
if len(prange) <= 3:
def do_donated_photo(catalog):
"""Import donated photometry."""
task_str = catalog.get_current_task_str()
# Private donations here #
if not catalog.args.travis:
pass
# End private donations #
# Ponder 05-12-17 donation
with open(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Ponder-05-12-17', 'meta.json'), 'r') as f:
metadict = json.loads(f.read())
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Ponder-05-12-17', '*.dat'))
for path in file_names:
with open(path, 'r') as f:
tsvin = list(csv.reader(f, delimiter=' ', skipinitialspace=True))
oname = path.split('/')[-1].split('.')[0]
name, source = catalog.new_entry(
oname, bibcode=metadict[oname]['bibcode'])
for row in pbar(tsvin, task_str + ': Ponder ' + oname):
if row[0][0] == '#' or not is_number(row[-1]):
continue
mjd = row[1]
bandinst = row[2].split('_')
band = bandinst[0]
inst = ''
if len(bandinst) > 1:
inst = bandinst[1]
mag = row[3]
uerr = row[4]
lerr = row[5]
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.BAND: band,
PHOTOMETRY.MAGNITUDE: mag,
PHOTOMETRY.E_LOWER_MAGNITUDE: lerr,
PHOTOMETRY.E_UPPER_MAGNITUDE: uerr,
PHOTOMETRY.SOURCE: source
}
if inst:
photodict[PHOTOMETRY.INSTRUMENT] = inst
catalog.entries[name].add_photometry(**photodict)
# Benetti 03-08-17 donation
path = os.path.join(catalog.get_current_task_repo(), 'Donations',
'Benetti-03-08-17', '1999E.dat')
with open(path, 'r') as f:
tsvin = list(csv.reader(f, delimiter=' ', skipinitialspace=True))
name, source = catalog.new_entry(
'SN1999E', bibcode='2003MNRAS.340..191R')
bands = None
for row in tsvin:
if not row or row[0][0] == '#':
continue
if not bands:
bands = row[2:-2]
continue
mjd = row[1]
tel = row[-1] if 'IAUC' not in row[-1] else None
for bi, band in enumerate(bands):
mag = row[2 + 2 * bi]
if mag == '9999':
continue
err = row[2 + 2 * bi + 1]
limit = row[6] == 'True'
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.TELESCOPE: tel,
PHOTOMETRY.BAND: band,
PHOTOMETRY.MAGNITUDE: mag,
PHOTOMETRY.SOURCE: source
}
if err != '.00':
photodict[PHOTOMETRY.E_MAGNITUDE] = str(Decimal(err))
if tel:
photodict[PHOTOMETRY.TELESCOPE] = tel
catalog.entries[name].add_photometry(**photodict)
# Nicholl 01-29-17 donation
with open(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-01-29-17', 'meta.json'), 'r') as f:
metadict = json.loads(f.read())
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-01-29-17', '*.txt'))
for path in file_names:
data = read(path, format='cds')
oname = path.split('/')[-1].split('_')[0]
name, source = catalog.new_entry(
oname, bibcode=metadict[oname]['bibcode'])
for row in pbar(data, task_str + ': Nicholl ' + oname):
photodict = {
PHOTOMETRY.TIME: str(row['MJD']),
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.MAGNITUDE: str(row['mag']),
PHOTOMETRY.BAND: row['Filter'],
PHOTOMETRY.SOURCE: source
}
if 'system' in metadict[oname]:
photodict[PHOTOMETRY.SYSTEM] = metadict[oname]['system']
if 'l_mag' in row.columns and row['l_mag'] == '>':
photodict[PHOTOMETRY.UPPER_LIMIT] = True
elif 'e_mag' in row.columns:
photodict[PHOTOMETRY.E_MAGNITUDE] = str(row['e_mag'])
if 'Telescope' in row.columns:
photodict[PHOTOMETRY.TELESCOPE] = row['Telescope']
catalog.entries[name].add_photometry(**photodict)
# Arcavi 2016gkg donation
path = os.path.join(catalog.get_current_task_repo(), 'Donations',
'Arcavi-01-24-17', 'SN2016gkg.txt')
with open(path, 'r') as f:
tsvin = list(csv.reader(f, delimiter=' ', skipinitialspace=True))
name, source = catalog.new_entry(
'SN2016gkg', bibcode='2016arXiv161106451A')
for row in tsvin:
if row[0][0] == '#':
continue
mjd = str(jd_to_mjd(Decimal(row[0])))
tel = row[1]
band = row[3]
mag = row[4]
err = row[5]
limit = row[6] == 'True'
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.TELESCOPE: tel,
PHOTOMETRY.BAND: band,
PHOTOMETRY.MAGNITUDE: mag,
PHOTOMETRY.SOURCE: source
}
if limit:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
else:
photodict[PHOTOMETRY.E_MAGNITUDE] = err
catalog.entries[name].add_photometry(**photodict)
# Nicholl Gaia16apd donation
path = os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-01-20-17', 'gaia16apd_phot.txt')
data = read(path, format='cds')
name, source = catalog.new_entry(
'Gaia16apd', bibcode='2017ApJ...835L...8N')
for row in pbar(data, task_str + ': Nicholl Gaia16apd'):
photodict = {
PHOTOMETRY.TIME: str(row['MJD']),
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.MAGNITUDE: str(row['mag']),
PHOTOMETRY.BAND: row['Filter'],
PHOTOMETRY.TELESCOPE: row['Telescope'],
PHOTOMETRY.SOURCE: source
}
if row['l_mag'] == '>':
photodict[PHOTOMETRY.UPPER_LIMIT] = True
else:
photodict[PHOTOMETRY.E_MAGNITUDE] = str(row['e_mag'])
catalog.entries[name].add_photometry(**photodict)
# Kuncarayakti-01-09-17
datafile = os.path.join(catalog.get_current_task_repo(), 'Donations',
'Kuncarayakti-01-09-17', 'SN1978K.dat')
inpname = os.path.basename(datafile).split('.')[0]
with open(datafile, 'r') as f:
tsvin = csv.reader(f, delimiter=' ', skipinitialspace=True)
host = False
for ri, row in enumerate(tsvin):
if ri == 0:
continue
if row[0][0] == '#':
rsplit = [x.strip('# ') for x in ' '.join(row).split(',')]
bc = rsplit[0]
tel, ins = '', ''
if len(rsplit) > 1:
tel = rsplit[1]
if len(rsplit) > 2:
ins = rsplit[2]
continue
(name, source) = catalog.new_entry(inpname, bibcode=bc)
mag = row[4]
err = row[5]
mjd = str(astrotime('-'.join(row[:3]), format='iso').mjd)
photodict = {
PHOTOMETRY.BAND: row[3],
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.MAGNITUDE: mag.strip('>s'),
PHOTOMETRY.SOURCE: source
}
if is_number(err):
photodict[PHOTOMETRY.E_MAGNITUDE] = err
if tel:
photodict[PHOTOMETRY.TELESCOPE] = tel
if ins:
photodict[PHOTOMETRY.INSTRUMENT] = ins
if '>' in mag:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
if 's' in mag:
photodict[PHOTOMETRY.SYNTHETIC] = True
catalog.entries[name].add_photometry(**photodict)
# Nugent 01-09-17 donation
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nugent-01-09-17', '*.dat'))
for datafile in pbar_strings(file_names, task_str + ': Nugent-01-09-17'):
inpname = os.path.basename(datafile).split('.')[0]
(name, source) = catalog.new_entry(
inpname, bibcode='2006ApJ...645..841N')
with open(datafile, 'r') as f:
tsvin = csv.reader(f, delimiter=' ', skipinitialspace=True)
host = False
for urow in tsvin:
row = list(filter(None, urow))
counts = row[2]
e_counts = row[3]
zp = row[4]
photodict = {
PHOTOMETRY.BAND: row[1],
PHOTOMETRY.TIME: row[0],
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.COUNT_RATE: counts,
PHOTOMETRY.E_COUNT_RATE: e_counts,
PHOTOMETRY.ZERO_POINT: zp,
PHOTOMETRY.TELESCOPE: 'CFHT',
PHOTOMETRY.SURVEY: 'SNLS',
PHOTOMETRY.SOURCE: source
}
set_pd_mag_from_counts(photodict, counts, ec=e_counts, zp=zp,
sig=5.0)
catalog.entries[name].add_photometry(**photodict)
# Inserra 09-04-16 donation
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Inserra-09-04-16', '*.txt'))
for datafile in pbar_strings(file_names, task_str + ': Inserra-09-04-16'):
inpname = os.path.basename(datafile).split('.')[0]
(name, source) = catalog.new_entry(
inpname, bibcode='2013ApJ...770..128I')
with open(datafile, 'r') as f:
tsvin = csv.reader(f, delimiter=' ', skipinitialspace=True)
host = False
for row in tsvin:
if row[0][0] == '#':
if row[0] == '#Host':
host = True
continue
host = False
bands = row[3:-1]
continue
for bi, ba in enumerate(bands):
mag = row[5 + 2 * bi]
if not is_number(mag):
continue
system = 'AB'
if ba in ['U', 'B', 'V', 'R', 'I', 'J', 'H', 'K']:
system = 'Vega'
photodict = {
PHOTOMETRY.TIME: row[3],
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.BAND: ba,
PHOTOMETRY.MAGNITUDE: mag.strip('< '),
PHOTOMETRY.SOURCE: source,
PHOTOMETRY.SYSTEM: system
}
if 'ATel' not in row[-1]:
photodict[PHOTOMETRY.TELESCOPE] = row[-1]
if host:
photodict[PHOTOMETRY.HOST] = True
if '<' in mag:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
e_mag = row[5 + 2 * bi + 1].strip('() ')
if is_number(e_mag):
photodict[PHOTOMETRY.E_MAGNITUDE] = e_mag
catalog.entries[name].add_photometry(**photodict)
# Nicholl 04-01-16 donation
with open(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-04-01-16', 'bibcodes.json'), 'r') as f:
bcs = json.loads(f.read())
kcorrected = ['SN2011ke', 'SN2011kf', 'SN2012il', 'PTF10hgi', 'PTF11rks']
ignorephoto = ['PTF10hgi', 'PTF11rks', 'SN2011ke', 'SN2011kf', 'SN2012il']
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-04-01-16/*.txt'))
for datafile in pbar_strings(file_names, task_str + ': Nicholl-04-01-16'):
inpname = os.path.basename(datafile).split('_')[0]
isk = inpname in kcorrected
name = catalog.add_entry(inpname)
bibcode = ''
for bc in bcs:
if inpname in bcs[bc]:
bibcode = bc
if not bibcode:
raise ValueError('Bibcode not found!')
source = catalog.entries[name].add_source(bibcode=bibcode)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, inpname, source)
if inpname in ignorephoto:
continue
with open(datafile, 'r') as f:
tsvin = csv.reader(f, delimiter='\t', skipinitialspace=True)
rtelescope = ''
for r, rrow in enumerate(tsvin):
row = list(filter(None, rrow))
if not row:
continue
if row[0] == '#MJD':
bands = [x for x in row[1:] if x and 'err' not in x]
elif row[0][0] == '#' and len(row[0]) > 1:
rtelescope = row[0][1:]
if row[0][0] == '#':
continue
mjd = row[0]
if not is_number(mjd):
continue
for v, val in enumerate(row[1::2]):
upperlimit = ''
mag = val.strip('>')
emag = row[2 * v + 2]
if '>' in val or (is_number(emag) and float(emag) == 0.0):
upperlimit = True
if (not is_number(mag) or isnan(float(mag)) or
float(mag) > 90.0):
continue
band = bands[v]
instrument = ''
survey = ''
system = ''
telescope = rtelescope
if telescope == 'LSQ':
instrument = 'QUEST'
elif telescope == 'PS1':
instrument = 'GPC'
elif telescope == 'NTT':
instrument = 'EFOSC'
elif telescope == 'GROND':
instrument = 'GROND'
telescope = 'MPI/ESO 2.2m'
else:
if band == 'NUV':
instrument = 'GALEX'
telescope = 'GALEX'
elif band in ['u', 'g', 'r', 'i', 'z']:
if inpname.startswith('PS1'):
instrument = 'GPC'
telescope = 'PS1'
survey = 'Pan-STARRS'
elif inpname.startswith('PTF'):
telescope = 'P60'
survey = 'PTF'
elif band.upper() in ['UVW2', 'UVW1', 'UVM2']:
instrument = 'UVOT'
telescope = 'Swift'
if inpname in ['PTF12dam']:
system = 'AB'
if inpname in ['SCP-06F6']:
system = 'Vega'
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.BAND: band,
PHOTOMETRY.MAGNITUDE: mag,
PHOTOMETRY.UPPER_LIMIT: upperlimit,
PHOTOMETRY.SOURCE: source
}
if instrument:
photodict[PHOTOMETRY.INSTRUMENT] = instrument
if telescope:
photodict[PHOTOMETRY.TELESCOPE] = telescope
if survey:
photodict[PHOTOMETRY.SURVEY] = survey
if system:
photodict[PHOTOMETRY.SYSTEM] = system
if (is_number(emag) and
not isnan(float(emag)) and float(emag) > 0.0):
photodict[PHOTOMETRY.E_MAGNITUDE] = emag
if isk:
photodict[PHOTOMETRY.KCORRECTED] = True
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
# Maggi 04-11-16 donation (MC SNRs)
with open(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Maggi-04-11-16', 'LMCSNRs_OpenSNe.csv')) as f:
tsvin = csv.reader(f, delimiter=',')
for row in pbar(list(tsvin), task_str + ': Maggi-04-11-16/LMCSNRs'):
name = 'MCSNR ' + row[0]
name = catalog.add_entry(name)
ra = row[2]
dec = row[3]
source = (catalog.entries[name]
.add_source(bibcode='2016A&A...585A.162M'))
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS,
'LMCSNR J' + rep_chars(ra, ' :.') + rep_chars(dec, ' :.'),
source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
if row[1] != 'noname':
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[1],
source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[2], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[3], source)
catalog.entries[name].add_quantity(SUPERNOVA.HOST, 'LMC', source)
if row[4] == '1':
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE,
'Ia', source)
elif row[4] == '2':
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE,
'CC', source)
with open(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Maggi-04-11-16', 'SMCSNRs_OpenSNe.csv')) as f:
tsvin = csv.reader(f, delimiter=',')
for row in pbar(list(tsvin), task_str + ': Maggi-04-11-16/SMCSNRs'):
name = 'MCSNR ' + row[0]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(name='Pierre Maggi')
ra = row[3]
dec = row[4]
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, 'SMCSNR J' + ra.replace(
':', '')[:6] + dec.replace(':', '')[:7], source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[1], source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[2], source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[3], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[4], source)
catalog.entries[name].add_quantity(SUPERNOVA.HOST, 'SMC', source)
catalog.journal_entries()
# Galbany 04-18-16 donation
folders = next(
os.walk(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Galbany-04-18-16/')))[1]
bibcode = '2016AJ....151...33G'
for folder in folders:
infofiles = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Galbany-04-18-16/') + folder + '/*.info')
photfiles = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Galbany-04-18-16/') + folder + '/*.out*')
zhel = ''
zcmb = ''
zerr = ''
for path in infofiles:
with open(path, 'r') as f:
lines = f.read().splitlines()
for line in lines:
splitline = line.split(':')
field = splitline[0].strip().lower()
value = splitline[1].strip()
if field == 'name':
name = value[:6].upper()
name += (value[6].upper()
if len(value) == 7 else value[6:])
name = catalog.add_entry(name)
source = (catalog.entries[name]
.add_source(bibcode=bibcode))
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS,
name, source)
elif field == 'type':
claimedtype = value.replace('SN', '')
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, claimedtype, source)
elif field == 'zhel':
zhel = value
elif field == 'redshift_error':
zerr = value
elif field == 'zcmb':
zcmb = value
elif field == 'ra':
catalog.entries[name].add_quantity(
SUPERNOVA.RA,
value,
source,
u_value='floatdegrees')
elif field == 'dec':
catalog.entries[name].add_quantity(
SUPERNOVA.DEC,
value,
source,
u_value='floatdegrees')
elif field == 'host':
value = value.replace('- ', '-').replace('G ', 'G')
catalog.entries[name].add_quantity(SUPERNOVA.HOST,
value, source)
elif field == 'e(b-v)_mw':
catalog.entries[name].add_quantity(SUPERNOVA.EBV,
value, source)
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT,
zhel,
source,
e_value=zerr,
kind='heliocentric')
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, zcmb, source, e_value=zerr, kind='cmb')
for path in photfiles:
with open(path, 'r') as f:
band = ''
lines = f.read().splitlines()
for li, line in enumerate(lines):
if li in [0, 2, 3]:
continue
if li == 1:
band = line.split(':')[-1].strip()
else:
cols = list(filter(None, line.split()))
if not cols:
continue
catalog.entries[name].add_photometry(
time=cols[0],
u_time='MJD',
magnitude=cols[1],
e_magnitude=cols[2],
band=band,
system=cols[3],
telescope=cols[4],
source=source)
catalog.journal_entries()
# Nicholl 05-03-16
files = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-05-03-16', '*.txt'))
name = catalog.add_entry('SN2015bn')
for fi in pbar(files, task_str + ': Nicholl-05-03-16'):
if 'late' in fi:
bc = '2016ApJ...828L..18N'
else:
bc = '2016ApJ...826...39N'
source = catalog.entries[name].add_source(bibcode=bc)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, 'PS15ae', source)
telescope = os.path.basename(fi).split('_')[1]
with open(fi, 'r') as f:
lines = f.read().splitlines()
for li, line in enumerate(lines):
if not line or (line[0] == '#' and li != 0):
continue
cols = list(filter(None, line.split()))
if not cols:
continue
if li == 0:
bands = cols[1:]
continue
mjd = cols[0]
for ci, col in enumerate(cols[1::2]):
if not is_number(col) or np.isnan(float(col)):
continue
band = bands[ci]
band_set = ''
system = 'Vega'
if bands[ci] in ["u'", "g'", "r'", "i'", "z'"]:
band_set = 'SDSS'
system = 'SDSS'
elif telescope == 'ASASSN':
band_set = 'ASASSN'
system = 'Vega'
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.MAGNITUDE: col,
PHOTOMETRY.BAND: bands[ci],
PHOTOMETRY.SOURCE: source,
PHOTOMETRY.TELESCOPE: telescope,
PHOTOMETRY.SYSTEM: system
}
if band_set:
photodict[PHOTOMETRY.BAND_SET] = band_set
emag = cols[2 * ci + 2]
if is_number(emag):
photodict[PHOTOMETRY.E_MAGNITUDE] = emag
else:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
if telescope == 'Swift':
photodict[PHOTOMETRY.INSTRUMENT] = 'UVOT'
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
return
def do_donated_spectra(catalog):
"""Import donated spectra."""
task_str = catalog.get_current_task_str()
fpath = os.path.join(catalog.get_current_task_repo(), 'donations')
with open(os.path.join(fpath, 'meta.json'), 'r') as f:
metadict = json.loads(f.read())
donationscnt = 0
oldname = ''
for fname in pbar(list(metadict.keys()), task_str):
name = metadict[fname]['name']
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
sec_bibc = metadict[fname]['bibcode']
name, source = catalog.new_entry(name, bibcode=sec_bibc)
date = metadict[fname].get('date', '')
year, month, day = date.split('/')
sig = get_sig_digits(day) + 5
day_fmt = str(floor(float(day))).zfill(2)
time = astrotime(year + '-' + month + '-' + day_fmt).mjd
time = time + float(day) - floor(float(day))
time = pretty_num(time, sig=sig)
with open(os.path.join(fpath, fname), 'r') as f:
specdata = list(
csv.reader(
f, delimiter=' ', skipinitialspace=True))
specdata = list(filter(None, specdata))
newspec = []
oldval = ''
for row in specdata:
if row[0][0] == '#':
continue
if row[1] == oldval:
continue
newspec.append(row)
oldval = row[1]
specdata = newspec
haserrors = len(specdata[0]) == 3 and specdata[0][2] and specdata[0][
2] != 'NaN'
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.U_TIME: 'MJD',
SPECTRUM.TIME: time,
SPECTRUM.WAVELENGTHS: wavelengths,
SPECTRUM.FLUXES: fluxes,
SPECTRUM.ERRORS: errors,
SPECTRUM.SOURCE: source,
SPECTRUM.FILENAME: fname
}
if 'instrument' in metadict[fname]:
specdict[SPECTRUM.INSTRUMENT] = metadict[fname]['instrument']
if 'telescope' in metadict[fname]:
specdict[SPECTRUM.TELESCOPE] = metadict[fname]['telescope']
if 'yunit' in metadict[fname]:
specdict[SPECTRUM.U_FLUXES] = metadict[fname]['yunit']
specdict[SPECTRUM.U_ERRORS] = metadict[fname]['yunit']
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict[SPECTRUM.U_FLUXES] = fluxunit
specdict[SPECTRUM.U_ERRORS] = fluxunit
catalog.entries[name].add_spectrum(**specdict)
donationscnt = donationscnt + 1
if (catalog.args.travis and
donationscnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
def do_essence_spectra(catalog):
task_str = catalog.get_current_task_str()
insdict = {
"lris": "LRIS",
"esi": "ESI",
"deimos": "DEIMOS",
"gmos": "GMOS",
"fors1": "FORS1",
"bluechannel": "Blue Channel",
"ldss2": "LDSS-2",
"ldss3": "LDSS-3",
"imacs": "IMACS",
"fast": "FAST"
}
teldict = {
"lris": "Keck",
"esi": "Keck",
"deimos": "Keck",
"gmos": "Gemini",
"fors1": "VLT",
"bluechannel": "MMT",
"ldss2": "Magellan Clay & Baade",
"ldss3": "Magellan Clay & Baade",
"imacs": "Magellan Clay & Baade",
"fast": "FLWO 1.5m"
}
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'ESSENCE', '*'))
oldname = ''
for fi, fname in enumerate(pbar_strings(file_names, task_str)):
filename = os.path.basename(fname)
fileparts = filename.split('_')
name = 'ESSENCE ' + fileparts[0]
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
if is_number(fileparts[1]):
doffset = 1
else:
if fileparts[1] != 'comb':
continue
doffset = 2
dstr = fileparts[doffset]
mjd = str(
astrotime(
datetime.datetime(year=int(dstr[:4]),
month=int(dstr[4:6]),
day=int(dstr[6:8])) +
datetime.timedelta(days=float(dstr[8:]))).mjd)
instrument = fileparts[-1].split('.')[0]
telescope = teldict.get(instrument, '')
instrument = insdict.get(instrument, '')
with open(fname, 'r') as f:
data = csv.reader(f, delimiter=' ', skipinitialspace=True)
data = [list(i) for i in zip(*data)]
wavelengths = data[0]
fluxes = [str(Decimal('1.0e-15') * Decimal(x)) for x in data[1]]
name, source = catalog.new_entry(name, bibcode='2016ApJS..224....3N')
specdict = {
SPECTRUM.TIME: mjd,
SPECTRUM.U_TIME: 'MJD',
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.WAVELENGTHS: wavelengths,
SPECTRUM.FLUXES: fluxes,
SPECTRUM.U_FLUXES: 'erg/s/cm^2/Angstrom',
SPECTRUM.FILENAME: filename,
SPECTRUM.SOURCE: source
}
if instrument:
specdict[SPECTRUM.INSTRUMENT] = instrument
if telescope:
specdict[SPECTRUM.TELESCOPE] = telescope
catalog.entries[name].add_spectrum(**specdict)
if catalog.args.travis and fi >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
return
def do_external_fits_spectra(catalog):
fpath = catalog.get_current_task_repo()
with open(os.path.join(fpath, 'fits', 'meta.json'), 'r') as f:
metadict = json.loads(f.read())
fureps = {'erg/cm2/s/A': 'erg/s/cm^2/Angstrom'}
task_str = catalog.get_current_task_str()
path_pattern = os.path.join(catalog.get_current_task_repo(), 'fits',
'*.fits')
files = glob(path_pattern)
for datafile in files:
filename = datafile.split('/')[-1]
if filename == 'meta.json':
continue
hdulist = fits.open(datafile)
for oi, obj in enumerate(hdulist[0].header):
if any(x in ['.', '/'] for x in obj):
del (hdulist[0].header[oi])
hdulist[0].verify('silentfix')
hdrkeys = list(hdulist[0].header.keys())
# print(hdrkeys)
name = ''
if filename in metadict:
if 'name' in metadict[filename]:
name = metadict[filename]['name']
if not name:
name = hdulist[0].header['OBJECT']
if 'bibcode' in metadict[filename]:
name, source = catalog.new_entry(
name, bibcode=metadict[filename]['bibcode'])
elif 'donator' in metadict[filename]:
name, source = catalog.new_entry(
name, srcname=metadict[filename]['donator'])
else:
if 'OBSERVER' in hdrkeys:
name, source = catalog.new_entry(
name, srcname=hdulist[0].header['OBSERVER'])
else:
name = catalog.add_entry(name)
source = catalog.entries[name].add_self_source()
# for key in hdulist[0].header.keys():
# print(key, hdulist[0].header[key])
if hdulist[0].header['SIMPLE']:
if 'JD' in hdrkeys:
mjd = str(jd_to_mjd(Decimal(str(hdulist[0].header['JD']))))
elif 'MJD' in hdrkeys:
mjd = str(hdulist[0].header['MJD'])
elif 'DATE-OBS' in hdrkeys:
if 'T' in hdulist[0].header['DATE-OBS']:
dateobs = hdulist[0].header['DATE-OBS'].strip()
elif 'UTC-OBS' in hdrkeys:
dateobs = hdulist[0].header['DATE-OBS'].strip(
) + 'T' + hdulist[0].header['UTC-OBS'].strip()
mjd = str(astrotime(dateobs, format='isot').mjd)
else:
raise ValueError("Couldn't find JD/MJD for spectrum.")
w0 = hdulist[0].header['CRVAL1']
if hdulist[0].header['NAXIS'] == 1:
wd = hdulist[0].header['CDELT1']
fluxes = [str(x) for x in list(hdulist[0].data)]
errors = False
elif hdulist[0].header['NAXIS'] == 2:
wd = hdulist[0].header['CD1_1']
fluxes = [str(x) for x in list(hdulist[0].data)[0]]
errors = False
elif hdulist[0].header['NAXIS'] == 3:
wd = hdulist[0].header['CD1_1']
fluxes = [str(x) for x in list(hdulist[0].data)[0][0]]
errors = [str(x) for x in list(hdulist[0].data)[3][0]]
else:
print('Warning: Skipping FITS spectrum `{}`.'.format(filename))
continue
waves = [str(w0 + wd * x) for x in range(0, len(fluxes))]
else:
raise ValueError('Non-simple FITS import not yet supported.')
if 'BUNIT' in hdrkeys:
fluxunit = hdulist[0].header['BUNIT']
if fluxunit in fureps:
fluxunit = fureps[fluxunit]
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.WAVELENGTHS: waves,
SPECTRUM.TIME: mjd,
SPECTRUM.U_TIME: 'MJD',
SPECTRUM.FLUXES: fluxes,
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.FILENAME: filename,
SPECTRUM.SOURCE: source
}
if 'TELESCOP' in hdrkeys:
specdict[SPECTRUM.TELESCOPE] = hdulist[0].header['TELESCOP']
if 'INSTRUME' in hdrkeys:
specdict[SPECTRUM.INSTRUMENT] = hdulist[0].header['INSTRUME']
if 'AIRMASS' in hdrkeys:
specdict[SPECTRUM.AIRMASS] = hdulist[0].header['AIRMASS']
if errors:
specdict[SPECTRUM.ERRORS] = errors
specdict[SPECTRUM.U_ERRORS] = fluxunit
if 'SITENAME' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['SITENAME']
elif 'OBSERVAT' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['OBSERVAT']
if 'OBSERVER' in hdrkeys:
specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER']
catalog.entries[name].add_spectrum(**specdict)
hdulist.close()
catalog.journal_entries()
return
def do_rochester(catalog):
rochestermirrors = ['http://www.rochesterastronomy.org/',
'http://www.supernova.thistlethwaites.com/']
rochesterpaths = ['snimages/snredshiftall.html',
'sn2016/snredshift.html', 'snimages/snredboneyard.html']
rochesterupdate = [False, True, True]
task_str = catalog.get_current_task_str()
for pp, path in enumerate(pbar(rochesterpaths, task_str)):
if catalog.args.update and not rochesterupdate[pp]:
continue
filepath = (os.path.join(
catalog.get_current_task_repo(), 'rochester/') +
os.path.basename(path))
for mirror in rochestermirrors:
html = catalog.load_cached_url(
mirror + path, filepath,
failhard=(mirror != rochestermirrors[-1]))
if html:
break
if not html:
continue
soup = BeautifulSoup(html, 'html5lib')
rows = soup.findAll('tr')
sec_ref = 'Latest Supernovae'
sec_refurl = ('http://www.rochesterastronomy.org/'
'snimages/snredshiftall.html')
for rr, row in enumerate(pbar(rows, task_str)):
if rr == 0:
continue
cols = row.findAll('td')
if not len(cols):
continue
name = ''
if cols[14].contents:
aka = str(cols[14].contents[0]).strip()
if is_number(aka.strip('?')):
aka = 'SN' + aka.strip('?') + 'A'
oldname = aka
name = catalog.add_entry(aka)
elif len(aka) == 4 and is_number(aka[:4]):
aka = 'SN' + aka
oldname = aka
name = catalog.add_entry(aka)
ra = str(cols[3].contents[0]).strip()
dec = str(cols[4].contents[0]).strip()
sn = re.sub('<[^<]+?>', '', str(cols[0].contents[0])).strip()
if is_number(sn.strip('?')):
sn = 'SN' + sn.strip('?') + 'A'
elif len(sn) == 4 and is_number(sn[:4]):
sn = 'SN' + sn
if not name:
if not sn:
continue
if sn[:8] == 'MASTER J':
sn = sn.replace('MASTER J', 'MASTER OT J').replace(
'SNHunt', 'SNhunt')
if 'POSSIBLE' in sn.upper() and ra and dec:
sn = 'PSN J' + ra.replace(':', '').replace('.', '')
sn += dec.replace(':', '').replace('.', '')
oldname = sn
name = catalog.add_entry(sn)
reference = cols[12].findAll('a')[0].contents[0].strip()
refurl = cols[12].findAll('a')[0]['href'].strip()
source = catalog.entries[name].add_source(
name=reference, url=refurl)
sec_source = catalog.entries[name].add_source(
name=sec_ref, url=sec_refurl, secondary=True)
sources = uniq_cdl(list(filter(None, [source, sec_source])))
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, oldname, sources)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, sn, sources)
if cols[14].contents:
if aka == 'SNR G1.9+0.3':
aka = 'G001.9+00.3'
if aka[:4] == 'PS1 ':
aka = 'PS1-' + aka[4:]
if aka[:8] == 'MASTER J':
aka = aka.replace('MASTER J', 'MASTER OT J').replace(
'SNHunt', 'SNhunt')
if 'POSSIBLE' in aka.upper() and ra and dec:
aka = 'PSN J' + ra.replace(':', '').replace('.', '')
aka += dec.replace(':', '').replace('.', '')
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, aka, sources)
if str(cols[1].contents[0]).strip() != 'unk':
type = str(cols[1].contents[0]).strip(' :,')
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, type, sources)
if str(cols[2].contents[0]).strip() != 'anonymous':
catalog.entries[name].add_quantity(SUPERNOVA.HOST, str(
cols[2].contents[0]).strip(), sources)
catalog.entries[name].add_quantity(SUPERNOVA.RA, ra, sources)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, dec, sources)
if (str(cols[6].contents[0]).strip() not in
['2440587', '2440587.292']):
astrot = astrotime(
float(str(cols[6].contents[0]).strip()),
format='jd').datetime
ddate = make_date_string(astrot.year, astrot.month, astrot.day)
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, ddate, sources)
if (str(cols[7].contents[0]).strip() not in
['2440587', '2440587.292']):
astrot = astrotime(
float(str(cols[7].contents[0]).strip()), format='jd')
if ((float(str(cols[8].contents[0]).strip()) <= 90.0 and
not any('GRB' in xx for xx in
catalog.entries[name].get_aliases()))):
mag = str(cols[8].contents[0]).strip()
catalog.entries[name].add_photometry(
time=str(astrot.mjd), magnitude=mag,
source=sources)
if cols[11].contents[0] != 'n/a':
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, str(
cols[11].contents[0]).strip(), sources)
catalog.entries[name].add_quantity('discoverer', str(
cols[13].contents[0]).strip(), sources)
if catalog.args.update:
catalog.journal_entries()
if not catalog.args.update:
vsnetfiles = ['latestsne.dat']
for vsnetfile in vsnetfiles:
file_name = os.path.join(
catalog.get_current_task_repo(), "" + vsnetfile)
with open(file_name, 'r', encoding='latin1') as csv_file:
tsvin = csv.reader(csv_file, delimiter=' ',
skipinitialspace=True)
for rr, row in enumerate(tsvin):
if (not row or row[0][:4] in ['http', 'www.'] or
len(row) < 3):
continue
name = row[0].strip()
if name[:4].isdigit():
name = 'SN' + name
if name.startswith('PSNJ'):
name = 'PSN J' + name[4:]
if name.startswith('MASTEROTJ'):
name = name.replace('MASTEROTJ', 'MASTER OT J')
name = catalog.add_entry(name)
sec_source = catalog.entries[name].add_source(
name=sec_ref, url=sec_refurl, secondary=True)
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, name, sec_source)
if not is_number(row[1]):
continue
year = row[1][:4]
month = row[1][4:6]
day = row[1][6:]
if '.' not in day:
day = day[:2] + '.' + day[2:]
mjd = astrotime(year + '-' + month + '-' +
str(floor(float(day))).zfill(2)).mjd
mjd += float(day) - floor(float(day))
magnitude = row[2].rstrip(ascii_letters)
if not is_number(magnitude):
continue
if magnitude.isdigit():
if int(magnitude) > 100:
magnitude = magnitude[:2] + '.' + magnitude[2:]
if float(str(cols[8].contents[0]).strip()) >= 90.0:
continue
if len(row) >= 4:
if is_number(row[3]):
e_magnitude = row[3]
refind = 4
else:
e_magnitude = ''
refind = 3
if refind >= len(row):
sources = sec_source
else:
reference = ' '.join(row[refind:])
source = catalog.entries[
name].add_source(name=reference)
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, name, sec_source)
sources = uniq_cdl([source, sec_source])
else:
sources = sec_source
band = row[2].lstrip('1234567890.')
catalog.entries[name].add_photometry(
time=mjd, band=band, magnitude=magnitude,
e_magnitude=e_magnitude, source=sources)
catalog.journal_entries()
return
