def check(self, val):
"""Make sure given value is consistent with this `Key` specification.
NOTE: if `type` is 'None', then `listable` also is *not* checked.
"""
# If there is no `type` requirement, everything is allowed
if self.type is None:
return True
is_list = isinstance(val, list)
# If lists are not allowed, and this is a list --> false
if not self.listable and is_list:
return False
# `is_number` already checks for either list or single value
if self.type == KEY_TYPES.NUMERIC and not is_number(val):
return False
elif (self.type == KEY_TYPES.TIME and not is_number(val)
and '-' not in val and '/' not in val):
return False
elif self.type == KEY_TYPES.STRING:
# If its a list, check first element
if is_list:
if not isinstance(val[0], basestring):
return False
# Otherwise, check it
elif not isinstance(val, basestring):
return False
elif self.type == KEY_TYPES.BOOL:
if is_list and not isinstance(val[0], bool):
return False
elif not isinstance(val, bool):
return False
return True
def add_source(self, **kwargs):
# Sanitize some fields before adding source
# Replace reference names and URLs using dictionaries.
if (kwargs.get(SOURCE.BIBCODE, []) and
len(kwargs[SOURCE.BIBCODE]) != 19):
raise ValueError(
"Bibcode '{}' must be exactly 19 characters "
"long".format(kwargs[SOURCE.BIBCODE]))
# if SOURCE.NAME not in kwargs:
# kwargs[SOURCE.NAME] = kwargs[SOURCE.BIBCODE]
if SOURCE.NAME in kwargs:
if (kwargs[SOURCE.NAME].upper().startswith('ATEL') and
SOURCE.BIBCODE not in kwargs):
kwargs[SOURCE.NAME] = (kwargs[SOURCE.NAME]
.replace('ATEL', 'ATel')
.replace('Atel', 'ATel')
.replace('ATel #', 'ATel ')
.replace('ATel#', 'ATel')
.replace('ATel', 'ATel '))
kwargs[SOURCE.NAME] = ' '.join(kwargs[SOURCE.NAME].split())
atelnum = kwargs[SOURCE.NAME].split()[-1]
if is_number(atelnum) and atelnum in self.catalog.atels_dict:
kwargs[SOURCE.BIBCODE] = self.catalog.atels_dict[atelnum]
if (kwargs[SOURCE.NAME].upper().startswith('CBET') and
SOURCE.BIBCODE not in kwargs):
kwargs[SOURCE.NAME] = kwargs[SOURCE.NAME].replace('CBET',
'CBET ')
kwargs[SOURCE.NAME] = ' '.join(kwargs[SOURCE.NAME].split())
cbetnum = kwargs[SOURCE.NAME].split()[-1]
if is_number(cbetnum) and cbetnum in self.catalog.cbets_dict:
kwargs[SOURCE.BIBCODE] = self.catalog.cbets_dict[cbetnum]
if (kwargs[SOURCE.NAME].upper().startswith('IAUC') and
SOURCE.BIBCODE not in kwargs):
kwargs[SOURCE.NAME] = kwargs[SOURCE.NAME].replace('IAUC',
'IAUC ')
kwargs[SOURCE.NAME] = ' '.join(kwargs[SOURCE.NAME].split())
iaucnum = kwargs[SOURCE.NAME].split()[-1]
if is_number(iaucnum) and iaucnum in self.catalog.iaucs_dict:
kwargs[SOURCE.BIBCODE] = self.catalog.iaucs_dict[iaucnum]
for rep in self.catalog.source_syns:
if kwargs[SOURCE.NAME] in self.catalog.source_syns[rep]:
kwargs[SOURCE.NAME] = rep
break
if SOURCE.URL in kwargs:
for rep in self.catalog.url_redirs:
if kwargs[SOURCE.URL] in self.catalog.url_redirs[rep]:
kwargs[SOURCE.URL] = rep
break
return super().add_source(**kwargs)
def add_source(self, **kwargs):
# Sanitize some fields before adding source
# Replace reference names and URLs using dictionaries.
if (kwargs.get(SOURCE.BIBCODE, [])
and len(kwargs[SOURCE.BIBCODE]) != 19):
raise ValueError("Bibcode '{}' must be exactly 19 characters "
"long".format(kwargs[SOURCE.BIBCODE]))
# if SOURCE.NAME not in kwargs:
# kwargs[SOURCE.NAME] = kwargs[SOURCE.BIBCODE]
if SOURCE.NAME in kwargs:
if (kwargs[SOURCE.NAME].upper().startswith('ATEL')
and SOURCE.BIBCODE not in kwargs):
kwargs[SOURCE.NAME] = (kwargs[SOURCE.NAME].replace(
'ATEL', 'ATel').replace('Atel', 'ATel').replace(
'ATel #',
'ATel ').replace('ATel#',
'ATel').replace('ATel', 'ATel '))
kwargs[SOURCE.NAME] = ' '.join(kwargs[SOURCE.NAME].split())
atelnum = kwargs[SOURCE.NAME].split()[-1]
if is_number(atelnum) and atelnum in self.catalog.atels_dict:
kwargs[SOURCE.BIBCODE] = self.catalog.atels_dict[atelnum]
if (kwargs[SOURCE.NAME].upper().startswith('CBET')
and SOURCE.BIBCODE not in kwargs):
kwargs[SOURCE.NAME] = kwargs[SOURCE.NAME].replace(
'CBET', 'CBET ')
kwargs[SOURCE.NAME] = ' '.join(kwargs[SOURCE.NAME].split())
cbetnum = kwargs[SOURCE.NAME].split()[-1]
if is_number(cbetnum) and cbetnum in self.catalog.cbets_dict:
kwargs[SOURCE.BIBCODE] = self.catalog.cbets_dict[cbetnum]
if (kwargs[SOURCE.NAME].upper().startswith('IAUC')
and SOURCE.BIBCODE not in kwargs):
kwargs[SOURCE.NAME] = kwargs[SOURCE.NAME].replace(
'IAUC', 'IAUC ')
kwargs[SOURCE.NAME] = ' '.join(kwargs[SOURCE.NAME].split())
iaucnum = kwargs[SOURCE.NAME].split()[-1]
if is_number(iaucnum) and iaucnum in self.catalog.iaucs_dict:
kwargs[SOURCE.BIBCODE] = self.catalog.iaucs_dict[iaucnum]
for rep in self.catalog.source_syns:
if kwargs[SOURCE.NAME] in self.catalog.source_syns[rep]:
kwargs[SOURCE.NAME] = rep
break
if SOURCE.URL in kwargs:
for rep in self.catalog.url_redirs:
if kwargs[SOURCE.URL] in self.catalog.url_redirs[rep]:
kwargs[SOURCE.URL] = rep
break
return super(Supernova, self).add_source(**kwargs)
def _clean_quantity(self, quantity):
"""Clean quantity value before it is added to entry."""
value = quantity.get(QUANTITY.VALUE, '').strip()
error = quantity.get(QUANTITY.E_VALUE, '').strip()
unit = quantity.get(QUANTITY.U_VALUE, '').strip()
kind = quantity.get(QUANTITY.KIND, '')
if isinstance(kind, list) and not isinstance(kind, string_types):
kind = [x.strip() for x in kind]
else:
kind = kind.strip()
if not value:
return False
if is_number(value):
value = '%g' % Decimal(value)
if error:
error = '%g' % Decimal(error)
if value:
quantity[QUANTITY.VALUE] = value
if error:
quantity[QUANTITY.E_VALUE] = error
if unit:
quantity[QUANTITY.U_VALUE] = unit
if kind:
quantity[QUANTITY.KIND] = kind
return True
def _clean_quantity(self, quantity):
"""Clean quantity value before it is added to entry.
"""
value = quantity.get(QUANTITY.VALUE, '').strip()
error = quantity.get(QUANTITY.E_VALUE, '').strip()
unit = quantity.get(QUANTITY.U_VALUE, '').strip()
kind = quantity.get(QUANTITY.KIND, '').strip()
if not value:
return False
if is_number(value):
value = '%g' % Decimal(value)
if error:
error = '%g' % Decimal(error)
if value:
quantity[QUANTITY.VALUE] = value
if error:
quantity[QUANTITY.E_VALUE] = error
if unit:
quantity[QUANTITY.U_VALUE] = unit
if kind:
quantity[QUANTITY.KIND] = kind
return True
def __init__(self, parent, **kwargs):
self._REQ_KEY_SETS = [[QUANTITY.VALUE], [QUANTITY.SOURCE]]
super().__init__(parent, **kwargs)
# Aliases not added if in DISTINCT_FROM
if self._key == parent._KEYS.ALIAS:
value = parent.catalog.clean_entry_name(self[QUANTITY.VALUE])
for df in parent.get(parent._KEYS.DISTINCT_FROM, []):
if value == df[QUANTITY.VALUE]:
raise CatDictError("Alias '{}' in '{}'\' '{}' list".format(
value, parent[parent._KEYS.NAME],
parent._KEYS.DISTINCT_FROM))
# Check that value exists
if (not self[QUANTITY.VALUE] or self[QUANTITY.VALUE] == '--'
or self[QUANTITY.VALUE] == '-'):
raise CatDictError(
"Value '{}' is empty, not adding to '{}'".format(
self[QUANTITY.VALUE], parent[parent._KEYS.NAME]))
if not parent._clean_quantity(self):
raise CatDictError(
"Value '{}' did not survive cleaning process, not adding to "
" '{}'.".format(self[QUANTITY.VALUE],
parent[parent._KEYS.NAME]))
# Check that quantity value matches type after cleaning
if (isinstance(self._key, Key) and self._key.type == KEY_TYPES.NUMERIC
and not is_number(self[QUANTITY.VALUE])):
raise CatDictError(
"Value '{}' is not numeric, not adding to '{}'".format(
self[QUANTITY.VALUE], parent[parent._KEYS.NAME]))
def check(self, val):
"""Make sure given value is consistent with this `Key` specification.
NOTE: if `type` is 'None', then `listable` also is *not* checked.
"""
# If there is no `type` requirement, everything is allowed
if self.type is None:
return True
is_list = isinstance(val, list)
# If lists are not allowed, and this is a list --> false
if not self.listable and is_list:
return False
# `is_number` already checks for either list or single value
if self.type == KEY_TYPES.NUMERIC and not is_number(val):
return False
elif self.type == KEY_TYPES.STRING:
# If its a list, check first element
if is_list:
if not isinstance(val[0], str):
return False
# Otherwise, check it
elif not isinstance(val, str):
return False
elif self.type == KEY_TYPES.BOOL:
if is_list and not isinstance(val[0], bool):
return False
elif not isinstance(val, bool):
return False
return True
def extra_aliases(self):
"""These aliases are considered when merging duplicates only, but are
not added to the list of aliases that would be included with the event
"""
if (self[SUPERNOVA.NAME].startswith('SN') and
is_number(self[SUPERNOVA.NAME][2:6])):
return ['AT' + self[SUPERNOVA.NAME][2:]]
return []
def extra_aliases(self):
"""These aliases are considered when merging duplicates only, but are
not added to the list of aliases that would be included with the event
"""
if (self[SUPERNOVA.NAME].startswith('SN')
and is_number(self[SUPERNOVA.NAME][2:6])):
return ['AT' + self[SUPERNOVA.NAME][2:]]
return []
def _clean_quantity(self, quantity):
"""Clean quantity value before it is added to entry."""
value = quantity.get(QUANTITY.VALUE, '').strip()
error = quantity.get(QUANTITY.E_VALUE, '').strip()
unit = quantity.get(QUANTITY.U_VALUE, '').strip()
kinds = [x.strip() for x in listify(quantity.get(QUANTITY.KIND, []))]
key = quantity._key
if not value:
return False
if error and (not is_number(error) or float(error) < 0):
raise ValueError(self[self._KEYS.NAME] + "'s quanta " + key +
' error value must be a number and positive.')
# Set default units
if not unit and key == self._KEYS.VELOCITY:
unit = 'km/s'
if not unit and key == self._KEYS.RA:
unit = 'hours'
if not unit and key == self._KEYS.DEC:
unit = 'degrees'
if not unit and key in [self._KEYS.LUM_DIST, self._KEYS.COMOVING_DIST]:
unit = 'Mpc'
if is_number(value):
value = '%g' % Decimal(value)
if error:
error = '%g' % Decimal(error)
if value:
quantity[QUANTITY.VALUE] = value
if error:
quantity[QUANTITY.E_VALUE] = error
if unit:
quantity[QUANTITY.U_VALUE] = unit
if kinds:
quantity[QUANTITY.KIND] = kinds if len(kinds) > 1 else kinds[0]
elif QUANTITY.KIND in quantity:
del (quantity[QUANTITY.KIND])
return True
def get_rep_folder(entry):
if 'discoverdate' not in entry:
return repofolders[0]
if not is_number(entry['discoverdate'][0]['value'].split('/')[0]):
warnings.warn('Discovery year is not a number')
return repofolders[0]
for r, repoyear in enumerate(repoyears):
if int(entry['discoverdate'][0]['value'].split('/')[0]) <= repoyear:
return repofolders[r]
return repofolders[0]
def get_rep_folder(entry, repofolders):
if 'discoverdate' not in entry:
return repofolders[0]
if not is_number(entry['discoverdate'][0]['value'].split('/')[0]):
warnings.warn('Discovery year is not a number')
return repofolders[0]
repoyears = get_rep_years(repofolders)
for r, repoyear in enumerate(repoyears):
if int(entry['discoverdate'][0]['value'].split('/')[0]) <= repoyear:
return repofolders[r]
return repofolders[0]
def do_hst(catalog):
task_str = catalog.get_current_task_str()
url = 'http://archive.stsci.edu/hst/search.php'
reference = 'Hubble Pointings'
jtxt = catalog.load_url(
url,
os.path.join(catalog.get_current_task_repo(), 'HST.json'),
post={
'sci_target_descrip': '*supernova*',
'outputformat': 'JSON_file',
'action': 'Search',
'max_records': '50000',
'max_rpp': '50000'
},
verify=False)
rows = json.loads(jtxt)
allowed_prefixes = ('PS1', 'DES', 'GAIA', 'ASASSN', 'AT', 'IPTF', 'LSQ',
'PTF')
loopcnt = 0
for row in pbar(rows, task_str):
oldname = name_clean(row['Target Name'])
if not oldname.upper().startswith(allowed_prefixes):
continue
if oldname.startswith('PS1-') and not is_number(oldname[4]):
continue
name, source = catalog.new_entry(oldname, srcname=reference, url=url)
if (ENTRY.RA in catalog.entries[name] and
ENTRY.DEC in catalog.entries[name]):
continue
catalog.entries[name].add_quantity(
ENTRY.RA, row['RA (J2000)'], source=source)
catalog.entries[name].add_quantity(
ENTRY.DEC, row['Dec (J2000)'], source=source)
catalog.journal_entries()
loopcnt = loopcnt + 1
if (catalog.args.travis and loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
def __init__(self, parent, **kwargs):
self._REQ_KEY_SETS = [
[QUANTITY.VALUE],
[QUANTITY.SOURCE]
]
super().__init__(parent, **kwargs)
# Aliases not added if in DISTINCT_FROM
if self._key == parent._KEYS.ALIAS:
value = parent.catalog.clean_entry_name(self[QUANTITY.VALUE])
for df in parent.get(parent._KEYS.DISTINCT_FROM, []):
if value == df[QUANTITY.VALUE]:
raise CatDictError(
"Alias '{}' in '{}'\' '{}' list".format(
value, parent[parent._KEYS.NAME],
parent._KEYS.DISTINCT_FROM))
# Check that value exists
if (not self[QUANTITY.VALUE] or self[QUANTITY.VALUE] == '--' or
self[QUANTITY.VALUE] == '-'):
raise CatDictError(
"Value '{}' is empty, not adding to '{}'".format(
self[QUANTITY.VALUE], parent[parent._KEYS.NAME]))
if not parent._clean_quantity(self):
raise CatDictError(
"Value '{}' did not survive cleaning process, not adding to "
" '{}'.".format(self[QUANTITY.VALUE],
parent[parent._KEYS.NAME]))
# Check that quantity value matches type after cleaning
if (isinstance(self._key, Key) and
self._key.type == KEY_TYPES.NUMERIC and not
is_number(self[QUANTITY.VALUE])):
raise CatDictError(
"Value '{}' is not numeric, not adding to '{}'".format(
self[QUANTITY.VALUE], parent[parent._KEYS.NAME]))
def retrieve_objects(self,
catalog_name,
event_name=None,
quantity_name=None,
attribute_name=None,
full=False):
"""Retrieve data, first trying catalog file then event files."""
event = None
use_full = full
search_all = False
ename = event_name
qname = quantity_name
aname = attribute_name
req_vals = request.get_json()
if not req_vals:
req_vals = request.values
# Load event/quantity/attribute if provided by request.
event_req = req_vals.get('event')
quantity_req = req_vals.get('quantity')
attribute_req = req_vals.get('attribute')
if ename is None and event_req is not None:
if not isinstance(event_req, string_types):
ename = '+'.join(listify(event_req))
else:
ename = event_req
if qname is None and quantity_req is not None:
if not isinstance(quantity_req, string_types):
qname = '+'.join(listify(quantity_req))
else:
qname = quantity_req
if aname is None and attribute_req is not None:
if not isinstance(attribute_req, string_types):
aname = '+'.join(listify(attribute_req))
else:
aname = attribute_req
if ename is None:
return msg('no_root_data')
# Options
if not use_full:
rfull = req_vals.get('full')
if rfull is not None:
return self.retrieve_objects(catalog_name,
event_name=ename,
quantity_name=qname,
attribute_name=aname,
full=True)
fmt = req_vals.get('format')
fmt = fmt.lower() if fmt is not None else fmt
fmt = None if fmt == 'json' else fmt
ra = req_vals.get('ra')
dec = req_vals.get('dec')
radius = req_vals.get('radius')
width = req_vals.get('width')
height = req_vals.get('height')
complete = req_vals.get('complete')
first = req_vals.get('first')
closest = req_vals.get('closest')
sortby = req_vals.get('sortby')
sortby = sortby.lower() if sortby is not None else sortby
include_keys = list(sorted(set(req_vals.keys()) - self._SPECIAL_ATTR))
includes = OrderedDict()
iincludes = OrderedDict()
for key in include_keys:
val = req_vals.get(key, '')
if not is_number(val) and val != '':
val = '^' + val + '$'
try:
includes[key] = re.compile(val)
iincludes[key] = re.compile(val, re.IGNORECASE)
except Exception:
return msg('invalid_regex', [req_vals.get(key, ''), key])
excludes = OrderedDict([('realization', '')])
if first is None:
item = req_vals.get('item')
try:
item = int(item)
except Exception:
item = None
else:
item = 0
if radius is not None:
try:
radius = float(radius)
except Exception:
radius = 0.0
if radius >= self._ANGLE_LIMIT:
return msg('radius_limited', self._ANGLE_LIMIT / 3600.)
if width is not None:
try:
width = float(width)
except Exception:
width = 0.0
if width >= self._ANGLE_LIMIT:
return msg('width_limited', self._ANGLE_LIMIT / 3600.)
if height is not None:
try:
height = float(height)
except Exception:
height = 0.0
if height >= self._ANGLE_LIMIT:
return msg('height_limited', self._ANGLE_LIMIT / 3600.)
if ename and ename.lower() in ['catalog', 'all']:
if ra is not None and dec is not None:
try:
ldec = str(dec).lower().strip(' .')
if is_number(ldec) or decregex.match(ldec):
sra = str(ra)
lra = sra.lower().replace('h', '').strip(' .')
if raregex.match(lra) or (is_number(lra)
and 'h' in sra):
lcoo = coord(lra,
ldec,
unit=(un.hourangle, un.deg))
elif is_number(lra):
lcoo = coord(lra, ldec, unit=(un.deg, un.deg))
else:
raise Exception
else:
raise Exception
except Exception:
return msg('bad_coordinates')
if (width is not None and height is not None and width > 0.0
and height > 0.0):
idxcat = np.where((
abs(lcoo.ra - apidata._coo.ra) <= width * un.arcsecond)
& (abs(lcoo.dec - apidata._coo.dec) <=
height * un.arcsecond))[0]
elif width is not None and width > 0.0:
idxcat = np.where(
abs(lcoo.ra - apidata._coo.ra) <= width *
un.arcsecond)[0]
elif height is not None and height > 0.0:
idxcat = np.where(
abs(lcoo.dec - apidata._coo.dec) <= height *
un.arcsecond)[0]
else:
if radius is None or radius == 0.0:
radius = 1.0
idxcat = np.where(
lcoo.separation(apidata._coo) <= radius *
un.arcsecond)[0]
if len(idxcat):
ename_arr = [
apidata._rdnames[i].replace('+', '$PLUS$')
for i in idxcat
]
else:
return msg('no_objects')
elif catalog_name in apidata._catalogs:
ename_arr = [
i.replace('+', '$PLUS$')
for i in apidata._catalogs[catalog_name]
]
search_all = True
else:
ename_arr = [
a for b in [[
i.replace('+', '$PLUS$')
for i in apidata._catalogs[cat]
] for cat in apidata._catalogs] for a in b
]
search_all = True
ename = '+'.join(list(sorted(set(ename_arr))))
if qname is None:
# Short circuit to full if keyword is present.
if full:
return self.retrieve_objects(catalog_name,
event_name=ename,
full=True)
search_all = True
if catalog_name not in apidata._CATS:
qname = '+'.join(
list(
set(
sorted([
a for b in [
apidata._cat_keys[x]
for x in apidata._cat_keys
] for a in b
]))))
else:
qname = '+'.join(
list(sorted(set(apidata._cat_keys[catalog_name]))))
# if fmt is not None and qname is None:
# return Response((
# 'Error: \'{}\' format only supported if quantity '
# 'is specified.').format(
# fmt), mimetype='text/plain')
# Events
event_names = [] if ename is None else ename.split('+')
# Check for + in names
nevent_names = []
joined = False
for ni, name in enumerate(event_names):
if joined:
joined = False
continue
if ni < len(event_names) - 1:
jname = '+'.join(event_names[ni:ni + 2])
if jname.lower().replace(' ', '') in apidata._aliases:
nevent_names.append(jname)
joined = True
continue
nevent_names.append(name)
event_names = nevent_names
event_names = [x.replace('$PLUS$', '+') for x in event_names]
if not len(event_names):
search_all = True
event_names = apidata._all
# Quantities
quantity_names = [] if qname is None else qname.split('+')
# Attributes. Always append source.
attribute_names = [] if aname is None else aname.split('+')
if use_full and len(event_names) > self._FULL_LIMIT:
return msg('max_events', self._FULL_LIMIT)
if fmt is not None and any(
[n in attribute_names for n in self._NO_CSV]):
return msg('no_delimited')
if len(event_names) > self._EXPENSIVE_LIMIT:
for quantity in quantity_names:
for exp in self._EXPENSIVE:
if any([e in attribute_names for e in self._EXPENSIVE]):
return msg('too_expensive')
edict = OrderedDict()
fcatalogs = OrderedDict()
sources = OrderedDict()
new_event_names = []
for event in event_names:
skip_entry = False
my_cat, my_event = None, None
alopts = apidata._aliases.get(event.lower().replace(' ', ''), [])
for opt in alopts:
if opt[0] == catalog_name:
my_cat, my_event, my_alias = tuple(opt)
break
if not my_cat:
for opt in alopts:
if opt[0] != catalog_name:
my_cat, my_event, my_alias = tuple(opt)
break
if not my_cat:
if len(event_names) == 1:
return msg('event_not_found', event)
continue
if full:
fcatalogs.update(
json.load(open(
os.path.join(apidata._AC_PATH,
apidata._CATS[my_cat][0], 'output',
'json', get_filename(my_event)), 'r'),
object_pairs_hook=OrderedDict))
sources[my_event] = [
x.get('bibcode', x.get('arxivid', x.get('name')))
for x in fcatalogs[my_event].get('sources')
]
if qname is None:
if full:
edict[event] = fcatalogs.get(my_event, {})
else:
edict[event] = apidata._catalogs.get(my_cat,
{}).get(my_event, {})
else:
# Check if user passed quantity or attribute names to filter
# by.
qdict = OrderedDict()
if full:
my_event_dict = fcatalogs.get(my_event, {})
else:
my_event_dict = apidata._catalogs.get(my_cat, {}).get(
my_event, {})
if aname is None:
for incl in iincludes:
incll = incl.lower()
if incll not in my_event_dict or (
iincludes[incl].pattern != '' and not any([
bool(iincludes[incl].match(
x.get('value', ''
) if isinstance(x, dict) else x))
for x in my_event_dict.get(incll, [{}])
])):
skip_entry = True
break
if not skip_entry:
for quantity in quantity_names:
my_quantity = listify(my_event_dict.get(quantity, {}))
closest_locs = []
if closest is not None:
closest_locs = list(
sorted(
list(
set([
np.argmin([
abs(
np.mean([
float(y)
for y in listify(
x.get(i))
]) -
float(includes[i].pattern))
for x in my_quantity
]) for i in includes
if len(my_quantity) and is_number(
includes[i].pattern) and all([
is_number(x.get(i, ''))
for x in my_quantity
])
]))))
if aname is None and quantity in my_event_dict:
qdict[quantity] = [
x for xi, x in enumerate(my_quantity)
if not len(closest_locs) or xi in closest_locs
]
if item is not None:
try:
qdict[quantity] = qdict[quantity][item]
except Exception:
pass
else:
qdict[quantity] = self.get_attributes(
attribute_names,
my_quantity,
complete=complete,
full=use_full,
item=item,
includes=includes,
iincludes=iincludes,
excludes=excludes,
closest_locs=closest_locs,
sources=np.array(sources.get(my_event, [])))
if not search_all and not qdict.get(quantity):
use_full = True
break
if not full and use_full:
new_event_names = list(event_names)
break
if qdict:
edict[event] = qdict
if not full and use_full:
break
if not (skip_entry and (full or search_all)) and event in edict:
new_event_names.append(event)
event_names = new_event_names
ename = '+'.join([i.replace('+', '$PLUS$') for i in event_names])
if not full and use_full:
return self.retrieve_objects(catalog_name,
event_name=ename,
quantity_name=qname,
attribute_name=aname,
full=True)
if fmt is not None:
return self.get_event_dsv(edict, event_names, quantity_names,
attribute_names, fmt, sortby)
return edict
def do_csp_fits_spectra(catalog):
from astropy.io import fits
fpath = catalog.get_current_task_repo()
fureps = {'erg/cm2/s/A': 'erg/s/cm^2/Angstrom'}
task_str = catalog.get_current_task_str()
dirs = [x[0] for x in os.walk(
os.path.join(fpath, 'Gutierrez_et_al_2017'))]
files = []
for dir in dirs:
files.extend(glob(os.path.join(dir, '*.fits')))
for datafile in pbar(files, task_str):
filename = datafile.split('/')[-1]
hdulist = fits.open(datafile)
for oi, obj in enumerate(hdulist[0].header):
if any(x in ['.', '/'] for x in obj):
del (hdulist[0].header[oi])
try:
hdulist[0].verify('silentfix')
except Exception as e:
print(e)
hdrkeys = list(hdulist[0].header.keys())
# print(hdrkeys)
name = datafile.split('/')[-2]
if name[2] in '6789':
name = 'SN19' + name[2:]
elif name != 'SN210':
name = 'SN20' + name[2:]
name, source = catalog.new_entry(name, bibcode='2017ApJ...850...89G')
# for key in hdulist[0].header.keys():
# print(key, hdulist[0].header[key])
mjd = None
if hdulist[0].header['SIMPLE']:
if 'JD' in hdrkeys:
mjd = str(jd_to_mjd(Decimal(str(hdulist[0].header['JD']))))
elif 'MJD' in hdrkeys:
mjd = str(hdulist[0].header['MJD'])
elif 'DATE-OBS' in hdrkeys or 'DATE' in hdrkeys:
dkey = 'DATE-OBS' if 'DATE-OBS' in hdrkeys else 'DATE'
dval = hdulist[0].header[dkey]
if is_number(dval):
dkey = 'DATE' if dkey == 'DATE-OBS' else 'DATE-OBS'
dval = hdulist[0].header[dkey]
dateobs = None
if 'T' in dval:
dateobs = dval.strip()
elif 'UTC-OBS' in hdrkeys:
dateobs = dval.strip(
) + 'T' + hdulist[0].header['UTC-OBS'].strip()
if dateobs is not None:
mjd = str(astrotime(dateobs, format='isot').mjd)
# print(hdulist[0].header)
if 'CRVAL1' in hdulist[0].header:
w0 = hdulist[0].header['CRVAL1']
elif hdulist[0].header['CTYPE1'] == 'MULTISPE':
w0 = float(hdulist[0].header['WAT2_001'].split(
'"')[-1].split()[3])
else:
raise ValueError('Unsupported spectrum format.')
if hdulist[0].header['NAXIS'] == 1:
wd = hdulist[0].header['CDELT1']
fluxes = [str(x) for x in list(hdulist[0].data)]
errors = False
elif hdulist[0].header['NAXIS'] == 3:
wd = hdulist[0].header['CD1_1']
fluxes = [str(x) for x in list(hdulist[0].data)[0][0]]
errors = [str(x) for x in list(hdulist[0].data)[-1][0]]
else:
print('Warning: Skipping FITS spectrum `{}`.'.format(filename))
continue
waves = [str(w0 + wd * x) for x in range(0, len(fluxes))]
else:
raise ValueError('Non-simple FITS import not yet supported.')
if 'BUNIT' in hdrkeys:
fluxunit = hdulist[0].header['BUNIT']
if fluxunit in fureps:
fluxunit = fureps[fluxunit]
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.WAVELENGTHS: waves,
SPECTRUM.FLUXES: fluxes,
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.FILENAME: filename,
SPECTRUM.SOURCE: source
}
if mjd is not None:
specdict[SPECTRUM.TIME] = mjd
specdict[SPECTRUM.U_TIME] = 'MJD'
if 'TELESCOP' in hdrkeys:
specdict[SPECTRUM.TELESCOPE] = hdulist[0].header['TELESCOP']
if 'INSTRUME' in hdrkeys:
specdict[SPECTRUM.INSTRUMENT] = hdulist[0].header['INSTRUME']
if 'AIRMASS' in hdrkeys:
specdict[SPECTRUM.AIRMASS] = hdulist[0].header['AIRMASS']
if errors:
specdict[SPECTRUM.ERRORS] = errors
specdict[SPECTRUM.U_ERRORS] = fluxunit
if 'SITENAME' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['SITENAME']
elif 'OBSERVAT' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['OBSERVAT']
if 'OBSERVER' in hdrkeys:
specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER']
catalog.entries[name].add_spectrum(**specdict)
hdulist.close()
catalog.journal_entries()
return
def set_preferred_name(self):
"""Set preferred name of supernova.
Highest preference goes to names of the form 'SN####AA'.
Otherwise base the name on whichever survey is the 'discoverer'.
FIX: create function to match SN####AA type names.
"""
name = self[self._KEYS.NAME]
newname = ''
aliases = self.get_aliases()
# if there are no other options to choose from, skip
if len(aliases) <= 1:
return name
# If the name is already in the form 'SN####AA' then keep using
# that
if (name.startswith('SN') and
((is_number(name[2:6]) and not is_number(name[6:])) or
(is_number(name[2:5]) and not is_number(name[5:])))):
return name
# If one of the aliases is in the form 'SN####AA' then use that
for alias in aliases:
if (alias.startswith('SN') and
((is_number(alias[2:6]) and not is_number(alias[6:])) or
(is_number(alias[2:5]) and not is_number(alias[5:])))):
newname = alias
break
# If not, name based on the 'discoverer' survey
if not newname and SUPERNOVA.DISCOVERER in self:
discoverer = ','.join(
[x['value'].upper() for x in self[SUPERNOVA.DISCOVERER]])
if 'ASAS' in discoverer:
for alias in aliases:
if 'ASASSN' in alias.upper():
newname = alias
break
if not newname and 'OGLE' in discoverer:
for alias in aliases:
if 'OGLE' in alias.upper():
newname = alias
break
if not newname and 'CRTS' in discoverer:
for alias in aliases:
if True in [
x in alias.upper()
for x in ['CSS', 'MLS', 'SSS', 'SNHUNT']
]:
newname = alias
break
if not newname and 'PS1' in discoverer:
for alias in aliases:
if 'PS1' in alias.upper():
newname = alias
break
if not newname and 'PTF' in discoverer:
for alias in aliases:
if 'PTF' in alias.upper():
newname = alias
break
if not newname and 'la silla-quest' in discoverer.lower():
for alias in aliases:
if 'LSQ' in alias.upper():
newname = alias
break
if not newname and 'GAIA' in discoverer:
for alias in aliases:
if 'GAIA' in alias.upper():
newname = alias
break
# If one of the aliases is in the form 'AT####AA' then use that
if not newname:
for alias in aliases:
if (alias.startswith('AT') and
((is_number(alias[2:6]) and not is_number(alias[6:])) or
(is_number(alias[2:5]) and not is_number(alias[5:])))):
newname = alias
break
# Otherwise, use the shortest name.
if not newname:
newname = min(aliases, key=len)
# Always prefer another alias over PSN
if not newname and name.startswith('PSN'):
for alias in aliases:
if not alias.startswith('PSN'):
newname = alias
if newname and name != newname:
file_entry = None
# Make sure new name doesn't already exist
if newname in self.catalog.entries:
if self.catalog.entries[newname]._stub:
file_entry = self.init_from_file(
self.catalog, name=newname)
else:
file_entry = self.catalog.entries[newname]
if file_entry:
self._log.info("`{}` already exists, copying `{}` to it".
format(newname, name))
self.catalog.copy_entry_to_entry(
self.catalog.entries[name], file_entry)
self.catalog.entries[newname] = file_entry
else:
self._log.info("Changing entry from name '{}' to preferred"
" name '{}'".format(name, newname))
self.catalog.entries[newname] = self.catalog.entries[name]
self.catalog.entries[newname][self._KEYS.NAME] = newname
del self.catalog.entries[name]
return newname
return name
def merge_duplicates(self):
"""Merge and remove duplicate entries.
Compares each entry ('name') in `stubs` to all later entries to check
for duplicates in name or alias. If a duplicate is found, they are
merged and written to file.
"""
if len(self.entries) == 0:
self.log.error("WARNING: `entries` is empty, loading stubs")
if self.args.update:
self.log.warning(
"No sources changed, entry files unchanged in update."
" Skipping merge.")
return
self.entries = self.load_stubs()
task_str = self.get_current_task_str()
keys = list(sorted(self.entries.keys()))
for n1, name1 in enumerate(pbar(keys, task_str)):
allnames1 = set(self.entries[name1].get_aliases())
if name1.startswith('SN') and is_number(name1[2:6]):
allnames1 = allnames1.union(['AT' + name1[2:]])
# Search all later names
for name2 in keys[n1 + 1:]:
if name1 == name2:
continue
allnames2 = set(self.entries[name2].get_aliases())
if name2.startswith('SN') and is_number(name2[2:6]):
allnames2.union(['AT' + name2[2:]])
# If there are any common names or aliases, merge
if len(allnames1 & allnames2):
self.log.warning(
"Found single entry with multiple entries "
"('{}' and '{}'), merging.".format(name1, name2))
load1 = self.proto.init_from_file(
self, name=name1)
load2 = self.proto.init_from_file(
self, name=name2)
if load1 is not None and load2 is not None:
# Delete old files
self._delete_entry_file(entry=load1)
self._delete_entry_file(entry=load2)
self.entries[name1] = load1
self.entries[name2] = load2
priority1 = 0
priority2 = 0
for an in allnames1:
if an.startswith(('SN', 'AT')):
priority1 += 1
for an in allnames2:
if an.startswith(('SN', 'AT')):
priority2 += 1
if priority1 > priority2:
self.copy_to_entry(name2, name1)
keys.append(name1)
del self.entries[name2]
else:
self.copy_to_entry(name1, name2)
keys.append(name2)
del self.entries[name1]
else:
self.log.warning('Duplicate already deleted')
# if len(self.entries) != 1:
# self.log.error(
# "WARNING: len(entries) = {}, expected 1. "
# "Still journaling...".format(len(self.entries)))
self.journal_entries()
if self.args.travis and n1 > self.TRAVIS_QUERY_LIMIT:
break
def do_suspect_spectra(catalog):
task_str = catalog.get_current_task_str()
with open(
os.path.join(catalog.get_current_task_repo(),
'Suspect/sources.json'), 'r') as f:
sourcedict = json.loads(f.read())
with open(
os.path.join(catalog.get_current_task_repo(),
'Suspect/filename-changes.txt'), 'r') as f:
rows = f.readlines()
changedict = {}
for row in rows:
if not row.strip() or row[0] == "#":
continue
items = row.strip().split(' ')
changedict[items[1]] = items[0]
suspectcnt = 0
folders = next(
os.walk(os.path.join(catalog.get_current_task_repo(), 'Suspect')))[1]
for folder in pbar(folders, task_str):
eventfolders = next(
os.walk(
os.path.join(catalog.get_current_task_repo(), 'Suspect/') +
folder))[1]
oldname = ''
for eventfolder in pbar(eventfolders, task_str):
name = eventfolder
if is_number(name[:4]):
name = 'SN' + name
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
sec_ref = 'SUSPECT'
sec_refurl = 'https://www.nhn.ou.edu/~suspect/'
sec_bibc = '2001AAS...199.8408R'
sec_source = catalog.entries[name].add_source(name=sec_ref,
url=sec_refurl,
bibcode=sec_bibc,
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name,
sec_source)
fpath = os.path.join(catalog.get_current_task_repo(), 'Suspect',
folder, eventfolder)
eventspectra = next(os.walk(fpath))[2]
for spectrum in eventspectra:
sources = [sec_source]
bibcode = ''
if spectrum in changedict:
specalias = changedict[spectrum]
else:
specalias = spectrum
if specalias in sourcedict:
bibcode = sourcedict[specalias]
elif name in sourcedict:
bibcode = sourcedict[name]
if bibcode:
source = catalog.entries[name].add_source(
bibcode=unescape(bibcode))
sources += [source]
sources = uniq_cdl(sources)
date = spectrum.split('_')[1]
year = date[:4]
month = date[4:6]
day = date[6:]
sig = get_sig_digits(day) + 5
day_fmt = str(floor(float(day))).zfill(2)
time = astrotime(year + '-' + month + '-' + day_fmt).mjd
time = time + float(day) - floor(float(day))
time = pretty_num(time, sig=sig)
fpath = os.path.join(catalog.get_current_task_repo(),
'Suspect', folder, eventfolder, spectrum)
with open(fpath, 'r') as f:
specdata = list(
csv.reader(f, delimiter=' ', skipinitialspace=True))
specdata = list(filter(None, specdata))
newspec = []
oldval = ''
for row in specdata:
if row[1] == oldval:
continue
newspec.append(row)
oldval = row[1]
specdata = newspec
haserrors = len(
specdata[0]
) == 3 and specdata[0][2] and specdata[0][2] != 'NaN'
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
catalog.entries[name].add_spectrum(u_wavelengths='Angstrom',
u_fluxes='Uncalibrated',
u_time='MJD',
time=time,
wavelengths=wavelengths,
fluxes=fluxes,
errors=errors,
u_errors='Uncalibrated',
source=sources,
filename=spectrum)
suspectcnt = suspectcnt + 1
if (catalog.args.travis
and suspectcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
def _clean_quantity(self, quantity):
"""Clean quantity value before it is added to entry."""
value = quantity.get(QUANTITY.VALUE, '').strip()
error = quantity.get(QUANTITY.E_VALUE, '').strip()
unit = quantity.get(QUANTITY.U_VALUE, '').strip()
kinds = [x.strip() for x in listify(quantity.get(QUANTITY.KIND, []))]
key = quantity._key
if not value:
return False
if error and (not is_number(error) or float(error) < 0):
raise ValueError(self[self._KEYS.NAME] + "'s quanta " + key +
' error value must be a number and positive.')
# Set default units
if not unit and key == self._KEYS.VELOCITY:
unit = 'km/s'
if not unit and key == self._KEYS.RA:
unit = 'hours'
if not unit and key == self._KEYS.DEC:
unit = 'degrees'
if not unit and key in [self._KEYS.LUM_DIST, self._KEYS.COMOVING_DIST]:
unit = 'Mpc'
# Handle certain name
if key == self._KEYS.ALIAS:
value = self.catalog.clean_entry_name(value)
for df in quantity.get(self._KEYS.DISTINCT_FROM, []):
if value == df[QUANTITY.VALUE]:
return False
elif key == self._KEYS.HOST:
if is_number(value):
return False
if value.lower() in [
'anonymous', 'anon.', 'anon', 'intergalactic'
]:
return False
value = host_clean(value)
if ((not kinds and ((value.lower().startswith('abell') and
is_number(value[5:].strip())) or
'cluster' in value.lower()))):
kinds = ['cluster']
elif key == self._KEYS.HOST_REDSHIFT:
kinds = list(filter(lambda x: x != 'host', kinds))
elif key == self._KEYS.CLAIMED_TYPE:
isq = False
if value.startswith('SN '):
value = value.replace('SN ', '', 1)
value = value.replace('young', '')
if '?' in value:
isq = True
value = value.strip(' ?')
for rep in self.catalog.type_syns:
if value in self.catalog.type_syns[rep]:
value = rep
break
if isq:
value = value + '?'
if not value:
return False
elif key in [
self._KEYS.RA, self._KEYS.DEC, self._KEYS.HOST_RA,
self._KEYS.HOST_DEC
]:
(value, unit) = radec_clean(value, key, unit=unit)
elif key == self._KEYS.MAX_DATE or key == self._KEYS.DISCOVER_DATE:
# Make sure month and day have leading zeroes
sparts = value.split('/')
if len(sparts[0]) > 5:
self._log.warn("Date year {} greater than four "
"digits.".format(sparts[0]))
if len(sparts) >= 2:
value = sparts[0] + '/' + sparts[1].zfill(2)
if len(sparts) == 3:
value = value + '/' + sparts[2].zfill(2)
# for ii, ct in enumerate(self.parent[key]):
# # Only add dates if they have more information
# if len(ct[QUANTITY.VALUE].split('/')) >
# len(value.split('/')):
# return False
if is_number(value):
value = '%g' % Decimal(value)
if error:
error = '%g' % Decimal(error)
if value:
quantity[QUANTITY.VALUE] = value
if error:
quantity[QUANTITY.E_VALUE] = error
if unit:
quantity[QUANTITY.U_VALUE] = unit
if kinds:
quantity[QUANTITY.KIND] = kinds if len(kinds) > 1 else kinds[0]
elif QUANTITY.KIND in quantity:
del (quantity[QUANTITY.KIND])
return True
def do_cpcs(catalog):
task_str = catalog.get_current_task_str()
cpcs_url = ('http://gsaweb.ast.cam.ac.uk/'
'followup/list_of_alerts?format=json&num=100000&'
'published=1&observed_only=1&'
'hashtag=JG_530ad9462a0b8785bfb385614bf178c6')
jsontxt = catalog.load_cached_url(
cpcs_url, os.path.join(catalog.get_current_task_repo(),
'CPCS/index.json'))
if not jsontxt:
return
alertindex = json.loads(jsontxt, object_pairs_hook=OrderedDict)
ids = [xx['id'] for xx in alertindex]
for ii, ai in enumerate(pbar(ids, task_str)):
name = alertindex[ii]['ivorn'].split('/')[-1].strip()
# Skip aa few weird entries
if name == 'ASASSNli':
continue
# Just use aa whitelist for now since naming seems inconsistent
white_list = ['GAIA', 'OGLE', 'ASASSN', 'MASTER', 'OTJ', 'PS1', 'IPTF']
if True in [xx in name.upper() for xx in white_list]:
name = name.replace('Verif', '').replace('_', ' ')
if 'ASASSN' in name and name[6] != '-':
name = 'ASASSN-' + name[6:]
if 'MASTEROTJ' in name:
name = name.replace('MASTEROTJ', 'MASTER OT J')
if 'OTJ' in name:
name = name.replace('OTJ', 'MASTER OT J')
if name.upper().startswith('IPTF'):
name = 'iPTF' + name[4:]
# Only add events that are classified as SN.
if catalog.entry_exists(name):
continue
oldname = name
name = catalog.add_entry(name)
else:
continue
sec_source = catalog.entries[name].add_source(
name='Cambridge Photometric Calibration Server',
url='http://gsaweb.ast.cam.ac.uk/followup/', secondary=True)
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, oldname, sec_source)
unit_deg = 'floatdegrees'
catalog.entries[name].add_quantity(
SUPERNOVA.RA, str(alertindex[ii][SUPERNOVA.RA]), sec_source,
u_value=unit_deg)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, str(
alertindex[ii][SUPERNOVA.DEC]), sec_source, u_value=unit_deg)
alerturl = ('http://gsaweb.ast.cam.ac.uk/'
'followup/get_alert_lc_data?alert_id=' +
str(ai))
source = catalog.entries[name].add_source(
name='CPCS Alert ' + str(ai), url=alerturl)
fname = os.path.join(catalog.get_current_task_repo(),
'CPCS/alert-') + str(ai).zfill(2) + '.json'
if (catalog.current_task.load_archive(catalog.args) and
os.path.isfile(fname)):
with open(fname, 'r') as ff:
jsonstr = ff.read()
else:
session = requests.Session()
response = session.get(
alerturl + '&hashtag=JG_530ad9462a0b8785bfb385614bf178c6')
with open(fname, 'w') as ff:
jsonstr = response.text
ff.write(jsonstr)
try:
cpcsalert = json.loads(jsonstr)
except:
continue
mjds = [round_sig(xx, sig=9) for xx in cpcsalert['mjd']]
mags = [round_sig(xx, sig=6) for xx in cpcsalert['mag']]
errs = [round_sig(xx, sig=6) if (is_number(xx) and float(xx) > 0.0)
else '' for xx in cpcsalert['magerr']]
bnds = cpcsalert['filter']
obs = cpcsalert['observatory']
for mi, mjd in enumerate(mjds):
(catalog.entries[name]
.add_photometry(time=mjd, magnitude=mags[mi],
e_magnitude=errs[mi],
band=bnds[mi], observatory=obs[mi],
source=uniq_cdl([source, sec_source])))
if catalog.args.update:
catalog.journal_entries()
catalog.journal_entries()
return
def do_cfa_spectra(catalog):
"""Import spectra from the CfA archive."""
task_str = catalog.get_current_task_str()
# II spectra
oldname = ''
file_names = next(
os.walk(os.path.join(catalog.get_current_task_repo(), 'CfA_SNII')))[1]
for ni, name in enumerate(pbar_strings(file_names, task_str)):
fullpath = os.path.join(catalog.get_current_task_repo(),
'CfA_SNII/') + name
origname = name
if name.startswith('sn') and is_number(name[2:6]):
name = 'SN' + name[2:]
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
reference = 'CfA Supernova Archive'
refurl = 'https://www.cfa.harvard.edu/supernova/SNarchive.html'
source = catalog.entries[name].add_source(
name=reference,
url=refurl,
secondary=True,
acknowledgment=ACKN_CFA)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
for fi, fname in enumerate(
sorted(
glob(fullpath + '/*'), key=lambda s: s.lower())):
filename = os.path.basename(fname)
fileparts = filename.split('-')
if origname.startswith('sn') and is_number(origname[2:6]):
year = fileparts[1][:4]
month = fileparts[1][4:6]
day = fileparts[1][6:]
instrument = fileparts[2].split('.')[0]
else:
year = fileparts[2][:4]
month = fileparts[2][4:6]
day = fileparts[2][6:]
instrument = fileparts[3].split('.')[0]
time = str(
astrotime(year + '-' + month + '-' + str(floor(float(day)))
.zfill(2)).mjd + float(day) - floor(float(day)))
f = open(fname, 'r')
data = csv.reader(f, delimiter=' ', skipinitialspace=True)
data = [list(i) for i in zip(*data)]
wavelengths = data[0]
fluxes = data[1]
errors = data[2]
sources = uniq_cdl([
source,
(catalog.entries[name]
.add_source(bibcode='2017arXiv170601030H'))
])
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom',
u_fluxes='erg/s/cm^2/Angstrom',
filename=filename,
wavelengths=wavelengths,
fluxes=fluxes,
u_time='MJD' if time else '',
time=time,
instrument=instrument,
u_errors='ergs/s/cm^2/Angstrom',
errors=errors,
source=sources,
dereddened=False,
deredshifted=False)
if catalog.args.travis and ni >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
# Ia spectra
oldname = ''
file_names = next(
os.walk(os.path.join(catalog.get_current_task_repo(), 'CfA_SNIa')))[1]
for ni, name in enumerate(pbar_strings(file_names, task_str)):
fullpath = os.path.join(catalog.get_current_task_repo(),
'CfA_SNIa/') + name
origname = name
if name.startswith('sn') and is_number(name[2:6]):
name = 'SN' + name[2:]
if name.startswith('snf') and is_number(name[3:7]):
name = 'SNF' + name[3:]
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
reference = 'CfA Supernova Archive'
refurl = 'https://www.cfa.harvard.edu/supernova/SNarchive.html'
source = catalog.entries[name].add_source(
name=reference,
url=refurl,
secondary=True,
acknowledgment=ACKN_CFA)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
for fi, fname in enumerate(
sorted(
glob(fullpath + '/*'), key=lambda s: s.lower())):
filename = os.path.basename(fname)
fileparts = filename.split('-')
if origname.startswith('sn') and is_number(origname[2:6]):
year = fileparts[1][:4]
month = fileparts[1][4:6]
day = fileparts[1][6:]
instrument = fileparts[2].split('.')[0]
else:
year = fileparts[2][:4]
month = fileparts[2][4:6]
day = fileparts[2][6:]
instrument = fileparts[3].split('.')[0]
time = str(
astrotime(year + '-' + month + '-' + str(floor(float(day)))
.zfill(2)).mjd + float(day) - floor(float(day)))
f = open(fname, 'r')
data = csv.reader(f, delimiter=' ', skipinitialspace=True)
data = [list(i) for i in zip(*data)]
wavelengths = data[0]
fluxes = data[1]
errors = data[2]
sources = uniq_cdl([
source, (catalog.entries[name]
.add_source(bibcode='2012AJ....143..126B')),
(catalog.entries[name]
.add_source(bibcode='2008AJ....135.1598M'))
])
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom',
u_fluxes='erg/s/cm^2/Angstrom',
filename=filename,
wavelengths=wavelengths,
fluxes=fluxes,
u_time='MJD' if time else '',
time=time,
instrument=instrument,
u_errors='ergs/s/cm^2/Angstrom',
errors=errors,
source=sources,
dereddened=False,
deredshifted=False)
if catalog.args.travis and ni >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
# Ibc spectra
oldname = ''
file_names = next(
os.walk(os.path.join(catalog.get_current_task_repo(), 'CfA_SNIbc')))[1]
for ni, name in enumerate(pbar(file_names, task_str)):
fullpath = os.path.join(catalog.get_current_task_repo(),
'CfA_SNIbc/') + name
if name.startswith('sn') and is_number(name[2:6]):
name = 'SN' + name[2:]
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
reference = 'CfA Supernova Archive'
refurl = 'https://www.cfa.harvard.edu/supernova/SNarchive.html'
source = catalog.entries[name].add_source(
name=reference,
url=refurl,
secondary=True,
acknowledgment=ACKN_CFA)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
for fi, fname in enumerate(
sorted(
glob(fullpath + '/*'), key=lambda s: s.lower())):
filename = os.path.basename(fname)
fileparts = filename.split('-')
instrument = ''
year = fileparts[1][:4]
month = fileparts[1][4:6]
day = fileparts[1][6:].split('.')[0]
if len(fileparts) > 2:
instrument = fileparts[-1].split('.')[0]
time = str(
astrotime(year + '-' + month + '-' + str(floor(float(day)))
.zfill(2)).mjd + float(day) - floor(float(day)))
f = open(fname, 'r')
data = csv.reader(f, delimiter=' ', skipinitialspace=True)
data = [list(i) for i in zip(*data)]
wavelengths = data[0]
fluxes = data[1]
sources = uniq_cdl([
source, catalog.entries[name]
.add_source(bibcode='2014AJ....147...99M')
])
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom',
u_fluxes='erg/s/cm^2/Angstrom',
wavelengths=wavelengths,
filename=filename,
fluxes=fluxes,
u_time='MJD' if time else '',
time=time,
instrument=instrument,
source=sources,
dereddened=False,
deredshifted=False)
if catalog.args.travis and ni >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
# Other spectra
oldname = ''
file_names = next(
os.walk(os.path.join(catalog.get_current_task_repo(), 'CfA_Extra')))[1]
for ni, name in enumerate(pbar_strings(file_names, task_str)):
fullpath = os.path.join(catalog.get_current_task_repo(),
'CfA_Extra/') + name
if name.startswith('sn') and is_number(name[2:6]):
name = 'SN' + name[2:]
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
reference = 'CfA Supernova Archive'
refurl = 'https://www.cfa.harvard.edu/supernova/SNarchive.html'
source = catalog.entries[name].add_source(
name=reference,
url=refurl,
secondary=True,
acknowledgment=ACKN_CFA)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
for fi, fname in enumerate(
sorted(
glob(fullpath + '/*'), key=lambda s: s.lower())):
if not os.path.isfile(fname):
continue
filename = os.path.basename(fname)
if ((not filename.startswith('sn') or
not filename.endswith('flm') or any(
x in filename
for x in ['-interp', '-z', '-dered', '-obj', '-gal']))):
continue
fileparts = filename.split('.')[0].split('-')
instrument = ''
time = ''
if len(fileparts) > 1:
year = fileparts[1][:4]
month = fileparts[1][4:6]
day = fileparts[1][6:]
if is_number(year) and is_number(month) and is_number(day):
if len(fileparts) > 2:
instrument = fileparts[-1]
time = str(
astrotime(year + '-' + month + '-' + str(
floor(float(day))).zfill(2)).mjd + float(day) -
floor(float(day)))
f = open(fname, 'r')
data = csv.reader(f, delimiter=' ', skipinitialspace=True)
data = [list(i) for i in zip(*data)]
wavelengths = data[0]
fluxes = [str(Decimal(x) * Decimal(1.0e-15)) for x in data[1]]
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom',
u_fluxes='erg/s/cm^2/Angstrom',
wavelengths=wavelengths,
filename=filename,
fluxes=fluxes,
u_time='MJD' if time else '',
time=time,
instrument=instrument,
source=source,
dereddened=False,
deredshifted=False)
if catalog.args.travis and ni >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
return
def host_clean(name):
newname = name.strip(' ;,*')
# Handle some special cases
hostcases = {'M051a': 'M51A', 'M051b': 'M51B'}
for k in hostcases:
if newname == k:
newname = hostcases[k]
# Some general cases
newname = newname.strip("()").replace(' ', ' ', 1)
newname = newname.replace("ABELL", "Abell", 1)
newname = newname.replace("Abell", "Abell ", 1)
newname = newname.replace("APMUKS(BJ)", "APMUKS(BJ) ", 1)
newname = newname.replace("ARP", "ARP ", 1)
newname = newname.replace("CGCG", "CGCG ", 1)
newname = newname.replace("HOLM", "HOLM ", 1)
newname = newname.replace("IC", "IC ", 1)
newname = newname.replace("Intergal.", "Intergalactic", 1)
newname = newname.replace("MCG+", "MCG +", 1)
newname = newname.replace("MCG-", "MCG -", 1)
newname = newname.replace("M+", "MCG +", 1)
newname = newname.replace("M-", "MCG -", 1)
newname = newname.replace("MGC ", "MCG ", 1)
newname = newname.replace("Mrk", "MRK", 1)
newname = newname.replace("MRK", "MRK ", 1)
newname = newname.replace("NGC", "NGC ", 1)
newname = newname.replace("PGC", "PGC ", 1)
newname = newname.replace("SDSS", "SDSS ", 1)
newname = newname.replace("UGC", "UGC ", 1)
if newname.startswith('MESSIER '):
newname = newname.replace('MESSIER ', 'M', 1)
if newname.startswith('M ') and is_number(newname[2:]):
newname = newname.replace('M ', 'M', 1)
if newname.startswith('M') and is_number(newname[1:]):
newname = 'M' + newname[1:].lstrip(" 0")
if len(newname) > 4 and newname.startswith("PGC "):
newname = newname[:4] + newname[4:].lstrip(" 0")
if len(newname) > 4 and newname.startswith("UGC "):
newname = newname[:4] + newname[4:].lstrip(" 0")
if len(newname) > 5 and newname.startswith(("MCG +", "MCG -")):
newname = newname[:5] + '-'.join([x.zfill(2)
for x in
newname[5:].strip().split("-")])
if len(newname) > 5 and newname.startswith("CGCG "):
newname = newname[:5] + '-'.join([x.zfill(3)
for x in
newname[5:].strip().split("-")])
if ((len(newname) > 1 and newname.startswith("E")) or
(len(newname) > 3 and newname.startswith('ESO'))):
if newname[0] == "E":
esplit = newname[1:].split("-")
else:
esplit = newname[3:].split("-")
if len(esplit) == 2 and is_number(esplit[0].strip()):
if esplit[1].strip()[0] == 'G':
parttwo = esplit[1][1:].strip()
else:
parttwo = esplit[1].strip()
if is_number(parttwo.strip()):
newname = 'ESO ' + \
esplit[0].lstrip('0') + '-G' + parttwo.lstrip('0')
newname = ' '.join(newname.split())
return newname
def set_preferred_name(self):
"""Highest preference goes to names of the form 'SN####AA'.
Otherwise base the name on whichever survey is the 'discoverer'.
FIX: create function to match SN####AA type names.
"""
name = self[self._KEYS.NAME]
newname = ''
aliases = self.get_aliases()
# if there are no other options to choose from, skip
if len(aliases) <= 1:
return name
# If the name is already in the form 'SN####AA' then keep using
# that
if (name.startswith('SN')
and ((is_number(name[2:6]) and not is_number(name[6:])) or
(is_number(name[2:5]) and not is_number(name[5:])))):
return name
# If one of the aliases is in the form 'SN####AA' then use that
for alias in aliases:
if (alias.startswith('SN') and
((is_number(alias[2:6]) and not is_number(alias[6:])) or
(is_number(alias[2:5]) and not is_number(alias[5:])))):
newname = alias
break
# Otherwise, name based on the 'discoverer' survey
if not newname and SUPERNOVA.DISCOVERER in self:
discoverer = ','.join(
[x['value'].upper() for x in self[SUPERNOVA.DISCOVERER]])
if 'ASAS' in discoverer:
for alias in aliases:
if 'ASASSN' in alias.upper():
newname = alias
break
if not newname and 'OGLE' in discoverer:
for alias in aliases:
if 'OGLE' in alias.upper():
newname = alias
break
if not newname and 'CRTS' in discoverer:
for alias in aliases:
if True in [
x in alias.upper()
for x in ['CSS', 'MLS', 'SSS', 'SNHUNT']
]:
newname = alias
break
if not newname and 'PS1' in discoverer:
for alias in aliases:
if 'PS1' in alias.upper():
newname = alias
break
if not newname and 'PTF' in discoverer:
for alias in aliases:
if 'PTF' in alias.upper():
newname = alias
break
if not newname and 'GAIA' in discoverer:
for alias in aliases:
if 'GAIA' in alias.upper():
newname = alias
break
# Always prefer another alias over PSN
if not newname and name.startswith('PSN'):
newname = aliases[0]
if newname and name != newname:
file_entry = None
# Make sure new name doesn't already exist
if newname in self.catalog.entries:
if self.catalog.entries[newname]._stub:
file_entry = self.init_from_file(self.catalog,
name=newname)
else:
file_entry = self.catalog.entries[newname]
if file_entry:
self._log.info(
"`{}` already exists, copying `{}` to it".format(
newname, name))
self.catalog.copy_entry_to_entry(self.catalog.entries[name],
file_entry)
self.catalog.entries[newname] = file_entry
else:
self._log.info("Changing entry from name '{}' to preferred"
" name '{}'".format(name, newname))
self.catalog.entries[newname] = self.catalog.entries[name]
self.catalog.entries[newname][self._KEYS.NAME] = newname
del self.catalog.entries[name]
return newname
return name
def sanitize(self):
super(Supernova, self).sanitize()
# Calculate some columns based on imported data, sanitize some fields
name = self[self._KEYS.NAME]
aliases = self.get_aliases()
if ((name.startswith('SN') and is_number(name[2:6])
and self._KEYS.DISCOVER_DATE in self
and int(self[self._KEYS.DISCOVER_DATE][0][QUANTITY.VALUE].split(
'/')[0]) >= 2016 and not any(['AT' in x for x in aliases]))):
source = self.add_self_source()
self.add_quantity(self._KEYS.ALIAS, 'AT' + name[2:], source)
if self._KEYS.CLAIMED_TYPE in self:
# FIX: this is something that should be done completely internally
# i.e. add it to `clean` or something??
self[self._KEYS.CLAIMED_TYPE] = self.ct_list_prioritized()
if self._KEYS.CLAIMED_TYPE in self:
self[self._KEYS.CLAIMED_TYPE][:] = [
ct for ct in self[self._KEYS.CLAIMED_TYPE]
if (ct[QUANTITY.VALUE] != '?' and ct[QUANTITY.VALUE] != '-')
]
if not len(self[self._KEYS.CLAIMED_TYPE]):
del (self[self._KEYS.CLAIMED_TYPE])
if self._KEYS.CLAIMED_TYPE not in self and name.startswith('AT'):
source = self.add_self_source()
self.add_quantity(self._KEYS.CLAIMED_TYPE, 'Candidate', source)
if self._KEYS.SOURCES in self:
for source in self[self._KEYS.SOURCES]:
if SOURCE.BIBCODE in source:
import urllib
from html import unescape
# First sanitize the bibcode
if len(source[SOURCE.BIBCODE]) != 19:
source[SOURCE.BIBCODE] = urllib.parse.unquote(
unescape(source[SOURCE.BIBCODE])).replace(
'A.A.', 'A&A')
if source[SOURCE.BIBCODE] in self.catalog.biberror_dict:
source[SOURCE.BIBCODE] = \
self.catalog.biberror_dict[source[SOURCE.BIBCODE]]
if (source[SOURCE.BIBCODE]
not in self.catalog.bibauthor_dict):
bibcode = source[SOURCE.BIBCODE]
adsquery = (self.catalog.ADS_BIB_URL +
urllib.parse.quote(bibcode) +
'&data_type=Custom&format=%253m%20%25(y)')
bibcodeauthor = ''
try:
response = urllib.request.urlopen(adsquery)
html = response.read().decode('utf-8')
hsplit = html.split("\n")
if len(hsplit) > 5:
bibcodeauthor = hsplit[5]
except:
pass
if not bibcodeauthor:
warnings.warn(
"Bibcode didn't return authors, not converting"
"this bibcode.")
self.catalog.bibauthor_dict[bibcode] = unescape(
bibcodeauthor).strip()
for source in self[self._KEYS.SOURCES]:
if (SOURCE.BIBCODE in source and source[SOURCE.BIBCODE]
in self.catalog.bibauthor_dict and
self.catalog.bibauthor_dict[source[SOURCE.BIBCODE]]):
source[SOURCE.REFERENCE] = self.catalog.bibauthor_dict[
source[SOURCE.BIBCODE]]
if (SOURCE.NAME not in source and SOURCE.BIBCODE in source
and source[SOURCE.BIBCODE]):
source[SOURCE.NAME] = source[SOURCE.BIBCODE]
if self._KEYS.REDSHIFT in self:
self[self._KEYS.REDSHIFT] = list(
sorted(self[self._KEYS.REDSHIFT],
key=lambda q: frame_priority(q, self._KEYS.REDSHIFT)))
if self._KEYS.VELOCITY in self:
self[self._KEYS.VELOCITY] = list(
sorted(self[self._KEYS.VELOCITY],
key=lambda q: frame_priority(q, self._KEYS.VELOCITY)))
if self._KEYS.CLAIMED_TYPE in self:
self[self._KEYS.CLAIMED_TYPE] = self.ct_list_prioritized()
# Renumber and reorder sources
if self._KEYS.SOURCES in self:
# Sort sources reverse-chronologically
self[self._KEYS.SOURCES] = sorted(self[self._KEYS.SOURCES],
key=lambda x: bib_priority(x))
# Assign new aliases to match new order
source_reps = OrderedDict(
[[x[SOURCE.ALIAS], str(i + 1)]
for i, x in enumerate(self[self._KEYS.SOURCES])])
for i, source in enumerate(self[self._KEYS.SOURCES]):
self[self._KEYS.SOURCES][i][SOURCE.ALIAS] = source_reps[source[
SOURCE.ALIAS]]
# Change sources to match new aliases
for key in self.keys():
if self._KEYS.get_key_by_name(key).no_source:
continue
for item in self[key]:
aliases = [
str(y) for y in sorted(
int(source_reps[x])
for x in item[item._KEYS.SOURCE].split(','))
]
item[item._KEYS.SOURCE] = ','.join(aliases)
def do_nedd(catalog):
task_str = catalog.get_current_task_str()
nedd_path = os.path.join(catalog.get_current_task_repo(),
'NED26.10.1-D-13.1.0-20160930.csv')
f = open(nedd_path, 'r')
data = sorted(list(csv.reader(f, delimiter=',', quotechar='"'))[13:],
key=lambda x: (x[9], x[3]))
reference = "NED-D v" + nedd_path.split('-')[-2]
refurl = "http://ned.ipac.caltech.edu/Library/Distances/"
nedbib = "1991ASSL..171...89H"
olddistname = ''
loopcnt = 0
for r, row in enumerate(pbar(data, task_str)):
if r <= 12:
continue
distname = row[3]
name = name_clean(distname)
# distmod = row[4]
# moderr = row[5]
dist = row[6]
bibcode = unescape(row[8])
snname = name_clean(row[9])
redshift = row[10]
cleanhost = ''
if name != snname and (name + ' HOST' != snname):
cleanhost = host_clean(distname)
if cleanhost.endswith(' HOST'):
cleanhost = ''
if not is_number(dist):
print(dist)
if dist:
catalog.nedd_dict.setdefault(cleanhost,
[]).append(Decimal(dist))
if snname and 'HOST' not in snname:
snname, secondarysource = catalog.new_entry(snname,
srcname=reference,
bibcode=nedbib,
url=refurl,
secondary=True)
if bibcode:
source = catalog.entries[snname].add_source(bibcode=bibcode)
sources = uniq_cdl([source, secondarysource])
else:
sources = secondarysource
if name == snname:
if redshift:
catalog.entries[snname].add_quantity(
SUPERNOVA.REDSHIFT, redshift, sources)
if dist:
catalog.entries[snname].add_quantity(
SUPERNOVA.COMOVING_DIST, dist, sources)
if not redshift:
try:
zatval = z_at_value(cosmo.comoving_distance,
float(dist) * un.Mpc,
zmax=5.0)
sigd = get_sig_digits(str(dist))
redshift = pretty_num(zatval, sig=sigd)
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
pass
else:
cosmosource = catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
combsources = uniq_cdl(
sources.split(',') + [cosmosource])
catalog.entries[snname].add_quantity(
SUPERNOVA.REDSHIFT,
redshift,
combsources,
derived=True)
if cleanhost:
catalog.entries[snname].add_quantity(SUPERNOVA.HOST, cleanhost,
sources)
if catalog.args.update and olddistname != distname:
catalog.journal_entries()
olddistname = distname
loopcnt = loopcnt + 1
if catalog.args.travis and loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0:
break
catalog.journal_entries()
f.close()
return
def do_suspect_photo(catalog):
task_str = catalog.get_current_task_str()
with open(
os.path.join(catalog.get_current_task_repo(),
'suspectreferences.csv'), 'r') as f:
tsvin = csv.reader(f, delimiter=',', skipinitialspace=True)
suspectrefdict = {}
for row in tsvin:
suspectrefdict[row[0]] = row[1]
file_names = list(
sorted(
glob(
os.path.join(catalog.get_current_task_repo(),
'SUSPECT/*.html'))))
for datafile in pbar_strings(file_names, task_str):
basename = os.path.basename(datafile)
basesplit = basename.split('-')
oldname = basesplit[1]
name = catalog.add_entry(oldname)
if name.startswith('SN') and is_number(name[2:]):
name = name + 'A'
band = basesplit[3].split('.')[0]
ei = int(basesplit[2])
bandlink = 'file://' + os.path.abspath(datafile)
bandresp = urllib.request.urlopen(bandlink)
bandsoup = BeautifulSoup(bandresp, 'html5lib')
bandtable = bandsoup.find('table')
names = bandsoup.body.findAll(text=re.compile('Name'))
reference = ''
for link in bandsoup.body.findAll('a'):
if 'adsabs' in link['href']:
reference = str(link).replace('"', "'")
bibcode = unescape(suspectrefdict[reference])
source = catalog.entries[name].add_source(bibcode=bibcode)
sec_ref = 'SUSPECT'
sec_refurl = 'https://www.nhn.ou.edu/~suspect/'
sec_source = catalog.entries[name].add_source(name=sec_ref,
url=sec_refurl,
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oldname,
sec_source)
if ei == 1:
year = re.findall(r'\d+', name)[0]
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, year,
sec_source)
catalog.entries[name].add_quantity(SUPERNOVA.HOST,
names[1].split(':')[1].strip(),
sec_source)
redshifts = bandsoup.body.findAll(text=re.compile('Redshift'))
if redshifts:
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT,
redshifts[0].split(':')[1].strip(),
sec_source,
kind='heliocentric')
# hvels = bandsoup.body.findAll(text=re.compile('Heliocentric
# Velocity'))
# if hvels:
# vel = hvels[0].split(':')[1].strip().split(' ')[0]
# catalog.entries[name].add_quantity(SUPERNOVA.VELOCITY, vel,
# sec_source,
# kind='heliocentric')
types = bandsoup.body.findAll(text=re.compile('Type'))
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE,
types[0].split(':')[1].strip().split(' ')[0], sec_source)
for r, row in enumerate(bandtable.findAll('tr')):
if r == 0:
continue
col = row.findAll('td')
mjd = str(jd_to_mjd(Decimal(col[0].contents[0])))
mag = col[3].contents[0]
if mag.isspace():
mag = ''
else:
mag = str(mag)
e_magnitude = col[4].contents[0]
if e_magnitude.isspace():
e_magnitude = ''
else:
e_magnitude = str(e_magnitude)
catalog.entries[name].add_photometry(time=mjd,
u_time='MJD',
band=band,
magnitude=mag,
e_magnitude=e_magnitude,
source=sec_source + ',' +
source)
catalog.journal_entries()
return
def do_ps_alerts(catalog):
task_str = catalog.get_current_task_str()
alertstables = ['alertstable-2010', 'alertstable-2011', 'alertstable']
rows = []
for at in alertstables:
with open(
os.path.join(catalog.get_current_task_repo(), 'ps1-clean',
at)) as f:
rows.extend(
list(csv.reader(
f, delimiter=' ', skipinitialspace=True))[1:])
alertfiles = glob(
os.path.join(catalog.get_current_task_repo(), 'ps1-clean/*.dat'))
alertfilestag = dict(
[(x.split('/')[-1].split('.')[0], x) for x in alertfiles])
alertfilesid = dict([(x.split('/')[-1].split('.')[0].split('-')[-1], x)
for x in alertfiles])
with open(
os.path.join(catalog.get_current_task_repo(),
'ps1-clean/whitelist')) as f:
whitelist = list(csv.reader(f, delimiter=' ', skipinitialspace=True))
wlnames = [x[0] for x in whitelist]
wlnamesleft = list(wlnames)
wlra = [x[1] for x in whitelist]
missing_confirmed = []
# already_collected = []
for ri, row in enumerate(pbar(rows, task_str)):
psname = row[50]
if psname == '-':
if row[4] in wlra:
psname = wlnames[wlra.index(row[4])]
else:
continue
sntype = row[21].replace('SN', '')
skip_photo = False
if psname not in wlnamesleft:
if row[1] == 'confirmed':
missing_confirmed.append((psname, row[21]))
# if 'II' in sntype:
# pass
# elif (sntype != 'Ia' or
# not (psname.startswith(('PS1-12', 'PS1-13')) or
# (psname.startswith('PS1-10') and
# len(psname.replace('PS1-10', '')) == 3 and
# psname[-3:] > 'ams'))):
# if sntype == 'Ia' and psname.startswith('PS1-10'):
# already_collected.append((psname, row[21]))
# else:
# missing_confirmed.append((psname, row[21]))
skip_photo = True
else:
continue
if psname in wlnamesleft:
wlnamesleft.remove(psname)
name, source = catalog.new_entry(
psname,
srcname='Pan-STARRS Alerts',
url='http://telescopes.rc.fas.harvard.edu/ps1/')
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[4], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[5], source)
if sntype != '-':
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, sntype,
source)
if row[22] != '-':
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, row[22],
source)
# Disabling photometry import
continue
if skip_photo:
continue
psinternal = row[-1].split('.')[0]
if not is_number(psinternal.split('-')[0]):
psid = row[0].zfill(6)
if psid not in alertfilesid:
continue
pspath = alertfilesid[psid]
else:
if psinternal not in alertfilestag:
continue
pspath = alertfilestag[psinternal]
with open(pspath) as f:
photrows = list(
csv.reader(
f, delimiter=' ', skipinitialspace=True))
for pi, prow in enumerate(photrows):
if pi == 0 or prow[3] == '-':
continue
counts = prow[13]
e_counts = prow[14]
photodict = {
PHOTOMETRY.TIME: prow[1],
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.BAND: prow[2],
PHOTOMETRY.MAGNITUDE: prow[3],
PHOTOMETRY.E_MAGNITUDE: prow[4],
PHOTOMETRY.COUNT_RATE: counts,
PHOTOMETRY.E_COUNT_RATE: e_counts,
PHOTOMETRY.ZERO_POINT: prow[15],
PHOTOMETRY.INSTRUMENT: 'GPC1',
PHOTOMETRY.OBSERVATORY: 'PS1',
PHOTOMETRY.SURVEY: 'MDS',
PHOTOMETRY.TELESCOPE: 'PS1',
PHOTOMETRY.SYSTEM: 'PS1',
PHOTOMETRY.SOURCE: source
}
ul_sigma = 3.0
if float(counts) < ul_sigma * float(e_counts):
photodict[PHOTOMETRY.UPPER_LIMIT] = True
photodict[PHOTOMETRY.UPPER_LIMIT_SIGMA] = ul_sigma
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
# print(already_collected)
# print(missing_confirmed)
return
def do_ptf(catalog):
# response =
# urllib.request.urlopen('http://wiserep.weizmann.ac.il/objects/list')
# bs = BeautifulSoup(response, 'html5lib')
# select = bs.find('select', {'name': 'objid'})
# options = select.findAll('option')
# for option in options:
# print(option.text)
# name = option.text
# if ((name.startswith('PTF') and is_number(name[3:5])) or
# name.startswith('PTFS') or name.startswith('iPTF')):
# name = catalog.add_entry(name)
task_str = catalog.get_current_task_str()
if catalog.current_task.load_archive(catalog.args):
with open(os.path.join(catalog.get_current_task_repo(),
'PTF/update.html'), 'r') as f:
html = f.read()
else:
session = requests.Session()
response = session.get('http://wiserep.weizmann.ac.il/spectra/update')
html = response.text
with open(os.path.join(catalog.get_current_task_repo(),
'PTF/update.html'), 'w') as f:
f.write(html)
bs = BeautifulSoup(html, 'html5lib')
select = bs.find('select', {'name': 'objid'})
options = select.findAll('option')
for option in pbar(options, task_str):
name = option.text
if (((name.startswith('PTF') and is_number(name[3:5])) or
name.startswith('PTFS') or name.startswith('iPTF'))):
if '(' in name:
alias = name.split('(')[0].strip(' ')
name = name.split('(')[-1].strip(') ').replace('sn', 'SN')
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2012PASP..124..668Y')
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, alias, source)
else:
# name = catalog.add_entry(name)
name, source = catalog.new_entry(name,
bibcode='2012PASP..124..668Y')
with open(os.path.join(
catalog.get_current_task_repo(), 'PTF/old-ptf-events.csv')) as f:
for suffix in pbar(f.read().splitlines(), task_str):
name = catalog.add_entry('PTF' + suffix)
with open(os.path.join(
catalog.get_current_task_repo(), 'PTF/perly-2016.csv')) as f:
for row in pbar(f.read().splitlines(), task_str):
cols = [x.strip() for x in row.split(',')]
alias = ''
if cols[8]:
name = cols[8]
alias = 'PTF' + cols[0]
else:
name = 'PTF' + cols[0]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2016arXiv160408207P')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
if alias:
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, alias, source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, cols[1], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, cols[2], source)
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, 'SLSN-' + cols[3], source)
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, cols[4], source, kind='spectroscopic')
maxdate = cols[6].replace('-', '/')
upl = maxdate.startswith('<')
catalog.entries[name].add_quantity(
SUPERNOVA.MAX_DATE, maxdate.lstrip('<'), source,
upperlimit=upl)
catalog.entries[name].add_quantity(
SUPERNOVA.EBV, cols[7], source, kind='spectroscopic')
name = catalog.add_entry('PTF' + suffix)
catalog.journal_entries()
return
def do_donated_photo(catalog):
"""Import donated photometry."""
task_str = catalog.get_current_task_str()
# Private donations here #
if not catalog.args.travis:
pass
# End private donations #
# Ponder 05-12-17 donation
with open(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Ponder-05-12-17', 'meta.json'), 'r') as f:
metadict = json.loads(f.read())
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Ponder-05-12-17', '*.dat'))
for path in file_names:
with open(path, 'r') as f:
tsvin = list(csv.reader(f, delimiter=' ', skipinitialspace=True))
oname = path.split('/')[-1].split('.')[0]
name, source = catalog.new_entry(
oname, bibcode=metadict[oname]['bibcode'])
for row in pbar(tsvin, task_str + ': Ponder ' + oname):
if row[0][0] == '#' or not is_number(row[-1]):
continue
mjd = row[1]
bandinst = row[2].split('_')
band = bandinst[0]
inst = ''
if len(bandinst) > 1:
inst = bandinst[1]
mag = row[3]
uerr = row[4]
lerr = row[5]
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.BAND: band,
PHOTOMETRY.MAGNITUDE: mag,
PHOTOMETRY.E_LOWER_MAGNITUDE: lerr,
PHOTOMETRY.E_UPPER_MAGNITUDE: uerr,
PHOTOMETRY.SOURCE: source
}
if inst:
photodict[PHOTOMETRY.INSTRUMENT] = inst
catalog.entries[name].add_photometry(**photodict)
# Benetti 03-08-17 donation
path = os.path.join(catalog.get_current_task_repo(), 'Donations',
'Benetti-03-08-17', '1999E.dat')
with open(path, 'r') as f:
tsvin = list(csv.reader(f, delimiter=' ', skipinitialspace=True))
name, source = catalog.new_entry(
'SN1999E', bibcode='2003MNRAS.340..191R')
bands = None
for row in tsvin:
if not row or row[0][0] == '#':
continue
if not bands:
bands = row[2:-2]
continue
mjd = row[1]
tel = row[-1] if 'IAUC' not in row[-1] else None
for bi, band in enumerate(bands):
mag = row[2 + 2 * bi]
if mag == '9999':
continue
err = row[2 + 2 * bi + 1]
limit = row[6] == 'True'
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.TELESCOPE: tel,
PHOTOMETRY.BAND: band,
PHOTOMETRY.MAGNITUDE: mag,
PHOTOMETRY.SOURCE: source
}
if err != '.00':
photodict[PHOTOMETRY.E_MAGNITUDE] = str(Decimal(err))
if tel:
photodict[PHOTOMETRY.TELESCOPE] = tel
catalog.entries[name].add_photometry(**photodict)
# Nicholl 01-29-17 donation
with open(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-01-29-17', 'meta.json'), 'r') as f:
metadict = json.loads(f.read())
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-01-29-17', '*.txt'))
for path in file_names:
data = read(path, format='cds')
oname = path.split('/')[-1].split('_')[0]
name, source = catalog.new_entry(
oname, bibcode=metadict[oname]['bibcode'])
for row in pbar(data, task_str + ': Nicholl ' + oname):
photodict = {
PHOTOMETRY.TIME: str(row['MJD']),
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.MAGNITUDE: str(row['mag']),
PHOTOMETRY.BAND: row['Filter'],
PHOTOMETRY.SOURCE: source
}
if 'system' in metadict[oname]:
photodict[PHOTOMETRY.SYSTEM] = metadict[oname]['system']
if 'l_mag' in row.columns and row['l_mag'] == '>':
photodict[PHOTOMETRY.UPPER_LIMIT] = True
elif 'e_mag' in row.columns:
photodict[PHOTOMETRY.E_MAGNITUDE] = str(row['e_mag'])
if 'Telescope' in row.columns:
photodict[PHOTOMETRY.TELESCOPE] = row['Telescope']
catalog.entries[name].add_photometry(**photodict)
# Arcavi 2016gkg donation
path = os.path.join(catalog.get_current_task_repo(), 'Donations',
'Arcavi-01-24-17', 'SN2016gkg.txt')
with open(path, 'r') as f:
tsvin = list(csv.reader(f, delimiter=' ', skipinitialspace=True))
name, source = catalog.new_entry(
'SN2016gkg', bibcode='2016arXiv161106451A')
for row in tsvin:
if row[0][0] == '#':
continue
mjd = str(jd_to_mjd(Decimal(row[0])))
tel = row[1]
band = row[3]
mag = row[4]
err = row[5]
limit = row[6] == 'True'
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.TELESCOPE: tel,
PHOTOMETRY.BAND: band,
PHOTOMETRY.MAGNITUDE: mag,
PHOTOMETRY.SOURCE: source
}
if limit:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
else:
photodict[PHOTOMETRY.E_MAGNITUDE] = err
catalog.entries[name].add_photometry(**photodict)
# Nicholl Gaia16apd donation
path = os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-01-20-17', 'gaia16apd_phot.txt')
data = read(path, format='cds')
name, source = catalog.new_entry(
'Gaia16apd', bibcode='2017ApJ...835L...8N')
for row in pbar(data, task_str + ': Nicholl Gaia16apd'):
photodict = {
PHOTOMETRY.TIME: str(row['MJD']),
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.MAGNITUDE: str(row['mag']),
PHOTOMETRY.BAND: row['Filter'],
PHOTOMETRY.TELESCOPE: row['Telescope'],
PHOTOMETRY.SOURCE: source
}
if row['l_mag'] == '>':
photodict[PHOTOMETRY.UPPER_LIMIT] = True
else:
photodict[PHOTOMETRY.E_MAGNITUDE] = str(row['e_mag'])
catalog.entries[name].add_photometry(**photodict)
# Kuncarayakti-01-09-17
datafile = os.path.join(catalog.get_current_task_repo(), 'Donations',
'Kuncarayakti-01-09-17', 'SN1978K.dat')
inpname = os.path.basename(datafile).split('.')[0]
with open(datafile, 'r') as f:
tsvin = csv.reader(f, delimiter=' ', skipinitialspace=True)
host = False
for ri, row in enumerate(tsvin):
if ri == 0:
continue
if row[0][0] == '#':
rsplit = [x.strip('# ') for x in ' '.join(row).split(',')]
bc = rsplit[0]
tel, ins = '', ''
if len(rsplit) > 1:
tel = rsplit[1]
if len(rsplit) > 2:
ins = rsplit[2]
continue
(name, source) = catalog.new_entry(inpname, bibcode=bc)
mag = row[4]
err = row[5]
mjd = str(astrotime('-'.join(row[:3]), format='iso').mjd)
photodict = {
PHOTOMETRY.BAND: row[3],
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.MAGNITUDE: mag.strip('>s'),
PHOTOMETRY.SOURCE: source
}
if is_number(err):
photodict[PHOTOMETRY.E_MAGNITUDE] = err
if tel:
photodict[PHOTOMETRY.TELESCOPE] = tel
if ins:
photodict[PHOTOMETRY.INSTRUMENT] = ins
if '>' in mag:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
if 's' in mag:
photodict[PHOTOMETRY.SYNTHETIC] = True
catalog.entries[name].add_photometry(**photodict)
# Nugent 01-09-17 donation
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nugent-01-09-17', '*.dat'))
for datafile in pbar_strings(file_names, task_str + ': Nugent-01-09-17'):
inpname = os.path.basename(datafile).split('.')[0]
(name, source) = catalog.new_entry(
inpname, bibcode='2006ApJ...645..841N')
with open(datafile, 'r') as f:
tsvin = csv.reader(f, delimiter=' ', skipinitialspace=True)
host = False
for urow in tsvin:
row = list(filter(None, urow))
counts = row[2]
e_counts = row[3]
zp = row[4]
photodict = {
PHOTOMETRY.BAND: row[1],
PHOTOMETRY.TIME: row[0],
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.COUNT_RATE: counts,
PHOTOMETRY.E_COUNT_RATE: e_counts,
PHOTOMETRY.ZERO_POINT: zp,
PHOTOMETRY.TELESCOPE: 'CFHT',
PHOTOMETRY.SURVEY: 'SNLS',
PHOTOMETRY.SOURCE: source
}
set_pd_mag_from_counts(photodict, counts, ec=e_counts, zp=zp,
sig=5.0)
catalog.entries[name].add_photometry(**photodict)
# Inserra 09-04-16 donation
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Inserra-09-04-16', '*.txt'))
for datafile in pbar_strings(file_names, task_str + ': Inserra-09-04-16'):
inpname = os.path.basename(datafile).split('.')[0]
(name, source) = catalog.new_entry(
inpname, bibcode='2013ApJ...770..128I')
with open(datafile, 'r') as f:
tsvin = csv.reader(f, delimiter=' ', skipinitialspace=True)
host = False
for row in tsvin:
if row[0][0] == '#':
if row[0] == '#Host':
host = True
continue
host = False
bands = row[3:-1]
continue
for bi, ba in enumerate(bands):
mag = row[5 + 2 * bi]
if not is_number(mag):
continue
system = 'AB'
if ba in ['U', 'B', 'V', 'R', 'I', 'J', 'H', 'K']:
system = 'Vega'
photodict = {
PHOTOMETRY.TIME: row[3],
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.BAND: ba,
PHOTOMETRY.MAGNITUDE: mag.strip('< '),
PHOTOMETRY.SOURCE: source,
PHOTOMETRY.SYSTEM: system
}
if 'ATel' not in row[-1]:
photodict[PHOTOMETRY.TELESCOPE] = row[-1]
if host:
photodict[PHOTOMETRY.HOST] = True
if '<' in mag:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
e_mag = row[5 + 2 * bi + 1].strip('() ')
if is_number(e_mag):
photodict[PHOTOMETRY.E_MAGNITUDE] = e_mag
catalog.entries[name].add_photometry(**photodict)
# Nicholl 04-01-16 donation
with open(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-04-01-16', 'bibcodes.json'), 'r') as f:
bcs = json.loads(f.read())
kcorrected = ['SN2011ke', 'SN2011kf', 'SN2012il', 'PTF10hgi', 'PTF11rks']
ignorephoto = ['PTF10hgi', 'PTF11rks', 'SN2011ke', 'SN2011kf', 'SN2012il']
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-04-01-16/*.txt'))
for datafile in pbar_strings(file_names, task_str + ': Nicholl-04-01-16'):
inpname = os.path.basename(datafile).split('_')[0]
isk = inpname in kcorrected
name = catalog.add_entry(inpname)
bibcode = ''
for bc in bcs:
if inpname in bcs[bc]:
bibcode = bc
if not bibcode:
raise ValueError('Bibcode not found!')
source = catalog.entries[name].add_source(bibcode=bibcode)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, inpname, source)
if inpname in ignorephoto:
continue
with open(datafile, 'r') as f:
tsvin = csv.reader(f, delimiter='\t', skipinitialspace=True)
rtelescope = ''
for r, rrow in enumerate(tsvin):
row = list(filter(None, rrow))
if not row:
continue
if row[0] == '#MJD':
bands = [x for x in row[1:] if x and 'err' not in x]
elif row[0][0] == '#' and len(row[0]) > 1:
rtelescope = row[0][1:]
if row[0][0] == '#':
continue
mjd = row[0]
if not is_number(mjd):
continue
for v, val in enumerate(row[1::2]):
upperlimit = ''
mag = val.strip('>')
emag = row[2 * v + 2]
if '>' in val or (is_number(emag) and float(emag) == 0.0):
upperlimit = True
if (not is_number(mag) or isnan(float(mag)) or
float(mag) > 90.0):
continue
band = bands[v]
instrument = ''
survey = ''
system = ''
telescope = rtelescope
if telescope == 'LSQ':
instrument = 'QUEST'
elif telescope == 'PS1':
instrument = 'GPC'
elif telescope == 'NTT':
instrument = 'EFOSC'
elif telescope == 'GROND':
instrument = 'GROND'
telescope = 'MPI/ESO 2.2m'
else:
if band == 'NUV':
instrument = 'GALEX'
telescope = 'GALEX'
elif band in ['u', 'g', 'r', 'i', 'z']:
if inpname.startswith('PS1'):
instrument = 'GPC'
telescope = 'PS1'
survey = 'Pan-STARRS'
elif inpname.startswith('PTF'):
telescope = 'P60'
survey = 'PTF'
elif band.upper() in ['UVW2', 'UVW1', 'UVM2']:
instrument = 'UVOT'
telescope = 'Swift'
if inpname in ['PTF12dam']:
system = 'AB'
if inpname in ['SCP-06F6']:
system = 'Vega'
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.BAND: band,
PHOTOMETRY.MAGNITUDE: mag,
PHOTOMETRY.UPPER_LIMIT: upperlimit,
PHOTOMETRY.SOURCE: source
}
if instrument:
photodict[PHOTOMETRY.INSTRUMENT] = instrument
if telescope:
photodict[PHOTOMETRY.TELESCOPE] = telescope
if survey:
photodict[PHOTOMETRY.SURVEY] = survey
if system:
photodict[PHOTOMETRY.SYSTEM] = system
if (is_number(emag) and
not isnan(float(emag)) and float(emag) > 0.0):
photodict[PHOTOMETRY.E_MAGNITUDE] = emag
if isk:
photodict[PHOTOMETRY.KCORRECTED] = True
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
# Maggi 04-11-16 donation (MC SNRs)
with open(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Maggi-04-11-16', 'LMCSNRs_OpenSNe.csv')) as f:
tsvin = csv.reader(f, delimiter=',')
for row in pbar(list(tsvin), task_str + ': Maggi-04-11-16/LMCSNRs'):
name = 'MCSNR ' + row[0]
name = catalog.add_entry(name)
ra = row[2]
dec = row[3]
source = (catalog.entries[name]
.add_source(bibcode='2016A&A...585A.162M'))
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS,
'LMCSNR J' + rep_chars(ra, ' :.') + rep_chars(dec, ' :.'),
source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
if row[1] != 'noname':
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[1],
source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[2], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[3], source)
catalog.entries[name].add_quantity(SUPERNOVA.HOST, 'LMC', source)
if row[4] == '1':
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE,
'Ia', source)
elif row[4] == '2':
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE,
'CC', source)
with open(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Maggi-04-11-16', 'SMCSNRs_OpenSNe.csv')) as f:
tsvin = csv.reader(f, delimiter=',')
for row in pbar(list(tsvin), task_str + ': Maggi-04-11-16/SMCSNRs'):
name = 'MCSNR ' + row[0]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(name='Pierre Maggi')
ra = row[3]
dec = row[4]
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, 'SMCSNR J' + ra.replace(
':', '')[:6] + dec.replace(':', '')[:7], source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[1], source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[2], source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[3], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[4], source)
catalog.entries[name].add_quantity(SUPERNOVA.HOST, 'SMC', source)
catalog.journal_entries()
# Galbany 04-18-16 donation
folders = next(
os.walk(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Galbany-04-18-16/')))[1]
bibcode = '2016AJ....151...33G'
for folder in folders:
infofiles = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Galbany-04-18-16/') + folder + '/*.info')
photfiles = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Galbany-04-18-16/') + folder + '/*.out*')
zhel = ''
zcmb = ''
zerr = ''
for path in infofiles:
with open(path, 'r') as f:
lines = f.read().splitlines()
for line in lines:
splitline = line.split(':')
field = splitline[0].strip().lower()
value = splitline[1].strip()
if field == 'name':
name = value[:6].upper()
name += (value[6].upper()
if len(value) == 7 else value[6:])
name = catalog.add_entry(name)
source = (catalog.entries[name]
.add_source(bibcode=bibcode))
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS,
name, source)
elif field == 'type':
claimedtype = value.replace('SN', '')
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, claimedtype, source)
elif field == 'zhel':
zhel = value
elif field == 'redshift_error':
zerr = value
elif field == 'zcmb':
zcmb = value
elif field == 'ra':
catalog.entries[name].add_quantity(
SUPERNOVA.RA,
value,
source,
u_value='floatdegrees')
elif field == 'dec':
catalog.entries[name].add_quantity(
SUPERNOVA.DEC,
value,
source,
u_value='floatdegrees')
elif field == 'host':
value = value.replace('- ', '-').replace('G ', 'G')
catalog.entries[name].add_quantity(SUPERNOVA.HOST,
value, source)
elif field == 'e(b-v)_mw':
catalog.entries[name].add_quantity(SUPERNOVA.EBV,
value, source)
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT,
zhel,
source,
e_value=zerr,
kind='heliocentric')
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, zcmb, source, e_value=zerr, kind='cmb')
for path in photfiles:
with open(path, 'r') as f:
band = ''
lines = f.read().splitlines()
for li, line in enumerate(lines):
if li in [0, 2, 3]:
continue
if li == 1:
band = line.split(':')[-1].strip()
else:
cols = list(filter(None, line.split()))
if not cols:
continue
catalog.entries[name].add_photometry(
time=cols[0],
u_time='MJD',
magnitude=cols[1],
e_magnitude=cols[2],
band=band,
system=cols[3],
telescope=cols[4],
source=source)
catalog.journal_entries()
# Nicholl 05-03-16
files = glob(
os.path.join(catalog.get_current_task_repo(), 'Donations',
'Nicholl-05-03-16', '*.txt'))
name = catalog.add_entry('SN2015bn')
for fi in pbar(files, task_str + ': Nicholl-05-03-16'):
if 'late' in fi:
bc = '2016ApJ...828L..18N'
else:
bc = '2016ApJ...826...39N'
source = catalog.entries[name].add_source(bibcode=bc)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, 'PS15ae', source)
telescope = os.path.basename(fi).split('_')[1]
with open(fi, 'r') as f:
lines = f.read().splitlines()
for li, line in enumerate(lines):
if not line or (line[0] == '#' and li != 0):
continue
cols = list(filter(None, line.split()))
if not cols:
continue
if li == 0:
bands = cols[1:]
continue
mjd = cols[0]
for ci, col in enumerate(cols[1::2]):
if not is_number(col) or np.isnan(float(col)):
continue
band = bands[ci]
band_set = ''
system = 'Vega'
if bands[ci] in ["u'", "g'", "r'", "i'", "z'"]:
band_set = 'SDSS'
system = 'SDSS'
elif telescope == 'ASASSN':
band_set = 'ASASSN'
system = 'Vega'
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.MAGNITUDE: col,
PHOTOMETRY.BAND: bands[ci],
PHOTOMETRY.SOURCE: source,
PHOTOMETRY.TELESCOPE: telescope,
PHOTOMETRY.SYSTEM: system
}
if band_set:
photodict[PHOTOMETRY.BAND_SET] = band_set
emag = cols[2 * ci + 2]
if is_number(emag):
photodict[PHOTOMETRY.E_MAGNITUDE] = emag
else:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
if telescope == 'Swift':
photodict[PHOTOMETRY.INSTRUMENT] = 'UVOT'
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
return
def do_cleanup(catalog):
"""Cleanup catalog after importing all data."""
task_str = catalog.get_current_task_str()
# Set preferred names, calculate some columns based on imported data,
# sanitize some fields
keys = list(catalog.entries.keys())
cleanupcnt = 0
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
try:
name = catalog.add_entry(oname)
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
# Set the preferred name, switching to that name if name changed.
name = catalog.entries[name].set_preferred_name()
aliases = catalog.entries[name].get_aliases()
catalog.entries[name].purge_bandless_photometry()
catalog.entries[name].set_first_max_light()
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['MLS', 'SSS', 'CSS', 'GRB ']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:2])):
discoverdate = ('/'.join([
'20' + alias.replace(prefix, '')[:2],
alias.replace(prefix, '')[2:4],
alias.replace(prefix, '')[4:6]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = [
'ASASSN-', 'PS1-', 'PS1', 'PS', 'iPTF', 'PTF', 'SCP-', 'SNLS-',
'SPIRITS', 'LSQ', 'DES', 'SNHiTS', 'Gaia', 'GND', 'GNW', 'GSD',
'GSW', 'EGS', 'COS', 'OGLE', 'HST'
]
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:2])
and is_number(alias.replace(prefix, '')[:1])):
discoverdate = '20' + alias.replace(prefix, '')[:2]
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['SNF']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:4])):
discoverdate = ('/'.join([
alias.replace(prefix, '')[:4],
alias.replace(prefix, '')[4:6],
alias.replace(prefix, '')[6:8]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['PTFS', 'SNSDF']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:2])):
discoverdate = ('/'.join([
'20' + alias.replace(prefix, '')[:2],
alias.replace(prefix, '')[2:4]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['AT', 'SN', 'OGLE-', 'SM ', 'KSN']
for alias in aliases:
for prefix in prefixes:
if alias.startswith(prefix):
year = re.findall(r'\d+', alias)
if len(year) == 1:
year = year[0]
else:
continue
if alias.replace(prefix, '').index(year) != 0:
continue
if (year and is_number(year) and '.' not in year
and len(year) <= 4):
discoverdate = year
if catalog.args.verbose:
tprint('Added discoverdate from name [' +
alias + ']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if (SUPERNOVA.RA not in catalog.entries[name]
or SUPERNOVA.DEC not in catalog.entries[name]):
prefixes = [
'PSN J', 'MASJ', 'CSS', 'SSS', 'MASTER OT J', 'HST J', 'TCP J',
'MACS J', '2MASS J', 'EQ J', 'CRTS J', 'SMT J'
]
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:6])):
noprefix = alias.split(':')[-1].replace(prefix,
'').replace(
'.', '')
decsign = '+' if '+' in noprefix else '-'
noprefix = noprefix.replace('+', '|').replace('-', '|')
nops = noprefix.split('|')
if len(nops) < 2:
continue
rastr = nops[0]
decstr = nops[1]
ra = ':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + \
('.' + rastr[6:] if len(rastr) > 6 else '')
dec = (
decsign +
':'.join([decstr[:2], decstr[2:4], decstr[4:6]]) +
('.' + decstr[6:] if len(decstr) > 6 else ''))
if catalog.args.verbose:
tprint('Added ra/dec from name: ' + ra + ' ' + dec)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(SUPERNOVA.RA,
ra,
source,
derived=True)
catalog.entries[name].add_quantity(SUPERNOVA.DEC,
dec,
source,
derived=True)
break
if SUPERNOVA.RA in catalog.entries[name]:
break
no_host = (SUPERNOVA.HOST not in catalog.entries[name] or not any([
x[QUANTITY.VALUE] == 'Milky Way'
for x in catalog.entries[name][SUPERNOVA.HOST]
]))
if (SUPERNOVA.RA in catalog.entries[name]
and SUPERNOVA.DEC in catalog.entries[name] and no_host):
from astroquery.irsa_dust import IrsaDust
if name not in catalog.extinctions_dict:
try:
ra_dec = catalog.entries[name][
SUPERNOVA.RA][0][QUANTITY.VALUE] + \
" " + \
catalog.entries[name][SUPERNOVA.DEC][0][QUANTITY.VALUE]
result = IrsaDust.get_query_table(ra_dec, section='ebv')
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
warnings.warn("Coordinate lookup for " + name +
" failed in IRSA.")
else:
ebv = result['ext SandF mean'][0]
ebverr = result['ext SandF std'][0]
catalog.extinctions_dict[name] = [ebv, ebverr]
if name in catalog.extinctions_dict:
sources = uniq_cdl([
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2011ApJ...737..103S')
])
(catalog.entries[name].add_quantity(
SUPERNOVA.EBV,
str(catalog.extinctions_dict[name][0]),
sources,
e_value=str(catalog.extinctions_dict[name][1]),
derived=True))
if ((SUPERNOVA.HOST in catalog.entries[name]
and (SUPERNOVA.HOST_RA not in catalog.entries[name]
or SUPERNOVA.HOST_DEC not in catalog.entries[name]))):
for host in catalog.entries[name][SUPERNOVA.HOST]:
alias = host[QUANTITY.VALUE]
if ' J' in alias and is_number(alias.split(' J')[-1][:6]):
noprefix = alias.split(' J')[-1].split(':')[-1].replace(
'.', '')
decsign = '+' if '+' in noprefix else '-'
noprefix = noprefix.replace('+', '|').replace('-', '|')
nops = noprefix.split('|')
if len(nops) < 2:
continue
rastr = nops[0]
decstr = nops[1]
hostra = (':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) +
('.' + rastr[6:] if len(rastr) > 6 else ''))
hostdec = decsign + ':'.join([
decstr[:2], decstr[2:4], decstr[4:6]
]) + ('.' + decstr[6:] if len(decstr) > 6 else '')
if catalog.args.verbose:
tprint('Added hostra/hostdec from name: ' + hostra +
' ' + hostdec)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(SUPERNOVA.HOST_RA,
hostra,
source,
derived=True)
catalog.entries[name].add_quantity(SUPERNOVA.HOST_DEC,
hostdec,
source,
derived=True)
break
if SUPERNOVA.HOST_RA in catalog.entries[name]:
break
if (SUPERNOVA.REDSHIFT not in catalog.entries[name]
and SUPERNOVA.VELOCITY in catalog.entries[name]):
# Find the "best" velocity to use for this
bestsig = 0
for hv in catalog.entries[name][SUPERNOVA.VELOCITY]:
sig = get_sig_digits(hv[QUANTITY.VALUE])
if sig > bestsig:
besthv = hv[QUANTITY.VALUE]
bestsrc = hv['source']
bestsig = sig
if bestsig > 0 and is_number(besthv):
voc = float(besthv) * 1.e5 / CLIGHT
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
(catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT,
pretty_num(sqrt((1. + voc) / (1. - voc)) - 1.,
sig=bestsig),
sources,
kind='heliocentric',
derived=True))
if (SUPERNOVA.REDSHIFT not in catalog.entries[name]
and len(catalog.nedd_dict) > 0
and SUPERNOVA.HOST in catalog.entries[name]):
reference = "NED-D"
refurl = "http://ned.ipac.caltech.edu/Library/Distances/"
refbib = "1991ASSL..171...89H"
for host in catalog.entries[name][SUPERNOVA.HOST]:
if host[QUANTITY.VALUE] in catalog.nedd_dict:
source = catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
secondarysource = catalog.entries[name].add_source(
name=reference,
url=refurl,
bibcode=refbib,
secondary=True)
meddist = statistics.median(
catalog.nedd_dict[host[QUANTITY.VALUE]])
redz = z_at_value(cosmo.comoving_distance,
float(meddist) * un.Mpc)
redshift = pretty_num(redz,
sig=get_sig_digits(str(meddist)))
catalog.entries[name].add_quantity(
[SUPERNOVA.REDSHIFT, SUPERNOVA.HOST_REDSHIFT],
redshift,
uniq_cdl([source, secondarysource]),
kind='host',
derived=True)
if (SUPERNOVA.MAX_ABS_MAG not in catalog.entries[name]
and SUPERNOVA.MAX_APP_MAG in catalog.entries[name]
and SUPERNOVA.LUM_DIST in catalog.entries[name]):
# Find the "best" distance to use for this
bestsig = 0
for ld in catalog.entries[name][SUPERNOVA.LUM_DIST]:
sig = get_sig_digits(ld[QUANTITY.VALUE])
if sig > bestsig:
bestld = ld[QUANTITY.VALUE]
bestsrc = ld[QUANTITY.SOURCE]
bestsig = sig
if bestsig > 0 and is_number(bestld) and float(bestld) > 0.:
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
bestldz = z_at_value(cosmo.luminosity_distance,
float(bestld) * un.Mpc)
pnum = (float(catalog.entries[name][SUPERNOVA.MAX_APP_MAG][0][
QUANTITY.VALUE]) - 5.0 *
(log10(float(bestld) * 1.0e6) - 1.0) +
2.5 * log10(1.0 + bestldz))
pnum = pretty_num(pnum, sig=bestsig + 1)
catalog.entries[name].add_quantity(SUPERNOVA.MAX_ABS_MAG,
pnum,
sources,
derived=True)
if (SUPERNOVA.MAX_VISUAL_ABS_MAG not in catalog.entries[name]
and SUPERNOVA.MAX_VISUAL_APP_MAG in catalog.entries[name]
and SUPERNOVA.LUM_DIST in catalog.entries[name]):
# Find the "best" distance to use for this
bestsig = 0
for ld in catalog.entries[name][SUPERNOVA.LUM_DIST]:
sig = get_sig_digits(ld[QUANTITY.VALUE])
if sig > bestsig:
bestld = ld[QUANTITY.VALUE]
bestsrc = ld[QUANTITY.SOURCE]
bestsig = sig
if bestsig > 0 and is_number(bestld) and float(bestld) > 0.:
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
# FIX: what's happening here?!
pnum = (float(catalog.entries[name][
SUPERNOVA.MAX_VISUAL_APP_MAG][0][QUANTITY.VALUE]) - 5.0 *
(log10(float(bestld) * 1.0e6) - 1.0))
pnum = pretty_num(pnum, sig=bestsig + 1)
catalog.entries[name].add_quantity(
SUPERNOVA.MAX_VISUAL_ABS_MAG, pnum, sources, derived=True)
if SUPERNOVA.REDSHIFT in catalog.entries[name]:
# Find the "best" redshift to use for this
bestz, bestkind, bestsig, bestsrc = catalog.entries[
name].get_best_redshift()
if bestsig > 0:
try:
bestz = float(bestz)
except Exception:
print(catalog.entries[name])
raise
if SUPERNOVA.VELOCITY not in catalog.entries[name]:
source = catalog.entries[name].add_self_source()
# FIX: what's happening here?!
pnum = CLIGHT / KM * \
((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.)
pnum = pretty_num(pnum, sig=bestsig)
catalog.entries[name].add_quantity(
SUPERNOVA.VELOCITY,
pnum,
source,
kind=(SUPERNOVA.VELOCITY.kind_preference[bestkind]
if bestkind else ''))
if bestz > 0.:
if SUPERNOVA.LUM_DIST not in catalog.entries[name]:
dl = cosmo.luminosity_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
SUPERNOVA.LUM_DIST,
pretty_num(dl.value, sig=bestsig + 1),
sources,
kind=(SUPERNOVA.LUM_DIST.kind_preference[bestkind]
if bestkind else ''),
derived=True)
if (SUPERNOVA.MAX_ABS_MAG not in catalog.entries[name]
and SUPERNOVA.MAX_APP_MAG
in catalog.entries[name]):
source = catalog.entries[name].add_self_source()
pnum = pretty_num(
float(catalog.entries[name][
SUPERNOVA.MAX_APP_MAG][0][QUANTITY.VALUE])
- 5.0 * (log10(dl.to('pc').value) - 1.0) +
2.5 * log10(1.0 + bestz),
sig=bestsig + 1)
catalog.entries[name].add_quantity(
SUPERNOVA.MAX_ABS_MAG,
pnum,
sources,
derived=True)
if (SUPERNOVA.MAX_VISUAL_ABS_MAG
not in catalog.entries[name]
and SUPERNOVA.MAX_VISUAL_APP_MAG
in catalog.entries[name]):
source = catalog.entries[name].add_self_source()
pnum = pretty_num(float(catalog.entries[name][
SUPERNOVA.MAX_VISUAL_APP_MAG][0][
QUANTITY.VALUE]) - 5.0 *
(log10(dl.to('pc').value) - 1.0),
sig=bestsig + 1)
catalog.entries[name].add_quantity(
SUPERNOVA.MAX_VISUAL_ABS_MAG,
pnum,
sources,
derived=True)
if SUPERNOVA.COMOVING_DIST not in catalog.entries[name]:
cd = cosmo.comoving_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
SUPERNOVA.COMOVING_DIST,
pretty_num(cd.value, sig=bestsig),
sources,
derived=True)
if SUPERNOVA.HOST_REDSHIFT in catalog.entries[name]:
# Find the "best" redshift to use for this
bestz, bestkind, bestsig, bestsrc = catalog.entries[
name].get_best_redshift(SUPERNOVA.HOST_REDSHIFT)
if bestsig > 0:
try:
bestz = float(bestz)
except Exception:
print(catalog.entries[name])
raise
if SUPERNOVA.HOST_VELOCITY not in catalog.entries[name]:
source = catalog.entries[name].add_self_source()
# FIX: what's happening here?!
pnum = CLIGHT / KM * \
((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.)
pnum = pretty_num(pnum, sig=bestsig)
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_VELOCITY,
pnum,
source,
kind=(SUPERNOVA.HOST_VELOCITY.kind_preference[bestkind]
if bestkind else ''))
if bestz > 0.:
if SUPERNOVA.HOST_LUM_DIST not in catalog.entries[name]:
dl = cosmo.luminosity_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_LUM_DIST,
pretty_num(dl.value, sig=bestsig + 1),
sources,
kind=(SUPERNOVA.HOST_LUM_DIST.
kind_preference[bestkind]
if bestkind else ''),
derived=True)
if SUPERNOVA.HOST_COMOVING_DIST not in catalog.entries[
name]:
cd = cosmo.comoving_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_COMOVING_DIST,
pretty_num(cd.value, sig=bestsig),
sources,
derived=True)
if all([
x in catalog.entries[name] for x in [
SUPERNOVA.RA, SUPERNOVA.DEC, SUPERNOVA.HOST_RA,
SUPERNOVA.HOST_DEC
]
]):
# For now just using first coordinates that appear in entry
try:
c1 = coord(
ra=catalog.entries[name][SUPERNOVA.RA][0][QUANTITY.VALUE],
dec=catalog.entries[name][SUPERNOVA.DEC][0][
QUANTITY.VALUE],
unit=(un.hourangle, un.deg))
c2 = coord(ra=catalog.entries[name][SUPERNOVA.HOST_RA][0][
QUANTITY.VALUE],
dec=catalog.entries[name][SUPERNOVA.HOST_DEC][0][
QUANTITY.VALUE],
unit=(un.hourangle, un.deg))
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
pass
else:
sources = uniq_cdl([catalog.entries[name].add_self_source()] +
catalog.entries[name][SUPERNOVA.RA][0][
QUANTITY.SOURCE].split(',') +
catalog.entries[name][SUPERNOVA.DEC][0][
QUANTITY.SOURCE].split(',') +
catalog.entries[name][SUPERNOVA.HOST_RA][0][
QUANTITY.SOURCE].split(',') +
catalog.entries[name][SUPERNOVA.HOST_DEC][0]
[QUANTITY.SOURCE].split(','))
if SUPERNOVA.HOST_OFFSET_ANG not in catalog.entries[name]:
hosa = Decimal(c1.separation(c2).arcsecond)
hosa = pretty_num(hosa)
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_OFFSET_ANG,
hosa,
sources,
derived=True,
u_value='arcseconds')
if (SUPERNOVA.COMOVING_DIST in catalog.entries[name]
and SUPERNOVA.REDSHIFT in catalog.entries[name]
and SUPERNOVA.HOST_OFFSET_DIST
not in catalog.entries[name]):
offsetsig = get_sig_digits(catalog.entries[name][
SUPERNOVA.HOST_OFFSET_ANG][0][QUANTITY.VALUE])
sources = uniq_cdl(
sources.split(',') +
(catalog.entries[name][SUPERNOVA.COMOVING_DIST][0][
QUANTITY.SOURCE]).split(',') +
(catalog.entries[name][SUPERNOVA.REDSHIFT][0][
QUANTITY.SOURCE]).split(','))
(catalog.entries[name].add_quantity(
SUPERNOVA.HOST_OFFSET_DIST,
pretty_num(
float(catalog.entries[name][
SUPERNOVA.HOST_OFFSET_ANG][0][QUANTITY.VALUE])
/ 3600. * (pi / 180.) *
float(catalog.entries[name][
SUPERNOVA.COMOVING_DIST][0][QUANTITY.VALUE]) *
1000. / (1.0 + float(catalog.entries[name][
SUPERNOVA.REDSHIFT][0][QUANTITY.VALUE])),
sig=offsetsig), sources))
catalog.entries[name].sanitize()
catalog.journal_entries(bury=True, final=True, gz=True)
cleanupcnt = cleanupcnt + 1
if catalog.args.travis and cleanupcnt % 1000 == 0:
break
catalog.save_caches()
return
def radec_clean(svalue, quantity, unit=''):
"""Clean R.A. and Dec."""
svalue = svalue.strip()
if unit == 'floatdegrees':
if not is_number(svalue):
return (svalue, unit)
deg = float('%g' % Decimal(svalue))
sig = get_sig_digits(svalue)
if 'ra' in quantity:
flhours = deg / 360.0 * 24.0
hours = floor(flhours)
minutes = floor((flhours - hours) * 60.0)
seconds = (flhours * 60.0 - (hours * 60.0 + minutes)) * 60.0
hours = 0 if hours < 1.e-6 else hours
minutes = 0 if minutes < 1.e-6 else minutes
seconds = 0.0 if seconds < 1.e-6 else seconds
if seconds > 60.0:
raise (ValueError('Invalid seconds value for ' + quantity))
svalue = str(hours).zfill(2) + ':' + str(minutes).zfill(2) + \
':' + zpad(pretty_num(seconds, sig=sig - 1))
elif 'dec' in quantity:
fldeg = abs(deg)
degree = floor(fldeg)
minutes = floor((fldeg - degree) * 60.0)
seconds = (fldeg * 60.0 - (degree * 60.0 + minutes)) * 60.0
minutes = 0 if minutes < 1.e-6 else minutes
seconds = 0.0 if seconds < 1.e-6 else seconds
if seconds > 60.0:
raise (ValueError('Invalid seconds value for ' + quantity))
svalue = (('+' if deg >= 0.0 else '-') +
str(degree).strip('+-').zfill(2) + ':' +
str(minutes).zfill(2) + ':' +
zpad(pretty_num(seconds, sig=sig - 1)))
elif unit == 'nospace' and 'ra' in quantity:
svalue = svalue[:2] + ':' + svalue[2:4] + \
((':' + zpad(svalue[4:])) if len(svalue) > 4 else '')
elif unit == 'nospace' and 'dec' in quantity:
if svalue.startswith(('+', '-')):
svalue = svalue[:3] + ':' + svalue[3:5] + \
((':' + zpad(svalue[5:])) if len(svalue) > 5 else '')
else:
svalue = '+' + svalue[:2] + ':' + svalue[2:4] + \
((':' + zpad(svalue[4:])) if len(svalue) > 4 else '')
else:
svalue = svalue.replace(' ', ':')
if 'dec' in quantity:
valuesplit = svalue.split(':')
svalue = (
('-' if valuesplit[0].startswith('-') else '+') +
valuesplit[0].strip('+-').zfill(2) +
(':' + valuesplit[1].zfill(2) if len(valuesplit) > 1 else '') +
(':' + zpad(valuesplit[2]) if len(valuesplit) > 2 else ''))
if 'ra' in quantity:
sunit = 'hours'
elif 'dec' in quantity:
sunit = 'degrees'
# Correct case of arcseconds = 60.0.
valuesplit = svalue.split(':')
if len(valuesplit) == 3 and valuesplit[-1] in ["60.0", "60.", "60"]:
svalue = valuesplit[0] + ':' + str(
Decimal(valuesplit[1]) + Decimal(1.0)) + ':' + "00.0"
# Strip trailing dots.
svalue = svalue.rstrip('.')
return (svalue, sunit)
def do_cleanup(catalog):
"""Task to cleanup catalog before final write."""
task_str = catalog.get_current_task_str()
# Set preferred names, calculate some columns based on imported data,
# sanitize some fields
keys = catalog.entries.copy().keys()
cleanupcnt = 0
for oname in pbar(keys, task_str):
name = catalog.add_entry(oname)
# Set the preferred name, switching to that name if name changed.
name = catalog.entries[name].set_preferred_name()
aliases = catalog.entries[name].get_aliases()
catalog.entries[name].set_first_max_light()
if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['MLS', 'SSS', 'CSS', 'GRB ']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix) and
is_number(alias.replace(prefix, '')[:2])):
discoverdate = ('/'.join([
'20' + alias.replace(prefix, '')[:2],
alias.replace(prefix, '')[2:4],
alias.replace(prefix, '')[4:6]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]:
break
if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]:
prefixes = [
'ASASSN-', 'PS1-', 'PS1', 'PS', 'iPTF', 'PTF', 'SCP-', 'SNLS-',
'SPIRITS', 'LSQ', 'DES', 'SNHiTS', 'Gaia', 'GND', 'GNW', 'GSD',
'GSW', 'EGS', 'COS', 'OGLE', 'HST'
]
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix) and
is_number(alias.replace(prefix, '')[:2]) and
is_number(alias.replace(prefix, '')[:1])):
discoverdate = '20' + alias.replace(prefix, '')[:2]
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]:
break
if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['SNF']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix) and
is_number(alias.replace(prefix, '')[:4])):
discoverdate = ('/'.join([
alias.replace(prefix, '')[:4],
alias.replace(prefix, '')[4:6],
alias.replace(prefix, '')[6:8]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]:
break
if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['PTFS', 'SNSDF']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix) and
is_number(alias.replace(prefix, '')[:2])):
discoverdate = ('/'.join([
'20' + alias.replace(prefix, '')[:2],
alias.replace(prefix, '')[2:4]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]:
break
if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['AT', 'SN', 'OGLE-', 'SM ', 'KSN-']
for alias in aliases:
for prefix in prefixes:
if alias.startswith(prefix):
year = re.findall(r'\d+', alias)
if len(year) == 1:
year = year[0]
else:
continue
if alias.replace(prefix, '').index(year) != 0:
continue
if (year and is_number(year) and '.' not in year and
len(year) <= 4):
discoverdate = year
if catalog.args.verbose:
tprint('Added discoverdate from name [' +
alias + ']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]:
break
if (TIDALDISRUPTION.RA not in catalog.entries[name] or
TIDALDISRUPTION.DEC not in catalog.entries[name]):
prefixes = [
'PSN J', 'MASJ', 'CSS', 'SSS', 'MASTER OT J', 'HST J', 'TCP J',
'MACS J', '2MASS J', 'EQ J', 'CRTS J', 'SMT J'
]
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix) and
is_number(alias.replace(prefix, '')[:6])):
noprefix = alias.split(':')[-1].replace(
prefix, '').replace('.', '')
decsign = '+' if '+' in noprefix else '-'
noprefix = noprefix.replace('+', '|').replace('-', '|')
nops = noprefix.split('|')
if len(nops) < 2:
continue
rastr = nops[0]
decstr = nops[1]
ra = ':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + \
('.' + rastr[6:] if len(rastr) > 6 else '')
dec = (decsign + ':'.join(
[decstr[:2], decstr[2:4], decstr[4:6]]) +
('.' + decstr[6:] if len(decstr) > 6 else ''))
if catalog.args.verbose:
tprint('Added ra/dec from name: ' + ra + ' ' + dec)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.RA, ra, source, derived=True)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DEC, dec, source, derived=True)
break
if TIDALDISRUPTION.RA in catalog.entries[name]:
break
no_host = (TIDALDISRUPTION.HOST not in catalog.entries[name] or
not any([
x[QUANTITY.VALUE] == 'Milky Way'
for x in catalog.entries[name][TIDALDISRUPTION.HOST]
]))
if (TIDALDISRUPTION.RA in catalog.entries[name] and
TIDALDISRUPTION.DEC in catalog.entries[name] and no_host):
from astroquery.irsa_dust import IrsaDust
if name not in catalog.extinctions_dict:
try:
ra_dec = (catalog.entries[name][TIDALDISRUPTION.RA][0][
QUANTITY.VALUE] + " " + catalog.entries[name][
TIDALDISRUPTION.DEC][0][QUANTITY.VALUE])
result = IrsaDust.get_query_table(ra_dec, section='ebv')
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
warnings.warn("Coordinate lookup for " + name +
" failed in IRSA.")
else:
ebv = result['ext SandF mean'][0]
ebverr = result['ext SandF std'][0]
catalog.extinctions_dict[name] = [ebv, ebverr]
if name in catalog.extinctions_dict:
sources = uniq_cdl([
catalog.entries[name].add_self_source(),
catalog.entries[name]
.add_source(bibcode='2011ApJ...737..103S')
])
(catalog.entries[name].add_quantity(
TIDALDISRUPTION.EBV,
str(catalog.extinctions_dict[name][0]),
sources,
e_value=str(catalog.extinctions_dict[name][1]),
derived=True))
if ((TIDALDISRUPTION.HOST in catalog.entries[name] and
(TIDALDISRUPTION.HOST_RA not in catalog.entries[name] or
TIDALDISRUPTION.HOST_DEC not in catalog.entries[name]))):
for host in catalog.entries[name][TIDALDISRUPTION.HOST]:
alias = host[QUANTITY.VALUE]
if ' J' in alias and is_number(alias.split(' J')[-1][:6]):
noprefix = alias.split(' J')[-1].split(':')[-1].replace(
'.', '')
decsign = '+' if '+' in noprefix else '-'
noprefix = noprefix.replace('+', '|').replace('-', '|')
nops = noprefix.split('|')
if len(nops) < 2:
continue
rastr = nops[0]
decstr = nops[1]
hostra = (':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) +
('.' + rastr[6:] if len(rastr) > 6 else ''))
hostdec = decsign + ':'.join([
decstr[:2], decstr[2:4], decstr[4:6]
]) + ('.' + decstr[6:] if len(decstr) > 6 else '')
if catalog.args.verbose:
tprint('Added hostra/hostdec from name: ' + hostra +
' ' + hostdec)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.HOST_RA, hostra, source, derived=True)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.HOST_DEC,
hostdec,
source,
derived=True)
break
if TIDALDISRUPTION.HOST_RA in catalog.entries[name]:
break
if (TIDALDISRUPTION.REDSHIFT not in catalog.entries[name] and
TIDALDISRUPTION.VELOCITY in catalog.entries[name]):
# Find the "best" velocity to use for this
bestsig = 0
for hv in catalog.entries[name][TIDALDISRUPTION.VELOCITY]:
sig = get_sig_digits(hv[QUANTITY.VALUE])
if sig > bestsig:
besthv = hv[QUANTITY.VALUE]
bestsrc = hv['source']
bestsig = sig
if bestsig > 0 and is_number(besthv):
voc = float(besthv) * 1.e5 / CLIGHT
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
(catalog.entries[name].add_quantity(
TIDALDISRUPTION.REDSHIFT,
pretty_num(
sqrt((1. + voc) / (1. - voc)) - 1., sig=bestsig),
sources,
kind='heliocentric',
derived=True))
if (TIDALDISRUPTION.REDSHIFT not in catalog.entries[name] and
len(catalog.nedd_dict) > 0 and
TIDALDISRUPTION.HOST in catalog.entries[name]):
reference = "NED-D"
refurl = "http://ned.ipac.caltech.edu/Library/Distances/"
for host in catalog.entries[name][TIDALDISRUPTION.HOST]:
if host[QUANTITY.VALUE] in catalog.nedd_dict:
source = catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
secondarysource = catalog.entries[name].add_source(
name=reference, url=refurl, secondary=True)
meddist = statistics.median(catalog.nedd_dict[host[
QUANTITY.VALUE]])
redz = z_at_value(cosmo.comoving_distance,
float(meddist) * un.Mpc)
redshift = pretty_num(
redz, sig=get_sig_digits(str(meddist)))
catalog.entries[name].add_quantity(
TIDALDISRUPTION.REDSHIFT,
redshift,
uniq_cdl([source, secondarysource]),
kind='host',
derived=True)
if (TIDALDISRUPTION.MAX_ABS_MAG not in catalog.entries[name] and
TIDALDISRUPTION.MAX_APP_MAG in catalog.entries[name] and
TIDALDISRUPTION.LUM_DIST in catalog.entries[name]):
# Find the "best" distance to use for this
bestsig = 0
for ld in catalog.entries[name][TIDALDISRUPTION.LUM_DIST]:
sig = get_sig_digits(ld[QUANTITY.VALUE])
if sig > bestsig:
bestld = ld[QUANTITY.VALUE]
bestsrc = ld['source']
bestsig = sig
if bestsig > 0 and is_number(bestld) and float(bestld) > 0.:
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
bestldz = z_at_value(cosmo.luminosity_distance,
float(bestld) * un.Mpc)
pnum = (float(catalog.entries[name][
TIDALDISRUPTION.MAX_APP_MAG][0][QUANTITY.VALUE]) - 5.0 *
(log10(float(bestld) * 1.0e6) - 1.0
) + 2.5 * log10(1.0 + bestldz))
pnum = pretty_num(pnum, sig=bestsig)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.MAX_ABS_MAG, pnum, sources, derived=True)
if TIDALDISRUPTION.REDSHIFT in catalog.entries[name]:
# Find the "best" redshift to use for this
bestz, bestkind, bestsig, bestsrc = catalog.entries[
name].get_best_redshift()
if bestsig > 0:
try:
bestz = float(bestz)
except Exception:
print(catalog.entries[name])
raise
if TIDALDISRUPTION.VELOCITY not in catalog.entries[name]:
source = catalog.entries[name].add_self_source()
# FIX: what's happening here?!
pnum = CLIGHT / KM * \
((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.)
pnum = pretty_num(pnum, sig=bestsig)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.VELOCITY,
pnum,
source,
kind=PREF_KINDS[bestkind],
derived=True)
if bestz > 0.:
from astropy.cosmology import Planck15 as cosmo
if TIDALDISRUPTION.LUM_DIST not in catalog.entries[name]:
dl = cosmo.luminosity_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name]
.add_source(bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
TIDALDISRUPTION.LUM_DIST,
pretty_num(
dl.value, sig=bestsig),
sources,
kind=PREF_KINDS[bestkind],
derived=True)
if (TIDALDISRUPTION.MAX_ABS_MAG not in
catalog.entries[name] and
TIDALDISRUPTION.MAX_APP_MAG in
catalog.entries[name]):
source = catalog.entries[name].add_self_source()
pnum = pretty_num(
float(catalog.entries[name][
TIDALDISRUPTION.MAX_APP_MAG][0][
QUANTITY.VALUE]) - 5.0 *
(log10(dl.to('pc').value) - 1.0
) + 2.5 * log10(1.0 + bestz),
sig=bestsig + 1)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.MAX_ABS_MAG,
pnum,
sources,
derived=True)
if TIDALDISRUPTION.COMOVING_DIST not in catalog.entries[
name]:
cd = cosmo.comoving_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name]
.add_source(bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
TIDALDISRUPTION.COMOVING_DIST,
pretty_num(
cd.value, sig=bestsig),
sources,
derived=True)
if all([
x in catalog.entries[name]
for x in [
TIDALDISRUPTION.RA, TIDALDISRUPTION.DEC,
TIDALDISRUPTION.HOST_RA, TIDALDISRUPTION.HOST_DEC
]
]):
# For now just using first coordinates that appear in entry
try:
c1 = coord(
ra=catalog.entries[name][TIDALDISRUPTION.RA][0][
QUANTITY.VALUE],
dec=catalog.entries[name][TIDALDISRUPTION.DEC][0][
QUANTITY.VALUE],
unit=(un.hourangle, un.deg))
c2 = coord(
ra=catalog.entries[name][TIDALDISRUPTION.HOST_RA][0][
QUANTITY.VALUE],
dec=catalog.entries[name][TIDALDISRUPTION.HOST_DEC][0][
QUANTITY.VALUE],
unit=(un.hourangle, un.deg))
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
pass
else:
sources = uniq_cdl(
[catalog.entries[name].add_self_source()] + catalog.
entries[name][TIDALDISRUPTION.RA][0]['source'].split(',') +
catalog.entries[name][TIDALDISRUPTION.DEC][0]['source'].
split(',') + catalog.entries[name][TIDALDISRUPTION.HOST_RA]
[0]['source'].split(',') + catalog.entries[name][
TIDALDISRUPTION.HOST_DEC][0]['source'].split(','))
if 'hostoffsetang' not in catalog.entries[name]:
hosa = Decimal(
hypot(c1.ra.degree - c2.ra.degree, c1.dec.degree -
c2.dec.degree))
hosa = pretty_num(hosa * Decimal(3600.))
catalog.entries[name].add_quantity(
TIDALDISRUPTION.HOST_OFFSET_ANG,
hosa,
sources,
derived=True,
u_value='arcseconds')
if (TIDALDISRUPTION.COMOVING_DIST in catalog.entries[name] and
TIDALDISRUPTION.REDSHIFT in catalog.entries[name] and
TIDALDISRUPTION.HOST_OFFSET_DIST not in
catalog.entries[name]):
offsetsig = get_sig_digits(catalog.entries[name][
TIDALDISRUPTION.HOST_OFFSET_ANG][0][QUANTITY.VALUE])
sources = uniq_cdl(
sources.split(',') + (catalog.entries[name][
TIDALDISRUPTION.COMOVING_DIST][0]['source']).
split(',') + (catalog.entries[name][
TIDALDISRUPTION.REDSHIFT][0]['source']).split(','))
(catalog.entries[name].add_quantity(
TIDALDISRUPTION.HOST_OFFSET_DIST,
pretty_num(
float(catalog.entries[name][
TIDALDISRUPTION.HOST_OFFSET_ANG][0][
QUANTITY.VALUE]) / 3600. * (pi / 180.) *
float(catalog.entries[name][
TIDALDISRUPTION.COMOVING_DIST][0][
QUANTITY.VALUE]) * 1000. /
(1.0 + float(catalog.entries[name][
TIDALDISRUPTION.REDSHIFT][0][QUANTITY.VALUE])),
sig=offsetsig),
sources))
catalog.entries[name].sanitize()
catalog.journal_entries(bury=True, final=True, gz=True)
cleanupcnt = cleanupcnt + 1
if catalog.args.travis and cleanupcnt % 1000 == 0:
break
catalog.save_caches()
return
def do_ptf(catalog):
# response =
# urllib.request.urlopen('http://wiserep.weizmann.ac.il/objects/list')
# bs = BeautifulSoup(response, 'html5lib')
# select = bs.find('select', {'name': 'objid'})
# options = select.findAll('option')
# for option in options:
# print(option.text)
# name = option.text
# if ((name.startswith('PTF') and is_number(name[3:5])) or
# name.startswith('PTFS') or name.startswith('iPTF')):
# name = catalog.add_entry(name)
task_str = catalog.get_current_task_str()
html = catalog.load_url(
'http://wiserep.weizmann.ac.il/spectra/update',
os.path.join(catalog.get_current_task_repo(), 'PTF/update.html'))
bs = BeautifulSoup(html, 'html5lib')
select = bs.find('select', {'name': 'objid'})
options = select.findAll('option')
for option in pbar(options, task_str):
name = option.text
if (((name.startswith('PTF') and is_number(name[3:5]))
or name.startswith('PTFS') or name.startswith('iPTF'))):
if '(' in name:
alias = name.split('(')[0].strip(' ')
name = name.split('(')[-1].strip(') ').replace('sn', 'SN')
if name == 'SNiauname': # A misentered entry
continue
name, source = catalog.new_entry(name,
bibcode='2012PASP..124..668Y')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, alias,
source)
else:
# name = catalog.add_entry(name)
name, source = catalog.new_entry(name,
bibcode='2012PASP..124..668Y')
with open(
os.path.join(catalog.get_current_task_repo(),
'PTF/old-ptf-events.csv')) as f:
for suffix in pbar(f.read().splitlines(), task_str):
name = catalog.add_entry('PTF' + suffix)
with open(
os.path.join(catalog.get_current_task_repo(),
'PTF/perly-2016.csv')) as f:
for row in pbar(f.read().splitlines(), task_str):
cols = [x.strip() for x in row.split(',')]
alias = ''
if cols[8]:
name = cols[8]
alias = 'PTF' + cols[0]
else:
name = 'PTF' + cols[0]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2016ApJ...830...13P')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
if alias:
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, alias,
source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, cols[1], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, cols[2], source)
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE,
'SLSN-' + cols[3], source)
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT,
cols[4],
source,
kind='spectroscopic')
maxdate = cols[6].replace('-', '/')
upl = maxdate.startswith('<')
catalog.entries[name].add_quantity(SUPERNOVA.MAX_DATE,
maxdate.lstrip('<'),
source,
upperlimit=upl)
catalog.entries[name].add_quantity(SUPERNOVA.EBV,
cols[7],
source,
kind='spectroscopic')
name = catalog.add_entry('PTF' + suffix)
catalog.journal_entries()
return
def _clean_quantity(self, quantity):
"""Clean quantity value before it is added to entry.
"""
value = quantity.get(QUANTITY.VALUE, '').strip()
error = quantity.get(QUANTITY.E_VALUE, '').strip()
unit = quantity.get(QUANTITY.U_VALUE, '').strip()
kinds = [x.strip() for x in listify(quantity.get(QUANTITY.KIND, []))]
key = quantity._key
if not value:
return False
if error and (not is_number(error) or float(error) < 0):
raise ValueError(self[self._KEYS.NAME] + "'s quanta " + key +
' error value must be a number and positive.')
# Set default units
if not unit and key == self._KEYS.VELOCITY:
unit = 'km/s'
if not unit and key == self._KEYS.RA:
unit = 'hours'
if not unit and key == self._KEYS.DEC:
unit = 'degrees'
if not unit and key in [self._KEYS.LUM_DIST, self._KEYS.COMOVING_DIST]:
unit = 'Mpc'
# Handle certain name
if key == self._KEYS.ALIAS:
value = self.catalog.clean_entry_name(value)
for df in quantity.get(self._KEYS.DISTINCT_FROM, []):
if value == df[QUANTITY.VALUE]:
return False
elif key == self._KEYS.HOST:
if is_number(value):
return False
if value.lower() in [
'anonymous', 'anon.', 'anon', 'intergalactic'
]:
return False
value = host_clean(value)
if ((not kinds and ((value.lower().startswith('abell')
and is_number(value[5:].strip()))
or 'cluster' in value.lower()))):
kinds = ['cluster']
elif key == self._KEYS.HOST_REDSHIFT:
kinds = list(filter(lambda x: x != 'host', kinds))
elif key == self._KEYS.CLAIMED_TYPE:
isq = False
if value.startswith('SN '):
value = value.replace('SN ', '', 1)
value = value.replace('young', '')
if '?' in value:
isq = True
value = value.strip(' ?')
for rep in self.catalog.type_syns:
if value in self.catalog.type_syns[rep]:
value = rep
break
if isq:
value = value + '?'
if not value:
return False
elif key in [
self._KEYS.RA, self._KEYS.DEC, self._KEYS.HOST_RA,
self._KEYS.HOST_DEC
]:
(value, unit) = radec_clean(value, key, unit=unit)
elif key == self._KEYS.MAX_DATE or key == self._KEYS.DISCOVER_DATE:
# Make sure month and day have leading zeroes
sparts = value.split('/')
if len(sparts[0]) > 5:
self._log.warn("Date year {} greater than four "
"digits.".format(sparts[0]))
if len(sparts) >= 2:
value = sparts[0] + '/' + sparts[1].zfill(2)
if len(sparts) == 3:
value = value + '/' + sparts[2].zfill(2)
# for ii, ct in enumerate(self.parent[key]):
# # Only add dates if they have more information
# if len(ct[QUANTITY.VALUE].split('/')) >
# len(value.split('/')):
# return False
if is_number(value):
value = '%g' % Decimal(value)
if error:
error = '%g' % Decimal(error)
if value:
quantity[QUANTITY.VALUE] = value
if error:
quantity[QUANTITY.E_VALUE] = error
if unit:
quantity[QUANTITY.U_VALUE] = unit
if kinds:
quantity[QUANTITY.KIND] = kinds if len(kinds) > 1 else kinds[0]
elif QUANTITY.KIND in quantity:
del (quantity[QUANTITY.KIND])
return True
def do_rochester(catalog):
"""Import data from latest supernova page."""
rochestermirrors = [
'http://www.rochesterastronomy.org/',
'http://www.supernova.thistlethwaites.com/'
]
rochesterpaths = [
'snimages/snredshiftall.html', 'sn2020/snredshift.html',
'snimages/snredboneyard.html', 'snimages/snredboneyard-old.html'
]
rochesterupdate = [False, True, True, False]
task_str = catalog.get_current_task_str()
baddates = ['2440587', '2440587.292', '0001/01/01']
for pp, path in enumerate(pbar(rochesterpaths, task_str)):
if catalog.args.update and not rochesterupdate[pp]:
continue
if 'snredboneyard.html' in path:
cns = {
'name': 0,
'host': 1,
'ra': 2,
'dec': 3,
'type': 7,
'z': 8,
'mmag': 9,
'max': 10,
'disc': 11,
'ref': 12,
'dver': 13,
'aka': 14
}
else:
cns = {
'name': 0,
'type': 1,
'host': 2,
'ra': 3,
'dec': 4,
'disc': 6,
'max': 7,
'mmag': 8,
'z': 11,
'zh': 12,
'ref': 13,
'dver': 14,
'aka': 15
}
filepath = (
os.path.join(catalog.get_current_task_repo(), 'rochester/') +
path.replace('/', '-'))
for mirror in rochestermirrors:
html = catalog.load_url(mirror + path,
filepath,
fail=(mirror != rochestermirrors[-1]))
if html:
break
if not html:
continue
soup = BeautifulSoup(html, 'html5lib')
rows = soup.findAll('tr')
sec_ref = 'Latest Supernovae'
sec_refurl = ('http://www.rochesterastronomy.org/'
'snimages/snredshiftall.html')
loopcnt = 0
for rr, row in enumerate(pbar(rows, task_str)):
if rr == 0:
continue
cols = row.findAll('td')
if not len(cols):
continue
name = ''
if cols[cns['aka']].contents:
for rawaka in str(cols[cns['aka']].contents[0]).split(','):
aka = rawaka.strip()
if is_number(aka.strip('?')):
aka = 'SN' + aka.strip('?') + 'A'
oldname = aka
name = catalog.add_entry(aka)
elif len(aka) == 4 and is_number(aka[:4]):
aka = 'SN' + aka
oldname = aka
name = catalog.add_entry(aka)
sn = re.sub('<[^<]+?>', '',
str(cols[cns['name']].contents[0])).strip()
if is_number(sn.strip('?')):
sn = 'SN' + sn.strip('?') + 'A'
elif len(sn) == 4 and is_number(sn[:4]):
sn = 'SN' + sn
if not name:
if not sn or sn in ['Transient']:
continue
ra = str(cols[cns['ra']].contents[0]).strip().replace(':.', '.')
dec = str(cols[cns['dec']].contents[0]).strip().replace(':.', '.')
if not name:
if sn[:8] == 'MASTER J':
sn = sn.replace('MASTER J',
'MASTER OT J').replace('SNHunt', 'SNhunt')
if 'POSSIBLE' in sn.upper() and ra and dec:
sn = 'PSN J' + ra.replace(':', '').replace('.', '')
sn += dec.replace(':', '').replace('.', '')
oldname = sn
name = catalog.add_entry(sn)
sec_source = catalog.entries[name].add_source(name=sec_ref,
url=sec_refurl,
secondary=True)
sources = []
if 'ref' in cns:
reftag = reference = cols[cns['ref']].findAll('a')
if len(reftag) and len(reftag[0].contents):
reference = reftag[0].contents[0].strip()
refurl = reftag[0]['href'].strip()
sources.append(catalog.entries[name].add_source(
name=reference, url=refurl))
sources.append(sec_source)
sources = uniq_cdl(list(filter(None, sources)))
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oldname,
sources)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, sn, sources)
if cols[cns['aka']].contents:
for rawaka in str(cols[cns['aka']].contents[0]).split(','):
aka = rawaka.strip()
if aka == 'SNR G1.9+0.3':
aka = 'G001.9+00.3'
if aka[:4] == 'PS1 ':
aka = 'PS1-' + aka[4:]
if aka[:8] == 'MASTER J':
aka = aka.replace('MASTER J', 'MASTER OT J').replace(
'SNHunt', 'SNhunt')
if 'POSSIBLE' in aka.upper() and ra and dec:
aka = 'PSN J' + ra.replace(':', '').replace('.', '')
aka += dec.replace(':', '').replace('.', '')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, aka,
sources)
if (len(cols[cns['type']].contents) > 0
and str(cols[cns['type']].contents[0]).strip() != 'unk'):
type = str(cols[cns['type']].contents[0]).strip(' :,')
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE,
type, sources)
if (len(cols[cns['host']].contents) > 0 and
str(cols[cns['host']].contents[0]).strip() != 'anonymous'):
catalog.entries[name].add_quantity(
SUPERNOVA.HOST,
str(cols[cns['host']].contents[0]).strip(), sources)
catalog.entries[name].add_quantity(SUPERNOVA.RA, ra, sources)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, dec, sources)
discstr = str(cols[cns['disc']].contents[0]).strip()
if discstr and discstr not in baddates:
if '/' not in discstr:
astrot = astrotime(float(discstr), format='jd').datetime
ddate = make_date_string(astrot.year, astrot.month,
astrot.day)
else:
ddate = discstr
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE,
ddate, sources)
maxstr = str(cols[cns.get('max', '')].contents[0]).strip()
if maxstr and maxstr not in baddates:
try:
if '/' not in maxstr:
astrot = astrotime(float(maxstr), format='jd')
else:
astrot = astrotime(maxstr.replace('/', '-'),
format='iso')
except:
catalog.log.info(
'Max date conversion failed for `{}`.'.format(maxstr))
if ((float(str(cols[cns['mmag']].contents[0]).strip()) <= 90.0
and
not any('GRB' in xx
for xx in catalog.entries[name].get_aliases()))):
mag = str(cols[cns['mmag']].contents[0]).strip()
catalog.entries[name].add_photometry(time=str(astrot.mjd),
u_time='MJD',
magnitude=mag,
source=sources)
if 'z' in cns and cols[cns['z']].contents[0] != 'n/a':
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT,
str(cols[cns['z']].contents[0]).strip(), sources)
if 'zh' in cns:
zhost = str(cols[cns['zh']].contents[0]).strip()
if is_number(zhost):
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT,
zhost, sources)
if 'dver' in cns:
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVERER,
str(cols[cns['dver']].contents[0]).strip(), sources)
if catalog.args.update:
catalog.journal_entries()
loopcnt = loopcnt + 1
if (catalog.args.travis
and loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
if not catalog.args.update:
vsnetfiles = ['latestsne.dat']
for vsnetfile in vsnetfiles:
file_name = os.path.join(catalog.get_current_task_repo(),
"" + vsnetfile)
with open(file_name, 'r', encoding='latin1') as csv_file:
tsvin = csv.reader(csv_file,
delimiter=' ',
skipinitialspace=True)
loopcnt = 0
for rr, row in enumerate(tsvin):
if (not row or row[0] in ['Transient']
or row[0][:4] in ['http', 'www.'] or len(row) < 3):
continue
name = row[0].strip()
if name[:4].isdigit():
name = 'SN' + name
if name.startswith('PSNJ'):
name = 'PSN J' + name[4:]
if name.startswith('MASTEROTJ'):
name = name.replace('MASTEROTJ', 'MASTER OT J')
name = catalog.add_entry(name)
sec_source = catalog.entries[name].add_source(
name=sec_ref, url=sec_refurl, secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name,
sec_source)
if not is_number(row[1]):
continue
year = row[1][:4]
month = row[1][4:6]
day = row[1][6:]
if '.' not in day:
day = day[:2] + '.' + day[2:]
mjd = astrotime(year + '-' + month + '-' +
str(floor(float(day))).zfill(2)).mjd
mjd += float(day) - floor(float(day))
magnitude = row[2].rstrip(ascii_letters)
if not is_number(magnitude):
continue
if magnitude.isdigit():
if int(magnitude) > 100:
magnitude = magnitude[:2] + '.' + magnitude[2:]
if float(str(cols[8].contents[0]).strip()) >= 90.0:
continue
if len(row) >= 4:
if is_number(row[3]):
e_magnitude = row[3]
refind = 4
else:
e_magnitude = ''
refind = 3
if refind >= len(row):
sources = sec_source
else:
reference = ' '.join(row[refind:])
source = catalog.entries[name].add_source(
name=reference)
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, name, sec_source)
sources = uniq_cdl([source, sec_source])
else:
sources = sec_source
band = row[2].lstrip('1234567890.')
catalog.entries[name].add_photometry(
time=mjd,
u_time='MJD',
band=band,
magnitude=magnitude,
e_magnitude=e_magnitude,
source=sources)
if (catalog.args.travis
and loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
def radec_clean(svalue, quantity, unit=''):
if unit == 'floatdegrees':
if not is_number(svalue):
return (svalue, unit)
deg = float('%g' % Decimal(svalue))
sig = get_sig_digits(svalue)
if 'ra' in quantity:
flhours = deg / 360.0 * 24.0
hours = floor(flhours)
minutes = floor((flhours - hours) * 60.0)
seconds = (flhours * 60.0 - (hours * 60.0 + minutes)) * 60.0
hours = 0 if hours < 1.e-6 else hours
minutes = 0 if minutes < 1.e-6 else minutes
seconds = 0.0 if seconds < 1.e-6 else seconds
if seconds > 60.0:
raise(ValueError('Invalid seconds value for ' + quantity))
svalue = str(hours).zfill(2) + ':' + str(minutes).zfill(2) + \
':' + zpad(pretty_num(seconds, sig=sig - 1))
elif 'dec' in quantity:
fldeg = abs(deg)
degree = floor(fldeg)
minutes = floor((fldeg - degree) * 60.0)
seconds = (fldeg * 60.0 - (degree * 60.0 + minutes)) * 60.0
if seconds > 60.0:
raise(ValueError('Invalid seconds value for ' + quantity))
svalue = (('+' if deg >= 0.0 else '-') +
str(degree).strip('+-').zfill(2) + ':' +
str(minutes).zfill(2) + ':' +
zpad(pretty_num(seconds, sig=sig - 1)))
elif unit == 'nospace' and 'ra' in quantity:
svalue = svalue[:2] + ':' + svalue[2:4] + \
((':' + zpad(svalue[4:])) if len(svalue) > 4 else '')
elif unit == 'nospace' and 'dec' in quantity:
if svalue.startswith(('+', '-')):
svalue = svalue[:3] + ':' + svalue[3:5] + \
((':' + zpad(svalue[5:])) if len(svalue) > 5 else '')
else:
svalue = '+' + svalue[:2] + ':' + svalue[2:4] + \
((':' + zpad(svalue[4:])) if len(svalue) > 4 else '')
else:
svalue = svalue.replace(' ', ':')
if 'dec' in quantity:
valuesplit = svalue.split(':')
svalue = (('-' if valuesplit[0].startswith('-') else '+') +
valuesplit[0].strip('+-').zfill(2) +
(':' + valuesplit[1].zfill(2) if
len(valuesplit) > 1 else '') +
(':' + zpad(valuesplit[2]) if
len(valuesplit) > 2 else ''))
if 'ra' in quantity:
sunit = 'hours'
elif 'dec' in quantity:
sunit = 'degrees'
# Correct case of arcseconds = 60.0.
valuesplit = svalue.split(':')
if len(valuesplit) == 3 and valuesplit[-1] in ["60.0", "60.", "60"]:
svalue = valuesplit[0] + ':' + str(Decimal(valuesplit[1]) +
Decimal(1.0)) + ':' + "00.0"
# Strip trailing dots.
svalue = svalue.rstrip('.')
return (svalue, sunit)
def do_wiserep_spectra(catalog):
#if not catalog.args.travis:
# from ..input.WISeWEBSpider.wisewebspider import spider
# try:
# spider(update=True, daysago=7, path="/../../sne-external-WISEREP/")
# except:
# catalog.log.warning(
# 'Spider errored, continuing without letting it complete.')
task_str = catalog.get_current_task_str()
secondaryreference = 'WISeREP'
secondaryrefurl = 'http://wiserep.weizmann.ac.il/'
secondarybibcode = '2012PASP..124..668Y'
wiserepcnt = 0
# These are known to be in error on the WISeREP page, either fix or ignore
# them.
wiserepbibcorrectdict = {
'2000AJ....120..367G]': '2000AJ....120..367G',
'Harutyunyan et al. 2008': '2008A&A...488..383H',
'0609268': '2007AJ....133...58K',
'2006ApJ...636...400Q': '2006ApJ...636..400Q',
'2011ApJ...741...76': '2011ApJ...741...76C',
'2016PASP...128...961': '2016PASP..128...961',
'2002AJ....1124..417H': '2002AJ....1124.417H',
'2013ApJ…774…58D': '2013ApJ...774...58D',
'2011Sci.333..856S': '2011Sci...333..856S',
'2014MNRAS.438,368': '2014MNRAS.438..368T',
'2012MNRAS.420.1135': '2012MNRAS.420.1135S',
'2012Sci..337..942D': '2012Sci...337..942D',
'stt1839': '2013MNRAS.436.3614S',
'arXiv:1605.03136': '2016MNRAS.460.3447T',
'10.1093/mnras/stt1839': '2013MNRAS.436.3614S'
}
file_names = list(glob(os.path.join(catalog.get_current_task_repo(), '*')))
for folder in pbar_strings(file_names, task_str):
if '.txt' in folder or '.json' in folder:
continue
name = os.path.basename(folder).strip()
if name.startswith('sn'):
name = 'SN' + name[2:]
if (name.startswith(('CSS', 'SSS', 'MLS')) and ':' not in name):
name = name.replace('-', ':', 1)
if name.startswith('MASTERJ'):
name = name.replace('MASTERJ', 'MASTER OT J')
if name.startswith('PSNJ'):
name = name.replace('PSNJ', 'PSN J')
name = catalog.add_entry(name)
secondarysource = catalog.entries[name].add_source(
name=secondaryreference,
url=secondaryrefurl,
bibcode=secondarybibcode,
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name,
secondarysource)
readme_path = os.path.join(folder, 'README.json')
if not os.path.exists(readme_path):
catalog.log.warning(
'Metadata file not found for event "{}"'.format(name))
continue
with open(readme_path, 'r') as f:
fileinfo = json.loads(f.read())
files = list(
set(glob(folder + '/*')) - set(glob(folder + '/README.json')))
for fname in pbar(files, task_str):
specfile = os.path.basename(fname)
if specfile not in fileinfo:
catalog.log.warning(
'Metadata not found for "{}"'.format(fname))
continue
claimedtype = fileinfo[specfile]["Type"]
instrument = fileinfo[specfile]["Instrument"]
epoch = fileinfo[specfile]["Obs. Date"]
observer = fileinfo[specfile]["Observer"]
reducer = fileinfo[specfile]["Reducer"]
bibcode = fileinfo[specfile]["Bibcode"]
redshift = fileinfo[specfile]["Redshift"]
survey = fileinfo[specfile]["Program"]
reduction = fileinfo[specfile]["Reduction Status"]
if bibcode:
newbibcode = bibcode
if bibcode in wiserepbibcorrectdict:
newbibcode = wiserepbibcorrectdict[bibcode]
if newbibcode and len(newbibcode) == 19:
source = catalog.entries[name].add_source(
bibcode=unescape(newbibcode))
else:
bibname = unescape(bibcode)
source = catalog.entries[name].add_source(name=bibname)
catalog.log.warning('Bibcode "{}" is invalid, using as '
'`{}` instead'.format(
bibname, SOURCE.NAME))
sources = uniq_cdl([source, secondarysource])
else:
sources = secondarysource
if claimedtype not in ['Other']:
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE,
claimedtype,
secondarysource)
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, redshift,
secondarysource)
with open(fname, 'r') as f:
data = [x.split() for x in f]
skipspec = False
newdata = []
oldval = ''
for row in data:
if row and '#' not in row[0]:
if (len(row) >= 2 and is_number(row[0])
and is_number(row[1]) and row[1] != oldval):
newdata.append(row)
oldval = row[1]
if skipspec or not newdata:
warnings.warn('Skipped adding spectrum file ' + specfile)
continue
data = [list(i) for i in zip(*newdata)]
wavelengths = data[0]
fluxes = data[1]
errors = ''
if len(data) == 3:
errors = data[1]
time = str(astrotime(epoch).mjd)
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom',
errors=errors,
u_fluxes=fluxunit,
u_errors=fluxunit if errors else '',
wavelengths=wavelengths,
fluxes=fluxes,
u_time='MJD',
time=time,
instrument=instrument,
source=sources,
observer=observer,
reducer=reducer,
reduction=reduction,
filename=specfile,
survey=survey,
redshift=redshift)
catalog.journal_entries()
wiserepcnt = wiserepcnt + 1
if (catalog.args.travis
and wiserepcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
return
def name_clean(name):
newname = name.strip(' ;,*')
if newname.startswith('NAME '):
newname = newname.replace('NAME ', '', 1)
if newname.endswith(' SN'):
newname = newname.replace(' SN', '')
if newname.endswith(':SN'):
newname = newname.replace(':SN', '')
if newname.startswith('MASJ'):
newname = newname.replace('MASJ', 'MASTER OT J', 1)
if newname.startswith('MASTER') and is_number(newname[7]):
newname = newname.replace('MASTER', 'MASTER OT J', 1)
if newname.startswith('MASTER OT J '):
newname = newname.replace('MASTER OT J ', 'MASTER OT J', 1)
if newname.startswith('OGLE '):
newname = newname.replace('OGLE ', 'OGLE-', 1)
if newname.startswith('OGLE-') and len(newname) != 16:
namesp = newname.split('-')
if (len(namesp[1]) == 4 and is_number(namesp[1]) and
is_number(namesp[3])):
newname = 'OGLE-' + namesp[1] + '-SN-' + namesp[3].zfill(3)
if newname.startswith('SN SDSS'):
newname = newname.replace('SN SDSS ', 'SDSS', 1)
if newname.startswith('SDSS '):
newname = newname.replace('SDSS ', 'SDSS', 1)
if newname.startswith('SDSS'):
namesp = newname.split('-')
if (len(namesp) == 3 and is_number(namesp[0][4:]) and
is_number(namesp[1]) and is_number(namesp[2])):
newname = namesp[0] + '-' + namesp[1] + '-' + namesp[2].zfill(3)
if newname.startswith('SDSS-II SN'):
namesp = newname.split()
if len(namesp) == 3 and is_number(namesp[2]):
newname = 'SDSS-II SN ' + namesp[2].lstrip('0')
if newname.startswith('SN CL'):
newname = newname.replace('SN CL', 'CL', 1)
if newname.startswith('SN HiTS '):
newname = newname.replace('SN HiTS ', 'SNHiTS', 1)
if newname.startswith('GAIA'):
newname = newname.replace('GAIA', 'Gaia', 1)
if newname.startswith('Gaia '):
newname = newname.replace('Gaia ', 'Gaia', 1)
if newname.startswith('Gaia'):
newname = 'Gaia' + newname[4:].lower()
if newname.startswith('GRB'):
newname = newname.replace('GRB', 'GRB ', 1)
if newname.startswith('GRB ') and is_number(newname[4:].strip()):
newname = 'GRB ' + newname[4:].strip() + 'A'
if newname.startswith('LSQ '):
newname = newname.replace('LSQ ', 'LSQ', 1)
if newname.startswith('KSN '):
newname = newname.replace('KSN ', 'KSN-', 1)
if newname.startswith('SNSDF '):
newname = newname.replace(' ', '')
if newname.startswith('SNSDF'):
namesp = newname.split('.')
if len(namesp[0]) == 9:
newname = namesp[0] + '-' + namesp[1].zfill(2)
if newname.startswith('HFF '):
newname = newname.replace(' ', '')
if newname.startswith('SN HST'):
newname = newname.replace('SN HST', 'HST', 1)
if newname.startswith('HST ') and newname[4] != 'J':
newname = newname.replace('HST ', 'HST J', 1)
if newname.startswith('SNLS') and newname[4] != '-':
newname = newname.replace('SNLS', 'SNLS-', 1)
if newname.startswith('SNLS- '):
newname = newname.replace('SNLS- ', 'SNLS-', 1)
if newname.startswith('CRTS CSS'):
newname = newname.replace('CRTS CSS', 'CSS', 1)
if newname.startswith('CRTS MLS'):
newname = newname.replace('CRTS MLS', 'MLS', 1)
if newname.startswith('CRTS SSS'):
newname = newname.replace('CRTS SSS', 'SSS', 1)
if newname.startswith(('CSS', 'MLS', 'SSS')):
newname = newname.replace(' ', ':').replace('J', '')
if newname.startswith('SN HFF'):
newname = newname.replace('SN HFF', 'HFF', 1)
if newname.startswith('SN GND'):
newname = newname.replace('SN GND', 'GND', 1)
if newname.startswith('SN SCP'):
newname = newname.replace('SN SCP', 'SCP', 1)
if newname.startswith('SN UDS'):
newname = newname.replace('SN UDS', 'UDS', 1)
if newname.startswith('SCP') and newname[3] != '-':
newname = newname.replace('SCP', 'SCP-', 1)
if newname.startswith('SCP- '):
newname = newname.replace('SCP- ', 'SCP-', 1)
if newname.startswith('PS 1'):
newname = newname.replace('PS 1', 'PS1', 1)
if newname.startswith('PS1 SN PS'):
newname = newname.replace('PS1 SN PS', 'PS', 1)
if newname.startswith('PS1 SN'):
newname = newname.replace('PS1 SN', 'PS1', 1)
if newname.startswith('PSN K'):
newname = newname.replace('PSN K', 'K', 1)
if newname.startswith('K') and is_number(newname[1:5]):
namesp = newname.split('-')
if len(namesp[0]) == 5:
newname = namesp[0] + '-' + namesp[1].zfill(3)
if newname.startswith('Psn'):
newname = newname.replace('Psn', 'PSN', 1)
if newname.startswith('PSNJ'):
newname = newname.replace('PSNJ', 'PSN J', 1)
if newname.startswith('TCPJ'):
newname = newname.replace('TCPJ', 'TCP J', 1)
if newname.startswith('SMTJ'):
newname = newname.replace('SMTJ', 'SMT J', 1)
if newname.startswith('PSN20J'):
newname = newname.replace('PSN20J', 'PSN J', 1)
if newname.startswith('SN ASASSN'):
newname = newname.replace('SN ASASSN', 'ASASSN', 1)
if newname.startswith('ASASSN '):
newname = newname.replace('ASASSN ', 'ASASSN-', 1).replace('--', '-')
if newname.startswith('ASASSN') and newname[6] != '-':
newname = newname.replace('ASASSN', 'ASASSN-', 1)
if newname.startswith('ROTSE3J'):
newname = newname.replace('ROTSE3J', 'ROTSE3 J', 1)
if newname.startswith('MACSJ'):
newname = newname.replace('MACSJ', 'MACS J', 1)
if newname.startswith('MWSNR'):
newname = newname.replace('MWSNR', 'MWSNR ', 1)
if newname.startswith('SN HUNT'):
newname = newname.replace('SN HUNT', 'SNhunt', 1)
if newname.startswith('SN Hunt'):
newname = newname.replace(' ', '')
if newname.startswith('SNHunt'):
newname = newname.replace('SNHunt', 'SNhunt', 1)
if newname.startswith('SNhunt '):
newname = newname.replace('SNhunt ', 'SNhunt', 1)
if newname.startswith('ptf'):
newname = newname.replace('ptf', 'PTF', 1)
if newname.startswith('SN PTF'):
newname = newname.replace('SN PTF', 'PTF', 1)
if newname.startswith('PTF '):
newname = newname.replace('PTF ', 'PTF', 1)
if newname.startswith('iPTF '):
newname = newname.replace('iPTF ', 'iPTF', 1)
if newname.startswith('PTF-'):
newname = newname.replace('PTF-', 'PTF', 1)
if newname.startswith('PESSTOESO'):
newname = newname.replace('PESSTOESO', 'PESSTO ESO ', 1)
if newname.startswith('snf'):
newname = newname.replace('snf', 'SNF', 1)
if newname.startswith('SNF '):
newname = newname.replace('SNF ', 'SNF', 1)
if (newname.startswith('SNF') and
is_number(newname[3:]) and len(newname) >= 12):
newname = 'SNF' + newname[3:11] + '-' + newname[11:]
if newname.startswith(('MASTER OT J', 'ROTSE3 J')):
prefix = newname.split('J')[0]
coords = newname.split('J')[-1].strip()
decsign = '+' if '+' in coords else '-'
coordsplit = coords.replace('+', '-').split('-')
if ('.' not in coordsplit[0] and
len(coordsplit[0]) > 6 and '.' not in coordsplit[1] and
len(coordsplit[1]) > 6):
newname = (prefix + 'J' + coordsplit[0][:6] + '.' +
coordsplit[0][6:] + decsign + coordsplit[1][:6] +
'.' + coordsplit[1][6:])
if (newname.startswith('Gaia ') and
is_number(newname[3:4]) and len(newname) > 5):
newname = newname.replace('Gaia ', 'Gaia', 1)
if len(newname) <= 4 and is_number(newname):
newname = 'SN' + newname + 'A'
if (len(newname) > 4 and is_number(newname[:4]) and not
is_number(newname[4:])):
newname = 'SN' + newname
if (newname.startswith('Sn ') and
is_number(newname[3:7]) and len(newname) > 7):
newname = newname.replace('Sn ', 'SN', 1)
if (newname.startswith('sn') and
is_number(newname[2:6]) and len(newname) > 6):
newname = newname.replace('sn', 'SN', 1)
if (newname.startswith('SN ') and
is_number(newname[3:7]) and len(newname) > 7):
newname = newname.replace('SN ', 'SN', 1)
if (newname.startswith('SN') and
is_number(newname[2:6]) and len(newname) == 7 and
newname[6].islower()):
newname = 'SN' + newname[2:6] + newname[6].upper()
elif (newname.startswith('SN') and is_number(newname[2:6]) and
(len(newname) == 8 or len(newname) == 9) and newname[6:].isupper()):
newname = 'SN' + newname[2:6] + newname[6:].lower()
newname = (' '.join(newname.split())).strip()
return newname
def do_ascii(catalog):
"""Process ASCII files that were extracted from datatables appearing in
published works.
"""
task_str = catalog.get_current_task_str()
# 2006ApJ...645..841N
file_path = os.path.join(
catalog.get_current_task_repo(), '2006ApJ...645..841N-table3.csv')
tsvin = list(csv.reader(open(file_path, 'r'), delimiter=','))
for ri, row in enumerate(pbar(tsvin, task_str)):
name = 'SNLS-' + row[0]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2006ApJ...645..841N')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, row[1], source, kind='spectroscopic')
astrot = astrotime(float(row[4]) + 2450000., format='jd').datetime
date_str = make_date_string(astrot.year, astrot.month, astrot.day)
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, date_str, source)
catalog.journal_entries()
# Anderson 2014
file_names = list(
glob(os.path.join(
catalog.get_current_task_repo(), 'SNII_anderson2014/*.dat')))
for datafile in pbar_strings(file_names, task_str):
basename = os.path.basename(datafile)
if not is_number(basename[:2]):
continue
if basename == '0210_V.dat':
name = 'SN0210'
else:
name = ('SN20' if int(basename[:2]) <
50 else 'SN19') + basename.split('_')[0]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2014ApJ...786...67A')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
if name in ['SN1999ca', 'SN2003dq', 'SN2008aw']:
system = 'Swope'
else:
system = 'Landolt'
with open(datafile, 'r') as ff:
tsvin = csv.reader(ff, delimiter=' ', skipinitialspace=True)
for row in tsvin:
if not row[0]:
continue
time = str(jd_to_mjd(Decimal(row[0])))
catalog.entries[name].add_photometry(
time=time, band='V',
magnitude=row[1], e_magnitude=row[2],
system=system, source=source)
catalog.journal_entries()
# stromlo
stromlobands = ['B', 'V', 'R', 'I', 'VM', 'RM']
file_path = os.path.join(
catalog.get_current_task_repo(), 'J_A+A_415_863-1/photometry.csv')
tsvin = list(csv.reader(open(file_path, 'r'), delimiter=','))
for row in pbar(tsvin, task_str):
name = row[0]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2004A&A...415..863G')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
mjd = str(jd_to_mjd(Decimal(row[1])))
for ri, ci in enumerate(range(2, len(row), 3)):
if not row[ci]:
continue
band = stromlobands[ri]
upperlimit = True if (not row[ci + 1] and row[ci + 2]) else False
e_upper_magnitude = str(
abs(Decimal(row[ci + 1]))) if row[ci + 1] else ''
e_lower_magnitude = str(
abs(Decimal(row[ci + 2]))) if row[ci + 2] else ''
teles = 'MSSSO 1.3m' if band in ['VM', 'RM'] else 'CTIO'
instr = 'MaCHO' if band in ['VM', 'RM'] else ''
catalog.entries[name].add_photometry(
time=mjd, band=band, magnitude=row[ci],
e_upper_magnitude=e_upper_magnitude,
e_lower_magnitude=e_lower_magnitude,
upperlimit=upperlimit, telescope=teles,
instrument=instr, source=source)
catalog.journal_entries()
# 2015MNRAS.449..451W
file_path = os.path.join(
catalog.get_current_task_repo(), '2015MNRAS.449..451W.dat')
data = list(csv.reader(open(file_path, 'r'), delimiter='\t',
quotechar='"', skipinitialspace=True))
for rr, row in enumerate(pbar(data, task_str)):
if rr == 0:
continue
namesplit = row[0].split('/')
name = namesplit[-1]
if name.startswith('SN'):
name = name.replace(' ', '')
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2015MNRAS.449..451W')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
if len(namesplit) > 1:
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, namesplit[0], source)
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, row[1], source)
catalog.entries[name].add_photometry(
time=row[2], band=row[4], magnitude=row[3], source=source)
catalog.journal_entries()
# 2016MNRAS.459.1039T
file_path = os.path.join(
catalog.get_current_task_repo(), '2016MNRAS.459.1039T.tsv')
data = list(csv.reader(open(file_path, 'r'), delimiter='\t',
quotechar='"', skipinitialspace=True))
name = catalog.add_entry('LSQ13zm')
source = catalog.entries[name].add_source(bibcode='2016MNRAS.459.1039T')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
for rr, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
bands = [xx.replace('(err)', '') for xx in row[3:-1]]
continue
mjd = row[1]
mags = [re.sub(r'\([^)]*\)', '', xx) for xx in row[3:-1]]
upps = [True if '>' in xx else '' for xx in mags]
mags = [xx.replace('>', '') for xx in mags]
errs = [xx[xx.find('(') + 1:xx.find(')')]
if '(' in xx else '' for xx in row[3:-1]]
for mi, mag in enumerate(mags):
if not is_number(mag):
continue
catalog.entries[name].add_photometry(
time=mjd, band=bands[mi], magnitude=mag, e_magnitude=errs[mi],
instrument=row[-1], upperlimit=upps[mi], source=source)
catalog.journal_entries()
# 2015ApJ...804...28G
file_path = os.path.join(
catalog.get_current_task_repo(), '2015ApJ...804...28G.tsv')
data = list(csv.reader(open(file_path, 'r'), delimiter='\t',
quotechar='"', skipinitialspace=True))
name = catalog.add_entry('PS1-13arp')
source = catalog.entries[name].add_source(bibcode='2015ApJ...804...28G')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
for rr, row in enumerate(pbar(data, task_str)):
if rr == 0:
continue
mjd = row[1]
mag = row[3]
upp = True if '<' in mag else ''
mag = mag.replace('<', '')
err = row[4] if is_number(row[4]) else ''
ins = row[5]
catalog.entries[name].add_photometry(
time=mjd, band=row[0], magnitude=mag, e_magnitude=err,
instrument=ins, upperlimit=upp, source=source)
catalog.journal_entries()
# 2016ApJ...819...35A
file_path = os.path.join(
catalog.get_current_task_repo(), '2016ApJ...819...35A.tsv')
data = list(csv.reader(open(file_path, 'r'), delimiter='\t',
quotechar='"', skipinitialspace=True))
for rr, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
continue
name = catalog.add_entry(row[0])
source = catalog.entries[name].add_source(
bibcode='2016ApJ...819...35A')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[1], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[2], source)
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, row[3], source)
disc_date = datetime.strptime(row[4], '%Y %b %d').isoformat()
disc_date = disc_date.split('T')[0].replace('-', '/')
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, disc_date, source)
catalog.journal_entries()
# 2014ApJ...784..105W
file_path = os.path.join(
catalog.get_current_task_repo(), '2014ApJ...784..105W.tsv')
data = list(csv.reader(open(file_path, 'r'), delimiter='\t',
quotechar='"', skipinitialspace=True))
for rr, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
continue
name = catalog.add_entry(row[0])
source = catalog.entries[name].add_source(
bibcode='2014ApJ...784..105W')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
mjd = row[1]
band = row[2]
mag = row[3]
err = row[4]
catalog.entries[name].add_photometry(
time=mjd, band=row[2], magnitude=mag, e_magnitude=err,
instrument='WHIRC', telescope='WIYN 3.5 m', observatory='NOAO',
system='WHIRC', source=source)
catalog.journal_entries()
# 2012MNRAS.425.1007B
file_path = os.path.join(
catalog.get_current_task_repo(), '2012MNRAS.425.1007B.tsv')
data = list(csv.reader(open(file_path, 'r'), delimiter='\t',
quotechar='"', skipinitialspace=True))
for rr, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
bands = row[2:]
continue
name = catalog.add_entry(row[0])
source = catalog.entries[name].add_source(
bibcode='2012MNRAS.425.1007B')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
mjd = row[1]
mags = [xx.split('±')[0].strip() for xx in row[2:]]
errs = [xx.split('±')[1].strip()
if '±' in xx else '' for xx in row[2:]]
if row[0] == 'PTF09dlc':
ins = 'HAWK-I'
tel = 'VLT 8.1m'
obs = 'ESO'
else:
ins = 'NIRI'
tel = 'Gemini North 8.2m'
obs = 'Gemini'
for mi, mag in enumerate(mags):
if not is_number(mag):
continue
catalog.entries[name].add_photometry(
time=mjd, band=bands[mi], magnitude=mag, e_magnitude=errs[mi],
instrument=ins, telescope=tel, observatory=obs,
system='Natural', source=source)
catalog.journal_entries()
# 2014ApJ...783...28G
file_path = os.path.join(
catalog.get_current_task_repo(), 'apj490105t2_ascii.txt')
with open(file_path, 'r') as f:
data = list(csv.reader(f, delimiter='\t',
quotechar='"', skipinitialspace=True))
for r, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
continue
name, source = catalog.new_entry(
row[0], bibcode='2014ApJ...783...28G')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[1], source)
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, '20' + row[0][3:5], source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[2], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[3], source)
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, row[13] if is_number(row[13]) else
row[10], source)
catalog.journal_entries()
# 2005ApJ...634.1190H
file_path = os.path.join(
catalog.get_current_task_repo(), '2005ApJ...634.1190H.tsv')
with open(file_path, 'r') as f:
data = list(csv.reader(f, delimiter='\t',
quotechar='"', skipinitialspace=True))
for r, row in enumerate(pbar(data, task_str)):
name, source = catalog.new_entry(
'SNLS-' + row[0], bibcode='2005ApJ...634.1190H')
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, '20' + row[0][:2], source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[1], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[2], source)
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, row[5].replace('?', ''), source,
e_value=row[6], kind='host')
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, row[7].replace('SN', '').strip(':* '),
source)
catalog.journal_entries()
# 2014MNRAS.444.2133S
file_path = os.path.join(
catalog.get_current_task_repo(), '2014MNRAS.444.2133S.tsv')
with open(file_path, 'r') as f:
data = list(csv.reader(f, delimiter='\t',
quotechar='"', skipinitialspace=True))
for r, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
continue
name = row[0]
if is_number(name[:4]):
name = 'SN' + name
name, source = catalog.new_entry(
name, bibcode='2014MNRAS.444.2133S')
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[1], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[2], source)
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, row[3],
source,
kind='host')
catalog.journal_entries()
# 2009MNRAS.398.1041B
file_path = os.path.join(
catalog.get_current_task_repo(), '2009MNRAS.398.1041B.tsv')
with open(file_path, 'r') as f:
data = list(csv.reader(f, delimiter='\t',
quotechar='"', skipinitialspace=True))
for r, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
bands = row[2:-1]
continue
name, source = catalog.new_entry(
'SN2008S', bibcode='2009MNRAS.398.1041B')
mjd = str(jd_to_mjd(Decimal(row[0])))
mags = [x.split('±')[0].strip() for x in row[2:]]
upps = [('<' in x.split('±')[0]) for x in row[2:]]
errs = [x.split('±')[1].strip()
if '±' in x else '' for x in row[2:]]
instrument = row[-1]
for mi, mag in enumerate(mags):
if not is_number(mag):
continue
catalog.entries[name].add_photometry(
time=mjd, band=bands[mi],
magnitude=mag, e_magnitude=errs[mi],
instrument=instrument, source=source)
catalog.journal_entries()
# 2010arXiv1007.0011P
file_path = os.path.join(
catalog.get_current_task_repo(), '2010arXiv1007.0011P.tsv')
with open(file_path, 'r') as f:
data = list(csv.reader(f, delimiter='\t',
quotechar='"', skipinitialspace=True))
for r, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
bands = row[1:]
continue
name, source = catalog.new_entry(
'SN2008S', bibcode='2010arXiv1007.0011P')
mjd = row[0]
mags = [x.split('±')[0].strip() for x in row[1:]]
errs = [x.split('±')[1].strip()
if '±' in x else '' for x in row[1:]]
for mi, mag in enumerate(mags):
if not is_number(mag):
continue
catalog.entries[name].add_photometry(
time=mjd, band=bands[mi],
magnitude=mag, e_magnitude=errs[mi],
instrument='LBT', source=source)
catalog.journal_entries()
# 2000ApJ...533..320G
file_path = os.path.join(
catalog.get_current_task_repo(), '2000ApJ...533..320G.tsv')
with open(file_path, 'r') as f:
data = list(csv.reader(f, delimiter='\t',
quotechar='"', skipinitialspace=True))
name, source = catalog.new_entry(
'SN1997cy', bibcode='2000ApJ...533..320G')
for r, row in enumerate(pbar(data, task_str)):
if row[0][0] == '#':
bands = row[1:-1]
continue
mjd = str(jd_to_mjd(Decimal(row[0])))
mags = row[1:len(bands)]
for mi, mag in enumerate(mags):
if not is_number(mag):
continue
catalog.entries[name].add_photometry(
time=mjd, band=bands[mi],
magnitude=mag,
observatory='Mount Stromlo', telescope='MSSSO',
source=source, kcorrected=True)
catalog.journal_entries()
return
def do_crts(catalog):
"""Import data from the Catalina Real-Time Transient Survey."""
crtsnameerrors = ['2011ax']
task_str = catalog.get_current_task_str()
folders = ['catalina', 'MLS', 'MLS', 'SSS']
files = ['AllSN.html', 'AllSN.arch.html', 'CRTSII_SN.html', 'AllSN.html']
for fi, fold in enumerate(pbar(folders, task_str)):
html = catalog.load_url(
'http://nesssi.cacr.caltech.edu/' + fold + '/' + files[fi],
os.path.join(catalog.get_current_task_repo(), 'CRTS', fold + '-' +
files[fi]), archived_mode=('arch' in files[fi]))
html = html.replace('<ahref=', '<a href=')
if not html:
continue
bs = BeautifulSoup(html, 'html5lib')
trs = bs.findAll('tr')
for tri, tr in enumerate(pbar(trs, task_str)):
tds = tr.findAll('td')
if not tds:
continue
# refs = []
aliases = []
crtsname = ''
ra = ''
dec = ''
lclink = ''
# ttype = ''
# ctype = ''
for tdi, td in enumerate(tds):
if tdi == 0:
crtsname = td.contents[0].text.strip()
elif tdi == 1:
ra = td.contents[0]
elif tdi == 2:
dec = td.contents[0]
elif tdi == (8 if files[fi] == 'CRTSII_SN.html' else 11):
lclink = td.find('a')['onclick']
lclink = lclink.split("'")[1]
elif tdi == (10 if files[fi] == 'CRTSII_SN.html' else 13):
aliases = re.sub('[()]', '', re.sub(
'<[^<]+?>', '', td.contents[-1].strip()))
aliases = [xx.strip('; ') for xx in list(
filter(None, aliases.split(' ')))]
name = ''
hostmag = ''
hostupper = False
validaliases = []
for ai, alias in enumerate(aliases):
if alias in ['SN', 'SDSS']:
continue
if alias in crtsnameerrors:
continue
if alias == 'mag':
if ai < len(aliases) - 1:
ind = ai + 1
if aliases[ai + 1] in ['SDSS']:
ind = ai + 2
elif aliases[ai + 1] in ['gal', 'obj', 'object',
'source']:
ind = ai - 1
if '>' in aliases[ind]:
hostupper = True
hostmag = aliases[ind].strip('>~').replace(
',', '.').replace('m', '.')
continue
if (is_number(alias[:4]) and alias[:2] == '20' and
len(alias) > 4):
name = 'SN' + alias
if ((('asassn' in alias and len(alias) > 6) or
('ptf' in alias and len(alias) > 3) or
('ps1' in alias and len(alias) > 3) or
'snhunt' in alias or
('mls' in alias and len(alias) > 3) or
'gaia' in alias or
('lsq' in alias and len(alias) > 3))):
alias = alias.replace('SNHunt', 'SNhunt')
validaliases.append(alias)
if not name:
name = crtsname
name, source = catalog.new_entry(
name, srcname='Catalina Sky Survey',
bibcode='2009ApJ...696..870D',
url='http://nesssi.cacr.caltech.edu/catalina/AllSN.html')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
for alias in validaliases:
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, alias, source)
catalog.entries[name].add_quantity(
SUPERNOVA.RA, ra.strip(), source, u_value='floatdegrees')
catalog.entries[name].add_quantity(
SUPERNOVA.DEC, dec.strip(), source, u_value='floatdegrees')
if SUPERNOVA.CLAIMED_TYPE not in catalog.entries[name]:
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, 'Candidate', source)
if hostmag:
# 1.0 magnitude error based on Drake 2009 assertion that SN are
# only considered
# real if they are 2 mags brighter than host.
photodict = {
PHOTOMETRY.BAND: 'C',
PHOTOMETRY.MAGNITUDE: hostmag,
PHOTOMETRY.E_MAGNITUDE: '1.0',
PHOTOMETRY.SOURCE: source,
PHOTOMETRY.HOST: True,
PHOTOMETRY.TELESCOPE: 'Catalina Schmidt',
PHOTOMETRY.UPPER_LIMIT: hostupper
}
catalog.entries[name].add_photometry(**photodict)
fname2 = (catalog.get_current_task_repo() + '/' + fold + '/' +
lclink.split('.')[-2].rstrip('p').split('/')[-1] +
'.html')
html2 = catalog.load_url(lclink, fname2)
if not html2:
continue
lines = html2.splitlines()
teles = 'Catalina Schmidt'
for line in lines:
if 'javascript:showx' in line:
search = re.search("showx\('(.*?)'\)", line)
if not search:
continue
mjdstr = search.group(1).split('(')[0].strip()
if not is_number(mjdstr):
continue
mjd = str(Decimal(mjdstr) + Decimal(53249.0))
else:
continue
mag = ''
err = ''
if 'javascript:showy' in line:
mag = re.search("showy\('(.*?)'\)", line).group(1)
if 'javascript:showz' in line:
err = re.search("showz\('(.*?)'\)", line).group(1)
if not is_number(mag) or (err and not is_number(err)):
continue
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.E_TIME: '0.125', # 3 hr error
PHOTOMETRY.BAND: 'C',
PHOTOMETRY.MAGNITUDE: mag,
PHOTOMETRY.SOURCE: source,
PHOTOMETRY.INCLUDES_HOST: True,
PHOTOMETRY.TELESCOPE: teles
}
if float(err) > 0.0:
photodict[PHOTOMETRY.E_MAGNITUDE] = err
if float(err) == 0.0:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
catalog.entries[name].add_photometry(**photodict)
if catalog.args.update:
catalog.journal_entries()
if catalog.args.travis and tri > catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
return
def do_nedd(catalog):
task_str = catalog.get_current_task_str()
nedd_path = os.path.join(
catalog.get_current_task_repo(), 'NED26.05.1-D-12.1.0-20160501.csv')
f = open(nedd_path, 'r')
data = sorted(list(csv.reader(f, delimiter=',', quotechar='"'))[
13:], key=lambda x: (x[9], x[3]))
reference = "NED-D"
refurl = "http://ned.ipac.caltech.edu/Library/Distances/"
nedd_dict = OrderedDict()
olddistname = ''
for r, row in enumerate(pbar(data, task_str)):
if r <= 12:
continue
distname = row[3]
name = name_clean(distname)
# distmod = row[4]
# moderr = row[5]
dist = row[6]
bibcode = unescape(row[8])
snname = name_clean(row[9])
redshift = row[10]
cleanhost = ''
if name != snname and (name + ' HOST' != snname):
cleanhost = host_clean(distname)
if cleanhost.endswith(' HOST'):
cleanhost = ''
if not is_number(dist):
print(dist)
if dist:
nedd_dict.setdefault(cleanhost, []).append(Decimal(dist))
if snname and 'HOST' not in snname:
snname, secondarysource = catalog.new_entry(
snname, srcname=reference, url=refurl, secondary=True)
if bibcode:
source = catalog.entries[snname].add_source(bibcode=bibcode)
sources = uniq_cdl([source, secondarysource])
else:
sources = secondarysource
if name == snname:
if redshift:
catalog.entries[snname].add_quantity(
'redshift', redshift, sources)
if dist:
catalog.entries[snname].add_quantity(
'comovingdist', dist, sources)
if not redshift:
try:
zatval = z_at_value(cosmo.comoving_distance,
float(dist) * un.Mpc, zmax=5.0)
sigd = get_sig_digits(str(dist))
redshift = pretty_num(zatval, sig=sigd)
except (KeyboardInterrupt, SystemExit):
raise
except:
pass
else:
cosmosource = catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
combsources = uniq_cdl(sources.split(',') +
[cosmosource])
catalog.entries[snname].add_quantity('redshift',
redshift,
combsources)
if cleanhost:
catalog.entries[snname].add_quantity(
'host', cleanhost, sources)
if catalog.args.update and olddistname != distname:
catalog.journal_entries()
olddistname = distname
catalog.journal_entries()
f.close()
return
def set_data(self,
all_data,
req_key_values={},
subtract_minimum_keys=[],
smooth_times=-1,
extrapolate_time=0.0,
limit_fitting_mjds=False,
exclude_bands=[],
exclude_instruments=[],
exclude_systems=[],
exclude_sources=[],
exclude_kinds=[],
time_unit=None,
time_list=[],
band_list=[],
band_telescopes=[],
band_systems=[],
band_instruments=[],
band_modes=[],
band_bandsets=[]):
"""Set transient data."""
prt = self._printer
self._all_data = all_data
self._data = OrderedDict()
if not self._all_data:
return
name = list(self._all_data.keys())[0]
self._data['name'] = name
numeric_keys = set()
ex_kinds = [
self._EX_REPS.get(x.lower(), x.lower()) for x in exclude_kinds
]
# Construct some source dictionaries for exclusion rules
src_dict = OrderedDict()
sources = self._all_data[name].get('sources', [])
for src in sources:
if SOURCE.BIBCODE in src:
src_dict[src[SOURCE.ALIAS]] = src[SOURCE.BIBCODE]
if SOURCE.ARXIVID in src:
src_dict[src[SOURCE.ALIAS]] = src[SOURCE.ARXIVID]
if SOURCE.NAME in src:
src_dict[src[SOURCE.ALIAS]] = src[SOURCE.NAME]
for key in self._keys:
subkeys = self._keys[key]
req_subkeys = [
x for x in subkeys if not isinstance(subkeys, dict)
or 'required' in listify(subkeys[x])
]
num_subkeys = [
x for x in subkeys if 'numeric' in listify(subkeys[x])
]
boo_subkeys = [
x for x in subkeys if 'boolean' in listify(subkeys[x])
]
exc_subkeys = [
x for x in subkeys if 'exclude' in listify(subkeys[x])
]
if (key not in self._all_data[name]
and not self._model.is_parameter_fixed_by_user(key)):
if subkeys.get('value', None) == 'recommended':
self._unset_recommended_keys.add(key)
continue
subdata = self._all_data[name].get(key)
if subdata is None:
continue
# Only include data that contains all subkeys
for entry in subdata:
if any([x not in entry for x in req_subkeys]):
continue
if any([x in entry for x in exc_subkeys]):
continue
if any([
x in entry and
((isinstance(entry[x], list) and any([
not is_number(y) or np.isnan(float(y))
for y in entry[x]
])) or not isinstance(entry[x], list) and
(not is_number(entry[x]) or np.isnan(float(entry[x]))))
for x in num_subkeys
]):
continue
skip_key = False
if 'frequency' not in entry:
for qkey in req_key_values:
if qkey in entry and entry[qkey] != '':
if entry[qkey] not in req_key_values[qkey]:
skip_key = True
break
if key == 'photometry':
if ('fluxdensity' in entry and 'magnitude' not in entry
and 'countrate' not in entry):
self._kinds_needed.add('radio')
if ('radio' in ex_kinds or
(not len(ex_kinds) or 'none' not in ex_kinds) and
'radio' not in self._model._kinds_supported):
continue
if (('countrate' in entry or 'unabsorbedflux' in entry
or 'flux' in entry) and 'magnitude' not in entry
and 'fluxdensity' not in entry):
self._kinds_needed.add('x-ray')
if ('x-ray' in ex_kinds or
(not len(ex_kinds) or 'none' not in ex_kinds) and
'x-ray' not in self._model._kinds_supported):
continue
if 'magnitude' in entry:
# For now, magnitudes are not excludable.
self._kinds_needed |= set(
['infrared', 'optical', 'ultraviolet'])
skip_entry = False
for x in subkeys:
if limit_fitting_mjds is not False and x == 'time':
val = np.mean([
float(x) for x in listify(entry.get(x, None))
])
if (val < limit_fitting_mjds[0]
or val > limit_fitting_mjds[1]):
skip_entry = True
break
if exclude_bands is not False and x == 'band':
if (entry.get(x, '') in exclude_bands and
(not exclude_instruments or entry.get(
'instrument', '') in exclude_instruments)
and (not exclude_systems or entry.get(
'system', '') in exclude_systems)):
skip_entry = True
break
if (exclude_instruments is not False
and x == 'instrument'):
if (entry.get(x, '') in exclude_instruments and
(not exclude_bands
or entry.get('band', '') in exclude_bands)
and (not exclude_systems or entry.get(
'system', '') in exclude_systems)):
skip_entry = True
break
if (exclude_systems is not False and x == 'system'):
if (entry.get(x, '') in exclude_systems and
(not exclude_bands
or entry.get('band', '') in exclude_bands) and
(not exclude_instruments or entry.get(
'instrument', '') in exclude_instruments)):
skip_entry = True
break
if (exclude_sources is not False and x == 'source'):
val = entry.get(x, '')
if (any(
[x in exclude_sources for x in val.split(',')])
or any([
src_dict.get(x, '') in exclude_sources
for x in val.split(',')
])):
skip_entry = True
break
if skip_entry:
continue
if ((('magnitude' in entry) !=
('band' in entry)) or ((('fluxdensity' in entry) !=
('frequency' in entry)) and
('magnitude' not in entry))
or (('countrate' in entry) and
('magnitude' not in entry) and
('instrument' not in entry))):
continue
for x in subkeys:
falseval = (False if x in boo_subkeys else
None if x in num_subkeys else '')
if x == 'value':
if not skip_key:
self._data[key] = entry.get(x, falseval)
else:
plural = self._model.plural(x)
val = entry.get(x, falseval)
if x in num_subkeys:
val = None if val is None else np.mean(
[float(x) for x in listify(val)])
if not skip_key:
self._data.setdefault(plural, []).append(val)
if x in num_subkeys:
numeric_keys.add(plural)
else:
self._data.setdefault('unmatched_' + plural,
[]).append(val)
if 'times' not in self._data or not any([
x in self._data
for x in ['magnitudes', 'frequencies', 'countrates']
]):
prt.message('no_fittable_data', [name])
return False
for key in list(self._data.keys()):
if isinstance(self._data[key], list):
self._data[key] = np.array(self._data[key])
if key not in numeric_keys:
continue
num_values = [
x for x in self._data[key] if isinstance(x, float)
]
if len(num_values):
self._data['min_' + key] = min(num_values)
self._data['max_' + key] = max(num_values)
else:
if is_number(self._data[key]):
self._data[key] = float(self._data[key])
self._data_determined_parameters.append(key)
if any(x in self._data
for x in ['magnitudes', 'countrates', 'fluxdensities']):
# Add a list of tags for each observation to indicate what unit
# observation is provided in.
self._data['measures'] = [
((['magnitude'] if x else []) + (['countrate'] if y else []) +
(['fluxdensity'] if x else []))
for x, y, z in zip(*(self._data['magnitudes'],
self._data['countrates'],
self._data['fluxdensities']))
]
if 'times' in self._data and (smooth_times >= 0 or time_list):
# Build an observation array out of the real data first.
obs = list(
zip(*(self._data[x] for x in self._OBS_KEYS if x != 'times')))
# Append extra observations if requested.
if len(band_list):
b_teles = band_telescopes if len(band_telescopes) == len(
band_list) else (
[band_telescopes[0] for x in band_list]
if len(band_telescopes) else ['' for x in band_list])
b_systs = band_systems if len(band_systems) == len(
band_list) else ([band_systems[0]
for x in band_list] if len(band_systems)
else ['' for x in band_list])
b_modes = band_modes if len(band_modes) == len(
band_list) else ([band_modes[0]
for x in band_list] if len(band_modes)
else ['' for x in band_list])
b_insts = band_instruments if len(band_instruments) == len(
band_list) else (
[band_instruments[0] for x in band_list]
if len(band_instruments) else ['' for x in band_list])
b_bsets = band_bandsets if len(band_bandsets) == len(
band_list) else ([band_bandsets[0]
for x in band_list] if len(band_bandsets)
else ['' for x in band_list])
b_freqs = [None for x in band_list]
b_u_freqs = ['' for x in band_list]
b_zerops = [None for x in band_list]
b_measures = [[] for x in band_list]
obs.extend(
list(
zip(*(b_teles, b_systs, b_modes, b_insts, b_bsets,
band_list, b_freqs, b_u_freqs, b_zerops,
b_measures))))
# Prune extra observations if they are duplicitous to existing.
uniqueobs = []
for o in obs:
to = tuple(o)
if to not in uniqueobs:
uniqueobs.append(to)
# Preprend times to real observations list.
minet, maxet = (extrapolate_time, extrapolate_time) if isinstance(
extrapolate_time, (float, int)) else (
(tuple(extrapolate_time) if len(extrapolate_time) == 2 else
(extrapolate_time[0], extrapolate_time[0])))
mint, maxt = (min(self._data['times']) - minet,
max(self._data['times']) + maxet)
if time_unit is None:
alltimes = time_list + [x for x in self._data['times']]
elif time_unit == 'mjd':
alltimes = [x - min(self._data['times']) for x in time_list
] + [x for x in self._data['times']]
elif time_unit == 'phase':
if 'maxdate' not in self._data:
raise (prt.message('no_maxdate', name))
max_mjd = astrotime(self._data['maxdate'].replace('/',
'-')).mjd
alltimes = [x + max_mjd for x in time_list
] + [x for x in self._data['times']]
else:
raise ('Unknown `time_unit`.')
if smooth_times >= 0:
alltimes += list(np.linspace(mint, maxt, max(smooth_times, 2)))
alltimes = list(sorted(set(alltimes)))
# Create additional fake observations.
currobslist = list(zip(*(self._data[x] for x in self._OBS_KEYS)))
obslist = []
for ti, t in enumerate(alltimes):
new_per = np.round(100.0 * float(ti) / len(alltimes), 1)
prt.message('construct_obs_array', [new_per], inline=True)
for o in uniqueobs:
newobs = tuple([t] + list(o))
if newobs not in currobslist:
obslist.append(newobs)
obslist.sort()
# Save these fake observations under keys with `extra_` prefix.
if obslist:
for x, y in zip(self._OBS_KEYS, zip(*obslist)):
self._data['extra_' + x] = y
for qkey in subtract_minimum_keys:
if 'upperlimits' in self._data:
new_vals = np.array(self._data[qkey])[np.array(
self._data['upperlimits']) != True] # noqa E712
if new_vals.size:
self._data['min_' + qkey] = min(new_vals)
self._data['max_' + qkey] = max(new_vals)
minv = self._data['min_' + qkey]
self._data[qkey] = [x - minv for x in self._data[qkey]]
if 'extra_' + qkey in self._data:
self._data['extra_' + qkey] = [
x - minv for x in self._data['extra_' + qkey]
]
return True
def do_cpcs(catalog):
"""Import from the CPCS."""
task_str = catalog.get_current_task_str()
cpcs_url = ('http://gsaweb.ast.cam.ac.uk/'
'followup/list_of_alerts?format=json&num=100000&'
'published=1&observed_only=1'
'&hashtag=JG_530ad9462a0b8785bfb385614bf178c6')
jsontxt = catalog.load_url(cpcs_url, os.path.join(
catalog.get_current_task_repo(), 'CPCS', 'index.json'))
if not jsontxt:
return
alertindex = json.loads(jsontxt, object_pairs_hook=OrderedDict)
ids = [xx['id'] for xx in alertindex]
for ii, ai in enumerate(pbar(ids, task_str)):
name = alertindex[ii]['ivorn'].split('/')[-1].strip()
# Skip aa few weird entries
if name == 'ASASSNli':
continue
# Just use aa whitelist for now since naming seems inconsistent
white_list = [
'GAIA', 'OGLE', 'ASASSN', 'MASTER', 'OTJ', 'PS1', 'IPTF', 'CSS']
if True in [xx in name.upper() for xx in white_list]:
name = name.replace('Verif', '').replace('_', ' ')
if 'ASASSN' in name and name[6] != '-':
name = 'ASASSN-' + name[6:].lower()
if 'MASTEROTJ' in name:
name = name.replace('MASTEROTJ', 'MASTER OT J')
if 'OTJ' in name:
name = name.replace('OTJ', 'MASTER OT J')
if name.upper().startswith('IPTF'):
name = 'iPTF' + name[4:].lower()
if name.upper().startswith('PS1'):
name = 'PS1' + name[3:].lower()
# Only add events that already exist.
if not catalog.entry_exists(name):
continue
oldname = name
name = catalog.add_entry(name)
else:
continue
sec_source = catalog.entries[name].add_source(
name='Cambridge Photometric Calibration Server',
url='http://gsaweb.ast.cam.ac.uk/followup/',
secondary=True)
catalog.entries[name].add_quantity(TIDALDISRUPTION.ALIAS, oldname,
sec_source)
unit_deg = 'floatdegrees'
catalog.entries[name].add_quantity(
TIDALDISRUPTION.RA,
str(alertindex[ii][TIDALDISRUPTION.RA]),
sec_source,
u_value=unit_deg)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DEC,
str(alertindex[ii][TIDALDISRUPTION.DEC]),
sec_source,
u_value=unit_deg)
alerturl = ('http://gsaweb.ast.cam.ac.uk/'
'followup/get_alert_lc_data?alert_id=' + str(ai))
source = catalog.entries[name].add_source(
name='CPCS Alert ' + str(ai), url=alerturl)
fname = os.path.join(catalog.get_current_task_repo(),
'CPCS/alert-') + str(ai).zfill(2) + '.json'
jsonstr = catalog.load_url(
alerturl + '&hashtag=JG_530ad9462a0b8785bfb385614bf178c6', fname)
try:
cpcsalert = json.loads(jsonstr)
except Exception:
catalog.log.warning('Mangled CPCS data for alert {}.'.format(ai))
continue
mjds = [round_sig(xx, sig=9) for xx in cpcsalert['mjd']]
mags = [round_sig(xx, sig=6) for xx in cpcsalert['mag']]
errs = [round_sig(
xx, sig=6) if (is_number(xx) and float(xx) > 0.0) else ''
for xx in cpcsalert['magerr']]
bnds = cpcsalert['filter']
obs = cpcsalert['observatory']
for mi, mjd in enumerate(mjds):
catalog.entries[name].add_photometry(
time=mjd,
u_time='MJD',
magnitude=mags[mi],
e_magnitude=errs[mi],
band=bnds[mi],
observatory=obs[mi],
source=uniq_cdl([source, sec_source]))
if catalog.args.update:
catalog.journal_entries()
if catalog.args.travis and ii >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
return
def name_clean(name):
"""Apply list of renaming rules for supernova names."""
newname = name.strip(' ;,*.')
if newname.startswith('NAME '):
newname = newname.replace('NAME ', '', 1)
if newname.endswith(' SN'):
newname = newname.replace(' SN', '')
if newname.endswith(':SN'):
newname = newname.replace(':SN', '')
if newname.startswith('MASJ'):
newname = newname.replace('MASJ', 'MASTER OT J', 1)
if (newname.startswith('MASTER') and len(newname) > 7
and is_number(newname[7])):
newname = newname.replace('MASTER', 'MASTER OT J', 1)
if (newname.startswith('MASTER OT') and len(newname) > 10
and is_number(newname[10])):
newname = newname.replace('MASTER OT', 'MASTER OT J', 1)
if newname.startswith('MASTER OT J '):
newname = newname.replace('MASTER OT J ', 'MASTER OT J', 1)
if newname.startswith('PTSS '):
newname = newname.replace('PTSS ', 'PTSS-', 1)
if newname.startswith('SPIRITS '):
newname = newname.replace('SPIRITS ', 'SPIRITS', 1)
if newname.startswith('OGLE '):
newname = newname.replace('OGLE ', 'OGLE-', 1)
if newname.startswith('OGLE-') and len(newname) != 16:
namesp = newname.split('-')
if (len(namesp) == 4 and len(namesp[1]) == 4 and is_number(namesp[1])
and is_number(namesp[3])):
newname = 'OGLE-' + namesp[1] + '-SN-' + namesp[3].zfill(3)
elif (len(namesp) == 2 and is_number(namesp[1][:2])
and not is_number(namesp[1][2:])):
newname = 'OGLE' + namesp[1]
if newname.startswith('SN SDSS'):
newname = newname.replace('SN SDSS ', 'SDSS', 1)
if newname.startswith('SDSS '):
newname = newname.replace('SDSS ', 'SDSS', 1)
if newname.startswith('SDSS'):
namesp = newname.split('-')
if (len(namesp) == 3 and is_number(namesp[0][4:])
and is_number(namesp[1]) and is_number(namesp[2])):
newname = namesp[0] + '-' + namesp[1] + '-' + namesp[2].zfill(3)
if newname.startswith('SDSS-II SN'):
namesp = newname.split()
if len(namesp) == 3 and is_number(namesp[2]):
newname = 'SDSS-II SN ' + namesp[2].lstrip('0')
if newname.startswith('SN CL'):
newname = newname.replace('SN CL', 'CL', 1)
if newname.startswith('SN HiTS'):
newname = newname.replace('SN HiTS', 'SNHiTS', 1)
if newname.startswith('SNHiTS '):
newname = newname.replace('SNHiTS ', 'SNHiTS', 1)
if newname.startswith('GAIA'):
newname = newname.replace('GAIA', 'Gaia', 1)
if newname.startswith('KSN-'):
newname = newname.replace('KSN-', 'KSN', 1)
if newname.startswith('KSN'):
newname = 'KSN' + newname[3:].lower()
if newname.startswith('Gaia '):
newname = newname.replace('Gaia ', 'Gaia', 1)
if newname.startswith('Gaia'):
newname = 'Gaia' + newname[4:].lower()
if newname.startswith('GRB'):
newname = newname.replace('GRB', 'GRB ', 1)
# if newname.startswith('GRB ') and is_number(newname[4:].strip()):
# newname = 'GRB ' + newname[4:].strip() + 'A'
if newname.startswith('ESSENCE '):
newname = newname.replace('ESSENCE ', 'ESSENCE', 1)
if newname.startswith('LSQ '):
newname = newname.replace('LSQ ', 'LSQ', 1)
if newname.startswith('LSQ') and len(newname) > 3 and is_number(
newname[3]):
newname = newname[:3] + newname[3:].lower()
if newname.startswith('DES') and len(newname) > 3 and is_number(
newname[3]):
newname = newname[:7] + newname[7:].lower()
if newname.startswith('SNSDF '):
newname = newname.replace(' ', '')
if newname.startswith('SNSDF'):
namesp = newname.split('.')
if len(namesp[0]) == 9:
newname = namesp[0] + '-' + namesp[1].zfill(2)
if newname.startswith('HFF '):
newname = newname.replace(' ', '')
if newname.startswith('SN HST'):
newname = newname.replace('SN HST', 'HST', 1)
if newname.startswith('HST ') and newname[4] != 'J':
newname = newname.replace('HST ', 'HST J', 1)
if newname.startswith('SNLS') and newname[4] != '-':
newname = newname.replace('SNLS', 'SNLS-', 1)
if newname.startswith('SNLS- '):
newname = newname.replace('SNLS- ', 'SNLS-', 1)
if newname.startswith('CRTS CSS'):
newname = newname.replace('CRTS CSS', 'CSS', 1)
if newname.startswith('CRTS MLS'):
newname = newname.replace('CRTS MLS', 'MLS', 1)
if newname.startswith('CRTS SSS'):
newname = newname.replace('CRTS SSS', 'SSS', 1)
if newname.startswith(('CSS', 'MLS', 'SSS')):
newname = newname.replace(' ', ':').replace('J', '')
if newname.startswith('SN HFF'):
newname = newname.replace('SN HFF', 'HFF', 1)
if newname.startswith('SN GND'):
newname = newname.replace('SN GND', 'GND', 1)
if newname.startswith('SN SCP'):
newname = newname.replace('SN SCP', 'SCP', 1)
if newname.startswith('SN UDS'):
newname = newname.replace('SN UDS', 'UDS', 1)
if newname.startswith('SCP') and newname[3] != '-':
newname = newname.replace('SCP', 'SCP-', 1)
if newname.startswith('SCP- '):
newname = newname.replace('SCP- ', 'SCP-', 1)
if newname.startswith('SCP-') and is_integer(newname[7:]):
newname = 'SCP-' + newname[4:7] + str(int(newname[7:]))
if newname.startswith('PS 1'):
newname = newname.replace('PS 1', 'PS1', 1)
if newname.startswith('PS1 SN PS'):
newname = newname.replace('PS1 SN PS', 'PS', 1)
if newname.startswith('PS1 SN'):
newname = newname.replace('PS1 SN', 'PS1', 1)
if newname.startswith('PS1') and len(newname) > 3 and is_number(
newname[3]):
newname = newname[:3] + newname[3:].lower()
elif newname.startswith('PS1-') and len(newname) > 4 and is_number(
newname[4]):
newname = newname[:4] + newname[4:].lower()
if newname.startswith('PSN K'):
newname = newname.replace('PSN K', 'K', 1)
if newname.startswith('K') and len(newname) > 5 and is_number(
newname[1:5]):
namesp = newname.split('-')
if len(namesp[0]) == 5:
newname = namesp[0] + '-' + namesp[1].zfill(3)
if newname.startswith('Psn'):
newname = newname.replace('Psn', 'PSN', 1)
if newname.startswith('PSNJ'):
newname = newname.replace('PSNJ', 'PSN J', 1)
if newname.startswith('TCPJ'):
newname = newname.replace('TCPJ', 'TCP J', 1)
if newname.startswith('SMTJ'):
newname = newname.replace('SMTJ', 'SMT J', 1)
if newname.startswith('PSN20J'):
newname = newname.replace('PSN20J', 'PSN J', 1)
if newname.startswith('kait'):
newname = newname.replace('kait', 'KAIT', 1)
if newname.startswith('SN ASASSN'):
newname = newname.replace('SN ASASSN', 'ASASSN', 1)
if newname.startswith('ASASSN-20') and is_number(newname[9]):
newname = newname.replace('ASASSN-20', 'ASASSN-', 1)
if newname.startswith('ASASSN '):
newname = newname.replace('ASASSN ', 'ASASSN-', 1).replace('--', '-')
if newname.startswith('ASASSN') and newname[6] != '-':
newname = newname.replace('ASASSN', 'ASASSN-', 1)
if newname.startswith('ASASSN-') and len(newname) > 7 and is_number(
newname[7]):
newname = newname[:7] + newname[7:].lower()
if newname.startswith('ROTSE3J'):
newname = newname.replace('ROTSE3J', 'ROTSE3 J', 1)
if newname.startswith('MACSJ'):
newname = newname.replace('MACSJ', 'MACS J', 1)
if newname.startswith('MWSNR'):
newname = newname.replace('MWSNR', 'MWSNR ', 1)
if newname.startswith('SN HUNT'):
newname = newname.replace('SN HUNT', 'SNhunt', 1)
if newname.startswith('SN Hunt'):
newname = newname.replace(' ', '')
if newname.startswith('SNHunt'):
newname = newname.replace('SNHunt', 'SNhunt', 1)
if newname.startswith('SNhunt '):
newname = newname.replace('SNhunt ', 'SNhunt', 1)
if newname.startswith('ptf'):
newname = newname.replace('ptf', 'PTF', 1)
if newname.startswith('SN PTF'):
newname = newname.replace('SN PTF', 'PTF', 1)
if newname.startswith('PTF '):
newname = newname.replace('PTF ', 'PTF', 1)
if newname.startswith('PTF') and len(newname) > 3 and is_number(
newname[3]):
newname = newname[:3] + newname[3:].lower()
if newname.startswith('IPTF'):
newname = newname.replace('IPTF', 'iPTF', 1)
if newname.startswith('iPTF '):
newname = newname.replace('iPTF ', 'iPTF', 1)
if newname.startswith('iPTF') and len(newname) > 4 and is_number(
newname[4]):
newname = newname[:4] + newname[4:].lower()
if newname.startswith('PESSTOESO'):
newname = newname.replace('PESSTOESO', 'PESSTO ESO ', 1)
if newname.startswith('snf'):
newname = newname.replace('snf', 'SNF', 1)
if newname.startswith('SNF '):
newname = newname.replace('SNF ', 'SNF', 1)
if (newname.startswith('SNF') and is_number(newname[3:])
and len(newname) >= 12):
newname = 'SNF' + newname[3:11] + '-' + newname[11:]
if newname.startswith(('MASTER OT J', 'ROTSE3 J')):
prefix = newname.split('J')[0]
coords = newname.split('J')[-1].strip()
decsign = '+' if '+' in coords else '-'
coordsplit = coords.replace('+', '-').split('-')
if ('.' not in coordsplit[0] and len(coordsplit[0]) > 6
and '.' not in coordsplit[1] and len(coordsplit[1]) > 6):
newname = (prefix + 'J' + coordsplit[0][:6] + '.' +
coordsplit[0][6:] + decsign + coordsplit[1][:6] + '.' +
coordsplit[1][6:])
if (newname.startswith('Gaia ') and is_number(newname[3:4])
and len(newname) > 5):
newname = newname.replace('Gaia ', 'Gaia', 1)
if (newname.startswith('AT ') and len(newname) > 7
and is_number(newname[3:7])):
newname = newname.replace('AT ', 'AT', 1)
if len(newname) <= 4 and is_number(newname):
newname = 'SN' + newname + 'A'
if (len(newname) > 4 and is_number(newname[:4])
and not is_number(newname[4:])):
newname = 'SN' + newname
if (newname.startswith('Sn ') and is_number(newname[3:7])
and len(newname) > 7):
newname = newname.replace('Sn ', 'SN', 1)
if (newname.startswith('sn') and is_number(newname[2:6])
and len(newname) > 6):
newname = newname.replace('sn', 'SN', 1)
if (newname.startswith('SN ') and is_number(newname[3:7])
and len(newname) > 7):
newname = newname.replace('SN ', 'SN', 1)
if (newname.startswith('SN') and is_number(newname[2:6])
and len(newname) == 7 and newname[6].islower()):
newname = 'SN' + newname[2:6] + newname[6].upper()
elif (newname.startswith('SN') and is_number(newname[2:6])
and (len(newname) == 8 or len(newname) == 9)
and newname[6:].isupper()):
newname = 'SN' + newname[2:6] + newname[6:].lower()
if (newname.startswith('AT') and is_number(newname[2:6])
and len(newname) == 7 and newname[6].islower()):
newname = 'AT' + newname[2:6] + newname[6].upper()
elif (newname.startswith('AT') and is_number(newname[2:6])
and (len(newname) == 8 or len(newname) == 9)
and newname[6:].isupper()):
newname = 'AT' + newname[2:6] + newname[6:].lower()
newname = (' '.join(newname.split())).strip()
return newname
def fetch(self, event_list, offline=False):
"""Fetch a list of events from the open catalogs."""
dir_path = os.path.dirname(os.path.realpath(__file__))
prt = self._printer
levent_list = listify(event_list)
events = [None for x in levent_list]
catalogs = OrderedDict([
(x, self._catalogs[x]) for x in self._catalogs
if x not in self._excluded_catalogs])
for ei, event in enumerate(levent_list):
if not event:
continue
events[ei] = OrderedDict()
path = ''
# If the event name ends in .json, assume event is a path.
if event.endswith('.json'):
path = event
events[ei]['name'] = event.replace('.json',
'').split('/')[-1]
# If not (or the file doesn't exist), download from an open
# catalog.
if not path or not os.path.exists(path):
names_paths = [
os.path.join(dir_path, 'cache', x +
'.names.min.json') for x in catalogs]
input_name = event.replace('.json', '')
if offline:
prt.message('event_interp', [input_name])
else:
prt.message('dling_aliases', [input_name])
for ci, catalog in enumerate(catalogs):
try:
response = get_url_file_handle(
catalogs[catalog]['json'] +
'/names.min.json',
timeout=10)
except Exception:
prt.message(
'cant_dl_names', [catalog], warning=True)
else:
with open_atomic(
names_paths[ci], 'wb') as f:
shutil.copyfileobj(response, f)
names = OrderedDict()
for ci, catalog in enumerate(catalogs):
if os.path.exists(names_paths[ci]):
with open(names_paths[ci], 'r') as f:
names[catalog] = json.load(
f, object_pairs_hook=OrderedDict)
else:
prt.message('cant_read_names', [catalog],
warning=True)
if offline:
prt.message('omit_offline')
continue
if input_name in names[catalog]:
events[ei]['name'] = input_name
events[ei]['catalog'] = catalog
else:
for name in names[catalog]:
if (input_name in names[catalog][name] or
'SN' + input_name in
names[catalog][name]):
events[ei]['name'] = name
events[ei]['catalog'] = catalog
break
if not events[ei].get('name', None):
for ci, catalog in enumerate(catalogs):
namekeys = []
for name in names[catalog]:
namekeys.extend(names[catalog][name])
namekeys = list(sorted(set(namekeys)))
matches = get_close_matches(
event, namekeys, n=5, cutoff=0.8)
# matches = []
if len(matches) < 5 and is_number(event[0]):
prt.message('pef_ext_search')
snprefixes = set(('SN19', 'SN20'))
for name in names[catalog]:
ind = re.search("\d", name)
if ind and ind.start() > 0:
snprefixes.add(name[:ind.start()])
snprefixes = list(sorted(snprefixes))
for prefix in snprefixes:
testname = prefix + event
new_matches = get_close_matches(
testname, namekeys, cutoff=0.95,
n=1)
if (len(new_matches) and
new_matches[0] not in matches):
matches.append(new_matches[0])
if len(matches) == 5:
break
if len(matches):
if self._test:
response = matches[0]
else:
response = prt.prompt(
'no_exact_match',
kind='select',
options=matches,
none_string=(
'None of the above, ' +
('skip this event.' if
ci == len(catalogs) - 1
else
'try the next catalog.')))
if response:
for name in names[catalog]:
if response in names[
catalog][name]:
events[ei]['name'] = name
events[ei]['catalog'] = catalog
break
if events[ei]['name']:
break
if not events[ei].get('name', None):
prt.message('no_event_by_name')
events[ei]['name'] = input_name
continue
urlname = events[ei]['name'] + '.json'
name_path = os.path.join(dir_path, 'cache', urlname)
if offline:
prt.message('cached_event', [
events[ei]['name'], events[ei]['catalog']])
else:
prt.message('dling_event', [
events[ei]['name'], events[ei]['catalog']])
try:
response = get_url_file_handle(
catalogs[events[ei]['catalog']][
'json'] + '/json/' + urlname,
timeout=10)
except Exception:
prt.message('cant_dl_event', [
events[ei]['name']], warning=True)
else:
with open_atomic(name_path, 'wb') as f:
shutil.copyfileobj(response, f)
path = name_path
if os.path.exists(path):
events[ei]['path'] = path
if self._open_in_browser:
webbrowser.open(
catalogs[events[ei]['catalog']]['web'] +
events[ei]['name'])
with open(path, 'r') as f:
events[ei]['data'] = json.load(
f, object_pairs_hook=OrderedDict)
prt.message('event_file', [path], wrapped=True)
else:
prt.message('no_data', [
events[ei]['name'],
'/'.join(catalogs.keys())])
if offline:
prt.message('omit_offline')
raise RuntimeError
return events
def host_clean(name):
"""Clean host name."""
newname = name.strip(' ;,*')
# Handle some special cases
hostcases = {'M051a': 'M51A', 'M051b': 'M51B'}
for k in hostcases:
if newname == k:
newname = hostcases[k]
# Some general cases
newname = newname.strip("()").replace(' ', ' ', 1)
newname = newname.replace("ABELL", "Abell", 1)
newname = newname.replace("Abell", "Abell ", 1)
newname = newname.replace("APMUKS(BJ)", "APMUKS(BJ) ", 1)
newname = newname.replace("ARP", "ARP ", 1)
newname = newname.replace("CGCG", "CGCG ", 1)
newname = newname.replace("HOLM", "HOLM ", 1)
newname = newname.replace("ESO", "ESO ", 1)
newname = newname.replace("IC", "IC ", 1)
newname = newname.replace("Intergal.", "Intergalactic", 1)
newname = newname.replace("MCG+", "MCG +", 1)
newname = newname.replace("MCG-", "MCG -", 1)
newname = newname.replace("M+", "MCG +", 1)
newname = newname.replace("M-", "MCG -", 1)
newname = newname.replace("MGC ", "MCG ", 1)
newname = newname.replace("Mrk", "MRK", 1)
newname = newname.replace("MRK", "MRK ", 1)
newname = newname.replace("NGC", "NGC ", 1)
newname = newname.replace("PGC", "PGC ", 1)
newname = newname.replace("SDSS", "SDSS ", 1)
newname = newname.replace("UGC", "UGC ", 1)
if newname.startswith('MESSIER '):
newname = newname.replace('MESSIER ', 'M', 1)
if newname.startswith('M ') and is_number(newname[2:]):
newname = newname.replace('M ', 'M', 1)
if newname.startswith('M') and is_number(newname[1:]):
newname = 'M' + newname[1:].lstrip(" 0")
if len(newname) > 4 and newname.startswith("PGC "):
newname = newname[:4] + newname[4:].lstrip(" 0")
if len(newname) > 4 and newname.startswith("UGC "):
newname = newname[:4] + newname[4:].lstrip(" 0")
if len(newname) > 5 and newname.startswith(("MCG +", "MCG -")):
newname = newname[:5] + '-'.join(
[x.zfill(2) for x in newname[5:].strip().split("-")])
if len(newname) > 5 and newname.startswith("CGCG "):
newname = newname[:5] + '-'.join(
[x.zfill(3) for x in newname[5:].strip().split("-")])
if ((len(newname) > 1 and newname.startswith("E"))
or (len(newname) > 3 and newname.startswith('ESO'))):
if newname[0] == "E":
esplit = newname[1:].split("-")
else:
esplit = newname[3:].split("-")
if len(esplit) == 2 and is_number(esplit[0].strip()):
if esplit[1].strip()[0] == 'G':
parttwo = esplit[1][1:].strip()
else:
parttwo = esplit[1].strip()
if is_number(parttwo.strip()):
newname = 'ESO ' + \
esplit[0].lstrip('0') + '-G' + parttwo.lstrip('0')
newname = ' '.join(newname.split())
return newname
def do_tns_spectra(catalog):
"""Load TNS spectra."""
requests.packages.urllib3.disable_warnings()
task_str = catalog.get_current_task_str()
tns_url = 'https://wis-tns.weizmann.ac.il/'
try:
with open('tns.key', 'r') as f:
tnskey = f.read().splitlines()[0]
except Exception:
catalog.log.warning('TNS API key not found, make sure a file named '
'`tns.key` containing the key is placed the '
'astrocats directory.')
tnskey = ''
fails = 0
for name in pbar(list(catalog.entries.keys()), task_str):
if name not in catalog.entries:
continue
aliases = catalog.entries[name].get_aliases()
oname = ''
for alias in aliases:
if (alias.startswith(('SN', 'AT')) and is_integer(alias[2:6]) and
int(alias[2:6]) >= 2016) and alias[6:].isalpha():
oname = alias
break
if not oname:
continue
reqname = oname[2:]
jsonpath = os.path.join(catalog.get_current_task_repo(), 'TNS', 'meta',
reqname + '.json')
download_json = True
if os.path.isfile(jsonpath):
with open(jsonpath, 'r') as f:
objdict = json.load(f)
if ('discoverydate' in objdict and
(datetime.now() - datetime.strptime(objdict['discoverydate'],
'%Y-%m-%d %H:%M:%S')
).days > 90):
download_json = False
if download_json:
data = urllib.parse.urlencode({
'api_key': tnskey,
'data': json.dumps({
'objname': reqname,
'spectra': '1'
})
}).encode('ascii')
req = urllib.request.Request(
'https://wis-tns.weizmann.ac.il/api/get/object', data=data)
trys = 0
objdict = None
while trys < 3 and not objdict:
try:
objdict = json.loads(
urllib.request.urlopen(req, timeout=30).read().decode('ascii'))[
'data']['reply']
except KeyboardInterrupt:
raise
except Exception:
catalog.log.warning('API request failed for `{}`.'.format(
name))
time.sleep(5)
trys = trys + 1
if (not objdict or 'objname' not in objdict or
not isinstance(objdict['objname'], str)):
fails = fails + 1
catalog.log.warning('Object `{}` not found!'.format(name))
if fails >= 5:
break
continue
# Cache object here
with open(jsonpath, 'w') as f:
json.dump(sortOD(objdict), f, indent='\t',
separators=(',', ':'), ensure_ascii=False,
sort_keys=True)
if 'spectra' not in objdict:
continue
specarr = objdict['spectra']
name, source = catalog.new_entry(
oname, srcname='Transient Name Server', url=tns_url)
for spectrum in specarr:
spectrumdict = {
PHOTOMETRY.SOURCE: source
}
if 'jd' in spectrum:
spectrumdict[SPECTRUM.TIME] = str(
jd_to_mjd(Decimal(str(spectrum['jd']))))
spectrumdict[SPECTRUM.U_TIME] = 'MJD'
if spectrum.get('observer', ''):
spectrumdict[SPECTRUM.OBSERVER] = spectrum['observer']
if spectrum.get('reducer', ''):
spectrumdict[SPECTRUM.OBSERVER] = spectrum['observer']
if 'source_group' in spectrum:
survey = spectrum['source_group']['name']
if survey:
spectrumdict[SPECTRUM.SURVEY] = survey
if 'telescope' in spectrum:
telescope = spectrum['telescope']['name']
if telescope and telescope != 'Other':
spectrumdict[SPECTRUM.TELESCOPE] = telescope
if 'instrument' in spectrum:
instrument = spectrum['instrument']['name']
if instrument and instrument != 'Other':
spectrumdict[SPECTRUM.INSTRUMENT] = instrument
if 'asciifile' in spectrum:
fname = urllib.parse.unquote(
spectrum['asciifile'].split('/')[-1])
spectxt = catalog.load_url(
spectrum['asciifile'],
os.path.join(
catalog.get_current_task_repo(), 'TNS', 'spectra',
fname), archived_mode=True)
data = [x.split() for x in spectxt.splitlines()]
skipspec = False
newdata = []
oldval = ''
for row in data:
if row and '#' not in row[0]:
if (len(row) >= 2 and is_number(row[0]) and
is_number(row[1]) and row[1] != oldval):
newdata.append(row)
oldval = row[1]
if skipspec or not newdata:
warnings.warn('Skipped adding spectrum file ' + fname)
continue
data = [list(i) for i in zip(*newdata)]
wavelengths = data[0]
fluxes = data[1]
errors = ''
if len(data) == 3:
errors = data[1]
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
spectrumdict.update({
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.ERRORS: errors,
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.U_ERRORS: fluxunit if errors else '',
SPECTRUM.WAVELENGTHS: wavelengths,
SPECTRUM.FLUXES: fluxes
})
catalog.entries[name].add_spectrum(**spectrumdict)
catalog.journal_entries()
return
def set_preferred_names(self):
"""Choose between each entries given name and its possible aliases for
the best one.
Highest preference goes to names of the form 'SN####AA'.
Otherwise base the name on whichever survey is the 'discoverer'.
FIX: create function to match SN####AA type names.
"""
if len(self.entries) == 0:
self.log.error("WARNING: `entries` is empty, loading stubs")
self.load_stubs()
task_str = self.get_current_task_str()
for ni, name in enumerate(pbar(list(
sorted(self.entries.keys())), task_str)):
newname = ''
aliases = self.entries[name].get_aliases()
# if there are no other options to choose from, skip
if len(aliases) <= 1:
continue
# If the name is already in the form 'SN####AA' then keep using
# that
if (name.startswith('SN') and
((is_number(name[2:6]) and not is_number(name[6:])) or
(is_number(name[2:5]) and not is_number(name[5:])))):
continue
# If one of the aliases is in the form 'SN####AA' then use that
for alias in aliases:
if (alias[:2] == 'SN' and
((is_number(alias[2:6]) and not is_number(alias[6:])) or
(is_number(alias[2:5]) and not is_number(alias[5:])))):
newname = alias
break
# Otherwise, name based on the 'discoverer' survey
if not newname and 'discoverer' in self.entries[name]:
discoverer = ','.join(
[x['value'].upper() for x in
self.entries[name]['discoverer']])
if 'ASAS' in discoverer:
for alias in aliases:
if 'ASASSN' in alias.upper():
newname = alias
break
if not newname and 'OGLE' in discoverer:
for alias in aliases:
if 'OGLE' in alias.upper():
newname = alias
break
if not newname and 'CRTS' in discoverer:
for alias in aliases:
if True in [x in alias.upper()
for x in ['CSS', 'MLS', 'SSS', 'SNHUNT']]:
newname = alias
break
if not newname and 'PS1' in discoverer:
for alias in aliases:
if 'PS1' in alias.upper():
newname = alias
break
if not newname and 'PTF' in discoverer:
for alias in aliases:
if 'PTF' in alias.upper():
newname = alias
break
if not newname and 'GAIA' in discoverer:
for alias in aliases:
if 'GAIA' in alias.upper():
newname = alias
break
# Always prefer another alias over PSN
if not newname and name.startswith('PSN'):
newname = aliases[0]
if newname and name != newname:
# Make sure new name doesn't already exist
if self.proto.init_from_file(self, name=newname):
self.log.error("WARNING: `newname` already exists... "
"should do something about that...")
continue
new_entry = self.proto.init_from_file(self, name=name)
if new_entry is None:
self.log.error(
"Could not load `new_entry` with name '{}'"
.format(name))
else:
self.log.info("Changing entry from name '{}' to preferred"
" name '{}'".format(name, newname))
self._delete_entry_file(entry=new_entry)
del self.entries[name]
self.entries[newname] = new_entry
self.entries[newname][self.proto._KEYS.NAME] = newname
if name in self.entries:
self.log.error(
"WARNING: `name` = '{}' is in `entries` "
"shouldnt happen?".format(name))
del self.entries[name]
self.journal_entries()
if self.args.travis and ni > self.TRAVIS_QUERY_LIMIT:
break
return
def do_simbad(catalog):
# Simbad.list_votable_fields()
# Some coordinates that SIMBAD claims belong to the SNe actually belong to
# the host.
task_str = catalog.get_current_task_str()
simbadmirrors = ['http://simbad.harvard.edu/simbad/sim-script',
'http://simbad.u-strasbg.fr/simbad/sim-script']
simbadbadcoordbib = ['2013ApJ...770..107C']
simbadbadnamebib = ['2004AJ....127.2809W', '2005MNRAS.364.1419Z',
'2015A&A...574A.112D', '2011MNRAS.417..916G',
'2002ApJ...566..880G']
simbadbannedcats = ['[TBV2008]', 'OGLE-MBR']
customSimbad = Simbad()
customSimbad.ROW_LIMIT = -1
customSimbad.TIMEOUT = 120
customSimbad.add_votable_fields('otype', 'sptype', 'sp_bibcode', 'id')
table = []
for mirror in simbadmirrors:
customSimbad.SIMBAD_URL = mirror
try:
table = customSimbad.query_criteria('maintype=No* | maintype="No?"')
except:
continue
else:
break
if not table:
catalog.log.warning('SIMBAD unable to load, probably offline.')
# 2000A&AS..143....9W
for brow in pbar(table, task_str):
row = {x: re.sub(r'b\'(.*)\'', r'\1',
str(brow[x])) for x in brow.colnames}
# Skip items with no bibliographic info aside from SIMBAD, too
# error-prone
if row['OTYPE'] == 'Candidate_No*' and not row['SP_TYPE']:
continue
if (not row['COO_BIBCODE'] and not row['SP_BIBCODE'] and
not row['SP_BIBCODE_2']):
continue
if any([x in row['MAIN_ID'] for x in simbadbannedcats]):
continue
if row['COO_BIBCODE'] and row['COO_BIBCODE'] in simbadbadnamebib:
continue
name = single_spaces(re.sub(r'\[[^)]*\]', '', row['MAIN_ID']).strip())
if name == 'SN':
continue
if is_number(name):
continue
name = catalog.add_entry(name)
source = (catalog.entries[name]
.add_source(name='SIMBAD astronomical database',
bibcode="2000A&AS..143....9W",
url="http://simbad.u-strasbg.fr/",
secondary=True))
aliases = row['ID'].split(',')
for alias in aliases:
if any([x in alias for x in simbadbannedcats]):
continue
ali = single_spaces(re.sub(r'\[[^)]*\]', '', alias).strip())
if is_number(ali):
continue
ali = name_clean(ali)
catalog.entries[name].add_quantity(NOVA.ALIAS,
ali, source)
if row['COO_BIBCODE'] and row['COO_BIBCODE'] not in simbadbadcoordbib:
csources = ','.join(
[source, catalog.entries[name].add_source(
bibcode=row['COO_BIBCODE'])])
catalog.entries[name].add_quantity(NOVA.RA,
row['RA'], csources)
catalog.entries[name].add_quantity(NOVA.DEC,
row['DEC'], csources)
if row['SP_BIBCODE']:
ssources = uniq_cdl([source,
catalog.entries[name]
.add_source(bibcode=row['SP_BIBCODE'])] +
([catalog.entries[name]
.add_source(bibcode=row['SP_BIBCODE_2'])] if
row['SP_BIBCODE_2'] else []))
catalog.entries[name].add_quantity(
NOVA.CLAIMED_TYPE,
(row['SP_TYPE']
.replace('SN.', '')
.replace('SN', '')
.replace('(~)', '')
.strip(': ')), ssources)
catalog.journal_entries()
return
def do_simbad(catalog):
task_str = catalog.get_current_task_str()
keys = list(catalog.entries.keys())
customSimbad = Simbad()
customSimbad.ROW_LIMIT = -1
customSimbad.TIMEOUT = 120
customSimbad.add_votable_fields('otype', 'sptype', 'sp_bibcode', 'id')
warnings.filterwarnings("ignore")
Mnames, Mradec = [], []
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
try:
name = catalog.add_entry(oname)
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
if (FASTSTARS.RA not in catalog.entries[name]
or FASTSTARS.DEC not in catalog.entries[name]):
continue
else:
Mnames.append(name)
radec = str(catalog.entries[name][FASTSTARS.RA][0]['value']) + str(
catalog.entries[name][FASTSTARS.DEC][0]['value'])
c = coord(radec, unit=(un.hourangle, un.deg), frame='icrs')
cnttry = 0
foundstar = False
while foundstar == False and cnttry < 10:
try:
cnttry += 1
time.sleep(0.1)
result = customSimbad.query_region(c, radius='0d0m5s')
aliases = re.sub(r'b\'(.*)\'', r'\1',
str(result['ID'].tolist()[0])).split(',')
except TypeError:
#print(radec,cnttry)
continue
foundstar = True
if foundstar == True:
source = (catalog.entries[name].add_source(
name='SIMBAD astronomical database',
bibcode="2000A&AS..143....9W",
url="http://simbad.u-strasbg.fr/",
secondary=True))
for alias in aliases:
ali = single_spaces(
re.sub(r'\[[^)]*\]', '', alias).strip())
if is_number(ali.replace(' ', '')):
continue
if ali[:4] == "HVS ":
continue
ali = name_clean(ali)
catalog.entries[name].add_quantity(FASTSTARS.ALIAS, ali,
source)
catalog.journal_entries()
return
def do_csp_fits_spectra(catalog):
from astropy.io import fits
fpath = catalog.get_current_task_repo()
fureps = {'erg/cm2/s/A': 'erg/s/cm^2/Angstrom'}
task_str = catalog.get_current_task_str()
dirs = [x[0] for x in os.walk(os.path.join(fpath, 'Gutierrez_et_al_2017'))]
files = []
for dir in dirs:
files.extend(glob(os.path.join(dir, '*.fits')))
for datafile in pbar(files, task_str):
filename = datafile.split('/')[-1]
hdulist = fits.open(datafile)
for oi, obj in enumerate(hdulist[0].header):
if any(x in ['.', '/'] for x in obj):
del (hdulist[0].header[oi])
try:
hdulist[0].verify('silentfix')
except Exception as e:
print(e)
hdrkeys = list(hdulist[0].header.keys())
# print(hdrkeys)
name = datafile.split('/')[-2]
if name[2] in '6789':
name = 'SN19' + name[2:]
elif name != 'SN210':
name = 'SN20' + name[2:]
name, source = catalog.new_entry(name, bibcode='2017ApJ...850...89G')
# for key in hdulist[0].header.keys():
# print(key, hdulist[0].header[key])
mjd = None
if hdulist[0].header['SIMPLE']:
if 'JD' in hdrkeys:
mjd = str(jd_to_mjd(Decimal(str(hdulist[0].header['JD']))))
elif 'MJD' in hdrkeys:
mjd = str(hdulist[0].header['MJD'])
elif 'DATE-OBS' in hdrkeys or 'DATE' in hdrkeys:
dkey = 'DATE-OBS' if 'DATE-OBS' in hdrkeys else 'DATE'
dval = hdulist[0].header[dkey]
if is_number(dval):
dkey = 'DATE' if dkey == 'DATE-OBS' else 'DATE-OBS'
dval = hdulist[0].header[dkey]
dateobs = None
if 'T' in dval:
dateobs = dval.strip()
elif 'UTC-OBS' in hdrkeys:
dateobs = dval.strip(
) + 'T' + hdulist[0].header['UTC-OBS'].strip()
if dateobs is not None:
mjd = str(astrotime(dateobs, format='isot').mjd)
# print(hdulist[0].header)
if 'CRVAL1' in hdulist[0].header:
w0 = hdulist[0].header['CRVAL1']
elif hdulist[0].header['CTYPE1'] == 'MULTISPE':
w0 = float(
hdulist[0].header['WAT2_001'].split('"')[-1].split()[3])
else:
raise ValueError('Unsupported spectrum format.')
if hdulist[0].header['NAXIS'] == 1:
wd = hdulist[0].header['CDELT1']
fluxes = [str(x) for x in list(hdulist[0].data)]
errors = False
elif hdulist[0].header['NAXIS'] == 3:
wd = hdulist[0].header['CD1_1']
fluxes = [str(x) for x in list(hdulist[0].data)[0][0]]
errors = [str(x) for x in list(hdulist[0].data)[-1][0]]
else:
print('Warning: Skipping FITS spectrum `{}`.'.format(filename))
continue
waves = [str(w0 + wd * x) for x in range(0, len(fluxes))]
else:
raise ValueError('Non-simple FITS import not yet supported.')
if 'BUNIT' in hdrkeys:
fluxunit = hdulist[0].header['BUNIT']
if fluxunit in fureps:
fluxunit = fureps[fluxunit]
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.WAVELENGTHS: waves,
SPECTRUM.FLUXES: fluxes,
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.FILENAME: filename,
SPECTRUM.SOURCE: source
}
if mjd is not None:
specdict[SPECTRUM.TIME] = mjd
specdict[SPECTRUM.U_TIME] = 'MJD'
if 'TELESCOP' in hdrkeys:
specdict[SPECTRUM.TELESCOPE] = hdulist[0].header['TELESCOP']
if 'INSTRUME' in hdrkeys:
specdict[SPECTRUM.INSTRUMENT] = hdulist[0].header['INSTRUME']
if 'AIRMASS' in hdrkeys:
specdict[SPECTRUM.AIRMASS] = hdulist[0].header['AIRMASS']
if errors:
specdict[SPECTRUM.ERRORS] = errors
specdict[SPECTRUM.U_ERRORS] = fluxunit
if 'SITENAME' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['SITENAME']
elif 'OBSERVAT' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['OBSERVAT']
if 'OBSERVER' in hdrkeys:
specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER']
catalog.entries[name].add_spectrum(**specdict)
hdulist.close()
catalog.journal_entries()
return
def do_asiago_spectra(catalog):
task_str = catalog.get_current_task_str()
html = catalog.load_cached_url(
('http://sngroup.oapd.inaf.it./'
'cgi-bin/output_class.cgi?sn=1990'),
os.path.join(catalog.get_current_task_repo(),
'Asiago/spectra.html'))
if not html:
return
bs = BeautifulSoup(html, 'html5lib')
trs = bs.findAll('tr')
for tr in pbar(trs, task_str):
tds = tr.findAll('td')
name = ''
host = ''
# fitsurl = ''
source = ''
reference = ''
for tdi, td in enumerate(tds):
if tdi == 0:
butt = td.find('button')
if not butt:
break
alias = butt.text.strip()
alias = alias.replace('PSNJ', 'PSN J').replace('GAIA', 'Gaia')
elif tdi == 1:
name = (td.text.strip()
.replace('PSNJ', 'PSN J')
.replace('GAIA', 'Gaia'))
if name.startswith('SN '):
name = 'SN' + name[3:]
if not name:
name = alias
if is_number(name[:4]):
name = 'SN' + name
oldname = name
name = catalog.add_entry(name)
reference = 'Asiago Supernova Catalogue'
refurl = 'http://graspa.oapd.inaf.it/cgi-bin/sncat.php'
secondarysource = catalog.entries[name].add_source(
name=reference, url=refurl, secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oldname,
secondarysource)
if alias != name:
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, alias,
secondarysource)
elif tdi == 2:
host = td.text.strip()
if host == 'anonymous':
host = ''
elif tdi == 3:
discoverer = td.text.strip()
elif tdi == 5:
ra = td.text.strip()
elif tdi == 6:
dec = td.text.strip()
elif tdi == 7:
claimedtype = td.text.strip()
elif tdi == 8:
redshift = td.text.strip()
# elif tdi == 9:
# epochstr = td.text.strip()
# if epochstr:
# mjd = (astrotime(epochstr[:4] + '-' + epochstr[4:6] +
# '-' +
# str(floor(float(epochstr[6:]))).zfill(2)).mjd +
# float(epochstr[6:]) - floor(float(epochstr[6:])))
# else:
# mjd = ''
elif tdi == 10:
refs = td.findAll('a')
source = ''
reference = ''
refurl = ''
for ref in refs:
if ref.text != 'REF':
reference = ref.text
refurl = ref['href']
if reference:
source = catalog.entries[name].add_source(
name=reference, url=refurl)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name,
secondarysource)
sources = uniq_cdl(
list(filter(None, [source, secondarysource])))
elif tdi == 12:
pass
# fitslink = td.find('a')
# if fitslink:
# fitsurl = fitslink['href']
if name:
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE,
claimedtype,
sources)
catalog.entries[name].add_quantity(SUPERNOVA.RA, ra, sources)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, dec, sources)
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, redshift,
sources)
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVERER,
discoverer,
sources)
catalog.entries[name].add_quantity(SUPERNOVA.HOST, host, sources)
# if fitsurl:
# response = urllib.request.urlopen(
# 'http://sngroup.oapd.inaf.it./' + fitsurl)
# compressed = io.BytesIO(response.read())
# decompressed = gzip.GzipFile(fileobj=compressed)
# hdulist = fits.open(decompressed)
# scidata = hdulist[0].data
# print(hdulist[0].header)
#
# print(scidata[3])
# sys.exit()
catalog.journal_entries()
return
def do_suspect_spectra(catalog):
task_str = catalog.get_current_task_str()
with open(os.path.join(catalog.get_current_task_repo(),
'Suspect/sources.json'), 'r') as f:
sourcedict = json.loads(f.read())
with open(os.path.join(catalog.get_current_task_repo(),
'Suspect/filename-changes.txt'), 'r') as f:
rows = f.readlines()
changedict = {}
for row in rows:
if not row.strip() or row[0] == "#":
continue
items = row.strip().split(' ')
changedict[items[1]] = items[0]
suspectcnt = 0
folders = next(os.walk(os.path.join(
catalog.get_current_task_repo(), 'Suspect')))[1]
for folder in pbar(folders, task_str):
eventfolders = next(os.walk(os.path.join(
catalog.get_current_task_repo(), 'Suspect/') + folder))[1]
oldname = ''
for eventfolder in pbar(eventfolders, task_str):
name = eventfolder
if is_number(name[:4]):
name = 'SN' + name
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
sec_ref = 'SUSPECT'
sec_refurl = 'https://www.nhn.ou.edu/~suspect/'
sec_bibc = '2001AAS...199.8408R'
sec_source = catalog.entries[name].add_source(
name=sec_ref, url=sec_refurl, bibcode=sec_bibc,
secondary=True)
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, name, sec_source)
fpath = os.path.join(catalog.get_current_task_repo(),
'Suspect', folder, eventfolder)
eventspectra = next(os.walk(fpath))[2]
for spectrum in eventspectra:
sources = [sec_source]
bibcode = ''
if spectrum in changedict:
specalias = changedict[spectrum]
else:
specalias = spectrum
if specalias in sourcedict:
bibcode = sourcedict[specalias]
elif name in sourcedict:
bibcode = sourcedict[name]
if bibcode:
source = catalog.entries[name].add_source(
bibcode=unescape(bibcode))
sources += [source]
sources = uniq_cdl(sources)
date = spectrum.split('_')[1]
year = date[:4]
month = date[4:6]
day = date[6:]
sig = get_sig_digits(day) + 5
day_fmt = str(floor(float(day))).zfill(2)
time = astrotime(year + '-' + month + '-' + day_fmt).mjd
time = time + float(day) - floor(float(day))
time = pretty_num(time, sig=sig)
fpath = os.path.join(catalog.get_current_task_repo(),
'Suspect',
folder,
eventfolder, spectrum)
with open(fpath, 'r') as f:
specdata = list(csv.reader(
f, delimiter=' ', skipinitialspace=True))
specdata = list(filter(None, specdata))
newspec = []
oldval = ''
for row in specdata:
if row[1] == oldval:
continue
newspec.append(row)
oldval = row[1]
specdata = newspec
haserrors = len(specdata[0]) == 3 and specdata[
0][2] and specdata[0][2] != 'NaN'
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom', u_fluxes='Uncalibrated',
u_time='MJD',
time=time,
wavelengths=wavelengths, fluxes=fluxes, errors=errors,
u_errors='Uncalibrated',
source=sources, filename=spectrum)
suspectcnt = suspectcnt + 1
if (catalog.args.travis and
suspectcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
def do_ps_threepi(catalog):
"""Import data from Pan-STARRS' 3pi page."""
task_str = catalog.get_current_task_str()
bad_aliases = ['SN1994J']
teles = 'Pan-STARRS1'
fname = os.path.join(catalog.get_current_task_repo(), '3pi/page00.html')
ps_url = ("http://psweb.mp.qub.ac.uk/"
"ps1threepi/psdb/public/?page=1&sort=followup_flag_date")
html = catalog.load_url(ps_url, fname, write=False)
if not html:
return
# Clean some common HTML manglings
html = html.replace('ahref=', 'a href=')
bs = BeautifulSoup(html, 'html5lib')
div = bs.find('div', {'class': 'pagination'})
offline = False
if not div:
offline = True
else:
links = div.findAll('a')
if not links:
offline = True
if offline:
if catalog.args.update:
return
warnings.warn('Pan-STARRS 3pi offline, using local files only.')
with open(fname, 'r') as f:
html = f.read()
bs = BeautifulSoup(html, 'html5lib')
div = bs.find('div', {'class': 'pagination'})
links = div.findAll('a')
else:
with open(fname, 'w') as f:
f.write(html)
numpages = int(links[-2].contents[0])
oldnumpages = len(
glob(os.path.join(catalog.get_current_task_repo(), '3pi/page*')))
for page in pbar(range(1, numpages), task_str):
fname = os.path.join(catalog.get_current_task_repo(), '3pi/page') + \
str(page).zfill(2) + '.html'
if offline:
if not os.path.isfile(fname):
continue
with open(fname, 'r') as f:
html = f.read()
else:
if (catalog.current_task.load_archive(catalog.args) and
page < oldnumpages and os.path.isfile(fname)):
with open(fname, 'r') as f:
html = f.read()
else:
response = urllib.request.urlopen(
"http://psweb.mp.qub.ac.uk/ps1threepi/psdb/public/?page=" +
str(page) + "&sort=followup_flag_date")
with open(fname, 'w') as f:
html = response.read().decode('utf-8')
f.write(html)
bs = BeautifulSoup(html, 'html5lib')
trs = bs.findAll('tr')
for tr in pbar(trs, task_str):
tds = tr.findAll('td')
if not tds:
continue
refs = []
aliases = []
ttype = ''
ctype = ''
for tdi, td in enumerate(tds):
if tdi == 0:
psname = td.contents[0]
pslink = psname['href']
psname = psname.text
elif tdi == 1:
ra = td.contents[0]
elif tdi == 2:
dec = td.contents[0]
elif tdi == 3:
ttype = td.contents[0]
if ttype != 'sn' and ttype != 'orphan':
break
elif tdi == 6:
if not td.contents:
continue
ctype = td.contents[0]
if ctype == 'Observed':
ctype = ''
elif tdi == 17:
if td.contents:
crossrefs = td.findAll('a')
for cref in crossrefs:
if 'atel' in cref.contents[0].lower():
refs.append([cref.contents[0], cref['href']])
elif is_number(cref.contents[0][:4]):
continue
else:
aliases.append(cref.contents[0])
if ttype != 'sn' and ttype != 'orphan':
continue
name = ''
for alias in aliases:
if alias in bad_aliases:
continue
if alias[:2] == 'SN':
name = alias
if not name:
name = psname
name = catalog.add_entry(name)
sources = [
catalog.entries[name].add_source(
name='Pan-STARRS 3Pi',
url=('http://psweb.mp.qub.ac.uk/'
'ps1threepi/psdb/'))
]
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name,
sources[0])
for ref in refs:
sources.append(catalog.entries[name].add_source(
name=ref[0], url=ref[1]))
source = uniq_cdl(sources)
for alias in aliases:
newalias = alias
if alias[:3] in ['CSS', 'SSS', 'MLS']:
newalias = alias.replace('-', ':', 1)
newalias = newalias.replace('PSNJ', 'PSN J')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, newalias,
source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, ra, source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, dec, source)
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, ctype,
source)
fname2 = os.path.join(catalog.get_current_task_repo(),
'3pi/candidate-')
fname2 += pslink.rstrip('/').split('/')[-1] + '.html'
if offline:
if not os.path.isfile(fname2):
continue
with open(fname2, 'r') as f:
html2 = f.read()
else:
if (catalog.current_task.load_archive(catalog.args) and
os.path.isfile(fname2)):
with open(fname2, 'r') as f:
html2 = f.read()
else:
pslink = ('http://psweb.mp.qub.ac.uk/'
'ps1threepi/psdb/public/') + pslink
try:
session2 = requests.Session()
response2 = session2.get(pslink)
except Exception:
offline = True
if not os.path.isfile(fname2):
continue
with open(fname2, 'r') as f:
html2 = f.read()
else:
html2 = response2.text
with open(fname2, 'w') as f:
f.write(html2)
bs2 = BeautifulSoup(html2, 'html5lib')
scripts = bs2.findAll('script')
nslines = []
nslabels = []
for script in scripts:
if 'jslcdata.push' not in script.text:
continue
slines = script.text.splitlines()
for line in slines:
if 'jslcdata.push' in line:
json_fname = (line.strip()
.replace('jslcdata.push(', '')
.replace(');', ''))
nslines.append(json.loads(json_fname))
if ('jslabels.push' in line and 'blanks' not in line and
'non det' not in line):
json_fname = (line.strip()
.replace('jslabels.push(', '')
.replace(');', ''))
nslabels.append(json.loads(json_fname)['label'])
for li, line in enumerate(nslines[:len(nslabels)]):
if not line:
continue
for obs in line:
catalog.entries[name].add_photometry(
time=str(obs[0]),
u_time='MJD',
band=nslabels[li],
instrument='GPC',
magnitude=str(obs[1]),
e_magnitude=str(obs[2]),
source=source,
telescope=teles)
# Ignoring upper limits as they are usually spurious chip gaps.
# for li, line in enumerate(nslines[2 * len(nslabels):]):
# if not line:
# continue
# for obs in line:
# catalog.entries[name].add_photometry(
# time=str(obs[0]),
# u_time='MJD',
# band=nslabels[li],
# instrument='GPC',
# magnitude=str(obs[1]),
# upperlimit=True,
# source=source,
# telescope=teles)
assoctab = bs2.find('table', {'class': 'generictable'})
hostname = ''
redshift = ''
if assoctab:
trs = assoctab.findAll('tr')
headertds = [x.contents[0] for x in trs[1].findAll('td')]
tds = trs[1].findAll('td')
for tdi, td in enumerate(tds):
if tdi == 1:
hostname = td.contents[0].strip()
elif tdi == 4:
if 'z' in headertds:
redshift = td.contents[0].strip()
# Skip galaxies with just SDSS id
if is_number(hostname):
continue
catalog.entries[name].add_quantity(SUPERNOVA.HOST, hostname,
source)
if redshift:
catalog.entries[name].add_quantity(
[SUPERNOVA.REDSHIFT, SUPERNOVA.HOST_REDSHIFT],
redshift,
source,
kind='host')
if catalog.args.update:
catalog.journal_entries()
catalog.journal_entries()
# Only run first page for Travis
if catalog.args.travis:
break
return
def do_suspect_photo(catalog):
task_str = catalog.get_current_task_str()
with open(os.path.join(catalog.get_current_task_repo(),
'suspectreferences.csv'), 'r') as f:
tsvin = csv.reader(f, delimiter=',', skipinitialspace=True)
suspectrefdict = {}
for row in tsvin:
suspectrefdict[row[0]] = row[1]
file_names = list(sorted(glob(os.path.join(
catalog.get_current_task_repo(), 'SUSPECT/*.html'))))
for datafile in pbar_strings(file_names, task_str):
basename = os.path.basename(datafile)
basesplit = basename.split('-')
oldname = basesplit[1]
name = catalog.add_entry(oldname)
if name.startswith('SN') and is_number(name[2:]):
name = name + 'A'
band = basesplit[3].split('.')[0]
ei = int(basesplit[2])
bandlink = 'file://' + os.path.abspath(datafile)
bandresp = urllib.request.urlopen(bandlink)
bandsoup = BeautifulSoup(bandresp, 'html5lib')
bandtable = bandsoup.find('table')
names = bandsoup.body.findAll(text=re.compile('Name'))
reference = ''
for link in bandsoup.body.findAll('a'):
if 'adsabs' in link['href']:
reference = str(link).replace('"', "'")
bibcode = unescape(suspectrefdict[reference])
source = catalog.entries[name].add_source(bibcode=bibcode)
sec_ref = 'SUSPECT'
sec_refurl = 'https://www.nhn.ou.edu/~suspect/'
sec_source = catalog.entries[name].add_source(
name=sec_ref, url=sec_refurl, secondary=True)
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, oldname, sec_source)
if ei == 1:
year = re.findall(r'\d+', name)[0]
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, year, sec_source)
catalog.entries[name].add_quantity(
SUPERNOVA.HOST, names[1].split(':')[1].strip(), sec_source)
redshifts = bandsoup.body.findAll(text=re.compile('Redshift'))
if redshifts:
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, redshifts[0].split(':')[1].strip(),
sec_source, kind='heliocentric')
# hvels = bandsoup.body.findAll(text=re.compile('Heliocentric
# Velocity'))
# if hvels:
# vel = hvels[0].split(':')[1].strip().split(' ')[0]
# catalog.entries[name].add_quantity(SUPERNOVA.VELOCITY, vel,
# sec_source,
# kind='heliocentric')
types = bandsoup.body.findAll(text=re.compile('Type'))
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, types[0].split(
':')[1].strip().split(' ')[0],
sec_source)
for r, row in enumerate(bandtable.findAll('tr')):
if r == 0:
continue
col = row.findAll('td')
mjd = str(jd_to_mjd(Decimal(col[0].contents[0])))
mag = col[3].contents[0]
if mag.isspace():
mag = ''
else:
mag = str(mag)
e_magnitude = col[4].contents[0]
if e_magnitude.isspace():
e_magnitude = ''
else:
e_magnitude = str(e_magnitude)
catalog.entries[name].add_photometry(
time=mjd, band=band, magnitude=mag, e_magnitude=e_magnitude,
source=sec_source + ',' + source)
catalog.journal_entries()
return
def do_ogle(catalog):
task_str = catalog.get_current_task_str()
basenames = ['transients', 'transients/2014b', 'transients/2014',
'transients/2013', 'transients/2012']
oglenames = []
ogleupdate = [True, False, False, False, False]
for b, bn in enumerate(pbar(basenames, task_str)):
if catalog.args.update and not ogleupdate[b]:
continue
filepath = os.path.join(catalog.get_current_task_repo(), 'OGLE-')
filepath += bn.replace('/', '-') + '-transients.html'
htmltxt = catalog.load_cached_url(
'http://ogle.astrouw.edu.pl/ogle4/' + bn +
'/transients.html', filepath)
if not htmltxt:
continue
soup = BeautifulSoup(htmltxt, 'html5lib')
links = soup.findAll('a')
breaks = soup.findAll('br')
datalinks = []
datafnames = []
for a in links:
if a.has_attr('href'):
if '.dat' in a['href']:
datalinks.append(
'http://ogle.astrouw.edu.pl/ogle4/' + bn + '/' +
a['href'])
datafnames.append(bn.replace('/', '-') +
'-' + a['href'].replace('/', '-'))
ec = -1
reference = 'OGLE-IV Transient Detection System'
refurl = 'http://ogle.astrouw.edu.pl/ogle4/transients/transients.html'
for br in pbar(breaks, task_str):
sibling = br.nextSibling
if 'Ra,Dec=' in sibling:
line = sibling.replace('\n', '').split('Ra,Dec=')
name = line[0].strip()
ec += 1
if 'NOVA' in name or 'dupl' in name:
continue
if name in oglenames:
continue
oglenames.append(name)
name = catalog.add_entry(name)
mySibling = sibling.nextSibling
atelref = ''
claimedtype = ''
while 'Ra,Dec=' not in mySibling:
if isinstance(mySibling, NavigableString):
if 'Phot.class=' in str(mySibling):
claimedtype = re.sub(
r'\([^)]*\)', '',
str(mySibling).split('=')[-1])
claimedtype = claimedtype.replace('SN', '').strip()
if isinstance(mySibling, Tag):
atela = mySibling
if (atela and atela.has_attr('href') and
'astronomerstelegram' in atela['href']):
atelref = atela.contents[0].strip()
atelurl = atela['href']
mySibling = mySibling.nextSibling
if mySibling is None:
break
# nextSibling = sibling.nextSibling
# if ((isinstance(nextSibling, Tag) and
# nextSibling.has_attr('alt') and
# nextSibling.contents[0].strip() != 'NED')):
# radec = nextSibling.contents[0].strip().split()
# else:
# radec = line[-1].split()
# ra = radec[0]
# dec = radec[1]
fname = os.path.join(catalog.get_current_task_repo(),
'OGLE/') + datafnames[ec]
if (catalog.current_task.load_archive(catalog.args) and
os.path.isfile(fname)):
with open(fname, 'r') as f:
csvtxt = f.read()
else:
response = urllib.request.urlopen(datalinks[ec])
with open(fname, 'w') as f:
csvtxt = response.read().decode('utf-8')
f.write(csvtxt)
lcdat = csvtxt.splitlines()
sources = [catalog.entries[name].add_source(
name=reference, url=refurl)]
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, name, sources[0])
if atelref and atelref != 'ATel#----':
sources.append(catalog.entries[name].add_source(
name=atelref, url=atelurl))
sources = uniq_cdl(sources)
if name.startswith('OGLE'):
if name[4] == '-':
if is_number(name[5:9]):
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, name[5:9], sources)
else:
if is_number(name[4:6]):
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, '20' + name[4:6],
sources)
# RA and Dec from OGLE pages currently not reliable
# catalog.entries[name].add_quantity(SUPERNOVA.RA, ra, sources)
# catalog.entries[name].add_quantity(SUPERNOVA.DEC, dec,
# sources)
if claimedtype and claimedtype != '-':
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, claimedtype, sources)
elif ('SN' not in name and SUPERNOVA.CLAIMED_TYPE not in
catalog.entries[name]):
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, 'Candidate', sources)
for row in lcdat:
row = row.split()
mjd = str(jd_to_mjd(Decimal(row[0])))
magnitude = row[1]
if float(magnitude) > 90.0:
continue
e_mag = row[2]
upperlimit = False
if e_mag == '-1' or float(e_mag) > 10.0:
e_mag = ''
upperlimit = True
catalog.entries[name].add_photometry(
time=mjd, band='I', magnitude=magnitude,
e_magnitude=e_mag,
system='Vega', source=sources, upperlimit=upperlimit)
if catalog.args.update:
catalog.journal_entries()
catalog.journal_entries()
return
def do_cpcs(catalog):
"""Import data from CPCS."""
task_str = catalog.get_current_task_str()
cpcs_url = ('http://gsaweb.ast.cam.ac.uk/'
'followup/list_of_alerts?format=json&num=100000&'
'published=1&observed_only=1'
'&hashtag=JG_530ad9462a0b8785bfb385614bf178c6')
jsontxt = catalog.load_url(
cpcs_url,
os.path.join(catalog.get_current_task_repo(), 'CPCS/index.json'))
if not jsontxt:
return
alertindex = json.loads(jsontxt, object_pairs_hook=OrderedDict)
ids = [xx['id'] for xx in alertindex]
for ii, ai in enumerate(pbar(ids, task_str)):
name = alertindex[ii]['ivorn'].split('/')[-1].strip()
# Skip aa few weird entries
if name == 'ASASSNli':
continue
# Just use aa whitelist for now since naming seems inconsistent
white_list = [
'GAIA', 'OGLE', 'ASASSN', 'MASTER', 'OTJ', 'PS1', 'IPTF', 'CSS'
]
if True in [xx in name.upper() for xx in white_list]:
name = name.replace('Verif', '').replace('_', ' ')
if 'ASASSN' in name and name[6] != '-':
name = 'ASASSN-' + name[6:].lower()
if 'MASTEROTJ' in name:
name = name.replace('MASTEROTJ', 'MASTER OT J')
if 'OTJ' in name:
name = name.replace('OTJ', 'MASTER OT J')
if name.upper().startswith('IPTF'):
name = 'iPTF' + name[4:].lower()
if name.upper().startswith('PS1'):
name = 'PS1' + name[3:].lower()
# Only add events that are classified as SN.
if not catalog.entry_exists(name):
continue
oldname = name
name = catalog.add_entry(name)
else:
continue
sec_source = catalog.entries[name].add_source(
name='Cambridge Photometric Calibration Server',
url='http://gsaweb.ast.cam.ac.uk/followup/',
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oldname,
sec_source)
unit_deg = 'floatdegrees'
catalog.entries[name].add_quantity(SUPERNOVA.RA,
str(alertindex[ii][SUPERNOVA.RA]),
sec_source,
u_value=unit_deg)
catalog.entries[name].add_quantity(SUPERNOVA.DEC,
str(alertindex[ii][SUPERNOVA.DEC]),
sec_source,
u_value=unit_deg)
alerturl = ('http://gsaweb.ast.cam.ac.uk/'
'followup/get_alert_lc_data?alert_id=' + str(ai))
source = catalog.entries[name].add_source(name='CPCS Alert ' + str(ai),
url=alerturl)
fname = os.path.join(catalog.get_current_task_repo(),
'CPCS/alert-') + str(ai).zfill(2) + '.json'
jsonstr = catalog.load_url(
alerturl + '&hashtag=JG_530ad9462a0b8785bfb385614bf178c6', fname)
try:
cpcsalert = json.loads(jsonstr)
except Exception:
catalog.log.warning('Mangled CPCS data for alert {}.'.format(ai))
continue
mjds = [round_sig(xx, sig=9) for xx in cpcsalert['mjd']]
mags = [round_sig(xx, sig=6) for xx in cpcsalert['mag']]
errs = [
round_sig(xx, sig=6) if (is_number(xx) and float(xx) > 0.0) else ''
for xx in cpcsalert['magerr']
]
bnds = cpcsalert['filter']
obs = cpcsalert['observatory']
for mi, mjd in enumerate(mjds):
catalog.entries[name].add_photometry(time=mjd,
u_time='MJD',
magnitude=mags[mi],
e_magnitude=errs[mi],
band=bnds[mi],
observatory=obs[mi],
source=uniq_cdl(
[source, sec_source]))
if catalog.args.update:
catalog.journal_entries()
if catalog.args.travis and ii >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
return
