def do_ps_mds(catalog):
task_str = catalog.get_current_task_str()
with open(
os.path.join(catalog.get_current_task_repo(),
'MDS/apj506838t1_mrt.txt')) as f:
for ri, row in enumerate(pbar(f.read().splitlines(), task_str)):
if ri < 35:
continue
cols = [x.strip() for x in row.split(',')]
name = catalog.add_entry(cols[0])
source = catalog.entries[name].add_source(
bibcode='2015ApJ...799..208S')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, cols[2], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, cols[3], source)
astrot = astrotime(float(cols[4]), format='mjd').datetime
ddate = make_date_string(astrot.year, astrot.month, astrot.day)
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, ddate,
source)
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, cols[5], source, kind='spectroscopic')
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, 'II P',
source)
catalog.journal_entries()
return
def do_fermi(catalog):
task_str = catalog.get_current_task_str()
with open(os.path.join(catalog.get_current_task_repo(),
'1SC_catalog_v01.asc'), 'r') as ff:
tsvin = list(csv.reader(ff, delimiter=','))
for ri, row in enumerate(pbar(tsvin, task_str)):
if row[0].startswith('#'):
if len(row) > 1 and 'UPPER_LIMITS' in row[1]:
break
continue
if 'Classified' not in row[1]:
continue
name = row[0].replace('SNR', 'G')
name = catalog.add_entry(name)
source = (catalog.entries[name]
.add_source(bibcode='2016ApJS..224....8A'))
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, row[0].replace('SNR', 'MWSNR'), source)
catalog.entries[name].add_quantity(
SUPERNOVA.RA, row[2], source, u_value='floatdegrees')
catalog.entries[name].add_quantity(
SUPERNOVA.DEC, row[3], source, u_value='floatdegrees')
catalog.journal_entries()
return
def do_pessto(catalog):
task_str = catalog.get_current_task_str()
pessto_path = os.path.join(
catalog.get_current_task_repo(), 'PESSTO_MPHOT.csv')
tsvin = list(csv.reader(open(pessto_path, 'r'), delimiter=','))
for ri, row in enumerate(pbar(tsvin, task_str)):
if ri == 0:
bands = [xx.split('_')[0] for xx in row[3::2]]
systems = [xx.split('_')[1].capitalize().replace(
'Ab', 'AB') for xx in row[3::2]]
continue
name = row[1]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2015A&A...579A..40S')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
for hi, ci in enumerate(range(3, len(row) - 1, 2)):
if not row[ci]:
continue
teles = 'Swift' if systems[hi] == 'Swift' else ''
(catalog.entries[name]
.add_photometry(time=row[2], magnitude=row[ci],
e_magnitude=row[ci + 1],
band=bands[hi], system=systems[hi],
telescope=teles,
source=source))
catalog.journal_entries()
return
def do_smt(catalog):
task_str = catalog.get_current_task_str()
smt_url = 'http://www.mso.anu.edu.au/skymapper/smt/transients/tns/'
html = catalog.load_url(
smt_url,
os.path.join(catalog.get_current_task_repo(), 'SMT', 'index.html'))
if not html:
return
bs = BeautifulSoup(html, 'html5lib')
trs = bs.find('table').findAll('tr')
for tr in pbar(trs, task_str):
cols = [str(xx.text) for xx in tr.findAll('td')]
if not cols:
continue
name = 'AT' + cols[0]
name, source = catalog.new_entry(name, srcname='SMT', url=smt_url)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, cols[1], source)
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, cols[2],
source)
catalog.entries[name].add_quantity(SUPERNOVA.RA,
cols[3],
source,
u_value='floatdegrees')
catalog.entries[name].add_quantity(SUPERNOVA.DEC,
cols[4],
source,
u_value='floatdegrees')
if catalog.args.update:
catalog.journal_entries()
catalog.journal_entries()
return
def do_agn_bhm_database(catalog):
"""Load data from the 'AGN Blackhole Mass Database': 2015PASP..127...67B.
FIX: Add archiving flags!
"""
log = catalog.log
log.debug("do_agn_bhm_database()")
task_str = catalog.get_current_task_str()
# Load data from URL or cached copy of it
cached_path = SOURCE_BIBCODE + '.txt'
html = catalog.load_url(DATA_URL, cached_path, fail=True)
if html is None:
return False
# Get this line for description of mass calculation
# 'M<sub>BH</sub> calculated using <i>< f ></i> =  2.8'
mass_scale_factor = re.search("<i>< f ></i> =  (.*)", html).groups()
# If pattern matches expectations, store it
if len(mass_scale_factor) == 1:
mass_scale_factor = mass_scale_factor[0]
# Otherwise, forget it
else:
err_msg = "Could not load `mass_scale_factor` from '{}'".format(mass_scale_factor)
catalog.log.error(err_msg)
return False
soup = BeautifulSoup(html, 'html5lib')
# The whole table is nested in a `<table class="hovertable">`
full_table = soup.find('table', attrs={'class': 'hovertable'})
# Each line in the file is separated with `'tr'`
div_lines = full_table.find_all('tr')
# Go through each element of the tables
entries = 0
for div in utils.pbar(div_lines, task_str):
# Get the `varname` -- ID number for each row
# The first element of the `contents` contains an href with the 'varname'
cell_text = str(div.contents[0])
groups = re.search('varname=([0-9]*)', cell_text)
# If no match is found, this is one of the header lines (not an entry line, skip)
if groups is not None:
varname = groups.groups()[0]
try:
name = _add_entry_for_data_line(catalog, div.text, varname, mass_scale_factor)
except Exception:
log.error("Failed `_add_entry_for_data_line()`")
log.error("`div.text`: '{}'".format(div.text))
log.error("`varname`: '{}'".format(varname))
raise
if name is not None:
entries += 1
if catalog.args.travis and (entries > catalog.TRAVIS_QUERY_LIMIT):
log.warning("Exiting on travis limit")
break
return True
def do_tns(catalog):
"""Load TNS metadata."""
session = requests.Session()
task_str = catalog.get_current_task_str()
tns_url = 'https://wis-tns.weizmann.ac.il/'
search_url = (
'https://wis-tns.weizmann.ac.il/' +
'search?&discovered_period_value=30&discovered_period_units=years' +
'&unclassified_at=0&classified_sne=0&include_frb=0&name=&name_like=0' +
'&isTNS_AT=all&public=all&ra=&decl=&radius=&coords_unit=arcsec&reporting_groupid%5B%5D=null'
+
'&groupid%5B%5D=null&classifier_groupid%5B%5D=null&objtype%5B%5D=27' +
'&at_type%5B%5D=null&date_start%5Bdate%5D=&date_end%5Bdate%5D=&discovery_mag_min=&discovery_mag_max='
+
'&internal_name=&discoverer=&classifier=&spectra_count=&redshift_min=&redshift_max=&hostname='
+
'&ext_catid=&ra_range_min=&ra_range_max=&decl_range_min=&decl_range_max='
+
'&discovery_instrument%5B%5D=null&classification_instrument%5B%5D=null&associated_groups%5B%5D=null'
+
'&at_rep_remarks=&class_rep_remarks=&frb_repeat=all&frb_repeater_of_objid=&frb_measured_redshift=0'
+
'&frb_dm_range_min=&frb_dm_range_max=&frb_rm_range_min=&frb_rm_range_max=&frb_snr_range_min='
+
'&frb_snr_range_max=&frb_flux_range_min=&frb_flux_range_max=&num_page=500&display%5Bredshift%5D=1'
+
'&display%5Bhostname%5D=1&display%5Bhost_redshift%5D=1&display%5Bsource_group_name%5D=1'
+
'&display%5Bclassifying_source_group_name%5D=1&display%5Bdiscovering_instrument_name%5D=0'
+
'&display%5Bclassifing_instrument_name%5D=0&display%5Bprograms_name%5D=0&display%5Binternal_name%5D=1'
+
'&display%5BisTNS_AT%5D=0&display%5Bpublic%5D=1&display%5Bend_pop_period%5D=0'
+
'&display%5Bspectra_count%5D=1&display%5Bdiscoverymag%5D=1&display%5Bdiscmagfilter%5D=1&display'
+
'%5Bdiscoverydate%5D=1&display%5Bdiscoverer%5D=1&display%5Bremarks%5D=0&display%5Bsources%5D=0'
+ '&display%5Bbibcode%5D=0&display%5Bext_catalogs%5D=0&format=csv')
csvtxt = catalog.load_url(search_url,
os.path.join(catalog.get_current_task_repo(),
'TNS', 'index.csv'),
timeout=220)
data = read(csvtxt, format='csv')
for rrow in pbar(data, task_str):
row = dict((x, str(rrow[x])) for x in rrow.columns)
name = catalog.add_entry(row['Name'])
source = catalog.entries[name].add_source(name='Transient Name Server',
url=tns_url)
if int(float(row['Discovery Mag/Flux'])) >= 8:
catalog.entries[name].add_quantity(CATACLYSMIC.MAX_VISUAL_APP_MAG,
row['Discovery Mag/Flux'],
source)
catalog.entries[name].add_quantity(CATACLYSMIC.RA, row['RA'], source)
catalog.entries[name].add_quantity(CATACLYSMIC.DEC, row['DEC'], source)
catalog.entries[name].add_quantity(
CATACLYSMIC.DISCOVER_DATE,
row['Discovery Date (UT)'].replace('-', '/'), source)
catalog.entries[name].add_quantity(CATACLYSMIC.CLAIMED_TYPE,
'Cataclysmic_Variable', source)
catalog.journal_entries()
def do_grb(catalog):
task_str = catalog.get_current_task_str()
file_path = os.path.join(catalog.get_current_task_repo(),
'GRB-catalog/catalog.csv')
csvtxt = catalog.load_url(
'http://www.grbcatalog.com/'
'download_data?cut_0_min=3&cut_0=BAT%20T90'
'&cut_0_max=100000&num_cuts=1&no_date_cut=True', file_path)
if not csvtxt:
return
csvtxt = csvtxt.replace('\x0C', '').splitlines()
data = list(
csv.reader(csvtxt, delimiter=',', quotechar='"',
skipinitialspace=True))
for r, row in enumerate(pbar(data, task_str)):
if r == 0:
continue
(name, source) = catalog.new_entry('GRB ' + row[0],
srcname='Gamma-ray Bursts Catalog',
url='http://www.grbcatalog.com')
catalog.entries[name].add_quantity(SUPERNOVA.RA,
row[2],
source,
u_value='floatdegrees')
catalog.entries[name].add_quantity(SUPERNOVA.DEC,
row[3],
source,
u_value='floatdegrees')
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, 'LGRB',
source)
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, row[8], source)
catalog.journal_entries()
return
def do_snls_photo(catalog):
task_str = catalog.get_current_task_str()
snls_path = os.path.join(catalog.get_current_task_repo(), 'SNLS-ugriz.dat')
data = list(csv.reader(open(snls_path, 'r'), delimiter=' ',
quotechar='"', skipinitialspace=True))
for row in pbar(data, task_str):
flux = row[3]
err = row[4]
# Being extra strict here with the flux constraint, see note below.
if float(flux) < 3.0 * float(err):
continue
name = 'SNLS-' + row[0]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2010A&A...523A...7G')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
band = row[1]
mjd = row[2]
sig = get_sig_digits(flux.split('E')[0]) + 1
# Conversion comes from SNLS-Readme
# NOTE: Datafiles avail for download suggest diff zeropoints than 30,
# need to inquire.
magnitude = pretty_num(30.0 - 2.5 * log10(float(flux)), sig=sig)
e_mag = pretty_num(
2.5 * log10(1.0 + float(err) / float(flux)), sig=sig)
# e_mag = pretty_num(2.5*(log10(float(flux) + float(err)) -
# log10(float(flux))), sig=sig)
catalog.entries[name].add_photometry(
time=mjd, band=band, magnitude=magnitude, e_magnitude=e_mag,
counts=flux, e_counts=err, source=source)
catalog.journal_entries()
return
def load_stubs(self):
"""
"""
currenttask = 'Loading entry stubs'
files = self.PATHS.get_repo_output_file_list()
for fi in pbar(files, currenttask):
fname = fi
# FIX: should this be ``fi.endswith(``.gz')`` ?
if '.gz' in fi:
fname = uncompress_gz(fi)
name = os.path.basename(
os.path.splitext(fname)[0]).replace('.json', '')
new_entry = self.proto.init_from_file(
self, path=fname, delete=False)
# Make sure a non-stub entry doesnt already exist with this name
if name in self.entries and not self.entries[name]._stub:
err_str = (
"ERROR: non-stub entry already exists with name '{}'"
.format(name))
self.log.error(err_str)
raise RuntimeError(err_str)
self.entries[name] = new_entry.get_stub()
self.log.debug("Added stub for '{}'".format(name))
return self.entries
def do_smt(catalog):
task_str = catalog.get_current_task_str()
smt_url = 'http://www.mso.anu.edu.au/skymapper/smt/transients/tns/'
html = catalog.load_url(smt_url,
os.path.join(catalog.get_current_task_repo(),
'SMT', 'index.html'))
if not html:
return
bs = BeautifulSoup(html, 'html5lib')
trs = bs.find('table').findAll('tr')
for tr in pbar(trs, task_str):
cols = [str(xx.text) for xx in tr.findAll('td')]
if not cols:
continue
name = 'AT' + cols[0]
name, source = catalog.new_entry(name, srcname='SMT', url=smt_url)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, cols[1], source)
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, cols[2],
source)
catalog.entries[name].add_quantity(
SUPERNOVA.RA, cols[3], source, u_value='floatdegrees')
catalog.entries[name].add_quantity(
SUPERNOVA.DEC, cols[4], source, u_value='floatdegrees')
if catalog.args.update:
catalog.journal_entries()
catalog.journal_entries()
return
def do_grb(catalog):
task_str = catalog.get_current_task_str()
file_path = os.path.join(
catalog.get_current_task_repo(), 'GRB-catalog/catalog.csv')
csvtxt = catalog.load_cached_url(
'http://grb.pa.msu.edu/grbcatalog/'
'download_data?cut_0_min=10&cut_0=BAT%20T90'
'&cut_0_max=100000&num_cuts=1&no_date_cut=True',
file_path)
if not csvtxt:
return
data = list(csv.reader(csvtxt.splitlines(), delimiter=',',
quotechar='"', skipinitialspace=True))
for r, row in enumerate(pbar(data, task_str)):
if r == 0:
continue
(name,
source) = catalog.new_entry('GRB ' +
row[0],
srcname='Gamma-ray Bursts Catalog',
url='http://grbcatalog.org')
catalog.entries[name].add_quantity(
SUPERNOVA.RA, row[2], source, u_value='floatdegrees')
catalog.entries[name].add_quantity(
SUPERNOVA.DEC, row[3], source, u_value='floatdegrees')
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, row[8], source)
catalog.journal_entries()
return
def do_fermi(catalog):
task_str = catalog.get_current_task_str()
with open(
os.path.join(catalog.get_current_task_repo(),
'1SC_catalog_v01.asc'), 'r') as ff:
tsvin = list(csv.reader(ff, delimiter=','))
for ri, row in enumerate(pbar(tsvin, task_str)):
if row[0].startswith('#'):
if len(row) > 1 and 'UPPER_LIMITS' in row[1]:
break
continue
if 'Classified' not in row[1]:
continue
name = row[0].replace('SNR', 'G')
name = catalog.add_entry(name)
source = (catalog.entries[name].add_source(
bibcode='2016ApJS..224....8A'))
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS,
row[0].replace('SNR',
'MWSNR'), source)
catalog.entries[name].add_quantity(SUPERNOVA.RA,
row[2],
source,
u_value='floatdegrees')
catalog.entries[name].add_quantity(SUPERNOVA.DEC,
row[3],
source,
u_value='floatdegrees')
catalog.journal_entries()
return
def do_itep(catalog):
task_str = catalog.get_current_task_str()
itepbadsources = ['2004ApJ...602..571B']
needsbib = []
with open(os.path.join(catalog.get_current_task_repo(),
'itep-refs.txt'), 'r') as refs_file:
refrep = refs_file.read().splitlines()
refrepf = dict(list(zip(refrep[1::2], refrep[::2])))
fname = os.path.join(catalog.get_current_task_repo(),
'itep-lc-cat-28dec2015.txt')
tsvin = list(csv.reader(open(fname, 'r'),
delimiter='|', skipinitialspace=True))
curname = ''
for rr, row in enumerate(pbar(tsvin, task_str)):
if rr <= 1 or len(row) < 7:
continue
oldname = 'SN' + row[0].strip()
mjd = str(jd_to_mjd(Decimal(row[1].strip())))
band = row[2].strip()
magnitude = row[3].strip()
e_magnitude = row[4].strip()
reference = row[6].strip().strip(',')
if curname != oldname:
curname = oldname
name = catalog.add_entry(oldname)
sec_reference = ('Sternberg Astronomical Institute '
'Supernova Light Curve Catalogue')
sec_refurl = 'http://dau.itep.ru/sn/node/72'
sec_source = catalog.entries[name].add_source(
name=sec_reference, url=sec_refurl, secondary=True)
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, oldname, sec_source)
year = re.findall(r'\d+', name)[0]
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, year, sec_source)
if reference in refrepf:
bibcode = unescape(refrepf[reference])
source = catalog.entries[name].add_source(bibcode=bibcode)
else:
needsbib.append(reference)
source = catalog.entries[name].add_source(
name=reference) if reference else ''
if bibcode not in itepbadsources:
catalog.entries[name].add_photometry(time=mjd, band=band,
magnitude=magnitude,
e_magnitude=e_magnitude,
source=sec_source + ',' +
source)
# Write out references that could use aa bibcode
needsbib = list(OrderedDict.fromkeys(needsbib))
with open('../itep-needsbib.txt', 'w') as bib_file:
bib_file.writelines(['%ss\n' % ii for ii in needsbib])
catalog.journal_entries()
return
def do_snhunt(catalog):
task_str = catalog.get_current_task_str()
snh_url = 'http://nesssi.cacr.caltech.edu/catalina/current.html'
html = catalog.load_cached_url(snh_url, os.path.join(
catalog.get_current_task_repo(), 'SNhunt/current.html'))
if not html:
return
text = html.splitlines()
findtable = False
for ri, row in enumerate(text):
if 'Supernova Discoveries' in row:
findtable = True
if findtable and '<table' in row:
tstart = ri + 1
if findtable and '</table>' in row:
tend = ri - 1
tablestr = '<html><body><table>'
for row in text[tstart:tend]:
if row[:3] == 'tr>':
tablestr = tablestr + '<tr>' + row[3:]
else:
tablestr = tablestr + row
tablestr = tablestr + '</table></body></html>'
bs = BeautifulSoup(tablestr, 'html5lib')
trs = bs.find('table').findAll('tr')
for tr in pbar(trs, task_str):
cols = [str(xx.text) for xx in tr.findAll('td')]
if not cols:
continue
name = re.sub('<[^<]+?>', '', cols[4]
).strip().replace(' ', '').replace('SNHunt', 'SNhunt')
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
name='Supernova Hunt', url=snh_url)
catalog.entries[name].add_quantity('alias', name, source)
host = re.sub('<[^<]+?>', '', cols[1]).strip().replace('_', ' ')
catalog.entries[name].add_quantity('host', host, source)
catalog.entries[name].add_quantity(
'ra', cols[2], source, u_value='floatdegrees')
catalog.entries[name].add_quantity(
'dec', cols[3], source, u_value='floatdegrees')
dd = cols[0]
discoverdate = dd[:4] + '/' + dd[4:6] + '/' + dd[6:8]
catalog.entries[name].add_quantity(
'discoverdate', discoverdate, source)
discoverers = cols[5].split('/')
for discoverer in discoverers:
catalog.entries[name].add_quantity('discoverer', 'CRTS', source)
catalog.entries[name].add_quantity(
'discoverer', discoverer, source)
if catalog.args.update:
catalog.journal_entries()
catalog.journal_entries()
return
def do_simbad_novae(catalog):
task_str = catalog.get_current_task_str()
simbad_mirrors = ['http://simbad.harvard.edu/simbad/sim-script',
'http://simbad.u-strasbg.fr/simbad/sim-script']
customSimbad = Simbad()
customSimbad.ROW_LIMIT = -1
customSimbad.TIMEOUT = 120
for mirror in simbad_mirrors:
customSimbad.SIMBAD_URL = mirror
try:
table = customSimbad.query_criteria('maintype=No* | maintype="No?"')
except:
continue
else:
break
if not table:
catalog.log.warning('SIMBAD unable to load, probably offline.')
for name in pbar(catalog.entries, task_str):
try:
nova_name = "V* " + get_nova_name(name)
aliases = customSimbad.query_objectids(nova_name)
except:
#THROW WARNING HERE
tprint("Could not find " + nova_name)
continue
table = customSimbad.query_object(nova_name)
name = catalog.add_entry(name)
bibcode = table[0]['COO_BIBCODE'].decode()
ra = str(table[0]['RA'])
dec = str(table[0]['DEC'])
source = catalog.entries[name].add_source(name='SIMBAD astronomical database',
bibcode=bibcode,
url="http://simbad.u-strasbg.fr/",
secondary=True)
catalog.entries[name].add_quantity(NOVA.RA, ra, source)
catalog.entries[name].add_quantity(NOVA.DEC, dec, source)
for i in range(len(aliases)):
try: alias = aliases[i][0].decode()
except: alias = str(aliases[i][0])
catalog.entries[name].add_quantity(NOVA.ALIAS, alias, source)
catalog.journal_entries()
def delete_old_entry_files(self):
if len(self.entries):
err_str = "`delete_old_entry_files` with `entries` not empty!"
self.log.error(err_str)
raise RuntimeError(err_str)
# Delete all old entry JSON files
repo_files = self.PATHS.get_repo_output_file_list()
for rfil in pbar(repo_files, desc='Deleting old entries'):
os.remove(rfil)
self.log.debug("Deleted '{}'".format(os.path.split(rfil)[-1]))
return
def do_gaiaviatap(catalog):
# Disable warnings
warnings.filterwarnings("ignore")
#catalog.log.warning(
# 'Some warnings are thrown by the Gaia module which do not affect the result of the Gaia queries.')
task_str = catalog.get_current_task_str()
keys = list(catalog.entries.keys())
cntL = 0
for oname in pbar(keys, task_str):
try:
name = catalog.add_entry(oname)
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
### Check if has DR2 ID
dr2id = ''
for alias in catalog.entries[name][FASTSTARS.ALIAS]:
if alias['value'][:8] == 'Gaia DR2' and len(
alias['value']) > len(dr2id):
dr2id = alias['value']
if len(dr2id) < 8 or FASTSTARS.PARALLAX not in catalog.entries[name]:
#print(name)
continue
else:
with HiddenPrints():
source = catalog.entries[name].add_source(
bibcode='2018arXiv180410121B')
job = silentari(
"SELECT r_len FROM gaiadr2_complements.geometric_distance where source_id = "
+ dr2id[8:],
dump_to_file=False,
verbose=False)
lengthscale = str(job.get_results()['col_0'][0] / 1e3)
catalog.entries[name].add_quantity(
FASTSTARS.DISTANCE_PRIOR_LENGTH_SCALE,
lengthscale,
source,
u_value='kpc')
cntL += 1
catalog.log.warning(
'"{}" have a more accurate scale length from Bailer-Jones et al. (2018).'
.format(cntL))
catalog.journal_entries()
# Reactivate warnings
warnings.filterwarnings("default")
return
def do_ppmxl(catalog):
task_str = catalog.get_current_task_str()
keys = list(catalog.entries.keys())
warnings.filterwarnings("ignore")
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
try:
name = catalog.add_entry(oname)
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
if (FASTSTARS.RA not in catalog.entries[name] or
FASTSTARS.DEC not in catalog.entries[name]):
continue
else:
radec= str(catalog.entries[name][FASTSTARS.RA][0]['value'])+str(catalog.entries[name][FASTSTARS.DEC][0]['value'])
c=coord(radec,unit=(un.hourangle, un.deg),frame='icrs')
cnttry = 0
foundstar = False
while foundstar == False and cnttry < 1:
try:
cnttry += 1
time.sleep(0.1)
result = Irsa.query_region(c,catalog='ppmxl',radius='0d0m10s')
except TypeError:
#print(radec,cnttry)
continue
if len(result) > 1:
foundstar = True
if foundstar == True:
source = (catalog.entries[name]
.add_source(name='The PPMXL Catalog',
bibcode="2010AJ....139.2440R",
url="https://irsa.ipac.caltech.edu/Missions/ppmxl.html",
secondary=True))
catalog.entries[name].add_quantity(FASTSTARS.PROPER_MOTION_RA, str(result['pmra'][0]*degperyrtomasperyear), source, e_value=str(result['e_pmra'][0]*degperyrtomasperyear), u_value='mas/yr')
catalog.entries[name].add_quantity(FASTSTARS.PROPER_MOTION_DEC, str(result['pmde'][0]*degperyrtomasperyear), source, e_value=str(result['e_pmde'][0]*degperyrtomasperyear), u_value='mas/yr')
catalog.journal_entries()
return
def do_ptss_meta(catalog):
"""Import metadata from PTSS webpage."""
task_str = catalog.get_current_task_str()
years = list(range(2015, datetime.today().year + 1))
for year in years:
jsontxt = None
while jsontxt is None:
try:
jsontxt = catalog.load_url(
'http://www.cneost.org/ptss/fetchlist.php?vip=sn&gdate=' +
str(year),
os.path.join(catalog.get_current_task_repo(),
'PTSS/catalog-' + str(year) + '.json'),
json_sort='name',
timeout=5)
except Exception:
pass
meta = json.loads(jsontxt)
for met in pbar(meta, task_str + ' - ' + str(year)):
oldname = met['name']
name, source = catalog.new_entry(
oldname,
srcname='PMO & Tsinghua Supernova Survey (PTSS)',
url='http://www.cneost.org/ptss/index.php')
coo = coord(met['ra'], met['dec'], unit=(un.deg, un.deg))
catalog.entries[name].add_quantity(
SUPERNOVA.RA, coo.ra.to_string(unit=un.hour, sep=':'), source)
catalog.entries[name].add_quantity(
SUPERNOVA.DEC, coo.dec.to_string(unit=un.degree, sep=':'),
source)
if met['filter'] is not None:
mjd = str(astrotime(met['obsdate'], format='isot').mjd)
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.MAGNITUDE: str(met['mag']),
PHOTOMETRY.E_MAGNITUDE: str(met['magerr']),
PHOTOMETRY.BAND: met['filter'].replace('sdss-', ''),
PHOTOMETRY.SOURCE: source
}
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
return
def set_preferred_names(self):
"""Choose between each entries given name and its possible aliases for
the best one.
"""
if len(self.entries) == 0:
self.log.error("WARNING: `entries` is empty, loading stubs")
self.load_stubs()
task_str = self.get_current_task_str()
for ni, oname in enumerate(pbar(self.entries, task_str)):
name = self.add_entry(oname)
self.entries[name].set_preferred_name()
if self.args.travis and ni > self.TRAVIS_QUERY_LIMIT:
break
return
def do_ptss_meta(catalog):
"""Import metadata from PTSS webpage."""
task_str = catalog.get_current_task_str()
years = list(range(2015, datetime.today().year + 1))
for year in years:
jsontxt = None
while jsontxt is None:
try:
jsontxt = catalog.load_url(
'http://www.cneost.org/ptss/fetchlist.php?vip=sn&gdate=' +
str(year),
os.path.join(catalog.get_current_task_repo(),
'PTSS/catalog-' + str(year) + '.json'),
json_sort='name', timeout=5)
except Exception:
pass
meta = json.loads(jsontxt)
for met in pbar(meta, task_str + ' - ' + str(year)):
oldname = met['name']
name, source = catalog.new_entry(
oldname, srcname='PMO & Tsinghua Supernova Survey (PTSS)',
url='http://www.cneost.org/ptss/index.php')
coo = coord(met['ra'], met['dec'], unit=(un.deg, un.deg))
catalog.entries[name].add_quantity(
SUPERNOVA.RA, coo.ra.to_string(unit=un.hour, sep=':'), source)
catalog.entries[name].add_quantity(
SUPERNOVA.DEC, coo.dec.to_string(unit=un.degree, sep=':'),
source)
if met['filter'] is not None:
mjd = str(astrotime(met['obsdate'], format='isot').mjd)
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.MAGNITUDE: str(met['mag']),
PHOTOMETRY.E_MAGNITUDE: str(met['magerr']),
PHOTOMETRY.BAND: met['filter'].replace('sdss-', ''),
PHOTOMETRY.SOURCE: source
}
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
return
def do_dust(catalog):
task_str = catalog.get_current_task_str()
# Set preferred names, calculate some columns based on imported data,
# sanitize some fields
keys = list(catalog.entries.keys())
check_dustmaps(catalog.get_current_task_repo())
sfd = SFDQuery()
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
try:
name = catalog.add_entry(oname)
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
if (FASTSTARS.RA not in catalog.entries[name]
or FASTSTARS.DEC not in catalog.entries[name]):
continue
else:
Mname = name
Mradec = str(
catalog.entries[name][FASTSTARS.RA][0]['value']) + str(
catalog.entries[name][FASTSTARS.DEC][0]['value'])
Mdist = '-1'
c = coord(Mradec, unit=(un.hourangle, un.deg), frame='icrs')
reddening = sfd(c)
source = catalog.entries[name].add_source(
bibcode='1998ApJ...500..525S')
catalog.entries[name].add_quantity(FASTSTARS.EBV,
str(reddening),
source,
upperlimit=True,
derived=True)
catalog.journal_entries()
return
def do_hst(catalog):
task_str = catalog.get_current_task_str()
url = 'http://archive.stsci.edu/hst/search.php'
reference = 'Hubble Pointings'
jtxt = catalog.load_url(
url,
os.path.join(catalog.get_current_task_repo(), 'HST.json'),
post={
'sci_target_descrip': '*supernova*',
'outputformat': 'JSON_file',
'action': 'Search',
'max_records': '50000',
'max_rpp': '50000'
},
verify=False)
rows = json.loads(jtxt)
allowed_prefixes = ('PS1', 'DES', 'GAIA', 'ASASSN', 'AT', 'IPTF', 'LSQ',
'PTF')
loopcnt = 0
for row in pbar(rows, task_str):
oldname = name_clean(row['Target Name'])
if not oldname.upper().startswith(allowed_prefixes):
continue
if oldname.startswith('PS1-') and not is_number(oldname[4]):
continue
name, source = catalog.new_entry(oldname, srcname=reference, url=url)
if (ENTRY.RA in catalog.entries[name] and
ENTRY.DEC in catalog.entries[name]):
continue
catalog.entries[name].add_quantity(
ENTRY.RA, row['RA (J2000)'], source=source)
catalog.entries[name].add_quantity(
ENTRY.DEC, row['Dec (J2000)'], source=source)
catalog.journal_entries()
loopcnt = loopcnt + 1
if (catalog.args.travis and loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
def do_pessto(catalog):
task_str = catalog.get_current_task_str()
pessto_path = os.path.join(
catalog.get_current_task_repo(), 'PESSTO_MPHOT.csv')
tsvin = list(csv.reader(open(pessto_path, 'r'), delimiter=','))
for ri, row in enumerate(pbar(tsvin, task_str)):
if ri == 0:
bands = [xx.split('_')[0] for xx in row[3::2]]
systems = [xx.split('_')[1].capitalize().replace(
'Ab', 'AB') for xx in row[3::2]]
continue
name = row[1]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2015A&A...579A..40S')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
for hi, ci in enumerate(range(3, len(row) - 1, 2)):
if not row[ci]:
continue
if systems[hi] == 'Swift':
teles = 'Swift'
instrument = 'UVOT'
bandset = 'Swift'
else:
teles = 'NTT'
instrument = 'EFOSC'
bandset = 'Johnson'
photodict = {
PHOTOMETRY.TIME: row[2],
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.MAGNITUDE: row[ci],
PHOTOMETRY.E_MAGNITUDE: row[ci + 1],
PHOTOMETRY.BAND: bands[hi],
PHOTOMETRY.SYSTEM: systems[hi],
PHOTOMETRY.BAND_SET: bandset,
PHOTOMETRY.TELESCOPE: teles,
PHOTOMETRY.INSTRUMENT: instrument,
PHOTOMETRY.SOURCE: source
}
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
return
def do_scp(catalog):
task_str = catalog.get_current_task_str()
tsvin = list(
csv.reader(open(
os.path.join(catalog.get_current_task_repo(), 'SCP09.csv'), 'r'),
delimiter=','))
for ri, row in enumerate(pbar(tsvin, task_str)):
if ri == 0:
continue
name = row[0].replace('SCP', 'SCP-')
name = catalog.add_entry(name)
source = (catalog.entries[name].add_source(
name='Supernova Cosmology Project',
url=('http://supernova.lbl.gov/'
'2009ClusterSurvey/')))
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
if row[1]:
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[1], source)
if row[2]:
kind = 'spectroscopic' if row[3] == 'sn' else SUPERNOVA.HOST
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT,
row[2],
source,
kind=kind)
if row[4]:
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT,
row[2],
source,
kind='cluster')
if row[6]:
claimedtype = row[6].replace('SN ', '')
kind = ('spectroscopic/light curve'
if 'a' in row[7] and 'c' in row[7] else 'spectroscopic' if
'a' in row[7] else 'light curve' if 'c' in row[7] else '')
if claimedtype != '?':
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE,
claimedtype,
source,
kind=kind)
catalog.journal_entries()
return
def do_snls_photo(catalog):
task_str = catalog.get_current_task_str()
snls_path = os.path.join(catalog.get_current_task_repo(), 'SNLS-ugriz.dat')
data = list(
csv.reader(
open(snls_path, 'r'),
delimiter=' ',
quotechar='"',
skipinitialspace=True))
for row in pbar(data, task_str):
counts = row[3]
err = row[4]
name = 'SNLS-' + row[0]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2010A&A...523A...7G')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
# Conversion comes from SNLS-Readme
# NOTE: Datafiles avail for download suggest diff zeropoints than 30,
# but README states mags should be calculated assuming 30. Need to
# inquire.
zp = 30.0
photodict = {
PHOTOMETRY.TIME: row[2],
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.BAND: row[1],
PHOTOMETRY.COUNT_RATE: counts,
PHOTOMETRY.E_COUNT_RATE: err,
PHOTOMETRY.ZERO_POINT: str(zp),
PHOTOMETRY.SOURCE: source,
PHOTOMETRY.TELESCOPE: 'CFHT',
PHOTOMETRY.INSTRUMENT: 'MegaCam',
PHOTOMETRY.BAND_SET: 'MegaCam',
PHOTOMETRY.SYSTEM: 'BD17'
}
set_pd_mag_from_counts(photodict, counts, ec=err, zp=zp, sig=5.0)
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
return
def do_snls_photo(catalog):
task_str = catalog.get_current_task_str()
snls_path = os.path.join(catalog.get_current_task_repo(), 'SNLS-ugriz.dat')
data = list(
csv.reader(open(snls_path, 'r'),
delimiter=' ',
quotechar='"',
skipinitialspace=True))
for row in pbar(data, task_str):
counts = row[3]
err = row[4]
name = 'SNLS-' + row[0]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2010A&A...523A...7G')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
# Conversion comes from SNLS-Readme
# NOTE: Datafiles avail for download suggest diff zeropoints than 30,
# but README states mags should be calculated assuming 30. Need to
# inquire.
zp = 30.0
photodict = {
PHOTOMETRY.TIME: row[2],
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.BAND: row[1],
PHOTOMETRY.COUNT_RATE: counts,
PHOTOMETRY.E_COUNT_RATE: err,
PHOTOMETRY.ZERO_POINT: str(zp),
PHOTOMETRY.SOURCE: source,
PHOTOMETRY.TELESCOPE: 'CFHT',
PHOTOMETRY.INSTRUMENT: 'MegaCam',
PHOTOMETRY.BAND_SET: 'MegaCam',
PHOTOMETRY.SYSTEM: 'BD17'
}
set_pd_mag_from_counts(photodict, counts, ec=err, zp=zp, sig=5.0)
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
return
def do_scp(catalog):
task_str = catalog.get_current_task_str()
tsvin = list(csv.reader(open(
os.path.join(catalog.get_current_task_repo(), 'SCP09.csv'), 'r'),
delimiter=','))
for ri, row in enumerate(pbar(tsvin, task_str)):
if ri == 0:
continue
name = row[0].replace('SCP', 'SCP-')
name = catalog.add_entry(name)
source = (catalog.entries[name]
.add_source(name='Supernova Cosmology Project',
url=('http://supernova.lbl.gov/'
'2009ClusterSurvey/')))
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
if row[1]:
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[1], source)
if row[2]:
kind = 'spectroscopic' if row[3] == 'sn' else SUPERNOVA.HOST
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, row[2], source, kind=kind)
if row[4]:
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, row[2], source, kind='cluster')
if row[6]:
claimedtype = row[6].replace('SN ', '')
kind = ('spectroscopic/light curve' if 'a' in row[7] and 'c' in
row[7] else
'spectroscopic' if 'a' in row[7] else
'light curve' if 'c' in row[7]
else '')
if claimedtype != '?':
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, claimedtype, source, kind=kind)
catalog.journal_entries()
return
def merge_duplicates(self):
"""Merge and remove duplicate entries.
Compares each entry ('name') in `stubs` to all later entries to check
for duplicates in name or alias. If a duplicate is found, they are
merged and written to file.
"""
if len(self.entries) == 0:
self.log.error("WARNING: `entries` is empty, loading stubs")
if self.args.update:
self.log.warning(
"No sources changed, entry files unchanged in update."
" Skipping merge.")
return
self.entries = self.load_stubs()
task_str = self.get_current_task_str()
keys = list(sorted(self.entries.keys()))
for n1, name1 in enumerate(pbar(keys, task_str)):
allnames1 = set(self.entries[name1].get_aliases())
if name1.startswith('SN') and is_number(name1[2:6]):
allnames1 = allnames1.union(['AT' + name1[2:]])
# Search all later names
for name2 in keys[n1 + 1:]:
if name1 == name2:
continue
allnames2 = set(self.entries[name2].get_aliases())
if name2.startswith('SN') and is_number(name2[2:6]):
allnames2.union(['AT' + name2[2:]])
# If there are any common names or aliases, merge
if len(allnames1 & allnames2):
self.log.warning(
"Found single entry with multiple entries "
"('{}' and '{}'), merging.".format(name1, name2))
load1 = self.proto.init_from_file(
self, name=name1)
load2 = self.proto.init_from_file(
self, name=name2)
if load1 is not None and load2 is not None:
# Delete old files
self._delete_entry_file(entry=load1)
self._delete_entry_file(entry=load2)
self.entries[name1] = load1
self.entries[name2] = load2
priority1 = 0
priority2 = 0
for an in allnames1:
if an.startswith(('SN', 'AT')):
priority1 += 1
for an in allnames2:
if an.startswith(('SN', 'AT')):
priority2 += 1
if priority1 > priority2:
self.copy_to_entry(name2, name1)
keys.append(name1)
del self.entries[name2]
else:
self.copy_to_entry(name1, name2)
keys.append(name2)
del self.entries[name1]
else:
self.log.warning('Duplicate already deleted')
# if len(self.entries) != 1:
# self.log.error(
# "WARNING: len(entries) = {}, expected 1. "
# "Still journaling...".format(len(self.entries)))
self.journal_entries()
if self.args.travis and n1 > self.TRAVIS_QUERY_LIMIT:
break
def do_crts(catalog):
"""Import data from the Catalina Real-Time Transient Survey."""
crtsnameerrors = ['2011ax']
task_str = catalog.get_current_task_str()
folders = ['catalina', 'MLS', 'MLS', 'SSS']
files = ['AllSN.html', 'AllSN.arch.html', 'CRTSII_SN.html', 'AllSN.html']
for fi, fold in enumerate(pbar(folders, task_str)):
html = catalog.load_url(
'http://nesssi.cacr.caltech.edu/' + fold + '/' + files[fi],
os.path.join(catalog.get_current_task_repo(), 'CRTS', fold + '-' +
files[fi]), archived_mode=('arch' in files[fi]))
html = html.replace('<ahref=', '<a href=')
if not html:
continue
bs = BeautifulSoup(html, 'html5lib')
trs = bs.findAll('tr')
for tri, tr in enumerate(pbar(trs, task_str)):
tds = tr.findAll('td')
if not tds:
continue
# refs = []
aliases = []
crtsname = ''
ra = ''
dec = ''
lclink = ''
# ttype = ''
# ctype = ''
for tdi, td in enumerate(tds):
if tdi == 0:
crtsname = td.contents[0].text.strip()
elif tdi == 1:
ra = td.contents[0]
elif tdi == 2:
dec = td.contents[0]
elif tdi == (8 if files[fi] == 'CRTSII_SN.html' else 11):
lclink = td.find('a')['onclick']
lclink = lclink.split("'")[1]
elif tdi == (10 if files[fi] == 'CRTSII_SN.html' else 13):
aliases = re.sub('[()]', '', re.sub(
'<[^<]+?>', '', td.contents[-1].strip()))
aliases = [xx.strip('; ') for xx in list(
filter(None, aliases.split(' ')))]
name = ''
hostmag = ''
hostupper = False
validaliases = []
for ai, alias in enumerate(aliases):
if alias in ['SN', 'SDSS']:
continue
if alias in crtsnameerrors:
continue
if alias == 'mag':
if ai < len(aliases) - 1:
ind = ai + 1
if aliases[ai + 1] in ['SDSS']:
ind = ai + 2
elif aliases[ai + 1] in ['gal', 'obj', 'object',
'source']:
ind = ai - 1
if '>' in aliases[ind]:
hostupper = True
hostmag = aliases[ind].strip('>~').replace(
',', '.').replace('m', '.')
continue
if (is_number(alias[:4]) and alias[:2] == '20' and
len(alias) > 4):
name = 'SN' + alias
if ((('asassn' in alias and len(alias) > 6) or
('ptf' in alias and len(alias) > 3) or
('ps1' in alias and len(alias) > 3) or
'snhunt' in alias or
('mls' in alias and len(alias) > 3) or
'gaia' in alias or
('lsq' in alias and len(alias) > 3))):
alias = alias.replace('SNHunt', 'SNhunt')
validaliases.append(alias)
if not name:
name = crtsname
name, source = catalog.new_entry(
name, srcname='Catalina Sky Survey',
bibcode='2009ApJ...696..870D',
url='http://nesssi.cacr.caltech.edu/catalina/AllSN.html')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
for alias in validaliases:
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, alias, source)
catalog.entries[name].add_quantity(
SUPERNOVA.RA, ra.strip(), source, u_value='floatdegrees')
catalog.entries[name].add_quantity(
SUPERNOVA.DEC, dec.strip(), source, u_value='floatdegrees')
if SUPERNOVA.CLAIMED_TYPE not in catalog.entries[name]:
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, 'Candidate', source)
if hostmag:
# 1.0 magnitude error based on Drake 2009 assertion that SN are
# only considered
# real if they are 2 mags brighter than host.
photodict = {
PHOTOMETRY.BAND: 'C',
PHOTOMETRY.MAGNITUDE: hostmag,
PHOTOMETRY.E_MAGNITUDE: '1.0',
PHOTOMETRY.SOURCE: source,
PHOTOMETRY.HOST: True,
PHOTOMETRY.TELESCOPE: 'Catalina Schmidt',
PHOTOMETRY.UPPER_LIMIT: hostupper
}
catalog.entries[name].add_photometry(**photodict)
fname2 = (catalog.get_current_task_repo() + '/' + fold + '/' +
lclink.split('.')[-2].rstrip('p').split('/')[-1] +
'.html')
html2 = catalog.load_url(lclink, fname2)
if not html2:
continue
lines = html2.splitlines()
teles = 'Catalina Schmidt'
for line in lines:
if 'javascript:showx' in line:
search = re.search("showx\('(.*?)'\)", line)
if not search:
continue
mjdstr = search.group(1).split('(')[0].strip()
if not is_number(mjdstr):
continue
mjd = str(Decimal(mjdstr) + Decimal(53249.0))
else:
continue
mag = ''
err = ''
if 'javascript:showy' in line:
mag = re.search("showy\('(.*?)'\)", line).group(1)
if 'javascript:showz' in line:
err = re.search("showz\('(.*?)'\)", line).group(1)
if not is_number(mag) or (err and not is_number(err)):
continue
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.E_TIME: '0.125', # 3 hr error
PHOTOMETRY.BAND: 'C',
PHOTOMETRY.MAGNITUDE: mag,
PHOTOMETRY.SOURCE: source,
PHOTOMETRY.INCLUDES_HOST: True,
PHOTOMETRY.TELESCOPE: teles
}
if float(err) > 0.0:
photodict[PHOTOMETRY.E_MAGNITUDE] = err
if float(err) == 0.0:
photodict[PHOTOMETRY.UPPER_LIMIT] = True
catalog.entries[name].add_photometry(**photodict)
if catalog.args.update:
catalog.journal_entries()
if catalog.args.travis and tri > catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
return
def do_ptf(catalog):
# response =
# urllib.request.urlopen('http://wiserep.weizmann.ac.il/objects/list')
# bs = BeautifulSoup(response, 'html5lib')
# select = bs.find('select', {'name': 'objid'})
# options = select.findAll('option')
# for option in options:
# print(option.text)
# name = option.text
# if ((name.startswith('PTF') and is_number(name[3:5])) or
# name.startswith('PTFS') or name.startswith('iPTF')):
# name = catalog.add_entry(name)
task_str = catalog.get_current_task_str()
if catalog.current_task.load_archive(catalog.args):
with open(os.path.join(catalog.get_current_task_repo(),
'PTF/update.html'), 'r') as f:
html = f.read()
else:
session = requests.Session()
response = session.get('http://wiserep.weizmann.ac.il/spectra/update')
html = response.text
with open(os.path.join(catalog.get_current_task_repo(),
'PTF/update.html'), 'w') as f:
f.write(html)
bs = BeautifulSoup(html, 'html5lib')
select = bs.find('select', {'name': 'objid'})
options = select.findAll('option')
for option in pbar(options, task_str):
name = option.text
if (((name.startswith('PTF') and is_number(name[3:5])) or
name.startswith('PTFS') or name.startswith('iPTF'))):
if '(' in name:
alias = name.split('(')[0].strip(' ')
name = name.split('(')[-1].strip(') ').replace('sn', 'SN')
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2012PASP..124..668Y')
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, alias, source)
else:
# name = catalog.add_entry(name)
name, source = catalog.new_entry(name,
bibcode='2012PASP..124..668Y')
with open(os.path.join(
catalog.get_current_task_repo(), 'PTF/old-ptf-events.csv')) as f:
for suffix in pbar(f.read().splitlines(), task_str):
name = catalog.add_entry('PTF' + suffix)
with open(os.path.join(
catalog.get_current_task_repo(), 'PTF/perly-2016.csv')) as f:
for row in pbar(f.read().splitlines(), task_str):
cols = [x.strip() for x in row.split(',')]
alias = ''
if cols[8]:
name = cols[8]
alias = 'PTF' + cols[0]
else:
name = 'PTF' + cols[0]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2016arXiv160408207P')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
if alias:
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, alias, source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, cols[1], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, cols[2], source)
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, 'SLSN-' + cols[3], source)
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, cols[4], source, kind='spectroscopic')
maxdate = cols[6].replace('-', '/')
upl = maxdate.startswith('<')
catalog.entries[name].add_quantity(
SUPERNOVA.MAX_DATE, maxdate.lstrip('<'), source,
upperlimit=upl)
catalog.entries[name].add_quantity(
SUPERNOVA.EBV, cols[7], source, kind='spectroscopic')
name = catalog.add_entry('PTF' + suffix)
catalog.journal_entries()
return
def do_cfa_photo(catalog):
"""Import photometry from the CfA archive."""
from html import unescape
import re
task_str = catalog.get_current_task_str()
file_names = glob(
os.path.join(catalog.get_current_task_repo(), 'cfa-input/*.dat'))
for fname in pbar_strings(file_names, task_str):
f = open(fname, 'r')
tsvin = csv.reader(f, delimiter=' ', skipinitialspace=True)
csv_data = []
for r, row in enumerate(tsvin):
new = []
for item in row:
new.extend(item.split('\t'))
csv_data.append(new)
for r, row in enumerate(csv_data):
for c, col in enumerate(row):
csv_data[r][c] = col.strip()
csv_data[r] = [_f for _f in csv_data[r] if _f]
eventname = os.path.basename(os.path.splitext(fname)[0])
eventparts = eventname.split('_')
name = clean_snname(eventparts[0])
name = catalog.add_entry(name)
secondaryname = 'CfA Supernova Archive'
secondaryurl = 'https://www.cfa.harvard.edu/supernova/SNarchive.html'
secondarysource = catalog.entries[name].add_source(
name=secondaryname,
url=secondaryurl,
secondary=True,
acknowledgment=ACKN_CFA)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name,
secondarysource)
year = re.findall(r'\d+', name)[0]
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, year,
secondarysource)
eventbands = list(eventparts[1])
tu = 'MJD'
jdoffset = Decimal(0.)
for rc, row in enumerate(csv_data):
if len(row) > 0 and row[0][0] == "#":
if len(row[0]) > 2 and row[0][:3] == '#JD':
tu = 'JD'
rowparts = row[0].split('-')
jdoffset = Decimal(rowparts[1])
elif len(row[0]) > 6 and row[0][:7] == '#Julian':
tu = 'JD'
jdoffset = Decimal(0.)
elif len(row) > 1 and row[1].lower() == 'photometry':
for ci, col in enumerate(row[2:]):
if col[0] == "(":
refstr = ' '.join(row[2 + ci:])
refstr = refstr.replace('(', '').replace(')', '')
bibcode = unescape(refstr)
source = catalog.entries[name].add_source(
bibcode=bibcode)
elif len(row) > 1 and row[1] == 'HJD':
tu = 'HJD'
continue
elif len(row) > 0:
mjd = row[0]
for v, val in enumerate(row):
if v == 0:
if tu == 'JD':
mjd = str(jd_to_mjd(Decimal(val) + jdoffset))
tuout = 'MJD'
elif tu == 'HJD':
mjd = str(jd_to_mjd(Decimal(val)))
tuout = 'MJD'
else:
mjd = val
tuout = tu
elif v % 2 != 0:
if float(row[v]) < 90.0:
src = secondarysource + ',' + source
photodict = {
PHOTOMETRY.U_TIME: tuout,
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.BAND_SET: 'Standard',
PHOTOMETRY.BAND: eventbands[(v - 1) // 2],
PHOTOMETRY.MAGNITUDE: row[v],
PHOTOMETRY.E_MAGNITUDE: row[v + 1],
PHOTOMETRY.SOURCE: src
}
catalog.entries[name].add_photometry(**photodict)
f.close()
# Hicken 2012
with open(
os.path.join(catalog.get_current_task_repo(),
'hicken-2012-standard.dat'), 'r') as infile:
tsvin = list(csv.reader(infile, delimiter='|', skipinitialspace=True))
for r, row in enumerate(pbar(tsvin, task_str)):
if r <= 47:
continue
if row[0][:2] != 'sn':
name = 'SN' + row[0].strip()
else:
name = row[0].strip()
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2012ApJS..200...12H')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, 'Ia',
source)
photodict = {
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.TIME: row[2].strip(),
PHOTOMETRY.BAND: row[1].strip(),
PHOTOMETRY.BAND_SET: 'Standard',
PHOTOMETRY.MAGNITUDE: row[6].strip(),
PHOTOMETRY.E_MAGNITUDE: row[7].strip(),
PHOTOMETRY.SOURCE: source
}
catalog.entries[name].add_photometry(**photodict)
# Bianco 2014
with open(
os.path.join(catalog.get_current_task_repo(),
'bianco-2014-standard.dat'), 'r') as infile:
tsvin = list(csv.reader(infile, delimiter=' ', skipinitialspace=True))
for row in pbar(tsvin, task_str):
name = 'SN' + row[0]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2014ApJS..213...19B')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
photodict = {
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.TIME: row[2],
PHOTOMETRY.BAND: row[1],
PHOTOMETRY.MAGNITUDE: row[3],
PHOTOMETRY.E_MAGNITUDE: row[4],
PHOTOMETRY.TELESCOPE: row[5],
PHOTOMETRY.BAND_SET: 'Standard',
PHOTOMETRY.SOURCE: source
}
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
return
def do_asassn(catalog):
"""Import list of ASASSN events."""
task_str = catalog.get_current_task_str()
asn_url = 'http://www.astronomy.ohio-state.edu/~assassin/transients.html'
html = catalog.load_url(
asn_url,
os.path.join(catalog.get_current_task_repo(),
'ASASSN/transients.html'))
if not html:
return
bs = BeautifulSoup(html, 'html5lib')
trs = bs.find('table').findAll('tr')
for tri, tr in enumerate(pbar(trs, task_str)):
name = ''
ra = ''
dec = ''
discdate = ''
mag = ''
atellink = ''
sdsslink = ''
comment = '' #store to check if entry is a CV later
if tri == 1:
continue
tds = tr.findAll('td')
atex = 0
for tdi, td in enumerate(tds):
if tdi == 0 and td.text.strip() != '---' and td.text.strip(
) != '' and td.text.strip() != '----':
try:
# name = catalog.add_entry(td.text.strip().replace("?","_"))
star_name = td.text.strip().replace("?", "_")
# atellink = td.find('a')
atex = 1
# if atellink:
# atellink = atellink['href']
# else:
# atellink = ''
except ValueError:
pass
if tdi == 1 and td.text.strip() != '---':
if atex == 0:
# name = catalog.add_entry(td.text.strip().replace("?","_").replace(':','_'))
star_name = td.text.strip().replace("?",
"_").replace(":", "_")
# atellink = td.find('a')
# if atellink:
# atellink = atellink['href']
# else:
# atellink = ''
else:
pass
if tdi == 3:
ra = td.text
if tdi == 4:
dec = td.text
if tdi == 5:
discdate = td.text.replace('-', '/')
if not discdate.startswith('2') or discdate.startswith('1'):
discdate = ''
# if discdate == 'bc094//0':
# discdate = '' #ASASSN-15co was having bc094--0 as a date
if tdi == 6:
mag_place = td.text
if td.text == '' or mag_place[-1:] == ')':
mag = td.text
elif int(float(td.text)) >= 8:
mag = td.text
if tdi == 7:
sdsslink = td.find('a')
if sdsslink:
sdsslink = sdsslink['href']
else:
sdsslink = ''
if tdi == 11:
comment = td.text
if 'CV' in comment:
name = catalog.add_entry(star_name)
sources = [
catalog.entries[name].add_source(name='ASAS-CV Transients',
bibcode="2014ApJ...788...48S",
url=asn_url)
]
typesources = sources[:]
if atellink:
sources.append((catalog.entries[name].add_source(
name='ATel ' + atellink.split('=')[-1], url=atellink)))
# if typelink:
# typesources.append(
# (catalog.entries[name]
# .add_source(name='ATel ' +
# typelink.split('=')[-1], url=typelink)))
sources = ','.join(sources)
typesources = ','.join(typesources)
catalog.entries[name].add_quantity(CATACLYSMIC.ALIAS, name,
sources)
catalog.entries[name].add_quantity(CATACLYSMIC.DISCOVER_DATE,
discdate, sources)
catalog.entries[name].add_quantity(CATACLYSMIC.RA,
ra,
sources,
u_value='floatdegrees')
catalog.entries[name].add_quantity(CATACLYSMIC.DEC,
dec,
sources,
u_value='floatdegrees')
catalog.entries[name].add_quantity(CATACLYSMIC.MAX_VISUAL_APP_MAG,
mag, sources)
if 'known CV' in comment and not 'known CV candidate' in comment:
catalog.entries[name].add_quantity(CATACLYSMIC.CLAIMED_TYPE,
'known CV', sources)
if 'CV candidate' in comment:
catalog.entries[name].add_quantity(CATACLYSMIC.CLAIMED_TYPE,
'Candidate', sources)
if 'possible Nova' in comment or 'Possible Nova' in comment or 'Nova candidate' in comment or 'nova candidate' in comment and not 'CV' in comment:
name = catalog.add_entry(star_name)
sources = [
catalog.entries[name].add_source(url=asn_url,
name='ASAS-CV Transients')
]
typesources = sources[:]
if atellink:
sources.append((catalog.entries[name].add_source(
name='ATel ' + atellink.split('=')[-1], url=atellink)))
# if typelink:
# typesources.append(
# (catalog.entries[name]
# .add_source(name='ATel ' +
# typelink.split('=')[-1], url=typelink)))
sources = ','.join(sources)
typesources = ','.join(typesources)
catalog.entries[name].add_quantity(CATACLYSMIC.ALIAS, name,
sources)
catalog.entries[name].add_quantity(CATACLYSMIC.DISCOVER_DATE,
discdate, sources)
catalog.entries[name].add_quantity(CATACLYSMIC.RA,
ra,
sources,
u_value='floatdegrees')
catalog.entries[name].add_quantity(CATACLYSMIC.DEC,
dec,
sources,
u_value='floatdegrees')
catalog.entries[name].add_quantity(CATACLYSMIC.MAX_VISUAL_APP_MAG,
mag, sources)
catalog.entries[name].add_quantity(CATACLYSMIC.CLAIMED_TYPE,
'Candidate Nova', sources)
# for ct in claimedtype.split('/'):
# if ct != 'Unk':
# catalog.entries[name].add_quantity(CATACLYSMIC.CLAIMED_TYPE, ct,
# typesources)
else:
pass
catalog.journal_entries()
return
#talk to professor about light curves data
#def do_asas_atels(catalog):
# """Import LCs exposed in ASASSN Atels."""
# with open('/root/better-atel/atels.json') as f:
# ateljson = json.load(f)
# for entry in ateljson:
# if ('asas-sn.osu.edu/light_curve' in entry['body'] and
# 'Cataclysmic' in entry['subjects']):
# matches = re.findall(r'<a\s+[^>]*?href="([^"]*)".*?>(.*?)<\/a>',
# entry['body'], re.DOTALL)
# lcurl = ''
# objname = ''
# for match in matches:
# if 'asas-sn.osu.edu/light_curve' in match[0]:
# lcurl = match[0]
# objname = re.findall(
# r'\bASASSN-[0-9][0-9].*?\b', match[1])
# if len(objname):
# objname = objname[0]
# if objname and lcurl:
# name, source = catalog.new_entry(
# objname, srcname='ASAS-SN Sky Patrol',
# bibcode='2017arXiv170607060K',
# url='https://asas-sn.osu.edu')
# csv = catalog.load_url(lcurl + '.csv', os.path.join(
# catalog.get_current_task_repo(), os.path.join(
# 'ASASSN', objname + '.csv')))
# data = read(csv, format='csv')
# for row in data:
# mag = str(row['mag'])
# if float(mag.strip('>')) > 50.0:
# continue
# photodict = {
# PHOTOMETRY.TIME: str(jd_to_mjd(
# Decimal(str(row['HJD'])))),
# PHOTOMETRY.MAGNITUDE: mag.strip('>'),
# PHOTOMETRY.SURVEY: 'ASASSN',
# PHOTOMETRY.SOURCE: source
# }
# if '>' in mag:
# photodict[PHOTOMETRY.UPPER_LIMIT] = True
# else:
# photodict[PHOTOMETRY.E_MAGNITUDE] = str(row['mag_err'])
# catalog.entries[name].add_photometry(**photodict)
# catalog.journal_entries()
# return
def do_ps_threepi(catalog):
"""Import data from Pan-STARRS' 3pi page."""
task_str = catalog.get_current_task_str()
bad_aliases = ['SN1994J']
teles = 'Pan-STARRS1'
fname = os.path.join(catalog.get_current_task_repo(), '3pi/page00.html')
ps_url = ("http://psweb.mp.qub.ac.uk/"
"ps1threepi/psdb/public/?page=1&sort=followup_flag_date")
html = catalog.load_url(ps_url, fname, write=False)
if not html:
return
# Clean some common HTML manglings
html = html.replace('ahref=', 'a href=')
bs = BeautifulSoup(html, 'html5lib')
div = bs.find('div', {'class': 'pagination'})
offline = False
if not div:
offline = True
else:
links = div.findAll('a')
if not links:
offline = True
if offline:
if catalog.args.update:
return
warnings.warn('Pan-STARRS 3pi offline, using local files only.')
with open(fname, 'r') as f:
html = f.read()
bs = BeautifulSoup(html, 'html5lib')
div = bs.find('div', {'class': 'pagination'})
links = div.findAll('a')
else:
with open(fname, 'w') as f:
f.write(html)
numpages = int(links[-2].contents[0])
oldnumpages = len(
glob(os.path.join(catalog.get_current_task_repo(), '3pi/page*')))
for page in pbar(range(1, numpages), task_str):
fname = os.path.join(catalog.get_current_task_repo(), '3pi/page') + \
str(page).zfill(2) + '.html'
if offline:
if not os.path.isfile(fname):
continue
with open(fname, 'r') as f:
html = f.read()
else:
if (catalog.current_task.load_archive(catalog.args) and
page < oldnumpages and os.path.isfile(fname)):
with open(fname, 'r') as f:
html = f.read()
else:
response = urllib.request.urlopen(
"http://psweb.mp.qub.ac.uk/ps1threepi/psdb/public/?page=" +
str(page) + "&sort=followup_flag_date")
with open(fname, 'w') as f:
html = response.read().decode('utf-8')
f.write(html)
bs = BeautifulSoup(html, 'html5lib')
trs = bs.findAll('tr')
for tr in pbar(trs, task_str):
tds = tr.findAll('td')
if not tds:
continue
refs = []
aliases = []
ttype = ''
ctype = ''
for tdi, td in enumerate(tds):
if tdi == 0:
psname = td.contents[0]
pslink = psname['href']
psname = psname.text
elif tdi == 1:
ra = td.contents[0]
elif tdi == 2:
dec = td.contents[0]
elif tdi == 3:
ttype = td.contents[0]
if ttype != 'sn' and ttype != 'orphan':
break
elif tdi == 6:
if not td.contents:
continue
ctype = td.contents[0]
if ctype == 'Observed':
ctype = ''
elif tdi == 17:
if td.contents:
crossrefs = td.findAll('a')
for cref in crossrefs:
if 'atel' in cref.contents[0].lower():
refs.append([cref.contents[0], cref['href']])
elif is_number(cref.contents[0][:4]):
continue
else:
aliases.append(cref.contents[0])
if ttype != 'sn' and ttype != 'orphan':
continue
name = ''
for alias in aliases:
if alias in bad_aliases:
continue
if alias[:2] == 'SN':
name = alias
if not name:
name = psname
name = catalog.add_entry(name)
sources = [
catalog.entries[name].add_source(
name='Pan-STARRS 3Pi',
url=('http://psweb.mp.qub.ac.uk/'
'ps1threepi/psdb/'))
]
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name,
sources[0])
for ref in refs:
sources.append(catalog.entries[name].add_source(
name=ref[0], url=ref[1]))
source = uniq_cdl(sources)
for alias in aliases:
newalias = alias
if alias[:3] in ['CSS', 'SSS', 'MLS']:
newalias = alias.replace('-', ':', 1)
newalias = newalias.replace('PSNJ', 'PSN J')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, newalias,
source)
catalog.entries[name].add_quantity(SUPERNOVA.RA, ra, source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, dec, source)
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, ctype,
source)
fname2 = os.path.join(catalog.get_current_task_repo(),
'3pi/candidate-')
fname2 += pslink.rstrip('/').split('/')[-1] + '.html'
if offline:
if not os.path.isfile(fname2):
continue
with open(fname2, 'r') as f:
html2 = f.read()
else:
if (catalog.current_task.load_archive(catalog.args) and
os.path.isfile(fname2)):
with open(fname2, 'r') as f:
html2 = f.read()
else:
pslink = ('http://psweb.mp.qub.ac.uk/'
'ps1threepi/psdb/public/') + pslink
try:
session2 = requests.Session()
response2 = session2.get(pslink)
except Exception:
offline = True
if not os.path.isfile(fname2):
continue
with open(fname2, 'r') as f:
html2 = f.read()
else:
html2 = response2.text
with open(fname2, 'w') as f:
f.write(html2)
bs2 = BeautifulSoup(html2, 'html5lib')
scripts = bs2.findAll('script')
nslines = []
nslabels = []
for script in scripts:
if 'jslcdata.push' not in script.text:
continue
slines = script.text.splitlines()
for line in slines:
if 'jslcdata.push' in line:
json_fname = (line.strip()
.replace('jslcdata.push(', '')
.replace(');', ''))
nslines.append(json.loads(json_fname))
if ('jslabels.push' in line and 'blanks' not in line and
'non det' not in line):
json_fname = (line.strip()
.replace('jslabels.push(', '')
.replace(');', ''))
nslabels.append(json.loads(json_fname)['label'])
for li, line in enumerate(nslines[:len(nslabels)]):
if not line:
continue
for obs in line:
catalog.entries[name].add_photometry(
time=str(obs[0]),
u_time='MJD',
band=nslabels[li],
instrument='GPC',
magnitude=str(obs[1]),
e_magnitude=str(obs[2]),
source=source,
telescope=teles)
# Ignoring upper limits as they are usually spurious chip gaps.
# for li, line in enumerate(nslines[2 * len(nslabels):]):
# if not line:
# continue
# for obs in line:
# catalog.entries[name].add_photometry(
# time=str(obs[0]),
# u_time='MJD',
# band=nslabels[li],
# instrument='GPC',
# magnitude=str(obs[1]),
# upperlimit=True,
# source=source,
# telescope=teles)
assoctab = bs2.find('table', {'class': 'generictable'})
hostname = ''
redshift = ''
if assoctab:
trs = assoctab.findAll('tr')
headertds = [x.contents[0] for x in trs[1].findAll('td')]
tds = trs[1].findAll('td')
for tdi, td in enumerate(tds):
if tdi == 1:
hostname = td.contents[0].strip()
elif tdi == 4:
if 'z' in headertds:
redshift = td.contents[0].strip()
# Skip galaxies with just SDSS id
if is_number(hostname):
continue
catalog.entries[name].add_quantity(SUPERNOVA.HOST, hostname,
source)
if redshift:
catalog.entries[name].add_quantity(
[SUPERNOVA.REDSHIFT, SUPERNOVA.HOST_REDSHIFT],
redshift,
source,
kind='host')
if catalog.args.update:
catalog.journal_entries()
catalog.journal_entries()
# Only run first page for Travis
if catalog.args.travis:
break
return
def do_nedd(catalog):
task_str = catalog.get_current_task_str()
nedd_path = os.path.join(catalog.get_current_task_repo(),
'NED26.10.1-D-13.1.0-20160930.csv')
f = open(nedd_path, 'r')
data = sorted(list(csv.reader(f, delimiter=',', quotechar='"'))[13:],
key=lambda x: (x[9], x[3]))
reference = "NED-D v" + nedd_path.split('-')[-2]
refurl = "http://ned.ipac.caltech.edu/Library/Distances/"
nedbib = "1991ASSL..171...89H"
olddistname = ''
loopcnt = 0
for r, row in enumerate(pbar(data, task_str)):
if r <= 12:
continue
distname = row[3]
name = name_clean(distname)
# distmod = row[4]
# moderr = row[5]
dist = row[6]
bibcode = unescape(row[8])
snname = name_clean(row[9])
redshift = row[10]
cleanhost = ''
if name != snname and (name + ' HOST' != snname):
cleanhost = host_clean(distname)
if cleanhost.endswith(' HOST'):
cleanhost = ''
if not is_number(dist):
print(dist)
if dist:
catalog.nedd_dict.setdefault(cleanhost,
[]).append(Decimal(dist))
if snname and 'HOST' not in snname:
snname, secondarysource = catalog.new_entry(snname,
srcname=reference,
bibcode=nedbib,
url=refurl,
secondary=True)
if bibcode:
source = catalog.entries[snname].add_source(bibcode=bibcode)
sources = uniq_cdl([source, secondarysource])
else:
sources = secondarysource
if name == snname:
if redshift:
catalog.entries[snname].add_quantity(
SUPERNOVA.REDSHIFT, redshift, sources)
if dist:
catalog.entries[snname].add_quantity(
SUPERNOVA.COMOVING_DIST, dist, sources)
if not redshift:
try:
zatval = z_at_value(cosmo.comoving_distance,
float(dist) * un.Mpc,
zmax=5.0)
sigd = get_sig_digits(str(dist))
redshift = pretty_num(zatval, sig=sigd)
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
pass
else:
cosmosource = catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
combsources = uniq_cdl(
sources.split(',') + [cosmosource])
catalog.entries[snname].add_quantity(
SUPERNOVA.REDSHIFT,
redshift,
combsources,
derived=True)
if cleanhost:
catalog.entries[snname].add_quantity(SUPERNOVA.HOST, cleanhost,
sources)
if catalog.args.update and olddistname != distname:
catalog.journal_entries()
olddistname = distname
loopcnt = loopcnt + 1
if catalog.args.travis and loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0:
break
catalog.journal_entries()
f.close()
return
def do_csp_fits_spectra(catalog):
from astropy.io import fits
fpath = catalog.get_current_task_repo()
fureps = {'erg/cm2/s/A': 'erg/s/cm^2/Angstrom'}
task_str = catalog.get_current_task_str()
dirs = [x[0] for x in os.walk(
os.path.join(fpath, 'Gutierrez_et_al_2017'))]
files = []
for dir in dirs:
files.extend(glob(os.path.join(dir, '*.fits')))
for datafile in pbar(files, task_str):
filename = datafile.split('/')[-1]
hdulist = fits.open(datafile)
for oi, obj in enumerate(hdulist[0].header):
if any(x in ['.', '/'] for x in obj):
del (hdulist[0].header[oi])
try:
hdulist[0].verify('silentfix')
except Exception as e:
print(e)
hdrkeys = list(hdulist[0].header.keys())
# print(hdrkeys)
name = datafile.split('/')[-2]
if name[2] in '6789':
name = 'SN19' + name[2:]
elif name != 'SN210':
name = 'SN20' + name[2:]
name, source = catalog.new_entry(name, bibcode='2017ApJ...850...89G')
# for key in hdulist[0].header.keys():
# print(key, hdulist[0].header[key])
mjd = None
if hdulist[0].header['SIMPLE']:
if 'JD' in hdrkeys:
mjd = str(jd_to_mjd(Decimal(str(hdulist[0].header['JD']))))
elif 'MJD' in hdrkeys:
mjd = str(hdulist[0].header['MJD'])
elif 'DATE-OBS' in hdrkeys or 'DATE' in hdrkeys:
dkey = 'DATE-OBS' if 'DATE-OBS' in hdrkeys else 'DATE'
dval = hdulist[0].header[dkey]
if is_number(dval):
dkey = 'DATE' if dkey == 'DATE-OBS' else 'DATE-OBS'
dval = hdulist[0].header[dkey]
dateobs = None
if 'T' in dval:
dateobs = dval.strip()
elif 'UTC-OBS' in hdrkeys:
dateobs = dval.strip(
) + 'T' + hdulist[0].header['UTC-OBS'].strip()
if dateobs is not None:
mjd = str(astrotime(dateobs, format='isot').mjd)
# print(hdulist[0].header)
if 'CRVAL1' in hdulist[0].header:
w0 = hdulist[0].header['CRVAL1']
elif hdulist[0].header['CTYPE1'] == 'MULTISPE':
w0 = float(hdulist[0].header['WAT2_001'].split(
'"')[-1].split()[3])
else:
raise ValueError('Unsupported spectrum format.')
if hdulist[0].header['NAXIS'] == 1:
wd = hdulist[0].header['CDELT1']
fluxes = [str(x) for x in list(hdulist[0].data)]
errors = False
elif hdulist[0].header['NAXIS'] == 3:
wd = hdulist[0].header['CD1_1']
fluxes = [str(x) for x in list(hdulist[0].data)[0][0]]
errors = [str(x) for x in list(hdulist[0].data)[-1][0]]
else:
print('Warning: Skipping FITS spectrum `{}`.'.format(filename))
continue
waves = [str(w0 + wd * x) for x in range(0, len(fluxes))]
else:
raise ValueError('Non-simple FITS import not yet supported.')
if 'BUNIT' in hdrkeys:
fluxunit = hdulist[0].header['BUNIT']
if fluxunit in fureps:
fluxunit = fureps[fluxunit]
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.WAVELENGTHS: waves,
SPECTRUM.FLUXES: fluxes,
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.FILENAME: filename,
SPECTRUM.SOURCE: source
}
if mjd is not None:
specdict[SPECTRUM.TIME] = mjd
specdict[SPECTRUM.U_TIME] = 'MJD'
if 'TELESCOP' in hdrkeys:
specdict[SPECTRUM.TELESCOPE] = hdulist[0].header['TELESCOP']
if 'INSTRUME' in hdrkeys:
specdict[SPECTRUM.INSTRUMENT] = hdulist[0].header['INSTRUME']
if 'AIRMASS' in hdrkeys:
specdict[SPECTRUM.AIRMASS] = hdulist[0].header['AIRMASS']
if errors:
specdict[SPECTRUM.ERRORS] = errors
specdict[SPECTRUM.U_ERRORS] = fluxunit
if 'SITENAME' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['SITENAME']
elif 'OBSERVAT' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['OBSERVAT']
if 'OBSERVER' in hdrkeys:
specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER']
catalog.entries[name].add_spectrum(**specdict)
hdulist.close()
catalog.journal_entries()
return
def do_asiago_photo(catalog):
task_str = catalog.get_current_task_str()
# response = (urllib.request
# .urlopen('http://graspa.oapd.inaf.it/cgi-bin/sncat.php'))
path = os.path.abspath(os.path.join(
catalog.get_current_task_repo(), 'asiago-cat.php'))
response = urllib.request.urlopen('file://' + path)
html = response.read().decode('utf-8')
html = html.replace('\r', "")
soup = BeautifulSoup(html, 'html5lib')
table = soup.find('table')
records = []
for r, row in enumerate(table.findAll('tr')):
if r == 0:
continue
col = row.findAll('td')
records.append([utf8(x.renderContents()) for x in col])
for record in pbar(records, task_str):
if len(record) > 1 and record[1] != '':
oldname = clean_snname("SN" + record[1]).strip('?')
reference = 'Asiago Supernova Catalogue'
refurl = 'http://graspa.oapd.inaf.it/cgi-bin/sncat.php'
refbib = '1989A&AS...81..421B'
name, source = catalog.new_entry(
oldname, srcname=reference, url=refurl, bibcode=refbib,
secondary=True)
year = re.findall(r'\d+', oldname)[0]
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, year,
source)
hostname = record[2]
hostra = record[3]
hostdec = record[4]
ra = record[5].strip(':')
dec = record[6].strip(':')
redvel = record[11].strip(':')
discoverer = record[19]
datestring = year
monthday = record[18]
if "*" in monthday:
datekey = SUPERNOVA.DISCOVER_DATE
else:
datekey = SUPERNOVA.MAX_DATE
if monthday.strip() != '':
monthstr = ''.join(re.findall('[a-zA-Z]+', monthday))
monthstr = str(list(calendar.month_abbr).index(monthstr))
datestring = datestring + '/' + monthstr
dayarr = re.findall(r'\d+', monthday)
if dayarr:
daystr = dayarr[0]
datestring = datestring + '/' + daystr
catalog.entries[name].add_quantity(datekey, datestring,
source)
velocity = ''
redshift = ''
if redvel != '':
if round(float(redvel)) == float(redvel):
velocity = int(redvel)
else:
redshift = float(redvel)
redshift = str(redshift)
velocity = str(velocity)
claimedtype = record[17].replace(':', '').replace('*', '').strip()
if (hostname != ''):
catalog.entries[name].add_quantity(SUPERNOVA.HOST, hostname,
source)
if (claimedtype != ''):
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE,
claimedtype,
source)
if (redshift != ''):
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT, redshift, source, kind='host')
if (velocity != ''):
catalog.entries[name].add_quantity(
SUPERNOVA.VELOCITY, velocity, source, kind='host')
if (hostra != ''):
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_RA, hostra, source, u_value='nospace')
if (hostdec != ''):
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_DEC, hostdec, source, u_value='nospace')
if (ra != ''):
catalog.entries[name].add_quantity(SUPERNOVA.RA, ra, source,
u_value='nospace')
if (dec != ''):
catalog.entries[name].add_quantity(SUPERNOVA.DEC, dec, source,
u_value='nospace')
if (discoverer != ''):
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVERER,
discoverer,
source)
catalog.journal_entries()
return
def do_rochester(catalog):
"""Import data from latest supernova page."""
rochestermirrors = [
'http://www.rochesterastronomy.org/',
'http://www.supernova.thistlethwaites.com/'
]
rochesterpaths = [
'snimages/snredshiftall.html', 'sn2020/snredshift.html',
'snimages/snredboneyard.html', 'snimages/snredboneyard-old.html'
]
rochesterupdate = [False, True, True, False]
task_str = catalog.get_current_task_str()
baddates = ['2440587', '2440587.292', '0001/01/01']
for pp, path in enumerate(pbar(rochesterpaths, task_str)):
if catalog.args.update and not rochesterupdate[pp]:
continue
if 'snredboneyard.html' in path:
cns = {
'name': 0,
'host': 1,
'ra': 2,
'dec': 3,
'type': 7,
'z': 8,
'mmag': 9,
'max': 10,
'disc': 11,
'ref': 12,
'dver': 13,
'aka': 14
}
else:
cns = {
'name': 0,
'type': 1,
'host': 2,
'ra': 3,
'dec': 4,
'disc': 6,
'max': 7,
'mmag': 8,
'z': 11,
'zh': 12,
'ref': 13,
'dver': 14,
'aka': 15
}
filepath = (
os.path.join(catalog.get_current_task_repo(), 'rochester/') +
path.replace('/', '-'))
for mirror in rochestermirrors:
html = catalog.load_url(mirror + path,
filepath,
fail=(mirror != rochestermirrors[-1]))
if html:
break
if not html:
continue
soup = BeautifulSoup(html, 'html5lib')
rows = soup.findAll('tr')
sec_ref = 'Latest Supernovae'
sec_refurl = ('http://www.rochesterastronomy.org/'
'snimages/snredshiftall.html')
loopcnt = 0
for rr, row in enumerate(pbar(rows, task_str)):
if rr == 0:
continue
cols = row.findAll('td')
if not len(cols):
continue
name = ''
if cols[cns['aka']].contents:
for rawaka in str(cols[cns['aka']].contents[0]).split(','):
aka = rawaka.strip()
if is_number(aka.strip('?')):
aka = 'SN' + aka.strip('?') + 'A'
oldname = aka
name = catalog.add_entry(aka)
elif len(aka) == 4 and is_number(aka[:4]):
aka = 'SN' + aka
oldname = aka
name = catalog.add_entry(aka)
sn = re.sub('<[^<]+?>', '',
str(cols[cns['name']].contents[0])).strip()
if is_number(sn.strip('?')):
sn = 'SN' + sn.strip('?') + 'A'
elif len(sn) == 4 and is_number(sn[:4]):
sn = 'SN' + sn
if not name:
if not sn or sn in ['Transient']:
continue
ra = str(cols[cns['ra']].contents[0]).strip().replace(':.', '.')
dec = str(cols[cns['dec']].contents[0]).strip().replace(':.', '.')
if not name:
if sn[:8] == 'MASTER J':
sn = sn.replace('MASTER J',
'MASTER OT J').replace('SNHunt', 'SNhunt')
if 'POSSIBLE' in sn.upper() and ra and dec:
sn = 'PSN J' + ra.replace(':', '').replace('.', '')
sn += dec.replace(':', '').replace('.', '')
oldname = sn
name = catalog.add_entry(sn)
sec_source = catalog.entries[name].add_source(name=sec_ref,
url=sec_refurl,
secondary=True)
sources = []
if 'ref' in cns:
reftag = reference = cols[cns['ref']].findAll('a')
if len(reftag) and len(reftag[0].contents):
reference = reftag[0].contents[0].strip()
refurl = reftag[0]['href'].strip()
sources.append(catalog.entries[name].add_source(
name=reference, url=refurl))
sources.append(sec_source)
sources = uniq_cdl(list(filter(None, sources)))
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oldname,
sources)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, sn, sources)
if cols[cns['aka']].contents:
for rawaka in str(cols[cns['aka']].contents[0]).split(','):
aka = rawaka.strip()
if aka == 'SNR G1.9+0.3':
aka = 'G001.9+00.3'
if aka[:4] == 'PS1 ':
aka = 'PS1-' + aka[4:]
if aka[:8] == 'MASTER J':
aka = aka.replace('MASTER J', 'MASTER OT J').replace(
'SNHunt', 'SNhunt')
if 'POSSIBLE' in aka.upper() and ra and dec:
aka = 'PSN J' + ra.replace(':', '').replace('.', '')
aka += dec.replace(':', '').replace('.', '')
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, aka,
sources)
if (len(cols[cns['type']].contents) > 0
and str(cols[cns['type']].contents[0]).strip() != 'unk'):
type = str(cols[cns['type']].contents[0]).strip(' :,')
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE,
type, sources)
if (len(cols[cns['host']].contents) > 0 and
str(cols[cns['host']].contents[0]).strip() != 'anonymous'):
catalog.entries[name].add_quantity(
SUPERNOVA.HOST,
str(cols[cns['host']].contents[0]).strip(), sources)
catalog.entries[name].add_quantity(SUPERNOVA.RA, ra, sources)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, dec, sources)
discstr = str(cols[cns['disc']].contents[0]).strip()
if discstr and discstr not in baddates:
if '/' not in discstr:
astrot = astrotime(float(discstr), format='jd').datetime
ddate = make_date_string(astrot.year, astrot.month,
astrot.day)
else:
ddate = discstr
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE,
ddate, sources)
maxstr = str(cols[cns.get('max', '')].contents[0]).strip()
if maxstr and maxstr not in baddates:
try:
if '/' not in maxstr:
astrot = astrotime(float(maxstr), format='jd')
else:
astrot = astrotime(maxstr.replace('/', '-'),
format='iso')
except:
catalog.log.info(
'Max date conversion failed for `{}`.'.format(maxstr))
if ((float(str(cols[cns['mmag']].contents[0]).strip()) <= 90.0
and
not any('GRB' in xx
for xx in catalog.entries[name].get_aliases()))):
mag = str(cols[cns['mmag']].contents[0]).strip()
catalog.entries[name].add_photometry(time=str(astrot.mjd),
u_time='MJD',
magnitude=mag,
source=sources)
if 'z' in cns and cols[cns['z']].contents[0] != 'n/a':
catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT,
str(cols[cns['z']].contents[0]).strip(), sources)
if 'zh' in cns:
zhost = str(cols[cns['zh']].contents[0]).strip()
if is_number(zhost):
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT,
zhost, sources)
if 'dver' in cns:
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVERER,
str(cols[cns['dver']].contents[0]).strip(), sources)
if catalog.args.update:
catalog.journal_entries()
loopcnt = loopcnt + 1
if (catalog.args.travis
and loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
if not catalog.args.update:
vsnetfiles = ['latestsne.dat']
for vsnetfile in vsnetfiles:
file_name = os.path.join(catalog.get_current_task_repo(),
"" + vsnetfile)
with open(file_name, 'r', encoding='latin1') as csv_file:
tsvin = csv.reader(csv_file,
delimiter=' ',
skipinitialspace=True)
loopcnt = 0
for rr, row in enumerate(tsvin):
if (not row or row[0] in ['Transient']
or row[0][:4] in ['http', 'www.'] or len(row) < 3):
continue
name = row[0].strip()
if name[:4].isdigit():
name = 'SN' + name
if name.startswith('PSNJ'):
name = 'PSN J' + name[4:]
if name.startswith('MASTEROTJ'):
name = name.replace('MASTEROTJ', 'MASTER OT J')
name = catalog.add_entry(name)
sec_source = catalog.entries[name].add_source(
name=sec_ref, url=sec_refurl, secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name,
sec_source)
if not is_number(row[1]):
continue
year = row[1][:4]
month = row[1][4:6]
day = row[1][6:]
if '.' not in day:
day = day[:2] + '.' + day[2:]
mjd = astrotime(year + '-' + month + '-' +
str(floor(float(day))).zfill(2)).mjd
mjd += float(day) - floor(float(day))
magnitude = row[2].rstrip(ascii_letters)
if not is_number(magnitude):
continue
if magnitude.isdigit():
if int(magnitude) > 100:
magnitude = magnitude[:2] + '.' + magnitude[2:]
if float(str(cols[8].contents[0]).strip()) >= 90.0:
continue
if len(row) >= 4:
if is_number(row[3]):
e_magnitude = row[3]
refind = 4
else:
e_magnitude = ''
refind = 3
if refind >= len(row):
sources = sec_source
else:
reference = ' '.join(row[refind:])
source = catalog.entries[name].add_source(
name=reference)
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, name, sec_source)
sources = uniq_cdl([source, sec_source])
else:
sources = sec_source
band = row[2].lstrip('1234567890.')
catalog.entries[name].add_photometry(
time=mjd,
u_time='MJD',
band=band,
magnitude=magnitude,
e_magnitude=e_magnitude,
source=sources)
if (catalog.args.travis
and loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
def do_itep(catalog):
task_str = catalog.get_current_task_str()
itepignoresources = ['2004ApJ...602..571B', '2013NewA...20...30M']
itepignorephot = ['SN2006gy']
needsbib = []
with open(os.path.join(catalog.get_current_task_repo(), 'itep-refs.txt'),
'r') as refs_file:
refrep = refs_file.read().splitlines()
refrepf = dict(list(zip(refrep[1::2], refrep[::2])))
fname = os.path.join(catalog.get_current_task_repo(),
'itep-lc-cat-28dec2015.txt')
tsvin = list(
csv.reader(open(fname, 'r'), delimiter='|', skipinitialspace=True))
curname = ''
for rr, row in enumerate(pbar(tsvin, task_str)):
if rr <= 1 or len(row) < 7:
continue
oldname = 'SN' + row[0].strip()
mjd = str(jd_to_mjd(Decimal(row[1].strip())))
band = row[2].strip()
magnitude = row[3].strip()
e_magnitude = row[4].strip()
reference = row[6].strip().strip(',')
if curname != oldname:
curname = oldname
name = catalog.add_entry(oldname)
sec_reference = ('Sternberg Astronomical Institute '
'Supernova Light Curve Catalogue')
sec_refurl = 'http://dau.itep.ru/sn/node/72'
sec_source = catalog.entries[name].add_source(name=sec_reference,
url=sec_refurl,
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oldname,
sec_source)
year = re.findall(r'\d+', name)[0]
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, year,
sec_source)
if reference in refrepf:
bibcode = unescape(refrepf[reference])
source = catalog.entries[name].add_source(bibcode=bibcode)
else:
needsbib.append(reference)
source = catalog.entries[name].add_source(
name=reference) if reference else ''
if oldname in itepignorephot or bibcode in itepignoresources:
continue
photodict = {
PHOTOMETRY.TIME: mjd,
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.MAGNITUDE: magnitude,
PHOTOMETRY.SOURCE: uniq_cdl([sec_source, source])
}
if e_magnitude:
photodict[PHOTOMETRY.E_MAGNITUDE] = e_magnitude
if band.endswith('_SDSS'):
photodict[PHOTOMETRY.BAND_SET] = 'SDSS'
photodict[PHOTOMETRY.SYSTEM] = 'SDSS'
band = band.replace('_SDSS', "'")
photodict[PHOTOMETRY.BAND] = band
catalog.entries[name].add_photometry(**photodict)
if catalog.args.travis and rr >= catalog.TRAVIS_QUERY_LIMIT:
break
# Write out references that could use aa bibcode
needsbib = list(OrderedDict.fromkeys(needsbib))
with open('../itep-needsbib.txt', 'w') as bib_file:
bib_file.writelines(['%ss\n' % ii for ii in needsbib])
catalog.journal_entries()
return
def set_preferred_names(self):
"""Choose between each entries given name and its possible aliases for
the best one.
Highest preference goes to names of the form 'SN####AA'.
Otherwise base the name on whichever survey is the 'discoverer'.
FIX: create function to match SN####AA type names.
"""
if len(self.entries) == 0:
self.log.error("WARNING: `entries` is empty, loading stubs")
self.load_stubs()
task_str = self.get_current_task_str()
for ni, name in enumerate(pbar(list(
sorted(self.entries.keys())), task_str)):
newname = ''
aliases = self.entries[name].get_aliases()
# if there are no other options to choose from, skip
if len(aliases) <= 1:
continue
# If the name is already in the form 'SN####AA' then keep using
# that
if (name.startswith('SN') and
((is_number(name[2:6]) and not is_number(name[6:])) or
(is_number(name[2:5]) and not is_number(name[5:])))):
continue
# If one of the aliases is in the form 'SN####AA' then use that
for alias in aliases:
if (alias[:2] == 'SN' and
((is_number(alias[2:6]) and not is_number(alias[6:])) or
(is_number(alias[2:5]) and not is_number(alias[5:])))):
newname = alias
break
# Otherwise, name based on the 'discoverer' survey
if not newname and 'discoverer' in self.entries[name]:
discoverer = ','.join(
[x['value'].upper() for x in
self.entries[name]['discoverer']])
if 'ASAS' in discoverer:
for alias in aliases:
if 'ASASSN' in alias.upper():
newname = alias
break
if not newname and 'OGLE' in discoverer:
for alias in aliases:
if 'OGLE' in alias.upper():
newname = alias
break
if not newname and 'CRTS' in discoverer:
for alias in aliases:
if True in [x in alias.upper()
for x in ['CSS', 'MLS', 'SSS', 'SNHUNT']]:
newname = alias
break
if not newname and 'PS1' in discoverer:
for alias in aliases:
if 'PS1' in alias.upper():
newname = alias
break
if not newname and 'PTF' in discoverer:
for alias in aliases:
if 'PTF' in alias.upper():
newname = alias
break
if not newname and 'GAIA' in discoverer:
for alias in aliases:
if 'GAIA' in alias.upper():
newname = alias
break
# Always prefer another alias over PSN
if not newname and name.startswith('PSN'):
newname = aliases[0]
if newname and name != newname:
# Make sure new name doesn't already exist
if self.proto.init_from_file(self, name=newname):
self.log.error("WARNING: `newname` already exists... "
"should do something about that...")
continue
new_entry = self.proto.init_from_file(self, name=name)
if new_entry is None:
self.log.error(
"Could not load `new_entry` with name '{}'"
.format(name))
else:
self.log.info("Changing entry from name '{}' to preferred"
" name '{}'".format(name, newname))
self._delete_entry_file(entry=new_entry)
del self.entries[name]
self.entries[newname] = new_entry
self.entries[newname][self.proto._KEYS.NAME] = newname
if name in self.entries:
self.log.error(
"WARNING: `name` = '{}' is in `entries` "
"shouldnt happen?".format(name))
del self.entries[name]
self.journal_entries()
if self.args.travis and ni > self.TRAVIS_QUERY_LIMIT:
break
return
def do_wiserep_spectra(catalog):
#if not catalog.args.travis:
# from ..input.WISeWEBSpider.wisewebspider import spider
# try:
# spider(update=True, daysago=7, path="/../../sne-external-WISEREP/")
# except:
# catalog.log.warning(
# 'Spider errored, continuing without letting it complete.')
task_str = catalog.get_current_task_str()
secondaryreference = 'WISeREP'
secondaryrefurl = 'http://wiserep.weizmann.ac.il/'
secondarybibcode = '2012PASP..124..668Y'
wiserepcnt = 0
# These are known to be in error on the WISeREP page, either fix or ignore
# them.
wiserepbibcorrectdict = {
'2000AJ....120..367G]': '2000AJ....120..367G',
'Harutyunyan et al. 2008': '2008A&A...488..383H',
'0609268': '2007AJ....133...58K',
'2006ApJ...636...400Q': '2006ApJ...636..400Q',
'2011ApJ...741...76': '2011ApJ...741...76C',
'2016PASP...128...961': '2016PASP..128...961',
'2002AJ....1124..417H': '2002AJ....1124.417H',
'2013ApJ…774…58D': '2013ApJ...774...58D',
'2011Sci.333..856S': '2011Sci...333..856S',
'2014MNRAS.438,368': '2014MNRAS.438..368T',
'2012MNRAS.420.1135': '2012MNRAS.420.1135S',
'2012Sci..337..942D': '2012Sci...337..942D',
'stt1839': '2013MNRAS.436.3614S',
'arXiv:1605.03136': '2016MNRAS.460.3447T',
'10.1093/mnras/stt1839': '2013MNRAS.436.3614S'
}
file_names = list(glob(os.path.join(catalog.get_current_task_repo(), '*')))
for folder in pbar_strings(file_names, task_str):
if '.txt' in folder or '.json' in folder:
continue
name = os.path.basename(folder).strip()
if name.startswith('sn'):
name = 'SN' + name[2:]
if (name.startswith(('CSS', 'SSS', 'MLS')) and ':' not in name):
name = name.replace('-', ':', 1)
if name.startswith('MASTERJ'):
name = name.replace('MASTERJ', 'MASTER OT J')
if name.startswith('PSNJ'):
name = name.replace('PSNJ', 'PSN J')
name = catalog.add_entry(name)
secondarysource = catalog.entries[name].add_source(
name=secondaryreference,
url=secondaryrefurl,
bibcode=secondarybibcode,
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name,
secondarysource)
readme_path = os.path.join(folder, 'README.json')
if not os.path.exists(readme_path):
catalog.log.warning(
'Metadata file not found for event "{}"'.format(name))
continue
with open(readme_path, 'r') as f:
fileinfo = json.loads(f.read())
files = list(
set(glob(folder + '/*')) - set(glob(folder + '/README.json')))
for fname in pbar(files, task_str):
specfile = os.path.basename(fname)
if specfile not in fileinfo:
catalog.log.warning(
'Metadata not found for "{}"'.format(fname))
continue
claimedtype = fileinfo[specfile]["Type"]
instrument = fileinfo[specfile]["Instrument"]
epoch = fileinfo[specfile]["Obs. Date"]
observer = fileinfo[specfile]["Observer"]
reducer = fileinfo[specfile]["Reducer"]
bibcode = fileinfo[specfile]["Bibcode"]
redshift = fileinfo[specfile]["Redshift"]
survey = fileinfo[specfile]["Program"]
reduction = fileinfo[specfile]["Reduction Status"]
if bibcode:
newbibcode = bibcode
if bibcode in wiserepbibcorrectdict:
newbibcode = wiserepbibcorrectdict[bibcode]
if newbibcode and len(newbibcode) == 19:
source = catalog.entries[name].add_source(
bibcode=unescape(newbibcode))
else:
bibname = unescape(bibcode)
source = catalog.entries[name].add_source(name=bibname)
catalog.log.warning('Bibcode "{}" is invalid, using as '
'`{}` instead'.format(
bibname, SOURCE.NAME))
sources = uniq_cdl([source, secondarysource])
else:
sources = secondarysource
if claimedtype not in ['Other']:
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE,
claimedtype,
secondarysource)
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, redshift,
secondarysource)
with open(fname, 'r') as f:
data = [x.split() for x in f]
skipspec = False
newdata = []
oldval = ''
for row in data:
if row and '#' not in row[0]:
if (len(row) >= 2 and is_number(row[0])
and is_number(row[1]) and row[1] != oldval):
newdata.append(row)
oldval = row[1]
if skipspec or not newdata:
warnings.warn('Skipped adding spectrum file ' + specfile)
continue
data = [list(i) for i in zip(*newdata)]
wavelengths = data[0]
fluxes = data[1]
errors = ''
if len(data) == 3:
errors = data[1]
time = str(astrotime(epoch).mjd)
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom',
errors=errors,
u_fluxes=fluxunit,
u_errors=fluxunit if errors else '',
wavelengths=wavelengths,
fluxes=fluxes,
u_time='MJD',
time=time,
instrument=instrument,
source=sources,
observer=observer,
reducer=reducer,
reduction=reduction,
filename=specfile,
survey=survey,
redshift=redshift)
catalog.journal_entries()
wiserepcnt = wiserepcnt + 1
if (catalog.args.travis
and wiserepcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
return
def do_ps_alerts(catalog):
task_str = catalog.get_current_task_str()
alertstables = ['alertstable-2010', 'alertstable-2011', 'alertstable']
rows = []
for at in alertstables:
with open(
os.path.join(catalog.get_current_task_repo(), 'ps1-clean',
at)) as f:
rows.extend(
list(csv.reader(
f, delimiter=' ', skipinitialspace=True))[1:])
alertfiles = glob(
os.path.join(catalog.get_current_task_repo(), 'ps1-clean/*.dat'))
alertfilestag = dict(
[(x.split('/')[-1].split('.')[0], x) for x in alertfiles])
alertfilesid = dict([(x.split('/')[-1].split('.')[0].split('-')[-1], x)
for x in alertfiles])
with open(
os.path.join(catalog.get_current_task_repo(),
'ps1-clean/whitelist')) as f:
whitelist = list(csv.reader(f, delimiter=' ', skipinitialspace=True))
wlnames = [x[0] for x in whitelist]
wlnamesleft = list(wlnames)
wlra = [x[1] for x in whitelist]
missing_confirmed = []
# already_collected = []
for ri, row in enumerate(pbar(rows, task_str)):
psname = row[50]
if psname == '-':
if row[4] in wlra:
psname = wlnames[wlra.index(row[4])]
else:
continue
sntype = row[21].replace('SN', '')
skip_photo = False
if psname not in wlnamesleft:
if row[1] == 'confirmed':
missing_confirmed.append((psname, row[21]))
# if 'II' in sntype:
# pass
# elif (sntype != 'Ia' or
# not (psname.startswith(('PS1-12', 'PS1-13')) or
# (psname.startswith('PS1-10') and
# len(psname.replace('PS1-10', '')) == 3 and
# psname[-3:] > 'ams'))):
# if sntype == 'Ia' and psname.startswith('PS1-10'):
# already_collected.append((psname, row[21]))
# else:
# missing_confirmed.append((psname, row[21]))
skip_photo = True
else:
continue
if psname in wlnamesleft:
wlnamesleft.remove(psname)
name, source = catalog.new_entry(
psname,
srcname='Pan-STARRS Alerts',
url='http://telescopes.rc.fas.harvard.edu/ps1/')
catalog.entries[name].add_quantity(SUPERNOVA.RA, row[4], source)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[5], source)
if sntype != '-':
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, sntype,
source)
if row[22] != '-':
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, row[22],
source)
# Disabling photometry import
continue
if skip_photo:
continue
psinternal = row[-1].split('.')[0]
if not is_number(psinternal.split('-')[0]):
psid = row[0].zfill(6)
if psid not in alertfilesid:
continue
pspath = alertfilesid[psid]
else:
if psinternal not in alertfilestag:
continue
pspath = alertfilestag[psinternal]
with open(pspath) as f:
photrows = list(
csv.reader(
f, delimiter=' ', skipinitialspace=True))
for pi, prow in enumerate(photrows):
if pi == 0 or prow[3] == '-':
continue
counts = prow[13]
e_counts = prow[14]
photodict = {
PHOTOMETRY.TIME: prow[1],
PHOTOMETRY.U_TIME: 'MJD',
PHOTOMETRY.BAND: prow[2],
PHOTOMETRY.MAGNITUDE: prow[3],
PHOTOMETRY.E_MAGNITUDE: prow[4],
PHOTOMETRY.COUNT_RATE: counts,
PHOTOMETRY.E_COUNT_RATE: e_counts,
PHOTOMETRY.ZERO_POINT: prow[15],
PHOTOMETRY.INSTRUMENT: 'GPC1',
PHOTOMETRY.OBSERVATORY: 'PS1',
PHOTOMETRY.SURVEY: 'MDS',
PHOTOMETRY.TELESCOPE: 'PS1',
PHOTOMETRY.SYSTEM: 'PS1',
PHOTOMETRY.SOURCE: source
}
ul_sigma = 3.0
if float(counts) < ul_sigma * float(e_counts):
photodict[PHOTOMETRY.UPPER_LIMIT] = True
photodict[PHOTOMETRY.UPPER_LIMIT_SIGMA] = ul_sigma
catalog.entries[name].add_photometry(**photodict)
catalog.journal_entries()
# print(already_collected)
# print(missing_confirmed)
return
def do_ascii(catalog):
"""Process ASCII files extracted from datatables of published works."""
task_str = catalog.get_current_task_str()
# Howerton Catalog
datafile = os.path.join(
catalog.get_current_task_repo(), 'ASCII',
'vizier_J_MNRAS_441_1186_table1 J_MNRAS_441_1186_table3_20200403.csv')
data = read(datafile, format='csv')
for rrow in pbar(data, task_str):
row = dict((x, str(rrow[x])) for x in rrow.columns)
# if any(x in row['Notes'].lower() for x in ['artifact']):
# continue
# ctypes = row['Type'].split('/')
# nonsne = False
# for ct in ctypes:
# if ct.replace('?', '') in catalog.nonsnetypes:
# nonsne = True
# else:
# nonsne = False
# break
# if nonsne:
# continue
name, source = catalog.new_entry(row['CRTS'],
srcname='CRTS',
bibcode='2014MNRAS.441.1186D')
# if row['IAU des.'] != '--':
# catalog.entries[name].add_quantity(SUPERNOVA.ALIAS,
# row['IAU des.'], source)
# for ct in ctypes:
# catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, ct,
# source)
# catalog.entries[name].add_quantity(SUPERNOVA.DISCOVERER,
# row['Discoverer'], source)
# date = row['Discovery'].split('/')
# date = '/'.join([date[-1].zfill(2), date[0].zfill(2), date[1]])
# catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, date,
# source)
catalog.entries[name].add_quantity(CATACLYSMIC.MAX_VISUAL_APP_MAG,
row['Vmin'], source)
catalog.entries[name].add_quantity(CATACLYSMIC.RA, row['RAJ2000'],
source)
catalog.entries[name].add_quantity(CATACLYSMIC.DEC, row['DEJ2000'],
source)
catalog.journal_entries()
# Howerton Catalog
# datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII',
# 'howerton-catalog.csv')
# data = read(datafile, format='csv')
# for rrow in pbar(data, task_str):
# row = dict((x, str(rrow[x])) for x in rrow.columns)
# if any(x in row['Notes'].lower() for x in ['artifact']):
# continue
# ctypes = row['Type'].split('/')
# nonsne = False
# for ct in ctypes:
# if ct.replace('?', '') in catalog.nonsnetypes:
# nonsne = True
# else:
# nonsne = False
# break
# if nonsne:
# continue
# name, source = catalog.new_entry(
# row['SNHunt des.'],
# srcname='CRTS SNhunt',
# bibcode='2017csnh.book.....H')
# if row['IAU des.'] != '--':
# catalog.entries[name].add_quantity(SUPERNOVA.ALIAS,
# row['IAU des.'], source)
# for ct in ctypes:
# catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, ct,
# source)
# catalog.entries[name].add_quantity(SUPERNOVA.DISCOVERER,
# row['Discoverer'], source)
# date = row['Discovery'].split('/')
# date = '/'.join([date[-1].zfill(2), date[0].zfill(2), date[1]])
# catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, date,
# source)
# catalog.entries[name].add_quantity(SUPERNOVA.HOST, row['Host galaxy'],
# source)
# catalog.entries[name].add_quantity(SUPERNOVA.RA, row['RA'].replace(
# 'h', ':').replace('m', ':').replace('s', ''), source)
# catalog.entries[name].add_quantity(SUPERNOVA.DEC, row['Dec'], source)
# catalog.journal_entries()
def do_csp_fits_spectra(catalog):
from astropy.io import fits
fpath = catalog.get_current_task_repo()
fureps = {'erg/cm2/s/A': 'erg/s/cm^2/Angstrom'}
task_str = catalog.get_current_task_str()
dirs = [x[0] for x in os.walk(os.path.join(fpath, 'Gutierrez_et_al_2017'))]
files = []
for dir in dirs:
files.extend(glob(os.path.join(dir, '*.fits')))
for datafile in pbar(files, task_str):
filename = datafile.split('/')[-1]
hdulist = fits.open(datafile)
for oi, obj in enumerate(hdulist[0].header):
if any(x in ['.', '/'] for x in obj):
del (hdulist[0].header[oi])
try:
hdulist[0].verify('silentfix')
except Exception as e:
print(e)
hdrkeys = list(hdulist[0].header.keys())
# print(hdrkeys)
name = datafile.split('/')[-2]
if name[2] in '6789':
name = 'SN19' + name[2:]
elif name != 'SN210':
name = 'SN20' + name[2:]
name, source = catalog.new_entry(name, bibcode='2017ApJ...850...89G')
# for key in hdulist[0].header.keys():
# print(key, hdulist[0].header[key])
mjd = None
if hdulist[0].header['SIMPLE']:
if 'JD' in hdrkeys:
mjd = str(jd_to_mjd(Decimal(str(hdulist[0].header['JD']))))
elif 'MJD' in hdrkeys:
mjd = str(hdulist[0].header['MJD'])
elif 'DATE-OBS' in hdrkeys or 'DATE' in hdrkeys:
dkey = 'DATE-OBS' if 'DATE-OBS' in hdrkeys else 'DATE'
dval = hdulist[0].header[dkey]
if is_number(dval):
dkey = 'DATE' if dkey == 'DATE-OBS' else 'DATE-OBS'
dval = hdulist[0].header[dkey]
dateobs = None
if 'T' in dval:
dateobs = dval.strip()
elif 'UTC-OBS' in hdrkeys:
dateobs = dval.strip(
) + 'T' + hdulist[0].header['UTC-OBS'].strip()
if dateobs is not None:
mjd = str(astrotime(dateobs, format='isot').mjd)
# print(hdulist[0].header)
if 'CRVAL1' in hdulist[0].header:
w0 = hdulist[0].header['CRVAL1']
elif hdulist[0].header['CTYPE1'] == 'MULTISPE':
w0 = float(
hdulist[0].header['WAT2_001'].split('"')[-1].split()[3])
else:
raise ValueError('Unsupported spectrum format.')
if hdulist[0].header['NAXIS'] == 1:
wd = hdulist[0].header['CDELT1']
fluxes = [str(x) for x in list(hdulist[0].data)]
errors = False
elif hdulist[0].header['NAXIS'] == 3:
wd = hdulist[0].header['CD1_1']
fluxes = [str(x) for x in list(hdulist[0].data)[0][0]]
errors = [str(x) for x in list(hdulist[0].data)[-1][0]]
else:
print('Warning: Skipping FITS spectrum `{}`.'.format(filename))
continue
waves = [str(w0 + wd * x) for x in range(0, len(fluxes))]
else:
raise ValueError('Non-simple FITS import not yet supported.')
if 'BUNIT' in hdrkeys:
fluxunit = hdulist[0].header['BUNIT']
if fluxunit in fureps:
fluxunit = fureps[fluxunit]
else:
if max([float(x) for x in fluxes]) < 1.0e-5:
fluxunit = 'erg/s/cm^2/Angstrom'
else:
fluxunit = 'Uncalibrated'
specdict = {
SPECTRUM.U_WAVELENGTHS: 'Angstrom',
SPECTRUM.WAVELENGTHS: waves,
SPECTRUM.FLUXES: fluxes,
SPECTRUM.U_FLUXES: fluxunit,
SPECTRUM.FILENAME: filename,
SPECTRUM.SOURCE: source
}
if mjd is not None:
specdict[SPECTRUM.TIME] = mjd
specdict[SPECTRUM.U_TIME] = 'MJD'
if 'TELESCOP' in hdrkeys:
specdict[SPECTRUM.TELESCOPE] = hdulist[0].header['TELESCOP']
if 'INSTRUME' in hdrkeys:
specdict[SPECTRUM.INSTRUMENT] = hdulist[0].header['INSTRUME']
if 'AIRMASS' in hdrkeys:
specdict[SPECTRUM.AIRMASS] = hdulist[0].header['AIRMASS']
if errors:
specdict[SPECTRUM.ERRORS] = errors
specdict[SPECTRUM.U_ERRORS] = fluxunit
if 'SITENAME' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['SITENAME']
elif 'OBSERVAT' in hdrkeys:
specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['OBSERVAT']
if 'OBSERVER' in hdrkeys:
specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER']
catalog.entries[name].add_spectrum(**specdict)
hdulist.close()
catalog.journal_entries()
return
def do_spectra(catalog):
"""
"""
#path to root ONC directory
ONC = catalog.get_current_task_repo()
novae_directory = os.path.join(ONC, 'Individual_Novae')
task_str = catalog.get_current_task_str()
for indiv_nova in pbar(os.listdir(novae_directory), task_str):
nova_dir = os.path.join(novae_directory, indiv_nova)
indiv_nova = get_nova_name(indiv_nova)
if not os.path.isdir(nova_dir) or indiv_nova is None: continue
ticket_directory = os.path.join(nova_dir, "Tickets", "CompletedTickets")
for ticket in os.listdir(ticket_directory):
ticket_raw_dict, ticket_column_dict = {}, {}
if not re.search(r'spectra', ticket.lower()): continue
with open(os.path.join(ticket_directory, ticket)) as spec_ticket:
try:
ticket_raw_dict, ticket_column_dict = read_spectra_ticket(spec_ticket.read())
spec_ticket.close()
except AttributeError:
True == True
print(ticket)
continue
#THROW ERROR TO LOG HERE
meta_filename = ticket_raw_dict['METADATA FILENAME']
file_path = os.path.join(nova_dir, "Data", meta_filename)
name = indiv_nova.replace('_', ' ')
name = catalog.add_entry(name)
if type(ticket_raw_dict['BIBCODE']) is str and len(ticket_raw_dict['BIBCODE']) == 19:
source = catalog.entries[name].add_source(bibcode=ticket_raw_dict['BIBCODE'], reference=ticket_raw_dict['REFERENCE'])
else:
source = catalog.entries[name].add_source(bibcode='couldnotfindbibcode', reference=ticket_raw_dict['REFERENCE'])
catalog.entries[name].add_quantity(NOVA.ALIAS, name, source)
time_offset = 0
if re.match(r'(days)|(ut)', ticket_raw_dict['TIME SYSTEM'].lower()):
t = Time(convert_date_UTC(ticket_raw_dict['ASSUMED DATE OF OUTBURST']))
offset = t.jd
mag = 'magnitude' if re.search(r'mag', ticket_raw_dict['FLUX UNITS'].lower()) else 'flux'
e_mag = 'e_magnitude' if re.search(r'mag', ticket_raw_dict['FLUX ERROR UNITS'].lower()) else 'e_flux'
with open(file_path, "r") as metadata_file:
data_file_list = read_spectra_metadata(metadata_file.read(), ticket_column_dict)
metadata_file.close()
for metadata_raw_dict, metadata_column_dict in data_file_list:
data_filename = metadata_raw_dict['FILENAME']
csvfile = open(os.path.join(nova_dir, "Data", data_filename))
contents = list(csv.reader(csvfile, delimiter=',', quotechar='"'))
csvfile.close()
flux_col = metadata_column_dict['FLUX']
wavelength_col = metadata_column_dict['WAVE']
flux_err_col = metadata_column_dict['FLUX ERR']
if flux_col is None or wavelength_col is None: continue
flux_err_arr = None if flux_err_col is None else []
flux_arr, wavelength_arr = [],[]
for line in contents:
line = [value.strip() for value in line]
if line[0].startswith("#"): continue
try:
flux = line[flux_col]
wavelength = line[wavelength_col]
except (IndexError, ValueError):
continue
try: flux_err = None if flux_err_arr is None else line[flux_err_col]
except (IndexError, ValueError): flux_err = None
if flux_err is not None: flux_err_arr.append(flux_err)
flux_arr.append(flux)
wavelength_arr.append(flux)
data_dict = {'wavelengths': wavelength_arr, 'fluxes': flux_arr, 'source': source, 'time': metadata_raw_dict['DATE'], 'observer': metadata_raw_dict['OBSERVER'], 'telescope': metadata_raw_dict['TELESCOPE'], 'instrument': metadata_raw_dict['INSTRUMENT'], 'u_wavelengths': 'Angstrom', 'u_time': 'JD', 'u_fluxes': 'erg/s/cm^2'}
if not flux_err_arr is None and len(flux_err_arr) != len(wavelength_arr):
data_dict['errors'] = flux_err_arr
key_list = list(data_dict.keys())
for key in key_list:
if data_dict[key] is None:
del data_dict[key]
catalog.entries[name].add_spectrum(**data_dict)
return
def do_cpcs(catalog):
"""Import data from CPCS."""
task_str = catalog.get_current_task_str()
cpcs_url = ('http://gsaweb.ast.cam.ac.uk/'
'followup/list_of_alerts?format=json&num=100000&'
'published=1&observed_only=1'
'&hashtag=JG_530ad9462a0b8785bfb385614bf178c6')
jsontxt = catalog.load_url(
cpcs_url,
os.path.join(catalog.get_current_task_repo(), 'CPCS/index.json'))
if not jsontxt:
return
alertindex = json.loads(jsontxt, object_pairs_hook=OrderedDict)
ids = [xx['id'] for xx in alertindex]
for ii, ai in enumerate(pbar(ids, task_str)):
name = alertindex[ii]['ivorn'].split('/')[-1].strip()
# Skip aa few weird entries
if name == 'ASASSNli':
continue
# Just use aa whitelist for now since naming seems inconsistent
white_list = [
'GAIA', 'OGLE', 'ASASSN', 'MASTER', 'OTJ', 'PS1', 'IPTF', 'CSS'
]
if True in [xx in name.upper() for xx in white_list]:
name = name.replace('Verif', '').replace('_', ' ')
if 'ASASSN' in name and name[6] != '-':
name = 'ASASSN-' + name[6:].lower()
if 'MASTEROTJ' in name:
name = name.replace('MASTEROTJ', 'MASTER OT J')
if 'OTJ' in name:
name = name.replace('OTJ', 'MASTER OT J')
if name.upper().startswith('IPTF'):
name = 'iPTF' + name[4:].lower()
if name.upper().startswith('PS1'):
name = 'PS1' + name[3:].lower()
# Only add events that are classified as SN.
if not catalog.entry_exists(name):
continue
oldname = name
name = catalog.add_entry(name)
else:
continue
sec_source = catalog.entries[name].add_source(
name='Cambridge Photometric Calibration Server',
url='http://gsaweb.ast.cam.ac.uk/followup/',
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oldname,
sec_source)
unit_deg = 'floatdegrees'
catalog.entries[name].add_quantity(SUPERNOVA.RA,
str(alertindex[ii][SUPERNOVA.RA]),
sec_source,
u_value=unit_deg)
catalog.entries[name].add_quantity(SUPERNOVA.DEC,
str(alertindex[ii][SUPERNOVA.DEC]),
sec_source,
u_value=unit_deg)
alerturl = ('http://gsaweb.ast.cam.ac.uk/'
'followup/get_alert_lc_data?alert_id=' + str(ai))
source = catalog.entries[name].add_source(name='CPCS Alert ' + str(ai),
url=alerturl)
fname = os.path.join(catalog.get_current_task_repo(),
'CPCS/alert-') + str(ai).zfill(2) + '.json'
jsonstr = catalog.load_url(
alerturl + '&hashtag=JG_530ad9462a0b8785bfb385614bf178c6', fname)
try:
cpcsalert = json.loads(jsonstr)
except Exception:
catalog.log.warning('Mangled CPCS data for alert {}.'.format(ai))
continue
mjds = [round_sig(xx, sig=9) for xx in cpcsalert['mjd']]
mags = [round_sig(xx, sig=6) for xx in cpcsalert['mag']]
errs = [
round_sig(xx, sig=6) if (is_number(xx) and float(xx) > 0.0) else ''
for xx in cpcsalert['magerr']
]
bnds = cpcsalert['filter']
obs = cpcsalert['observatory']
for mi, mjd in enumerate(mjds):
catalog.entries[name].add_photometry(time=mjd,
u_time='MJD',
magnitude=mags[mi],
e_magnitude=errs[mi],
band=bnds[mi],
observatory=obs[mi],
source=uniq_cdl(
[source, sec_source]))
if catalog.args.update:
catalog.journal_entries()
if catalog.args.travis and ii >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
return
def do_simbad(catalog):
task_str = catalog.get_current_task_str()
keys = list(catalog.entries.keys())
customSimbad = Simbad()
customSimbad.ROW_LIMIT = -1
customSimbad.TIMEOUT = 120
customSimbad.add_votable_fields('otype', 'sptype', 'sp_bibcode', 'id')
warnings.filterwarnings("ignore")
Mnames, Mradec = [], []
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
try:
name = catalog.add_entry(oname)
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
if (FASTSTARS.RA not in catalog.entries[name]
or FASTSTARS.DEC not in catalog.entries[name]):
continue
else:
Mnames.append(name)
radec = str(catalog.entries[name][FASTSTARS.RA][0]['value']) + str(
catalog.entries[name][FASTSTARS.DEC][0]['value'])
c = coord(radec, unit=(un.hourangle, un.deg), frame='icrs')
cnttry = 0
foundstar = False
while foundstar == False and cnttry < 10:
try:
cnttry += 1
time.sleep(0.1)
result = customSimbad.query_region(c, radius='0d0m5s')
aliases = re.sub(r'b\'(.*)\'', r'\1',
str(result['ID'].tolist()[0])).split(',')
except TypeError:
#print(radec,cnttry)
continue
foundstar = True
if foundstar == True:
source = (catalog.entries[name].add_source(
name='SIMBAD astronomical database',
bibcode="2000A&AS..143....9W",
url="http://simbad.u-strasbg.fr/",
secondary=True))
for alias in aliases:
ali = single_spaces(
re.sub(r'\[[^)]*\]', '', alias).strip())
if is_number(ali.replace(' ', '')):
continue
if ali[:4] == "HVS ":
continue
ali = name_clean(ali)
catalog.entries[name].add_quantity(FASTSTARS.ALIAS, ali,
source)
catalog.journal_entries()
return
def do_cccp(catalog):
task_str = catalog.get_current_task_str()
cccpbands = ['B', 'V', 'R', 'I']
file_names = list(
glob(os.path.join(catalog.get_current_task_repo(),
'CCCP/apj407397*.txt')))
for datafile in pbar_strings(file_names, task_str + ': apj407397...'):
with open(datafile, 'r') as ff:
tsvin = csv.reader(ff, delimiter='\t', skipinitialspace=True)
for rr, row in enumerate(tsvin):
if rr == 0:
continue
elif rr == 1:
name = 'SN' + row[0].split('SN ')[-1]
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(
bibcode='2012ApJ...744...10K')
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, name, source)
elif rr >= 5:
mjd = str(Decimal(row[0]) + 53000)
for bb, band in enumerate(cccpbands):
if row[2 * bb + 1]:
mag = row[2 * bb + 1].strip('>')
upl = (not row[2 * bb + 2])
(catalog.entries[name]
.add_photometry(time=mjd, band=band,
magnitude=mag,
e_magnitude=row[2 * bb + 2],
upperlimit=upl, source=source))
if catalog.current_task.load_archive(catalog.args):
with open(os.path.join(catalog.get_current_task_repo(),
'CCCP/sc_cccp.html'), 'r') as ff:
html = ff.read()
else:
session = requests.Session()
response = session.get(
'https://webhome.weizmann.ac.il/home/iair/sc_cccp.html')
html = response.text
with open(os.path.join(catalog.get_current_task_repo(),
'CCCP/sc_cccp.html'), 'w') as ff:
ff.write(html)
soup = BeautifulSoup(html, 'html5lib')
links = soup.body.findAll("a")
for link in pbar(links, task_str + ': links'):
if 'sc_sn' in link['href']:
name = catalog.add_entry(link.text.replace(' ', ''))
source = (catalog.entries[name]
.add_source(name='CCCP',
url=('https://webhome.weizmann.ac.il'
'/home/iair/sc_cccp.html')))
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
if catalog.current_task.load_archive(catalog.args):
fname = os.path.join(catalog.get_current_task_repo(),
'CCCP/') + link['href'].split('/')[-1]
with open(fname, 'r') as ff:
html2 = ff.read()
else:
response2 = session.get(
'https://webhome.weizmann.ac.il/home/iair/' + link['href'])
html2 = response2.text
fname = os.path.join(catalog.get_current_task_repo(),
'CCCP/') + link['href'].split('/')[-1]
with open(fname, 'w') as ff:
ff.write(html2)
soup2 = BeautifulSoup(html2, 'html5lib')
links2 = soup2.body.findAll("a")
for link2 in links2:
if '.txt' in link2['href'] and '_' in link2['href']:
band = link2['href'].split('_')[1].split('.')[0].upper()
if catalog.current_task.load_archive(catalog.args):
fname = os.path.join(
catalog.get_current_task_repo(), 'CCCP/')
fname += link2['href'].split('/')[-1]
if not os.path.isfile(fname):
continue
with open(fname, 'r') as ff:
html3 = ff.read()
else:
response3 = (session
.get('https://webhome.weizmann.ac.il'
'/home/iair/cccp/' +
link2['href']))
if response3.status_code == 404:
continue
html3 = response3.text
fname = os.path.join(
catalog.get_current_task_repo(), 'CCCP/')
fname += link2['href'].split('/')[-1]
with open(fname, 'w') as ff:
ff.write(html3)
table = [[str(Decimal(yy.strip())).rstrip('0') for yy in
xx.split(',')]
for xx in list(filter(None, html3.split('\n')))]
for row in table:
catalog.entries[name].add_photometry(
time=str(Decimal(row[0]) + 53000),
band=band, magnitude=row[1],
e_magnitude=row[2], source=source)
catalog.journal_entries()
return
def do_asiago_spectra(catalog):
task_str = catalog.get_current_task_str()
html = catalog.load_cached_url(
('http://sngroup.oapd.inaf.it./'
'cgi-bin/output_class.cgi?sn=1990'),
os.path.join(catalog.get_current_task_repo(),
'Asiago/spectra.html'))
if not html:
return
bs = BeautifulSoup(html, 'html5lib')
trs = bs.findAll('tr')
for tr in pbar(trs, task_str):
tds = tr.findAll('td')
name = ''
host = ''
# fitsurl = ''
source = ''
reference = ''
for tdi, td in enumerate(tds):
if tdi == 0:
butt = td.find('button')
if not butt:
break
alias = butt.text.strip()
alias = alias.replace('PSNJ', 'PSN J').replace('GAIA', 'Gaia')
elif tdi == 1:
name = (td.text.strip()
.replace('PSNJ', 'PSN J')
.replace('GAIA', 'Gaia'))
if name.startswith('SN '):
name = 'SN' + name[3:]
if not name:
name = alias
if is_number(name[:4]):
name = 'SN' + name
oldname = name
name = catalog.add_entry(name)
reference = 'Asiago Supernova Catalogue'
refurl = 'http://graspa.oapd.inaf.it/cgi-bin/sncat.php'
secondarysource = catalog.entries[name].add_source(
name=reference, url=refurl, secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oldname,
secondarysource)
if alias != name:
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, alias,
secondarysource)
elif tdi == 2:
host = td.text.strip()
if host == 'anonymous':
host = ''
elif tdi == 3:
discoverer = td.text.strip()
elif tdi == 5:
ra = td.text.strip()
elif tdi == 6:
dec = td.text.strip()
elif tdi == 7:
claimedtype = td.text.strip()
elif tdi == 8:
redshift = td.text.strip()
# elif tdi == 9:
# epochstr = td.text.strip()
# if epochstr:
# mjd = (astrotime(epochstr[:4] + '-' + epochstr[4:6] +
# '-' +
# str(floor(float(epochstr[6:]))).zfill(2)).mjd +
# float(epochstr[6:]) - floor(float(epochstr[6:])))
# else:
# mjd = ''
elif tdi == 10:
refs = td.findAll('a')
source = ''
reference = ''
refurl = ''
for ref in refs:
if ref.text != 'REF':
reference = ref.text
refurl = ref['href']
if reference:
source = catalog.entries[name].add_source(
name=reference, url=refurl)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name,
secondarysource)
sources = uniq_cdl(
list(filter(None, [source, secondarysource])))
elif tdi == 12:
pass
# fitslink = td.find('a')
# if fitslink:
# fitsurl = fitslink['href']
if name:
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE,
claimedtype,
sources)
catalog.entries[name].add_quantity(SUPERNOVA.RA, ra, sources)
catalog.entries[name].add_quantity(SUPERNOVA.DEC, dec, sources)
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, redshift,
sources)
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVERER,
discoverer,
sources)
catalog.entries[name].add_quantity(SUPERNOVA.HOST, host, sources)
# if fitsurl:
# response = urllib.request.urlopen(
# 'http://sngroup.oapd.inaf.it./' + fitsurl)
# compressed = io.BytesIO(response.read())
# decompressed = gzip.GzipFile(fileobj=compressed)
# hdulist = fits.open(decompressed)
# scidata = hdulist[0].data
# print(hdulist[0].header)
#
# print(scidata[3])
# sys.exit()
catalog.journal_entries()
return
def do_cleanup(catalog):
"""Cleanup catalog after importing all data."""
task_str = catalog.get_current_task_str()
# Set preferred names, calculate some columns based on imported data,
# sanitize some fields
keys = list(catalog.entries.keys())
cleanupcnt = 0
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
try:
name = catalog.add_entry(oname)
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
# Set the preferred name, switching to that name if name changed.
name = catalog.entries[name].set_preferred_name()
aliases = catalog.entries[name].get_aliases()
catalog.entries[name].purge_bandless_photometry()
catalog.entries[name].set_first_max_light()
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['MLS', 'SSS', 'CSS', 'GRB ']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:2])):
discoverdate = ('/'.join([
'20' + alias.replace(prefix, '')[:2],
alias.replace(prefix, '')[2:4],
alias.replace(prefix, '')[4:6]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = [
'ASASSN-', 'PS1-', 'PS1', 'PS', 'iPTF', 'PTF', 'SCP-', 'SNLS-',
'SPIRITS', 'LSQ', 'DES', 'SNHiTS', 'Gaia', 'GND', 'GNW', 'GSD',
'GSW', 'EGS', 'COS', 'OGLE', 'HST'
]
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:2])
and is_number(alias.replace(prefix, '')[:1])):
discoverdate = '20' + alias.replace(prefix, '')[:2]
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['SNF']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:4])):
discoverdate = ('/'.join([
alias.replace(prefix, '')[:4],
alias.replace(prefix, '')[4:6],
alias.replace(prefix, '')[6:8]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['PTFS', 'SNSDF']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:2])):
discoverdate = ('/'.join([
'20' + alias.replace(prefix, '')[:2],
alias.replace(prefix, '')[2:4]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['AT', 'SN', 'OGLE-', 'SM ', 'KSN']
for alias in aliases:
for prefix in prefixes:
if alias.startswith(prefix):
year = re.findall(r'\d+', alias)
if len(year) == 1:
year = year[0]
else:
continue
if alias.replace(prefix, '').index(year) != 0:
continue
if (year and is_number(year) and '.' not in year
and len(year) <= 4):
discoverdate = year
if catalog.args.verbose:
tprint('Added discoverdate from name [' +
alias + ']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if (SUPERNOVA.RA not in catalog.entries[name]
or SUPERNOVA.DEC not in catalog.entries[name]):
prefixes = [
'PSN J', 'MASJ', 'CSS', 'SSS', 'MASTER OT J', 'HST J', 'TCP J',
'MACS J', '2MASS J', 'EQ J', 'CRTS J', 'SMT J'
]
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:6])):
noprefix = alias.split(':')[-1].replace(prefix,
'').replace(
'.', '')
decsign = '+' if '+' in noprefix else '-'
noprefix = noprefix.replace('+', '|').replace('-', '|')
nops = noprefix.split('|')
if len(nops) < 2:
continue
rastr = nops[0]
decstr = nops[1]
ra = ':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + \
('.' + rastr[6:] if len(rastr) > 6 else '')
dec = (
decsign +
':'.join([decstr[:2], decstr[2:4], decstr[4:6]]) +
('.' + decstr[6:] if len(decstr) > 6 else ''))
if catalog.args.verbose:
tprint('Added ra/dec from name: ' + ra + ' ' + dec)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(SUPERNOVA.RA,
ra,
source,
derived=True)
catalog.entries[name].add_quantity(SUPERNOVA.DEC,
dec,
source,
derived=True)
break
if SUPERNOVA.RA in catalog.entries[name]:
break
no_host = (SUPERNOVA.HOST not in catalog.entries[name] or not any([
x[QUANTITY.VALUE] == 'Milky Way'
for x in catalog.entries[name][SUPERNOVA.HOST]
]))
if (SUPERNOVA.RA in catalog.entries[name]
and SUPERNOVA.DEC in catalog.entries[name] and no_host):
from astroquery.irsa_dust import IrsaDust
if name not in catalog.extinctions_dict:
try:
ra_dec = catalog.entries[name][
SUPERNOVA.RA][0][QUANTITY.VALUE] + \
" " + \
catalog.entries[name][SUPERNOVA.DEC][0][QUANTITY.VALUE]
result = IrsaDust.get_query_table(ra_dec, section='ebv')
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
warnings.warn("Coordinate lookup for " + name +
" failed in IRSA.")
else:
ebv = result['ext SandF mean'][0]
ebverr = result['ext SandF std'][0]
catalog.extinctions_dict[name] = [ebv, ebverr]
if name in catalog.extinctions_dict:
sources = uniq_cdl([
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2011ApJ...737..103S')
])
(catalog.entries[name].add_quantity(
SUPERNOVA.EBV,
str(catalog.extinctions_dict[name][0]),
sources,
e_value=str(catalog.extinctions_dict[name][1]),
derived=True))
if ((SUPERNOVA.HOST in catalog.entries[name]
and (SUPERNOVA.HOST_RA not in catalog.entries[name]
or SUPERNOVA.HOST_DEC not in catalog.entries[name]))):
for host in catalog.entries[name][SUPERNOVA.HOST]:
alias = host[QUANTITY.VALUE]
if ' J' in alias and is_number(alias.split(' J')[-1][:6]):
noprefix = alias.split(' J')[-1].split(':')[-1].replace(
'.', '')
decsign = '+' if '+' in noprefix else '-'
noprefix = noprefix.replace('+', '|').replace('-', '|')
nops = noprefix.split('|')
if len(nops) < 2:
continue
rastr = nops[0]
decstr = nops[1]
hostra = (':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) +
('.' + rastr[6:] if len(rastr) > 6 else ''))
hostdec = decsign + ':'.join([
decstr[:2], decstr[2:4], decstr[4:6]
]) + ('.' + decstr[6:] if len(decstr) > 6 else '')
if catalog.args.verbose:
tprint('Added hostra/hostdec from name: ' + hostra +
' ' + hostdec)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(SUPERNOVA.HOST_RA,
hostra,
source,
derived=True)
catalog.entries[name].add_quantity(SUPERNOVA.HOST_DEC,
hostdec,
source,
derived=True)
break
if SUPERNOVA.HOST_RA in catalog.entries[name]:
break
if (SUPERNOVA.REDSHIFT not in catalog.entries[name]
and SUPERNOVA.VELOCITY in catalog.entries[name]):
# Find the "best" velocity to use for this
bestsig = 0
for hv in catalog.entries[name][SUPERNOVA.VELOCITY]:
sig = get_sig_digits(hv[QUANTITY.VALUE])
if sig > bestsig:
besthv = hv[QUANTITY.VALUE]
bestsrc = hv['source']
bestsig = sig
if bestsig > 0 and is_number(besthv):
voc = float(besthv) * 1.e5 / CLIGHT
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
(catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT,
pretty_num(sqrt((1. + voc) / (1. - voc)) - 1.,
sig=bestsig),
sources,
kind='heliocentric',
derived=True))
if (SUPERNOVA.REDSHIFT not in catalog.entries[name]
and len(catalog.nedd_dict) > 0
and SUPERNOVA.HOST in catalog.entries[name]):
reference = "NED-D"
refurl = "http://ned.ipac.caltech.edu/Library/Distances/"
refbib = "1991ASSL..171...89H"
for host in catalog.entries[name][SUPERNOVA.HOST]:
if host[QUANTITY.VALUE] in catalog.nedd_dict:
source = catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
secondarysource = catalog.entries[name].add_source(
name=reference,
url=refurl,
bibcode=refbib,
secondary=True)
meddist = statistics.median(
catalog.nedd_dict[host[QUANTITY.VALUE]])
redz = z_at_value(cosmo.comoving_distance,
float(meddist) * un.Mpc)
redshift = pretty_num(redz,
sig=get_sig_digits(str(meddist)))
catalog.entries[name].add_quantity(
[SUPERNOVA.REDSHIFT, SUPERNOVA.HOST_REDSHIFT],
redshift,
uniq_cdl([source, secondarysource]),
kind='host',
derived=True)
if (SUPERNOVA.MAX_ABS_MAG not in catalog.entries[name]
and SUPERNOVA.MAX_APP_MAG in catalog.entries[name]
and SUPERNOVA.LUM_DIST in catalog.entries[name]):
# Find the "best" distance to use for this
bestsig = 0
for ld in catalog.entries[name][SUPERNOVA.LUM_DIST]:
sig = get_sig_digits(ld[QUANTITY.VALUE])
if sig > bestsig:
bestld = ld[QUANTITY.VALUE]
bestsrc = ld[QUANTITY.SOURCE]
bestsig = sig
if bestsig > 0 and is_number(bestld) and float(bestld) > 0.:
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
bestldz = z_at_value(cosmo.luminosity_distance,
float(bestld) * un.Mpc)
pnum = (float(catalog.entries[name][SUPERNOVA.MAX_APP_MAG][0][
QUANTITY.VALUE]) - 5.0 *
(log10(float(bestld) * 1.0e6) - 1.0) +
2.5 * log10(1.0 + bestldz))
pnum = pretty_num(pnum, sig=bestsig + 1)
catalog.entries[name].add_quantity(SUPERNOVA.MAX_ABS_MAG,
pnum,
sources,
derived=True)
if (SUPERNOVA.MAX_VISUAL_ABS_MAG not in catalog.entries[name]
and SUPERNOVA.MAX_VISUAL_APP_MAG in catalog.entries[name]
and SUPERNOVA.LUM_DIST in catalog.entries[name]):
# Find the "best" distance to use for this
bestsig = 0
for ld in catalog.entries[name][SUPERNOVA.LUM_DIST]:
sig = get_sig_digits(ld[QUANTITY.VALUE])
if sig > bestsig:
bestld = ld[QUANTITY.VALUE]
bestsrc = ld[QUANTITY.SOURCE]
bestsig = sig
if bestsig > 0 and is_number(bestld) and float(bestld) > 0.:
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
# FIX: what's happening here?!
pnum = (float(catalog.entries[name][
SUPERNOVA.MAX_VISUAL_APP_MAG][0][QUANTITY.VALUE]) - 5.0 *
(log10(float(bestld) * 1.0e6) - 1.0))
pnum = pretty_num(pnum, sig=bestsig + 1)
catalog.entries[name].add_quantity(
SUPERNOVA.MAX_VISUAL_ABS_MAG, pnum, sources, derived=True)
if SUPERNOVA.REDSHIFT in catalog.entries[name]:
# Find the "best" redshift to use for this
bestz, bestkind, bestsig, bestsrc = catalog.entries[
name].get_best_redshift()
if bestsig > 0:
try:
bestz = float(bestz)
except Exception:
print(catalog.entries[name])
raise
if SUPERNOVA.VELOCITY not in catalog.entries[name]:
source = catalog.entries[name].add_self_source()
# FIX: what's happening here?!
pnum = CLIGHT / KM * \
((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.)
pnum = pretty_num(pnum, sig=bestsig)
catalog.entries[name].add_quantity(
SUPERNOVA.VELOCITY,
pnum,
source,
kind=(SUPERNOVA.VELOCITY.kind_preference[bestkind]
if bestkind else ''))
if bestz > 0.:
if SUPERNOVA.LUM_DIST not in catalog.entries[name]:
dl = cosmo.luminosity_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
SUPERNOVA.LUM_DIST,
pretty_num(dl.value, sig=bestsig + 1),
sources,
kind=(SUPERNOVA.LUM_DIST.kind_preference[bestkind]
if bestkind else ''),
derived=True)
if (SUPERNOVA.MAX_ABS_MAG not in catalog.entries[name]
and SUPERNOVA.MAX_APP_MAG
in catalog.entries[name]):
source = catalog.entries[name].add_self_source()
pnum = pretty_num(
float(catalog.entries[name][
SUPERNOVA.MAX_APP_MAG][0][QUANTITY.VALUE])
- 5.0 * (log10(dl.to('pc').value) - 1.0) +
2.5 * log10(1.0 + bestz),
sig=bestsig + 1)
catalog.entries[name].add_quantity(
SUPERNOVA.MAX_ABS_MAG,
pnum,
sources,
derived=True)
if (SUPERNOVA.MAX_VISUAL_ABS_MAG
not in catalog.entries[name]
and SUPERNOVA.MAX_VISUAL_APP_MAG
in catalog.entries[name]):
source = catalog.entries[name].add_self_source()
pnum = pretty_num(float(catalog.entries[name][
SUPERNOVA.MAX_VISUAL_APP_MAG][0][
QUANTITY.VALUE]) - 5.0 *
(log10(dl.to('pc').value) - 1.0),
sig=bestsig + 1)
catalog.entries[name].add_quantity(
SUPERNOVA.MAX_VISUAL_ABS_MAG,
pnum,
sources,
derived=True)
if SUPERNOVA.COMOVING_DIST not in catalog.entries[name]:
cd = cosmo.comoving_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
SUPERNOVA.COMOVING_DIST,
pretty_num(cd.value, sig=bestsig),
sources,
derived=True)
if SUPERNOVA.HOST_REDSHIFT in catalog.entries[name]:
# Find the "best" redshift to use for this
bestz, bestkind, bestsig, bestsrc = catalog.entries[
name].get_best_redshift(SUPERNOVA.HOST_REDSHIFT)
if bestsig > 0:
try:
bestz = float(bestz)
except Exception:
print(catalog.entries[name])
raise
if SUPERNOVA.HOST_VELOCITY not in catalog.entries[name]:
source = catalog.entries[name].add_self_source()
# FIX: what's happening here?!
pnum = CLIGHT / KM * \
((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.)
pnum = pretty_num(pnum, sig=bestsig)
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_VELOCITY,
pnum,
source,
kind=(SUPERNOVA.HOST_VELOCITY.kind_preference[bestkind]
if bestkind else ''))
if bestz > 0.:
if SUPERNOVA.HOST_LUM_DIST not in catalog.entries[name]:
dl = cosmo.luminosity_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_LUM_DIST,
pretty_num(dl.value, sig=bestsig + 1),
sources,
kind=(SUPERNOVA.HOST_LUM_DIST.
kind_preference[bestkind]
if bestkind else ''),
derived=True)
if SUPERNOVA.HOST_COMOVING_DIST not in catalog.entries[
name]:
cd = cosmo.comoving_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_COMOVING_DIST,
pretty_num(cd.value, sig=bestsig),
sources,
derived=True)
if all([
x in catalog.entries[name] for x in [
SUPERNOVA.RA, SUPERNOVA.DEC, SUPERNOVA.HOST_RA,
SUPERNOVA.HOST_DEC
]
]):
# For now just using first coordinates that appear in entry
try:
c1 = coord(
ra=catalog.entries[name][SUPERNOVA.RA][0][QUANTITY.VALUE],
dec=catalog.entries[name][SUPERNOVA.DEC][0][
QUANTITY.VALUE],
unit=(un.hourangle, un.deg))
c2 = coord(ra=catalog.entries[name][SUPERNOVA.HOST_RA][0][
QUANTITY.VALUE],
dec=catalog.entries[name][SUPERNOVA.HOST_DEC][0][
QUANTITY.VALUE],
unit=(un.hourangle, un.deg))
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
pass
else:
sources = uniq_cdl([catalog.entries[name].add_self_source()] +
catalog.entries[name][SUPERNOVA.RA][0][
QUANTITY.SOURCE].split(',') +
catalog.entries[name][SUPERNOVA.DEC][0][
QUANTITY.SOURCE].split(',') +
catalog.entries[name][SUPERNOVA.HOST_RA][0][
QUANTITY.SOURCE].split(',') +
catalog.entries[name][SUPERNOVA.HOST_DEC][0]
[QUANTITY.SOURCE].split(','))
if SUPERNOVA.HOST_OFFSET_ANG not in catalog.entries[name]:
hosa = Decimal(c1.separation(c2).arcsecond)
hosa = pretty_num(hosa)
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_OFFSET_ANG,
hosa,
sources,
derived=True,
u_value='arcseconds')
if (SUPERNOVA.COMOVING_DIST in catalog.entries[name]
and SUPERNOVA.REDSHIFT in catalog.entries[name]
and SUPERNOVA.HOST_OFFSET_DIST
not in catalog.entries[name]):
offsetsig = get_sig_digits(catalog.entries[name][
SUPERNOVA.HOST_OFFSET_ANG][0][QUANTITY.VALUE])
sources = uniq_cdl(
sources.split(',') +
(catalog.entries[name][SUPERNOVA.COMOVING_DIST][0][
QUANTITY.SOURCE]).split(',') +
(catalog.entries[name][SUPERNOVA.REDSHIFT][0][
QUANTITY.SOURCE]).split(','))
(catalog.entries[name].add_quantity(
SUPERNOVA.HOST_OFFSET_DIST,
pretty_num(
float(catalog.entries[name][
SUPERNOVA.HOST_OFFSET_ANG][0][QUANTITY.VALUE])
/ 3600. * (pi / 180.) *
float(catalog.entries[name][
SUPERNOVA.COMOVING_DIST][0][QUANTITY.VALUE]) *
1000. / (1.0 + float(catalog.entries[name][
SUPERNOVA.REDSHIFT][0][QUANTITY.VALUE])),
sig=offsetsig), sources))
catalog.entries[name].sanitize()
catalog.journal_entries(bury=True, final=True, gz=True)
cleanupcnt = cleanupcnt + 1
if catalog.args.travis and cleanupcnt % 1000 == 0:
break
catalog.save_caches()
return
def do_snf_specta(catalog):
task_str = catalog.get_current_task_str()
bibcodes = {'SN2005gj': '2006ApJ...650..510A',
'SN2006D': '2007ApJ...654L..53T',
'SN2007if': '2010ApJ...713.1073S',
'SN2011fe': '2013A&A...554A..27P'}
oldname = ''
snfcnt = 0
eventfolders = next(os.walk(os.path.join(
catalog.get_current_task_repo(), 'SNFactory')))[1]
for eventfolder in pbar(eventfolders, task_str):
oname = eventfolder
name = catalog.get_preferred_name(oname)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
sec_reference = 'Nearby Supernova Factory'
sec_refurl = 'http://snfactory.lbl.gov/'
sec_bibcode = '2002SPIE.4836...61A'
sec_source = catalog.entries[name].add_source(
name=sec_reference, url=sec_refurl, bibcode=sec_bibcode,
secondary=True)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oname, sec_source)
bibcode = bibcodes[oname]
source = catalog.entries[name].add_source(bibcode=bibcode)
sources = uniq_cdl([source, sec_source])
use_path = os.path.join(
catalog.get_current_task_repo(), 'SNFactory', eventfolder, '*.dat')
eventspectra = glob(use_path)
for spectrum in pbar(eventspectra, task_str):
filename = os.path.basename(spectrum)
with open(spectrum) as spec_file:
specdata = list(csv.reader(
spec_file, delimiter=' ', skipinitialspace=True))
specdata = list(filter(None, specdata))
newspec = []
time = ''
telescope = ''
instrument = ''
observer = ''
observatory = ''
if 'Keck_20060202_R' in spectrum:
time = '53768.23469'
elif 'Spectrum05_276' in spectrum:
time = pretty_num(astrotime('2005-10-03').mjd, sig=5)
elif 'Spectrum05_329' in spectrum:
time = pretty_num(astrotime('2005-11-25').mjd, sig=5)
elif 'Spectrum05_336' in spectrum:
time = pretty_num(astrotime('2005-12-02').mjd, sig=5)
for row in specdata:
if row[0][0] == '#':
joinrow = (' '.join(row)).split('=')
if len(joinrow) < 2:
continue
field = joinrow[0].strip('# ')
value = joinrow[1].split('/')[0].strip('\' ')
if not time:
if field == 'JD':
time = str(jd_to_mjd(Decimal(value)))
elif field == 'MJD':
time = value
elif field == 'MJD-OBS':
time = value
if field == 'OBSERVER':
observer = value.capitalize()
if field == 'OBSERVAT':
observatory = value.capitalize()
if field == 'TELESCOP':
telescope = value.capitalize()
if field == 'INSTRUME':
instrument = value.capitalize()
else:
newspec.append(row)
if not time:
raise ValueError('Time missing from spectrum.')
specdata = newspec
haserrors = len(specdata[0]) == 3 and specdata[
0][2] and specdata[0][2] != 'NaN'
specdata = [list(i) for i in zip(*specdata)]
wavelengths = specdata[0]
fluxes = specdata[1]
errors = ''
if haserrors:
errors = specdata[2]
unit_err = ('Variance' if oldname == 'SN2011fe' else
'erg/s/cm^2/Angstrom')
unit_flx = 'erg/s/cm^2/Angstrom'
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom', u_fluxes=unit_flx, u_time='MJD',
time=time,
wavelengths=wavelengths, fluxes=fluxes, errors=errors,
observer=observer, observatory=observatory,
telescope=telescope, instrument=instrument, u_errors=unit_err,
source=sources, filename=filename)
snfcnt = snfcnt + 1
if (catalog.args.travis and
snfcnt % catalog.TRAVIS_QUERY_LIMIT == 0):
break
catalog.journal_entries()
return
def do_asassn(catalog):
"""Import list of ASASSN events."""
task_str = catalog.get_current_task_str()
asn_url = 'http://www.astronomy.ohio-state.edu/~assassin/sn_list.html'
html = catalog.load_url(
asn_url,
os.path.join(catalog.get_current_task_repo(), 'ASASSN/sn_list.html'))
if not html:
return
bs = BeautifulSoup(html, 'html5lib')
trs = bs.find('table').findAll('tr')
for tri, tr in enumerate(pbar(trs, task_str)):
name = ''
ra = ''
dec = ''
redshift = ''
hostoff = ''
claimedtype = ''
host = ''
atellink = ''
typelink = ''
if tri == 0:
continue
tds = tr.findAll('td')
for tdi, td in enumerate(tds):
if tdi == 1:
name = catalog.add_entry(td.text.strip())
atellink = td.find('a')
if atellink:
atellink = atellink['href']
else:
atellink = ''
if tdi == 2:
discdate = td.text.replace('-', '/')
if tdi == 3:
ra = td.text
if tdi == 4:
dec = td.text
if tdi == 5:
redshift = td.text
if tdi == 8:
hostoff = td.text
if tdi == 9:
claimedtype = td.text
typelink = td.find('a')
if typelink:
typelink = typelink['href']
else:
typelink = ''
if tdi == 12:
host = td.text
sources = [
catalog.entries[name].add_source(url=asn_url,
name='ASAS-SN Supernovae')
]
typesources = sources[:]
if atellink:
sources.append((catalog.entries[name].add_source(
name='ATel ' + atellink.split('=')[-1], url=atellink)))
if typelink:
typesources.append((catalog.entries[name].add_source(
name='ATel ' + typelink.split('=')[-1], url=typelink)))
sources = ','.join(sources)
typesources = ','.join(typesources)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, sources)
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, discdate,
sources)
catalog.entries[name].add_quantity(SUPERNOVA.RA,
ra,
sources,
u_value='floatdegrees')
catalog.entries[name].add_quantity(SUPERNOVA.DEC,
dec,
sources,
u_value='floatdegrees')
catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, redshift,
sources)
catalog.entries[name].add_quantity(SUPERNOVA.HOST_OFFSET_ANG,
hostoff,
sources,
u_value='arcseconds')
for ct in claimedtype.split('/'):
if ct != 'Unk':
catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, ct,
typesources)
if host != 'Uncatalogued':
catalog.entries[name].add_quantity(SUPERNOVA.HOST, host, sources)
catalog.journal_entries()
return
def do_mosfit(catalog):
task_str = catalog.get_current_task_str()
try:
with open('mosfit.key', 'r') as f:
mosfitkey = f.read().splitlines()[0]
except:
catalog.log.warning('MOSFiT key not found, make sure a file named '
'`mosfit.key` containing the key is placed the '
'astrocats directory.')
mosfitkey = ''
dbx = dropbox.Dropbox(mosfitkey)
files = list(
sorted([
x.name for x in dbx.files_list_folder('').entries
if not x.name.startswith('.')
]))
fdir = os.path.join(catalog.get_current_task_repo(), 'MOSFiT')
if not os.path.isdir(fdir):
os.mkdir(fdir)
efiles = [x.split('/')[-1] for x in glob(os.path.join(fdir, '*'))]
old_name = ''
for fname in pbar(files, desc=task_str):
if fname in efiles:
efiles.remove(fname)
fpath = os.path.join(fdir, fname)
if not os.path.isfile(fpath):
md, res = dbx.files_download('/' + fname)
jtxt = res.content
with open(fpath, 'wb') as f:
f.write(jtxt)
name = fname.split('_')[-3]
new_entry = Supernova.init_from_file(catalog,
path=fpath,
compare_to_existing=False)
old_entry = None
if name in catalog.entries:
if catalog.entries[name]._stub:
old_entry = Supernova.init_from_file(catalog,
name=name,
compare_to_existing=False)
else:
old_entry = catalog.entries[name]
if old_entry:
catalog.copy_entry_to_entry(new_entry,
old_entry,
compare_to_existing=False)
catalog.entries[name] = old_entry
else:
catalog.entries[name] = new_entry
if old_name != name:
catalog.journal_entries()
old_name = name
for fname in efiles:
os.remove(os.path.join(fdir, fname))
return
def do_cfa_spectra(catalog):
"""Import spectra from the CfA archive."""
task_str = catalog.get_current_task_str()
# II spectra
oldname = ''
file_names = next(
os.walk(os.path.join(catalog.get_current_task_repo(), 'CfA_SNII')))[1]
for ni, name in enumerate(pbar_strings(file_names, task_str)):
fullpath = os.path.join(catalog.get_current_task_repo(),
'CfA_SNII/') + name
origname = name
if name.startswith('sn') and is_number(name[2:6]):
name = 'SN' + name[2:]
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
reference = 'CfA Supernova Archive'
refurl = 'https://www.cfa.harvard.edu/supernova/SNarchive.html'
source = catalog.entries[name].add_source(
name=reference,
url=refurl,
secondary=True,
acknowledgment=ACKN_CFA)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
for fi, fname in enumerate(
sorted(
glob(fullpath + '/*'), key=lambda s: s.lower())):
filename = os.path.basename(fname)
fileparts = filename.split('-')
if origname.startswith('sn') and is_number(origname[2:6]):
year = fileparts[1][:4]
month = fileparts[1][4:6]
day = fileparts[1][6:]
instrument = fileparts[2].split('.')[0]
else:
year = fileparts[2][:4]
month = fileparts[2][4:6]
day = fileparts[2][6:]
instrument = fileparts[3].split('.')[0]
time = str(
astrotime(year + '-' + month + '-' + str(floor(float(day)))
.zfill(2)).mjd + float(day) - floor(float(day)))
f = open(fname, 'r')
data = csv.reader(f, delimiter=' ', skipinitialspace=True)
data = [list(i) for i in zip(*data)]
wavelengths = data[0]
fluxes = data[1]
errors = data[2]
sources = uniq_cdl([
source,
(catalog.entries[name]
.add_source(bibcode='2017arXiv170601030H'))
])
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom',
u_fluxes='erg/s/cm^2/Angstrom',
filename=filename,
wavelengths=wavelengths,
fluxes=fluxes,
u_time='MJD' if time else '',
time=time,
instrument=instrument,
u_errors='ergs/s/cm^2/Angstrom',
errors=errors,
source=sources,
dereddened=False,
deredshifted=False)
if catalog.args.travis and ni >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
# Ia spectra
oldname = ''
file_names = next(
os.walk(os.path.join(catalog.get_current_task_repo(), 'CfA_SNIa')))[1]
for ni, name in enumerate(pbar_strings(file_names, task_str)):
fullpath = os.path.join(catalog.get_current_task_repo(),
'CfA_SNIa/') + name
origname = name
if name.startswith('sn') and is_number(name[2:6]):
name = 'SN' + name[2:]
if name.startswith('snf') and is_number(name[3:7]):
name = 'SNF' + name[3:]
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
reference = 'CfA Supernova Archive'
refurl = 'https://www.cfa.harvard.edu/supernova/SNarchive.html'
source = catalog.entries[name].add_source(
name=reference,
url=refurl,
secondary=True,
acknowledgment=ACKN_CFA)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
for fi, fname in enumerate(
sorted(
glob(fullpath + '/*'), key=lambda s: s.lower())):
filename = os.path.basename(fname)
fileparts = filename.split('-')
if origname.startswith('sn') and is_number(origname[2:6]):
year = fileparts[1][:4]
month = fileparts[1][4:6]
day = fileparts[1][6:]
instrument = fileparts[2].split('.')[0]
else:
year = fileparts[2][:4]
month = fileparts[2][4:6]
day = fileparts[2][6:]
instrument = fileparts[3].split('.')[0]
time = str(
astrotime(year + '-' + month + '-' + str(floor(float(day)))
.zfill(2)).mjd + float(day) - floor(float(day)))
f = open(fname, 'r')
data = csv.reader(f, delimiter=' ', skipinitialspace=True)
data = [list(i) for i in zip(*data)]
wavelengths = data[0]
fluxes = data[1]
errors = data[2]
sources = uniq_cdl([
source, (catalog.entries[name]
.add_source(bibcode='2012AJ....143..126B')),
(catalog.entries[name]
.add_source(bibcode='2008AJ....135.1598M'))
])
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom',
u_fluxes='erg/s/cm^2/Angstrom',
filename=filename,
wavelengths=wavelengths,
fluxes=fluxes,
u_time='MJD' if time else '',
time=time,
instrument=instrument,
u_errors='ergs/s/cm^2/Angstrom',
errors=errors,
source=sources,
dereddened=False,
deredshifted=False)
if catalog.args.travis and ni >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
# Ibc spectra
oldname = ''
file_names = next(
os.walk(os.path.join(catalog.get_current_task_repo(), 'CfA_SNIbc')))[1]
for ni, name in enumerate(pbar(file_names, task_str)):
fullpath = os.path.join(catalog.get_current_task_repo(),
'CfA_SNIbc/') + name
if name.startswith('sn') and is_number(name[2:6]):
name = 'SN' + name[2:]
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
reference = 'CfA Supernova Archive'
refurl = 'https://www.cfa.harvard.edu/supernova/SNarchive.html'
source = catalog.entries[name].add_source(
name=reference,
url=refurl,
secondary=True,
acknowledgment=ACKN_CFA)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
for fi, fname in enumerate(
sorted(
glob(fullpath + '/*'), key=lambda s: s.lower())):
filename = os.path.basename(fname)
fileparts = filename.split('-')
instrument = ''
year = fileparts[1][:4]
month = fileparts[1][4:6]
day = fileparts[1][6:].split('.')[0]
if len(fileparts) > 2:
instrument = fileparts[-1].split('.')[0]
time = str(
astrotime(year + '-' + month + '-' + str(floor(float(day)))
.zfill(2)).mjd + float(day) - floor(float(day)))
f = open(fname, 'r')
data = csv.reader(f, delimiter=' ', skipinitialspace=True)
data = [list(i) for i in zip(*data)]
wavelengths = data[0]
fluxes = data[1]
sources = uniq_cdl([
source, catalog.entries[name]
.add_source(bibcode='2014AJ....147...99M')
])
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom',
u_fluxes='erg/s/cm^2/Angstrom',
wavelengths=wavelengths,
filename=filename,
fluxes=fluxes,
u_time='MJD' if time else '',
time=time,
instrument=instrument,
source=sources,
dereddened=False,
deredshifted=False)
if catalog.args.travis and ni >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
# Other spectra
oldname = ''
file_names = next(
os.walk(os.path.join(catalog.get_current_task_repo(), 'CfA_Extra')))[1]
for ni, name in enumerate(pbar_strings(file_names, task_str)):
fullpath = os.path.join(catalog.get_current_task_repo(),
'CfA_Extra/') + name
if name.startswith('sn') and is_number(name[2:6]):
name = 'SN' + name[2:]
name = catalog.get_preferred_name(name)
if oldname and name != oldname:
catalog.journal_entries()
oldname = name
name = catalog.add_entry(name)
reference = 'CfA Supernova Archive'
refurl = 'https://www.cfa.harvard.edu/supernova/SNarchive.html'
source = catalog.entries[name].add_source(
name=reference,
url=refurl,
secondary=True,
acknowledgment=ACKN_CFA)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
for fi, fname in enumerate(
sorted(
glob(fullpath + '/*'), key=lambda s: s.lower())):
if not os.path.isfile(fname):
continue
filename = os.path.basename(fname)
if ((not filename.startswith('sn') or
not filename.endswith('flm') or any(
x in filename
for x in ['-interp', '-z', '-dered', '-obj', '-gal']))):
continue
fileparts = filename.split('.')[0].split('-')
instrument = ''
time = ''
if len(fileparts) > 1:
year = fileparts[1][:4]
month = fileparts[1][4:6]
day = fileparts[1][6:]
if is_number(year) and is_number(month) and is_number(day):
if len(fileparts) > 2:
instrument = fileparts[-1]
time = str(
astrotime(year + '-' + month + '-' + str(
floor(float(day))).zfill(2)).mjd + float(day) -
floor(float(day)))
f = open(fname, 'r')
data = csv.reader(f, delimiter=' ', skipinitialspace=True)
data = [list(i) for i in zip(*data)]
wavelengths = data[0]
fluxes = [str(Decimal(x) * Decimal(1.0e-15)) for x in data[1]]
catalog.entries[name].add_spectrum(
u_wavelengths='Angstrom',
u_fluxes='erg/s/cm^2/Angstrom',
wavelengths=wavelengths,
filename=filename,
fluxes=fluxes,
u_time='MJD' if time else '',
time=time,
instrument=instrument,
source=source,
dereddened=False,
deredshifted=False)
if catalog.args.travis and ni >= catalog.TRAVIS_QUERY_LIMIT:
break
catalog.journal_entries()
return
def do_ogle(catalog):
task_str = catalog.get_current_task_str()
basenames = ['transients', 'transients/2014b', 'transients/2014',
'transients/2013', 'transients/2012']
oglenames = []
ogleupdate = [True, False, False, False, False]
for b, bn in enumerate(pbar(basenames, task_str)):
if catalog.args.update and not ogleupdate[b]:
continue
filepath = os.path.join(catalog.get_current_task_repo(), 'OGLE-')
filepath += bn.replace('/', '-') + '-transients.html'
htmltxt = catalog.load_cached_url(
'http://ogle.astrouw.edu.pl/ogle4/' + bn +
'/transients.html', filepath)
if not htmltxt:
continue
soup = BeautifulSoup(htmltxt, 'html5lib')
links = soup.findAll('a')
breaks = soup.findAll('br')
datalinks = []
datafnames = []
for a in links:
if a.has_attr('href'):
if '.dat' in a['href']:
datalinks.append(
'http://ogle.astrouw.edu.pl/ogle4/' + bn + '/' +
a['href'])
datafnames.append(bn.replace('/', '-') +
'-' + a['href'].replace('/', '-'))
ec = -1
reference = 'OGLE-IV Transient Detection System'
refurl = 'http://ogle.astrouw.edu.pl/ogle4/transients/transients.html'
for br in pbar(breaks, task_str):
sibling = br.nextSibling
if 'Ra,Dec=' in sibling:
line = sibling.replace('\n', '').split('Ra,Dec=')
name = line[0].strip()
ec += 1
if 'NOVA' in name or 'dupl' in name:
continue
if name in oglenames:
continue
oglenames.append(name)
name = catalog.add_entry(name)
mySibling = sibling.nextSibling
atelref = ''
claimedtype = ''
while 'Ra,Dec=' not in mySibling:
if isinstance(mySibling, NavigableString):
if 'Phot.class=' in str(mySibling):
claimedtype = re.sub(
r'\([^)]*\)', '',
str(mySibling).split('=')[-1])
claimedtype = claimedtype.replace('SN', '').strip()
if isinstance(mySibling, Tag):
atela = mySibling
if (atela and atela.has_attr('href') and
'astronomerstelegram' in atela['href']):
atelref = atela.contents[0].strip()
atelurl = atela['href']
mySibling = mySibling.nextSibling
if mySibling is None:
break
# nextSibling = sibling.nextSibling
# if ((isinstance(nextSibling, Tag) and
# nextSibling.has_attr('alt') and
# nextSibling.contents[0].strip() != 'NED')):
# radec = nextSibling.contents[0].strip().split()
# else:
# radec = line[-1].split()
# ra = radec[0]
# dec = radec[1]
fname = os.path.join(catalog.get_current_task_repo(),
'OGLE/') + datafnames[ec]
if (catalog.current_task.load_archive(catalog.args) and
os.path.isfile(fname)):
with open(fname, 'r') as f:
csvtxt = f.read()
else:
response = urllib.request.urlopen(datalinks[ec])
with open(fname, 'w') as f:
csvtxt = response.read().decode('utf-8')
f.write(csvtxt)
lcdat = csvtxt.splitlines()
sources = [catalog.entries[name].add_source(
name=reference, url=refurl)]
catalog.entries[name].add_quantity(
SUPERNOVA.ALIAS, name, sources[0])
if atelref and atelref != 'ATel#----':
sources.append(catalog.entries[name].add_source(
name=atelref, url=atelurl))
sources = uniq_cdl(sources)
if name.startswith('OGLE'):
if name[4] == '-':
if is_number(name[5:9]):
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, name[5:9], sources)
else:
if is_number(name[4:6]):
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE, '20' + name[4:6],
sources)
# RA and Dec from OGLE pages currently not reliable
# catalog.entries[name].add_quantity(SUPERNOVA.RA, ra, sources)
# catalog.entries[name].add_quantity(SUPERNOVA.DEC, dec,
# sources)
if claimedtype and claimedtype != '-':
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, claimedtype, sources)
elif ('SN' not in name and SUPERNOVA.CLAIMED_TYPE not in
catalog.entries[name]):
catalog.entries[name].add_quantity(
SUPERNOVA.CLAIMED_TYPE, 'Candidate', sources)
for row in lcdat:
row = row.split()
mjd = str(jd_to_mjd(Decimal(row[0])))
magnitude = row[1]
if float(magnitude) > 90.0:
continue
e_mag = row[2]
upperlimit = False
if e_mag == '-1' or float(e_mag) > 10.0:
e_mag = ''
upperlimit = True
catalog.entries[name].add_photometry(
time=mjd, band='I', magnitude=magnitude,
e_magnitude=e_mag,
system='Vega', source=sources, upperlimit=upperlimit)
if catalog.args.update:
catalog.journal_entries()
catalog.journal_entries()
return
def do_snhunt(catalog):
task_str = catalog.get_current_task_str()
snh_url = 'http://nesssi.cacr.caltech.edu/catalina/current.html'
html = catalog.load_url(
snh_url,
os.path.join(catalog.get_current_task_repo(), 'SNhunt/current.html'))
if not html:
return
text = html.splitlines()
findtable = False
for ri, row in enumerate(text):
if 'Supernova Discoveries' in row:
findtable = True
if findtable and '<table' in row:
tstart = ri + 1
if findtable and '</table>' in row:
tend = ri - 1
tablestr = '<html><body><table>'
for row in text[tstart:tend]:
if row[:3] == 'tr>':
tablestr = tablestr + '<tr>' + row[3:]
else:
tablestr = tablestr + row
tablestr = tablestr + '</table></body></html>'
bs = BeautifulSoup(tablestr, 'html5lib')
trs = bs.find('table').findAll('tr')
for tr in pbar(trs, task_str):
cols = [str(xx.text) for xx in tr.findAll('td')]
if not cols:
continue
name = re.sub('<[^<]+?>', '',
cols[4]).strip().replace(' ',
'').replace('SNHunt', 'SNhunt')
name = catalog.add_entry(name)
source = catalog.entries[name].add_source(name='Supernova Hunt',
url=snh_url)
catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source)
host = re.sub('<[^<]+?>', '', cols[1]).strip().replace('_', ' ')
catalog.entries[name].add_quantity(SUPERNOVA.HOST, host, source)
catalog.entries[name].add_quantity(SUPERNOVA.RA,
cols[2],
source,
u_value='floatdegrees')
catalog.entries[name].add_quantity(SUPERNOVA.DEC,
cols[3],
source,
u_value='floatdegrees')
dd = cols[0]
discoverdate = dd[:4] + '/' + dd[4:6] + '/' + dd[6:8]
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE,
discoverdate, source)
discoverers = cols[5].split('/')
for discoverer in discoverers:
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVERER, 'CRTS',
source)
catalog.entries[name].add_quantity(SUPERNOVA.DISCOVERER,
discoverer, source)
if catalog.args.update:
catalog.journal_entries()
catalog.journal_entries()
return
