def do_ps_mds(catalog): task_str = catalog.get_current_task_str() with open( os.path.join(catalog.get_current_task_repo(), 'MDS/apj506838t1_mrt.txt')) as f: for ri, row in enumerate(pbar(f.read().splitlines(), task_str)): if ri < 35: continue cols = [x.strip() for x in row.split(',')] name = catalog.add_entry(cols[0]) source = catalog.entries[name].add_source( bibcode='2015ApJ...799..208S') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) catalog.entries[name].add_quantity(SUPERNOVA.RA, cols[2], source) catalog.entries[name].add_quantity(SUPERNOVA.DEC, cols[3], source) astrot = astrotime(float(cols[4]), format='mjd').datetime ddate = make_date_string(astrot.year, astrot.month, astrot.day) catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, ddate, source) catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, cols[5], source, kind='spectroscopic') catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, 'II P', source) catalog.journal_entries() return
def do_fermi(catalog): task_str = catalog.get_current_task_str() with open(os.path.join(catalog.get_current_task_repo(), '1SC_catalog_v01.asc'), 'r') as ff: tsvin = list(csv.reader(ff, delimiter=',')) for ri, row in enumerate(pbar(tsvin, task_str)): if row[0].startswith('#'): if len(row) > 1 and 'UPPER_LIMITS' in row[1]: break continue if 'Classified' not in row[1]: continue name = row[0].replace('SNR', 'G') name = catalog.add_entry(name) source = (catalog.entries[name] .add_source(bibcode='2016ApJS..224....8A')) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, row[0].replace('SNR', 'MWSNR'), source) catalog.entries[name].add_quantity( SUPERNOVA.RA, row[2], source, u_value='floatdegrees') catalog.entries[name].add_quantity( SUPERNOVA.DEC, row[3], source, u_value='floatdegrees') catalog.journal_entries() return
def do_pessto(catalog): task_str = catalog.get_current_task_str() pessto_path = os.path.join( catalog.get_current_task_repo(), 'PESSTO_MPHOT.csv') tsvin = list(csv.reader(open(pessto_path, 'r'), delimiter=',')) for ri, row in enumerate(pbar(tsvin, task_str)): if ri == 0: bands = [xx.split('_')[0] for xx in row[3::2]] systems = [xx.split('_')[1].capitalize().replace( 'Ab', 'AB') for xx in row[3::2]] continue name = row[1] name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2015A&A...579A..40S') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) for hi, ci in enumerate(range(3, len(row) - 1, 2)): if not row[ci]: continue teles = 'Swift' if systems[hi] == 'Swift' else '' (catalog.entries[name] .add_photometry(time=row[2], magnitude=row[ci], e_magnitude=row[ci + 1], band=bands[hi], system=systems[hi], telescope=teles, source=source)) catalog.journal_entries() return
def do_smt(catalog): task_str = catalog.get_current_task_str() smt_url = 'http://www.mso.anu.edu.au/skymapper/smt/transients/tns/' html = catalog.load_url( smt_url, os.path.join(catalog.get_current_task_repo(), 'SMT', 'index.html')) if not html: return bs = BeautifulSoup(html, 'html5lib') trs = bs.find('table').findAll('tr') for tr in pbar(trs, task_str): cols = [str(xx.text) for xx in tr.findAll('td')] if not cols: continue name = 'AT' + cols[0] name, source = catalog.new_entry(name, srcname='SMT', url=smt_url) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, cols[1], source) catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, cols[2], source) catalog.entries[name].add_quantity(SUPERNOVA.RA, cols[3], source, u_value='floatdegrees') catalog.entries[name].add_quantity(SUPERNOVA.DEC, cols[4], source, u_value='floatdegrees') if catalog.args.update: catalog.journal_entries() catalog.journal_entries() return
def do_agn_bhm_database(catalog): """Load data from the 'AGN Blackhole Mass Database': 2015PASP..127...67B. FIX: Add archiving flags! """ log = catalog.log log.debug("do_agn_bhm_database()") task_str = catalog.get_current_task_str() # Load data from URL or cached copy of it cached_path = SOURCE_BIBCODE + '.txt' html = catalog.load_url(DATA_URL, cached_path, fail=True) if html is None: return False # Get this line for description of mass calculation # 'M<sub>BH</sub> calculated using <i>< f ></i> =  2.8' mass_scale_factor = re.search("<i>< f ></i> =  (.*)", html).groups() # If pattern matches expectations, store it if len(mass_scale_factor) == 1: mass_scale_factor = mass_scale_factor[0] # Otherwise, forget it else: err_msg = "Could not load `mass_scale_factor` from '{}'".format(mass_scale_factor) catalog.log.error(err_msg) return False soup = BeautifulSoup(html, 'html5lib') # The whole table is nested in a `<table class="hovertable">` full_table = soup.find('table', attrs={'class': 'hovertable'}) # Each line in the file is separated with `'tr'` div_lines = full_table.find_all('tr') # Go through each element of the tables entries = 0 for div in utils.pbar(div_lines, task_str): # Get the `varname` -- ID number for each row # The first element of the `contents` contains an href with the 'varname' cell_text = str(div.contents[0]) groups = re.search('varname=([0-9]*)', cell_text) # If no match is found, this is one of the header lines (not an entry line, skip) if groups is not None: varname = groups.groups()[0] try: name = _add_entry_for_data_line(catalog, div.text, varname, mass_scale_factor) except Exception: log.error("Failed `_add_entry_for_data_line()`") log.error("`div.text`: '{}'".format(div.text)) log.error("`varname`: '{}'".format(varname)) raise if name is not None: entries += 1 if catalog.args.travis and (entries > catalog.TRAVIS_QUERY_LIMIT): log.warning("Exiting on travis limit") break return True
def do_tns(catalog): """Load TNS metadata.""" session = requests.Session() task_str = catalog.get_current_task_str() tns_url = 'https://wis-tns.weizmann.ac.il/' search_url = ( 'https://wis-tns.weizmann.ac.il/' + 'search?&discovered_period_value=30&discovered_period_units=years' + '&unclassified_at=0&classified_sne=0&include_frb=0&name=&name_like=0' + '&isTNS_AT=all&public=all&ra=&decl=&radius=&coords_unit=arcsec&reporting_groupid%5B%5D=null' + '&groupid%5B%5D=null&classifier_groupid%5B%5D=null&objtype%5B%5D=27' + '&at_type%5B%5D=null&date_start%5Bdate%5D=&date_end%5Bdate%5D=&discovery_mag_min=&discovery_mag_max=' + '&internal_name=&discoverer=&classifier=&spectra_count=&redshift_min=&redshift_max=&hostname=' + '&ext_catid=&ra_range_min=&ra_range_max=&decl_range_min=&decl_range_max=' + '&discovery_instrument%5B%5D=null&classification_instrument%5B%5D=null&associated_groups%5B%5D=null' + '&at_rep_remarks=&class_rep_remarks=&frb_repeat=all&frb_repeater_of_objid=&frb_measured_redshift=0' + '&frb_dm_range_min=&frb_dm_range_max=&frb_rm_range_min=&frb_rm_range_max=&frb_snr_range_min=' + '&frb_snr_range_max=&frb_flux_range_min=&frb_flux_range_max=&num_page=500&display%5Bredshift%5D=1' + '&display%5Bhostname%5D=1&display%5Bhost_redshift%5D=1&display%5Bsource_group_name%5D=1' + '&display%5Bclassifying_source_group_name%5D=1&display%5Bdiscovering_instrument_name%5D=0' + '&display%5Bclassifing_instrument_name%5D=0&display%5Bprograms_name%5D=0&display%5Binternal_name%5D=1' + '&display%5BisTNS_AT%5D=0&display%5Bpublic%5D=1&display%5Bend_pop_period%5D=0' + '&display%5Bspectra_count%5D=1&display%5Bdiscoverymag%5D=1&display%5Bdiscmagfilter%5D=1&display' + '%5Bdiscoverydate%5D=1&display%5Bdiscoverer%5D=1&display%5Bremarks%5D=0&display%5Bsources%5D=0' + '&display%5Bbibcode%5D=0&display%5Bext_catalogs%5D=0&format=csv') csvtxt = catalog.load_url(search_url, os.path.join(catalog.get_current_task_repo(), 'TNS', 'index.csv'), timeout=220) data = read(csvtxt, format='csv') for rrow in pbar(data, task_str): row = dict((x, str(rrow[x])) for x in rrow.columns) name = catalog.add_entry(row['Name']) source = catalog.entries[name].add_source(name='Transient Name Server', url=tns_url) if int(float(row['Discovery Mag/Flux'])) >= 8: catalog.entries[name].add_quantity(CATACLYSMIC.MAX_VISUAL_APP_MAG, row['Discovery Mag/Flux'], source) catalog.entries[name].add_quantity(CATACLYSMIC.RA, row['RA'], source) catalog.entries[name].add_quantity(CATACLYSMIC.DEC, row['DEC'], source) catalog.entries[name].add_quantity( CATACLYSMIC.DISCOVER_DATE, row['Discovery Date (UT)'].replace('-', '/'), source) catalog.entries[name].add_quantity(CATACLYSMIC.CLAIMED_TYPE, 'Cataclysmic_Variable', source) catalog.journal_entries()
def do_grb(catalog): task_str = catalog.get_current_task_str() file_path = os.path.join(catalog.get_current_task_repo(), 'GRB-catalog/catalog.csv') csvtxt = catalog.load_url( 'http://www.grbcatalog.com/' 'download_data?cut_0_min=3&cut_0=BAT%20T90' '&cut_0_max=100000&num_cuts=1&no_date_cut=True', file_path) if not csvtxt: return csvtxt = csvtxt.replace('\x0C', '').splitlines() data = list( csv.reader(csvtxt, delimiter=',', quotechar='"', skipinitialspace=True)) for r, row in enumerate(pbar(data, task_str)): if r == 0: continue (name, source) = catalog.new_entry('GRB ' + row[0], srcname='Gamma-ray Bursts Catalog', url='http://www.grbcatalog.com') catalog.entries[name].add_quantity(SUPERNOVA.RA, row[2], source, u_value='floatdegrees') catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[3], source, u_value='floatdegrees') catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, 'LGRB', source) catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, row[8], source) catalog.journal_entries() return
def do_snls_photo(catalog): task_str = catalog.get_current_task_str() snls_path = os.path.join(catalog.get_current_task_repo(), 'SNLS-ugriz.dat') data = list(csv.reader(open(snls_path, 'r'), delimiter=' ', quotechar='"', skipinitialspace=True)) for row in pbar(data, task_str): flux = row[3] err = row[4] # Being extra strict here with the flux constraint, see note below. if float(flux) < 3.0 * float(err): continue name = 'SNLS-' + row[0] name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2010A&A...523A...7G') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) band = row[1] mjd = row[2] sig = get_sig_digits(flux.split('E')[0]) + 1 # Conversion comes from SNLS-Readme # NOTE: Datafiles avail for download suggest diff zeropoints than 30, # need to inquire. magnitude = pretty_num(30.0 - 2.5 * log10(float(flux)), sig=sig) e_mag = pretty_num( 2.5 * log10(1.0 + float(err) / float(flux)), sig=sig) # e_mag = pretty_num(2.5*(log10(float(flux) + float(err)) - # log10(float(flux))), sig=sig) catalog.entries[name].add_photometry( time=mjd, band=band, magnitude=magnitude, e_magnitude=e_mag, counts=flux, e_counts=err, source=source) catalog.journal_entries() return
def load_stubs(self): """ """ currenttask = 'Loading entry stubs' files = self.PATHS.get_repo_output_file_list() for fi in pbar(files, currenttask): fname = fi # FIX: should this be ``fi.endswith(``.gz')`` ? if '.gz' in fi: fname = uncompress_gz(fi) name = os.path.basename( os.path.splitext(fname)[0]).replace('.json', '') new_entry = self.proto.init_from_file( self, path=fname, delete=False) # Make sure a non-stub entry doesnt already exist with this name if name in self.entries and not self.entries[name]._stub: err_str = ( "ERROR: non-stub entry already exists with name '{}'" .format(name)) self.log.error(err_str) raise RuntimeError(err_str) self.entries[name] = new_entry.get_stub() self.log.debug("Added stub for '{}'".format(name)) return self.entries
def do_smt(catalog): task_str = catalog.get_current_task_str() smt_url = 'http://www.mso.anu.edu.au/skymapper/smt/transients/tns/' html = catalog.load_url(smt_url, os.path.join(catalog.get_current_task_repo(), 'SMT', 'index.html')) if not html: return bs = BeautifulSoup(html, 'html5lib') trs = bs.find('table').findAll('tr') for tr in pbar(trs, task_str): cols = [str(xx.text) for xx in tr.findAll('td')] if not cols: continue name = 'AT' + cols[0] name, source = catalog.new_entry(name, srcname='SMT', url=smt_url) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, cols[1], source) catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, cols[2], source) catalog.entries[name].add_quantity( SUPERNOVA.RA, cols[3], source, u_value='floatdegrees') catalog.entries[name].add_quantity( SUPERNOVA.DEC, cols[4], source, u_value='floatdegrees') if catalog.args.update: catalog.journal_entries() catalog.journal_entries() return
def do_grb(catalog): task_str = catalog.get_current_task_str() file_path = os.path.join( catalog.get_current_task_repo(), 'GRB-catalog/catalog.csv') csvtxt = catalog.load_cached_url( 'http://grb.pa.msu.edu/grbcatalog/' 'download_data?cut_0_min=10&cut_0=BAT%20T90' '&cut_0_max=100000&num_cuts=1&no_date_cut=True', file_path) if not csvtxt: return data = list(csv.reader(csvtxt.splitlines(), delimiter=',', quotechar='"', skipinitialspace=True)) for r, row in enumerate(pbar(data, task_str)): if r == 0: continue (name, source) = catalog.new_entry('GRB ' + row[0], srcname='Gamma-ray Bursts Catalog', url='http://grbcatalog.org') catalog.entries[name].add_quantity( SUPERNOVA.RA, row[2], source, u_value='floatdegrees') catalog.entries[name].add_quantity( SUPERNOVA.DEC, row[3], source, u_value='floatdegrees') catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, row[8], source) catalog.journal_entries() return
def do_fermi(catalog): task_str = catalog.get_current_task_str() with open( os.path.join(catalog.get_current_task_repo(), '1SC_catalog_v01.asc'), 'r') as ff: tsvin = list(csv.reader(ff, delimiter=',')) for ri, row in enumerate(pbar(tsvin, task_str)): if row[0].startswith('#'): if len(row) > 1 and 'UPPER_LIMITS' in row[1]: break continue if 'Classified' not in row[1]: continue name = row[0].replace('SNR', 'G') name = catalog.add_entry(name) source = (catalog.entries[name].add_source( bibcode='2016ApJS..224....8A')) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[0].replace('SNR', 'MWSNR'), source) catalog.entries[name].add_quantity(SUPERNOVA.RA, row[2], source, u_value='floatdegrees') catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[3], source, u_value='floatdegrees') catalog.journal_entries() return
def do_itep(catalog): task_str = catalog.get_current_task_str() itepbadsources = ['2004ApJ...602..571B'] needsbib = [] with open(os.path.join(catalog.get_current_task_repo(), 'itep-refs.txt'), 'r') as refs_file: refrep = refs_file.read().splitlines() refrepf = dict(list(zip(refrep[1::2], refrep[::2]))) fname = os.path.join(catalog.get_current_task_repo(), 'itep-lc-cat-28dec2015.txt') tsvin = list(csv.reader(open(fname, 'r'), delimiter='|', skipinitialspace=True)) curname = '' for rr, row in enumerate(pbar(tsvin, task_str)): if rr <= 1 or len(row) < 7: continue oldname = 'SN' + row[0].strip() mjd = str(jd_to_mjd(Decimal(row[1].strip()))) band = row[2].strip() magnitude = row[3].strip() e_magnitude = row[4].strip() reference = row[6].strip().strip(',') if curname != oldname: curname = oldname name = catalog.add_entry(oldname) sec_reference = ('Sternberg Astronomical Institute ' 'Supernova Light Curve Catalogue') sec_refurl = 'http://dau.itep.ru/sn/node/72' sec_source = catalog.entries[name].add_source( name=sec_reference, url=sec_refurl, secondary=True) catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, oldname, sec_source) year = re.findall(r'\d+', name)[0] catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, year, sec_source) if reference in refrepf: bibcode = unescape(refrepf[reference]) source = catalog.entries[name].add_source(bibcode=bibcode) else: needsbib.append(reference) source = catalog.entries[name].add_source( name=reference) if reference else '' if bibcode not in itepbadsources: catalog.entries[name].add_photometry(time=mjd, band=band, magnitude=magnitude, e_magnitude=e_magnitude, source=sec_source + ',' + source) # Write out references that could use aa bibcode needsbib = list(OrderedDict.fromkeys(needsbib)) with open('../itep-needsbib.txt', 'w') as bib_file: bib_file.writelines(['%ss\n' % ii for ii in needsbib]) catalog.journal_entries() return
def do_snhunt(catalog): task_str = catalog.get_current_task_str() snh_url = 'http://nesssi.cacr.caltech.edu/catalina/current.html' html = catalog.load_cached_url(snh_url, os.path.join( catalog.get_current_task_repo(), 'SNhunt/current.html')) if not html: return text = html.splitlines() findtable = False for ri, row in enumerate(text): if 'Supernova Discoveries' in row: findtable = True if findtable and '<table' in row: tstart = ri + 1 if findtable and '</table>' in row: tend = ri - 1 tablestr = '<html><body><table>' for row in text[tstart:tend]: if row[:3] == 'tr>': tablestr = tablestr + '<tr>' + row[3:] else: tablestr = tablestr + row tablestr = tablestr + '</table></body></html>' bs = BeautifulSoup(tablestr, 'html5lib') trs = bs.find('table').findAll('tr') for tr in pbar(trs, task_str): cols = [str(xx.text) for xx in tr.findAll('td')] if not cols: continue name = re.sub('<[^<]+?>', '', cols[4] ).strip().replace(' ', '').replace('SNHunt', 'SNhunt') name = catalog.add_entry(name) source = catalog.entries[name].add_source( name='Supernova Hunt', url=snh_url) catalog.entries[name].add_quantity('alias', name, source) host = re.sub('<[^<]+?>', '', cols[1]).strip().replace('_', ' ') catalog.entries[name].add_quantity('host', host, source) catalog.entries[name].add_quantity( 'ra', cols[2], source, u_value='floatdegrees') catalog.entries[name].add_quantity( 'dec', cols[3], source, u_value='floatdegrees') dd = cols[0] discoverdate = dd[:4] + '/' + dd[4:6] + '/' + dd[6:8] catalog.entries[name].add_quantity( 'discoverdate', discoverdate, source) discoverers = cols[5].split('/') for discoverer in discoverers: catalog.entries[name].add_quantity('discoverer', 'CRTS', source) catalog.entries[name].add_quantity( 'discoverer', discoverer, source) if catalog.args.update: catalog.journal_entries() catalog.journal_entries() return
def do_simbad_novae(catalog): task_str = catalog.get_current_task_str() simbad_mirrors = ['http://simbad.harvard.edu/simbad/sim-script', 'http://simbad.u-strasbg.fr/simbad/sim-script'] customSimbad = Simbad() customSimbad.ROW_LIMIT = -1 customSimbad.TIMEOUT = 120 for mirror in simbad_mirrors: customSimbad.SIMBAD_URL = mirror try: table = customSimbad.query_criteria('maintype=No* | maintype="No?"') except: continue else: break if not table: catalog.log.warning('SIMBAD unable to load, probably offline.') for name in pbar(catalog.entries, task_str): try: nova_name = "V* " + get_nova_name(name) aliases = customSimbad.query_objectids(nova_name) except: #THROW WARNING HERE tprint("Could not find " + nova_name) continue table = customSimbad.query_object(nova_name) name = catalog.add_entry(name) bibcode = table[0]['COO_BIBCODE'].decode() ra = str(table[0]['RA']) dec = str(table[0]['DEC']) source = catalog.entries[name].add_source(name='SIMBAD astronomical database', bibcode=bibcode, url="http://simbad.u-strasbg.fr/", secondary=True) catalog.entries[name].add_quantity(NOVA.RA, ra, source) catalog.entries[name].add_quantity(NOVA.DEC, dec, source) for i in range(len(aliases)): try: alias = aliases[i][0].decode() except: alias = str(aliases[i][0]) catalog.entries[name].add_quantity(NOVA.ALIAS, alias, source) catalog.journal_entries()
def delete_old_entry_files(self): if len(self.entries): err_str = "`delete_old_entry_files` with `entries` not empty!" self.log.error(err_str) raise RuntimeError(err_str) # Delete all old entry JSON files repo_files = self.PATHS.get_repo_output_file_list() for rfil in pbar(repo_files, desc='Deleting old entries'): os.remove(rfil) self.log.debug("Deleted '{}'".format(os.path.split(rfil)[-1])) return
def do_gaiaviatap(catalog): # Disable warnings warnings.filterwarnings("ignore") #catalog.log.warning( # 'Some warnings are thrown by the Gaia module which do not affect the result of the Gaia queries.') task_str = catalog.get_current_task_str() keys = list(catalog.entries.keys()) cntL = 0 for oname in pbar(keys, task_str): try: name = catalog.add_entry(oname) except Exception: catalog.log.warning( '"{}" was not found, suggests merge occurred in cleanup ' 'process.'.format(oname)) continue ### Check if has DR2 ID dr2id = '' for alias in catalog.entries[name][FASTSTARS.ALIAS]: if alias['value'][:8] == 'Gaia DR2' and len( alias['value']) > len(dr2id): dr2id = alias['value'] if len(dr2id) < 8 or FASTSTARS.PARALLAX not in catalog.entries[name]: #print(name) continue else: with HiddenPrints(): source = catalog.entries[name].add_source( bibcode='2018arXiv180410121B') job = silentari( "SELECT r_len FROM gaiadr2_complements.geometric_distance where source_id = " + dr2id[8:], dump_to_file=False, verbose=False) lengthscale = str(job.get_results()['col_0'][0] / 1e3) catalog.entries[name].add_quantity( FASTSTARS.DISTANCE_PRIOR_LENGTH_SCALE, lengthscale, source, u_value='kpc') cntL += 1 catalog.log.warning( '"{}" have a more accurate scale length from Bailer-Jones et al. (2018).' .format(cntL)) catalog.journal_entries() # Reactivate warnings warnings.filterwarnings("default") return
def do_ppmxl(catalog): task_str = catalog.get_current_task_str() keys = list(catalog.entries.keys()) warnings.filterwarnings("ignore") for oname in pbar(keys, task_str): # Some events may be merged in cleanup process, skip them if # non-existent. try: name = catalog.add_entry(oname) except Exception: catalog.log.warning( '"{}" was not found, suggests merge occurred in cleanup ' 'process.'.format(oname)) continue if (FASTSTARS.RA not in catalog.entries[name] or FASTSTARS.DEC not in catalog.entries[name]): continue else: radec= str(catalog.entries[name][FASTSTARS.RA][0]['value'])+str(catalog.entries[name][FASTSTARS.DEC][0]['value']) c=coord(radec,unit=(un.hourangle, un.deg),frame='icrs') cnttry = 0 foundstar = False while foundstar == False and cnttry < 1: try: cnttry += 1 time.sleep(0.1) result = Irsa.query_region(c,catalog='ppmxl',radius='0d0m10s') except TypeError: #print(radec,cnttry) continue if len(result) > 1: foundstar = True if foundstar == True: source = (catalog.entries[name] .add_source(name='The PPMXL Catalog', bibcode="2010AJ....139.2440R", url="https://irsa.ipac.caltech.edu/Missions/ppmxl.html", secondary=True)) catalog.entries[name].add_quantity(FASTSTARS.PROPER_MOTION_RA, str(result['pmra'][0]*degperyrtomasperyear), source, e_value=str(result['e_pmra'][0]*degperyrtomasperyear), u_value='mas/yr') catalog.entries[name].add_quantity(FASTSTARS.PROPER_MOTION_DEC, str(result['pmde'][0]*degperyrtomasperyear), source, e_value=str(result['e_pmde'][0]*degperyrtomasperyear), u_value='mas/yr') catalog.journal_entries() return
def do_ptss_meta(catalog): """Import metadata from PTSS webpage.""" task_str = catalog.get_current_task_str() years = list(range(2015, datetime.today().year + 1)) for year in years: jsontxt = None while jsontxt is None: try: jsontxt = catalog.load_url( 'http://www.cneost.org/ptss/fetchlist.php?vip=sn&gdate=' + str(year), os.path.join(catalog.get_current_task_repo(), 'PTSS/catalog-' + str(year) + '.json'), json_sort='name', timeout=5) except Exception: pass meta = json.loads(jsontxt) for met in pbar(meta, task_str + ' - ' + str(year)): oldname = met['name'] name, source = catalog.new_entry( oldname, srcname='PMO & Tsinghua Supernova Survey (PTSS)', url='http://www.cneost.org/ptss/index.php') coo = coord(met['ra'], met['dec'], unit=(un.deg, un.deg)) catalog.entries[name].add_quantity( SUPERNOVA.RA, coo.ra.to_string(unit=un.hour, sep=':'), source) catalog.entries[name].add_quantity( SUPERNOVA.DEC, coo.dec.to_string(unit=un.degree, sep=':'), source) if met['filter'] is not None: mjd = str(astrotime(met['obsdate'], format='isot').mjd) photodict = { PHOTOMETRY.TIME: mjd, PHOTOMETRY.MAGNITUDE: str(met['mag']), PHOTOMETRY.E_MAGNITUDE: str(met['magerr']), PHOTOMETRY.BAND: met['filter'].replace('sdss-', ''), PHOTOMETRY.SOURCE: source } catalog.entries[name].add_photometry(**photodict) catalog.journal_entries() return
def set_preferred_names(self): """Choose between each entries given name and its possible aliases for the best one. """ if len(self.entries) == 0: self.log.error("WARNING: `entries` is empty, loading stubs") self.load_stubs() task_str = self.get_current_task_str() for ni, oname in enumerate(pbar(self.entries, task_str)): name = self.add_entry(oname) self.entries[name].set_preferred_name() if self.args.travis and ni > self.TRAVIS_QUERY_LIMIT: break return
def do_ptss_meta(catalog): """Import metadata from PTSS webpage.""" task_str = catalog.get_current_task_str() years = list(range(2015, datetime.today().year + 1)) for year in years: jsontxt = None while jsontxt is None: try: jsontxt = catalog.load_url( 'http://www.cneost.org/ptss/fetchlist.php?vip=sn&gdate=' + str(year), os.path.join(catalog.get_current_task_repo(), 'PTSS/catalog-' + str(year) + '.json'), json_sort='name', timeout=5) except Exception: pass meta = json.loads(jsontxt) for met in pbar(meta, task_str + ' - ' + str(year)): oldname = met['name'] name, source = catalog.new_entry( oldname, srcname='PMO & Tsinghua Supernova Survey (PTSS)', url='http://www.cneost.org/ptss/index.php') coo = coord(met['ra'], met['dec'], unit=(un.deg, un.deg)) catalog.entries[name].add_quantity( SUPERNOVA.RA, coo.ra.to_string(unit=un.hour, sep=':'), source) catalog.entries[name].add_quantity( SUPERNOVA.DEC, coo.dec.to_string(unit=un.degree, sep=':'), source) if met['filter'] is not None: mjd = str(astrotime(met['obsdate'], format='isot').mjd) photodict = { PHOTOMETRY.TIME: mjd, PHOTOMETRY.MAGNITUDE: str(met['mag']), PHOTOMETRY.E_MAGNITUDE: str(met['magerr']), PHOTOMETRY.BAND: met['filter'].replace('sdss-', ''), PHOTOMETRY.SOURCE: source } catalog.entries[name].add_photometry(**photodict) catalog.journal_entries() return
def do_dust(catalog): task_str = catalog.get_current_task_str() # Set preferred names, calculate some columns based on imported data, # sanitize some fields keys = list(catalog.entries.keys()) check_dustmaps(catalog.get_current_task_repo()) sfd = SFDQuery() for oname in pbar(keys, task_str): # Some events may be merged in cleanup process, skip them if # non-existent. try: name = catalog.add_entry(oname) except Exception: catalog.log.warning( '"{}" was not found, suggests merge occurred in cleanup ' 'process.'.format(oname)) continue if (FASTSTARS.RA not in catalog.entries[name] or FASTSTARS.DEC not in catalog.entries[name]): continue else: Mname = name Mradec = str( catalog.entries[name][FASTSTARS.RA][0]['value']) + str( catalog.entries[name][FASTSTARS.DEC][0]['value']) Mdist = '-1' c = coord(Mradec, unit=(un.hourangle, un.deg), frame='icrs') reddening = sfd(c) source = catalog.entries[name].add_source( bibcode='1998ApJ...500..525S') catalog.entries[name].add_quantity(FASTSTARS.EBV, str(reddening), source, upperlimit=True, derived=True) catalog.journal_entries() return
def do_hst(catalog): task_str = catalog.get_current_task_str() url = 'http://archive.stsci.edu/hst/search.php' reference = 'Hubble Pointings' jtxt = catalog.load_url( url, os.path.join(catalog.get_current_task_repo(), 'HST.json'), post={ 'sci_target_descrip': '*supernova*', 'outputformat': 'JSON_file', 'action': 'Search', 'max_records': '50000', 'max_rpp': '50000' }, verify=False) rows = json.loads(jtxt) allowed_prefixes = ('PS1', 'DES', 'GAIA', 'ASASSN', 'AT', 'IPTF', 'LSQ', 'PTF') loopcnt = 0 for row in pbar(rows, task_str): oldname = name_clean(row['Target Name']) if not oldname.upper().startswith(allowed_prefixes): continue if oldname.startswith('PS1-') and not is_number(oldname[4]): continue name, source = catalog.new_entry(oldname, srcname=reference, url=url) if (ENTRY.RA in catalog.entries[name] and ENTRY.DEC in catalog.entries[name]): continue catalog.entries[name].add_quantity( ENTRY.RA, row['RA (J2000)'], source=source) catalog.entries[name].add_quantity( ENTRY.DEC, row['Dec (J2000)'], source=source) catalog.journal_entries() loopcnt = loopcnt + 1 if (catalog.args.travis and loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0): break catalog.journal_entries() return
def do_pessto(catalog): task_str = catalog.get_current_task_str() pessto_path = os.path.join( catalog.get_current_task_repo(), 'PESSTO_MPHOT.csv') tsvin = list(csv.reader(open(pessto_path, 'r'), delimiter=',')) for ri, row in enumerate(pbar(tsvin, task_str)): if ri == 0: bands = [xx.split('_')[0] for xx in row[3::2]] systems = [xx.split('_')[1].capitalize().replace( 'Ab', 'AB') for xx in row[3::2]] continue name = row[1] name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2015A&A...579A..40S') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) for hi, ci in enumerate(range(3, len(row) - 1, 2)): if not row[ci]: continue if systems[hi] == 'Swift': teles = 'Swift' instrument = 'UVOT' bandset = 'Swift' else: teles = 'NTT' instrument = 'EFOSC' bandset = 'Johnson' photodict = { PHOTOMETRY.TIME: row[2], PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.MAGNITUDE: row[ci], PHOTOMETRY.E_MAGNITUDE: row[ci + 1], PHOTOMETRY.BAND: bands[hi], PHOTOMETRY.SYSTEM: systems[hi], PHOTOMETRY.BAND_SET: bandset, PHOTOMETRY.TELESCOPE: teles, PHOTOMETRY.INSTRUMENT: instrument, PHOTOMETRY.SOURCE: source } catalog.entries[name].add_photometry(**photodict) catalog.journal_entries() return
def do_scp(catalog): task_str = catalog.get_current_task_str() tsvin = list( csv.reader(open( os.path.join(catalog.get_current_task_repo(), 'SCP09.csv'), 'r'), delimiter=',')) for ri, row in enumerate(pbar(tsvin, task_str)): if ri == 0: continue name = row[0].replace('SCP', 'SCP-') name = catalog.add_entry(name) source = (catalog.entries[name].add_source( name='Supernova Cosmology Project', url=('http://supernova.lbl.gov/' '2009ClusterSurvey/'))) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) if row[1]: catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[1], source) if row[2]: kind = 'spectroscopic' if row[3] == 'sn' else SUPERNOVA.HOST catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, row[2], source, kind=kind) if row[4]: catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, row[2], source, kind='cluster') if row[6]: claimedtype = row[6].replace('SN ', '') kind = ('spectroscopic/light curve' if 'a' in row[7] and 'c' in row[7] else 'spectroscopic' if 'a' in row[7] else 'light curve' if 'c' in row[7] else '') if claimedtype != '?': catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, claimedtype, source, kind=kind) catalog.journal_entries() return
def do_snls_photo(catalog): task_str = catalog.get_current_task_str() snls_path = os.path.join(catalog.get_current_task_repo(), 'SNLS-ugriz.dat') data = list( csv.reader( open(snls_path, 'r'), delimiter=' ', quotechar='"', skipinitialspace=True)) for row in pbar(data, task_str): counts = row[3] err = row[4] name = 'SNLS-' + row[0] name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2010A&A...523A...7G') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) # Conversion comes from SNLS-Readme # NOTE: Datafiles avail for download suggest diff zeropoints than 30, # but README states mags should be calculated assuming 30. Need to # inquire. zp = 30.0 photodict = { PHOTOMETRY.TIME: row[2], PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.BAND: row[1], PHOTOMETRY.COUNT_RATE: counts, PHOTOMETRY.E_COUNT_RATE: err, PHOTOMETRY.ZERO_POINT: str(zp), PHOTOMETRY.SOURCE: source, PHOTOMETRY.TELESCOPE: 'CFHT', PHOTOMETRY.INSTRUMENT: 'MegaCam', PHOTOMETRY.BAND_SET: 'MegaCam', PHOTOMETRY.SYSTEM: 'BD17' } set_pd_mag_from_counts(photodict, counts, ec=err, zp=zp, sig=5.0) catalog.entries[name].add_photometry(**photodict) catalog.journal_entries() return
def do_snls_photo(catalog): task_str = catalog.get_current_task_str() snls_path = os.path.join(catalog.get_current_task_repo(), 'SNLS-ugriz.dat') data = list( csv.reader(open(snls_path, 'r'), delimiter=' ', quotechar='"', skipinitialspace=True)) for row in pbar(data, task_str): counts = row[3] err = row[4] name = 'SNLS-' + row[0] name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2010A&A...523A...7G') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) # Conversion comes from SNLS-Readme # NOTE: Datafiles avail for download suggest diff zeropoints than 30, # but README states mags should be calculated assuming 30. Need to # inquire. zp = 30.0 photodict = { PHOTOMETRY.TIME: row[2], PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.BAND: row[1], PHOTOMETRY.COUNT_RATE: counts, PHOTOMETRY.E_COUNT_RATE: err, PHOTOMETRY.ZERO_POINT: str(zp), PHOTOMETRY.SOURCE: source, PHOTOMETRY.TELESCOPE: 'CFHT', PHOTOMETRY.INSTRUMENT: 'MegaCam', PHOTOMETRY.BAND_SET: 'MegaCam', PHOTOMETRY.SYSTEM: 'BD17' } set_pd_mag_from_counts(photodict, counts, ec=err, zp=zp, sig=5.0) catalog.entries[name].add_photometry(**photodict) catalog.journal_entries() return
def do_scp(catalog): task_str = catalog.get_current_task_str() tsvin = list(csv.reader(open( os.path.join(catalog.get_current_task_repo(), 'SCP09.csv'), 'r'), delimiter=',')) for ri, row in enumerate(pbar(tsvin, task_str)): if ri == 0: continue name = row[0].replace('SCP', 'SCP-') name = catalog.add_entry(name) source = (catalog.entries[name] .add_source(name='Supernova Cosmology Project', url=('http://supernova.lbl.gov/' '2009ClusterSurvey/'))) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) if row[1]: catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[1], source) if row[2]: kind = 'spectroscopic' if row[3] == 'sn' else SUPERNOVA.HOST catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, row[2], source, kind=kind) if row[4]: catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, row[2], source, kind='cluster') if row[6]: claimedtype = row[6].replace('SN ', '') kind = ('spectroscopic/light curve' if 'a' in row[7] and 'c' in row[7] else 'spectroscopic' if 'a' in row[7] else 'light curve' if 'c' in row[7] else '') if claimedtype != '?': catalog.entries[name].add_quantity( SUPERNOVA.CLAIMED_TYPE, claimedtype, source, kind=kind) catalog.journal_entries() return
def merge_duplicates(self): """Merge and remove duplicate entries. Compares each entry ('name') in `stubs` to all later entries to check for duplicates in name or alias. If a duplicate is found, they are merged and written to file. """ if len(self.entries) == 0: self.log.error("WARNING: `entries` is empty, loading stubs") if self.args.update: self.log.warning( "No sources changed, entry files unchanged in update." " Skipping merge.") return self.entries = self.load_stubs() task_str = self.get_current_task_str() keys = list(sorted(self.entries.keys())) for n1, name1 in enumerate(pbar(keys, task_str)): allnames1 = set(self.entries[name1].get_aliases()) if name1.startswith('SN') and is_number(name1[2:6]): allnames1 = allnames1.union(['AT' + name1[2:]]) # Search all later names for name2 in keys[n1 + 1:]: if name1 == name2: continue allnames2 = set(self.entries[name2].get_aliases()) if name2.startswith('SN') and is_number(name2[2:6]): allnames2.union(['AT' + name2[2:]]) # If there are any common names or aliases, merge if len(allnames1 & allnames2): self.log.warning( "Found single entry with multiple entries " "('{}' and '{}'), merging.".format(name1, name2)) load1 = self.proto.init_from_file( self, name=name1) load2 = self.proto.init_from_file( self, name=name2) if load1 is not None and load2 is not None: # Delete old files self._delete_entry_file(entry=load1) self._delete_entry_file(entry=load2) self.entries[name1] = load1 self.entries[name2] = load2 priority1 = 0 priority2 = 0 for an in allnames1: if an.startswith(('SN', 'AT')): priority1 += 1 for an in allnames2: if an.startswith(('SN', 'AT')): priority2 += 1 if priority1 > priority2: self.copy_to_entry(name2, name1) keys.append(name1) del self.entries[name2] else: self.copy_to_entry(name1, name2) keys.append(name2) del self.entries[name1] else: self.log.warning('Duplicate already deleted') # if len(self.entries) != 1: # self.log.error( # "WARNING: len(entries) = {}, expected 1. " # "Still journaling...".format(len(self.entries))) self.journal_entries() if self.args.travis and n1 > self.TRAVIS_QUERY_LIMIT: break
def do_crts(catalog): """Import data from the Catalina Real-Time Transient Survey.""" crtsnameerrors = ['2011ax'] task_str = catalog.get_current_task_str() folders = ['catalina', 'MLS', 'MLS', 'SSS'] files = ['AllSN.html', 'AllSN.arch.html', 'CRTSII_SN.html', 'AllSN.html'] for fi, fold in enumerate(pbar(folders, task_str)): html = catalog.load_url( 'http://nesssi.cacr.caltech.edu/' + fold + '/' + files[fi], os.path.join(catalog.get_current_task_repo(), 'CRTS', fold + '-' + files[fi]), archived_mode=('arch' in files[fi])) html = html.replace('<ahref=', '<a href=') if not html: continue bs = BeautifulSoup(html, 'html5lib') trs = bs.findAll('tr') for tri, tr in enumerate(pbar(trs, task_str)): tds = tr.findAll('td') if not tds: continue # refs = [] aliases = [] crtsname = '' ra = '' dec = '' lclink = '' # ttype = '' # ctype = '' for tdi, td in enumerate(tds): if tdi == 0: crtsname = td.contents[0].text.strip() elif tdi == 1: ra = td.contents[0] elif tdi == 2: dec = td.contents[0] elif tdi == (8 if files[fi] == 'CRTSII_SN.html' else 11): lclink = td.find('a')['onclick'] lclink = lclink.split("'")[1] elif tdi == (10 if files[fi] == 'CRTSII_SN.html' else 13): aliases = re.sub('[()]', '', re.sub( '<[^<]+?>', '', td.contents[-1].strip())) aliases = [xx.strip('; ') for xx in list( filter(None, aliases.split(' ')))] name = '' hostmag = '' hostupper = False validaliases = [] for ai, alias in enumerate(aliases): if alias in ['SN', 'SDSS']: continue if alias in crtsnameerrors: continue if alias == 'mag': if ai < len(aliases) - 1: ind = ai + 1 if aliases[ai + 1] in ['SDSS']: ind = ai + 2 elif aliases[ai + 1] in ['gal', 'obj', 'object', 'source']: ind = ai - 1 if '>' in aliases[ind]: hostupper = True hostmag = aliases[ind].strip('>~').replace( ',', '.').replace('m', '.') continue if (is_number(alias[:4]) and alias[:2] == '20' and len(alias) > 4): name = 'SN' + alias if ((('asassn' in alias and len(alias) > 6) or ('ptf' in alias and len(alias) > 3) or ('ps1' in alias and len(alias) > 3) or 'snhunt' in alias or ('mls' in alias and len(alias) > 3) or 'gaia' in alias or ('lsq' in alias and len(alias) > 3))): alias = alias.replace('SNHunt', 'SNhunt') validaliases.append(alias) if not name: name = crtsname name, source = catalog.new_entry( name, srcname='Catalina Sky Survey', bibcode='2009ApJ...696..870D', url='http://nesssi.cacr.caltech.edu/catalina/AllSN.html') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) for alias in validaliases: catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, alias, source) catalog.entries[name].add_quantity( SUPERNOVA.RA, ra.strip(), source, u_value='floatdegrees') catalog.entries[name].add_quantity( SUPERNOVA.DEC, dec.strip(), source, u_value='floatdegrees') if SUPERNOVA.CLAIMED_TYPE not in catalog.entries[name]: catalog.entries[name].add_quantity( SUPERNOVA.CLAIMED_TYPE, 'Candidate', source) if hostmag: # 1.0 magnitude error based on Drake 2009 assertion that SN are # only considered # real if they are 2 mags brighter than host. photodict = { PHOTOMETRY.BAND: 'C', PHOTOMETRY.MAGNITUDE: hostmag, PHOTOMETRY.E_MAGNITUDE: '1.0', PHOTOMETRY.SOURCE: source, PHOTOMETRY.HOST: True, PHOTOMETRY.TELESCOPE: 'Catalina Schmidt', PHOTOMETRY.UPPER_LIMIT: hostupper } catalog.entries[name].add_photometry(**photodict) fname2 = (catalog.get_current_task_repo() + '/' + fold + '/' + lclink.split('.')[-2].rstrip('p').split('/')[-1] + '.html') html2 = catalog.load_url(lclink, fname2) if not html2: continue lines = html2.splitlines() teles = 'Catalina Schmidt' for line in lines: if 'javascript:showx' in line: search = re.search("showx\('(.*?)'\)", line) if not search: continue mjdstr = search.group(1).split('(')[0].strip() if not is_number(mjdstr): continue mjd = str(Decimal(mjdstr) + Decimal(53249.0)) else: continue mag = '' err = '' if 'javascript:showy' in line: mag = re.search("showy\('(.*?)'\)", line).group(1) if 'javascript:showz' in line: err = re.search("showz\('(.*?)'\)", line).group(1) if not is_number(mag) or (err and not is_number(err)): continue photodict = { PHOTOMETRY.TIME: mjd, PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.E_TIME: '0.125', # 3 hr error PHOTOMETRY.BAND: 'C', PHOTOMETRY.MAGNITUDE: mag, PHOTOMETRY.SOURCE: source, PHOTOMETRY.INCLUDES_HOST: True, PHOTOMETRY.TELESCOPE: teles } if float(err) > 0.0: photodict[PHOTOMETRY.E_MAGNITUDE] = err if float(err) == 0.0: photodict[PHOTOMETRY.UPPER_LIMIT] = True catalog.entries[name].add_photometry(**photodict) if catalog.args.update: catalog.journal_entries() if catalog.args.travis and tri > catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() return
def do_ptf(catalog): # response = # urllib.request.urlopen('http://wiserep.weizmann.ac.il/objects/list') # bs = BeautifulSoup(response, 'html5lib') # select = bs.find('select', {'name': 'objid'}) # options = select.findAll('option') # for option in options: # print(option.text) # name = option.text # if ((name.startswith('PTF') and is_number(name[3:5])) or # name.startswith('PTFS') or name.startswith('iPTF')): # name = catalog.add_entry(name) task_str = catalog.get_current_task_str() if catalog.current_task.load_archive(catalog.args): with open(os.path.join(catalog.get_current_task_repo(), 'PTF/update.html'), 'r') as f: html = f.read() else: session = requests.Session() response = session.get('http://wiserep.weizmann.ac.il/spectra/update') html = response.text with open(os.path.join(catalog.get_current_task_repo(), 'PTF/update.html'), 'w') as f: f.write(html) bs = BeautifulSoup(html, 'html5lib') select = bs.find('select', {'name': 'objid'}) options = select.findAll('option') for option in pbar(options, task_str): name = option.text if (((name.startswith('PTF') and is_number(name[3:5])) or name.startswith('PTFS') or name.startswith('iPTF'))): if '(' in name: alias = name.split('(')[0].strip(' ') name = name.split('(')[-1].strip(') ').replace('sn', 'SN') name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2012PASP..124..668Y') catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, alias, source) else: # name = catalog.add_entry(name) name, source = catalog.new_entry(name, bibcode='2012PASP..124..668Y') with open(os.path.join( catalog.get_current_task_repo(), 'PTF/old-ptf-events.csv')) as f: for suffix in pbar(f.read().splitlines(), task_str): name = catalog.add_entry('PTF' + suffix) with open(os.path.join( catalog.get_current_task_repo(), 'PTF/perly-2016.csv')) as f: for row in pbar(f.read().splitlines(), task_str): cols = [x.strip() for x in row.split(',')] alias = '' if cols[8]: name = cols[8] alias = 'PTF' + cols[0] else: name = 'PTF' + cols[0] name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2016arXiv160408207P') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) if alias: catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, alias, source) catalog.entries[name].add_quantity(SUPERNOVA.RA, cols[1], source) catalog.entries[name].add_quantity(SUPERNOVA.DEC, cols[2], source) catalog.entries[name].add_quantity( SUPERNOVA.CLAIMED_TYPE, 'SLSN-' + cols[3], source) catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, cols[4], source, kind='spectroscopic') maxdate = cols[6].replace('-', '/') upl = maxdate.startswith('<') catalog.entries[name].add_quantity( SUPERNOVA.MAX_DATE, maxdate.lstrip('<'), source, upperlimit=upl) catalog.entries[name].add_quantity( SUPERNOVA.EBV, cols[7], source, kind='spectroscopic') name = catalog.add_entry('PTF' + suffix) catalog.journal_entries() return
def do_cfa_photo(catalog): """Import photometry from the CfA archive.""" from html import unescape import re task_str = catalog.get_current_task_str() file_names = glob( os.path.join(catalog.get_current_task_repo(), 'cfa-input/*.dat')) for fname in pbar_strings(file_names, task_str): f = open(fname, 'r') tsvin = csv.reader(f, delimiter=' ', skipinitialspace=True) csv_data = [] for r, row in enumerate(tsvin): new = [] for item in row: new.extend(item.split('\t')) csv_data.append(new) for r, row in enumerate(csv_data): for c, col in enumerate(row): csv_data[r][c] = col.strip() csv_data[r] = [_f for _f in csv_data[r] if _f] eventname = os.path.basename(os.path.splitext(fname)[0]) eventparts = eventname.split('_') name = clean_snname(eventparts[0]) name = catalog.add_entry(name) secondaryname = 'CfA Supernova Archive' secondaryurl = 'https://www.cfa.harvard.edu/supernova/SNarchive.html' secondarysource = catalog.entries[name].add_source( name=secondaryname, url=secondaryurl, secondary=True, acknowledgment=ACKN_CFA) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, secondarysource) year = re.findall(r'\d+', name)[0] catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, year, secondarysource) eventbands = list(eventparts[1]) tu = 'MJD' jdoffset = Decimal(0.) for rc, row in enumerate(csv_data): if len(row) > 0 and row[0][0] == "#": if len(row[0]) > 2 and row[0][:3] == '#JD': tu = 'JD' rowparts = row[0].split('-') jdoffset = Decimal(rowparts[1]) elif len(row[0]) > 6 and row[0][:7] == '#Julian': tu = 'JD' jdoffset = Decimal(0.) elif len(row) > 1 and row[1].lower() == 'photometry': for ci, col in enumerate(row[2:]): if col[0] == "(": refstr = ' '.join(row[2 + ci:]) refstr = refstr.replace('(', '').replace(')', '') bibcode = unescape(refstr) source = catalog.entries[name].add_source( bibcode=bibcode) elif len(row) > 1 and row[1] == 'HJD': tu = 'HJD' continue elif len(row) > 0: mjd = row[0] for v, val in enumerate(row): if v == 0: if tu == 'JD': mjd = str(jd_to_mjd(Decimal(val) + jdoffset)) tuout = 'MJD' elif tu == 'HJD': mjd = str(jd_to_mjd(Decimal(val))) tuout = 'MJD' else: mjd = val tuout = tu elif v % 2 != 0: if float(row[v]) < 90.0: src = secondarysource + ',' + source photodict = { PHOTOMETRY.U_TIME: tuout, PHOTOMETRY.TIME: mjd, PHOTOMETRY.BAND_SET: 'Standard', PHOTOMETRY.BAND: eventbands[(v - 1) // 2], PHOTOMETRY.MAGNITUDE: row[v], PHOTOMETRY.E_MAGNITUDE: row[v + 1], PHOTOMETRY.SOURCE: src } catalog.entries[name].add_photometry(**photodict) f.close() # Hicken 2012 with open( os.path.join(catalog.get_current_task_repo(), 'hicken-2012-standard.dat'), 'r') as infile: tsvin = list(csv.reader(infile, delimiter='|', skipinitialspace=True)) for r, row in enumerate(pbar(tsvin, task_str)): if r <= 47: continue if row[0][:2] != 'sn': name = 'SN' + row[0].strip() else: name = row[0].strip() name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2012ApJS..200...12H') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, 'Ia', source) photodict = { PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.TIME: row[2].strip(), PHOTOMETRY.BAND: row[1].strip(), PHOTOMETRY.BAND_SET: 'Standard', PHOTOMETRY.MAGNITUDE: row[6].strip(), PHOTOMETRY.E_MAGNITUDE: row[7].strip(), PHOTOMETRY.SOURCE: source } catalog.entries[name].add_photometry(**photodict) # Bianco 2014 with open( os.path.join(catalog.get_current_task_repo(), 'bianco-2014-standard.dat'), 'r') as infile: tsvin = list(csv.reader(infile, delimiter=' ', skipinitialspace=True)) for row in pbar(tsvin, task_str): name = 'SN' + row[0] name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2014ApJS..213...19B') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) photodict = { PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.TIME: row[2], PHOTOMETRY.BAND: row[1], PHOTOMETRY.MAGNITUDE: row[3], PHOTOMETRY.E_MAGNITUDE: row[4], PHOTOMETRY.TELESCOPE: row[5], PHOTOMETRY.BAND_SET: 'Standard', PHOTOMETRY.SOURCE: source } catalog.entries[name].add_photometry(**photodict) catalog.journal_entries() return
def do_asassn(catalog): """Import list of ASASSN events.""" task_str = catalog.get_current_task_str() asn_url = 'http://www.astronomy.ohio-state.edu/~assassin/transients.html' html = catalog.load_url( asn_url, os.path.join(catalog.get_current_task_repo(), 'ASASSN/transients.html')) if not html: return bs = BeautifulSoup(html, 'html5lib') trs = bs.find('table').findAll('tr') for tri, tr in enumerate(pbar(trs, task_str)): name = '' ra = '' dec = '' discdate = '' mag = '' atellink = '' sdsslink = '' comment = '' #store to check if entry is a CV later if tri == 1: continue tds = tr.findAll('td') atex = 0 for tdi, td in enumerate(tds): if tdi == 0 and td.text.strip() != '---' and td.text.strip( ) != '' and td.text.strip() != '----': try: # name = catalog.add_entry(td.text.strip().replace("?","_")) star_name = td.text.strip().replace("?", "_") # atellink = td.find('a') atex = 1 # if atellink: # atellink = atellink['href'] # else: # atellink = '' except ValueError: pass if tdi == 1 and td.text.strip() != '---': if atex == 0: # name = catalog.add_entry(td.text.strip().replace("?","_").replace(':','_')) star_name = td.text.strip().replace("?", "_").replace(":", "_") # atellink = td.find('a') # if atellink: # atellink = atellink['href'] # else: # atellink = '' else: pass if tdi == 3: ra = td.text if tdi == 4: dec = td.text if tdi == 5: discdate = td.text.replace('-', '/') if not discdate.startswith('2') or discdate.startswith('1'): discdate = '' # if discdate == 'bc094//0': # discdate = '' #ASASSN-15co was having bc094--0 as a date if tdi == 6: mag_place = td.text if td.text == '' or mag_place[-1:] == ')': mag = td.text elif int(float(td.text)) >= 8: mag = td.text if tdi == 7: sdsslink = td.find('a') if sdsslink: sdsslink = sdsslink['href'] else: sdsslink = '' if tdi == 11: comment = td.text if 'CV' in comment: name = catalog.add_entry(star_name) sources = [ catalog.entries[name].add_source(name='ASAS-CV Transients', bibcode="2014ApJ...788...48S", url=asn_url) ] typesources = sources[:] if atellink: sources.append((catalog.entries[name].add_source( name='ATel ' + atellink.split('=')[-1], url=atellink))) # if typelink: # typesources.append( # (catalog.entries[name] # .add_source(name='ATel ' + # typelink.split('=')[-1], url=typelink))) sources = ','.join(sources) typesources = ','.join(typesources) catalog.entries[name].add_quantity(CATACLYSMIC.ALIAS, name, sources) catalog.entries[name].add_quantity(CATACLYSMIC.DISCOVER_DATE, discdate, sources) catalog.entries[name].add_quantity(CATACLYSMIC.RA, ra, sources, u_value='floatdegrees') catalog.entries[name].add_quantity(CATACLYSMIC.DEC, dec, sources, u_value='floatdegrees') catalog.entries[name].add_quantity(CATACLYSMIC.MAX_VISUAL_APP_MAG, mag, sources) if 'known CV' in comment and not 'known CV candidate' in comment: catalog.entries[name].add_quantity(CATACLYSMIC.CLAIMED_TYPE, 'known CV', sources) if 'CV candidate' in comment: catalog.entries[name].add_quantity(CATACLYSMIC.CLAIMED_TYPE, 'Candidate', sources) if 'possible Nova' in comment or 'Possible Nova' in comment or 'Nova candidate' in comment or 'nova candidate' in comment and not 'CV' in comment: name = catalog.add_entry(star_name) sources = [ catalog.entries[name].add_source(url=asn_url, name='ASAS-CV Transients') ] typesources = sources[:] if atellink: sources.append((catalog.entries[name].add_source( name='ATel ' + atellink.split('=')[-1], url=atellink))) # if typelink: # typesources.append( # (catalog.entries[name] # .add_source(name='ATel ' + # typelink.split('=')[-1], url=typelink))) sources = ','.join(sources) typesources = ','.join(typesources) catalog.entries[name].add_quantity(CATACLYSMIC.ALIAS, name, sources) catalog.entries[name].add_quantity(CATACLYSMIC.DISCOVER_DATE, discdate, sources) catalog.entries[name].add_quantity(CATACLYSMIC.RA, ra, sources, u_value='floatdegrees') catalog.entries[name].add_quantity(CATACLYSMIC.DEC, dec, sources, u_value='floatdegrees') catalog.entries[name].add_quantity(CATACLYSMIC.MAX_VISUAL_APP_MAG, mag, sources) catalog.entries[name].add_quantity(CATACLYSMIC.CLAIMED_TYPE, 'Candidate Nova', sources) # for ct in claimedtype.split('/'): # if ct != 'Unk': # catalog.entries[name].add_quantity(CATACLYSMIC.CLAIMED_TYPE, ct, # typesources) else: pass catalog.journal_entries() return #talk to professor about light curves data #def do_asas_atels(catalog): # """Import LCs exposed in ASASSN Atels.""" # with open('/root/better-atel/atels.json') as f: # ateljson = json.load(f) # for entry in ateljson: # if ('asas-sn.osu.edu/light_curve' in entry['body'] and # 'Cataclysmic' in entry['subjects']): # matches = re.findall(r'<a\s+[^>]*?href="([^"]*)".*?>(.*?)<\/a>', # entry['body'], re.DOTALL) # lcurl = '' # objname = '' # for match in matches: # if 'asas-sn.osu.edu/light_curve' in match[0]: # lcurl = match[0] # objname = re.findall( # r'\bASASSN-[0-9][0-9].*?\b', match[1]) # if len(objname): # objname = objname[0] # if objname and lcurl: # name, source = catalog.new_entry( # objname, srcname='ASAS-SN Sky Patrol', # bibcode='2017arXiv170607060K', # url='https://asas-sn.osu.edu') # csv = catalog.load_url(lcurl + '.csv', os.path.join( # catalog.get_current_task_repo(), os.path.join( # 'ASASSN', objname + '.csv'))) # data = read(csv, format='csv') # for row in data: # mag = str(row['mag']) # if float(mag.strip('>')) > 50.0: # continue # photodict = { # PHOTOMETRY.TIME: str(jd_to_mjd( # Decimal(str(row['HJD'])))), # PHOTOMETRY.MAGNITUDE: mag.strip('>'), # PHOTOMETRY.SURVEY: 'ASASSN', # PHOTOMETRY.SOURCE: source # } # if '>' in mag: # photodict[PHOTOMETRY.UPPER_LIMIT] = True # else: # photodict[PHOTOMETRY.E_MAGNITUDE] = str(row['mag_err']) # catalog.entries[name].add_photometry(**photodict) # catalog.journal_entries() # return
def do_ps_threepi(catalog): """Import data from Pan-STARRS' 3pi page.""" task_str = catalog.get_current_task_str() bad_aliases = ['SN1994J'] teles = 'Pan-STARRS1' fname = os.path.join(catalog.get_current_task_repo(), '3pi/page00.html') ps_url = ("http://psweb.mp.qub.ac.uk/" "ps1threepi/psdb/public/?page=1&sort=followup_flag_date") html = catalog.load_url(ps_url, fname, write=False) if not html: return # Clean some common HTML manglings html = html.replace('ahref=', 'a href=') bs = BeautifulSoup(html, 'html5lib') div = bs.find('div', {'class': 'pagination'}) offline = False if not div: offline = True else: links = div.findAll('a') if not links: offline = True if offline: if catalog.args.update: return warnings.warn('Pan-STARRS 3pi offline, using local files only.') with open(fname, 'r') as f: html = f.read() bs = BeautifulSoup(html, 'html5lib') div = bs.find('div', {'class': 'pagination'}) links = div.findAll('a') else: with open(fname, 'w') as f: f.write(html) numpages = int(links[-2].contents[0]) oldnumpages = len( glob(os.path.join(catalog.get_current_task_repo(), '3pi/page*'))) for page in pbar(range(1, numpages), task_str): fname = os.path.join(catalog.get_current_task_repo(), '3pi/page') + \ str(page).zfill(2) + '.html' if offline: if not os.path.isfile(fname): continue with open(fname, 'r') as f: html = f.read() else: if (catalog.current_task.load_archive(catalog.args) and page < oldnumpages and os.path.isfile(fname)): with open(fname, 'r') as f: html = f.read() else: response = urllib.request.urlopen( "http://psweb.mp.qub.ac.uk/ps1threepi/psdb/public/?page=" + str(page) + "&sort=followup_flag_date") with open(fname, 'w') as f: html = response.read().decode('utf-8') f.write(html) bs = BeautifulSoup(html, 'html5lib') trs = bs.findAll('tr') for tr in pbar(trs, task_str): tds = tr.findAll('td') if not tds: continue refs = [] aliases = [] ttype = '' ctype = '' for tdi, td in enumerate(tds): if tdi == 0: psname = td.contents[0] pslink = psname['href'] psname = psname.text elif tdi == 1: ra = td.contents[0] elif tdi == 2: dec = td.contents[0] elif tdi == 3: ttype = td.contents[0] if ttype != 'sn' and ttype != 'orphan': break elif tdi == 6: if not td.contents: continue ctype = td.contents[0] if ctype == 'Observed': ctype = '' elif tdi == 17: if td.contents: crossrefs = td.findAll('a') for cref in crossrefs: if 'atel' in cref.contents[0].lower(): refs.append([cref.contents[0], cref['href']]) elif is_number(cref.contents[0][:4]): continue else: aliases.append(cref.contents[0]) if ttype != 'sn' and ttype != 'orphan': continue name = '' for alias in aliases: if alias in bad_aliases: continue if alias[:2] == 'SN': name = alias if not name: name = psname name = catalog.add_entry(name) sources = [ catalog.entries[name].add_source( name='Pan-STARRS 3Pi', url=('http://psweb.mp.qub.ac.uk/' 'ps1threepi/psdb/')) ] catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, sources[0]) for ref in refs: sources.append(catalog.entries[name].add_source( name=ref[0], url=ref[1])) source = uniq_cdl(sources) for alias in aliases: newalias = alias if alias[:3] in ['CSS', 'SSS', 'MLS']: newalias = alias.replace('-', ':', 1) newalias = newalias.replace('PSNJ', 'PSN J') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, newalias, source) catalog.entries[name].add_quantity(SUPERNOVA.RA, ra, source) catalog.entries[name].add_quantity(SUPERNOVA.DEC, dec, source) catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, ctype, source) fname2 = os.path.join(catalog.get_current_task_repo(), '3pi/candidate-') fname2 += pslink.rstrip('/').split('/')[-1] + '.html' if offline: if not os.path.isfile(fname2): continue with open(fname2, 'r') as f: html2 = f.read() else: if (catalog.current_task.load_archive(catalog.args) and os.path.isfile(fname2)): with open(fname2, 'r') as f: html2 = f.read() else: pslink = ('http://psweb.mp.qub.ac.uk/' 'ps1threepi/psdb/public/') + pslink try: session2 = requests.Session() response2 = session2.get(pslink) except Exception: offline = True if not os.path.isfile(fname2): continue with open(fname2, 'r') as f: html2 = f.read() else: html2 = response2.text with open(fname2, 'w') as f: f.write(html2) bs2 = BeautifulSoup(html2, 'html5lib') scripts = bs2.findAll('script') nslines = [] nslabels = [] for script in scripts: if 'jslcdata.push' not in script.text: continue slines = script.text.splitlines() for line in slines: if 'jslcdata.push' in line: json_fname = (line.strip() .replace('jslcdata.push(', '') .replace(');', '')) nslines.append(json.loads(json_fname)) if ('jslabels.push' in line and 'blanks' not in line and 'non det' not in line): json_fname = (line.strip() .replace('jslabels.push(', '') .replace(');', '')) nslabels.append(json.loads(json_fname)['label']) for li, line in enumerate(nslines[:len(nslabels)]): if not line: continue for obs in line: catalog.entries[name].add_photometry( time=str(obs[0]), u_time='MJD', band=nslabels[li], instrument='GPC', magnitude=str(obs[1]), e_magnitude=str(obs[2]), source=source, telescope=teles) # Ignoring upper limits as they are usually spurious chip gaps. # for li, line in enumerate(nslines[2 * len(nslabels):]): # if not line: # continue # for obs in line: # catalog.entries[name].add_photometry( # time=str(obs[0]), # u_time='MJD', # band=nslabels[li], # instrument='GPC', # magnitude=str(obs[1]), # upperlimit=True, # source=source, # telescope=teles) assoctab = bs2.find('table', {'class': 'generictable'}) hostname = '' redshift = '' if assoctab: trs = assoctab.findAll('tr') headertds = [x.contents[0] for x in trs[1].findAll('td')] tds = trs[1].findAll('td') for tdi, td in enumerate(tds): if tdi == 1: hostname = td.contents[0].strip() elif tdi == 4: if 'z' in headertds: redshift = td.contents[0].strip() # Skip galaxies with just SDSS id if is_number(hostname): continue catalog.entries[name].add_quantity(SUPERNOVA.HOST, hostname, source) if redshift: catalog.entries[name].add_quantity( [SUPERNOVA.REDSHIFT, SUPERNOVA.HOST_REDSHIFT], redshift, source, kind='host') if catalog.args.update: catalog.journal_entries() catalog.journal_entries() # Only run first page for Travis if catalog.args.travis: break return
def do_nedd(catalog): task_str = catalog.get_current_task_str() nedd_path = os.path.join(catalog.get_current_task_repo(), 'NED26.10.1-D-13.1.0-20160930.csv') f = open(nedd_path, 'r') data = sorted(list(csv.reader(f, delimiter=',', quotechar='"'))[13:], key=lambda x: (x[9], x[3])) reference = "NED-D v" + nedd_path.split('-')[-2] refurl = "http://ned.ipac.caltech.edu/Library/Distances/" nedbib = "1991ASSL..171...89H" olddistname = '' loopcnt = 0 for r, row in enumerate(pbar(data, task_str)): if r <= 12: continue distname = row[3] name = name_clean(distname) # distmod = row[4] # moderr = row[5] dist = row[6] bibcode = unescape(row[8]) snname = name_clean(row[9]) redshift = row[10] cleanhost = '' if name != snname and (name + ' HOST' != snname): cleanhost = host_clean(distname) if cleanhost.endswith(' HOST'): cleanhost = '' if not is_number(dist): print(dist) if dist: catalog.nedd_dict.setdefault(cleanhost, []).append(Decimal(dist)) if snname and 'HOST' not in snname: snname, secondarysource = catalog.new_entry(snname, srcname=reference, bibcode=nedbib, url=refurl, secondary=True) if bibcode: source = catalog.entries[snname].add_source(bibcode=bibcode) sources = uniq_cdl([source, secondarysource]) else: sources = secondarysource if name == snname: if redshift: catalog.entries[snname].add_quantity( SUPERNOVA.REDSHIFT, redshift, sources) if dist: catalog.entries[snname].add_quantity( SUPERNOVA.COMOVING_DIST, dist, sources) if not redshift: try: zatval = z_at_value(cosmo.comoving_distance, float(dist) * un.Mpc, zmax=5.0) sigd = get_sig_digits(str(dist)) redshift = pretty_num(zatval, sig=sigd) except (KeyboardInterrupt, SystemExit): raise except Exception: pass else: cosmosource = catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') combsources = uniq_cdl( sources.split(',') + [cosmosource]) catalog.entries[snname].add_quantity( SUPERNOVA.REDSHIFT, redshift, combsources, derived=True) if cleanhost: catalog.entries[snname].add_quantity(SUPERNOVA.HOST, cleanhost, sources) if catalog.args.update and olddistname != distname: catalog.journal_entries() olddistname = distname loopcnt = loopcnt + 1 if catalog.args.travis and loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0: break catalog.journal_entries() f.close() return
def do_csp_fits_spectra(catalog): from astropy.io import fits fpath = catalog.get_current_task_repo() fureps = {'erg/cm2/s/A': 'erg/s/cm^2/Angstrom'} task_str = catalog.get_current_task_str() dirs = [x[0] for x in os.walk( os.path.join(fpath, 'Gutierrez_et_al_2017'))] files = [] for dir in dirs: files.extend(glob(os.path.join(dir, '*.fits'))) for datafile in pbar(files, task_str): filename = datafile.split('/')[-1] hdulist = fits.open(datafile) for oi, obj in enumerate(hdulist[0].header): if any(x in ['.', '/'] for x in obj): del (hdulist[0].header[oi]) try: hdulist[0].verify('silentfix') except Exception as e: print(e) hdrkeys = list(hdulist[0].header.keys()) # print(hdrkeys) name = datafile.split('/')[-2] if name[2] in '6789': name = 'SN19' + name[2:] elif name != 'SN210': name = 'SN20' + name[2:] name, source = catalog.new_entry(name, bibcode='2017ApJ...850...89G') # for key in hdulist[0].header.keys(): # print(key, hdulist[0].header[key]) mjd = None if hdulist[0].header['SIMPLE']: if 'JD' in hdrkeys: mjd = str(jd_to_mjd(Decimal(str(hdulist[0].header['JD'])))) elif 'MJD' in hdrkeys: mjd = str(hdulist[0].header['MJD']) elif 'DATE-OBS' in hdrkeys or 'DATE' in hdrkeys: dkey = 'DATE-OBS' if 'DATE-OBS' in hdrkeys else 'DATE' dval = hdulist[0].header[dkey] if is_number(dval): dkey = 'DATE' if dkey == 'DATE-OBS' else 'DATE-OBS' dval = hdulist[0].header[dkey] dateobs = None if 'T' in dval: dateobs = dval.strip() elif 'UTC-OBS' in hdrkeys: dateobs = dval.strip( ) + 'T' + hdulist[0].header['UTC-OBS'].strip() if dateobs is not None: mjd = str(astrotime(dateobs, format='isot').mjd) # print(hdulist[0].header) if 'CRVAL1' in hdulist[0].header: w0 = hdulist[0].header['CRVAL1'] elif hdulist[0].header['CTYPE1'] == 'MULTISPE': w0 = float(hdulist[0].header['WAT2_001'].split( '"')[-1].split()[3]) else: raise ValueError('Unsupported spectrum format.') if hdulist[0].header['NAXIS'] == 1: wd = hdulist[0].header['CDELT1'] fluxes = [str(x) for x in list(hdulist[0].data)] errors = False elif hdulist[0].header['NAXIS'] == 3: wd = hdulist[0].header['CD1_1'] fluxes = [str(x) for x in list(hdulist[0].data)[0][0]] errors = [str(x) for x in list(hdulist[0].data)[-1][0]] else: print('Warning: Skipping FITS spectrum `{}`.'.format(filename)) continue waves = [str(w0 + wd * x) for x in range(0, len(fluxes))] else: raise ValueError('Non-simple FITS import not yet supported.') if 'BUNIT' in hdrkeys: fluxunit = hdulist[0].header['BUNIT'] if fluxunit in fureps: fluxunit = fureps[fluxunit] else: if max([float(x) for x in fluxes]) < 1.0e-5: fluxunit = 'erg/s/cm^2/Angstrom' else: fluxunit = 'Uncalibrated' specdict = { SPECTRUM.U_WAVELENGTHS: 'Angstrom', SPECTRUM.WAVELENGTHS: waves, SPECTRUM.FLUXES: fluxes, SPECTRUM.U_FLUXES: fluxunit, SPECTRUM.FILENAME: filename, SPECTRUM.SOURCE: source } if mjd is not None: specdict[SPECTRUM.TIME] = mjd specdict[SPECTRUM.U_TIME] = 'MJD' if 'TELESCOP' in hdrkeys: specdict[SPECTRUM.TELESCOPE] = hdulist[0].header['TELESCOP'] if 'INSTRUME' in hdrkeys: specdict[SPECTRUM.INSTRUMENT] = hdulist[0].header['INSTRUME'] if 'AIRMASS' in hdrkeys: specdict[SPECTRUM.AIRMASS] = hdulist[0].header['AIRMASS'] if errors: specdict[SPECTRUM.ERRORS] = errors specdict[SPECTRUM.U_ERRORS] = fluxunit if 'SITENAME' in hdrkeys: specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['SITENAME'] elif 'OBSERVAT' in hdrkeys: specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['OBSERVAT'] if 'OBSERVER' in hdrkeys: specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER'] catalog.entries[name].add_spectrum(**specdict) hdulist.close() catalog.journal_entries() return
def do_asiago_photo(catalog): task_str = catalog.get_current_task_str() # response = (urllib.request # .urlopen('http://graspa.oapd.inaf.it/cgi-bin/sncat.php')) path = os.path.abspath(os.path.join( catalog.get_current_task_repo(), 'asiago-cat.php')) response = urllib.request.urlopen('file://' + path) html = response.read().decode('utf-8') html = html.replace('\r', "") soup = BeautifulSoup(html, 'html5lib') table = soup.find('table') records = [] for r, row in enumerate(table.findAll('tr')): if r == 0: continue col = row.findAll('td') records.append([utf8(x.renderContents()) for x in col]) for record in pbar(records, task_str): if len(record) > 1 and record[1] != '': oldname = clean_snname("SN" + record[1]).strip('?') reference = 'Asiago Supernova Catalogue' refurl = 'http://graspa.oapd.inaf.it/cgi-bin/sncat.php' refbib = '1989A&AS...81..421B' name, source = catalog.new_entry( oldname, srcname=reference, url=refurl, bibcode=refbib, secondary=True) year = re.findall(r'\d+', oldname)[0] catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, year, source) hostname = record[2] hostra = record[3] hostdec = record[4] ra = record[5].strip(':') dec = record[6].strip(':') redvel = record[11].strip(':') discoverer = record[19] datestring = year monthday = record[18] if "*" in monthday: datekey = SUPERNOVA.DISCOVER_DATE else: datekey = SUPERNOVA.MAX_DATE if monthday.strip() != '': monthstr = ''.join(re.findall('[a-zA-Z]+', monthday)) monthstr = str(list(calendar.month_abbr).index(monthstr)) datestring = datestring + '/' + monthstr dayarr = re.findall(r'\d+', monthday) if dayarr: daystr = dayarr[0] datestring = datestring + '/' + daystr catalog.entries[name].add_quantity(datekey, datestring, source) velocity = '' redshift = '' if redvel != '': if round(float(redvel)) == float(redvel): velocity = int(redvel) else: redshift = float(redvel) redshift = str(redshift) velocity = str(velocity) claimedtype = record[17].replace(':', '').replace('*', '').strip() if (hostname != ''): catalog.entries[name].add_quantity(SUPERNOVA.HOST, hostname, source) if (claimedtype != ''): catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, claimedtype, source) if (redshift != ''): catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, redshift, source, kind='host') if (velocity != ''): catalog.entries[name].add_quantity( SUPERNOVA.VELOCITY, velocity, source, kind='host') if (hostra != ''): catalog.entries[name].add_quantity( SUPERNOVA.HOST_RA, hostra, source, u_value='nospace') if (hostdec != ''): catalog.entries[name].add_quantity( SUPERNOVA.HOST_DEC, hostdec, source, u_value='nospace') if (ra != ''): catalog.entries[name].add_quantity(SUPERNOVA.RA, ra, source, u_value='nospace') if (dec != ''): catalog.entries[name].add_quantity(SUPERNOVA.DEC, dec, source, u_value='nospace') if (discoverer != ''): catalog.entries[name].add_quantity(SUPERNOVA.DISCOVERER, discoverer, source) catalog.journal_entries() return
def do_rochester(catalog): """Import data from latest supernova page.""" rochestermirrors = [ 'http://www.rochesterastronomy.org/', 'http://www.supernova.thistlethwaites.com/' ] rochesterpaths = [ 'snimages/snredshiftall.html', 'sn2020/snredshift.html', 'snimages/snredboneyard.html', 'snimages/snredboneyard-old.html' ] rochesterupdate = [False, True, True, False] task_str = catalog.get_current_task_str() baddates = ['2440587', '2440587.292', '0001/01/01'] for pp, path in enumerate(pbar(rochesterpaths, task_str)): if catalog.args.update and not rochesterupdate[pp]: continue if 'snredboneyard.html' in path: cns = { 'name': 0, 'host': 1, 'ra': 2, 'dec': 3, 'type': 7, 'z': 8, 'mmag': 9, 'max': 10, 'disc': 11, 'ref': 12, 'dver': 13, 'aka': 14 } else: cns = { 'name': 0, 'type': 1, 'host': 2, 'ra': 3, 'dec': 4, 'disc': 6, 'max': 7, 'mmag': 8, 'z': 11, 'zh': 12, 'ref': 13, 'dver': 14, 'aka': 15 } filepath = ( os.path.join(catalog.get_current_task_repo(), 'rochester/') + path.replace('/', '-')) for mirror in rochestermirrors: html = catalog.load_url(mirror + path, filepath, fail=(mirror != rochestermirrors[-1])) if html: break if not html: continue soup = BeautifulSoup(html, 'html5lib') rows = soup.findAll('tr') sec_ref = 'Latest Supernovae' sec_refurl = ('http://www.rochesterastronomy.org/' 'snimages/snredshiftall.html') loopcnt = 0 for rr, row in enumerate(pbar(rows, task_str)): if rr == 0: continue cols = row.findAll('td') if not len(cols): continue name = '' if cols[cns['aka']].contents: for rawaka in str(cols[cns['aka']].contents[0]).split(','): aka = rawaka.strip() if is_number(aka.strip('?')): aka = 'SN' + aka.strip('?') + 'A' oldname = aka name = catalog.add_entry(aka) elif len(aka) == 4 and is_number(aka[:4]): aka = 'SN' + aka oldname = aka name = catalog.add_entry(aka) sn = re.sub('<[^<]+?>', '', str(cols[cns['name']].contents[0])).strip() if is_number(sn.strip('?')): sn = 'SN' + sn.strip('?') + 'A' elif len(sn) == 4 and is_number(sn[:4]): sn = 'SN' + sn if not name: if not sn or sn in ['Transient']: continue ra = str(cols[cns['ra']].contents[0]).strip().replace(':.', '.') dec = str(cols[cns['dec']].contents[0]).strip().replace(':.', '.') if not name: if sn[:8] == 'MASTER J': sn = sn.replace('MASTER J', 'MASTER OT J').replace('SNHunt', 'SNhunt') if 'POSSIBLE' in sn.upper() and ra and dec: sn = 'PSN J' + ra.replace(':', '').replace('.', '') sn += dec.replace(':', '').replace('.', '') oldname = sn name = catalog.add_entry(sn) sec_source = catalog.entries[name].add_source(name=sec_ref, url=sec_refurl, secondary=True) sources = [] if 'ref' in cns: reftag = reference = cols[cns['ref']].findAll('a') if len(reftag) and len(reftag[0].contents): reference = reftag[0].contents[0].strip() refurl = reftag[0]['href'].strip() sources.append(catalog.entries[name].add_source( name=reference, url=refurl)) sources.append(sec_source) sources = uniq_cdl(list(filter(None, sources))) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oldname, sources) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, sn, sources) if cols[cns['aka']].contents: for rawaka in str(cols[cns['aka']].contents[0]).split(','): aka = rawaka.strip() if aka == 'SNR G1.9+0.3': aka = 'G001.9+00.3' if aka[:4] == 'PS1 ': aka = 'PS1-' + aka[4:] if aka[:8] == 'MASTER J': aka = aka.replace('MASTER J', 'MASTER OT J').replace( 'SNHunt', 'SNhunt') if 'POSSIBLE' in aka.upper() and ra and dec: aka = 'PSN J' + ra.replace(':', '').replace('.', '') aka += dec.replace(':', '').replace('.', '') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, aka, sources) if (len(cols[cns['type']].contents) > 0 and str(cols[cns['type']].contents[0]).strip() != 'unk'): type = str(cols[cns['type']].contents[0]).strip(' :,') catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, type, sources) if (len(cols[cns['host']].contents) > 0 and str(cols[cns['host']].contents[0]).strip() != 'anonymous'): catalog.entries[name].add_quantity( SUPERNOVA.HOST, str(cols[cns['host']].contents[0]).strip(), sources) catalog.entries[name].add_quantity(SUPERNOVA.RA, ra, sources) catalog.entries[name].add_quantity(SUPERNOVA.DEC, dec, sources) discstr = str(cols[cns['disc']].contents[0]).strip() if discstr and discstr not in baddates: if '/' not in discstr: astrot = astrotime(float(discstr), format='jd').datetime ddate = make_date_string(astrot.year, astrot.month, astrot.day) else: ddate = discstr catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, ddate, sources) maxstr = str(cols[cns.get('max', '')].contents[0]).strip() if maxstr and maxstr not in baddates: try: if '/' not in maxstr: astrot = astrotime(float(maxstr), format='jd') else: astrot = astrotime(maxstr.replace('/', '-'), format='iso') except: catalog.log.info( 'Max date conversion failed for `{}`.'.format(maxstr)) if ((float(str(cols[cns['mmag']].contents[0]).strip()) <= 90.0 and not any('GRB' in xx for xx in catalog.entries[name].get_aliases()))): mag = str(cols[cns['mmag']].contents[0]).strip() catalog.entries[name].add_photometry(time=str(astrot.mjd), u_time='MJD', magnitude=mag, source=sources) if 'z' in cns and cols[cns['z']].contents[0] != 'n/a': catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, str(cols[cns['z']].contents[0]).strip(), sources) if 'zh' in cns: zhost = str(cols[cns['zh']].contents[0]).strip() if is_number(zhost): catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, zhost, sources) if 'dver' in cns: catalog.entries[name].add_quantity( SUPERNOVA.DISCOVERER, str(cols[cns['dver']].contents[0]).strip(), sources) if catalog.args.update: catalog.journal_entries() loopcnt = loopcnt + 1 if (catalog.args.travis and loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0): break if not catalog.args.update: vsnetfiles = ['latestsne.dat'] for vsnetfile in vsnetfiles: file_name = os.path.join(catalog.get_current_task_repo(), "" + vsnetfile) with open(file_name, 'r', encoding='latin1') as csv_file: tsvin = csv.reader(csv_file, delimiter=' ', skipinitialspace=True) loopcnt = 0 for rr, row in enumerate(tsvin): if (not row or row[0] in ['Transient'] or row[0][:4] in ['http', 'www.'] or len(row) < 3): continue name = row[0].strip() if name[:4].isdigit(): name = 'SN' + name if name.startswith('PSNJ'): name = 'PSN J' + name[4:] if name.startswith('MASTEROTJ'): name = name.replace('MASTEROTJ', 'MASTER OT J') name = catalog.add_entry(name) sec_source = catalog.entries[name].add_source( name=sec_ref, url=sec_refurl, secondary=True) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, sec_source) if not is_number(row[1]): continue year = row[1][:4] month = row[1][4:6] day = row[1][6:] if '.' not in day: day = day[:2] + '.' + day[2:] mjd = astrotime(year + '-' + month + '-' + str(floor(float(day))).zfill(2)).mjd mjd += float(day) - floor(float(day)) magnitude = row[2].rstrip(ascii_letters) if not is_number(magnitude): continue if magnitude.isdigit(): if int(magnitude) > 100: magnitude = magnitude[:2] + '.' + magnitude[2:] if float(str(cols[8].contents[0]).strip()) >= 90.0: continue if len(row) >= 4: if is_number(row[3]): e_magnitude = row[3] refind = 4 else: e_magnitude = '' refind = 3 if refind >= len(row): sources = sec_source else: reference = ' '.join(row[refind:]) source = catalog.entries[name].add_source( name=reference) catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, name, sec_source) sources = uniq_cdl([source, sec_source]) else: sources = sec_source band = row[2].lstrip('1234567890.') catalog.entries[name].add_photometry( time=mjd, u_time='MJD', band=band, magnitude=magnitude, e_magnitude=e_magnitude, source=sources) if (catalog.args.travis and loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0): break catalog.journal_entries() return
def do_itep(catalog): task_str = catalog.get_current_task_str() itepignoresources = ['2004ApJ...602..571B', '2013NewA...20...30M'] itepignorephot = ['SN2006gy'] needsbib = [] with open(os.path.join(catalog.get_current_task_repo(), 'itep-refs.txt'), 'r') as refs_file: refrep = refs_file.read().splitlines() refrepf = dict(list(zip(refrep[1::2], refrep[::2]))) fname = os.path.join(catalog.get_current_task_repo(), 'itep-lc-cat-28dec2015.txt') tsvin = list( csv.reader(open(fname, 'r'), delimiter='|', skipinitialspace=True)) curname = '' for rr, row in enumerate(pbar(tsvin, task_str)): if rr <= 1 or len(row) < 7: continue oldname = 'SN' + row[0].strip() mjd = str(jd_to_mjd(Decimal(row[1].strip()))) band = row[2].strip() magnitude = row[3].strip() e_magnitude = row[4].strip() reference = row[6].strip().strip(',') if curname != oldname: curname = oldname name = catalog.add_entry(oldname) sec_reference = ('Sternberg Astronomical Institute ' 'Supernova Light Curve Catalogue') sec_refurl = 'http://dau.itep.ru/sn/node/72' sec_source = catalog.entries[name].add_source(name=sec_reference, url=sec_refurl, secondary=True) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oldname, sec_source) year = re.findall(r'\d+', name)[0] catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, year, sec_source) if reference in refrepf: bibcode = unescape(refrepf[reference]) source = catalog.entries[name].add_source(bibcode=bibcode) else: needsbib.append(reference) source = catalog.entries[name].add_source( name=reference) if reference else '' if oldname in itepignorephot or bibcode in itepignoresources: continue photodict = { PHOTOMETRY.TIME: mjd, PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.MAGNITUDE: magnitude, PHOTOMETRY.SOURCE: uniq_cdl([sec_source, source]) } if e_magnitude: photodict[PHOTOMETRY.E_MAGNITUDE] = e_magnitude if band.endswith('_SDSS'): photodict[PHOTOMETRY.BAND_SET] = 'SDSS' photodict[PHOTOMETRY.SYSTEM] = 'SDSS' band = band.replace('_SDSS', "'") photodict[PHOTOMETRY.BAND] = band catalog.entries[name].add_photometry(**photodict) if catalog.args.travis and rr >= catalog.TRAVIS_QUERY_LIMIT: break # Write out references that could use aa bibcode needsbib = list(OrderedDict.fromkeys(needsbib)) with open('../itep-needsbib.txt', 'w') as bib_file: bib_file.writelines(['%ss\n' % ii for ii in needsbib]) catalog.journal_entries() return
def set_preferred_names(self): """Choose between each entries given name and its possible aliases for the best one. Highest preference goes to names of the form 'SN####AA'. Otherwise base the name on whichever survey is the 'discoverer'. FIX: create function to match SN####AA type names. """ if len(self.entries) == 0: self.log.error("WARNING: `entries` is empty, loading stubs") self.load_stubs() task_str = self.get_current_task_str() for ni, name in enumerate(pbar(list( sorted(self.entries.keys())), task_str)): newname = '' aliases = self.entries[name].get_aliases() # if there are no other options to choose from, skip if len(aliases) <= 1: continue # If the name is already in the form 'SN####AA' then keep using # that if (name.startswith('SN') and ((is_number(name[2:6]) and not is_number(name[6:])) or (is_number(name[2:5]) and not is_number(name[5:])))): continue # If one of the aliases is in the form 'SN####AA' then use that for alias in aliases: if (alias[:2] == 'SN' and ((is_number(alias[2:6]) and not is_number(alias[6:])) or (is_number(alias[2:5]) and not is_number(alias[5:])))): newname = alias break # Otherwise, name based on the 'discoverer' survey if not newname and 'discoverer' in self.entries[name]: discoverer = ','.join( [x['value'].upper() for x in self.entries[name]['discoverer']]) if 'ASAS' in discoverer: for alias in aliases: if 'ASASSN' in alias.upper(): newname = alias break if not newname and 'OGLE' in discoverer: for alias in aliases: if 'OGLE' in alias.upper(): newname = alias break if not newname and 'CRTS' in discoverer: for alias in aliases: if True in [x in alias.upper() for x in ['CSS', 'MLS', 'SSS', 'SNHUNT']]: newname = alias break if not newname and 'PS1' in discoverer: for alias in aliases: if 'PS1' in alias.upper(): newname = alias break if not newname and 'PTF' in discoverer: for alias in aliases: if 'PTF' in alias.upper(): newname = alias break if not newname and 'GAIA' in discoverer: for alias in aliases: if 'GAIA' in alias.upper(): newname = alias break # Always prefer another alias over PSN if not newname and name.startswith('PSN'): newname = aliases[0] if newname and name != newname: # Make sure new name doesn't already exist if self.proto.init_from_file(self, name=newname): self.log.error("WARNING: `newname` already exists... " "should do something about that...") continue new_entry = self.proto.init_from_file(self, name=name) if new_entry is None: self.log.error( "Could not load `new_entry` with name '{}'" .format(name)) else: self.log.info("Changing entry from name '{}' to preferred" " name '{}'".format(name, newname)) self._delete_entry_file(entry=new_entry) del self.entries[name] self.entries[newname] = new_entry self.entries[newname][self.proto._KEYS.NAME] = newname if name in self.entries: self.log.error( "WARNING: `name` = '{}' is in `entries` " "shouldnt happen?".format(name)) del self.entries[name] self.journal_entries() if self.args.travis and ni > self.TRAVIS_QUERY_LIMIT: break return
def do_wiserep_spectra(catalog): #if not catalog.args.travis: # from ..input.WISeWEBSpider.wisewebspider import spider # try: # spider(update=True, daysago=7, path="/../../sne-external-WISEREP/") # except: # catalog.log.warning( # 'Spider errored, continuing without letting it complete.') task_str = catalog.get_current_task_str() secondaryreference = 'WISeREP' secondaryrefurl = 'http://wiserep.weizmann.ac.il/' secondarybibcode = '2012PASP..124..668Y' wiserepcnt = 0 # These are known to be in error on the WISeREP page, either fix or ignore # them. wiserepbibcorrectdict = { '2000AJ....120..367G]': '2000AJ....120..367G', 'Harutyunyan et al. 2008': '2008A&A...488..383H', '0609268': '2007AJ....133...58K', '2006ApJ...636...400Q': '2006ApJ...636..400Q', '2011ApJ...741...76': '2011ApJ...741...76C', '2016PASP...128...961': '2016PASP..128...961', '2002AJ....1124..417H': '2002AJ....1124.417H', '2013ApJ…774…58D': '2013ApJ...774...58D', '2011Sci.333..856S': '2011Sci...333..856S', '2014MNRAS.438,368': '2014MNRAS.438..368T', '2012MNRAS.420.1135': '2012MNRAS.420.1135S', '2012Sci..337..942D': '2012Sci...337..942D', 'stt1839': '2013MNRAS.436.3614S', 'arXiv:1605.03136': '2016MNRAS.460.3447T', '10.1093/mnras/stt1839': '2013MNRAS.436.3614S' } file_names = list(glob(os.path.join(catalog.get_current_task_repo(), '*'))) for folder in pbar_strings(file_names, task_str): if '.txt' in folder or '.json' in folder: continue name = os.path.basename(folder).strip() if name.startswith('sn'): name = 'SN' + name[2:] if (name.startswith(('CSS', 'SSS', 'MLS')) and ':' not in name): name = name.replace('-', ':', 1) if name.startswith('MASTERJ'): name = name.replace('MASTERJ', 'MASTER OT J') if name.startswith('PSNJ'): name = name.replace('PSNJ', 'PSN J') name = catalog.add_entry(name) secondarysource = catalog.entries[name].add_source( name=secondaryreference, url=secondaryrefurl, bibcode=secondarybibcode, secondary=True) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, secondarysource) readme_path = os.path.join(folder, 'README.json') if not os.path.exists(readme_path): catalog.log.warning( 'Metadata file not found for event "{}"'.format(name)) continue with open(readme_path, 'r') as f: fileinfo = json.loads(f.read()) files = list( set(glob(folder + '/*')) - set(glob(folder + '/README.json'))) for fname in pbar(files, task_str): specfile = os.path.basename(fname) if specfile not in fileinfo: catalog.log.warning( 'Metadata not found for "{}"'.format(fname)) continue claimedtype = fileinfo[specfile]["Type"] instrument = fileinfo[specfile]["Instrument"] epoch = fileinfo[specfile]["Obs. Date"] observer = fileinfo[specfile]["Observer"] reducer = fileinfo[specfile]["Reducer"] bibcode = fileinfo[specfile]["Bibcode"] redshift = fileinfo[specfile]["Redshift"] survey = fileinfo[specfile]["Program"] reduction = fileinfo[specfile]["Reduction Status"] if bibcode: newbibcode = bibcode if bibcode in wiserepbibcorrectdict: newbibcode = wiserepbibcorrectdict[bibcode] if newbibcode and len(newbibcode) == 19: source = catalog.entries[name].add_source( bibcode=unescape(newbibcode)) else: bibname = unescape(bibcode) source = catalog.entries[name].add_source(name=bibname) catalog.log.warning('Bibcode "{}" is invalid, using as ' '`{}` instead'.format( bibname, SOURCE.NAME)) sources = uniq_cdl([source, secondarysource]) else: sources = secondarysource if claimedtype not in ['Other']: catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, claimedtype, secondarysource) catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, redshift, secondarysource) with open(fname, 'r') as f: data = [x.split() for x in f] skipspec = False newdata = [] oldval = '' for row in data: if row and '#' not in row[0]: if (len(row) >= 2 and is_number(row[0]) and is_number(row[1]) and row[1] != oldval): newdata.append(row) oldval = row[1] if skipspec or not newdata: warnings.warn('Skipped adding spectrum file ' + specfile) continue data = [list(i) for i in zip(*newdata)] wavelengths = data[0] fluxes = data[1] errors = '' if len(data) == 3: errors = data[1] time = str(astrotime(epoch).mjd) if max([float(x) for x in fluxes]) < 1.0e-5: fluxunit = 'erg/s/cm^2/Angstrom' else: fluxunit = 'Uncalibrated' catalog.entries[name].add_spectrum( u_wavelengths='Angstrom', errors=errors, u_fluxes=fluxunit, u_errors=fluxunit if errors else '', wavelengths=wavelengths, fluxes=fluxes, u_time='MJD', time=time, instrument=instrument, source=sources, observer=observer, reducer=reducer, reduction=reduction, filename=specfile, survey=survey, redshift=redshift) catalog.journal_entries() wiserepcnt = wiserepcnt + 1 if (catalog.args.travis and wiserepcnt % catalog.TRAVIS_QUERY_LIMIT == 0): break return
def do_ps_alerts(catalog): task_str = catalog.get_current_task_str() alertstables = ['alertstable-2010', 'alertstable-2011', 'alertstable'] rows = [] for at in alertstables: with open( os.path.join(catalog.get_current_task_repo(), 'ps1-clean', at)) as f: rows.extend( list(csv.reader( f, delimiter=' ', skipinitialspace=True))[1:]) alertfiles = glob( os.path.join(catalog.get_current_task_repo(), 'ps1-clean/*.dat')) alertfilestag = dict( [(x.split('/')[-1].split('.')[0], x) for x in alertfiles]) alertfilesid = dict([(x.split('/')[-1].split('.')[0].split('-')[-1], x) for x in alertfiles]) with open( os.path.join(catalog.get_current_task_repo(), 'ps1-clean/whitelist')) as f: whitelist = list(csv.reader(f, delimiter=' ', skipinitialspace=True)) wlnames = [x[0] for x in whitelist] wlnamesleft = list(wlnames) wlra = [x[1] for x in whitelist] missing_confirmed = [] # already_collected = [] for ri, row in enumerate(pbar(rows, task_str)): psname = row[50] if psname == '-': if row[4] in wlra: psname = wlnames[wlra.index(row[4])] else: continue sntype = row[21].replace('SN', '') skip_photo = False if psname not in wlnamesleft: if row[1] == 'confirmed': missing_confirmed.append((psname, row[21])) # if 'II' in sntype: # pass # elif (sntype != 'Ia' or # not (psname.startswith(('PS1-12', 'PS1-13')) or # (psname.startswith('PS1-10') and # len(psname.replace('PS1-10', '')) == 3 and # psname[-3:] > 'ams'))): # if sntype == 'Ia' and psname.startswith('PS1-10'): # already_collected.append((psname, row[21])) # else: # missing_confirmed.append((psname, row[21])) skip_photo = True else: continue if psname in wlnamesleft: wlnamesleft.remove(psname) name, source = catalog.new_entry( psname, srcname='Pan-STARRS Alerts', url='http://telescopes.rc.fas.harvard.edu/ps1/') catalog.entries[name].add_quantity(SUPERNOVA.RA, row[4], source) catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[5], source) if sntype != '-': catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, sntype, source) if row[22] != '-': catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, row[22], source) # Disabling photometry import continue if skip_photo: continue psinternal = row[-1].split('.')[0] if not is_number(psinternal.split('-')[0]): psid = row[0].zfill(6) if psid not in alertfilesid: continue pspath = alertfilesid[psid] else: if psinternal not in alertfilestag: continue pspath = alertfilestag[psinternal] with open(pspath) as f: photrows = list( csv.reader( f, delimiter=' ', skipinitialspace=True)) for pi, prow in enumerate(photrows): if pi == 0 or prow[3] == '-': continue counts = prow[13] e_counts = prow[14] photodict = { PHOTOMETRY.TIME: prow[1], PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.BAND: prow[2], PHOTOMETRY.MAGNITUDE: prow[3], PHOTOMETRY.E_MAGNITUDE: prow[4], PHOTOMETRY.COUNT_RATE: counts, PHOTOMETRY.E_COUNT_RATE: e_counts, PHOTOMETRY.ZERO_POINT: prow[15], PHOTOMETRY.INSTRUMENT: 'GPC1', PHOTOMETRY.OBSERVATORY: 'PS1', PHOTOMETRY.SURVEY: 'MDS', PHOTOMETRY.TELESCOPE: 'PS1', PHOTOMETRY.SYSTEM: 'PS1', PHOTOMETRY.SOURCE: source } ul_sigma = 3.0 if float(counts) < ul_sigma * float(e_counts): photodict[PHOTOMETRY.UPPER_LIMIT] = True photodict[PHOTOMETRY.UPPER_LIMIT_SIGMA] = ul_sigma catalog.entries[name].add_photometry(**photodict) catalog.journal_entries() # print(already_collected) # print(missing_confirmed) return
def do_ascii(catalog): """Process ASCII files extracted from datatables of published works.""" task_str = catalog.get_current_task_str() # Howerton Catalog datafile = os.path.join( catalog.get_current_task_repo(), 'ASCII', 'vizier_J_MNRAS_441_1186_table1 J_MNRAS_441_1186_table3_20200403.csv') data = read(datafile, format='csv') for rrow in pbar(data, task_str): row = dict((x, str(rrow[x])) for x in rrow.columns) # if any(x in row['Notes'].lower() for x in ['artifact']): # continue # ctypes = row['Type'].split('/') # nonsne = False # for ct in ctypes: # if ct.replace('?', '') in catalog.nonsnetypes: # nonsne = True # else: # nonsne = False # break # if nonsne: # continue name, source = catalog.new_entry(row['CRTS'], srcname='CRTS', bibcode='2014MNRAS.441.1186D') # if row['IAU des.'] != '--': # catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, # row['IAU des.'], source) # for ct in ctypes: # catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, ct, # source) # catalog.entries[name].add_quantity(SUPERNOVA.DISCOVERER, # row['Discoverer'], source) # date = row['Discovery'].split('/') # date = '/'.join([date[-1].zfill(2), date[0].zfill(2), date[1]]) # catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, date, # source) catalog.entries[name].add_quantity(CATACLYSMIC.MAX_VISUAL_APP_MAG, row['Vmin'], source) catalog.entries[name].add_quantity(CATACLYSMIC.RA, row['RAJ2000'], source) catalog.entries[name].add_quantity(CATACLYSMIC.DEC, row['DEJ2000'], source) catalog.journal_entries() # Howerton Catalog # datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII', # 'howerton-catalog.csv') # data = read(datafile, format='csv') # for rrow in pbar(data, task_str): # row = dict((x, str(rrow[x])) for x in rrow.columns) # if any(x in row['Notes'].lower() for x in ['artifact']): # continue # ctypes = row['Type'].split('/') # nonsne = False # for ct in ctypes: # if ct.replace('?', '') in catalog.nonsnetypes: # nonsne = True # else: # nonsne = False # break # if nonsne: # continue # name, source = catalog.new_entry( # row['SNHunt des.'], # srcname='CRTS SNhunt', # bibcode='2017csnh.book.....H') # if row['IAU des.'] != '--': # catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, # row['IAU des.'], source) # for ct in ctypes: # catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, ct, # source) # catalog.entries[name].add_quantity(SUPERNOVA.DISCOVERER, # row['Discoverer'], source) # date = row['Discovery'].split('/') # date = '/'.join([date[-1].zfill(2), date[0].zfill(2), date[1]]) # catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, date, # source) # catalog.entries[name].add_quantity(SUPERNOVA.HOST, row['Host galaxy'], # source) # catalog.entries[name].add_quantity(SUPERNOVA.RA, row['RA'].replace( # 'h', ':').replace('m', ':').replace('s', ''), source) # catalog.entries[name].add_quantity(SUPERNOVA.DEC, row['Dec'], source) # catalog.journal_entries()
def do_csp_fits_spectra(catalog): from astropy.io import fits fpath = catalog.get_current_task_repo() fureps = {'erg/cm2/s/A': 'erg/s/cm^2/Angstrom'} task_str = catalog.get_current_task_str() dirs = [x[0] for x in os.walk(os.path.join(fpath, 'Gutierrez_et_al_2017'))] files = [] for dir in dirs: files.extend(glob(os.path.join(dir, '*.fits'))) for datafile in pbar(files, task_str): filename = datafile.split('/')[-1] hdulist = fits.open(datafile) for oi, obj in enumerate(hdulist[0].header): if any(x in ['.', '/'] for x in obj): del (hdulist[0].header[oi]) try: hdulist[0].verify('silentfix') except Exception as e: print(e) hdrkeys = list(hdulist[0].header.keys()) # print(hdrkeys) name = datafile.split('/')[-2] if name[2] in '6789': name = 'SN19' + name[2:] elif name != 'SN210': name = 'SN20' + name[2:] name, source = catalog.new_entry(name, bibcode='2017ApJ...850...89G') # for key in hdulist[0].header.keys(): # print(key, hdulist[0].header[key]) mjd = None if hdulist[0].header['SIMPLE']: if 'JD' in hdrkeys: mjd = str(jd_to_mjd(Decimal(str(hdulist[0].header['JD'])))) elif 'MJD' in hdrkeys: mjd = str(hdulist[0].header['MJD']) elif 'DATE-OBS' in hdrkeys or 'DATE' in hdrkeys: dkey = 'DATE-OBS' if 'DATE-OBS' in hdrkeys else 'DATE' dval = hdulist[0].header[dkey] if is_number(dval): dkey = 'DATE' if dkey == 'DATE-OBS' else 'DATE-OBS' dval = hdulist[0].header[dkey] dateobs = None if 'T' in dval: dateobs = dval.strip() elif 'UTC-OBS' in hdrkeys: dateobs = dval.strip( ) + 'T' + hdulist[0].header['UTC-OBS'].strip() if dateobs is not None: mjd = str(astrotime(dateobs, format='isot').mjd) # print(hdulist[0].header) if 'CRVAL1' in hdulist[0].header: w0 = hdulist[0].header['CRVAL1'] elif hdulist[0].header['CTYPE1'] == 'MULTISPE': w0 = float( hdulist[0].header['WAT2_001'].split('"')[-1].split()[3]) else: raise ValueError('Unsupported spectrum format.') if hdulist[0].header['NAXIS'] == 1: wd = hdulist[0].header['CDELT1'] fluxes = [str(x) for x in list(hdulist[0].data)] errors = False elif hdulist[0].header['NAXIS'] == 3: wd = hdulist[0].header['CD1_1'] fluxes = [str(x) for x in list(hdulist[0].data)[0][0]] errors = [str(x) for x in list(hdulist[0].data)[-1][0]] else: print('Warning: Skipping FITS spectrum `{}`.'.format(filename)) continue waves = [str(w0 + wd * x) for x in range(0, len(fluxes))] else: raise ValueError('Non-simple FITS import not yet supported.') if 'BUNIT' in hdrkeys: fluxunit = hdulist[0].header['BUNIT'] if fluxunit in fureps: fluxunit = fureps[fluxunit] else: if max([float(x) for x in fluxes]) < 1.0e-5: fluxunit = 'erg/s/cm^2/Angstrom' else: fluxunit = 'Uncalibrated' specdict = { SPECTRUM.U_WAVELENGTHS: 'Angstrom', SPECTRUM.WAVELENGTHS: waves, SPECTRUM.FLUXES: fluxes, SPECTRUM.U_FLUXES: fluxunit, SPECTRUM.FILENAME: filename, SPECTRUM.SOURCE: source } if mjd is not None: specdict[SPECTRUM.TIME] = mjd specdict[SPECTRUM.U_TIME] = 'MJD' if 'TELESCOP' in hdrkeys: specdict[SPECTRUM.TELESCOPE] = hdulist[0].header['TELESCOP'] if 'INSTRUME' in hdrkeys: specdict[SPECTRUM.INSTRUMENT] = hdulist[0].header['INSTRUME'] if 'AIRMASS' in hdrkeys: specdict[SPECTRUM.AIRMASS] = hdulist[0].header['AIRMASS'] if errors: specdict[SPECTRUM.ERRORS] = errors specdict[SPECTRUM.U_ERRORS] = fluxunit if 'SITENAME' in hdrkeys: specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['SITENAME'] elif 'OBSERVAT' in hdrkeys: specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['OBSERVAT'] if 'OBSERVER' in hdrkeys: specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER'] catalog.entries[name].add_spectrum(**specdict) hdulist.close() catalog.journal_entries() return
def do_spectra(catalog): """ """ #path to root ONC directory ONC = catalog.get_current_task_repo() novae_directory = os.path.join(ONC, 'Individual_Novae') task_str = catalog.get_current_task_str() for indiv_nova in pbar(os.listdir(novae_directory), task_str): nova_dir = os.path.join(novae_directory, indiv_nova) indiv_nova = get_nova_name(indiv_nova) if not os.path.isdir(nova_dir) or indiv_nova is None: continue ticket_directory = os.path.join(nova_dir, "Tickets", "CompletedTickets") for ticket in os.listdir(ticket_directory): ticket_raw_dict, ticket_column_dict = {}, {} if not re.search(r'spectra', ticket.lower()): continue with open(os.path.join(ticket_directory, ticket)) as spec_ticket: try: ticket_raw_dict, ticket_column_dict = read_spectra_ticket(spec_ticket.read()) spec_ticket.close() except AttributeError: True == True print(ticket) continue #THROW ERROR TO LOG HERE meta_filename = ticket_raw_dict['METADATA FILENAME'] file_path = os.path.join(nova_dir, "Data", meta_filename) name = indiv_nova.replace('_', ' ') name = catalog.add_entry(name) if type(ticket_raw_dict['BIBCODE']) is str and len(ticket_raw_dict['BIBCODE']) == 19: source = catalog.entries[name].add_source(bibcode=ticket_raw_dict['BIBCODE'], reference=ticket_raw_dict['REFERENCE']) else: source = catalog.entries[name].add_source(bibcode='couldnotfindbibcode', reference=ticket_raw_dict['REFERENCE']) catalog.entries[name].add_quantity(NOVA.ALIAS, name, source) time_offset = 0 if re.match(r'(days)|(ut)', ticket_raw_dict['TIME SYSTEM'].lower()): t = Time(convert_date_UTC(ticket_raw_dict['ASSUMED DATE OF OUTBURST'])) offset = t.jd mag = 'magnitude' if re.search(r'mag', ticket_raw_dict['FLUX UNITS'].lower()) else 'flux' e_mag = 'e_magnitude' if re.search(r'mag', ticket_raw_dict['FLUX ERROR UNITS'].lower()) else 'e_flux' with open(file_path, "r") as metadata_file: data_file_list = read_spectra_metadata(metadata_file.read(), ticket_column_dict) metadata_file.close() for metadata_raw_dict, metadata_column_dict in data_file_list: data_filename = metadata_raw_dict['FILENAME'] csvfile = open(os.path.join(nova_dir, "Data", data_filename)) contents = list(csv.reader(csvfile, delimiter=',', quotechar='"')) csvfile.close() flux_col = metadata_column_dict['FLUX'] wavelength_col = metadata_column_dict['WAVE'] flux_err_col = metadata_column_dict['FLUX ERR'] if flux_col is None or wavelength_col is None: continue flux_err_arr = None if flux_err_col is None else [] flux_arr, wavelength_arr = [],[] for line in contents: line = [value.strip() for value in line] if line[0].startswith("#"): continue try: flux = line[flux_col] wavelength = line[wavelength_col] except (IndexError, ValueError): continue try: flux_err = None if flux_err_arr is None else line[flux_err_col] except (IndexError, ValueError): flux_err = None if flux_err is not None: flux_err_arr.append(flux_err) flux_arr.append(flux) wavelength_arr.append(flux) data_dict = {'wavelengths': wavelength_arr, 'fluxes': flux_arr, 'source': source, 'time': metadata_raw_dict['DATE'], 'observer': metadata_raw_dict['OBSERVER'], 'telescope': metadata_raw_dict['TELESCOPE'], 'instrument': metadata_raw_dict['INSTRUMENT'], 'u_wavelengths': 'Angstrom', 'u_time': 'JD', 'u_fluxes': 'erg/s/cm^2'} if not flux_err_arr is None and len(flux_err_arr) != len(wavelength_arr): data_dict['errors'] = flux_err_arr key_list = list(data_dict.keys()) for key in key_list: if data_dict[key] is None: del data_dict[key] catalog.entries[name].add_spectrum(**data_dict) return
def do_cpcs(catalog): """Import data from CPCS.""" task_str = catalog.get_current_task_str() cpcs_url = ('http://gsaweb.ast.cam.ac.uk/' 'followup/list_of_alerts?format=json&num=100000&' 'published=1&observed_only=1' '&hashtag=JG_530ad9462a0b8785bfb385614bf178c6') jsontxt = catalog.load_url( cpcs_url, os.path.join(catalog.get_current_task_repo(), 'CPCS/index.json')) if not jsontxt: return alertindex = json.loads(jsontxt, object_pairs_hook=OrderedDict) ids = [xx['id'] for xx in alertindex] for ii, ai in enumerate(pbar(ids, task_str)): name = alertindex[ii]['ivorn'].split('/')[-1].strip() # Skip aa few weird entries if name == 'ASASSNli': continue # Just use aa whitelist for now since naming seems inconsistent white_list = [ 'GAIA', 'OGLE', 'ASASSN', 'MASTER', 'OTJ', 'PS1', 'IPTF', 'CSS' ] if True in [xx in name.upper() for xx in white_list]: name = name.replace('Verif', '').replace('_', ' ') if 'ASASSN' in name and name[6] != '-': name = 'ASASSN-' + name[6:].lower() if 'MASTEROTJ' in name: name = name.replace('MASTEROTJ', 'MASTER OT J') if 'OTJ' in name: name = name.replace('OTJ', 'MASTER OT J') if name.upper().startswith('IPTF'): name = 'iPTF' + name[4:].lower() if name.upper().startswith('PS1'): name = 'PS1' + name[3:].lower() # Only add events that are classified as SN. if not catalog.entry_exists(name): continue oldname = name name = catalog.add_entry(name) else: continue sec_source = catalog.entries[name].add_source( name='Cambridge Photometric Calibration Server', url='http://gsaweb.ast.cam.ac.uk/followup/', secondary=True) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oldname, sec_source) unit_deg = 'floatdegrees' catalog.entries[name].add_quantity(SUPERNOVA.RA, str(alertindex[ii][SUPERNOVA.RA]), sec_source, u_value=unit_deg) catalog.entries[name].add_quantity(SUPERNOVA.DEC, str(alertindex[ii][SUPERNOVA.DEC]), sec_source, u_value=unit_deg) alerturl = ('http://gsaweb.ast.cam.ac.uk/' 'followup/get_alert_lc_data?alert_id=' + str(ai)) source = catalog.entries[name].add_source(name='CPCS Alert ' + str(ai), url=alerturl) fname = os.path.join(catalog.get_current_task_repo(), 'CPCS/alert-') + str(ai).zfill(2) + '.json' jsonstr = catalog.load_url( alerturl + '&hashtag=JG_530ad9462a0b8785bfb385614bf178c6', fname) try: cpcsalert = json.loads(jsonstr) except Exception: catalog.log.warning('Mangled CPCS data for alert {}.'.format(ai)) continue mjds = [round_sig(xx, sig=9) for xx in cpcsalert['mjd']] mags = [round_sig(xx, sig=6) for xx in cpcsalert['mag']] errs = [ round_sig(xx, sig=6) if (is_number(xx) and float(xx) > 0.0) else '' for xx in cpcsalert['magerr'] ] bnds = cpcsalert['filter'] obs = cpcsalert['observatory'] for mi, mjd in enumerate(mjds): catalog.entries[name].add_photometry(time=mjd, u_time='MJD', magnitude=mags[mi], e_magnitude=errs[mi], band=bnds[mi], observatory=obs[mi], source=uniq_cdl( [source, sec_source])) if catalog.args.update: catalog.journal_entries() if catalog.args.travis and ii >= catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() return
def do_simbad(catalog): task_str = catalog.get_current_task_str() keys = list(catalog.entries.keys()) customSimbad = Simbad() customSimbad.ROW_LIMIT = -1 customSimbad.TIMEOUT = 120 customSimbad.add_votable_fields('otype', 'sptype', 'sp_bibcode', 'id') warnings.filterwarnings("ignore") Mnames, Mradec = [], [] for oname in pbar(keys, task_str): # Some events may be merged in cleanup process, skip them if # non-existent. try: name = catalog.add_entry(oname) except Exception: catalog.log.warning( '"{}" was not found, suggests merge occurred in cleanup ' 'process.'.format(oname)) continue if (FASTSTARS.RA not in catalog.entries[name] or FASTSTARS.DEC not in catalog.entries[name]): continue else: Mnames.append(name) radec = str(catalog.entries[name][FASTSTARS.RA][0]['value']) + str( catalog.entries[name][FASTSTARS.DEC][0]['value']) c = coord(radec, unit=(un.hourangle, un.deg), frame='icrs') cnttry = 0 foundstar = False while foundstar == False and cnttry < 10: try: cnttry += 1 time.sleep(0.1) result = customSimbad.query_region(c, radius='0d0m5s') aliases = re.sub(r'b\'(.*)\'', r'\1', str(result['ID'].tolist()[0])).split(',') except TypeError: #print(radec,cnttry) continue foundstar = True if foundstar == True: source = (catalog.entries[name].add_source( name='SIMBAD astronomical database', bibcode="2000A&AS..143....9W", url="http://simbad.u-strasbg.fr/", secondary=True)) for alias in aliases: ali = single_spaces( re.sub(r'\[[^)]*\]', '', alias).strip()) if is_number(ali.replace(' ', '')): continue if ali[:4] == "HVS ": continue ali = name_clean(ali) catalog.entries[name].add_quantity(FASTSTARS.ALIAS, ali, source) catalog.journal_entries() return
def do_cccp(catalog): task_str = catalog.get_current_task_str() cccpbands = ['B', 'V', 'R', 'I'] file_names = list( glob(os.path.join(catalog.get_current_task_repo(), 'CCCP/apj407397*.txt'))) for datafile in pbar_strings(file_names, task_str + ': apj407397...'): with open(datafile, 'r') as ff: tsvin = csv.reader(ff, delimiter='\t', skipinitialspace=True) for rr, row in enumerate(tsvin): if rr == 0: continue elif rr == 1: name = 'SN' + row[0].split('SN ')[-1] name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2012ApJ...744...10K') catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, name, source) elif rr >= 5: mjd = str(Decimal(row[0]) + 53000) for bb, band in enumerate(cccpbands): if row[2 * bb + 1]: mag = row[2 * bb + 1].strip('>') upl = (not row[2 * bb + 2]) (catalog.entries[name] .add_photometry(time=mjd, band=band, magnitude=mag, e_magnitude=row[2 * bb + 2], upperlimit=upl, source=source)) if catalog.current_task.load_archive(catalog.args): with open(os.path.join(catalog.get_current_task_repo(), 'CCCP/sc_cccp.html'), 'r') as ff: html = ff.read() else: session = requests.Session() response = session.get( 'https://webhome.weizmann.ac.il/home/iair/sc_cccp.html') html = response.text with open(os.path.join(catalog.get_current_task_repo(), 'CCCP/sc_cccp.html'), 'w') as ff: ff.write(html) soup = BeautifulSoup(html, 'html5lib') links = soup.body.findAll("a") for link in pbar(links, task_str + ': links'): if 'sc_sn' in link['href']: name = catalog.add_entry(link.text.replace(' ', '')) source = (catalog.entries[name] .add_source(name='CCCP', url=('https://webhome.weizmann.ac.il' '/home/iair/sc_cccp.html'))) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) if catalog.current_task.load_archive(catalog.args): fname = os.path.join(catalog.get_current_task_repo(), 'CCCP/') + link['href'].split('/')[-1] with open(fname, 'r') as ff: html2 = ff.read() else: response2 = session.get( 'https://webhome.weizmann.ac.il/home/iair/' + link['href']) html2 = response2.text fname = os.path.join(catalog.get_current_task_repo(), 'CCCP/') + link['href'].split('/')[-1] with open(fname, 'w') as ff: ff.write(html2) soup2 = BeautifulSoup(html2, 'html5lib') links2 = soup2.body.findAll("a") for link2 in links2: if '.txt' in link2['href'] and '_' in link2['href']: band = link2['href'].split('_')[1].split('.')[0].upper() if catalog.current_task.load_archive(catalog.args): fname = os.path.join( catalog.get_current_task_repo(), 'CCCP/') fname += link2['href'].split('/')[-1] if not os.path.isfile(fname): continue with open(fname, 'r') as ff: html3 = ff.read() else: response3 = (session .get('https://webhome.weizmann.ac.il' '/home/iair/cccp/' + link2['href'])) if response3.status_code == 404: continue html3 = response3.text fname = os.path.join( catalog.get_current_task_repo(), 'CCCP/') fname += link2['href'].split('/')[-1] with open(fname, 'w') as ff: ff.write(html3) table = [[str(Decimal(yy.strip())).rstrip('0') for yy in xx.split(',')] for xx in list(filter(None, html3.split('\n')))] for row in table: catalog.entries[name].add_photometry( time=str(Decimal(row[0]) + 53000), band=band, magnitude=row[1], e_magnitude=row[2], source=source) catalog.journal_entries() return
def do_asiago_spectra(catalog): task_str = catalog.get_current_task_str() html = catalog.load_cached_url( ('http://sngroup.oapd.inaf.it./' 'cgi-bin/output_class.cgi?sn=1990'), os.path.join(catalog.get_current_task_repo(), 'Asiago/spectra.html')) if not html: return bs = BeautifulSoup(html, 'html5lib') trs = bs.findAll('tr') for tr in pbar(trs, task_str): tds = tr.findAll('td') name = '' host = '' # fitsurl = '' source = '' reference = '' for tdi, td in enumerate(tds): if tdi == 0: butt = td.find('button') if not butt: break alias = butt.text.strip() alias = alias.replace('PSNJ', 'PSN J').replace('GAIA', 'Gaia') elif tdi == 1: name = (td.text.strip() .replace('PSNJ', 'PSN J') .replace('GAIA', 'Gaia')) if name.startswith('SN '): name = 'SN' + name[3:] if not name: name = alias if is_number(name[:4]): name = 'SN' + name oldname = name name = catalog.add_entry(name) reference = 'Asiago Supernova Catalogue' refurl = 'http://graspa.oapd.inaf.it/cgi-bin/sncat.php' secondarysource = catalog.entries[name].add_source( name=reference, url=refurl, secondary=True) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oldname, secondarysource) if alias != name: catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, alias, secondarysource) elif tdi == 2: host = td.text.strip() if host == 'anonymous': host = '' elif tdi == 3: discoverer = td.text.strip() elif tdi == 5: ra = td.text.strip() elif tdi == 6: dec = td.text.strip() elif tdi == 7: claimedtype = td.text.strip() elif tdi == 8: redshift = td.text.strip() # elif tdi == 9: # epochstr = td.text.strip() # if epochstr: # mjd = (astrotime(epochstr[:4] + '-' + epochstr[4:6] + # '-' + # str(floor(float(epochstr[6:]))).zfill(2)).mjd + # float(epochstr[6:]) - floor(float(epochstr[6:]))) # else: # mjd = '' elif tdi == 10: refs = td.findAll('a') source = '' reference = '' refurl = '' for ref in refs: if ref.text != 'REF': reference = ref.text refurl = ref['href'] if reference: source = catalog.entries[name].add_source( name=reference, url=refurl) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, secondarysource) sources = uniq_cdl( list(filter(None, [source, secondarysource]))) elif tdi == 12: pass # fitslink = td.find('a') # if fitslink: # fitsurl = fitslink['href'] if name: catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, claimedtype, sources) catalog.entries[name].add_quantity(SUPERNOVA.RA, ra, sources) catalog.entries[name].add_quantity(SUPERNOVA.DEC, dec, sources) catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, redshift, sources) catalog.entries[name].add_quantity(SUPERNOVA.DISCOVERER, discoverer, sources) catalog.entries[name].add_quantity(SUPERNOVA.HOST, host, sources) # if fitsurl: # response = urllib.request.urlopen( # 'http://sngroup.oapd.inaf.it./' + fitsurl) # compressed = io.BytesIO(response.read()) # decompressed = gzip.GzipFile(fileobj=compressed) # hdulist = fits.open(decompressed) # scidata = hdulist[0].data # print(hdulist[0].header) # # print(scidata[3]) # sys.exit() catalog.journal_entries() return
def do_cleanup(catalog): """Cleanup catalog after importing all data.""" task_str = catalog.get_current_task_str() # Set preferred names, calculate some columns based on imported data, # sanitize some fields keys = list(catalog.entries.keys()) cleanupcnt = 0 for oname in pbar(keys, task_str): # Some events may be merged in cleanup process, skip them if # non-existent. try: name = catalog.add_entry(oname) except Exception: catalog.log.warning( '"{}" was not found, suggests merge occurred in cleanup ' 'process.'.format(oname)) continue # Set the preferred name, switching to that name if name changed. name = catalog.entries[name].set_preferred_name() aliases = catalog.entries[name].get_aliases() catalog.entries[name].purge_bandless_photometry() catalog.entries[name].set_first_max_light() if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['MLS', 'SSS', 'CSS', 'GRB '] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:2])): discoverdate = ('/'.join([ '20' + alias.replace(prefix, '')[:2], alias.replace(prefix, '')[2:4], alias.replace(prefix, '')[4:6] ])) if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, discoverdate, source, derived=True) break if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]: break if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]: prefixes = [ 'ASASSN-', 'PS1-', 'PS1', 'PS', 'iPTF', 'PTF', 'SCP-', 'SNLS-', 'SPIRITS', 'LSQ', 'DES', 'SNHiTS', 'Gaia', 'GND', 'GNW', 'GSD', 'GSW', 'EGS', 'COS', 'OGLE', 'HST' ] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:2]) and is_number(alias.replace(prefix, '')[:1])): discoverdate = '20' + alias.replace(prefix, '')[:2] if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, discoverdate, source, derived=True) break if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]: break if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['SNF'] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:4])): discoverdate = ('/'.join([ alias.replace(prefix, '')[:4], alias.replace(prefix, '')[4:6], alias.replace(prefix, '')[6:8] ])) if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, discoverdate, source, derived=True) break if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]: break if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['PTFS', 'SNSDF'] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:2])): discoverdate = ('/'.join([ '20' + alias.replace(prefix, '')[:2], alias.replace(prefix, '')[2:4] ])) if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, discoverdate, source, derived=True) break if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]: break if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['AT', 'SN', 'OGLE-', 'SM ', 'KSN'] for alias in aliases: for prefix in prefixes: if alias.startswith(prefix): year = re.findall(r'\d+', alias) if len(year) == 1: year = year[0] else: continue if alias.replace(prefix, '').index(year) != 0: continue if (year and is_number(year) and '.' not in year and len(year) <= 4): discoverdate = year if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, discoverdate, source, derived=True) break if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]: break if (SUPERNOVA.RA not in catalog.entries[name] or SUPERNOVA.DEC not in catalog.entries[name]): prefixes = [ 'PSN J', 'MASJ', 'CSS', 'SSS', 'MASTER OT J', 'HST J', 'TCP J', 'MACS J', '2MASS J', 'EQ J', 'CRTS J', 'SMT J' ] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:6])): noprefix = alias.split(':')[-1].replace(prefix, '').replace( '.', '') decsign = '+' if '+' in noprefix else '-' noprefix = noprefix.replace('+', '|').replace('-', '|') nops = noprefix.split('|') if len(nops) < 2: continue rastr = nops[0] decstr = nops[1] ra = ':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + \ ('.' + rastr[6:] if len(rastr) > 6 else '') dec = ( decsign + ':'.join([decstr[:2], decstr[2:4], decstr[4:6]]) + ('.' + decstr[6:] if len(decstr) > 6 else '')) if catalog.args.verbose: tprint('Added ra/dec from name: ' + ra + ' ' + dec) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity(SUPERNOVA.RA, ra, source, derived=True) catalog.entries[name].add_quantity(SUPERNOVA.DEC, dec, source, derived=True) break if SUPERNOVA.RA in catalog.entries[name]: break no_host = (SUPERNOVA.HOST not in catalog.entries[name] or not any([ x[QUANTITY.VALUE] == 'Milky Way' for x in catalog.entries[name][SUPERNOVA.HOST] ])) if (SUPERNOVA.RA in catalog.entries[name] and SUPERNOVA.DEC in catalog.entries[name] and no_host): from astroquery.irsa_dust import IrsaDust if name not in catalog.extinctions_dict: try: ra_dec = catalog.entries[name][ SUPERNOVA.RA][0][QUANTITY.VALUE] + \ " " + \ catalog.entries[name][SUPERNOVA.DEC][0][QUANTITY.VALUE] result = IrsaDust.get_query_table(ra_dec, section='ebv') except (KeyboardInterrupt, SystemExit): raise except Exception: warnings.warn("Coordinate lookup for " + name + " failed in IRSA.") else: ebv = result['ext SandF mean'][0] ebverr = result['ext SandF std'][0] catalog.extinctions_dict[name] = [ebv, ebverr] if name in catalog.extinctions_dict: sources = uniq_cdl([ catalog.entries[name].add_self_source(), catalog.entries[name].add_source( bibcode='2011ApJ...737..103S') ]) (catalog.entries[name].add_quantity( SUPERNOVA.EBV, str(catalog.extinctions_dict[name][0]), sources, e_value=str(catalog.extinctions_dict[name][1]), derived=True)) if ((SUPERNOVA.HOST in catalog.entries[name] and (SUPERNOVA.HOST_RA not in catalog.entries[name] or SUPERNOVA.HOST_DEC not in catalog.entries[name]))): for host in catalog.entries[name][SUPERNOVA.HOST]: alias = host[QUANTITY.VALUE] if ' J' in alias and is_number(alias.split(' J')[-1][:6]): noprefix = alias.split(' J')[-1].split(':')[-1].replace( '.', '') decsign = '+' if '+' in noprefix else '-' noprefix = noprefix.replace('+', '|').replace('-', '|') nops = noprefix.split('|') if len(nops) < 2: continue rastr = nops[0] decstr = nops[1] hostra = (':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + ('.' + rastr[6:] if len(rastr) > 6 else '')) hostdec = decsign + ':'.join([ decstr[:2], decstr[2:4], decstr[4:6] ]) + ('.' + decstr[6:] if len(decstr) > 6 else '') if catalog.args.verbose: tprint('Added hostra/hostdec from name: ' + hostra + ' ' + hostdec) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity(SUPERNOVA.HOST_RA, hostra, source, derived=True) catalog.entries[name].add_quantity(SUPERNOVA.HOST_DEC, hostdec, source, derived=True) break if SUPERNOVA.HOST_RA in catalog.entries[name]: break if (SUPERNOVA.REDSHIFT not in catalog.entries[name] and SUPERNOVA.VELOCITY in catalog.entries[name]): # Find the "best" velocity to use for this bestsig = 0 for hv in catalog.entries[name][SUPERNOVA.VELOCITY]: sig = get_sig_digits(hv[QUANTITY.VALUE]) if sig > bestsig: besthv = hv[QUANTITY.VALUE] bestsrc = hv['source'] bestsig = sig if bestsig > 0 and is_number(besthv): voc = float(besthv) * 1.e5 / CLIGHT source = catalog.entries[name].add_self_source() sources = uniq_cdl([source] + bestsrc.split(',')) (catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, pretty_num(sqrt((1. + voc) / (1. - voc)) - 1., sig=bestsig), sources, kind='heliocentric', derived=True)) if (SUPERNOVA.REDSHIFT not in catalog.entries[name] and len(catalog.nedd_dict) > 0 and SUPERNOVA.HOST in catalog.entries[name]): reference = "NED-D" refurl = "http://ned.ipac.caltech.edu/Library/Distances/" refbib = "1991ASSL..171...89H" for host in catalog.entries[name][SUPERNOVA.HOST]: if host[QUANTITY.VALUE] in catalog.nedd_dict: source = catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') secondarysource = catalog.entries[name].add_source( name=reference, url=refurl, bibcode=refbib, secondary=True) meddist = statistics.median( catalog.nedd_dict[host[QUANTITY.VALUE]]) redz = z_at_value(cosmo.comoving_distance, float(meddist) * un.Mpc) redshift = pretty_num(redz, sig=get_sig_digits(str(meddist))) catalog.entries[name].add_quantity( [SUPERNOVA.REDSHIFT, SUPERNOVA.HOST_REDSHIFT], redshift, uniq_cdl([source, secondarysource]), kind='host', derived=True) if (SUPERNOVA.MAX_ABS_MAG not in catalog.entries[name] and SUPERNOVA.MAX_APP_MAG in catalog.entries[name] and SUPERNOVA.LUM_DIST in catalog.entries[name]): # Find the "best" distance to use for this bestsig = 0 for ld in catalog.entries[name][SUPERNOVA.LUM_DIST]: sig = get_sig_digits(ld[QUANTITY.VALUE]) if sig > bestsig: bestld = ld[QUANTITY.VALUE] bestsrc = ld[QUANTITY.SOURCE] bestsig = sig if bestsig > 0 and is_number(bestld) and float(bestld) > 0.: source = catalog.entries[name].add_self_source() sources = uniq_cdl([source] + bestsrc.split(',')) bestldz = z_at_value(cosmo.luminosity_distance, float(bestld) * un.Mpc) pnum = (float(catalog.entries[name][SUPERNOVA.MAX_APP_MAG][0][ QUANTITY.VALUE]) - 5.0 * (log10(float(bestld) * 1.0e6) - 1.0) + 2.5 * log10(1.0 + bestldz)) pnum = pretty_num(pnum, sig=bestsig + 1) catalog.entries[name].add_quantity(SUPERNOVA.MAX_ABS_MAG, pnum, sources, derived=True) if (SUPERNOVA.MAX_VISUAL_ABS_MAG not in catalog.entries[name] and SUPERNOVA.MAX_VISUAL_APP_MAG in catalog.entries[name] and SUPERNOVA.LUM_DIST in catalog.entries[name]): # Find the "best" distance to use for this bestsig = 0 for ld in catalog.entries[name][SUPERNOVA.LUM_DIST]: sig = get_sig_digits(ld[QUANTITY.VALUE]) if sig > bestsig: bestld = ld[QUANTITY.VALUE] bestsrc = ld[QUANTITY.SOURCE] bestsig = sig if bestsig > 0 and is_number(bestld) and float(bestld) > 0.: source = catalog.entries[name].add_self_source() sources = uniq_cdl([source] + bestsrc.split(',')) # FIX: what's happening here?! pnum = (float(catalog.entries[name][ SUPERNOVA.MAX_VISUAL_APP_MAG][0][QUANTITY.VALUE]) - 5.0 * (log10(float(bestld) * 1.0e6) - 1.0)) pnum = pretty_num(pnum, sig=bestsig + 1) catalog.entries[name].add_quantity( SUPERNOVA.MAX_VISUAL_ABS_MAG, pnum, sources, derived=True) if SUPERNOVA.REDSHIFT in catalog.entries[name]: # Find the "best" redshift to use for this bestz, bestkind, bestsig, bestsrc = catalog.entries[ name].get_best_redshift() if bestsig > 0: try: bestz = float(bestz) except Exception: print(catalog.entries[name]) raise if SUPERNOVA.VELOCITY not in catalog.entries[name]: source = catalog.entries[name].add_self_source() # FIX: what's happening here?! pnum = CLIGHT / KM * \ ((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.) pnum = pretty_num(pnum, sig=bestsig) catalog.entries[name].add_quantity( SUPERNOVA.VELOCITY, pnum, source, kind=(SUPERNOVA.VELOCITY.kind_preference[bestkind] if bestkind else '')) if bestz > 0.: if SUPERNOVA.LUM_DIST not in catalog.entries[name]: dl = cosmo.luminosity_distance(bestz) sources = [ catalog.entries[name].add_self_source(), catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') ] sources = uniq_cdl(sources + bestsrc.split(',')) catalog.entries[name].add_quantity( SUPERNOVA.LUM_DIST, pretty_num(dl.value, sig=bestsig + 1), sources, kind=(SUPERNOVA.LUM_DIST.kind_preference[bestkind] if bestkind else ''), derived=True) if (SUPERNOVA.MAX_ABS_MAG not in catalog.entries[name] and SUPERNOVA.MAX_APP_MAG in catalog.entries[name]): source = catalog.entries[name].add_self_source() pnum = pretty_num( float(catalog.entries[name][ SUPERNOVA.MAX_APP_MAG][0][QUANTITY.VALUE]) - 5.0 * (log10(dl.to('pc').value) - 1.0) + 2.5 * log10(1.0 + bestz), sig=bestsig + 1) catalog.entries[name].add_quantity( SUPERNOVA.MAX_ABS_MAG, pnum, sources, derived=True) if (SUPERNOVA.MAX_VISUAL_ABS_MAG not in catalog.entries[name] and SUPERNOVA.MAX_VISUAL_APP_MAG in catalog.entries[name]): source = catalog.entries[name].add_self_source() pnum = pretty_num(float(catalog.entries[name][ SUPERNOVA.MAX_VISUAL_APP_MAG][0][ QUANTITY.VALUE]) - 5.0 * (log10(dl.to('pc').value) - 1.0), sig=bestsig + 1) catalog.entries[name].add_quantity( SUPERNOVA.MAX_VISUAL_ABS_MAG, pnum, sources, derived=True) if SUPERNOVA.COMOVING_DIST not in catalog.entries[name]: cd = cosmo.comoving_distance(bestz) sources = [ catalog.entries[name].add_self_source(), catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') ] sources = uniq_cdl(sources + bestsrc.split(',')) catalog.entries[name].add_quantity( SUPERNOVA.COMOVING_DIST, pretty_num(cd.value, sig=bestsig), sources, derived=True) if SUPERNOVA.HOST_REDSHIFT in catalog.entries[name]: # Find the "best" redshift to use for this bestz, bestkind, bestsig, bestsrc = catalog.entries[ name].get_best_redshift(SUPERNOVA.HOST_REDSHIFT) if bestsig > 0: try: bestz = float(bestz) except Exception: print(catalog.entries[name]) raise if SUPERNOVA.HOST_VELOCITY not in catalog.entries[name]: source = catalog.entries[name].add_self_source() # FIX: what's happening here?! pnum = CLIGHT / KM * \ ((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.) pnum = pretty_num(pnum, sig=bestsig) catalog.entries[name].add_quantity( SUPERNOVA.HOST_VELOCITY, pnum, source, kind=(SUPERNOVA.HOST_VELOCITY.kind_preference[bestkind] if bestkind else '')) if bestz > 0.: if SUPERNOVA.HOST_LUM_DIST not in catalog.entries[name]: dl = cosmo.luminosity_distance(bestz) sources = [ catalog.entries[name].add_self_source(), catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') ] sources = uniq_cdl(sources + bestsrc.split(',')) catalog.entries[name].add_quantity( SUPERNOVA.HOST_LUM_DIST, pretty_num(dl.value, sig=bestsig + 1), sources, kind=(SUPERNOVA.HOST_LUM_DIST. kind_preference[bestkind] if bestkind else ''), derived=True) if SUPERNOVA.HOST_COMOVING_DIST not in catalog.entries[ name]: cd = cosmo.comoving_distance(bestz) sources = [ catalog.entries[name].add_self_source(), catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') ] sources = uniq_cdl(sources + bestsrc.split(',')) catalog.entries[name].add_quantity( SUPERNOVA.HOST_COMOVING_DIST, pretty_num(cd.value, sig=bestsig), sources, derived=True) if all([ x in catalog.entries[name] for x in [ SUPERNOVA.RA, SUPERNOVA.DEC, SUPERNOVA.HOST_RA, SUPERNOVA.HOST_DEC ] ]): # For now just using first coordinates that appear in entry try: c1 = coord( ra=catalog.entries[name][SUPERNOVA.RA][0][QUANTITY.VALUE], dec=catalog.entries[name][SUPERNOVA.DEC][0][ QUANTITY.VALUE], unit=(un.hourangle, un.deg)) c2 = coord(ra=catalog.entries[name][SUPERNOVA.HOST_RA][0][ QUANTITY.VALUE], dec=catalog.entries[name][SUPERNOVA.HOST_DEC][0][ QUANTITY.VALUE], unit=(un.hourangle, un.deg)) except (KeyboardInterrupt, SystemExit): raise except Exception: pass else: sources = uniq_cdl([catalog.entries[name].add_self_source()] + catalog.entries[name][SUPERNOVA.RA][0][ QUANTITY.SOURCE].split(',') + catalog.entries[name][SUPERNOVA.DEC][0][ QUANTITY.SOURCE].split(',') + catalog.entries[name][SUPERNOVA.HOST_RA][0][ QUANTITY.SOURCE].split(',') + catalog.entries[name][SUPERNOVA.HOST_DEC][0] [QUANTITY.SOURCE].split(',')) if SUPERNOVA.HOST_OFFSET_ANG not in catalog.entries[name]: hosa = Decimal(c1.separation(c2).arcsecond) hosa = pretty_num(hosa) catalog.entries[name].add_quantity( SUPERNOVA.HOST_OFFSET_ANG, hosa, sources, derived=True, u_value='arcseconds') if (SUPERNOVA.COMOVING_DIST in catalog.entries[name] and SUPERNOVA.REDSHIFT in catalog.entries[name] and SUPERNOVA.HOST_OFFSET_DIST not in catalog.entries[name]): offsetsig = get_sig_digits(catalog.entries[name][ SUPERNOVA.HOST_OFFSET_ANG][0][QUANTITY.VALUE]) sources = uniq_cdl( sources.split(',') + (catalog.entries[name][SUPERNOVA.COMOVING_DIST][0][ QUANTITY.SOURCE]).split(',') + (catalog.entries[name][SUPERNOVA.REDSHIFT][0][ QUANTITY.SOURCE]).split(',')) (catalog.entries[name].add_quantity( SUPERNOVA.HOST_OFFSET_DIST, pretty_num( float(catalog.entries[name][ SUPERNOVA.HOST_OFFSET_ANG][0][QUANTITY.VALUE]) / 3600. * (pi / 180.) * float(catalog.entries[name][ SUPERNOVA.COMOVING_DIST][0][QUANTITY.VALUE]) * 1000. / (1.0 + float(catalog.entries[name][ SUPERNOVA.REDSHIFT][0][QUANTITY.VALUE])), sig=offsetsig), sources)) catalog.entries[name].sanitize() catalog.journal_entries(bury=True, final=True, gz=True) cleanupcnt = cleanupcnt + 1 if catalog.args.travis and cleanupcnt % 1000 == 0: break catalog.save_caches() return
def do_snf_specta(catalog): task_str = catalog.get_current_task_str() bibcodes = {'SN2005gj': '2006ApJ...650..510A', 'SN2006D': '2007ApJ...654L..53T', 'SN2007if': '2010ApJ...713.1073S', 'SN2011fe': '2013A&A...554A..27P'} oldname = '' snfcnt = 0 eventfolders = next(os.walk(os.path.join( catalog.get_current_task_repo(), 'SNFactory')))[1] for eventfolder in pbar(eventfolders, task_str): oname = eventfolder name = catalog.get_preferred_name(oname) if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) sec_reference = 'Nearby Supernova Factory' sec_refurl = 'http://snfactory.lbl.gov/' sec_bibcode = '2002SPIE.4836...61A' sec_source = catalog.entries[name].add_source( name=sec_reference, url=sec_refurl, bibcode=sec_bibcode, secondary=True) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oname, sec_source) bibcode = bibcodes[oname] source = catalog.entries[name].add_source(bibcode=bibcode) sources = uniq_cdl([source, sec_source]) use_path = os.path.join( catalog.get_current_task_repo(), 'SNFactory', eventfolder, '*.dat') eventspectra = glob(use_path) for spectrum in pbar(eventspectra, task_str): filename = os.path.basename(spectrum) with open(spectrum) as spec_file: specdata = list(csv.reader( spec_file, delimiter=' ', skipinitialspace=True)) specdata = list(filter(None, specdata)) newspec = [] time = '' telescope = '' instrument = '' observer = '' observatory = '' if 'Keck_20060202_R' in spectrum: time = '53768.23469' elif 'Spectrum05_276' in spectrum: time = pretty_num(astrotime('2005-10-03').mjd, sig=5) elif 'Spectrum05_329' in spectrum: time = pretty_num(astrotime('2005-11-25').mjd, sig=5) elif 'Spectrum05_336' in spectrum: time = pretty_num(astrotime('2005-12-02').mjd, sig=5) for row in specdata: if row[0][0] == '#': joinrow = (' '.join(row)).split('=') if len(joinrow) < 2: continue field = joinrow[0].strip('# ') value = joinrow[1].split('/')[0].strip('\' ') if not time: if field == 'JD': time = str(jd_to_mjd(Decimal(value))) elif field == 'MJD': time = value elif field == 'MJD-OBS': time = value if field == 'OBSERVER': observer = value.capitalize() if field == 'OBSERVAT': observatory = value.capitalize() if field == 'TELESCOP': telescope = value.capitalize() if field == 'INSTRUME': instrument = value.capitalize() else: newspec.append(row) if not time: raise ValueError('Time missing from spectrum.') specdata = newspec haserrors = len(specdata[0]) == 3 and specdata[ 0][2] and specdata[0][2] != 'NaN' specdata = [list(i) for i in zip(*specdata)] wavelengths = specdata[0] fluxes = specdata[1] errors = '' if haserrors: errors = specdata[2] unit_err = ('Variance' if oldname == 'SN2011fe' else 'erg/s/cm^2/Angstrom') unit_flx = 'erg/s/cm^2/Angstrom' catalog.entries[name].add_spectrum( u_wavelengths='Angstrom', u_fluxes=unit_flx, u_time='MJD', time=time, wavelengths=wavelengths, fluxes=fluxes, errors=errors, observer=observer, observatory=observatory, telescope=telescope, instrument=instrument, u_errors=unit_err, source=sources, filename=filename) snfcnt = snfcnt + 1 if (catalog.args.travis and snfcnt % catalog.TRAVIS_QUERY_LIMIT == 0): break catalog.journal_entries() return
def do_asassn(catalog): """Import list of ASASSN events.""" task_str = catalog.get_current_task_str() asn_url = 'http://www.astronomy.ohio-state.edu/~assassin/sn_list.html' html = catalog.load_url( asn_url, os.path.join(catalog.get_current_task_repo(), 'ASASSN/sn_list.html')) if not html: return bs = BeautifulSoup(html, 'html5lib') trs = bs.find('table').findAll('tr') for tri, tr in enumerate(pbar(trs, task_str)): name = '' ra = '' dec = '' redshift = '' hostoff = '' claimedtype = '' host = '' atellink = '' typelink = '' if tri == 0: continue tds = tr.findAll('td') for tdi, td in enumerate(tds): if tdi == 1: name = catalog.add_entry(td.text.strip()) atellink = td.find('a') if atellink: atellink = atellink['href'] else: atellink = '' if tdi == 2: discdate = td.text.replace('-', '/') if tdi == 3: ra = td.text if tdi == 4: dec = td.text if tdi == 5: redshift = td.text if tdi == 8: hostoff = td.text if tdi == 9: claimedtype = td.text typelink = td.find('a') if typelink: typelink = typelink['href'] else: typelink = '' if tdi == 12: host = td.text sources = [ catalog.entries[name].add_source(url=asn_url, name='ASAS-SN Supernovae') ] typesources = sources[:] if atellink: sources.append((catalog.entries[name].add_source( name='ATel ' + atellink.split('=')[-1], url=atellink))) if typelink: typesources.append((catalog.entries[name].add_source( name='ATel ' + typelink.split('=')[-1], url=typelink))) sources = ','.join(sources) typesources = ','.join(typesources) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, sources) catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, discdate, sources) catalog.entries[name].add_quantity(SUPERNOVA.RA, ra, sources, u_value='floatdegrees') catalog.entries[name].add_quantity(SUPERNOVA.DEC, dec, sources, u_value='floatdegrees') catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, redshift, sources) catalog.entries[name].add_quantity(SUPERNOVA.HOST_OFFSET_ANG, hostoff, sources, u_value='arcseconds') for ct in claimedtype.split('/'): if ct != 'Unk': catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, ct, typesources) if host != 'Uncatalogued': catalog.entries[name].add_quantity(SUPERNOVA.HOST, host, sources) catalog.journal_entries() return
def do_mosfit(catalog): task_str = catalog.get_current_task_str() try: with open('mosfit.key', 'r') as f: mosfitkey = f.read().splitlines()[0] except: catalog.log.warning('MOSFiT key not found, make sure a file named ' '`mosfit.key` containing the key is placed the ' 'astrocats directory.') mosfitkey = '' dbx = dropbox.Dropbox(mosfitkey) files = list( sorted([ x.name for x in dbx.files_list_folder('').entries if not x.name.startswith('.') ])) fdir = os.path.join(catalog.get_current_task_repo(), 'MOSFiT') if not os.path.isdir(fdir): os.mkdir(fdir) efiles = [x.split('/')[-1] for x in glob(os.path.join(fdir, '*'))] old_name = '' for fname in pbar(files, desc=task_str): if fname in efiles: efiles.remove(fname) fpath = os.path.join(fdir, fname) if not os.path.isfile(fpath): md, res = dbx.files_download('/' + fname) jtxt = res.content with open(fpath, 'wb') as f: f.write(jtxt) name = fname.split('_')[-3] new_entry = Supernova.init_from_file(catalog, path=fpath, compare_to_existing=False) old_entry = None if name in catalog.entries: if catalog.entries[name]._stub: old_entry = Supernova.init_from_file(catalog, name=name, compare_to_existing=False) else: old_entry = catalog.entries[name] if old_entry: catalog.copy_entry_to_entry(new_entry, old_entry, compare_to_existing=False) catalog.entries[name] = old_entry else: catalog.entries[name] = new_entry if old_name != name: catalog.journal_entries() old_name = name for fname in efiles: os.remove(os.path.join(fdir, fname)) return
def do_cfa_spectra(catalog): """Import spectra from the CfA archive.""" task_str = catalog.get_current_task_str() # II spectra oldname = '' file_names = next( os.walk(os.path.join(catalog.get_current_task_repo(), 'CfA_SNII')))[1] for ni, name in enumerate(pbar_strings(file_names, task_str)): fullpath = os.path.join(catalog.get_current_task_repo(), 'CfA_SNII/') + name origname = name if name.startswith('sn') and is_number(name[2:6]): name = 'SN' + name[2:] name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) reference = 'CfA Supernova Archive' refurl = 'https://www.cfa.harvard.edu/supernova/SNarchive.html' source = catalog.entries[name].add_source( name=reference, url=refurl, secondary=True, acknowledgment=ACKN_CFA) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) for fi, fname in enumerate( sorted( glob(fullpath + '/*'), key=lambda s: s.lower())): filename = os.path.basename(fname) fileparts = filename.split('-') if origname.startswith('sn') and is_number(origname[2:6]): year = fileparts[1][:4] month = fileparts[1][4:6] day = fileparts[1][6:] instrument = fileparts[2].split('.')[0] else: year = fileparts[2][:4] month = fileparts[2][4:6] day = fileparts[2][6:] instrument = fileparts[3].split('.')[0] time = str( astrotime(year + '-' + month + '-' + str(floor(float(day))) .zfill(2)).mjd + float(day) - floor(float(day))) f = open(fname, 'r') data = csv.reader(f, delimiter=' ', skipinitialspace=True) data = [list(i) for i in zip(*data)] wavelengths = data[0] fluxes = data[1] errors = data[2] sources = uniq_cdl([ source, (catalog.entries[name] .add_source(bibcode='2017arXiv170601030H')) ]) catalog.entries[name].add_spectrum( u_wavelengths='Angstrom', u_fluxes='erg/s/cm^2/Angstrom', filename=filename, wavelengths=wavelengths, fluxes=fluxes, u_time='MJD' if time else '', time=time, instrument=instrument, u_errors='ergs/s/cm^2/Angstrom', errors=errors, source=sources, dereddened=False, deredshifted=False) if catalog.args.travis and ni >= catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() # Ia spectra oldname = '' file_names = next( os.walk(os.path.join(catalog.get_current_task_repo(), 'CfA_SNIa')))[1] for ni, name in enumerate(pbar_strings(file_names, task_str)): fullpath = os.path.join(catalog.get_current_task_repo(), 'CfA_SNIa/') + name origname = name if name.startswith('sn') and is_number(name[2:6]): name = 'SN' + name[2:] if name.startswith('snf') and is_number(name[3:7]): name = 'SNF' + name[3:] name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) reference = 'CfA Supernova Archive' refurl = 'https://www.cfa.harvard.edu/supernova/SNarchive.html' source = catalog.entries[name].add_source( name=reference, url=refurl, secondary=True, acknowledgment=ACKN_CFA) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) for fi, fname in enumerate( sorted( glob(fullpath + '/*'), key=lambda s: s.lower())): filename = os.path.basename(fname) fileparts = filename.split('-') if origname.startswith('sn') and is_number(origname[2:6]): year = fileparts[1][:4] month = fileparts[1][4:6] day = fileparts[1][6:] instrument = fileparts[2].split('.')[0] else: year = fileparts[2][:4] month = fileparts[2][4:6] day = fileparts[2][6:] instrument = fileparts[3].split('.')[0] time = str( astrotime(year + '-' + month + '-' + str(floor(float(day))) .zfill(2)).mjd + float(day) - floor(float(day))) f = open(fname, 'r') data = csv.reader(f, delimiter=' ', skipinitialspace=True) data = [list(i) for i in zip(*data)] wavelengths = data[0] fluxes = data[1] errors = data[2] sources = uniq_cdl([ source, (catalog.entries[name] .add_source(bibcode='2012AJ....143..126B')), (catalog.entries[name] .add_source(bibcode='2008AJ....135.1598M')) ]) catalog.entries[name].add_spectrum( u_wavelengths='Angstrom', u_fluxes='erg/s/cm^2/Angstrom', filename=filename, wavelengths=wavelengths, fluxes=fluxes, u_time='MJD' if time else '', time=time, instrument=instrument, u_errors='ergs/s/cm^2/Angstrom', errors=errors, source=sources, dereddened=False, deredshifted=False) if catalog.args.travis and ni >= catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() # Ibc spectra oldname = '' file_names = next( os.walk(os.path.join(catalog.get_current_task_repo(), 'CfA_SNIbc')))[1] for ni, name in enumerate(pbar(file_names, task_str)): fullpath = os.path.join(catalog.get_current_task_repo(), 'CfA_SNIbc/') + name if name.startswith('sn') and is_number(name[2:6]): name = 'SN' + name[2:] name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) reference = 'CfA Supernova Archive' refurl = 'https://www.cfa.harvard.edu/supernova/SNarchive.html' source = catalog.entries[name].add_source( name=reference, url=refurl, secondary=True, acknowledgment=ACKN_CFA) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) for fi, fname in enumerate( sorted( glob(fullpath + '/*'), key=lambda s: s.lower())): filename = os.path.basename(fname) fileparts = filename.split('-') instrument = '' year = fileparts[1][:4] month = fileparts[1][4:6] day = fileparts[1][6:].split('.')[0] if len(fileparts) > 2: instrument = fileparts[-1].split('.')[0] time = str( astrotime(year + '-' + month + '-' + str(floor(float(day))) .zfill(2)).mjd + float(day) - floor(float(day))) f = open(fname, 'r') data = csv.reader(f, delimiter=' ', skipinitialspace=True) data = [list(i) for i in zip(*data)] wavelengths = data[0] fluxes = data[1] sources = uniq_cdl([ source, catalog.entries[name] .add_source(bibcode='2014AJ....147...99M') ]) catalog.entries[name].add_spectrum( u_wavelengths='Angstrom', u_fluxes='erg/s/cm^2/Angstrom', wavelengths=wavelengths, filename=filename, fluxes=fluxes, u_time='MJD' if time else '', time=time, instrument=instrument, source=sources, dereddened=False, deredshifted=False) if catalog.args.travis and ni >= catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() # Other spectra oldname = '' file_names = next( os.walk(os.path.join(catalog.get_current_task_repo(), 'CfA_Extra')))[1] for ni, name in enumerate(pbar_strings(file_names, task_str)): fullpath = os.path.join(catalog.get_current_task_repo(), 'CfA_Extra/') + name if name.startswith('sn') and is_number(name[2:6]): name = 'SN' + name[2:] name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) reference = 'CfA Supernova Archive' refurl = 'https://www.cfa.harvard.edu/supernova/SNarchive.html' source = catalog.entries[name].add_source( name=reference, url=refurl, secondary=True, acknowledgment=ACKN_CFA) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) for fi, fname in enumerate( sorted( glob(fullpath + '/*'), key=lambda s: s.lower())): if not os.path.isfile(fname): continue filename = os.path.basename(fname) if ((not filename.startswith('sn') or not filename.endswith('flm') or any( x in filename for x in ['-interp', '-z', '-dered', '-obj', '-gal']))): continue fileparts = filename.split('.')[0].split('-') instrument = '' time = '' if len(fileparts) > 1: year = fileparts[1][:4] month = fileparts[1][4:6] day = fileparts[1][6:] if is_number(year) and is_number(month) and is_number(day): if len(fileparts) > 2: instrument = fileparts[-1] time = str( astrotime(year + '-' + month + '-' + str( floor(float(day))).zfill(2)).mjd + float(day) - floor(float(day))) f = open(fname, 'r') data = csv.reader(f, delimiter=' ', skipinitialspace=True) data = [list(i) for i in zip(*data)] wavelengths = data[0] fluxes = [str(Decimal(x) * Decimal(1.0e-15)) for x in data[1]] catalog.entries[name].add_spectrum( u_wavelengths='Angstrom', u_fluxes='erg/s/cm^2/Angstrom', wavelengths=wavelengths, filename=filename, fluxes=fluxes, u_time='MJD' if time else '', time=time, instrument=instrument, source=source, dereddened=False, deredshifted=False) if catalog.args.travis and ni >= catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() return
def do_ogle(catalog): task_str = catalog.get_current_task_str() basenames = ['transients', 'transients/2014b', 'transients/2014', 'transients/2013', 'transients/2012'] oglenames = [] ogleupdate = [True, False, False, False, False] for b, bn in enumerate(pbar(basenames, task_str)): if catalog.args.update and not ogleupdate[b]: continue filepath = os.path.join(catalog.get_current_task_repo(), 'OGLE-') filepath += bn.replace('/', '-') + '-transients.html' htmltxt = catalog.load_cached_url( 'http://ogle.astrouw.edu.pl/ogle4/' + bn + '/transients.html', filepath) if not htmltxt: continue soup = BeautifulSoup(htmltxt, 'html5lib') links = soup.findAll('a') breaks = soup.findAll('br') datalinks = [] datafnames = [] for a in links: if a.has_attr('href'): if '.dat' in a['href']: datalinks.append( 'http://ogle.astrouw.edu.pl/ogle4/' + bn + '/' + a['href']) datafnames.append(bn.replace('/', '-') + '-' + a['href'].replace('/', '-')) ec = -1 reference = 'OGLE-IV Transient Detection System' refurl = 'http://ogle.astrouw.edu.pl/ogle4/transients/transients.html' for br in pbar(breaks, task_str): sibling = br.nextSibling if 'Ra,Dec=' in sibling: line = sibling.replace('\n', '').split('Ra,Dec=') name = line[0].strip() ec += 1 if 'NOVA' in name or 'dupl' in name: continue if name in oglenames: continue oglenames.append(name) name = catalog.add_entry(name) mySibling = sibling.nextSibling atelref = '' claimedtype = '' while 'Ra,Dec=' not in mySibling: if isinstance(mySibling, NavigableString): if 'Phot.class=' in str(mySibling): claimedtype = re.sub( r'\([^)]*\)', '', str(mySibling).split('=')[-1]) claimedtype = claimedtype.replace('SN', '').strip() if isinstance(mySibling, Tag): atela = mySibling if (atela and atela.has_attr('href') and 'astronomerstelegram' in atela['href']): atelref = atela.contents[0].strip() atelurl = atela['href'] mySibling = mySibling.nextSibling if mySibling is None: break # nextSibling = sibling.nextSibling # if ((isinstance(nextSibling, Tag) and # nextSibling.has_attr('alt') and # nextSibling.contents[0].strip() != 'NED')): # radec = nextSibling.contents[0].strip().split() # else: # radec = line[-1].split() # ra = radec[0] # dec = radec[1] fname = os.path.join(catalog.get_current_task_repo(), 'OGLE/') + datafnames[ec] if (catalog.current_task.load_archive(catalog.args) and os.path.isfile(fname)): with open(fname, 'r') as f: csvtxt = f.read() else: response = urllib.request.urlopen(datalinks[ec]) with open(fname, 'w') as f: csvtxt = response.read().decode('utf-8') f.write(csvtxt) lcdat = csvtxt.splitlines() sources = [catalog.entries[name].add_source( name=reference, url=refurl)] catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, name, sources[0]) if atelref and atelref != 'ATel#----': sources.append(catalog.entries[name].add_source( name=atelref, url=atelurl)) sources = uniq_cdl(sources) if name.startswith('OGLE'): if name[4] == '-': if is_number(name[5:9]): catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, name[5:9], sources) else: if is_number(name[4:6]): catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, '20' + name[4:6], sources) # RA and Dec from OGLE pages currently not reliable # catalog.entries[name].add_quantity(SUPERNOVA.RA, ra, sources) # catalog.entries[name].add_quantity(SUPERNOVA.DEC, dec, # sources) if claimedtype and claimedtype != '-': catalog.entries[name].add_quantity( SUPERNOVA.CLAIMED_TYPE, claimedtype, sources) elif ('SN' not in name and SUPERNOVA.CLAIMED_TYPE not in catalog.entries[name]): catalog.entries[name].add_quantity( SUPERNOVA.CLAIMED_TYPE, 'Candidate', sources) for row in lcdat: row = row.split() mjd = str(jd_to_mjd(Decimal(row[0]))) magnitude = row[1] if float(magnitude) > 90.0: continue e_mag = row[2] upperlimit = False if e_mag == '-1' or float(e_mag) > 10.0: e_mag = '' upperlimit = True catalog.entries[name].add_photometry( time=mjd, band='I', magnitude=magnitude, e_magnitude=e_mag, system='Vega', source=sources, upperlimit=upperlimit) if catalog.args.update: catalog.journal_entries() catalog.journal_entries() return
def do_snhunt(catalog): task_str = catalog.get_current_task_str() snh_url = 'http://nesssi.cacr.caltech.edu/catalina/current.html' html = catalog.load_url( snh_url, os.path.join(catalog.get_current_task_repo(), 'SNhunt/current.html')) if not html: return text = html.splitlines() findtable = False for ri, row in enumerate(text): if 'Supernova Discoveries' in row: findtable = True if findtable and '<table' in row: tstart = ri + 1 if findtable and '</table>' in row: tend = ri - 1 tablestr = '<html><body><table>' for row in text[tstart:tend]: if row[:3] == 'tr>': tablestr = tablestr + '<tr>' + row[3:] else: tablestr = tablestr + row tablestr = tablestr + '</table></body></html>' bs = BeautifulSoup(tablestr, 'html5lib') trs = bs.find('table').findAll('tr') for tr in pbar(trs, task_str): cols = [str(xx.text) for xx in tr.findAll('td')] if not cols: continue name = re.sub('<[^<]+?>', '', cols[4]).strip().replace(' ', '').replace('SNHunt', 'SNhunt') name = catalog.add_entry(name) source = catalog.entries[name].add_source(name='Supernova Hunt', url=snh_url) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) host = re.sub('<[^<]+?>', '', cols[1]).strip().replace('_', ' ') catalog.entries[name].add_quantity(SUPERNOVA.HOST, host, source) catalog.entries[name].add_quantity(SUPERNOVA.RA, cols[2], source, u_value='floatdegrees') catalog.entries[name].add_quantity(SUPERNOVA.DEC, cols[3], source, u_value='floatdegrees') dd = cols[0] discoverdate = dd[:4] + '/' + dd[4:6] + '/' + dd[6:8] catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, discoverdate, source) discoverers = cols[5].split('/') for discoverer in discoverers: catalog.entries[name].add_quantity(SUPERNOVA.DISCOVERER, 'CRTS', source) catalog.entries[name].add_quantity(SUPERNOVA.DISCOVERER, discoverer, source) if catalog.args.update: catalog.journal_entries() catalog.journal_entries() return