def do_internal(catalog): """Load events from files in the 'internal' repository, and save them.""" task_str = catalog.get_current_task_str() path_pattern = os.path.join(catalog.get_current_task_repo(), '*.json') files = glob(path_pattern) catalog.log.debug("found {} files matching '{}'".format( len(files), path_pattern)) for datafile in pbar_strings(files, task_str): new_entry = Supernova.init_from_file( catalog, path=datafile, clean=True, merge=True) name = new_entry[SUPERNOVA.NAME] old_name = None for alias in new_entry.get_aliases(): if catalog.entry_exists(alias): old_name = catalog.get_preferred_name(alias) if catalog.entries[old_name]._stub: catalog.add_entry(old_name) break if old_name: old_entry = deepcopy(catalog.entries[old_name]) catalog.copy_entry_to_entry(new_entry, old_entry) catalog.entries[old_name] = old_entry else: catalog.entries[name] = new_entry catalog.journal_entries() return
def do_internal(catalog): """Load events from files in the 'internal' repository, and save them.""" task_str = catalog.get_current_task_str() path_pattern = os.path.join(catalog.get_current_task_repo(), '*.json') files = glob(path_pattern) catalog.log.debug("found {} files matching '{}'".format( len(files), path_pattern)) for datafile in pbar_strings(files, task_str): new_entry = FastStars.init_from_file(catalog, path=datafile, clean=True, merge=True) name = new_entry[FASTSTARS.NAME] old_name = None for alias in new_entry.get_aliases(): if catalog.entry_exists(alias): old_name = catalog.get_preferred_name(alias) if catalog.entries[old_name]._stub: catalog.add_entry(old_name) break if old_name: old_entry = deepcopy(catalog.entries[old_name]) catalog.copy_entry_to_entry(new_entry, old_entry) catalog.entries[old_name] = old_entry else: catalog.entries[name] = new_entry catalog.journal_entries() return
def do_csp_photo(catalog): import re cspbands = ['u', 'B', 'V', 'g', 'r', 'i', 'Y', 'J', 'H', 'K'] file_names = glob( os.path.join(catalog.get_current_task_repo(), 'CSP/*.dat')) task_str = catalog.get_current_task_str() for fname in pbar_strings(file_names, task_str): tsvin = csv.reader(open(fname, 'r'), delimiter='\t', skipinitialspace=True) eventname = os.path.basename(os.path.splitext(fname)[0]) eventparts = eventname.split('opt+') name = clean_snname(eventparts[0]) name = catalog.add_entry(name) reference = 'Carnegie Supernova Project' refbib = '2010AJ....139..519C' refurl = 'http://csp.obs.carnegiescience.edu/data' source = catalog.entries[name].add_source(bibcode=refbib, name=reference, url=refurl) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) year = re.findall(r'\d+', name)[0] catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, year, source) for r, row in enumerate(tsvin): if len(row) > 0 and row[0][0] == "#": if r == 2: redz = row[0].split(' ')[-1] catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, redz, source, kind='cmb') catalog.entries[name].add_quantity(SUPERNOVA.RA, row[1].split(' ')[-1], source) catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[2].split(' ')[-1], source) continue for v, val in enumerate(row): if v == 0: mjd = val elif v % 2 != 0: if float(row[v]) < 90.0: catalog.entries[name].add_photometry( time=mjd, u_time='MJD', observatory='LCO', band=cspbands[(v - 1) // 2], system='CSP', magnitude=row[v], e_magnitude=row[v + 1], source=source) catalog.journal_entries() return
def do_csp_spectra(catalog): """Import CSP spectra.""" oldname = '' task_str = catalog.get_current_task_str() file_names = glob(os.path.join(catalog.get_current_task_repo(), 'CSP/*')) for fi, fname in enumerate(pbar_strings(file_names, task_str)): filename = os.path.basename(fname) sfile = filename.split('.') if sfile[1] == 'txt': continue sfile = sfile[0] fileparts = sfile.split('_') name = 'SN20' + fileparts[0][2:] name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) telescope = fileparts[-2] instrument = fileparts[-1] source = catalog.entries[name].add_source( bibcode='2013ApJ...773...53F') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) data = csv.reader(open(fname, 'r'), delimiter=' ', skipinitialspace=True) specdata = [] for r, row in enumerate(data): if row[0] == '#JDate_of_observation:': jd = row[1].strip() time = str(jd_to_mjd(Decimal(jd))) elif row[0] == '#Redshift:': catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, row[1].strip(), source) if r < 7: continue specdata.append(list(filter(None, [x.strip(' ') for x in row]))) specdata = [list(i) for i in zip(*specdata)] wavelengths = specdata[0] fluxes = specdata[1] catalog.entries[name].add_spectrum(u_wavelengths='Angstrom', u_fluxes='erg/s/cm^2/Angstrom', u_time='MJD', time=time, wavelengths=wavelengths, fluxes=fluxes, telescope=telescope, instrument=instrument, source=source, deredshifted=True, filename=filename) if catalog.args.travis and fi >= catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() return
def do_csp_spectra(catalog): """Import CSP spectra.""" oldname = '' task_str = catalog.get_current_task_str() file_names = glob(os.path.join(catalog.get_current_task_repo(), 'CSP/*')) for fi, fname in enumerate(pbar_strings(file_names, task_str)): filename = os.path.basename(fname) sfile = filename.split('.') if sfile[1] == 'txt': continue sfile = sfile[0] fileparts = sfile.split('_') name = 'SN20' + fileparts[0][2:] name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) telescope = fileparts[-2] instrument = fileparts[-1] source = catalog.entries[name].add_source( bibcode='2013ApJ...773...53F') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) data = csv.reader(open(fname, 'r'), delimiter=' ', skipinitialspace=True) specdata = [] for r, row in enumerate(data): if row[0] == '#JDate_of_observation:': jd = row[1].strip() time = str(jd_to_mjd(Decimal(jd))) elif row[0] == '#Redshift:': catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, row[1].strip(), source) if r < 7: continue specdata.append(list(filter(None, [x.strip(' ') for x in row]))) specdata = [list(i) for i in zip(*specdata)] wavelengths = specdata[0] fluxes = specdata[1] catalog.entries[name].add_spectrum( u_wavelengths='Angstrom', u_fluxes='erg/s/cm^2/Angstrom', u_time='MJD', time=time, wavelengths=wavelengths, fluxes=fluxes, telescope=telescope, instrument=instrument, source=source, deredshifted=True, filename=filename) if catalog.args.travis and fi >= catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() return
def do_csp_photo(catalog): """Import CSP photometry.""" import re file_names = glob( os.path.join(catalog.get_current_task_repo(), 'CSP/*.dat')) task_str = catalog.get_current_task_str() for fname in pbar_strings(file_names, task_str): tsvin = csv.reader(open(fname, 'r'), delimiter='\t', skipinitialspace=True) eventname = os.path.basename(os.path.splitext(fname)[0]) eventparts = eventname.split('opt+') name = clean_snname(eventparts[0]) name = catalog.add_entry(name) reference = 'Carnegie Supernova Project' refbib = '2010AJ....139..519C' refurl = 'http://csp.obs.carnegiescience.edu/data' source = catalog.entries[name].add_source( bibcode=refbib, name=reference, url=refurl) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) year = re.findall(r'\d+', name)[0] catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, year, source) for r, row in enumerate(tsvin): if len(row) > 0 and row[0][0] == "#": if r == 2: redz = row[0].split(' ')[-1] catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, redz, source, kind='cmb') catalog.entries[name].add_quantity( SUPERNOVA.RA, row[1].split(' ')[-1], source) catalog.entries[name].add_quantity( SUPERNOVA.DEC, row[2].split(' ')[-1], source) if 'MJD' in ''.join(row): cspbands = list(filter(None, [ x.strip() for x in ''.join(row).split('MJD')[-1].split('+/-')])) continue for v, val in enumerate(row): if v == 0: mjd = val elif v % 2 != 0: if float(row[v]) < 90.0: catalog.entries[name].add_photometry( time=mjd, u_time='MJD', observatory='LCO', band=cspbands[(v - 1) // 2], system='CSP', magnitude=row[v], e_magnitude=row[v + 1], source=source) catalog.journal_entries() return
def do_internal(catalog): """Load events from files in the 'internal' repository, and save them.""" task_str = catalog.get_current_task_str() path_pattern = os.path.join(catalog.get_current_task_repo(), '*.json') files = glob(path_pattern) catalog.log.debug("found {} files matching '{}'".format( len(files), path_pattern)) for datafile in pbar_strings(files, task_str): new_event = Cataclysmic.init_from_file( catalog, path=datafile, clean=True) catalog.entries.update({new_event[CATACLYSMIC.NAME]: new_event}) return
def do_internal(catalog): """Load events from files in the 'internal' repository, and save them. """ task_str = catalog.get_current_task_str() path_pattern = os.path.join(catalog.get_current_task_repo(), '*.json') files = glob(path_pattern) catalog.log.debug("found {} files matching '{}'".format( len(files), path_pattern)) for datafile in pbar_strings(files, task_str): new_event = TidalDisruption.init_from_file( catalog, path=datafile, clean=True) catalog.entries.update({new_event[TIDALDISRUPTION.NAME]: new_event}) return
def do_external_radio(catalog): task_str = catalog.get_current_task_str() path_pattern = os.path.join(catalog.get_current_task_repo(), '*.txt') for datafile in pbar_strings(glob(path_pattern), task_str): oldname = os.path.basename(datafile).split('.')[0] name = catalog.add_entry(oldname) radiosourcedict = OrderedDict() with open(datafile, 'r') as ff: for li, line in enumerate( [xx.strip() for xx in ff.read().splitlines()]): if line.startswith('(') and li <= len(radiosourcedict): key = line.split()[0] bibc = line.split()[-1] radiosourcedict[key] = catalog.entries[name].add_source( bibcode=bibc) elif li in [xx + len(radiosourcedict) for xx in range(3)]: continue else: cols = list(filter(None, line.split())) source = radiosourcedict[cols[6]] if float(cols[4]) == 0.0: eflux = '' upp = True else: eflux = cols[4] upp = False photodict = { PHOTOMETRY.TIME: cols[0], PHOTOMETRY.FREQUENCY: cols[2], PHOTOMETRY.U_FREQUENCY: 'GHz', PHOTOMETRY.FLUX_DENSITY: cols[3], PHOTOMETRY.E_FLUX_DENSITY: eflux, PHOTOMETRY.U_FLUX_DENSITY: 'µJy', PHOTOMETRY.UPPER_LIMIT: upp, PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.INSTRUMENT: cols[5], PHOTOMETRY.SOURCE: source } catalog.entries[name].add_photometry(**photodict) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oldname, source) catalog.journal_entries() return
def do_external_radio(catalog): task_str = catalog.get_current_task_str() path_pattern = os.path.join(catalog.get_current_task_repo(), '*.txt') for datafile in pbar_strings(glob(path_pattern), task_str): oldname = os.path.basename(datafile).split('.')[0] name = catalog.add_entry(oldname) radiosourcedict = OrderedDict() with open(datafile, 'r') as ff: for li, line in enumerate( [xx.strip() for xx in ff.read().splitlines()]): if line.startswith('(') and li <= len(radiosourcedict): key = line.split()[0] bibc = line.split()[-1] radiosourcedict[key] = catalog.entries[name].add_source( bibcode=bibc) elif li in [xx + len(radiosourcedict) for xx in range(3)]: continue else: cols = list(filter(None, line.split())) source = radiosourcedict[cols[6]] if float(cols[4]) == 0.0: eflux = '' upp = True else: eflux = cols[4] upp = False photodict = { PHOTOMETRY.TIME: cols[0], PHOTOMETRY.FREQUENCY: cols[2], PHOTOMETRY.U_FREQUENCY: 'GHz', PHOTOMETRY.FLUX_DENSITY: cols[3], PHOTOMETRY.E_FLUX_DENSITY: eflux, PHOTOMETRY.U_FLUX_DENSITY: 'µJy', PHOTOMETRY.UPPER_LIMIT: upp, PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.INSTRUMENT: cols[5], PHOTOMETRY.SOURCE: source } catalog.entries[name].add_photometry(**photodict) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oldname, source) catalog.journal_entries() return
def do_external_xray(catalog): """Import supernova X-ray data.""" task_str = catalog.get_current_task_str() path_pattern = os.path.join(catalog.get_current_task_repo(), '*.txt') for datafile in pbar_strings(glob(path_pattern), task_str): oldname = os.path.basename(datafile).split('.')[0] name = catalog.add_entry(oldname) with open(datafile, 'r') as ff: for li, line in enumerate(ff.read().splitlines()): if li == 0: source = catalog.entries[name].add_source( bibcode=line.split()[-1]) elif li in [1, 2, 3]: continue else: cols = list(filter(None, line.split())) photodict = { PHOTOMETRY.TIME: cols[:2], PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.ENERGY: cols[2:4], PHOTOMETRY.U_ENERGY: 'keV', PHOTOMETRY.COUNT_RATE: cols[4], PHOTOMETRY.FLUX: cols[6], PHOTOMETRY.UNABSORBED_FLUX: cols[8], PHOTOMETRY.U_FLUX: 'ergs/s/cm^2', PHOTOMETRY.PHOTON_INDEX: cols[15], PHOTOMETRY.INSTRUMENT: cols[17], PHOTOMETRY.NHMW: cols[11], PHOTOMETRY.UPPER_LIMIT: (float(cols[5]) < 0), PHOTOMETRY.SOURCE: source } catalog.entries[name].add_photometry(**photodict) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oldname, source) catalog.journal_entries() return
def do_external_xray(catalog): """Import supernova X-ray data.""" task_str = catalog.get_current_task_str() path_pattern = os.path.join(catalog.get_current_task_repo(), '*.txt') for datafile in pbar_strings(glob(path_pattern), task_str): oldname = os.path.basename(datafile).split('.')[0] name = catalog.add_entry(oldname) with open(datafile, 'r') as ff: for li, line in enumerate(ff.read().splitlines()): if li == 0: source = catalog.entries[name].add_source( bibcode=line.split()[-1]) elif li in [1, 2, 3]: continue else: cols = list(filter(None, line.split())) photodict = { PHOTOMETRY.TIME: cols[:2], PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.ENERGY: cols[2:4], PHOTOMETRY.U_ENERGY: 'keV', PHOTOMETRY.COUNT_RATE: cols[4], PHOTOMETRY.FLUX: cols[6], PHOTOMETRY.UNABSORBED_FLUX: cols[8], PHOTOMETRY.U_FLUX: 'ergs/s/cm^2', PHOTOMETRY.PHOTON_INDEX: cols[15], PHOTOMETRY.INSTRUMENT: cols[17], PHOTOMETRY.NHMW: cols[11], PHOTOMETRY.UPPER_LIMIT: (float(cols[5]) < 0), PHOTOMETRY.SOURCE: source } catalog.entries[name].add_photometry(**photodict) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oldname, source) catalog.journal_entries() return
def do_snls_spectra(catalog): """ """ task_str = catalog.get_current_task_str() result = Vizier.get_catalogs('J/A+A/507/85/table1') table = result[list(result.keys())[0]] table.convert_bytestring_to_unicode() datedict = {} for row in table: datedict['SNLS-' + row['SN']] = str(astrotime(row['Date']).mjd) oldname = '' file_names = glob(os.path.join(catalog.get_current_task_repo(), 'SNLS/*')) for fi, fname in enumerate(pbar_strings(file_names, task_str)): filename = os.path.basename(fname) fileparts = filename.split('_') name = 'SNLS-' + fileparts[1] name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2009A&A...507...85B') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, '20' + fileparts[1][:2], source) f = open(fname, 'r') data = csv.reader(f, delimiter=' ', skipinitialspace=True) specdata = [] for r, row in enumerate(data): if row[0] == '@TELESCOPE': telescope = row[1].strip() elif row[0] == '@REDSHIFT': catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, row[1].strip(), source) if r < 14: continue specdata.append(list(filter(None, [x.strip(' \t') for x in row]))) specdata = [list(i) for i in zip(*specdata)] wavelengths = specdata[1] fluxes = [ pretty_num(float(x) * 1.e-16, sig=get_sig_digits(x)) for x in specdata[2] ] # FIX: this isnt being used errors = [ pretty_num(float(x) * 1.e-16, sig=get_sig_digits(x)) for x in specdata[3] ] fluxunit = 'erg/s/cm^2/Angstrom' specdict = { SPECTRUM.WAVELENGTHS: wavelengths, SPECTRUM.FLUXES: fluxes, SPECTRUM.ERRORS: errors, SPECTRUM.U_WAVELENGTHS: 'Angstrom', SPECTRUM.U_FLUXES: fluxunit, SPECTRUM.U_ERRORS: fluxunit, SPECTRUM.TELESCOPE: telescope, SPECTRUM.FILENAME: filename, SPECTRUM.SOURCE: source } if name in datedict: specdict[SPECTRUM.TIME] = datedict[name] specdict[SPECTRUM.U_TIME] = 'MJD' catalog.entries[name].add_spectrum(**specdict) if catalog.args.travis and fi >= catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() return
def do_ascii(catalog): """Process ASCII files that were extracted from datatables appearing in published works. """ task_str = catalog.get_current_task_str() # 2006ApJ...645..841N file_path = os.path.join( catalog.get_current_task_repo(), '2006ApJ...645..841N-table3.csv') tsvin = list(csv.reader(open(file_path, 'r'), delimiter=',')) for ri, row in enumerate(pbar(tsvin, task_str)): name = 'SNLS-' + row[0] name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2006ApJ...645..841N') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, row[1], source, kind='spectroscopic') astrot = astrotime(float(row[4]) + 2450000., format='jd').datetime date_str = make_date_string(astrot.year, astrot.month, astrot.day) catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, date_str, source) catalog.journal_entries() # Anderson 2014 file_names = list( glob(os.path.join( catalog.get_current_task_repo(), 'SNII_anderson2014/*.dat'))) for datafile in pbar_strings(file_names, task_str): basename = os.path.basename(datafile) if not is_number(basename[:2]): continue if basename == '0210_V.dat': name = 'SN0210' else: name = ('SN20' if int(basename[:2]) < 50 else 'SN19') + basename.split('_')[0] name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2014ApJ...786...67A') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) if name in ['SN1999ca', 'SN2003dq', 'SN2008aw']: system = 'Swope' else: system = 'Landolt' with open(datafile, 'r') as ff: tsvin = csv.reader(ff, delimiter=' ', skipinitialspace=True) for row in tsvin: if not row[0]: continue time = str(jd_to_mjd(Decimal(row[0]))) catalog.entries[name].add_photometry( time=time, band='V', magnitude=row[1], e_magnitude=row[2], system=system, source=source) catalog.journal_entries() # stromlo stromlobands = ['B', 'V', 'R', 'I', 'VM', 'RM'] file_path = os.path.join( catalog.get_current_task_repo(), 'J_A+A_415_863-1/photometry.csv') tsvin = list(csv.reader(open(file_path, 'r'), delimiter=',')) for row in pbar(tsvin, task_str): name = row[0] name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2004A&A...415..863G') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) mjd = str(jd_to_mjd(Decimal(row[1]))) for ri, ci in enumerate(range(2, len(row), 3)): if not row[ci]: continue band = stromlobands[ri] upperlimit = True if (not row[ci + 1] and row[ci + 2]) else False e_upper_magnitude = str( abs(Decimal(row[ci + 1]))) if row[ci + 1] else '' e_lower_magnitude = str( abs(Decimal(row[ci + 2]))) if row[ci + 2] else '' teles = 'MSSSO 1.3m' if band in ['VM', 'RM'] else 'CTIO' instr = 'MaCHO' if band in ['VM', 'RM'] else '' catalog.entries[name].add_photometry( time=mjd, band=band, magnitude=row[ci], e_upper_magnitude=e_upper_magnitude, e_lower_magnitude=e_lower_magnitude, upperlimit=upperlimit, telescope=teles, instrument=instr, source=source) catalog.journal_entries() # 2015MNRAS.449..451W file_path = os.path.join( catalog.get_current_task_repo(), '2015MNRAS.449..451W.dat') data = list(csv.reader(open(file_path, 'r'), delimiter='\t', quotechar='"', skipinitialspace=True)) for rr, row in enumerate(pbar(data, task_str)): if rr == 0: continue namesplit = row[0].split('/') name = namesplit[-1] if name.startswith('SN'): name = name.replace(' ', '') name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2015MNRAS.449..451W') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) if len(namesplit) > 1: catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, namesplit[0], source) catalog.entries[name].add_quantity( SUPERNOVA.CLAIMED_TYPE, row[1], source) catalog.entries[name].add_photometry( time=row[2], band=row[4], magnitude=row[3], source=source) catalog.journal_entries() # 2016MNRAS.459.1039T file_path = os.path.join( catalog.get_current_task_repo(), '2016MNRAS.459.1039T.tsv') data = list(csv.reader(open(file_path, 'r'), delimiter='\t', quotechar='"', skipinitialspace=True)) name = catalog.add_entry('LSQ13zm') source = catalog.entries[name].add_source(bibcode='2016MNRAS.459.1039T') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) for rr, row in enumerate(pbar(data, task_str)): if row[0][0] == '#': bands = [xx.replace('(err)', '') for xx in row[3:-1]] continue mjd = row[1] mags = [re.sub(r'\([^)]*\)', '', xx) for xx in row[3:-1]] upps = [True if '>' in xx else '' for xx in mags] mags = [xx.replace('>', '') for xx in mags] errs = [xx[xx.find('(') + 1:xx.find(')')] if '(' in xx else '' for xx in row[3:-1]] for mi, mag in enumerate(mags): if not is_number(mag): continue catalog.entries[name].add_photometry( time=mjd, band=bands[mi], magnitude=mag, e_magnitude=errs[mi], instrument=row[-1], upperlimit=upps[mi], source=source) catalog.journal_entries() # 2015ApJ...804...28G file_path = os.path.join( catalog.get_current_task_repo(), '2015ApJ...804...28G.tsv') data = list(csv.reader(open(file_path, 'r'), delimiter='\t', quotechar='"', skipinitialspace=True)) name = catalog.add_entry('PS1-13arp') source = catalog.entries[name].add_source(bibcode='2015ApJ...804...28G') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) for rr, row in enumerate(pbar(data, task_str)): if rr == 0: continue mjd = row[1] mag = row[3] upp = True if '<' in mag else '' mag = mag.replace('<', '') err = row[4] if is_number(row[4]) else '' ins = row[5] catalog.entries[name].add_photometry( time=mjd, band=row[0], magnitude=mag, e_magnitude=err, instrument=ins, upperlimit=upp, source=source) catalog.journal_entries() # 2016ApJ...819...35A file_path = os.path.join( catalog.get_current_task_repo(), '2016ApJ...819...35A.tsv') data = list(csv.reader(open(file_path, 'r'), delimiter='\t', quotechar='"', skipinitialspace=True)) for rr, row in enumerate(pbar(data, task_str)): if row[0][0] == '#': continue name = catalog.add_entry(row[0]) source = catalog.entries[name].add_source( bibcode='2016ApJ...819...35A') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) catalog.entries[name].add_quantity(SUPERNOVA.RA, row[1], source) catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[2], source) catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, row[3], source) disc_date = datetime.strptime(row[4], '%Y %b %d').isoformat() disc_date = disc_date.split('T')[0].replace('-', '/') catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, disc_date, source) catalog.journal_entries() # 2014ApJ...784..105W file_path = os.path.join( catalog.get_current_task_repo(), '2014ApJ...784..105W.tsv') data = list(csv.reader(open(file_path, 'r'), delimiter='\t', quotechar='"', skipinitialspace=True)) for rr, row in enumerate(pbar(data, task_str)): if row[0][0] == '#': continue name = catalog.add_entry(row[0]) source = catalog.entries[name].add_source( bibcode='2014ApJ...784..105W') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) mjd = row[1] band = row[2] mag = row[3] err = row[4] catalog.entries[name].add_photometry( time=mjd, band=row[2], magnitude=mag, e_magnitude=err, instrument='WHIRC', telescope='WIYN 3.5 m', observatory='NOAO', system='WHIRC', source=source) catalog.journal_entries() # 2012MNRAS.425.1007B file_path = os.path.join( catalog.get_current_task_repo(), '2012MNRAS.425.1007B.tsv') data = list(csv.reader(open(file_path, 'r'), delimiter='\t', quotechar='"', skipinitialspace=True)) for rr, row in enumerate(pbar(data, task_str)): if row[0][0] == '#': bands = row[2:] continue name = catalog.add_entry(row[0]) source = catalog.entries[name].add_source( bibcode='2012MNRAS.425.1007B') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) mjd = row[1] mags = [xx.split('±')[0].strip() for xx in row[2:]] errs = [xx.split('±')[1].strip() if '±' in xx else '' for xx in row[2:]] if row[0] == 'PTF09dlc': ins = 'HAWK-I' tel = 'VLT 8.1m' obs = 'ESO' else: ins = 'NIRI' tel = 'Gemini North 8.2m' obs = 'Gemini' for mi, mag in enumerate(mags): if not is_number(mag): continue catalog.entries[name].add_photometry( time=mjd, band=bands[mi], magnitude=mag, e_magnitude=errs[mi], instrument=ins, telescope=tel, observatory=obs, system='Natural', source=source) catalog.journal_entries() # 2014ApJ...783...28G file_path = os.path.join( catalog.get_current_task_repo(), 'apj490105t2_ascii.txt') with open(file_path, 'r') as f: data = list(csv.reader(f, delimiter='\t', quotechar='"', skipinitialspace=True)) for r, row in enumerate(pbar(data, task_str)): if row[0][0] == '#': continue name, source = catalog.new_entry( row[0], bibcode='2014ApJ...783...28G') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[1], source) catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, '20' + row[0][3:5], source) catalog.entries[name].add_quantity(SUPERNOVA.RA, row[2], source) catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[3], source) catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, row[13] if is_number(row[13]) else row[10], source) catalog.journal_entries() # 2005ApJ...634.1190H file_path = os.path.join( catalog.get_current_task_repo(), '2005ApJ...634.1190H.tsv') with open(file_path, 'r') as f: data = list(csv.reader(f, delimiter='\t', quotechar='"', skipinitialspace=True)) for r, row in enumerate(pbar(data, task_str)): name, source = catalog.new_entry( 'SNLS-' + row[0], bibcode='2005ApJ...634.1190H') catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, '20' + row[0][:2], source) catalog.entries[name].add_quantity(SUPERNOVA.RA, row[1], source) catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[2], source) catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, row[5].replace('?', ''), source, e_value=row[6], kind='host') catalog.entries[name].add_quantity( SUPERNOVA.CLAIMED_TYPE, row[7].replace('SN', '').strip(':* '), source) catalog.journal_entries() # 2014MNRAS.444.2133S file_path = os.path.join( catalog.get_current_task_repo(), '2014MNRAS.444.2133S.tsv') with open(file_path, 'r') as f: data = list(csv.reader(f, delimiter='\t', quotechar='"', skipinitialspace=True)) for r, row in enumerate(pbar(data, task_str)): if row[0][0] == '#': continue name = row[0] if is_number(name[:4]): name = 'SN' + name name, source = catalog.new_entry( name, bibcode='2014MNRAS.444.2133S') catalog.entries[name].add_quantity(SUPERNOVA.RA, row[1], source) catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[2], source) catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, row[3], source, kind='host') catalog.journal_entries() # 2009MNRAS.398.1041B file_path = os.path.join( catalog.get_current_task_repo(), '2009MNRAS.398.1041B.tsv') with open(file_path, 'r') as f: data = list(csv.reader(f, delimiter='\t', quotechar='"', skipinitialspace=True)) for r, row in enumerate(pbar(data, task_str)): if row[0][0] == '#': bands = row[2:-1] continue name, source = catalog.new_entry( 'SN2008S', bibcode='2009MNRAS.398.1041B') mjd = str(jd_to_mjd(Decimal(row[0]))) mags = [x.split('±')[0].strip() for x in row[2:]] upps = [('<' in x.split('±')[0]) for x in row[2:]] errs = [x.split('±')[1].strip() if '±' in x else '' for x in row[2:]] instrument = row[-1] for mi, mag in enumerate(mags): if not is_number(mag): continue catalog.entries[name].add_photometry( time=mjd, band=bands[mi], magnitude=mag, e_magnitude=errs[mi], instrument=instrument, source=source) catalog.journal_entries() # 2010arXiv1007.0011P file_path = os.path.join( catalog.get_current_task_repo(), '2010arXiv1007.0011P.tsv') with open(file_path, 'r') as f: data = list(csv.reader(f, delimiter='\t', quotechar='"', skipinitialspace=True)) for r, row in enumerate(pbar(data, task_str)): if row[0][0] == '#': bands = row[1:] continue name, source = catalog.new_entry( 'SN2008S', bibcode='2010arXiv1007.0011P') mjd = row[0] mags = [x.split('±')[0].strip() for x in row[1:]] errs = [x.split('±')[1].strip() if '±' in x else '' for x in row[1:]] for mi, mag in enumerate(mags): if not is_number(mag): continue catalog.entries[name].add_photometry( time=mjd, band=bands[mi], magnitude=mag, e_magnitude=errs[mi], instrument='LBT', source=source) catalog.journal_entries() # 2000ApJ...533..320G file_path = os.path.join( catalog.get_current_task_repo(), '2000ApJ...533..320G.tsv') with open(file_path, 'r') as f: data = list(csv.reader(f, delimiter='\t', quotechar='"', skipinitialspace=True)) name, source = catalog.new_entry( 'SN1997cy', bibcode='2000ApJ...533..320G') for r, row in enumerate(pbar(data, task_str)): if row[0][0] == '#': bands = row[1:-1] continue mjd = str(jd_to_mjd(Decimal(row[0]))) mags = row[1:len(bands)] for mi, mag in enumerate(mags): if not is_number(mag): continue catalog.entries[name].add_photometry( time=mjd, band=bands[mi], magnitude=mag, observatory='Mount Stromlo', telescope='MSSSO', source=source, kcorrected=True) catalog.journal_entries() return
def do_cfa_photo(catalog): """Import photometry from the CfA archive.""" from html import unescape import re task_str = catalog.get_current_task_str() file_names = glob( os.path.join(catalog.get_current_task_repo(), 'cfa-input/*.dat')) for fname in pbar_strings(file_names, task_str): f = open(fname, 'r') tsvin = csv.reader(f, delimiter=' ', skipinitialspace=True) csv_data = [] for r, row in enumerate(tsvin): new = [] for item in row: new.extend(item.split('\t')) csv_data.append(new) for r, row in enumerate(csv_data): for c, col in enumerate(row): csv_data[r][c] = col.strip() csv_data[r] = [_f for _f in csv_data[r] if _f] eventname = os.path.basename(os.path.splitext(fname)[0]) eventparts = eventname.split('_') name = clean_snname(eventparts[0]) name = catalog.add_entry(name) secondaryname = 'CfA Supernova Archive' secondaryurl = 'https://www.cfa.harvard.edu/supernova/SNarchive.html' secondarysource = catalog.entries[name].add_source( name=secondaryname, url=secondaryurl, secondary=True, acknowledgment=ACKN_CFA) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, secondarysource) year = re.findall(r'\d+', name)[0] catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, year, secondarysource) eventbands = list(eventparts[1]) tu = 'MJD' jdoffset = Decimal(0.) for rc, row in enumerate(csv_data): if len(row) > 0 and row[0][0] == "#": if len(row[0]) > 2 and row[0][:3] == '#JD': tu = 'JD' rowparts = row[0].split('-') jdoffset = Decimal(rowparts[1]) elif len(row[0]) > 6 and row[0][:7] == '#Julian': tu = 'JD' jdoffset = Decimal(0.) elif len(row) > 1 and row[1].lower() == 'photometry': for ci, col in enumerate(row[2:]): if col[0] == "(": refstr = ' '.join(row[2 + ci:]) refstr = refstr.replace('(', '').replace(')', '') bibcode = unescape(refstr) source = catalog.entries[name].add_source( bibcode=bibcode) elif len(row) > 1 and row[1] == 'HJD': tu = 'HJD' continue elif len(row) > 0: mjd = row[0] for v, val in enumerate(row): if v == 0: if tu == 'JD': mjd = str(jd_to_mjd(Decimal(val) + jdoffset)) tuout = 'MJD' elif tu == 'HJD': mjd = str(jd_to_mjd(Decimal(val))) tuout = 'MJD' else: mjd = val tuout = tu elif v % 2 != 0: if float(row[v]) < 90.0: src = secondarysource + ',' + source photodict = { PHOTOMETRY.U_TIME: tuout, PHOTOMETRY.TIME: mjd, PHOTOMETRY.BAND_SET: 'Standard', PHOTOMETRY.BAND: eventbands[(v - 1) // 2], PHOTOMETRY.MAGNITUDE: row[v], PHOTOMETRY.E_MAGNITUDE: row[v + 1], PHOTOMETRY.SOURCE: src } catalog.entries[name].add_photometry(**photodict) f.close() # Hicken 2012 with open( os.path.join(catalog.get_current_task_repo(), 'hicken-2012-standard.dat'), 'r') as infile: tsvin = list(csv.reader(infile, delimiter='|', skipinitialspace=True)) for r, row in enumerate(pbar(tsvin, task_str)): if r <= 47: continue if row[0][:2] != 'sn': name = 'SN' + row[0].strip() else: name = row[0].strip() name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2012ApJS..200...12H') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, 'Ia', source) photodict = { PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.TIME: row[2].strip(), PHOTOMETRY.BAND: row[1].strip(), PHOTOMETRY.BAND_SET: 'Standard', PHOTOMETRY.MAGNITUDE: row[6].strip(), PHOTOMETRY.E_MAGNITUDE: row[7].strip(), PHOTOMETRY.SOURCE: source } catalog.entries[name].add_photometry(**photodict) # Bianco 2014 with open( os.path.join(catalog.get_current_task_repo(), 'bianco-2014-standard.dat'), 'r') as infile: tsvin = list(csv.reader(infile, delimiter=' ', skipinitialspace=True)) for row in pbar(tsvin, task_str): name = 'SN' + row[0] name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2014ApJS..213...19B') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) photodict = { PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.TIME: row[2], PHOTOMETRY.BAND: row[1], PHOTOMETRY.MAGNITUDE: row[3], PHOTOMETRY.E_MAGNITUDE: row[4], PHOTOMETRY.TELESCOPE: row[5], PHOTOMETRY.BAND_SET: 'Standard', PHOTOMETRY.SOURCE: source } catalog.entries[name].add_photometry(**photodict) catalog.journal_entries() return
def do_cccp(catalog): task_str = catalog.get_current_task_str() cccpbands = ['B', 'V', 'R', 'I'] file_names = list( glob(os.path.join(catalog.get_current_task_repo(), 'CCCP/apj407397*.txt'))) for datafile in pbar_strings(file_names, task_str + ': apj407397...'): with open(datafile, 'r') as ff: tsvin = csv.reader(ff, delimiter='\t', skipinitialspace=True) for rr, row in enumerate(tsvin): if rr == 0: continue elif rr == 1: name = 'SN' + row[0].split('SN ')[-1] name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2012ApJ...744...10K') catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, name, source) elif rr >= 5: mjd = str(Decimal(row[0]) + 53000) for bb, band in enumerate(cccpbands): if row[2 * bb + 1]: mag = row[2 * bb + 1].strip('>') upl = (not row[2 * bb + 2]) (catalog.entries[name] .add_photometry(time=mjd, band=band, magnitude=mag, e_magnitude=row[2 * bb + 2], upperlimit=upl, source=source)) if catalog.current_task.load_archive(catalog.args): with open(os.path.join(catalog.get_current_task_repo(), 'CCCP/sc_cccp.html'), 'r') as ff: html = ff.read() else: session = requests.Session() response = session.get( 'https://webhome.weizmann.ac.il/home/iair/sc_cccp.html') html = response.text with open(os.path.join(catalog.get_current_task_repo(), 'CCCP/sc_cccp.html'), 'w') as ff: ff.write(html) soup = BeautifulSoup(html, 'html5lib') links = soup.body.findAll("a") for link in pbar(links, task_str + ': links'): if 'sc_sn' in link['href']: name = catalog.add_entry(link.text.replace(' ', '')) source = (catalog.entries[name] .add_source(name='CCCP', url=('https://webhome.weizmann.ac.il' '/home/iair/sc_cccp.html'))) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) if catalog.current_task.load_archive(catalog.args): fname = os.path.join(catalog.get_current_task_repo(), 'CCCP/') + link['href'].split('/')[-1] with open(fname, 'r') as ff: html2 = ff.read() else: response2 = session.get( 'https://webhome.weizmann.ac.il/home/iair/' + link['href']) html2 = response2.text fname = os.path.join(catalog.get_current_task_repo(), 'CCCP/') + link['href'].split('/')[-1] with open(fname, 'w') as ff: ff.write(html2) soup2 = BeautifulSoup(html2, 'html5lib') links2 = soup2.body.findAll("a") for link2 in links2: if '.txt' in link2['href'] and '_' in link2['href']: band = link2['href'].split('_')[1].split('.')[0].upper() if catalog.current_task.load_archive(catalog.args): fname = os.path.join( catalog.get_current_task_repo(), 'CCCP/') fname += link2['href'].split('/')[-1] if not os.path.isfile(fname): continue with open(fname, 'r') as ff: html3 = ff.read() else: response3 = (session .get('https://webhome.weizmann.ac.il' '/home/iair/cccp/' + link2['href'])) if response3.status_code == 404: continue html3 = response3.text fname = os.path.join( catalog.get_current_task_repo(), 'CCCP/') fname += link2['href'].split('/')[-1] with open(fname, 'w') as ff: ff.write(html3) table = [[str(Decimal(yy.strip())).rstrip('0') for yy in xx.split(',')] for xx in list(filter(None, html3.split('\n')))] for row in table: catalog.entries[name].add_photometry( time=str(Decimal(row[0]) + 53000), band=band, magnitude=row[1], e_magnitude=row[2], source=source) catalog.journal_entries() return
def do_external(catalog): task_str = catalog.get_current_task_str() oldbanddict = { "Pg": { "instrument": "Pan-STARRS1", "band": "g" }, "Pr": { "instrument": "Pan-STARRS1", "band": "r" }, "Pi": { "instrument": "Pan-STARRS1", "band": "i" }, "Pz": { "instrument": "Pan-STARRS1", "band": "z" }, "Mu": { "instrument": "MegaCam", "band": "u" }, "Mg": { "instrument": "MegaCam", "band": "g" }, "Mr": { "instrument": "MegaCam", "band": "r" }, "Mi": { "instrument": "MegaCam", "band": "i" }, "Mz": { "instrument": "MegaCam", "band": "z" }, "Su": { "instrument": "SDSS", "band": "u" }, "Sg": { "instrument": "SDSS", "band": "g" }, "Sr": { "instrument": "SDSS", "band": "r" }, "Si": { "instrument": "SDSS", "band": "i" }, "Sz": { "instrument": "SDSS", "band": "z" }, "bU": { "instrument": "Bessel", "band": "U" }, "bB": { "instrument": "Bessel", "band": "B" }, "bV": { "instrument": "Bessel", "band": "V" }, "bR": { "instrument": "Bessel", "band": "R" }, "bI": { "instrument": "Bessel", "band": "I" }, "4g": { "instrument": "PTF 48-Inch", "band": "g" }, "4r": { "instrument": "PTF 48-Inch", "band": "r" }, "6g": { "instrument": "PTF 60-Inch", "band": "g" }, "6r": { "instrument": "PTF 60-Inch", "band": "r" }, "6i": { "instrument": "PTF 60-Inch", "band": "i" }, "Uu": { "instrument": "UVOT", "band": "U" }, "Ub": { "instrument": "UVOT", "band": "B" }, "Uv": { "instrument": "UVOT", "band": "V" }, "Um": { "instrument": "UVOT", "band": "M2" }, "U1": { "instrument": "UVOT", "band": "W1" }, "U2": { "instrument": "UVOT", "band": "W2" }, "GN": { "instrument": "GALEX", "band": "NUV" }, "GF": { "instrument": "GALEX", "band": "FUV" }, "CR": { "instrument": "Clear", "band": "r" }, "RO": { "instrument": "ROTSE" }, "X1": { "instrument": "Chandra" }, "X2": { "instrument": "XRT" }, "Xs": { "instrument": "XRT", "band": "soft" }, "Xm": { "instrument": "XRT", "band": "hard" }, "XM": { "instrument": "XMM" } } path_pattern = os.path.join(catalog.get_current_task_repo(), 'old-tdefit/*.dat') for datafile in pbar_strings(glob(path_pattern), task_str): f = open(datafile, 'r') tsvin = csv.reader(f, delimiter='\t', skipinitialspace=True) sources = [] yrsmjdoffset = 0. for row in tsvin: if row[0] == 'name': name = re.sub('<[^<]+?>', '', row[1].split(',')[0].strip()) name = catalog.add_entry(name) elif row[0] == 'citations': citarr = row[1].split(',') for cite in citarr: if '*' in cite: bibcode = urllib.parse.unquote( cite.split('/')[-2].split("'")[0]) sources.append(catalog.entries[name].add_source( bibcode=bibcode)) elif row[0] == 'nhcorr': hostnhcorr = True if row[1] == 'T' else False elif row[0] == 'restframe': restframe = True if row[1] == 'T' else False elif row[0] == 'yrsmjdoffset': yrsmjdoffset = float(row[1]) if row[0] == 'redshift': redshift = float(row[1].split(',')[0].strip(' *')) source = ','.join(sources) if not source: source = catalog.entries[name].add_self_source() f.seek(0) for row in tsvin: if not row or len(row) < 2 or not row[1]: continue if row[0] == 'redshift': for rs in [x.strip() for x in row[1].split(',')]: catalog.entries[name].add_quantity( TIDALDISRUPTION.REDSHIFT, rs.strip(' *'), source) elif row[0] == 'host': hostname = re.sub('<[^<]+?>', '', row[1]) catalog.entries[name].add_quantity( TIDALDISRUPTION.HOST, hostname, source) elif row[0] == 'claimedtype' and row[1] != 'TDE': cts = row[1].split(',') for ct in cts: ctype = ct.strip() catalog.entries[name].add_quantity( TIDALDISRUPTION.CLAIMED_TYPE, ctype, source) elif row[0] == 'citations': catalog.entries[name].add_quantity( Key('citations', KEY_TYPES.STRING), row[1], source) elif row[0] == 'notes': catalog.entries[name].add_quantity( Key('notes', KEY_TYPES.STRING), row[1], source) elif row[0] == 'nh': catalog.entries[name].add_quantity( Key('nhhost', KEY_TYPES.STRING), row[1], source) elif row[0] == 'photometry': timeunit = row[1] if timeunit == 'yrs': timeunit = 'MJD' if restframe: # Currently presume only the time, not the flux, has # been affected by redshifting. time = str(yrsmjdoffset + float(row[2]) * 365.25 * ( 1.0 + redshift)) else: time = str(yrsmjdoffset + float(row[2]) * 365.25) lrestframe = False else: time = row[2] if timeunit == 'floatyr': timeunit = 'MJD' time = str(astrotime(float(time), format='jyear').mjd) lrestframe = restframe instrument = '' iband = row[3] if iband in oldbanddict: if 'band' in oldbanddict[iband]: band = oldbanddict[iband]['band'] if 'instrument' in oldbanddict[iband]: instrument = oldbanddict[iband]['instrument'] else: band = iband upperlimit = True if row[6] == '1' else False if 'X' in iband: counts = Decimal(10.0)**Decimal(row[4]) photodict = { PHOTOMETRY.TIME: time, PHOTOMETRY.U_TIME: timeunit, PHOTOMETRY.BAND: band, PHOTOMETRY.COUNT_RATE: counts, PHOTOMETRY.UPPER_LIMIT: upperlimit, PHOTOMETRY.REST_FRAME: lrestframe, PHOTOMETRY.HOST_NH_CORR: hostnhcorr, PHOTOMETRY.INSTRUMENT: instrument, PHOTOMETRY.SOURCE: source } # Old TDEFit stored counts in log if float(row[5]) != 0.0: photodict[PHOTOMETRY.E_COUNT_RATE] = str( (Decimal(10.0)**(Decimal(row[4]) + Decimal(row[5])) - Decimal(10.0)**Decimal(row[4]))) else: magnitude = row[4] photodict = { PHOTOMETRY.TIME: time, PHOTOMETRY.U_TIME: timeunit, PHOTOMETRY.BAND: band, PHOTOMETRY.MAGNITUDE: magnitude, PHOTOMETRY.UPPER_LIMIT: upperlimit, PHOTOMETRY.REST_FRAME: lrestframe, PHOTOMETRY.HOST_NH_CORR: hostnhcorr, PHOTOMETRY.INSTRUMENT: instrument, PHOTOMETRY.SOURCE: source } if float(row[5]) != 0.0: photodict[PHOTOMETRY.E_MAGNITUDE] = row[5] catalog.entries[name].add_photometry(**photodict) catalog.journal_entries() return
def do_snls_spectra(catalog): """ """ task_str = catalog.get_current_task_str() result = Vizier.get_catalogs('J/A+A/507/85/table1') table = result[list(result.keys())[0]] table.convert_bytestring_to_unicode(python3_only=True) datedict = {} for row in table: datedict['SNLS-' + row['SN']] = str(astrotime(row['Date']).mjd) oldname = '' file_names = glob(os.path.join(catalog.get_current_task_repo(), 'SNLS/*')) for fi, fname in enumerate(pbar_strings(file_names, task_str)): filename = os.path.basename(fname) fileparts = filename.split('_') name = 'SNLS-' + fileparts[1] name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2009A&A...507...85B') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, '20' + fileparts[1][:2], source) f = open(fname, 'r') data = csv.reader(f, delimiter=' ', skipinitialspace=True) specdata = [] for r, row in enumerate(data): if row[0] == '@TELESCOPE': telescope = row[1].strip() elif row[0] == '@REDSHIFT': catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, row[1].strip(), source) if r < 14: continue specdata.append(list(filter(None, [x.strip(' \t') for x in row]))) specdata = [list(i) for i in zip(*specdata)] wavelengths = specdata[1] fluxes = [pretty_num(float(x) * 1.e-16, sig=get_sig_digits(x)) for x in specdata[2]] # FIX: this isnt being used # errors = [pretty_num(float(x)*1.e-16, sig=get_sig_digits(x)) for x in # specdata[3]] catalog.entries[name].add_spectrum( u_wavelengths='Angstrom', u_fluxes='erg/s/cm^2/Angstrom', wavelengths=wavelengths, fluxes=fluxes, u_time='MJD' if name in datedict else '', time=datedict[name] if name in datedict else '', telescope=telescope, source=source, filename=filename) if catalog.args.travis and fi >= catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() return
def do_essence_spectra(catalog): task_str = catalog.get_current_task_str() insdict = { "lris": "LRIS", "esi": "ESI", "deimos": "DEIMOS", "gmos": "GMOS", "fors1": "FORS1", "bluechannel": "Blue Channel", "ldss2": "LDSS-2", "ldss3": "LDSS-3", "imacs": "IMACS", "fast": "FAST" } teldict = { "lris": "Keck", "esi": "Keck", "deimos": "Keck", "gmos": "Gemini", "fors1": "VLT", "bluechannel": "MMT", "ldss2": "Magellan Clay & Baade", "ldss3": "Magellan Clay & Baade", "imacs": "Magellan Clay & Baade", "fast": "FLWO 1.5m" } file_names = glob( os.path.join(catalog.get_current_task_repo(), 'ESSENCE', '*')) oldname = '' for fi, fname in enumerate(pbar_strings(file_names, task_str)): filename = os.path.basename(fname) fileparts = filename.split('_') name = 'ESSENCE ' + fileparts[0] name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name if is_number(fileparts[1]): doffset = 1 else: if fileparts[1] != 'comb': continue doffset = 2 dstr = fileparts[doffset] mjd = str( astrotime( datetime.datetime( year=int(dstr[:4]), month=int(dstr[4:6]), day=int(dstr[6:8])) + datetime.timedelta(days=float(dstr[ 8:]))).mjd) instrument = fileparts[-1].split('.')[0] telescope = teldict.get(instrument, '') instrument = insdict.get(instrument, '') with open(fname, 'r') as f: data = csv.reader(f, delimiter=' ', skipinitialspace=True) data = [list(i) for i in zip(*data)] wavelengths = data[0] fluxes = [str(Decimal('1.0e-15') * Decimal(x)) for x in data[1]] name, source = catalog.new_entry(name, bibcode='2016ApJS..224....3N') specdict = { SPECTRUM.TIME: mjd, SPECTRUM.U_TIME: 'MJD', SPECTRUM.U_WAVELENGTHS: 'Angstrom', SPECTRUM.WAVELENGTHS: wavelengths, SPECTRUM.FLUXES: fluxes, SPECTRUM.U_FLUXES: 'erg/s/cm^2/Angstrom', SPECTRUM.FILENAME: filename, SPECTRUM.SOURCE: source } if instrument: specdict[SPECTRUM.INSTRUMENT] = instrument if telescope: specdict[SPECTRUM.TELESCOPE] = telescope catalog.entries[name].add_spectrum(**specdict) if catalog.args.travis and fi >= catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() return
def do_wiserep_spectra(catalog): task_str = catalog.get_current_task_str() secondaryreference = 'WISeREP' secondaryrefurl = 'http://wiserep.weizmann.ac.il/' secondarybibcode = '2012PASP..124..668Y' wiserepcnt = 0 # These are known to be in error on the WISeREP page, either fix or ignore # them. wiserepbibcorrectdict = {'2000AJ....120..367G]': '2000AJ....120..367G', 'Harutyunyan et al. 2008': '2008A&A...488..383H', '0609268': '2007AJ....133...58K', '2006ApJ...636...400Q': '2006ApJ...636..400Q', '2011ApJ...741...76': '2011ApJ...741...76C', '2016PASP...128...961': '2016PASP..128...961', '2002AJ....1124..417H': '2002AJ....1124.417H', '2013ApJ…774…58D': '2013ApJ...774...58D', '2011Sci.333..856S': '2011Sci...333..856S', '2014MNRAS.438,368': '2014MNRAS.438..368T', '2012MNRAS.420.1135': '2012MNRAS.420.1135S', '2012Sci..337..942D': '2012Sci...337..942D', 'stt1839': '2013MNRAS.436.3614S', 'arXiv:1605.03136': '2016arXiv160503136T', '10.1093/mnras/stt1839': '2013MNRAS.436.3614S'} file_names = list( glob(os.path.join( catalog.get_current_task_repo(), '*'))) for folder in pbar_strings(file_names, task_str): if '.txt' in folder: continue name = os.path.basename(folder).strip() if name.startswith('sn'): name = 'SN' + name[2:] if (name.startswith(('CSS', 'SSS', 'MLS')) and ':' not in name): name = name.replace('-', ':', 1) if name.startswith('MASTERJ'): name = name.replace('MASTERJ', 'MASTER OT J') if name.startswith('PSNJ'): name = name.replace('PSNJ', 'PSN J') name = catalog.add_entry(name) secondarysource = catalog.entries[name].add_source( name=secondaryreference, url=secondaryrefurl, bibcode=secondarybibcode, secondary=True) catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, name, secondarysource) with open(os.path.join(folder, 'README.json'), 'r') as f: fileinfo = json.loads(f.read()) files = list(set(glob(folder + '/*')) - set(glob(folder + '/README.json'))) for fname in pbar(files, task_str): specfile = os.path.basename(fname) claimedtype = fileinfo[specfile]["Type"] instrument = fileinfo[specfile]["Instrument"] epoch = fileinfo[specfile]["Obs. Date"] observer = fileinfo[specfile]["Observer"] reducer = fileinfo[specfile]["Reducer"] bibcode = fileinfo[specfile]["Bibcode"] redshift = fileinfo[specfile]["Redshift"] survey = fileinfo[specfile]["Program"] reduction = fileinfo[specfile]["Reduction Status"] if bibcode: newbibcode = bibcode if bibcode in wiserepbibcorrectdict: newbibcode = wiserepbibcorrectdict[bibcode] if newbibcode and len(newbibcode) == 19: source = catalog.entries[name].add_source( bibcode=unescape(newbibcode)) else: bibname = unescape(bibcode) source = catalog.entries[name].add_source( name=bibname) catalog.log.warning('Bibcode "{}" is invalid, using as ' '`{}` instead'.format(bibname, SOURCE.NAME)) sources = uniq_cdl([source, secondarysource]) else: sources = secondarysource if claimedtype not in ['Other']: catalog.entries[name].add_quantity( SUPERNOVA.CLAIMED_TYPE, claimedtype, secondarysource) catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, redshift, secondarysource) with open(fname, 'r') as f: data = [x.split() for x in f] skipspec = False newdata = [] oldval = '' for row in data: if row and '#' not in row[0]: if (len(row) >= 2 and is_number(row[0]) and is_number(row[1]) and row[1] != oldval): newdata.append(row) oldval = row[1] if skipspec or not newdata: warnings.warn( 'Skipped adding spectrum file ' + specfile) continue data = [list(i) for i in zip(*newdata)] wavelengths = data[0] fluxes = data[1] errors = '' if len(data) == 3: errors = data[1] time = str(astrotime(epoch).mjd) if max([float(x) for x in fluxes]) < 1.0e-5: fluxunit = 'erg/s/cm^2/Angstrom' else: fluxunit = 'Uncalibrated' catalog.entries[name].add_spectrum( u_wavelengths='Angstrom', errors=errors, u_fluxes=fluxunit, u_errors=fluxunit if errors else '', wavelengths=wavelengths, fluxes=fluxes, u_time='MJD', time=time, instrument=instrument, source=sources, observer=observer, reducer=reducer, reduction=reduction, filename=specfile, survey=survey, redshift=redshift) catalog.journal_entries() wiserepcnt = wiserepcnt + 1 if (catalog.args.travis and wiserepcnt % catalog.TRAVIS_QUERY_LIMIT == 0): break return
def do_donations(catalog): task_str = catalog.get_current_task_str() # Nicholl 04-01-16 donation with open(os.path.join(catalog.get_current_task_repo(), 'Nicholl-04-01-16/bibcodes.json'), 'r') as f: bcs = json.loads(f.read()) file_names = glob(os.path.join( catalog.get_current_task_repo(), 'Nicholl-04-01-16/*.txt')) for datafile in pbar_strings(file_names, task_str + ': Nicholl-04-01-16'): inpname = os.path.basename(datafile).split('_')[0] name = catalog.add_entry(inpname) bibcode = '' for bc in bcs: if inpname in bcs[bc]: bibcode = bc if not bibcode: raise ValueError('Bibcode not found!') source = catalog.entries[name].add_source(bibcode=bibcode) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, inpname, source) with open(datafile, 'r') as f: tsvin = csv.reader(f, delimiter='\t', skipinitialspace=True) for r, rrow in enumerate(tsvin): row = list(filter(None, rrow)) if not row: continue if row[0][0] == '#' and row[0] != '#MJD': continue if row[0] == '#MJD': bands = [x for x in row[1:] if x and 'err' not in x] continue mjd = row[0] if not is_number(mjd): continue for v, val in enumerate(row[1::2]): upperlimit = '' if '>' in val: upperlimit = True mag = val.strip('>') if (not is_number(mag) or isnan(float(mag)) or float(mag) > 90.0): continue err = '' if (is_number(row[2 * v + 2]) and not isnan(float(row[2 * v + 2]))): err = row[2 * v + 2] catalog.entries[name].add_photometry( time=mjd, band=bands[v], magnitude=mag, e_magnitude=err, upperlimit=upperlimit, source=source) catalog.journal_entries() # Maggi 04-11-16 donation (MC SNRs) with open(os.path.join(catalog.get_current_task_repo(), 'Maggi-04-11-16/LMCSNRs_OpenSNe.csv')) as f: tsvin = csv.reader(f, delimiter=',') for row in pbar(list(tsvin), task_str + ': Maggi-04-11-16/LMCSNRs'): name = 'MCSNR ' + row[0] name = catalog.add_entry(name) ra = row[2] dec = row[3] source = (catalog.entries[name] .add_source(bibcode='2016A&A...585A.162M')) catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, 'LMCSNR J' + rep_chars(ra, ' :.') + rep_chars(dec, ' :.'), source) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) if row[1] != 'noname': catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, row[1], source) catalog.entries[name].add_quantity(SUPERNOVA.RA, row[2], source) catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[3], source) catalog.entries[name].add_quantity(SUPERNOVA.HOST, 'LMC', source) if row[4] == '1': catalog.entries[name].add_quantity( SUPERNOVA.CLAIMED_TYPE, 'Ia', source) elif row[4] == '2': catalog.entries[name].add_quantity( SUPERNOVA.CLAIMED_TYPE, 'CC', source) with open(os.path.join(catalog.get_current_task_repo(), 'Maggi-04-11-16/SMCSNRs_OpenSNe.csv')) as f: tsvin = csv.reader(f, delimiter=',') for row in pbar(list(tsvin), task_str + ': Maggi-04-11-16/SMCSNRs'): name = 'MCSNR ' + row[0] name = catalog.add_entry(name) source = catalog.entries[name].add_source(name='Pierre Maggi') ra = row[3] dec = row[4] catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, 'SMCSNR J' + ra.replace(':', '')[:6] + dec.replace(':', '')[:7], source) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[1], source) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[2], source) catalog.entries[name].add_quantity(SUPERNOVA.RA, row[3], source) catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[4], source) catalog.entries[name].add_quantity(SUPERNOVA.HOST, 'SMC', source) catalog.journal_entries() # Galbany 04-18-16 donation folders = next(os.walk(os.path.join( catalog.get_current_task_repo(), 'galbany-04-18-16/')))[1] bibcode = '2016AJ....151...33G' for folder in folders: infofiles = glob(os.path.join(catalog.get_current_task_repo(), 'galbany-04-18-16/') + folder + '/*.info') photfiles = glob(os.path.join(catalog.get_current_task_repo(), 'galbany-04-18-16/') + folder + '/*.out*') zhel = '' zcmb = '' zerr = '' for path in infofiles: with open(path, 'r') as f: lines = f.read().splitlines() for line in lines: splitline = line.split(':') field = splitline[0].strip().lower() value = splitline[1].strip() if field == 'name': name = value[:6].upper() name += (value[6].upper() if len(value) == 7 else value[6:]) name = catalog.add_entry(name) source = (catalog.entries[name] .add_source(bibcode=bibcode)) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) elif field == 'type': claimedtype = value.replace('SN', '') catalog.entries[name].add_quantity( SUPERNOVA.CLAIMED_TYPE, claimedtype, source) elif field == 'zhel': zhel = value elif field == 'redshift_error': zerr = value elif field == 'zcmb': zcmb = value elif field == 'ra': catalog.entries[name].add_quantity( SUPERNOVA.RA, value, source, u_value='floatdegrees') elif field == 'dec': catalog.entries[name].add_quantity( SUPERNOVA.DEC, value, source, u_value='floatdegrees') elif field == 'host': value = value.replace('- ', '-').replace('G ', 'G') catalog.entries[name].add_quantity(SUPERNOVA.HOST, value, source) elif field == 'e(b-v)_mw': catalog.entries[name].add_quantity( SUPERNOVA.EBV, value, source) catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, zhel, source, e_value=zerr, kind='heliocentric') catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, zcmb, source, e_value=zerr, kind='cmb') for path in photfiles: with open(path, 'r') as f: band = '' lines = f.read().splitlines() for li, line in enumerate(lines): if li in [0, 2, 3]: continue if li == 1: band = line.split(':')[-1].strip() else: cols = list(filter(None, line.split())) if not cols: continue catalog.entries[name].add_photometry( time=cols[0], magnitude=cols[1], e_magnitude=cols[2], band=band, system=cols[3], telescope=cols[4], source=source) catalog.journal_entries() # Brown 05-14-16 files = glob(os.path.join( catalog.get_current_task_repo(), 'brown-05-14-16/*.dat')) for fi in pbar(files, task_str): name = os.path.basename(fi).split('_')[0] name = catalog.add_entry(name) source = catalog.entries[name].add_source( name='Swift Supernovae', bibcode='2014Ap&SS.354...89B', url='http://people.physics.tamu.edu/pbrown/SwiftSN/swift_sn.html') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) with open(fi, 'r') as f: lines = f.read().splitlines() for line in lines: if not line or line[0] == '#': continue cols = list(filter(None, line.split())) band = cols[0] mjd = cols[1] # Skip lower limit entries for now if cols[2] == 'NULL' and cols[6] == 'NULL': continue isupp = cols[2] == 'NULL' and cols[6] != 'NULL' mag = cols[2] if not isupp else cols[4] e_mag = cols[3] if not isupp else '' upp = '' if not isupp else True (catalog.entries[name] .add_photometry(time=mjd, magnitude=mag, e_magnitude=e_mag, upperlimit=upp, band=band, source=source, telescope='Swift', instrument='UVOT', system='Vega')) catalog.journal_entries() # Nicholl 05-03-16 files = glob(os.path.join( catalog.get_current_task_repo(), 'nicholl-05-03-16/*.txt')) name = catalog.add_entry('SN2015bn') source = catalog.entries[name].add_source(bibcode='2016arXiv160304748N') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, 'PS15ae', source) for fi in pbar(files, task_str): telescope = os.path.basename(fi).split('_')[1] with open(fi, 'r') as f: lines = f.read().splitlines() for li, line in enumerate(lines): if not line or (line[0] == '#' and li != 0): continue cols = list(filter(None, line.split())) if not cols: continue if li == 0: bands = cols[1:] continue mjd = cols[0] for ci, col in enumerate(cols[1::2]): if not is_number(col): continue emag = cols[2 * ci + 2] upp = '' if not is_number(emag): emag = '' upp = True instrument = 'UVOT' if telescope == 'Swift' else '' (catalog.entries[name] .add_photometry(time=mjd, magnitude=col, e_magnitude=emag, upperlimit=upp, band=bands[ci], source=source, telescope=telescope, instrument=instrument, system='Vega' if telescope == 'Swift' else 'AB')) catalog.journal_entries() return
def do_suspect_photo(catalog): task_str = catalog.get_current_task_str() with open(os.path.join(catalog.get_current_task_repo(), 'suspectreferences.csv'), 'r') as f: tsvin = csv.reader(f, delimiter=',', skipinitialspace=True) suspectrefdict = {} for row in tsvin: suspectrefdict[row[0]] = row[1] file_names = list(sorted(glob(os.path.join( catalog.get_current_task_repo(), 'SUSPECT/*.html')))) for datafile in pbar_strings(file_names, task_str): basename = os.path.basename(datafile) basesplit = basename.split('-') oldname = basesplit[1] name = catalog.add_entry(oldname) if name.startswith('SN') and is_number(name[2:]): name = name + 'A' band = basesplit[3].split('.')[0] ei = int(basesplit[2]) bandlink = 'file://' + os.path.abspath(datafile) bandresp = urllib.request.urlopen(bandlink) bandsoup = BeautifulSoup(bandresp, 'html5lib') bandtable = bandsoup.find('table') names = bandsoup.body.findAll(text=re.compile('Name')) reference = '' for link in bandsoup.body.findAll('a'): if 'adsabs' in link['href']: reference = str(link).replace('"', "'") bibcode = unescape(suspectrefdict[reference]) source = catalog.entries[name].add_source(bibcode=bibcode) sec_ref = 'SUSPECT' sec_refurl = 'https://www.nhn.ou.edu/~suspect/' sec_source = catalog.entries[name].add_source( name=sec_ref, url=sec_refurl, secondary=True) catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, oldname, sec_source) if ei == 1: year = re.findall(r'\d+', name)[0] catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, year, sec_source) catalog.entries[name].add_quantity( SUPERNOVA.HOST, names[1].split(':')[1].strip(), sec_source) redshifts = bandsoup.body.findAll(text=re.compile('Redshift')) if redshifts: catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, redshifts[0].split(':')[1].strip(), sec_source, kind='heliocentric') # hvels = bandsoup.body.findAll(text=re.compile('Heliocentric # Velocity')) # if hvels: # vel = hvels[0].split(':')[1].strip().split(' ')[0] # catalog.entries[name].add_quantity(SUPERNOVA.VELOCITY, vel, # sec_source, # kind='heliocentric') types = bandsoup.body.findAll(text=re.compile('Type')) catalog.entries[name].add_quantity( SUPERNOVA.CLAIMED_TYPE, types[0].split( ':')[1].strip().split(' ')[0], sec_source) for r, row in enumerate(bandtable.findAll('tr')): if r == 0: continue col = row.findAll('td') mjd = str(jd_to_mjd(Decimal(col[0].contents[0]))) mag = col[3].contents[0] if mag.isspace(): mag = '' else: mag = str(mag) e_magnitude = col[4].contents[0] if e_magnitude.isspace(): e_magnitude = '' else: e_magnitude = str(e_magnitude) catalog.entries[name].add_photometry( time=mjd, band=band, magnitude=mag, e_magnitude=e_magnitude, source=sec_source + ',' + source) catalog.journal_entries() return
def do_donated_photo(catalog): """Import donated photometry.""" task_str = catalog.get_current_task_str() # Private donations here # if not catalog.args.travis: pass # End private donations # # Ponder 05-12-17 donation with open( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Ponder-05-12-17', 'meta.json'), 'r') as f: metadict = json.loads(f.read()) file_names = glob( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Ponder-05-12-17', '*.dat')) for path in file_names: with open(path, 'r') as f: tsvin = list(csv.reader(f, delimiter=' ', skipinitialspace=True)) oname = path.split('/')[-1].split('.')[0] name, source = catalog.new_entry(oname, bibcode=metadict[oname]['bibcode']) for row in pbar(tsvin, task_str + ': Ponder ' + oname): if row[0][0] == '#' or not is_number(row[-1]): continue mjd = row[1] bandinst = row[2].split('_') band = bandinst[0] inst = '' if len(bandinst) > 1: inst = bandinst[1] mag = row[3] uerr = row[4] lerr = row[5] photodict = { PHOTOMETRY.TIME: mjd, PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.BAND: band, PHOTOMETRY.MAGNITUDE: mag, PHOTOMETRY.E_LOWER_MAGNITUDE: lerr, PHOTOMETRY.E_UPPER_MAGNITUDE: uerr, PHOTOMETRY.SOURCE: source } if inst: photodict[PHOTOMETRY.INSTRUMENT] = inst catalog.entries[name].add_photometry(**photodict) # Benetti 03-08-17 donation path = os.path.join(catalog.get_current_task_repo(), 'Donations', 'Benetti-03-08-17', '1999E.dat') with open(path, 'r') as f: tsvin = list(csv.reader(f, delimiter=' ', skipinitialspace=True)) name, source = catalog.new_entry('SN1999E', bibcode='2003MNRAS.340..191R') bands = None for row in tsvin: if not row or row[0][0] == '#': continue if not bands: bands = row[2:-2] continue mjd = row[1] tel = row[-1] if 'IAUC' not in row[-1] else None for bi, band in enumerate(bands): mag = row[2 + 2 * bi] if mag == '9999': continue err = row[2 + 2 * bi + 1] limit = row[6] == 'True' photodict = { PHOTOMETRY.TIME: mjd, PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.TELESCOPE: tel, PHOTOMETRY.BAND: band, PHOTOMETRY.MAGNITUDE: mag, PHOTOMETRY.SOURCE: source } if err != '.00': photodict[PHOTOMETRY.E_MAGNITUDE] = str(Decimal(err)) if tel: photodict[PHOTOMETRY.TELESCOPE] = tel catalog.entries[name].add_photometry(**photodict) # Nicholl 01-29-17 donation with open( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Nicholl-01-29-17', 'meta.json'), 'r') as f: metadict = json.loads(f.read()) file_names = glob( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Nicholl-01-29-17', '*.txt')) for path in file_names: data = read(path, format='cds') oname = path.split('/')[-1].split('_')[0] name, source = catalog.new_entry(oname, bibcode=metadict[oname]['bibcode']) for row in pbar(data, task_str + ': Nicholl ' + oname): photodict = { PHOTOMETRY.TIME: str(row['MJD']), PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.MAGNITUDE: str(row['mag']), PHOTOMETRY.BAND: row['Filter'], PHOTOMETRY.SOURCE: source } if 'system' in metadict[oname]: photodict[PHOTOMETRY.SYSTEM] = metadict[oname]['system'] if 'l_mag' in row.columns and row['l_mag'] == '>': photodict[PHOTOMETRY.UPPER_LIMIT] = True elif 'e_mag' in row.columns: photodict[PHOTOMETRY.E_MAGNITUDE] = str(row['e_mag']) if 'Telescope' in row.columns: photodict[PHOTOMETRY.TELESCOPE] = row['Telescope'] catalog.entries[name].add_photometry(**photodict) # Arcavi 2016gkg donation path = os.path.join(catalog.get_current_task_repo(), 'Donations', 'Arcavi-01-24-17', 'SN2016gkg.txt') with open(path, 'r') as f: tsvin = list(csv.reader(f, delimiter=' ', skipinitialspace=True)) name, source = catalog.new_entry('SN2016gkg', bibcode='2016arXiv161106451A') for row in tsvin: if row[0][0] == '#': continue mjd = str(jd_to_mjd(Decimal(row[0]))) tel = row[1] band = row[3] mag = row[4] err = row[5] limit = row[6] == 'True' photodict = { PHOTOMETRY.TIME: mjd, PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.TELESCOPE: tel, PHOTOMETRY.BAND: band, PHOTOMETRY.MAGNITUDE: mag, PHOTOMETRY.SOURCE: source } if limit: photodict[PHOTOMETRY.UPPER_LIMIT] = True else: photodict[PHOTOMETRY.E_MAGNITUDE] = err catalog.entries[name].add_photometry(**photodict) # Nicholl Gaia16apd donation path = os.path.join(catalog.get_current_task_repo(), 'Donations', 'Nicholl-01-20-17', 'gaia16apd_phot.txt') data = read(path, format='cds') name, source = catalog.new_entry('Gaia16apd', bibcode='2017ApJ...835L...8N') for row in pbar(data, task_str + ': Nicholl Gaia16apd'): photodict = { PHOTOMETRY.TIME: str(row['MJD']), PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.MAGNITUDE: str(row['mag']), PHOTOMETRY.BAND: row['Filter'], PHOTOMETRY.TELESCOPE: row['Telescope'], PHOTOMETRY.SOURCE: source } if row['l_mag'] == '>': photodict[PHOTOMETRY.UPPER_LIMIT] = True else: photodict[PHOTOMETRY.E_MAGNITUDE] = str(row['e_mag']) catalog.entries[name].add_photometry(**photodict) # Kuncarayakti-01-09-17 datafile = os.path.join(catalog.get_current_task_repo(), 'Donations', 'Kuncarayakti-01-09-17', 'SN1978K.dat') inpname = os.path.basename(datafile).split('.')[0] with open(datafile, 'r') as f: tsvin = csv.reader(f, delimiter=' ', skipinitialspace=True) host = False for ri, row in enumerate(tsvin): if ri == 0: continue if row[0][0] == '#': rsplit = [x.strip('# ') for x in ' '.join(row).split(',')] bc = rsplit[0] tel, ins = '', '' if len(rsplit) > 1: tel = rsplit[1] if len(rsplit) > 2: ins = rsplit[2] continue (name, source) = catalog.new_entry(inpname, bibcode=bc) mag = row[4] err = row[5] mjd = str(astrotime('-'.join(row[:3]), format='iso').mjd) photodict = { PHOTOMETRY.BAND: row[3], PHOTOMETRY.TIME: mjd, PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.MAGNITUDE: mag.strip('>s'), PHOTOMETRY.SOURCE: source } if is_number(err): photodict[PHOTOMETRY.E_MAGNITUDE] = err if tel: photodict[PHOTOMETRY.TELESCOPE] = tel if ins: photodict[PHOTOMETRY.INSTRUMENT] = ins if '>' in mag: photodict[PHOTOMETRY.UPPER_LIMIT] = True if 's' in mag: photodict[PHOTOMETRY.SYNTHETIC] = True catalog.entries[name].add_photometry(**photodict) # Nugent 01-09-17 donation file_names = glob( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Nugent-01-09-17', '*.dat')) for datafile in pbar_strings(file_names, task_str + ': Nugent-01-09-17'): inpname = os.path.basename(datafile).split('.')[0] (name, source) = catalog.new_entry(inpname, bibcode='2006ApJ...645..841N') with open(datafile, 'r') as f: tsvin = csv.reader(f, delimiter=' ', skipinitialspace=True) host = False for urow in tsvin: row = list(filter(None, urow)) counts = row[2] e_counts = row[3] zp = row[4] photodict = { PHOTOMETRY.BAND: row[1], PHOTOMETRY.TIME: row[0], PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.COUNT_RATE: counts, PHOTOMETRY.E_COUNT_RATE: e_counts, PHOTOMETRY.ZERO_POINT: zp, PHOTOMETRY.TELESCOPE: 'CFHT', PHOTOMETRY.SURVEY: 'SNLS', PHOTOMETRY.SOURCE: source } set_pd_mag_from_counts(photodict, counts, ec=e_counts, zp=zp) catalog.entries[name].add_photometry(**photodict) # Inserra 09-04-16 donation file_names = glob( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Inserra-09-04-16', '*.txt')) for datafile in pbar_strings(file_names, task_str + ': Inserra-09-04-16'): inpname = os.path.basename(datafile).split('.')[0] (name, source) = catalog.new_entry(inpname, bibcode='2013ApJ...770..128I') with open(datafile, 'r') as f: tsvin = csv.reader(f, delimiter=' ', skipinitialspace=True) host = False for row in tsvin: if row[0][0] == '#': if row[0] == '#Host': host = True continue host = False bands = row[3:-1] continue for bi, ba in enumerate(bands): mag = row[5 + 2 * bi] if not is_number(mag): continue system = 'AB' if ba in ['U', 'B', 'V', 'R', 'I', 'J', 'H', 'K']: system = 'Vega' photodict = { PHOTOMETRY.TIME: row[3], PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.BAND: ba, PHOTOMETRY.MAGNITUDE: mag.strip('< '), PHOTOMETRY.SOURCE: source, PHOTOMETRY.SYSTEM: system } if 'ATel' not in row[-1]: photodict[PHOTOMETRY.TELESCOPE] = row[-1] if host: photodict[PHOTOMETRY.HOST] = True if '<' in mag: photodict[PHOTOMETRY.UPPER_LIMIT] = True e_mag = row[5 + 2 * bi + 1].strip('() ') if is_number(e_mag): photodict[PHOTOMETRY.E_MAGNITUDE] = e_mag catalog.entries[name].add_photometry(**photodict) # Nicholl 04-01-16 donation with open( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Nicholl-04-01-16', 'bibcodes.json'), 'r') as f: bcs = json.loads(f.read()) kcorrected = ['SN2011ke', 'SN2011kf', 'SN2012il', 'PTF10hgi', 'PTF11rks'] ignorephoto = ['PTF10hgi', 'PTF11rks', 'SN2011ke', 'SN2011kf', 'SN2012il'] file_names = glob( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Nicholl-04-01-16/*.txt')) for datafile in pbar_strings(file_names, task_str + ': Nicholl-04-01-16'): inpname = os.path.basename(datafile).split('_')[0] isk = inpname in kcorrected name = catalog.add_entry(inpname) bibcode = '' for bc in bcs: if inpname in bcs[bc]: bibcode = bc if not bibcode: raise ValueError('Bibcode not found!') source = catalog.entries[name].add_source(bibcode=bibcode) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, inpname, source) if inpname in ignorephoto: continue with open(datafile, 'r') as f: tsvin = csv.reader(f, delimiter='\t', skipinitialspace=True) rtelescope = '' for r, rrow in enumerate(tsvin): row = list(filter(None, rrow)) if not row: continue if row[0] == '#MJD': bands = [x for x in row[1:] if x and 'err' not in x] elif row[0][0] == '#' and len(row[0]) > 1: rtelescope = row[0][1:] if row[0][0] == '#': continue mjd = row[0] if not is_number(mjd): continue for v, val in enumerate(row[1::2]): upperlimit = '' mag = val.strip('>') emag = row[2 * v + 2] if '>' in val or (is_number(emag) and float(emag) == 0.0): upperlimit = True if (not is_number(mag) or isnan(float(mag)) or float(mag) > 90.0): continue band = bands[v] instrument = '' survey = '' system = '' telescope = rtelescope if telescope == 'LSQ': instrument = 'QUEST' elif telescope == 'PS1': instrument = 'GPC' elif telescope == 'NTT': instrument = 'EFOSC' elif telescope == 'GROND': instrument = 'GROND' telescope = 'MPI/ESO 2.2m' else: if band == 'NUV': instrument = 'GALEX' telescope = 'GALEX' elif band in ['u', 'g', 'r', 'i', 'z']: if inpname.startswith('PS1'): instrument = 'GPC' telescope = 'PS1' survey = 'Pan-STARRS' elif inpname.startswith('PTF'): telescope = 'P60' survey = 'PTF' elif band.upper() in ['UVW2', 'UVW1', 'UVM2']: instrument = 'UVOT' telescope = 'Swift' if inpname in ['PTF12dam']: system = 'AB' if inpname in ['SCP-06F6']: system = 'Vega' photodict = { PHOTOMETRY.TIME: mjd, PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.BAND: band, PHOTOMETRY.MAGNITUDE: mag, PHOTOMETRY.UPPER_LIMIT: upperlimit, PHOTOMETRY.SOURCE: source } if instrument: photodict[PHOTOMETRY.INSTRUMENT] = instrument if telescope: photodict[PHOTOMETRY.TELESCOPE] = telescope if survey: photodict[PHOTOMETRY.SURVEY] = survey if system: photodict[PHOTOMETRY.SYSTEM] = system if (is_number(emag) and not isnan(float(emag)) and float(emag) > 0.0): photodict[PHOTOMETRY.E_MAGNITUDE] = emag if isk: photodict[PHOTOMETRY.KCORRECTED] = True catalog.entries[name].add_photometry(**photodict) catalog.journal_entries() # Maggi 04-11-16 donation (MC SNRs) with open( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Maggi-04-11-16', 'LMCSNRs_OpenSNe.csv')) as f: tsvin = csv.reader(f, delimiter=',') for row in pbar(list(tsvin), task_str + ': Maggi-04-11-16/LMCSNRs'): name = 'MCSNR ' + row[0] name = catalog.add_entry(name) ra = row[2] dec = row[3] source = (catalog.entries[name].add_source( bibcode='2016A&A...585A.162M')) catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, 'LMCSNR J' + rep_chars(ra, ' :.') + rep_chars(dec, ' :.'), source) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) if row[1] != 'noname': catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[1], source) catalog.entries[name].add_quantity(SUPERNOVA.RA, row[2], source) catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[3], source) catalog.entries[name].add_quantity(SUPERNOVA.HOST, 'LMC', source) if row[4] == '1': catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, 'Ia', source) elif row[4] == '2': catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, 'CC', source) with open( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Maggi-04-11-16', 'SMCSNRs_OpenSNe.csv')) as f: tsvin = csv.reader(f, delimiter=',') for row in pbar(list(tsvin), task_str + ': Maggi-04-11-16/SMCSNRs'): name = 'MCSNR ' + row[0] name = catalog.add_entry(name) source = catalog.entries[name].add_source(name='Pierre Maggi') ra = row[3] dec = row[4] catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, 'SMCSNR J' + ra.replace(':', '')[:6] + dec.replace(':', '')[:7], source) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[1], source) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[2], source) catalog.entries[name].add_quantity(SUPERNOVA.RA, row[3], source) catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[4], source) catalog.entries[name].add_quantity(SUPERNOVA.HOST, 'SMC', source) catalog.journal_entries() # Galbany 04-18-16 donation folders = next( os.walk( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Galbany-04-18-16/')))[1] bibcode = '2016AJ....151...33G' for folder in folders: infofiles = glob( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Galbany-04-18-16/') + folder + '/*.info') photfiles = glob( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Galbany-04-18-16/') + folder + '/*.out*') zhel = '' zcmb = '' zerr = '' for path in infofiles: with open(path, 'r') as f: lines = f.read().splitlines() for line in lines: splitline = line.split(':') field = splitline[0].strip().lower() value = splitline[1].strip() if field == 'name': name = value[:6].upper() name += (value[6].upper() if len(value) == 7 else value[6:]) name = catalog.add_entry(name) source = (catalog.entries[name].add_source( bibcode=bibcode)) catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, name, source) elif field == 'type': claimedtype = value.replace('SN', '') catalog.entries[name].add_quantity( SUPERNOVA.CLAIMED_TYPE, claimedtype, source) elif field == 'zhel': zhel = value elif field == 'redshift_error': zerr = value elif field == 'zcmb': zcmb = value elif field == 'ra': catalog.entries[name].add_quantity( SUPERNOVA.RA, value, source, u_value='floatdegrees') elif field == 'dec': catalog.entries[name].add_quantity( SUPERNOVA.DEC, value, source, u_value='floatdegrees') elif field == 'host': value = value.replace('- ', '-').replace('G ', 'G') catalog.entries[name].add_quantity( SUPERNOVA.HOST, value, source) elif field == 'e(b-v)_mw': catalog.entries[name].add_quantity( SUPERNOVA.EBV, value, source) catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, zhel, source, e_value=zerr, kind='heliocentric') catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, zcmb, source, e_value=zerr, kind='cmb') for path in photfiles: with open(path, 'r') as f: band = '' lines = f.read().splitlines() for li, line in enumerate(lines): if li in [0, 2, 3]: continue if li == 1: band = line.split(':')[-1].strip() else: cols = list(filter(None, line.split())) if not cols: continue catalog.entries[name].add_photometry( time=cols[0], u_time='MJD', magnitude=cols[1], e_magnitude=cols[2], band=band, system=cols[3], telescope=cols[4], source=source) catalog.journal_entries() # Nicholl 05-03-16 files = glob( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Nicholl-05-03-16', '*.txt')) name = catalog.add_entry('SN2015bn') for fi in pbar(files, task_str + ': Nicholl-05-03-16'): if 'late' in fi: bc = '2016ApJ...828L..18N' else: bc = '2016ApJ...826...39N' source = catalog.entries[name].add_source(bibcode=bc) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, 'PS15ae', source) telescope = os.path.basename(fi).split('_')[1] with open(fi, 'r') as f: lines = f.read().splitlines() for li, line in enumerate(lines): if not line or (line[0] == '#' and li != 0): continue cols = list(filter(None, line.split())) if not cols: continue if li == 0: bands = cols[1:] continue mjd = cols[0] for ci, col in enumerate(cols[1::2]): if not is_number(col) or np.isnan(float(col)): continue band = bands[ci] band_set = '' system = 'Vega' if bands[ci] in ["u'", "g'", "r'", "i'", "z'"]: band_set = 'SDSS' system = 'SDSS' elif telescope == 'ASASSN': band_set = 'ASASSN' system = 'Vega' photodict = { PHOTOMETRY.TIME: mjd, PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.MAGNITUDE: col, PHOTOMETRY.BAND: bands[ci], PHOTOMETRY.SOURCE: source, PHOTOMETRY.TELESCOPE: telescope, PHOTOMETRY.SYSTEM: system } if band_set: photodict[PHOTOMETRY.BAND_SET] = band_set emag = cols[2 * ci + 2] if is_number(emag): photodict[PHOTOMETRY.E_MAGNITUDE] = emag else: photodict[PHOTOMETRY.UPPER_LIMIT] = True if telescope == 'Swift': photodict[PHOTOMETRY.INSTRUMENT] = 'UVOT' catalog.entries[name].add_photometry(**photodict) catalog.journal_entries() return
def do_cfa_spectra(catalog): """Import spectra from the CfA archive.""" task_str = catalog.get_current_task_str() # II spectra oldname = '' file_names = next( os.walk(os.path.join(catalog.get_current_task_repo(), 'CfA_SNII')))[1] for ni, name in enumerate(pbar_strings(file_names, task_str)): fullpath = os.path.join(catalog.get_current_task_repo(), 'CfA_SNII/') + name origname = name if name.startswith('sn') and is_number(name[2:6]): name = 'SN' + name[2:] name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) reference = 'CfA Supernova Archive' refurl = 'https://www.cfa.harvard.edu/supernova/SNarchive.html' source = catalog.entries[name].add_source( name=reference, url=refurl, secondary=True, acknowledgment=ACKN_CFA) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) for fi, fname in enumerate( sorted( glob(fullpath + '/*'), key=lambda s: s.lower())): filename = os.path.basename(fname) fileparts = filename.split('-') if origname.startswith('sn') and is_number(origname[2:6]): year = fileparts[1][:4] month = fileparts[1][4:6] day = fileparts[1][6:] instrument = fileparts[2].split('.')[0] else: year = fileparts[2][:4] month = fileparts[2][4:6] day = fileparts[2][6:] instrument = fileparts[3].split('.')[0] time = str( astrotime(year + '-' + month + '-' + str(floor(float(day))) .zfill(2)).mjd + float(day) - floor(float(day))) f = open(fname, 'r') data = csv.reader(f, delimiter=' ', skipinitialspace=True) data = [list(i) for i in zip(*data)] wavelengths = data[0] fluxes = data[1] errors = data[2] sources = uniq_cdl([ source, (catalog.entries[name] .add_source(bibcode='2017arXiv170601030H')) ]) catalog.entries[name].add_spectrum( u_wavelengths='Angstrom', u_fluxes='erg/s/cm^2/Angstrom', filename=filename, wavelengths=wavelengths, fluxes=fluxes, u_time='MJD' if time else '', time=time, instrument=instrument, u_errors='ergs/s/cm^2/Angstrom', errors=errors, source=sources, dereddened=False, deredshifted=False) if catalog.args.travis and ni >= catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() # Ia spectra oldname = '' file_names = next( os.walk(os.path.join(catalog.get_current_task_repo(), 'CfA_SNIa')))[1] for ni, name in enumerate(pbar_strings(file_names, task_str)): fullpath = os.path.join(catalog.get_current_task_repo(), 'CfA_SNIa/') + name origname = name if name.startswith('sn') and is_number(name[2:6]): name = 'SN' + name[2:] if name.startswith('snf') and is_number(name[3:7]): name = 'SNF' + name[3:] name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) reference = 'CfA Supernova Archive' refurl = 'https://www.cfa.harvard.edu/supernova/SNarchive.html' source = catalog.entries[name].add_source( name=reference, url=refurl, secondary=True, acknowledgment=ACKN_CFA) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) for fi, fname in enumerate( sorted( glob(fullpath + '/*'), key=lambda s: s.lower())): filename = os.path.basename(fname) fileparts = filename.split('-') if origname.startswith('sn') and is_number(origname[2:6]): year = fileparts[1][:4] month = fileparts[1][4:6] day = fileparts[1][6:] instrument = fileparts[2].split('.')[0] else: year = fileparts[2][:4] month = fileparts[2][4:6] day = fileparts[2][6:] instrument = fileparts[3].split('.')[0] time = str( astrotime(year + '-' + month + '-' + str(floor(float(day))) .zfill(2)).mjd + float(day) - floor(float(day))) f = open(fname, 'r') data = csv.reader(f, delimiter=' ', skipinitialspace=True) data = [list(i) for i in zip(*data)] wavelengths = data[0] fluxes = data[1] errors = data[2] sources = uniq_cdl([ source, (catalog.entries[name] .add_source(bibcode='2012AJ....143..126B')), (catalog.entries[name] .add_source(bibcode='2008AJ....135.1598M')) ]) catalog.entries[name].add_spectrum( u_wavelengths='Angstrom', u_fluxes='erg/s/cm^2/Angstrom', filename=filename, wavelengths=wavelengths, fluxes=fluxes, u_time='MJD' if time else '', time=time, instrument=instrument, u_errors='ergs/s/cm^2/Angstrom', errors=errors, source=sources, dereddened=False, deredshifted=False) if catalog.args.travis and ni >= catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() # Ibc spectra oldname = '' file_names = next( os.walk(os.path.join(catalog.get_current_task_repo(), 'CfA_SNIbc')))[1] for ni, name in enumerate(pbar(file_names, task_str)): fullpath = os.path.join(catalog.get_current_task_repo(), 'CfA_SNIbc/') + name if name.startswith('sn') and is_number(name[2:6]): name = 'SN' + name[2:] name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) reference = 'CfA Supernova Archive' refurl = 'https://www.cfa.harvard.edu/supernova/SNarchive.html' source = catalog.entries[name].add_source( name=reference, url=refurl, secondary=True, acknowledgment=ACKN_CFA) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) for fi, fname in enumerate( sorted( glob(fullpath + '/*'), key=lambda s: s.lower())): filename = os.path.basename(fname) fileparts = filename.split('-') instrument = '' year = fileparts[1][:4] month = fileparts[1][4:6] day = fileparts[1][6:].split('.')[0] if len(fileparts) > 2: instrument = fileparts[-1].split('.')[0] time = str( astrotime(year + '-' + month + '-' + str(floor(float(day))) .zfill(2)).mjd + float(day) - floor(float(day))) f = open(fname, 'r') data = csv.reader(f, delimiter=' ', skipinitialspace=True) data = [list(i) for i in zip(*data)] wavelengths = data[0] fluxes = data[1] sources = uniq_cdl([ source, catalog.entries[name] .add_source(bibcode='2014AJ....147...99M') ]) catalog.entries[name].add_spectrum( u_wavelengths='Angstrom', u_fluxes='erg/s/cm^2/Angstrom', wavelengths=wavelengths, filename=filename, fluxes=fluxes, u_time='MJD' if time else '', time=time, instrument=instrument, source=sources, dereddened=False, deredshifted=False) if catalog.args.travis and ni >= catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() # Other spectra oldname = '' file_names = next( os.walk(os.path.join(catalog.get_current_task_repo(), 'CfA_Extra')))[1] for ni, name in enumerate(pbar_strings(file_names, task_str)): fullpath = os.path.join(catalog.get_current_task_repo(), 'CfA_Extra/') + name if name.startswith('sn') and is_number(name[2:6]): name = 'SN' + name[2:] name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) reference = 'CfA Supernova Archive' refurl = 'https://www.cfa.harvard.edu/supernova/SNarchive.html' source = catalog.entries[name].add_source( name=reference, url=refurl, secondary=True, acknowledgment=ACKN_CFA) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) for fi, fname in enumerate( sorted( glob(fullpath + '/*'), key=lambda s: s.lower())): if not os.path.isfile(fname): continue filename = os.path.basename(fname) if ((not filename.startswith('sn') or not filename.endswith('flm') or any( x in filename for x in ['-interp', '-z', '-dered', '-obj', '-gal']))): continue fileparts = filename.split('.')[0].split('-') instrument = '' time = '' if len(fileparts) > 1: year = fileparts[1][:4] month = fileparts[1][4:6] day = fileparts[1][6:] if is_number(year) and is_number(month) and is_number(day): if len(fileparts) > 2: instrument = fileparts[-1] time = str( astrotime(year + '-' + month + '-' + str( floor(float(day))).zfill(2)).mjd + float(day) - floor(float(day))) f = open(fname, 'r') data = csv.reader(f, delimiter=' ', skipinitialspace=True) data = [list(i) for i in zip(*data)] wavelengths = data[0] fluxes = [str(Decimal(x) * Decimal(1.0e-15)) for x in data[1]] catalog.entries[name].add_spectrum( u_wavelengths='Angstrom', u_fluxes='erg/s/cm^2/Angstrom', wavelengths=wavelengths, filename=filename, fluxes=fluxes, u_time='MJD' if time else '', time=time, instrument=instrument, source=source, dereddened=False, deredshifted=False) if catalog.args.travis and ni >= catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() return
def do_test(catalog): log = catalog.log log.info("do_test()") task_str = catalog.get_current_task_str() log.info("`task_str`: '{}'".format(task_str)) if len(catalog.entries) != 0: raise RuntimeError("Run test only with empty catalog.") # Test URL retrieve functions # --------------------------- log.info("`args.archived` = '{}', `current_task.archived` = '{}'".format( catalog.args.archived, catalog.current_task.archived)) test_load_url(catalog) # Test repo path functions # ------------------------ paths = catalog.PATHS.get_all_repo_folders() for path in tq(paths, currenttask='Test tq progress bar.'): tprint('Test tprint.') log.debug(path) paths = catalog.PATHS.get_repo_input_folders() for path in pbar_strings(paths, desc='Test pbar_strings progress bar.'): log.debug(path) boneyard = catalog.PATHS.get_repo_boneyard() log.debug(boneyard) # Create a Fake Entry, with some Fake Data # ---------------------------------------- _first_event_first_source(catalog) log_str = "ADDING SECOND SOURCE" log.info("\n\n{}\n{}\n{}\n\n".format("=" * 100, log_str, "=" * 100)) # Add new Data, from different source, to same fake entry # ------------------------------------------------------- _first_event_second_source(catalog) # Make sure output file for this test exists outdir, filename = catalog.entries[FAKE_ALIAS_1]._get_save_path() save_name = os.path.join(outdir, filename + '.json') if not os.path.exists(save_name): raise RuntimeError("File not found in '{}'".format(save_name)) # Delete created test file catalog._delete_entry_file(entry_name=FAKE_ALIAS_1) # Make sure it was deleted if os.path.exists(save_name): raise RuntimeError("File not deleted at '{}'".format(save_name)) # Delete entry in catalog del catalog.entries[FAKE_ALIAS_1] # Make sure entry was deleted if len(catalog.entries) != 0: raise RuntimeError("Error deleting test entry!") # Add entry back catalog to test later tasks _first_event_first_source(catalog) _first_event_second_source(catalog) # Test some utility functions log.debug("Preferred name for 2nd source: " + catalog.get_preferred_name(FAKE_ALIAS_2)) log.debug("Entry exists? " + str(catalog.entry_exists(FAKE_ALIAS_2))) log.debug("Entry text: " + catalog.entries[FAKE_ALIAS_1].get_entry_text( os.path.join(outdir, filename + '.json'))) # Third source is a duplicate that will be merged _first_event_third_source(catalog) # Add second event to perform different tests _second_event(catalog) # Delete name to test name re-addition in sanitize for ii, alias in enumerate( catalog.entries[FAKE_ALIAS_5][ENTRY.ALIAS].copy()): if alias[QUANTITY.VALUE] == FAKE_ALIAS_5: del catalog.entries[FAKE_ALIAS_1][ENTRY.ALIAS][ii] break return
def do_sdss_photo(catalog): task_str = catalog.get_current_task_str() # Load up metadata first with open(os.path.join(catalog.get_current_task_repo(), 'SDSS/sdsssn_master.dat2'), 'r') as f: rows = list(csv.reader(f.read().splitlines()[1:], delimiter=' ')) ignored_cids = [] columns = { SUPERNOVA.RA: 1, SUPERNOVA.DEC: 2, SUPERNOVA.ALIAS: 4, SUPERNOVA.CLAIMED_TYPE: 5, SUPERNOVA.REDSHIFT: 11, SUPERNOVA.MAX_DATE: 21, SUPERNOVA.HOST_RA: 99, SUPERNOVA.HOST_DEC: 100 } colnums = {v: k for k, v in columns.items()} rows = [[x.replace('\\N', '') for x in y] for y in rows] co = [[x[0], x[99], x[100]] for x in rows if x[99] and x[100]] coo = coord([x[1] for x in co], [x[2] for x in co], unit="deg") coo = [''.join([y[:9] for y in x.split()]) for x in coo.to_string('hmsdms', sep='')] hostdict = dict(zip([x[0] for x in co], ['SDSS J' + x[1:] for x in coo])) for ri, row in enumerate(pbar(rows, task_str + ": metadata")): name = '' # Check if type is non-SNe first ct = row[columns[SUPERNOVA.CLAIMED_TYPE]] al = row[columns[SUPERNOVA.ALIAS]] if ct in ['AGN', 'Variable'] and not al: catalog.log.info('`{}` is not a SN, not ' 'adding.'.format(row[0])) ignored_cids.append(row[0]) continue # Add entry (name, source) = catalog.new_entry( 'SDSS-II SN ' + row[0], bibcode='2014arXiv1401.3317S', url='http://data.sdss3.org/sas/dr10/boss/papers/supernova/') # Add host name if row[0] in hostdict: catalog.entries[name].add_quantity(SUPERNOVA.HOST, hostdict[row[0]], source) # Add other metadata for cn in colnums: key = colnums[cn] if not key: continue ic = int(cn) val = row[ic] if not val: continue kwargs = {} if key == SUPERNOVA.ALIAS: val = 'SN' + val elif key in [SUPERNOVA.RA, SUPERNOVA.DEC, SUPERNOVA.HOST_RA, SUPERNOVA.HOST_DEC]: kwargs = {QUANTITY.U_VALUE: 'floatdegrees'} if key in [SUPERNOVA.RA, SUPERNOVA.HOST_RA]: fval = float(val) if fval < 0.0: val = str(Decimal(360) + Decimal(fval)) elif key == SUPERNOVA.CLAIMED_TYPE: val = val.lstrip('pz').replace('SN', '') elif key == SUPERNOVA.REDSHIFT: kwargs[QUANTITY.KIND] = 'spectroscopic' if float(row[ic + 1]) > 0.0: kwargs[QUANTITY.E_VALUE] = row[ic + 1] elif key == SUPERNOVA.MAX_DATE: dt = astrotime(float(val), format='mjd').datetime val = make_date_string(dt.year, dt.month, dt.day) catalog.entries[name].add_quantity(key, val, source, **kwargs) with open(os.path.join(catalog.get_current_task_repo(), 'SDSS/2010ApJ...708..661D.txt'), 'r') as sdss_file: bibcodes2010 = sdss_file.read().split('\n') sdssbands = ['u', 'g', 'r', 'i', 'z'] file_names = (list(glob(os.path.join(catalog .get_current_task_repo(), 'SDSS/sum/*.sum'))) + list(glob(os.path.join(catalog .get_current_task_repo(), 'SDSS/SMP_Data/*.dat')))) for fi, fname in enumerate(pbar_strings(file_names, task_str)): tsvin = csv.reader(open(fname, 'r'), delimiter=' ', skipinitialspace=True) basename = os.path.basename(fname) hasred = True rst = 19 if '.dat' in fname: bibcode = '2014arXiv1401.3317S' hasred = False rst = 4 elif basename in bibcodes2010: bibcode = '2010ApJ...708..661D' else: bibcode = '2008AJ....136.2306H' skip_entry = False for rr, row in enumerate(tsvin): if skip_entry: break if rr == 0: # Ignore non-SNe objects and those not in metadata table above if row[3] in ignored_cids: skip_entry = True continue # Ignore IAU names from Sako 2014 as they are unreliable if row[5] == 'RA:' or bibcode == '2014arXiv1401.3317S': name = 'SDSS-II SN ' + row[3] else: name = 'SN' + row[5] name = catalog.add_entry(name) source = catalog.entries[name].add_source(bibcode=bibcode) catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, name, source) catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, 'SDSS-II SN ' + row[3], source) if row[5] != 'RA:' and bibcode == '2014arXiv1401.3317S': year = re.findall(r'\d+', name)[0] catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, year, source) catalog.entries[name].add_quantity( SUPERNOVA.RA, row[-4], source, u_value='floatdegrees') catalog.entries[name].add_quantity( SUPERNOVA.DEC, row[-2], source, u_value='floatdegrees') if hasred and rr == 1: error = row[4] if float(row[4]) >= 0.0 else '' (catalog.entries[name] .add_quantity(SUPERNOVA.REDSHIFT, row[2], source, e_value=error, kind='heliocentric')) if rr >= rst: # Skip bad measurements if int(row[0]) > 1024: continue mjd = row[1] band = sdssbands[int(row[2])] magnitude = row[3] e_mag = row[4] telescope = 'SDSS' (catalog.entries[name] .add_photometry(time=mjd, telescope=telescope, band=band, magnitude=magnitude, e_magnitude=e_mag, source=source, system='SDSS')) if not fi % 1000: catalog.journal_entries() catalog.journal_entries() return
def do_suspect_photo(catalog): task_str = catalog.get_current_task_str() with open( os.path.join(catalog.get_current_task_repo(), 'suspectreferences.csv'), 'r') as f: tsvin = csv.reader(f, delimiter=',', skipinitialspace=True) suspectrefdict = {} for row in tsvin: suspectrefdict[row[0]] = row[1] file_names = list( sorted( glob( os.path.join(catalog.get_current_task_repo(), 'SUSPECT/*.html')))) for datafile in pbar_strings(file_names, task_str): basename = os.path.basename(datafile) basesplit = basename.split('-') oldname = basesplit[1] name = catalog.add_entry(oldname) if name.startswith('SN') and is_number(name[2:]): name = name + 'A' band = basesplit[3].split('.')[0] ei = int(basesplit[2]) bandlink = 'file://' + os.path.abspath(datafile) bandresp = urllib.request.urlopen(bandlink) bandsoup = BeautifulSoup(bandresp, 'html5lib') bandtable = bandsoup.find('table') names = bandsoup.body.findAll(text=re.compile('Name')) reference = '' for link in bandsoup.body.findAll('a'): if 'adsabs' in link['href']: reference = str(link).replace('"', "'") bibcode = unescape(suspectrefdict[reference]) source = catalog.entries[name].add_source(bibcode=bibcode) sec_ref = 'SUSPECT' sec_refurl = 'https://www.nhn.ou.edu/~suspect/' sec_source = catalog.entries[name].add_source(name=sec_ref, url=sec_refurl, secondary=True) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oldname, sec_source) if ei == 1: year = re.findall(r'\d+', name)[0] catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, year, sec_source) catalog.entries[name].add_quantity(SUPERNOVA.HOST, names[1].split(':')[1].strip(), sec_source) redshifts = bandsoup.body.findAll(text=re.compile('Redshift')) if redshifts: catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, redshifts[0].split(':')[1].strip(), sec_source, kind='heliocentric') # hvels = bandsoup.body.findAll(text=re.compile('Heliocentric # Velocity')) # if hvels: # vel = hvels[0].split(':')[1].strip().split(' ')[0] # catalog.entries[name].add_quantity(SUPERNOVA.VELOCITY, vel, # sec_source, # kind='heliocentric') types = bandsoup.body.findAll(text=re.compile('Type')) catalog.entries[name].add_quantity( SUPERNOVA.CLAIMED_TYPE, types[0].split(':')[1].strip().split(' ')[0], sec_source) for r, row in enumerate(bandtable.findAll('tr')): if r == 0: continue col = row.findAll('td') mjd = str(jd_to_mjd(Decimal(col[0].contents[0]))) mag = col[3].contents[0] if mag.isspace(): mag = '' else: mag = str(mag) e_magnitude = col[4].contents[0] if e_magnitude.isspace(): e_magnitude = '' else: e_magnitude = str(e_magnitude) catalog.entries[name].add_photometry(time=mjd, u_time='MJD', band=band, magnitude=mag, e_magnitude=e_magnitude, source=sec_source + ',' + source) catalog.journal_entries() return
def do_test(catalog): log = catalog.log log.info("do_test()") task_str = catalog.get_current_task_str() log.info("`task_str`: '{}'".format(task_str)) if len(catalog.entries) != 0: raise RuntimeError("Run test only with empty catalog.") # Test URL retrieve functions # --------------------------- log.info("`args.archived` = '{}', `current_task.archived` = '{}'".format( catalog.args.archived, catalog.current_task.archived)) test_load_url(catalog) # Test repo path functions # ------------------------ paths = catalog.PATHS.get_all_repo_folders() for path in tq(paths, currenttask='Test tq progress bar.'): tprint('Test tprint.') log.debug(path) paths = catalog.PATHS.get_repo_input_folders() for path in pbar_strings(paths, desc='Test pbar_strings progress bar.'): log.debug(path) boneyard = catalog.PATHS.get_repo_boneyard() log.debug(boneyard) # Create a Fake Entry, with some Fake Data # ---------------------------------------- _first_event_first_source(catalog) log_str = "ADDING SECOND SOURCE" log.info("\n\n{}\n{}\n{}\n\n".format("=" * 100, log_str, "=" * 100)) # Add new Data, from different source, to same fake entry # ------------------------------------------------------- _first_event_second_source(catalog) # Make sure output file for this test exists outdir, filename = catalog.entries[FAKE_ALIAS_1]._get_save_path() save_name = os.path.join(outdir, filename + '.json') if not os.path.exists(save_name): raise RuntimeError("File not found in '{}'".format(save_name)) # Delete created test file catalog._delete_entry_file(entry_name=FAKE_ALIAS_1) # Make sure it was deleted if os.path.exists(save_name): raise RuntimeError("File not deleted at '{}'".format(save_name)) # Delete entry in catalog del catalog.entries[FAKE_ALIAS_1] # Make sure entry was deleted if len(catalog.entries) != 0: raise RuntimeError("Error deleting test entry!") # Add entry back catalog to test later tasks _first_event_first_source(catalog) _first_event_second_source(catalog) # Test some utility functions log.debug("Preferred name for 2nd source: " + catalog.get_preferred_name(FAKE_ALIAS_2)) log.debug("Entry exists? " + str(catalog.entry_exists(FAKE_ALIAS_2))) log.debug("Entry text: " + catalog.entries[FAKE_ALIAS_1].get_entry_text( os.path.join(outdir, filename + '.json'))) # Third source is a duplicate that will be merged _first_event_third_source(catalog) # Add second event to perform different tests _second_event(catalog) # Delete name to test name re-addition in sanitize for ii, alias in enumerate( catalog.entries[FAKE_ALIAS_5][ENTRY.ALIAS].copy()): if alias[QUANTITY.VALUE] == FAKE_ALIAS_5: del catalog.entries[FAKE_ALIAS_1][ENTRY.ALIAS][ii] break return
def do_wiserep_spectra(catalog): #if not catalog.args.travis: # from ..input.WISeWEBSpider.wisewebspider import spider # try: # spider(update=True, daysago=7, path="/../../sne-external-WISEREP/") # except: # catalog.log.warning( # 'Spider errored, continuing without letting it complete.') task_str = catalog.get_current_task_str() secondaryreference = 'WISeREP' secondaryrefurl = 'http://wiserep.weizmann.ac.il/' secondarybibcode = '2012PASP..124..668Y' wiserepcnt = 0 # These are known to be in error on the WISeREP page, either fix or ignore # them. wiserepbibcorrectdict = { '2000AJ....120..367G]': '2000AJ....120..367G', 'Harutyunyan et al. 2008': '2008A&A...488..383H', '0609268': '2007AJ....133...58K', '2006ApJ...636...400Q': '2006ApJ...636..400Q', '2011ApJ...741...76': '2011ApJ...741...76C', '2016PASP...128...961': '2016PASP..128...961', '2002AJ....1124..417H': '2002AJ....1124.417H', '2013ApJ…774…58D': '2013ApJ...774...58D', '2011Sci.333..856S': '2011Sci...333..856S', '2014MNRAS.438,368': '2014MNRAS.438..368T', '2012MNRAS.420.1135': '2012MNRAS.420.1135S', '2012Sci..337..942D': '2012Sci...337..942D', 'stt1839': '2013MNRAS.436.3614S', 'arXiv:1605.03136': '2016MNRAS.460.3447T', '10.1093/mnras/stt1839': '2013MNRAS.436.3614S' } file_names = list(glob(os.path.join(catalog.get_current_task_repo(), '*'))) for folder in pbar_strings(file_names, task_str): if '.txt' in folder or '.json' in folder: continue name = os.path.basename(folder).strip() if name.startswith('sn'): name = 'SN' + name[2:] if (name.startswith(('CSS', 'SSS', 'MLS')) and ':' not in name): name = name.replace('-', ':', 1) if name.startswith('MASTERJ'): name = name.replace('MASTERJ', 'MASTER OT J') if name.startswith('PSNJ'): name = name.replace('PSNJ', 'PSN J') name = catalog.add_entry(name) secondarysource = catalog.entries[name].add_source( name=secondaryreference, url=secondaryrefurl, bibcode=secondarybibcode, secondary=True) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, secondarysource) readme_path = os.path.join(folder, 'README.json') if not os.path.exists(readme_path): catalog.log.warning( 'Metadata file not found for event "{}"'.format(name)) continue with open(readme_path, 'r') as f: fileinfo = json.loads(f.read()) files = list( set(glob(folder + '/*')) - set(glob(folder + '/README.json'))) for fname in pbar(files, task_str): specfile = os.path.basename(fname) if specfile not in fileinfo: catalog.log.warning( 'Metadata not found for "{}"'.format(fname)) continue claimedtype = fileinfo[specfile]["Type"] instrument = fileinfo[specfile]["Instrument"] epoch = fileinfo[specfile]["Obs. Date"] observer = fileinfo[specfile]["Observer"] reducer = fileinfo[specfile]["Reducer"] bibcode = fileinfo[specfile]["Bibcode"] redshift = fileinfo[specfile]["Redshift"] survey = fileinfo[specfile]["Program"] reduction = fileinfo[specfile]["Reduction Status"] if bibcode: newbibcode = bibcode if bibcode in wiserepbibcorrectdict: newbibcode = wiserepbibcorrectdict[bibcode] if newbibcode and len(newbibcode) == 19: source = catalog.entries[name].add_source( bibcode=unescape(newbibcode)) else: bibname = unescape(bibcode) source = catalog.entries[name].add_source(name=bibname) catalog.log.warning('Bibcode "{}" is invalid, using as ' '`{}` instead'.format( bibname, SOURCE.NAME)) sources = uniq_cdl([source, secondarysource]) else: sources = secondarysource if claimedtype not in ['Other']: catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, claimedtype, secondarysource) catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, redshift, secondarysource) with open(fname, 'r') as f: data = [x.split() for x in f] skipspec = False newdata = [] oldval = '' for row in data: if row and '#' not in row[0]: if (len(row) >= 2 and is_number(row[0]) and is_number(row[1]) and row[1] != oldval): newdata.append(row) oldval = row[1] if skipspec or not newdata: warnings.warn('Skipped adding spectrum file ' + specfile) continue data = [list(i) for i in zip(*newdata)] wavelengths = data[0] fluxes = data[1] errors = '' if len(data) == 3: errors = data[1] time = str(astrotime(epoch).mjd) if max([float(x) for x in fluxes]) < 1.0e-5: fluxunit = 'erg/s/cm^2/Angstrom' else: fluxunit = 'Uncalibrated' catalog.entries[name].add_spectrum( u_wavelengths='Angstrom', errors=errors, u_fluxes=fluxunit, u_errors=fluxunit if errors else '', wavelengths=wavelengths, fluxes=fluxes, u_time='MJD', time=time, instrument=instrument, source=sources, observer=observer, reducer=reducer, reduction=reduction, filename=specfile, survey=survey, redshift=redshift) catalog.journal_entries() wiserepcnt = wiserepcnt + 1 if (catalog.args.travis and wiserepcnt % catalog.TRAVIS_QUERY_LIMIT == 0): break return
def do_donated_photo(catalog): """Import donated photometry.""" task_str = catalog.get_current_task_str() # Private donations here # if not catalog.args.travis: pass # End private donations # # Ponder 05-12-17 donation with open( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Ponder-05-12-17', 'meta.json'), 'r') as f: metadict = json.loads(f.read()) file_names = glob( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Ponder-05-12-17', '*.dat')) for path in file_names: with open(path, 'r') as f: tsvin = list(csv.reader(f, delimiter=' ', skipinitialspace=True)) oname = path.split('/')[-1].split('.')[0] name, source = catalog.new_entry( oname, bibcode=metadict[oname]['bibcode']) for row in pbar(tsvin, task_str + ': Ponder ' + oname): if row[0][0] == '#' or not is_number(row[-1]): continue mjd = row[1] bandinst = row[2].split('_') band = bandinst[0] inst = '' if len(bandinst) > 1: inst = bandinst[1] mag = row[3] uerr = row[4] lerr = row[5] photodict = { PHOTOMETRY.TIME: mjd, PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.BAND: band, PHOTOMETRY.MAGNITUDE: mag, PHOTOMETRY.E_LOWER_MAGNITUDE: lerr, PHOTOMETRY.E_UPPER_MAGNITUDE: uerr, PHOTOMETRY.SOURCE: source } if inst: photodict[PHOTOMETRY.INSTRUMENT] = inst catalog.entries[name].add_photometry(**photodict) # Benetti 03-08-17 donation path = os.path.join(catalog.get_current_task_repo(), 'Donations', 'Benetti-03-08-17', '1999E.dat') with open(path, 'r') as f: tsvin = list(csv.reader(f, delimiter=' ', skipinitialspace=True)) name, source = catalog.new_entry( 'SN1999E', bibcode='2003MNRAS.340..191R') bands = None for row in tsvin: if not row or row[0][0] == '#': continue if not bands: bands = row[2:-2] continue mjd = row[1] tel = row[-1] if 'IAUC' not in row[-1] else None for bi, band in enumerate(bands): mag = row[2 + 2 * bi] if mag == '9999': continue err = row[2 + 2 * bi + 1] limit = row[6] == 'True' photodict = { PHOTOMETRY.TIME: mjd, PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.TELESCOPE: tel, PHOTOMETRY.BAND: band, PHOTOMETRY.MAGNITUDE: mag, PHOTOMETRY.SOURCE: source } if err != '.00': photodict[PHOTOMETRY.E_MAGNITUDE] = str(Decimal(err)) if tel: photodict[PHOTOMETRY.TELESCOPE] = tel catalog.entries[name].add_photometry(**photodict) # Nicholl 01-29-17 donation with open( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Nicholl-01-29-17', 'meta.json'), 'r') as f: metadict = json.loads(f.read()) file_names = glob( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Nicholl-01-29-17', '*.txt')) for path in file_names: data = read(path, format='cds') oname = path.split('/')[-1].split('_')[0] name, source = catalog.new_entry( oname, bibcode=metadict[oname]['bibcode']) for row in pbar(data, task_str + ': Nicholl ' + oname): photodict = { PHOTOMETRY.TIME: str(row['MJD']), PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.MAGNITUDE: str(row['mag']), PHOTOMETRY.BAND: row['Filter'], PHOTOMETRY.SOURCE: source } if 'system' in metadict[oname]: photodict[PHOTOMETRY.SYSTEM] = metadict[oname]['system'] if 'l_mag' in row.columns and row['l_mag'] == '>': photodict[PHOTOMETRY.UPPER_LIMIT] = True elif 'e_mag' in row.columns: photodict[PHOTOMETRY.E_MAGNITUDE] = str(row['e_mag']) if 'Telescope' in row.columns: photodict[PHOTOMETRY.TELESCOPE] = row['Telescope'] catalog.entries[name].add_photometry(**photodict) # Arcavi 2016gkg donation path = os.path.join(catalog.get_current_task_repo(), 'Donations', 'Arcavi-01-24-17', 'SN2016gkg.txt') with open(path, 'r') as f: tsvin = list(csv.reader(f, delimiter=' ', skipinitialspace=True)) name, source = catalog.new_entry( 'SN2016gkg', bibcode='2016arXiv161106451A') for row in tsvin: if row[0][0] == '#': continue mjd = str(jd_to_mjd(Decimal(row[0]))) tel = row[1] band = row[3] mag = row[4] err = row[5] limit = row[6] == 'True' photodict = { PHOTOMETRY.TIME: mjd, PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.TELESCOPE: tel, PHOTOMETRY.BAND: band, PHOTOMETRY.MAGNITUDE: mag, PHOTOMETRY.SOURCE: source } if limit: photodict[PHOTOMETRY.UPPER_LIMIT] = True else: photodict[PHOTOMETRY.E_MAGNITUDE] = err catalog.entries[name].add_photometry(**photodict) # Nicholl Gaia16apd donation path = os.path.join(catalog.get_current_task_repo(), 'Donations', 'Nicholl-01-20-17', 'gaia16apd_phot.txt') data = read(path, format='cds') name, source = catalog.new_entry( 'Gaia16apd', bibcode='2017ApJ...835L...8N') for row in pbar(data, task_str + ': Nicholl Gaia16apd'): photodict = { PHOTOMETRY.TIME: str(row['MJD']), PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.MAGNITUDE: str(row['mag']), PHOTOMETRY.BAND: row['Filter'], PHOTOMETRY.TELESCOPE: row['Telescope'], PHOTOMETRY.SOURCE: source } if row['l_mag'] == '>': photodict[PHOTOMETRY.UPPER_LIMIT] = True else: photodict[PHOTOMETRY.E_MAGNITUDE] = str(row['e_mag']) catalog.entries[name].add_photometry(**photodict) # Kuncarayakti-01-09-17 datafile = os.path.join(catalog.get_current_task_repo(), 'Donations', 'Kuncarayakti-01-09-17', 'SN1978K.dat') inpname = os.path.basename(datafile).split('.')[0] with open(datafile, 'r') as f: tsvin = csv.reader(f, delimiter=' ', skipinitialspace=True) host = False for ri, row in enumerate(tsvin): if ri == 0: continue if row[0][0] == '#': rsplit = [x.strip('# ') for x in ' '.join(row).split(',')] bc = rsplit[0] tel, ins = '', '' if len(rsplit) > 1: tel = rsplit[1] if len(rsplit) > 2: ins = rsplit[2] continue (name, source) = catalog.new_entry(inpname, bibcode=bc) mag = row[4] err = row[5] mjd = str(astrotime('-'.join(row[:3]), format='iso').mjd) photodict = { PHOTOMETRY.BAND: row[3], PHOTOMETRY.TIME: mjd, PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.MAGNITUDE: mag.strip('>s'), PHOTOMETRY.SOURCE: source } if is_number(err): photodict[PHOTOMETRY.E_MAGNITUDE] = err if tel: photodict[PHOTOMETRY.TELESCOPE] = tel if ins: photodict[PHOTOMETRY.INSTRUMENT] = ins if '>' in mag: photodict[PHOTOMETRY.UPPER_LIMIT] = True if 's' in mag: photodict[PHOTOMETRY.SYNTHETIC] = True catalog.entries[name].add_photometry(**photodict) # Nugent 01-09-17 donation file_names = glob( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Nugent-01-09-17', '*.dat')) for datafile in pbar_strings(file_names, task_str + ': Nugent-01-09-17'): inpname = os.path.basename(datafile).split('.')[0] (name, source) = catalog.new_entry( inpname, bibcode='2006ApJ...645..841N') with open(datafile, 'r') as f: tsvin = csv.reader(f, delimiter=' ', skipinitialspace=True) host = False for urow in tsvin: row = list(filter(None, urow)) counts = row[2] e_counts = row[3] zp = row[4] photodict = { PHOTOMETRY.BAND: row[1], PHOTOMETRY.TIME: row[0], PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.COUNT_RATE: counts, PHOTOMETRY.E_COUNT_RATE: e_counts, PHOTOMETRY.ZERO_POINT: zp, PHOTOMETRY.TELESCOPE: 'CFHT', PHOTOMETRY.SURVEY: 'SNLS', PHOTOMETRY.SOURCE: source } set_pd_mag_from_counts(photodict, counts, ec=e_counts, zp=zp, sig=5.0) catalog.entries[name].add_photometry(**photodict) # Inserra 09-04-16 donation file_names = glob( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Inserra-09-04-16', '*.txt')) for datafile in pbar_strings(file_names, task_str + ': Inserra-09-04-16'): inpname = os.path.basename(datafile).split('.')[0] (name, source) = catalog.new_entry( inpname, bibcode='2013ApJ...770..128I') with open(datafile, 'r') as f: tsvin = csv.reader(f, delimiter=' ', skipinitialspace=True) host = False for row in tsvin: if row[0][0] == '#': if row[0] == '#Host': host = True continue host = False bands = row[3:-1] continue for bi, ba in enumerate(bands): mag = row[5 + 2 * bi] if not is_number(mag): continue system = 'AB' if ba in ['U', 'B', 'V', 'R', 'I', 'J', 'H', 'K']: system = 'Vega' photodict = { PHOTOMETRY.TIME: row[3], PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.BAND: ba, PHOTOMETRY.MAGNITUDE: mag.strip('< '), PHOTOMETRY.SOURCE: source, PHOTOMETRY.SYSTEM: system } if 'ATel' not in row[-1]: photodict[PHOTOMETRY.TELESCOPE] = row[-1] if host: photodict[PHOTOMETRY.HOST] = True if '<' in mag: photodict[PHOTOMETRY.UPPER_LIMIT] = True e_mag = row[5 + 2 * bi + 1].strip('() ') if is_number(e_mag): photodict[PHOTOMETRY.E_MAGNITUDE] = e_mag catalog.entries[name].add_photometry(**photodict) # Nicholl 04-01-16 donation with open( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Nicholl-04-01-16', 'bibcodes.json'), 'r') as f: bcs = json.loads(f.read()) kcorrected = ['SN2011ke', 'SN2011kf', 'SN2012il', 'PTF10hgi', 'PTF11rks'] ignorephoto = ['PTF10hgi', 'PTF11rks', 'SN2011ke', 'SN2011kf', 'SN2012il'] file_names = glob( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Nicholl-04-01-16/*.txt')) for datafile in pbar_strings(file_names, task_str + ': Nicholl-04-01-16'): inpname = os.path.basename(datafile).split('_')[0] isk = inpname in kcorrected name = catalog.add_entry(inpname) bibcode = '' for bc in bcs: if inpname in bcs[bc]: bibcode = bc if not bibcode: raise ValueError('Bibcode not found!') source = catalog.entries[name].add_source(bibcode=bibcode) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, inpname, source) if inpname in ignorephoto: continue with open(datafile, 'r') as f: tsvin = csv.reader(f, delimiter='\t', skipinitialspace=True) rtelescope = '' for r, rrow in enumerate(tsvin): row = list(filter(None, rrow)) if not row: continue if row[0] == '#MJD': bands = [x for x in row[1:] if x and 'err' not in x] elif row[0][0] == '#' and len(row[0]) > 1: rtelescope = row[0][1:] if row[0][0] == '#': continue mjd = row[0] if not is_number(mjd): continue for v, val in enumerate(row[1::2]): upperlimit = '' mag = val.strip('>') emag = row[2 * v + 2] if '>' in val or (is_number(emag) and float(emag) == 0.0): upperlimit = True if (not is_number(mag) or isnan(float(mag)) or float(mag) > 90.0): continue band = bands[v] instrument = '' survey = '' system = '' telescope = rtelescope if telescope == 'LSQ': instrument = 'QUEST' elif telescope == 'PS1': instrument = 'GPC' elif telescope == 'NTT': instrument = 'EFOSC' elif telescope == 'GROND': instrument = 'GROND' telescope = 'MPI/ESO 2.2m' else: if band == 'NUV': instrument = 'GALEX' telescope = 'GALEX' elif band in ['u', 'g', 'r', 'i', 'z']: if inpname.startswith('PS1'): instrument = 'GPC' telescope = 'PS1' survey = 'Pan-STARRS' elif inpname.startswith('PTF'): telescope = 'P60' survey = 'PTF' elif band.upper() in ['UVW2', 'UVW1', 'UVM2']: instrument = 'UVOT' telescope = 'Swift' if inpname in ['PTF12dam']: system = 'AB' if inpname in ['SCP-06F6']: system = 'Vega' photodict = { PHOTOMETRY.TIME: mjd, PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.BAND: band, PHOTOMETRY.MAGNITUDE: mag, PHOTOMETRY.UPPER_LIMIT: upperlimit, PHOTOMETRY.SOURCE: source } if instrument: photodict[PHOTOMETRY.INSTRUMENT] = instrument if telescope: photodict[PHOTOMETRY.TELESCOPE] = telescope if survey: photodict[PHOTOMETRY.SURVEY] = survey if system: photodict[PHOTOMETRY.SYSTEM] = system if (is_number(emag) and not isnan(float(emag)) and float(emag) > 0.0): photodict[PHOTOMETRY.E_MAGNITUDE] = emag if isk: photodict[PHOTOMETRY.KCORRECTED] = True catalog.entries[name].add_photometry(**photodict) catalog.journal_entries() # Maggi 04-11-16 donation (MC SNRs) with open( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Maggi-04-11-16', 'LMCSNRs_OpenSNe.csv')) as f: tsvin = csv.reader(f, delimiter=',') for row in pbar(list(tsvin), task_str + ': Maggi-04-11-16/LMCSNRs'): name = 'MCSNR ' + row[0] name = catalog.add_entry(name) ra = row[2] dec = row[3] source = (catalog.entries[name] .add_source(bibcode='2016A&A...585A.162M')) catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, 'LMCSNR J' + rep_chars(ra, ' :.') + rep_chars(dec, ' :.'), source) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) if row[1] != 'noname': catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[1], source) catalog.entries[name].add_quantity(SUPERNOVA.RA, row[2], source) catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[3], source) catalog.entries[name].add_quantity(SUPERNOVA.HOST, 'LMC', source) if row[4] == '1': catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, 'Ia', source) elif row[4] == '2': catalog.entries[name].add_quantity(SUPERNOVA.CLAIMED_TYPE, 'CC', source) with open( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Maggi-04-11-16', 'SMCSNRs_OpenSNe.csv')) as f: tsvin = csv.reader(f, delimiter=',') for row in pbar(list(tsvin), task_str + ': Maggi-04-11-16/SMCSNRs'): name = 'MCSNR ' + row[0] name = catalog.add_entry(name) source = catalog.entries[name].add_source(name='Pierre Maggi') ra = row[3] dec = row[4] catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, 'SMCSNR J' + ra.replace( ':', '')[:6] + dec.replace(':', '')[:7], source) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[1], source) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[2], source) catalog.entries[name].add_quantity(SUPERNOVA.RA, row[3], source) catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[4], source) catalog.entries[name].add_quantity(SUPERNOVA.HOST, 'SMC', source) catalog.journal_entries() # Galbany 04-18-16 donation folders = next( os.walk( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Galbany-04-18-16/')))[1] bibcode = '2016AJ....151...33G' for folder in folders: infofiles = glob( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Galbany-04-18-16/') + folder + '/*.info') photfiles = glob( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Galbany-04-18-16/') + folder + '/*.out*') zhel = '' zcmb = '' zerr = '' for path in infofiles: with open(path, 'r') as f: lines = f.read().splitlines() for line in lines: splitline = line.split(':') field = splitline[0].strip().lower() value = splitline[1].strip() if field == 'name': name = value[:6].upper() name += (value[6].upper() if len(value) == 7 else value[6:]) name = catalog.add_entry(name) source = (catalog.entries[name] .add_source(bibcode=bibcode)) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) elif field == 'type': claimedtype = value.replace('SN', '') catalog.entries[name].add_quantity( SUPERNOVA.CLAIMED_TYPE, claimedtype, source) elif field == 'zhel': zhel = value elif field == 'redshift_error': zerr = value elif field == 'zcmb': zcmb = value elif field == 'ra': catalog.entries[name].add_quantity( SUPERNOVA.RA, value, source, u_value='floatdegrees') elif field == 'dec': catalog.entries[name].add_quantity( SUPERNOVA.DEC, value, source, u_value='floatdegrees') elif field == 'host': value = value.replace('- ', '-').replace('G ', 'G') catalog.entries[name].add_quantity(SUPERNOVA.HOST, value, source) elif field == 'e(b-v)_mw': catalog.entries[name].add_quantity(SUPERNOVA.EBV, value, source) catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, zhel, source, e_value=zerr, kind='heliocentric') catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, zcmb, source, e_value=zerr, kind='cmb') for path in photfiles: with open(path, 'r') as f: band = '' lines = f.read().splitlines() for li, line in enumerate(lines): if li in [0, 2, 3]: continue if li == 1: band = line.split(':')[-1].strip() else: cols = list(filter(None, line.split())) if not cols: continue catalog.entries[name].add_photometry( time=cols[0], u_time='MJD', magnitude=cols[1], e_magnitude=cols[2], band=band, system=cols[3], telescope=cols[4], source=source) catalog.journal_entries() # Nicholl 05-03-16 files = glob( os.path.join(catalog.get_current_task_repo(), 'Donations', 'Nicholl-05-03-16', '*.txt')) name = catalog.add_entry('SN2015bn') for fi in pbar(files, task_str + ': Nicholl-05-03-16'): if 'late' in fi: bc = '2016ApJ...828L..18N' else: bc = '2016ApJ...826...39N' source = catalog.entries[name].add_source(bibcode=bc) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, 'PS15ae', source) telescope = os.path.basename(fi).split('_')[1] with open(fi, 'r') as f: lines = f.read().splitlines() for li, line in enumerate(lines): if not line or (line[0] == '#' and li != 0): continue cols = list(filter(None, line.split())) if not cols: continue if li == 0: bands = cols[1:] continue mjd = cols[0] for ci, col in enumerate(cols[1::2]): if not is_number(col) or np.isnan(float(col)): continue band = bands[ci] band_set = '' system = 'Vega' if bands[ci] in ["u'", "g'", "r'", "i'", "z'"]: band_set = 'SDSS' system = 'SDSS' elif telescope == 'ASASSN': band_set = 'ASASSN' system = 'Vega' photodict = { PHOTOMETRY.TIME: mjd, PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.MAGNITUDE: col, PHOTOMETRY.BAND: bands[ci], PHOTOMETRY.SOURCE: source, PHOTOMETRY.TELESCOPE: telescope, PHOTOMETRY.SYSTEM: system } if band_set: photodict[PHOTOMETRY.BAND_SET] = band_set emag = cols[2 * ci + 2] if is_number(emag): photodict[PHOTOMETRY.E_MAGNITUDE] = emag else: photodict[PHOTOMETRY.UPPER_LIMIT] = True if telescope == 'Swift': photodict[PHOTOMETRY.INSTRUMENT] = 'UVOT' catalog.entries[name].add_photometry(**photodict) catalog.journal_entries() return
def do_sdss_photo(catalog): task_str = catalog.get_current_task_str() D25 = Decimal('2.5') # fits_path = os.path.join(catalog.get_current_task_repo(), # 'SDSS/SDSS_allCandidates+BOSS_HEAD.FITS') # # hdulist = fits.open(fits_path) # print(hdulist[1].columns) # for ri, row in enumerate(hdulist[1].data['SNID']): # print([[tag, hdulist[1].data[tag][ri]] for tag in hdulist[1].data]) # print(hdulist[1].data['SNID'][ri], hdulist[1].data['IAUC'][ri], # hdulist[1].data['REDSHIFT_HELIO'][ri]) # # # print(hdulist[1].data['MJD']) # hdulist.close() # return # Load up metadata first with open( os.path.join(catalog.get_current_task_repo(), 'SDSS/sdsssn_master.dat2'), 'r') as f: rows = list(csv.reader(f.read().splitlines()[1:], delimiter=' ')) ignored_cids = [] columns = { SUPERNOVA.RA: 1, SUPERNOVA.DEC: 2, SUPERNOVA.ALIAS: 4, SUPERNOVA.CLAIMED_TYPE: 5, SUPERNOVA.REDSHIFT: 11, SUPERNOVA.MAX_DATE: 21, SUPERNOVA.HOST_RA: 99, SUPERNOVA.HOST_DEC: 100 } colnums = {v: k for k, v in columns.items()} rows = [[x.replace('\\N', '') for x in y] for y in rows] co = [[x[0], x[99], x[100]] for x in rows if x[99] and x[100]] coo = coord([x[1] for x in co], [x[2] for x in co], unit="deg") coo = [ ''.join([y[:9] for y in x.split()]) for x in coo.to_string('hmsdms', sep='') ] hostdict = dict( zip([x[0] for x in co], ['SDSS J' + x[1:] for x in coo])) for ri, row in enumerate(pbar(rows, task_str + ": metadata")): name = '' # Check if type is non-SNe first ct = row[columns[SUPERNOVA.CLAIMED_TYPE]] al = row[columns[SUPERNOVA.ALIAS]] if ct in ['AGN', 'Variable', 'Unknown'] and not al: catalog.log.info('`{}` is not a SN, not ' 'adding.'.format(row[0])) ignored_cids.append(row[0]) continue # Add entry (name, source) = catalog.new_entry( 'SDSS-II SN ' + row[0], bibcode='2014arXiv1401.3317S', url='http://data.sdss3.org/sas/dr10/boss/papers/supernova/') # Add host name if row[0] in hostdict: catalog.entries[name].add_quantity(SUPERNOVA.HOST, hostdict[row[0]], source) # Add other metadata for cn in colnums: key = colnums[cn] if not key: continue ic = int(cn) val = row[ic] if not val: continue kwargs = {} if key == SUPERNOVA.ALIAS: val = 'SN' + val elif key in [ SUPERNOVA.RA, SUPERNOVA.DEC, SUPERNOVA.HOST_RA, SUPERNOVA.HOST_DEC ]: kwargs = {QUANTITY.U_VALUE: 'floatdegrees'} if key in [SUPERNOVA.RA, SUPERNOVA.HOST_RA]: fval = float(val) if fval < 0.0: val = str(Decimal(360) + Decimal(fval)) elif key == SUPERNOVA.CLAIMED_TYPE: val = val.lstrip('pz').replace('SN', '') elif key == SUPERNOVA.REDSHIFT: kwargs[QUANTITY.KIND] = 'spectroscopic' if float(val) < -1.0: continue if float(row[ic + 1]) > 0.0: kwargs[QUANTITY.E_VALUE] = row[ic + 1] elif key == SUPERNOVA.MAX_DATE: dt = astrotime(float(val), format='mjd').datetime val = make_date_string(dt.year, dt.month, dt.day) catalog.entries[name].add_quantity(key, val, source, **kwargs) with open( os.path.join(catalog.get_current_task_repo(), 'SDSS/2010ApJ...708..661D.txt'), 'r') as sdss_file: bibcodes2010 = sdss_file.read().split('\n') sdssbands = ['u', 'g', 'r', 'i', 'z'] file_names = (list( glob(os.path.join(catalog.get_current_task_repo(), 'SDSS/sum/*.sum'))) + list( glob( os.path.join(catalog.get_current_task_repo(), 'SDSS/SMP_Data/*.dat')))) skipphoto = ['SDSS-II SN 15557'] for fi, fname in enumerate(pbar_strings(file_names, task_str)): tsvin = csv.reader(open(fname, 'r'), delimiter=' ', skipinitialspace=True) basename = os.path.basename(fname) hasred = True rst = 19 if '.dat' in fname: bibcode = '2014arXiv1401.3317S' hasred = False rst = 4 elif basename in bibcodes2010: bibcode = '2010ApJ...708..661D' else: bibcode = '2008AJ....136.2306H' skip_entry = False for rr, row in enumerate(tsvin): if skip_entry: break if rr == 0: # Ignore non-SNe objects and those not in metadata table above if row[3] in ignored_cids: skip_entry = True continue # Ignore IAU names from file headers as they are unreliable oname = 'SDSS-II SN ' + row[3] (name, source) = catalog.new_entry(oname, bibcode=bibcode) catalog.entries[name].add_quantity(SUPERNOVA.RA, row[-4], source, u_value='floatdegrees') catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[-2], source, u_value='floatdegrees') if hasred and rr == 1: error = row[4] if float(row[4]) >= 0.0 else '' val = row[2] if float(val) < -1.0: continue (catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, val, source, e_value=error, kind='heliocentric')) if rr >= rst: # Skip bad measurements if int(row[0]) > 1024: continue if oname in skipphoto: break mjd = row[1] band = sdssbands[int(row[2])] + "'" magnitude = row[3] e_mag = row[4] fluxd = row[7] e_fluxd = row[8] telescope = 'SDSS' photodict = { PHOTOMETRY.TIME: mjd, PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.TELESCOPE: telescope, PHOTOMETRY.BAND: band, PHOTOMETRY.MAGNITUDE: magnitude, PHOTOMETRY.E_MAGNITUDE: e_mag, PHOTOMETRY.FLUX_DENSITY: fluxd, PHOTOMETRY.E_FLUX_DENSITY: e_fluxd, PHOTOMETRY.U_FLUX_DENSITY: 'μJy', PHOTOMETRY.SOURCE: source, PHOTOMETRY.BAND_SET: 'SDSS', PHOTOMETRY.SYSTEM: 'SDSS' } if float(fluxd) > 0.0: photodict[PHOTOMETRY.ZERO_POINT] = str( D25 * Decimal(fluxd).log10() + Decimal(magnitude)) ul_sigma = 3.0 if int(row[0]) & 32 or float( fluxd) < ul_sigma * float(e_fluxd): photodict[PHOTOMETRY.UPPER_LIMIT] = True photodict[PHOTOMETRY.UPPER_LIMIT_SIGMA] = str(ul_sigma) catalog.entries[name].add_photometry(**photodict) if catalog.args.travis and fi >= catalog.TRAVIS_QUERY_LIMIT: break if not fi % 1000: catalog.journal_entries() catalog.journal_entries() return
def do_essence_spectra(catalog): task_str = catalog.get_current_task_str() insdict = { "lris": "LRIS", "esi": "ESI", "deimos": "DEIMOS", "gmos": "GMOS", "fors1": "FORS1", "bluechannel": "Blue Channel", "ldss2": "LDSS-2", "ldss3": "LDSS-3", "imacs": "IMACS", "fast": "FAST" } teldict = { "lris": "Keck", "esi": "Keck", "deimos": "Keck", "gmos": "Gemini", "fors1": "VLT", "bluechannel": "MMT", "ldss2": "Magellan Clay & Baade", "ldss3": "Magellan Clay & Baade", "imacs": "Magellan Clay & Baade", "fast": "FLWO 1.5m" } file_names = glob( os.path.join(catalog.get_current_task_repo(), 'ESSENCE', '*')) oldname = '' for fi, fname in enumerate(pbar_strings(file_names, task_str)): filename = os.path.basename(fname) fileparts = filename.split('_') name = 'ESSENCE ' + fileparts[0] name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name if is_number(fileparts[1]): doffset = 1 else: if fileparts[1] != 'comb': continue doffset = 2 dstr = fileparts[doffset] mjd = str( astrotime( datetime.datetime(year=int(dstr[:4]), month=int(dstr[4:6]), day=int(dstr[6:8])) + datetime.timedelta(days=float(dstr[8:]))).mjd) instrument = fileparts[-1].split('.')[0] telescope = teldict.get(instrument, '') instrument = insdict.get(instrument, '') with open(fname, 'r') as f: data = csv.reader(f, delimiter=' ', skipinitialspace=True) data = [list(i) for i in zip(*data)] wavelengths = data[0] fluxes = [str(Decimal('1.0e-15') * Decimal(x)) for x in data[1]] name, source = catalog.new_entry(name, bibcode='2016ApJS..224....3N') specdict = { SPECTRUM.TIME: mjd, SPECTRUM.U_TIME: 'MJD', SPECTRUM.U_WAVELENGTHS: 'Angstrom', SPECTRUM.WAVELENGTHS: wavelengths, SPECTRUM.FLUXES: fluxes, SPECTRUM.U_FLUXES: 'erg/s/cm^2/Angstrom', SPECTRUM.FILENAME: filename, SPECTRUM.SOURCE: source } if instrument: specdict[SPECTRUM.INSTRUMENT] = instrument if telescope: specdict[SPECTRUM.TELESCOPE] = telescope catalog.entries[name].add_spectrum(**specdict) if catalog.args.travis and fi >= catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() return
def do_cccp(catalog): task_str = catalog.get_current_task_str() cccpbands = ['B', 'V', 'R', 'I'] file_names = list( glob(os.path.join(catalog.get_current_task_repo(), 'CCCP/apj407397*.txt'))) for datafile in pbar_strings(file_names, task_str + ': apj407397...'): with open(datafile, 'r') as ff: tsvin = csv.reader(ff, delimiter='\t', skipinitialspace=True) for rr, row in enumerate(tsvin): if rr == 0: continue elif rr == 1: name = 'SN' + row[0].split('SN ')[-1] name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2012ApJ...744...10K') catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, name, source) elif rr >= 5: mjd = str(Decimal(row[0]) + 53000) for bb, band in enumerate(cccpbands): if row[2 * bb + 1]: mag = row[2 * bb + 1].strip('>') upl = (not row[2 * bb + 2]) (catalog.entries[name] .add_photometry(time=mjd, u_time='MJD', band=band, magnitude=mag, e_magnitude=row[2 * bb + 2], upperlimit=upl, source=source)) html = catalog.load_url( 'https://webhome.weizmann.ac.il/home/iair/sc_cccp.html', os.path.join(catalog.get_current_task_repo(), 'CCCP/sc_cccp.html')) soup = BeautifulSoup(html, 'html5lib') links = soup.body.findAll("a") for link in pbar(links, task_str + ': links'): if 'sc_sn' in link['href']: name = catalog.add_entry(link.text.replace(' ', '')) source = (catalog.entries[name] .add_source(name='CCCP', url=('https://webhome.weizmann.ac.il' '/home/iair/sc_cccp.html'))) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) html2 = catalog.load_url( 'https://webhome.weizmann.ac.il/home/iair/' + link['href'], os.path.join(catalog.get_current_task_repo(), 'CCCP/') + link['href'].split('/')[-1]) soup2 = BeautifulSoup(html2, 'html5lib') links2 = soup2.body.findAll("a") for link2 in links2: if '.txt' in link2['href'] and '_' in link2['href']: band = link2['href'].split('_')[1].split('.')[0].upper() # Many 404s in photometry, set cache_only = True unless # attempting complete rebuild. html3 = catalog.load_url( 'https://webhome.weizmann.ac.il/home/iair/cccp/' + link2['href'], os.path.join(catalog.get_current_task_repo(), 'CCCP/') + link2['href'].split('/')[-1], cache_only=True) if html3 is None: continue table = [[str(Decimal(yy.strip())).rstrip('0') for yy in xx.split(',')] for xx in list(filter(None, html3.split('\n')))] for row in table: catalog.entries[name].add_photometry( time=str(Decimal(row[0]) + 53000), u_time='MJD', band=band, magnitude=row[1], e_magnitude=row[2], source=source) catalog.journal_entries() return