def do_snls_photo(catalog): task_str = catalog.get_current_task_str() snls_path = os.path.join(catalog.get_current_task_repo(), 'SNLS-ugriz.dat') data = list(csv.reader(open(snls_path, 'r'), delimiter=' ', quotechar='"', skipinitialspace=True)) for row in pbar(data, task_str): flux = row[3] err = row[4] # Being extra strict here with the flux constraint, see note below. if float(flux) < 3.0 * float(err): continue name = 'SNLS-' + row[0] name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2010A&A...523A...7G') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) band = row[1] mjd = row[2] sig = get_sig_digits(flux.split('E')[0]) + 1 # Conversion comes from SNLS-Readme # NOTE: Datafiles avail for download suggest diff zeropoints than 30, # need to inquire. magnitude = pretty_num(30.0 - 2.5 * log10(float(flux)), sig=sig) e_mag = pretty_num( 2.5 * log10(1.0 + float(err) / float(flux)), sig=sig) # e_mag = pretty_num(2.5*(log10(float(flux) + float(err)) - # log10(float(flux))), sig=sig) catalog.entries[name].add_photometry( time=mjd, band=band, magnitude=magnitude, e_magnitude=e_mag, counts=flux, e_counts=err, source=source) catalog.journal_entries() return
def set_first_max_light(self): if ENTRY.MAX_APP_MAG not in self: mldt, mlmag, mlband, mlsource = self._get_max_light() if mldt or mlmag or mlband: source = self.add_self_source() uniq_src = uniq_cdl([source] + mlsource.split(',')) if mldt: max_date = make_date_string(mldt.year, mldt.month, mldt.day) self.add_quantity(ENTRY.MAX_DATE, max_date, uniq_src, derived=True) if mlmag: mlmag = pretty_num(mlmag) self.add_quantity(ENTRY.MAX_APP_MAG, mlmag, uniq_src, derived=True) if mlband: self.add_quantity(ENTRY.MAX_BAND, mlband, uniq_src, derived=True) if (self._KEYS.DISCOVER_DATE not in self or max([len(x[QUANTITY.VALUE].split('/')) for x in self[self._KEYS.DISCOVER_DATE]]) < 3): fldt, flsource = self._get_first_light() if fldt: source = self.add_self_source() disc_date = make_date_string(fldt.year, fldt.month, fldt.day) self.add_quantity( self._KEYS.DISCOVER_DATE, disc_date, uniq_cdl([source] + flsource.split(',')), derived=True) if self._KEYS.DISCOVER_DATE not in self and self._KEYS.SPECTRA in self: minspecmjd = float("+inf") for spectrum in self[self._KEYS.SPECTRA]: if 'time' in spectrum and 'u_time' in spectrum: if spectrum['u_time'] == 'MJD': mjd = float(spectrum['time']) elif spectrum['u_time'] == 'JD': mjd = float(jd_to_mjd(Decimal(spectrum['time']))) else: continue if mjd < minspecmjd: minspecmjd = mjd minspecsource = spectrum['source'] if minspecmjd < float("+inf"): fldt = astrotime(minspecmjd, format='mjd').datetime source = self.add_self_source() disc_date = make_date_string(fldt.year, fldt.month, fldt.day) self.add_quantity( self._KEYS.DISCOVER_DATE, disc_date, uniq_cdl([source] + minspecsource.split(',')), derived=True) return
def set_first_max_light(self): if FASTSTARS.MAX_APP_MAG not in self: # Get the maximum amongst all bands mldt, mlmag, mlband, mlsource = self._get_max_light() if mldt or mlmag or mlband: source = self.add_self_source() uniq_src = uniq_cdl([source] + mlsource.split(',')) if mldt: max_date = make_date_string(mldt.year, mldt.month, mldt.day) self.add_quantity( FASTSTARS.MAX_DATE, max_date, uniq_src, derived=True) if mlmag: mlmag = pretty_num(mlmag) self.add_quantity( FASTSTARS.MAX_APP_MAG, mlmag, uniq_src, derived=True) if mlband: self.add_quantity( FASTSTARS.MAX_BAND, mlband, uniq_src, derived=True) return
def do_donated_spectra(catalog): """Import donated spectra.""" task_str = catalog.get_current_task_str() fpath = os.path.join(catalog.get_current_task_repo(), 'donations') with open(os.path.join(fpath, 'meta.json'), 'r') as f: metadict = json.loads(f.read()) donationscnt = 0 oldname = '' for fname in pbar(list(metadict.keys()), task_str): name = metadict[fname]['name'] name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name sec_bibc = metadict[fname]['bibcode'] name, source = catalog.new_entry(name, bibcode=sec_bibc) date = metadict[fname].get('date', '') year, month, day = date.split('/') sig = get_sig_digits(day) + 5 day_fmt = str(floor(float(day))).zfill(2) time = astrotime(year + '-' + month + '-' + day_fmt).mjd time = time + float(day) - floor(float(day)) time = pretty_num(time, sig=sig) with open(os.path.join(fpath, fname), 'r') as f: specdata = list( csv.reader( f, delimiter=' ', skipinitialspace=True)) specdata = list(filter(None, specdata)) newspec = [] oldval = '' for row in specdata: if row[0][0] == '#': continue if row[1] == oldval: continue newspec.append(row) oldval = row[1] specdata = newspec haserrors = len(specdata[0]) == 3 and specdata[0][2] and specdata[0][ 2] != 'NaN' specdata = [list(i) for i in zip(*specdata)] wavelengths = specdata[0] fluxes = specdata[1] errors = '' if haserrors: errors = specdata[2] specdict = { SPECTRUM.U_WAVELENGTHS: 'Angstrom', SPECTRUM.U_TIME: 'MJD', SPECTRUM.TIME: time, SPECTRUM.WAVELENGTHS: wavelengths, SPECTRUM.FLUXES: fluxes, SPECTRUM.ERRORS: errors, SPECTRUM.SOURCE: source, SPECTRUM.FILENAME: fname } if 'instrument' in metadict[fname]: specdict[SPECTRUM.INSTRUMENT] = metadict[fname]['instrument'] if 'telescope' in metadict[fname]: specdict[SPECTRUM.TELESCOPE] = metadict[fname]['telescope'] if 'yunit' in metadict[fname]: specdict[SPECTRUM.U_FLUXES] = metadict[fname]['yunit'] specdict[SPECTRUM.U_ERRORS] = metadict[fname]['yunit'] else: if max([float(x) for x in fluxes]) < 1.0e-5: fluxunit = 'erg/s/cm^2/Angstrom' else: fluxunit = 'Uncalibrated' specdict[SPECTRUM.U_FLUXES] = fluxunit specdict[SPECTRUM.U_ERRORS] = fluxunit catalog.entries[name].add_spectrum(**specdict) donationscnt = donationscnt + 1 if (catalog.args.travis and donationscnt % catalog.TRAVIS_QUERY_LIMIT == 0): break catalog.journal_entries() return
def radec_clean(svalue, quantity, unit=''): """Clean R.A. and Dec.""" svalue = svalue.strip() if unit == 'floatdegrees': if not is_number(svalue): return (svalue, unit) deg = float('%g' % Decimal(svalue)) sig = get_sig_digits(svalue) if 'ra' in quantity: flhours = deg / 360.0 * 24.0 hours = floor(flhours) minutes = floor((flhours - hours) * 60.0) seconds = (flhours * 60.0 - (hours * 60.0 + minutes)) * 60.0 hours = 0 if hours < 1.e-6 else hours minutes = 0 if minutes < 1.e-6 else minutes seconds = 0.0 if seconds < 1.e-6 else seconds if seconds > 60.0: raise (ValueError('Invalid seconds value for ' + quantity)) svalue = str(hours).zfill(2) + ':' + str(minutes).zfill(2) + \ ':' + zpad(pretty_num(seconds, sig=sig - 1)) elif 'dec' in quantity: fldeg = abs(deg) degree = floor(fldeg) minutes = floor((fldeg - degree) * 60.0) seconds = (fldeg * 60.0 - (degree * 60.0 + minutes)) * 60.0 minutes = 0 if minutes < 1.e-6 else minutes seconds = 0.0 if seconds < 1.e-6 else seconds if seconds > 60.0: raise (ValueError('Invalid seconds value for ' + quantity)) svalue = (('+' if deg >= 0.0 else '-') + str(degree).strip('+-').zfill(2) + ':' + str(minutes).zfill(2) + ':' + zpad(pretty_num(seconds, sig=sig - 1))) elif unit == 'nospace' and 'ra' in quantity: svalue = svalue[:2] + ':' + svalue[2:4] + \ ((':' + zpad(svalue[4:])) if len(svalue) > 4 else '') elif unit == 'nospace' and 'dec' in quantity: if svalue.startswith(('+', '-')): svalue = svalue[:3] + ':' + svalue[3:5] + \ ((':' + zpad(svalue[5:])) if len(svalue) > 5 else '') else: svalue = '+' + svalue[:2] + ':' + svalue[2:4] + \ ((':' + zpad(svalue[4:])) if len(svalue) > 4 else '') else: svalue = svalue.replace(' ', ':') if 'dec' in quantity: valuesplit = svalue.split(':') svalue = ( ('-' if valuesplit[0].startswith('-') else '+') + valuesplit[0].strip('+-').zfill(2) + (':' + valuesplit[1].zfill(2) if len(valuesplit) > 1 else '') + (':' + zpad(valuesplit[2]) if len(valuesplit) > 2 else '')) if 'ra' in quantity: sunit = 'hours' elif 'dec' in quantity: sunit = 'degrees' # Correct case of arcseconds = 60.0. valuesplit = svalue.split(':') if len(valuesplit) == 3 and valuesplit[-1] in ["60.0", "60.", "60"]: svalue = valuesplit[0] + ':' + str( Decimal(valuesplit[1]) + Decimal(1.0)) + ':' + "00.0" # Strip trailing dots. svalue = svalue.rstrip('.') return (svalue, sunit)
def do_ucb_spectra(catalog): task_str = catalog.get_current_task_str() sec_reference = 'UCB Filippenko Group\'s Supernova Database (SNDB)' sec_refurl = 'http://heracles.astro.berkeley.edu/sndb/info' sec_refbib = '2012MNRAS.425.1789S' ucbspectracnt = 0 jsontxt = catalog.load_url( 'http://heracles.astro.berkeley.edu/sndb/download?id=allpubspec', os.path.join(catalog.get_current_task_repo(), 'UCB/allpubspec.json'), json_sort='SpecID') if not jsontxt: return spectra = json.loads(jsontxt) spectra = sorted(spectra, key=lambda kk: kk['SpecID']) oldname = '' for spectrum in pbar(spectra, task_str): name = spectrum['ObjName'] if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) sec_source = catalog.entries[name].add_source( name=sec_reference, url=sec_refurl, bibcode=sec_refbib, secondary=True) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, sec_source) sources = [sec_source] if spectrum['Reference']: sources += [catalog.entries[name] .add_source(bibcode=spectrum['Reference'])] sources = uniq_cdl(sources) if spectrum['Type'] and spectrum['Type'].strip() != 'NoMatch': for ct in spectrum['Type'].strip().split(','): catalog.entries[name].add_quantity( SUPERNOVA.CLAIMED_TYPE, ct.replace('-norm', '').strip(), sources) if spectrum['DiscDate']: ddate = spectrum['DiscDate'].replace('-', '/') catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, ddate, sources) if spectrum['HostName']: host = urllib.parse.unquote(spectrum['HostName']).replace('*', '') catalog.entries[name].add_quantity(SUPERNOVA.HOST, host, sources) if spectrum['UT_Date']: epoch = str(spectrum['UT_Date']) year = epoch[:4] month = epoch[4:6] day = epoch[6:] sig = get_sig_digits(day) + 5 mjd = astrotime(year + '-' + month + '-' + str(floor(float( day))).zfill(2)).mjd mjd = pretty_num(mjd + float(day) - floor(float(day)), sig=sig) filename = spectrum['Filename'] if spectrum['Filename'] else '' instrument = spectrum['Instrument'] if spectrum['Instrument'] else '' reducer = spectrum['Reducer'] if spectrum['Reducer'] else '' observer = spectrum['Observer'] if spectrum['Observer'] else '' snr = str(spectrum['SNR']) if spectrum['SNR'] else '' if not filename: raise ValueError('Filename not found for SNDB spectrum!') if not spectrum['SpecID']: raise ValueError('ID not found for SNDB spectrum!') filepath = os.path.join(catalog.get_current_task_repo(), 'UCB/') + filename spectxt = catalog.load_url( 'http://heracles.astro.berkeley.edu/sndb/download?id=ds:' + str(spectrum['SpecID']), filepath, archived_mode=True) specdata = list( csv.reader( spectxt.splitlines(), delimiter=' ', skipinitialspace=True)) newspecdata = [] for row in specdata: if row[0][0] == '#': continue else: newspecdata.append(row) specdata = newspecdata haserrors = len(specdata[0]) == 3 and specdata[0][2] and specdata[0][ 2] != 'NaN' specdata = [list(ii) for ii in zip(*specdata)] wavelengths = specdata[0] fluxes = specdata[1] errors = '' if haserrors: errors = specdata[2] if not list(filter(None, errors)): errors = '' units = 'Uncalibrated' catalog.entries[name].add_spectrum( u_wavelengths='Angstrom', u_fluxes=units, u_time='MJD', time=mjd, wavelengths=wavelengths, filename=filename, fluxes=fluxes, errors=errors, u_errors=units, instrument=instrument, source=sources, snr=snr, observer=observer, reducer=reducer, deredshifted=('-noz' in filename)) ucbspectracnt = ucbspectracnt + 1 if catalog.args.travis and ucbspectracnt >= catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() return
def do_suspect_spectra(catalog): task_str = catalog.get_current_task_str() with open(os.path.join(catalog.get_current_task_repo(), 'Suspect/sources.json'), 'r') as f: sourcedict = json.loads(f.read()) with open(os.path.join(catalog.get_current_task_repo(), 'Suspect/filename-changes.txt'), 'r') as f: rows = f.readlines() changedict = {} for row in rows: if not row.strip() or row[0] == "#": continue items = row.strip().split(' ') changedict[items[1]] = items[0] suspectcnt = 0 folders = next(os.walk(os.path.join( catalog.get_current_task_repo(), 'Suspect')))[1] for folder in pbar(folders, task_str): eventfolders = next(os.walk(os.path.join( catalog.get_current_task_repo(), 'Suspect/') + folder))[1] oldname = '' for eventfolder in pbar(eventfolders, task_str): name = eventfolder if is_number(name[:4]): name = 'SN' + name name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) sec_ref = 'SUSPECT' sec_refurl = 'https://www.nhn.ou.edu/~suspect/' sec_bibc = '2001AAS...199.8408R' sec_source = catalog.entries[name].add_source( name=sec_ref, url=sec_refurl, bibcode=sec_bibc, secondary=True) catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, name, sec_source) fpath = os.path.join(catalog.get_current_task_repo(), 'Suspect', folder, eventfolder) eventspectra = next(os.walk(fpath))[2] for spectrum in eventspectra: sources = [sec_source] bibcode = '' if spectrum in changedict: specalias = changedict[spectrum] else: specalias = spectrum if specalias in sourcedict: bibcode = sourcedict[specalias] elif name in sourcedict: bibcode = sourcedict[name] if bibcode: source = catalog.entries[name].add_source( bibcode=unescape(bibcode)) sources += [source] sources = uniq_cdl(sources) date = spectrum.split('_')[1] year = date[:4] month = date[4:6] day = date[6:] sig = get_sig_digits(day) + 5 day_fmt = str(floor(float(day))).zfill(2) time = astrotime(year + '-' + month + '-' + day_fmt).mjd time = time + float(day) - floor(float(day)) time = pretty_num(time, sig=sig) fpath = os.path.join(catalog.get_current_task_repo(), 'Suspect', folder, eventfolder, spectrum) with open(fpath, 'r') as f: specdata = list(csv.reader( f, delimiter=' ', skipinitialspace=True)) specdata = list(filter(None, specdata)) newspec = [] oldval = '' for row in specdata: if row[1] == oldval: continue newspec.append(row) oldval = row[1] specdata = newspec haserrors = len(specdata[0]) == 3 and specdata[ 0][2] and specdata[0][2] != 'NaN' specdata = [list(i) for i in zip(*specdata)] wavelengths = specdata[0] fluxes = specdata[1] errors = '' if haserrors: errors = specdata[2] catalog.entries[name].add_spectrum( u_wavelengths='Angstrom', u_fluxes='Uncalibrated', u_time='MJD', time=time, wavelengths=wavelengths, fluxes=fluxes, errors=errors, u_errors='Uncalibrated', source=sources, filename=spectrum) suspectcnt = suspectcnt + 1 if (catalog.args.travis and suspectcnt % catalog.TRAVIS_QUERY_LIMIT == 0): break catalog.journal_entries() return
def set_first_max_light(self): if SUPERNOVA.MAX_APP_MAG not in self: # Get the maximum amongst all bands mldt, mlmag, mlband, mlsource = self._get_max_light() if mldt or mlmag or mlband: source = self.add_self_source() uniq_src = uniq_cdl([source] + mlsource.split(',')) if mldt: max_date = make_date_string(mldt.year, mldt.month, mldt.day) self.add_quantity(SUPERNOVA.MAX_DATE, max_date, uniq_src, derived=True) if mlmag: mlmag = pretty_num(mlmag) self.add_quantity(SUPERNOVA.MAX_APP_MAG, mlmag, uniq_src, derived=True) if mlband: self.add_quantity(SUPERNOVA.MAX_BAND, mlband, uniq_src, derived=True) if SUPERNOVA.MAX_VISUAL_APP_MAG not in self: # Get the "visual" maximum mldt, mlmag, mlband, mlsource = self._get_max_light(visual=True) if mldt or mlmag or mlband: source = self.add_self_source() uniq_src = uniq_cdl([source] + mlsource.split(',')) if mldt: max_date = make_date_string(mldt.year, mldt.month, mldt.day) self.add_quantity(SUPERNOVA.MAX_VISUAL_DATE, max_date, uniq_src, derived=True) if mlmag: mlmag = pretty_num(mlmag) self.add_quantity(SUPERNOVA.MAX_VISUAL_APP_MAG, mlmag, uniq_src, derived=True) if mlband: self.add_quantity(SUPERNOVA.MAX_VISUAL_BAND, mlband, uniq_src, derived=True) if (self._KEYS.DISCOVER_DATE not in self or max([ len(x[QUANTITY.VALUE].split('/')) for x in self[self._KEYS.DISCOVER_DATE] ]) < 3): fldt, flsource = self._get_first_light() if fldt: source = self.add_self_source() disc_date = make_date_string(fldt.year, fldt.month, fldt.day) self.add_quantity(self._KEYS.DISCOVER_DATE, disc_date, uniq_cdl([source] + flsource.split(',')), derived=True) if self._KEYS.DISCOVER_DATE not in self and self._KEYS.SPECTRA in self: minspecmjd = float("+inf") for spectrum in self[self._KEYS.SPECTRA]: if 'time' in spectrum and 'u_time' in spectrum: if spectrum['u_time'] == 'MJD': mjd = float(spectrum['time']) elif spectrum['u_time'] == 'JD': mjd = float(jd_to_mjd(Decimal(spectrum['time']))) else: continue if mjd < minspecmjd: minspecmjd = mjd minspecsource = spectrum['source'] if minspecmjd < float("+inf"): fldt = astrotime(minspecmjd, format='mjd').datetime source = self.add_self_source() disc_date = make_date_string(fldt.year, fldt.month, fldt.day) self.add_quantity(self._KEYS.DISCOVER_DATE, disc_date, uniq_cdl([source] + minspecsource.split(',')), derived=True) return
def do_cleanup(catalog): """Task to cleanup catalog before final write.""" task_str = catalog.get_current_task_str() # Set preferred names, calculate some columns based on imported data, # sanitize some fields keys = catalog.entries.copy().keys() cleanupcnt = 0 for oname in pbar(keys, task_str): name = catalog.add_entry(oname) # Set the preferred name, switching to that name if name changed. name = catalog.entries[name].set_preferred_name() aliases = catalog.entries[name].get_aliases() catalog.entries[name].set_first_max_light() if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['MLS', 'SSS', 'CSS', 'GRB '] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:2])): discoverdate = ('/'.join([ '20' + alias.replace(prefix, '')[:2], alias.replace(prefix, '')[2:4], alias.replace(prefix, '')[4:6] ])) if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( TIDALDISRUPTION.DISCOVER_DATE, discoverdate, source, derived=True) break if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]: break if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]: prefixes = [ 'ASASSN-', 'PS1-', 'PS1', 'PS', 'iPTF', 'PTF', 'SCP-', 'SNLS-', 'SPIRITS', 'LSQ', 'DES', 'SNHiTS', 'Gaia', 'GND', 'GNW', 'GSD', 'GSW', 'EGS', 'COS', 'OGLE', 'HST' ] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:2]) and is_number(alias.replace(prefix, '')[:1])): discoverdate = '20' + alias.replace(prefix, '')[:2] if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( TIDALDISRUPTION.DISCOVER_DATE, discoverdate, source, derived=True) break if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]: break if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['SNF'] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:4])): discoverdate = ('/'.join([ alias.replace(prefix, '')[:4], alias.replace(prefix, '')[4:6], alias.replace(prefix, '')[6:8] ])) if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( TIDALDISRUPTION.DISCOVER_DATE, discoverdate, source, derived=True) break if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]: break if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['PTFS', 'SNSDF'] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:2])): discoverdate = ('/'.join([ '20' + alias.replace(prefix, '')[:2], alias.replace(prefix, '')[2:4] ])) if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( TIDALDISRUPTION.DISCOVER_DATE, discoverdate, source, derived=True) break if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]: break if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['AT', 'SN', 'OGLE-', 'SM ', 'KSN-'] for alias in aliases: for prefix in prefixes: if alias.startswith(prefix): year = re.findall(r'\d+', alias) if len(year) == 1: year = year[0] else: continue if alias.replace(prefix, '').index(year) != 0: continue if (year and is_number(year) and '.' not in year and len(year) <= 4): discoverdate = year if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( TIDALDISRUPTION.DISCOVER_DATE, discoverdate, source, derived=True) break if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]: break if (TIDALDISRUPTION.RA not in catalog.entries[name] or TIDALDISRUPTION.DEC not in catalog.entries[name]): prefixes = [ 'PSN J', 'MASJ', 'CSS', 'SSS', 'MASTER OT J', 'HST J', 'TCP J', 'MACS J', '2MASS J', 'EQ J', 'CRTS J', 'SMT J' ] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:6])): noprefix = alias.split(':')[-1].replace( prefix, '').replace('.', '') decsign = '+' if '+' in noprefix else '-' noprefix = noprefix.replace('+', '|').replace('-', '|') nops = noprefix.split('|') if len(nops) < 2: continue rastr = nops[0] decstr = nops[1] ra = ':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + \ ('.' + rastr[6:] if len(rastr) > 6 else '') dec = (decsign + ':'.join( [decstr[:2], decstr[2:4], decstr[4:6]]) + ('.' + decstr[6:] if len(decstr) > 6 else '')) if catalog.args.verbose: tprint('Added ra/dec from name: ' + ra + ' ' + dec) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( TIDALDISRUPTION.RA, ra, source, derived=True) catalog.entries[name].add_quantity( TIDALDISRUPTION.DEC, dec, source, derived=True) break if TIDALDISRUPTION.RA in catalog.entries[name]: break no_host = (TIDALDISRUPTION.HOST not in catalog.entries[name] or not any([ x[QUANTITY.VALUE] == 'Milky Way' for x in catalog.entries[name][TIDALDISRUPTION.HOST] ])) if (TIDALDISRUPTION.RA in catalog.entries[name] and TIDALDISRUPTION.DEC in catalog.entries[name] and no_host): from astroquery.irsa_dust import IrsaDust if name not in catalog.extinctions_dict: try: ra_dec = (catalog.entries[name][TIDALDISRUPTION.RA][0][ QUANTITY.VALUE] + " " + catalog.entries[name][ TIDALDISRUPTION.DEC][0][QUANTITY.VALUE]) result = IrsaDust.get_query_table(ra_dec, section='ebv') except (KeyboardInterrupt, SystemExit): raise except Exception: warnings.warn("Coordinate lookup for " + name + " failed in IRSA.") else: ebv = result['ext SandF mean'][0] ebverr = result['ext SandF std'][0] catalog.extinctions_dict[name] = [ebv, ebverr] if name in catalog.extinctions_dict: sources = uniq_cdl([ catalog.entries[name].add_self_source(), catalog.entries[name] .add_source(bibcode='2011ApJ...737..103S') ]) (catalog.entries[name].add_quantity( TIDALDISRUPTION.EBV, str(catalog.extinctions_dict[name][0]), sources, e_value=str(catalog.extinctions_dict[name][1]), derived=True)) if ((TIDALDISRUPTION.HOST in catalog.entries[name] and (TIDALDISRUPTION.HOST_RA not in catalog.entries[name] or TIDALDISRUPTION.HOST_DEC not in catalog.entries[name]))): for host in catalog.entries[name][TIDALDISRUPTION.HOST]: alias = host[QUANTITY.VALUE] if ' J' in alias and is_number(alias.split(' J')[-1][:6]): noprefix = alias.split(' J')[-1].split(':')[-1].replace( '.', '') decsign = '+' if '+' in noprefix else '-' noprefix = noprefix.replace('+', '|').replace('-', '|') nops = noprefix.split('|') if len(nops) < 2: continue rastr = nops[0] decstr = nops[1] hostra = (':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + ('.' + rastr[6:] if len(rastr) > 6 else '')) hostdec = decsign + ':'.join([ decstr[:2], decstr[2:4], decstr[4:6] ]) + ('.' + decstr[6:] if len(decstr) > 6 else '') if catalog.args.verbose: tprint('Added hostra/hostdec from name: ' + hostra + ' ' + hostdec) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( TIDALDISRUPTION.HOST_RA, hostra, source, derived=True) catalog.entries[name].add_quantity( TIDALDISRUPTION.HOST_DEC, hostdec, source, derived=True) break if TIDALDISRUPTION.HOST_RA in catalog.entries[name]: break if (TIDALDISRUPTION.REDSHIFT not in catalog.entries[name] and TIDALDISRUPTION.VELOCITY in catalog.entries[name]): # Find the "best" velocity to use for this bestsig = 0 for hv in catalog.entries[name][TIDALDISRUPTION.VELOCITY]: sig = get_sig_digits(hv[QUANTITY.VALUE]) if sig > bestsig: besthv = hv[QUANTITY.VALUE] bestsrc = hv['source'] bestsig = sig if bestsig > 0 and is_number(besthv): voc = float(besthv) * 1.e5 / CLIGHT source = catalog.entries[name].add_self_source() sources = uniq_cdl([source] + bestsrc.split(',')) (catalog.entries[name].add_quantity( TIDALDISRUPTION.REDSHIFT, pretty_num( sqrt((1. + voc) / (1. - voc)) - 1., sig=bestsig), sources, kind='heliocentric', derived=True)) if (TIDALDISRUPTION.REDSHIFT not in catalog.entries[name] and len(catalog.nedd_dict) > 0 and TIDALDISRUPTION.HOST in catalog.entries[name]): reference = "NED-D" refurl = "http://ned.ipac.caltech.edu/Library/Distances/" for host in catalog.entries[name][TIDALDISRUPTION.HOST]: if host[QUANTITY.VALUE] in catalog.nedd_dict: source = catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') secondarysource = catalog.entries[name].add_source( name=reference, url=refurl, secondary=True) meddist = statistics.median(catalog.nedd_dict[host[ QUANTITY.VALUE]]) redz = z_at_value(cosmo.comoving_distance, float(meddist) * un.Mpc) redshift = pretty_num( redz, sig=get_sig_digits(str(meddist))) catalog.entries[name].add_quantity( TIDALDISRUPTION.REDSHIFT, redshift, uniq_cdl([source, secondarysource]), kind='host', derived=True) if (TIDALDISRUPTION.MAX_ABS_MAG not in catalog.entries[name] and TIDALDISRUPTION.MAX_APP_MAG in catalog.entries[name] and TIDALDISRUPTION.LUM_DIST in catalog.entries[name]): # Find the "best" distance to use for this bestsig = 0 for ld in catalog.entries[name][TIDALDISRUPTION.LUM_DIST]: sig = get_sig_digits(ld[QUANTITY.VALUE]) if sig > bestsig: bestld = ld[QUANTITY.VALUE] bestsrc = ld['source'] bestsig = sig if bestsig > 0 and is_number(bestld) and float(bestld) > 0.: source = catalog.entries[name].add_self_source() sources = uniq_cdl([source] + bestsrc.split(',')) bestldz = z_at_value(cosmo.luminosity_distance, float(bestld) * un.Mpc) pnum = (float(catalog.entries[name][ TIDALDISRUPTION.MAX_APP_MAG][0][QUANTITY.VALUE]) - 5.0 * (log10(float(bestld) * 1.0e6) - 1.0 ) + 2.5 * log10(1.0 + bestldz)) pnum = pretty_num(pnum, sig=bestsig) catalog.entries[name].add_quantity( TIDALDISRUPTION.MAX_ABS_MAG, pnum, sources, derived=True) if TIDALDISRUPTION.REDSHIFT in catalog.entries[name]: # Find the "best" redshift to use for this bestz, bestkind, bestsig, bestsrc = catalog.entries[ name].get_best_redshift() if bestsig > 0: try: bestz = float(bestz) except Exception: print(catalog.entries[name]) raise if TIDALDISRUPTION.VELOCITY not in catalog.entries[name]: source = catalog.entries[name].add_self_source() # FIX: what's happening here?! pnum = CLIGHT / KM * \ ((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.) pnum = pretty_num(pnum, sig=bestsig) catalog.entries[name].add_quantity( TIDALDISRUPTION.VELOCITY, pnum, source, kind=PREF_KINDS[bestkind], derived=True) if bestz > 0.: from astropy.cosmology import Planck15 as cosmo if TIDALDISRUPTION.LUM_DIST not in catalog.entries[name]: dl = cosmo.luminosity_distance(bestz) sources = [ catalog.entries[name].add_self_source(), catalog.entries[name] .add_source(bibcode='2016A&A...594A..13P') ] sources = uniq_cdl(sources + bestsrc.split(',')) catalog.entries[name].add_quantity( TIDALDISRUPTION.LUM_DIST, pretty_num( dl.value, sig=bestsig), sources, kind=PREF_KINDS[bestkind], derived=True) if (TIDALDISRUPTION.MAX_ABS_MAG not in catalog.entries[name] and TIDALDISRUPTION.MAX_APP_MAG in catalog.entries[name]): source = catalog.entries[name].add_self_source() pnum = pretty_num( float(catalog.entries[name][ TIDALDISRUPTION.MAX_APP_MAG][0][ QUANTITY.VALUE]) - 5.0 * (log10(dl.to('pc').value) - 1.0 ) + 2.5 * log10(1.0 + bestz), sig=bestsig + 1) catalog.entries[name].add_quantity( TIDALDISRUPTION.MAX_ABS_MAG, pnum, sources, derived=True) if TIDALDISRUPTION.COMOVING_DIST not in catalog.entries[ name]: cd = cosmo.comoving_distance(bestz) sources = [ catalog.entries[name].add_self_source(), catalog.entries[name] .add_source(bibcode='2016A&A...594A..13P') ] sources = uniq_cdl(sources + bestsrc.split(',')) catalog.entries[name].add_quantity( TIDALDISRUPTION.COMOVING_DIST, pretty_num( cd.value, sig=bestsig), sources, derived=True) if all([ x in catalog.entries[name] for x in [ TIDALDISRUPTION.RA, TIDALDISRUPTION.DEC, TIDALDISRUPTION.HOST_RA, TIDALDISRUPTION.HOST_DEC ] ]): # For now just using first coordinates that appear in entry try: c1 = coord( ra=catalog.entries[name][TIDALDISRUPTION.RA][0][ QUANTITY.VALUE], dec=catalog.entries[name][TIDALDISRUPTION.DEC][0][ QUANTITY.VALUE], unit=(un.hourangle, un.deg)) c2 = coord( ra=catalog.entries[name][TIDALDISRUPTION.HOST_RA][0][ QUANTITY.VALUE], dec=catalog.entries[name][TIDALDISRUPTION.HOST_DEC][0][ QUANTITY.VALUE], unit=(un.hourangle, un.deg)) except (KeyboardInterrupt, SystemExit): raise except Exception: pass else: sources = uniq_cdl( [catalog.entries[name].add_self_source()] + catalog. entries[name][TIDALDISRUPTION.RA][0]['source'].split(',') + catalog.entries[name][TIDALDISRUPTION.DEC][0]['source']. split(',') + catalog.entries[name][TIDALDISRUPTION.HOST_RA] [0]['source'].split(',') + catalog.entries[name][ TIDALDISRUPTION.HOST_DEC][0]['source'].split(',')) if 'hostoffsetang' not in catalog.entries[name]: hosa = Decimal( hypot(c1.ra.degree - c2.ra.degree, c1.dec.degree - c2.dec.degree)) hosa = pretty_num(hosa * Decimal(3600.)) catalog.entries[name].add_quantity( TIDALDISRUPTION.HOST_OFFSET_ANG, hosa, sources, derived=True, u_value='arcseconds') if (TIDALDISRUPTION.COMOVING_DIST in catalog.entries[name] and TIDALDISRUPTION.REDSHIFT in catalog.entries[name] and TIDALDISRUPTION.HOST_OFFSET_DIST not in catalog.entries[name]): offsetsig = get_sig_digits(catalog.entries[name][ TIDALDISRUPTION.HOST_OFFSET_ANG][0][QUANTITY.VALUE]) sources = uniq_cdl( sources.split(',') + (catalog.entries[name][ TIDALDISRUPTION.COMOVING_DIST][0]['source']). split(',') + (catalog.entries[name][ TIDALDISRUPTION.REDSHIFT][0]['source']).split(',')) (catalog.entries[name].add_quantity( TIDALDISRUPTION.HOST_OFFSET_DIST, pretty_num( float(catalog.entries[name][ TIDALDISRUPTION.HOST_OFFSET_ANG][0][ QUANTITY.VALUE]) / 3600. * (pi / 180.) * float(catalog.entries[name][ TIDALDISRUPTION.COMOVING_DIST][0][ QUANTITY.VALUE]) * 1000. / (1.0 + float(catalog.entries[name][ TIDALDISRUPTION.REDSHIFT][0][QUANTITY.VALUE])), sig=offsetsig), sources)) catalog.entries[name].sanitize() catalog.journal_entries(bury=True, final=True, gz=True) cleanupcnt = cleanupcnt + 1 if catalog.args.travis and cleanupcnt % 1000 == 0: break catalog.save_caches() return
def do_snls_spectra(catalog): """ """ task_str = catalog.get_current_task_str() result = Vizier.get_catalogs('J/A+A/507/85/table1') table = result[list(result.keys())[0]] table.convert_bytestring_to_unicode() datedict = {} for row in table: datedict['SNLS-' + row['SN']] = str(astrotime(row['Date']).mjd) oldname = '' file_names = glob(os.path.join(catalog.get_current_task_repo(), 'SNLS/*')) for fi, fname in enumerate(pbar_strings(file_names, task_str)): filename = os.path.basename(fname) fileparts = filename.split('_') name = 'SNLS-' + fileparts[1] name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2009A&A...507...85B') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, '20' + fileparts[1][:2], source) f = open(fname, 'r') data = csv.reader(f, delimiter=' ', skipinitialspace=True) specdata = [] for r, row in enumerate(data): if row[0] == '@TELESCOPE': telescope = row[1].strip() elif row[0] == '@REDSHIFT': catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, row[1].strip(), source) if r < 14: continue specdata.append(list(filter(None, [x.strip(' \t') for x in row]))) specdata = [list(i) for i in zip(*specdata)] wavelengths = specdata[1] fluxes = [ pretty_num(float(x) * 1.e-16, sig=get_sig_digits(x)) for x in specdata[2] ] # FIX: this isnt being used errors = [ pretty_num(float(x) * 1.e-16, sig=get_sig_digits(x)) for x in specdata[3] ] fluxunit = 'erg/s/cm^2/Angstrom' specdict = { SPECTRUM.WAVELENGTHS: wavelengths, SPECTRUM.FLUXES: fluxes, SPECTRUM.ERRORS: errors, SPECTRUM.U_WAVELENGTHS: 'Angstrom', SPECTRUM.U_FLUXES: fluxunit, SPECTRUM.U_ERRORS: fluxunit, SPECTRUM.TELESCOPE: telescope, SPECTRUM.FILENAME: filename, SPECTRUM.SOURCE: source } if name in datedict: specdict[SPECTRUM.TIME] = datedict[name] specdict[SPECTRUM.U_TIME] = 'MJD' catalog.entries[name].add_spectrum(**specdict) if catalog.args.travis and fi >= catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() return
def do_tns(catalog): from datetime import timedelta session = requests.Session() task_str = catalog.get_current_task_str() tns_url = 'https://wis-tns.weizmann.ac.il/' search_url = tns_url + \ 'search?&num_page=1&format=html&sort=desc&order=id&format=csv&page=0' csvtxt = catalog.load_cached_url( search_url, os.path.join(catalog.get_current_task_repo(), 'TNS/index.csv')) if not csvtxt: return maxid = csvtxt.splitlines()[1].split(',')[0].strip('"') maxpages = ceil(int(maxid) / 1000.) for page in pbar(range(maxpages), task_str): fname = os.path.join(catalog.get_current_task_repo(), 'TNS/page-') + \ str(page).zfill(2) + '.csv' if (catalog.current_task.load_archive(catalog.args) and os.path.isfile(fname) and page < 7): with open(fname, 'r') as tns_file: csvtxt = tns_file.read() else: with open(fname, 'w') as tns_file: session = requests.Session() ses_url = (tns_url + 'search?&num_page=1000&format=html&edit' '[type]=&edit[objname]=&edit[id]=&sort=asc&order=id' '&display[redshift]=1' '&display[hostname]=1&display[host_redshift]=1' '&display[source_group_name]=1' '&display[programs_name]=1' '&display[internal_name]=1' '&display[isTNS_AT]=1' '&display[public]=1' '&display[end_pop_period]=0' '&display[spectra_count]=1' '&display[discoverymag]=1&display[discmagfilter]=1' '&display[discoverydate]=1&display[discoverer]=1' '&display[sources]=1' '&display[bibcode]=1&format=csv&page=' + str(page)) try: response = session.get(ses_url) csvtxt = response.text except: if os.path.isfile(fname): with open(fname, 'r') as tns_file: csvtxt = tns_file.read() else: continue else: tns_file.write(csvtxt) tsvin = list(csv.reader(csvtxt.splitlines(), delimiter=',')) for ri, row in enumerate(pbar(tsvin, task_str, leave=False)): if ri == 0: continue if row[4] and 'SN' not in row[4]: continue name = row[1].replace(' ', '') name = catalog.add_entry(name) source = catalog.entries[name].add_source( name='Transient Name Server', url=tns_url) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) if row[2] and row[2] != '00:00:00.00': catalog.entries[name].add_quantity( SUPERNOVA.RA, row[2], source) if row[3] and row[3] != '+00:00:00.00': catalog.entries[name].add_quantity( SUPERNOVA.DEC, row[3], source) if row[4]: catalog.entries[name].add_quantity( SUPERNOVA.CLAIMED_TYPE, row[4].replace('SN', '').strip(), source) if row[5]: catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, row[5], source, kind='spectroscopic') if row[6]: catalog.entries[name].add_quantity( SUPERNOVA.HOST, row[6], source) if row[7]: catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, row[7], source, kind='host') if row[8]: catalog.entries[name].add_quantity( 'discoverer', row[8], source) # Currently, all events listing all possible observers. TNS bug? # if row[9]: # observers = row[9].split(',') # for observer in observers: # catalog.entries[name].add_quantity('observer', # observer.strip(), # source) if row[10]: catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, row[10], source) if row[8] and row[14] and row[15] and row[16]: survey = row[8] magnitude = row[14] band = row[15].split('-')[0] mjd = astrotime(row[16]).mjd (catalog.entries[name] .add_photometry(time=mjd, magnitude=magnitude, band=band, survey=survey, source=source)) if row[16]: date = row[16].split()[0].replace('-', '/') if date != '0000/00/00': date = date.replace('/00', '') time = row[16].split()[1] if time != '00:00:00': ts = time.split(':') dt = timedelta(hours=int(ts[0]), minutes=int( ts[1]), seconds=int(ts[2])) date += pretty_num(dt.total_seconds() / (24 * 60 * 60), sig=6).lstrip('0') catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, date, source) if catalog.args.update: catalog.journal_entries() catalog.journal_entries() return
def do_suspect_spectra(catalog): task_str = catalog.get_current_task_str() with open( os.path.join(catalog.get_current_task_repo(), 'Suspect/sources.json'), 'r') as f: sourcedict = json.loads(f.read()) with open( os.path.join(catalog.get_current_task_repo(), 'Suspect/filename-changes.txt'), 'r') as f: rows = f.readlines() changedict = {} for row in rows: if not row.strip() or row[0] == "#": continue items = row.strip().split(' ') changedict[items[1]] = items[0] suspectcnt = 0 folders = next( os.walk(os.path.join(catalog.get_current_task_repo(), 'Suspect')))[1] for folder in pbar(folders, task_str): eventfolders = next( os.walk( os.path.join(catalog.get_current_task_repo(), 'Suspect/') + folder))[1] oldname = '' for eventfolder in pbar(eventfolders, task_str): name = eventfolder if is_number(name[:4]): name = 'SN' + name name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) sec_ref = 'SUSPECT' sec_refurl = 'https://www.nhn.ou.edu/~suspect/' sec_bibc = '2001AAS...199.8408R' sec_source = catalog.entries[name].add_source(name=sec_ref, url=sec_refurl, bibcode=sec_bibc, secondary=True) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, sec_source) fpath = os.path.join(catalog.get_current_task_repo(), 'Suspect', folder, eventfolder) eventspectra = next(os.walk(fpath))[2] for spectrum in eventspectra: sources = [sec_source] bibcode = '' if spectrum in changedict: specalias = changedict[spectrum] else: specalias = spectrum if specalias in sourcedict: bibcode = sourcedict[specalias] elif name in sourcedict: bibcode = sourcedict[name] if bibcode: source = catalog.entries[name].add_source( bibcode=unescape(bibcode)) sources += [source] sources = uniq_cdl(sources) date = spectrum.split('_')[1] year = date[:4] month = date[4:6] day = date[6:] sig = get_sig_digits(day) + 5 day_fmt = str(floor(float(day))).zfill(2) time = astrotime(year + '-' + month + '-' + day_fmt).mjd time = time + float(day) - floor(float(day)) time = pretty_num(time, sig=sig) fpath = os.path.join(catalog.get_current_task_repo(), 'Suspect', folder, eventfolder, spectrum) with open(fpath, 'r') as f: specdata = list( csv.reader(f, delimiter=' ', skipinitialspace=True)) specdata = list(filter(None, specdata)) newspec = [] oldval = '' for row in specdata: if row[1] == oldval: continue newspec.append(row) oldval = row[1] specdata = newspec haserrors = len( specdata[0] ) == 3 and specdata[0][2] and specdata[0][2] != 'NaN' specdata = [list(i) for i in zip(*specdata)] wavelengths = specdata[0] fluxes = specdata[1] errors = '' if haserrors: errors = specdata[2] catalog.entries[name].add_spectrum(u_wavelengths='Angstrom', u_fluxes='Uncalibrated', u_time='MJD', time=time, wavelengths=wavelengths, fluxes=fluxes, errors=errors, u_errors='Uncalibrated', source=sources, filename=spectrum) suspectcnt = suspectcnt + 1 if (catalog.args.travis and suspectcnt % catalog.TRAVIS_QUERY_LIMIT == 0): break catalog.journal_entries() return
def do_nedd(catalog): task_str = catalog.get_current_task_str() nedd_path = os.path.join( catalog.get_current_task_repo(), 'NED26.10.1-D-13.1.0-20160930.csv') f = open(nedd_path, 'r') data = sorted(list(csv.reader(f, delimiter=',', quotechar='"'))[ 13:], key=lambda x: (x[9], x[3])) reference = "NED-D v" + nedd_path.split('-')[-2] refurl = "http://ned.ipac.caltech.edu/Library/Distances/" nedbib = "1991ASSL..171...89H" olddistname = '' loopcnt = 0 for r, row in enumerate(pbar(data, task_str)): if r <= 12: continue distname = row[3] name = name_clean(distname) # distmod = row[4] # moderr = row[5] dist = row[6] bibcode = unescape(row[8]) snname = name_clean(row[9]) redshift = row[10] cleanhost = '' if name != snname and (name + ' HOST' != snname): cleanhost = host_clean(distname) if cleanhost.endswith(' HOST') or cleanhost.startswith('SN'): cleanhost = '' if not is_number(dist): print(dist) if dist and cleanhost: catalog.nedd_dict.setdefault( cleanhost, []).append(Decimal(dist)) if snname and 'HOST' not in snname: snname, secondarysource = catalog.new_entry( snname, srcname=reference, bibcode=nedbib, url=refurl, secondary=True) if bibcode: source = catalog.entries[snname].add_source(bibcode=bibcode) sources = uniq_cdl([source, secondarysource]) else: sources = secondarysource if name == snname: if redshift: catalog.entries[snname].add_quantity( SUPERNOVA.REDSHIFT, redshift, sources) if dist: catalog.entries[snname].add_quantity( SUPERNOVA.COMOVING_DIST, dist, sources) if not redshift: try: zatval = z_at_value(cosmo.comoving_distance, float(dist) * un.Mpc, zmax=5.0) sigd = get_sig_digits(str(dist)) redshift = pretty_num(zatval, sig=sigd) except (KeyboardInterrupt, SystemExit): raise except Exception: pass else: cosmosource = catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') combsources = uniq_cdl(sources.split(',') + [cosmosource]) catalog.entries[snname].add_quantity( SUPERNOVA.REDSHIFT, redshift, combsources, derived=True) if cleanhost: catalog.entries[snname].add_quantity( SUPERNOVA.HOST, cleanhost, sources) if catalog.args.update and olddistname != distname: catalog.journal_entries() olddistname = distname loopcnt = loopcnt + 1 if catalog.args.travis and loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0: break catalog.journal_entries() f.close() return
def do_sdss_spectra(catalog): """Import spectra from LAMOST.""" task_str = catalog.get_current_task_str() # Set preferred names, calculate some columns based on imported data, # sanitize some fields keys = list(catalog.entries.keys()) fureps = { 'erg/cm2/s/A': 'erg/s/cm^2/Angstrom', '1E-17 erg/cm^2/s/Ang': 'erg/s/cm^2/Angstrom' } c_kms = con.c.cgs.value / 1.0e5 cntsdss = 0 for oname in pbar(keys, task_str): # Some events may be merged in cleanup process, skip them if # non-existent. if (FASTSTARS.RA not in catalog.entries[oname] or FASTSTARS.DEC not in catalog.entries[oname]): continue else: xid = SDSS.query_region(coord.SkyCoord( ra=catalog.entries[oname][FASTSTARS.RA][0]['value'], dec=catalog.entries[oname][FASTSTARS.DEC][0]['value'], unit=(un.hourangle, un.deg), frame='icrs'), spectro=True) # xid = SDSS.query_region(coord.SkyCoord( # ra='14:34:06.17', # dec='+56:30:47.24', # unit=(un.hourangle, un.deg), frame='icrs'), spectro=True) if xid is None: continue while len(xid) > 1: notstar = xid['z'].argmax() xid.remove_row(notstar) #print(xid) # star = None # for row in tab: # if (row['objType'] == 'Star' and # row['Class'].lower() in ['star', 'unknown']): # star = row # break # if not star: # continue try: name, source = catalog.new_entry(oname, bibcode='2015ApJS..219...12A', srcname='SDSS', url='http://www.sdss.org/') except Exception: catalog.log.warning( '"{}" was not found, suggests merge occurred in cleanup ' 'process.'.format(oname)) continue ffile = ('spec-' + str(xid['specobjid'][0]) + '.fits.gz') # furl = 'http://dr3.lamost.org/sas/fits/' + vname + '/' + ffile datafile = os.path.join(catalog.get_current_task_repo(), 'SDSS', ffile) if not os.path.exists(datafile): # Download spectra try: sp = SDSS.get_spectra(matches=xid)[0] except urllib.error.HTTPError: catalog.log.warning( '"{}" threw an HTTP 404, must be error upstream. Will likely go away on the next run.' .format(oname)) continue # Identify star # assert sp[2].data['class'][0]=='STAR' # Write spectra # print(catalog.entries[oname][FASTSTARS.RA][0]['value'],catalog.entries[oname][FASTSTARS.DEC][0]['value']) sp.writeto(datafile, overwrite=True) # open(datafile, 'wb').write(fr.content) # Open spectra hdulist = fits.open(datafile) # sp contains a list of fits datafiles, identify main one i_primary = 0 i_coadd = 1 i_specobj = 2 assert hdulist[i_primary].name == 'PRIMARY' assert hdulist[i_coadd].name == 'COADD' assert (hdulist[i_specobj].name == 'SPECOBJ' or hdulist[i_specobj].name == 'SPALL') # xid = SDSS.query_region(coord.SkyCoord( # ra='12:11:50.27', # dec='+14:37:16.2', # unit=(un.hourangle, un.deg), frame='icrs'), spectro=True) # from SPECOBJ # print('.'+hdulist[i_specobj].data['ELODIE_SPTYPE'][0]+'.') if hdulist[i_specobj].data['ELODIE_SPTYPE'][0] != 'unknown': ST, SCfull = hdulist[i_specobj].data['ELODIE_SPTYPE'][ 0][:2], hdulist[i_specobj].data['ELODIE_SPTYPE'][0][2:] if len(SCfull) > 0: if 'IV' in SCfull: SC = 'sg' elif 'III' in SCfull: SC = 'g' elif 'V' in SCfull: SC = 'd' elif 'I' in SCfull: SC = 'Sg' else: SC = False if SC is not False: catalog.entries[name].add_quantity( FASTSTARS.STELLAR_CLASS, SC, source=source) catalog.entries[name].add_quantity(FASTSTARS.SPECTRAL_TYPE, ST, source=source) if hdulist[i_specobj].data['Z'][0] != 0.0: catalog.entries[name].add_quantity( FASTSTARS.REDSHIFT, str(hdulist[i_specobj].data['Z'][0]), e_value=str(hdulist[i_specobj].data['Z_ERR'][0]), source=source) catalog.entries[name].add_quantity( FASTSTARS.VELOCITY, pretty_num(float(hdulist[i_specobj].data['Z'][0]) * c_kms, sig=5), e_value=pretty_num(float( hdulist[i_specobj].data['Z_ERR'][0] * c_kms), sig=5), source=source) for oi, obj in enumerate(hdulist[0].header): if any(x in ['.', '/'] for x in obj): del (hdulist[0].header[oi]) hdulist[0].verify('silentfix') hdrkeys = list(hdulist[0].header.keys()) # print(hdrkeys) # for key in hdulist[0].header.keys(): # print(key, hdulist[0].header[key]) if hdulist[0].header['SIMPLE']: if 'JD' in hdrkeys: mjd = str(jd_to_mjd(Decimal(str(hdulist[0].header['JD'])))) elif 'MJD' in hdrkeys: mjd = str(hdulist[0].header['MJD']) elif 'DATE-OBS' in hdrkeys: if 'T' in hdulist[0].header['DATE-OBS']: dateobs = hdulist[0].header['DATE-OBS'].strip() elif 'UTC-OBS' in hdrkeys: dateobs = hdulist[0].header['DATE-OBS'].strip( ) + 'T' + hdulist[0].header['UTC-OBS'].strip() mjd = str(astrotime(dateobs, format='isot').mjd) else: raise ValueError("Couldn't find JD/MJD for spectrum.") if hdulist[i_coadd].header['NAXIS'] == 2: waves = [ str(x) for x in list(10**hdulist[i_coadd].data['loglam']) ] fluxes = [ str(x) for x in list(hdulist[i_coadd].data['flux']) ] else: print('Warning: Skipping FITS spectrum `{}`.'.format( datafile)) continue else: raise ValueError('Non-simple FITS import not yet supported.') if 'BUNIT' in hdrkeys: fluxunit = hdulist[0].header['BUNIT'] if fluxunit in fureps: fluxunit = fureps[fluxunit] if fluxunit[:3] == '1E-17': fluxes = [ str(x * 1e-17) for x in list(hdulist[i_coadd].data['flux']) ] else: if max([float(x) for x in fluxes]) < 1.0e-5: fluxunit = 'erg/s/cm^2/Angstrom' else: fluxunit = 'Uncalibrated' specdict = { SPECTRUM.U_WAVELENGTHS: 'Angstrom', SPECTRUM.WAVELENGTHS: waves, SPECTRUM.TIME: mjd, SPECTRUM.U_TIME: 'MJD', SPECTRUM.FLUXES: fluxes, SPECTRUM.U_FLUXES: fluxunit, SPECTRUM.FILENAME: ffile, SPECTRUM.SOURCE: source } if 'TELESCOP' in hdrkeys: specdict[SPECTRUM.TELESCOPE] = hdulist[0].header['TELESCOP'] if 'INSTRUME' in hdrkeys: specdict[SPECTRUM.INSTRUMENT] = hdulist[0].header['INSTRUME'] if 'SITENAME' in hdrkeys: specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['SITENAME'] elif 'OBSERVAT' in hdrkeys: specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['OBSERVAT'] if 'OBSERVER' in hdrkeys: specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER'] if 'AIRMASS' in hdrkeys: specdict[SPECTRUM.AIRMASS] = hdulist[0].header['AIRMASS'] catalog.entries[name].add_spectrum(**specdict) cntsdss += 1 hdulist.close() catalog.journal_entries() print('`{}` have SDSS spectra.'.format(cntsdss)) return
def do_snf_specta(catalog): task_str = catalog.get_current_task_str() bibcodes = { 'SN2005gj': '2006ApJ...650..510A', 'SN2006D': '2007ApJ...654L..53T', 'SN2007if': '2010ApJ...713.1073S', 'SN2011fe': '2013A&A...554A..27P' } oldname = '' snfcnt = 0 eventfolders = next( os.walk(os.path.join(catalog.get_current_task_repo(), 'SNFactory')))[1] for eventfolder in pbar(eventfolders, task_str): oname = eventfolder name = catalog.get_preferred_name(oname) if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) sec_reference = 'Nearby Supernova Factory' sec_refurl = 'http://snfactory.lbl.gov/' sec_bibcode = '2002SPIE.4836...61A' sec_source = catalog.entries[name].add_source(name=sec_reference, url=sec_refurl, bibcode=sec_bibcode, secondary=True) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oname, sec_source) bibcode = bibcodes[oname] source = catalog.entries[name].add_source(bibcode=bibcode) sources = uniq_cdl([source, sec_source]) use_path = os.path.join(catalog.get_current_task_repo(), 'SNFactory', eventfolder, '*.dat') eventspectra = glob(use_path) for spectrum in pbar(eventspectra, task_str): filename = os.path.basename(spectrum) with open(spectrum) as spec_file: specdata = list( csv.reader(spec_file, delimiter=' ', skipinitialspace=True)) specdata = list(filter(None, specdata)) newspec = [] time = '' telescope = '' instrument = '' observer = '' observatory = '' if 'Keck_20060202_R' in spectrum: time = '53768.23469' elif 'Spectrum05_276' in spectrum: time = pretty_num(astrotime('2005-10-03').mjd, sig=5) elif 'Spectrum05_329' in spectrum: time = pretty_num(astrotime('2005-11-25').mjd, sig=5) elif 'Spectrum05_336' in spectrum: time = pretty_num(astrotime('2005-12-02').mjd, sig=5) for row in specdata: if row[0][0] == '#': joinrow = (' '.join(row)).split('=') if len(joinrow) < 2: continue field = joinrow[0].strip('# ') value = joinrow[1].split('/')[0].strip('\' ') if not time: if field == 'JD': time = str(jd_to_mjd(Decimal(value))) elif field == 'MJD': time = value elif field == 'MJD-OBS': time = value if field == 'OBSERVER': observer = value.capitalize() if field == 'OBSERVAT': observatory = value.capitalize() if field == 'TELESCOP': telescope = value.capitalize() if field == 'INSTRUME': instrument = value.capitalize() else: newspec.append(row) if not time: raise ValueError('Time missing from spectrum.') specdata = newspec haserrors = len( specdata[0] ) == 3 and specdata[0][2] and specdata[0][2] != 'NaN' specdata = [list(i) for i in zip(*specdata)] wavelengths = specdata[0] fluxes = specdata[1] errors = '' if haserrors: errors = specdata[2] unit_err = ('Variance' if oldname == 'SN2011fe' else 'erg/s/cm^2/Angstrom') unit_flx = 'erg/s/cm^2/Angstrom' catalog.entries[name].add_spectrum(u_wavelengths='Angstrom', u_fluxes=unit_flx, u_time='MJD', time=time, wavelengths=wavelengths, fluxes=fluxes, errors=errors, observer=observer, observatory=observatory, telescope=telescope, instrument=instrument, u_errors=unit_err, source=sources, filename=filename) snfcnt = snfcnt + 1 if (catalog.args.travis and snfcnt % catalog.TRAVIS_QUERY_LIMIT == 0): break catalog.journal_entries() return
def do_ucb_spectra(catalog): task_str = catalog.get_current_task_str() sec_reference = 'UCB Filippenko Group\'s Supernova Database (SNDB)' sec_refurl = 'http://heracles.astro.berkeley.edu/sndb/info' sec_refbib = '2012MNRAS.425.1789S' ucbspectracnt = 0 jsontxt = catalog.load_url( 'http://heracles.astro.berkeley.edu/sndb/download?id=allpubspec', os.path.join(catalog.get_current_task_repo(), 'UCB/allpubspec.json'), json_sort='SpecID') if not jsontxt: return spectra = json.loads(jsontxt) spectra = sorted(spectra, key=lambda kk: kk['SpecID']) oldname = '' for spectrum in pbar(spectra, task_str): name = spectrum['ObjName'] if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) sec_source = catalog.entries[name].add_source(name=sec_reference, url=sec_refurl, bibcode=sec_refbib, secondary=True) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, sec_source) sources = [sec_source] if spectrum['Reference']: sources += [ catalog.entries[name].add_source(bibcode=spectrum['Reference']) ] sources = uniq_cdl(sources) if spectrum['Type'] and spectrum['Type'].strip() != 'NoMatch': for ct in spectrum['Type'].strip().split(','): catalog.entries[name].add_quantity( SUPERNOVA.CLAIMED_TYPE, ct.replace('-norm', '').strip(), sources) if spectrum['DiscDate']: ddate = spectrum['DiscDate'].replace('-', '/') catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, ddate, sources) if spectrum['HostName']: host = urllib.parse.unquote(spectrum['HostName']).replace('*', '') catalog.entries[name].add_quantity(SUPERNOVA.HOST, host, sources) if spectrum['UT_Date']: epoch = str(spectrum['UT_Date']) year = epoch[:4] month = epoch[4:6] day = epoch[6:] sig = get_sig_digits(day) + 5 mjd = astrotime(year + '-' + month + '-' + str(floor(float(day))).zfill(2)).mjd mjd = pretty_num(mjd + float(day) - floor(float(day)), sig=sig) filename = spectrum['Filename'] if spectrum['Filename'] else '' instrument = spectrum['Instrument'] if spectrum['Instrument'] else '' reducer = spectrum['Reducer'] if spectrum['Reducer'] else '' observer = spectrum['Observer'] if spectrum['Observer'] else '' snr = str(spectrum['SNR']) if spectrum['SNR'] else '' if not filename: raise ValueError('Filename not found for SNDB spectrum!') if not spectrum['SpecID']: raise ValueError('ID not found for SNDB spectrum!') filepath = os.path.join(catalog.get_current_task_repo(), 'UCB/') + filename spectxt = catalog.load_url( 'http://heracles.astro.berkeley.edu/sndb/download?id=ds:' + str(spectrum['SpecID']), filepath, archived_mode=True) specdata = list( csv.reader(spectxt.splitlines(), delimiter=' ', skipinitialspace=True)) newspecdata = [] for row in specdata: if not row or not row[0] or row[0][0] == '#': continue else: newspecdata.append(row) specdata = newspecdata haserrors = len( specdata[0]) == 3 and specdata[0][2] and specdata[0][2] != 'NaN' specdata = [list(ii) for ii in zip(*specdata)] wavelengths = specdata[0] fluxes = specdata[1] errors = '' if haserrors: errors = specdata[2] if not list(filter(None, errors)): errors = '' units = 'Uncalibrated' catalog.entries[name].add_spectrum(u_wavelengths='Angstrom', u_fluxes=units, u_time='MJD', time=mjd, wavelengths=wavelengths, filename=filename, fluxes=fluxes, errors=errors, u_errors=units, instrument=instrument, source=sources, snr=snr, observer=observer, reducer=reducer, deredshifted=('-noz' in filename)) ucbspectracnt = ucbspectracnt + 1 if catalog.args.travis and ucbspectracnt >= catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() return
hosts[hn]['photocount'] += len(item['photometry']) if 'spectra' in item: hosts[hn]['spectracount'] += len(item['spectra']) curtime = time.time() centrate = 100.0 * 365.25 * 24.0 * 60.0 * 60.0 for hn in hosts: finitedates = sorted( [x for x in hosts[hn]['eventdates'] if x != float("inf")]) if len(finitedates) >= 2: datediff = curtime - finitedates[0] lamb = float(len(finitedates)) / (curtime - finitedates[0]) * centrate hosts[hn]['rate'] = ( pretty_num(lamb, sig=3) + ',' + pretty_num(lamb / sqrt(float(len(finitedates))), sig=3)) else: hosts[hn]['rate'] = '' hosts[hn]['events'] = [ x for (y, x) in sorted(zip(hosts[hn]['eventdates'], hosts[hn]['events']), key=lambda ev: ev[0]) ] del hosts[hn]['eventdates'] # Convert to array since that's what datatables expects hosts = list(hosts.values()) jsonstring = json.dumps(hosts, indent='\t',
def do_nedd(catalog): task_str = catalog.get_current_task_str() nedd_path = os.path.join(catalog.get_current_task_repo(), 'NED26.10.1-D-13.1.0-20160930.csv') f = open(nedd_path, 'r') data = sorted(list(csv.reader(f, delimiter=',', quotechar='"'))[13:], key=lambda x: (x[9], x[3])) reference = "NED-D v" + nedd_path.split('-')[-2] refurl = "http://ned.ipac.caltech.edu/Library/Distances/" nedbib = "1991ASSL..171...89H" olddistname = '' loopcnt = 0 for r, row in enumerate(pbar(data, task_str)): if r <= 12: continue distname = row[3] name = name_clean(distname) # distmod = row[4] # moderr = row[5] dist = row[6] bibcode = unescape(row[8]) snname = name_clean(row[9]) redshift = row[10] cleanhost = '' if name != snname and (name + ' HOST' != snname): cleanhost = host_clean(distname) if cleanhost.endswith(' HOST'): cleanhost = '' if not is_number(dist): print(dist) if dist: catalog.nedd_dict.setdefault(cleanhost, []).append(Decimal(dist)) if snname and 'HOST' not in snname: snname, secondarysource = catalog.new_entry(snname, srcname=reference, bibcode=nedbib, url=refurl, secondary=True) if bibcode: source = catalog.entries[snname].add_source(bibcode=bibcode) sources = uniq_cdl([source, secondarysource]) else: sources = secondarysource if name == snname: if redshift: catalog.entries[snname].add_quantity( SUPERNOVA.REDSHIFT, redshift, sources) if dist: catalog.entries[snname].add_quantity( SUPERNOVA.COMOVING_DIST, dist, sources) if not redshift: try: zatval = z_at_value(cosmo.comoving_distance, float(dist) * un.Mpc, zmax=5.0) sigd = get_sig_digits(str(dist)) redshift = pretty_num(zatval, sig=sigd) except (KeyboardInterrupt, SystemExit): raise except Exception: pass else: cosmosource = catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') combsources = uniq_cdl( sources.split(',') + [cosmosource]) catalog.entries[snname].add_quantity( SUPERNOVA.REDSHIFT, redshift, combsources, derived=True) if cleanhost: catalog.entries[snname].add_quantity(SUPERNOVA.HOST, cleanhost, sources) if catalog.args.update and olddistname != distname: catalog.journal_entries() olddistname = distname loopcnt = loopcnt + 1 if catalog.args.travis and loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0: break catalog.journal_entries() f.close() return
def do_snls_spectra(catalog): """ """ task_str = catalog.get_current_task_str() result = Vizier.get_catalogs('J/A+A/507/85/table1') table = result[list(result.keys())[0]] table.convert_bytestring_to_unicode(python3_only=True) datedict = {} for row in table: datedict['SNLS-' + row['SN']] = str(astrotime(row['Date']).mjd) oldname = '' file_names = glob(os.path.join(catalog.get_current_task_repo(), 'SNLS/*')) for fi, fname in enumerate(pbar_strings(file_names, task_str)): filename = os.path.basename(fname) fileparts = filename.split('_') name = 'SNLS-' + fileparts[1] name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2009A&A...507...85B') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, '20' + fileparts[1][:2], source) f = open(fname, 'r') data = csv.reader(f, delimiter=' ', skipinitialspace=True) specdata = [] for r, row in enumerate(data): if row[0] == '@TELESCOPE': telescope = row[1].strip() elif row[0] == '@REDSHIFT': catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, row[1].strip(), source) if r < 14: continue specdata.append(list(filter(None, [x.strip(' \t') for x in row]))) specdata = [list(i) for i in zip(*specdata)] wavelengths = specdata[1] fluxes = [pretty_num(float(x) * 1.e-16, sig=get_sig_digits(x)) for x in specdata[2]] # FIX: this isnt being used # errors = [pretty_num(float(x)*1.e-16, sig=get_sig_digits(x)) for x in # specdata[3]] catalog.entries[name].add_spectrum( u_wavelengths='Angstrom', u_fluxes='erg/s/cm^2/Angstrom', wavelengths=wavelengths, fluxes=fluxes, u_time='MJD' if name in datedict else '', time=datedict[name] if name in datedict else '', telescope=telescope, source=source, filename=filename) if catalog.args.travis and fi >= catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() return
def do_nedd(catalog): task_str = catalog.get_current_task_str() nedd_path = os.path.join( catalog.get_current_task_repo(), 'NED26.05.1-D-12.1.0-20160501.csv') f = open(nedd_path, 'r') data = sorted(list(csv.reader(f, delimiter=',', quotechar='"'))[ 13:], key=lambda x: (x[9], x[3])) reference = "NED-D" refurl = "http://ned.ipac.caltech.edu/Library/Distances/" nedd_dict = OrderedDict() olddistname = '' for r, row in enumerate(pbar(data, task_str)): if r <= 12: continue distname = row[3] name = name_clean(distname) # distmod = row[4] # moderr = row[5] dist = row[6] bibcode = unescape(row[8]) snname = name_clean(row[9]) redshift = row[10] cleanhost = '' if name != snname and (name + ' HOST' != snname): cleanhost = host_clean(distname) if cleanhost.endswith(' HOST'): cleanhost = '' if not is_number(dist): print(dist) if dist: nedd_dict.setdefault(cleanhost, []).append(Decimal(dist)) if snname and 'HOST' not in snname: snname, secondarysource = catalog.new_entry( snname, srcname=reference, url=refurl, secondary=True) if bibcode: source = catalog.entries[snname].add_source(bibcode=bibcode) sources = uniq_cdl([source, secondarysource]) else: sources = secondarysource if name == snname: if redshift: catalog.entries[snname].add_quantity( 'redshift', redshift, sources) if dist: catalog.entries[snname].add_quantity( 'comovingdist', dist, sources) if not redshift: try: zatval = z_at_value(cosmo.comoving_distance, float(dist) * un.Mpc, zmax=5.0) sigd = get_sig_digits(str(dist)) redshift = pretty_num(zatval, sig=sigd) except (KeyboardInterrupt, SystemExit): raise except: pass else: cosmosource = catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') combsources = uniq_cdl(sources.split(',') + [cosmosource]) catalog.entries[snname].add_quantity('redshift', redshift, combsources) if cleanhost: catalog.entries[snname].add_quantity( 'host', cleanhost, sources) if catalog.args.update and olddistname != distname: catalog.journal_entries() olddistname = distname catalog.journal_entries() f.close() return
def do_lamost(catalog): """Import spectra from LAMOST.""" task_str = catalog.get_current_task_str() # Set preferred names, calculate some columns based on imported data, # sanitize some fields keys = list(catalog.entries.keys()) viz = Vizier(columns=["**"]) fureps = {'erg/cm2/s/A': 'erg/s/cm^2/Angstrom'} c_kms = con.c.cgs.value / 1.0e5 for oname in pbar(keys, task_str): # Some events may be merged in cleanup process, skip them if # non-existent. if (FASTSTARS.RA not in catalog.entries[oname] or FASTSTARS.DEC not in catalog.entries[oname]): continue else: result = viz.query_region(coord.SkyCoord( ra=catalog.entries[oname][FASTSTARS.RA][0]['value'], dec=catalog.entries[oname][FASTSTARS.DEC][0]['value'], unit=(un.hourangle, un.deg), frame='icrs'), width="2s", catalog="V/149/dr2") if not result.keys(): continue tab = result['V/149/dr2'] star = None for row in tab: if (row['objType'] == 'Star' and row['Class'].lower() in ['star', 'unknown']): star = row break if not star: continue try: name, source = catalog.new_entry(oname, bibcode='2016yCat.5149....0L', srcname='LAMOST', url='http://dr3.lamost.org/') except Exception: catalog.log.warning( '"{}" was not found, suggests merge occurred in cleanup ' 'process.'.format(oname)) continue if row['SubClass'] is not 'Non': #catalog.entries[name].add_quantity( # FASTSTARS.SPECTRAL_TYPE, row['SubClass'], source=source) ST, SCfull = row['SubClass'][:2], row['SubClass'][2:] if len(SCfull) > 0: if 'IV' in SCfull: SC = 'sg' elif 'III' in SCfull: SC = 'g' elif 'V' in SCfull: SC = 'd' elif 'I' in SCfull: SC = 'Sg' catalog.entries[name].add_quantity(FASTSTARS.STELLAR_CLASS, SC, source=source) catalog.entries[name].add_quantity(FASTSTARS.SPECTRAL_TYPE, ST, source=source) if row['z'] and is_number(row['z']): catalog.entries[name].add_quantity(FASTSTARS.REDSHIFT, str(row['z']), e_value=str(row['e_z']), source=source) catalog.entries[name].add_quantity( FASTSTARS.VELOCITY, pretty_num(float(row['z']) * c_kms, sig=5), e_value=pretty_num(float(row['e_z'] * c_kms), sig=5), source=source) mag_types = list(row['magType'].replace('psf_', '')) nmt = [] nmi = 0 for mt in mag_types: if is_number(mt): nmt[nmi - 1] += mt else: nmt += mt nmi += 1 mag_types = [ x.upper() if x in ['b', 'v', 'j', 'h'] else x for x in nmt ] for mi, mt in enumerate(mag_types): snrf = 'snr' + mt.lower() if snrf in row.columns and float(row[snrf]) < 3: continue photodict = { PHOTOMETRY.TIME: str(row['MJD']), PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.BAND: mt, PHOTOMETRY.TELESCOPE: 'LAMOST', PHOTOMETRY.MAGNITUDE: str(row['mag' + str(mi + 1)]), PHOTOMETRY.SOURCE: source } if snrf in row.columns: photodict[PHOTOMETRY.E_MAGNITUDE] = str( Decimal('2.5') * (Decimal('1') + Decimal('1') / Decimal(str(row[snrf]))).log10())[:5] catalog.entries[name].add_photometry(**photodict) vname = row['PlanId'] ffile = ('spec-' + row['LMJD'] + '-' + vname + '_sp' + row['spId'] + '-' + row['FiberId'] + '.fits.gz') furl = 'http://dr3.lamost.org/sas/fits/' + vname + '/' + ffile datafile = os.path.join(catalog.get_current_task_repo(), 'LAMOST', ffile) if not os.path.exists(datafile): fr = requests.get(furl) open(datafile, 'wb').write(fr.content) hdulist = fits.open(datafile) for oi, obj in enumerate(hdulist[0].header): if any(x in ['.', '/'] for x in obj): del (hdulist[0].header[oi]) hdulist[0].verify('silentfix') hdrkeys = list(hdulist[0].header.keys()) # print(hdrkeys) # for key in hdulist[0].header.keys(): # print(key, hdulist[0].header[key]) if hdulist[0].header['SIMPLE']: if 'JD' in hdrkeys: mjd = str(jd_to_mjd(Decimal(str(hdulist[0].header['JD'])))) elif 'MJD' in hdrkeys: mjd = str(hdulist[0].header['MJD']) elif 'DATE-OBS' in hdrkeys: if 'T' in hdulist[0].header['DATE-OBS']: dateobs = hdulist[0].header['DATE-OBS'].strip() elif 'UTC-OBS' in hdrkeys: dateobs = hdulist[0].header['DATE-OBS'].strip( ) + 'T' + hdulist[0].header['UTC-OBS'].strip() mjd = str(astrotime(dateobs, format='isot').mjd) else: raise ValueError("Couldn't find JD/MJD for spectrum.") if hdulist[0].header['NAXIS'] == 2: waves = [str(x) for x in list(hdulist[0].data)[2]] fluxes = [str(x) for x in list(hdulist[0].data)[0]] else: print('Warning: Skipping FITS spectrum `{}`.'.format( datafile)) continue else: raise ValueError('Non-simple FITS import not yet supported.') if 'BUNIT' in hdrkeys: fluxunit = hdulist[0].header['BUNIT'] if fluxunit in fureps: fluxunit = fureps[fluxunit] else: if max([float(x) for x in fluxes]) < 1.0e-5: fluxunit = 'erg/s/cm^2/Angstrom' else: fluxunit = 'Uncalibrated' specdict = { SPECTRUM.U_WAVELENGTHS: 'Angstrom', SPECTRUM.WAVELENGTHS: waves, SPECTRUM.TIME: mjd, SPECTRUM.U_TIME: 'MJD', SPECTRUM.FLUXES: fluxes, SPECTRUM.U_FLUXES: fluxunit, SPECTRUM.FILENAME: ffile, SPECTRUM.SOURCE: source } if 'TELESCOP' in hdrkeys: specdict[SPECTRUM.TELESCOPE] = hdulist[0].header['TELESCOP'] if 'INSTRUME' in hdrkeys: specdict[SPECTRUM.INSTRUMENT] = hdulist[0].header['INSTRUME'] if 'SITENAME' in hdrkeys: specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['SITENAME'] elif 'OBSERVAT' in hdrkeys: specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header['OBSERVAT'] if 'OBSERVER' in hdrkeys: specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER'] if 'AIRMASS' in hdrkeys: specdict[SPECTRUM.AIRMASS] = hdulist[0].header['AIRMASS'] catalog.entries[name].add_spectrum(**specdict) hdulist.close() catalog.journal_entries() return
def do_tns(catalog): """Load TNS metadata.""" session = requests.Session() task_str = catalog.get_current_task_str() tns_url = 'https://wis-tns.weizmann.ac.il/' search_url = tns_url + \ 'search?&num_page=1&format=html&sort=desc&order=id&format=csv&page=0' csvtxt = catalog.load_url(search_url, os.path.join(catalog.get_current_task_repo(), 'TNS', 'index.csv')) if not csvtxt: return maxid = csvtxt.splitlines()[1].split(',')[0].strip('"') maxpages = ceil(int(maxid) / 1000.) for page in pbar(range(maxpages), task_str): fname = os.path.join( catalog.get_current_task_repo(), 'TNS', 'page-') + str(page).zfill(2) + '.csv' if (catalog.current_task.load_archive(catalog.args) and os.path.isfile(fname) and page < 7): with open(fname, 'r') as tns_file: csvtxt = tns_file.read() else: session = requests.Session() ses_url = (tns_url + 'search?&num_page=1000&format=html&edit' '[type]=&edit[objname]=&edit[id]=&sort=asc&order=id' '&display[redshift]=1' '&display[hostname]=1&display[host_redshift]=1' '&display[source_group_name]=1' '&display[programs_name]=1' '&display[internal_name]=1' '&display[isTNS_AT]=1' '&display[public]=1' '&display[end_pop_period]=0' '&display[spectra_count]=1' '&display[discoverymag]=1&display[discmagfilter]=1' '&display[discoverydate]=1&display[discoverer]=1' '&display[sources]=1' '&display[bibcode]=1&format=csv&page=' + str(page)) try: response = session.get(ses_url, timeout=40) csvtxt = response.text except Exception as e: catalog.log.warning( 'Could not download TNS page #{}.'.format(str(page))) if os.path.isfile(fname): with open(fname, 'r') as tns_file: csvtxt = tns_file.read() else: continue else: with open(fname, 'w') as tns_file: tns_file.write(csvtxt) tsvin = list(csv.reader([x.replace('[data validation error, please contact site admin]', '""') for x in csvtxt.splitlines()], delimiter=',')) for ri, row in enumerate(pbar(tsvin, task_str, leave=False)): if ri == 0: continue if row[4] and 'SN' not in row[4]: continue name = row[1].replace(' ', '') if len(name) < 5: continue name, source = catalog.new_entry( name, srcname='Transient Name Server', url=tns_url) if row[2] and row[2] != '00:00:00.00': catalog.entries[name].add_quantity(SUPERNOVA.RA, row[2], source) if row[3] and row[3] != '+00:00:00.00': catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[3], source) if row[4]: catalog.entries[name].add_quantity( SUPERNOVA.CLAIMED_TYPE, row[4].replace('SN', '').strip(), source) if row[5]: catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, row[5], source, kind='spectroscopic') if row[6]: catalog.entries[name].add_quantity(SUPERNOVA.HOST, row[6], source) if row[7]: catalog.entries[name].add_quantity( [SUPERNOVA.REDSHIFT, SUPERNOVA.HOST_REDSHIFT], row[7], source, kind='host') if row[8]: catalog.entries[name].add_quantity(SUPERNOVA.DISCOVERER, row[8], source) # Currently, all events listing all possible observers. TNS bug? # if row[9]: # observers = row[9].split(',') # for observer in observers: # catalog.entries[name].add_quantity('observer', # observer.strip(), # source) if row[11]: catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[11], source) if row[19]: date = row[19].split()[0].replace('-', '/') if 'clear' in row[19].lower(): print(row) if date != '0000/00/00' and 'clear' not in row[19].lower(): date = date.replace('/00', '') dsplit = row[19].split() if len(dsplit) >= 2: t = dsplit[1] if t != '00:00:00': ts = t.split(':') dt = timedelta( hours=int(ts[0]), minutes=int(ts[1]), seconds=int(ts[2])) date += pretty_num( dt.total_seconds() / (24 * 60 * 60), sig=6).lstrip('0') catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, date, source) if catalog.args.travis and ri >= catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() catalog.journal_entries()
if 'photometry' in item: hosts[hn]['photocount'] += len(item['photometry']) if 'spectra' in item: hosts[hn]['spectracount'] += len(item['spectra']) curtime = time.time() centrate = 100.0 * 365.25 * 24.0 * 60.0 * 60.0 for hn in hosts: finitedates = sorted( [x for x in hosts[hn]['eventdates'] if x != float("inf")]) if len(finitedates) >= 2: datediff = curtime - finitedates[0] lamb = float(len(finitedates)) / (curtime - finitedates[0]) * centrate hosts[hn]['rate'] = (pretty_num(lamb, sig=3) + ',' + pretty_num(lamb / sqrt(float(len(finitedates))), sig=3)) else: hosts[hn]['rate'] = '' hosts[hn]['events'] = [x for (y, x) in sorted( zip(hosts[hn]['eventdates'], hosts[hn]['events']), key=lambda ev: ev[0])] del hosts[hn]['eventdates'] # Convert to array since that's what datatables expects hosts = list(hosts.values()) jsonstring = json.dumps( hosts, indent='\t', separators=(',', ':'), ensure_ascii=False) with open(outdir + 'hosts.json', 'w') as f: f.write(jsonstring)
def do_cleanup(catalog): """Cleanup catalog after importing all data.""" task_str = catalog.get_current_task_str() # Set preferred names, calculate some columns based on imported data, # sanitize some fields keys = list(catalog.entries.keys()) cleanupcnt = 0 for oname in pbar(keys, task_str): # Some events may be merged in cleanup process, skip them if # non-existent. try: name = catalog.add_entry(oname) except Exception: catalog.log.warning( '"{}" was not found, suggests merge occurred in cleanup ' 'process.'.format(oname)) continue # Set the preferred name, switching to that name if name changed. name = catalog.entries[name].set_preferred_name() aliases = catalog.entries[name].get_aliases() catalog.entries[name].purge_bandless_photometry() catalog.entries[name].set_first_max_light() if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['MLS', 'SSS', 'CSS', 'GRB '] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:2])): discoverdate = ('/'.join([ '20' + alias.replace(prefix, '')[:2], alias.replace(prefix, '')[2:4], alias.replace(prefix, '')[4:6] ])) if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, discoverdate, source, derived=True) break if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]: break if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]: prefixes = [ 'ASASSN-', 'PS1-', 'PS1', 'PS', 'iPTF', 'PTF', 'SCP-', 'SNLS-', 'SPIRITS', 'LSQ', 'DES', 'SNHiTS', 'Gaia', 'GND', 'GNW', 'GSD', 'GSW', 'EGS', 'COS', 'OGLE', 'HST' ] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:2]) and is_number(alias.replace(prefix, '')[:1])): discoverdate = '20' + alias.replace(prefix, '')[:2] if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, discoverdate, source, derived=True) break if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]: break if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['SNF'] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:4])): discoverdate = ('/'.join([ alias.replace(prefix, '')[:4], alias.replace(prefix, '')[4:6], alias.replace(prefix, '')[6:8] ])) if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, discoverdate, source, derived=True) break if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]: break if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['PTFS', 'SNSDF'] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:2])): discoverdate = ('/'.join([ '20' + alias.replace(prefix, '')[:2], alias.replace(prefix, '')[2:4] ])) if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, discoverdate, source, derived=True) break if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]: break if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['AT', 'SN', 'OGLE-', 'SM ', 'KSN'] for alias in aliases: for prefix in prefixes: if alias.startswith(prefix): year = re.findall(r'\d+', alias) if len(year) == 1: year = year[0] else: continue if alias.replace(prefix, '').index(year) != 0: continue if (year and is_number(year) and '.' not in year and len(year) <= 4): discoverdate = year if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, discoverdate, source, derived=True) break if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]: break if (SUPERNOVA.RA not in catalog.entries[name] or SUPERNOVA.DEC not in catalog.entries[name]): prefixes = [ 'PSN J', 'MASJ', 'CSS', 'SSS', 'MASTER OT J', 'HST J', 'TCP J', 'MACS J', '2MASS J', 'EQ J', 'CRTS J', 'SMT J' ] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:6])): noprefix = alias.split(':')[-1].replace(prefix, '').replace( '.', '') decsign = '+' if '+' in noprefix else '-' noprefix = noprefix.replace('+', '|').replace('-', '|') nops = noprefix.split('|') if len(nops) < 2: continue rastr = nops[0] decstr = nops[1] ra = ':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + \ ('.' + rastr[6:] if len(rastr) > 6 else '') dec = ( decsign + ':'.join([decstr[:2], decstr[2:4], decstr[4:6]]) + ('.' + decstr[6:] if len(decstr) > 6 else '')) if catalog.args.verbose: tprint('Added ra/dec from name: ' + ra + ' ' + dec) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity(SUPERNOVA.RA, ra, source, derived=True) catalog.entries[name].add_quantity(SUPERNOVA.DEC, dec, source, derived=True) break if SUPERNOVA.RA in catalog.entries[name]: break no_host = (SUPERNOVA.HOST not in catalog.entries[name] or not any([ x[QUANTITY.VALUE] == 'Milky Way' for x in catalog.entries[name][SUPERNOVA.HOST] ])) if (SUPERNOVA.RA in catalog.entries[name] and SUPERNOVA.DEC in catalog.entries[name] and no_host): from astroquery.irsa_dust import IrsaDust if name not in catalog.extinctions_dict: try: ra_dec = catalog.entries[name][ SUPERNOVA.RA][0][QUANTITY.VALUE] + \ " " + \ catalog.entries[name][SUPERNOVA.DEC][0][QUANTITY.VALUE] result = IrsaDust.get_query_table(ra_dec, section='ebv') except (KeyboardInterrupt, SystemExit): raise except Exception: warnings.warn("Coordinate lookup for " + name + " failed in IRSA.") else: ebv = result['ext SandF mean'][0] ebverr = result['ext SandF std'][0] catalog.extinctions_dict[name] = [ebv, ebverr] if name in catalog.extinctions_dict: sources = uniq_cdl([ catalog.entries[name].add_self_source(), catalog.entries[name].add_source( bibcode='2011ApJ...737..103S') ]) (catalog.entries[name].add_quantity( SUPERNOVA.EBV, str(catalog.extinctions_dict[name][0]), sources, e_value=str(catalog.extinctions_dict[name][1]), derived=True)) if ((SUPERNOVA.HOST in catalog.entries[name] and (SUPERNOVA.HOST_RA not in catalog.entries[name] or SUPERNOVA.HOST_DEC not in catalog.entries[name]))): for host in catalog.entries[name][SUPERNOVA.HOST]: alias = host[QUANTITY.VALUE] if ' J' in alias and is_number(alias.split(' J')[-1][:6]): noprefix = alias.split(' J')[-1].split(':')[-1].replace( '.', '') decsign = '+' if '+' in noprefix else '-' noprefix = noprefix.replace('+', '|').replace('-', '|') nops = noprefix.split('|') if len(nops) < 2: continue rastr = nops[0] decstr = nops[1] hostra = (':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + ('.' + rastr[6:] if len(rastr) > 6 else '')) hostdec = decsign + ':'.join([ decstr[:2], decstr[2:4], decstr[4:6] ]) + ('.' + decstr[6:] if len(decstr) > 6 else '') if catalog.args.verbose: tprint('Added hostra/hostdec from name: ' + hostra + ' ' + hostdec) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity(SUPERNOVA.HOST_RA, hostra, source, derived=True) catalog.entries[name].add_quantity(SUPERNOVA.HOST_DEC, hostdec, source, derived=True) break if SUPERNOVA.HOST_RA in catalog.entries[name]: break if (SUPERNOVA.REDSHIFT not in catalog.entries[name] and SUPERNOVA.VELOCITY in catalog.entries[name]): # Find the "best" velocity to use for this bestsig = 0 for hv in catalog.entries[name][SUPERNOVA.VELOCITY]: sig = get_sig_digits(hv[QUANTITY.VALUE]) if sig > bestsig: besthv = hv[QUANTITY.VALUE] bestsrc = hv['source'] bestsig = sig if bestsig > 0 and is_number(besthv): voc = float(besthv) * 1.e5 / CLIGHT source = catalog.entries[name].add_self_source() sources = uniq_cdl([source] + bestsrc.split(',')) (catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, pretty_num(sqrt((1. + voc) / (1. - voc)) - 1., sig=bestsig), sources, kind='heliocentric', derived=True)) if (SUPERNOVA.REDSHIFT not in catalog.entries[name] and len(catalog.nedd_dict) > 0 and SUPERNOVA.HOST in catalog.entries[name]): reference = "NED-D" refurl = "http://ned.ipac.caltech.edu/Library/Distances/" refbib = "1991ASSL..171...89H" for host in catalog.entries[name][SUPERNOVA.HOST]: if host[QUANTITY.VALUE] in catalog.nedd_dict: source = catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') secondarysource = catalog.entries[name].add_source( name=reference, url=refurl, bibcode=refbib, secondary=True) meddist = statistics.median( catalog.nedd_dict[host[QUANTITY.VALUE]]) redz = z_at_value(cosmo.comoving_distance, float(meddist) * un.Mpc) redshift = pretty_num(redz, sig=get_sig_digits(str(meddist))) catalog.entries[name].add_quantity( [SUPERNOVA.REDSHIFT, SUPERNOVA.HOST_REDSHIFT], redshift, uniq_cdl([source, secondarysource]), kind='host', derived=True) if (SUPERNOVA.MAX_ABS_MAG not in catalog.entries[name] and SUPERNOVA.MAX_APP_MAG in catalog.entries[name] and SUPERNOVA.LUM_DIST in catalog.entries[name]): # Find the "best" distance to use for this bestsig = 0 for ld in catalog.entries[name][SUPERNOVA.LUM_DIST]: sig = get_sig_digits(ld[QUANTITY.VALUE]) if sig > bestsig: bestld = ld[QUANTITY.VALUE] bestsrc = ld[QUANTITY.SOURCE] bestsig = sig if bestsig > 0 and is_number(bestld) and float(bestld) > 0.: source = catalog.entries[name].add_self_source() sources = uniq_cdl([source] + bestsrc.split(',')) bestldz = z_at_value(cosmo.luminosity_distance, float(bestld) * un.Mpc) pnum = (float(catalog.entries[name][SUPERNOVA.MAX_APP_MAG][0][ QUANTITY.VALUE]) - 5.0 * (log10(float(bestld) * 1.0e6) - 1.0) + 2.5 * log10(1.0 + bestldz)) pnum = pretty_num(pnum, sig=bestsig + 1) catalog.entries[name].add_quantity(SUPERNOVA.MAX_ABS_MAG, pnum, sources, derived=True) if (SUPERNOVA.MAX_VISUAL_ABS_MAG not in catalog.entries[name] and SUPERNOVA.MAX_VISUAL_APP_MAG in catalog.entries[name] and SUPERNOVA.LUM_DIST in catalog.entries[name]): # Find the "best" distance to use for this bestsig = 0 for ld in catalog.entries[name][SUPERNOVA.LUM_DIST]: sig = get_sig_digits(ld[QUANTITY.VALUE]) if sig > bestsig: bestld = ld[QUANTITY.VALUE] bestsrc = ld[QUANTITY.SOURCE] bestsig = sig if bestsig > 0 and is_number(bestld) and float(bestld) > 0.: source = catalog.entries[name].add_self_source() sources = uniq_cdl([source] + bestsrc.split(',')) # FIX: what's happening here?! pnum = (float(catalog.entries[name][ SUPERNOVA.MAX_VISUAL_APP_MAG][0][QUANTITY.VALUE]) - 5.0 * (log10(float(bestld) * 1.0e6) - 1.0)) pnum = pretty_num(pnum, sig=bestsig + 1) catalog.entries[name].add_quantity( SUPERNOVA.MAX_VISUAL_ABS_MAG, pnum, sources, derived=True) if SUPERNOVA.REDSHIFT in catalog.entries[name]: # Find the "best" redshift to use for this bestz, bestkind, bestsig, bestsrc = catalog.entries[ name].get_best_redshift() if bestsig > 0: try: bestz = float(bestz) except Exception: print(catalog.entries[name]) raise if SUPERNOVA.VELOCITY not in catalog.entries[name]: source = catalog.entries[name].add_self_source() # FIX: what's happening here?! pnum = CLIGHT / KM * \ ((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.) pnum = pretty_num(pnum, sig=bestsig) catalog.entries[name].add_quantity( SUPERNOVA.VELOCITY, pnum, source, kind=(SUPERNOVA.VELOCITY.kind_preference[bestkind] if bestkind else '')) if bestz > 0.: if SUPERNOVA.LUM_DIST not in catalog.entries[name]: dl = cosmo.luminosity_distance(bestz) sources = [ catalog.entries[name].add_self_source(), catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') ] sources = uniq_cdl(sources + bestsrc.split(',')) catalog.entries[name].add_quantity( SUPERNOVA.LUM_DIST, pretty_num(dl.value, sig=bestsig + 1), sources, kind=(SUPERNOVA.LUM_DIST.kind_preference[bestkind] if bestkind else ''), derived=True) if (SUPERNOVA.MAX_ABS_MAG not in catalog.entries[name] and SUPERNOVA.MAX_APP_MAG in catalog.entries[name]): source = catalog.entries[name].add_self_source() pnum = pretty_num( float(catalog.entries[name][ SUPERNOVA.MAX_APP_MAG][0][QUANTITY.VALUE]) - 5.0 * (log10(dl.to('pc').value) - 1.0) + 2.5 * log10(1.0 + bestz), sig=bestsig + 1) catalog.entries[name].add_quantity( SUPERNOVA.MAX_ABS_MAG, pnum, sources, derived=True) if (SUPERNOVA.MAX_VISUAL_ABS_MAG not in catalog.entries[name] and SUPERNOVA.MAX_VISUAL_APP_MAG in catalog.entries[name]): source = catalog.entries[name].add_self_source() pnum = pretty_num(float(catalog.entries[name][ SUPERNOVA.MAX_VISUAL_APP_MAG][0][ QUANTITY.VALUE]) - 5.0 * (log10(dl.to('pc').value) - 1.0), sig=bestsig + 1) catalog.entries[name].add_quantity( SUPERNOVA.MAX_VISUAL_ABS_MAG, pnum, sources, derived=True) if SUPERNOVA.COMOVING_DIST not in catalog.entries[name]: cd = cosmo.comoving_distance(bestz) sources = [ catalog.entries[name].add_self_source(), catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') ] sources = uniq_cdl(sources + bestsrc.split(',')) catalog.entries[name].add_quantity( SUPERNOVA.COMOVING_DIST, pretty_num(cd.value, sig=bestsig), sources, derived=True) if SUPERNOVA.HOST_REDSHIFT in catalog.entries[name]: # Find the "best" redshift to use for this bestz, bestkind, bestsig, bestsrc = catalog.entries[ name].get_best_redshift(SUPERNOVA.HOST_REDSHIFT) if bestsig > 0: try: bestz = float(bestz) except Exception: print(catalog.entries[name]) raise if SUPERNOVA.HOST_VELOCITY not in catalog.entries[name]: source = catalog.entries[name].add_self_source() # FIX: what's happening here?! pnum = CLIGHT / KM * \ ((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.) pnum = pretty_num(pnum, sig=bestsig) catalog.entries[name].add_quantity( SUPERNOVA.HOST_VELOCITY, pnum, source, kind=(SUPERNOVA.HOST_VELOCITY.kind_preference[bestkind] if bestkind else '')) if bestz > 0.: if SUPERNOVA.HOST_LUM_DIST not in catalog.entries[name]: dl = cosmo.luminosity_distance(bestz) sources = [ catalog.entries[name].add_self_source(), catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') ] sources = uniq_cdl(sources + bestsrc.split(',')) catalog.entries[name].add_quantity( SUPERNOVA.HOST_LUM_DIST, pretty_num(dl.value, sig=bestsig + 1), sources, kind=(SUPERNOVA.HOST_LUM_DIST. kind_preference[bestkind] if bestkind else ''), derived=True) if SUPERNOVA.HOST_COMOVING_DIST not in catalog.entries[ name]: cd = cosmo.comoving_distance(bestz) sources = [ catalog.entries[name].add_self_source(), catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') ] sources = uniq_cdl(sources + bestsrc.split(',')) catalog.entries[name].add_quantity( SUPERNOVA.HOST_COMOVING_DIST, pretty_num(cd.value, sig=bestsig), sources, derived=True) if all([ x in catalog.entries[name] for x in [ SUPERNOVA.RA, SUPERNOVA.DEC, SUPERNOVA.HOST_RA, SUPERNOVA.HOST_DEC ] ]): # For now just using first coordinates that appear in entry try: c1 = coord( ra=catalog.entries[name][SUPERNOVA.RA][0][QUANTITY.VALUE], dec=catalog.entries[name][SUPERNOVA.DEC][0][ QUANTITY.VALUE], unit=(un.hourangle, un.deg)) c2 = coord(ra=catalog.entries[name][SUPERNOVA.HOST_RA][0][ QUANTITY.VALUE], dec=catalog.entries[name][SUPERNOVA.HOST_DEC][0][ QUANTITY.VALUE], unit=(un.hourangle, un.deg)) except (KeyboardInterrupt, SystemExit): raise except Exception: pass else: sources = uniq_cdl([catalog.entries[name].add_self_source()] + catalog.entries[name][SUPERNOVA.RA][0][ QUANTITY.SOURCE].split(',') + catalog.entries[name][SUPERNOVA.DEC][0][ QUANTITY.SOURCE].split(',') + catalog.entries[name][SUPERNOVA.HOST_RA][0][ QUANTITY.SOURCE].split(',') + catalog.entries[name][SUPERNOVA.HOST_DEC][0] [QUANTITY.SOURCE].split(',')) if SUPERNOVA.HOST_OFFSET_ANG not in catalog.entries[name]: hosa = Decimal(c1.separation(c2).arcsecond) hosa = pretty_num(hosa) catalog.entries[name].add_quantity( SUPERNOVA.HOST_OFFSET_ANG, hosa, sources, derived=True, u_value='arcseconds') if (SUPERNOVA.COMOVING_DIST in catalog.entries[name] and SUPERNOVA.REDSHIFT in catalog.entries[name] and SUPERNOVA.HOST_OFFSET_DIST not in catalog.entries[name]): offsetsig = get_sig_digits(catalog.entries[name][ SUPERNOVA.HOST_OFFSET_ANG][0][QUANTITY.VALUE]) sources = uniq_cdl( sources.split(',') + (catalog.entries[name][SUPERNOVA.COMOVING_DIST][0][ QUANTITY.SOURCE]).split(',') + (catalog.entries[name][SUPERNOVA.REDSHIFT][0][ QUANTITY.SOURCE]).split(',')) (catalog.entries[name].add_quantity( SUPERNOVA.HOST_OFFSET_DIST, pretty_num( float(catalog.entries[name][ SUPERNOVA.HOST_OFFSET_ANG][0][QUANTITY.VALUE]) / 3600. * (pi / 180.) * float(catalog.entries[name][ SUPERNOVA.COMOVING_DIST][0][QUANTITY.VALUE]) * 1000. / (1.0 + float(catalog.entries[name][ SUPERNOVA.REDSHIFT][0][QUANTITY.VALUE])), sig=offsetsig), sources)) catalog.entries[name].sanitize() catalog.journal_entries(bury=True, final=True, gz=True) cleanupcnt = cleanupcnt + 1 if catalog.args.travis and cleanupcnt % 1000 == 0: break catalog.save_caches() return
def do_donated_spectra(catalog): """Import donated spectra.""" task_str = catalog.get_current_task_str() fpath = os.path.join(catalog.get_current_task_repo(), 'donations') with open(os.path.join(fpath, 'meta.json'), 'r') as f: metadict = json.loads(f.read()) donationscnt = 0 oldname = '' for fname in pbar(metadict, task_str): name = metadict[fname]['name'] name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name sec_bibc = metadict[fname]['bibcode'] name, source = catalog.new_entry(name, bibcode=sec_bibc) date = metadict[fname].get('date', '') year, month, day = date.split('/') sig = get_sig_digits(day) + 5 day_fmt = str(floor(float(day))).zfill(2) time = astrotime(year + '-' + month + '-' + day_fmt).mjd time = time + float(day) - floor(float(day)) time = pretty_num(time, sig=sig) with open(os.path.join(fpath, fname), 'r') as f: specdata = list(csv.reader(f, delimiter=' ', skipinitialspace=True)) specdata = list(filter(None, specdata)) newspec = [] oldval = '' for row in specdata: if row[0][0] == '#': continue if row[1] == oldval: continue newspec.append(row) oldval = row[1] specdata = newspec haserrors = len( specdata[0]) == 3 and specdata[0][2] and specdata[0][2] != 'NaN' specdata = [list(i) for i in zip(*specdata)] wavelengths = specdata[0] fluxes = specdata[1] errors = '' if haserrors: errors = specdata[2] specdict = { SPECTRUM.U_WAVELENGTHS: 'Angstrom', SPECTRUM.U_TIME: 'MJD', SPECTRUM.TIME: time, SPECTRUM.WAVELENGTHS: wavelengths, SPECTRUM.FLUXES: fluxes, SPECTRUM.ERRORS: errors, SPECTRUM.SOURCE: source, SPECTRUM.FILENAME: fname } if 'instrument' in metadict[fname]: specdict[SPECTRUM.INSTRUMENT] = metadict[fname]['instrument'] if 'telescope' in metadict[fname]: specdict[SPECTRUM.TELESCOPE] = metadict[fname]['telescope'] if 'yunit' in metadict[fname]: specdict[SPECTRUM.U_FLUXES] = metadict[fname]['yunit'] specdict[SPECTRUM.U_ERRORS] = metadict[fname]['yunit'] else: if max([float(x) for x in fluxes]) < 1.0e-5: fluxunit = 'erg/s/cm^2/Angstrom' else: fluxunit = 'Uncalibrated' specdict[SPECTRUM.U_FLUXES] = fluxunit specdict[SPECTRUM.U_ERRORS] = fluxunit catalog.entries[name].add_spectrum(**specdict) donationscnt = donationscnt + 1 if (catalog.args.travis and donationscnt % catalog.TRAVIS_QUERY_LIMIT == 0): break catalog.journal_entries() return
def radec_clean(svalue, quantity, unit=''): if unit == 'floatdegrees': if not is_number(svalue): return (svalue, unit) deg = float('%g' % Decimal(svalue)) sig = get_sig_digits(svalue) if 'ra' in quantity: flhours = deg / 360.0 * 24.0 hours = floor(flhours) minutes = floor((flhours - hours) * 60.0) seconds = (flhours * 60.0 - (hours * 60.0 + minutes)) * 60.0 hours = 0 if hours < 1.e-6 else hours minutes = 0 if minutes < 1.e-6 else minutes seconds = 0.0 if seconds < 1.e-6 else seconds if seconds > 60.0: raise(ValueError('Invalid seconds value for ' + quantity)) svalue = str(hours).zfill(2) + ':' + str(minutes).zfill(2) + \ ':' + zpad(pretty_num(seconds, sig=sig - 1)) elif 'dec' in quantity: fldeg = abs(deg) degree = floor(fldeg) minutes = floor((fldeg - degree) * 60.0) seconds = (fldeg * 60.0 - (degree * 60.0 + minutes)) * 60.0 if seconds > 60.0: raise(ValueError('Invalid seconds value for ' + quantity)) svalue = (('+' if deg >= 0.0 else '-') + str(degree).strip('+-').zfill(2) + ':' + str(minutes).zfill(2) + ':' + zpad(pretty_num(seconds, sig=sig - 1))) elif unit == 'nospace' and 'ra' in quantity: svalue = svalue[:2] + ':' + svalue[2:4] + \ ((':' + zpad(svalue[4:])) if len(svalue) > 4 else '') elif unit == 'nospace' and 'dec' in quantity: if svalue.startswith(('+', '-')): svalue = svalue[:3] + ':' + svalue[3:5] + \ ((':' + zpad(svalue[5:])) if len(svalue) > 5 else '') else: svalue = '+' + svalue[:2] + ':' + svalue[2:4] + \ ((':' + zpad(svalue[4:])) if len(svalue) > 4 else '') else: svalue = svalue.replace(' ', ':') if 'dec' in quantity: valuesplit = svalue.split(':') svalue = (('-' if valuesplit[0].startswith('-') else '+') + valuesplit[0].strip('+-').zfill(2) + (':' + valuesplit[1].zfill(2) if len(valuesplit) > 1 else '') + (':' + zpad(valuesplit[2]) if len(valuesplit) > 2 else '')) if 'ra' in quantity: sunit = 'hours' elif 'dec' in quantity: sunit = 'degrees' # Correct case of arcseconds = 60.0. valuesplit = svalue.split(':') if len(valuesplit) == 3 and valuesplit[-1] in ["60.0", "60.", "60"]: svalue = valuesplit[0] + ':' + str(Decimal(valuesplit[1]) + Decimal(1.0)) + ':' + "00.0" # Strip trailing dots. svalue = svalue.rstrip('.') return (svalue, sunit)
def do_tns(catalog): """Load TNS metadata.""" session = requests.Session() task_str = catalog.get_current_task_str() tns_url = 'https://www.wis-tns.org/' search_url = tns_url + \ 'search?&num_page=1&format=html&sort=desc&order=id&format=csv&page=0' csvtxt = catalog.load_url(search_url, os.path.join(catalog.get_current_task_repo(), 'TNS', 'index.csv')) if not csvtxt: return maxid = csvtxt.splitlines()[1].split(',')[0].strip('"') maxpages = ceil(int(maxid) / 1000.) for page in pbar(range(maxpages), task_str): fname = os.path.join( catalog.get_current_task_repo(), 'TNS', 'page-') + str(page).zfill(2) + '.csv' if (catalog.current_task.load_archive(catalog.args) and os.path.isfile(fname) and page < 7): with open(fname, 'r') as tns_file: csvtxt = tns_file.read() else: session = requests.Session() ses_url = (tns_url + 'search?&num_page=1000&format=html&edit' '[type]=&edit[objname]=&edit[id]=&sort=asc&order=id' '&display[redshift]=1' '&display[hostname]=1&display[host_redshift]=1' '&display[source_group_name]=1' '&display[programs_name]=1' '&display[internal_name]=1' '&display[isTNS_AT]=1' '&display[public]=1' '&display[end_pop_period]=0' '&display[spectra_count]=1' '&display[discoverymag]=1&display[discmagfilter]=1' '&display[discoverydate]=1&display[discoverer]=1' '&display[sources]=1' '&display[bibcode]=1&format=csv&page=' + str(page)) try: response = session.get(ses_url, timeout=40) csvtxt = response.text except Exception as e: catalog.log.warning( 'Could not download TNS page #{}.'.format(str(page))) if os.path.isfile(fname): with open(fname, 'r') as tns_file: csvtxt = tns_file.read() else: continue else: with open(fname, 'w') as tns_file: tns_file.write(csvtxt) tsvin = list(csv.reader([x.replace('[data validation error, please contact site admin]', '""') for x in csvtxt.splitlines()], delimiter=',')) for ri, row in enumerate(pbar(tsvin, task_str, leave=False)): if ri == 0: continue if row[4] and 'SN' not in row[4]: continue name = row[1].replace(' ', '') if len(name) < 5: continue name, source = catalog.new_entry( name, srcname='Transient Name Server', url=tns_url) if row[2] and row[2] != '00:00:00.00': catalog.entries[name].add_quantity(SUPERNOVA.RA, row[2], source) if row[3] and row[3] != '+00:00:00.00': catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[3], source) if row[4]: catalog.entries[name].add_quantity( SUPERNOVA.CLAIMED_TYPE, row[4].replace('SN', '').strip(), source) if row[5]: catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, row[5], source, kind='spectroscopic') if row[6]: catalog.entries[name].add_quantity(SUPERNOVA.HOST, row[6], source) if row[7]: catalog.entries[name].add_quantity( [SUPERNOVA.REDSHIFT, SUPERNOVA.HOST_REDSHIFT], row[7], source, kind='host') if row[8]: catalog.entries[name].add_quantity(SUPERNOVA.DISCOVERER, row[8], source) # Currently, all events listing all possible observers. TNS bug? # if row[9]: # observers = row[9].split(',') # for observer in observers: # catalog.entries[name].add_quantity('observer', # observer.strip(), # source) if row[12]: catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, row[12], source) if row[20]: date = row[20].split()[0].replace('-', '/') if 'clear' in row[20].lower(): print(row) if date != '0000/00/00' and 'clear' not in row[20].lower(): date = date.replace('/00', '') dsplit = row[20].split() if len(dsplit) >= 2: t = dsplit[1] if t != '00:00:00': ts = t.split(':') dt = timedelta( hours=int(ts[0]), minutes=int(ts[1]), seconds=float(ts[2])) date += pretty_num( dt.total_seconds() / (24 * 60 * 60), sig=6).lstrip('0') catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, date, source) if catalog.args.travis and ri >= catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() catalog.journal_entries()
def do_snf_specta(catalog): task_str = catalog.get_current_task_str() bibcodes = {'SN2005gj': '2006ApJ...650..510A', 'SN2006D': '2007ApJ...654L..53T', 'SN2007if': '2010ApJ...713.1073S', 'SN2011fe': '2013A&A...554A..27P'} oldname = '' snfcnt = 0 eventfolders = next(os.walk(os.path.join( catalog.get_current_task_repo(), 'SNFactory')))[1] for eventfolder in pbar(eventfolders, task_str): oname = eventfolder name = catalog.get_preferred_name(oname) if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) sec_reference = 'Nearby Supernova Factory' sec_refurl = 'http://snfactory.lbl.gov/' sec_bibcode = '2002SPIE.4836...61A' sec_source = catalog.entries[name].add_source( name=sec_reference, url=sec_refurl, bibcode=sec_bibcode, secondary=True) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, oname, sec_source) bibcode = bibcodes[oname] source = catalog.entries[name].add_source(bibcode=bibcode) sources = uniq_cdl([source, sec_source]) use_path = os.path.join( catalog.get_current_task_repo(), 'SNFactory', eventfolder, '*.dat') eventspectra = glob(use_path) for spectrum in pbar(eventspectra, task_str): filename = os.path.basename(spectrum) with open(spectrum) as spec_file: specdata = list(csv.reader( spec_file, delimiter=' ', skipinitialspace=True)) specdata = list(filter(None, specdata)) newspec = [] time = '' telescope = '' instrument = '' observer = '' observatory = '' if 'Keck_20060202_R' in spectrum: time = '53768.23469' elif 'Spectrum05_276' in spectrum: time = pretty_num(astrotime('2005-10-03').mjd, sig=5) elif 'Spectrum05_329' in spectrum: time = pretty_num(astrotime('2005-11-25').mjd, sig=5) elif 'Spectrum05_336' in spectrum: time = pretty_num(astrotime('2005-12-02').mjd, sig=5) for row in specdata: if row[0][0] == '#': joinrow = (' '.join(row)).split('=') if len(joinrow) < 2: continue field = joinrow[0].strip('# ') value = joinrow[1].split('/')[0].strip('\' ') if not time: if field == 'JD': time = str(jd_to_mjd(Decimal(value))) elif field == 'MJD': time = value elif field == 'MJD-OBS': time = value if field == 'OBSERVER': observer = value.capitalize() if field == 'OBSERVAT': observatory = value.capitalize() if field == 'TELESCOP': telescope = value.capitalize() if field == 'INSTRUME': instrument = value.capitalize() else: newspec.append(row) if not time: raise ValueError('Time missing from spectrum.') specdata = newspec haserrors = len(specdata[0]) == 3 and specdata[ 0][2] and specdata[0][2] != 'NaN' specdata = [list(i) for i in zip(*specdata)] wavelengths = specdata[0] fluxes = specdata[1] errors = '' if haserrors: errors = specdata[2] unit_err = ('Variance' if oldname == 'SN2011fe' else 'erg/s/cm^2/Angstrom') unit_flx = 'erg/s/cm^2/Angstrom' catalog.entries[name].add_spectrum( u_wavelengths='Angstrom', u_fluxes=unit_flx, u_time='MJD', time=time, wavelengths=wavelengths, fluxes=fluxes, errors=errors, observer=observer, observatory=observatory, telescope=telescope, instrument=instrument, u_errors=unit_err, source=sources, filename=filename) snfcnt = snfcnt + 1 if (catalog.args.travis and snfcnt % catalog.TRAVIS_QUERY_LIMIT == 0): break catalog.journal_entries() return
def do_cleanup(catalog): """Cleanup catalog after importing all data.""" task_str = catalog.get_current_task_str() # Set preferred names, calculate some columns based on imported data, # sanitize some fields keys = list(catalog.entries.keys()) cleanupcnt = 0 for oname in pbar(keys, task_str): # Some events may be merged in cleanup process, skip them if # non-existent. try: name = catalog.add_entry(oname) except Exception: catalog.log.warning( '"{}" was not found, suggests merge occurred in cleanup ' 'process.'.format(oname)) continue # Set the preferred name, switching to that name if name changed. name = catalog.entries[name].set_preferred_name() aliases = catalog.entries[name].get_aliases() catalog.entries[name].set_first_max_light() # Clean discoverer field if FASTSTARS.DISCOVERER in catalog.entries[name]: if len(catalog.entries[name][FASTSTARS.DISCOVERER]) > 1: POSSIBLEDISCOVERER = [ catalog.entries[name][FASTSTARS.DISCOVERER][i]['value'] for i in range( len(catalog.entries[name][FASTSTARS.DISCOVERER])) ] POSSIBLEDISCOVERER_DATE = [ int(DATE['value']) for DATE in catalog.entries[name][FASTSTARS.DISCOVER_DATE] ] POSSIBLEDISCOVERER_DATE_SOURCES = [ DATE['source'] for DATE in catalog.entries[name][FASTSTARS.DISCOVER_DATE] ] EARLIESTSOURCE = POSSIBLEDISCOVERER_DATE_SOURCES[np.argmin( POSSIBLEDISCOVERER_DATE)] EARLIESTDISCOVER_DATE = catalog.entries[name][ FASTSTARS.DISCOVER_DATE][np.argmin( POSSIBLEDISCOVERER_DATE)] # Deal with case where a star was 'discovered' multiple times in one year if ',' in EARLIESTSOURCE: EARLIESTSOURCE = EARLIESTSOURCE.split(',')[0] for DISCOVERER in catalog.entries[name][FASTSTARS.DISCOVERER]: for DISCOVERERSOURCE in DISCOVERER['source'].split(','): if DISCOVERERSOURCE == EARLIESTSOURCE: EARLIESTDISCOVERER = DISCOVERER for DISCOVERER in catalog.entries[name][FASTSTARS.DISCOVERER]: for DISCOVERERSOURCE in DISCOVERER['source'].split(','): if DISCOVERERSOURCE == EARLIESTSOURCE: EARLIESTDISCOVERER = DISCOVERER catalog.entries[name][FASTSTARS.DISCOVERER] = [ EARLIESTDISCOVERER ] catalog.entries[name][FASTSTARS.DISCOVER_DATE] = [ EARLIESTDISCOVER_DATE ] # Convert all distances to kpc. if FASTSTARS.LUM_DIST in catalog.entries[name]: for li, ld in enumerate(catalog.entries[name][FASTSTARS.LUM_DIST]): if ld.get('u_value') != 'kpc': if ld.get('u_value') == 'pc': catalog.entries[name][ FASTSTARS.LUM_DIST][li]['value'] = str( Decimal(catalog.entries[name][ FASTSTARS.LUM_DIST][li]['value']) * Decimal('0.001')) elif ld.get('u_value') == 'Mpc': catalog.entries[name][ FASTSTARS.LUM_DIST][li]['value'] = str( Decimal(catalog.entries[name][ FASTSTARS.LUM_DIST][li]['value']) * Decimal('1000')) else: raise ValueError('unknown distance unit') catalog.entries[name][ FASTSTARS.LUM_DIST][li]['u_value'] = 'kpc' if (FASTSTARS.RA not in catalog.entries[name] or FASTSTARS.DEC not in catalog.entries[name]): prefixes = ['SDSS'] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:6])): noprefix = alias.split(':')[-1].replace(prefix, '').replace( '.', '') decsign = '+' if '+' in noprefix else '-' noprefix = noprefix.replace('+', '|').replace('-', '|') nops = noprefix.split('|') if len(nops) < 2: continue rastr = nops[0] decstr = nops[1] ra = ':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + \ ('.' + rastr[6:] if len(rastr) > 6 else '') dec = ( decsign + ':'.join([decstr[:2], decstr[2:4], decstr[4:6]]) + ('.' + decstr[6:] if len(decstr) > 6 else '')) if catalog.args.verbose: tprint('Added ra/dec from name: ' + ra + ' ' + dec) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity(FASTSTARS.RA, ra, source, derived=True) catalog.entries[name].add_quantity(FASTSTARS.DEC, dec, source, derived=True) break if FASTSTARS.RA in catalog.entries[name]: break if (FASTSTARS.MAX_ABS_MAG not in catalog.entries[name] and FASTSTARS.MAX_APP_MAG in catalog.entries[name] and FASTSTARS.LUM_DIST in catalog.entries[name]): # Find the "best" distance to use for this bestsig = 0 for ld in catalog.entries[name][FASTSTARS.LUM_DIST]: sig = get_sig_digits(ld[QUANTITY.VALUE]) if sig > bestsig: bestld = ld[QUANTITY.VALUE] bestsrc = ld[QUANTITY.SOURCE] bestsig = sig if bestsig > 0 and is_number(bestld) and float(bestld) > 0.: source = catalog.entries[name].add_self_source() sources = uniq_cdl([source] + bestsrc.split(',')) bestldz = z_at_value(cosmo.luminosity_distance, float(bestld) * un.Mpc) pnum = (float(catalog.entries[name][FASTSTARS.MAX_APP_MAG][0][ QUANTITY.VALUE]) - 5.0 * (log10(float(bestld) * 1.0e6) - 1.0) + 2.5 * log10(1.0 + bestldz)) pnum = pretty_num(pnum, sig=bestsig + 1) catalog.entries[name].add_quantity(FASTSTARS.MAX_ABS_MAG, pnum, sources, derived=True) catalog.entries[name].sanitize() catalog.journal_entries(bury=True, final=True, gz=True) cleanupcnt = cleanupcnt + 1 if catalog.args.travis and cleanupcnt % 1000 == 0: break catalog.save_caches() return