def do_snls_photo(catalog): task_str = catalog.get_current_task_str() snls_path = os.path.join(catalog.get_current_task_repo(), 'SNLS-ugriz.dat') data = list(csv.reader(open(snls_path, 'r'), delimiter=' ', quotechar='"', skipinitialspace=True)) for row in pbar(data, task_str): flux = row[3] err = row[4] # Being extra strict here with the flux constraint, see note below. if float(flux) < 3.0 * float(err): continue name = 'SNLS-' + row[0] name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2010A&A...523A...7G') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) band = row[1] mjd = row[2] sig = get_sig_digits(flux.split('E')[0]) + 1 # Conversion comes from SNLS-Readme # NOTE: Datafiles avail for download suggest diff zeropoints than 30, # need to inquire. magnitude = pretty_num(30.0 - 2.5 * log10(float(flux)), sig=sig) e_mag = pretty_num( 2.5 * log10(1.0 + float(err) / float(flux)), sig=sig) # e_mag = pretty_num(2.5*(log10(float(flux) + float(err)) - # log10(float(flux))), sig=sig) catalog.entries[name].add_photometry( time=mjd, band=band, magnitude=magnitude, e_magnitude=e_mag, counts=flux, e_counts=err, source=source) catalog.journal_entries() return
def get_best_redshift(self): bestsig = -1 bestkind = 10 for z in self['redshift']: kind = PREF_KINDS.index(z['kind'] if 'kind' in z else '') sig = get_sig_digits(z[QUANTITY.VALUE]) if sig > bestsig and kind <= bestkind: bestz = z[QUANTITY.VALUE] bestkind = kind bestsig = sig bestsrc = z['source'] return bestz, bestkind, bestsig, bestsrc
def set_pd_mag_from_flux_density(photodict, fd='', efd='', lefd='', uefd='', sig=DEFAULT_UL_SIGMA): """Set photometry dictionary from a flux density measurement. `fd` is assumed to be in microjanskys. """ with localcontext() as ctx: if lefd == '' or uefd == '': lefd = efd uefd = efd prec = max( get_sig_digits(str(fd), strip_zeroes=False), get_sig_digits(str(lefd), strip_zeroes=False), get_sig_digits(str(uefd), strip_zeroes=False)) + 1 ctx.prec = prec dlefd = Decimal(str(lefd)) duefd = Decimal(str(uefd)) if fd != '': dfd = Decimal(str(fd)) dsig = Decimal(str(sig)) if fd == '' or float(fd) < DEFAULT_UL_SIGMA * float(uefd): photodict[PHOTOMETRY.UPPER_LIMIT] = True photodict[PHOTOMETRY.UPPER_LIMIT_SIGMA] = str(sig) photodict[PHOTOMETRY.MAGNITUDE] = str(Decimal('23.9') - D25 * ( dsig * duefd).log10()) if fd: photodict[PHOTOMETRY.E_UPPER_MAGNITUDE] = str(D25 * ( (dfd + duefd).log10() - dfd.log10())) else: photodict[PHOTOMETRY.MAGNITUDE] = str(Decimal('23.9') - D25 * dfd.log10()) photodict[PHOTOMETRY.E_UPPER_MAGNITUDE] = str(D25 * ( (dfd + duefd).log10() - dfd.log10())) photodict[PHOTOMETRY.E_LOWER_MAGNITUDE] = str(D25 * ( dfd.log10() - (dfd - dlefd).log10()))
def set_pd_mag_from_counts(photodict, c='', ec='', lec='', uec='', zp=DEFAULT_ZP, sig=DEFAULT_UL_SIGMA): """Set photometry dictionary from a counts measurement.""" with localcontext() as ctx: if lec == '' or uec == '': lec = ec uec = ec prec = max(get_sig_digits(str(c), strip_zeroes=False), get_sig_digits(str(lec), strip_zeroes=False), get_sig_digits(str(uec), strip_zeroes=False)) + 1 ctx.prec = prec dlec = Decimal(str(lec)) duec = Decimal(str(uec)) if c != '': dc = Decimal(str(c)) dzp = Decimal(str(zp)) dsig = Decimal(str(sig)) photodict[PHOTOMETRY.ZERO_POINT] = str(zp) if c == '' or float(c) < DEFAULT_UL_SIGMA * float(uec): photodict[PHOTOMETRY.UPPER_LIMIT] = True photodict[PHOTOMETRY.UPPER_LIMIT_SIGMA] = str(sig) photodict[PHOTOMETRY.MAGNITUDE] = str(dzp - (D25 * (dsig * duec).log10())) dnec = Decimal('10.0')**( (dzp - Decimal(photodict[PHOTOMETRY.MAGNITUDE])) / D25) photodict[PHOTOMETRY.E_UPPER_MAGNITUDE] = str( D25 * ((dnec + duec).log10() - dnec.log10())) else: photodict[PHOTOMETRY.MAGNITUDE] = str(dzp - D25 * dc.log10()) photodict[PHOTOMETRY.E_UPPER_MAGNITUDE] = str( D25 * ((dc + duec).log10() - dc.log10())) photodict[PHOTOMETRY.E_LOWER_MAGNITUDE] = str( D25 * (dc.log10() - (dc - dlec).log10()))
def set_pd_mag_from_counts(photodict, c, ec='', lec='', uec='', zp=DEFAULT_ZP, sig=DEFAULT_UL_SIGMA): with localcontext() as ctx: if lec == '' or uec == '': lec = ec uec = ec prec = max( get_sig_digits(str(c)), get_sig_digits(str(lec)), get_sig_digits(str(uec))) ctx.prec = prec dlec = Decimal(str(lec)) duec = Decimal(str(uec)) dc = Decimal(str(c)) dzp = Decimal(str(zp)) dsig = Decimal(str(sig)) photodict[PHOTOMETRY.ZERO_POINT] = str(zp) if float(c) < DEFAULT_UL_SIGMA * float(uec): photodict[PHOTOMETRY.UPPER_LIMIT] = True photodict[PHOTOMETRY.UPPER_LIMIT_SIGMA] = str(sig) photodict[PHOTOMETRY.MAGNITUDE] = str(dzp - (D25 * (dsig * duec ).log10())) dnec = Decimal('10.0')**( (dzp - Decimal(photodict[PHOTOMETRY.MAGNITUDE])) / D25) photodict[PHOTOMETRY.E_UPPER_MAGNITUDE] = str(D25 * ( (dnec + duec).log10() - dnec.log10())) else: photodict[PHOTOMETRY.MAGNITUDE] = str(dzp - D25 * dc.log10()) photodict[PHOTOMETRY.E_UPPER_MAGNITUDE] = str(D25 * ( (dc + duec).log10() - dc.log10())) photodict[PHOTOMETRY.E_LOWER_MAGNITUDE] = str(D25 * ( dc.log10() - (dc - dlec).log10()))
def set_pd_mag_from_counts(photodict, c, ec='', lec='', uec='', zp=DEFAULT_ZP, sig=DEFAULT_UL_SIGMA): with localcontext() as ctx: if lec == '' or uec == '': lec = ec uec = ec prec = max(get_sig_digits(str(c)), get_sig_digits(str(lec)), get_sig_digits(str(uec))) ctx.prec = prec dlec = Decimal(str(lec)) duec = Decimal(str(uec)) dc = Decimal(str(c)) dzp = Decimal(str(zp)) dsig = Decimal(str(sig)) photodict[PHOTOMETRY.ZERO_POINT] = str(zp) if float(c) < DEFAULT_UL_SIGMA * float(uec): photodict[PHOTOMETRY.UPPER_LIMIT] = True photodict[PHOTOMETRY.UPPER_LIMIT_SIGMA] = str(sig) photodict[PHOTOMETRY.MAGNITUDE] = str(dzp - (D25 * (dsig * duec).log10())) dnec = Decimal('10.0')**( (dzp - Decimal(photodict[PHOTOMETRY.MAGNITUDE])) / D25) photodict[PHOTOMETRY.E_UPPER_MAGNITUDE] = str( D25 * ((dnec + duec).log10() - dnec.log10())) else: photodict[PHOTOMETRY.MAGNITUDE] = str(dzp - D25 * dc.log10()) photodict[PHOTOMETRY.E_UPPER_MAGNITUDE] = str( D25 * ((dc + duec).log10() - dc.log10())) photodict[PHOTOMETRY.E_LOWER_MAGNITUDE] = str( D25 * (dc.log10() - (dc - dlec).log10()))
def get_best_redshift(self, key=SUPERNOVA.REDSHIFT): bestsig = -1 bestkind = None for z in self[key]: try: kind = min([ z.kind_preference.index(x) for x in listify(z.get(QUANTITY.KIND, [])) ]) except: kind = None sig = get_sig_digits(z[QUANTITY.VALUE]) if (sig > bestsig and ((kind is None and bestkind is None) or kind <= bestkind)): bestz = z[QUANTITY.VALUE] bestkind = kind bestsig = sig bestsrc = z[QUANTITY.SOURCE] return bestz, bestkind, bestsig, bestsrc
def get_best_redshift(self, key=SUPERNOVA.REDSHIFT): bestsig = -1 bestkind = None for z in self[key]: try: kind = min([ z.kind_preference.index(x) for x in listify(z.get(QUANTITY.KIND, [])) ]) except: kind = None sig = get_sig_digits(z[QUANTITY.VALUE]) if (sig > bestsig and ((kind is None and bestkind is None) or kind <= bestkind)): bestz = z[QUANTITY.VALUE] bestkind = kind bestsig = sig bestsrc = z[QUANTITY.SOURCE] return bestz, bestkind, bestsig, bestsrc
def do_snls_spectra(catalog): """ """ task_str = catalog.get_current_task_str() result = Vizier.get_catalogs('J/A+A/507/85/table1') table = result[list(result.keys())[0]] table.convert_bytestring_to_unicode() datedict = {} for row in table: datedict['SNLS-' + row['SN']] = str(astrotime(row['Date']).mjd) oldname = '' file_names = glob(os.path.join(catalog.get_current_task_repo(), 'SNLS/*')) for fi, fname in enumerate(pbar_strings(file_names, task_str)): filename = os.path.basename(fname) fileparts = filename.split('_') name = 'SNLS-' + fileparts[1] name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2009A&A...507...85B') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, '20' + fileparts[1][:2], source) f = open(fname, 'r') data = csv.reader(f, delimiter=' ', skipinitialspace=True) specdata = [] for r, row in enumerate(data): if row[0] == '@TELESCOPE': telescope = row[1].strip() elif row[0] == '@REDSHIFT': catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, row[1].strip(), source) if r < 14: continue specdata.append(list(filter(None, [x.strip(' \t') for x in row]))) specdata = [list(i) for i in zip(*specdata)] wavelengths = specdata[1] fluxes = [ pretty_num(float(x) * 1.e-16, sig=get_sig_digits(x)) for x in specdata[2] ] # FIX: this isnt being used errors = [ pretty_num(float(x) * 1.e-16, sig=get_sig_digits(x)) for x in specdata[3] ] fluxunit = 'erg/s/cm^2/Angstrom' specdict = { SPECTRUM.WAVELENGTHS: wavelengths, SPECTRUM.FLUXES: fluxes, SPECTRUM.ERRORS: errors, SPECTRUM.U_WAVELENGTHS: 'Angstrom', SPECTRUM.U_FLUXES: fluxunit, SPECTRUM.U_ERRORS: fluxunit, SPECTRUM.TELESCOPE: telescope, SPECTRUM.FILENAME: filename, SPECTRUM.SOURCE: source } if name in datedict: specdict[SPECTRUM.TIME] = datedict[name] specdict[SPECTRUM.U_TIME] = 'MJD' catalog.entries[name].add_spectrum(**specdict) if catalog.args.travis and fi >= catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() return
def do_nedd(catalog): task_str = catalog.get_current_task_str() nedd_path = os.path.join( catalog.get_current_task_repo(), 'NED26.05.1-D-12.1.0-20160501.csv') f = open(nedd_path, 'r') data = sorted(list(csv.reader(f, delimiter=',', quotechar='"'))[ 13:], key=lambda x: (x[9], x[3])) reference = "NED-D" refurl = "http://ned.ipac.caltech.edu/Library/Distances/" nedd_dict = OrderedDict() olddistname = '' for r, row in enumerate(pbar(data, task_str)): if r <= 12: continue distname = row[3] name = name_clean(distname) # distmod = row[4] # moderr = row[5] dist = row[6] bibcode = unescape(row[8]) snname = name_clean(row[9]) redshift = row[10] cleanhost = '' if name != snname and (name + ' HOST' != snname): cleanhost = host_clean(distname) if cleanhost.endswith(' HOST'): cleanhost = '' if not is_number(dist): print(dist) if dist: nedd_dict.setdefault(cleanhost, []).append(Decimal(dist)) if snname and 'HOST' not in snname: snname, secondarysource = catalog.new_entry( snname, srcname=reference, url=refurl, secondary=True) if bibcode: source = catalog.entries[snname].add_source(bibcode=bibcode) sources = uniq_cdl([source, secondarysource]) else: sources = secondarysource if name == snname: if redshift: catalog.entries[snname].add_quantity( 'redshift', redshift, sources) if dist: catalog.entries[snname].add_quantity( 'comovingdist', dist, sources) if not redshift: try: zatval = z_at_value(cosmo.comoving_distance, float(dist) * un.Mpc, zmax=5.0) sigd = get_sig_digits(str(dist)) redshift = pretty_num(zatval, sig=sigd) except (KeyboardInterrupt, SystemExit): raise except: pass else: cosmosource = catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') combsources = uniq_cdl(sources.split(',') + [cosmosource]) catalog.entries[snname].add_quantity('redshift', redshift, combsources) if cleanhost: catalog.entries[snname].add_quantity( 'host', cleanhost, sources) if catalog.args.update and olddistname != distname: catalog.journal_entries() olddistname = distname catalog.journal_entries() f.close() return
def do_cleanup(catalog): """Task to cleanup catalog before final write.""" task_str = catalog.get_current_task_str() # Set preferred names, calculate some columns based on imported data, # sanitize some fields keys = catalog.entries.copy().keys() cleanupcnt = 0 for oname in pbar(keys, task_str): name = catalog.add_entry(oname) # Set the preferred name, switching to that name if name changed. name = catalog.entries[name].set_preferred_name() aliases = catalog.entries[name].get_aliases() catalog.entries[name].set_first_max_light() if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['MLS', 'SSS', 'CSS', 'GRB '] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:2])): discoverdate = ('/'.join([ '20' + alias.replace(prefix, '')[:2], alias.replace(prefix, '')[2:4], alias.replace(prefix, '')[4:6] ])) if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( TIDALDISRUPTION.DISCOVER_DATE, discoverdate, source, derived=True) break if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]: break if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]: prefixes = [ 'ASASSN-', 'PS1-', 'PS1', 'PS', 'iPTF', 'PTF', 'SCP-', 'SNLS-', 'SPIRITS', 'LSQ', 'DES', 'SNHiTS', 'Gaia', 'GND', 'GNW', 'GSD', 'GSW', 'EGS', 'COS', 'OGLE', 'HST' ] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:2]) and is_number(alias.replace(prefix, '')[:1])): discoverdate = '20' + alias.replace(prefix, '')[:2] if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( TIDALDISRUPTION.DISCOVER_DATE, discoverdate, source, derived=True) break if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]: break if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['SNF'] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:4])): discoverdate = ('/'.join([ alias.replace(prefix, '')[:4], alias.replace(prefix, '')[4:6], alias.replace(prefix, '')[6:8] ])) if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( TIDALDISRUPTION.DISCOVER_DATE, discoverdate, source, derived=True) break if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]: break if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['PTFS', 'SNSDF'] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:2])): discoverdate = ('/'.join([ '20' + alias.replace(prefix, '')[:2], alias.replace(prefix, '')[2:4] ])) if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( TIDALDISRUPTION.DISCOVER_DATE, discoverdate, source, derived=True) break if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]: break if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['AT', 'SN', 'OGLE-', 'SM ', 'KSN-'] for alias in aliases: for prefix in prefixes: if alias.startswith(prefix): year = re.findall(r'\d+', alias) if len(year) == 1: year = year[0] else: continue if alias.replace(prefix, '').index(year) != 0: continue if (year and is_number(year) and '.' not in year and len(year) <= 4): discoverdate = year if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( TIDALDISRUPTION.DISCOVER_DATE, discoverdate, source, derived=True) break if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]: break if (TIDALDISRUPTION.RA not in catalog.entries[name] or TIDALDISRUPTION.DEC not in catalog.entries[name]): prefixes = [ 'PSN J', 'MASJ', 'CSS', 'SSS', 'MASTER OT J', 'HST J', 'TCP J', 'MACS J', '2MASS J', 'EQ J', 'CRTS J', 'SMT J' ] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:6])): noprefix = alias.split(':')[-1].replace( prefix, '').replace('.', '') decsign = '+' if '+' in noprefix else '-' noprefix = noprefix.replace('+', '|').replace('-', '|') nops = noprefix.split('|') if len(nops) < 2: continue rastr = nops[0] decstr = nops[1] ra = ':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + \ ('.' + rastr[6:] if len(rastr) > 6 else '') dec = (decsign + ':'.join( [decstr[:2], decstr[2:4], decstr[4:6]]) + ('.' + decstr[6:] if len(decstr) > 6 else '')) if catalog.args.verbose: tprint('Added ra/dec from name: ' + ra + ' ' + dec) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( TIDALDISRUPTION.RA, ra, source, derived=True) catalog.entries[name].add_quantity( TIDALDISRUPTION.DEC, dec, source, derived=True) break if TIDALDISRUPTION.RA in catalog.entries[name]: break no_host = (TIDALDISRUPTION.HOST not in catalog.entries[name] or not any([ x[QUANTITY.VALUE] == 'Milky Way' for x in catalog.entries[name][TIDALDISRUPTION.HOST] ])) if (TIDALDISRUPTION.RA in catalog.entries[name] and TIDALDISRUPTION.DEC in catalog.entries[name] and no_host): from astroquery.irsa_dust import IrsaDust if name not in catalog.extinctions_dict: try: ra_dec = (catalog.entries[name][TIDALDISRUPTION.RA][0][ QUANTITY.VALUE] + " " + catalog.entries[name][ TIDALDISRUPTION.DEC][0][QUANTITY.VALUE]) result = IrsaDust.get_query_table(ra_dec, section='ebv') except (KeyboardInterrupt, SystemExit): raise except Exception: warnings.warn("Coordinate lookup for " + name + " failed in IRSA.") else: ebv = result['ext SandF mean'][0] ebverr = result['ext SandF std'][0] catalog.extinctions_dict[name] = [ebv, ebverr] if name in catalog.extinctions_dict: sources = uniq_cdl([ catalog.entries[name].add_self_source(), catalog.entries[name] .add_source(bibcode='2011ApJ...737..103S') ]) (catalog.entries[name].add_quantity( TIDALDISRUPTION.EBV, str(catalog.extinctions_dict[name][0]), sources, e_value=str(catalog.extinctions_dict[name][1]), derived=True)) if ((TIDALDISRUPTION.HOST in catalog.entries[name] and (TIDALDISRUPTION.HOST_RA not in catalog.entries[name] or TIDALDISRUPTION.HOST_DEC not in catalog.entries[name]))): for host in catalog.entries[name][TIDALDISRUPTION.HOST]: alias = host[QUANTITY.VALUE] if ' J' in alias and is_number(alias.split(' J')[-1][:6]): noprefix = alias.split(' J')[-1].split(':')[-1].replace( '.', '') decsign = '+' if '+' in noprefix else '-' noprefix = noprefix.replace('+', '|').replace('-', '|') nops = noprefix.split('|') if len(nops) < 2: continue rastr = nops[0] decstr = nops[1] hostra = (':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + ('.' + rastr[6:] if len(rastr) > 6 else '')) hostdec = decsign + ':'.join([ decstr[:2], decstr[2:4], decstr[4:6] ]) + ('.' + decstr[6:] if len(decstr) > 6 else '') if catalog.args.verbose: tprint('Added hostra/hostdec from name: ' + hostra + ' ' + hostdec) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( TIDALDISRUPTION.HOST_RA, hostra, source, derived=True) catalog.entries[name].add_quantity( TIDALDISRUPTION.HOST_DEC, hostdec, source, derived=True) break if TIDALDISRUPTION.HOST_RA in catalog.entries[name]: break if (TIDALDISRUPTION.REDSHIFT not in catalog.entries[name] and TIDALDISRUPTION.VELOCITY in catalog.entries[name]): # Find the "best" velocity to use for this bestsig = 0 for hv in catalog.entries[name][TIDALDISRUPTION.VELOCITY]: sig = get_sig_digits(hv[QUANTITY.VALUE]) if sig > bestsig: besthv = hv[QUANTITY.VALUE] bestsrc = hv['source'] bestsig = sig if bestsig > 0 and is_number(besthv): voc = float(besthv) * 1.e5 / CLIGHT source = catalog.entries[name].add_self_source() sources = uniq_cdl([source] + bestsrc.split(',')) (catalog.entries[name].add_quantity( TIDALDISRUPTION.REDSHIFT, pretty_num( sqrt((1. + voc) / (1. - voc)) - 1., sig=bestsig), sources, kind='heliocentric', derived=True)) if (TIDALDISRUPTION.REDSHIFT not in catalog.entries[name] and len(catalog.nedd_dict) > 0 and TIDALDISRUPTION.HOST in catalog.entries[name]): reference = "NED-D" refurl = "http://ned.ipac.caltech.edu/Library/Distances/" for host in catalog.entries[name][TIDALDISRUPTION.HOST]: if host[QUANTITY.VALUE] in catalog.nedd_dict: source = catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') secondarysource = catalog.entries[name].add_source( name=reference, url=refurl, secondary=True) meddist = statistics.median(catalog.nedd_dict[host[ QUANTITY.VALUE]]) redz = z_at_value(cosmo.comoving_distance, float(meddist) * un.Mpc) redshift = pretty_num( redz, sig=get_sig_digits(str(meddist))) catalog.entries[name].add_quantity( TIDALDISRUPTION.REDSHIFT, redshift, uniq_cdl([source, secondarysource]), kind='host', derived=True) if (TIDALDISRUPTION.MAX_ABS_MAG not in catalog.entries[name] and TIDALDISRUPTION.MAX_APP_MAG in catalog.entries[name] and TIDALDISRUPTION.LUM_DIST in catalog.entries[name]): # Find the "best" distance to use for this bestsig = 0 for ld in catalog.entries[name][TIDALDISRUPTION.LUM_DIST]: sig = get_sig_digits(ld[QUANTITY.VALUE]) if sig > bestsig: bestld = ld[QUANTITY.VALUE] bestsrc = ld['source'] bestsig = sig if bestsig > 0 and is_number(bestld) and float(bestld) > 0.: source = catalog.entries[name].add_self_source() sources = uniq_cdl([source] + bestsrc.split(',')) bestldz = z_at_value(cosmo.luminosity_distance, float(bestld) * un.Mpc) pnum = (float(catalog.entries[name][ TIDALDISRUPTION.MAX_APP_MAG][0][QUANTITY.VALUE]) - 5.0 * (log10(float(bestld) * 1.0e6) - 1.0 ) + 2.5 * log10(1.0 + bestldz)) pnum = pretty_num(pnum, sig=bestsig) catalog.entries[name].add_quantity( TIDALDISRUPTION.MAX_ABS_MAG, pnum, sources, derived=True) if TIDALDISRUPTION.REDSHIFT in catalog.entries[name]: # Find the "best" redshift to use for this bestz, bestkind, bestsig, bestsrc = catalog.entries[ name].get_best_redshift() if bestsig > 0: try: bestz = float(bestz) except Exception: print(catalog.entries[name]) raise if TIDALDISRUPTION.VELOCITY not in catalog.entries[name]: source = catalog.entries[name].add_self_source() # FIX: what's happening here?! pnum = CLIGHT / KM * \ ((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.) pnum = pretty_num(pnum, sig=bestsig) catalog.entries[name].add_quantity( TIDALDISRUPTION.VELOCITY, pnum, source, kind=PREF_KINDS[bestkind], derived=True) if bestz > 0.: from astropy.cosmology import Planck15 as cosmo if TIDALDISRUPTION.LUM_DIST not in catalog.entries[name]: dl = cosmo.luminosity_distance(bestz) sources = [ catalog.entries[name].add_self_source(), catalog.entries[name] .add_source(bibcode='2016A&A...594A..13P') ] sources = uniq_cdl(sources + bestsrc.split(',')) catalog.entries[name].add_quantity( TIDALDISRUPTION.LUM_DIST, pretty_num( dl.value, sig=bestsig), sources, kind=PREF_KINDS[bestkind], derived=True) if (TIDALDISRUPTION.MAX_ABS_MAG not in catalog.entries[name] and TIDALDISRUPTION.MAX_APP_MAG in catalog.entries[name]): source = catalog.entries[name].add_self_source() pnum = pretty_num( float(catalog.entries[name][ TIDALDISRUPTION.MAX_APP_MAG][0][ QUANTITY.VALUE]) - 5.0 * (log10(dl.to('pc').value) - 1.0 ) + 2.5 * log10(1.0 + bestz), sig=bestsig + 1) catalog.entries[name].add_quantity( TIDALDISRUPTION.MAX_ABS_MAG, pnum, sources, derived=True) if TIDALDISRUPTION.COMOVING_DIST not in catalog.entries[ name]: cd = cosmo.comoving_distance(bestz) sources = [ catalog.entries[name].add_self_source(), catalog.entries[name] .add_source(bibcode='2016A&A...594A..13P') ] sources = uniq_cdl(sources + bestsrc.split(',')) catalog.entries[name].add_quantity( TIDALDISRUPTION.COMOVING_DIST, pretty_num( cd.value, sig=bestsig), sources, derived=True) if all([ x in catalog.entries[name] for x in [ TIDALDISRUPTION.RA, TIDALDISRUPTION.DEC, TIDALDISRUPTION.HOST_RA, TIDALDISRUPTION.HOST_DEC ] ]): # For now just using first coordinates that appear in entry try: c1 = coord( ra=catalog.entries[name][TIDALDISRUPTION.RA][0][ QUANTITY.VALUE], dec=catalog.entries[name][TIDALDISRUPTION.DEC][0][ QUANTITY.VALUE], unit=(un.hourangle, un.deg)) c2 = coord( ra=catalog.entries[name][TIDALDISRUPTION.HOST_RA][0][ QUANTITY.VALUE], dec=catalog.entries[name][TIDALDISRUPTION.HOST_DEC][0][ QUANTITY.VALUE], unit=(un.hourangle, un.deg)) except (KeyboardInterrupt, SystemExit): raise except Exception: pass else: sources = uniq_cdl( [catalog.entries[name].add_self_source()] + catalog. entries[name][TIDALDISRUPTION.RA][0]['source'].split(',') + catalog.entries[name][TIDALDISRUPTION.DEC][0]['source']. split(',') + catalog.entries[name][TIDALDISRUPTION.HOST_RA] [0]['source'].split(',') + catalog.entries[name][ TIDALDISRUPTION.HOST_DEC][0]['source'].split(',')) if 'hostoffsetang' not in catalog.entries[name]: hosa = Decimal( hypot(c1.ra.degree - c2.ra.degree, c1.dec.degree - c2.dec.degree)) hosa = pretty_num(hosa * Decimal(3600.)) catalog.entries[name].add_quantity( TIDALDISRUPTION.HOST_OFFSET_ANG, hosa, sources, derived=True, u_value='arcseconds') if (TIDALDISRUPTION.COMOVING_DIST in catalog.entries[name] and TIDALDISRUPTION.REDSHIFT in catalog.entries[name] and TIDALDISRUPTION.HOST_OFFSET_DIST not in catalog.entries[name]): offsetsig = get_sig_digits(catalog.entries[name][ TIDALDISRUPTION.HOST_OFFSET_ANG][0][QUANTITY.VALUE]) sources = uniq_cdl( sources.split(',') + (catalog.entries[name][ TIDALDISRUPTION.COMOVING_DIST][0]['source']). split(',') + (catalog.entries[name][ TIDALDISRUPTION.REDSHIFT][0]['source']).split(',')) (catalog.entries[name].add_quantity( TIDALDISRUPTION.HOST_OFFSET_DIST, pretty_num( float(catalog.entries[name][ TIDALDISRUPTION.HOST_OFFSET_ANG][0][ QUANTITY.VALUE]) / 3600. * (pi / 180.) * float(catalog.entries[name][ TIDALDISRUPTION.COMOVING_DIST][0][ QUANTITY.VALUE]) * 1000. / (1.0 + float(catalog.entries[name][ TIDALDISRUPTION.REDSHIFT][0][QUANTITY.VALUE])), sig=offsetsig), sources)) catalog.entries[name].sanitize() catalog.journal_entries(bury=True, final=True, gz=True) cleanupcnt = cleanupcnt + 1 if catalog.args.travis and cleanupcnt % 1000 == 0: break catalog.save_caches() return
def do_snls_spectra(catalog): """ """ task_str = catalog.get_current_task_str() result = Vizier.get_catalogs('J/A+A/507/85/table1') table = result[list(result.keys())[0]] table.convert_bytestring_to_unicode(python3_only=True) datedict = {} for row in table: datedict['SNLS-' + row['SN']] = str(astrotime(row['Date']).mjd) oldname = '' file_names = glob(os.path.join(catalog.get_current_task_repo(), 'SNLS/*')) for fi, fname in enumerate(pbar_strings(file_names, task_str)): filename = os.path.basename(fname) fileparts = filename.split('_') name = 'SNLS-' + fileparts[1] name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) source = catalog.entries[name].add_source( bibcode='2009A&A...507...85B') catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, source) catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, '20' + fileparts[1][:2], source) f = open(fname, 'r') data = csv.reader(f, delimiter=' ', skipinitialspace=True) specdata = [] for r, row in enumerate(data): if row[0] == '@TELESCOPE': telescope = row[1].strip() elif row[0] == '@REDSHIFT': catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, row[1].strip(), source) if r < 14: continue specdata.append(list(filter(None, [x.strip(' \t') for x in row]))) specdata = [list(i) for i in zip(*specdata)] wavelengths = specdata[1] fluxes = [pretty_num(float(x) * 1.e-16, sig=get_sig_digits(x)) for x in specdata[2]] # FIX: this isnt being used # errors = [pretty_num(float(x)*1.e-16, sig=get_sig_digits(x)) for x in # specdata[3]] catalog.entries[name].add_spectrum( u_wavelengths='Angstrom', u_fluxes='erg/s/cm^2/Angstrom', wavelengths=wavelengths, fluxes=fluxes, u_time='MJD' if name in datedict else '', time=datedict[name] if name in datedict else '', telescope=telescope, source=source, filename=filename) if catalog.args.travis and fi >= catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() return
def radec_clean(svalue, quantity, unit=''): if unit == 'floatdegrees': if not is_number(svalue): return (svalue, unit) deg = float('%g' % Decimal(svalue)) sig = get_sig_digits(svalue) if 'ra' in quantity: flhours = deg / 360.0 * 24.0 hours = floor(flhours) minutes = floor((flhours - hours) * 60.0) seconds = (flhours * 60.0 - (hours * 60.0 + minutes)) * 60.0 hours = 0 if hours < 1.e-6 else hours minutes = 0 if minutes < 1.e-6 else minutes seconds = 0.0 if seconds < 1.e-6 else seconds if seconds > 60.0: raise(ValueError('Invalid seconds value for ' + quantity)) svalue = str(hours).zfill(2) + ':' + str(minutes).zfill(2) + \ ':' + zpad(pretty_num(seconds, sig=sig - 1)) elif 'dec' in quantity: fldeg = abs(deg) degree = floor(fldeg) minutes = floor((fldeg - degree) * 60.0) seconds = (fldeg * 60.0 - (degree * 60.0 + minutes)) * 60.0 if seconds > 60.0: raise(ValueError('Invalid seconds value for ' + quantity)) svalue = (('+' if deg >= 0.0 else '-') + str(degree).strip('+-').zfill(2) + ':' + str(minutes).zfill(2) + ':' + zpad(pretty_num(seconds, sig=sig - 1))) elif unit == 'nospace' and 'ra' in quantity: svalue = svalue[:2] + ':' + svalue[2:4] + \ ((':' + zpad(svalue[4:])) if len(svalue) > 4 else '') elif unit == 'nospace' and 'dec' in quantity: if svalue.startswith(('+', '-')): svalue = svalue[:3] + ':' + svalue[3:5] + \ ((':' + zpad(svalue[5:])) if len(svalue) > 5 else '') else: svalue = '+' + svalue[:2] + ':' + svalue[2:4] + \ ((':' + zpad(svalue[4:])) if len(svalue) > 4 else '') else: svalue = svalue.replace(' ', ':') if 'dec' in quantity: valuesplit = svalue.split(':') svalue = (('-' if valuesplit[0].startswith('-') else '+') + valuesplit[0].strip('+-').zfill(2) + (':' + valuesplit[1].zfill(2) if len(valuesplit) > 1 else '') + (':' + zpad(valuesplit[2]) if len(valuesplit) > 2 else '')) if 'ra' in quantity: sunit = 'hours' elif 'dec' in quantity: sunit = 'degrees' # Correct case of arcseconds = 60.0. valuesplit = svalue.split(':') if len(valuesplit) == 3 and valuesplit[-1] in ["60.0", "60.", "60"]: svalue = valuesplit[0] + ':' + str(Decimal(valuesplit[1]) + Decimal(1.0)) + ':' + "00.0" # Strip trailing dots. svalue = svalue.rstrip('.') return (svalue, sunit)
def add_quantity(self, quantities, value, source, forcereplacebetter=False, **kwargs): success = super(Supernova, self).add_quantity(quantities, value, source, **kwargs) if not success: return for quantity in listify(quantities): my_quantity_list = self.get(quantity, []) if ((forcereplacebetter or quantity.replace_better) and len(my_quantity_list) > 1): # The quantity that was just added should be last in the list added_quantity = my_quantity_list.pop() newquantities = [] isworse = True if quantity in [self._KEYS.DISCOVER_DATE, self._KEYS.MAX_DATE]: for ct in my_quantity_list: ctsplit = ct[QUANTITY.VALUE].split('/') svsplit = added_quantity[QUANTITY.VALUE].split('/') if len(ctsplit) < len(svsplit): isworse = False continue elif len(ctsplit) < len(svsplit) and len(svsplit) == 3: val_one = max( 2, get_sig_digits(ctsplit[-1].lstrip('0'))) val_two = max( 2, get_sig_digits(svsplit[-1].lstrip('0'))) if val_one < val_two: isworse = False continue newquantities.append(ct) else: if type(quantity) != Key: isworse = False elif quantity.type == KEY_TYPES.NUMERIC: newsig = get_sig_digits(added_quantity[QUANTITY.VALUE]) for ct in my_quantity_list: addct = False checke = False if (len(quantity.kind_preference) > 0 and not set(listify(ct.get( QUANTITY.KIND, []))).isdisjoint( quantity.kind_preference) and not set( listify( added_quantity.get( QUANTITY.KIND, []))).isdisjoint( quantity.kind_preference)): aqi = min([ quantity.kind_preference.index(x) for x in listify(added_quantity[QUANTITY.KIND]) ]) qqi = min([ quantity.kind_preference.index(x) for x in listify(ct[QUANTITY.KIND]) ]) if aqi > qqi: addct = True if aqi == qqi: checke = True if aqi <= qqi: isworse = False else: checke = True if checke and QUANTITY.E_VALUE in ct: if QUANTITY.E_VALUE in added_quantity: if (float(added_quantity[QUANTITY.E_VALUE]) >= float(ct[QUANTITY.E_VALUE])): addct = True if (float(added_quantity[QUANTITY.E_VALUE]) <= float(ct[QUANTITY.E_VALUE])): isworse = False else: if (checke and QUANTITY.E_VALUE in added_quantity): isworse = False else: oldsig = get_sig_digits(ct[QUANTITY.VALUE]) if oldsig >= newsig: addct = True if newsig >= oldsig: isworse = False if addct: newquantities.append(ct) elif quantity.type == KEY_TYPES.STRING: for ct in my_quantity_list: addct = False if (len(quantity.kind_preference) > 0 and not set(listify(ct.get( QUANTITY.KIND, []))).isdisjoint( quantity.kind_preference) and not set( listify( added_quantity.get( QUANTITY.KIND, []))).isdisjoint( quantity.kind_preference)): aqi = min([ quantity.kind_preference.index(x) for x in listify(added_quantity[QUANTITY.KIND]) ]) qqi = min([ quantity.kind_preference.index(x) for x in listify(ct[QUANTITY.KIND]) ]) if aqi >= qqi: addct = True if aqi <= qqi: isworse = False else: addct = True isworse = False if addct: newquantities.append(ct) if isworse: self._log.info("Removing quantity '{}' with value '{}' " "and kind '{}' determined to be worse than " "existing alternative values.".format( quantity, added_quantity[QUANTITY.VALUE], added_quantity.get(QUANTITY.KIND, ''))) else: newquantities.append(added_quantity) if len(newquantities) > 0: self[quantity] = newquantities # As all SN####xx designations for 2016+ have corresponding AT # designations, add the AT alias when the SN alias is added. if quantity == self._KEYS.ALIAS: for alias in self.get(quantity, []): cleaned_value = alias[QUANTITY.VALUE] if (cleaned_value.startswith('SN') and is_integer(cleaned_value[2:6]) and int(cleaned_value[2:6]) >= 2016): success = super(Supernova, self).add_quantity( SUPERNOVA.ALIAS, 'AT' + cleaned_value[2:], source, **kwargs) return True
def radec_clean(svalue, quantity, unit=''): """Clean R.A. and Dec.""" svalue = svalue.strip() if unit == 'floatdegrees': if not is_number(svalue): return (svalue, unit) deg = float('%g' % Decimal(svalue)) sig = get_sig_digits(svalue) if 'ra' in quantity: flhours = deg / 360.0 * 24.0 hours = floor(flhours) minutes = floor((flhours - hours) * 60.0) seconds = (flhours * 60.0 - (hours * 60.0 + minutes)) * 60.0 hours = 0 if hours < 1.e-6 else hours minutes = 0 if minutes < 1.e-6 else minutes seconds = 0.0 if seconds < 1.e-6 else seconds if seconds > 60.0: raise (ValueError('Invalid seconds value for ' + quantity)) svalue = str(hours).zfill(2) + ':' + str(minutes).zfill(2) + \ ':' + zpad(pretty_num(seconds, sig=sig - 1)) elif 'dec' in quantity: fldeg = abs(deg) degree = floor(fldeg) minutes = floor((fldeg - degree) * 60.0) seconds = (fldeg * 60.0 - (degree * 60.0 + minutes)) * 60.0 minutes = 0 if minutes < 1.e-6 else minutes seconds = 0.0 if seconds < 1.e-6 else seconds if seconds > 60.0: raise (ValueError('Invalid seconds value for ' + quantity)) svalue = (('+' if deg >= 0.0 else '-') + str(degree).strip('+-').zfill(2) + ':' + str(minutes).zfill(2) + ':' + zpad(pretty_num(seconds, sig=sig - 1))) elif unit == 'nospace' and 'ra' in quantity: svalue = svalue[:2] + ':' + svalue[2:4] + \ ((':' + zpad(svalue[4:])) if len(svalue) > 4 else '') elif unit == 'nospace' and 'dec' in quantity: if svalue.startswith(('+', '-')): svalue = svalue[:3] + ':' + svalue[3:5] + \ ((':' + zpad(svalue[5:])) if len(svalue) > 5 else '') else: svalue = '+' + svalue[:2] + ':' + svalue[2:4] + \ ((':' + zpad(svalue[4:])) if len(svalue) > 4 else '') else: svalue = svalue.replace(' ', ':') if 'dec' in quantity: valuesplit = svalue.split(':') svalue = ( ('-' if valuesplit[0].startswith('-') else '+') + valuesplit[0].strip('+-').zfill(2) + (':' + valuesplit[1].zfill(2) if len(valuesplit) > 1 else '') + (':' + zpad(valuesplit[2]) if len(valuesplit) > 2 else '')) if 'ra' in quantity: sunit = 'hours' elif 'dec' in quantity: sunit = 'degrees' # Correct case of arcseconds = 60.0. valuesplit = svalue.split(':') if len(valuesplit) == 3 and valuesplit[-1] in ["60.0", "60.", "60"]: svalue = valuesplit[0] + ':' + str( Decimal(valuesplit[1]) + Decimal(1.0)) + ':' + "00.0" # Strip trailing dots. svalue = svalue.rstrip('.') return (svalue, sunit)
def do_donated_spectra(catalog): """Import donated spectra.""" task_str = catalog.get_current_task_str() fpath = os.path.join(catalog.get_current_task_repo(), 'donations') with open(os.path.join(fpath, 'meta.json'), 'r') as f: metadict = json.loads(f.read()) donationscnt = 0 oldname = '' for fname in pbar(list(metadict.keys()), task_str): name = metadict[fname]['name'] name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name sec_bibc = metadict[fname]['bibcode'] name, source = catalog.new_entry(name, bibcode=sec_bibc) date = metadict[fname].get('date', '') year, month, day = date.split('/') sig = get_sig_digits(day) + 5 day_fmt = str(floor(float(day))).zfill(2) time = astrotime(year + '-' + month + '-' + day_fmt).mjd time = time + float(day) - floor(float(day)) time = pretty_num(time, sig=sig) with open(os.path.join(fpath, fname), 'r') as f: specdata = list( csv.reader( f, delimiter=' ', skipinitialspace=True)) specdata = list(filter(None, specdata)) newspec = [] oldval = '' for row in specdata: if row[0][0] == '#': continue if row[1] == oldval: continue newspec.append(row) oldval = row[1] specdata = newspec haserrors = len(specdata[0]) == 3 and specdata[0][2] and specdata[0][ 2] != 'NaN' specdata = [list(i) for i in zip(*specdata)] wavelengths = specdata[0] fluxes = specdata[1] errors = '' if haserrors: errors = specdata[2] specdict = { SPECTRUM.U_WAVELENGTHS: 'Angstrom', SPECTRUM.U_TIME: 'MJD', SPECTRUM.TIME: time, SPECTRUM.WAVELENGTHS: wavelengths, SPECTRUM.FLUXES: fluxes, SPECTRUM.ERRORS: errors, SPECTRUM.SOURCE: source, SPECTRUM.FILENAME: fname } if 'instrument' in metadict[fname]: specdict[SPECTRUM.INSTRUMENT] = metadict[fname]['instrument'] if 'telescope' in metadict[fname]: specdict[SPECTRUM.TELESCOPE] = metadict[fname]['telescope'] if 'yunit' in metadict[fname]: specdict[SPECTRUM.U_FLUXES] = metadict[fname]['yunit'] specdict[SPECTRUM.U_ERRORS] = metadict[fname]['yunit'] else: if max([float(x) for x in fluxes]) < 1.0e-5: fluxunit = 'erg/s/cm^2/Angstrom' else: fluxunit = 'Uncalibrated' specdict[SPECTRUM.U_FLUXES] = fluxunit specdict[SPECTRUM.U_ERRORS] = fluxunit catalog.entries[name].add_spectrum(**specdict) donationscnt = donationscnt + 1 if (catalog.args.travis and donationscnt % catalog.TRAVIS_QUERY_LIMIT == 0): break catalog.journal_entries() return
def add_quantity(self, quantities, value, source, forcereplacebetter=False, **kwargs): """Add `Quantity` to `Supernova`.""" success = super(Supernova, self).add_quantity( quantities, value, source, **kwargs) if not success: return for quantity in listify(quantities): my_quantity_list = self.get(quantity, []) if ((forcereplacebetter or quantity.replace_better) and len(my_quantity_list) > 1): # The quantity that was just added should be last in the list added_quantity = my_quantity_list.pop() newquantities = [] isworse = True if quantity in [self._KEYS.DISCOVER_DATE, self._KEYS.MAX_DATE]: for ct in my_quantity_list: ctsplit = ct[QUANTITY.VALUE].split('/') svsplit = added_quantity[QUANTITY.VALUE].split('/') if len(ctsplit) < len(svsplit): isworse = False continue elif len(ctsplit) < len(svsplit) and len(svsplit) == 3: val_one = max( 2, get_sig_digits(ctsplit[-1].lstrip('0'))) val_two = max( 2, get_sig_digits(svsplit[-1].lstrip('0'))) if val_one < val_two: isworse = False continue newquantities.append(ct) else: if type(quantity) != Key: isworse = False elif quantity.type == KEY_TYPES.NUMERIC: newsig = get_sig_digits(added_quantity[QUANTITY.VALUE]) for ct in my_quantity_list: addct = False checke = False if (len(quantity.kind_preference) > 0 and not set( listify(ct.get(QUANTITY.KIND, []))) .isdisjoint(quantity.kind_preference) and not set( listify( added_quantity.get(QUANTITY.KIND, []))) .isdisjoint(quantity.kind_preference)): aqi = min([ quantity.kind_preference.index(x) for x in listify(added_quantity[ QUANTITY.KIND]) ]) qqi = min([ quantity.kind_preference.index(x) for x in listify(ct[QUANTITY.KIND]) ]) if aqi > qqi: addct = True if aqi == qqi: checke = True if aqi <= qqi: isworse = False else: checke = True if checke and QUANTITY.E_VALUE in ct: if QUANTITY.E_VALUE in added_quantity: if (float(added_quantity[QUANTITY.E_VALUE]) >= float(ct[QUANTITY.E_VALUE])): addct = True if (float(added_quantity[QUANTITY.E_VALUE]) <= float(ct[QUANTITY.E_VALUE])): isworse = False else: if (checke and QUANTITY.E_VALUE in added_quantity): isworse = False else: oldsig = get_sig_digits(ct[QUANTITY.VALUE]) if oldsig >= newsig: addct = True if newsig >= oldsig: isworse = False if addct: newquantities.append(ct) elif quantity.type == KEY_TYPES.STRING: for ct in my_quantity_list: addct = False if (len(quantity.kind_preference) > 0 and not set( listify(ct.get(QUANTITY.KIND, []))) .isdisjoint(quantity.kind_preference) and not set( listify( added_quantity.get(QUANTITY.KIND, []))) .isdisjoint(quantity.kind_preference)): aqi = min([ quantity.kind_preference.index(x) for x in listify(added_quantity[ QUANTITY.KIND]) ]) qqi = min([ quantity.kind_preference.index(x) for x in listify(ct[QUANTITY.KIND]) ]) if aqi >= qqi: addct = True if aqi <= qqi: isworse = False else: addct = True isworse = False if addct: newquantities.append(ct) if isworse: self._log.info("Removing quantity '{}' with value '{}' " "and kind '{}' determined to be worse than " "existing alternative values.".format( quantity, added_quantity[ QUANTITY.VALUE], added_quantity.get(QUANTITY.KIND, ''))) else: newquantities.append(added_quantity) if len(newquantities) > 0: self[quantity] = newquantities # As all SN####xx designations for 2016+ have corresponding AT # designations, add the AT alias when the SN alias is added. if quantity == self._KEYS.ALIAS: for alias in self.get(quantity, []): cleaned_value = alias[QUANTITY.VALUE] if (cleaned_value.startswith('SN') and is_integer(cleaned_value[2:6]) and int(cleaned_value[2:6]) >= 2016): success = super(Supernova, self).add_quantity( SUPERNOVA.ALIAS, 'AT' + cleaned_value[2:], source, **kwargs) return True
def do_ucb_spectra(catalog): task_str = catalog.get_current_task_str() sec_reference = 'UCB Filippenko Group\'s Supernova Database (SNDB)' sec_refurl = 'http://heracles.astro.berkeley.edu/sndb/info' sec_refbib = '2012MNRAS.425.1789S' ucbspectracnt = 0 jsontxt = catalog.load_url( 'http://heracles.astro.berkeley.edu/sndb/download?id=allpubspec', os.path.join(catalog.get_current_task_repo(), 'UCB/allpubspec.json'), json_sort='SpecID') if not jsontxt: return spectra = json.loads(jsontxt) spectra = sorted(spectra, key=lambda kk: kk['SpecID']) oldname = '' for spectrum in pbar(spectra, task_str): name = spectrum['ObjName'] if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) sec_source = catalog.entries[name].add_source(name=sec_reference, url=sec_refurl, bibcode=sec_refbib, secondary=True) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, sec_source) sources = [sec_source] if spectrum['Reference']: sources += [ catalog.entries[name].add_source(bibcode=spectrum['Reference']) ] sources = uniq_cdl(sources) if spectrum['Type'] and spectrum['Type'].strip() != 'NoMatch': for ct in spectrum['Type'].strip().split(','): catalog.entries[name].add_quantity( SUPERNOVA.CLAIMED_TYPE, ct.replace('-norm', '').strip(), sources) if spectrum['DiscDate']: ddate = spectrum['DiscDate'].replace('-', '/') catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, ddate, sources) if spectrum['HostName']: host = urllib.parse.unquote(spectrum['HostName']).replace('*', '') catalog.entries[name].add_quantity(SUPERNOVA.HOST, host, sources) if spectrum['UT_Date']: epoch = str(spectrum['UT_Date']) year = epoch[:4] month = epoch[4:6] day = epoch[6:] sig = get_sig_digits(day) + 5 mjd = astrotime(year + '-' + month + '-' + str(floor(float(day))).zfill(2)).mjd mjd = pretty_num(mjd + float(day) - floor(float(day)), sig=sig) filename = spectrum['Filename'] if spectrum['Filename'] else '' instrument = spectrum['Instrument'] if spectrum['Instrument'] else '' reducer = spectrum['Reducer'] if spectrum['Reducer'] else '' observer = spectrum['Observer'] if spectrum['Observer'] else '' snr = str(spectrum['SNR']) if spectrum['SNR'] else '' if not filename: raise ValueError('Filename not found for SNDB spectrum!') if not spectrum['SpecID']: raise ValueError('ID not found for SNDB spectrum!') filepath = os.path.join(catalog.get_current_task_repo(), 'UCB/') + filename spectxt = catalog.load_url( 'http://heracles.astro.berkeley.edu/sndb/download?id=ds:' + str(spectrum['SpecID']), filepath, archived_mode=True) specdata = list( csv.reader(spectxt.splitlines(), delimiter=' ', skipinitialspace=True)) newspecdata = [] for row in specdata: if not row or not row[0] or row[0][0] == '#': continue else: newspecdata.append(row) specdata = newspecdata haserrors = len( specdata[0]) == 3 and specdata[0][2] and specdata[0][2] != 'NaN' specdata = [list(ii) for ii in zip(*specdata)] wavelengths = specdata[0] fluxes = specdata[1] errors = '' if haserrors: errors = specdata[2] if not list(filter(None, errors)): errors = '' units = 'Uncalibrated' catalog.entries[name].add_spectrum(u_wavelengths='Angstrom', u_fluxes=units, u_time='MJD', time=mjd, wavelengths=wavelengths, filename=filename, fluxes=fluxes, errors=errors, u_errors=units, instrument=instrument, source=sources, snr=snr, observer=observer, reducer=reducer, deredshifted=('-noz' in filename)) ucbspectracnt = ucbspectracnt + 1 if catalog.args.travis and ucbspectracnt >= catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() return
def add_quantity(self, quantity, value, source, forcereplacebetter=False, **kwargs): success = super().add_quantity(quantity, value, source, **kwargs) if not success: return my_quantity_list = self.get(quantity, []) if (forcereplacebetter or quantity in REPR_BETTER_QUANTITY) and \ len(my_quantity_list) > 1: # The quantity that was just added should be last in the list added_quantity = my_quantity_list.pop() newquantities = [] isworse = True if quantity in [self._KEYS.DISCOVER_DATE, self._KEYS.MAX_DATE]: for ct in my_quantity_list: ctsplit = ct[QUANTITY.VALUE].split('/') svsplit = added_quantity[QUANTITY.VALUE].split('/') if len(ctsplit) < len(svsplit): isworse = False continue elif len(ctsplit) < len(svsplit) and len(svsplit) == 3: val_one = max(2, get_sig_digits( ctsplit[-1].lstrip('0'))) val_two = max(2, get_sig_digits( svsplit[-1].lstrip('0'))) if val_one < val_two: isworse = False continue newquantities.append(ct) else: newsig = get_sig_digits(added_quantity[QUANTITY.VALUE]) for ct in my_quantity_list: if QUANTITY.E_VALUE in ct: if QUANTITY.E_VALUE in added_quantity: if (float(added_quantity[QUANTITY.E_VALUE]) <= float(ct[QUANTITY.E_VALUE])): isworse = False continue newquantities.append(ct) else: if QUANTITY.E_VALUE in added_quantity: isworse = False continue oldsig = get_sig_digits(ct[QUANTITY.VALUE]) if oldsig >= newsig: newquantities.append(ct) if newsig >= oldsig: isworse = False if not isworse: newquantities.append(added_quantity) self[quantity] = newquantities # As all SN####xx designations have corresponding AT designations, add # the AT alias when the SN alias is added. if quantity == self._KEYS.ALIAS: cleaned_quantity = quantity.strip() if (cleaned_quantity.startswith('SN') and is_integer(cleaned_quantity[2:6]) and int(cleaned_quantity[2:6]) >= 2016): success = super().add_quantity( 'AT' + cleaned_quantity[2:], value, source, **kwargs) return True
def do_donated_spectra(catalog): """Import donated spectra.""" task_str = catalog.get_current_task_str() fpath = os.path.join(catalog.get_current_task_repo(), 'donations') with open(os.path.join(fpath, 'meta.json'), 'r') as f: metadict = json.loads(f.read()) donationscnt = 0 oldname = '' for fname in pbar(metadict, task_str): name = metadict[fname]['name'] name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name sec_bibc = metadict[fname]['bibcode'] name, source = catalog.new_entry(name, bibcode=sec_bibc) date = metadict[fname].get('date', '') year, month, day = date.split('/') sig = get_sig_digits(day) + 5 day_fmt = str(floor(float(day))).zfill(2) time = astrotime(year + '-' + month + '-' + day_fmt).mjd time = time + float(day) - floor(float(day)) time = pretty_num(time, sig=sig) with open(os.path.join(fpath, fname), 'r') as f: specdata = list(csv.reader(f, delimiter=' ', skipinitialspace=True)) specdata = list(filter(None, specdata)) newspec = [] oldval = '' for row in specdata: if row[0][0] == '#': continue if row[1] == oldval: continue newspec.append(row) oldval = row[1] specdata = newspec haserrors = len( specdata[0]) == 3 and specdata[0][2] and specdata[0][2] != 'NaN' specdata = [list(i) for i in zip(*specdata)] wavelengths = specdata[0] fluxes = specdata[1] errors = '' if haserrors: errors = specdata[2] specdict = { SPECTRUM.U_WAVELENGTHS: 'Angstrom', SPECTRUM.U_TIME: 'MJD', SPECTRUM.TIME: time, SPECTRUM.WAVELENGTHS: wavelengths, SPECTRUM.FLUXES: fluxes, SPECTRUM.ERRORS: errors, SPECTRUM.SOURCE: source, SPECTRUM.FILENAME: fname } if 'instrument' in metadict[fname]: specdict[SPECTRUM.INSTRUMENT] = metadict[fname]['instrument'] if 'telescope' in metadict[fname]: specdict[SPECTRUM.TELESCOPE] = metadict[fname]['telescope'] if 'yunit' in metadict[fname]: specdict[SPECTRUM.U_FLUXES] = metadict[fname]['yunit'] specdict[SPECTRUM.U_ERRORS] = metadict[fname]['yunit'] else: if max([float(x) for x in fluxes]) < 1.0e-5: fluxunit = 'erg/s/cm^2/Angstrom' else: fluxunit = 'Uncalibrated' specdict[SPECTRUM.U_FLUXES] = fluxunit specdict[SPECTRUM.U_ERRORS] = fluxunit catalog.entries[name].add_spectrum(**specdict) donationscnt = donationscnt + 1 if (catalog.args.travis and donationscnt % catalog.TRAVIS_QUERY_LIMIT == 0): break catalog.journal_entries() return
def do_suspect_spectra(catalog): task_str = catalog.get_current_task_str() with open(os.path.join(catalog.get_current_task_repo(), 'Suspect/sources.json'), 'r') as f: sourcedict = json.loads(f.read()) with open(os.path.join(catalog.get_current_task_repo(), 'Suspect/filename-changes.txt'), 'r') as f: rows = f.readlines() changedict = {} for row in rows: if not row.strip() or row[0] == "#": continue items = row.strip().split(' ') changedict[items[1]] = items[0] suspectcnt = 0 folders = next(os.walk(os.path.join( catalog.get_current_task_repo(), 'Suspect')))[1] for folder in pbar(folders, task_str): eventfolders = next(os.walk(os.path.join( catalog.get_current_task_repo(), 'Suspect/') + folder))[1] oldname = '' for eventfolder in pbar(eventfolders, task_str): name = eventfolder if is_number(name[:4]): name = 'SN' + name name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) sec_ref = 'SUSPECT' sec_refurl = 'https://www.nhn.ou.edu/~suspect/' sec_bibc = '2001AAS...199.8408R' sec_source = catalog.entries[name].add_source( name=sec_ref, url=sec_refurl, bibcode=sec_bibc, secondary=True) catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, name, sec_source) fpath = os.path.join(catalog.get_current_task_repo(), 'Suspect', folder, eventfolder) eventspectra = next(os.walk(fpath))[2] for spectrum in eventspectra: sources = [sec_source] bibcode = '' if spectrum in changedict: specalias = changedict[spectrum] else: specalias = spectrum if specalias in sourcedict: bibcode = sourcedict[specalias] elif name in sourcedict: bibcode = sourcedict[name] if bibcode: source = catalog.entries[name].add_source( bibcode=unescape(bibcode)) sources += [source] sources = uniq_cdl(sources) date = spectrum.split('_')[1] year = date[:4] month = date[4:6] day = date[6:] sig = get_sig_digits(day) + 5 day_fmt = str(floor(float(day))).zfill(2) time = astrotime(year + '-' + month + '-' + day_fmt).mjd time = time + float(day) - floor(float(day)) time = pretty_num(time, sig=sig) fpath = os.path.join(catalog.get_current_task_repo(), 'Suspect', folder, eventfolder, spectrum) with open(fpath, 'r') as f: specdata = list(csv.reader( f, delimiter=' ', skipinitialspace=True)) specdata = list(filter(None, specdata)) newspec = [] oldval = '' for row in specdata: if row[1] == oldval: continue newspec.append(row) oldval = row[1] specdata = newspec haserrors = len(specdata[0]) == 3 and specdata[ 0][2] and specdata[0][2] != 'NaN' specdata = [list(i) for i in zip(*specdata)] wavelengths = specdata[0] fluxes = specdata[1] errors = '' if haserrors: errors = specdata[2] catalog.entries[name].add_spectrum( u_wavelengths='Angstrom', u_fluxes='Uncalibrated', u_time='MJD', time=time, wavelengths=wavelengths, fluxes=fluxes, errors=errors, u_errors='Uncalibrated', source=sources, filename=spectrum) suspectcnt = suspectcnt + 1 if (catalog.args.travis and suspectcnt % catalog.TRAVIS_QUERY_LIMIT == 0): break catalog.journal_entries() return
def do_cleanup(catalog): """Cleanup catalog after importing all data.""" task_str = catalog.get_current_task_str() # Set preferred names, calculate some columns based on imported data, # sanitize some fields keys = list(catalog.entries.keys()) cleanupcnt = 0 for oname in pbar(keys, task_str): # Some events may be merged in cleanup process, skip them if # non-existent. try: name = catalog.add_entry(oname) except Exception: catalog.log.warning( '"{}" was not found, suggests merge occurred in cleanup ' 'process.'.format(oname)) continue # Set the preferred name, switching to that name if name changed. name = catalog.entries[name].set_preferred_name() aliases = catalog.entries[name].get_aliases() catalog.entries[name].purge_bandless_photometry() catalog.entries[name].set_first_max_light() if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['MLS', 'SSS', 'CSS', 'GRB '] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:2])): discoverdate = ('/'.join([ '20' + alias.replace(prefix, '')[:2], alias.replace(prefix, '')[2:4], alias.replace(prefix, '')[4:6] ])) if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, discoverdate, source, derived=True) break if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]: break if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]: prefixes = [ 'ASASSN-', 'PS1-', 'PS1', 'PS', 'iPTF', 'PTF', 'SCP-', 'SNLS-', 'SPIRITS', 'LSQ', 'DES', 'SNHiTS', 'Gaia', 'GND', 'GNW', 'GSD', 'GSW', 'EGS', 'COS', 'OGLE', 'HST' ] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:2]) and is_number(alias.replace(prefix, '')[:1])): discoverdate = '20' + alias.replace(prefix, '')[:2] if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, discoverdate, source, derived=True) break if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]: break if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['SNF'] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:4])): discoverdate = ('/'.join([ alias.replace(prefix, '')[:4], alias.replace(prefix, '')[4:6], alias.replace(prefix, '')[6:8] ])) if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, discoverdate, source, derived=True) break if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]: break if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['PTFS', 'SNSDF'] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:2])): discoverdate = ('/'.join([ '20' + alias.replace(prefix, '')[:2], alias.replace(prefix, '')[2:4] ])) if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, discoverdate, source, derived=True) break if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]: break if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['AT', 'SN', 'OGLE-', 'SM ', 'KSN'] for alias in aliases: for prefix in prefixes: if alias.startswith(prefix): year = re.findall(r'\d+', alias) if len(year) == 1: year = year[0] else: continue if alias.replace(prefix, '').index(year) != 0: continue if (year and is_number(year) and '.' not in year and len(year) <= 4): discoverdate = year if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, discoverdate, source, derived=True) break if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]: break if (SUPERNOVA.RA not in catalog.entries[name] or SUPERNOVA.DEC not in catalog.entries[name]): prefixes = [ 'PSN J', 'MASJ', 'CSS', 'SSS', 'MASTER OT J', 'HST J', 'TCP J', 'MACS J', '2MASS J', 'EQ J', 'CRTS J', 'SMT J' ] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:6])): noprefix = alias.split(':')[-1].replace(prefix, '').replace( '.', '') decsign = '+' if '+' in noprefix else '-' noprefix = noprefix.replace('+', '|').replace('-', '|') nops = noprefix.split('|') if len(nops) < 2: continue rastr = nops[0] decstr = nops[1] ra = ':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + \ ('.' + rastr[6:] if len(rastr) > 6 else '') dec = ( decsign + ':'.join([decstr[:2], decstr[2:4], decstr[4:6]]) + ('.' + decstr[6:] if len(decstr) > 6 else '')) if catalog.args.verbose: tprint('Added ra/dec from name: ' + ra + ' ' + dec) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity(SUPERNOVA.RA, ra, source, derived=True) catalog.entries[name].add_quantity(SUPERNOVA.DEC, dec, source, derived=True) break if SUPERNOVA.RA in catalog.entries[name]: break no_host = (SUPERNOVA.HOST not in catalog.entries[name] or not any([ x[QUANTITY.VALUE] == 'Milky Way' for x in catalog.entries[name][SUPERNOVA.HOST] ])) if (SUPERNOVA.RA in catalog.entries[name] and SUPERNOVA.DEC in catalog.entries[name] and no_host): from astroquery.irsa_dust import IrsaDust if name not in catalog.extinctions_dict: try: ra_dec = catalog.entries[name][ SUPERNOVA.RA][0][QUANTITY.VALUE] + \ " " + \ catalog.entries[name][SUPERNOVA.DEC][0][QUANTITY.VALUE] result = IrsaDust.get_query_table(ra_dec, section='ebv') except (KeyboardInterrupt, SystemExit): raise except Exception: warnings.warn("Coordinate lookup for " + name + " failed in IRSA.") else: ebv = result['ext SandF mean'][0] ebverr = result['ext SandF std'][0] catalog.extinctions_dict[name] = [ebv, ebverr] if name in catalog.extinctions_dict: sources = uniq_cdl([ catalog.entries[name].add_self_source(), catalog.entries[name].add_source( bibcode='2011ApJ...737..103S') ]) (catalog.entries[name].add_quantity( SUPERNOVA.EBV, str(catalog.extinctions_dict[name][0]), sources, e_value=str(catalog.extinctions_dict[name][1]), derived=True)) if ((SUPERNOVA.HOST in catalog.entries[name] and (SUPERNOVA.HOST_RA not in catalog.entries[name] or SUPERNOVA.HOST_DEC not in catalog.entries[name]))): for host in catalog.entries[name][SUPERNOVA.HOST]: alias = host[QUANTITY.VALUE] if ' J' in alias and is_number(alias.split(' J')[-1][:6]): noprefix = alias.split(' J')[-1].split(':')[-1].replace( '.', '') decsign = '+' if '+' in noprefix else '-' noprefix = noprefix.replace('+', '|').replace('-', '|') nops = noprefix.split('|') if len(nops) < 2: continue rastr = nops[0] decstr = nops[1] hostra = (':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + ('.' + rastr[6:] if len(rastr) > 6 else '')) hostdec = decsign + ':'.join([ decstr[:2], decstr[2:4], decstr[4:6] ]) + ('.' + decstr[6:] if len(decstr) > 6 else '') if catalog.args.verbose: tprint('Added hostra/hostdec from name: ' + hostra + ' ' + hostdec) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity(SUPERNOVA.HOST_RA, hostra, source, derived=True) catalog.entries[name].add_quantity(SUPERNOVA.HOST_DEC, hostdec, source, derived=True) break if SUPERNOVA.HOST_RA in catalog.entries[name]: break if (SUPERNOVA.REDSHIFT not in catalog.entries[name] and SUPERNOVA.VELOCITY in catalog.entries[name]): # Find the "best" velocity to use for this bestsig = 0 for hv in catalog.entries[name][SUPERNOVA.VELOCITY]: sig = get_sig_digits(hv[QUANTITY.VALUE]) if sig > bestsig: besthv = hv[QUANTITY.VALUE] bestsrc = hv['source'] bestsig = sig if bestsig > 0 and is_number(besthv): voc = float(besthv) * 1.e5 / CLIGHT source = catalog.entries[name].add_self_source() sources = uniq_cdl([source] + bestsrc.split(',')) (catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, pretty_num(sqrt((1. + voc) / (1. - voc)) - 1., sig=bestsig), sources, kind='heliocentric', derived=True)) if (SUPERNOVA.REDSHIFT not in catalog.entries[name] and len(catalog.nedd_dict) > 0 and SUPERNOVA.HOST in catalog.entries[name]): reference = "NED-D" refurl = "http://ned.ipac.caltech.edu/Library/Distances/" refbib = "1991ASSL..171...89H" for host in catalog.entries[name][SUPERNOVA.HOST]: if host[QUANTITY.VALUE] in catalog.nedd_dict: source = catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') secondarysource = catalog.entries[name].add_source( name=reference, url=refurl, bibcode=refbib, secondary=True) meddist = statistics.median( catalog.nedd_dict[host[QUANTITY.VALUE]]) redz = z_at_value(cosmo.comoving_distance, float(meddist) * un.Mpc) redshift = pretty_num(redz, sig=get_sig_digits(str(meddist))) catalog.entries[name].add_quantity( [SUPERNOVA.REDSHIFT, SUPERNOVA.HOST_REDSHIFT], redshift, uniq_cdl([source, secondarysource]), kind='host', derived=True) if (SUPERNOVA.MAX_ABS_MAG not in catalog.entries[name] and SUPERNOVA.MAX_APP_MAG in catalog.entries[name] and SUPERNOVA.LUM_DIST in catalog.entries[name]): # Find the "best" distance to use for this bestsig = 0 for ld in catalog.entries[name][SUPERNOVA.LUM_DIST]: sig = get_sig_digits(ld[QUANTITY.VALUE]) if sig > bestsig: bestld = ld[QUANTITY.VALUE] bestsrc = ld[QUANTITY.SOURCE] bestsig = sig if bestsig > 0 and is_number(bestld) and float(bestld) > 0.: source = catalog.entries[name].add_self_source() sources = uniq_cdl([source] + bestsrc.split(',')) bestldz = z_at_value(cosmo.luminosity_distance, float(bestld) * un.Mpc) pnum = (float(catalog.entries[name][SUPERNOVA.MAX_APP_MAG][0][ QUANTITY.VALUE]) - 5.0 * (log10(float(bestld) * 1.0e6) - 1.0) + 2.5 * log10(1.0 + bestldz)) pnum = pretty_num(pnum, sig=bestsig + 1) catalog.entries[name].add_quantity(SUPERNOVA.MAX_ABS_MAG, pnum, sources, derived=True) if (SUPERNOVA.MAX_VISUAL_ABS_MAG not in catalog.entries[name] and SUPERNOVA.MAX_VISUAL_APP_MAG in catalog.entries[name] and SUPERNOVA.LUM_DIST in catalog.entries[name]): # Find the "best" distance to use for this bestsig = 0 for ld in catalog.entries[name][SUPERNOVA.LUM_DIST]: sig = get_sig_digits(ld[QUANTITY.VALUE]) if sig > bestsig: bestld = ld[QUANTITY.VALUE] bestsrc = ld[QUANTITY.SOURCE] bestsig = sig if bestsig > 0 and is_number(bestld) and float(bestld) > 0.: source = catalog.entries[name].add_self_source() sources = uniq_cdl([source] + bestsrc.split(',')) # FIX: what's happening here?! pnum = (float(catalog.entries[name][ SUPERNOVA.MAX_VISUAL_APP_MAG][0][QUANTITY.VALUE]) - 5.0 * (log10(float(bestld) * 1.0e6) - 1.0)) pnum = pretty_num(pnum, sig=bestsig + 1) catalog.entries[name].add_quantity( SUPERNOVA.MAX_VISUAL_ABS_MAG, pnum, sources, derived=True) if SUPERNOVA.REDSHIFT in catalog.entries[name]: # Find the "best" redshift to use for this bestz, bestkind, bestsig, bestsrc = catalog.entries[ name].get_best_redshift() if bestsig > 0: try: bestz = float(bestz) except Exception: print(catalog.entries[name]) raise if SUPERNOVA.VELOCITY not in catalog.entries[name]: source = catalog.entries[name].add_self_source() # FIX: what's happening here?! pnum = CLIGHT / KM * \ ((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.) pnum = pretty_num(pnum, sig=bestsig) catalog.entries[name].add_quantity( SUPERNOVA.VELOCITY, pnum, source, kind=(SUPERNOVA.VELOCITY.kind_preference[bestkind] if bestkind else '')) if bestz > 0.: if SUPERNOVA.LUM_DIST not in catalog.entries[name]: dl = cosmo.luminosity_distance(bestz) sources = [ catalog.entries[name].add_self_source(), catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') ] sources = uniq_cdl(sources + bestsrc.split(',')) catalog.entries[name].add_quantity( SUPERNOVA.LUM_DIST, pretty_num(dl.value, sig=bestsig + 1), sources, kind=(SUPERNOVA.LUM_DIST.kind_preference[bestkind] if bestkind else ''), derived=True) if (SUPERNOVA.MAX_ABS_MAG not in catalog.entries[name] and SUPERNOVA.MAX_APP_MAG in catalog.entries[name]): source = catalog.entries[name].add_self_source() pnum = pretty_num( float(catalog.entries[name][ SUPERNOVA.MAX_APP_MAG][0][QUANTITY.VALUE]) - 5.0 * (log10(dl.to('pc').value) - 1.0) + 2.5 * log10(1.0 + bestz), sig=bestsig + 1) catalog.entries[name].add_quantity( SUPERNOVA.MAX_ABS_MAG, pnum, sources, derived=True) if (SUPERNOVA.MAX_VISUAL_ABS_MAG not in catalog.entries[name] and SUPERNOVA.MAX_VISUAL_APP_MAG in catalog.entries[name]): source = catalog.entries[name].add_self_source() pnum = pretty_num(float(catalog.entries[name][ SUPERNOVA.MAX_VISUAL_APP_MAG][0][ QUANTITY.VALUE]) - 5.0 * (log10(dl.to('pc').value) - 1.0), sig=bestsig + 1) catalog.entries[name].add_quantity( SUPERNOVA.MAX_VISUAL_ABS_MAG, pnum, sources, derived=True) if SUPERNOVA.COMOVING_DIST not in catalog.entries[name]: cd = cosmo.comoving_distance(bestz) sources = [ catalog.entries[name].add_self_source(), catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') ] sources = uniq_cdl(sources + bestsrc.split(',')) catalog.entries[name].add_quantity( SUPERNOVA.COMOVING_DIST, pretty_num(cd.value, sig=bestsig), sources, derived=True) if SUPERNOVA.HOST_REDSHIFT in catalog.entries[name]: # Find the "best" redshift to use for this bestz, bestkind, bestsig, bestsrc = catalog.entries[ name].get_best_redshift(SUPERNOVA.HOST_REDSHIFT) if bestsig > 0: try: bestz = float(bestz) except Exception: print(catalog.entries[name]) raise if SUPERNOVA.HOST_VELOCITY not in catalog.entries[name]: source = catalog.entries[name].add_self_source() # FIX: what's happening here?! pnum = CLIGHT / KM * \ ((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.) pnum = pretty_num(pnum, sig=bestsig) catalog.entries[name].add_quantity( SUPERNOVA.HOST_VELOCITY, pnum, source, kind=(SUPERNOVA.HOST_VELOCITY.kind_preference[bestkind] if bestkind else '')) if bestz > 0.: if SUPERNOVA.HOST_LUM_DIST not in catalog.entries[name]: dl = cosmo.luminosity_distance(bestz) sources = [ catalog.entries[name].add_self_source(), catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') ] sources = uniq_cdl(sources + bestsrc.split(',')) catalog.entries[name].add_quantity( SUPERNOVA.HOST_LUM_DIST, pretty_num(dl.value, sig=bestsig + 1), sources, kind=(SUPERNOVA.HOST_LUM_DIST. kind_preference[bestkind] if bestkind else ''), derived=True) if SUPERNOVA.HOST_COMOVING_DIST not in catalog.entries[ name]: cd = cosmo.comoving_distance(bestz) sources = [ catalog.entries[name].add_self_source(), catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') ] sources = uniq_cdl(sources + bestsrc.split(',')) catalog.entries[name].add_quantity( SUPERNOVA.HOST_COMOVING_DIST, pretty_num(cd.value, sig=bestsig), sources, derived=True) if all([ x in catalog.entries[name] for x in [ SUPERNOVA.RA, SUPERNOVA.DEC, SUPERNOVA.HOST_RA, SUPERNOVA.HOST_DEC ] ]): # For now just using first coordinates that appear in entry try: c1 = coord( ra=catalog.entries[name][SUPERNOVA.RA][0][QUANTITY.VALUE], dec=catalog.entries[name][SUPERNOVA.DEC][0][ QUANTITY.VALUE], unit=(un.hourangle, un.deg)) c2 = coord(ra=catalog.entries[name][SUPERNOVA.HOST_RA][0][ QUANTITY.VALUE], dec=catalog.entries[name][SUPERNOVA.HOST_DEC][0][ QUANTITY.VALUE], unit=(un.hourangle, un.deg)) except (KeyboardInterrupt, SystemExit): raise except Exception: pass else: sources = uniq_cdl([catalog.entries[name].add_self_source()] + catalog.entries[name][SUPERNOVA.RA][0][ QUANTITY.SOURCE].split(',') + catalog.entries[name][SUPERNOVA.DEC][0][ QUANTITY.SOURCE].split(',') + catalog.entries[name][SUPERNOVA.HOST_RA][0][ QUANTITY.SOURCE].split(',') + catalog.entries[name][SUPERNOVA.HOST_DEC][0] [QUANTITY.SOURCE].split(',')) if SUPERNOVA.HOST_OFFSET_ANG not in catalog.entries[name]: hosa = Decimal(c1.separation(c2).arcsecond) hosa = pretty_num(hosa) catalog.entries[name].add_quantity( SUPERNOVA.HOST_OFFSET_ANG, hosa, sources, derived=True, u_value='arcseconds') if (SUPERNOVA.COMOVING_DIST in catalog.entries[name] and SUPERNOVA.REDSHIFT in catalog.entries[name] and SUPERNOVA.HOST_OFFSET_DIST not in catalog.entries[name]): offsetsig = get_sig_digits(catalog.entries[name][ SUPERNOVA.HOST_OFFSET_ANG][0][QUANTITY.VALUE]) sources = uniq_cdl( sources.split(',') + (catalog.entries[name][SUPERNOVA.COMOVING_DIST][0][ QUANTITY.SOURCE]).split(',') + (catalog.entries[name][SUPERNOVA.REDSHIFT][0][ QUANTITY.SOURCE]).split(',')) (catalog.entries[name].add_quantity( SUPERNOVA.HOST_OFFSET_DIST, pretty_num( float(catalog.entries[name][ SUPERNOVA.HOST_OFFSET_ANG][0][QUANTITY.VALUE]) / 3600. * (pi / 180.) * float(catalog.entries[name][ SUPERNOVA.COMOVING_DIST][0][QUANTITY.VALUE]) * 1000. / (1.0 + float(catalog.entries[name][ SUPERNOVA.REDSHIFT][0][QUANTITY.VALUE])), sig=offsetsig), sources)) catalog.entries[name].sanitize() catalog.journal_entries(bury=True, final=True, gz=True) cleanupcnt = cleanupcnt + 1 if catalog.args.travis and cleanupcnt % 1000 == 0: break catalog.save_caches() return
def do_cleanup(catalog): """Cleanup catalog after importing all data.""" task_str = catalog.get_current_task_str() # Set preferred names, calculate some columns based on imported data, # sanitize some fields keys = list(catalog.entries.keys()) cleanupcnt = 0 for oname in pbar(keys, task_str): # Some events may be merged in cleanup process, skip them if # non-existent. try: name = catalog.add_entry(oname) except Exception: catalog.log.warning( '"{}" was not found, suggests merge occurred in cleanup ' 'process.'.format(oname)) continue # Set the preferred name, switching to that name if name changed. name = catalog.entries[name].set_preferred_name() aliases = catalog.entries[name].get_aliases() catalog.entries[name].set_first_max_light() # Clean discoverer field if FASTSTARS.DISCOVERER in catalog.entries[name]: if len(catalog.entries[name][FASTSTARS.DISCOVERER]) > 1: POSSIBLEDISCOVERER = [ catalog.entries[name][FASTSTARS.DISCOVERER][i]['value'] for i in range( len(catalog.entries[name][FASTSTARS.DISCOVERER])) ] POSSIBLEDISCOVERER_DATE = [ int(DATE['value']) for DATE in catalog.entries[name][FASTSTARS.DISCOVER_DATE] ] POSSIBLEDISCOVERER_DATE_SOURCES = [ DATE['source'] for DATE in catalog.entries[name][FASTSTARS.DISCOVER_DATE] ] EARLIESTSOURCE = POSSIBLEDISCOVERER_DATE_SOURCES[np.argmin( POSSIBLEDISCOVERER_DATE)] EARLIESTDISCOVER_DATE = catalog.entries[name][ FASTSTARS.DISCOVER_DATE][np.argmin( POSSIBLEDISCOVERER_DATE)] # Deal with case where a star was 'discovered' multiple times in one year if ',' in EARLIESTSOURCE: EARLIESTSOURCE = EARLIESTSOURCE.split(',')[0] for DISCOVERER in catalog.entries[name][FASTSTARS.DISCOVERER]: for DISCOVERERSOURCE in DISCOVERER['source'].split(','): if DISCOVERERSOURCE == EARLIESTSOURCE: EARLIESTDISCOVERER = DISCOVERER for DISCOVERER in catalog.entries[name][FASTSTARS.DISCOVERER]: for DISCOVERERSOURCE in DISCOVERER['source'].split(','): if DISCOVERERSOURCE == EARLIESTSOURCE: EARLIESTDISCOVERER = DISCOVERER catalog.entries[name][FASTSTARS.DISCOVERER] = [ EARLIESTDISCOVERER ] catalog.entries[name][FASTSTARS.DISCOVER_DATE] = [ EARLIESTDISCOVER_DATE ] # Convert all distances to kpc. if FASTSTARS.LUM_DIST in catalog.entries[name]: for li, ld in enumerate(catalog.entries[name][FASTSTARS.LUM_DIST]): if ld.get('u_value') != 'kpc': if ld.get('u_value') == 'pc': catalog.entries[name][ FASTSTARS.LUM_DIST][li]['value'] = str( Decimal(catalog.entries[name][ FASTSTARS.LUM_DIST][li]['value']) * Decimal('0.001')) elif ld.get('u_value') == 'Mpc': catalog.entries[name][ FASTSTARS.LUM_DIST][li]['value'] = str( Decimal(catalog.entries[name][ FASTSTARS.LUM_DIST][li]['value']) * Decimal('1000')) else: raise ValueError('unknown distance unit') catalog.entries[name][ FASTSTARS.LUM_DIST][li]['u_value'] = 'kpc' if (FASTSTARS.RA not in catalog.entries[name] or FASTSTARS.DEC not in catalog.entries[name]): prefixes = ['SDSS'] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:6])): noprefix = alias.split(':')[-1].replace(prefix, '').replace( '.', '') decsign = '+' if '+' in noprefix else '-' noprefix = noprefix.replace('+', '|').replace('-', '|') nops = noprefix.split('|') if len(nops) < 2: continue rastr = nops[0] decstr = nops[1] ra = ':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + \ ('.' + rastr[6:] if len(rastr) > 6 else '') dec = ( decsign + ':'.join([decstr[:2], decstr[2:4], decstr[4:6]]) + ('.' + decstr[6:] if len(decstr) > 6 else '')) if catalog.args.verbose: tprint('Added ra/dec from name: ' + ra + ' ' + dec) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity(FASTSTARS.RA, ra, source, derived=True) catalog.entries[name].add_quantity(FASTSTARS.DEC, dec, source, derived=True) break if FASTSTARS.RA in catalog.entries[name]: break if (FASTSTARS.MAX_ABS_MAG not in catalog.entries[name] and FASTSTARS.MAX_APP_MAG in catalog.entries[name] and FASTSTARS.LUM_DIST in catalog.entries[name]): # Find the "best" distance to use for this bestsig = 0 for ld in catalog.entries[name][FASTSTARS.LUM_DIST]: sig = get_sig_digits(ld[QUANTITY.VALUE]) if sig > bestsig: bestld = ld[QUANTITY.VALUE] bestsrc = ld[QUANTITY.SOURCE] bestsig = sig if bestsig > 0 and is_number(bestld) and float(bestld) > 0.: source = catalog.entries[name].add_self_source() sources = uniq_cdl([source] + bestsrc.split(',')) bestldz = z_at_value(cosmo.luminosity_distance, float(bestld) * un.Mpc) pnum = (float(catalog.entries[name][FASTSTARS.MAX_APP_MAG][0][ QUANTITY.VALUE]) - 5.0 * (log10(float(bestld) * 1.0e6) - 1.0) + 2.5 * log10(1.0 + bestldz)) pnum = pretty_num(pnum, sig=bestsig + 1) catalog.entries[name].add_quantity(FASTSTARS.MAX_ABS_MAG, pnum, sources, derived=True) catalog.entries[name].sanitize() catalog.journal_entries(bury=True, final=True, gz=True) cleanupcnt = cleanupcnt + 1 if catalog.args.travis and cleanupcnt % 1000 == 0: break catalog.save_caches() return
def do_ucb_spectra(catalog): task_str = catalog.get_current_task_str() sec_reference = 'UCB Filippenko Group\'s Supernova Database (SNDB)' sec_refurl = 'http://heracles.astro.berkeley.edu/sndb/info' sec_refbib = '2012MNRAS.425.1789S' ucbspectracnt = 0 jsontxt = catalog.load_url( 'http://heracles.astro.berkeley.edu/sndb/download?id=allpubspec', os.path.join(catalog.get_current_task_repo(), 'UCB/allpubspec.json'), json_sort='SpecID') if not jsontxt: return spectra = json.loads(jsontxt) spectra = sorted(spectra, key=lambda kk: kk['SpecID']) oldname = '' for spectrum in pbar(spectra, task_str): name = spectrum['ObjName'] if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) sec_source = catalog.entries[name].add_source( name=sec_reference, url=sec_refurl, bibcode=sec_refbib, secondary=True) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, sec_source) sources = [sec_source] if spectrum['Reference']: sources += [catalog.entries[name] .add_source(bibcode=spectrum['Reference'])] sources = uniq_cdl(sources) if spectrum['Type'] and spectrum['Type'].strip() != 'NoMatch': for ct in spectrum['Type'].strip().split(','): catalog.entries[name].add_quantity( SUPERNOVA.CLAIMED_TYPE, ct.replace('-norm', '').strip(), sources) if spectrum['DiscDate']: ddate = spectrum['DiscDate'].replace('-', '/') catalog.entries[name].add_quantity(SUPERNOVA.DISCOVER_DATE, ddate, sources) if spectrum['HostName']: host = urllib.parse.unquote(spectrum['HostName']).replace('*', '') catalog.entries[name].add_quantity(SUPERNOVA.HOST, host, sources) if spectrum['UT_Date']: epoch = str(spectrum['UT_Date']) year = epoch[:4] month = epoch[4:6] day = epoch[6:] sig = get_sig_digits(day) + 5 mjd = astrotime(year + '-' + month + '-' + str(floor(float( day))).zfill(2)).mjd mjd = pretty_num(mjd + float(day) - floor(float(day)), sig=sig) filename = spectrum['Filename'] if spectrum['Filename'] else '' instrument = spectrum['Instrument'] if spectrum['Instrument'] else '' reducer = spectrum['Reducer'] if spectrum['Reducer'] else '' observer = spectrum['Observer'] if spectrum['Observer'] else '' snr = str(spectrum['SNR']) if spectrum['SNR'] else '' if not filename: raise ValueError('Filename not found for SNDB spectrum!') if not spectrum['SpecID']: raise ValueError('ID not found for SNDB spectrum!') filepath = os.path.join(catalog.get_current_task_repo(), 'UCB/') + filename spectxt = catalog.load_url( 'http://heracles.astro.berkeley.edu/sndb/download?id=ds:' + str(spectrum['SpecID']), filepath, archived_mode=True) specdata = list( csv.reader( spectxt.splitlines(), delimiter=' ', skipinitialspace=True)) newspecdata = [] for row in specdata: if row[0][0] == '#': continue else: newspecdata.append(row) specdata = newspecdata haserrors = len(specdata[0]) == 3 and specdata[0][2] and specdata[0][ 2] != 'NaN' specdata = [list(ii) for ii in zip(*specdata)] wavelengths = specdata[0] fluxes = specdata[1] errors = '' if haserrors: errors = specdata[2] if not list(filter(None, errors)): errors = '' units = 'Uncalibrated' catalog.entries[name].add_spectrum( u_wavelengths='Angstrom', u_fluxes=units, u_time='MJD', time=mjd, wavelengths=wavelengths, filename=filename, fluxes=fluxes, errors=errors, u_errors=units, instrument=instrument, source=sources, snr=snr, observer=observer, reducer=reducer, deredshifted=('-noz' in filename)) ucbspectracnt = ucbspectracnt + 1 if catalog.args.travis and ucbspectracnt >= catalog.TRAVIS_QUERY_LIMIT: break catalog.journal_entries() return
def do_nedd(catalog): task_str = catalog.get_current_task_str() nedd_path = os.path.join(catalog.get_current_task_repo(), 'NED26.10.1-D-13.1.0-20160930.csv') f = open(nedd_path, 'r') data = sorted(list(csv.reader(f, delimiter=',', quotechar='"'))[13:], key=lambda x: (x[9], x[3])) reference = "NED-D v" + nedd_path.split('-')[-2] refurl = "http://ned.ipac.caltech.edu/Library/Distances/" nedbib = "1991ASSL..171...89H" olddistname = '' loopcnt = 0 for r, row in enumerate(pbar(data, task_str)): if r <= 12: continue distname = row[3] name = name_clean(distname) # distmod = row[4] # moderr = row[5] dist = row[6] bibcode = unescape(row[8]) snname = name_clean(row[9]) redshift = row[10] cleanhost = '' if name != snname and (name + ' HOST' != snname): cleanhost = host_clean(distname) if cleanhost.endswith(' HOST'): cleanhost = '' if not is_number(dist): print(dist) if dist: catalog.nedd_dict.setdefault(cleanhost, []).append(Decimal(dist)) if snname and 'HOST' not in snname: snname, secondarysource = catalog.new_entry(snname, srcname=reference, bibcode=nedbib, url=refurl, secondary=True) if bibcode: source = catalog.entries[snname].add_source(bibcode=bibcode) sources = uniq_cdl([source, secondarysource]) else: sources = secondarysource if name == snname: if redshift: catalog.entries[snname].add_quantity( SUPERNOVA.REDSHIFT, redshift, sources) if dist: catalog.entries[snname].add_quantity( SUPERNOVA.COMOVING_DIST, dist, sources) if not redshift: try: zatval = z_at_value(cosmo.comoving_distance, float(dist) * un.Mpc, zmax=5.0) sigd = get_sig_digits(str(dist)) redshift = pretty_num(zatval, sig=sigd) except (KeyboardInterrupt, SystemExit): raise except Exception: pass else: cosmosource = catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') combsources = uniq_cdl( sources.split(',') + [cosmosource]) catalog.entries[snname].add_quantity( SUPERNOVA.REDSHIFT, redshift, combsources, derived=True) if cleanhost: catalog.entries[snname].add_quantity(SUPERNOVA.HOST, cleanhost, sources) if catalog.args.update and olddistname != distname: catalog.journal_entries() olddistname = distname loopcnt = loopcnt + 1 if catalog.args.travis and loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0: break catalog.journal_entries() f.close() return
def add_quantity(self, quantities, value, source, forcereplacebetter=False, **kwargs): """Add `Quantity` to `FastStars`.""" success = super(FastStars, self).add_quantity( quantities, value, source, **kwargs) if not success: return for quantity in listify(quantities): my_quantity_list = self.get(quantity, []) if ((forcereplacebetter or quantity.replace_better) and len(my_quantity_list) > 1): # The quantity that was just added should be last in the list added_quantity = my_quantity_list.pop() newquantities = [] isworse = True if type(quantity) != Key: isworse = False elif quantity.type == KEY_TYPES.NUMERIC: newsig = get_sig_digits(added_quantity[QUANTITY.VALUE]) for ct in my_quantity_list: addct = False checke = False if (len(quantity.kind_preference) > 0 and not set( listify(ct.get(QUANTITY.KIND, []))) .isdisjoint(quantity.kind_preference) and not set( listify( added_quantity.get(QUANTITY.KIND, []))) .isdisjoint(quantity.kind_preference)): aqi = min([ quantity.kind_preference.index(x) for x in listify(added_quantity[ QUANTITY.KIND]) ]) qqi = min([ quantity.kind_preference.index(x) for x in listify(ct[QUANTITY.KIND]) ]) if aqi > qqi: addct = True if aqi == qqi: checke = True if aqi <= qqi: isworse = False else: checke = True if checke and QUANTITY.CORRELATIONS in ct: if QUANTITY.CORRELATIONS in added_quantity: if len(ct[QUANTITY.CORRELATIONS]) > len(added_quantity[QUANTITY.CORRELATIONS]): addct = True checke = False else: addct = True checke = False if checke and QUANTITY.E_VALUE in ct: if QUANTITY.E_VALUE in added_quantity: if (float(added_quantity[QUANTITY.E_VALUE]) >= float(ct[QUANTITY.E_VALUE])): addct = True if (float(added_quantity[QUANTITY.E_VALUE]) <= float(ct[QUANTITY.E_VALUE])): isworse = False if QUANTITY.E_LOWER_VALUE in added_quantity and QUANTITY.E_UPPER_VALUE in added_quantity: if (0.5*float(added_quantity[QUANTITY.E_LOWER_VALUE])+0.5*float(added_quantity[QUANTITY.E_UPPER_VALUE]) >= float(ct[QUANTITY.E_VALUE])): addct = True if (0.5*float(added_quantity[QUANTITY.E_LOWER_VALUE])+0.5*float(added_quantity[QUANTITY.E_UPPER_VALUE]) <= float(ct[QUANTITY.E_VALUE])): isworse = False elif checke and QUANTITY.E_LOWER_VALUE in ct and QUANTITY.E_UPPER_VALUE in ct: if QUANTITY.E_VALUE in added_quantity: if (float(added_quantity[QUANTITY.E_VALUE]) >= 0.5*float(ct[QUANTITY.E_LOWER_VALUE])+0.5*float(ct[QUANTITY.E_UPPER_VALUE])): addct = True if (float(added_quantity[QUANTITY.E_VALUE]) <= 0.5*float(ct[QUANTITY.E_LOWER_VALUE])+0.5*float(ct[QUANTITY.E_UPPER_VALUE])): isworse = False if QUANTITY.E_LOWER_VALUE in added_quantity and QUANTITY.E_UPPER_VALUE in added_quantity: if (0.5*float(added_quantity[QUANTITY.E_LOWER_VALUE])+0.5*float(added_quantity[QUANTITY.E_UPPER_VALUE]) >= 0.5*float(ct[QUANTITY.E_LOWER_VALUE])+0.5*float(ct[QUANTITY.E_UPPER_VALUE])): addct = True if (0.5*float(added_quantity[QUANTITY.E_LOWER_VALUE])+0.5*float(added_quantity[QUANTITY.E_UPPER_VALUE]) <= 0.5*float(ct[QUANTITY.E_LOWER_VALUE])+0.5*float(ct[QUANTITY.E_UPPER_VALUE])): isworse = False else: if (checke and ((QUANTITY.E_VALUE in added_quantity) or (QUANTITY.E_LOWER_VALUE in added_quantity and QUANTITY.E_UPPER_VALUE in added_quantity))): isworse = False elif (checke and QUANTITY.CORRELATIONS in added_quantity): isworse = False else: oldsig = get_sig_digits(ct[QUANTITY.VALUE]) if oldsig >= newsig: addct = True if newsig >= oldsig: isworse = False if addct: newquantities.append(ct) elif quantity.type == KEY_TYPES.STRING: for ct in my_quantity_list: addct = False if (len(quantity.kind_preference) > 0 and not set( listify(ct.get(QUANTITY.KIND, []))) .isdisjoint(quantity.kind_preference) and not set( listify( added_quantity.get(QUANTITY.KIND, []))) .isdisjoint(quantity.kind_preference)): aqi = min([ quantity.kind_preference.index(x) for x in listify(added_quantity[ QUANTITY.KIND]) ]) qqi = min([ quantity.kind_preference.index(x) for x in listify(ct[QUANTITY.KIND]) ]) if aqi >= qqi: addct = True if aqi <= qqi: isworse = False else: addct = True isworse = False if addct: newquantities.append(ct) if isworse: self._log.info("Removing quantity '{}' with value '{}' " "and kind '{}' determined to be worse than " "existing alternative values.".format( quantity, added_quantity[ QUANTITY.VALUE], added_quantity.get(QUANTITY.KIND, ''))) else: newquantities.append(added_quantity) if len(newquantities) > 0: self[quantity] = newquantities return True
def do_nedd(catalog): task_str = catalog.get_current_task_str() nedd_path = os.path.join( catalog.get_current_task_repo(), 'NED26.10.1-D-13.1.0-20160930.csv') f = open(nedd_path, 'r') data = sorted(list(csv.reader(f, delimiter=',', quotechar='"'))[ 13:], key=lambda x: (x[9], x[3])) reference = "NED-D v" + nedd_path.split('-')[-2] refurl = "http://ned.ipac.caltech.edu/Library/Distances/" nedbib = "1991ASSL..171...89H" olddistname = '' loopcnt = 0 for r, row in enumerate(pbar(data, task_str)): if r <= 12: continue distname = row[3] name = name_clean(distname) # distmod = row[4] # moderr = row[5] dist = row[6] bibcode = unescape(row[8]) snname = name_clean(row[9]) redshift = row[10] cleanhost = '' if name != snname and (name + ' HOST' != snname): cleanhost = host_clean(distname) if cleanhost.endswith(' HOST') or cleanhost.startswith('SN'): cleanhost = '' if not is_number(dist): print(dist) if dist and cleanhost: catalog.nedd_dict.setdefault( cleanhost, []).append(Decimal(dist)) if snname and 'HOST' not in snname: snname, secondarysource = catalog.new_entry( snname, srcname=reference, bibcode=nedbib, url=refurl, secondary=True) if bibcode: source = catalog.entries[snname].add_source(bibcode=bibcode) sources = uniq_cdl([source, secondarysource]) else: sources = secondarysource if name == snname: if redshift: catalog.entries[snname].add_quantity( SUPERNOVA.REDSHIFT, redshift, sources) if dist: catalog.entries[snname].add_quantity( SUPERNOVA.COMOVING_DIST, dist, sources) if not redshift: try: zatval = z_at_value(cosmo.comoving_distance, float(dist) * un.Mpc, zmax=5.0) sigd = get_sig_digits(str(dist)) redshift = pretty_num(zatval, sig=sigd) except (KeyboardInterrupt, SystemExit): raise except Exception: pass else: cosmosource = catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') combsources = uniq_cdl(sources.split(',') + [cosmosource]) catalog.entries[snname].add_quantity( SUPERNOVA.REDSHIFT, redshift, combsources, derived=True) if cleanhost: catalog.entries[snname].add_quantity( SUPERNOVA.HOST, cleanhost, sources) if catalog.args.update and olddistname != distname: catalog.journal_entries() olddistname = distname loopcnt = loopcnt + 1 if catalog.args.travis and loopcnt % catalog.TRAVIS_QUERY_LIMIT == 0: break catalog.journal_entries() f.close() return
def do_suspect_spectra(catalog): task_str = catalog.get_current_task_str() with open( os.path.join(catalog.get_current_task_repo(), 'Suspect/sources.json'), 'r') as f: sourcedict = json.loads(f.read()) with open( os.path.join(catalog.get_current_task_repo(), 'Suspect/filename-changes.txt'), 'r') as f: rows = f.readlines() changedict = {} for row in rows: if not row.strip() or row[0] == "#": continue items = row.strip().split(' ') changedict[items[1]] = items[0] suspectcnt = 0 folders = next( os.walk(os.path.join(catalog.get_current_task_repo(), 'Suspect')))[1] for folder in pbar(folders, task_str): eventfolders = next( os.walk( os.path.join(catalog.get_current_task_repo(), 'Suspect/') + folder))[1] oldname = '' for eventfolder in pbar(eventfolders, task_str): name = eventfolder if is_number(name[:4]): name = 'SN' + name name = catalog.get_preferred_name(name) if oldname and name != oldname: catalog.journal_entries() oldname = name name = catalog.add_entry(name) sec_ref = 'SUSPECT' sec_refurl = 'https://www.nhn.ou.edu/~suspect/' sec_bibc = '2001AAS...199.8408R' sec_source = catalog.entries[name].add_source(name=sec_ref, url=sec_refurl, bibcode=sec_bibc, secondary=True) catalog.entries[name].add_quantity(SUPERNOVA.ALIAS, name, sec_source) fpath = os.path.join(catalog.get_current_task_repo(), 'Suspect', folder, eventfolder) eventspectra = next(os.walk(fpath))[2] for spectrum in eventspectra: sources = [sec_source] bibcode = '' if spectrum in changedict: specalias = changedict[spectrum] else: specalias = spectrum if specalias in sourcedict: bibcode = sourcedict[specalias] elif name in sourcedict: bibcode = sourcedict[name] if bibcode: source = catalog.entries[name].add_source( bibcode=unescape(bibcode)) sources += [source] sources = uniq_cdl(sources) date = spectrum.split('_')[1] year = date[:4] month = date[4:6] day = date[6:] sig = get_sig_digits(day) + 5 day_fmt = str(floor(float(day))).zfill(2) time = astrotime(year + '-' + month + '-' + day_fmt).mjd time = time + float(day) - floor(float(day)) time = pretty_num(time, sig=sig) fpath = os.path.join(catalog.get_current_task_repo(), 'Suspect', folder, eventfolder, spectrum) with open(fpath, 'r') as f: specdata = list( csv.reader(f, delimiter=' ', skipinitialspace=True)) specdata = list(filter(None, specdata)) newspec = [] oldval = '' for row in specdata: if row[1] == oldval: continue newspec.append(row) oldval = row[1] specdata = newspec haserrors = len( specdata[0] ) == 3 and specdata[0][2] and specdata[0][2] != 'NaN' specdata = [list(i) for i in zip(*specdata)] wavelengths = specdata[0] fluxes = specdata[1] errors = '' if haserrors: errors = specdata[2] catalog.entries[name].add_spectrum(u_wavelengths='Angstrom', u_fluxes='Uncalibrated', u_time='MJD', time=time, wavelengths=wavelengths, fluxes=fluxes, errors=errors, u_errors='Uncalibrated', source=sources, filename=spectrum) suspectcnt = suspectcnt + 1 if (catalog.args.travis and suspectcnt % catalog.TRAVIS_QUERY_LIMIT == 0): break catalog.journal_entries() return
def add_quantity(self, quantities, value, source, forcereplacebetter=False, **kwargs): success = super().add_quantity(quantities, value, source, **kwargs) if not success: return for quantity in listify(quantities): my_quantity_list = self.get(quantity, []) if ((forcereplacebetter or quantity.replace_better) and len(my_quantity_list) > 1): # The quantity that was just added should be last in the list added_quantity = my_quantity_list.pop() newquantities = [] isworse = True if quantity in [self._KEYS.DISCOVER_DATE, self._KEYS.MAX_DATE]: for ct in my_quantity_list: ctsplit = ct[QUANTITY.VALUE].split('/') svsplit = added_quantity[QUANTITY.VALUE].split('/') if len(ctsplit) < len(svsplit): isworse = False continue elif len(ctsplit) < len(svsplit) and len(svsplit) == 3: val_one = max( 2, get_sig_digits(ctsplit[-1].lstrip('0'))) val_two = max( 2, get_sig_digits(svsplit[-1].lstrip('0'))) if val_one < val_two: isworse = False continue newquantities.append(ct) else: if type(quantity) != Key: isworse = False elif quantity.type == KEY_TYPES.NUMERIC: newsig = get_sig_digits(added_quantity[QUANTITY.VALUE]) for ct in my_quantity_list: addct = False checke = False if (len(quantity.kind_preference) > 0 and not set( listify(ct.get(QUANTITY.KIND, []))) .isdisjoint(quantity.kind_preference) and not set( listify( added_quantity.get(QUANTITY.KIND, []))) .isdisjoint(quantity.kind_preference)): aqi = min([ quantity.kind_preference.index(x) for x in listify(added_quantity[ QUANTITY.KIND]) ]) qqi = min([ quantity.kind_preference.index(x) for x in listify(ct[QUANTITY.KIND]) ]) if aqi > qqi: addct = True if aqi == qqi: checke = True if aqi <= qqi: isworse = False else: checke = True if checke and QUANTITY.E_VALUE in ct: if QUANTITY.E_VALUE in added_quantity: if (float(added_quantity[QUANTITY.E_VALUE]) >= float(ct[QUANTITY.E_VALUE])): addct = True if (float(added_quantity[QUANTITY.E_VALUE]) <= float(ct[QUANTITY.E_VALUE])): isworse = False else: if (checke and QUANTITY.E_VALUE in added_quantity): isworse = False else: oldsig = get_sig_digits(ct[QUANTITY.VALUE]) if oldsig >= newsig: addct = True if newsig >= oldsig: isworse = False if addct: newquantities.append(ct) elif quantity.type == KEY_TYPES.STRING: for ct in my_quantity_list: addct = False if (len(quantity.kind_preference) > 0 and not set( listify(ct.get(QUANTITY.KIND, []))) .isdisjoint(quantity.kind_preference) and not set( listify( added_quantity.get(QUANTITY.KIND, []))) .isdisjoint(quantity.kind_preference)): aqi = min([ quantity.kind_preference.index(x) for x in listify(added_quantity[ QUANTITY.KIND]) ]) qqi = min([ quantity.kind_preference.index(x) for x in listify(ct[QUANTITY.KIND]) ]) if aqi >= qqi: addct = True if aqi <= qqi: isworse = False else: isworse = False if addct: newquantities.append(ct) if isworse: self._log.info("Removing quantity '{}' with value '{}' " "and kind '{}' determined to be worse than " "existing alternative values.".format( quantity, added_quantity[ QUANTITY.VALUE], added_quantity.get(QUANTITY.KIND, ''))) else: newquantities.append(added_quantity) if len(newquantities) > 0: self[quantity] = newquantities return