def Pbdot_Gal(pf): try: name = pf.PSRJ except: name = pf.PSR ra = RA(pf.RAJ[0]) dec = Dec(pf.DECJ[0]) #pos = coord.FK5Coordinates(str(ra) +' '+ str(dec)) pos = coord(str(ra), str(dec), unit=(u.deg, u.hourangle) ) l = pos.galactic.l.radian b = pos.galactic.b.radian Pb = float(pf.PB[0] * secperday) #d = AU * 180 / PI * 3600 / pf.PX[0] * 1000 d = float(1/pf.PX[0]) pf.DIST = d z_kpc = float(d)*(abs(sin(b)))#/kpc must be positive a_z = ((2.27)*z_kpc + (3.68)*(1 - exp((-4.31)*z_kpc)))*(1.e-9) #cm s^-2 #print 'a_z:', a_z A_z = -1 * a_z *abs(sin(b))/c pf.A_z = A_z R0 = (8.34) #* kpc # Reid et al. 2014 beta = float(d/R0) * cos(b) - cos(l) Omega0 = (240. * 1.e5) #240+/-8 km/s; Reid et al 2014 #print b,l, cos(b), cos(l), beta A_x = -1/c * (cos(b)) * (Omega0**2/R0/kpc) * (cos(l) + beta/(sin(l)**2 + beta**2)) pf.A_x = A_x #print 'Ax, Az: ',A_x, A_z fac1 = float(pf.PX[1]/pf.PX[0]) fac2 = 8/240 #Omega_0 fac3 = 0.16/8.34 #R0 Reid et al 2014 val = float(Pb*(A_z + A_x)) err1 = A_x * fac1 err2 = A_z * sqrt(fac1**2 + fac2**2 + fac3**2) err = sqrt(err1**2 + err2**2) * float(Pb) return val, err
def radecs_to_lb(ras, decs): """ Transformation between lists of ras, decs, to ls, bs. Assumes ra, dec in degrees Conforms to astropy 0.4.3 taken from https://github.com/seclark/FITSHandling/commit/f04a6e54c6624741e4f3077ba8ba96af620871ac for lb masks """ obj = coord(ras, decs, unit="deg", frame="icrs") obj = obj.galactic ls = obj.l.degree bs = obj.b.degree return ls, bs
def do_ppmxl(catalog): task_str = catalog.get_current_task_str() keys = list(catalog.entries.keys()) warnings.filterwarnings("ignore") for oname in pbar(keys, task_str): # Some events may be merged in cleanup process, skip them if # non-existent. try: name = catalog.add_entry(oname) except Exception: catalog.log.warning( '"{}" was not found, suggests merge occurred in cleanup ' 'process.'.format(oname)) continue if (FASTSTARS.RA not in catalog.entries[name] or FASTSTARS.DEC not in catalog.entries[name]): continue else: radec= str(catalog.entries[name][FASTSTARS.RA][0]['value'])+str(catalog.entries[name][FASTSTARS.DEC][0]['value']) c=coord(radec,unit=(un.hourangle, un.deg),frame='icrs') cnttry = 0 foundstar = False while foundstar == False and cnttry < 1: try: cnttry += 1 time.sleep(0.1) result = Irsa.query_region(c,catalog='ppmxl',radius='0d0m10s') except TypeError: #print(radec,cnttry) continue if len(result) > 1: foundstar = True if foundstar == True: source = (catalog.entries[name] .add_source(name='The PPMXL Catalog', bibcode="2010AJ....139.2440R", url="https://irsa.ipac.caltech.edu/Missions/ppmxl.html", secondary=True)) catalog.entries[name].add_quantity(FASTSTARS.PROPER_MOTION_RA, str(result['pmra'][0]*degperyrtomasperyear), source, e_value=str(result['e_pmra'][0]*degperyrtomasperyear), u_value='mas/yr') catalog.entries[name].add_quantity(FASTSTARS.PROPER_MOTION_DEC, str(result['pmde'][0]*degperyrtomasperyear), source, e_value=str(result['e_pmde'][0]*degperyrtomasperyear), u_value='mas/yr') catalog.journal_entries() return
def do_ptss_meta(catalog): """Import metadata from PTSS webpage.""" task_str = catalog.get_current_task_str() years = list(range(2015, datetime.today().year + 1)) for year in years: jsontxt = None while jsontxt is None: try: jsontxt = catalog.load_url( 'http://www.cneost.org/ptss/fetchlist.php?vip=sn&gdate=' + str(year), os.path.join(catalog.get_current_task_repo(), 'PTSS/catalog-' + str(year) + '.json'), json_sort='name', timeout=5) except Exception: pass meta = json.loads(jsontxt) for met in pbar(meta, task_str + ' - ' + str(year)): oldname = met['name'] name, source = catalog.new_entry( oldname, srcname='PMO & Tsinghua Supernova Survey (PTSS)', url='http://www.cneost.org/ptss/index.php') coo = coord(met['ra'], met['dec'], unit=(un.deg, un.deg)) catalog.entries[name].add_quantity( SUPERNOVA.RA, coo.ra.to_string(unit=un.hour, sep=':'), source) catalog.entries[name].add_quantity( SUPERNOVA.DEC, coo.dec.to_string(unit=un.degree, sep=':'), source) if met['filter'] is not None: mjd = str(astrotime(met['obsdate'], format='isot').mjd) photodict = { PHOTOMETRY.TIME: mjd, PHOTOMETRY.MAGNITUDE: str(met['mag']), PHOTOMETRY.E_MAGNITUDE: str(met['magerr']), PHOTOMETRY.BAND: met['filter'].replace('sdss-', ''), PHOTOMETRY.SOURCE: source } catalog.entries[name].add_photometry(**photodict) catalog.journal_entries() return
def load_cats(): """Reload the catalog dictionaries.""" logger.info('Loading catalog...') for cat in apidata._CATS: apidata._catalogs[cat] = json.load(open( os.path.join(apidata._AC_PATH, apidata._CATS[cat][0], 'output', apidata._CATS[cat][1]), 'r'), object_pairs_hook=OrderedDict) # Add some API-specific fields to each catalog. for i, x in enumerate(apidata._catalogs[cat]): apidata._catalogs[cat][i]['catalog'] = cat apidata._catalogs[cat] = OrderedDict( sorted(dict( zip([x['name'] for x in apidata._catalogs[cat]], apidata._catalogs[cat])).items(), key=lambda s: (s[0].upper(), s[0]))) if cat not in apidata._extras: apidata._extras[cat] = OrderedDict() logger.info('Creating alias dictionary and position arrays...') apidata._rdnames = [] apidata._ras = [] apidata._decs = [] apidata._all = [] # Load object apidata._catalogs. for cat in apidata._CATS: apidata._cat_keys[cat] = set() for event in apidata._catalogs[cat]: add_event(cat, event, convert_coords=False) apidata._all = list(sorted(set(apidata._all), key=lambda s: (s.upper(), s))) apidata._coo = coord(apidata._ras, apidata._decs, unit=(un.hourangle, un.deg)) del (apidata._ras, apidata._decs) # Re-append events that were added later. for cat in apidata._extras: for event in apidata._extras[cat]: if event not in apidata._catalogs[cat]: apidata._catalogs[cat][event] = apidata._extras[cat][event] add_event(cat, event)
def do_dust(catalog): task_str = catalog.get_current_task_str() # Set preferred names, calculate some columns based on imported data, # sanitize some fields keys = list(catalog.entries.keys()) check_dustmaps(catalog.get_current_task_repo()) sfd = SFDQuery() for oname in pbar(keys, task_str): # Some events may be merged in cleanup process, skip them if # non-existent. try: name = catalog.add_entry(oname) except Exception: catalog.log.warning( '"{}" was not found, suggests merge occurred in cleanup ' 'process.'.format(oname)) continue if (FASTSTARS.RA not in catalog.entries[name] or FASTSTARS.DEC not in catalog.entries[name]): continue else: Mname = name Mradec = str( catalog.entries[name][FASTSTARS.RA][0]['value']) + str( catalog.entries[name][FASTSTARS.DEC][0]['value']) Mdist = '-1' c = coord(Mradec, unit=(un.hourangle, un.deg), frame='icrs') reddening = sfd(c) source = catalog.entries[name].add_source( bibcode='1998ApJ...500..525S') catalog.entries[name].add_quantity(FASTSTARS.EBV, str(reddening), source, upperlimit=True, derived=True) catalog.journal_entries() return
def lbs_to_radecs(ls, bs, remin=False): """ Transformation between lists of ls, bs, to ras, decs. Assumes all in degrees Conforms to astropy 0.4.3 taken from https://github.com/seclark/FITSHandling/commit/f04a6e54c6624741e4f3077ba8ba96af620871ac """ assert len(ls) == len(bs) obj = coord(ls, bs, unit="deg", frame="galactic") obj = obj.icrs ras = obj.ra.degree decs = obj.dec.degree if remin: ra_min = np.argmin(ras) ras = np.hstack((ras[ra_min:], ras[:ra_min])) decs = np.hstack((decs[ra_min:], decs[:ra_min])) return ras, decs return ras, decs
def add_event(cat, event, convert_coords=True): """Add event to global arrays.""" apidata._all.append(event) apidata._cat_keys[cat].update(list(apidata._catalogs[cat][event].keys())) levent = apidata._catalogs[cat].get(event, {}) laliases = levent.get('alias', []) lev = event.lower() laliases = list( set([ lev, replace_multiple(lev, ['sn', 'at']) if lev.startswith( ('sn', 'at')) else lev ] + [x['value'].lower() for x in laliases] + [ replace_multiple(x['value'].lower(), ['sn', 'at']) for x in laliases if x['value'].lower().startswith(('sn', 'at')) ])) for alias in laliases: apidata._aliases.setdefault(alias.lower().replace(' ', ''), []).append([cat, event, alias]) apidata._all_aliases.add(alias) lra = levent.get('ra') ldec = levent.get('dec') if lra is None and ldec is None: return lra = lra[0].get('value') ldec = ldec[0].get('value') if lra is None or ldec is None: return if not raregex.match(lra) or not decregex.match(ldec): return apidata._rdnames.append(event) if convert_coords: apidata._coo = coord_concat( (apidata._coo, coord(lra, ldec, unit=(un.hourangle, un.deg)))) else: apidata._ras.append(lra) apidata._decs.append(ldec)
def do_cleanup(catalog): """Task to cleanup catalog before final write.""" task_str = catalog.get_current_task_str() # Set preferred names, calculate some columns based on imported data, # sanitize some fields keys = catalog.entries.copy().keys() cleanupcnt = 0 for oname in pbar(keys, task_str): name = catalog.add_entry(oname) # Set the preferred name, switching to that name if name changed. name = catalog.entries[name].set_preferred_name() aliases = catalog.entries[name].get_aliases() catalog.entries[name].set_first_max_light() if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['MLS', 'SSS', 'CSS', 'GRB '] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:2])): discoverdate = ('/'.join([ '20' + alias.replace(prefix, '')[:2], alias.replace(prefix, '')[2:4], alias.replace(prefix, '')[4:6] ])) if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( TIDALDISRUPTION.DISCOVER_DATE, discoverdate, source, derived=True) break if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]: break if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]: prefixes = [ 'ASASSN-', 'PS1-', 'PS1', 'PS', 'iPTF', 'PTF', 'SCP-', 'SNLS-', 'SPIRITS', 'LSQ', 'DES', 'SNHiTS', 'Gaia', 'GND', 'GNW', 'GSD', 'GSW', 'EGS', 'COS', 'OGLE', 'HST' ] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:2]) and is_number(alias.replace(prefix, '')[:1])): discoverdate = '20' + alias.replace(prefix, '')[:2] if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( TIDALDISRUPTION.DISCOVER_DATE, discoverdate, source, derived=True) break if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]: break if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['SNF'] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:4])): discoverdate = ('/'.join([ alias.replace(prefix, '')[:4], alias.replace(prefix, '')[4:6], alias.replace(prefix, '')[6:8] ])) if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( TIDALDISRUPTION.DISCOVER_DATE, discoverdate, source, derived=True) break if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]: break if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['PTFS', 'SNSDF'] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:2])): discoverdate = ('/'.join([ '20' + alias.replace(prefix, '')[:2], alias.replace(prefix, '')[2:4] ])) if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( TIDALDISRUPTION.DISCOVER_DATE, discoverdate, source, derived=True) break if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]: break if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['AT', 'SN', 'OGLE-', 'SM ', 'KSN-'] for alias in aliases: for prefix in prefixes: if alias.startswith(prefix): year = re.findall(r'\d+', alias) if len(year) == 1: year = year[0] else: continue if alias.replace(prefix, '').index(year) != 0: continue if (year and is_number(year) and '.' not in year and len(year) <= 4): discoverdate = year if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( TIDALDISRUPTION.DISCOVER_DATE, discoverdate, source, derived=True) break if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]: break if (TIDALDISRUPTION.RA not in catalog.entries[name] or TIDALDISRUPTION.DEC not in catalog.entries[name]): prefixes = [ 'PSN J', 'MASJ', 'CSS', 'SSS', 'MASTER OT J', 'HST J', 'TCP J', 'MACS J', '2MASS J', 'EQ J', 'CRTS J', 'SMT J' ] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:6])): noprefix = alias.split(':')[-1].replace( prefix, '').replace('.', '') decsign = '+' if '+' in noprefix else '-' noprefix = noprefix.replace('+', '|').replace('-', '|') nops = noprefix.split('|') if len(nops) < 2: continue rastr = nops[0] decstr = nops[1] ra = ':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + \ ('.' + rastr[6:] if len(rastr) > 6 else '') dec = (decsign + ':'.join( [decstr[:2], decstr[2:4], decstr[4:6]]) + ('.' + decstr[6:] if len(decstr) > 6 else '')) if catalog.args.verbose: tprint('Added ra/dec from name: ' + ra + ' ' + dec) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( TIDALDISRUPTION.RA, ra, source, derived=True) catalog.entries[name].add_quantity( TIDALDISRUPTION.DEC, dec, source, derived=True) break if TIDALDISRUPTION.RA in catalog.entries[name]: break no_host = (TIDALDISRUPTION.HOST not in catalog.entries[name] or not any([ x[QUANTITY.VALUE] == 'Milky Way' for x in catalog.entries[name][TIDALDISRUPTION.HOST] ])) if (TIDALDISRUPTION.RA in catalog.entries[name] and TIDALDISRUPTION.DEC in catalog.entries[name] and no_host): from astroquery.irsa_dust import IrsaDust if name not in catalog.extinctions_dict: try: ra_dec = (catalog.entries[name][TIDALDISRUPTION.RA][0][ QUANTITY.VALUE] + " " + catalog.entries[name][ TIDALDISRUPTION.DEC][0][QUANTITY.VALUE]) result = IrsaDust.get_query_table(ra_dec, section='ebv') except (KeyboardInterrupt, SystemExit): raise except Exception: warnings.warn("Coordinate lookup for " + name + " failed in IRSA.") else: ebv = result['ext SandF mean'][0] ebverr = result['ext SandF std'][0] catalog.extinctions_dict[name] = [ebv, ebverr] if name in catalog.extinctions_dict: sources = uniq_cdl([ catalog.entries[name].add_self_source(), catalog.entries[name] .add_source(bibcode='2011ApJ...737..103S') ]) (catalog.entries[name].add_quantity( TIDALDISRUPTION.EBV, str(catalog.extinctions_dict[name][0]), sources, e_value=str(catalog.extinctions_dict[name][1]), derived=True)) if ((TIDALDISRUPTION.HOST in catalog.entries[name] and (TIDALDISRUPTION.HOST_RA not in catalog.entries[name] or TIDALDISRUPTION.HOST_DEC not in catalog.entries[name]))): for host in catalog.entries[name][TIDALDISRUPTION.HOST]: alias = host[QUANTITY.VALUE] if ' J' in alias and is_number(alias.split(' J')[-1][:6]): noprefix = alias.split(' J')[-1].split(':')[-1].replace( '.', '') decsign = '+' if '+' in noprefix else '-' noprefix = noprefix.replace('+', '|').replace('-', '|') nops = noprefix.split('|') if len(nops) < 2: continue rastr = nops[0] decstr = nops[1] hostra = (':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + ('.' + rastr[6:] if len(rastr) > 6 else '')) hostdec = decsign + ':'.join([ decstr[:2], decstr[2:4], decstr[4:6] ]) + ('.' + decstr[6:] if len(decstr) > 6 else '') if catalog.args.verbose: tprint('Added hostra/hostdec from name: ' + hostra + ' ' + hostdec) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( TIDALDISRUPTION.HOST_RA, hostra, source, derived=True) catalog.entries[name].add_quantity( TIDALDISRUPTION.HOST_DEC, hostdec, source, derived=True) break if TIDALDISRUPTION.HOST_RA in catalog.entries[name]: break if (TIDALDISRUPTION.REDSHIFT not in catalog.entries[name] and TIDALDISRUPTION.VELOCITY in catalog.entries[name]): # Find the "best" velocity to use for this bestsig = 0 for hv in catalog.entries[name][TIDALDISRUPTION.VELOCITY]: sig = get_sig_digits(hv[QUANTITY.VALUE]) if sig > bestsig: besthv = hv[QUANTITY.VALUE] bestsrc = hv['source'] bestsig = sig if bestsig > 0 and is_number(besthv): voc = float(besthv) * 1.e5 / CLIGHT source = catalog.entries[name].add_self_source() sources = uniq_cdl([source] + bestsrc.split(',')) (catalog.entries[name].add_quantity( TIDALDISRUPTION.REDSHIFT, pretty_num( sqrt((1. + voc) / (1. - voc)) - 1., sig=bestsig), sources, kind='heliocentric', derived=True)) if (TIDALDISRUPTION.REDSHIFT not in catalog.entries[name] and len(catalog.nedd_dict) > 0 and TIDALDISRUPTION.HOST in catalog.entries[name]): reference = "NED-D" refurl = "http://ned.ipac.caltech.edu/Library/Distances/" for host in catalog.entries[name][TIDALDISRUPTION.HOST]: if host[QUANTITY.VALUE] in catalog.nedd_dict: source = catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') secondarysource = catalog.entries[name].add_source( name=reference, url=refurl, secondary=True) meddist = statistics.median(catalog.nedd_dict[host[ QUANTITY.VALUE]]) redz = z_at_value(cosmo.comoving_distance, float(meddist) * un.Mpc) redshift = pretty_num( redz, sig=get_sig_digits(str(meddist))) catalog.entries[name].add_quantity( TIDALDISRUPTION.REDSHIFT, redshift, uniq_cdl([source, secondarysource]), kind='host', derived=True) if (TIDALDISRUPTION.MAX_ABS_MAG not in catalog.entries[name] and TIDALDISRUPTION.MAX_APP_MAG in catalog.entries[name] and TIDALDISRUPTION.LUM_DIST in catalog.entries[name]): # Find the "best" distance to use for this bestsig = 0 for ld in catalog.entries[name][TIDALDISRUPTION.LUM_DIST]: sig = get_sig_digits(ld[QUANTITY.VALUE]) if sig > bestsig: bestld = ld[QUANTITY.VALUE] bestsrc = ld['source'] bestsig = sig if bestsig > 0 and is_number(bestld) and float(bestld) > 0.: source = catalog.entries[name].add_self_source() sources = uniq_cdl([source] + bestsrc.split(',')) bestldz = z_at_value(cosmo.luminosity_distance, float(bestld) * un.Mpc) pnum = (float(catalog.entries[name][ TIDALDISRUPTION.MAX_APP_MAG][0][QUANTITY.VALUE]) - 5.0 * (log10(float(bestld) * 1.0e6) - 1.0 ) + 2.5 * log10(1.0 + bestldz)) pnum = pretty_num(pnum, sig=bestsig) catalog.entries[name].add_quantity( TIDALDISRUPTION.MAX_ABS_MAG, pnum, sources, derived=True) if TIDALDISRUPTION.REDSHIFT in catalog.entries[name]: # Find the "best" redshift to use for this bestz, bestkind, bestsig, bestsrc = catalog.entries[ name].get_best_redshift() if bestsig > 0: try: bestz = float(bestz) except Exception: print(catalog.entries[name]) raise if TIDALDISRUPTION.VELOCITY not in catalog.entries[name]: source = catalog.entries[name].add_self_source() # FIX: what's happening here?! pnum = CLIGHT / KM * \ ((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.) pnum = pretty_num(pnum, sig=bestsig) catalog.entries[name].add_quantity( TIDALDISRUPTION.VELOCITY, pnum, source, kind=PREF_KINDS[bestkind], derived=True) if bestz > 0.: from astropy.cosmology import Planck15 as cosmo if TIDALDISRUPTION.LUM_DIST not in catalog.entries[name]: dl = cosmo.luminosity_distance(bestz) sources = [ catalog.entries[name].add_self_source(), catalog.entries[name] .add_source(bibcode='2016A&A...594A..13P') ] sources = uniq_cdl(sources + bestsrc.split(',')) catalog.entries[name].add_quantity( TIDALDISRUPTION.LUM_DIST, pretty_num( dl.value, sig=bestsig), sources, kind=PREF_KINDS[bestkind], derived=True) if (TIDALDISRUPTION.MAX_ABS_MAG not in catalog.entries[name] and TIDALDISRUPTION.MAX_APP_MAG in catalog.entries[name]): source = catalog.entries[name].add_self_source() pnum = pretty_num( float(catalog.entries[name][ TIDALDISRUPTION.MAX_APP_MAG][0][ QUANTITY.VALUE]) - 5.0 * (log10(dl.to('pc').value) - 1.0 ) + 2.5 * log10(1.0 + bestz), sig=bestsig + 1) catalog.entries[name].add_quantity( TIDALDISRUPTION.MAX_ABS_MAG, pnum, sources, derived=True) if TIDALDISRUPTION.COMOVING_DIST not in catalog.entries[ name]: cd = cosmo.comoving_distance(bestz) sources = [ catalog.entries[name].add_self_source(), catalog.entries[name] .add_source(bibcode='2016A&A...594A..13P') ] sources = uniq_cdl(sources + bestsrc.split(',')) catalog.entries[name].add_quantity( TIDALDISRUPTION.COMOVING_DIST, pretty_num( cd.value, sig=bestsig), sources, derived=True) if all([ x in catalog.entries[name] for x in [ TIDALDISRUPTION.RA, TIDALDISRUPTION.DEC, TIDALDISRUPTION.HOST_RA, TIDALDISRUPTION.HOST_DEC ] ]): # For now just using first coordinates that appear in entry try: c1 = coord( ra=catalog.entries[name][TIDALDISRUPTION.RA][0][ QUANTITY.VALUE], dec=catalog.entries[name][TIDALDISRUPTION.DEC][0][ QUANTITY.VALUE], unit=(un.hourangle, un.deg)) c2 = coord( ra=catalog.entries[name][TIDALDISRUPTION.HOST_RA][0][ QUANTITY.VALUE], dec=catalog.entries[name][TIDALDISRUPTION.HOST_DEC][0][ QUANTITY.VALUE], unit=(un.hourangle, un.deg)) except (KeyboardInterrupt, SystemExit): raise except Exception: pass else: sources = uniq_cdl( [catalog.entries[name].add_self_source()] + catalog. entries[name][TIDALDISRUPTION.RA][0]['source'].split(',') + catalog.entries[name][TIDALDISRUPTION.DEC][0]['source']. split(',') + catalog.entries[name][TIDALDISRUPTION.HOST_RA] [0]['source'].split(',') + catalog.entries[name][ TIDALDISRUPTION.HOST_DEC][0]['source'].split(',')) if 'hostoffsetang' not in catalog.entries[name]: hosa = Decimal( hypot(c1.ra.degree - c2.ra.degree, c1.dec.degree - c2.dec.degree)) hosa = pretty_num(hosa * Decimal(3600.)) catalog.entries[name].add_quantity( TIDALDISRUPTION.HOST_OFFSET_ANG, hosa, sources, derived=True, u_value='arcseconds') if (TIDALDISRUPTION.COMOVING_DIST in catalog.entries[name] and TIDALDISRUPTION.REDSHIFT in catalog.entries[name] and TIDALDISRUPTION.HOST_OFFSET_DIST not in catalog.entries[name]): offsetsig = get_sig_digits(catalog.entries[name][ TIDALDISRUPTION.HOST_OFFSET_ANG][0][QUANTITY.VALUE]) sources = uniq_cdl( sources.split(',') + (catalog.entries[name][ TIDALDISRUPTION.COMOVING_DIST][0]['source']). split(',') + (catalog.entries[name][ TIDALDISRUPTION.REDSHIFT][0]['source']).split(',')) (catalog.entries[name].add_quantity( TIDALDISRUPTION.HOST_OFFSET_DIST, pretty_num( float(catalog.entries[name][ TIDALDISRUPTION.HOST_OFFSET_ANG][0][ QUANTITY.VALUE]) / 3600. * (pi / 180.) * float(catalog.entries[name][ TIDALDISRUPTION.COMOVING_DIST][0][ QUANTITY.VALUE]) * 1000. / (1.0 + float(catalog.entries[name][ TIDALDISRUPTION.REDSHIFT][0][QUANTITY.VALUE])), sig=offsetsig), sources)) catalog.entries[name].sanitize() catalog.journal_entries(bury=True, final=True, gz=True) cleanupcnt = cleanupcnt + 1 if catalog.args.travis and cleanupcnt % 1000 == 0: break catalog.save_caches() return
newitem = OrderedDict() if 'redshift' in item and 'host' not in item and 'ra' in item and 'dec' in item and item['ra'] and item['dec']: #if 'ra' in item and 'dec' in item and item['ra'] and item['dec']: newitem['name'] = item['name'] newitem['alias'] = [x['value'] for x in item['alias']] newitem['ra'] = item['ra'][0]['value'] newitem['dec'] = item['dec'][0]['value'] if 'redshift' in item: newitem['redshift'] = item['redshift'][0]['value'] # Temporary fix for David's typo if newitem['dec'].count('.') == 2: newitem['dec'] = newitem['dec'][:newitem['dec'].rfind('.')] newcatalog.append(newitem) coo = coord([x['ra'] for x in newcatalog], [x['dec'] for x in newcatalog], unit = (un.hourangle, un.deg)) for ci, co in enumerate(tqdm(coo)): customSimbad = Simbad() customSimbad.add_votable_fields('otype', 'z_value') regstr = 'region(ICRS, ' + co.to_string('hmsdms') + ', 1m)' print(regstr) result_table = customSimbad.query_criteria(regstr, otype='Galaxy') if result_table: print(newcatalog[ci]) if 'redshift' in newcatalog[ci]: print(newcatalog[ci]['redshift']) print(result_table) # Convert to array since that's what datatables expects
thistype = ct.replace('?', '').replace('*', '') if thistype.upper() in nonsnetypes: continue elif thistype in ('Other', 'not Ia', 'SN', 'unconf', 'Radio', 'CC', 'CCSN', 'Candidate', 'nIa'): sntypes.append('Unknown') break else: sntypes.append(thistype) break else: sntypes.append('Unknown') tprint(thisevent['name']) try: c = coord(ra=thisevent['ra'][0]['value'], dec=thisevent[ 'dec'][0]['value'], unit=(un.hourangle, un.deg)) except (KeyboardInterrupt, SystemExit): raise except: warnings.warn('Mangled coordinate, skipping') continue else: snnames.append(thisevent['name']) rarad = c.ra.radian - pi decrad = c.dec.radian snhx = 2.0**1.5 * cos(decrad) * sin(rarad / 2.0) / \ sqrt(1.0 + cos(decrad) * cos(rarad / 2.0)) snhy = sqrt(2.0) * sin(decrad) / \ sqrt(1.0 + cos(decrad) * cos(rarad / 2.0)) snras.append(c.ra.deg) sndecs.append(c.dec.deg)
if not is_number(newitem['ra'].split(':')[0]): continue newitem['dec'] = item['dec'][0]['value'] if not is_number(newitem['dec'].split(':')[0]): continue newitem['raerr'] = float(item['ra'][0].get('e_value', 0)) newitem['decerr'] = float(item['dec'][0].get('e_value', 0)) # Temporary fix for David's typo if newitem['dec'].count('.') == 2: newitem['dec'] = newitem['dec'][:newitem['dec'].rfind('.')] if 'distinctfrom' in item: newitem['distinctfrom'] = [x['value'] for x in item['distinctfrom']] newcatalog.append(newitem) coo = coord([x['ra'] for x in newcatalog], [x['dec'] for x in newcatalog], unit=(un.hourangle, un.deg)) response = urllib.request.urlopen('http://www.astronomy.swin.edu.au/pulsar/frbcat/table.php?format=text&sep=comma') frbtxt = response.read().decode('utf-8').splitlines() with open(utmost, 'r') as f: utmosttable = f.read().splitlines() frbcatalog = [] frbnames = [] utmostcoords = {} for row in utmosttable: if row[0] == '#': frbname = row[1:].strip() utmostcoords[frbname] = []
def do_dust(catalog): task_str = catalog.get_current_task_str() # Set preferred names, calculate some columns based on imported data, # sanitize some fields keys = list(catalog.entries.keys()) for oname in pbar(keys, task_str): # Some events may be merged in cleanup process, skip them if # non-existent. try: name = catalog.add_entry(oname) except Exception: catalog.log.warning( '"{}" was not found, suggests merge occurred in cleanup ' 'process.'.format(oname)) continue if (FASTSTARS.RA not in catalog.entries[name] or FASTSTARS.DEC not in catalog.entries[name]): continue elif (FASTSTARS.LUM_DIST not in catalog.entries[name]): Mname = name Mradec = str( catalog.entries[name][FASTSTARS.RA][0]['value']) + str( catalog.entries[name][FASTSTARS.DEC][0]['value']) Mdist = '-1' c = coord(Mradec, unit=(un.hourangle, un.deg), frame='icrs') reddening = sfd(c) source = catalog.entries[name].add_source( bibcode='1998ApJ...500..525S') catalog.entries[name].add_quantity(FASTSTARS.EBV, str(reddening), source, upperlimit=True, derived=True) else: Mname = name Mradec = str( catalog.entries[name][FASTSTARS.RA][0]['value']) + str( catalog.entries[name][FASTSTARS.DEC][0]['value']) Mdist = str(catalog.entries[name][FASTSTARS.LUM_DIST][0]['value']) Mdist_u = str( catalog.entries[name][FASTSTARS.LUM_DIST][0]['u_value']) if Mdist_u == 'kpc': c = coord(Mradec, unit=(un.hourangle, un.deg), distance=float(Mdist) * un.kpc, frame='icrs') elif Mdist_u == 'pc': c = coord(Mradec, unit=(un.hourangle, un.deg), distance=float(Mdist) * un.pc, frame='icrs') else: catalog.log.warning( '"{}" has a distance but not an attached unit.'.format( oname)) continue reddening = bayestar(c, mode='median') # If that reddening is a nan (ie. the star lies outside PanSTARRS) switch to SFD if np.isnan(reddening) == True: c = coord(Mradec, unit=(un.hourangle, un.deg), frame='icrs') reddening = sfd(c) source = catalog.entries[name].add_source( bibcode='1998ApJ...500..525S') catalog.entries[name].add_quantity(FASTSTARS.EBV, str(reddening), source, upperlimit=True, derived=True) else: source = catalog.entries[name].add_source( bibcode='2018arXiv180103555G') catalog.entries[name].add_quantity(FASTSTARS.EBV, str(reddening), source, derived=True) catalog.journal_entries() return
continue newitem['dec'] = item['dec'][0]['value'] if not is_number(newitem['dec'].split(':')[0]): continue newitem['raerr'] = float(item['ra'][0].get('e_value', 0)) newitem['decerr'] = float(item['dec'][0].get('e_value', 0)) # Temporary fix for David's typo if newitem['dec'].count('.') == 2: newitem['dec'] = newitem['dec'][:newitem['dec'].rfind('.')] if 'distinctfrom' in item: newitem['distinctfrom'] = [ x['value'] for x in item['distinctfrom'] ] newcatalog.append(newitem) coo = coord([x['ra'] for x in newcatalog], [x['dec'] for x in newcatalog], unit=(un.hourangle, un.deg)) response = urllib.request.urlopen( 'http://www.astronomy.swin.edu.au/pulsar/frbcat/table.php?format=text&sep=comma' ) frbtxt = response.read().decode('utf-8').splitlines() with open(utmost, 'r') as f: utmosttable = f.read().splitlines() frbcatalog = [] frbnames = [] utmostcoords = {} for row in utmosttable: if row[0] == '#':
def do_sdss_photo(catalog): task_str = catalog.get_current_task_str() D25 = Decimal('2.5') # fits_path = os.path.join(catalog.get_current_task_repo(), # 'SDSS/SDSS_allCandidates+BOSS_HEAD.FITS') # # hdulist = fits.open(fits_path) # print(hdulist[1].columns) # for ri, row in enumerate(hdulist[1].data['SNID']): # print([[tag, hdulist[1].data[tag][ri]] for tag in hdulist[1].data]) # print(hdulist[1].data['SNID'][ri], hdulist[1].data['IAUC'][ri], # hdulist[1].data['REDSHIFT_HELIO'][ri]) # # # print(hdulist[1].data['MJD']) # hdulist.close() # return # Load up metadata first with open( os.path.join(catalog.get_current_task_repo(), 'SDSS/sdsssn_master.dat2'), 'r') as f: rows = list(csv.reader(f.read().splitlines()[1:], delimiter=' ')) ignored_cids = [] columns = { SUPERNOVA.RA: 1, SUPERNOVA.DEC: 2, SUPERNOVA.ALIAS: 4, SUPERNOVA.CLAIMED_TYPE: 5, SUPERNOVA.REDSHIFT: 11, SUPERNOVA.MAX_DATE: 21, SUPERNOVA.HOST_RA: 99, SUPERNOVA.HOST_DEC: 100 } colnums = {v: k for k, v in columns.items()} rows = [[x.replace('\\N', '') for x in y] for y in rows] co = [[x[0], x[99], x[100]] for x in rows if x[99] and x[100]] coo = coord([x[1] for x in co], [x[2] for x in co], unit="deg") coo = [ ''.join([y[:9] for y in x.split()]) for x in coo.to_string('hmsdms', sep='') ] hostdict = dict( zip([x[0] for x in co], ['SDSS J' + x[1:] for x in coo])) for ri, row in enumerate(pbar(rows, task_str + ": metadata")): name = '' # Check if type is non-SNe first ct = row[columns[SUPERNOVA.CLAIMED_TYPE]] al = row[columns[SUPERNOVA.ALIAS]] if ct in ['AGN', 'Variable', 'Unknown'] and not al: catalog.log.info('`{}` is not a SN, not ' 'adding.'.format(row[0])) ignored_cids.append(row[0]) continue # Add entry (name, source) = catalog.new_entry( 'SDSS-II SN ' + row[0], bibcode='2014arXiv1401.3317S', url='http://data.sdss3.org/sas/dr10/boss/papers/supernova/') # Add host name if row[0] in hostdict: catalog.entries[name].add_quantity(SUPERNOVA.HOST, hostdict[row[0]], source) # Add other metadata for cn in colnums: key = colnums[cn] if not key: continue ic = int(cn) val = row[ic] if not val: continue kwargs = {} if key == SUPERNOVA.ALIAS: val = 'SN' + val elif key in [ SUPERNOVA.RA, SUPERNOVA.DEC, SUPERNOVA.HOST_RA, SUPERNOVA.HOST_DEC ]: kwargs = {QUANTITY.U_VALUE: 'floatdegrees'} if key in [SUPERNOVA.RA, SUPERNOVA.HOST_RA]: fval = float(val) if fval < 0.0: val = str(Decimal(360) + Decimal(fval)) elif key == SUPERNOVA.CLAIMED_TYPE: val = val.lstrip('pz').replace('SN', '') elif key == SUPERNOVA.REDSHIFT: kwargs[QUANTITY.KIND] = 'spectroscopic' if float(val) < -1.0: continue if float(row[ic + 1]) > 0.0: kwargs[QUANTITY.E_VALUE] = row[ic + 1] elif key == SUPERNOVA.MAX_DATE: dt = astrotime(float(val), format='mjd').datetime val = make_date_string(dt.year, dt.month, dt.day) catalog.entries[name].add_quantity(key, val, source, **kwargs) with open( os.path.join(catalog.get_current_task_repo(), 'SDSS/2010ApJ...708..661D.txt'), 'r') as sdss_file: bibcodes2010 = sdss_file.read().split('\n') sdssbands = ['u', 'g', 'r', 'i', 'z'] file_names = (list( glob(os.path.join(catalog.get_current_task_repo(), 'SDSS/sum/*.sum'))) + list( glob( os.path.join(catalog.get_current_task_repo(), 'SDSS/SMP_Data/*.dat')))) skipphoto = ['SDSS-II SN 15557'] for fi, fname in enumerate(pbar_strings(file_names, task_str)): tsvin = csv.reader(open(fname, 'r'), delimiter=' ', skipinitialspace=True) basename = os.path.basename(fname) hasred = True rst = 19 if '.dat' in fname: bibcode = '2014arXiv1401.3317S' hasred = False rst = 4 elif basename in bibcodes2010: bibcode = '2010ApJ...708..661D' else: bibcode = '2008AJ....136.2306H' skip_entry = False for rr, row in enumerate(tsvin): if skip_entry: break if rr == 0: # Ignore non-SNe objects and those not in metadata table above if row[3] in ignored_cids: skip_entry = True continue # Ignore IAU names from file headers as they are unreliable oname = 'SDSS-II SN ' + row[3] (name, source) = catalog.new_entry(oname, bibcode=bibcode) catalog.entries[name].add_quantity(SUPERNOVA.RA, row[-4], source, u_value='floatdegrees') catalog.entries[name].add_quantity(SUPERNOVA.DEC, row[-2], source, u_value='floatdegrees') if hasred and rr == 1: error = row[4] if float(row[4]) >= 0.0 else '' val = row[2] if float(val) < -1.0: continue (catalog.entries[name].add_quantity(SUPERNOVA.REDSHIFT, val, source, e_value=error, kind='heliocentric')) if rr >= rst: # Skip bad measurements if int(row[0]) > 1024: continue if oname in skipphoto: break mjd = row[1] band = sdssbands[int(row[2])] + "'" magnitude = row[3] e_mag = row[4] fluxd = row[7] e_fluxd = row[8] telescope = 'SDSS' photodict = { PHOTOMETRY.TIME: mjd, PHOTOMETRY.U_TIME: 'MJD', PHOTOMETRY.TELESCOPE: telescope, PHOTOMETRY.BAND: band, PHOTOMETRY.MAGNITUDE: magnitude, PHOTOMETRY.E_MAGNITUDE: e_mag, PHOTOMETRY.FLUX_DENSITY: fluxd, PHOTOMETRY.E_FLUX_DENSITY: e_fluxd, PHOTOMETRY.U_FLUX_DENSITY: 'μJy', PHOTOMETRY.SOURCE: source, PHOTOMETRY.BAND_SET: 'SDSS', PHOTOMETRY.SYSTEM: 'SDSS' } if float(fluxd) > 0.0: photodict[PHOTOMETRY.ZERO_POINT] = str( D25 * Decimal(fluxd).log10() + Decimal(magnitude)) ul_sigma = 3.0 if int(row[0]) & 32 or float( fluxd) < ul_sigma * float(e_fluxd): photodict[PHOTOMETRY.UPPER_LIMIT] = True photodict[PHOTOMETRY.UPPER_LIMIT_SIGMA] = str(ul_sigma) catalog.entries[name].add_photometry(**photodict) if catalog.args.travis and fi >= catalog.TRAVIS_QUERY_LIMIT: break if not fi % 1000: catalog.journal_entries() catalog.journal_entries() return
def do_mergeradec(catalog): """Merge stars based on close ra/dec prior to cleanup.""" task_str = catalog.get_current_task_str() # Set preferred names, calculate some columns based on imported data, # sanitize some fields keys = list(catalog.entries.keys()) Mnames, Mradec = [], [] for oname in pbar(keys, task_str): # Some events may be merged in cleanup process, skip them if # non-existent. try: name = catalog.add_entry(oname) except Exception: catalog.log.warning( '"{}" was not found, suggests merge occurred in cleanup ' 'process.'.format(oname)) continue # Set the preferred name, switching to that name if name changed. #name = catalog.entries[name].set_preferred_name() #aliases = catalog.entries[name].get_aliases() if (FASTSTARS.RA not in catalog.entries[name] or FASTSTARS.DEC not in catalog.entries[name]): continue else: Mnames.append(name) Mradec.append( str(catalog.entries[name][FASTSTARS.RA][0]['value']) + str(catalog.entries[name][FASTSTARS.DEC][0]['value'])) if len(Mradec) > 1: # Only can merge if we have at least two stars! c = coord(Mradec, unit=(un.hourangle, un.deg)) ### Construct tree to look for duplicates # Convert to pixel space pixx = np.cos(c.dec.rad) * np.cos(c.ra.rad) pixy = np.cos(c.dec.rad) * np.sin(c.ra.rad) pixz = np.sin(c.dec.rad) pix = np.vstack([pixx, pixy, pixz]).T # Construct tree pixtree = cKDTree(pix) # Query pairs pixarcsec = np.tan(5. * np.pi / 180. / 3600.) pairs = pixtree.query_pairs(pixarcsec) arrpairs = np.array(list(pairs)) # Attempt to merge pairs for i, j in arrpairs: try: catalog.copy_entry_to_entry(catalog.entries[Mnames[i]], catalog.entries[Mnames[j]]) del catalog.entries[Mnames[i]] print("`{}` and `{}` merged".format(Mnames[i], Mnames[j])) except KeyError: print("`{}` and `{}` pair already broken".format( Mnames[i], Mnames[j])) catalog.save_caches() return
def do_simbad(catalog): task_str = catalog.get_current_task_str() keys = list(catalog.entries.keys()) customSimbad = Simbad() customSimbad.ROW_LIMIT = -1 customSimbad.TIMEOUT = 120 customSimbad.add_votable_fields('otype', 'sptype', 'sp_bibcode', 'id') warnings.filterwarnings("ignore") Mnames, Mradec = [], [] for oname in pbar(keys, task_str): # Some events may be merged in cleanup process, skip them if # non-existent. try: name = catalog.add_entry(oname) except Exception: catalog.log.warning( '"{}" was not found, suggests merge occurred in cleanup ' 'process.'.format(oname)) continue if (FASTSTARS.RA not in catalog.entries[name] or FASTSTARS.DEC not in catalog.entries[name]): continue else: Mnames.append(name) radec = str(catalog.entries[name][FASTSTARS.RA][0]['value']) + str( catalog.entries[name][FASTSTARS.DEC][0]['value']) c = coord(radec, unit=(un.hourangle, un.deg), frame='icrs') cnttry = 0 foundstar = False while foundstar == False and cnttry < 10: try: cnttry += 1 time.sleep(0.1) result = customSimbad.query_region(c, radius='0d0m5s') aliases = re.sub(r'b\'(.*)\'', r'\1', str(result['ID'].tolist()[0])).split(',') except TypeError: #print(radec,cnttry) continue foundstar = True if foundstar == True: source = (catalog.entries[name].add_source( name='SIMBAD astronomical database', bibcode="2000A&AS..143....9W", url="http://simbad.u-strasbg.fr/", secondary=True)) for alias in aliases: ali = single_spaces( re.sub(r'\[[^)]*\]', '', alias).strip()) if is_number(ali.replace(' ', '')): continue if ali[:4] == "HVS ": continue ali = name_clean(ali) catalog.entries[name].add_quantity(FASTSTARS.ALIAS, ali, source) catalog.journal_entries() return
def do_cleanup(catalog): """Cleanup catalog after importing all data.""" task_str = catalog.get_current_task_str() # Set preferred names, calculate some columns based on imported data, # sanitize some fields keys = list(catalog.entries.keys()) cleanupcnt = 0 for oname in pbar(keys, task_str): # Some events may be merged in cleanup process, skip them if # non-existent. try: name = catalog.add_entry(oname) except Exception: catalog.log.warning( '"{}" was not found, suggests merge occurred in cleanup ' 'process.'.format(oname)) continue # Set the preferred name, switching to that name if name changed. name = catalog.entries[name].set_preferred_name() aliases = catalog.entries[name].get_aliases() catalog.entries[name].purge_bandless_photometry() catalog.entries[name].set_first_max_light() if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['MLS', 'SSS', 'CSS', 'GRB '] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:2])): discoverdate = ('/'.join([ '20' + alias.replace(prefix, '')[:2], alias.replace(prefix, '')[2:4], alias.replace(prefix, '')[4:6] ])) if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, discoverdate, source, derived=True) break if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]: break if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]: prefixes = [ 'ASASSN-', 'PS1-', 'PS1', 'PS', 'iPTF', 'PTF', 'SCP-', 'SNLS-', 'SPIRITS', 'LSQ', 'DES', 'SNHiTS', 'Gaia', 'GND', 'GNW', 'GSD', 'GSW', 'EGS', 'COS', 'OGLE', 'HST' ] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:2]) and is_number(alias.replace(prefix, '')[:1])): discoverdate = '20' + alias.replace(prefix, '')[:2] if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, discoverdate, source, derived=True) break if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]: break if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['SNF'] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:4])): discoverdate = ('/'.join([ alias.replace(prefix, '')[:4], alias.replace(prefix, '')[4:6], alias.replace(prefix, '')[6:8] ])) if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, discoverdate, source, derived=True) break if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]: break if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['PTFS', 'SNSDF'] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:2])): discoverdate = ('/'.join([ '20' + alias.replace(prefix, '')[:2], alias.replace(prefix, '')[2:4] ])) if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, discoverdate, source, derived=True) break if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]: break if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]: prefixes = ['AT', 'SN', 'OGLE-', 'SM ', 'KSN'] for alias in aliases: for prefix in prefixes: if alias.startswith(prefix): year = re.findall(r'\d+', alias) if len(year) == 1: year = year[0] else: continue if alias.replace(prefix, '').index(year) != 0: continue if (year and is_number(year) and '.' not in year and len(year) <= 4): discoverdate = year if catalog.args.verbose: tprint('Added discoverdate from name [' + alias + ']: ' + discoverdate) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, discoverdate, source, derived=True) break if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]: break if (SUPERNOVA.RA not in catalog.entries[name] or SUPERNOVA.DEC not in catalog.entries[name]): prefixes = [ 'PSN J', 'MASJ', 'CSS', 'SSS', 'MASTER OT J', 'HST J', 'TCP J', 'MACS J', '2MASS J', 'EQ J', 'CRTS J', 'SMT J' ] for alias in aliases: for prefix in prefixes: if (alias.startswith(prefix) and is_number(alias.replace(prefix, '')[:6])): noprefix = alias.split(':')[-1].replace(prefix, '').replace( '.', '') decsign = '+' if '+' in noprefix else '-' noprefix = noprefix.replace('+', '|').replace('-', '|') nops = noprefix.split('|') if len(nops) < 2: continue rastr = nops[0] decstr = nops[1] ra = ':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + \ ('.' + rastr[6:] if len(rastr) > 6 else '') dec = ( decsign + ':'.join([decstr[:2], decstr[2:4], decstr[4:6]]) + ('.' + decstr[6:] if len(decstr) > 6 else '')) if catalog.args.verbose: tprint('Added ra/dec from name: ' + ra + ' ' + dec) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity(SUPERNOVA.RA, ra, source, derived=True) catalog.entries[name].add_quantity(SUPERNOVA.DEC, dec, source, derived=True) break if SUPERNOVA.RA in catalog.entries[name]: break no_host = (SUPERNOVA.HOST not in catalog.entries[name] or not any([ x[QUANTITY.VALUE] == 'Milky Way' for x in catalog.entries[name][SUPERNOVA.HOST] ])) if (SUPERNOVA.RA in catalog.entries[name] and SUPERNOVA.DEC in catalog.entries[name] and no_host): from astroquery.irsa_dust import IrsaDust if name not in catalog.extinctions_dict: try: ra_dec = catalog.entries[name][ SUPERNOVA.RA][0][QUANTITY.VALUE] + \ " " + \ catalog.entries[name][SUPERNOVA.DEC][0][QUANTITY.VALUE] result = IrsaDust.get_query_table(ra_dec, section='ebv') except (KeyboardInterrupt, SystemExit): raise except Exception: warnings.warn("Coordinate lookup for " + name + " failed in IRSA.") else: ebv = result['ext SandF mean'][0] ebverr = result['ext SandF std'][0] catalog.extinctions_dict[name] = [ebv, ebverr] if name in catalog.extinctions_dict: sources = uniq_cdl([ catalog.entries[name].add_self_source(), catalog.entries[name].add_source( bibcode='2011ApJ...737..103S') ]) (catalog.entries[name].add_quantity( SUPERNOVA.EBV, str(catalog.extinctions_dict[name][0]), sources, e_value=str(catalog.extinctions_dict[name][1]), derived=True)) if ((SUPERNOVA.HOST in catalog.entries[name] and (SUPERNOVA.HOST_RA not in catalog.entries[name] or SUPERNOVA.HOST_DEC not in catalog.entries[name]))): for host in catalog.entries[name][SUPERNOVA.HOST]: alias = host[QUANTITY.VALUE] if ' J' in alias and is_number(alias.split(' J')[-1][:6]): noprefix = alias.split(' J')[-1].split(':')[-1].replace( '.', '') decsign = '+' if '+' in noprefix else '-' noprefix = noprefix.replace('+', '|').replace('-', '|') nops = noprefix.split('|') if len(nops) < 2: continue rastr = nops[0] decstr = nops[1] hostra = (':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + ('.' + rastr[6:] if len(rastr) > 6 else '')) hostdec = decsign + ':'.join([ decstr[:2], decstr[2:4], decstr[4:6] ]) + ('.' + decstr[6:] if len(decstr) > 6 else '') if catalog.args.verbose: tprint('Added hostra/hostdec from name: ' + hostra + ' ' + hostdec) source = catalog.entries[name].add_self_source() catalog.entries[name].add_quantity(SUPERNOVA.HOST_RA, hostra, source, derived=True) catalog.entries[name].add_quantity(SUPERNOVA.HOST_DEC, hostdec, source, derived=True) break if SUPERNOVA.HOST_RA in catalog.entries[name]: break if (SUPERNOVA.REDSHIFT not in catalog.entries[name] and SUPERNOVA.VELOCITY in catalog.entries[name]): # Find the "best" velocity to use for this bestsig = 0 for hv in catalog.entries[name][SUPERNOVA.VELOCITY]: sig = get_sig_digits(hv[QUANTITY.VALUE]) if sig > bestsig: besthv = hv[QUANTITY.VALUE] bestsrc = hv['source'] bestsig = sig if bestsig > 0 and is_number(besthv): voc = float(besthv) * 1.e5 / CLIGHT source = catalog.entries[name].add_self_source() sources = uniq_cdl([source] + bestsrc.split(',')) (catalog.entries[name].add_quantity( SUPERNOVA.REDSHIFT, pretty_num(sqrt((1. + voc) / (1. - voc)) - 1., sig=bestsig), sources, kind='heliocentric', derived=True)) if (SUPERNOVA.REDSHIFT not in catalog.entries[name] and len(catalog.nedd_dict) > 0 and SUPERNOVA.HOST in catalog.entries[name]): reference = "NED-D" refurl = "http://ned.ipac.caltech.edu/Library/Distances/" refbib = "1991ASSL..171...89H" for host in catalog.entries[name][SUPERNOVA.HOST]: if host[QUANTITY.VALUE] in catalog.nedd_dict: source = catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') secondarysource = catalog.entries[name].add_source( name=reference, url=refurl, bibcode=refbib, secondary=True) meddist = statistics.median( catalog.nedd_dict[host[QUANTITY.VALUE]]) redz = z_at_value(cosmo.comoving_distance, float(meddist) * un.Mpc) redshift = pretty_num(redz, sig=get_sig_digits(str(meddist))) catalog.entries[name].add_quantity( [SUPERNOVA.REDSHIFT, SUPERNOVA.HOST_REDSHIFT], redshift, uniq_cdl([source, secondarysource]), kind='host', derived=True) if (SUPERNOVA.MAX_ABS_MAG not in catalog.entries[name] and SUPERNOVA.MAX_APP_MAG in catalog.entries[name] and SUPERNOVA.LUM_DIST in catalog.entries[name]): # Find the "best" distance to use for this bestsig = 0 for ld in catalog.entries[name][SUPERNOVA.LUM_DIST]: sig = get_sig_digits(ld[QUANTITY.VALUE]) if sig > bestsig: bestld = ld[QUANTITY.VALUE] bestsrc = ld[QUANTITY.SOURCE] bestsig = sig if bestsig > 0 and is_number(bestld) and float(bestld) > 0.: source = catalog.entries[name].add_self_source() sources = uniq_cdl([source] + bestsrc.split(',')) bestldz = z_at_value(cosmo.luminosity_distance, float(bestld) * un.Mpc) pnum = (float(catalog.entries[name][SUPERNOVA.MAX_APP_MAG][0][ QUANTITY.VALUE]) - 5.0 * (log10(float(bestld) * 1.0e6) - 1.0) + 2.5 * log10(1.0 + bestldz)) pnum = pretty_num(pnum, sig=bestsig + 1) catalog.entries[name].add_quantity(SUPERNOVA.MAX_ABS_MAG, pnum, sources, derived=True) if (SUPERNOVA.MAX_VISUAL_ABS_MAG not in catalog.entries[name] and SUPERNOVA.MAX_VISUAL_APP_MAG in catalog.entries[name] and SUPERNOVA.LUM_DIST in catalog.entries[name]): # Find the "best" distance to use for this bestsig = 0 for ld in catalog.entries[name][SUPERNOVA.LUM_DIST]: sig = get_sig_digits(ld[QUANTITY.VALUE]) if sig > bestsig: bestld = ld[QUANTITY.VALUE] bestsrc = ld[QUANTITY.SOURCE] bestsig = sig if bestsig > 0 and is_number(bestld) and float(bestld) > 0.: source = catalog.entries[name].add_self_source() sources = uniq_cdl([source] + bestsrc.split(',')) # FIX: what's happening here?! pnum = (float(catalog.entries[name][ SUPERNOVA.MAX_VISUAL_APP_MAG][0][QUANTITY.VALUE]) - 5.0 * (log10(float(bestld) * 1.0e6) - 1.0)) pnum = pretty_num(pnum, sig=bestsig + 1) catalog.entries[name].add_quantity( SUPERNOVA.MAX_VISUAL_ABS_MAG, pnum, sources, derived=True) if SUPERNOVA.REDSHIFT in catalog.entries[name]: # Find the "best" redshift to use for this bestz, bestkind, bestsig, bestsrc = catalog.entries[ name].get_best_redshift() if bestsig > 0: try: bestz = float(bestz) except Exception: print(catalog.entries[name]) raise if SUPERNOVA.VELOCITY not in catalog.entries[name]: source = catalog.entries[name].add_self_source() # FIX: what's happening here?! pnum = CLIGHT / KM * \ ((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.) pnum = pretty_num(pnum, sig=bestsig) catalog.entries[name].add_quantity( SUPERNOVA.VELOCITY, pnum, source, kind=(SUPERNOVA.VELOCITY.kind_preference[bestkind] if bestkind else '')) if bestz > 0.: if SUPERNOVA.LUM_DIST not in catalog.entries[name]: dl = cosmo.luminosity_distance(bestz) sources = [ catalog.entries[name].add_self_source(), catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') ] sources = uniq_cdl(sources + bestsrc.split(',')) catalog.entries[name].add_quantity( SUPERNOVA.LUM_DIST, pretty_num(dl.value, sig=bestsig + 1), sources, kind=(SUPERNOVA.LUM_DIST.kind_preference[bestkind] if bestkind else ''), derived=True) if (SUPERNOVA.MAX_ABS_MAG not in catalog.entries[name] and SUPERNOVA.MAX_APP_MAG in catalog.entries[name]): source = catalog.entries[name].add_self_source() pnum = pretty_num( float(catalog.entries[name][ SUPERNOVA.MAX_APP_MAG][0][QUANTITY.VALUE]) - 5.0 * (log10(dl.to('pc').value) - 1.0) + 2.5 * log10(1.0 + bestz), sig=bestsig + 1) catalog.entries[name].add_quantity( SUPERNOVA.MAX_ABS_MAG, pnum, sources, derived=True) if (SUPERNOVA.MAX_VISUAL_ABS_MAG not in catalog.entries[name] and SUPERNOVA.MAX_VISUAL_APP_MAG in catalog.entries[name]): source = catalog.entries[name].add_self_source() pnum = pretty_num(float(catalog.entries[name][ SUPERNOVA.MAX_VISUAL_APP_MAG][0][ QUANTITY.VALUE]) - 5.0 * (log10(dl.to('pc').value) - 1.0), sig=bestsig + 1) catalog.entries[name].add_quantity( SUPERNOVA.MAX_VISUAL_ABS_MAG, pnum, sources, derived=True) if SUPERNOVA.COMOVING_DIST not in catalog.entries[name]: cd = cosmo.comoving_distance(bestz) sources = [ catalog.entries[name].add_self_source(), catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') ] sources = uniq_cdl(sources + bestsrc.split(',')) catalog.entries[name].add_quantity( SUPERNOVA.COMOVING_DIST, pretty_num(cd.value, sig=bestsig), sources, derived=True) if SUPERNOVA.HOST_REDSHIFT in catalog.entries[name]: # Find the "best" redshift to use for this bestz, bestkind, bestsig, bestsrc = catalog.entries[ name].get_best_redshift(SUPERNOVA.HOST_REDSHIFT) if bestsig > 0: try: bestz = float(bestz) except Exception: print(catalog.entries[name]) raise if SUPERNOVA.HOST_VELOCITY not in catalog.entries[name]: source = catalog.entries[name].add_self_source() # FIX: what's happening here?! pnum = CLIGHT / KM * \ ((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.) pnum = pretty_num(pnum, sig=bestsig) catalog.entries[name].add_quantity( SUPERNOVA.HOST_VELOCITY, pnum, source, kind=(SUPERNOVA.HOST_VELOCITY.kind_preference[bestkind] if bestkind else '')) if bestz > 0.: if SUPERNOVA.HOST_LUM_DIST not in catalog.entries[name]: dl = cosmo.luminosity_distance(bestz) sources = [ catalog.entries[name].add_self_source(), catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') ] sources = uniq_cdl(sources + bestsrc.split(',')) catalog.entries[name].add_quantity( SUPERNOVA.HOST_LUM_DIST, pretty_num(dl.value, sig=bestsig + 1), sources, kind=(SUPERNOVA.HOST_LUM_DIST. kind_preference[bestkind] if bestkind else ''), derived=True) if SUPERNOVA.HOST_COMOVING_DIST not in catalog.entries[ name]: cd = cosmo.comoving_distance(bestz) sources = [ catalog.entries[name].add_self_source(), catalog.entries[name].add_source( bibcode='2016A&A...594A..13P') ] sources = uniq_cdl(sources + bestsrc.split(',')) catalog.entries[name].add_quantity( SUPERNOVA.HOST_COMOVING_DIST, pretty_num(cd.value, sig=bestsig), sources, derived=True) if all([ x in catalog.entries[name] for x in [ SUPERNOVA.RA, SUPERNOVA.DEC, SUPERNOVA.HOST_RA, SUPERNOVA.HOST_DEC ] ]): # For now just using first coordinates that appear in entry try: c1 = coord( ra=catalog.entries[name][SUPERNOVA.RA][0][QUANTITY.VALUE], dec=catalog.entries[name][SUPERNOVA.DEC][0][ QUANTITY.VALUE], unit=(un.hourangle, un.deg)) c2 = coord(ra=catalog.entries[name][SUPERNOVA.HOST_RA][0][ QUANTITY.VALUE], dec=catalog.entries[name][SUPERNOVA.HOST_DEC][0][ QUANTITY.VALUE], unit=(un.hourangle, un.deg)) except (KeyboardInterrupt, SystemExit): raise except Exception: pass else: sources = uniq_cdl([catalog.entries[name].add_self_source()] + catalog.entries[name][SUPERNOVA.RA][0][ QUANTITY.SOURCE].split(',') + catalog.entries[name][SUPERNOVA.DEC][0][ QUANTITY.SOURCE].split(',') + catalog.entries[name][SUPERNOVA.HOST_RA][0][ QUANTITY.SOURCE].split(',') + catalog.entries[name][SUPERNOVA.HOST_DEC][0] [QUANTITY.SOURCE].split(',')) if SUPERNOVA.HOST_OFFSET_ANG not in catalog.entries[name]: hosa = Decimal(c1.separation(c2).arcsecond) hosa = pretty_num(hosa) catalog.entries[name].add_quantity( SUPERNOVA.HOST_OFFSET_ANG, hosa, sources, derived=True, u_value='arcseconds') if (SUPERNOVA.COMOVING_DIST in catalog.entries[name] and SUPERNOVA.REDSHIFT in catalog.entries[name] and SUPERNOVA.HOST_OFFSET_DIST not in catalog.entries[name]): offsetsig = get_sig_digits(catalog.entries[name][ SUPERNOVA.HOST_OFFSET_ANG][0][QUANTITY.VALUE]) sources = uniq_cdl( sources.split(',') + (catalog.entries[name][SUPERNOVA.COMOVING_DIST][0][ QUANTITY.SOURCE]).split(',') + (catalog.entries[name][SUPERNOVA.REDSHIFT][0][ QUANTITY.SOURCE]).split(',')) (catalog.entries[name].add_quantity( SUPERNOVA.HOST_OFFSET_DIST, pretty_num( float(catalog.entries[name][ SUPERNOVA.HOST_OFFSET_ANG][0][QUANTITY.VALUE]) / 3600. * (pi / 180.) * float(catalog.entries[name][ SUPERNOVA.COMOVING_DIST][0][QUANTITY.VALUE]) * 1000. / (1.0 + float(catalog.entries[name][ SUPERNOVA.REDSHIFT][0][QUANTITY.VALUE])), sig=offsetsig), sources)) catalog.entries[name].sanitize() catalog.journal_entries(bury=True, final=True, gz=True) cleanupcnt = cleanupcnt + 1 if catalog.args.travis and cleanupcnt % 1000 == 0: break catalog.save_caches() return
def do_sdss_photo(catalog): task_str = catalog.get_current_task_str() # Load up metadata first with open(os.path.join(catalog.get_current_task_repo(), 'SDSS/sdsssn_master.dat2'), 'r') as f: rows = list(csv.reader(f.read().splitlines()[1:], delimiter=' ')) ignored_cids = [] columns = { SUPERNOVA.RA: 1, SUPERNOVA.DEC: 2, SUPERNOVA.ALIAS: 4, SUPERNOVA.CLAIMED_TYPE: 5, SUPERNOVA.REDSHIFT: 11, SUPERNOVA.MAX_DATE: 21, SUPERNOVA.HOST_RA: 99, SUPERNOVA.HOST_DEC: 100 } colnums = {v: k for k, v in columns.items()} rows = [[x.replace('\\N', '') for x in y] for y in rows] co = [[x[0], x[99], x[100]] for x in rows if x[99] and x[100]] coo = coord([x[1] for x in co], [x[2] for x in co], unit="deg") coo = [''.join([y[:9] for y in x.split()]) for x in coo.to_string('hmsdms', sep='')] hostdict = dict(zip([x[0] for x in co], ['SDSS J' + x[1:] for x in coo])) for ri, row in enumerate(pbar(rows, task_str + ": metadata")): name = '' # Check if type is non-SNe first ct = row[columns[SUPERNOVA.CLAIMED_TYPE]] al = row[columns[SUPERNOVA.ALIAS]] if ct in ['AGN', 'Variable'] and not al: catalog.log.info('`{}` is not a SN, not ' 'adding.'.format(row[0])) ignored_cids.append(row[0]) continue # Add entry (name, source) = catalog.new_entry( 'SDSS-II SN ' + row[0], bibcode='2014arXiv1401.3317S', url='http://data.sdss3.org/sas/dr10/boss/papers/supernova/') # Add host name if row[0] in hostdict: catalog.entries[name].add_quantity(SUPERNOVA.HOST, hostdict[row[0]], source) # Add other metadata for cn in colnums: key = colnums[cn] if not key: continue ic = int(cn) val = row[ic] if not val: continue kwargs = {} if key == SUPERNOVA.ALIAS: val = 'SN' + val elif key in [SUPERNOVA.RA, SUPERNOVA.DEC, SUPERNOVA.HOST_RA, SUPERNOVA.HOST_DEC]: kwargs = {QUANTITY.U_VALUE: 'floatdegrees'} if key in [SUPERNOVA.RA, SUPERNOVA.HOST_RA]: fval = float(val) if fval < 0.0: val = str(Decimal(360) + Decimal(fval)) elif key == SUPERNOVA.CLAIMED_TYPE: val = val.lstrip('pz').replace('SN', '') elif key == SUPERNOVA.REDSHIFT: kwargs[QUANTITY.KIND] = 'spectroscopic' if float(row[ic + 1]) > 0.0: kwargs[QUANTITY.E_VALUE] = row[ic + 1] elif key == SUPERNOVA.MAX_DATE: dt = astrotime(float(val), format='mjd').datetime val = make_date_string(dt.year, dt.month, dt.day) catalog.entries[name].add_quantity(key, val, source, **kwargs) with open(os.path.join(catalog.get_current_task_repo(), 'SDSS/2010ApJ...708..661D.txt'), 'r') as sdss_file: bibcodes2010 = sdss_file.read().split('\n') sdssbands = ['u', 'g', 'r', 'i', 'z'] file_names = (list(glob(os.path.join(catalog .get_current_task_repo(), 'SDSS/sum/*.sum'))) + list(glob(os.path.join(catalog .get_current_task_repo(), 'SDSS/SMP_Data/*.dat')))) for fi, fname in enumerate(pbar_strings(file_names, task_str)): tsvin = csv.reader(open(fname, 'r'), delimiter=' ', skipinitialspace=True) basename = os.path.basename(fname) hasred = True rst = 19 if '.dat' in fname: bibcode = '2014arXiv1401.3317S' hasred = False rst = 4 elif basename in bibcodes2010: bibcode = '2010ApJ...708..661D' else: bibcode = '2008AJ....136.2306H' skip_entry = False for rr, row in enumerate(tsvin): if skip_entry: break if rr == 0: # Ignore non-SNe objects and those not in metadata table above if row[3] in ignored_cids: skip_entry = True continue # Ignore IAU names from Sako 2014 as they are unreliable if row[5] == 'RA:' or bibcode == '2014arXiv1401.3317S': name = 'SDSS-II SN ' + row[3] else: name = 'SN' + row[5] name = catalog.add_entry(name) source = catalog.entries[name].add_source(bibcode=bibcode) catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, name, source) catalog.entries[name].add_quantity( SUPERNOVA.ALIAS, 'SDSS-II SN ' + row[3], source) if row[5] != 'RA:' and bibcode == '2014arXiv1401.3317S': year = re.findall(r'\d+', name)[0] catalog.entries[name].add_quantity( SUPERNOVA.DISCOVER_DATE, year, source) catalog.entries[name].add_quantity( SUPERNOVA.RA, row[-4], source, u_value='floatdegrees') catalog.entries[name].add_quantity( SUPERNOVA.DEC, row[-2], source, u_value='floatdegrees') if hasred and rr == 1: error = row[4] if float(row[4]) >= 0.0 else '' (catalog.entries[name] .add_quantity(SUPERNOVA.REDSHIFT, row[2], source, e_value=error, kind='heliocentric')) if rr >= rst: # Skip bad measurements if int(row[0]) > 1024: continue mjd = row[1] band = sdssbands[int(row[2])] magnitude = row[3] e_mag = row[4] telescope = 'SDSS' (catalog.entries[name] .add_photometry(time=mjd, telescope=telescope, band=band, magnitude=magnitude, e_magnitude=e_mag, source=source, system='SDSS')) if not fi % 1000: catalog.journal_entries() catalog.journal_entries() return
def do_gaiaviavizier(catalog): # Disable warnings warnings.filterwarnings("ignore") catalog.log.warning( 'Some warnings are thrown by the Gaia module which do not affect the result of the Gaia queries.' ) task_str = catalog.get_current_task_str() keys = list(catalog.entries.keys()) ##### Gaia parallax offset gaiaparallaxoffset = -0.029 datafile = os.path.join(catalog.get_current_task_repo(), 'AUXILIARY', 'lindegren_error_inflation.csv') lei_data = pd.read_csv(datafile, header=None) lei_G = lei_data[0].values # Gaia magnitude lei_EIR = lei_data[1].values # ratio of external to internal error lei_EIR_interp = interp1d(lei_G, lei_EIR) def add_lindegren_inflation(AST_ERRORS, GMAG): _EIR_factor = lei_EIR_interp(float(GMAG)) if isinstance(AST_ERRORS, list): return [str(float(AE) * _EIR_factor) for AE in AST_ERRORS] else: return str(float(AST_ERRORS) * _EIR_factor) ##### Boubert et al. (2019) RVS error with open( './astrocats/faststars/input/faststars-external/AUXILIARY/boubert_2019_gaiarvscuts.p', 'rb') as fp: b19_valid_rvs = pickle.load(fp)['source_id'] _caught_by_cut = [] cntgphot = 0 cntgast = 0 for oname in pbar(keys, task_str): try: name = catalog.add_entry(oname) except Exception: catalog.log.warning( '"{}" was not found, suggests merge occurred in cleanup ' 'process.'.format(oname)) continue ### Check if has DR1 or DR2 ID gaiaid = '' try: for alias in catalog.entries[name][FASTSTARS.ALIAS]: if alias['value'][:8] == 'Gaia DR2' and len( alias['value']) > len(gaiaid): gaiaid = alias['value'] elif alias['value'][:8] == 'Gaia DR1' and len( alias['value']) > len(gaiaid): gaiaid = alias['value'] except KeyError: print(name) hasgaiaid = True if len(gaiaid) > 0 else False if (FASTSTARS.RA not in catalog.entries[name] or FASTSTARS.DEC not in catalog.entries[name]) and not hasgaiaid: continue else: if hasgaiaid: result = silentgaiaobjectquery(gaiaid, radius=un.Quantity( 10.0, un.arcsecond), catalog='I/345/gaia2') if len(result) < 1: catalog.log.warning( '"{}" should really have had a cross-match.'.format( name)) print(gaiaid) else: Mradec = str( catalog.entries[name][FASTSTARS.RA][0]['value']) + str( catalog.entries[name][FASTSTARS.DEC][0]['value']) #print(name,Mradec) c = coord(Mradec, unit=(un.hourangle, un.deg), frame='icrs') queryradius = 0. if (FASTSTARS.PROPER_MOTION_RA in catalog.entries[name] and FASTSTARS.PROPER_MOTION_DEC in catalog.entries[name]): queryradius = 1e-3 * 3. * 10. * np.sqrt( float(catalog.entries[name][FASTSTARS.PROPER_MOTION_RA] [0]['value'])**2. + float(catalog.entries[name][ FASTSTARS.PROPER_MOTION_DEC][0]['value'])**2.) queryradius = max(queryradius, 3.) if name in []: result = silentgaiaregionquery(c, radius=un.Quantity( 30.0, un.arcsecond), catalog='I/345/gaia2') else: result = silentgaiaregionquery(c, radius=un.Quantity( queryradius, un.arcsecond), catalog='I/345/gaia2') if len(result) > 0: if len(result[0]) > 1: indexclosest = np.argmin(result[0]['_r']) result = result[0][indexclosest] else: result = result[0][0] # Utility function for accessing the Vizier table's columns def gtab(parstring): return str(result[parstring]) cntgphot += 1 source = catalog.entries[name].add_source( bibcode='2016A&A...595A...2G') catalog.entries[name].add_photometry( time=57023, u_time='MJD', telescope='Gaia', band='G', magnitude=gtab('Gmag'), e_magnitude=gtab('e_Gmag'), source=source) catalog.entries[name].add_photometry( time=57023, u_time='MJD', telescope='Gaia', band='GBP', magnitude=gtab('BPmag'), e_magnitude=gtab('e_BPmag'), source=source) catalog.entries[name].add_photometry( time=57023, u_time='MJD', telescope='Gaia', band='GRP', magnitude=gtab('RPmag'), e_magnitude=gtab('e_RPmag'), source=source) if not hasgaiaid: catalog.entries[name].add_quantity(FASTSTARS.ALIAS, gtab('DR2Name'), source=source) gra, gde = coord(ra=float(gtab('RA_ICRS')) * un.deg, dec=float(gtab('DE_ICRS')) * un.deg, frame='icrs').to_string('hmsdms', sep=':').split() if gtab('Plx') == '--': catalog.entries[name].add_quantity( FASTSTARS.RA, gra, source, e_value=add_lindegren_inflation( gtab('e_RA_ICRS'), gtab('Gmag')), u_e_value='mas', correlations=[{ CORRELATION.VALUE: gtab('RADEcor'), CORRELATION.QUANTITY: FASTSTARS.DEC, CORRELATION.KIND: 'Pearson' }]) catalog.entries[name].add_quantity( FASTSTARS.DEC, gde, source, e_value=add_lindegren_inflation( gtab('e_DE_ICRS'), gtab('Gmag')), u_e_value='mas', correlations=[{ CORRELATION.VALUE: gtab('RADEcor'), CORRELATION.QUANTITY: FASTSTARS.RA, CORRELATION.KIND: 'Pearson' }]) else: #print(gtab('Plx')) cntgast += 1 #catalog.log.warning( # '"{}" has Gaia astrometry.'.format(name)) ast_keys = [ FASTSTARS.RA, FASTSTARS.DEC, FASTSTARS.PARALLAX, FASTSTARS.PROPER_MOTION_RA, FASTSTARS.PROPER_MOTION_DEC ] ast_values = [ gra, gde, str(float(gtab('Plx')) - gaiaparallaxoffset), gtab('pmRA'), gtab('pmDE') ] ast_errors = add_lindegren_inflation([ gtab('e_RA_ICRS'), gtab('e_DE_ICRS'), gtab('e_Plx'), gtab('e_pmRA'), gtab('e_pmDE') ], gtab('Gmag')) ast_corr = [[ '1.0', gtab('RADEcor'), gtab('RAPlxcor'), gtab('RApmRAcor'), gtab('RApmDEcor') ], [ gtab('RADEcor'), '1.0', gtab('DEPlxcor'), gtab('DEpmRAcor'), gtab('DEpmDEcor') ], [ gtab('RAPlxcor'), gtab('DEPlxcor'), '1.0', gtab('PlxpmRAcor'), gtab('PlxpmDEcor') ], [ gtab('RApmRAcor'), gtab('DEpmRAcor'), gtab('PlxpmRAcor'), '1.0', gtab('pmRApmDEcor') ], [ gtab('RApmDEcor'), gtab('DEpmDEcor'), gtab('PlxpmDEcor'), gtab('pmRApmDEcor'), '1.0' ]] ast_names = [ 'ra', 'dec', 'parallax', 'propermotionra', 'propermotiondec' ] ast_units = ['hms', 'dms', 'mas', 'mas/yr', 'mas/yr'] ast_error_units = ['mas', 'mas', 'mas', 'mas/yr', 'mas/yr'] #print(ast_values[0]) n_ast = len(ast_keys) # number of columns for i in range(n_ast): corr_dict = [{ CORRELATION.VALUE: ast_corr[i][(i + j) % n_ast], CORRELATION.QUANTITY: ast_names[(i + j) % n_ast], CORRELATION.KIND: 'Pearson' } for j in list(range(1, n_ast))] catalog.entries[name].add_quantity( ast_keys[i], ast_values[i], source, e_value=ast_errors[i], u_value=ast_units[i], e_u_value=ast_error_units[i], correlations=corr_dict) if gtab('RV') != '--': if int(gtab('Source')) in b19_valid_rvs: catalog.entries[name].add_quantity( FASTSTARS.VELOCITY, gtab('RV'), source, e_value=gtab('e_RV'), u_value='km/s') else: #print(int(gtab('Source'))) _caught_by_cut.append(int(gtab('Source'))) ##### This has been moved to boundprobability.py # Convert parallax to distance #if (FASTSTARS.LUM_DIST in catalog.entries[name]): # catalog.log.warning( # '"{}" has distance from photmetric distance prior.'.format(name)) # distance, distance_error = parallax_to_distance(name,result['Plx'][0],result['e_Plx'][0],float(catalog.entries[name][FASTSTARS.LUM_DIST][0]['value']),float(catalog.entries[name][FASTSTARS.LUM_DIST][0]['e_value'])) # catalog.entries[name].add_quantity(FASTSTARS.LUM_DIST, str(distance), e_value=str(distance_error), u_value='kpc', source=source, derived=True) #else: # catalog.log.warning( # '"{}" has distance from Astraatmadja & Bailer-Jones (2016) prior.'.format(name)) # distance, distance_error = parallax_to_distance(name,result['Plx'][0],result['e_Plx'][0]) # catalog.entries[name].add_quantity(FASTSTARS.LUM_DIST, str(distance), e_value=str(distance_error), u_value='kpc', source=source, derived=True) _box = {'source_id': _caught_by_cut} with open('./boubert_2019_gaiarvscuts_caught.p', "wb") as f: pickle.dump(_box, f, protocol=3) catalog.log.warning( '"{}" have Gaia photometry and "{}" have Gaia astrometry.'.format( cntgphot, cntgast)) catalog.journal_entries() # Reactivate warnings warnings.filterwarnings("default") return
# 2018arXiv180410607M datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII', 'marchetti2018.txt') data = read(datafile) for row in pbar(data, task_str): oname = 'Gaia DR2 '+str(row['source_id']).strip()[:6] name, source = catalog.new_entry(oname, bibcode='2018arXiv180410607M') lname = 'Gaia DR2 '+str(row['source_id']).strip() catalog.entries[name].add_quantity(FASTSTARS.ALIAS, lname, source=source) if (FASTSTARS.DISCOVERER not in catalog.entries[name]): catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'T. Marchetti, E. M. Rossi, A. G. A. Brown', source) catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2018), source) #sourcegaia = catalog.entries[name].add_source(bibcode='2016A&A...595A...2G') ra, dec = coord(ra=float(row['ra'])*u.deg,dec=float(row['dec'])*u.deg,frame='icrs').to_string( 'hmsdms', sep=':').split() catalog.entries[name].add_quantity( FASTSTARS.RA, ra, source=source) catalog.entries[name].add_quantity( FASTSTARS.DEC, dec, source=source) # 2018arXiv180503194H datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII','hattori2018.csv') data = read(datafile) for row in pbar(data, task_str): oname = str(row['ID']).strip(' ') name, source = catalog.new_entry(oname, bibcode='2018arXiv180503194H') lgname = 'Gaia DR2 '+str(row['GaiaDR2']).strip(' ') sgname = 'Gaia DR2 '+str(row['GaiaDR2']).strip(' ')[:6] catalog.entries[name].add_quantity(FASTSTARS.ALIAS, lgname, source=source)
if not is_number(newitem['ra'].split(':')[0]): continue newitem['dec'] = item['dec'][0]['value'] if not is_number(newitem['dec'].split(':')[0]): continue newitem['raerr'] = float(item['ra'][0].get('e_value', 0)) newitem['decerr'] = float(item['dec'][0].get('e_value', 0)) # Temporary fix for David's typo if newitem['dec'].count('.') == 2: newitem['dec'] = newitem['dec'][:newitem['dec'].rfind('.')] if 'distinctfrom' in item: newitem['distinctfrom'] = [x['value'] for x in item['distinctfrom']] newcatalog.append(newitem) coo = coord([x['ra'] for x in newcatalog], [x['dec'] for x in newcatalog], unit=(un.hourangle, un.deg)) path = download_file( 'http://cxc.harvard.edu/csc2/preliminary/preliminary_detlist.fits', 'astrocats/supernovae/output/cache') cxctable = fits.getdata(path, 1) cxccatalog = [] for ri, row in enumerate(tqdm(cxctable)): if ri > 1000: break name = row[0] if row[2] != 'TRUE' or row[34] != 'POINT' or row[5] == 'T': continue cxcdict = { 'name': name,
def do_boundprobability(catalog): task_str = catalog.get_current_task_str() keys = list(catalog.entries.keys()) if vescmethod == 'galpy': from galpy.potential import MWPotential2014, vesc elif vescmethod == 'williams2017': datafile = os.path.join(catalog.get_current_task_repo(), 'spherical_powerlaw.dat') chain = np.genfromtxt(datafile) VESCSOLAR = chain[:, 5] ALPHA = chain[:, 6] def vescsolar_sampler(NAME, NSAMP): # Set random number generator based on name seed = NAME def craft_seed(SEED): nSEED = len(SEED) # The modulo ensures we don't overrun. indSEED = range(2 * (nSEED + 3)) return np.prod( np.array([ord(SEED[i % nSEED]) for i in indSEED])) % 4294967296 np.random.seed(seed=craft_seed(seed)) choice_chain = np.random.choice(range(len(VESCSOLAR)), NSAMP) return VESCSOLAR[choice_chain], ALPHA[choice_chain] else: catalog.log.warning( 'vescmethod was not properly defined, please check.') for oname in pbar(keys, task_str): # Some events may be merged in cleanup process, skip them if # non-existent. try: name = catalog.add_entry(oname) except Exception: catalog.log.warning( '"{}" was not found, suggests merge occurred in cleanup ' 'process.'.format(oname)) continue if (FASTSTARS.RA not in catalog.entries[name] or FASTSTARS.DEC not in catalog.entries[name] or (FASTSTARS.LUM_DIST not in catalog.entries[name] and FASTSTARS.PARALLAX not in catalog.entries[name])): # Don't need to worry about not having any velocities. Can't be in the catalogue if it doesn't have velocities! continue else: # Obtain highest precision position #best_position_i = best_parameter(catalog.entries[name][FASTSTARS.RA]) Mradec = str( catalog.entries[name][FASTSTARS.RA][0]['value']) + str( catalog.entries[name][FASTSTARS.DEC][0]['value']) c = coord(Mradec, unit=(un.hourangle, un.deg), frame='icrs') # Obtain galactic coordinates ra = c.ra.rad dec = c.dec.rad l = c.galactic.l.rad b = c.galactic.b.rad # Storage for means, errors and correlation kine_n = 4 kine_values = np.zeros(kine_n) kine_errors = 9999.9 * np.ones(kine_n) kine_corr = np.diag(np.ones(kine_n)) kine_L = 1.35 kine_k = 3.0 # Do we have distance prior length scale? if FASTSTARS.DISTANCE_PRIOR_LENGTH_SCALE in catalog.entries[name]: kine_L = float(catalog.entries[name][ FASTSTARS.DISTANCE_PRIOR_LENGTH_SCALE][0]['value']) # Flags for having proper motions and radial velocities haveparallax = False havepropermotions = False havevelocities = False # Do we have a parallax? if FASTSTARS.PARALLAX in catalog.entries[name]: # Yes. haveparallax = True kine_values[0] = float( catalog.entries[name][FASTSTARS.PARALLAX][0]['value']) kine_errors[0] = float( catalog.entries[name][FASTSTARS.PARALLAX][0]['e_value']) # Is the parallax correlated with the proper motions? if QUANTITY.CORRELATIONS in catalog.entries[name][ FASTSTARS.PARALLAX]: # Yes. Loop through correlations. parallax_corr = catalog.entries[name][ FASTSTARS.PARALLAX][0]['correlations'] n_corr = len(parallax_corr) for i in range(n_corr): if parallax_corr[i]['quantity'] == 'propermotionra': kine_corr[0, 1] = kine_corr[1, 0] = float( parallax_corr[i]['value']) if parallax_corr[i]['quantity'] == 'propermotiondec': kine_corr[0, 2] = kine_corr[2, 0] = float( parallax_corr[i]['value']) # Do we have proper motions? if (FASTSTARS.PROPER_MOTION_RA in catalog.entries[name] and FASTSTARS.PROPER_MOTION_DEC in catalog.entries[name]): # Yes. havepropermotions = True best_propermotion_i = best_parameter( catalog.entries[name][FASTSTARS.PROPER_MOTION_RA])[0] kine_values[1] = float(catalog.entries[name][ FASTSTARS.PROPER_MOTION_RA][best_propermotion_i]['value']) kine_values[2] = float(catalog.entries[name][ FASTSTARS.PROPER_MOTION_DEC][best_propermotion_i]['value']) # Do the proper motions have errors? if (QUANTITY.E_VALUE in catalog.entries[name][ FASTSTARS.PROPER_MOTION_RA][best_propermotion_i] and QUANTITY.E_VALUE in catalog.entries[name][ FASTSTARS.PROPER_MOTION_DEC][best_propermotion_i]): kine_errors[1] = float( catalog.entries[name][FASTSTARS.PROPER_MOTION_RA] [best_propermotion_i]['e_value']) kine_errors[2] = float( catalog.entries[name][FASTSTARS.PROPER_MOTION_DEC] [best_propermotion_i]['e_value']) else: kine_errors[1] = errorifmissing * kine_values[1] kine_errors[2] = errorifmissing * kine_values[2] # Are the proper motions correlated? if (QUANTITY.CORRELATIONS in catalog.entries[name][ FASTSTARS.PROPER_MOTION_RA][best_propermotion_i] and QUANTITY.CORRELATIONS in catalog.entries[name][ FASTSTARS.PROPER_MOTION_DEC][best_propermotion_i]): # Yes. Loop through correlations. propermotionra_corr = catalog.entries[name][ FASTSTARS. PROPER_MOTION_RA][best_propermotion_i]['correlations'] n_corr = len(propermotionra_corr) for i in range(n_corr): if propermotionra_corr[i][ 'quantity'] == 'propermotiondec': kine_corr[1, 2] = kine_corr[2, 1] = float( propermotionra_corr[i]['value']) else: # No. Reset correlations with parallax. kine_corr[0, 1] = kine_corr[1, 0] = kine_corr[ 0, 2] = kine_corr[2, 0] = 0.0 # Do we have a radial velocity? if FASTSTARS.VELOCITY in catalog.entries[name]: # Yes. havevelocities = True # What is the best radial velocity? best_velocity_i, best_velocity_err = best_parameter( catalog.entries[name][FASTSTARS.VELOCITY]) kine_values[3] = float(catalog.entries[name][ FASTSTARS.VELOCITY][best_velocity_i]['value']) # Does the radial velocity have an error? #if QUANTITY.E_VALUE in catalog.entries[name][FASTSTARS.VELOCITY][best_velocity_i]: #kine_errors[3] = float(catalog.entries[name][FASTSTARS.VELOCITY][best_velocity_i]['e_value']) if best_velocity_err < 9999.0: kine_errors[3] = best_velocity_err else: kine_errors[3] = errorifmissing * kine_values[3] # Do we have a previous photometric distance? if FASTSTARS.LUM_DIST in catalog.entries[name]: # Yes. # What is the best photometric distance? best_lumdist_i, best_lumdist_err = best_parameter( catalog.entries[name][FASTSTARS.LUM_DIST]) kine_mu = float(catalog.entries[name][FASTSTARS.LUM_DIST] [best_lumdist_i]['value']) # Does the photometric distance have an error? #if QUANTITY.E_VALUE in catalog.entries[name][FASTSTARS.LUM_DIST][best_lumdist_i]: # kine_sigma = float(catalog.entries[name][FASTSTARS.LUM_DIST][best_lumdist_i]['e_value']) if best_lumdist_err < 9999.0: kine_sigma = best_lumdist_err else: kine_sigma = errorifmissing * kine_mu kine_L = kine_sigma**2 / kine_mu kine_k = kine_mu / kine_L # Sample kine_samples = kinematic_sampler(name, kine_values, kine_errors, kine_corr, KINE_L=kine_L, KINE_K=kine_k) kine_samples_corrected = np.copy(kine_samples) kine_samples_corrected[:, 1] *= k * kine_samples[:, 0] kine_samples_corrected[:, 2] *= k * kine_samples[:, 0] kine_samples_n = kine_samples.shape[0] # Calculate solar reflex correction cosl = cos(l) sinl = sin(l) cosb = cos(b) sinb = sin(b) cosdec = cos(dec) sindec = sin(dec) cosra = cos(ra) sinra = sin(ra) A = np.array([[cosra * cosdec, -sinra, -cosra * sindec], [sinra * cosdec, cosra, -sinra * sindec], [sindec, 0., cosdec]]) B = np.dot(T, A) # T*A #invB = np.linalg.inv(B) kine_samples_solar = solarmotion_sampler(name, kine_samples_n) kine_vhel = kine_samples_solar[:, 2:] # samples in UVW_solar kine_vhel[:, 1] += kine_samples_solar[:, 1] # add the vdisk component kine_D = np.einsum('ei,ni->ne', B.T, kine_vhel) # If had no proper motions, then assume had exactly solar reflex if havepropermotions == False: kine_samples_corrected[:, 1] = -1.0 * kine_D[:, 1] kine_samples_corrected[:, 2] = -1.0 * kine_D[:, 2] # If had no radial velocity, then assume had exactly solar reflex if havevelocities == False: kine_samples_corrected[:, 3] = -1.0 * kine_D[:, 0] kine_samples_vhel = kine_vhel + np.einsum( 'ij,nj->ni', B, np.vstack([ kine_samples_corrected[:, 3], kine_samples_corrected[:, 1], kine_samples_corrected[:, 2] ]).T) kine_vgrf = np.sqrt(kine_samples_vhel[:, 0]**2 + kine_samples_vhel[:, 1]**2 + kine_samples_vhel[:, 2]**2) # Calculate escape velocity kine_galrad = np.sqrt(kine_samples_solar[:, 0]**2 + (kine_samples_corrected[:, 0])**2 - 2. * kine_samples_solar[:, 0] * kine_samples_corrected[:, 0] * cosb * cosl) if vescmethod == 'galpy': kine_vesc = vesc(MWPotential2014, kine_galrad * un.kpc) * kine_samples_solar[:, 1] elif vescmethod == 'williams2017': kine_vesc_solar, kine_alpha = vescsolar_sampler( name, kine_samples_n) kine_vesc = kine_vesc_solar * np.power( kine_galrad / kine_samples_solar[:, 0], -kine_alpha / 2.0) # Store samples for each star if False: if FASTSTARS.DISCOVERER in catalog.entries[ name] and FASTSTARS.DISCOVER_DATE in catalog.entries[ name]: _discoverer = catalog.entries[name][ FASTSTARS.DISCOVERER][0]['value'] _discover_date = catalog.entries[name][ FASTSTARS.DISCOVER_DATE][0]['value'] if ('Bromley' in _discoverer or 'Scholz' in _discoverer) and _discover_date == '2018': kine_samples_output = np.copy(kine_samples_corrected) kine_samples_output[:, 1] /= (k * kine_samples[:, 0]) kine_samples_output[:, 2] /= (k * kine_samples[:, 0]) np.savez_compressed( '/Users/douglasboubert/Documents/Science/GaiaDR2/LiteratureRV/data/' + name + '.npz', radec=Mradec, kine_samples_output=kine_samples_output, kine_samples_solar=kine_samples_solar, kine_vgrf=kine_vgrf) #np.savez_compressed('/data/dpb33/GaiaHypervelocity/WhiteDwarfs/mainsequence/samples/'+name+'.npz',radec=Mradec,kine_samples_output=kine_samples_output,kine_samples_solar=kine_samples_solar,kine_vgrf=kine_vgrf) # Bound probability kine_bound_n = np.where(kine_vesc > kine_vgrf)[0].shape[0] kine_betaa = kine_bound_n + 0.5 kine_betab = kine_samples_n - kine_bound_n + 0.5 kine_bound_percentiles = [ beta.ppf(confidenceintervalcoverage / 2.0, kine_betaa, kine_betab), beta.ppf(0.5, kine_betaa, kine_betab), beta.ppf(1.0 - confidenceintervalcoverage / 2.0, kine_betaa, kine_betab) ] # Store all outcomes. source = catalog.entries[name].add_self_source() boundprobability_upperlimit = (havepropermotions == False or havevelocities == False) catalog.entries[name].add_quantity( FASTSTARS.BOUND_PROBABILITY, str(kine_bound_percentiles[1]), e_lower_value=str(kine_bound_percentiles[1] - kine_bound_percentiles[0]), e_upper_value=str(kine_bound_percentiles[2] - kine_bound_percentiles[1]), upperlimit=boundprobability_upperlimit, source=source, derived=True) catalog.entries[name].add_quantity( FASTSTARS.VELOCITY, str(kine_vesc.mean()), e_value=str(kine_vesc.std()), u_value='km/s', source=source, derived=True, kind=['galactocentric', 'escape']) catalog.entries[name].add_quantity( FASTSTARS.VELOCITY, str(kine_vgrf.mean()), e_value=str(kine_vgrf.std()), u_value='km/s', lowerlimit=boundprobability_upperlimit, source=source, derived=True, kind=['galactocentric', 'total']) if haveparallax == True: catalog.entries[name].add_quantity( FASTSTARS.LUM_DIST, str(kine_samples_corrected[:, 0].mean()), e_value=str(kine_samples_corrected[:, 0].std()), u_value='kpc', source=source, derived=True) catalog.journal_entries() return
def retrieve_objects(self, catalog_name, event_name=None, quantity_name=None, attribute_name=None, full=False): """Retrieve data, first trying catalog file then event files.""" event = None use_full = full search_all = False ename = event_name qname = quantity_name aname = attribute_name req_vals = request.get_json() if not req_vals: req_vals = request.values # Load event/quantity/attribute if provided by request. event_req = req_vals.get('event') quantity_req = req_vals.get('quantity') attribute_req = req_vals.get('attribute') if ename is None and event_req is not None: if not isinstance(event_req, string_types): ename = '+'.join(listify(event_req)) else: ename = event_req if qname is None and quantity_req is not None: if not isinstance(quantity_req, string_types): qname = '+'.join(listify(quantity_req)) else: qname = quantity_req if aname is None and attribute_req is not None: if not isinstance(attribute_req, string_types): aname = '+'.join(listify(attribute_req)) else: aname = attribute_req if ename is None: return msg('no_root_data') # Options if not use_full: rfull = req_vals.get('full') if rfull is not None: return self.retrieve_objects(catalog_name, event_name=ename, quantity_name=qname, attribute_name=aname, full=True) fmt = req_vals.get('format') fmt = fmt.lower() if fmt is not None else fmt fmt = None if fmt == 'json' else fmt ra = req_vals.get('ra') dec = req_vals.get('dec') radius = req_vals.get('radius') width = req_vals.get('width') height = req_vals.get('height') complete = req_vals.get('complete') first = req_vals.get('first') closest = req_vals.get('closest') sortby = req_vals.get('sortby') sortby = sortby.lower() if sortby is not None else sortby include_keys = list(sorted(set(req_vals.keys()) - self._SPECIAL_ATTR)) includes = OrderedDict() iincludes = OrderedDict() for key in include_keys: val = req_vals.get(key, '') if not is_number(val) and val != '': val = '^' + val + '$' try: includes[key] = re.compile(val) iincludes[key] = re.compile(val, re.IGNORECASE) except Exception: return msg('invalid_regex', [req_vals.get(key, ''), key]) excludes = OrderedDict([('realization', '')]) if first is None: item = req_vals.get('item') try: item = int(item) except Exception: item = None else: item = 0 if radius is not None: try: radius = float(radius) except Exception: radius = 0.0 if radius >= self._ANGLE_LIMIT: return msg('radius_limited', self._ANGLE_LIMIT / 3600.) if width is not None: try: width = float(width) except Exception: width = 0.0 if width >= self._ANGLE_LIMIT: return msg('width_limited', self._ANGLE_LIMIT / 3600.) if height is not None: try: height = float(height) except Exception: height = 0.0 if height >= self._ANGLE_LIMIT: return msg('height_limited', self._ANGLE_LIMIT / 3600.) if ename and ename.lower() in ['catalog', 'all']: if ra is not None and dec is not None: try: ldec = str(dec).lower().strip(' .') if is_number(ldec) or decregex.match(ldec): sra = str(ra) lra = sra.lower().replace('h', '').strip(' .') if raregex.match(lra) or (is_number(lra) and 'h' in sra): lcoo = coord(lra, ldec, unit=(un.hourangle, un.deg)) elif is_number(lra): lcoo = coord(lra, ldec, unit=(un.deg, un.deg)) else: raise Exception else: raise Exception except Exception: return msg('bad_coordinates') if (width is not None and height is not None and width > 0.0 and height > 0.0): idxcat = np.where(( abs(lcoo.ra - apidata._coo.ra) <= width * un.arcsecond) & (abs(lcoo.dec - apidata._coo.dec) <= height * un.arcsecond))[0] elif width is not None and width > 0.0: idxcat = np.where( abs(lcoo.ra - apidata._coo.ra) <= width * un.arcsecond)[0] elif height is not None and height > 0.0: idxcat = np.where( abs(lcoo.dec - apidata._coo.dec) <= height * un.arcsecond)[0] else: if radius is None or radius == 0.0: radius = 1.0 idxcat = np.where( lcoo.separation(apidata._coo) <= radius * un.arcsecond)[0] if len(idxcat): ename_arr = [ apidata._rdnames[i].replace('+', '$PLUS$') for i in idxcat ] else: return msg('no_objects') elif catalog_name in apidata._catalogs: ename_arr = [ i.replace('+', '$PLUS$') for i in apidata._catalogs[catalog_name] ] search_all = True else: ename_arr = [ a for b in [[ i.replace('+', '$PLUS$') for i in apidata._catalogs[cat] ] for cat in apidata._catalogs] for a in b ] search_all = True ename = '+'.join(list(sorted(set(ename_arr)))) if qname is None: # Short circuit to full if keyword is present. if full: return self.retrieve_objects(catalog_name, event_name=ename, full=True) search_all = True if catalog_name not in apidata._CATS: qname = '+'.join( list( set( sorted([ a for b in [ apidata._cat_keys[x] for x in apidata._cat_keys ] for a in b ])))) else: qname = '+'.join( list(sorted(set(apidata._cat_keys[catalog_name])))) # if fmt is not None and qname is None: # return Response(( # 'Error: \'{}\' format only supported if quantity ' # 'is specified.').format( # fmt), mimetype='text/plain') # Events event_names = [] if ename is None else ename.split('+') # Check for + in names nevent_names = [] joined = False for ni, name in enumerate(event_names): if joined: joined = False continue if ni < len(event_names) - 1: jname = '+'.join(event_names[ni:ni + 2]) if jname.lower().replace(' ', '') in apidata._aliases: nevent_names.append(jname) joined = True continue nevent_names.append(name) event_names = nevent_names event_names = [x.replace('$PLUS$', '+') for x in event_names] if not len(event_names): search_all = True event_names = apidata._all # Quantities quantity_names = [] if qname is None else qname.split('+') # Attributes. Always append source. attribute_names = [] if aname is None else aname.split('+') if use_full and len(event_names) > self._FULL_LIMIT: return msg('max_events', self._FULL_LIMIT) if fmt is not None and any( [n in attribute_names for n in self._NO_CSV]): return msg('no_delimited') if len(event_names) > self._EXPENSIVE_LIMIT: for quantity in quantity_names: for exp in self._EXPENSIVE: if any([e in attribute_names for e in self._EXPENSIVE]): return msg('too_expensive') edict = OrderedDict() fcatalogs = OrderedDict() sources = OrderedDict() new_event_names = [] for event in event_names: skip_entry = False my_cat, my_event = None, None alopts = apidata._aliases.get(event.lower().replace(' ', ''), []) for opt in alopts: if opt[0] == catalog_name: my_cat, my_event, my_alias = tuple(opt) break if not my_cat: for opt in alopts: if opt[0] != catalog_name: my_cat, my_event, my_alias = tuple(opt) break if not my_cat: if len(event_names) == 1: return msg('event_not_found', event) continue if full: fcatalogs.update( json.load(open( os.path.join(apidata._AC_PATH, apidata._CATS[my_cat][0], 'output', 'json', get_filename(my_event)), 'r'), object_pairs_hook=OrderedDict)) sources[my_event] = [ x.get('bibcode', x.get('arxivid', x.get('name'))) for x in fcatalogs[my_event].get('sources') ] if qname is None: if full: edict[event] = fcatalogs.get(my_event, {}) else: edict[event] = apidata._catalogs.get(my_cat, {}).get(my_event, {}) else: # Check if user passed quantity or attribute names to filter # by. qdict = OrderedDict() if full: my_event_dict = fcatalogs.get(my_event, {}) else: my_event_dict = apidata._catalogs.get(my_cat, {}).get( my_event, {}) if aname is None: for incl in iincludes: incll = incl.lower() if incll not in my_event_dict or ( iincludes[incl].pattern != '' and not any([ bool(iincludes[incl].match( x.get('value', '' ) if isinstance(x, dict) else x)) for x in my_event_dict.get(incll, [{}]) ])): skip_entry = True break if not skip_entry: for quantity in quantity_names: my_quantity = listify(my_event_dict.get(quantity, {})) closest_locs = [] if closest is not None: closest_locs = list( sorted( list( set([ np.argmin([ abs( np.mean([ float(y) for y in listify( x.get(i)) ]) - float(includes[i].pattern)) for x in my_quantity ]) for i in includes if len(my_quantity) and is_number( includes[i].pattern) and all([ is_number(x.get(i, '')) for x in my_quantity ]) ])))) if aname is None and quantity in my_event_dict: qdict[quantity] = [ x for xi, x in enumerate(my_quantity) if not len(closest_locs) or xi in closest_locs ] if item is not None: try: qdict[quantity] = qdict[quantity][item] except Exception: pass else: qdict[quantity] = self.get_attributes( attribute_names, my_quantity, complete=complete, full=use_full, item=item, includes=includes, iincludes=iincludes, excludes=excludes, closest_locs=closest_locs, sources=np.array(sources.get(my_event, []))) if not search_all and not qdict.get(quantity): use_full = True break if not full and use_full: new_event_names = list(event_names) break if qdict: edict[event] = qdict if not full and use_full: break if not (skip_entry and (full or search_all)) and event in edict: new_event_names.append(event) event_names = new_event_names ename = '+'.join([i.replace('+', '$PLUS$') for i in event_names]) if not full and use_full: return self.retrieve_objects(catalog_name, event_name=ename, quantity_name=qname, attribute_name=aname, full=True) if fmt is not None: return self.get_event_dsv(edict, event_names, quantity_names, attribute_names, fmt, sortby) return edict
def do_ascii(catalog): """Process ASCII files extracted from datatables of published works.""" task_str = catalog.get_current_task_str() # catalog.journal_entries() # return #def holding(): # 2007ApJ...660..311B datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII', 'ApJ_660_311_table1.csv') data = read(datafile, format='csv') for row in pbar(data, task_str): oname = 'SDSS'+str(row['SDSS']) name, source = catalog.new_entry(oname, bibcode='2007ApJ...660..311B') if (FASTSTARS.DISCOVERER not in catalog.entries[name]): catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'Warren R. Brown, Margaret J. Geller, Scott J. Kenyon, Michael J. Kurtz, Benjamin C. Bromley', source) catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2007), source) gallon = float(str(row['Glon'])) gallat = float(str(row['Glat'])) radec = oname.strip('SDSSJ') radec = radec[0:2]+' '+radec[2:4]+' '+radec[4:9]+' '+radec[9:12]+' '+radec[12:14]+' '+radec[14:18] ra, dec = coord(radec, unit=(u.hourangle, u.deg)).to_string( 'hmsdms', sep=':').split() catalog.entries[name].add_quantity( FASTSTARS.RA, ra, source=source) catalog.entries[name].add_quantity( FASTSTARS.DEC, dec, source=source) catalog.entries[name].add_quantity( FASTSTARS.VELOCITY, str(row['Vhelio']), source=source) galrad_MS = float(str(row['Ra'])) galrad_BHB = float(str(row['Rb'])) dhel_MS = rgc_to_dhel(galrad_MS,gallon,gallat) dhel_BHB = rgc_to_dhel(galrad_BHB,gallon,gallat) catalog.entries[name].add_quantity( FASTSTARS.LUM_DIST, str(dhel_MS), u_value='kpc', source=source,derived=True) catalog.entries[name].add_quantity( FASTSTARS.LUM_DIST, str(dhel_BHB), u_value='kpc', source=source,derived=True) catalog.entries[name].add_quantity( FASTSTARS.SPECTRAL_TYPE, str(row['Sp']), source=source) catalog.entries[name].add_quantity( FASTSTARS.STELLAR_CLASS, "d", source=source) catalog.entries[name].add_quantity( FASTSTARS.STELLAR_CLASS, "bhb", source=source) catalog.entries[name].add_quantity( FASTSTARS.STELLAR_CLASS, "g", source=source) catalog.journal_entries() # 2009ApJ...690.1639B datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII', 'apj292642t1_ascii.csv') data = read(datafile, format='csv') for row in pbar(data, task_str): oname = row['Catalog'] name, source = catalog.new_entry(oname, bibcode='2009ApJ...690.1639B') if (FASTSTARS.DISCOVERER not in catalog.entries[name]): catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'Warren R. Brown, Margaret J. Geller, Scott J. Kenyon', source) catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2009), source) gallon = float(str(row['Glon'])) gallat = float(str(row['Glat'])) if (oname!='US708') & (oname!='HE0437-5439'): radec = oname.strip('SDSSJ') radec = radec[0:2]+' '+radec[2:4]+' '+radec[4:9]+' '+radec[9:12]+' '+radec[12:14]+' '+radec[14:18] ra, dec = coord(radec, unit=(u.hourangle, u.deg)).to_string( 'hmsdms', sep=':').split() catalog.entries[name].add_quantity( FASTSTARS.RA, ra, source=source) catalog.entries[name].add_quantity( FASTSTARS.DEC, dec, source=source) if str(row['e_Vhel'])!='NA': catalog.entries[name].add_quantity( FASTSTARS.VELOCITY, str(row['Vhel']), e_value=str(row['e_Vhel']), source=source) else: catalog.entries[name].add_quantity( FASTSTARS.VELOCITY, str(row['Vhel']), source=source) galrad = float(str(row['RGC'])) dhel = rgc_to_dhel(galrad,gallon,gallat) catalog.entries[name].add_quantity( FASTSTARS.LUM_DIST, str(dhel), u_value='kpc', source=source,derived=True) sptype = str(row['Type']).split('/') for SPTYPE in sptype: if SPTYPE == "sdO": catalog.entries[name].add_quantity( FASTSTARS.SPECTRAL_TYPE, "O", source=source) catalog.entries[name].add_quantity( FASTSTARS.STELLAR_CLASS, "sd", source=source) elif SPTYPE == "BHB": catalog.entries[name].add_quantity( FASTSTARS.STELLAR_CLASS, "bhb", source=source) else: catalog.entries[name].add_quantity( FASTSTARS.SPECTRAL_TYPE, SPTYPE, source=source) catalog.entries[name].add_quantity( FASTSTARS.STELLAR_CLASS, "d", source=source) if str(row['ID'])[:3]=='HVS': catalog.entries[name].add_quantity( FASTSTARS.ALIAS, row['ID'], source=source) catalog.journal_entries() # 2012ApJ...744L..24L datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII', 'apjl415156t1t2.csv') data = read(datafile, format='csv') for row in pbar(data, task_str): oname = str(row['Catalog']) name, source = catalog.new_entry(oname, bibcode='2012ApJ...744L..24L') if (FASTSTARS.DISCOVERER not in catalog.entries[name]): catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'Yinbi Li, Ali Luo, Gang Zhao, Youjun Lu, Juanjuan Ren, Fang Zuo', source) catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2012), source) catalog.entries[name].add_quantity( FASTSTARS.ALIAS, row['ID'], source=source) radec = oname.strip('SDSSJ') radec = radec[0:2]+' '+radec[2:4]+' '+radec[4:9]+' '+radec[9:12]+' '+radec[12:14]+' '+radec[14:18] ra, dec = coord(radec, unit=(u.hourangle, u.deg)).to_string( 'hmsdms', sep=':').split() catalog.entries[name].add_quantity( FASTSTARS.RA, ra, source=source) catalog.entries[name].add_quantity( FASTSTARS.DEC, dec, source=source) catalog.entries[name].add_quantity( FASTSTARS.VELOCITY, str(row['Vhel']), e_value=str(row['e_Vhel']), source=source) catalog.entries[name].add_quantity( FASTSTARS.LUM_DIST, str(row['Dhel']), e_value=str(row['e_Dhel']), u_value='kpc', source=source) catalog.entries[name].add_quantity( FASTSTARS.PROPER_MOTION_RA, str(row['pmra']), e_value=str(row['e_pmra']), u_value='mas/yr', source=source) catalog.entries[name].add_quantity( FASTSTARS.PROPER_MOTION_DEC, str(row['pmdec']), e_value=str(row['e_pmdec']), u_value='mas/yr', source=source) catalog.entries[name].add_quantity( FASTSTARS.SPECTRAL_TYPE, "F", source=source) catalog.entries[name].add_quantity( FASTSTARS.STELLAR_CLASS, "d", source=source) catalog.journal_entries() # 2012ApJ...751...55B datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII', 'apj427101t1_ascii.csv') data = read(datafile, format='csv') for row in pbar(data, task_str): oname = str(row['Catalog']).replace(' ','') name, source = catalog.new_entry(oname, bibcode='2012ApJ...751...55B') if (FASTSTARS.DISCOVERER not in catalog.entries[name]): catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'Warren R. Brown, Margaret J. Geller, Scott J. Kenyon', source) catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2012), source) if name == 'SDSSJ112359.47+751807.73': sourcewarren = catalog.entries[name].add_source(name='Warren R. Brown (private comm.)',secondary=True) catalog.entries[name].add_quantity(FASTSTARS.SPECTRAL_TYPE, 'B9', source=sourcewarren) gallon = float(str(row['Glon'])) gallat = float(str(row['Glat'])) if (oname!='US708') & (oname!='HE0437-5439'): radec = oname.strip('SDSSJ') radec = radec[0:2]+' '+radec[2:4]+' '+radec[4:9]+' '+radec[9:12]+' '+radec[12:14]+' '+radec[14:] ra, dec = coord(radec, unit=(u.hourangle, u.deg)).to_string( 'hmsdms', sep=':').split() catalog.entries[name].add_quantity( FASTSTARS.RA, ra, source=source) catalog.entries[name].add_quantity( FASTSTARS.DEC, dec, source=source) if str(row['e_Vhel'])!='NA': catalog.entries[name].add_quantity( FASTSTARS.VELOCITY, str(row['Vhel']), e_value=str(row['e_Vhel']), source=source) else: catalog.entries[name].add_quantity( FASTSTARS.VELOCITY, str(row['Vhel']), source=source) galrad = float(str(row['RGC'])) dhel = rgc_to_dhel(galrad,gallon,gallat) catalog.entries[name].add_quantity( FASTSTARS.LUM_DIST, str(dhel), u_value='kpc', source=source, derived=True) sptype = str(row['Type']).split('/') for SPTYPE in sptype: if SPTYPE != "NA": catalog.entries[name].add_quantity( FASTSTARS.SPECTRAL_TYPE, SPTYPE, source=source) catalog.entries[name].add_quantity( FASTSTARS.STELLAR_CLASS, "d", source=source) if str(row['ID'])[:3]=='HVS': catalog.entries[name].add_quantity( FASTSTARS.ALIAS, row['ID'], source=source) catalog.journal_entries() # 2014ApJ...780....7P datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII', 'apj485719t1_ascii.csv') data = read(datafile, format='csv') for row in pbar(data, task_str): oname = 'SDSS'+str(row['Catalog']).replace(' ','') name, source = catalog.new_entry(oname, bibcode='2014ApJ...780....7P') if (FASTSTARS.DISCOVERER not in catalog.entries[name]): catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'Lauren E. Palladino, Katharine J. Schlesinger, Kelly Holley-Bockelmann, Carlos Allende Prieto, Timothy C. Beers, Young Sun Lee, Donald P. Schneider', source) catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2014), source) catalog.entries[name].add_quantity( FASTSTARS.ALIAS, 'Pal'+str(row['Pal']), source=source) radec = oname.strip('SDSSJ') radec = radec[0:2]+' '+radec[2:4]+' '+radec[4:9]+' '+radec[9:12]+' '+radec[12:14]+' '+radec[14:18] ra, dec = coord(radec, unit=(u.hourangle, u.deg)).to_string( 'hmsdms', sep=':').split() catalog.entries[name].add_quantity( FASTSTARS.RA, ra, source=source) catalog.entries[name].add_quantity( FASTSTARS.DEC, dec, source=source) catalog.entries[name].add_quantity( FASTSTARS.VELOCITY, str(row['Vhel']), source=source) catalog.entries[name].add_quantity( FASTSTARS.LUM_DIST, str(row['Dhel']), u_value='kpc', source=source) catalog.entries[name].add_quantity( FASTSTARS.SPECTRAL_TYPE, "G", source=source) catalog.entries[name].add_quantity( FASTSTARS.SPECTRAL_TYPE, "K", source=source) catalog.entries[name].add_quantity( FASTSTARS.STELLAR_CLASS, "d", source=source) catalog.journal_entries() # 2014ApJ...787...89B datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII', 'apj494602t1_ascii.csv') data = read(datafile, format='csv') for row in pbar(data, task_str): oname = str(row['Catalog']).replace(' ','') name, source = catalog.new_entry(oname, bibcode='2014ApJ...787...89B') if (FASTSTARS.DISCOVERER not in catalog.entries[name]): catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'Warren R. Brown, Margaret J. Geller, Scott J. Kenyon', source) catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2014), source) radec = oname.strip('SDSSJ') radec = radec[0:2]+' '+radec[2:4]+' '+radec[4:9]+' '+radec[9:12]+' '+radec[12:14]+' '+radec[14:19] ra, dec = coord(radec, unit=(u.hourangle, u.deg)).to_string( 'hmsdms', sep=':').split() catalog.entries[name].add_quantity( FASTSTARS.RA, ra, source=source) catalog.entries[name].add_quantity( FASTSTARS.DEC, dec, source=source) catalog.entries[name].add_quantity( FASTSTARS.VELOCITY, str(row['Vhel']), e_value=str(row['e_Vhel']), source=source) if FASTSTARS.SPECTRAL_TYPE not in catalog.entries[name]: sourceboubert = catalog.entries[name].add_source(bibcode='2018arXiv180410179B',secondary=True) catalog.entries[name].add_quantity(FASTSTARS.SPECTRAL_TYPE, 'A', source=sourceboubert) catalog.entries[name].add_quantity(FASTSTARS.SPECTRAL_TYPE, 'B', source=sourceboubert) galrad = float(str(row['RGC'])) errgalrad = float(str(row['e_RGC'])) dhel = rgc_to_dhel(galrad,gallon,gallat) dhel_lo = rgc_to_dhel(galrad-errgalrad,gallon,gallat) dhel_hi = rgc_to_dhel(galrad+errgalrad,gallon,gallat) catalog.entries[name].add_quantity( FASTSTARS.LUM_DIST, str(dhel), e_lower_value=dhel-dhel_lo, e_upper_value=dhel_hi-dhel, u_value='kpc', source=source, derived=True) if str(row['ID'])!='pBHVS': catalog.entries[name].add_quantity( FASTSTARS.ALIAS, 'HVS'+str(row['ID']), source=source) if str(row['ID'])=='22': catalog.entries[name].add_quantity( FASTSTARS.SPECTRAL_TYPE, "B", source=source) catalog.entries[name].add_quantity( FASTSTARS.STELLAR_CLASS, "d", source=source) catalog.entries[name].add_quantity( FASTSTARS.STELLAR_CLASS, "bhb", source=source) catalog.entries[name].add_quantity( FASTSTARS.STELLAR_CLASS, "g", source=source) elif str(row['ID'])=='23': catalog.entries[name].add_quantity( FASTSTARS.SPECTRAL_TYPE, "B", source=source) catalog.entries[name].add_quantity( FASTSTARS.STELLAR_CLASS, "d", source=source) catalog.entries[name].add_quantity( FASTSTARS.STELLAR_CLASS, "bhb", source=source) catalog.entries[name].add_quantity( FASTSTARS.STELLAR_CLASS, "g", source=source) elif str(row['ID'])=='24': catalog.entries[name].add_quantity( FASTSTARS.SPECTRAL_TYPE, "B", source=source) catalog.entries[name].add_quantity( FASTSTARS.STELLAR_CLASS, "d", source=source) catalog.journal_entries() # 2014EAS....67..255Z datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII', '1501.07824.csv') data = read(datafile, format='csv') for row in pbar(data, task_str): oname = str(row['ID']) name, source = catalog.new_entry(oname, bibcode='2014EAS....67..255Z') if (FASTSTARS.DISCOVERER not in catalog.entries[name]): catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER, 'Y. Q. Zhang, M. C. Smith, J. L. Carlin', source) catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2014), source) radeg = str(row['ra']) decdeg = str(row['dec']) ra, dec = coord(ra=radeg*u.deg, dec=decdeg*u.deg).to_string( 'hmsdms', sep=':').split() catalog.entries[name].add_quantity( FASTSTARS.RA, ra, source=source) catalog.entries[name].add_quantity( FASTSTARS.DEC, dec, source=source) catalog.entries[name].add_quantity( FASTSTARS.VELOCITY, str(row['Vhel']), e_value=str(row['e_Vhel']), source=source) catalog.entries[name].add_quantity( FASTSTARS.PROPER_MOTION_RA, str(row['pmra']), e_value=str(row['e_pmra']), u_value='mas/yr', source=source) catalog.entries[name].add_quantity( FASTSTARS.PROPER_MOTION_DEC, str(row['pmdec']), e_value=str(row['e_pmdec']), u_value='mas/yr', source=source) catalog.entries[name].add_quantity( FASTSTARS.LUM_DIST, str(row['Dhel']), u_value='kpc', source=source) # This distance may have some metallicity dependent uncertainty? catalog.entries[name].add_quantity( FASTSTARS.SPECTRAL_TYPE, 'K', source=source) catalog.entries[name].add_quantity( FASTSTARS.SPECTRAL_TYPE, 'M', source=source) catalog.entries[name].add_quantity( FASTSTARS.STELLAR_CLASS, 'd', source=source) catalog.journal_entries() # 2014ApJ...789L...2Z datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII', 'apjl496832t1t2t3_ascii.csv') data = read(datafile, format='csv') for row in pbar(data, task_str): oname = str(row['ID']) name, source = catalog.new_entry(oname, bibcode='2014ApJ...789L...2Z') if (FASTSTARS.DISCOVERER not in catalog.entries[name]): catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'Jing Zhong, Li Chen, Chao Liu, Richard de Grijs, Jinliang Hou, Shiyin Shen, Zhengyi Shao, Jing Li, Ali Luo, Jianrong Shi, Haotong Zhang, Ming Yang, Licai Deng, Ge Jin, Yong Zhang, Yonghui Hou, Zhenchao Zhang', source) catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2014), source) catalog.entries[name].add_quantity( FASTSTARS.ALIAS, str(row['Catalog']), source=source) radec = str(row['Catalog']).strip('J') radec = radec[0:2]+' '+radec[2:4]+' '+radec[4:9]+' '+radec[9:12]+' '+radec[12:14]+' '+radec[14:18] ra, dec = coord(radec, unit=(u.hourangle, u.deg)).to_string( 'hmsdms', sep=':').split() catalog.entries[name].add_quantity( FASTSTARS.RA, ra, source=source) catalog.entries[name].add_quantity( FASTSTARS.DEC, dec, source=source) catalog.entries[name].add_quantity( FASTSTARS.VELOCITY, str(row['Vrb']), source=source) catalog.entries[name].add_quantity( FASTSTARS.PROPER_MOTION_RA, str(row['pmra']), e_value=str(row['e_pmra']), u_value='mas/yr', source=source) catalog.entries[name].add_quantity( FASTSTARS.PROPER_MOTION_DEC, str(row['pmdec']), e_value=str(row['e_pmdec']), u_value='mas/yr', source=source) catalog.entries[name].add_quantity( FASTSTARS.LUM_DIST, str(row['Dhel']), u_value='kpc', source=source) # This distance may have some metallicity dependent uncertainty? catalog.entries[name].add_quantity( FASTSTARS.SPECTRAL_TYPE, str(row['Type']), source=source) catalog.entries[name].add_quantity( FASTSTARS.STELLAR_CLASS, 'd', source=source) catalog.journal_entries() # 2014ApJ...794..146T datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII', 'ApJ794146.csv') data = read(datafile, format='csv') for row in pbar(data, task_str): oname = str(row['Catalog']).replace(' ','') name, source = catalog.new_entry(oname, bibcode='2014ApJ...794..146T') if (FASTSTARS.DISCOVERER not in catalog.entries[name]): catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'Christopher A. Theissen, Andrew A. West', source) catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2014), source) radec = oname.strip('SDSSJ') radec = radec[0:2]+' '+radec[2:4]+' '+radec[4:9]+' '+radec[9:12]+' '+radec[12:14]+' '+radec[14:18] ra, dec = coord(radec, unit=(u.hourangle, u.deg)).to_string( 'hmsdms', sep=':').split() catalog.entries[name].add_quantity( FASTSTARS.RA, ra, source=source) catalog.entries[name].add_quantity( FASTSTARS.DEC, dec, source=source) catalog.entries[name].add_quantity( FASTSTARS.VELOCITY, str(row['Vhel']), e_value=str(row['e_Vhel']), source=source) catalog.entries[name].add_quantity( FASTSTARS.PROPER_MOTION_RA, str(row['pmra']), e_value=str(row['e_pmra']), u_value='mas/yr', source=source) catalog.entries[name].add_quantity( FASTSTARS.PROPER_MOTION_DEC, str(row['pmdec']), e_value=str(row['e_pmdec']), u_value='mas/yr', source=source) catalog.entries[name].add_quantity( FASTSTARS.LUM_DIST, str(row['Dhel']), e_value=str(row['e_Dhel']), u_value='kpc', source=source) catalog.entries[name].add_quantity( FASTSTARS.SPECTRAL_TYPE, str(row['Type']), source=source) catalog.entries[name].add_quantity( FASTSTARS.STELLAR_CLASS, 'd', source=source) catalog.journal_entries() # 2014ApJ...794..145S datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII', 'apj501503t8_ascii.csv') data = read(datafile, format='csv') for row in pbar(data, task_str): oname = str(row['Catalog']) oname = oname[:4]+'J'+oname[4:] name, source = catalog.new_entry(oname, bibcode='2014ApJ...794..145S') if (FASTSTARS.DISCOVERER not in catalog.entries[name]): catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'Antonia S. Savcheva, Andrew A. West, John J. Bochanski', source) catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2014), source) radec = oname.strip('SDSSJ') radec = radec[0:2]+' '+radec[2:4]+' '+radec[4:8]+' '+radec[8:11]+' '+radec[11:13]+' '+radec[13:17] ra, dec = coord(radec, unit=(u.hourangle, u.deg)).to_string( 'hmsdms', sep=':').split() catalog.entries[name].add_quantity( FASTSTARS.RA, ra, source=source) catalog.entries[name].add_quantity( FASTSTARS.DEC, dec, source=source) catalog.entries[name].add_quantity( FASTSTARS.VELOCITY, str(row['Vhel']), source=source) catalog.entries[name].add_quantity( FASTSTARS.PROPER_MOTION_RA, str(row['pmra']), e_value=str(row['e_pmra']), u_value='mas/yr', source=source) catalog.entries[name].add_quantity( FASTSTARS.PROPER_MOTION_DEC, str(row['pmdec']), e_value=str(row['e_pmdec']), u_value='mas/yr', source=source) catalog.entries[name].add_quantity( FASTSTARS.LUM_DIST, str(row['Dhel']), e_value=str(row['e_Dhel']), u_value='kpc', source=source) sptype = str(row['Type']) catalog.entries[name].add_quantity( FASTSTARS.SPECTRAL_TYPE, sptype[2:], source=source) catalog.entries[name].add_quantity( FASTSTARS.STELLAR_CLASS, "sd", source=source) catalog.journal_entries() # 2015MNRAS.447.2046H datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII', 'hawkins2015.csv') data = read(datafile, format='csv') for row in pbar(data, task_str): oname = str(row['Catalog']).strip(' ') oname = 'RAVE'+oname name, source = catalog.new_entry(oname, bibcode='2015MNRAS.447.2046H') if (FASTSTARS.DISCOVERER not in catalog.entries[name]): catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'K. Hawkins, G. Kordopatis, G. Gilmore, T. Masseron, R. F. G. Wyse, G. Ruchti, O. Bienaymé, J. Bland-Hawthorn, C. Boeche, K. Freeman, B. K. Gibson, E. K. Grebel, A. Helmi, A. Kunder, U. Munari, J. F. Navarro, Q. A. Parker, W. A. Reid, R. D. Scholz, G. Seabroke, A. Siebert, M. Steinmetz, F. Watson, T. Zwitter', source) catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2015), source) radec = oname.strip('RAVEJ') radec = radec[0:2]+' '+radec[2:4]+' '+radec[4:8]+' '+radec[8:11]+' '+radec[11:13]+' '+radec[13:15] ra, dec = coord(radec, unit=(u.hourangle, u.deg)).to_string( 'hmsdms', sep=':').split() catalog.entries[name].add_quantity( FASTSTARS.RA, ra, source=source) catalog.entries[name].add_quantity( FASTSTARS.DEC, dec, source=source) catalog.entries[name].add_quantity( FASTSTARS.VELOCITY, str(row['Vhel']).strip(' '), e_value=str(row['e_Vhel']).strip(' '), source=source) catalog.entries[name].add_quantity( FASTSTARS.PROPER_MOTION_RA, str(row['pmra']).strip(' '), e_value=str(row['e_pmra']).strip(' '), u_value='mas/yr', source=source) catalog.entries[name].add_quantity( FASTSTARS.PROPER_MOTION_DEC, str(row['pmdec']).strip(' '), e_value=str(row['e_pmdec']).strip(' '), u_value='mas/yr', source=source) catalog.entries[name].add_quantity( FASTSTARS.LUM_DIST, str(float(row['Dhel'])/1e3).strip(' '), e_value=str(float(row['e_Dhel'])/1e3).strip(' '), u_value='kpc', source=source) catalog.entries[name].add_quantity( FASTSTARS.STELLAR_CLASS, "g", source=source) catalog.journal_entries() # 2015A&A...576L..14Z datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII', 'zeigerer.csv') data = read(datafile, format='csv') for row in pbar(data, task_str): oname = 'Pal'+str(row['Pal']) name, source = catalog.new_entry(oname, bibcode='2015A&A...576L..14Z') if (FASTSTARS.DISCOVERER not in catalog.entries[name]): catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'E. Ziegerer, M. Volkert, U. Heber, A. Irrgang, B. T. Gänsicke, S. Geier', source) catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2015), source) catalog.entries[name].add_quantity( FASTSTARS.PROPER_MOTION_RA, str(row['pmra']), e_value=str(row['e_pmra']), u_value='mas/yr', source=source) catalog.entries[name].add_quantity( FASTSTARS.PROPER_MOTION_DEC, str(row['pmdec']), e_value=str(row['e_pmdec']), u_value='mas/yr', source=source) catalog.journal_entries() # 2015ApJ...804...49B datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII', 'apj510826t1_ascii.csv') data = read(datafile, format='csv') for row in pbar(data, task_str): oname = str(row['Catalog']).strip(' ') if oname[1]==':': # Add the leading 0 oname = '0'+oname oname = 'SDSSJ'+oname name, source = catalog.new_entry(oname.replace(':',''), bibcode='2015ApJ...804...49B') if (FASTSTARS.DISCOVERER not in catalog.entries[name]): catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'Warren R. Brown, Jay Anderson, Oleg Y. Gnedin, Howard E. Bond, Margaret J. Geller, Scott J. Kenyon', source) catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2015), source) catalog.entries[name].add_quantity( FASTSTARS.ALIAS, str(row['ID']), source=source) catalog.entries[name].add_quantity( FASTSTARS.PROPER_MOTION_RA, str(row['pmra']).strip(' '), e_value=str(row['e_pmra']).strip(' '), u_value='mas/yr', source=source) catalog.entries[name].add_quantity( FASTSTARS.PROPER_MOTION_DEC, str(row['pmdec']).strip(' '), e_value=str(row['e_pmdec']).strip(' '), u_value='mas/yr', source=source) catalog.entries[name].add_quantity( FASTSTARS.LUM_DIST, str(row['Dhel']).strip(' '), e_value=str(row['e_Dhel']).strip(' '), u_value='kpc', source=source) if str(row['newspec']) == 'y': radec = oname.strip('SDSSJ') ra, dec = radec[:11], radec[11:] catalog.entries[name].add_quantity( FASTSTARS.RA, ra, source=source) catalog.entries[name].add_quantity( FASTSTARS.DEC, dec, source=source) catalog.entries[name].add_quantity( FASTSTARS.VELOCITY, str(row['Vhel']), e_value=str(row['e_Vhel']), source=source) #catalog.entries[name].add_quantity( # FASTSTARS.LUM_DIST, str(row['Dhel']), e_value=str(row['e_Dhel']), u_value='kpc', source=source) catalog.entries[name].add_quantity( FASTSTARS.SPECTRAL_TYPE, str(row['Type']), source=source) catalog.journal_entries() # 2015RAA....15.1364L datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII', 'li2015.csv') data = read(datafile, format='csv') for row in pbar(data, task_str): oname = str(row['Catalog']) oname = 'LAMOST'+oname name, source = catalog.new_entry(oname, bibcode='2015RAA....15.1364L') if (FASTSTARS.DISCOVERER not in catalog.entries[name]): catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'Yin-Bi Li, A.-Li Luo, Gang Zhao, You-Jun Lu, Peng Wei, Bing Du, Xiang Li, Yong-Heng Zhao, Zhan-Wen Han, Bo Wang, Yue Wu, Yong Zhang, Yong-Hui Hou, Yue-Fei Wang, Ming Yang', source) catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2015), source) radec = oname.strip('LAMOSTJ') radec = radec[0:2]+' '+radec[2:4]+' '+radec[4:9]+' '+radec[9:12]+' '+radec[12:14]+' '+radec[14:] ra, dec = coord(radec, unit=(u.hourangle, u.deg)).to_string( 'hmsdms', sep=':').split() catalog.entries[name].add_quantity( FASTSTARS.ALIAS, 'Li'+str(row['ID']), source=source) catalog.entries[name].add_quantity( FASTSTARS.RA, ra, source=source) catalog.entries[name].add_quantity( FASTSTARS.DEC, dec, source=source) catalog.entries[name].add_quantity( FASTSTARS.VELOCITY, str(row['Vhel']), e_value=str(row['e_Vhel']), source=source) catalog.entries[name].add_quantity( FASTSTARS.PROPER_MOTION_RA, str(row['pmra']), e_value=str(row['e_pmra']), u_value='mas/yr', source=source) catalog.entries[name].add_quantity( FASTSTARS.PROPER_MOTION_DEC, str(row['pmdec']), e_value=str(row['e_pmdec']), u_value='mas/yr', source=source) catalog.entries[name].add_quantity( FASTSTARS.LUM_DIST, str(row['Dhel']), e_value=str(row['e_Dhel']), u_value='kpc', source=source) catalog.entries[name].add_quantity( FASTSTARS.SPECTRAL_TYPE, 'F', source=source) catalog.entries[name].add_quantity( FASTSTARS.SPECTRAL_TYPE, 'G', source=source) catalog.entries[name].add_quantity( FASTSTARS.SPECTRAL_TYPE, 'K', source=source) catalog.entries[name].add_quantity( FASTSTARS.STELLAR_CLASS, 'd', source=source) catalog.journal_entries() # 2015AJ....150...77V datafile = os.path.join(catalog.get_current_task_repo(), 'ASCII', 'vickers2015final.csv') data = read(datafile, format='csv') for row in pbar(data, task_str): oname = str(row['Catalog']) oname = 'SDSS'+oname name, source = catalog.new_entry(oname, bibcode='2015AJ....150...77V') if (FASTSTARS.DISCOVERER not in catalog.entries[name]): catalog.entries[name].add_quantity(FASTSTARS.DISCOVERER,'John J.Vickers, Martin C. Smith, Eva K. Grebel', source) catalog.entries[name].add_quantity(FASTSTARS.DISCOVER_DATE,str(2015), source) radec = oname.strip('SDSSJ') radec = radec[0:2]+' '+radec[2:4]+' '+radec[4:9]+' '+radec[9:12]+' '+radec[12:14]+' '+radec[14:] ra, dec = coord(radec, unit=(u.hourangle, u.deg)).to_string( 'hmsdms', sep=':').split() catalog.entries[name].add_quantity( FASTSTARS.RA, ra, source=source) catalog.entries[name].add_quantity( FASTSTARS.DEC, dec, source=source) if str(row['distance']) != '': catalog.entries[name].add_quantity( FASTSTARS.LUM_DIST, str(row['distance']), u_value='kpc', source=source)) catalog.entries[name].add_quantity( FASTSTARS.SPECTRAL_TYPE, 'F', source=source) catalog.entries[name].add_quantity( FASTSTARS.SPECTRAL_TYPE, 'G', source=source)
continue newitem['dec'] = item['dec'][0]['value'] if not is_number(newitem['dec'].split(':')[0]): continue newitem['raerr'] = float(item['ra'][0].get('e_value', 0)) newitem['decerr'] = float(item['dec'][0].get('e_value', 0)) # Temporary fix for David's typo if newitem['dec'].count('.') == 2: newitem['dec'] = newitem['dec'][:newitem['dec'].rfind('.')] if 'distinctfrom' in item: newitem['distinctfrom'] = [ x['value'] for x in item['distinctfrom'] ] newcatalog.append(newitem) coo = coord([x['ra'] for x in newcatalog], [x['dec'] for x in newcatalog], unit=(un.hourangle, un.deg)) path = download_file( 'http://cxc.harvard.edu/csc2/preliminary/preliminary_detlist.fits', 'astrocats/supernovae/output/cache') cxctable = fits.getdata(path, 1) cxccatalog = [] for ri, row in enumerate(tqdm(cxctable)): if ri > 1000: break name = row[0] if row[2] != 'TRUE' or row[34] != 'POINT' or row[5] == 'T': continue cxcdict = { 'name': name,
def do_mast_spectra(catalog): """Import HST spectra from MAST.""" task_str = catalog.get_current_task_str() masturl = 'https://mast.stsci.edu' mastref = 'MAST' fureps = {'erg/cm2/s/A': 'erg/s/cm^2/Angstrom'} histfile = os.path.join( catalog.get_current_task_repo(), 'MAST', 'history.json') if os.path.exists(histfile): histdict = json.load(open(histfile, 'r')) else: histdict = OrderedDict() # objectOfInterest = 'ASASSN-14li' # # resolverRequest = {'service': 'Mast.Name.Lookup', # 'params': {'input': objectOfInterest, # 'format': 'json'}, # } # # headers, resolvedObjectString = mastQuery(resolverRequest) # # resolvedObject = json.loads(resolvedObjectString) # # pp.pprint(resolvedObject) # objRa = resolvedObject['resolvedCoordinate'][0]['ra'] # objDec = resolvedObject['resolvedCoordinate'][0]['decl'] objs = [] for entry in catalog.entries: if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[entry]: dd = catalog.entries[entry][ TIDALDISRUPTION.DISCOVER_DATE][0][QUANTITY.VALUE] try: dt = datetime.strptime(dd, '%Y/%m/%d') if dt < datetime(1997, 1, 1): continue except Exception: pass if (TIDALDISRUPTION.RA in catalog.entries[entry] and TIDALDISRUPTION.DEC in catalog.entries[entry]): objs.append([ entry, catalog.entries[entry][ TIDALDISRUPTION.RA][0][QUANTITY.VALUE], catalog.entries[entry][ TIDALDISRUPTION.DEC][0][QUANTITY.VALUE]]) if not len(objs): return objs = np.array(objs).T coords = coord(objs[1], objs[2], unit=(un.hourangle, un.deg)) for ci, co in enumerate(pbar(coords, desc=task_str)): entry = objs[0][ci] use_cache = False if entry in histdict: if dt < datetime(datetime.now().year - 5, 1, 1): use_cache = True if (dt >= datetime(datetime.now().year - 5, 1, 1) and time.time() - histdict[entry][0] < 30. * 86400.): use_cache = True if use_cache: spectra = histdict[entry][1] else: objRa, objDec = tuple(co.to_string().split()) mastRequest = {'service': 'Mast.Caom.Cone', 'params': {'ra': objRa, 'dec': objDec, 'radius': 0.008}, 'format': 'json', 'pagesize': 1000, 'page': 1, 'removenullcolumns': True, 'removecache': True} try: headers, mastDataString = mastQuery(mastRequest) except Exception: print('`mastQuery` failed for `{}`, skipping.'.format(entry)) continue mastData = json.loads(mastDataString) # print(mastData.keys()) # print("Query status:", mastData['status']) mastDataTable = Table() if 'fields' not in mastData: print('`fields` not found for `{}`'.format(entry)) continue for col, atype in [( x['name'], x['type']) for x in mastData['fields']]: if atype == "string": atype = "str" if atype == "boolean": atype = "bool" mastDataTable[col] = np.array( [x.get(col, None) for x in mastData['data']], dtype=atype) # print(mastDataTable.columns) spectra = [ x for x in mastDataTable if x['dataproduct_type'] == 'spectrum' and x[ 'obs_collection'] == 'HST'] spectra = [{ 'target_classification': x['target_classification'], 'obsid': x['obsid'], 't_min': x['t_min'], 't_max': x['t_max'], 'instrument_name': x['instrument_name'], 'proposal_pi': x['proposal_pi'], } for x in spectra] mjd = '' instrument = '' observer = '' for si in range(len(spectra)): spec = spectra[si] # if all([x not in spec['target_classification'].upper() # for x in ['SUPERNOVA', 'UNIDENTIFIED']]): # continue obsid = spec['obsid'] mjd = str(Decimal('0.5') * ( Decimal(str(spec['t_min'])) + Decimal(str(spec['t_max'])))) instrument = spec['instrument_name'] if ('STIS' not in instrument.upper() and 'COS' not in instrument.upper()): continue observer = spec['proposal_pi'] if use_cache: scienceProducts = spec.get('sciProds', []) else: productRequest = {'service': 'Mast.Caom.Products', 'params': {'obsid': obsid}, 'format': 'json', 'pagesize': 100, 'page': 1} try: headers, obsProductsString = mastQuery(productRequest) except Exception: print( '`mastQuery` failed for `{}`, skipping.'.format(obsid)) continue obsProducts = json.loads(obsProductsString) if 'fields' not in obsProducts: print('`fields` not found for `{}`'.format(obsid)) print(obsProducts) continue sciProdArr = [x for x in obsProducts['data'] if x.get("productType", None) == 'SCIENCE'] scienceProducts = OrderedDict() for col, atype in [ (x['name'], x['type']) for x in obsProducts['fields']]: if atype == "string": atype = "str" if atype == "boolean": atype = "bool" if atype == "int": # array may contain nan values, and they do not exist # in numpy integer arrays. atype = "float" scienceProducts[col] = [ x.get(col, None) for x in sciProdArr] spectra[si]['sciProds'] = scienceProducts # print("Number of science products:", len(scienceProducts)) # print(scienceProducts) search_str = '_x1d.fits' summed = False if any(['_x1dsum.fits' in x for x in scienceProducts[ 'productFilename']]): search_str = '_x1dsum.fits' summed = True if any(['_sx1.fits' in x for x in scienceProducts[ 'productFilename']]): search_str = '_sx1.fits' summed = True for ri in range(len(scienceProducts['productFilename'])): if search_str not in scienceProducts['productFilename'][ri]: continue filename = str(obsid) + "_" + \ scienceProducts['productFilename'][ri] datafile = os.path.join( catalog.get_current_task_repo(), 'MAST', filename) if not os.path.exists(datafile): # link is url, so can just dl if "http" in scienceProducts['dataURI'][ri]: urlretrieve(scienceProducts['dataURI'][ri], datafile) else: # link is uri, need to go through direct dl request server = 'mast.stsci.edu' conn = httplib.HTTPSConnection(server) conn.request( "GET", "/api/v0/download/file/" + scienceProducts['dataURI'][ri].lstrip('mast:')) resp = conn.getresponse() fileContent = resp.read() with open(datafile, 'wb') as FLE: FLE.write(fileContent) conn.close() try: hdulist = fits.open(datafile) except Exception: print( "Couldn't read `{}`, maybe private.".format(filename)) os.remove(datafile) continue for oi, obj in enumerate(hdulist[0].header): if any(x in ['.', '/'] for x in obj): del (hdulist[0].header[oi]) hdulist[0].verify('silentfix') hdrkeys = list(hdulist[0].header.keys()) # print(hdrkeys) name = entry if not name: name = hdulist[0].header['OBJECT'] name, source = catalog.new_entry( name, srcname=mastref, url=masturl, secondary=True) sources = [source] if 'OBSERVER' in hdrkeys: sources.append( catalog.entries[name].add_source( name=hdulist[0].header['OBSERVER'])) if observer: source = catalog.entries[name].add_source(name=observer) sources.append(source) source = ','.join(sources) if summed: wcol = 3 fcol = 4 else: wcol = 2 fcol = 3 try: waves = [str(x) for x in list(hdulist[1].data)[0][wcol]] fluxes = [str(x) for x in list(hdulist[1].data)[0][fcol]] except Exception: print('Failed to find waves/fluxes for `{}`.'.format(filename)) continue if 'BUNIT' in hdrkeys: fluxunit = hdulist[0].header['BUNIT'] if fluxunit in fureps: fluxunit = fureps[fluxunit] else: if max([float(x) for x in fluxes]) < 1.0e-5: fluxunit = 'erg/s/cm^2/Angstrom' else: fluxunit = 'Uncalibrated' specdict = { SPECTRUM.U_WAVELENGTHS: 'Angstrom', SPECTRUM.WAVELENGTHS: waves, SPECTRUM.TIME: mjd, SPECTRUM.U_TIME: 'MJD', SPECTRUM.FLUXES: fluxes, SPECTRUM.U_FLUXES: fluxunit, SPECTRUM.FILENAME: filename, SPECTRUM.SOURCE: source } if 'TELESCOP' in hdrkeys: specdict[SPECTRUM.TELESCOPE] = hdulist[0].header[ 'TELESCOP'] if not instrument and 'INSTRUME' in hdrkeys: instrument = hdulist[0].header['INSTRUME'] if instrument: specdict[SPECTRUM.INSTRUMENT] = instrument if 'SITENAME' in hdrkeys: specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header[ 'SITENAME'] elif 'OBSERVAT' in hdrkeys: specdict[SPECTRUM.OBSERVATORY] = hdulist[0].header[ 'OBSERVAT'] if 'OBSERVER' in hdrkeys: specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER'] catalog.entries[name].add_spectrum(**specdict) if not use_cache and (ci % 100 == 0 or ci == len(coords) - 1): histdict[entry] = [time.time(), spectra] json.dump(histdict, open(histfile, 'w'), indent='\t', separators=(',', ':')) catalog.journal_entries() return
if thistype.upper() in nontypes: continue elif thistype in ('Other', 'not Ia', 'SN', 'unconf', 'Radio', 'CC', 'CCSN', 'Candidate', 'nIa'): evtypes.append('Unknown') break else: evtypes.append(thistype) break else: evtypes.append('Unknown') tprint(thisevent['name']) try: c = coord(ra=thisevent['ra'][0]['value'], dec=thisevent['dec'][0]['value'], unit=(un.hourangle, un.deg)) except (KeyboardInterrupt, SystemExit): raise except: warnings.warn('Mangled coordinate, skipping') continue else: evnames.append(thisevent['name']) rarad = c.ra.radian - pi decrad = c.dec.radian snhx = 2.0**1.5 * cos(decrad) * sin(rarad / 2.0) / \ sqrt(1.0 + cos(decrad) * cos(rarad / 2.0)) snhy = sqrt(2.0) * sin(decrad) / \ sqrt(1.0 + cos(decrad) * cos(rarad / 2.0)) evras.append(c.ra.deg)
thistype = thistype[0] if thistype not in spectral_ordering: continue levtypes.append(thistype) if not len(levtypes): evtype = untype elif moduletype == 'spectraltype': evtype = [x for x in spectral_ordering if x in levtypes][0] else: evtype = levtypes[0] else: evtype = untype try: c = coord(ra=thisevent['ra'][0]['value'], dec=thisevent[ 'dec'][0]['value'], unit=(un.hourangle, un.deg)) except (KeyboardInterrupt, SystemExit): raise except: warnings.warn('Mangled coordinate, skipping {}.'.format(thisevent['name'])) continue else: evtypes.append(evtype) evnames.append(thisevent['name']) rarad = c.ra.radian - pi decrad = c.dec.radian snhx = 2.0**1.5 * cos(decrad) * sin(rarad / 2.0) / \ sqrt(1.0 + cos(decrad) * cos(rarad / 2.0)) snhy = sqrt(2.0) * sin(decrad) / \ sqrt(1.0 + cos(decrad) * cos(rarad / 2.0)) rastr = str(c.ra.deg)
if newsources: cts.append(quantum['value']) ctsources.append({'idtype':','.join([x['idtype'] for x in newsources]), 'id':','.join([x['id'] for x in newsources])}) edit = True if os.path.isfile('../sne-internal/' + get_event_filename(item['name']) + '.json') else False if ras and decs and item['name'] and item['name'] not in ['SN2002fz']: oralen = len(ras) odeclen = len(decs) if len(ras) > len(decs): decs = decs + [decs[0] for x in range(len(ras) - len(decs))] elif len(ras) < len(decs): ras = ras + [ras[0] for x in range(len(decs) - len(ras))] try: coo = coord(ras, decs, unit = (un.hourangle, un.deg)) except: warnings.warn('Mangled coordinate, setting to 0') radegs = [] decdegs = [] else: radegs = coo.ra.deg[:oralen] decdegs = coo.dec.deg[:odeclen] ras = ras[:oralen] decs = decs[:odeclen] if len(ras) != len(radegs): tqdm.write('Mangled R.A. for ' + item['name']) conflicts.append(OrderedDict([('name', item['name']), ('alias', item['alias']), ('edit',edit), ('quantity', 'ra'), ('difference', '?'), ('values', ras), ('sources', rasources)]))
def do_sdss(catalog): task_str = catalog.get_current_task_str() keys = list(catalog.entries.keys()) Mnames, Mradec = [], [] for oname in pbar(keys, task_str): # Some events may be merged in cleanup process, skip them if # non-existent. try: name = catalog.add_entry(oname) except Exception: catalog.log.warning( '"{}" was not found, suggests merge occurred in cleanup ' 'process.'.format(oname)) continue if (FASTSTARS.RA not in catalog.entries[name] or FASTSTARS.DEC not in catalog.entries[name]): continue else: Mnames.append(name) Mradec.append( str(catalog.entries[name][FASTSTARS.RA][0]['value']) + str(catalog.entries[name][FASTSTARS.DEC][0]['value'])) c = coord(Mradec, unit=(un.hourangle, un.deg), frame='icrs') # We must step through the data to query it, >100 is too many maxstep = 100 # stepsize Nentries = len(Mradec) roundindex = np.zeros( 0 ) # the stepping round that this star's data was acquired in, needed to connect the obj_id in the query to the name of the star for i in range(int(Nentries / maxstep) + 1): result_tmp = SDSS.query_crossid( c[maxstep * i:min(maxstep * (i + 1), Nentries)], timeout=200., photoobj_fields=[ 'u', 'err_u', 'g', 'err_g', 'r', 'err_r', 'i', 'err_i', 'z', 'err_z', 'MJD' ]) roundindex = np.concatenate( [roundindex, i * np.ones(len(result_tmp['obj_id']))]) if i == 0: result = result_tmp else: result = vstack([result, result_tmp]) flagsuccess = result['obj_id'] listfilter = ['u', 'g', 'r', 'i', 'z'] for i in range(len(flagsuccess)): Mi = int(flagsuccess[i].strip('obj_')) + maxstep * int(roundindex[i]) name = Mnames[Mi] source = catalog.entries[name].add_source( bibcode='2015ApJS..219...12A') for j in range(5): catalog.entries[name].add_photometry( time=str(result[i]['MJD']), u_time='MJD', telescope='SDSS', band=listfilter[j], magnitude=str(result[i][listfilter[j]]), e_magnitude=str(result[i]['err_' + listfilter[j]]), source=source) catalog.journal_entries() return
def do_gaia(catalog): # Disable warnings warnings.filterwarnings("ignore") catalog.log.warning( 'Some warnings are thrown by the Gaia module which do not affect the result of the Gaia queries.' ) task_str = catalog.get_current_task_str() keys = list(catalog.entries.keys()) cntgphot = 0 cntgast = 0 for oname in pbar(keys, task_str): # Some events may be merged in cleanup process, skip them if # non-existent. try: name = catalog.add_entry(oname) except Exception: catalog.log.warning( '"{}" was not found, suggests merge occurred in cleanup ' 'process.'.format(oname)) continue if (FASTSTARS.RA not in catalog.entries[name] or FASTSTARS.DEC not in catalog.entries[name]): continue else: Mradec = str( catalog.entries[name][FASTSTARS.RA][0]['value']) + str( catalog.entries[name][FASTSTARS.DEC][0]['value']) print(Mradec) c = coord(Mradec, unit=(un.hourangle, un.deg), frame='icrs') result = silentgaiaquery(c, radius=un.Quantity(2.0, un.arcsecond)) if len(result['dist']) > 0: cntgphot += 1 source = catalog.entries[name].add_source( bibcode='2016A&A...595A...2G') catalog.entries[name].add_photometry( time=57023, u_time='MJD', telescope='Gaia', band='G', magnitude=str(result['phot_g_mean_mag'][0]), source=source) if result['parallax'][0] != '--': cntgast += 1 catalog.log.warning( '"{}" has Gaia astrometry.'.format(name)) catalog.entries[name].add_quantity( FASTSTARS.PROPER_MOTION_RA, str(result['pmra'][0]), source, e_value=str(result['pmra_error'][0]), u_value='mas/yr') catalog.entries[name].add_quantity( FASTSTARS.PROPER_MOTION_DEC, str(result['pmdec'][0]), source, e_value=str(result['pmdec_error'][0]), u_value='mas/yr') catalog.entries[name].add_quantity( FASTSTARS.PARALLAX, str(result['parallax'][0]), source, e_value=str(result['parallax_error'][0]), u_value='mas') # Convert parallax to distance if (FASTSTARS.LUM_DIST in catalog.entries[name]): catalog.log.warning( '"{}" has distance from photmetric distance prior.' .format(name)) distance, distance_error = parallax_to_distance( result['parallax'][0], result['parallax_error'][0], float(catalog.entries[name][FASTSTARS.LUM_DIST][0] ['value']), float(catalog.entries[name][FASTSTARS.LUM_DIST][0] ['e_value'])) catalog.entries[name].add_quantity( FASTSTARS.LUM_DIST, str(distance), e_value=str(distance_error), u_value='kpc', source=source, derived=True) else: catalog.log.warning( '"{}" has distance from Astraatmadja & Bailer-Jones (2016) prior.' .format(name)) distance, distance_error = parallax_to_distance( result['parallax'][0], result['parallax_error'][0]) catalog.entries[name].add_quantity( FASTSTARS.LUM_DIST, str(distance), e_value=str(distance_error), u_value='kpc', source=source, derived=True) catalog.log.warning( '"{}" have Gaia photometry and "{}" have Gaia astrometry.'.format( cntgphot, cntgast)) catalog.journal_entries() # Reactivate warnings warnings.filterwarnings("default") return
def do_mast_spectra(catalog): """Import HST spectra from MAST.""" task_str = catalog.get_current_task_str() masturl = 'https://mast.stsci.edu' mastref = 'MAST' fureps = {'erg/cm2/s/A': 'erg/s/cm^2/Angstrom'} histfile = os.path.join(catalog.get_current_task_repo(), 'MAST', 'history.json') if os.path.exists(histfile): histdict = json.load(open(histfile, 'r')) else: histdict = OrderedDict() # objectOfInterest = 'ASASSN-14li' # # resolverRequest = {'service': 'Mast.Name.Lookup', # 'params': {'input': objectOfInterest, # 'format': 'json'}, # } # # headers, resolvedObjectString = mastQuery(resolverRequest) # # resolvedObject = json.loads(resolvedObjectString) # # pp.pprint(resolvedObject) # objRa = resolvedObject['resolvedCoordinate'][0]['ra'] # objDec = resolvedObject['resolvedCoordinate'][0]['decl'] objs = [] for entry in catalog.entries: if SUPERNOVA.DISCOVER_DATE in catalog.entries[entry]: dd = catalog.entries[entry][SUPERNOVA.DISCOVER_DATE][0][ QUANTITY.VALUE] try: dt = datetime.strptime(dd, '%Y/%m/%d') if dt < datetime(1997, 1, 1): continue except Exception: pass if (SUPERNOVA.RA in catalog.entries[entry] and SUPERNOVA.DEC in catalog.entries[entry]): objs.append([ entry, catalog.entries[entry][SUPERNOVA.RA][0][QUANTITY.VALUE], catalog.entries[entry][SUPERNOVA.DEC][0][QUANTITY.VALUE] ]) if not len(objs): return objs = np.array(objs).T coords = coord(objs[1], objs[2], unit=(un.hourangle, un.deg)) for ci, co in enumerate(pbar(coords, desc=task_str)): entry = objs[0][ci] use_cache = False if entry in histdict: if dt < datetime(datetime.now().year - 5, 1, 1): use_cache = True if (dt >= datetime(datetime.now().year - 5, 1, 1) and time.time() - histdict[entry][0] < 30. * 86400.): use_cache = True if use_cache: spectra = histdict[entry][1] else: objRa, objDec = tuple(co.to_string().split()) mastRequest = { 'service': 'Mast.Caom.Cone', 'params': { 'ra': objRa, 'dec': objDec, 'radius': 0.008 }, 'format': 'json', 'pagesize': 1000, 'page': 1, 'removenullcolumns': True, 'removecache': True } try: headers, mastDataString = mastQuery(mastRequest) except Exception: print('`mastQuery` failed for `{}`, request failed, skipping.'. format(entry)) continue try: mastData = json.loads(mastDataString) except Exception: print( '`mastQuery` failed for `{}`, couldn\'t decode JSON, skipping.' .format(entry)) continue # print(mastData.keys()) # print("Query status:", mastData['status']) mastDataTable = Table() if 'fields' not in mastData: print('`fields` not found for `{}`'.format(entry)) continue for col, atype in [(x['name'], x['type']) for x in mastData['fields']]: if atype == "string": atype = "str" if atype == "boolean": atype = "bool" mastDataTable[col] = np.array( [x.get(col, None) for x in mastData['data']], dtype=atype) # print(mastDataTable.columns) spectra = [ x for x in mastDataTable if x['dataproduct_type'] == 'spectrum' and x['obs_collection'] == 'HST' ] spectra = [{ 'target_classification': x['target_classification'], 'obsid': x['obsid'] if 'obsid' in x else None, 't_min': x['t_min'], 't_max': x['t_max'], 'instrument_name': x['instrument_name'], 'proposal_pi': x['proposal_pi'], } for x in spectra] mjd = '' instrument = '' observer = '' for si in range(len(spectra)): spec = spectra[si] if all([ x not in spec['target_classification'].upper() for x in ['SUPERNOVA', 'UNIDENTIFIED'] ]): continue obsid = spec['obsid'] if obsid is None: continue mjd = str( Decimal('0.5') * (Decimal(str(spec['t_min'])) + Decimal(str(spec['t_max'])))) instrument = spec['instrument_name'] if ('STIS' not in instrument.upper() and 'COS' not in instrument.upper()): continue observer = spec['proposal_pi'] if use_cache: scienceProducts = spec.get('sciProds', []) else: productRequest = { 'service': 'Mast.Caom.Products', 'params': { 'obsid': obsid }, 'format': 'json', 'pagesize': 100, 'page': 1 } try: headers, obsProductsString = mastQuery(productRequest) except Exception: print( '`mastQuery` failed for `{}`, skipping.'.format(obsid)) continue obsProducts = json.loads(obsProductsString) if 'fields' not in obsProducts: print('`fields` not found for `{}`'.format(obsid)) print(obsProducts) continue sciProdArr = [ x for x in obsProducts['data'] if x.get("productType", None) == 'SCIENCE' ] scienceProducts = OrderedDict() for col, atype in [(x['name'], x['type']) for x in obsProducts['fields']]: if atype == "string": atype = "str" if atype == "boolean": atype = "bool" if atype == "int": # array may contain nan values, and they do not exist # in numpy integer arrays. atype = "float" scienceProducts[col] = [ x.get(col, None) for x in sciProdArr ] spectra[si]['sciProds'] = scienceProducts # print("Number of science products:", len(scienceProducts)) # print(scienceProducts) search_str = '_x1d.fits' summed = False if any([ '_x1dsum.fits' in x for x in scienceProducts['productFilename'] ]): search_str = '_x1dsum.fits' summed = True if any( ['_sx1.fits' in x for x in scienceProducts['productFilename']]): search_str = '_sx1.fits' summed = True for ri in range(len(scienceProducts['productFilename'])): if search_str not in scienceProducts['productFilename'][ri]: continue filename = str(obsid) + "_" + \ scienceProducts['productFilename'][ri] datafile = os.path.join(catalog.get_current_task_repo(), 'MAST', filename) if not os.path.exists(datafile): # link is url, so can just dl if "http" in scienceProducts['dataURI'][ri]: urlretrieve(scienceProducts['dataURI'][ri], datafile) else: # link is uri, need to go through direct dl request server = 'mast.stsci.edu' conn = httplib.HTTPSConnection(server) conn.request( "GET", "/api/v0/download/file/" + scienceProducts['dataURI'][ri].lstrip('mast:')) resp = conn.getresponse() fileContent = resp.read() with open(datafile, 'wb') as FLE: FLE.write(fileContent) conn.close() try: hdulist = fits.open(datafile) except Exception: print( "Couldn't read `{}`, maybe private.".format(filename)) os.remove(datafile) continue for oi, obj in enumerate(hdulist[0].header): if any(x in ['.', '/'] for x in obj): del (hdulist[0].header[oi]) hdulist[0].verify('silentfix') hdrkeys = list(hdulist[0].header.keys()) # print(hdrkeys) name = entry if not name: name = hdulist[0].header['OBJECT'] name, source = catalog.new_entry(name, srcname=mastref, url=masturl, secondary=True) sources = [source] if 'OBSERVER' in hdrkeys: sources.append(catalog.entries[name].add_source( name=hdulist[0].header['OBSERVER'])) if observer: source = catalog.entries[name].add_source(name=observer) sources.append(source) source = ','.join(sources) if summed: wcol = 3 fcol = 4 else: wcol = 2 fcol = 3 try: waves = [str(x) for x in list(hdulist[1].data)[0][wcol]] fluxes = [str(x) for x in list(hdulist[1].data)[0][fcol]] except Exception: print('Failed to find waves/fluxes for `{}`.'.format( filename)) continue if 'BUNIT' in hdrkeys: fluxunit = hdulist[0].header['BUNIT'] if fluxunit in fureps: fluxunit = fureps[fluxunit] else: if max([float(x) for x in fluxes]) < 1.0e-5: fluxunit = 'erg/s/cm^2/Angstrom' else: fluxunit = 'Uncalibrated' specdict = { SPECTRUM.U_WAVELENGTHS: 'Angstrom', SPECTRUM.WAVELENGTHS: waves, SPECTRUM.TIME: mjd, SPECTRUM.U_TIME: 'MJD', SPECTRUM.FLUXES: fluxes, SPECTRUM.U_FLUXES: fluxunit, SPECTRUM.FILENAME: filename, SPECTRUM.SOURCE: source } if 'TELESCOP' in hdrkeys: specdict[ SPECTRUM.TELESCOPE] = hdulist[0].header['TELESCOP'] if not instrument and 'INSTRUME' in hdrkeys: instrument = hdulist[0].header['INSTRUME'] if instrument: specdict[SPECTRUM.INSTRUMENT] = instrument if 'SITENAME' in hdrkeys: specdict[ SPECTRUM.OBSERVATORY] = hdulist[0].header['SITENAME'] elif 'OBSERVAT' in hdrkeys: specdict[ SPECTRUM.OBSERVATORY] = hdulist[0].header['OBSERVAT'] if 'OBSERVER' in hdrkeys: specdict[SPECTRUM.OBSERVER] = hdulist[0].header['OBSERVER'] catalog.entries[name].add_spectrum(**specdict) if not use_cache and (ci % 100 == 0 or ci == len(coords) - 1): histdict[entry] = [time.time(), spectra] json.dump(histdict, open(histfile, 'w'), indent='\t', separators=(',', ':')) catalog.journal_entries() return
if ('redshift' in item and 'host' not in item and 'ra' in item and 'dec' in item and item['ra'] and item['dec']): # if 'ra' in item and 'dec' in item and item['ra'] and item['dec']: newitem['name'] = item['name'] newitem['alias'] = [x['value'] for x in item['alias']] newitem['ra'] = item['ra'][0]['value'] newitem['dec'] = item['dec'][0]['value'] if 'redshift' in item: newitem['redshift'] = item['redshift'][0]['value'] # Temporary fix for David's typo if newitem['dec'].count('.') == 2: newitem['dec'] = newitem['dec'][:newitem['dec'].rfind('.')] newcatalog.append(newitem) coo = coord([x['ra'] for x in newcatalog], [x['dec'] for x in newcatalog], unit=(un.hourangle, un.deg)) for ci, co in enumerate(tqdm(coo)): customSimbad = Simbad() customSimbad.add_votable_fields('otype', 'z_value') regstr = 'region(ICRS, ' + co.to_string('hmsdms') + ', 1m)' print(regstr) result_table = customSimbad.query_criteria(regstr, otype='Galaxy') if result_table: print(newcatalog[ci]) if 'redshift' in newcatalog[ci]: print(newcatalog[ci]['redshift']) print(result_table) # Convert to array since that's what datatables expects
import gPhoton.gAperture import numpy as np from astropy import units as un from astropy.coordinates import SkyCoord as coord from astropy.time import Time as astrotime with open('all-bright-sne.csv', 'r') as f: rows = list(csv.reader(f, delimiter=',', quotechar='"')) datas = [] for ri, row in enumerate(rows[1:]): ddate = row[1].split(',')[0] ra = row[2].split(',')[0] dec = row[3].split(',')[0] if not ddate or not ra or not dec: continue coo = coord(ra=ra, dec=dec, unit=(un.hourangle, un.deg)) dra, ddec = coo.ra.deg, coo.dec.deg gtime = astrotime(ddate.replace('/', '-')).unix - 315964800. tmin = gtime - 86400. * 365.0 tmax = gtime + 86400. * 365.0 try: data = gPhoton.gAperture(band='NUV', skypos=[dra, ddec], radius=0.03, annulus=[0.03, 0.04], trange=[tmin, tmax], stepsz=600.) except: data = None if data is not None: for key in data: