def do_simbad_novae(catalog):
task_str = catalog.get_current_task_str()
simbad_mirrors = ['http://simbad.harvard.edu/simbad/sim-script',
'http://simbad.u-strasbg.fr/simbad/sim-script']
customSimbad = Simbad()
customSimbad.ROW_LIMIT = -1
customSimbad.TIMEOUT = 120
for mirror in simbad_mirrors:
customSimbad.SIMBAD_URL = mirror
try:
table = customSimbad.query_criteria('maintype=No* | maintype="No?"')
except:
continue
else:
break
if not table:
catalog.log.warning('SIMBAD unable to load, probably offline.')
for name in pbar(catalog.entries, task_str):
try:
nova_name = "V* " + get_nova_name(name)
aliases = customSimbad.query_objectids(nova_name)
except:
#THROW WARNING HERE
tprint("Could not find " + nova_name)
continue
table = customSimbad.query_object(nova_name)
name = catalog.add_entry(name)
bibcode = table[0]['COO_BIBCODE'].decode()
ra = str(table[0]['RA'])
dec = str(table[0]['DEC'])
source = catalog.entries[name].add_source(name='SIMBAD astronomical database',
bibcode=bibcode,
url="http://simbad.u-strasbg.fr/",
secondary=True)
catalog.entries[name].add_quantity(NOVA.RA, ra, source)
catalog.entries[name].add_quantity(NOVA.DEC, dec, source)
for i in range(len(aliases)):
try: alias = aliases[i][0].decode()
except: alias = str(aliases[i][0])
catalog.entries[name].add_quantity(NOVA.ALIAS, alias, source)
catalog.journal_entries()
def do_test(catalog):
log = catalog.log
log.info("do_test()")
task_str = catalog.get_current_task_str()
log.info("`task_str`: '{}'".format(task_str))
if len(catalog.entries) != 0:
raise RuntimeError("Run test only with empty catalog.")
# Test URL retrieve functions
# ---------------------------
log.info("`args.archived` = '{}', `current_task.archived` = '{}'".format(
catalog.args.archived, catalog.current_task.archived))
test_load_url(catalog)
# Test repo path functions
# ------------------------
paths = catalog.PATHS.get_all_repo_folders()
for path in tq(paths, currenttask='Test tq progress bar.'):
tprint('Test tprint.')
log.debug(path)
paths = catalog.PATHS.get_repo_input_folders()
for path in pbar_strings(paths, desc='Test pbar_strings progress bar.'):
log.debug(path)
boneyard = catalog.PATHS.get_repo_boneyard()
log.debug(boneyard)
# Create a Fake Entry, with some Fake Data
# ----------------------------------------
_first_event_first_source(catalog)
log_str = "ADDING SECOND SOURCE"
log.info("\n\n{}\n{}\n{}\n\n".format("=" * 100, log_str, "=" * 100))
# Add new Data, from different source, to same fake entry
# -------------------------------------------------------
_first_event_second_source(catalog)
# Make sure output file for this test exists
outdir, filename = catalog.entries[FAKE_ALIAS_1]._get_save_path()
save_name = os.path.join(outdir, filename + '.json')
if not os.path.exists(save_name):
raise RuntimeError("File not found in '{}'".format(save_name))
# Delete created test file
catalog._delete_entry_file(entry_name=FAKE_ALIAS_1)
# Make sure it was deleted
if os.path.exists(save_name):
raise RuntimeError("File not deleted at '{}'".format(save_name))
# Delete entry in catalog
del catalog.entries[FAKE_ALIAS_1]
# Make sure entry was deleted
if len(catalog.entries) != 0:
raise RuntimeError("Error deleting test entry!")
# Add entry back catalog to test later tasks
_first_event_first_source(catalog)
_first_event_second_source(catalog)
# Test some utility functions
log.debug("Preferred name for 2nd source: " +
catalog.get_preferred_name(FAKE_ALIAS_2))
log.debug("Entry exists? " +
str(catalog.entry_exists(FAKE_ALIAS_2)))
log.debug("Entry text: " + catalog.entries[FAKE_ALIAS_1].get_entry_text(
os.path.join(outdir, filename + '.json')))
# Third source is a duplicate that will be merged
_first_event_third_source(catalog)
# Add second event to perform different tests
_second_event(catalog)
# Delete name to test name re-addition in sanitize
for ii, alias in enumerate(
catalog.entries[FAKE_ALIAS_5][ENTRY.ALIAS].copy()):
if alias[QUANTITY.VALUE] == FAKE_ALIAS_5:
del catalog.entries[FAKE_ALIAS_1][ENTRY.ALIAS][ii]
break
return
break
if not foundtype:
continue
maxdate = astrotime(thisevent['maxdate'][0][
'value'].replace('/', '-')).mjd
discoverdate = astrotime(thisevent['discoverdate'][0][
'value'].replace('/', '-')).mjd
if maxdate == discoverdate:
continue
distmod = float(thisevent['maxappmag'][0]['value']) - \
float(thisevent['maxabsmag'][0]['value'])
tprint(thisevent['name'])
prange = list(range(len([x for x in thisevent['photometry'] if photo_cut(
x)]))) if 'photometry' in thisevent else []
if len(prange) <= 3:
continue
phototime += [float(x['time'][:-1] + str(0 * randint(0, 9)) if x['time'][-1] != '.' else x['time'] + '.' + str(0 * randint(0, 9))) - maxdate
for x in thisevent['photometry'] if photo_cut(x)]
phototimelowererrs += [float(x['e_lower_time']) if ('e_lower_time' in x and 'e_upper_time' in x)
else (float(x['e_time']) if 'e_time' in x else 0.) for x in thisevent['photometry'] if photo_cut(x)]
phototimeuppererrs += [float(x['e_upper_time']) if ('e_lower_time' in x and 'e_upper_time' in x) in x
else (float(x['e_time']) if 'e_time' in x else 0.) for x in thisevent['photometry'] if photo_cut(x)]
photoAB += [float(x['magnitude'] + str(0 * randint(0, 9)) if '.' in x['magnitude'] else x['magnitude'] + '.' +
str(0 * randint(0, 9))) - distmod for x in thisevent['photometry'] if photo_cut(x)]
if 'claimedtype' in thisevent and thisevent['claimedtype']:
for ct in [x['value'] for x in thisevent['claimedtype']]:
thistype = ct.replace('?', '').replace('*', '')
if thistype.upper() in nontypes:
continue
elif thistype in ('Other', 'not Ia', 'SN', 'unconf', 'Radio',
'CC', 'CCSN', 'Candidate', 'nIa'):
evtypes.append('Unknown')
break
else:
evtypes.append(thistype)
break
else:
evtypes.append('Unknown')
tprint(thisevent['name'])
try:
c = coord(ra=thisevent['ra'][0]['value'],
dec=thisevent['dec'][0]['value'],
unit=(un.hourangle, un.deg))
except (KeyboardInterrupt, SystemExit):
raise
except:
warnings.warn('Mangled coordinate, skipping')
continue
else:
evnames.append(thisevent['name'])
rarad = c.ra.radian - pi
decrad = c.dec.radian
snhx = 2.0**1.5 * cos(decrad) * sin(rarad / 2.0) / \
sqrt(1.0 + cos(decrad) * cos(rarad / 2.0))
def do_test(catalog):
log = catalog.log
log.info("do_test()")
task_str = catalog.get_current_task_str()
log.info("`task_str`: '{}'".format(task_str))
if len(catalog.entries) != 0:
raise RuntimeError("Run test only with empty catalog.")
# Test URL retrieve functions
# ---------------------------
log.info("`args.archived` = '{}', `current_task.archived` = '{}'".format(
catalog.args.archived, catalog.current_task.archived))
test_load_url(catalog)
# Test repo path functions
# ------------------------
paths = catalog.PATHS.get_all_repo_folders()
for path in tq(paths, currenttask='Test tq progress bar.'):
tprint('Test tprint.')
log.debug(path)
paths = catalog.PATHS.get_repo_input_folders()
for path in pbar_strings(paths, desc='Test pbar_strings progress bar.'):
log.debug(path)
boneyard = catalog.PATHS.get_repo_boneyard()
log.debug(boneyard)
# Create a Fake Entry, with some Fake Data
# ----------------------------------------
_first_event_first_source(catalog)
log_str = "ADDING SECOND SOURCE"
log.info("\n\n{}\n{}\n{}\n\n".format("=" * 100, log_str, "=" * 100))
# Add new Data, from different source, to same fake entry
# -------------------------------------------------------
_first_event_second_source(catalog)
# Make sure output file for this test exists
outdir, filename = catalog.entries[FAKE_ALIAS_1]._get_save_path()
save_name = os.path.join(outdir, filename + '.json')
if not os.path.exists(save_name):
raise RuntimeError("File not found in '{}'".format(save_name))
# Delete created test file
catalog._delete_entry_file(entry_name=FAKE_ALIAS_1)
# Make sure it was deleted
if os.path.exists(save_name):
raise RuntimeError("File not deleted at '{}'".format(save_name))
# Delete entry in catalog
del catalog.entries[FAKE_ALIAS_1]
# Make sure entry was deleted
if len(catalog.entries) != 0:
raise RuntimeError("Error deleting test entry!")
# Add entry back catalog to test later tasks
_first_event_first_source(catalog)
_first_event_second_source(catalog)
# Test some utility functions
log.debug("Preferred name for 2nd source: " +
catalog.get_preferred_name(FAKE_ALIAS_2))
log.debug("Entry exists? " + str(catalog.entry_exists(FAKE_ALIAS_2)))
log.debug("Entry text: " + catalog.entries[FAKE_ALIAS_1].get_entry_text(
os.path.join(outdir, filename + '.json')))
# Third source is a duplicate that will be merged
_first_event_third_source(catalog)
# Add second event to perform different tests
_second_event(catalog)
# Delete name to test name re-addition in sanitize
for ii, alias in enumerate(
catalog.entries[FAKE_ALIAS_5][ENTRY.ALIAS].copy()):
if alias[QUANTITY.VALUE] == FAKE_ALIAS_5:
del catalog.entries[FAKE_ALIAS_1][ENTRY.ALIAS][ii]
break
return
def do_cleanup(catalog):
"""Cleanup catalog after importing all data."""
task_str = catalog.get_current_task_str()
# Set preferred names, calculate some columns based on imported data,
# sanitize some fields
keys = list(catalog.entries.keys())
cleanupcnt = 0
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
try:
name = catalog.add_entry(oname)
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
# Set the preferred name, switching to that name if name changed.
name = catalog.entries[name].set_preferred_name()
aliases = catalog.entries[name].get_aliases()
catalog.entries[name].purge_bandless_photometry()
catalog.entries[name].set_first_max_light()
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['MLS', 'SSS', 'CSS', 'GRB ']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:2])):
discoverdate = ('/'.join([
'20' + alias.replace(prefix, '')[:2],
alias.replace(prefix, '')[2:4],
alias.replace(prefix, '')[4:6]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = [
'ASASSN-', 'PS1-', 'PS1', 'PS', 'iPTF', 'PTF', 'SCP-', 'SNLS-',
'SPIRITS', 'LSQ', 'DES', 'SNHiTS', 'Gaia', 'GND', 'GNW', 'GSD',
'GSW', 'EGS', 'COS', 'OGLE', 'HST'
]
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:2])
and is_number(alias.replace(prefix, '')[:1])):
discoverdate = '20' + alias.replace(prefix, '')[:2]
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['SNF']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:4])):
discoverdate = ('/'.join([
alias.replace(prefix, '')[:4],
alias.replace(prefix, '')[4:6],
alias.replace(prefix, '')[6:8]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['PTFS', 'SNSDF']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:2])):
discoverdate = ('/'.join([
'20' + alias.replace(prefix, '')[:2],
alias.replace(prefix, '')[2:4]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if SUPERNOVA.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['AT', 'SN', 'OGLE-', 'SM ', 'KSN']
for alias in aliases:
for prefix in prefixes:
if alias.startswith(prefix):
year = re.findall(r'\d+', alias)
if len(year) == 1:
year = year[0]
else:
continue
if alias.replace(prefix, '').index(year) != 0:
continue
if (year and is_number(year) and '.' not in year
and len(year) <= 4):
discoverdate = year
if catalog.args.verbose:
tprint('Added discoverdate from name [' +
alias + ']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
SUPERNOVA.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if SUPERNOVA.DISCOVER_DATE in catalog.entries[name]:
break
if (SUPERNOVA.RA not in catalog.entries[name]
or SUPERNOVA.DEC not in catalog.entries[name]):
prefixes = [
'PSN J', 'MASJ', 'CSS', 'SSS', 'MASTER OT J', 'HST J', 'TCP J',
'MACS J', '2MASS J', 'EQ J', 'CRTS J', 'SMT J'
]
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:6])):
noprefix = alias.split(':')[-1].replace(prefix,
'').replace(
'.', '')
decsign = '+' if '+' in noprefix else '-'
noprefix = noprefix.replace('+', '|').replace('-', '|')
nops = noprefix.split('|')
if len(nops) < 2:
continue
rastr = nops[0]
decstr = nops[1]
ra = ':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + \
('.' + rastr[6:] if len(rastr) > 6 else '')
dec = (
decsign +
':'.join([decstr[:2], decstr[2:4], decstr[4:6]]) +
('.' + decstr[6:] if len(decstr) > 6 else ''))
if catalog.args.verbose:
tprint('Added ra/dec from name: ' + ra + ' ' + dec)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(SUPERNOVA.RA,
ra,
source,
derived=True)
catalog.entries[name].add_quantity(SUPERNOVA.DEC,
dec,
source,
derived=True)
break
if SUPERNOVA.RA in catalog.entries[name]:
break
no_host = (SUPERNOVA.HOST not in catalog.entries[name] or not any([
x[QUANTITY.VALUE] == 'Milky Way'
for x in catalog.entries[name][SUPERNOVA.HOST]
]))
if (SUPERNOVA.RA in catalog.entries[name]
and SUPERNOVA.DEC in catalog.entries[name] and no_host):
from astroquery.irsa_dust import IrsaDust
if name not in catalog.extinctions_dict:
try:
ra_dec = catalog.entries[name][
SUPERNOVA.RA][0][QUANTITY.VALUE] + \
" " + \
catalog.entries[name][SUPERNOVA.DEC][0][QUANTITY.VALUE]
result = IrsaDust.get_query_table(ra_dec, section='ebv')
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
warnings.warn("Coordinate lookup for " + name +
" failed in IRSA.")
else:
ebv = result['ext SandF mean'][0]
ebverr = result['ext SandF std'][0]
catalog.extinctions_dict[name] = [ebv, ebverr]
if name in catalog.extinctions_dict:
sources = uniq_cdl([
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2011ApJ...737..103S')
])
(catalog.entries[name].add_quantity(
SUPERNOVA.EBV,
str(catalog.extinctions_dict[name][0]),
sources,
e_value=str(catalog.extinctions_dict[name][1]),
derived=True))
if ((SUPERNOVA.HOST in catalog.entries[name]
and (SUPERNOVA.HOST_RA not in catalog.entries[name]
or SUPERNOVA.HOST_DEC not in catalog.entries[name]))):
for host in catalog.entries[name][SUPERNOVA.HOST]:
alias = host[QUANTITY.VALUE]
if ' J' in alias and is_number(alias.split(' J')[-1][:6]):
noprefix = alias.split(' J')[-1].split(':')[-1].replace(
'.', '')
decsign = '+' if '+' in noprefix else '-'
noprefix = noprefix.replace('+', '|').replace('-', '|')
nops = noprefix.split('|')
if len(nops) < 2:
continue
rastr = nops[0]
decstr = nops[1]
hostra = (':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) +
('.' + rastr[6:] if len(rastr) > 6 else ''))
hostdec = decsign + ':'.join([
decstr[:2], decstr[2:4], decstr[4:6]
]) + ('.' + decstr[6:] if len(decstr) > 6 else '')
if catalog.args.verbose:
tprint('Added hostra/hostdec from name: ' + hostra +
' ' + hostdec)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(SUPERNOVA.HOST_RA,
hostra,
source,
derived=True)
catalog.entries[name].add_quantity(SUPERNOVA.HOST_DEC,
hostdec,
source,
derived=True)
break
if SUPERNOVA.HOST_RA in catalog.entries[name]:
break
if (SUPERNOVA.REDSHIFT not in catalog.entries[name]
and SUPERNOVA.VELOCITY in catalog.entries[name]):
# Find the "best" velocity to use for this
bestsig = 0
for hv in catalog.entries[name][SUPERNOVA.VELOCITY]:
sig = get_sig_digits(hv[QUANTITY.VALUE])
if sig > bestsig:
besthv = hv[QUANTITY.VALUE]
bestsrc = hv['source']
bestsig = sig
if bestsig > 0 and is_number(besthv):
voc = float(besthv) * 1.e5 / CLIGHT
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
(catalog.entries[name].add_quantity(
SUPERNOVA.REDSHIFT,
pretty_num(sqrt((1. + voc) / (1. - voc)) - 1.,
sig=bestsig),
sources,
kind='heliocentric',
derived=True))
if (SUPERNOVA.REDSHIFT not in catalog.entries[name]
and len(catalog.nedd_dict) > 0
and SUPERNOVA.HOST in catalog.entries[name]):
reference = "NED-D"
refurl = "http://ned.ipac.caltech.edu/Library/Distances/"
refbib = "1991ASSL..171...89H"
for host in catalog.entries[name][SUPERNOVA.HOST]:
if host[QUANTITY.VALUE] in catalog.nedd_dict:
source = catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
secondarysource = catalog.entries[name].add_source(
name=reference,
url=refurl,
bibcode=refbib,
secondary=True)
meddist = statistics.median(
catalog.nedd_dict[host[QUANTITY.VALUE]])
redz = z_at_value(cosmo.comoving_distance,
float(meddist) * un.Mpc)
redshift = pretty_num(redz,
sig=get_sig_digits(str(meddist)))
catalog.entries[name].add_quantity(
[SUPERNOVA.REDSHIFT, SUPERNOVA.HOST_REDSHIFT],
redshift,
uniq_cdl([source, secondarysource]),
kind='host',
derived=True)
if (SUPERNOVA.MAX_ABS_MAG not in catalog.entries[name]
and SUPERNOVA.MAX_APP_MAG in catalog.entries[name]
and SUPERNOVA.LUM_DIST in catalog.entries[name]):
# Find the "best" distance to use for this
bestsig = 0
for ld in catalog.entries[name][SUPERNOVA.LUM_DIST]:
sig = get_sig_digits(ld[QUANTITY.VALUE])
if sig > bestsig:
bestld = ld[QUANTITY.VALUE]
bestsrc = ld[QUANTITY.SOURCE]
bestsig = sig
if bestsig > 0 and is_number(bestld) and float(bestld) > 0.:
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
bestldz = z_at_value(cosmo.luminosity_distance,
float(bestld) * un.Mpc)
pnum = (float(catalog.entries[name][SUPERNOVA.MAX_APP_MAG][0][
QUANTITY.VALUE]) - 5.0 *
(log10(float(bestld) * 1.0e6) - 1.0) +
2.5 * log10(1.0 + bestldz))
pnum = pretty_num(pnum, sig=bestsig + 1)
catalog.entries[name].add_quantity(SUPERNOVA.MAX_ABS_MAG,
pnum,
sources,
derived=True)
if (SUPERNOVA.MAX_VISUAL_ABS_MAG not in catalog.entries[name]
and SUPERNOVA.MAX_VISUAL_APP_MAG in catalog.entries[name]
and SUPERNOVA.LUM_DIST in catalog.entries[name]):
# Find the "best" distance to use for this
bestsig = 0
for ld in catalog.entries[name][SUPERNOVA.LUM_DIST]:
sig = get_sig_digits(ld[QUANTITY.VALUE])
if sig > bestsig:
bestld = ld[QUANTITY.VALUE]
bestsrc = ld[QUANTITY.SOURCE]
bestsig = sig
if bestsig > 0 and is_number(bestld) and float(bestld) > 0.:
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
# FIX: what's happening here?!
pnum = (float(catalog.entries[name][
SUPERNOVA.MAX_VISUAL_APP_MAG][0][QUANTITY.VALUE]) - 5.0 *
(log10(float(bestld) * 1.0e6) - 1.0))
pnum = pretty_num(pnum, sig=bestsig + 1)
catalog.entries[name].add_quantity(
SUPERNOVA.MAX_VISUAL_ABS_MAG, pnum, sources, derived=True)
if SUPERNOVA.REDSHIFT in catalog.entries[name]:
# Find the "best" redshift to use for this
bestz, bestkind, bestsig, bestsrc = catalog.entries[
name].get_best_redshift()
if bestsig > 0:
try:
bestz = float(bestz)
except Exception:
print(catalog.entries[name])
raise
if SUPERNOVA.VELOCITY not in catalog.entries[name]:
source = catalog.entries[name].add_self_source()
# FIX: what's happening here?!
pnum = CLIGHT / KM * \
((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.)
pnum = pretty_num(pnum, sig=bestsig)
catalog.entries[name].add_quantity(
SUPERNOVA.VELOCITY,
pnum,
source,
kind=(SUPERNOVA.VELOCITY.kind_preference[bestkind]
if bestkind else ''))
if bestz > 0.:
if SUPERNOVA.LUM_DIST not in catalog.entries[name]:
dl = cosmo.luminosity_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
SUPERNOVA.LUM_DIST,
pretty_num(dl.value, sig=bestsig + 1),
sources,
kind=(SUPERNOVA.LUM_DIST.kind_preference[bestkind]
if bestkind else ''),
derived=True)
if (SUPERNOVA.MAX_ABS_MAG not in catalog.entries[name]
and SUPERNOVA.MAX_APP_MAG
in catalog.entries[name]):
source = catalog.entries[name].add_self_source()
pnum = pretty_num(
float(catalog.entries[name][
SUPERNOVA.MAX_APP_MAG][0][QUANTITY.VALUE])
- 5.0 * (log10(dl.to('pc').value) - 1.0) +
2.5 * log10(1.0 + bestz),
sig=bestsig + 1)
catalog.entries[name].add_quantity(
SUPERNOVA.MAX_ABS_MAG,
pnum,
sources,
derived=True)
if (SUPERNOVA.MAX_VISUAL_ABS_MAG
not in catalog.entries[name]
and SUPERNOVA.MAX_VISUAL_APP_MAG
in catalog.entries[name]):
source = catalog.entries[name].add_self_source()
pnum = pretty_num(float(catalog.entries[name][
SUPERNOVA.MAX_VISUAL_APP_MAG][0][
QUANTITY.VALUE]) - 5.0 *
(log10(dl.to('pc').value) - 1.0),
sig=bestsig + 1)
catalog.entries[name].add_quantity(
SUPERNOVA.MAX_VISUAL_ABS_MAG,
pnum,
sources,
derived=True)
if SUPERNOVA.COMOVING_DIST not in catalog.entries[name]:
cd = cosmo.comoving_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
SUPERNOVA.COMOVING_DIST,
pretty_num(cd.value, sig=bestsig),
sources,
derived=True)
if SUPERNOVA.HOST_REDSHIFT in catalog.entries[name]:
# Find the "best" redshift to use for this
bestz, bestkind, bestsig, bestsrc = catalog.entries[
name].get_best_redshift(SUPERNOVA.HOST_REDSHIFT)
if bestsig > 0:
try:
bestz = float(bestz)
except Exception:
print(catalog.entries[name])
raise
if SUPERNOVA.HOST_VELOCITY not in catalog.entries[name]:
source = catalog.entries[name].add_self_source()
# FIX: what's happening here?!
pnum = CLIGHT / KM * \
((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.)
pnum = pretty_num(pnum, sig=bestsig)
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_VELOCITY,
pnum,
source,
kind=(SUPERNOVA.HOST_VELOCITY.kind_preference[bestkind]
if bestkind else ''))
if bestz > 0.:
if SUPERNOVA.HOST_LUM_DIST not in catalog.entries[name]:
dl = cosmo.luminosity_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_LUM_DIST,
pretty_num(dl.value, sig=bestsig + 1),
sources,
kind=(SUPERNOVA.HOST_LUM_DIST.
kind_preference[bestkind]
if bestkind else ''),
derived=True)
if SUPERNOVA.HOST_COMOVING_DIST not in catalog.entries[
name]:
cd = cosmo.comoving_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_COMOVING_DIST,
pretty_num(cd.value, sig=bestsig),
sources,
derived=True)
if all([
x in catalog.entries[name] for x in [
SUPERNOVA.RA, SUPERNOVA.DEC, SUPERNOVA.HOST_RA,
SUPERNOVA.HOST_DEC
]
]):
# For now just using first coordinates that appear in entry
try:
c1 = coord(
ra=catalog.entries[name][SUPERNOVA.RA][0][QUANTITY.VALUE],
dec=catalog.entries[name][SUPERNOVA.DEC][0][
QUANTITY.VALUE],
unit=(un.hourangle, un.deg))
c2 = coord(ra=catalog.entries[name][SUPERNOVA.HOST_RA][0][
QUANTITY.VALUE],
dec=catalog.entries[name][SUPERNOVA.HOST_DEC][0][
QUANTITY.VALUE],
unit=(un.hourangle, un.deg))
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
pass
else:
sources = uniq_cdl([catalog.entries[name].add_self_source()] +
catalog.entries[name][SUPERNOVA.RA][0][
QUANTITY.SOURCE].split(',') +
catalog.entries[name][SUPERNOVA.DEC][0][
QUANTITY.SOURCE].split(',') +
catalog.entries[name][SUPERNOVA.HOST_RA][0][
QUANTITY.SOURCE].split(',') +
catalog.entries[name][SUPERNOVA.HOST_DEC][0]
[QUANTITY.SOURCE].split(','))
if SUPERNOVA.HOST_OFFSET_ANG not in catalog.entries[name]:
hosa = Decimal(c1.separation(c2).arcsecond)
hosa = pretty_num(hosa)
catalog.entries[name].add_quantity(
SUPERNOVA.HOST_OFFSET_ANG,
hosa,
sources,
derived=True,
u_value='arcseconds')
if (SUPERNOVA.COMOVING_DIST in catalog.entries[name]
and SUPERNOVA.REDSHIFT in catalog.entries[name]
and SUPERNOVA.HOST_OFFSET_DIST
not in catalog.entries[name]):
offsetsig = get_sig_digits(catalog.entries[name][
SUPERNOVA.HOST_OFFSET_ANG][0][QUANTITY.VALUE])
sources = uniq_cdl(
sources.split(',') +
(catalog.entries[name][SUPERNOVA.COMOVING_DIST][0][
QUANTITY.SOURCE]).split(',') +
(catalog.entries[name][SUPERNOVA.REDSHIFT][0][
QUANTITY.SOURCE]).split(','))
(catalog.entries[name].add_quantity(
SUPERNOVA.HOST_OFFSET_DIST,
pretty_num(
float(catalog.entries[name][
SUPERNOVA.HOST_OFFSET_ANG][0][QUANTITY.VALUE])
/ 3600. * (pi / 180.) *
float(catalog.entries[name][
SUPERNOVA.COMOVING_DIST][0][QUANTITY.VALUE]) *
1000. / (1.0 + float(catalog.entries[name][
SUPERNOVA.REDSHIFT][0][QUANTITY.VALUE])),
sig=offsetsig), sources))
catalog.entries[name].sanitize()
catalog.journal_entries(bury=True, final=True, gz=True)
cleanupcnt = cleanupcnt + 1
if catalog.args.travis and cleanupcnt % 1000 == 0:
break
catalog.save_caches()
return
def do_cleanup(catalog):
"""Task to cleanup catalog before final write."""
task_str = catalog.get_current_task_str()
# Set preferred names, calculate some columns based on imported data,
# sanitize some fields
keys = catalog.entries.copy().keys()
cleanupcnt = 0
for oname in pbar(keys, task_str):
name = catalog.add_entry(oname)
# Set the preferred name, switching to that name if name changed.
name = catalog.entries[name].set_preferred_name()
aliases = catalog.entries[name].get_aliases()
catalog.entries[name].set_first_max_light()
if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['MLS', 'SSS', 'CSS', 'GRB ']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix) and
is_number(alias.replace(prefix, '')[:2])):
discoverdate = ('/'.join([
'20' + alias.replace(prefix, '')[:2],
alias.replace(prefix, '')[2:4],
alias.replace(prefix, '')[4:6]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]:
break
if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]:
prefixes = [
'ASASSN-', 'PS1-', 'PS1', 'PS', 'iPTF', 'PTF', 'SCP-', 'SNLS-',
'SPIRITS', 'LSQ', 'DES', 'SNHiTS', 'Gaia', 'GND', 'GNW', 'GSD',
'GSW', 'EGS', 'COS', 'OGLE', 'HST'
]
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix) and
is_number(alias.replace(prefix, '')[:2]) and
is_number(alias.replace(prefix, '')[:1])):
discoverdate = '20' + alias.replace(prefix, '')[:2]
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]:
break
if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['SNF']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix) and
is_number(alias.replace(prefix, '')[:4])):
discoverdate = ('/'.join([
alias.replace(prefix, '')[:4],
alias.replace(prefix, '')[4:6],
alias.replace(prefix, '')[6:8]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]:
break
if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['PTFS', 'SNSDF']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix) and
is_number(alias.replace(prefix, '')[:2])):
discoverdate = ('/'.join([
'20' + alias.replace(prefix, '')[:2],
alias.replace(prefix, '')[2:4]
]))
if catalog.args.verbose:
tprint('Added discoverdate from name [' + alias +
']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]:
break
if TIDALDISRUPTION.DISCOVER_DATE not in catalog.entries[name]:
prefixes = ['AT', 'SN', 'OGLE-', 'SM ', 'KSN-']
for alias in aliases:
for prefix in prefixes:
if alias.startswith(prefix):
year = re.findall(r'\d+', alias)
if len(year) == 1:
year = year[0]
else:
continue
if alias.replace(prefix, '').index(year) != 0:
continue
if (year and is_number(year) and '.' not in year and
len(year) <= 4):
discoverdate = year
if catalog.args.verbose:
tprint('Added discoverdate from name [' +
alias + ']: ' + discoverdate)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DISCOVER_DATE,
discoverdate,
source,
derived=True)
break
if TIDALDISRUPTION.DISCOVER_DATE in catalog.entries[name]:
break
if (TIDALDISRUPTION.RA not in catalog.entries[name] or
TIDALDISRUPTION.DEC not in catalog.entries[name]):
prefixes = [
'PSN J', 'MASJ', 'CSS', 'SSS', 'MASTER OT J', 'HST J', 'TCP J',
'MACS J', '2MASS J', 'EQ J', 'CRTS J', 'SMT J'
]
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix) and
is_number(alias.replace(prefix, '')[:6])):
noprefix = alias.split(':')[-1].replace(
prefix, '').replace('.', '')
decsign = '+' if '+' in noprefix else '-'
noprefix = noprefix.replace('+', '|').replace('-', '|')
nops = noprefix.split('|')
if len(nops) < 2:
continue
rastr = nops[0]
decstr = nops[1]
ra = ':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + \
('.' + rastr[6:] if len(rastr) > 6 else '')
dec = (decsign + ':'.join(
[decstr[:2], decstr[2:4], decstr[4:6]]) +
('.' + decstr[6:] if len(decstr) > 6 else ''))
if catalog.args.verbose:
tprint('Added ra/dec from name: ' + ra + ' ' + dec)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.RA, ra, source, derived=True)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.DEC, dec, source, derived=True)
break
if TIDALDISRUPTION.RA in catalog.entries[name]:
break
no_host = (TIDALDISRUPTION.HOST not in catalog.entries[name] or
not any([
x[QUANTITY.VALUE] == 'Milky Way'
for x in catalog.entries[name][TIDALDISRUPTION.HOST]
]))
if (TIDALDISRUPTION.RA in catalog.entries[name] and
TIDALDISRUPTION.DEC in catalog.entries[name] and no_host):
from astroquery.irsa_dust import IrsaDust
if name not in catalog.extinctions_dict:
try:
ra_dec = (catalog.entries[name][TIDALDISRUPTION.RA][0][
QUANTITY.VALUE] + " " + catalog.entries[name][
TIDALDISRUPTION.DEC][0][QUANTITY.VALUE])
result = IrsaDust.get_query_table(ra_dec, section='ebv')
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
warnings.warn("Coordinate lookup for " + name +
" failed in IRSA.")
else:
ebv = result['ext SandF mean'][0]
ebverr = result['ext SandF std'][0]
catalog.extinctions_dict[name] = [ebv, ebverr]
if name in catalog.extinctions_dict:
sources = uniq_cdl([
catalog.entries[name].add_self_source(),
catalog.entries[name]
.add_source(bibcode='2011ApJ...737..103S')
])
(catalog.entries[name].add_quantity(
TIDALDISRUPTION.EBV,
str(catalog.extinctions_dict[name][0]),
sources,
e_value=str(catalog.extinctions_dict[name][1]),
derived=True))
if ((TIDALDISRUPTION.HOST in catalog.entries[name] and
(TIDALDISRUPTION.HOST_RA not in catalog.entries[name] or
TIDALDISRUPTION.HOST_DEC not in catalog.entries[name]))):
for host in catalog.entries[name][TIDALDISRUPTION.HOST]:
alias = host[QUANTITY.VALUE]
if ' J' in alias and is_number(alias.split(' J')[-1][:6]):
noprefix = alias.split(' J')[-1].split(':')[-1].replace(
'.', '')
decsign = '+' if '+' in noprefix else '-'
noprefix = noprefix.replace('+', '|').replace('-', '|')
nops = noprefix.split('|')
if len(nops) < 2:
continue
rastr = nops[0]
decstr = nops[1]
hostra = (':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) +
('.' + rastr[6:] if len(rastr) > 6 else ''))
hostdec = decsign + ':'.join([
decstr[:2], decstr[2:4], decstr[4:6]
]) + ('.' + decstr[6:] if len(decstr) > 6 else '')
if catalog.args.verbose:
tprint('Added hostra/hostdec from name: ' + hostra +
' ' + hostdec)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(
TIDALDISRUPTION.HOST_RA, hostra, source, derived=True)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.HOST_DEC,
hostdec,
source,
derived=True)
break
if TIDALDISRUPTION.HOST_RA in catalog.entries[name]:
break
if (TIDALDISRUPTION.REDSHIFT not in catalog.entries[name] and
TIDALDISRUPTION.VELOCITY in catalog.entries[name]):
# Find the "best" velocity to use for this
bestsig = 0
for hv in catalog.entries[name][TIDALDISRUPTION.VELOCITY]:
sig = get_sig_digits(hv[QUANTITY.VALUE])
if sig > bestsig:
besthv = hv[QUANTITY.VALUE]
bestsrc = hv['source']
bestsig = sig
if bestsig > 0 and is_number(besthv):
voc = float(besthv) * 1.e5 / CLIGHT
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
(catalog.entries[name].add_quantity(
TIDALDISRUPTION.REDSHIFT,
pretty_num(
sqrt((1. + voc) / (1. - voc)) - 1., sig=bestsig),
sources,
kind='heliocentric',
derived=True))
if (TIDALDISRUPTION.REDSHIFT not in catalog.entries[name] and
len(catalog.nedd_dict) > 0 and
TIDALDISRUPTION.HOST in catalog.entries[name]):
reference = "NED-D"
refurl = "http://ned.ipac.caltech.edu/Library/Distances/"
for host in catalog.entries[name][TIDALDISRUPTION.HOST]:
if host[QUANTITY.VALUE] in catalog.nedd_dict:
source = catalog.entries[name].add_source(
bibcode='2016A&A...594A..13P')
secondarysource = catalog.entries[name].add_source(
name=reference, url=refurl, secondary=True)
meddist = statistics.median(catalog.nedd_dict[host[
QUANTITY.VALUE]])
redz = z_at_value(cosmo.comoving_distance,
float(meddist) * un.Mpc)
redshift = pretty_num(
redz, sig=get_sig_digits(str(meddist)))
catalog.entries[name].add_quantity(
TIDALDISRUPTION.REDSHIFT,
redshift,
uniq_cdl([source, secondarysource]),
kind='host',
derived=True)
if (TIDALDISRUPTION.MAX_ABS_MAG not in catalog.entries[name] and
TIDALDISRUPTION.MAX_APP_MAG in catalog.entries[name] and
TIDALDISRUPTION.LUM_DIST in catalog.entries[name]):
# Find the "best" distance to use for this
bestsig = 0
for ld in catalog.entries[name][TIDALDISRUPTION.LUM_DIST]:
sig = get_sig_digits(ld[QUANTITY.VALUE])
if sig > bestsig:
bestld = ld[QUANTITY.VALUE]
bestsrc = ld['source']
bestsig = sig
if bestsig > 0 and is_number(bestld) and float(bestld) > 0.:
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
bestldz = z_at_value(cosmo.luminosity_distance,
float(bestld) * un.Mpc)
pnum = (float(catalog.entries[name][
TIDALDISRUPTION.MAX_APP_MAG][0][QUANTITY.VALUE]) - 5.0 *
(log10(float(bestld) * 1.0e6) - 1.0
) + 2.5 * log10(1.0 + bestldz))
pnum = pretty_num(pnum, sig=bestsig)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.MAX_ABS_MAG, pnum, sources, derived=True)
if TIDALDISRUPTION.REDSHIFT in catalog.entries[name]:
# Find the "best" redshift to use for this
bestz, bestkind, bestsig, bestsrc = catalog.entries[
name].get_best_redshift()
if bestsig > 0:
try:
bestz = float(bestz)
except Exception:
print(catalog.entries[name])
raise
if TIDALDISRUPTION.VELOCITY not in catalog.entries[name]:
source = catalog.entries[name].add_self_source()
# FIX: what's happening here?!
pnum = CLIGHT / KM * \
((bestz + 1.)**2. - 1.) / ((bestz + 1.)**2. + 1.)
pnum = pretty_num(pnum, sig=bestsig)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.VELOCITY,
pnum,
source,
kind=PREF_KINDS[bestkind],
derived=True)
if bestz > 0.:
from astropy.cosmology import Planck15 as cosmo
if TIDALDISRUPTION.LUM_DIST not in catalog.entries[name]:
dl = cosmo.luminosity_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name]
.add_source(bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
TIDALDISRUPTION.LUM_DIST,
pretty_num(
dl.value, sig=bestsig),
sources,
kind=PREF_KINDS[bestkind],
derived=True)
if (TIDALDISRUPTION.MAX_ABS_MAG not in
catalog.entries[name] and
TIDALDISRUPTION.MAX_APP_MAG in
catalog.entries[name]):
source = catalog.entries[name].add_self_source()
pnum = pretty_num(
float(catalog.entries[name][
TIDALDISRUPTION.MAX_APP_MAG][0][
QUANTITY.VALUE]) - 5.0 *
(log10(dl.to('pc').value) - 1.0
) + 2.5 * log10(1.0 + bestz),
sig=bestsig + 1)
catalog.entries[name].add_quantity(
TIDALDISRUPTION.MAX_ABS_MAG,
pnum,
sources,
derived=True)
if TIDALDISRUPTION.COMOVING_DIST not in catalog.entries[
name]:
cd = cosmo.comoving_distance(bestz)
sources = [
catalog.entries[name].add_self_source(),
catalog.entries[name]
.add_source(bibcode='2016A&A...594A..13P')
]
sources = uniq_cdl(sources + bestsrc.split(','))
catalog.entries[name].add_quantity(
TIDALDISRUPTION.COMOVING_DIST,
pretty_num(
cd.value, sig=bestsig),
sources,
derived=True)
if all([
x in catalog.entries[name]
for x in [
TIDALDISRUPTION.RA, TIDALDISRUPTION.DEC,
TIDALDISRUPTION.HOST_RA, TIDALDISRUPTION.HOST_DEC
]
]):
# For now just using first coordinates that appear in entry
try:
c1 = coord(
ra=catalog.entries[name][TIDALDISRUPTION.RA][0][
QUANTITY.VALUE],
dec=catalog.entries[name][TIDALDISRUPTION.DEC][0][
QUANTITY.VALUE],
unit=(un.hourangle, un.deg))
c2 = coord(
ra=catalog.entries[name][TIDALDISRUPTION.HOST_RA][0][
QUANTITY.VALUE],
dec=catalog.entries[name][TIDALDISRUPTION.HOST_DEC][0][
QUANTITY.VALUE],
unit=(un.hourangle, un.deg))
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
pass
else:
sources = uniq_cdl(
[catalog.entries[name].add_self_source()] + catalog.
entries[name][TIDALDISRUPTION.RA][0]['source'].split(',') +
catalog.entries[name][TIDALDISRUPTION.DEC][0]['source'].
split(',') + catalog.entries[name][TIDALDISRUPTION.HOST_RA]
[0]['source'].split(',') + catalog.entries[name][
TIDALDISRUPTION.HOST_DEC][0]['source'].split(','))
if 'hostoffsetang' not in catalog.entries[name]:
hosa = Decimal(
hypot(c1.ra.degree - c2.ra.degree, c1.dec.degree -
c2.dec.degree))
hosa = pretty_num(hosa * Decimal(3600.))
catalog.entries[name].add_quantity(
TIDALDISRUPTION.HOST_OFFSET_ANG,
hosa,
sources,
derived=True,
u_value='arcseconds')
if (TIDALDISRUPTION.COMOVING_DIST in catalog.entries[name] and
TIDALDISRUPTION.REDSHIFT in catalog.entries[name] and
TIDALDISRUPTION.HOST_OFFSET_DIST not in
catalog.entries[name]):
offsetsig = get_sig_digits(catalog.entries[name][
TIDALDISRUPTION.HOST_OFFSET_ANG][0][QUANTITY.VALUE])
sources = uniq_cdl(
sources.split(',') + (catalog.entries[name][
TIDALDISRUPTION.COMOVING_DIST][0]['source']).
split(',') + (catalog.entries[name][
TIDALDISRUPTION.REDSHIFT][0]['source']).split(','))
(catalog.entries[name].add_quantity(
TIDALDISRUPTION.HOST_OFFSET_DIST,
pretty_num(
float(catalog.entries[name][
TIDALDISRUPTION.HOST_OFFSET_ANG][0][
QUANTITY.VALUE]) / 3600. * (pi / 180.) *
float(catalog.entries[name][
TIDALDISRUPTION.COMOVING_DIST][0][
QUANTITY.VALUE]) * 1000. /
(1.0 + float(catalog.entries[name][
TIDALDISRUPTION.REDSHIFT][0][QUANTITY.VALUE])),
sig=offsetsig),
sources))
catalog.entries[name].sanitize()
catalog.journal_entries(bury=True, final=True, gz=True)
cleanupcnt = cleanupcnt + 1
if catalog.args.travis and cleanupcnt % 1000 == 0:
break
catalog.save_caches()
return
terms = []
if search_spectra:
terms += specterms
if search_photometry:
terms += photterms
qstr += '"' + '" or "'.join(terms) + '")'
allpapers = ads.SearchQuery(
q=qstr, fl=['id', 'bibcode', 'author'], max_pages=100)
except:
continue
try:
npapers = 0
for paper in allpapers:
if paper.bibstem in ['ATel', 'CBET', 'IAUC']:
tprint('ignored bibstem')
continue
bc = paper.bibcode
if bc not in sentinel:
allauthors = paper.author
sentinel[bc] = OrderedDict([('bibcode', bc), (
'allauthors', allauthors), ('events', [])])
sentinel[bc]['events'].append(fileeventname)
npapers += 1
if not npapers:
continue
if int(rate_limits['remaining']) <= 10:
print('ADS API limit reached, terminating early.')
break
rate_limits = allpapers.response.get_ratelimits()
if search_spectra:
terms += specterms
if search_photometry:
terms += photterms
qstr += '"' + '" or "'.join(terms) + '")'
allpapers = ads.SearchQuery(
q=qstr, fl=['id', 'bibstem', 'bibcode', 'author'], max_pages=100)
except:
print('ADS query failed for {}, skipping'.format(fileeventname))
continue
try:
npapers = 0
for paper in allpapers:
if paper.bibstem in ['ATel', 'CBET', 'IAUC']:
tprint('ignored bibstem')
continue
bc = paper.bibcode
if bc not in sentinel:
allauthors = paper.author
sentinel[bc] = OrderedDict([('bibcode', bc), (
'allauthors', allauthors), ('events', [])])
sentinel[bc]['events'].append(fileeventname)
npapers += 1
if not npapers:
continue
if int(rate_limits['remaining']) <= 10:
print('ADS API limit reached, terminating early.')
break
rate_limits = allpapers.response.get_ratelimits()
# if fcnt > 1000:
# break
fileeventname = os.path.splitext(os.path.basename(eventfile))[
0].replace('.json', '')
if eventfile.split('.')[-1] == 'gz':
with gzip.open(eventfile, 'rt') as f:
filetext = f.read()
else:
with open(eventfile, 'r') as f:
filetext = f.read()
item = json.loads(filetext, object_pairs_hook=OrderedDict)
namekey = list(item.keys())[0]
item = item[namekey]
tprint(namekey)
if 'claimedtype' in item and 'maxdate' in item:
maxyear = int(round(float(item['maxdate'][0]['value'].split('/')[0])))
for yi, year in enumerate(years):
if maxyear <= years[yi]:
for ct in item['claimedtype']:
if ct['value'] == "Ic":
ics[yi] += 1
elif ct['value'] == "Ib":
ibs[yi] += 1
break
print(list(years))
print(ics)
print(ibs)
def do_cleanup(catalog):
"""Cleanup catalog after importing all data."""
task_str = catalog.get_current_task_str()
# Set preferred names, calculate some columns based on imported data,
# sanitize some fields
keys = list(catalog.entries.keys())
cleanupcnt = 0
for oname in pbar(keys, task_str):
# Some events may be merged in cleanup process, skip them if
# non-existent.
try:
name = catalog.add_entry(oname)
except Exception:
catalog.log.warning(
'"{}" was not found, suggests merge occurred in cleanup '
'process.'.format(oname))
continue
# Set the preferred name, switching to that name if name changed.
name = catalog.entries[name].set_preferred_name()
aliases = catalog.entries[name].get_aliases()
catalog.entries[name].set_first_max_light()
# Clean discoverer field
if FASTSTARS.DISCOVERER in catalog.entries[name]:
if len(catalog.entries[name][FASTSTARS.DISCOVERER]) > 1:
POSSIBLEDISCOVERER = [
catalog.entries[name][FASTSTARS.DISCOVERER][i]['value']
for i in range(
len(catalog.entries[name][FASTSTARS.DISCOVERER]))
]
POSSIBLEDISCOVERER_DATE = [
int(DATE['value'])
for DATE in catalog.entries[name][FASTSTARS.DISCOVER_DATE]
]
POSSIBLEDISCOVERER_DATE_SOURCES = [
DATE['source']
for DATE in catalog.entries[name][FASTSTARS.DISCOVER_DATE]
]
EARLIESTSOURCE = POSSIBLEDISCOVERER_DATE_SOURCES[np.argmin(
POSSIBLEDISCOVERER_DATE)]
EARLIESTDISCOVER_DATE = catalog.entries[name][
FASTSTARS.DISCOVER_DATE][np.argmin(
POSSIBLEDISCOVERER_DATE)]
# Deal with case where a star was 'discovered' multiple times in one year
if ',' in EARLIESTSOURCE:
EARLIESTSOURCE = EARLIESTSOURCE.split(',')[0]
for DISCOVERER in catalog.entries[name][FASTSTARS.DISCOVERER]:
for DISCOVERERSOURCE in DISCOVERER['source'].split(','):
if DISCOVERERSOURCE == EARLIESTSOURCE:
EARLIESTDISCOVERER = DISCOVERER
for DISCOVERER in catalog.entries[name][FASTSTARS.DISCOVERER]:
for DISCOVERERSOURCE in DISCOVERER['source'].split(','):
if DISCOVERERSOURCE == EARLIESTSOURCE:
EARLIESTDISCOVERER = DISCOVERER
catalog.entries[name][FASTSTARS.DISCOVERER] = [
EARLIESTDISCOVERER
]
catalog.entries[name][FASTSTARS.DISCOVER_DATE] = [
EARLIESTDISCOVER_DATE
]
# Convert all distances to kpc.
if FASTSTARS.LUM_DIST in catalog.entries[name]:
for li, ld in enumerate(catalog.entries[name][FASTSTARS.LUM_DIST]):
if ld.get('u_value') != 'kpc':
if ld.get('u_value') == 'pc':
catalog.entries[name][
FASTSTARS.LUM_DIST][li]['value'] = str(
Decimal(catalog.entries[name][
FASTSTARS.LUM_DIST][li]['value']) *
Decimal('0.001'))
elif ld.get('u_value') == 'Mpc':
catalog.entries[name][
FASTSTARS.LUM_DIST][li]['value'] = str(
Decimal(catalog.entries[name][
FASTSTARS.LUM_DIST][li]['value']) *
Decimal('1000'))
else:
raise ValueError('unknown distance unit')
catalog.entries[name][
FASTSTARS.LUM_DIST][li]['u_value'] = 'kpc'
if (FASTSTARS.RA not in catalog.entries[name]
or FASTSTARS.DEC not in catalog.entries[name]):
prefixes = ['SDSS']
for alias in aliases:
for prefix in prefixes:
if (alias.startswith(prefix)
and is_number(alias.replace(prefix, '')[:6])):
noprefix = alias.split(':')[-1].replace(prefix,
'').replace(
'.', '')
decsign = '+' if '+' in noprefix else '-'
noprefix = noprefix.replace('+', '|').replace('-', '|')
nops = noprefix.split('|')
if len(nops) < 2:
continue
rastr = nops[0]
decstr = nops[1]
ra = ':'.join([rastr[:2], rastr[2:4], rastr[4:6]]) + \
('.' + rastr[6:] if len(rastr) > 6 else '')
dec = (
decsign +
':'.join([decstr[:2], decstr[2:4], decstr[4:6]]) +
('.' + decstr[6:] if len(decstr) > 6 else ''))
if catalog.args.verbose:
tprint('Added ra/dec from name: ' + ra + ' ' + dec)
source = catalog.entries[name].add_self_source()
catalog.entries[name].add_quantity(FASTSTARS.RA,
ra,
source,
derived=True)
catalog.entries[name].add_quantity(FASTSTARS.DEC,
dec,
source,
derived=True)
break
if FASTSTARS.RA in catalog.entries[name]:
break
if (FASTSTARS.MAX_ABS_MAG not in catalog.entries[name]
and FASTSTARS.MAX_APP_MAG in catalog.entries[name]
and FASTSTARS.LUM_DIST in catalog.entries[name]):
# Find the "best" distance to use for this
bestsig = 0
for ld in catalog.entries[name][FASTSTARS.LUM_DIST]:
sig = get_sig_digits(ld[QUANTITY.VALUE])
if sig > bestsig:
bestld = ld[QUANTITY.VALUE]
bestsrc = ld[QUANTITY.SOURCE]
bestsig = sig
if bestsig > 0 and is_number(bestld) and float(bestld) > 0.:
source = catalog.entries[name].add_self_source()
sources = uniq_cdl([source] + bestsrc.split(','))
bestldz = z_at_value(cosmo.luminosity_distance,
float(bestld) * un.Mpc)
pnum = (float(catalog.entries[name][FASTSTARS.MAX_APP_MAG][0][
QUANTITY.VALUE]) - 5.0 *
(log10(float(bestld) * 1.0e6) - 1.0) +
2.5 * log10(1.0 + bestldz))
pnum = pretty_num(pnum, sig=bestsig + 1)
catalog.entries[name].add_quantity(FASTSTARS.MAX_ABS_MAG,
pnum,
sources,
derived=True)
catalog.entries[name].sanitize()
catalog.journal_entries(bury=True, final=True, gz=True)
cleanupcnt = cleanupcnt + 1
if catalog.args.travis and cleanupcnt % 1000 == 0:
break
catalog.save_caches()
return
# if fcnt > 1000:
# break
fileeventname = os.path.splitext(os.path.basename(eventfile))[0].replace(
'.json', '')
if eventfile.split('.')[-1] == 'gz':
with gzip.open(eventfile, 'rt') as f:
filetext = f.read()
else:
with open(eventfile, 'r') as f:
filetext = f.read()
item = json.loads(filetext, object_pairs_hook=OrderedDict)
namekey = list(item.keys())[0]
item = item[namekey]
tprint(namekey)
if 'claimedtype' in item and 'maxdate' in item:
maxyear = int(round(float(item['maxdate'][0]['value'].split('/')[0])))
for yi, year in enumerate(years):
if maxyear <= years[yi]:
for ct in item['claimedtype']:
if ct['value'] == "Ic":
ics[yi] += 1
elif ct['value'] == "Ib":
ibs[yi] += 1
break
print(list(years))
print(ics)
print(ibs)