def fit_data(self,
event_name='',
method=None,
iterations=None,
frack_step=20,
num_walkers=None,
num_temps=1,
burn=None,
post_burn=None,
fracking=True,
gibbs=False,
pool=None,
output_path='',
suffix='',
write=False,
upload=False,
upload_token='',
check_upload_quality=True,
convergence_type=None,
convergence_criteria=None,
save_full_chain=False,
extra_outputs=None):
"""Fit the data for a given event.
Fitting performed using a combination of emcee and fracking.
"""
if self._speak:
speak('Fitting ' + event_name, self._speak)
from mosfit.__init__ import __version__
global model
model = self._model
prt = self._printer
upload_model = upload and iterations > 0
if pool is not None:
self._pool = pool
if upload:
try:
import dropbox
except ImportError:
if self._test:
pass
else:
prt.message('install_db', error=True)
raise
if not self._pool.is_master():
try:
self._pool.wait()
except (KeyboardInterrupt, SystemExit):
pass
return (None, None, None)
self._method = method
if self._method == 'nester':
self._sampler = Nester(self, model, iterations, burn, post_burn,
num_walkers, convergence_criteria,
convergence_type, gibbs, fracking,
frack_step)
else:
self._sampler = Ensembler(self, model, iterations, burn, post_burn,
num_temps, num_walkers,
convergence_criteria, convergence_type,
gibbs, fracking, frack_step)
self._sampler.run(self._walker_data)
prt.message('constructing')
if write:
if self._speak:
speak(prt._strings['saving_output'], self._speak)
if self._event_path:
entry = Entry.init_from_file(catalog=None,
name=self._event_name,
path=self._event_path,
merge=False,
pop_schema=False,
ignore_keys=[ENTRY.MODELS],
compare_to_existing=False)
new_photometry = []
for photo in entry.get(ENTRY.PHOTOMETRY, []):
if PHOTOMETRY.REALIZATION not in photo:
new_photometry.append(photo)
if len(new_photometry):
entry[ENTRY.PHOTOMETRY] = new_photometry
else:
entry = Entry(name=self._event_name)
uentry = Entry(name=self._event_name)
data_keys = set()
for task in model._call_stack:
if model._call_stack[task]['kind'] == 'data':
data_keys.update(
list(model._call_stack[task].get('keys', {}).keys()))
entryhash = entry.get_hash(keys=list(sorted(list(data_keys))))
# Accumulate all the sources and add them to each entry.
sources = []
for root in model._references:
for ref in model._references[root]:
sources.append(entry.add_source(**ref))
sources.append(entry.add_source(**self._DEFAULT_SOURCE))
source = ','.join(sources)
usources = []
for root in model._references:
for ref in model._references[root]:
usources.append(uentry.add_source(**ref))
usources.append(uentry.add_source(**self._DEFAULT_SOURCE))
usource = ','.join(usources)
model_setup = OrderedDict()
for ti, task in enumerate(model._call_stack):
task_copy = deepcopy(model._call_stack[task])
if (task_copy['kind'] == 'parameter'
and task in model._parameter_json):
task_copy.update(model._parameter_json[task])
model_setup[task] = task_copy
modeldict = OrderedDict([(MODEL.NAME, model._model_name),
(MODEL.SETUP, model_setup),
(MODEL.CODE, 'MOSFiT'),
(MODEL.DATE, time.strftime("%Y/%m/%d")),
(MODEL.VERSION, __version__),
(MODEL.SOURCE, source)])
self._sampler.prepare_output(check_upload_quality, upload)
self._sampler.append_output(modeldict)
umodeldict = deepcopy(modeldict)
umodeldict[MODEL.SOURCE] = usource
modelhash = get_model_hash(umodeldict,
ignore_keys=[MODEL.DATE, MODEL.SOURCE])
umodelnum = uentry.add_model(**umodeldict)
if self._sampler._upload_model is not None:
upload_model = self._sampler._upload_model
modelnum = entry.add_model(**modeldict)
samples, probs, weights = self._sampler.get_samples()
extras = OrderedDict()
samples_to_plot = self._sampler._nwalkers
if isinstance(self._sampler, Nester):
icdf = np.cumsum(np.concatenate(([0.0], weights)))
draws = np.random.rand(samples_to_plot)
indices = np.searchsorted(icdf, draws) - 1
else:
indices = list(range(samples_to_plot))
ri = 0
selected_extra = False
for xi, x in enumerate(samples):
ri = ri + 1
prt.message('outputting_walker', [ri, len(samples)],
inline=True,
min_time=0.2)
if xi in indices:
output = model.run_stack(x, root='output')
if extra_outputs is not None:
if not extra_outputs and not selected_extra:
extra_options = list(output.keys())
prt.message('available_keys')
for opt in extra_options:
prt.prt('- {}'.format(opt))
selected_extra = True
for key in extra_outputs:
new_val = output.get(key, [])
new_val = all_to_list(new_val)
extras.setdefault(key, []).append(new_val)
for i in range(len(output['times'])):
if not np.isfinite(output['model_observations'][i]):
continue
photodict = {
PHOTOMETRY.TIME:
output['times'][i] + output['min_times'],
PHOTOMETRY.MODEL: modelnum,
PHOTOMETRY.SOURCE: source,
PHOTOMETRY.REALIZATION: str(ri)
}
if output['observation_types'][i] == 'magnitude':
photodict[PHOTOMETRY.BAND] = output['bands'][i]
photodict[PHOTOMETRY.
MAGNITUDE] = output['model_observations'][i]
photodict[PHOTOMETRY.
E_MAGNITUDE] = output['model_variances'][i]
elif output['observation_types'][i] == 'magcount':
if output['model_observations'][i] == 0.0:
continue
photodict[PHOTOMETRY.BAND] = output['bands'][i]
photodict[PHOTOMETRY.
COUNT_RATE] = output['model_observations'][i]
photodict[PHOTOMETRY.
E_COUNT_RATE] = output['model_variances'][i]
photodict[PHOTOMETRY.MAGNITUDE] = -2.5 * np.log10(
output['model_observations']
[i]) + output['all_zeropoints'][i]
photodict[PHOTOMETRY.E_UPPER_MAGNITUDE] = 2.5 * (
np.log10(output['model_observations'][i] +
output['model_variances'][i]) -
np.log10(output['model_observations'][i]))
if (output['model_variances'][i] >
output['model_observations'][i]):
photodict[PHOTOMETRY.UPPER_LIMIT] = True
else:
photodict[PHOTOMETRY.E_LOWER_MAGNITUDE] = 2.5 * (
np.log10(output['model_observations'][i]) -
np.log10(output['model_observations'][i] -
output['model_variances'][i]))
elif output['observation_types'][i] == 'fluxdensity':
photodict[PHOTOMETRY.FREQUENCY] = output[
'frequencies'][i] * frequency_unit('GHz')
photodict[PHOTOMETRY.FLUX_DENSITY] = output[
'model_observations'][i] * flux_density_unit('µJy')
photodict[PHOTOMETRY.E_LOWER_FLUX_DENSITY] = (
photodict[PHOTOMETRY.FLUX_DENSITY] -
(10.0**
(np.log10(photodict[PHOTOMETRY.FLUX_DENSITY]) -
output['model_variances'][i] / 2.5)) *
flux_density_unit('µJy'))
photodict[PHOTOMETRY.E_UPPER_FLUX_DENSITY] = (
10.0**(np.log10(photodict[PHOTOMETRY.FLUX_DENSITY])
+ output['model_variances'][i] / 2.5) *
flux_density_unit('µJy') -
photodict[PHOTOMETRY.FLUX_DENSITY])
photodict[PHOTOMETRY.U_FREQUENCY] = 'GHz'
photodict[PHOTOMETRY.U_FLUX_DENSITY] = 'µJy'
elif output['observation_types'][i] == 'countrate':
photodict[PHOTOMETRY.
COUNT_RATE] = output['model_observations'][i]
photodict[PHOTOMETRY.E_LOWER_COUNT_RATE] = (
photodict[PHOTOMETRY.COUNT_RATE] -
(10.0**(np.log10(photodict[PHOTOMETRY.COUNT_RATE])
- output['model_variances'][i] / 2.5)))
photodict[PHOTOMETRY.E_UPPER_COUNT_RATE] = (
10.0**(np.log10(photodict[PHOTOMETRY.COUNT_RATE]) +
output['model_variances'][i] / 2.5) -
photodict[PHOTOMETRY.COUNT_RATE])
photodict[PHOTOMETRY.U_COUNT_RATE] = 's^-1'
if ('model_upper_limits' in output
and output['model_upper_limits'][i]):
photodict[PHOTOMETRY.UPPER_LIMIT] = bool(
output['model_upper_limits'][i])
if self._limiting_magnitude is not None:
photodict[PHOTOMETRY.SIMULATED] = True
if 'telescopes' in output and output['telescopes'][i]:
photodict[
PHOTOMETRY.TELESCOPE] = output['telescopes'][i]
if 'systems' in output and output['systems'][i]:
photodict[PHOTOMETRY.SYSTEM] = output['systems'][i]
if 'bandsets' in output and output['bandsets'][i]:
photodict[PHOTOMETRY.BAND_SET] = output['bandsets'][i]
if 'instruments' in output and output['instruments'][i]:
photodict[
PHOTOMETRY.INSTRUMENT] = output['instruments'][i]
if 'modes' in output and output['modes'][i]:
photodict[PHOTOMETRY.MODE] = output['modes'][i]
entry.add_photometry(compare_to_existing=False,
check_for_dupes=False,
**photodict)
uphotodict = deepcopy(photodict)
uphotodict[PHOTOMETRY.SOURCE] = umodelnum
uentry.add_photometry(compare_to_existing=False,
check_for_dupes=False,
**uphotodict)
else:
output = model.run_stack(x, root='objective')
parameters = OrderedDict()
derived_keys = set()
pi = 0
for ti, task in enumerate(model._call_stack):
# if task not in model._free_parameters:
# continue
if model._call_stack[task]['kind'] != 'parameter':
continue
paramdict = OrderedDict(
(('latex', model._modules[task].latex()),
('log', model._modules[task].is_log())))
if task in model._free_parameters:
poutput = model._modules[task].process(
**{'fraction': x[pi]})
value = list(poutput.values())[0]
paramdict['value'] = value
paramdict['fraction'] = x[pi]
pi = pi + 1
else:
if output.get(task, None) is not None:
paramdict['value'] = output[task]
parameters.update({model._modules[task].name(): paramdict})
# Dump out any derived parameter keys
derived_keys.update(model._modules[task].get_derived_keys())
for key in list(sorted(list(derived_keys))):
if (output.get(key, None) is not None
and key not in parameters):
parameters.update({key: {'value': output[key]}})
realdict = {REALIZATION.PARAMETERS: parameters}
if probs is not None:
realdict[REALIZATION.SCORE] = str(probs[xi])
else:
realdict[REALIZATION.SCORE] = str(
ln_likelihood(x) + ln_prior(x))
realdict[REALIZATION.ALIAS] = str(ri)
realdict[REALIZATION.WEIGHT] = str(weights[xi])
entry[ENTRY.MODELS][0].add_realization(check_for_dupes=False,
**realdict)
urealdict = deepcopy(realdict)
uentry[ENTRY.MODELS][0].add_realization(check_for_dupes=False,
**urealdict)
prt.message('all_walkers_written', inline=True)
entry.sanitize()
oentry = {self._event_name: entry._ordered(entry)}
uentry.sanitize()
ouentry = {self._event_name: uentry._ordered(uentry)}
uname = '_'.join([self._event_name, entryhash, modelhash])
if output_path and not os.path.exists(output_path):
os.makedirs(output_path)
if not os.path.exists(model.get_products_path()):
os.makedirs(model.get_products_path())
if write:
prt.message('writing_complete')
with open_atomic(
os.path.join(model.get_products_path(), 'walkers.json'),
'w') as flast, open_atomic(
os.path.join(
model.get_products_path(), self._event_name +
(('_' + suffix) if suffix else '') + '.json'),
'w') as feven:
entabbed_json_dump(oentry, flast, separators=(',', ':'))
entabbed_json_dump(oentry, feven, separators=(',', ':'))
if save_full_chain:
prt.message('writing_full_chain')
with open_atomic(
os.path.join(model.get_products_path(), 'chain.json'),
'w') as flast, open_atomic(
os.path.join(
model.get_products_path(),
self._event_name + '_chain' +
(('_' + suffix) if suffix else '') + '.json'),
'w') as feven:
entabbed_json_dump(self._sampler._all_chain.tolist(),
flast,
separators=(',', ':'))
entabbed_json_dump(self._sampler._all_chain.tolist(),
feven,
separators=(',', ':'))
if extra_outputs is not None:
prt.message('writing_extras')
with open_atomic(
os.path.join(model.get_products_path(), 'extras.json'),
'w') as flast, open_atomic(
os.path.join(
model.get_products_path(),
self._event_name + '_extras' +
(('_' + suffix) if suffix else '') + '.json'),
'w') as feven:
entabbed_json_dump(extras, flast, separators=(',', ':'))
entabbed_json_dump(extras, feven, separators=(',', ':'))
prt.message('writing_model')
with open_atomic(
os.path.join(model.get_products_path(), 'upload.json'),
'w') as flast, open_atomic(
os.path.join(
model.get_products_path(), uname +
(('_' + suffix) if suffix else '') + '.json'),
'w') as feven:
entabbed_json_dump(ouentry, flast, separators=(',', ':'))
entabbed_json_dump(ouentry, feven, separators=(',', ':'))
if upload_model:
prt.message('ul_fit', [entryhash, self._sampler._modelhash])
upayload = entabbed_json_dumps(ouentry, separators=(',', ':'))
try:
dbx = dropbox.Dropbox(upload_token)
dbx.files_upload(upayload.encode(),
'/' + uname + '.json',
mode=dropbox.files.WriteMode.overwrite)
prt.message('ul_complete')
except Exception:
if self._test:
pass
else:
raise
if upload:
for ce in self._converter.get_converted():
dentry = Entry.init_from_file(catalog=None,
name=ce[0],
path=ce[1],
merge=False,
pop_schema=False,
ignore_keys=[ENTRY.MODELS],
compare_to_existing=False)
dentry.sanitize()
odentry = {ce[0]: uentry._ordered(dentry)}
dpayload = entabbed_json_dumps(odentry, separators=(',', ':'))
text = prt.message('ul_devent', [ce[0]], prt=False)
ul_devent = prt.prompt(text, kind='bool', message=False)
if ul_devent:
dpath = '/' + slugify(
ce[0] + '_' + dentry[ENTRY.SOURCES][0].get(
SOURCE.BIBCODE, dentry[ENTRY.SOURCES][0].get(
SOURCE.NAME, 'NOSOURCE'))) + '.json'
try:
dbx = dropbox.Dropbox(upload_token)
dbx.files_upload(
dpayload.encode(),
dpath,
mode=dropbox.files.WriteMode.overwrite)
prt.message('ul_complete')
except Exception:
if self._test:
pass
else:
raise
return (entry, samples, probs)
def handle_tns(event):
"""Add a newly announced TNS event."""
from astrocats.catalog.entry import ENTRY, Entry
import time
import urllib
tns_name = 'Transient Name Server'
tns_url = 'https://wis-tns.weizmann.ac.il/'
# First, create the JSON file.
if event.startswith(('AT', 'SN', 'at', 'sn')):
name = event.upper()
else:
name = 'AT' + event
qname = replace_multiple(name.lower(), ['at', 'sn'])
cat = 'sne'
# Check if already in catalog, if so skip.
if name.lower() in apidata._all_aliases:
return False
new_event = Entry(name=name)
source = new_event.add_source(name=tns_name, url=tns_url)
data = urllib.parse.urlencode({
'api_key':
apidata._tnskey,
'data':
json.dumps({
'objname': qname,
'photometry': '1'
})
}).encode('ascii')
req = urllib.request.Request(
'https://wis-tns.weizmann.ac.il/api/get/object', data=data)
trys = 0
objdict = None
while trys < 3 and not objdict:
try:
objdict = json.loads(
urllib.request.urlopen(
req, timeout=30).read().decode('ascii'))['data']['reply']
except KeyboardInterrupt:
raise
except Exception:
logger.info('API request failed for `{}`.'.format(name))
time.sleep(5)
trys = trys + 1
logger.info(objdict)
if (not objdict or 'objname' not in objdict
or not isinstance(objdict['objname'], str)):
logger.info('Object `{}` not found!'.format(name))
return False
objdict = sortOD(objdict)
if objdict.get('ra'):
new_event.add_quantity(ENTRY.RA, objdict['ra'], source=source)
if objdict.get('dec'):
new_event.add_quantity(ENTRY.DEC, objdict['dec'], source=source)
if objdict.get('redshift'):
new_event.add_quantity(ENTRY.REDSHIFT,
objdict['redshift'],
source=source)
if objdict.get('internal_name'):
new_event.add_quantity(ENTRY.ALIAS,
objdict['internal_name'],
source=source)
new_event.sanitize()
oentry = new_event._ordered(new_event)
outfile = os.path.join(apidata._AC_PATH, apidata._CATS[cat][0], 'output',
apidata._CATS[cat][2], name + '.json')
if not os.path.exists(outfile):
entabbed_json_dump({name: oentry},
open(outfile, 'w'),
separators=(',', ':'))
# Then, load it into the API dicts.
if name not in apidata._catalogs[cat]:
apidata._catalogs[cat][name] = oentry
apidata._extras[cat][name] = oentry
add_event(cat, name)
return True
def generate_event_list(self, event_list):
"""Generate a list of events and/or convert events to JSON format."""
prt = self._printer
cidict = OrderedDict()
intro_shown = False
new_event_list = []
previous_file = None
for event in event_list:
rsource = {SOURCE.NAME: self._DEFAULT_SOURCE}
use_self_source = None
new_events = []
toffset = Decimal('0')
if ('.' in event and os.path.isfile(event) and
not event.endswith('.json')):
if not intro_shown:
prt.message('converter_info')
intro_shown = True
prt.message('converting_to_json', [event])
with open(event, 'r') as f:
ftxt = f.read()
# Try a couple of table formats from astropy.
table = None
try:
table = read(ftxt, Reader=Cds, guess=False)
except Exception:
pass
else:
prt.message('convert_cds')
flines = [table.colnames] + [
list(x) for x in np.array(table).tolist()]
for i in range(len(flines)):
flines[i] = [str(x) for x in flines[i]]
try:
table = read(ftxt, Reader=Latex, guess=False)
except Exception:
pass
else:
prt.message('convert_latex')
flines = [table.colnames] + [
list(x) for x in np.array(table).tolist()]
if table is None:
# Count to try and determine delimiter.
delims = [' ', '\t', ',', ';', '|', '&']
delimnames = [
'Space: ` `', 'Tab: `\t`', 'Comma: `,`',
'Semi-colon: `;`', 'Bar: `|`', 'Ampersand: `&`']
delim = None
delimcounts = [ftxt.count(x) for x in delims]
maxdelimcount = max(delimcounts)
delim = delims[delimcounts.index(maxdelimcount)]
# If two delimiter options are close in count, ask user.
for i, x in enumerate(delimcounts):
if x > 0.5 * maxdelimcount and delims[i] != delim:
delim = None
if delim is None:
odelims = list(np.array(delimnames)[
np.array(delimcounts) > 0])
delim = delims[prt.prompt(
'delim', kind='option', options=odelims) - 1]
ad = list(delims)
ad.remove(delim)
ad = ''.join(ad)
fsplit = ftxt.splitlines()
fsplit = [
x.replace('$', '').replace('\\pm', delim)
.replace('±', delim).replace('(', delim + '(')
.strip(ad + '()# ').replace('′', "'")
for x in fsplit]
flines = []
for fs in fsplit:
flines.append(list(
csv.reader([fs], delimiter=delim))[0])
flines = [[
x.strip(ad + '#$()\\')
for x in y] for y in flines]
# Find band columns if they exist and insert error columns
# if they don't exist.
for fi, fl in enumerate(list(flines)):
flcopy = list(fl)
offset = 0
if not any([is_number(x) for x in fl]):
for fci, fc in enumerate(fl):
if (fc in self._band_names and
(fci == len(fl) - 1 or
fl[fci + 1] not in self._emagstrs)):
flcopy.insert(fci + 1 + offset, 'e mag')
offset += 1
flines[fi] = flcopy
# Find the most frequent column count. These are probably
# the tables we wish to read.
flens = [len(x) for x in flines]
ncols = Counter(flens).most_common(1)[0][0]
newlines = []
potential_name = None
for fi, fl in enumerate(flines):
if (len(fl) and flens[fi] == 1 and
fi < len(flines) - 1 and
flens[fi + 1] == ncols and not len(newlines)):
potential_name = fl[0]
if flens[fi] == ncols:
if potential_name is not None and any(
[is_number(x) for x in fl]):
newlines.append([potential_name] + list(fl))
else:
newlines.append(list(fl))
flines = newlines
for fi, fl in enumerate(flines):
if len(fl) == ncols and potential_name is not None:
if not any([is_number(x) for x in fl]):
flines[fi] = ['name'] + list(fl)
# If none of the rows contain numeric data, the file
# is likely a list of transient names.
if (len(flines) and
(not any(any([is_number(x) or x == '' for x in y])
for y in flines) or
len(flines) == 1)):
new_events = [
it for s in flines for it in s]
# If last row is numeric, then likely this is a file with
# transient data.
elif (len(flines) > 1 and
any([is_number(x) for x in flines[-1]])):
# Check that each row has the same number of columns.
if len(set([len(x) for x in flines])) > 1:
print(set([len(x) for x in flines]))
raise ValueError(
'Number of columns in each row not '
'consistent!')
if len(cidict) and len(new_event_list):
msg = ('is_file_same' if
previous_file else 'is_event_same')
reps = [previous_file] if previous_file else [''.join(
new_event_list[-1].split('.')[:-1])]
text = prt.text(msg, reps)
is_same = prt.prompt(text, message=False,
kind='bool')
if not is_same:
cidict = OrderedDict()
# If the first row has no numbers it is likely a header.
if not len(cidict):
self.assign_columns(cidict, flines)
perms = 1
for key in cidict:
if isinstance(cidict[key], list) and not isinstance(
cidict[key], string_types):
if cidict[key][0] != 'j':
perms = len(cidict[key])
# Get event name (if single event) or list of names from
# table.
event_names = []
if ENTRY.NAME in cidict:
for fi, fl in enumerate(flines):
flines[fi][cidict[ENTRY.NAME]] = name_clean(
fl[cidict[ENTRY.NAME]])
event_names = list(sorted(set([
x[cidict[ENTRY.NAME]] for x in flines[
self._first_data:]])))
new_events = [x + '.json' for x in event_names]
else:
new_event_name = '.'.join(event.split(
'.')[:-1]).split('/')[-1]
text = prt.message(
'is_event_name', [new_event_name], prt=False)
is_name = prt.prompt(text, message=False,
kind='bool', default='y')
if not is_name:
new_event_name = ''
while new_event_name.strip() == '':
new_event_name = prt.prompt(
'enter_name', kind='string')
event_names.append(new_event_name)
new_events = [new_event_name + '.json']
# Create a new event, populate the photometry, and dump
# to a JSON file in the run directory.
entries = OrderedDict([(x, Entry(name=x))
for x in event_names])
# Clean up the data a bit now that we know the column
# identities.
# Strip common prefixes/suffixes from band names
if PHOTOMETRY.BAND in cidict:
bi = cidict[PHOTOMETRY.BAND]
for d in [True, False]:
if not isinstance(bi, (int, np.integer)):
break
strip_cols = []
lens = [len(x[bi])
for x in flines[self._first_data:]]
llen = min(lens)
ra = range(llen) if d else range(-1, -llen - 1, -1)
for li in ra:
letter = None
for row in list(flines[self._first_data:]):
if letter is None:
letter = row[bi][li]
elif row[bi][li] != letter:
letter = None
break
if letter is not None:
strip_cols.append(li)
else:
break
if len(strip_cols) == llen:
break
for ri in range(len(flines[self._first_data:])):
flines[self._first_data + ri][bi] = ''.join(
[c for i, c in enumerate(flines[
self._first_data + ri][bi])
if (i if d else i - len(flines[
self._first_data + ri][bi])) not in
strip_cols])
if (PHOTOMETRY.TIME in cidict and
(not isinstance(cidict[PHOTOMETRY.TIME], list) or
len(cidict[PHOTOMETRY.TIME]) <= 2)):
bi = cidict[PHOTOMETRY.TIME]
if isinstance(bi, list) and not isinstance(
bi, string_types) and isinstance(
bi[0], string_types) and bi[0] == 'jd':
bi = bi[-1]
mmtimes = [float(x[bi])
for x in flines[self._first_data:]]
mintime, maxtime = min(mmtimes), max(mmtimes)
if mintime < 10000:
while True:
try:
response = prt.prompt(
'small_time_offset', kind='string')
if response is not None:
toffset = Decimal(response)
break
except Exception:
pass
elif maxtime > 60000 and cidict[
PHOTOMETRY.TIME][0] != 'jd':
isjd = prt.prompt(
'large_time_offset',
kind='bool', default='y')
if isjd:
toffset = Decimal('-2400000.5')
for row in flines[self._first_data:]:
photodict = {}
rname = (row[cidict[ENTRY.NAME]]
if ENTRY.NAME in cidict else event_names[0])
for pi in range(perms):
sources = set()
for key in cidict:
if key in self._bool_keys:
rval = row[cidict[key]]
if rval in self._FALSE_VALS:
rval = False
elif rval in self._TRUE_VALS:
rval = True
if type(rval) != 'bool':
try:
rval = bool(rval)
except Exception:
pass
if type(rval) != 'bool':
try:
rval = bool(float(rval))
except Exception:
rval = True
if not rval:
continue
row[cidict[key]] = rval
elif key == 'reference':
if (isinstance(cidict[key],
string_types) and
len(cidict[key]) == 19):
new_src = entries[rname].add_source(
bibcode=cidict[key])
sources.update(new_src)
row[
cidict[key]] = new_src
elif key == ENTRY.NAME:
continue
elif (isinstance(key, Key) and
key.type == KEY_TYPES.TIME and
isinstance(cidict[key], list) and not
isinstance(cidict[key],
string_types)):
tval = np.array(row)[np.array(cidict[key][
1:], dtype=int)]
if cidict[key][0] == 'j':
date = '-'.join([x.zfill(2) for x in
tval])
date = self._month_rep.sub(
lambda x: self._MONTH_IDS[
x.group()], date)
photodict[key] = str(
astrotime(date, format='isot').mjd)
elif cidict[key][0] == 'jd':
photodict[key] = str(
jd_to_mjd(Decimal(tval[-1])))
continue
val = cidict[key]
if (isinstance(val, list) and not
isinstance(val, string_types)):
val = val[pi]
if isinstance(val, string_types):
if val != '':
photodict[key] = val
else:
photodict[key] = row[val]
else:
if isinstance(val, string_types):
if val != '':
photodict[key] = val
else:
photodict[key] = row[val]
if self._data_type == 2:
if self._zp:
photodict[PHOTOMETRY.ZERO_POINT] = self._zp
else:
photodict[PHOTOMETRY.ZERO_POINT] = (
row[cidict[PHOTOMETRY.ZERO_POINT][pi]]
if isinstance(cidict[
PHOTOMETRY.ZERO_POINT], list) else
row[cidict[PHOTOMETRY.ZERO_POINT]])
zpp = photodict[PHOTOMETRY.ZERO_POINT]
cc = (
row[cidict[PHOTOMETRY.COUNT_RATE][pi]] if
isinstance(cidict[
PHOTOMETRY.COUNT_RATE], list) else
row[cidict[PHOTOMETRY.COUNT_RATE]])
ecc = (
row[cidict[PHOTOMETRY.E_COUNT_RATE][pi]] if
isinstance(cidict[
PHOTOMETRY.E_COUNT_RATE], list) else
row[cidict[PHOTOMETRY.E_COUNT_RATE]])
if '<' in cc:
set_pd_mag_from_counts(
photodict, ec=cc.strip('<'), zp=zpp)
else:
set_pd_mag_from_counts(
photodict, c=cc, ec=ecc, zp=zpp)
elif self._data_type == 3:
photodict[
PHOTOMETRY.U_FLUX_DENSITY] = self._ufd
if PHOTOMETRY.U_FLUX_DENSITY in cidict:
photodict[PHOTOMETRY.U_FLUX_DENSITY] = (
row[cidict[
PHOTOMETRY.U_FLUX_DENSITY][pi]]
if isinstance(cidict[
PHOTOMETRY.
U_FLUX_DENSITY], list) else
row[cidict[PHOTOMETRY.U_FLUX_DENSITY]])
if photodict[
PHOTOMETRY.U_FLUX_DENSITY] == '':
photodict[
PHOTOMETRY.U_FLUX_DENSITY] = 'µJy'
fd = (
row[cidict[PHOTOMETRY.FLUX_DENSITY][pi]] if
isinstance(cidict[
PHOTOMETRY.FLUX_DENSITY], list) else
row[cidict[PHOTOMETRY.FLUX_DENSITY]])
efd = (
row[cidict[
PHOTOMETRY.E_FLUX_DENSITY][pi]] if
isinstance(cidict[
PHOTOMETRY.E_FLUX_DENSITY], list) else
row[cidict[PHOTOMETRY.E_FLUX_DENSITY]])
mult = Decimal('1')
ufd = photodict[PHOTOMETRY.U_FLUX_DENSITY]
if ufd.lower() in [
'mjy', 'millijy', 'millijansky']:
mult = Decimal('1e3')
elif ufd.lower() in ['jy', 'jansky']:
mult = Decimal('1e6')
if '<' in fd:
set_pd_mag_from_flux_density(
photodict, efd=str(
Decimal(fd.strip('<')) * mult))
else:
set_pd_mag_from_flux_density(
photodict, fd=Decimal(fd) * mult,
efd=Decimal(efd) * mult)
if not len(sources):
if use_self_source is None:
sopts = [
('Bibcode', 'b'), ('Last name', 'l')]
if self._require_source:
sel_str = 'must_select_source'
else:
sel_str = 'select_source'
text = prt.text(sel_str)
skind = prt.prompt(
text, kind='option',
options=sopts, default='b',
none_string=(
None if self._require_source else
'Neither, tag MOSFiT as source'))
if skind == 'b':
rsource = {}
bibcode = ''
while len(bibcode) != 19:
bibcode = prt.prompt(
'bibcode',
kind='string',
allow_blank=False
)
bibcode = bibcode.strip()
if (re.search(
'[0-9]{4}..........[\.0-9]{4}'
'[A-Za-z]', bibcode)
is None):
bibcode = ''
rsource[
SOURCE.BIBCODE] = bibcode
use_self_source = False
elif skind == 'l':
rsource = {}
last_name = prt.prompt(
'last_name', kind='string'
)
rsource[
SOURCE.NAME] = (
last_name.strip().title() +
' et al., in preparation')
use_self_source = False
elif skind == 'n':
use_self_source = True
photodict[
PHOTOMETRY.SOURCE] = entries[
rname].add_source(**rsource)
if any([x in photodict.get(
PHOTOMETRY.MAGNITUDE, '')
for x in ['<', '>']]):
photodict[PHOTOMETRY.UPPER_LIMIT] = True
photodict[
PHOTOMETRY.MAGNITUDE] = photodict[
PHOTOMETRY.MAGNITUDE].strip('<>')
if '<' in photodict.get(PHOTOMETRY.COUNT_RATE, ''):
photodict[PHOTOMETRY.UPPER_LIMIT] = True
photodict[
PHOTOMETRY.COUNT_RATE] = photodict[
PHOTOMETRY.COUNT_RATE].strip('<')
if PHOTOMETRY.E_COUNT_RATE in photodict:
del(photodict[PHOTOMETRY.E_COUNT_RATE])
if '<' in photodict.get(
PHOTOMETRY.FLUX_DENSITY, ''):
photodict[PHOTOMETRY.UPPER_LIMIT] = True
photodict[
PHOTOMETRY.FLUX_DENSITY] = photodict[
PHOTOMETRY.FLUX_DENSITY].strip('<')
if PHOTOMETRY.E_FLUX_DENSITY in photodict:
del(photodict[PHOTOMETRY.E_FLUX_DENSITY])
# Apply offset time if set.
if (PHOTOMETRY.TIME in photodict and
toffset != Decimal('0')):
photodict[PHOTOMETRY.TIME] = str(
Decimal(photodict[PHOTOMETRY.TIME]) +
toffset)
# Skip entries for which key values are not
# expected type.
if not all([
is_number(photodict.get(x, ''))
for x in photodict.keys() if
(PHOTOMETRY.get_key_by_name(x).type ==
KEY_TYPES.NUMERIC)]):
continue
# Skip placeholder values.
if float(photodict.get(
PHOTOMETRY.MAGNITUDE, 0.0)) > 50.0:
continue
# Add system if specified by user.
if (self._system is not None and
PHOTOMETRY.SYSTEM not in photodict):
photodict[PHOTOMETRY.SYSTEM] = self._system
# Remove keys not in the `PHOTOMETRY` class.
for key in list(photodict.keys()):
if key not in PHOTOMETRY.vals():
del(photodict[key])
# Add the photometry.
entries[rname].add_photometry(
**photodict)
merge_with_existing = None
for ei, entry in enumerate(entries):
entries[entry].sanitize()
if os.path.isfile(new_events[ei]):
if merge_with_existing is None:
merge_with_existing = prt.prompt(
'merge_with_existing', default='y')
if merge_with_existing:
existing = Entry.init_from_file(
catalog=None,
name=event_names[ei],
path=new_events[ei],
merge=False,
pop_schema=False,
ignore_keys=[ENTRY.MODELS],
compare_to_existing=False)
Catalog().copy_entry_to_entry(
existing, entries[entry])
oentry = entries[entry]._ordered(entries[entry])
entabbed_json_dump(
{entry: oentry}, open(new_events[ei], 'w'),
separators=(',', ':'))
self._converted.extend([
[event_names[x], new_events[x]]
for x in range(len(event_names))])
new_event_list.extend(new_events)
previous_file = event
else:
new_event_list.append(event)
return new_event_list