def __init__(self, **kwargs):
"""Initialize module."""
super(Parameter, self).__init__(**kwargs)
self._fixed = kwargs.get('fixed', False)
self._fixed_by_user = False
self._max_value = kwargs.get('max_value', None)
self._min_value = kwargs.get('min_value', None)
if (self._min_value is not None and self._max_value is not None
and self._min_value == self._max_value):
self._printer.message('min_max_same', [self._name], warning=True)
self._value = self._min_value
self._min_value, self._max_value = None, None
self._fixed = True
self._fixed_by_user = True
if self._min_value is None or self._max_value is None:
self._fixed = True
self._fixed_by_user = True
self._value = kwargs.get('value', None)
self._log = kwargs.get('log', False)
self._latex = kwargs.get('latex', self._name)
self._derived_keys = listify(kwargs.get(
'derived_keys', [])) + ['reference_' + self._name]
if (self._log and self._min_value is not None
and self._max_value is not None):
if self._min_value <= 0.0 or self._max_value <= 0.0:
raise ValueError(
'Parameter with log prior cannot have range values <= 0!')
self._min_value = np.log(self._min_value)
self._max_value = np.log(self._max_value)
self._reference_value = None
self._clipped_warning = False
def construct_trees(self,
d,
trees,
simple,
kinds=[],
name='',
roots=[],
depth=0):
"""Construct call trees for each root."""
leaf = kinds if len(kinds) else name
if depth > 100:
raise RuntimeError(
'Error: Tree depth greater than 100, suggests a recursive '
'input loop in `{}`.'.format(leaf))
for tag in d:
entry = deepcopy(d[tag])
new_roots = list(roots)
if entry['kind'] in kinds or tag == name:
entry['depth'] = depth
if entry['kind'] in kinds:
new_roots.append(entry['kind'])
entry['roots'] = list(sorted(set(new_roots)))
trees[tag] = entry
simple[tag] = OrderedDict()
inputs = listify(entry.get('inputs', []))
for inps in inputs:
conditional = False
if isinstance(inps, list) and not isinstance(
inps, string_types) and inps[-1] == "conditional":
inp = inps[0]
conditional = True
else:
inp = inps
if inp not in d:
suggests = get_close_matches(inp, d, n=1, cutoff=0.8)
warn_str = ('Module `{}` for input to `{}` '
'not found!'.format(inp, leaf))
if len(suggests):
warn_str += (' Did you perhaps mean `{}`?'.format(
suggests[0]))
raise RuntimeError(warn_str)
# Conditional inputs don't propagate down the tree.
if conditional:
continue
children = OrderedDict()
simple_children = OrderedDict()
self.construct_trees(d,
children,
simple_children,
name=inp,
roots=new_roots,
depth=depth + 1)
trees[tag].setdefault('children', OrderedDict())
trees[tag]['children'].update(children)
simple[tag].update(simple_children)
def fit_events(self,
events=[],
models=[],
max_time='',
time_list=[],
time_unit=None,
band_list=[],
band_systems=[],
band_instruments=[],
band_bandsets=[],
band_sampling_points=17,
iterations=10000,
num_walkers=None,
num_temps=1,
parameter_paths=['parameters.json'],
fracking=True,
frack_step=50,
burn=None,
post_burn=None,
gibbs=False,
smooth_times=-1,
extrapolate_time=0.0,
limit_fitting_mjds=False,
exclude_bands=[],
exclude_instruments=[],
exclude_systems=[],
exclude_sources=[],
exclude_kinds=[],
output_path='',
suffix='',
upload=False,
write=False,
upload_token='',
check_upload_quality=False,
variance_for_each=[],
user_fixed_parameters=[],
user_released_parameters=[],
convergence_type=None,
convergence_criteria=None,
save_full_chain=False,
draw_above_likelihood=False,
maximum_walltime=False,
start_time=False,
print_trees=False,
maximum_memory=np.inf,
speak=False,
return_fits=True,
extra_outputs=None,
walker_paths=[],
catalogs=[],
exit_on_prompt=False,
download_recommended_data=False,
local_data_only=False,
method=None,
seed=None,
**kwargs):
"""Fit a list of events with a list of models."""
global model
if start_time is False:
start_time = time.time()
self._seed = seed
if seed is not None:
np.random.seed(seed)
self._start_time = start_time
self._maximum_walltime = maximum_walltime
self._maximum_memory = maximum_memory
self._debug = False
self._speak = speak
self._download_recommended_data = download_recommended_data
self._local_data_only = local_data_only
self._draw_above_likelihood = draw_above_likelihood
prt = self._printer
event_list = listify(events)
model_list = listify(models)
if len(model_list) and not len(event_list):
event_list = ['']
# Exclude catalogs not included in catalog list.
self._fetcher.add_excluded_catalogs(catalogs)
if not len(event_list) and not len(model_list):
prt.message('no_events_models', warning=True)
# If the input is not a JSON file, assume it is either a list of
# transients or that it is the data from a single transient in tabular
# form. Try to guess the format first, and if that fails ask the user.
self._converter = Converter(prt, require_source=upload)
event_list = self._converter.generate_event_list(event_list)
event_list = [x.replace('‑', '-') for x in event_list]
entries = [[] for x in range(len(event_list))]
ps = [[] for x in range(len(event_list))]
lnprobs = [[] for x in range(len(event_list))]
# Load walker data if provided a list of walker paths.
walker_data = []
if len(walker_paths):
try:
pool = MPIPool()
except (ImportError, ValueError):
pool = SerialPool()
if pool.is_master():
prt.message('walker_file')
wfi = 0
for walker_path in walker_paths:
if os.path.exists(walker_path):
prt.prt(' {}'.format(walker_path))
with codecs.open(walker_path, 'r',
encoding='utf-8') as f:
all_walker_data = json.load(
f, object_pairs_hook=OrderedDict)
# Support both the format where all data stored in a
# single-item dictionary (the OAC format) and the older
# MOSFiT format where the data was stored in the
# top-level dictionary.
if ENTRY.NAME not in all_walker_data:
all_walker_data = all_walker_data[list(
all_walker_data.keys())[0]]
models = all_walker_data.get(ENTRY.MODELS, [])
choice = None
if len(models) > 1:
model_opts = [
'{}-{}-{}'.format(x['code'], x['name'],
x['date']) for x in models
]
choice = prt.prompt('select_model_walkers',
kind='select',
message=True,
options=model_opts)
choice = model_opts.index(choice)
elif len(models) == 1:
choice = 0
if choice is not None:
walker_data.extend([[
wfi, x[REALIZATION.PARAMETERS],
x.get(REALIZATION.WEIGHT)
] for x in models[choice][MODEL.REALIZATIONS]])
for i in range(len(walker_data)):
if walker_data[i][2] is not None:
walker_data[i][2] = float(walker_data[i][2])
if not len(walker_data):
prt.message('no_walker_data')
else:
prt.message('no_walker_data')
if self._offline:
prt.message('omit_offline')
raise RuntimeError
wfi = wfi + 1
for rank in range(1, pool.size + 1):
pool.comm.send(walker_data, dest=rank, tag=3)
else:
walker_data = pool.comm.recv(source=0, tag=3)
pool.wait()
if pool.is_master():
pool.close()
self._event_name = 'Batch'
self._event_path = ''
self._event_data = {}
try:
pool = MPIPool()
except (ImportError, ValueError):
pool = SerialPool()
if pool.is_master():
fetched_events = self._fetcher.fetch(
event_list,
offline=self._offline,
prefer_cache=self._prefer_cache)
for rank in range(1, pool.size + 1):
pool.comm.send(fetched_events, dest=rank, tag=0)
pool.close()
else:
fetched_events = pool.comm.recv(source=0, tag=0)
pool.wait()
for ei, event in enumerate(fetched_events):
if event is not None:
self._event_name = event.get('name', 'Batch')
self._event_path = event.get('path', '')
if not self._event_path:
continue
self._event_data = self._fetcher.load_data(event)
if not self._event_data:
continue
if model_list:
lmodel_list = model_list
else:
lmodel_list = ['']
entries[ei] = [None for y in range(len(lmodel_list))]
ps[ei] = [None for y in range(len(lmodel_list))]
lnprobs[ei] = [None for y in range(len(lmodel_list))]
if (event is not None and
(not self._event_data or ENTRY.PHOTOMETRY
not in self._event_data[list(self._event_data.keys())[0]])):
prt.message('no_photometry', [self._event_name])
continue
for mi, mod_name in enumerate(lmodel_list):
for parameter_path in parameter_paths:
try:
pool = MPIPool()
except (ImportError, ValueError):
pool = SerialPool()
self._model = Model(model=mod_name,
data=self._event_data,
parameter_path=parameter_path,
output_path=output_path,
wrap_length=self._wrap_length,
test=self._test,
printer=prt,
fitter=self,
pool=pool,
print_trees=print_trees)
if not self._model._model_name:
prt.message('no_models_avail', [self._event_name],
warning=True)
continue
if not event:
prt.message('gen_dummy')
self._event_name = mod_name
gen_args = {
'name': mod_name,
'max_time': max_time,
'time_list': time_list,
'band_list': band_list,
'band_systems': band_systems,
'band_instruments': band_instruments,
'band_bandsets': band_bandsets
}
self._event_data = self.generate_dummy_data(**gen_args)
success = False
alt_name = None
while not success:
self._model.reset_unset_recommended_keys()
success = self._model.load_data(
self._event_data,
event_name=self._event_name,
smooth_times=smooth_times,
extrapolate_time=extrapolate_time,
limit_fitting_mjds=limit_fitting_mjds,
exclude_bands=exclude_bands,
exclude_instruments=exclude_instruments,
exclude_systems=exclude_systems,
exclude_sources=exclude_sources,
exclude_kinds=exclude_kinds,
time_list=time_list,
time_unit=time_unit,
band_list=band_list,
band_systems=band_systems,
band_instruments=band_instruments,
band_bandsets=band_bandsets,
band_sampling_points=band_sampling_points,
variance_for_each=variance_for_each,
user_fixed_parameters=user_fixed_parameters,
user_released_parameters=user_released_parameters,
pool=pool)
if not success:
break
if self._local_data_only:
break
# If our data is missing recommended keys, offer the
# user option to pull the missing data from online and
# merge with existing data.
urk = self._model.get_unset_recommended_keys()
ptxt = prt.text('acquire_recommended',
[', '.join(list(urk))])
while event and len(urk) and (
alt_name or self._download_recommended_data
or prt.prompt(ptxt, [', '.join(urk)],
kind='bool')):
try:
pool = MPIPool()
except (ImportError, ValueError):
pool = SerialPool()
if pool.is_master():
en = (alt_name
if alt_name else self._event_name)
extra_event = self._fetcher.fetch(
en,
offline=self._offline,
prefer_cache=self._prefer_cache)[0]
extra_data = self._fetcher.load_data(
extra_event)
for rank in range(1, pool.size + 1):
pool.comm.send(extra_data,
dest=rank,
tag=4)
pool.close()
else:
extra_data = pool.comm.recv(source=0, tag=4)
pool.wait()
if extra_data is not None:
extra_data = extra_data[list(
extra_data.keys())[0]]
for key in urk:
new_val = extra_data.get(key)
self._event_data[list(
self._event_data.keys())
[0]][key] = new_val
if new_val is not None and len(new_val):
prt.message('extra_value', [
key,
str(new_val[0].get(QUANTITY.VALUE))
])
success = False
prt.message('reloading_merged')
break
else:
text = prt.text('extra_not_found',
[self._event_name])
alt_name = prt.prompt(text, kind='string')
if not alt_name:
break
if success:
self._walker_data = walker_data
entry, p, lnprob = self.fit_data(
event_name=self._event_name,
method=method,
iterations=iterations,
num_walkers=num_walkers,
num_temps=num_temps,
burn=burn,
post_burn=post_burn,
fracking=fracking,
frack_step=frack_step,
gibbs=gibbs,
pool=pool,
output_path=output_path,
suffix=suffix,
write=write,
upload=upload,
upload_token=upload_token,
check_upload_quality=check_upload_quality,
convergence_type=convergence_type,
convergence_criteria=convergence_criteria,
save_full_chain=save_full_chain,
extra_outputs=extra_outputs)
if return_fits:
entries[ei][mi] = deepcopy(entry)
ps[ei][mi] = deepcopy(p)
lnprobs[ei][mi] = deepcopy(lnprob)
if pool.is_master():
pool.close()
# Remove global model variable and garbage collect.
try:
model
except NameError:
pass
else:
del (model)
del (self._model)
gc.collect()
return (entries, ps, lnprobs)
def __init__(self, **kwargs):
"""Initialize module."""
super(Photometry, self).__init__(**kwargs)
bands = kwargs.get('bands', '')
bands = listify(bands)
self._dir_path = os.path.dirname(os.path.realpath(__file__))
band_list = []
if self._pool.is_master():
with open(os.path.join(self._dir_path, 'filterrules.json')) as f:
filterrules = json.load(f, object_pairs_hook=OrderedDict)
for rank in range(1, self._pool.size + 1):
self._pool.comm.send(filterrules, dest=rank, tag=5)
else:
filterrules = self._pool.comm.recv(source=0, tag=5)
for bi, band in enumerate(bands):
for rule in filterrules:
if '@note' in rule:
continue
sysinstperms = [
{
'systems': xx,
'instruments': yy,
'bandsets': zz,
'telescopes': tt,
'modes': mm
}
for xx in rule.get('systems', [''])
for yy in rule.get('instruments', [''])
for zz in rule.get('bandsets', [''])
for tt in rule.get('telescopes', [''])
for mm in rule.get('modes', [''])
]
for bnd in rule.get('filters', []):
if band == bnd or band == '':
for perm in sysinstperms:
new_band = deepcopy(rule['filters'][bnd])
new_band.update(deepcopy(perm))
new_band['name'] = bnd
band_list.append(new_band)
self._unique_bands = band_list
self._band_insts = np.array(
[x['instruments'] for x in self._unique_bands], dtype=object)
self._band_bsets = np.array(
[x['bandsets'] for x in self._unique_bands], dtype=object)
self._band_systs = np.array(
[x['systems'] for x in self._unique_bands], dtype=object)
self._band_teles = np.array(
[x['telescopes'] for x in self._unique_bands], dtype=object)
self._band_modes = np.array(
[x['modes'] for x in self._unique_bands], dtype=object)
self._band_names = np.array(
[x['name'] for x in self._unique_bands], dtype=object)
self._n_bands = len(self._unique_bands)
self._band_wavelengths = [[] for i in range(self._n_bands)]
self._band_energies = [[] for i in range(self._n_bands)]
self._transmissions = [[] for i in range(self._n_bands)]
self._band_areas = [[] for i in range(self._n_bands)]
self._min_waves = np.full(self._n_bands, 0.0)
self._max_waves = np.full(self._n_bands, 0.0)
self._imp_waves = [[0.0, 1.0] for i in range(self._n_bands)]
self._filter_integrals = np.full(self._n_bands, 0.0)
self._average_wavelengths = np.full(self._n_bands, 0.0)
self._band_offsets = np.full(self._n_bands, 0.0)
self._band_xunits = np.full(self._n_bands, 'Angstrom', dtype=object)
self._band_yunits = np.full(self._n_bands, '', dtype=object)
self._band_xu = np.full(self._n_bands, u.Angstrom.cgs.scale)
self._band_yu = np.full(self._n_bands, 1.0)
self._band_kinds = np.full(self._n_bands, 'magnitude', dtype=object)
self._band_index_cache = {}
for i, band in enumerate(self._unique_bands):
self._band_xunits[i] = band.get('xunit', 'Angstrom')
self._band_yunits[i] = band.get('yunit', '')
self._band_xu[i] = u.Unit(self._band_xunits[i]).cgs.scale
self._band_yu[i] = u.Unit(self._band_yunits[i]).cgs.scale
if '{0}'.format(self._band_yunits[i]) == 'cm2':
self._band_kinds[i] = 'countrate'
def set_data(self,
all_data,
req_key_values={},
subtract_minimum_keys=[],
smooth_times=-1,
extrapolate_time=0.0,
limit_fitting_mjds=False,
exclude_bands=[],
exclude_instruments=[],
exclude_systems=[],
exclude_sources=[],
exclude_kinds=[],
time_unit=None,
time_list=[],
band_list=[],
band_telescopes=[],
band_systems=[],
band_instruments=[],
band_modes=[],
band_bandsets=[]):
"""Set transient data."""
prt = self._printer
self._all_data = all_data
self._data = OrderedDict()
if not self._all_data:
return
name = list(self._all_data.keys())[0]
self._data['name'] = name
numeric_keys = set()
ex_kinds = [
self._EX_REPS.get(x.lower(), x.lower()) for x in exclude_kinds
]
# Construct some source dictionaries for exclusion rules
src_dict = OrderedDict()
sources = self._all_data[name].get('sources', [])
for src in sources:
if SOURCE.BIBCODE in src:
src_dict[src[SOURCE.ALIAS]] = src[SOURCE.BIBCODE]
if SOURCE.ARXIVID in src:
src_dict[src[SOURCE.ALIAS]] = src[SOURCE.ARXIVID]
if SOURCE.NAME in src:
src_dict[src[SOURCE.ALIAS]] = src[SOURCE.NAME]
for key in self._keys:
subkeys = self._keys[key]
req_subkeys = [
x for x in subkeys if not isinstance(subkeys, dict)
or 'required' in listify(subkeys[x])
]
num_subkeys = [
x for x in subkeys if 'numeric' in listify(subkeys[x])
]
boo_subkeys = [
x for x in subkeys if 'boolean' in listify(subkeys[x])
]
exc_subkeys = [
x for x in subkeys if 'exclude' in listify(subkeys[x])
]
if (key not in self._all_data[name]
and not self._model.is_parameter_fixed_by_user(key)):
if subkeys.get('value', None) == 'recommended':
self._unset_recommended_keys.add(key)
continue
subdata = self._all_data[name].get(key)
if subdata is None:
continue
# Only include data that contains all subkeys
for entry in subdata:
if any([x not in entry for x in req_subkeys]):
continue
if any([x in entry for x in exc_subkeys]):
continue
if any([
x in entry and
((isinstance(entry[x], list) and any([
not is_number(y) or np.isnan(float(y))
for y in entry[x]
])) or not isinstance(entry[x], list) and
(not is_number(entry[x]) or np.isnan(float(entry[x]))))
for x in num_subkeys
]):
continue
skip_key = False
if 'frequency' not in entry:
for qkey in req_key_values:
if qkey in entry and entry[qkey] != '':
if entry[qkey] not in req_key_values[qkey]:
skip_key = True
break
if key == 'photometry':
if ('fluxdensity' in entry and 'magnitude' not in entry
and 'countrate' not in entry):
self._kinds_needed.add('radio')
if ('radio' in ex_kinds or
(not len(ex_kinds) or 'none' not in ex_kinds) and
'radio' not in self._model._kinds_supported):
continue
if (('countrate' in entry or 'unabsorbedflux' in entry
or 'flux' in entry) and 'magnitude' not in entry
and 'fluxdensity' not in entry):
self._kinds_needed.add('x-ray')
if ('x-ray' in ex_kinds or
(not len(ex_kinds) or 'none' not in ex_kinds) and
'x-ray' not in self._model._kinds_supported):
continue
if 'magnitude' in entry:
# For now, magnitudes are not excludable.
self._kinds_needed |= set(
['infrared', 'optical', 'ultraviolet'])
skip_entry = False
for x in subkeys:
if limit_fitting_mjds is not False and x == 'time':
val = np.mean([
float(x) for x in listify(entry.get(x, None))
])
if (val < limit_fitting_mjds[0]
or val > limit_fitting_mjds[1]):
skip_entry = True
break
if exclude_bands is not False and x == 'band':
if (entry.get(x, '') in exclude_bands and
(not exclude_instruments or entry.get(
'instrument', '') in exclude_instruments)
and (not exclude_systems or entry.get(
'system', '') in exclude_systems)):
skip_entry = True
break
if (exclude_instruments is not False
and x == 'instrument'):
if (entry.get(x, '') in exclude_instruments and
(not exclude_bands
or entry.get('band', '') in exclude_bands)
and (not exclude_systems or entry.get(
'system', '') in exclude_systems)):
skip_entry = True
break
if (exclude_systems is not False and x == 'system'):
if (entry.get(x, '') in exclude_systems and
(not exclude_bands
or entry.get('band', '') in exclude_bands) and
(not exclude_instruments or entry.get(
'instrument', '') in exclude_instruments)):
skip_entry = True
break
if (exclude_sources is not False and x == 'source'):
val = entry.get(x, '')
if (any(
[x in exclude_sources for x in val.split(',')])
or any([
src_dict.get(x, '') in exclude_sources
for x in val.split(',')
])):
skip_entry = True
break
if skip_entry:
continue
if ((('magnitude' in entry) !=
('band' in entry)) or ((('fluxdensity' in entry) !=
('frequency' in entry)) and
('magnitude' not in entry))
or (('countrate' in entry) and
('magnitude' not in entry) and
('instrument' not in entry))):
continue
for x in subkeys:
falseval = (False if x in boo_subkeys else
None if x in num_subkeys else '')
if x == 'value':
if not skip_key:
self._data[key] = entry.get(x, falseval)
else:
plural = self._model.plural(x)
val = entry.get(x, falseval)
if x in num_subkeys:
val = None if val is None else np.mean(
[float(x) for x in listify(val)])
if not skip_key:
self._data.setdefault(plural, []).append(val)
if x in num_subkeys:
numeric_keys.add(plural)
else:
self._data.setdefault('unmatched_' + plural,
[]).append(val)
if 'times' not in self._data or not any([
x in self._data
for x in ['magnitudes', 'frequencies', 'countrates']
]):
prt.message('no_fittable_data', [name])
return False
for key in list(self._data.keys()):
if isinstance(self._data[key], list):
self._data[key] = np.array(self._data[key])
if key not in numeric_keys:
continue
num_values = [
x for x in self._data[key] if isinstance(x, float)
]
if len(num_values):
self._data['min_' + key] = min(num_values)
self._data['max_' + key] = max(num_values)
else:
if is_number(self._data[key]):
self._data[key] = float(self._data[key])
self._data_determined_parameters.append(key)
if any(x in self._data
for x in ['magnitudes', 'countrates', 'fluxdensities']):
# Add a list of tags for each observation to indicate what unit
# observation is provided in.
self._data['measures'] = [
((['magnitude'] if x else []) + (['countrate'] if y else []) +
(['fluxdensity'] if x else []))
for x, y, z in zip(*(self._data['magnitudes'],
self._data['countrates'],
self._data['fluxdensities']))
]
if 'times' in self._data and (smooth_times >= 0 or time_list):
# Build an observation array out of the real data first.
obs = list(
zip(*(self._data[x] for x in self._OBS_KEYS if x != 'times')))
# Append extra observations if requested.
if len(band_list):
b_teles = band_telescopes if len(band_telescopes) == len(
band_list) else (
[band_telescopes[0] for x in band_list]
if len(band_telescopes) else ['' for x in band_list])
b_systs = band_systems if len(band_systems) == len(
band_list) else ([band_systems[0]
for x in band_list] if len(band_systems)
else ['' for x in band_list])
b_modes = band_modes if len(band_modes) == len(
band_list) else ([band_modes[0]
for x in band_list] if len(band_modes)
else ['' for x in band_list])
b_insts = band_instruments if len(band_instruments) == len(
band_list) else (
[band_instruments[0] for x in band_list]
if len(band_instruments) else ['' for x in band_list])
b_bsets = band_bandsets if len(band_bandsets) == len(
band_list) else ([band_bandsets[0]
for x in band_list] if len(band_bandsets)
else ['' for x in band_list])
b_freqs = [None for x in band_list]
b_u_freqs = ['' for x in band_list]
b_zerops = [None for x in band_list]
b_measures = [[] for x in band_list]
obs.extend(
list(
zip(*(b_teles, b_systs, b_modes, b_insts, b_bsets,
band_list, b_freqs, b_u_freqs, b_zerops,
b_measures))))
# Prune extra observations if they are duplicitous to existing.
uniqueobs = []
for o in obs:
to = tuple(o)
if to not in uniqueobs:
uniqueobs.append(to)
# Preprend times to real observations list.
minet, maxet = (extrapolate_time, extrapolate_time) if isinstance(
extrapolate_time, (float, int)) else (
(tuple(extrapolate_time) if len(extrapolate_time) == 2 else
(extrapolate_time[0], extrapolate_time[0])))
mint, maxt = (min(self._data['times']) - minet,
max(self._data['times']) + maxet)
if time_unit is None:
alltimes = time_list + [x for x in self._data['times']]
elif time_unit == 'mjd':
alltimes = [x - min(self._data['times']) for x in time_list
] + [x for x in self._data['times']]
elif time_unit == 'phase':
if 'maxdate' not in self._data:
raise (prt.message('no_maxdate', name))
max_mjd = astrotime(self._data['maxdate'].replace('/',
'-')).mjd
alltimes = [x + max_mjd for x in time_list
] + [x for x in self._data['times']]
else:
raise ('Unknown `time_unit`.')
if smooth_times >= 0:
alltimes += list(np.linspace(mint, maxt, max(smooth_times, 2)))
alltimes = list(sorted(set(alltimes)))
# Create additional fake observations.
currobslist = list(zip(*(self._data[x] for x in self._OBS_KEYS)))
obslist = []
for ti, t in enumerate(alltimes):
new_per = np.round(100.0 * float(ti) / len(alltimes), 1)
prt.message('construct_obs_array', [new_per], inline=True)
for o in uniqueobs:
newobs = tuple([t] + list(o))
if newobs not in currobslist:
obslist.append(newobs)
obslist.sort()
# Save these fake observations under keys with `extra_` prefix.
if obslist:
for x, y in zip(self._OBS_KEYS, zip(*obslist)):
self._data['extra_' + x] = y
for qkey in subtract_minimum_keys:
if 'upperlimits' in self._data:
new_vals = np.array(self._data[qkey])[np.array(
self._data['upperlimits']) != True] # noqa E712
if new_vals.size:
self._data['min_' + qkey] = min(new_vals)
self._data['max_' + qkey] = max(new_vals)
minv = self._data['min_' + qkey]
self._data[qkey] = [x - minv for x in self._data[qkey]]
if 'extra_' + qkey in self._data:
self._data['extra_' + qkey] = [
x - minv for x in self._data['extra_' + qkey]
]
return True
def fetch(self, event_list, offline=False):
"""Fetch a list of events from the open catalogs."""
dir_path = os.path.dirname(os.path.realpath(__file__))
prt = self._printer
levent_list = listify(event_list)
events = [None for x in levent_list]
catalogs = OrderedDict([
(x, self._catalogs[x]) for x in self._catalogs
if x not in self._excluded_catalogs])
for ei, event in enumerate(levent_list):
if not event:
continue
events[ei] = OrderedDict()
path = ''
# If the event name ends in .json, assume event is a path.
if event.endswith('.json'):
path = event
events[ei]['name'] = event.replace('.json',
'').split('/')[-1]
# If not (or the file doesn't exist), download from an open
# catalog.
if not path or not os.path.exists(path):
names_paths = [
os.path.join(dir_path, 'cache', x +
'.names.min.json') for x in catalogs]
input_name = event.replace('.json', '')
if offline:
prt.message('event_interp', [input_name])
else:
prt.message('dling_aliases', [input_name])
for ci, catalog in enumerate(catalogs):
try:
response = get_url_file_handle(
catalogs[catalog]['json'] +
'/names.min.json',
timeout=10)
except Exception:
prt.message(
'cant_dl_names', [catalog], warning=True)
else:
with open_atomic(
names_paths[ci], 'wb') as f:
shutil.copyfileobj(response, f)
names = OrderedDict()
for ci, catalog in enumerate(catalogs):
if os.path.exists(names_paths[ci]):
with open(names_paths[ci], 'r') as f:
names[catalog] = json.load(
f, object_pairs_hook=OrderedDict)
else:
prt.message('cant_read_names', [catalog],
warning=True)
if offline:
prt.message('omit_offline')
continue
if input_name in names[catalog]:
events[ei]['name'] = input_name
events[ei]['catalog'] = catalog
else:
for name in names[catalog]:
if (input_name in names[catalog][name] or
'SN' + input_name in
names[catalog][name]):
events[ei]['name'] = name
events[ei]['catalog'] = catalog
break
if not events[ei].get('name', None):
for ci, catalog in enumerate(catalogs):
namekeys = []
for name in names[catalog]:
namekeys.extend(names[catalog][name])
namekeys = list(sorted(set(namekeys)))
matches = get_close_matches(
event, namekeys, n=5, cutoff=0.8)
# matches = []
if len(matches) < 5 and is_number(event[0]):
prt.message('pef_ext_search')
snprefixes = set(('SN19', 'SN20'))
for name in names[catalog]:
ind = re.search("\d", name)
if ind and ind.start() > 0:
snprefixes.add(name[:ind.start()])
snprefixes = list(sorted(snprefixes))
for prefix in snprefixes:
testname = prefix + event
new_matches = get_close_matches(
testname, namekeys, cutoff=0.95,
n=1)
if (len(new_matches) and
new_matches[0] not in matches):
matches.append(new_matches[0])
if len(matches) == 5:
break
if len(matches):
if self._test:
response = matches[0]
else:
response = prt.prompt(
'no_exact_match',
kind='select',
options=matches,
none_string=(
'None of the above, ' +
('skip this event.' if
ci == len(catalogs) - 1
else
'try the next catalog.')))
if response:
for name in names[catalog]:
if response in names[
catalog][name]:
events[ei]['name'] = name
events[ei]['catalog'] = catalog
break
if events[ei]['name']:
break
if not events[ei].get('name', None):
prt.message('no_event_by_name')
events[ei]['name'] = input_name
continue
urlname = events[ei]['name'] + '.json'
name_path = os.path.join(dir_path, 'cache', urlname)
if offline:
prt.message('cached_event', [
events[ei]['name'], events[ei]['catalog']])
else:
prt.message('dling_event', [
events[ei]['name'], events[ei]['catalog']])
try:
response = get_url_file_handle(
catalogs[events[ei]['catalog']][
'json'] + '/json/' + urlname,
timeout=10)
except Exception:
prt.message('cant_dl_event', [
events[ei]['name']], warning=True)
else:
with open_atomic(name_path, 'wb') as f:
shutil.copyfileobj(response, f)
path = name_path
if os.path.exists(path):
events[ei]['path'] = path
if self._open_in_browser:
webbrowser.open(
catalogs[events[ei]['catalog']]['web'] +
events[ei]['name'])
with open(path, 'r') as f:
events[ei]['data'] = json.load(
f, object_pairs_hook=OrderedDict)
prt.message('event_file', [path], wrapped=True)
else:
prt.message('no_data', [
events[ei]['name'],
'/'.join(catalogs.keys())])
if offline:
prt.message('omit_offline')
raise RuntimeError
return events
def add_excluded_catalogs(self, catalogs):
"""Add catalog name(s) to list of catalogs that will be excluded."""
if not isinstance(catalogs, list) or isinstance(
catalogs, string_types):
catalogs = listify(catalogs)
self._excluded_catalogs.extend([x.upper() for x in catalogs])
def assign_columns(self, cidict, flines):
"""Assign columns based on header."""
used_cis = OrderedDict()
akeys = list(self._critical_keys) + list(self._helpful_keys)
dkeys = list(self._dep_keys)
prt = self._printer
for fi, fl in enumerate(flines):
if not any([is_number(x) for x in fl]):
# Try to associate column names with common header keys.
conflict_keys = []
conflict_cis = []
for ci, col in enumerate(fl):
for key in self._header_keys:
if any([(x[0] if isinstance(x, tuple)
else x) == col.lower()
for x in self._header_keys[key]]):
if key in cidict or ci in used_cis:
# There is a conflict, ask user.
conflict_keys.append(key)
conflict_cis.append(ci)
else:
ind = [
(x[0] if isinstance(x, tuple) else x)
for x in self._header_keys[key]].index(
col.lower())
match = self._header_keys[key][ind]
cidict[key] = [match[-1], ci] if isinstance(
match, tuple) else ci
used_cis[ci] = key
break
for cki, ck in enumerate(conflict_keys):
if ck in cidict:
ci = cidict[ck]
del(cidict[ck])
del(used_cis[ci])
else:
self._first_data = fi
break
# Look for columns that are band names if no mag/counts/flux dens
# column was found.
if (not any([x in cidict for x in [
PHOTOMETRY.MAGNITUDE, PHOTOMETRY.COUNT_RATE,
PHOTOMETRY.FLUX_DENSITY]])):
# Delete `E_MAGNITUDE` and `BAND` if they exist (we'll need to find
# for each column).
key = PHOTOMETRY.MAGNITUDE
ekey = PHOTOMETRY.E_MAGNITUDE
bkey = PHOTOMETRY.BAND
if ekey in cidict:
ci = cidict[ekey]
del(cidict[used_cis[ci]])
del(used_cis[ci])
if bkey in cidict:
ci = cidict[bkey]
del(cidict[used_cis[ci]])
del(used_cis[ci])
for fi, fl in enumerate(flines):
if not any([is_number(x) for x in fl]):
# Try to associate column names with common header keys.
for ci, col in enumerate(fl):
if ci in used_cis:
continue
if col in self._band_names:
cidict.setdefault(key, []).append(ci)
used_cis[ci] = key
cidict.setdefault(bkey, []).append(col)
elif col in self._emagstrs:
cidict.setdefault(ekey, []).append(ci)
used_cis[ci] = ekey
# See which keys we collected. If we are missing any critical keys, ask
# the user which column they are.
# First ask the user if this data is in magnitudes or in counts.
self._data_type = 1
if (PHOTOMETRY.MAGNITUDE in cidict and
PHOTOMETRY.COUNT_RATE not in cidict and
PHOTOMETRY.FLUX_DENSITY not in cidict):
self._data_type = 1
elif (PHOTOMETRY.MAGNITUDE not in cidict and
PHOTOMETRY.COUNT_RATE in cidict and
PHOTOMETRY.FLUX_DENSITY not in cidict):
self._data_type = 2
elif (PHOTOMETRY.MAGNITUDE not in cidict and
PHOTOMETRY.COUNT_RATE not in cidict and
PHOTOMETRY.FLUX_DENSITY in cidict):
self._data_type = 3
else:
self._data_type = prt.prompt(
'counts_mags_fds', kind='option',
options=['Magnitudes', 'Counts (per second)',
'Flux Densities (Jansky)'],
none_string=None)
if self._data_type in [1, 3]:
akeys.remove(PHOTOMETRY.COUNT_RATE)
akeys.remove(PHOTOMETRY.E_COUNT_RATE)
akeys.remove(PHOTOMETRY.ZERO_POINT)
if (PHOTOMETRY.MAGNITUDE in akeys and
PHOTOMETRY.E_MAGNITUDE in akeys):
akeys.remove(PHOTOMETRY.E_MAGNITUDE)
akeys.insert(
akeys.index(PHOTOMETRY.MAGNITUDE) + 1,
PHOTOMETRY.E_MAGNITUDE)
if (PHOTOMETRY.E_LOWER_MAGNITUDE in cidict and
PHOTOMETRY.E_UPPER_MAGNITUDE in cidict):
akeys.remove(PHOTOMETRY.E_MAGNITUDE)
dkeys.remove(PHOTOMETRY.E_COUNT_RATE)
if self._data_type in [2, 3]:
akeys.remove(PHOTOMETRY.MAGNITUDE)
akeys.remove(PHOTOMETRY.E_MAGNITUDE)
dkeys.remove(PHOTOMETRY.E_MAGNITUDE)
if self._data_type in [1, 2]:
akeys.remove(PHOTOMETRY.FLUX_DENSITY)
akeys.remove(PHOTOMETRY.E_FLUX_DENSITY)
if (PHOTOMETRY.E_LOWER_FLUX_DENSITY in cidict and
PHOTOMETRY.E_UPPER_FLUX_DENSITY in cidict):
akeys.remove(PHOTOMETRY.E_FLUX_DENSITY)
dkeys.remove(PHOTOMETRY.E_FLUX_DENSITY)
dkeys.remove(PHOTOMETRY.U_FLUX_DENSITY)
columns = np.array(flines[self._first_data:]).T.tolist()
colstrs = np.array([
', '.join(x[:5]) + ', ...' for x in columns])
colinds = np.setdiff1d(np.arange(
len(colstrs)), list([x[-1] if (
isinstance(x, list) and not isinstance(
x, string_types)) else x for x in cidict.values()]))
ignore = prt.message('ignore_column', prt=False)
specify = prt.message('specify_column', prt=False)
for key in akeys:
selected_cols = [
y for y in [a for b in [
listify(x) for x in list(cidict.values())] for a in b]
if isinstance(y, (int, np.integer))]
if key in cidict:
continue
if key in dkeys and self._use_mc:
continue
if key.type == KEY_TYPES.NUMERIC:
lcolinds = [x for x in colinds
if any(is_number(y) for y in columns[x]) and
x not in selected_cols]
elif key.type == KEY_TYPES.TIME:
lcolinds = [x for x in colinds
if any(is_date(y) or is_number(y)
for y in columns[x]) and
x not in selected_cols]
elif key.type == KEY_TYPES.STRING:
lcolinds = [x for x in colinds
if any(not is_number(y) for y in columns[x]) and
x not in selected_cols]
else:
lcolinds = [x for x in colinds if x not in selected_cols]
select = False
selects = []
while select is False:
mc = 1
if key in self._mc_keys:
pkey = self._inflect.plural(key)
text = prt.message(
'one_per_line', [key, pkey, pkey],
prt=False)
mc = prt.prompt(
text, kind='option', message=False,
none_string=None,
options=[
'One `{}` per row'.format(key),
'Multiple `{}` per row'.format(pkey)])
if mc == 1:
text = prt.message(
'no_matching_column', [key], prt=False)
ns = (
ignore if key in (
self._optional_keys + self._helpful_keys) else
specify if key in self._specify_keys
else None)
if len(colstrs[lcolinds]):
select = prt.prompt(
text, message=False,
kind='option', none_string=ns,
default=('j' if ns is None and
len(colstrs[lcolinds]) > 1
else None if ns is None else 'n'),
options=colstrs[lcolinds].tolist() + (
[('Multiple columns need to be joined.', 'j')]
if len(colstrs[lcolinds]) > 1 else []))
else:
select = None
if select == 'j':
select = None
jsel = None
selects.append('j')
while jsel != 'd' and len(lcolinds):
jsel = prt.prompt(
'join_which_columns', default='d',
kind='option', none_string=None,
options=colstrs[lcolinds].tolist() + [
('All columns to be joined '
'have been selected.', 'd')
])
if jsel != 'd':
selects.append(lcolinds[jsel - 1])
lcolinds = np.delete(lcolinds, jsel - 1)
else:
self._use_mc = True
select = False
while select is not None:
text = prt.message(
'select_mc_column', [key], prt=False)
select = prt.prompt(
text, message=False,
kind='option', default='n',
none_string='No more `{}` columns.'.format(key),
options=colstrs[lcolinds].tolist())
if select is not None and select is not False:
selects.append(lcolinds[select - 1])
lcolinds = np.delete(lcolinds, select - 1)
else:
break
for dk in dkeys:
dksel = None
while dksel is None:
text = prt.message(
'select_dep_column', [dk, key], prt=False)
sk = dk in self._specify_keys
if not sk:
dksel = prt.prompt(
text, message=False,
kind='option', none_string=None,
options=colstrs[lcolinds].tolist())
if dksel is not None:
selects.append(lcolinds[dksel - 1])
lcolinds = np.delete(
lcolinds, dksel - 1)
else:
spectext = prt.message(
'specify_mc_value', [dk, key],
prt=False)
val = ''
while val.strip() is '':
val = prt.prompt(
spectext, message=False,
kind='string')
selects.append(val)
break
if select is not None:
iselect = int(select)
cidict[key] = lcolinds[iselect - 1]
colinds = np.delete(colinds, np.argwhere(
colinds == lcolinds[iselect - 1]))
elif len(selects):
if selects[0] == 'j':
cidict[key] = selects
else:
kdkeys = [key] + dkeys
allk = list(OrderedDict.fromkeys(kdkeys).keys())
for ki, k in enumerate(allk):
cidict[k] = [
colinds[s - 1] if isinstance(s, (
int, np.integer)) else s
for s in selects[ki::len(allk)]]
for s in selects:
if not isinstance(s, (int, np.integer)):
continue
colinds = np.delete(colinds, np.argwhere(
colinds == s - 1))
elif key in self._specify_keys:
msg = ('specify_value_blank' if key in self._helpful_keys else
'specify_value')
text = prt.message(msg, [key], prt=False)
cidict[key] = prt.prompt(
text, message=False, kind='string', allow_blank=(
key in self._helpful_keys))
self._zp = ''
if self._data_type == 2 and PHOTOMETRY.ZERO_POINT not in cidict:
while not is_number(self._zp):
self._zp = prt.prompt('zeropoint', kind='string')
self._ufd = None
if self._data_type == 3 and PHOTOMETRY.U_FLUX_DENSITY not in cidict:
while ((self._ufd.lower() if self._ufd is not None else None)
not in ['µjy', 'mjy', 'jy', 'microjy', 'millijy', 'jy',
'microjansky', 'millijansky', 'jansky', '']):
self._ufd = prt.prompt('u_flux_density', kind='string')
self._system = None
if self._data_type == 1 and PHOTOMETRY.SYSTEM not in cidict:
systems = ['AB', 'Vega']
self._system = prt.prompt(
'system', kind='option', options=systems,
none_string='Use default for all bands.',
default='n')
if self._system is not None:
self._system = systems[int(self._system) - 1]
if (PHOTOMETRY.INSTRUMENT not in cidict and
PHOTOMETRY.TELESCOPE not in cidict):
prt.message('instrument_recommended', warning=True)
def adjust_fixed_parameters(self, variance_for_each=[], output={}):
"""Create free parameters that depend on loaded data."""
unique_band_indices = list(
sorted(set(output.get('all_band_indices', []))))
needs_general_variance = any(
np.array(output.get('all_band_indices', [])) < 0)
new_call_stack = OrderedDict()
for task in self._call_stack:
cur_task = self._call_stack[task]
vfe = listify(variance_for_each)
if task == 'variance' and 'band' in vfe:
vfi = vfe.index('band') + 1
mwfd = float(vfe[vfi]) if (vfi < len(vfe) and is_number(
vfe[vfi])) else self.MIN_WAVE_FRAC_DIFF
# Find photometry in call stack.
ptask = None
for ptask in self._call_stack:
if ptask == 'photometry':
awaves = self._modules[ptask].average_wavelengths(
unique_band_indices)
abands = self._modules[ptask].bands(
unique_band_indices)
band_pairs = list(sorted(zip(awaves, abands)))
break
owav = 0.0
variance_bands = []
for (awav, band) in band_pairs:
wave_frac_diff = abs(awav - owav) / (awav + owav)
if wave_frac_diff < mwfd:
continue
new_task_name = '-'.join([task, 'band', band])
if new_task_name in self._call_stack:
continue
new_task = deepcopy(cur_task)
new_call_stack[new_task_name] = new_task
if 'latex' in new_task:
new_task['latex'] += '_{\\rm ' + band + '}'
new_call_stack[new_task_name] = new_task
self._modules[new_task_name] = self._load_task_module(
new_task_name, call_stack=new_call_stack)
owav = awav
variance_bands.append([awav, band])
if needs_general_variance:
new_call_stack[task] = deepcopy(cur_task)
if self._pool.is_master():
self._printer.message(
'anchoring_variances',
[', '.join([x[1] for x in variance_bands])],
wrapped=True)
self._modules[ptask].set_variance_bands(variance_bands)
else:
new_call_stack[task] = deepcopy(cur_task)
# Fixed any variables to be fixed if any conditional inputs are
# fixed by the data.
# if any([listify(x)[-1] == 'conditional'
# for x in cur_task.get('inputs', [])]):
self._call_stack = new_call_stack
for task in reversed(self._call_stack):
cur_task = self._call_stack[task]
for inp in cur_task.get('inputs', []):
other = listify(inp)[0]
if (cur_task['kind'] == 'parameter'
and output.get(other, None) is not None):
if (not self._modules[other]._fixed
or self._modules[other]._fixed_by_user):
self._modules[task]._fixed = True
self._modules[task]._derived_keys = list(
set(self._modules[task]._derived_keys + [task]))
def set_data(self,
all_data,
req_key_values={},
subtract_minimum_keys=[],
smooth_times=-1,
extrapolate_time=0.0,
limit_fitting_mjds=False,
exclude_bands=[],
exclude_instruments=[],
exclude_systems=[],
exclude_sources=[],
exclude_kinds=[],
band_list=[],
band_telescopes=[],
band_systems=[],
band_instruments=[],
band_modes=[],
band_bandsets=[]):
"""Set transient data."""
prt = self._printer
self._all_data = all_data
self._data = OrderedDict()
if not self._all_data:
return
name = list(self._all_data.keys())[0]
self._data['name'] = name
numeric_keys = set()
ex_kinds = [self._EX_REPS.get(
x.lower(), x.lower()) for x in exclude_kinds]
# Construct some source dictionaries for exclusion rules
src_dict = OrderedDict()
sources = self._all_data[name].get('sources', [])
for src in sources:
if SOURCE.BIBCODE in src:
src_dict[src[SOURCE.ALIAS]] = src[SOURCE.BIBCODE]
if SOURCE.ARXIVID in src:
src_dict[src[SOURCE.ALIAS]] = src[SOURCE.ARXIVID]
if SOURCE.NAME in src:
src_dict[src[SOURCE.ALIAS]] = src[SOURCE.NAME]
for key in self._keys:
subkeys = self._keys[key]
req_subkeys = [
x for x in subkeys
if not isinstance(subkeys, dict) or 'required' in listify(
subkeys[x])
]
num_subkeys = [
x for x in subkeys if 'numeric' in listify(subkeys[x])
]
boo_subkeys = [
x for x in subkeys if 'boolean' in listify(subkeys[x])
]
exc_subkeys = [
x for x in subkeys if 'exclude' in listify(subkeys[x])
]
if (key not in self._all_data[name] and not
self._model.is_parameter_fixed_by_user(key)):
if subkeys.get('value', None) == 'recommended':
self._unset_recommended_keys.add(key)
continue
subdata = self._all_data[name].get(key)
if subdata is None:
continue
# Only include data that contains all subkeys
for entry in subdata:
if any([x not in entry for x in req_subkeys]):
continue
if any([x in entry for x in exc_subkeys]):
continue
if any([
x in entry and ((isinstance(entry[x], list) and any([
not is_number(y) or np.isnan(float(y))
for y in entry[x]
])) or not isinstance(entry[x], list) and (
not is_number(entry[x]) or np.isnan(
float(entry[x]))))
for x in num_subkeys
]):
continue
skip_key = False
if 'frequency' not in entry:
for qkey in req_key_values:
if qkey in entry and entry[qkey] != '':
if entry[qkey] not in req_key_values[qkey]:
skip_key = True
break
if key == 'photometry':
if ex_kinds is not False:
if 'radio' in ex_kinds:
if ('fluxdensity' in entry and
'magnitude' not in entry and
'countrate' not in entry):
continue
if 'x-ray' in ex_kinds:
if (('countrate' in entry or
'unabsorbedflux' in entry or
'flux' in entry) and
'magnitude' not in entry and
'fluxdensity' not in entry):
continue
skip_entry = False
for x in subkeys:
if limit_fitting_mjds is not False and x == 'time':
val = np.mean([
float(x) for x in listify(
entry.get(x, None))])
if (val < limit_fitting_mjds[0] or
val > limit_fitting_mjds[1]):
skip_entry = True
break
if exclude_bands is not False and x == 'band':
if (entry.get(x, '') in exclude_bands and
(not exclude_instruments or entry.get(
'instrument', '') in
exclude_instruments) and (
not exclude_systems or entry.get(
'system', '') in exclude_systems)):
skip_entry = True
break
if (exclude_instruments is not False and
x == 'instrument'):
if (entry.get(x, '') in exclude_instruments and
(not exclude_bands or
entry.get('band', '') in
exclude_bands) and (
not exclude_systems or entry.get(
'system', '') in exclude_systems)):
skip_entry = True
break
if (exclude_systems is not False and
x == 'system'):
if (entry.get(x, '') in exclude_systems and
(not exclude_bands or
entry.get('band', '') in
exclude_bands) and (
not exclude_instruments or entry.get(
'instrument', '') in exclude_instruments)):
skip_entry = True
break
if (exclude_sources is not False and
x == 'source'):
val = entry.get(x, '')
if (any([x in exclude_sources
for x in val.split(',')]) or
any([src_dict.get(x, '') in exclude_sources
for x in val.split(',')])):
skip_entry = True
break
if skip_entry:
continue
if ((('magnitude' in entry) != ('band' in entry)) or
(('fluxdensity' in entry) != ('frequency' in entry)) or
(('countrate' in entry) and
('magnitude' not in entry) and
('instrument' not in entry))):
continue
for x in subkeys:
falseval = (
False if x in boo_subkeys else None if
x in num_subkeys else '')
if x == 'value':
if not skip_key:
self._data[key] = entry.get(x, falseval)
else:
plural = self._model.plural(x)
val = entry.get(x, falseval)
if x in num_subkeys:
val = None if val is None else np.mean([
float(x) for x in listify(val)])
if not skip_key:
self._data.setdefault(plural, []).append(val)
if x in num_subkeys:
numeric_keys.add(plural)
else:
self._data.setdefault(
'unmatched_' + plural, []).append(val)
if 'times' not in self._data or not any([x in self._data for x in [
'magnitudes', 'frequencies', 'countrates']]):
prt.message('no_fittable_data', [name])
return False
for key in list(self._data.keys()):
if isinstance(self._data[key], list):
self._data[key] = np.array(self._data[key])
if key not in numeric_keys:
continue
num_values = [
x for x in self._data[key] if isinstance(x, float)
]
if len(num_values):
self._data['min_' + key] = min(num_values)
self._data['max_' + key] = max(num_values)
else:
if is_number(self._data[key]):
self._data[key] = float(self._data[key])
self._data_determined_parameters.append(key)
if 'times' in self._data and smooth_times >= 0:
obs = list(
zip(*(self._data['telescopes'], self._data['systems'],
self._data['modes'], self._data['instruments'],
self._data['bandsets'], self._data['bands'], self._data[
'frequencies'], self._data['u_frequencies'])))
if len(band_list):
b_teles = band_telescopes if len(band_telescopes) == len(
band_list) else ([band_telescopes[0] for x in band_list]
if len(band_telescopes) else
['' for x in band_list])
b_systs = band_systems if len(band_systems) == len(
band_list) else ([band_systems[0] for x in band_list]
if len(band_systems) else
['' for x in band_list])
b_modes = band_modes if len(band_modes) == len(
band_list) else ([band_modes[0] for x in band_list]
if len(band_modes) else
['' for x in band_list])
b_insts = band_instruments if len(band_instruments) == len(
band_list) else ([band_instruments[0] for x in band_list]
if len(band_instruments) else
['' for x in band_list])
b_bsets = band_bandsets if len(band_bandsets) == len(
band_list) else ([band_bandsets[0] for x in band_list]
if len(band_bandsets) else
['' for x in band_list])
b_freqs = [None for x in band_list]
b_u_freqs = ['' for x in band_list]
obs.extend(
list(
zip(*(b_teles, b_systs, b_modes, b_insts, b_bsets,
band_list, b_freqs, b_u_freqs))))
uniqueobs = []
for o in obs:
to = tuple(o)
if to not in uniqueobs:
uniqueobs.append(to)
minet, maxet = (extrapolate_time, extrapolate_time) if isinstance(
extrapolate_time, (float, int)) else (
(tuple(extrapolate_time) if len(extrapolate_time) == 2 else
(extrapolate_time[0], extrapolate_time[0])))
mint, maxt = (min(self._data['times']) - minet,
max(self._data['times']) + maxet)
alltimes = list(
sorted(
set([x for x in self._data['times']] + list(
np.linspace(mint, maxt, max(smooth_times, 2))))))
currobslist = list(
zip(*(
self._data['times'], self._data['telescopes'],
self._data['systems'], self._data['modes'],
self._data['instruments'], self._data['bandsets'],
self._data['bands'], self._data['frequencies'],
self._data['u_frequencies'])))
obslist = []
for ti, t in enumerate(alltimes):
new_per = np.round(100.0 * float(ti) / len(alltimes), 1)
prt.message('construct_obs_array', [new_per], inline=True)
for o in uniqueobs:
newobs = tuple([t] + list(o))
if newobs not in currobslist:
obslist.append(newobs)
obslist.sort()
if len(obslist):
(self._data['extra_times'], self._data['extra_telescopes'],
self._data['extra_systems'], self._data['extra_modes'],
self._data['extra_instruments'], self._data['extra_bandsets'],
self._data['extra_bands'], self._data['extra_frequencies'],
self._data['extra_u_frequencies']) = zip(*obslist)
for qkey in subtract_minimum_keys:
if 'upperlimits' in self._data:
new_vals = np.array(self._data[qkey])[
np.array(self._data['upperlimits']) != True] # noqa E712
if len(new_vals):
self._data['min_' + qkey] = min(new_vals)
self._data['max_' + qkey] = max(new_vals)
minv = self._data['min_' + qkey]
self._data[qkey] = [x - minv for x in self._data[qkey]]
if 'extra_' + qkey in self._data:
self._data['extra_' + qkey] = [
x - minv for x in self._data['extra_' + qkey]
]
return True
