def produce_output(self): if not self.output_plan.use_plan: print 'skipping output...' return print 'producing output...' check_0 = time.time() #self.output_plan.flat_data = False pool = self.load_data_pool() data_ = lfu.data_container(data = pool.data) if self.output_plan.must_output(): self.output_plan(data_) if not pool.postproc_data is None: processes = self.postprocess_plan.post_processes for dex, proc in enumerate(processes): if proc.output.must_output(): proc.provide_axes_manager_input() data_ = lfu.data_container( data = pool.postproc_data[dex]) proc.determine_regime(self) proc.output(data_) if not pool.routine_data is None: routines = self.fitting_plan.routines for dex, rout in enumerate(routines): if rout.output.must_output(): rout.provide_axes_manager_input() data_ = lfu.data_container( data = pool.routine_data[dex]) rout.output(data_) print 'produced output: ', time.time() - check_0
def set_module_memory_(ensem): ensem._module_memory_ = [lfu.data_container( output_plan_selected_memory = 'Simulation', variable_selected_memory = 'None', function_selected_memory = 'None', reaction_selected_memory = 'None', species_selected_memory = 'None')]
def save_data_pool(self): print 'saving data pool...' check = time.time() proc_data = None if self.postprocess_plan.use_plan: proc_data = [proc.data for proc in self.postprocess_plan.post_processes] rout_data = None if self.fitting_plan.use_plan: rout_data = [rout.data for rout in self.fitting_plan.routines] self.data_pool = lfu.data_container(data = self.data_pool, postproc_data = proc_data, routine_data = rout_data) self.describe_data_pool(self.data_pool) if self.data_scheme == 'smart_batch': self.data_pool.data.live_pool = [] self.data_pool_pkl = os.path.join(os.getcwd(), 'data_pools', '.'.join(['data_pool', 'smart', self.data_pool_id, 'pkl'])) elif self.data_scheme == 'batch': self.data_pool_pkl = os.path.join(os.getcwd(), 'data_pools', '.'.join(['data_pool', self.data_pool_id, 'pkl'])) else: print 'data_scheme is unresolved... assuming "batch"' self.data_pool_pkl = os.path.join(os.getcwd(), 'data_pools', '.'.join(['data_pool', self.data_pool_id, 'pkl'])) lf.save_pkl_object(self.data_pool, self.data_pool_pkl) print 'saved data pool: ', time.time() - check
def set_module_memory_(ensem):
ensem._module_memory_ = [
lfu.data_container(output_plan_selected_memory='Simulation',
variable_selected_memory='None',
function_selected_memory='None',
reaction_selected_memory='None',
species_selected_memory='None')
]
def deep_parse(self, *args, **kwargs):
def empty_check(unic):
checked = [x for x in unic if not x == '']
return checked
def ovrflw_check(unic):
checked =\
['1000000.0' if x.count('OVRFLW') else x for x in unic]
return checked
def _all_checks_(unic):
unic = empty_check(unic)
checked = ovrflw_check(unic)
return checked
def hms_to_mins(hms):
convs = [60.0, 1.0, 1.0/60.0]
mins = [sum([float(pair[0])*pair[1] for pair
in zip(hms[x].split(':'), convs)])
for x in range(len(hms))]
return mins
def time_filter(hms):
hms0 = '0:00:00'
hms = [min_ for d, min_ in
enumerate(hms) if not
min_ == hms0 or d == 0]
mins = hms_to_mins(hms)
return mins
known_filters = OrderedDict()
known_filters['Time'] = time_filter
known_filters['_all_'] = _all_checks_
def filter_(key, dat):
kf = known_filters
dat = kf['_all_'](dat)
if key in kf.keys(): dat = kf[key](dat)
try: return np.array(dat,dtype='f')
except ValueError: pdb.set_trace()
measur = self._measurement_
weldat = self._well_data_
welkey = self._well_key_
condat = self._cond_data_
conkey = self._cond_key_
con_data = lgeo.scalars_from_labels(conkey)
for dex, key in enumerate(conkey):
con_data[dex].scalars = filter_(key, condat[:,dex])
wel_data = lgeo.scalars_from_labels(welkey)
for dex, key in enumerate(welkey):
wel_data[dex].scalars = filter_(key, weldat[:,dex])
all_data = con_data + wel_data
self._unreduced_ = lfu.data_container(data = all_data)
self._reduced_ = self.apply_reduction(self._unreduced_.data)
self.update_replicate_reduction()
def __init__(self, *args, **kwargs):
self.impose_default('parsed_data',lfu.data_container(),**kwargs)
self.impose_default('input_data_file','051214_P2R2.txt',**kwargs) # TO EXPEDITE TESTING!!
self.impose_default('input_tmpl_file','061114 template for calc.txt',**kwargs) # TO EXPEDITE TESTING!!
#self.impose_default('input_data_file','',**kwargs)
#self.impose_default('input_tmpl_file','',**kwargs)
self.settings_manager = lset.settings_manager(parent = self,
filename = "plate_reader_analyzer_settings.txt")
self.settings = self.settings_manager.read_settings()
self.current_tab_index = 0
self.current_tab_index_outputs = 0
self.postprocess_plan = lpp.post_process_plan(
label = 'Post Process Plan', parent = self)
self.postprocess_plan._display_for_children_ = True
lfu.modular_object_qt.__init__(self, *args, **kwargs)
self._children_ = [self.postprocess_plan]
def apply_reduction(self, unred): read = self.parent.parent.read['layout'].read flat = lfu.flatten(read['table']) well_cnt = len(flat) reduced = unred[:len(unred)-well_cnt] #list of replicate averaged scalers con_offset = len(reduced) uniq = lfu.uniqfy(flat) layout = OrderedDict() for dex, key in enumerate(flat): if key in layout.keys(): layout[key].append(dex + con_offset) else: layout[key] = [dex + con_offset] new = lgeo.scalars_from_labels(layout.keys()) for ndex, key in enumerate(layout.keys()): rel_dexes = layout[key] rel_dater = [unred[d] for d in rel_dexes] zi = zip(*[r.scalars for r in rel_dater]) new[ndex].scalars = np.array([np.mean(z) for z in zi]) reduced.extend(new) red = lfu.data_container(data = reduced) return red
def on_write_data(self, *args, **kwargs):
if not self.domain:
self.domain = ["x"]
self.codomains = ["y1", "y2", "y3"]
self.domain_data = np.linspace(0, 10, 100)
self.codomain_datas = [
np.exp(self.domain_data) / np.exp(self.domain_data[-1]),
np.sin(self.domain_data),
np.cos(2 * self.domain_data),
]
print "USED FAKE TEST DATA!!"
# the data should be normalized -1, 1 on both axes
x = self.domain_data
ys = self.codomain_datas
[plt.plot(x, y) for y in ys]
plt.show()
self.out_data = lfu.data_container(
data=[
lgeo.scalars(label=lab, scalars=dat)
for lab, dat in zip(self.domain + self.codomains, [self.domain_data] + self.codomain_datas)
]
)
lf.save_pkl_object(self.out_data, self.out_filename)
def get_single_well_data(self):
# THIS FUNCTION WILL JUST RETURN THE SELECTED WELLS PLOT DATA
od_dex = self.get_selected_well()
time_ = self.data.data[0]
od = self.data.data[od_dex]
dbl = self._doubling_times_[od_dex]
gro = self._growth_rates_[od_dex]
lo, hi = self._thresholds_[od_dex]
#od.scalars = od.scalars
y_low = lgeo.scalars('__skip__', [lo]*2, color = 'black')
# [self.OD_threshold_low]*2, color = 'black')
y_high = lgeo.scalars('__skip__', [hi]*2, color = 'black')
# [self.OD_threshold_high]*2, color = 'black')
x_stack = lgeo.scalars('__skip__',
[time_.scalars.min(), time_.scalars.max()])
# CALCULATE THE DOUBLING TIME AND ADD VERTICAL LINES FOR THE POINTS USED
# MAKE ALL THESE FLAT LINES DASHED OR SOMETHING
#pdb.set_trace()
data = lfu.data_container(
domains = [time_, x_stack, x_stack],
codomains = [od, y_low, y_high],
doubling = dbl, growth_rate = gro,
thresholds = (lo, hi))
return data
def on_write_data(self, *args, **kwargs):
if not self.domain:
self.domain = ['x']
self.codomains = ['y1', 'y2', 'y3']
self.domain_data = np.linspace(0, 10, 100)
self.codomain_datas = [
np.exp(self.domain_data) / np.exp(self.domain_data[-1]),
np.sin(self.domain_data),
np.cos(2 * self.domain_data)
]
print 'USED FAKE TEST DATA!!'
#the data should be normalized -1, 1 on both axes
x = self.domain_data
ys = self.codomain_datas
[plt.plot(x, y) for y in ys]
plt.show()
self.out_data = lfu.data_container(data=[
lgeo.scalars(label=lab, scalars=dat)
for lab, dat in zip(self.domain +
self.codomains, [self.domain_data] +
self.codomain_datas)
])
lf.save_pkl_object(self.out_data, self.out_filename)
def set_module_memory_(ensem): ensem._module_memory_ = [lfu.data_container( output_plan_selected_memory = 'Simulation', variable_selected_memory = 'None', vector_selected_memory = 'None')]
def set_module_memory_(ensem):
ensem._module_memory_ = [
lfu.data_container(
output_plan_selected_memory="Simulation", variable_selected_memory="None", vector_selected_memory="None"
)
]
def set_module_memory_(ensem): ensem._module_memory_ = [lfu.data_container( output_plan_selected_memory = 'Simulation')]
def __init__(self, *args, **kwargs):
#self.impose_default('current_tab_index', 0, **kwargs)
#self.current_tab_index = 0
self.aborted = False
self.data_pool = lfu.data_container(
data = [], postproc_data = [])
self.data_scheme = None
self.__dict__ = lfu.dictionary()
if 'parent' in kwargs.keys(): self.parent = kwargs['parent']
self.cancel_make = False
self.skip_simulation = False
self.mcfg_path = ''
#self.num_trajectories = 1
num_traj = lset.get_setting('trajectory_count')
if num_traj: self.num_trajectories = num_traj
else: self.num_trajectories = 1
self.data_pool_descr = ''
self.treebook_memory = [0, [], []]
self._module_memory_ = []
self.simulation_plan = simulation_plan(parent = self)
self.output_plan = lo.output_plan(
label = 'Simulation', parent = self)
self.fitting_plan = lfr.fit_routine_plan(parent = self)
self.cartographer_plan = lgeo.cartographer_plan(
label = 'Parameter Scan', parent = self)
self.postprocess_plan = lpp.post_process_plan(
label = 'Post Process Plan', parent = self,
_always_sourceable_ = ['simulation'])
self.multiprocess_plan = lmp.multiprocess_plan(parent = self)
self.run_params = lfu.dictionary(parent = self)
self.run_params['end_criteria'] = \
self.simulation_plan.end_criteria
self.run_params['capture_criteria'] = \
self.simulation_plan.capture_criteria
self.run_params['plot_targets'] = \
self.simulation_plan.plot_targets
self.run_params['output_plans'] = {
'Simulation' : self.output_plan}
self.run_params['fit_routines'] = \
self.fitting_plan.routines
self.run_params['post_processes'] = \
self.postprocess_plan.post_processes
self.run_params['p_space_map'] = None
self.run_params['multiprocessing'] = None
self.__dict__.create_partition('template owners',
['output_plan', 'fitting_plan', 'cartographer_plan',
'postprocess_plan', 'multiprocess_plan', 'run_params',
'simulation_plan'])
if 'label' not in kwargs.keys(): kwargs['label'] = 'ensemble'
if 'module_options' in kwargs.keys():
opts = kwargs['module_options']
else:
print 'no modules detected; requesting from manager'
opts = self.parent.find_module_options()
if len(opts) == 0:
lgd.message_dialog(None,
'No module options detected!', 'Problem')
self.cancel_make = True
return
elif len(opts) == 1: module = opts[0]
else:
module_choice_container = lfu.data_container(data = opts[0])
module_options_templates = [lgm.interface_template_gui(
layout = 'horizontal',
widgets = ['radio'],
verbosities = [0],
labels = [opts],
initials = [[module_choice_container.data]],
instances = [[module_choice_container]],
keys = [['data']],
box_labels = ['Ensemble Module'],
minimum_sizes = [[(250, 50)]])]
mod_dlg = lgd.create_dialog(title = 'Choose Ensemble Module',
templates = module_options_templates, variety = 'templated')
module = mod_dlg()
if module: module = module_choice_container.data
else:
self.cancel_make = True
return
self.impose_default('module', module, **kwargs)
self._children_ = [self.simulation_plan, self.output_plan,
self.fitting_plan, self.cartographer_plan,
self.postprocess_plan, self.multiprocess_plan]
self.load_module(reset_params = True)
self.mcfg_dir = os.path.join(os.getcwd(), self.module)
if not os.path.isdir(self.mcfg_dir): self.mcfg_dir = os.getcwd()
lfu.modular_object_qt.__init__(self, *args, **kwargs)
self.data_pool_id = lfu.get_new_pool_id()
self.data_pool_pkl = os.path.join(os.getcwd(), 'data_pools',
'.'.join(['data_pool', self.data_pool_id, 'pkl']))
def enact_plan(self, *args):
system = args[0]
proper_targets = self.find_proper_targets()
to_be_outted = []
if not self.flat_data: #if the list of data objects is not flat (system.data is the list)
#self.to_be_outted has a 3rd element pointing to system within non-flat pool
#system will be an object with attribute .data but .data is a non-flat list!
#put each data list into a flat list of objects with flat lists for data attributes
#targs = self.get_target_labels()
for traj in system.data:
data_container = lfu.data_container(data=traj)
self.update_filenames()
proper_paths = self.find_proper_paths()
if self.output_vtk:
to_be_outted.append(
(proper_paths['vtk'], 0, data_container))
if self.output_pkl:
to_be_outted.append(
(proper_paths['pkl'], 1, data_container))
if self.output_txt:
to_be_outted.append(
(proper_paths['txt'], 2, data_container))
if self.output_plt:
to_be_outted.append(
(proper_paths['plt'], 3, data_container))
if 3 in [out[1] for out in to_be_outted]:
self.writers[3].get_plt_window()
[
self.writers[out[1]](out[2], out[0], proper_targets[out[1]])
for out in to_be_outted
]
if 3 in [out[1] for out in to_be_outted]:
self.writers[3].plt_window()
else:
self.update_filenames()
proper_paths = self.find_proper_paths()
#if the list of data objects is flat (system.data is the list)
#self.to_be_outted has only 2 elements since data is already flat
if self.output_vtk:
to_be_outted.append((proper_paths['vtk'], 0))
if self.output_pkl:
to_be_outted.append((proper_paths['pkl'], 1))
if self.output_txt:
to_be_outted.append((proper_paths['txt'], 2))
if self.output_plt:
to_be_outted.append((proper_paths['plt'], 3))
if 3 in [out[1] for out in to_be_outted]:
self.writers[3].get_plt_window()
[
self.writers[out[1]](system, out[0], proper_targets[out[1]])
for out in to_be_outted
]
if 3 in [out[1] for out in to_be_outted]:
self.writers[3].plt_window()
