class config(HasTraits):
uuid = traits.Str(desc="UUID")
# Directories
working_dir = Directory(mandatory=True,
desc="Location of the Nipype working directory")
base_dir = Directory(
os.path.abspath('.'),
mandatory=True,
desc='Base directory of data. (Should be subject-independent)')
sink_dir = Directory(mandatory=True,
desc="Location where the BIP will store the results")
crash_dir = Directory(mandatory=False,
desc="Location to store crash files")
surf_dir = Directory(
desc="freesurfer directory. subject id's should be the same")
save_script_only = traits.Bool(False)
# Execution
run_using_plugin = Bool(
False,
usedefault=True,
desc="True to run pipeline with plugin, False to run serially")
plugin = traits.Enum("PBS",
"MultiProc",
"SGE",
"Condor",
usedefault=True,
desc="plugin to use, if run_using_plugin=True")
plugin_args = traits.Dict({"qsub_args": "-q many"},
usedefault=True,
desc='Plugin arguments.')
test_mode = Bool(
False,
mandatory=False,
usedefault=True,
desc='Affects whether where and if the workflow keeps its \
intermediary files. True to keep intermediary files. '
)
timeout = traits.Float(30.0)
# DataGrabber
datagrabber = traits.Instance(Data, ())
# segstats
use_reg = traits.Bool(True)
inverse_reg = traits.Bool(True)
use_standard_label = traits.Bool(
False, desc="use same label file for all subjects")
label_file = traits.File()
use_annotation = traits.Bool(
False,
desc=
"use same annotation file for all subjects (will warp to subject space"
)
use_subject_annotation = traits.Bool(
False,
desc="you need to change datragrabber to\
have outputs lh_annotation and rh_annotation"
)
annot_space = traits.String("fsaverage5",
desc="subject space of annot file")
lh_annotation = traits.File()
rh_annotation = traits.File()
color_table_file = traits.Enum("Default", "Color_Table", "GCA_color_table",
"None")
color_file = traits.File()
proj = traits.BaseTuple(("frac", 0, 1, 0.1), traits.Enum("abs", "frac"),
traits.Float(), traits.Float(), traits.Float())
statname = traits.Str('segstats1', desc="description of the segstat")
class Component(t.HasTraits):
__axes_manager = None
active = t.Property(t.CBool(True))
name = t.Property(t.Str(''))
def __init__(self, parameter_name_list):
self.events = Events()
self.events.active_changed = Event("""
Event that triggers when the `Component.active` changes.
The event triggers after the internal state of the `Component` has
been updated.
Arguments
---------
obj : Component
The `Component` that the event belongs to
active : bool
The new active state
""",
arguments=["obj", 'active'])
self.parameters = []
self.init_parameters(parameter_name_list)
self._update_free_parameters()
self.active = True
self._active_array = None
self.isbackground = False
self.convolved = True
self.parameters = tuple(self.parameters)
self._id_name = self.__class__.__name__
self._id_version = '1.0'
self._position = None
self.model = None
self.name = ''
self._whitelist = {
'_id_name': None,
'name': None,
'active_is_multidimensional': None,
'_active_array': None,
'active': None
}
self._slicing_whitelist = {'_active_array': 'inav'}
self._slicing_order = (
'active',
'active_is_multidimensional',
'_active_array',
)
_name = ''
_active_is_multidimensional = False
_active = True
@property
def active_is_multidimensional(self):
return self._active_is_multidimensional
@active_is_multidimensional.setter
def active_is_multidimensional(self, value):
if not isinstance(value, bool):
raise ValueError('Only boolean values are permitted')
if value == self.active_is_multidimensional:
return
if value: # Turn on
if self._axes_manager.navigation_size < 2:
_logger.info('`navigation_size` < 2, skipping')
return
# Store value at current position
self._create_active_array()
self._store_active_value_in_array(self._active)
self._active_is_multidimensional = True
else: # Turn off
# Get the value at the current position before switching it off
self._active = self.active
self._active_array = None
self._active_is_multidimensional = False
def _get_name(self):
return self._name
def _set_name(self, value):
old_value = self._name
if old_value == value:
return
if self.model:
for component in self.model:
if value == component.name:
raise ValueError("Another component already has "
"the name " + str(value))
self._name = value
setattr(self.model.components,
slugify(value, valid_variable_name=True), self)
self.model.components.__delattr__(
slugify(old_value, valid_variable_name=True))
else:
self._name = value
self.trait_property_changed('name', old_value, self._name)
@property
def _axes_manager(self):
return self.__axes_manager
@_axes_manager.setter
def _axes_manager(self, value):
for parameter in self.parameters:
parameter._axes_manager = value
self.__axes_manager = value
def _get_active(self):
if self.active_is_multidimensional is True:
# The following should set
self.active = self._active_array[self._axes_manager.indices[::-1]]
return self._active
def _store_active_value_in_array(self, value):
self._active_array[self._axes_manager.indices[::-1]] = value
def _set_active(self, arg):
if self._active == arg:
return
old_value = self._active
self._active = arg
if self.active_is_multidimensional is True:
self._store_active_value_in_array(arg)
self.events.active_changed.trigger(active=self._active, obj=self)
self.trait_property_changed('active', old_value, self._active)
def init_parameters(self, parameter_name_list):
for name in parameter_name_list:
parameter = Parameter()
self.parameters.append(parameter)
parameter.name = name
parameter._id_name = name
setattr(self, name, parameter)
if hasattr(self, 'grad_' + name):
parameter.grad = getattr(self, 'grad_' + name)
parameter.component = self
self.add_trait(name, t.Instance(Parameter))
def _get_long_description(self):
if self.name:
text = '%s (%s component)' % (self.name, self._id_name)
else:
text = '%s component' % self._id_name
return text
def _get_short_description(self):
text = ''
if self.name:
text += self.name
else:
text += self._id_name
text += ' component'
return text
def __repr__(self):
text = '<%s>' % self._get_long_description()
return text
def _update_free_parameters(self):
self.free_parameters = sorted(
[par for par in self.parameters if par.free], key=lambda x: x.name)
self._nfree_param = sum(
[par._number_of_elements for par in self.free_parameters])
def update_number_parameters(self):
i = 0
for parameter in self.parameters:
i += parameter._number_of_elements
self.nparam = i
self._update_free_parameters()
def fetch_values_from_array(self, p, p_std=None, onlyfree=False):
if onlyfree is True:
parameters = self.free_parameters
else:
parameters = self.parameters
i = 0
for parameter in sorted(parameters, key=lambda x: x.name):
length = parameter._number_of_elements
parameter.value = (p[i] if length == 1 else p[i:i + length])
if p_std is not None:
parameter.std = (p_std[i] if length == 1 else tuple(
p_std[i:i + length]))
i += length
def _create_active_array(self):
shape = self._axes_manager._navigation_shape_in_array
if len(shape) == 1 and shape[0] == 0:
shape = [
1,
]
if (not isinstance(self._active_array, np.ndarray)
or self._active_array.shape != shape):
_logger.debug('Creating _active_array for {}.\n\tCurrent array '
'is:\n{}'.format(self, self._active_array))
self._active_array = np.ones(shape, dtype=bool)
def _create_arrays(self):
if self.active_is_multidimensional:
self._create_active_array()
for parameter in self.parameters:
parameter._create_array()
def store_current_parameters_in_map(self):
for parameter in self.parameters:
parameter.store_current_value_in_array()
def fetch_stored_values(self, only_fixed=False):
if self.active_is_multidimensional:
# Store the stored value in self._active and trigger the connected
# functions.
self.active = self.active
if only_fixed is True:
parameters = (set(self.parameters) - set(self.free_parameters))
else:
parameters = self.parameters
parameters = [
parameter for parameter in parameters
if (parameter.twin is None
or not isinstance(parameter.twin, Parameter))
]
for parameter in parameters:
parameter.fetch()
def plot(self, only_free=True):
"""Plot the value of the parameters of the model
Parameters
----------
only_free : bool
If True, only the value of the parameters that are free will
be plotted
"""
if only_free:
parameters = self.free_parameters
else:
parameters = self.parameters
parameters = [k for k in parameters if k.twin is None]
for parameter in parameters:
parameter.plot()
def export(self,
folder=None,
format="hspy",
save_std=False,
only_free=True):
"""Plot the value of the parameters of the model
Parameters
----------
folder : str or None
The path to the folder where the file will be saved. If
`None` the
current folder is used by default.
format : str
The extension of the file format, default "hspy".
save_std : bool
If True, also the standard deviation will be saved.
only_free : bool
If True, only the value of the parameters that are free will
be
exported.
Notes
-----
The name of the files will be determined by each the Component
and
each Parameter name attributes. Therefore, it is possible to
customise
the file names modify the name attributes.
"""
if only_free:
parameters = self.free_parameters
else:
parameters = self.parameters
parameters = [k for k in parameters if k.twin is None]
for parameter in parameters:
parameter.export(
folder=folder,
format=format,
save_std=save_std,
)
def summary(self):
for parameter in self.parameters:
dim = len(parameter.map.squeeze().shape) if parameter.map \
is not None else 0
if parameter.twin is None:
if dim <= 1:
print('%s = %s ± %s %s' % (parameter.name, parameter.value,
parameter.std, parameter.units))
def __call__(self):
"""Returns the corresponding model for the current coordinates
Returns
-------
numpy array
"""
axis = self.model.axis.axis[self.model.channel_switches]
component_array = self.function(axis)
return component_array
def _component2plot(self, axes_manager, out_of_range2nans=True):
old_axes_manager = None
if axes_manager is not self.model.axes_manager:
old_axes_manager = self.model.axes_manager
self.model.axes_manager = axes_manager
self.fetch_stored_values()
s = self.__call__()
if not self.active:
s.fill(np.nan)
if self.model.signal.metadata.Signal.binned is True:
s *= self.model.signal.axes_manager.signal_axes[0].scale
if old_axes_manager is not None:
self.model.axes_manager = old_axes_manager
self.charge()
if out_of_range2nans is True:
ns = np.empty(self.model.axis.axis.shape)
ns.fill(np.nan)
ns[self.model.channel_switches] = s
s = ns
if old_axes_manager is not None:
self.model.axes_manager = old_axes_manager
self.fetch_stored_values()
return s
def set_parameters_free(self, parameter_name_list=None):
"""
Sets parameters in a component to free.
Parameters
----------
parameter_name_list : None or list of strings, optional
If None, will set all the parameters to free.
If list of strings, will set all the parameters with the same name
as the strings in parameter_name_list to free.
Examples
--------
>>> v1 = hs.model.components1D.Voigt()
>>> v1.set_parameters_free()
>>> v1.set_parameters_free(parameter_name_list=['area','centre'])
See also
--------
set_parameters_not_free
hyperspy.model.BaseModel.set_parameters_free
hyperspy.model.BaseModel.set_parameters_not_free
"""
parameter_list = []
if not parameter_name_list:
parameter_list = self.parameters
else:
for _parameter in self.parameters:
if _parameter.name in parameter_name_list:
parameter_list.append(_parameter)
for _parameter in parameter_list:
_parameter.free = True
def set_parameters_not_free(self, parameter_name_list=None):
"""
Sets parameters in a component to not free.
Parameters
----------
parameter_name_list : None or list of strings, optional
If None, will set all the parameters to not free.
If list of strings, will set all the parameters with the same name
as the strings in parameter_name_list to not free.
Examples
--------
>>> v1 = hs.model.components1D.Voigt()
>>> v1.set_parameters_not_free()
>>> v1.set_parameters_not_free(parameter_name_list=['area','centre'])
See also
--------
set_parameters_free
hyperspy.model.BaseModel.set_parameters_free
hyperspy.model.BaseModel.set_parameters_not_free
"""
parameter_list = []
if not parameter_name_list:
parameter_list = self.parameters
else:
for _parameter in self.parameters:
if _parameter.name in parameter_name_list:
parameter_list.append(_parameter)
for _parameter in parameter_list:
_parameter.free = False
def _estimate_parameters(self, signal):
if self._axes_manager != signal.axes_manager:
self._axes_manager = signal.axes_manager
self._create_arrays()
def as_dictionary(self, fullcopy=True):
"""Returns component as a dictionary
For more information on method and conventions, see
:meth:`hyperspy.misc.export_dictionary.export_to_dictionary`
Parameters
----------
fullcopy : Bool (optional, False)
Copies of objects are stored, not references. If any found,
functions will be pickled and signals converted to dictionaries
Returns
-------
dic : dictionary
A dictionary, containing at least the following fields:
parameters : list
a list of dictionaries of the parameters, one per
_whitelist : dictionary
a dictionary with keys used as references saved attributes, for
more information, see
:meth:`hyperspy.misc.export_dictionary.export_to_dictionary`
* any field from _whitelist.keys() *
"""
dic = {
'parameters': [p.as_dictionary(fullcopy) for p in self.parameters]
}
export_to_dictionary(self, self._whitelist, dic, fullcopy)
return dic
def _load_dictionary(self, dic):
"""Load data from dictionary.
Parameters
----------
dict : dictionary
A dictionary containing following items:
_id_name : string
_id_name of the original component, used to create the
dictionary. Has to match with the self._id_name
parameters : list
A list of dictionaries, one per parameter of the component (see
parameter.as_dictionary() documentation for more)
_whitelist : dictionary
a dictionary, which keys are used as keywords to match with the
component attributes. For more information see
:meth:`hyperspy.misc.export_dictionary.load_from_dictionary`
* any field from _whitelist.keys() *
Returns
-------
twin_dict : dictionary
Dictionary of 'id' values from input dictionary as keys with all of
the parameters of the component, to be later used for setting up
correct twins.
"""
if dic['_id_name'] == self._id_name:
load_from_dictionary(self, dic)
id_dict = {}
for p in dic['parameters']:
idname = p['_id_name']
if hasattr(self, idname):
par = getattr(self, idname)
t_id = par._load_dictionary(p)
id_dict[t_id] = par
else:
raise ValueError(
"_id_name of parameters in component and dictionary do not match"
)
return id_dict
else:
raise ValueError(
"_id_name of component and dictionary do not match, \ncomponent._id_name = %s\
\ndictionary['_id_name'] = %s" %
(self._id_name, dic['_id_name']))
class Smoothing(t.HasTraits):
# The following is disabled because as of traits 4.6 the Color trait
# imports traitsui (!)
# try:
# line_color = t.Color("blue")
# except ModuleNotFoundError:
# # traitsui is required to define this trait so it is not defined when
# # traitsui is not installed.
# pass
line_color_ipy = t.Str("blue")
differential_order = t.Int(0)
@property
def line_color_rgb(self):
if hasattr(self, "line_color"):
try:
# PyQt and WX
return np.array(self.line_color.Get()) / 255.
except AttributeError:
try:
# PySide
return np.array(self.line_color.getRgb()) / 255.
except BaseException:
return matplotlib.colors.to_rgb(self.line_color_ipy)
else:
return matplotlib.colors.to_rgb(self.line_color_ipy)
def __init__(self, signal):
self.ax = None
self.data_line = None
self.smooth_line = None
self.signal = signal
self.single_spectrum = self.signal.get_current_signal().deepcopy()
self.axis = self.signal.axes_manager.signal_axes[0].axis
self.plot()
def plot(self):
if self.signal._plot is None or not self.signal._plot.is_active:
self.signal.plot()
hse = self.signal._plot
l1 = hse.signal_plot.ax_lines[0]
self.original_color = l1.line.get_color()
l1.set_line_properties(color=self.original_color, type='scatter')
l2 = drawing.signal1d.Signal1DLine()
l2.data_function = self.model2plot
l2.set_line_properties(color=self.line_color_rgb, type='line')
# Add the line to the figure
hse.signal_plot.add_line(l2)
l2.plot()
self.data_line = l1
self.smooth_line = l2
self.smooth_diff_line = None
def update_lines(self):
self.smooth_line.update()
if self.smooth_diff_line is not None:
self.smooth_diff_line.update()
def turn_diff_line_on(self, diff_order):
self.signal._plot.signal_plot.create_right_axis()
self.smooth_diff_line = drawing.signal1d.Signal1DLine()
self.smooth_diff_line.axes_manager = self.signal.axes_manager
self.smooth_diff_line.data_function = self.diff_model2plot
self.smooth_diff_line.set_line_properties(color=self.line_color_rgb,
type='line')
self.signal._plot.signal_plot.add_line(self.smooth_diff_line,
ax='right')
def _line_color_ipy_changed(self):
if hasattr(self, "line_color"):
self.line_color = str(self.line_color_ipy)
else:
self._line_color_changed(None, None)
def turn_diff_line_off(self):
if self.smooth_diff_line is None:
return
self.smooth_diff_line.close()
self.smooth_diff_line = None
def _differential_order_changed(self, old, new):
if new == 0:
self.turn_diff_line_off()
return
if old == 0:
self.turn_diff_line_on(new)
self.smooth_diff_line.plot()
else:
self.smooth_diff_line.update(force_replot=False)
def _line_color_changed(self, old, new):
self.smooth_line.line_properties = {'color': self.line_color_rgb}
if self.smooth_diff_line is not None:
self.smooth_diff_line.line_properties = {
'color': self.line_color_rgb
}
try:
# it seems that changing the properties can be done before the
# first rendering event, which can cause issue with blitting
self.update_lines()
except AttributeError:
pass
def diff_model2plot(self, axes_manager=None):
smoothed = np.diff(self.model2plot(axes_manager),
self.differential_order)
return smoothed
def close(self):
if self.signal._plot.is_active:
if self.differential_order != 0:
self.turn_diff_line_off()
self.smooth_line.close()
self.data_line.set_line_properties(color=self.original_color,
type='line')
class Data(HasTraits):
fields = traits.List(traits.Instance(DataBase, ()))
base_directory = Directory(os.path.abspath('.'))
template = traits.Str('*')
template_args = traits.Dict({"a": "b"}, usedefault=True)
field_template = traits.Dict({"key": ["hi"]}, usedefault=True)
if use_view:
check = traits.Button("Check")
view = get_view()
def __init__(self, outfields=None):
if outfields:
d_ft = {}
d_ta = {}
for out in outfields:
d_ft[out] = '%s'
d_ta[out] = [['name']]
self.field_template = d_ft
self.template_args = d_ta
self.outfields = outfields
def _get_infields(self):
infields = []
for f in self.fields:
infields.append(f.name)
return infields
def _add_iterable(self, field):
import nipype.interfaces.utility as niu
import nipype.pipeline.engine as pe
it = pe.Node(niu.IdentityInterface(fields=[field.name]),
name=field.name + "_iterable")
it.iterables = (field.name, field.values)
return it
def _set_inputs(self):
self._node_added = False
set_dict = {}
for f in self.fields:
if not f.iterable:
set_dict[f.name] = f.values
else:
it = self._add_iterable(f)
self._node_added = True
self._wk.connect(it, f.name, self._dg, f.name)
self._dg.inputs.trait_set(**set_dict)
def create_dataflow(self):
import nipype.interfaces.io as nio
import nipype.pipeline.engine as pe
self._wk = pe.Workflow(name='custom_datagrabber')
self._dg = pe.Node(nio.DataGrabber(outfields=self.outfields,
infields=self._get_infields()),
name='datagrabber')
self._set_inputs()
self._dg.inputs.base_directory = self.base_directory
self._dg.inputs.field_template = self.field_template
self._dg.inputs.template_args = self.template_args
self._dg.inputs.template = self.template
if not self._node_added:
self._wk.add_nodes([self._dg])
return self._wk
def get_fields(self):
foo = self.get()
d = {}
for key, item in foo.iteritems():
if not key.startswith('_'):
if isinstance(item, list):
d[key] = []
for it in item:
if isinstance(it, DataBase):
d[key].append(it.get())
else:
d[key].append(it)
else:
d[key] = item
return d
def set_fields(self, d):
for key, item in d.iteritems():
if not key == "fields":
self.set(**{key: item})
else:
foo = []
for f in item:
tmp = DataBase()
tmp.set(**f)
foo.append(tmp)
self.set(**{key: foo})
def _check_fired(self):
dg = self.create_dataflow()
dg.run()
def configure_controller(cls):
c = Controller()
c.add_trait('param_type', traits.Str('Str'))
return c
class DACChannel(traits.HasTraits):
def __init__(self, setchannelName, port, connection, rpiADC):
self.channelName = setchannelName
self.x = ad.ADEvalBC()
self.port = port
self.connection = connection
self.rpiADC = rpiADC
self.wmLockTimer = Timer(1000, self.wmLock2)
self.wmLockTimer.Stop()
# time.sleep(5)
# self.start_timer()
update = traits.Button()
set = traits.Button()
pinMode = '0'
relockMode = traits.Enum("Manual Mode", "Doubling cavity Relock",
"Wavemeter Relock", "Wavemeter Lock")
#set_Relock = traits.Button()
#---- MANUAL MODE ----#
voltage = traits.Float(desc="Voltage of the Channel")
setVoltage = traits.Float(desc="set Voltage")
powerModeMult = 0
channelName = traits.Str(desc="Name")
channelDescription = traits.Str(desc="Description")
powerMode = traits.Enum(
5,
10,
10.8,
desc="power Mode",
)
bipolar = traits.Bool(desc="Bipolarity")
channelMessage = traits.Str()
bytecode = traits.Str()
port = traits.Str()
def pinSet(self, channel, mode):
cmd = "pin=" + channel + mode
self.connection.send(cmd)
#print cmd
def _update_fired(self):
if self.bipolar == True:
a = "bipolar"
bip = True
self.powerModeMult = -1
else:
a = "unipolar"
bip = False
self.powerModeMult = 0
cmd = "cmd=" + self.x.setMaxValue(self.port, self.powerMode, bip)
#print cmd
self.connection.send(cmd)
b = "Mode set to %.1f" % self.powerMode
self.channelMessage = b + ' ' + a
self._set_fired()
def _set_fired(self):
if ((self.setVoltage > self.powerMode) and (self.bipolar == False)):
print "setVoltage out of bounds. Not sending."
elif ((abs(self.setVoltage) > self.powerMode)
and (self.bipolar == True)):
print "setVoltage out of bounds. Not sending."
else:
cmd = "cmd=" + self.x.generate_voltage(
self.port, self.powerMode, self.powerModeMult * self.powerMode,
self.setVoltage)
self.connection.send(cmd)
self.bytecode = self.x.generate_voltage(
self.port, self.powerMode, self.powerModeMult * self.powerMode,
self.setVoltage)
self.voltage = self.setVoltage
#---- MANUAL MODE GUI ----#
voltageGroup = traitsui.HGroup(traitsui.VGroup(
traitsui.Item('voltage',
label="Measured Voltage",
style_sheet='* { font-size: 18px; }',
style="readonly"),
traitsui.Item('setVoltage', label="Set Value"),
),
traitsui.Item('set', show_label=False),
show_border=True)
powerGroup = traitsui.VGroup(traitsui.HGroup(
traitsui.Item('powerMode', label="Power Mode"),
traitsui.Item('bipolar'),
),
traitsui.HGroup(
traitsui.Item('update', show_label=False),
traitsui.Item('channelMessage',
show_label=False,
style="readonly"),
),
traitsui.Item('bytecode',
show_label=False,
style="readonly"),
traitsui.Item('switch_Lock'),
show_border=True)
manualGroup = traitsui.VGroup(traitsui.Item('channelName',
label="Channel Name",
style="readonly"),
voltageGroup,
show_border=True,
visible_when='relockMode == "Manual Mode"')
#---- DOUBLING CAVITY MODE GUI----#
adcChannel = traits.Enum(0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
desc="Channel of the rpiADC")
adcVoltage = traits.Float(desc="Voltage on rpiADC Channel")
DCscan_and_lock = traits.Button()
switch_Lock = traits.Button()
DCconnect = traits.Bool()
DCautolock = traits.Button()
DCadcVoltages = None
DCadcVoltagesMean = None
DCtolerance = 0.01 #Volt
DCmistakeCounter = 0
DCminBoundary = traits.Float()
DCmaxBoundary = traits.Float()
DCnoOfSteps = traits.Int()
#Updates voltage of the selected channel"
def _adcVoltage_update(self):
self.adcVoltage = self._adcVoltage_get()
#Gets voltage of the selected channel via rpiADC Client
def _adcVoltage_get(self):
return self.rpiADC.getResults()[self.adcChannel]
# print "latest results = %s " % self.rpiADC.latestResults
# self.rpiADC.getResults()
# print "latest results = %s " % self.rpiADC.latestResults
# if self.adcChannel in self.rpiADC.latestResults:
# return self.rpiADC.latestResults[self.adcChannel]
# else:
# return -999
#As soon as the connect button is checked, automatic updating of the adc voltage is initiated.
#When it is unchecked, the update stops
def _DCconnect_changed(self):
if self.DCconnect == True:
self._start_PD_timer()
else:
self.PDtimer.stop()
self.adcVoltage = -999
print "PD timer stopped."
#Starts the timer, that only updates the displayed voltage
def _start_PD_timer(self):
self.PDtimer = Timer(1000.0, self._adcVoltage_update)
#Starts a timer, that updates the displayed voltage, as well as does the "in lock" checking
def _start_PD_lock_timer(self):
self.PDtimer = Timer(1000.0, self.update_PD_and_Lock)
#Controls if everything is still in lock. Also updates displayed voltage. It counts still in lock, when the measured frequency
#is within DC tolerance of the mean of the last five measured frequencies.
def update_PD_and_Lock(self):
self._adcVoltage_update() #still display Voltage
pdVoltage = self._adcVoltage_get()
#print "Updated Frequency"
#mistakeCounter = 0
if len(self.DCadcVoltages
) < 5: #number of Measurements that will be compared
print "Getting Data for Lock. Do not unlock!"
self.DCadcVoltages = np.append(self.DCadcVoltages, pdVoltage)
else:
self.DCadcVoltagesMean = np.mean(
self.DCadcVoltages) #Mean Frequency to compare to
if (abs(pdVoltage - self.DCadcVoltagesMean) < self.DCtolerance):
self.DCadcVoltages = np.append(self.DCadcVoltages, pdVoltage)
self.DCadcVoltages = np.delete(
self.DCadcVoltages, 0) #keep Array at constant length
print "Still locked."
if self.DCmistakeCounter > 0:
self.DCmistakeCounter = 0
else:
self.DCmistakeCounter += 1
if self.DCmistakeCounter > 5:
self.PDtimer.stop()
self._start_PD_timer() #keep Frequency on display..
self._DCscan_and_lock_fired()
self._DCautolock_fired()
else:
print self.DCmistakeCounter
#This button is used, when everything is already locked. It prepares the voltage mean array, stops the PD timer and starts the update_PD_and_Lock timer routine
def _DCautolock_fired(self):
if self.DCconnect == True:
self.DCadcVoltages = np.array([])
self.PDtimer.stop()
self._start_PD_lock_timer()
else:
print "No adcRPi connected."
#This function (button) scans the voltage and reads the RPiADC voltage of the selected channel. It subsequently attempts
#to lock the Cavity at the voltage, where the RPiADC voltage is highest. The Algorithm for the lock is as follows:
#1) It does a coarse DAC voltage scan with the parameters selected in the input (minBoundary etc). This scan is terminated when
# the adc voltage goes above a threshold (3.2 V - hardcoded, maybe add input), or after scanning the whole range.
#2) A fine scan 0.3V(dac) around the maximum (or around the 3.2V dac voltage) is done. Currently the number of steps is hardcoded to 600,
# which worked well. This scan terminated either by going to a 3.3V (adc) threshhold or after finishing.
#3) The Cavity is locked at a dac voltage that corresponds to either the maximum after the whole scan or the 3.3V threshold.
def _DCscan_and_lock_fired(self):
self.pinSet(self.port, '1') #Unlock
time.sleep(
1
) #If the cavity was in lock before we can not directly start scanning, or else it will read 3.3V as first value
voltages = np.linspace(
self.DCminBoundary, self.DCmaxBoundary,
self.DCnoOfSteps) #Parameters for first scan set by GUI
diodeVoltages = np.array([])
for entry in voltages: #First scan
self.setVoltage = entry
self._set_fired() #update setVoltage
diodeVoltages = np.append(diodeVoltages, self._adcVoltage_get())
volt = self._adcVoltage_get()
print volt
if volt >= 3.2: #Threshold for first scan
break
time.sleep(
0.05
) #sleep time between every step for adc readout, maybe it is possible to make the scan faster
self.setVoltage = voltages[diodeVoltages.argmax(
axis=0)] #DAC Voltage corresponding to the highest adc voltage
print "Attempting to reduce scan Range"
print self.setVoltage
time.sleep(2.0)
if self.setVoltage < 0.3: #make sure we do not scan below zero, as the lock box does not like negative voltages.
auxSetVolt = 0
else:
auxSetVolt = self.setVoltage - 0.3
voltages = np.linspace(auxSetVolt, self.setVoltage + 0.3,
600) #parameters for second scan
diodeVoltages = np.array([])
for entry in voltages: #Second scan
if entry > self.powerMode: #Our voltage can not go above our maximum value
break
self.setVoltage = entry
self._set_fired()
diodeVoltages = np.append(diodeVoltages, self._adcVoltage_get())
volt = self._adcVoltage_get()
print volt
if volt >= 3.3: #threshold for second scan
print self.setVoltage
self.pinSet(self.port, '0')
return
time.sleep(0.1)
self.setVoltage = voltages[diodeVoltages.argmax(axis=0)]
print "DAC Voltage set to %f" % voltages[diodeVoltages.argmax(axis=0)]
print ".. this corresponds to a diode Voltage of %f" % diodeVoltages[
diodeVoltages.argmax(axis=0)]
self._set_fired()
time.sleep(0.2)
self.pinSet(self.port, '0') #Lock
print "Voltage set to %f to attempt relock." % self.setVoltage
return
#Changes from lock to dither and other way around
def _switch_Lock_fired(self):
if self.pinMode == '0':
self.pinMode = '1'
self.pinSet(self.port, self.pinMode)
else:
self.pinMode = '0'
self.pinSet(self.port, self.pinMode)
#Gui Stuff#
DCgroup = traitsui.VGroup(
traitsui.HGroup(
traitsui.VGroup(
traitsui.Item('adcChannel'),
traitsui.Item('adcVoltage', style='readonly'),
),
traitsui.VGroup(
traitsui.Item('DCmaxBoundary'),
traitsui.Item('DCminBoundary'),
traitsui.Item('DCnoOfSteps'),
),
),
#traitsui.Item('switch_Lock'),
traitsui.HGroup(
traitsui.VGroup(
traitsui.Item('DCscan_and_lock'),
traitsui.Item('DCautolock'),
), traitsui.Item('DCconnect')),
visible_when='relockMode == "Doubling cavity Relock"',
show_border=True,
)
#---- WAVEMETER GUI ----#
wavemeter = traits.Enum("Humphry", "Ion Cavity")
wmChannel = traits.Enum(1, 2, 3, 4, 5, 6, 7, 8)
wmFrequency = traits.Float()
wmConnected = traits.Bool()
wmHWI = None
wmIP = None
wmPort = None
wmReLock = traits.Button()
wmFrequencyLog = np.array([])
wmMeanFrequency = None
wmTolerance = 0.0000006 #Frequency tolerance in THz, set to 60 MHz in accordance with wm accuracy
mistakeCounter = 0
#wmPolarity = traits.Enum("+", "-")
wmEmptyMemory = traits.Button()
#When hitting wmConnected, a connection to the wavemeter and readout is established
def _wmConnected_changed(self):
if self.wmConnected == True:
if self.wavemeter == "Humphry":
self.wmIP = '192.168.0.111'
self.wmPort = 6101
elif self.wavemeter == "Ion Cavity":
self.wmIP = '192.168.32.2'
self.wmPort = 6100
self.wmHWI = PyHWI.DECADSClientConnection('WLM', self.wmIP,
self.wmPort)
#frequencyArray = wmHWI.getFrequency(True, self.wmChannel)
self.start_timer()
else:
self.wmFrequency = -999
self.timer.Stop() #stops either lock_timer or read_timer.
print "Timer stopped"
#GUI stuff
wmGroup = traitsui.VGroup(traitsui.Item('wmFrequency', style='readonly'),
traitsui.Item('wmConnected'),
traitsui.Item('wmReLock'),
traitsui.HGroup(
traitsui.Item('wavemeter'),
traitsui.Item('wmChannel'),
traitsui.Item('wmEmptyMemory',
show_label=False)),
visible_when='relockMode == "Wavemeter Relock"')
#Resets the memory of the lock (the array which saves the last read frequencies)
def _wmEmptyMemory_fired(self):
self.wmFrequencyLog = np.array([])
print "Memory empty."
#starts readout-only timer
def start_timer(self):
print "Timer started"
self.timer = Timer(1000.0, self.update_wm)
#starts readout-and-lock timer, analogue to the doubling cavity case
def start_lock_timer(self):
print "Lock timer started"
self.timer = Timer(1000.0, self.update_wm_and_lock)
#updates the displayed frequency
def update_wm(self):
frequencyArray = self.wmHWI.getFrequency(True, self.wmChannel)
self.wmFrequency = frequencyArray[1]
#updates frequency and checks if its still near the mean of the last five (hardcoded, see below) measured frequencies. If not, it attempts a relock.
def update_wm_and_lock(self):
self.update_wm()
frequencyArray = self.wmHWI.getFrequency(True, self.wmChannel)
#print "Updated Frequency"
#mistakeCounter = 0
if len(self.wmFrequencyLog
) < 5: #number of Measurements that will be compared
print "Getting Data for Lock. Do not unlock!"
self.wmFrequencyLog = np.append(self.wmFrequencyLog,
frequencyArray[1])
else:
self.wmMeanFrequency = np.mean(
self.wmFrequencyLog) #Mean Frequency to compare to
if (abs(frequencyArray[1] - self.wmMeanFrequency) <
self.wmTolerance):
self.wmFrequencyLog = np.append(self.wmFrequencyLog,
frequencyArray[1])
self.wmFrequencyLog = np.delete(
self.wmFrequencyLog, 0) #keep Array at constant length
print "Still locked."
if self.mistakeCounter > 0:
self.mistakeCounter = 0
else:
self.mistakeCounter += 1
if self.mistakeCounter > 5: #number of measurements that still count as locked, though the frequency is not within boundaries
self.timer.stop()
self.start_timer() #keep Frequency on display..
self.wmRelock(self.wmMeanFrequency)
else:
print self.mistakeCounter
#Relock procedure.
#For now this scans only one time, with a hardcoded number of steps. It might be worth modifying this to look like the doubling cavity relock procedure.
def wmRelock(self, wantedFrequency):
self.pinSet(self.port, '1')
voltages = np.linspace(self.powerModeMult * self.powerMode,
self.powerMode, 10)
wmRelockTry = 0
try:
while (wmRelockTry < 5): #attempt relock five times
for entry in voltages:
self.setVoltage = entry
self._set_fired()
time.sleep(1.0)
frequencyArray = self.wmHWI.getFrequency(
True, self.wmChannel)
if (abs(frequencyArray[1] - wantedFrequency) <
self.wmTolerance):
print "Relock_attempt!"
self.pinSet(self.port, '0')
self._wmLock_fired()
raise GetOutOfLoop #Opens the function again (inductively). Maybe fix that by going back
#to the level above somehow.
wmRelockTry += 1
print "Relock try %f not succesful" % wmRelockTry
print "Was not able to Relock."
except GetOutOfLoop:
print "gotOutOfLoop"
pass
def _wmReLock_fired(self):
#self.wmFrequencyLog = np.array([])
self.mistakeCounter = 0
if self.wmConnected == True:
self.timer.Stop(
) #Switch from read-only timer to read-and-log timer. If both run at the same time
self.start_lock_timer() #we run into timing problems.
else:
print "No Wavemeter connected!"
print "hi"
#---- WAVEMETERLOCK - DE 17092018 GUI ----#
wavemeter = traits.Enum("Humphry", "Ion Cavity")
wmChannel = traits.Enum(1, 2, 3, 4, 5, 6, 7, 8)
wmFrequency = traits.Float()
wmConnected = traits.Bool()
wmHWI = None
wmIP = None
wmPort = None
isRunning = traits.Bool(False)
wmVoltage = 0
wmLockTimer = traits.Instance(Timer)
wmLockStart = traits.Button()
wmLockStop = traits.Button()
wmFrequencyLog = np.array([])
wmMeanFrequency = None
wmTargetFrequency = traits.Float() #frequency to lock
wmGain = traits.Float() #Gain for wavemeterlock
wmTolerance = 0.0000006 #Frequency tolerance in THz, set to 60 MHz in accordance with wm accuracy
mistakeCounter = 0
#wmPolarity = traits.Enum("+", "-")
wmEmptyMemory = traits.Button()
#GUI stuff
wmlockGroup = traitsui.VGroup(
traitsui.HGroup(traitsui.Item('wavemeter'), traitsui.Item('wmChannel'),
traitsui.Item('wmEmptyMemory', show_label=False)),
traitsui.Item('wmFrequency', style='readonly'),
traitsui.Item('wmConnected'),
traitsui.Item('wmTargetFrequency'),
traitsui.Item('wmGain'),
traitsui.HGroup(
traitsui.Item('wmLockStart', visible_when="isRunning == False"),
traitsui.Item('wmLockStop', visible_when="isRunning == True")),
visible_when='relockMode == "Wavemeter Lock"')
def _wmLockStart_fired(self):
print "Start: Wavemeterlock"
self.isRunning = True
self.wmLockTimer.Start()
def wmLock2(self):
# Calculate error in MHz
error = (self.wmFrequency - self.wmTargetFrequency) * 10**3
if abs(error) < 5000:
self.wmVoltage = self.wmVoltage + error * self.wmGain
if self.wmVoltage > 10:
self.wmVoltage = 10
elif self.wmVoltage < -10:
self.wmVoltage = -10
cmd = "cmd=" + self.x.generate_voltage(
self.port, self.powerMode, self.powerModeMult * self.powerMode,
self.wmVoltage)
self.connection.sendwithoutcomment(cmd)
self.bytecode = self.x.generate_voltage(
self.port, self.powerMode, self.powerModeMult * self.powerMode,
self.wmVoltage)
#self.saveToFile( "frequency_data.csv" )
def _wmLockStop_fired(self):
print "Stop: Wavemeterlock"
self.isRunning = False
cmd = "cmd=" + self.x.generate_voltage(
self.port, self.powerMode, self.powerModeMult * self.powerMode,
0) #Spannung wieder auf 0
self.connection.sendwithoutcomment(cmd)
self.bytecode = self.x.generate_voltage(
self.port, self.powerMode, self.powerModeMult * self.powerMode, 0)
self.wmVoltage = 0
self.wmLockTimer.Stop()
def saveToFile(self, fileName):
f = open(fileName, "a")
f.write("%f \n" % self.wmFrequency)
f.close()
#---- PUT TOGETHER CHANNEL ----#
selectionGroup = traitsui.HGroup(traitsui.Item('relockMode'),
#traitsui.Item('set_Relock'),
)
channelGroup = traitsui.VGroup(
selectionGroup,
powerGroup,
wmGroup,
wmlockGroup,
DCgroup,
manualGroup,
)
traits_view = traitsui.View(channelGroup)
class config(HasTraits):
uuid = traits.Str(desc="UUID")
# Directories
working_dir = Directory(mandatory=True,
desc="Location of the Nipype working directory")
base_dir = Directory(
os.path.abspath('.'),
mandatory=True,
desc='Base directory of data. (Should be subject-independent)')
sink_dir = Directory(mandatory=True,
desc="Location where the BIP will store the results")
crash_dir = Directory(mandatory=False,
desc="Location to store crash files")
surf_dir = Directory(
desc="freesurfer directory. subject id's should be the same")
save_script_only = traits.Bool(False)
# Execution
run_using_plugin = Bool(
False,
usedefault=True,
desc="True to run pipeline with plugin, False to run serially")
plugin = traits.Enum("PBS",
"MultiProc",
"SGE",
"Condor",
usedefault=True,
desc="plugin to use, if run_using_plugin=True")
plugin_args = traits.Dict({"qsub_args": "-q many"},
usedefault=True,
desc='Plugin arguments.')
test_mode = Bool(
False,
mandatory=False,
usedefault=True,
desc='Affects whether where and if the workflow keeps its \
intermediary files. True to keep intermediary files. '
)
# Data
datagrabber = traits.Instance(Data, ())
#subject_id = traits.String()
#contrast = traits.File()
#mask_contrast = traits.File()
use_contrast_mask = traits.Bool(True)
#reg_file = traits.File()
#mean_image = traits.File()
background_thresh = traits.Float(0.5)
hemi = traits.List(['lh', 'rh'])
roi = traits.List(
['superiortemporal', 'bankssts'],
traits.Enum('superiortemporal', 'bankssts', 'caudalanteriorcingulate',
'caudalmiddlefrontal', 'corpuscallosum', 'cuneus',
'entorhinal', 'fusiform', 'inferiorparietal',
'inferiortemporal', 'isthmuscingulate', 'lateraloccipital',
'lateralorbitofrontal', 'lingual', 'medialorbitofrontal',
'middletemporal', 'parahippocampal', 'paracentral',
'parsopercularis', 'parsorbitalis', 'parstriangularis',
'pericalcarine', 'postcentral', 'posteriorcingulate',
'precentral', 'precuneus', 'rostralanteriorcingulate',
'rostralmiddlefrontal', 'superiorfrontal',
'superiorparietal', 'supramarginal', 'frontalpole',
'temporalpole', 'transversetemporal', 'insula'),
usedefault=True) #35 freesurfer regions,
thresh = traits.Float(1.5)
class DataAxis(t.HasTraits):
name = t.Str()
units = t.Str()
scale = t.Float()
offset = t.Float()
size = t.CInt()
low_value = t.Float()
high_value = t.Float()
value = t.Range('low_value', 'high_value')
low_index = t.Int(0)
high_index = t.Int()
slice = t.Instance(slice)
navigate = t.Bool(t.Undefined)
index = t.Range('low_index', 'high_index')
axis = t.Array()
continuous_value = t.Bool(False)
def __init__(self,
size,
index_in_array=None,
name=t.Undefined,
scale=1.,
offset=0.,
units=t.Undefined,
navigate=t.Undefined):
super(DataAxis, self).__init__()
self.name = name
self.units = units
self.scale = scale
self.offset = offset
self.size = size
self.high_index = self.size - 1
self.low_index = 0
self.index = 0
self.update_axis()
self.navigate = navigate
self.axes_manager = None
self.on_trait_change(self.update_axis, ['scale', 'offset', 'size'])
self.on_trait_change(self.update_value, 'index')
self.on_trait_change(self.set_index_from_value, 'value')
self.on_trait_change(self._update_slice, 'navigate')
self.on_trait_change(self.update_index_bounds, 'size')
# The slice must be updated even if the default value did not
# change to correctly set its value.
self._update_slice(self.navigate)
@property
def index_in_array(self):
if self.axes_manager is not None:
return self.axes_manager._axes.index(self)
else:
raise AttributeError(
"This DataAxis does not belong to an AxesManager"
" and therefore its index_in_array attribute "
" is not defined")
@property
def index_in_axes_manager(self):
if self.axes_manager is not None:
return self.axes_manager._get_axes_in_natural_order().\
index(self)
else:
raise AttributeError(
"This DataAxis does not belong to an AxesManager"
" and therefore its index_in_array attribute "
" is not defined")
def _get_positive_index(self, index):
if index < 0:
index = self.size + index
if index < 0:
raise IndexError("index out of bounds")
return index
def _get_index(self, value):
if isinstance(value, float):
return self.value2index(value)
else:
return value
def _slice_me(self, slice_):
"""Returns a slice to slice the corresponding data axis and
change the offset and scale of the DataAxis acordingly.
Parameters
----------
slice_ : {float, int, slice}
Returns
-------
my_slice : slice
"""
i2v = self.index2value
v2i = self.value2index
if isinstance(slice_, slice):
start = slice_.start
stop = slice_.stop
step = slice_.step
else:
if isinstance(slice_, float):
start = v2i(slice_)
else:
start = self._get_positive_index(slice_)
stop = start + 1
step = None
if isinstance(step, float):
step = int(round(step / self.scale))
if isinstance(start, float):
try:
start = v2i(start)
except ValueError:
# The value is below the axis limits
# we slice from the start.
start = None
if isinstance(stop, float):
try:
stop = v2i(stop)
except ValueError:
# The value is above the axes limits
# we slice up to the end.
stop = None
if step == 0:
raise ValueError("slice step cannot be zero")
my_slice = slice(start, stop, step)
if start is None:
if step > 0 or step is None:
start = 0
else:
start = self.size - 1
self.offset = i2v(start)
if step is not None:
self.scale *= step
return my_slice
def _get_name(self):
name = (self.name if self.name is not t.Undefined else
("Unnamed " + ordinal(self.index_in_axes_manager)))
return name
def __repr__(self):
text = '<%s axis, size: %i' % (
self._get_name(),
self.size,
)
if self.navigate is True:
text += ", index: %i" % self.index
text += ">"
return text
def __str__(self):
return self._get_name() + " axis"
def connect(self, f, trait='value'):
self.on_trait_change(f, trait)
def disconnect(self, f, trait='value'):
self.on_trait_change(f, trait, remove=True)
def update_index_bounds(self):
self.high_index = self.size - 1
def update_axis(self):
self.axis = generate_axis(self.offset, self.scale, self.size)
if len(self.axis) != 0:
self.low_value, self.high_value = (self.axis.min(),
self.axis.max())
def _update_slice(self, value):
if value is False:
self.slice = slice(None)
else:
self.slice = None
def get_axis_dictionary(self):
adict = {
'name': self.name,
'scale': self.scale,
'offset': self.offset,
'size': self.size,
'units': self.units,
'index_in_array': self.index_in_array,
'navigate': self.navigate
}
return adict
def copy(self):
return DataAxis(**self.get_axis_dictionary())
def update_value(self):
self.value = self.axis[self.index]
def value2index(self, value, rounding=round):
"""Return the closest index to the given value if between the limit.
Parameters
----------
value : float
Returns
-------
int
Raises
------
ValueError if value is out of the axis limits.
"""
if value is None:
return None
else:
index = int(rounding((value - self.offset) / self.scale))
if self.size > index >= 0:
return index
else:
raise ValueError("The value is out of the axis limits")
def index2value(self, index):
return self.axis[index]
def set_index_from_value(self, value):
self.index = self.value2index(value)
# If the value is above the limits we must correct the value
if self.continuous_value is False:
self.value = self.index2value(self.index)
def calibrate(self, value_tuple, index_tuple, modify_calibration=True):
scale = (value_tuple[1] - value_tuple[0]) /\
(index_tuple[1] - index_tuple[0])
offset = value_tuple[0] - scale * index_tuple[0]
if modify_calibration is True:
self.offset = offset
self.scale = scale
else:
return offset, scale
class HistoryEditor(EditorFactory):
"""
Progress bar running between 0 and 1 by default
"""
label = 'history'
var = tr.Str('t')
min_value = tr.Float(0)
max_value = tr.Float(1)
step = tr.Float(0.01)
min_var = tr.Str('')
max_var = tr.Str('')
step = tr.Str('')
tooltip = tr.Property(depends_on='time_var, time_max_var')
@tr.cached_property
def _get_tooltip(self):
return 'history slider 0 -> %s -> %s' % (self.var, self.max_var)
t_min = tr.Property
def _get_t_min(self):
if self.min_var == '':
t_min = self.min_value
else:
t_min = getattr(self.model, str(self.min_var))
return t_min
t_max = tr.Property
def _get_t_max(self):
if self.max_var == '':
t_max = self.max_value
else:
t_max = getattr(self.model, str(self.max_var))
return t_max
step = tr.Property
def _get_step(self):
if self.step == '':
step = self.step
else:
step = getattr(self.model, str(self.step))
return step
def render(self):
history_bar_widgets = []
eta = (getattr(self.model, str(self.var)) - self.t_min) / (self.t_max - self.t_min)
history_slider = ipw.FloatSlider(
value=eta,
min=0,
max=1,
step=0.01,
tooltip=self.tooltip,
continuous_update=False,
description=self.label,
disabled=self.disabled,
# readout=self.readout,
# readout_format=self.readout_format
layout = ipw.Layout(display='flex', width="100%")
)
def change_time_var(event):
eta = event['new']
# with bu.print_output:
# print('slider on',self.model)
t = self.t_min + (self.t_max - self.t_min) * eta
setattr(self.model, self.var, t)
app_window = self.controller.app_window
app_window.update_plot(self.model)
history_slider.observe(change_time_var,'value')
# if self.min_var != '':
# def change_t_min(event):
# t_min = event.new
# history_slider.min = t_min
# self.model.observe(change_t_min, self.min_var)
#
# if self.max_var != '':
# def change_t_max(event):
# t_max = event.new
# history_slider.max = t_max
# self.model.observe(change_t_max, self.max_var)
history_bar_widgets.append(history_slider)
history_box = ipw.HBox(history_bar_widgets,
layout=ipw.Layout(padding='0px'))
history_box.layout.align_items = 'center'
return history_box
class EnsembleTrainer(t.HasStrictTraits):
def __init__(self, config={}, **kwargs):
trainer_template = Trainer(**config)
super().__init__(trainer_template=trainer_template,
config=config,
**kwargs)
config: dict = t.Dict()
trainer_template: Trainer = t.Instance(Trainer)
trainers: ty.List[Trainer] = t.List(t.Instance(Trainer))
n_folds = t.Int(5)
dl_test: DataLoader = t.DelegatesTo("trainer_template")
data_spec: dict = t.DelegatesTo("trainer_template")
cuda: bool = t.DelegatesTo("trainer_template")
device: str = t.DelegatesTo("trainer_template")
loss_func: str = t.DelegatesTo("trainer_template")
batch_size: int = t.DelegatesTo("trainer_template")
win_len: int = t.DelegatesTo("trainer_template")
has_null_class: bool = t.DelegatesTo("trainer_template")
predict_null_class: bool = t.DelegatesTo("trainer_template")
name: str = t.Str()
def _name_default(self):
import time
modelstr = "Ensemble"
timestr = time.strftime("%Y%m%d-%H%M%S")
return f"{modelstr}_{timestr}"
X_folds = t.Tuple(transient=True)
ys_folds = t.Tuple(transient=True)
def _trainers_default(self):
# Temp trainer for grabbing datasets, etc
tt = self.trainer_template
tt.init_data()
# Combine official train & val sets
X = torch.cat(
[tt.dl_train.dataset.tensors[0], tt.dl_val.dataset.tensors[0]])
ys = [
torch.cat([yt, yv]) for yt, yv in zip(
tt.dl_train.dataset.tensors[1:], tt.dl_val.dataset.tensors[1:])
]
# make folds
fold_len = int(np.ceil(len(X) / self.n_folds))
self.X_folds = torch.split(X, fold_len)
self.ys_folds = [torch.split(y, fold_len) for y in ys]
trainers = []
for i_val_fold in range(self.n_folds):
trainer = Trainer(
validation_fold=i_val_fold,
name=f"{self.name}/{i_val_fold}",
**self.config,
)
trainer.dl_test = tt.dl_test
trainers.append(trainer)
return trainers
model: models.BaseNet = t.Instance(torch.nn.Module, transient=True)
def _model_default(self):
model = models.FilterNetEnsemble()
model.set_models([trainer.model for trainer in self.trainers])
return model
model_path: str = t.Str()
def _model_path_default(self):
return f"saved_models/{self.name}/"
def init_data(self):
# Initiate loading of datasets, model
pass
# for trainer in self.trainers:
# trainer.init_data()
def init_train(self):
pass
# for trainer in self.trainers:
# trainer.init_train()
def train(self, max_epochs=50):
""" A pretty standard training loop, constrained to stop in `max_epochs` but may stop early if our
custom stopping metric does not improve for `self.patience` epochs. Always checkpoints
when a new best stopping_metric is achieved. An alternative to using
ray.tune for training."""
for trainer in self.trainers:
# Add data to trainer
X_train = torch.cat([
arr for i, arr in enumerate(self.X_folds)
if i != trainer.validation_fold
])
ys_train = [
torch.cat([
arr for i, arr in enumerate(y)
if i != trainer.validation_fold
]) for y in self.ys_folds
]
X_val = torch.cat([
arr for i, arr in enumerate(self.X_folds)
if i == trainer.validation_fold
])
ys_val = [
torch.cat([
arr for i, arr in enumerate(y)
if i == trainer.validation_fold
]) for y in self.ys_folds
]
trainer.dl_train = DataLoader(
TensorDataset(torch.Tensor(X_train), *ys_train),
batch_size=trainer.batch_size,
shuffle=True,
)
trainer.data_spec = self.trainer_template.data_spec
trainer.epoch_iters = self.trainer_template.epoch_iters
trainer.dl_val = DataLoader(
TensorDataset(torch.Tensor(X_val), *ys_val),
batch_size=trainer.batch_size,
shuffle=False,
)
# Now clear local vars to save ranm
X_train = ys_train = X_val = ys_val = None
trainer.init_data()
trainer.init_train()
trainer.train(max_epochs=max_epochs)
# Clear trainer train and val datasets to save ram
trainer.dl_train = t.Undefined
trainer.dl_val = t.Undefined
print(f"RESTORING TO best model")
trainer._restore()
trainer._save()
trainer.print_train_summary()
em = EvalModel(trainer=trainer)
em.run_test_set()
em.calc_metrics()
em.calc_ward_metrics()
print(em.classification_report_df.to_string(float_format="%.3f"))
em._save()
def print_train_summary(self):
for trainer in self.trainers:
trainer.print_train_summary()
def _save(self, checkpoint_dir=None, save_model=True, save_trainer=True):
""" Saves/checkpoints model state and training state to disk. """
if checkpoint_dir is None:
checkpoint_dir = self.model_path
else:
self.model_path = checkpoint_dir
os.makedirs(checkpoint_dir, exist_ok=True)
# save model params
model_path = os.path.join(checkpoint_dir, "model.pth")
trainer_path = os.path.join(checkpoint_dir, "trainer.pth")
if save_model:
torch.save(self.model.state_dict(), model_path)
if save_trainer:
with open(trainer_path, "wb") as f:
pickle.dump(self, f)
return checkpoint_dir
def _restore(self, checkpoint_dir=None):
""" Restores model state and training state from disk. """
if checkpoint_dir is None:
checkpoint_dir = self.model_path
model_path = os.path.join(checkpoint_dir, "model.pth")
trainer_path = os.path.join(checkpoint_dir, "trainer.pth")
# Reconstitute old trainer and copy state to this trainer.
with open(trainer_path, "rb") as f:
other_trainer = pickle.load(f)
self.__setstate__(other_trainer.__getstate__())
# Load sub-models
for trainer in self.trainers:
trainer._restore()
# Load model (after loading state in case we need to re-initialize model from config)
self.model.load_state_dict(
torch.load(model_path, map_location=self.device))
class EvalModel(t.HasStrictTraits):
trainer: Trainer = t.Any()
model: mo.BaseNet = t.DelegatesTo("trainer")
dl_test: DataLoader = t.DelegatesTo("trainer")
data_spec: dict = t.DelegatesTo("trainer")
cuda: bool = t.DelegatesTo("trainer")
device: str = t.DelegatesTo("trainer")
loss_func: str = t.DelegatesTo("trainer")
model_path: str = t.DelegatesTo("trainer")
has_null_class: bool = t.DelegatesTo("trainer")
predict_null_class: bool = t.DelegatesTo("trainer")
# 'prediction' mode employs overlap and reconstructs signal
# as a contiguous timeseries w/ optional windowing.
# It aims for best accuracy/f1 by using overlap, and will
# typically outperform 'training' mode.
# 'training' mode does not average repeated point and does
# not window; it should product acc/loss/f1 similar to
# training mode.
run_mode: str = t.Enum(["prediction", "training"])
window: str = t.Enum(["hanning", "boxcar"])
eval_batch_size: int = t.Int(100)
target_names: ty.List[str] = t.ListStr()
def _target_names_default(self):
target_names = self.data_spec["output_spec"][0]["classes"]
if self.has_null_class:
assert target_names[0] in ("", "Null")
if not self.predict_null_class:
target_names = target_names[1:]
return target_names
def _run_model_on_batch(self, data, targets):
targets = torch.stack(targets)
if self.cuda:
data, targets = data.cuda(), targets.cuda()
output = self.model(data)
_targets = self.model.transform_targets(targets, one_hot=False)
if self.loss_func == "cross_entropy":
_losses = [F.cross_entropy(o, t) for o, t in zip(output, _targets)]
loss = sum(_losses)
elif self.loss_func == "binary_cross_entropy":
_targets_onehot = self.model.transform_targets(targets,
one_hot=True)
_losses = [
F.binary_cross_entropy_with_logits(o, t)
for o, t in zip(output, _targets_onehot)
]
loss = sum(_losses)
else:
raise NotImplementedError(self.loss)
# Assume only 1 output:
return loss, output[0], _targets[0], _losses[0]
def run_test_set(self, dl=None):
""" Runs `self.model` on `self.dl_test` (or a provided dl) and stores results for subsequent evaluation. """
if dl is None:
dl = self.dl_test
if self.cuda:
self.model.cuda()
self.model.eval()
if self.eval_batch_size:
dl = DataLoader(dl.dataset,
batch_size=self.eval_batch_size,
shuffle=False)
#
# # Xc, yc = data.get_x_y_contig('test')
X, *ys = dl.dataset.tensors
# X: [N, input_chans, win_len]
step = int(X.shape[2] / 2)
assert torch.equal(X[0, :, step], X[1, :, 0])
losses = []
outputsraw = []
outputs = []
targets = []
with Timer("run", log_output=False) as tr:
with Timer("infer", log_output=False) as ti:
for batch_idx, (data, *target) in enumerate(dl):
(
batch_loss,
batch_output,
batch_targets,
train_losses,
) = self._run_model_on_batch(data, target)
losses.append(batch_loss.detach().cpu().item())
outputsraw.append(batch_output.detach().cpu().data.numpy())
outputs.append(
torch.argmax(batch_output, 1,
False).detach().cpu().data.numpy())
targets.append(batch_targets.detach().cpu().data.numpy())
self.infer_time_s_cpu = ti.interval_cpu
self.infer_time_s_wall = ti.interval_wall
self.loss = np.mean(losses)
targets = np.concatenate(targets, axis=0) # [N, out_win_len]
outputsraw = np.concatenate(
outputsraw, axis=0) # [N, n_out_classes, out_win_len]
outputs = np.concatenate(outputs,
axis=0) # [N, n_out_classes, out_win_len]
# win_len = toutputsraw[0].shape[-1]
if (self.model.output_type == "many_to_one_takelast"
or self.run_mode == "training"):
self.targets = np.concatenate(targets, axis=-1) # [N,]
self.outputsraw = np.concatenate(
outputsraw, axis=-1) # [n_out_classes, N,]
self.outputs = np.concatenate(outputs, axis=-1) # [N,]
elif self.run_mode == "prediction":
n_segments, n_classes, out_win_len = outputsraw.shape
output_step = int(out_win_len / 2)
if self.window == "hanning":
EPS = 0.001 # prevents divide-by-zero
arr_window = (1 - EPS) * np.hanning(out_win_len) + EPS
elif self.window == "boxcar":
arr_window = np.ones((out_win_len, ))
else:
raise ValueError()
# Allocate space for merged predictions
if self.has_null_class and not self.predict_null_class:
outputsraw2 = np.zeros(
(n_segments + 1, n_classes - 1, output_step, 2))
window2 = np.zeros(
(n_segments + 1, n_classes - 1, output_step,
2)) # [N+1, out_win_len/2, 2]
# Drop in outputs/window vals in the two layers
outputsraw = outputsraw[:, 1:, :]
else:
outputsraw2 = np.zeros(
(n_segments + 1, n_classes, output_step, 2))
window2 = np.zeros((n_segments + 1, n_classes, output_step,
2)) # [N+1, out_win_len/2, 2]
# Drop in outputs/window vals in the two layers
outputsraw2[:-1, :, :, 0] = outputsraw[:, :, :output_step]
outputsraw2[1:, :, :,
1] = outputsraw[:, :, output_step:output_step * 2]
window2[:-1, :, :, 0] = arr_window[:output_step]
window2[1:, :, :, 1] = arr_window[output_step:output_step * 2]
merged_outputsraw = (outputsraw2 * window2).sum(
axis=3) / (window2).sum(axis=3)
softmaxed_merged_outputsraw = softmax(merged_outputsraw,
axis=1)
merged_outputs = np.argmax(softmaxed_merged_outputsraw, 1)
self.outputsraw = np.concatenate(merged_outputsraw, axis=-1)
self.outputs = np.concatenate(merged_outputs, axis=-1)
self.targets = np.concatenate(
np.concatenate(
[
targets[:, :output_step],
targets[[-1], output_step:output_step * 2],
],
axis=0,
),
axis=-1,
)
else:
raise ValueError()
if self.has_null_class and not self.predict_null_class:
not_null_mask = self.targets > 0
self.outputsraw = self.outputsraw[..., not_null_mask]
self.outputs = self.outputs[not_null_mask]
self.targets = self.targets[not_null_mask]
self.targets -= 1
self.n_samples_in = np.prod(dl.dataset.tensors[1].shape)
self.n_samples_out = len(self.outputs)
self.infer_samples_per_s = self.n_samples_in / self.infer_time_s_wall
self.run_time_s_cpu = tr.interval_cpu
self.run_time_s_wall = tr.interval_wall
loss: float = t.Float()
targets: np.ndarray = t.Array()
outputsraw: np.ndarray = t.Array()
outputs: np.ndarray = t.Array()
n_samples_in: int = t.Int()
n_samples_out: int = t.Int()
infer_samples_per_s: float = t.Float()
infer_time_s_cpu: float = t.Float()
infer_time_s_wall: float = t.Float()
run_time_s_cpu: float = t.Float()
run_time_s_wall: float = t.Float()
extra: dict = t.Dict({})
acc: float = t.Float()
f1: float = t.Float()
f1_mean: float = t.Float()
event_f1: float = t.Float()
classification_report_txt: str = t.Str()
classification_report_dict: dict = t.Dict()
classification_report_df: pd.DataFrame = t.Property(
t.Instance(pd.DataFrame))
confusion_matrix: np.ndarray = t.Array()
nonull_acc: float = t.Float()
nonull_f1: float = t.Float()
nonull_f1_mean: float = t.Float()
nonull_classification_report_txt: str = t.Str()
nonull_classification_report_dict: dict = t.Dict()
nonull_classification_report_df: pd.DataFrame = t.Property(
t.Instance(pd.DataFrame))
nonull_confusion_matrix: np.ndarray = t.Array()
def calc_metrics(self):
self.acc = sklearn.metrics.accuracy_score(self.targets, self.outputs)
self.f1 = sklearn.metrics.f1_score(self.targets,
self.outputs,
average="weighted")
self.f1_mean = sklearn.metrics.f1_score(self.targets,
self.outputs,
average="macro")
self.classification_report_txt = sklearn.metrics.classification_report(
self.targets,
self.outputs,
digits=3,
labels=np.arange(len(self.target_names)),
target_names=self.target_names,
)
self.classification_report_dict = sklearn.metrics.classification_report(
self.targets,
self.outputs,
digits=3,
output_dict=True,
labels=np.arange(len(self.target_names)),
target_names=self.target_names,
)
self.confusion_matrix = sklearn.metrics.confusion_matrix(
self.targets, self.outputs)
# Now, ignoring the null/none class:
if self.has_null_class and self.predict_null_class:
# assume null class comes fistnonull_mask = self.targets > 0
nonull_mask = self.targets > 0
nonull_targets = self.targets[nonull_mask]
# nonull_outputs = self.outputs[nonull_mask]
nonull_outputs = self.outputsraw[1:, :].argmax(
axis=0)[nonull_mask] + 1
self.nonull_acc = sklearn.metrics.accuracy_score(
nonull_targets, nonull_outputs)
self.nonull_f1 = sklearn.metrics.f1_score(nonull_targets,
nonull_outputs,
average="weighted")
self.nonull_f1_mean = sklearn.metrics.f1_score(nonull_targets,
nonull_outputs,
average="macro")
self.nonull_classification_report_txt = sklearn.metrics.classification_report(
nonull_targets,
nonull_outputs,
digits=3,
labels=np.arange(len(self.target_names)),
target_names=self.target_names,
)
self.nonull_classification_report_dict = sklearn.metrics.classification_report(
nonull_targets,
nonull_outputs,
digits=3,
output_dict=True,
labels=np.arange(len(self.target_names)),
target_names=self.target_names,
)
self.nonull_confusion_matrix = sklearn.metrics.confusion_matrix(
nonull_targets, nonull_outputs)
else:
self.nonull_acc = self.acc
self.nonull_f1 = self.f1
self.nonull_f1_mean = self.f1_mean
self.nonull_classification_report_txt = self.classification_report_txt
self.nonull_classification_report_dict = self.classification_report_dict
self.nonull_confusion_matrix = self.confusion_matrix
ward_metrics: WardMetrics = t.Instance(WardMetrics)
def calc_ward_metrics(self):
""" Do event-wise metrics, using the `wardmetrics` package which implements metrics from:
[1] J. A. Ward, P. Lukowicz, and H. W. Gellersen, “Performance metrics for activity recognition,”
ACM Trans. Intell. Syst. Technol., vol. 2, no. 1, pp. 1–23, Jan. 2011.
"""
import wardmetrics
# Must be in prediction mode -- otherwise, data is not contiguous, ward metrics will be bogus
assert self.run_mode == "prediction"
targets = self.targets
predictions = self.outputs
wmetrics = WardMetrics()
targets_events = wardmetrics.frame_results_to_events(targets)
preds_events = wardmetrics.frame_results_to_events(predictions)
for i, class_name in enumerate(self.target_names):
class_wmetrics = ClassWardMetrics()
t = targets_events.get(str(i), [])
p = preds_events.get(str(i), [])
# class_wmetrics['t'] = t
# class_wmetrics['p'] = p
try:
assert len(t) and len(p)
(
twoset_results,
segments_with_scores,
segment_counts,
normed_segment_counts,
) = wardmetrics.eval_segments(t, p)
class_wmetrics.segment_twoset_results = twoset_results
(
gt_event_scores,
det_event_scores,
detailed_scores,
standard_scores,
) = wardmetrics.eval_events(t, p)
class_wmetrics.event_detailed_scores = detailed_scores
class_wmetrics.event_standard_scores = standard_scores
except (AssertionError, ZeroDivisionError) as e:
class_wmetrics.segment_twoset_results = {}
class_wmetrics.event_detailed_scores = {}
class_wmetrics.event_standard_scores = {}
# print("Empty Results or targets for a class.")
# raise ValueError("Empty Results or targets for a class.")
wmetrics.class_ward_metrics.append(class_wmetrics)
tt = []
pp = []
for i, class_name in enumerate(self.target_names):
# skip null class for combined eventing:
if class_name in ("", "Null"):
continue
if len(tt) or len(pp):
offset = np.max(tt + pp) + 2
else:
offset = 0
[(a + offset, b + offset) for (a, b) in t]
t = targets_events.get(str(i), [])
p = preds_events.get(str(i), [])
tt += [(a + offset, b + offset) for (a, b) in t]
pp += [(a + offset, b + offset) for (a, b) in p]
t = tt
p = pp
class_wmetrics = ClassWardMetrics()
assert len(t) and len(p)
(
twoset_results,
segments_with_scores,
segment_counts,
normed_segment_counts,
) = wardmetrics.eval_segments(t, p)
class_wmetrics.segment_twoset_results = twoset_results
(
gt_event_scores,
det_event_scores,
detailed_scores,
standard_scores,
) = wardmetrics.eval_events(t, p)
class_wmetrics.event_detailed_scores = detailed_scores
class_wmetrics.event_standard_scores = standard_scores
# Reformat as dataframe for easier calculations
df = pd.DataFrame(
[cm.event_standard_scores for cm in wmetrics.class_ward_metrics],
index=self.target_names,
)
df.loc["all_nonull"] = class_wmetrics.event_standard_scores
# Calculate F1's to summarize recall/precision for each class
df["f1"] = (2 * (df["precision"] * df["recall"]) /
(df["precision"] + df["recall"]))
df["f1 (weighted)"] = (
2 * (df["precision (weighted)"] * df["recall (weighted)"]) /
(df["precision (weighted)"] + df["recall (weighted)"]))
# Load dataframes into dictionary output
wmetrics.df_event_scores = df
wmetrics.df_event_detailed_scores = pd.DataFrame(
[cm.event_detailed_scores for cm in wmetrics.class_ward_metrics],
index=self.target_names,
)
wmetrics.df_segment_2set_results = pd.DataFrame(
[cm.segment_twoset_results for cm in wmetrics.class_ward_metrics],
index=self.target_names,
)
wmetrics.overall_ward_metrics = class_wmetrics
self.ward_metrics = wmetrics
self.event_f1 = self.ward_metrics.df_event_scores.loc["all_nonull",
"f1"]
def _get_classification_report_df(self):
df = pd.DataFrame(self.classification_report_dict).T
# Include Ward-metrics-derived "Event F1 (unweighted by length)"
if self.ward_metrics:
df["event_f1"] = self.ward_metrics.df_event_scores["f1"]
else:
df["event_f1"] = np.nan
# Calculate various summary averages
df.loc["macro avg", "event_f1"] = df["event_f1"].iloc[:-3].mean()
df.loc["weighted avg", "event_f1"] = (
df["event_f1"].iloc[:-3] *
df["support"].iloc[:-3]).sum() / df["support"].iloc[:-3].sum()
df["support"] = df["support"].astype(int)
return df
def _get_nonull_classification_report_df(self):
target_names = self.target_names
if not (target_names[0] in ("", "Null")):
return None
df = pd.DataFrame(self.nonull_classification_report_dict).T
df["support"] = df["support"].astype(int)
return df
def _save(self, checkpoint_dir=None):
""" Saves/checkpoints model state and training state to disk. """
if checkpoint_dir is None:
checkpoint_dir = self.model_path
os.makedirs(checkpoint_dir, exist_ok=True)
# save model params
evalmodel_path = os.path.join(checkpoint_dir, "evalmodel.pth")
with open(evalmodel_path, "wb") as f:
pickle.dump(self, f)
return checkpoint_dir
def _restore(self, checkpoint_dir=None):
""" Restores model state and training state from disk. """
if checkpoint_dir is None:
checkpoint_dir = self.model_path
evalmodel_path = os.path.join(checkpoint_dir, "evalmodel.pth")
# Reconstitute old trainer and copy state to this trainer.
with open(evalmodel_path, "rb") as f:
other_evalmodel = pickle.load(f)
self.__setstate__(other_evalmodel.__getstate__())
self.trainer._restore(checkpoint_dir)
class Parameter(t.HasTraits):
"""Model parameter
Attributes
----------
value : float or array
The value of the parameter for the current location. The value
for other locations is stored in map.
bmin, bmax: float
Lower and upper bounds of the parameter value.
twin : {None, Parameter}
If it is not None, the value of the current parameter is
a function of the given Parameter. The function is by default
the identity function, but it can be defined by twin_function
twin_function : function
Function that, if selt.twin is not None, takes self.twin.value
as its only argument and returns a float or array that is
returned when getting Parameter.value
twin_inverse_function : function
The inverse of twin_function. If it is None then it is not
possible to set the value of the parameter twin by setting
the value of the current parameter.
ext_force_positive : bool
If True, the parameter value is set to be the absolute value
of the input value i.e. if we set Parameter.value = -3, the
value stored is 3 instead. This is useful to bound a value
to be positive in an optimization without actually using an
optimizer that supports bounding.
ext_bounded : bool
Similar to ext_force_positive, but in this case the bounds are
defined by bmin and bmax. It is a better idea to use
an optimizer that supports bounding though.
Methods
-------
as_signal(field = 'values')
Get a parameter map as a signal object
plot()
Plots the value of the Parameter at all locations.
export(folder=None, name=None, format=None, save_std=False)
Saves the value of the parameter map to the specified format
connect, disconnect(function)
Call the functions connected when the value attribute changes.
"""
__number_of_elements = 1
__value = 0
__free = True
_bounds = (None, None)
__twin = None
_axes_manager = None
__ext_bounded = False
__ext_force_positive = False
# traitsui bugs out trying to make an editor for this, so always specify!
# (it bugs out, because both editor shares the object, and Array editors
# don't like non-sequence objects). TextEditor() works well, so does
# RangeEditor() as it works with bmin/bmax.
value = t.Property(t.Either([t.CFloat(0), Array()]))
units = t.Str('')
free = t.Property(t.CBool(True))
bmin = t.Property(NoneFloat(), label="Lower bounds")
bmax = t.Property(NoneFloat(), label="Upper bounds")
def __init__(self):
self._twins = set()
self.events = Events()
self.events.value_changed = Event("""
Event that triggers when the `Parameter.value` changes.
The event triggers after the internal state of the `Parameter` has
been updated.
Arguments
---------
obj : Parameter
The `Parameter` that the event belongs to
value : {float | array}
The new value of the parameter
""",
arguments=["obj", 'value'])
self.twin_function = lambda x: x
self.twin_inverse_function = lambda x: x
self.std = None
self.component = None
self.grad = None
self.name = ''
self.units = ''
self.map = None
self.model = None
self._whitelist = {
'_id_name': None,
'value': None,
'std': None,
'free': None,
'units': None,
'map': None,
'_bounds': None,
'ext_bounded': None,
'name': None,
'ext_force_positive': None,
'self': ('id', None),
'twin_function': ('fn', None),
'twin_inverse_function': ('fn', None),
}
self._slicing_whitelist = {'map': 'inav'}
def _load_dictionary(self, dictionary):
"""Load data from dictionary
Parameters
----------
dict : dictionary
A dictionary containing at least the following items:
_id_name : string
_id_name of the original parameter, used to create the
dictionary. Has to match with the self._id_name
_whitelist : dictionary
a dictionary, which keys are used as keywords to match with the
parameter attributes. For more information see
:meth:`hyperspy.misc.export_dictionary.load_from_dictionary`
* any field from _whitelist.keys() *
Returns
-------
id_value : int
the ID value of the original parameter, to be later used for setting
up the correct twins
"""
if dictionary['_id_name'] == self._id_name:
load_from_dictionary(self, dictionary)
return dictionary['self']
else:
raise ValueError(
"_id_name of parameter and dictionary do not match, \nparameter._id_name = %s\
\ndictionary['_id_name'] = %s" %
(self._id_name, dictionary['_id_name']))
def __repr__(self):
text = ''
text += 'Parameter %s' % self.name
if self.component is not None:
text += ' of %s' % self.component._get_short_description()
text = '<' + text + '>'
return text
def __len__(self):
return self._number_of_elements
def connect(self, f):
warnings.warn(
"The method `Parameter.connect()` has been deprecated and will be "
"removed in HyperSpy 0.10. Please use "
"`Parameter.events.value_changed.connect()` instead.",
VisibleDeprecationWarning)
self.events.value_changed.connect(f, [])
def disconnect(self, f):
warnings.warn(
"The method `Parameter.disconnect()` has been deprecated and will "
"be removed in HyperSpy 0.10. Please use "
"`Parameter.events.value_changed.disconnect()` instead.",
VisibleDeprecationWarning)
self.events.value_changed.disconnect(f)
def _get_value(self):
if self.twin is None:
return self.__value
else:
return self.twin_function(self.twin.value)
def _set_value(self, value):
try:
# Use try/except instead of hasattr("__len__") because a numpy
# memmap has a __len__ wrapper even for numbers that raises a
# TypeError when calling. See issue #349.
if len(value) != self._number_of_elements:
raise ValueError("The length of the parameter must be ",
self._number_of_elements)
else:
if not isinstance(value, tuple):
value = tuple(value)
except TypeError:
if self._number_of_elements != 1:
raise ValueError("The length of the parameter must be ",
self._number_of_elements)
old_value = self.__value
if self.twin is not None:
if self.twin_inverse_function is not None:
self.twin.value = self.twin_inverse_function(value)
return
if self.ext_bounded is False:
self.__value = value
else:
if self.ext_force_positive is True:
value = np.abs(value)
if self._number_of_elements == 1:
if self.bmin is not None and value <= self.bmin:
self.__value = self.bmin
elif self.bmax is not None and value >= self.bmax:
self.__value = self.bmax
else:
self.__value = value
else:
bmin = (self.bmin if self.bmin is not None else -np.inf)
bmax = (self.bmax if self.bmin is not None else np.inf)
self.__value = np.clip(value, bmin, bmax)
if (self._number_of_elements != 1
and not isinstance(self.__value, tuple)):
self.__value = tuple(self.__value)
if old_value != self.__value:
self.events.value_changed.trigger(value=self.__value, obj=self)
self.trait_property_changed('value', old_value, self.__value)
# Fix the parameter when coupled
def _get_free(self):
if self.twin is None:
return self.__free
else:
return False
def _set_free(self, arg):
old_value = self.__free
self.__free = arg
if self.component is not None:
self.component._update_free_parameters()
self.trait_property_changed('free', old_value, self.__free)
def _on_twin_update(self, value, twin=None):
if (twin is not None and hasattr(twin, 'events')
and hasattr(twin.events, 'value_changed')):
with twin.events.value_changed.suppress_callback(
self._on_twin_update):
self.events.value_changed.trigger(value=value, obj=self)
else:
self.events.value_changed.trigger(value=value, obj=self)
def _set_twin(self, arg):
if arg is None:
if self.twin is not None:
# Store the value of the twin in order to set the
# value of the parameter when it is uncoupled
twin_value = self.value
if self in self.twin._twins:
self.twin._twins.remove(self)
self.twin.events.value_changed.disconnect(
self._on_twin_update)
self.__twin = arg
self.value = twin_value
else:
if self not in arg._twins:
arg._twins.add(self)
arg.events.value_changed.connect(self._on_twin_update,
["value"])
self.__twin = arg
if self.component is not None:
self.component._update_free_parameters()
def _get_twin(self):
return self.__twin
twin = property(_get_twin, _set_twin)
def _get_bmin(self):
if self._number_of_elements == 1:
return self._bounds[0]
else:
return self._bounds[0][0]
def _set_bmin(self, arg):
old_value = self.bmin
if self._number_of_elements == 1:
self._bounds = (arg, self.bmax)
else:
self._bounds = ((arg, self.bmax), ) * self._number_of_elements
# Update the value to take into account the new bounds
self.value = self.value
self.trait_property_changed('bmin', old_value, arg)
def _get_bmax(self):
if self._number_of_elements == 1:
return self._bounds[1]
else:
return self._bounds[0][1]
def _set_bmax(self, arg):
old_value = self.bmax
if self._number_of_elements == 1:
self._bounds = (self.bmin, arg)
else:
self._bounds = ((self.bmin, arg), ) * self._number_of_elements
# Update the value to take into account the new bounds
self.value = self.value
self.trait_property_changed('bmax', old_value, arg)
@property
def _number_of_elements(self):
return self.__number_of_elements
@_number_of_elements.setter
def _number_of_elements(self, arg):
# Do nothing if the number of arguments stays the same
if self.__number_of_elements == arg:
return
if arg <= 1:
raise ValueError("Please provide an integer number equal "
"or greater to 1")
self._bounds = ((self.bmin, self.bmax), ) * arg
self.__number_of_elements = arg
if arg == 1:
self._Parameter__value = 0
else:
self._Parameter__value = (0, ) * arg
if self.component is not None:
self.component.update_number_parameters()
@property
def ext_bounded(self):
return self.__ext_bounded
@ext_bounded.setter
def ext_bounded(self, arg):
if arg is not self.__ext_bounded:
self.__ext_bounded = arg
# Update the value to take into account the new bounds
self.value = self.value
@property
def ext_force_positive(self):
return self.__ext_force_positive
@ext_force_positive.setter
def ext_force_positive(self, arg):
if arg is not self.__ext_force_positive:
self.__ext_force_positive = arg
# Update the value to take into account the new bounds
self.value = self.value
def store_current_value_in_array(self):
"""Store the value and std attributes.
See also
--------
fetch, assign_current_value_to_all
"""
indices = self._axes_manager.indices[::-1]
# If it is a single spectrum indices is ()
if not indices:
indices = (0, )
self.map['values'][indices] = self.value
self.map['is_set'][indices] = True
if self.std is not None:
self.map['std'][indices] = self.std
def fetch(self):
"""Fetch the stored value and std attributes.
See Also
--------
store_current_value_in_array, assign_current_value_to_all
"""
indices = self._axes_manager.indices[::-1]
# If it is a single spectrum indices is ()
if not indices:
indices = (0, )
if self.map['is_set'][indices]:
self.value = self.map['values'][indices]
self.std = self.map['std'][indices]
def assign_current_value_to_all(self, mask=None):
"""Assign the current value attribute to all the indices
Parameters
----------
mask: {None, boolean numpy array}
Set only the indices that are not masked i.e. where
mask is False.
See Also
--------
store_current_value_in_array, fetch
"""
if mask is None:
mask = np.zeros(self.map.shape, dtype='bool')
self.map['values'][mask == False] = self.value
self.map['is_set'][mask == False] = True
def _create_array(self):
"""Create the map array to store the information in
multidimensional datasets.
"""
shape = self._axes_manager._navigation_shape_in_array
if not shape:
shape = [
1,
]
dtype_ = np.dtype([('values', 'float', self._number_of_elements),
('std', 'float', self._number_of_elements),
('is_set', 'bool', 1)])
if (self.map is None or self.map.shape != shape
or self.map.dtype != dtype_):
self.map = np.zeros(shape, dtype_)
self.map['std'].fill(np.nan)
# TODO: in the future this class should have access to
# axes manager and should be able to fetch its own
# values. Until then, the next line is necessary to avoid
# erros when self.std is defined and the shape is different
# from the newly defined arrays
self.std = None
def as_signal(self, field='values'):
"""Get a parameter map as a signal object.
Please note that this method only works when the navigation
dimension is greater than 0.
Parameters
----------
field : {'values', 'std', 'is_set'}
Raises
------
NavigationDimensionError : if the navigation dimension is 0
"""
from hyperspy.signal import BaseSignal
s = BaseSignal(data=self.map[field],
axes=self._axes_manager._get_navigation_axes_dicts())
if self.component is not None and \
self.component.active_is_multidimensional:
s.data[np.logical_not(self.component._active_array)] = np.nan
s.metadata.General.title = ("%s parameter" %
self.name if self.component is None else
"%s parameter of %s component" %
(self.name, self.component.name))
for axis in s.axes_manager._axes:
axis.navigate = False
if self._number_of_elements > 1:
s.axes_manager._append_axis(size=self._number_of_elements,
name=self.name,
navigate=True)
s._assign_subclass()
if field == "values":
# Add the variance if available
std = self.as_signal(field="std")
if not np.isnan(std.data).all():
std.data = std.data**2
std.metadata.General.title = "Variance"
s.metadata.set_item("Signal.Noise_properties.variance", std)
return s
def plot(self, **kwargs):
"""Plot parameter signal.
Parameters
----------
**kwargs
Any extra keyword arguments are passed to the signal plot.
Example
-------
>>> parameter.plot()
Set the minimum and maximum displayed values
>>> parameter.plot(vmin=0, vmax=1)
"""
self.as_signal().plot(**kwargs)
def export(self, folder=None, name=None, format=None, save_std=False):
"""Save the data to a file.
All the arguments are optional.
Parameters
----------
folder : str or None
The path to the folder where the file will be saved.
If `None` the current folder is used by default.
name : str or None
The name of the file. If `None` the Components name followed
by the Parameter `name` attributes will be used by default.
If a file with the same name exists the name will be
modified by appending a number to the file path.
save_std : bool
If True, also the standard deviation will be saved
"""
if format is None:
format = preferences.General.default_export_format
if name is None:
name = self.component.name + '_' + self.name
filename = incremental_filename(slugify(name) + '.' + format)
if folder is not None:
filename = os.path.join(folder, filename)
self.as_signal().save(filename)
if save_std is True:
self.as_signal(field='std').save(append2pathname(filename, '_std'))
def as_dictionary(self, fullcopy=True):
"""Returns parameter as a dictionary, saving all attributes from
self._whitelist.keys() For more information see
:meth:`hyperspy.misc.export_dictionary.export_to_dictionary`
Parameters
----------
fullcopy : Bool (optional, False)
Copies of objects are stored, not references. If any found,
functions will be pickled and signals converted to dictionaries
Returns
-------
dic : dictionary with the following keys:
_id_name : string
_id_name of the original parameter, used to create the
dictionary. Has to match with the self._id_name
_twins : list
a list of ids of the twins of the parameter
_whitelist : dictionary
a dictionary, which keys are used as keywords to match with the
parameter attributes. For more information see
:meth:`hyperspy.misc.export_dictionary.export_to_dictionary`
* any field from _whitelist.keys() *
"""
dic = {'_twins': [id(t) for t in self._twins]}
export_to_dictionary(self, self._whitelist, dic, fullcopy)
return dic
def default_traits_view(self):
# As mentioned above, the default editor for
# value = t.Property(t.Either([t.CFloat(0), Array()]))
# gives a ValueError. We therefore implement default_traits_view so
# that configure/edit_traits will still work straight out of the box.
# A whitelist controls which traits to include in this view.
from traitsui.api import RangeEditor, View, Item
whitelist = ['bmax', 'bmin', 'free', 'name', 'std', 'units', 'value']
editable_traits = [
trait for trait in self.editable_traits() if trait in whitelist
]
if 'value' in editable_traits:
i = editable_traits.index('value')
v = editable_traits.pop(i)
editable_traits.insert(
i,
Item(v, editor=RangeEditor(low_name='bmin', high_name='bmax')))
view = View(editable_traits, buttons=['OK', 'Cancel'])
return view
def _interactive_slider_bounds(self, index=None):
"""Guesstimates the bounds for the slider. They will probably have to
be changed later by the user.
"""
fraction = 10.
_min, _max, step = None, None, None
value = self.value if index is None else self.value[index]
if self.bmin is not None:
_min = self.bmin
if self.bmax is not None:
_max = self.bmax
if _max is None and _min is not None:
_max = value + fraction * (value - _min)
if _min is None and _max is not None:
_min = value - fraction * (_max - value)
if _min is None and _max is None:
if self is self.component._position:
axis = self._axes_manager.signal_axes[-1]
_min = axis.axis.min()
_max = axis.axis.max()
step = np.abs(axis.scale)
else:
_max = value + np.abs(value * fraction)
_min = value - np.abs(value * fraction)
if step is None:
step = (_max - _min) * 0.001
return {'min': _min, 'max': _max, 'step': step}
def _interactive_update(self, value=None, index=None):
"""Callback function for the widgets, to update the value
"""
if value is not None:
if index is None:
self.value = value['new']
else:
self.value = self.value[:index] + (value['new'],) +\
self.value[index + 1:]
def notebook_interaction(self, display=True):
"""Creates interactive notebook widgets for the parameter, if
available.
Requires `ipywidgets` to be installed.
Parameters
----------
display : bool
if True (default), attempts to display the parameter widget.
Otherwise returns the formatted widget object.
"""
from ipywidgets import VBox
from traitlets import TraitError as TraitletError
from IPython.display import display as ip_display
try:
if self._number_of_elements == 1:
container = self._create_notebook_widget()
else:
children = [
self._create_notebook_widget(index=i)
for i in range(self._number_of_elements)
]
container = VBox(children)
if not display:
return container
ip_display(container)
except TraitletError:
if display:
_logger.info('This function is only avialable when running in'
' a notebook')
else:
raise
def _create_notebook_widget(self, index=None):
from ipywidgets import (FloatSlider, FloatText, Layout, HBox)
widget_bounds = self._interactive_slider_bounds(index=index)
thismin = FloatText(
value=widget_bounds['min'],
description='min',
layout=Layout(flex='0 1 auto', width='auto'),
)
thismax = FloatText(
value=widget_bounds['max'],
description='max',
layout=Layout(flex='0 1 auto', width='auto'),
)
current_value = self.value if index is None else self.value[index]
current_name = self.name
if index is not None:
current_name += '_{}'.format(index)
widget = FloatSlider(value=current_value,
min=thismin.value,
max=thismax.value,
step=widget_bounds['step'],
description=current_name,
layout=Layout(flex='1 1 auto', width='auto'))
def on_min_change(change):
if widget.max > change['new']:
widget.min = change['new']
widget.step = np.abs(widget.max - widget.min) * 0.001
def on_max_change(change):
if widget.min < change['new']:
widget.max = change['new']
widget.step = np.abs(widget.max - widget.min) * 0.001
thismin.observe(on_min_change, names='value')
thismax.observe(on_max_change, names='value')
this_observed = functools.partial(self._interactive_update,
index=index)
widget.observe(this_observed, names='value')
container = HBox((thismin, widget, thismax))
return container
class HardwareAction(traits.HasTraits):
"""Parent class for all hardware actions. User must make a subclass of this for each
hardware action and overwrite init, close and callback methods where necessary. Other
functions can use the parent class implementation directly"""
callbackTime = traits.Float()
variables = traits.List()
variablesReference = {}# leave empty. This will be set to the experiment control variables before the call back is executed. This is all taken care of by the snake
hardwareActionName = traits.Str()
examineVariablesButton = traits.Button()
enabled = traits.Bool(True)
callbackTimeVariableDependent = False # if true the calbackTime argument is a variable to be parsed in the snake
callbackTimeString = None # gets populated if callbackTimeInSequence is a string
snakeReference = None# reference to the snake object so that we can call update functions for e.g. examineVariablesDict pane
def __init__(self, callbackTimeInSequence, **traitsDict):
super(HardwareAction,self).__init__(**traitsDict)
if type(callbackTimeInSequence) is float:
self.callbackTime = callbackTimeInSequence # time in the sequence when call back should be performed passed during constructions
elif type(callbackTimeInSequence) is str:#here we check if callback time is a timing edge or a variable
self.callbackTimeVariableDependent = True
self.callbackTimeString = callbackTimeInSequence
logger.info( "CallbackTime string detected attempting to parse string as timing edge or variable" )
else:
self.callbackTime = callbackTimeInSequence # time in the sequence when call back should be performed passed during constructions
self.awaitingCallback = True # goes to False after it has been called back for the final time in a sequence (usually once)
self.callbackCounter = 0 # number of times called back this sequence
self.initialised = False # set to true if init run. set to false if close run
logger.info( "HardwareAction Super class __init__ completed" )
def _variables_default(self):
"""uses the variable mappings dictionary defined in the subclass """
return self.variableMappings.keys()
def setVariablesDictionary(self, variables):
"""sets the variables reference to the latest variables dictionary. simply sets the variables reference attribute """
self.variablesReference=variables
def mapVariables(self):
"""returns a dictionary of python variable names used in the callback function
with their correct values for this run. Raises an error if a variable is missing.
Could potentially implement default values here"""
logger.debug( "variables in %s: %s" % (self.hardwareActionName,self.variablesReference))
try:
return {self.variableMappings[key]:self.variablesReference[key] for key in self.variableMappings.iterkeys()}
except KeyError as e:
raise e # defaults handling TODO
def parseCallbackTime(self):
"""if callback Time is a string we comprehend it as a timing edge name or variable name"""
if self.callbackTimeString in self.snakeReference.timingEdges:
self.callbackTime = self.snakeReference.timingEdges[self.callbackTimeString]
elif self.callbackTimeString in self.snakeReference.variables:
self.callbackTime = self.snakeReference.variables[self.callbackTimeString]
else:
raise KeyError("callbackTime %s was not found in either the timing edges or variables dictionary. Check Spelling? Could not initialise %s object" % (self.callbackTimeString, self.hardwareActionName))
#####USER SHOULD OVERWRITE THE BELOW FUNCTIONS IN SUBCLASS AS REQUIRED
def init(self):
"""only called once when the user presses the start button. This should perform
any hardware specific initialisation. E.g opening sockets / decads connections. Return
string is printed to main log terminal"""
self.initialised=True
logger.warning("Using default init as no init method has been defined in Hardware Action Subclass")
return "%s init successful" % self.hardwareActionName
def close(self):
"""called to close the hardware down when user stops Snake or exits. Should
safely close the hardware. It should be able to restart again when the
init function is called (e.g. user then presses start"""
logger.warning("Using default close as no close method has been defined in Hardware Action Subclass")
return "%s closed" % self.hardwareActionName
def callback(self):
"""This is the function that is called every sequence at the callbackTime.
IT SHOULD NOT HANG as this is a blocking function call. You need to handle
threading yourself if the callback function would take a long time to return.
Return value should be a string to be printed in terminal"""
logger.debug( "beginning %s callback" % self.hardwareActionName)
if not self.initialised:
return "%s not initialised with init function. Cannot be called back until initialised. Doing nothing" % self.hardwareActionName
try:#YOUR CALLBACK CODE SHOULD GO IN THIS TRY BLOCK!
self.finalVariables = self.mapVariables()
raise NotImplementedError("the callback function needs to be implemented in your subclass")
return "callback on %s completed" % (self.hardwareActionName)
except KeyboardInterrupt:
raise
except KeyError as e:
return "Failed to find variable %s in variables %s. Check variable is defined in experiment control " % (e.message, self.variablesReference.keys())
except Exception as e:
return "Failed to perform callback on %s. Error message %s" % (self.hardwareActionName, e.message)
def _enabled_changed(self):
"""traitsui handler function (is automatically called when enabled changes during interaction with user interface """
if self.enabled:
self.snakeReference.mainLog.addLine("%s was just enabled. Will perform its init method" % self.hardwareActionName,1)
self.awaitingCallback=False # by setting this to False we prevent the action being performed till the next sequence begins. This is usually desireable
self.init()
elif not self.enabled:
if self.snakeReference.isRunning:#only print to log if it's disabled while snake is running
self.snakeReference.mainLog.addLine("%s was just disabled. Will perform its close method" % self.hardwareActionName,1)
self.close()#close method always performed for safety
def _examineVariablesButton_fired(self):
"""Called when user clicks on book item near hardware action name. This makes a pop up
which shows all the variables that the hardware action defines. later it might let users
edit certain parameters"""
self.snakeReference.updateExamineVariablesDictionary(self)# pass the update this hardwareAction object as the argument
logger.info("variables = %s" % self.variables)
#traits_view for all hardware actions. Just shows the name and lets the user enable or disable
traits_view = traitsui.View(
traitsui.HGroup(traitsui.Item("hardwareActionName", show_label=False, style="readonly"),
traitsui.Item("enabled",show_label=False),
traitsui.Item("examineVariablesButton",show_label=False,
editor=traitsui.ButtonEditor(image = pyface.image_resource.ImageResource( os.path.join(os.getcwd(), 'icons', 'book.png' ))),
style="custom"),
)
)
class HCFT(tr.HasStrictTraits):
'''High-Cycle Fatigue Tool
'''
#=========================================================================
# Traits definitions
#=========================================================================
decimal = tr.Enum(',', '.')
delimiter = tr.Str(';')
records_per_second = tr.Float(100)
take_time_from_time_column = tr.Bool(True)
file_csv = tr.File
open_file_csv = tr.Button('Input file')
skip_first_rows = tr.Range(low=1, high=10**9, mode='spinner')
columns_headers_list = tr.List([])
x_axis = tr.Enum(values='columns_headers_list')
y_axis = tr.Enum(values='columns_headers_list')
force_column = tr.Enum(values='columns_headers_list')
time_column = tr.Enum(values='columns_headers_list')
x_axis_multiplier = tr.Enum(1, -1)
y_axis_multiplier = tr.Enum(-1, 1)
npy_folder_path = tr.Str
file_name = tr.Str
apply_filters = tr.Bool
plot_settings_btn = tr.Button
plot_settings = PlotSettings()
plot_settings_active = tr.Bool
normalize_cycles = tr.Bool
smooth = tr.Bool
plot_every_nth_point = tr.Range(low=1, high=1000000, mode='spinner')
old_peak_force_before_cycles = tr.Float
peak_force_before_cycles = tr.Float
window_length = tr.Range(low=1, high=10**9 - 1, value=31, mode='spinner')
polynomial_order = tr.Range(low=1, high=10**9, value=2, mode='spinner')
activate = tr.Bool(False)
add_plot = tr.Button
add_creep_plot = tr.Button(desc='Creep plot of X axis array')
clear_plot = tr.Button
parse_csv_to_npy = tr.Button
generate_filtered_and_creep_npy = tr.Button
add_columns_average = tr.Button
force_max = tr.Float(100)
force_min = tr.Float(40)
min_cycle_force_range = tr.Float(50)
cutting_method = tr.Enum(
'Define min cycle range(force difference)', 'Define Max, Min')
columns_to_be_averaged = tr.List
figure = tr.Instance(mpl.figure.Figure)
log = tr.Str('')
clear_log = tr.Button
def _figure_default(self):
figure = mpl.figure.Figure(facecolor='white')
figure.set_tight_layout(True)
return figure
#=========================================================================
# File management
#=========================================================================
def _open_file_csv_fired(self):
try:
self.reset()
""" Handles the user clicking the 'Open...' button.
"""
extns = ['*.csv', ] # seems to handle only one extension...
wildcard = '|'.join(extns)
dialog = FileDialog(title='Select text file',
action='open', wildcard=wildcard,
default_path=self.file_csv)
result = dialog.open()
""" Test if the user opened a file to avoid throwing an exception if he
doesn't """
if result == OK:
self.file_csv = dialog.path
else:
return
""" Filling x_axis and y_axis with values """
headers_array = np.array(
pd.read_csv(
self.file_csv, delimiter=self.delimiter, decimal=self.decimal,
nrows=1, header=None
)
)[0]
for i in range(len(headers_array)):
headers_array[i] = self.get_valid_file_name(headers_array[i])
self.columns_headers_list = list(headers_array)
""" Saving file name and path and creating NPY folder """
dir_path = os.path.dirname(self.file_csv)
self.npy_folder_path = os.path.join(dir_path, 'NPY')
if os.path.exists(self.npy_folder_path) == False:
os.makedirs(self.npy_folder_path)
self.file_name = os.path.splitext(
os.path.basename(self.file_csv))[0]
except Exception as e:
self.deal_with_exception(e)
def _parse_csv_to_npy_fired(self):
# Run method on different thread so GUI doesn't freeze
#thread = Thread(target = threaded_function, function_args = (10,))
thread = Thread(target=self.parse_csv_to_npy_fired)
thread.start()
def parse_csv_to_npy_fired(self):
try:
self.print_custom('Parsing csv into npy files...')
for i in range(len(self.columns_headers_list) -
len(self.columns_to_be_averaged)):
current_column_name = self.columns_headers_list[i]
column_array = np.array(pd.read_csv(
self.file_csv, delimiter=self.delimiter, decimal=self.decimal,
skiprows=self.skip_first_rows, usecols=[i]))
if current_column_name == self.time_column and \
self.take_time_from_time_column == False:
column_array = np.arange(start=0.0,
stop=len(column_array) /
self.records_per_second,
step=1.0 / self.records_per_second)
np.save(os.path.join(self.npy_folder_path, self.file_name +
'_' + current_column_name + '.npy'),
column_array)
""" Exporting npy arrays of averaged columns """
for columns_names in self.columns_to_be_averaged:
temp = np.zeros((1))
for column_name in columns_names:
temp = temp + np.load(os.path.join(self.npy_folder_path,
self.file_name +
'_' + column_name +
'.npy')).flatten()
avg = temp / len(columns_names)
avg_file_suffex = self.get_suffex_for_columns_to_be_averaged(
columns_names)
np.save(os.path.join(self.npy_folder_path, self.file_name +
'_' + avg_file_suffex + '.npy'), avg)
self.print_custom('Finsihed parsing csv into npy files.')
except Exception as e:
self.deal_with_exception(e)
def get_suffex_for_columns_to_be_averaged(self, columns_names):
suffex_for_saved_file_name = 'avg_' + '_'.join(columns_names)
return suffex_for_saved_file_name
def get_valid_file_name(self, original_file_name):
valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits)
new_valid_file_name = ''.join(
c for c in original_file_name if c in valid_chars)
return new_valid_file_name
def _clear_plot_fired(self):
self.figure.clear()
self.data_changed = True
def _add_columns_average_fired(self):
try:
columns_average = ColumnsAverage()
for name in self.columns_headers_list:
columns_average.columns.append(Column(column_name=name))
# kind='modal' pauses the implementation until the window is closed
columns_average.configure_traits(kind='modal')
columns_to_be_averaged_temp = []
for i in columns_average.columns:
if i.selected:
columns_to_be_averaged_temp.append(i.column_name)
if columns_to_be_averaged_temp: # If it's not empty
self.columns_to_be_averaged.append(columns_to_be_averaged_temp)
avg_file_suffex = self.get_suffex_for_columns_to_be_averaged(
columns_to_be_averaged_temp)
self.columns_headers_list.append(avg_file_suffex)
except Exception as e:
self.deal_with_exception(e)
def _generate_filtered_and_creep_npy_fired(self):
# Run method on different thread so GUI doesn't freeze
#thread = Thread(target = threaded_function, function_args = (10,))
thread = Thread(target=self.generate_filtered_and_creep_npy_fired)
thread.start()
def generate_filtered_and_creep_npy_fired(self):
try:
if self.npy_files_exist(os.path.join(
self.npy_folder_path, self.file_name + '_' + self.force_column
+ '.npy')) == False:
return
self.print_custom('Generating filtered and creep files...')
# 1- Export filtered force
force = np.load(os.path.join(self.npy_folder_path,
self.file_name + '_' + self.force_column
+ '.npy')).flatten()
peak_force_before_cycles_index = np.where(
abs((force)) > abs(self.peak_force_before_cycles))[0][0]
force_ascending = force[0:peak_force_before_cycles_index]
force_rest = force[peak_force_before_cycles_index:]
force_max_indices, force_min_indices = self.get_array_max_and_min_indices(
force_rest)
force_max_min_indices = np.concatenate(
(force_min_indices, force_max_indices))
force_max_min_indices.sort()
force_rest_filtered = force_rest[force_max_min_indices]
force_filtered = np.concatenate(
(force_ascending, force_rest_filtered))
np.save(os.path.join(self.npy_folder_path, self.file_name +
'_' + self.force_column + '_filtered.npy'),
force_filtered)
# 2- Export filtered displacements
for i in range(0, len(self.columns_headers_list)):
if self.columns_headers_list[i] != self.force_column and \
self.columns_headers_list[i] != self.time_column:
disp = np.load(os.path.join(self.npy_folder_path, self.file_name
+ '_' +
self.columns_headers_list[i]
+ '.npy')).flatten()
disp_ascending = disp[0:peak_force_before_cycles_index]
disp_rest = disp[peak_force_before_cycles_index:]
if self.activate == True:
disp_ascending = savgol_filter(
disp_ascending, window_length=self.window_length,
polyorder=self.polynomial_order)
disp_rest_filtered = disp_rest[force_max_min_indices]
filtered_disp = np.concatenate(
(disp_ascending, disp_rest_filtered))
np.save(os.path.join(self.npy_folder_path, self.file_name + '_'
+ self.columns_headers_list[i] +
'_filtered.npy'), filtered_disp)
# 3- Export creep for displacements
# Cutting unwanted max min values to get correct full cycles and remove
# false min/max values caused by noise
if self.cutting_method == "Define Max, Min":
force_max_indices_cutted, force_min_indices_cutted = \
self.cut_indices_of_min_max_range(force_rest,
force_max_indices,
force_min_indices,
self.force_max,
self.force_min)
elif self.cutting_method == "Define min cycle range(force difference)":
force_max_indices_cutted, force_min_indices_cutted = \
self.cut_indices_of_defined_range(force_rest,
force_max_indices,
force_min_indices,
self.min_cycle_force_range)
self.print_custom("Cycles number= ", len(force_min_indices))
self.print_custom("Cycles number after cutting fake cycles = ",
len(force_min_indices_cutted))
for i in range(0, len(self.columns_headers_list)):
if self.columns_headers_list[i] != self.time_column:
array = np.load(os.path.join(self.npy_folder_path, self.file_name +
'_' +
self.columns_headers_list[i]
+ '.npy')).flatten()
array_rest = array[peak_force_before_cycles_index:]
array_rest_maxima = array_rest[force_max_indices_cutted]
array_rest_minima = array_rest[force_min_indices_cutted]
np.save(os.path.join(self.npy_folder_path, self.file_name + '_' +
self.columns_headers_list[i] + '_max.npy'), array_rest_maxima)
np.save(os.path.join(self.npy_folder_path, self.file_name + '_' +
self.columns_headers_list[i] + '_min.npy'), array_rest_minima)
self.print_custom('Filtered and creep npy files are generated.')
except Exception as e:
self.deal_with_exception(e)
def cut_indices_of_min_max_range(self, array, max_indices, min_indices,
range_upper_value, range_lower_value):
cutted_max_indices = []
cutted_min_indices = []
for max_index in max_indices:
if abs(array[max_index]) > abs(range_upper_value):
cutted_max_indices.append(max_index)
for min_index in min_indices:
if abs(array[min_index]) < abs(range_lower_value):
cutted_min_indices.append(min_index)
return cutted_max_indices, cutted_min_indices
def cut_indices_of_defined_range(self, array, max_indices, min_indices, range_):
cutted_max_indices = []
cutted_min_indices = []
for max_index, min_index in zip(max_indices, min_indices):
if abs(array[max_index] - array[min_index]) > range_:
cutted_max_indices.append(max_index)
cutted_min_indices.append(min_index)
if max_indices.size > min_indices.size:
cutted_max_indices.append(max_indices[-1])
elif min_indices.size > max_indices.size:
cutted_min_indices.append(min_indices[-1])
return cutted_max_indices, cutted_min_indices
def get_array_max_and_min_indices(self, input_array):
# Checking dominant sign
positive_values_count = np.sum(np.array(input_array) >= 0)
negative_values_count = input_array.size - positive_values_count
# Getting max and min indices
if (positive_values_count > negative_values_count):
force_max_indices = self.get_max_indices(input_array)
force_min_indices = self.get_min_indices(input_array)
else:
force_max_indices = self.get_min_indices(input_array)
force_min_indices = self.get_max_indices(input_array)
return force_max_indices, force_min_indices
def get_max_indices(self, a):
# This method doesn't qualify first and last elements as max
max_indices = []
i = 1
while i < a.size - 1:
previous_element = a[i - 1]
# Skip repeated elements and record previous element value
first_repeated_element = True
while a[i] == a[i + 1] and i < a.size - 1:
if first_repeated_element:
previous_element = a[i - 1]
first_repeated_element = False
if i < a.size - 2:
i += 1
else:
break
if a[i] > a[i + 1] and a[i] > previous_element:
max_indices.append(i)
i += 1
return np.array(max_indices)
def get_min_indices(self, a):
# This method doesn't qualify first and last elements as min
min_indices = []
i = 1
while i < a.size - 1:
previous_element = a[i - 1]
# Skip repeated elements and record previous element value
first_repeated_element = True
while a[i] == a[i + 1]:
if first_repeated_element:
previous_element = a[i - 1]
first_repeated_element = False
if i < a.size - 2:
i += 1
else:
break
if a[i] < a[i + 1] and a[i] < previous_element:
min_indices.append(i)
i += 1
return np.array(min_indices)
def _activate_changed(self):
if self.activate == False:
self.old_peak_force_before_cycles = self.peak_force_before_cycles
self.peak_force_before_cycles = 0
else:
self.peak_force_before_cycles = self.old_peak_force_before_cycles
def _window_length_changed(self, new):
if new <= self.polynomial_order:
dialog = MessageDialog(
title='Attention!',
message='Window length must be bigger than polynomial order.')
dialog.open()
if new % 2 == 0 or new <= 0:
dialog = MessageDialog(
title='Attention!',
message='Window length must be odd positive integer.')
dialog.open()
def _polynomial_order_changed(self, new):
if new >= self.window_length:
dialog = MessageDialog(
title='Attention!',
message='Polynomial order must be smaller than window length.')
dialog.open()
#=========================================================================
# Plotting
#=========================================================================
def _plot_settings_btn_fired(self):
try:
self.plot_settings.configure_traits(kind='modal')
except Exception as e:
self.deal_with_exception(e)
def npy_files_exist(self, path):
if os.path.exists(path) == True:
return True
else:
# TODO fix this
self.print_custom(
'Please parse csv file to generate npy files first.')
# dialog = MessageDialog(
# title='Attention!',
# message='Please parse csv file to generate npy files first.')
# dialog.open()
return False
def filtered_and_creep_npy_files_exist(self, path):
if os.path.exists(path) == True:
return True
else:
# TODO fix this
self.print_custom(
'Please generate filtered and creep npy files first.')
# dialog = MessageDialog(
# title='Attention!',
# message='Please generate filtered and creep npy files first.')
# dialog.open()
return False
data_changed = tr.Event
def _add_plot_fired(self):
# Run method on different thread so GUI doesn't freeze
#thread = Thread(target = threaded_function, function_args = (10,))
thread = Thread(target=self.add_plot_fired)
thread.start()
def add_plot_fired(self):
try:
if self.apply_filters:
if self.filtered_and_creep_npy_files_exist(os.path.join(
self.npy_folder_path, self.file_name + '_' + self.x_axis
+ '_filtered.npy')) == False:
return
x_axis_name = self.x_axis + '_filtered'
y_axis_name = self.y_axis + '_filtered'
self.print_custom('Loading npy files...')
# when mmap_mode!=None, the array will be loaded as 'numpy.memmap'
# object which doesn't load the array to memory until it's
# indexed
x_axis_array = np.load(os.path.join(self.npy_folder_path,
self.file_name + '_' + self.x_axis
+ '_filtered.npy'), mmap_mode='r')
y_axis_array = np.load(os.path.join(self.npy_folder_path,
self.file_name + '_' + self.y_axis
+ '_filtered.npy'), mmap_mode='r')
else:
if self.npy_files_exist(os.path.join(
self.npy_folder_path, self.file_name + '_' + self.x_axis
+ '.npy')) == False:
return
x_axis_name = self.x_axis
y_axis_name = self.y_axis
self.print_custom('Loading npy files...')
# when mmap_mode!=None, the array will be loaded as 'numpy.memmap'
# object which doesn't load the array to memory until it's
# indexed
x_axis_array = np.load(os.path.join(self.npy_folder_path,
self.file_name + '_' + self.x_axis
+ '.npy'), mmap_mode='r')
y_axis_array = np.load(os.path.join(self.npy_folder_path,
self.file_name + '_' + self.y_axis
+ '.npy'), mmap_mode='r')
if self.plot_settings_active:
print(self.plot_settings.first_rows)
print(self.plot_settings.distance)
print(self.plot_settings.num_of_rows_after_each_distance)
print(np.size(x_axis_array))
indices = self.get_indices_array(np.size(x_axis_array),
self.plot_settings.first_rows,
self.plot_settings.distance,
self.plot_settings.num_of_rows_after_each_distance)
x_axis_array = self.x_axis_multiplier * x_axis_array[indices]
y_axis_array = self.y_axis_multiplier * y_axis_array[indices]
else:
x_axis_array = self.x_axis_multiplier * x_axis_array
y_axis_array = self.y_axis_multiplier * y_axis_array
self.print_custom('Adding Plot...')
mpl.rcParams['agg.path.chunksize'] = 10000
ax = self.figure.add_subplot(1, 1, 1)
ax.set_xlabel(x_axis_name)
ax.set_ylabel(y_axis_name)
ax.plot(x_axis_array, y_axis_array, 'k',
linewidth=1.2, color=np.random.rand(3), label=self.file_name +
', ' + x_axis_name)
ax.legend()
self.data_changed = True
self.print_custom('Finished adding plot.')
except Exception as e:
self.deal_with_exception(e)
def _add_creep_plot_fired(self):
# Run method on different thread so GUI doesn't freeze
#thread = Thread(target = threaded_function, function_args = (10,))
thread = Thread(target=self.add_creep_plot_fired)
thread.start()
def add_creep_plot_fired(self):
try:
if self.filtered_and_creep_npy_files_exist(os.path.join(
self.npy_folder_path, self.file_name + '_' + self.x_axis
+ '_max.npy')) == False:
return
self.print_custom('Loading npy files...')
disp_max = self.x_axis_multiplier * \
np.load(os.path.join(self.npy_folder_path,
self.file_name + '_' + self.x_axis + '_max.npy'))
disp_min = self.x_axis_multiplier * \
np.load(os.path.join(self.npy_folder_path,
self.file_name + '_' + self.x_axis + '_min.npy'))
complete_cycles_number = disp_max.size
self.print_custom('Adding creep-fatigue plot...')
mpl.rcParams['agg.path.chunksize'] = 10000
ax = self.figure.add_subplot(1, 1, 1)
ax.set_xlabel('Cycles number')
ax.set_ylabel(self.x_axis)
if self.plot_every_nth_point > 1:
disp_max = disp_max[0::self.plot_every_nth_point]
disp_min = disp_min[0::self.plot_every_nth_point]
if self.smooth:
# Keeping the first item of the array and filtering the rest
disp_max = np.concatenate((
np.array([disp_max[0]]),
savgol_filter(disp_max[1:],
window_length=self.window_length,
polyorder=self.polynomial_order)
))
disp_min = np.concatenate((
np.array([disp_min[0]]),
savgol_filter(disp_min[1:],
window_length=self.window_length,
polyorder=self.polynomial_order)
))
if self.normalize_cycles:
ax.plot(np.linspace(0, 1., disp_max.size), disp_max,
'k', linewidth=1.2, color=np.random.rand(3), label='Max'
+ ', ' + self.file_name + ', ' + self.x_axis)
ax.plot(np.linspace(0, 1., disp_min.size), disp_min,
'k', linewidth=1.2, color=np.random.rand(3), label='Min'
+ ', ' + self.file_name + ', ' + self.x_axis)
else:
ax.plot(np.linspace(0, complete_cycles_number,
disp_max.size), disp_max,
'k', linewidth=1.2, color=np.random.rand(3), label='Max'
+ ', ' + self.file_name + ', ' + self.x_axis)
ax.plot(np.linspace(0, complete_cycles_number,
disp_min.size), disp_min,
'k', linewidth=1.2, color=np.random.rand(3), label='Min'
+ ', ' + self.file_name + ', ' + self.x_axis)
ax.legend()
self.data_changed = True
self.print_custom('Finished adding creep-fatigue plot.')
except Exception as e:
self.deal_with_exception(e)
def get_indices_array(self,
array_size,
first_rows,
distance,
num_of_rows_after_each_distance):
result_1 = np.arange(first_rows)
result_2 = np.arange(start=first_rows, stop=array_size,
step=distance + num_of_rows_after_each_distance)
result_2_updated = np.array([], dtype=np.int_)
for result_2_value in result_2:
data_slice = np.arange(result_2_value, result_2_value +
num_of_rows_after_each_distance)
result_2_updated = np.concatenate((result_2_updated, data_slice))
result = np.concatenate((result_1, result_2_updated))
return result
def reset(self):
self.columns_to_be_averaged = []
self.log = ''
def print_custom(self, *input_args):
print(*input_args)
if self.log == '':
self.log = ''.join(str(e) for e in list(input_args))
else:
self.log = self.log + '\n' + \
''.join(str(e) for e in list(input_args))
def deal_with_exception(self, e):
self.print_custom('SOMETHING WENT WRONG!')
self.print_custom('--------- Error message: ---------')
self.print_custom(traceback.format_exc())
self.print_custom('----------------------------------')
def _clear_log_fired(self):
self.log = ''
#=========================================================================
# Configuration of the view
#=========================================================================
traits_view = ui.View(
ui.HSplit(
ui.VSplit(
ui.VGroup(
ui.VGroup(
ui.Item('decimal'),
ui.Item('delimiter'),
ui.HGroup(
ui.UItem('open_file_csv', has_focus=True),
ui.UItem('file_csv', style='readonly', width=0.1)),
label='Importing csv file',
show_border=True)),
ui.VGroup(
ui.VGroup(
ui.VGroup(
ui.Item('take_time_from_time_column'),
ui.Item('time_column',
enabled_when='take_time_from_time_column == True'),
ui.Item('records_per_second',
enabled_when='take_time_from_time_column == False'),
label='Time calculation',
show_border=True),
ui.UItem('add_columns_average'),
ui.Item('skip_first_rows'),
ui.UItem('parse_csv_to_npy', resizable=True),
label='Processing csv file',
show_border=True)),
ui.VGroup(
ui.VGroup(
ui.HGroup(ui.Item('x_axis'), ui.Item(
'x_axis_multiplier')),
ui.HGroup(ui.Item('y_axis'), ui.Item(
'y_axis_multiplier')),
ui.VGroup(
ui.HGroup(ui.UItem('add_plot'),
ui.Item('apply_filters'),
ui.Item('plot_settings_btn',
label='Settings',
show_label=False,
enabled_when='plot_settings_active == True'),
ui.Item('plot_settings_active',
show_label=False)
),
show_border=True,
label='Plotting X axis with Y axis'
),
ui.VGroup(
ui.HGroup(ui.UItem('add_creep_plot'),
ui.VGroup(
ui.Item('normalize_cycles'),
ui.Item('smooth'),
ui.Item('plot_every_nth_point'))
),
show_border=True,
label='Plotting Creep-fatigue of X axis variable'
),
ui.UItem('clear_plot', resizable=True),
show_border=True,
label='Plotting'))
),
ui.VGroup(
ui.Item('force_column'),
ui.VGroup(ui.VGroup(
ui.Item('window_length'),
ui.Item('polynomial_order'),
enabled_when='activate == True or smooth == True'),
show_border=True,
label='Smoothing parameters (Savitzky-Golay filter):'
),
ui.VGroup(ui.VGroup(
ui.Item('activate'),
ui.Item('peak_force_before_cycles',
enabled_when='activate == True')
),
show_border=True,
label='Smooth ascending branch for all displacements:'
),
ui.VGroup(ui.Item('cutting_method'),
ui.VGroup(ui.Item('force_max'),
ui.Item('force_min'),
label='Max, Min:',
show_border=True,
enabled_when='cutting_method == "Define Max, Min"'),
ui.VGroup(ui.Item('min_cycle_force_range'),
label='Min cycle force range:',
show_border=True,
enabled_when='cutting_method == "Define min cycle range(force difference)"'),
show_border=True,
label='Cut fake cycles for creep:'),
ui.VSplit(
ui.UItem('generate_filtered_and_creep_npy'),
ui.VGroup(
ui.Item('log',
width=0.1, style='custom'),
ui.UItem('clear_log'))),
show_border=True,
label='Filters'
),
ui.UItem('figure', editor=MPLFigureEditor(),
resizable=True,
springy=True,
width=0.8,
label='2d plots')
),
title='High-cycle fatigue tool',
resizable=True,
width=0.85,
height=0.7
)
class config(HasTraits):
uuid = traits.Str(desc="UUID")
desc = traits.Str(desc="Workflow Description")
# Directories
working_dir = Directory(mandatory=True,
desc="Location of the Nipype working directory")
sink_dir = Directory(os.path.abspath('.'),
mandatory=True,
desc="Location where the BIP will store the results")
crash_dir = Directory(mandatory=False,
desc="Location to store crash files")
json_sink = Directory(mandatory=False, desc="Location to store json_files")
surf_dir = Directory(mandatory=True, desc="Freesurfer subjects directory")
save_script_only = traits.Bool(False)
# Execution
run_using_plugin = Bool(
False,
usedefault=True,
desc="True to run pipeline with plugin, False to run serially")
plugin = traits.Enum("PBS",
"MultiProc",
"SGE",
"Condor",
usedefault=True,
desc="plugin to use, if run_using_plugin=True")
plugin_args = traits.Dict({"qsub_args": "-q many"},
usedefault=True,
desc='Plugin arguments.')
test_mode = Bool(
False,
mandatory=False,
usedefault=True,
desc='Affects whether where and if the workflow keeps its \
intermediary files. True to keep intermediary files. '
)
timeout = traits.Float(14.0)
# Subjects
#subjects= traits.List(traits.Str, mandatory=True, usedefault=True,
# desc="Subject id's. Note: These MUST match the subject id's in the \
# Freesurfer directory. For simplicity, the subject id's should \
# also match with the location of individual functional files.")
datagrabber = traits.Instance(Data, ())
# First Level
interscan_interval = traits.Float()
film_threshold = traits.Float()
input_units = traits.Enum('scans', 'secs')
is_sparse = traits.Bool(False)
model_hrf = traits.Bool(True)
stimuli_as_impulses = traits.Bool(True)
use_temporal_deriv = traits.Bool(True)
volumes_in_cluster = traits.Int(1)
ta = traits.Float()
tr = traits.Float()
hpcutoff = traits.Float()
scan_onset = traits.Int(0)
scale_regressors = traits.Bool(True)
#bases = traits.Dict({'dgamma':{'derivs': False}},use_default=True)
bases = traits.Dict(
{'dgamma': {
'derivs': False
}}, use_default=True
) #traits.Enum('dgamma','gamma','none'), traits.Enum(traits.Dict(traits.Enum('derivs',None), traits.Bool),None), desc="name of basis function and options e.g., {'dgamma': {'derivs': True}}")
# preprocessing info
preproc_config = traits.File(desc="preproc config file")
use_compcor = traits.Bool(desc="use noise components from CompCor")
#advanced_options
use_advanced_options = Bool(False)
advanced_options = traits.Code()
class Trainer(t.HasStrictTraits):
model: models.BaseNet = t.Instance(torch.nn.Module, transient=True)
def _model_default(self):
# Merge 'base config' (if requested) and any overrides in 'model_config'
if self.base_config:
model_config = get_ref_arch(self.base_config)
else:
model_config = {}
if self.model_config:
model_config.update(self.model_config)
if self.data_spec:
model_config.update(
{
"input_channels": self.data_spec["input_channels"],
"num_output_classes": [
s["num_classes"] for s in self.data_spec["output_spec"]
],
}
)
# create model accordingly
model_class = getattr(models, self.model_class)
return model_class(**model_config)
base_config: str = t.Str()
model_config: dict = t.Dict()
model_class: str = t.Enum("FilterNet", "DeepConvLSTM")
lr_exp: float = t.Float(-3.0)
batch_size: int = t.Int()
win_len: int = t.Int(512)
n_samples_per_batch: int = t.Int(5000)
train_step: int = t.Int(16)
seed: int = t.Int()
decimation: int = t.Int(1)
optim_type: str = t.Enum(["Adam", "SGD, RMSprop"])
loss_func: str = t.Enum(["cross_entropy", "binary_cross_entropy"])
patience: int = t.Int(10)
lr_decay: float = t.Float(0.95)
weight_decay: float = t.Float(1e-4)
alpha: float = t.Float(0.99)
momentum: float = t.Float(0.25)
validation_fold: int = t.Int()
epoch_size: float = t.Float(2.0)
y_cols: str = t.Str()
sensor_subset: str = t.Str()
has_null_class: bool = t.Bool()
def _has_null_class_default(self):
return self.data_spec["output_spec"][0]["classes"][0] in ("", "Null")
predict_null_class: bool = t.Bool(True)
_class_weights: torch.Tensor = t.Instance(torch.Tensor)
def __class_weights_default(self):
# Not weights for now because didn't seem to increase things significantly and
# added yet another hyper-parameter. Using zero didn't seem to work well.
if False and self.has_null_class and not self.predict_null_class:
cw = torch.ones(self.model.num_output_classes, device=self.device)
cw[0] = 0.01
cw /= cw.sum()
return cw
return None
dataset: str = t.Enum(
["opportunity", "smartphone_hapt", "har", "intention_recognition"]
)
name: str = t.Str()
def _name_default(self):
import time
modelstr = self.model.__class__.__name__
timestr = time.strftime("%Y%m%d-%H%M%S")
return f"{modelstr}_{timestr}"
model_path: str = t.Str()
def _model_path_default(self):
return f"saved_models/{self.name}/"
data_spec: dict = t.Any()
epoch_iters: int = t.Int(0)
train_state: TrainState = t.Instance(TrainState, ())
cp_iter: int = t.Int()
cuda: bool = t.Bool(transient=True)
def _cuda_default(self):
return torch.cuda.is_available()
device: str = t.Str(transient=True)
def _device_default(self):
return "cuda" if self.cuda else "cpu"
dl_train: DataLoader = t.Instance(DataLoader, transient=True)
def _dl_train_default(self):
return self._get_dl("train")
dl_val: DataLoader = t.Instance(DataLoader, transient=True)
def _dl_val_default(self):
return self._get_dl("val")
dl_test: DataLoader = t.Instance(DataLoader, transient=True)
def _dl_test_default(self):
return self._get_dl("test")
def _get_dl(self, s):
if self.dataset == "opportunity":
from filternet.datasets.opportunity import get_x_y_contig
elif self.dataset == "smartphone_hapt":
from filternet.datasets.smartphone_hapt import get_x_y_contig
elif self.dataset == "har":
from filternet.datasets.har import get_x_y_contig
elif self.dataset == "intention_recognition":
from filternet.datasets.intention_recognition import get_x_y_contig
else:
raise ValueError(f"Unknown dataset {self.dataset}")
kwargs = {}
if self.y_cols:
kwargs["y_cols"] = self.y_cols
if self.sensor_subset:
kwargs["sensor_subset"] = self.sensor_subset
Xc, ycs, data_spec = get_x_y_contig(s, **kwargs)
if s == "train":
# Training shuffles, and we set epoch size to length of the dataset. We can set train_step as
# small as we want to get more windows; we'll only run len(Sc)/win_len of them in each training
# epoch.
self.epoch_iters = int(len(Xc) / self.decimation)
X, ys = sliding_window_x_y(
Xc, ycs, win_len=self.win_len, step=self.train_step, shuffle=False
)
# Set the overall data spec using the training set,
# and modify later if more info is needed.
self.data_spec = data_spec
else:
# Val and test data are not shuffled.
# Each point is inferred ~twice b/c step = win_len/2
X, ys = sliding_window_x_y(
Xc,
ycs,
win_len=self.win_len,
step=int(self.win_len / 2),
shuffle=False, # Cannot be true with windows
)
dl = DataLoader(
TensorDataset(torch.Tensor(X), *[torch.Tensor(y).long() for y in ys]),
batch_size=self.batch_size,
shuffle=True if s == "train" else False,
)
return dl
def _batch_size_default(self):
batch_size = int(self.n_samples_per_batch / self.win_len)
print(f"Batch size: {batch_size}")
return batch_size
optimizer = t.Any(transient=True)
def _optimizer_default(self):
if self.optim_type == "SGD":
optimizer = torch.optim.SGD(
self.model.parameters(),
lr=10 ** (self.lr_exp),
momentum=self.momentum,
weight_decay=self.weight_decay,
)
elif self.optim_type == "Adam":
optimizer = torch.optim.Adam(
self.model.parameters(),
lr=10 ** (self.lr_exp),
weight_decay=self.weight_decay,
amsgrad=True,
)
elif self.optim_type == "RMSprop":
optimizer = torch.optim.RMSprop(
self.model.parameters(),
lr=10 ** (self.lr_exp),
alpha=self.alpha,
weight_decay=self.weight_decay,
momentum=self.momentum,
)
else:
raise NotImplementedError(self.optim_type)
return optimizer
iteration: int = t.Property(t.Int)
def _get_iteration(self):
return len(self.train_state.epoch_records) + 1
lr_scheduler = t.Any(transient=True)
def _lr_scheduler_default(self):
lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(
self.optimizer, self.lr_decay # , last_epoch=self._iteration
)
# If this is being re-instantiated in mid-training, then we must
# iterate scheduler forward to match the training step.
for i in range(self.iteration):
if self.lr_decay != 1:
lr_scheduler.step()
return lr_scheduler
#####
# Training Methods
##
def _train_batch(self, data, targets):
self.optimizer.zero_grad()
loss, output, _targets, _ = self._run_model_on_batch(data, targets)
loss.backward()
self.optimizer.step()
# if self.max_lr:
# self.lr_scheduler.step()
return loss, output, _targets
def _run_model_on_batch(self, data, targets):
targets = torch.stack(targets)
if self.cuda:
data, targets = data.cuda(), targets.cuda()
output = self.model(data)
_targets = self.model.transform_targets(targets, one_hot=False)
if self.loss_func == "cross_entropy":
_losses = [
F.cross_entropy(o, t, weight=self._class_weights)
for o, t in zip(output, _targets)
]
loss = sum(_losses)
elif self.loss_func == "binary_cross_entropy":
_targets_onehot = self.model.transform_targets(targets, one_hot=True)
_losses = [
F.binary_cross_entropy_with_logits(o, t, weight=self._class_weights)
for o, t in zip(output, _targets_onehot)
]
loss = sum(_losses)
else:
raise NotImplementedError(self.loss)
# Assume only 1 output:
return loss, output[0], _targets[0], _losses[0]
def _calc_validation_loss(self):
running_loss = 0
self.model.eval()
with torch.no_grad():
for batch_idx, (data, *targets) in enumerate(self.dl_val):
loss, _, _, _ = self._run_model_on_batch(data, targets)
running_loss += loss.item() * data.size(0)
return running_loss / len(self.dl_val.dataset)
def _train_epoch(self):
self.model.train()
train_losses = []
train_accs = []
for batch_idx, (data, *targets) in enumerate(self.dl_train):
if (
batch_idx * data.shape[0] * data.shape[2]
> self.epoch_iters * self.epoch_size
):
# we've effectively finished one epoch worth of data; break!
break
batch_loss, batch_output, batch_targets = self._train_batch(data, targets)
train_losses.append(batch_loss.detach().cpu().item())
batch_preds = torch.argmax(batch_output, 1, False)
train_accs.append(
(batch_preds == batch_targets).detach().cpu().float().mean().item()
)
if self.lr_decay != 1:
self.lr_scheduler.step()
return EpochMetrics(loss=np.mean(train_losses), acc=np.mean(train_accs))
def _val_epoch(self):
return self._eval_epoch(self.dl_val)
def _eval_epoch(self, data_loader):
# Validation
self.model.eval()
losses = []
outputs = []
targets = []
with torch.no_grad():
for batch_idx, (data, *target) in enumerate(data_loader):
(
batch_loss,
batch_output,
batch_targets,
train_losses,
) = self._run_model_on_batch(data, target)
losses.append(batch_loss.detach().cpu().item())
outputs.append(
torch.argmax(batch_output, 1, False)
.detach()
.cpu()
.data.numpy()
.flatten()
)
targets.append(batch_targets.detach().cpu().data.numpy().flatten())
targets = np.hstack(targets)
outputs = np.hstack(outputs)
acc = sklearn.metrics.accuracy_score(targets, outputs)
f1 = sklearn.metrics.f1_score(targets, outputs, average="weighted")
return EpochMetrics(loss=np.mean(losses), acc=acc, f1=f1)
def init_data(self):
# Initiate loading of datasets, model
_, _, _ = self.dl_train, self.dl_val, self.dl_test
_ = self.model
def init_train(self):
# initialization
if self.seed:
torch.manual_seed(self.seed)
if self.cuda:
if self.seed:
torch.cuda.manual_seed(self.seed)
self.model.to(self.device)
def train_one_epoch(self, verbose=True) -> EpochRecord:
""" traing a single epoch -- method tailored to the Ray.tune methodology."""
epoch_record = EpochRecord(epoch=len(self.train_state.epoch_records))
self.train_state.epoch_records.append(epoch_record)
with Timer("Train Epoch", log_output=verbose) as t:
epoch_record.train = self._train_epoch()
epoch_record.iter_s_cpu = t.interval_cpu
epoch_record.iter_s_wall = t.interval_wall
epoch_record.lr = self.optimizer.param_groups[0]["lr"]
with Timer("Val Epoch", log_output=verbose):
epoch_record.val = self._val_epoch()
df = self.train_state.to_df()
# Early stopping / checkpointing implementation
df["raw_metric"] = df.val_loss / df.val_f1
df["ewma_smoothed_loss"] = (
df["raw_metric"].ewm(ignore_na=False, halflife=3).mean()
)
df["instability_penalty"] = (
df["raw_metric"].rolling(5, min_periods=3).std().fillna(0.75)
)
stopping_metric = df["stopping_metric"] = (
df["ewma_smoothed_loss"] + df["instability_penalty"]
)
epoch_record.stopping_metric = df["stopping_metric"].iloc[-1]
idx_this_iter = stopping_metric.index.max()
idx_best_yet = stopping_metric.idxmin()
self.train_state.best_sm = df.loc[idx_best_yet, "stopping_metric"]
self.train_state.best_loss = df.loc[idx_best_yet, "val_loss"]
self.train_state.best_f1 = df.loc[idx_best_yet, "val_f1"]
if idx_best_yet == idx_this_iter:
# Best yet! Checkpoint.
epoch_record.should_checkpoint = True
self.cp_iter = epoch_record.epoch
else:
if self.patience is not None:
patience_counter = idx_this_iter - idx_best_yet
assert patience_counter >= 0
if patience_counter > self.patience:
if verbose:
print(
f"Early stop! Out of patience ( {patience_counter} > {self.patience} )"
)
epoch_record.done = True
if verbose:
self.print_train_summary()
return epoch_record
def train(self, max_epochs=50, verbose=True):
""" A pretty standard training loop, constrained to stop in `max_epochs` but may stop early if our
custom stopping metric does not improve for `self.patience` epochs. Always checkpoints
when a new best stopping_metric is achieved. An alternative to using
ray.tune for training."""
self.init_data()
self.init_train()
while True:
epoch_record = self.train_one_epoch(verbose=verbose)
if epoch_record.should_checkpoint:
last_cp = self._save()
if verbose:
print(f"<<<< Checkpointed ({last_cp}) >>>")
if epoch_record.done:
break
if epoch_record.epoch >= max_epochs:
break
# Save trainer state, but not model"
self._save(save_model=False)
if verbose:
print(self.model_path)
def print_train_summary(self):
df = self.train_state.to_df()
with pd.option_context(
"display.max_rows",
100,
"display.max_columns",
100,
"display.precision",
3,
"display.width",
180,
):
print(df.drop(["done"], axis=1, errors="ignore"))
def _save(self, checkpoint_dir=None, save_model=True, save_trainer=True):
""" Saves/checkpoints model state and training state to disk. """
if checkpoint_dir is None:
checkpoint_dir = self.model_path
else:
self.model_path = checkpoint_dir
os.makedirs(checkpoint_dir, exist_ok=True)
# save model params
model_path = os.path.join(checkpoint_dir, "model.pth")
trainer_path = os.path.join(checkpoint_dir, "trainer.pth")
if save_model:
torch.save(self.model.state_dict(), model_path)
if save_trainer:
with open(trainer_path, "wb") as f:
pickle.dump(self, f)
return checkpoint_dir
def _restore(self, checkpoint_dir=None):
""" Restores model state and training state from disk. """
if checkpoint_dir is None:
checkpoint_dir = self.model_path
model_path = os.path.join(checkpoint_dir, "model.pth")
trainer_path = os.path.join(checkpoint_dir, "trainer.pth")
# Reconstitute old trainer and copy state to this trainer.
with open(trainer_path, "rb") as f:
other_trainer = pickle.load(f)
self.__setstate__(other_trainer.__getstate__())
# Load model (after loading state in case we need to re-initialize model from config)
self.model.load_state_dict(torch.load(model_path, map_location=self.device))
# Be careful to reinitialize optimizer and lr scheduler
self.optimizer = self._optimizer_default()
self.lr_scheduler = self._lr_scheduler_default()
class BCSliceI(bu.Model):
'''
Implements the IBC functionality for a constrained dof.
'''
name = tr.Str('<unnamed>')
var = tr.Enum('u', 'f')
slice = tr.Instance(FEGridNodeSlice)
link_slice = tr.Instance(FEGridNodeSlice)
bcdof_list = tr.List(BCDof)
def reset(self):
self.bcdof_list = []
link_coeffs = tr.List(tr.Float)
'''
List of dofs that determine the value of the current dof
If this list is empty, then the current dof is
prescribed. Otherwise, the dof value is given by the
linear combination of DOFs in the list (see the example below)
link_dofs = List( Int )
Coefficients of the linear combination of DOFs specified in the
above list.
'''
dims = tr.List(tr.Int)
_link_dims = tr.List(tr.Int)
link_dims = tr.Property(tr.List(tr.Int))
def _get_link_dims(self):
if len(self._link_dims) == 0:
return self.dims
else:
return self._link_dims
def _set_link_dims(self, link_dims):
self._link_dims = link_dims
value = tr.Float
time_function = tr.Instance(TimeFunction, ())
def _time_function_default(self):
return TFMonotonic()
space_function = tr.Instance(MFnLineArray, ())
def _space_function_default(self):
return MFnLineArray(xdata=[0, 1], ydata=[1, 1], extrapolate='diff')
def is_essential(self):
return self.var == 'u'
def is_linked(self):
return self.link_dofs != []
def is_constrained(self):
'''
Return true if a DOF is either explicitly prescribed or it depends on other DOFS.
'''
return self.is_essential() or self.is_linked()
def is_natural(self):
return self.var == 'f'
def setup(self, sctx):
'''
Locate the spatial context.f
'''
if self.link_slice == None:
for node_dofs, dof_X in zip(self.slice.dofs, self.slice.dof_X):
for dof in node_dofs[self.dims]:
self.bcdof_list.append(
BCDof(
var=self.var,
dof=dof,
value=self.value,
# link_coeffs=self.link_coeffs,
time_function=self.time_function))
else:
# apply the linked slice
n_link_nodes = len(self.link_slice.dofs.flatten())
link_dofs = self.link_dofs
if n_link_nodes == 1:
#
link_dof = self.link_slice.dofs.flatten()[0]
link_coeffs = self.link_coeffs
for node_dofs, dof_X in zip(self.slice.dofs, self.slice.dof_X):
for dof, link_dof, link_coeff in zip(
node_dofs[self.dims], link_dofs, link_coeffs):
self.bcdof_list.append(
BCDof(var=self.var,
dof=dof,
link_dofs=[link_dof],
value=self.value,
link_coeffs=[link_coeff],
time_function=self.time_function))
else:
for node_dofs, dof_X, node_link_dofs, link_dof_X in \
zip(self.slice.dofs, self.slice.dof_X,
self.link_slice.dofs, self.link_slice.dof_X):
#print('node', node_dofs, node_link_dofs)
#print('node[dims]', node_dofs[self.dims],
# node_link_dofs[self.link_dims])
for dof, link_dof, link_coeff in zip(
node_dofs[self.dims],
node_link_dofs[self.link_dims], self.link_coeffs):
#print('dof, link, coeff', dof, link_dof, link_coeff)
self.bcdof_list.append(
BCDof(var=self.var,
dof=dof,
link_dofs=[link_dof],
value=self.value,
link_coeffs=[link_coeff],
time_function=self.time_function))
def register(self, K):
'''Register the boundary condition in the equation system.
'''
for bcond in self.bcdof_list:
bcond.register(K)
def apply_essential(self, K):
for bcond in self.bcdof_list:
bcond.apply_essential(K)
def apply(self, step_flag, sctx, K, R, t_n, t_n1):
for bcond in self.bcdof_list:
bcond.apply(step_flag, sctx, K, R, t_n, t_n1)
#-------------------------------------------------------------------------
# Ccnstrained DOFs
#-------------------------------------------------------------------------
dofs = tr.Property
def _get_dofs(self):
return np.unique(self.slice.dofs[..., self.dims].flatten())
dof_X = tr.Property
def _get_dof_X(self):
return self.slice.dof_X
n_dof_nodes = tr.Property
def _get_n_dof_nodes(self):
sliceshape = self.dofs.shape
return sliceshape[0] * sliceshape[1]
#-------------------------------------------------------------------------
# Link DOFs
#-------------------------------------------------------------------------
link_dofs = tr.Property(tr.List)
def _get_link_dofs(self):
if self.link_slice != None:
return np.unique(self.link_slice.dofs[...,
self.link_dims].flatten())
else:
return []
link_dof_X = tr.Property
def _get_link_dof_X(self):
return self.link_slice.dof_X
n_link_dof_nodes = tr.Property
def _get_n_link_dof_nodes(self):
sliceshape = self.link_dofs.shape
return sliceshape[0] * sliceshape[1]
class TStep(bu.Model):
'''Manage the data and metadata of a time step within an interation loop.
'''
title = tr.Str('<unnamed>')
tloop_type = tr.Type(ITLoop)
'''Type of time loop to be used with the model
'''
#=========================================================================
# HISTORY
#=========================================================================
hist_type = tr.Type(Hist)
hist = tr.Property(tr.Instance(IHist))
r'''History representation of the model response.
'''
@tr.cached_property
def _get_hist(self):
return self.hist_type(tstep_source=self)
debug = tr.Bool(False)
t_n1 = tr.Float(0.0, auto_set=False, enter_set=True)
'''Target value of the control variable.
'''
U_n = tr.Float(0.0, auto_set=False, enter_set=True)
'''Current fundamental value of the primary variable.
'''
U_k = tr.Float(0.0, auto_set=False, enter_set=True)
'''Current trial value of the primary variable.
'''
def init_state(self):
'''Initialize state.
'''
self.U_n = 0.0
self.t_n1 = 0.0
self.U_k = 0.0
def record_state(self):
'''Provide the current state for history recording.
'''
pass
_corr_pred = tr.Property(depends_on='U_k,t_n1')
@tr.cached_property
def _get__corr_pred(self):
return self.get_corr_pred(self.U_k, self.t_n1)
R = tr.Property
def _get_R(self):
R, _ = self._corr_pred
return R
dR = tr.Property
def _get_dR(self):
_, dR = self._corr_pred
return dR
R_norm = tr.Property
def _get_R_norm(self):
R = self.R
return np.sqrt(R * R)
def make_iter(self):
d_U = self.R / self.dR
self.U_k += d_U
def make_incr(self):
'''Update the control, primary and state variables..
'''
self.U_n = self.U_k
# self.hist.record_timestep()
sim = tr.Property()
'''Launch a simulator - currently only one simulator is allowed
for a model. Mutiple might also make sense when different solvers
are to be compared. The simulator pulls the time loop type
from the model.
'''
@tr.cached_property
def _get_sim(self):
return Simulator(self)
class Vis2DField(Vis2D):
model = tr.DelegatesTo('sim')
x_file = tr.File
file_list = tr.List(tr.File)
var = tr.Str('<unnamed>')
dir = tr.Directory
def new_dir(self):
self.dir = tempfile.mkdtemp()
def setup(self):
self.new_dir()
# make a loop over the DomainState
fe_domain = self.sim.tstep.fe_domain
domain = fe_domain[2]
xdomain = domain.xmodel
r_Eia = np.einsum(
'Eira,Eia->Eir',
xdomain.T_Emra[..., :xdomain.x_Eia.shape[-1]], xdomain.x_Eia
)
file_name = 'slice_x_%s' % (self.var,)
target_file = os.path.join(
self.dir, file_name.replace('.', '_') + '.npy'
)
#print('r', r_Eia[..., :-1])
np.save(target_file, r_Eia[..., :-1])
self.x_file = target_file
def get_x_Eir(self):
return np.load(self.x_file)
def update(self):
ts = self.sim.tstep
fe_domain = self.sim.tstep.fe_domain
domain = fe_domain[2]
xdomain = domain.xmodel
U = ts.U_k
t = ts.t_n1
s_Emr = xdomain.map_U_to_field(U)
var_function = domain.tmodel.var_dict.get(self.var, None)
if var_function == None:
raise ValueError('no such variable' % self.var)
state_k = copy.deepcopy(domain.state_n)
var_k = var_function(s_Emr, ts.t_n1, **state_k)
target_file = self.filename(t)
#np.save(target_file, s_Emr)
np.save(target_file, var_k)
self.file_list.append(target_file)
def filename(self, t):
file_name = 'slice_%s_step_%008.4f' % (self.var, t)
target_file = os.path.join(
self.dir, file_name.replace('.', '_')
) + '.npy'
return target_file
def edition_widget(engine, environment):
''' Edition GUI for SPM config - see
:class:`~capsul.qt_gui.widgets.settings_editor.SettingsEditor`
'''
from soma.qt_gui.controller_widget import ScrollControllerWidget
from soma.controller import Controller
import types
import traits.api as traits
def validate_config(widget):
controller = widget.controller_widget.controller
with widget.engine.settings as session:
values = {}
if controller.directory in (None, traits.Undefined, ''):
values['directory'] = None
else:
values['directory'] = controller.directory
values['standalone'] = controller.standalone
values['version'] = controller.version
id = 'spm%s%s' % (controller.version,
'-standalone' if controller.standalone else '')
values['config_id'] = id
query = 'config_id == "%s"' % id
conf = session.config('spm', 'global', selection=query)
if conf is None:
session.new_config('spm', widget.environment, values)
else:
for k in ('directory', 'standalone', 'version'):
setattr(conf, k, values[k])
controller = Controller()
controller.add_trait(
"directory",
traits.Directory(traits.Undefined,
output=False,
desc="Directory containing SPM."))
controller.add_trait(
"standalone",
traits.Bool(True, desc="If True, use the standalone version of SPM."))
controller.add_trait(
'version',
traits.Str(traits.Undefined,
output=False,
desc='Version string for SPM: "12", "8", etc.'))
conf = engine.settings.select_configurations(environment, {'spm': 'any'})
if conf:
controller.directory = conf.get('capsul.engine.module.spm',
{}).get('directory', traits.Undefined)
controller.standalone = conf.get('capsul.engine.module.spm',
{}).get('standalone', True)
controller.version = conf.get('capsul.engine.module.spm',
{}).get('version', '12')
# TODO handle several configs
widget = ScrollControllerWidget(controller, live=True)
widget.engine = engine
widget.environment = environment
widget.accept = types.MethodType(validate_config, widget)
return widget
class DataAxis(t.HasTraits):
name = t.Str()
units = t.Str()
scale = t.Float()
offset = t.Float()
size = t.CInt()
low_value = t.Float()
high_value = t.Float()
value = t.Range('low_value', 'high_value')
low_index = t.Int(0)
high_index = t.Int()
slice = t.Instance(slice)
navigate = t.Bool(t.Undefined)
index = t.Range('low_index', 'high_index')
axis = t.Array()
continuous_value = t.Bool(False)
def __init__(self,
size,
index_in_array=None,
name=t.Undefined,
scale=1.,
offset=0.,
units=t.Undefined,
navigate=t.Undefined):
super(DataAxis, self).__init__()
self.events = Events()
self.events.index_changed = Event("""
Event that triggers when the index of the `DataAxis` changes
Triggers after the internal state of the `DataAxis` has been
updated.
Arguments:
---------
obj : The DataAxis that the event belongs to.
index : The new index
""",
arguments=["obj", 'index'])
self.events.value_changed = Event("""
Event that triggers when the value of the `DataAxis` changes
Triggers after the internal state of the `DataAxis` has been
updated.
Arguments:
---------
obj : The DataAxis that the event belongs to.
value : The new value
""",
arguments=["obj", 'value'])
self._suppress_value_changed_trigger = False
self._suppress_update_value = False
self.name = name
self.units = units
self.scale = scale
self.offset = offset
self.size = size
self.high_index = self.size - 1
self.low_index = 0
self.index = 0
self.update_axis()
self.navigate = navigate
self.axes_manager = None
self.on_trait_change(self.update_axis, ['scale', 'offset', 'size'])
self.on_trait_change(self._update_slice, 'navigate')
self.on_trait_change(self.update_index_bounds, 'size')
# The slice must be updated even if the default value did not
# change to correctly set its value.
self._update_slice(self.navigate)
def _index_changed(self, name, old, new):
self.events.index_changed.trigger(obj=self, index=self.index)
if not self._suppress_update_value:
new_value = self.axis[self.index]
if new_value != self.value:
self.value = new_value
def _value_changed(self, name, old, new):
old_index = self.index
new_index = self.value2index(new)
if self.continuous_value is False: # Only values in the grid allowed
if old_index != new_index:
self.index = new_index
if new == self.axis[self.index]:
self.events.value_changed.trigger(obj=self, value=new)
elif old_index == new_index:
new_value = self.index2value(new_index)
if new_value == old:
self._suppress_value_changed_trigger = True
try:
self.value = new_value
finally:
self._suppress_value_changed_trigger = False
elif new_value == new and not\
self._suppress_value_changed_trigger:
self.events.value_changed.trigger(obj=self, value=new)
else: # Intergrid values are allowed. This feature is deprecated
self.events.value_changed.trigger(obj=self, value=new)
if old_index != new_index:
self._suppress_update_value = True
self.index = new_index
self._suppress_update_value = False
@property
def index_in_array(self):
if self.axes_manager is not None:
return self.axes_manager._axes.index(self)
else:
raise AttributeError(
"This DataAxis does not belong to an AxesManager"
" and therefore its index_in_array attribute "
" is not defined")
@property
def index_in_axes_manager(self):
if self.axes_manager is not None:
return self.axes_manager._get_axes_in_natural_order().\
index(self)
else:
raise AttributeError(
"This DataAxis does not belong to an AxesManager"
" and therefore its index_in_array attribute "
" is not defined")
def _get_positive_index(self, index):
if index < 0:
index = self.size + index
if index < 0:
raise IndexError("index out of bounds")
return index
def _get_index(self, value):
if isfloat(value):
return self.value2index(value)
else:
return value
def _get_array_slices(self, slice_):
"""Returns a slice to slice the corresponding data axis without
changing the offset and scale of the DataAxis.
Parameters
----------
slice_ : {float, int, slice}
Returns
-------
my_slice : slice
"""
v2i = self.value2index
if isinstance(slice_, slice):
start = slice_.start
stop = slice_.stop
step = slice_.step
else:
if isfloat(slice_):
start = v2i(slice_)
else:
start = self._get_positive_index(slice_)
stop = start + 1
step = None
if isfloat(step):
step = int(round(step / self.scale))
if isfloat(start):
try:
start = v2i(start)
except ValueError:
if start > self.high_value:
# The start value is above the axis limit
raise IndexError(
"Start value above axis high bound for axis %s."
"value: %f high_bound: %f" %
(repr(self), start, self.high_value))
else:
# The start value is below the axis limit,
# we slice from the start.
start = None
if isfloat(stop):
try:
stop = v2i(stop)
except ValueError:
if stop < self.low_value:
# The stop value is below the axis limits
raise IndexError(
"Stop value below axis low bound for axis %s."
"value: %f low_bound: %f" %
(repr(self), stop, self.low_value))
else:
# The stop value is below the axis limit,
# we slice until the end.
stop = None
if step == 0:
raise ValueError("slice step cannot be zero")
return slice(start, stop, step)
def _slice_me(self, slice_):
"""Returns a slice to slice the corresponding data axis and
change the offset and scale of the DataAxis accordingly.
Parameters
----------
slice_ : {float, int, slice}
Returns
-------
my_slice : slice
"""
i2v = self.index2value
my_slice = self._get_array_slices(slice_)
start, stop, step = my_slice.start, my_slice.stop, my_slice.step
if start is None:
if step is None or step > 0:
start = 0
else:
start = self.size - 1
self.offset = i2v(start)
if step is not None:
self.scale *= step
return my_slice
def _get_name(self):
if self.name is t.Undefined:
if self.axes_manager is None:
name = "Unnamed"
else:
name = "Unnamed " + ordinal(self.index_in_axes_manager)
else:
name = self.name
return name
def __repr__(self):
text = '<%s axis, size: %i' % (
self._get_name(),
self.size,
)
if self.navigate is True:
text += ", index: %i" % self.index
text += ">"
return text
def __str__(self):
return self._get_name() + " axis"
def update_index_bounds(self):
self.high_index = self.size - 1
def update_axis(self):
self.axis = generate_axis(self.offset, self.scale, self.size)
if len(self.axis) != 0:
self.low_value, self.high_value = (self.axis.min(),
self.axis.max())
def _update_slice(self, value):
if value is False:
self.slice = slice(None)
else:
self.slice = None
def get_axis_dictionary(self):
adict = {
'name': self.name,
'scale': self.scale,
'offset': self.offset,
'size': self.size,
'units': self.units,
'navigate': self.navigate
}
return adict
def copy(self):
return DataAxis(**self.get_axis_dictionary())
def __copy__(self):
return self.copy()
def __deepcopy__(self, memo):
cp = self.copy()
return cp
def value2index(self, value, rounding=round):
"""Return the closest index to the given value if between the limit.
Parameters
----------
value : number or numpy array
Returns
-------
index : integer or numpy array
Raises
------
ValueError if any value is out of the axis limits.
"""
if value is None:
return None
if isinstance(value, np.ndarray):
if rounding is round:
rounding = np.round
elif rounding is math.ceil:
rounding = np.ceil
elif rounding is math.floor:
rounding = np.floor
index = rounding((value - self.offset) / self.scale)
if isinstance(value, np.ndarray):
index = index.astype(int)
if np.all(self.size > index) and np.all(index >= 0):
return index
else:
raise ValueError("A value is out of the axis limits")
else:
index = int(index)
if self.size > index >= 0:
return index
else:
raise ValueError("The value is out of the axis limits")
def index2value(self, index):
if isinstance(index, np.ndarray):
return self.axis[index.ravel()].reshape(index.shape)
else:
return self.axis[index]
def calibrate(self, value_tuple, index_tuple, modify_calibration=True):
scale = (value_tuple[1] - value_tuple[0]) /\
(index_tuple[1] - index_tuple[0])
offset = value_tuple[0] - scale * index_tuple[0]
if modify_calibration is True:
self.offset = offset
self.scale = scale
else:
return offset, scale
def value_range_to_indices(self, v1, v2):
"""Convert the given range to index range.
When an out of the axis limits, the endpoint is used instead.
Parameters
----------
v1, v2 : float
The end points of the interval in the axis units. v2 must be
greater than v1.
"""
if v1 is not None and v2 is not None and v1 > v2:
raise ValueError("v2 must be greater than v1.")
if v1 is not None and self.low_value < v1 <= self.high_value:
i1 = self.value2index(v1)
else:
i1 = 0
if v2 is not None and self.high_value > v2 >= self.low_value:
i2 = self.value2index(v2)
else:
i2 = self.size - 1
return i1, i2
def update_from(self, axis, attributes=["scale", "offset", "units"]):
"""Copy values of specified axes fields from the passed AxesManager.
Parameters
----------
axis : DataAxis
The DataAxis instance to use as a source for values.
attributes : iterable container of strings.
The name of the attribute to update. If the attribute does not
exist in either of the AxesManagers, an AttributeError will be
raised.
Returns
-------
A boolean indicating whether any changes were made.
"""
any_changes = False
changed = {}
for f in attributes:
if getattr(self, f) != getattr(axis, f):
changed[f] = getattr(axis, f)
if len(changed) > 0:
self.trait_set(**changed)
any_changes = True
return any_changes
class config(HasTraits):
uuid = traits.Str(desc="UUID")
# Directories
working_dir = Directory(mandatory=True, desc="Location of the Nipype working directory")
base_dir = Directory(os.path.abspath('.'),exists=True, desc='Base directory of data. (Should be subject-independent)')
sink_dir = Directory(mandatory=True, desc="Location where the BIP will store the results")
crash_dir = Directory(mandatory=False, desc="Location to store crash files")
surf_dir = Directory(os.path.abspath('.'),mandatory=True, desc="Freesurfer subjects directory")
# Execution
run_using_plugin = Bool(False, usedefault=True, desc="True to run pipeline with plugin, False to run serially")
plugin = traits.Enum("PBS", "MultiProc", "SGE", "Condor",
usedefault=True,
desc="plugin to use, if run_using_plugin=True")
plugin_args = traits.Dict({"qsub_args": "-q many"},
usedefault=True, desc='Plugin arguments.')
test_mode = Bool(False, mandatory=False, usedefault=True,
desc='Affects whether where and if the workflow keeps its \
intermediary files. True to keep intermediary files. ')
# Subjects
datagrabber = traits.Instance(Data, ())
subjects = traits.List(traits.Str, mandatory=True, usedefault=True,
desc="Subject id's. Note: These MUST match the subject id's in the \
Freesurfer directory. For simplicity, the subject id's should \
also match with the location of individual functional files.")
func_template = traits.String('%s/cleaned_resting.nii.gz')
reg_template = traits.String('%s/cleaned_resting_reg.dat')
ref_template = traits.String('%s/cleaned_resting_ref.nii.gz')
combine_surfaces = traits.Bool()
# Target surface
target_surf = traits.Enum('fsaverage4', 'fsaverage3', 'fsaverage5',
'fsaverage6', 'fsaverage', 'subject',
desc='which average surface to map to')
surface_fwhm = traits.List([5], traits.Float(), mandatory=True,
usedefault=True,
desc="How much to smooth on target surface")
projection_stem = traits.Str('-projfrac-avg 0 1 0.1',
desc='how to project data onto the surface')
combine_surfaces = traits.Bool(desc=('compute correlation matrix across'
'both left and right surfaces'))
# Saving output
out_type = traits.Enum('mat', 'hdf5', desc='mat or hdf5')
hdf5_package = traits.Enum('h5py', 'pytables',
desc='which hdf5 package to use')
# Advanced Options
use_advanced_options = traits.Bool()
advanced_script = traits.Code()
save_script_only = traits.Bool(False)
# Atlas mapping
surface_atlas = traits.Str('None',
desc='Name of parcellation atlas')
# Buttons
check_func_datagrabber = Button("Check")
def _check_func_datagrabber_fired(self):
subs = self.subjects
for s in subs:
for template in [self.func_template, self.ref_template,
self.reg_template]:
check_path(os.path.join(self.base_dir, template % s))
check_path(os.path.join(self.surf_dir, s))
class config(HasTraits):
uuid = traits.Str(desc="UUID")
# Directories
base_dir = Directory(
os.path.abspath('.'),
exists=True,
desc='Base directory of data. (Should be subject-independent)')
sink_dir = Directory(mandatory=True,
desc="Location where the BIP will store the results")
surf_dir = Directory(os.path.abspath('.'),
mandatory=True,
desc="Freesurfer subjects directory")
# Subjects
datagrabber = traits.Instance(Data, ())
subjects = traits.List(
traits.Str,
mandatory=True,
usedefault=True,
desc="Subject id's. Note: These MUST match the subject id's in the \
Freesurfer directory. For simplicity, the subject id's should \
also match with the location of individual functional files."
)
func_template = traits.String('%s/cleaned_resting.nii.gz')
reg_template = traits.String('%s/cleaned_resting_reg.dat')
ref_template = traits.String('%s/cleaned_resting_ref.nii.gz')
combine_surfaces = traits.Bool()
# Target surface
target_surf = traits.Enum('fsaverage4',
'fsaverage3',
'fsaverage5',
'fsaverage6',
'fsaverage',
'subject',
desc='which average surface to map to')
surface_fwhm = traits.List([5],
traits.Float(),
mandatory=True,
usedefault=True,
desc="How much to smooth on target surface")
projection_stem = traits.Str('-projfrac-avg 0 1 0.1',
desc='how to project data onto the surface')
combine_surfaces = traits.Bool(desc=('compute correlation matrix across'
'both left and right surfaces'))
# Saving output
out_type = traits.Enum('mat', 'hdf5', desc='mat or hdf5')
hdf5_package = traits.Enum('h5py',
'pytables',
desc='which hdf5 package to use')
# Advanced Options
use_advanced_options = traits.Bool()
advanced_script = traits.Code()
save_script_only = traits.Bool(False)
# Atlas mapping
surface_atlas = traits.Str('None', desc='Name of parcellation atlas')
# Buttons
check_func_datagrabber = Button("Check")
def _check_func_datagrabber_fired(self):
subs = self.subjects
for s in subs:
for template in [
self.func_template, self.ref_template, self.reg_template
]:
check_path(os.path.join(self.base_dir, template % s))
check_path(os.path.join(self.surf_dir, s))
def configure_controller(cls):
c = Controller()
c.add_trait('param_type', traits.Str('Str'))
c.add_trait('is_output', traits.Bool(True))
return c
class Parameter(t.HasTraits):
"""Model parameter
Attributes
----------
value : float or array
The value of the parameter for the current location. The value
for other locations is stored in map.
bmin, bmax: float
Lower and upper bounds of the parameter value.
twin : {None, Parameter}
If it is not None, the value of the current parameter is
a function of the given Parameter. The function is by default
the identity function, but it can be defined by twin_function
twin_function_expr: str
Expression of the ``twin_function`` that enables setting a functional
relationship between the parameter and its twin. If ``twin`` is not
``None``, the parameter value is calculated as the output of calling the
twin function with the value of the twin parameter. The string is
parsed using sympy, so permitted values are any valid sympy expressions
of one variable. If the function is invertible the twin inverse function
is set automatically.
twin_inverse_function : str
Expression of the ``twin_inverse_function`` that enables setting the
value of the twin parameter. If ``twin`` is not
``None``, its value is set to the output of calling the
twin inverse function with the value provided. The string is
parsed using sympy, so permitted values are any valid sympy expressions
of one variable.
twin_function : function
**Setting this attribute manually
is deprecated in HyperSpy newer than 1.1.2. It will become private in
HyperSpy 2.0. Please use ``twin_function_expr`` instead.**
twin_inverse_function : function
**Setting this attribute manually
is deprecated in HyperSpy newer than 1.1.2. It will become private in
HyperSpy 2.0. Please use ``twin_inverse_function_expr`` instead.**
ext_force_positive : bool
If True, the parameter value is set to be the absolute value
of the input value i.e. if we set Parameter.value = -3, the
value stored is 3 instead. This is useful to bound a value
to be positive in an optimization without actually using an
optimizer that supports bounding.
ext_bounded : bool
Similar to ext_force_positive, but in this case the bounds are
defined by bmin and bmax. It is a better idea to use
an optimizer that supports bounding though.
Methods
-------
as_signal(field = 'values')
Get a parameter map as a signal object
plot()
Plots the value of the Parameter at all locations.
export(folder=None, name=None, format=None, save_std=False)
Saves the value of the parameter map to the specified format
connect, disconnect(function)
Call the functions connected when the value attribute changes.
"""
__number_of_elements = 1
__value = 0
__free = True
_bounds = (None, None)
__twin = None
_axes_manager = None
__ext_bounded = False
__ext_force_positive = False
# traitsui bugs out trying to make an editor for this, so always specify!
# (it bugs out, because both editor shares the object, and Array editors
# don't like non-sequence objects). TextEditor() works well, so does
# RangeEditor() as it works with bmin/bmax.
value = t.Property(t.Either([t.CFloat(0), Array()]))
units = t.Str('')
free = t.Property(t.CBool(True))
bmin = t.Property(NoneFloat(), label="Lower bounds")
bmax = t.Property(NoneFloat(), label="Upper bounds")
_twin_function_expr = ""
_twin_inverse_function_expr = ""
twin_function = None
_twin_inverse_function = None
_twin_inverse_sympy = None
def __init__(self):
self._twins = set()
self.events = Events()
self.events.value_changed = Event("""
Event that triggers when the `Parameter.value` changes.
The event triggers after the internal state of the `Parameter` has
been updated.
Arguments
---------
obj : Parameter
The `Parameter` that the event belongs to
value : {float | array}
The new value of the parameter
""",
arguments=["obj", 'value'])
self.std = None
self.component = None
self.grad = None
self.name = ''
self.units = ''
self.map = None
self.model = None
self._whitelist = {
'_id_name': None,
'value': None,
'std': None,
'free': None,
'units': None,
'map': None,
'_bounds': None,
'ext_bounded': None,
'name': None,
'ext_force_positive': None,
'twin_function_expr': None,
'twin_inverse_function_expr': None,
'self': ('id', None),
}
self._slicing_whitelist = {'map': 'inav'}
def _load_dictionary(self, dictionary):
"""Load data from dictionary
Parameters
----------
dict : dictionary
A dictionary containing at least the following items:
_id_name : string
_id_name of the original parameter, used to create the
dictionary. Has to match with the self._id_name
_whitelist : dictionary
a dictionary, which keys are used as keywords to match with the
parameter attributes. For more information see
:meth:`hyperspy.misc.export_dictionary.load_from_dictionary`
* any field from _whitelist.keys() *
Returns
-------
id_value : int
the ID value of the original parameter, to be later used for setting
up the correct twins
"""
if dictionary['_id_name'] == self._id_name:
load_from_dictionary(self, dictionary)
return dictionary['self']
else:
raise ValueError(
"_id_name of parameter and dictionary do not match, \nparameter._id_name = %s\
\ndictionary['_id_name'] = %s" %
(self._id_name, dictionary['_id_name']))
def __repr__(self):
text = ''
text += 'Parameter %s' % self.name
if self.component is not None:
text += ' of %s' % self.component._get_short_description()
text = '<' + text + '>'
return text
def __len__(self):
return self._number_of_elements
@property
def twin_function_expr(self):
return self._twin_function_expr
@twin_function_expr.setter
def twin_function_expr(self, value):
if not value:
self.twin_function = None
self.twin_inverse_function = None
self._twin_function_expr = ""
self._twin_inverse_sympy = None
return
expr = sympy.sympify(value)
if len(expr.free_symbols) > 1:
raise ValueError("The expression must contain only one variable.")
elif len(expr.free_symbols) == 0:
raise ValueError("The expression must contain one variable, "
"it contains none.")
x = tuple(expr.free_symbols)[0]
self.twin_function = lambdify(x, expr.evalf())
self._twin_function_expr = value
if not self.twin_inverse_function:
y = sympy.Symbol(x.name + "2")
try:
inv = sympy.solveset(sympy.Eq(y, expr), x)
self._twin_inverse_sympy = lambdify(y, inv)
self._twin_inverse_function = None
except:
# Not all may have a suitable solution.
self._twin_inverse_function = None
self._twin_inverse_sympy = None
_logger.warning(
"The function {} is not invertible. Setting the value of "
"{} will raise an AttributeError unless you set manually "
"``twin_inverse_function_expr``. Otherwise, set the "
"value of its twin parameter instead.".format(value, self))
@property
def twin_inverse_function_expr(self):
if self.twin:
return self._twin_inverse_function_expr
else:
return ""
@twin_inverse_function_expr.setter
def twin_inverse_function_expr(self, value):
if not value:
self.twin_inverse_function = None
self._twin_inverse_function_expr = ""
return
expr = sympy.sympify(value)
if len(expr.free_symbols) > 1:
raise ValueError("The expression must contain only one variable.")
elif len(expr.free_symbols) == 0:
raise ValueError("The expression must contain one variable, "
"it contains none.")
x = tuple(expr.free_symbols)[0]
self._twin_inverse_function = lambdify(x, expr.evalf())
self._twin_inverse_function_expr = value
@property
def twin_inverse_function(self):
if (not self.twin_inverse_function_expr and self.twin_function_expr
and self._twin_inverse_sympy):
return lambda x: self._twin_inverse_sympy(x).pop()
else:
return self._twin_inverse_function
@twin_inverse_function.setter
def twin_inverse_function(self, value):
self._twin_inverse_function = value
def _get_value(self):
if self.twin is None:
return self.__value
else:
if self.twin_function:
return self.twin_function(self.twin.value)
else:
return self.twin.value
def _set_value(self, value):
try:
# Use try/except instead of hasattr("__len__") because a numpy
# memmap has a __len__ wrapper even for numbers that raises a
# TypeError when calling. See issue #349.
if len(value) != self._number_of_elements:
raise ValueError("The length of the parameter must be ",
self._number_of_elements)
else:
if not isinstance(value, tuple):
value = tuple(value)
except TypeError:
if self._number_of_elements != 1:
raise ValueError("The length of the parameter must be ",
self._number_of_elements)
old_value = self.__value
if self.twin is not None:
if self.twin_function is not None:
if self.twin_inverse_function is not None:
self.twin.value = self.twin_inverse_function(value)
return
else:
raise AttributeError(
"This parameter has a ``twin_function`` but"
"its ``twin_inverse_function`` is not defined.")
else:
self.twin.value = value
return
if self.ext_bounded is False:
self.__value = value
else:
if self.ext_force_positive is True:
value = np.abs(value)
if self._number_of_elements == 1:
if self.bmin is not None and value <= self.bmin:
self.__value = self.bmin
elif self.bmax is not None and value >= self.bmax:
self.__value = self.bmax
else:
self.__value = value
else:
bmin = (self.bmin if self.bmin is not None else -np.inf)
bmax = (self.bmax if self.bmin is not None else np.inf)
self.__value = np.clip(value, bmin, bmax)
if (self._number_of_elements != 1
and not isinstance(self.__value, tuple)):
self.__value = tuple(self.__value)
if old_value != self.__value:
self.events.value_changed.trigger(value=self.__value, obj=self)
self.trait_property_changed('value', old_value, self.__value)
# Fix the parameter when coupled
def _get_free(self):
if self.twin is None:
return self.__free
else:
return False
def _set_free(self, arg):
old_value = self.__free
self.__free = arg
if self.component is not None:
self.component._update_free_parameters()
self.trait_property_changed('free', old_value, self.__free)
def _on_twin_update(self, value, twin=None):
if (twin is not None and hasattr(twin, 'events')
and hasattr(twin.events, 'value_changed')):
with twin.events.value_changed.suppress_callback(
self._on_twin_update):
self.events.value_changed.trigger(value=value, obj=self)
else:
self.events.value_changed.trigger(value=value, obj=self)
def _set_twin(self, arg):
if arg is None:
if self.twin is not None:
# Store the value of the twin in order to set the
# value of the parameter when it is uncoupled
twin_value = self.value
if self in self.twin._twins:
self.twin._twins.remove(self)
self.twin.events.value_changed.disconnect(
self._on_twin_update)
self.__twin = arg
self.value = twin_value
else:
if self not in arg._twins:
arg._twins.add(self)
arg.events.value_changed.connect(self._on_twin_update,
["value"])
self.__twin = arg
if self.component is not None:
self.component._update_free_parameters()
def _get_twin(self):
return self.__twin
twin = property(_get_twin, _set_twin)
def _get_bmin(self):
if self._number_of_elements == 1:
return self._bounds[0]
else:
return self._bounds[0][0]
def _set_bmin(self, arg):
old_value = self.bmin
if self._number_of_elements == 1:
self._bounds = (arg, self.bmax)
else:
self._bounds = ((arg, self.bmax), ) * self._number_of_elements
# Update the value to take into account the new bounds
self.value = self.value
self.trait_property_changed('bmin', old_value, arg)
def _get_bmax(self):
if self._number_of_elements == 1:
return self._bounds[1]
else:
return self._bounds[0][1]
def _set_bmax(self, arg):
old_value = self.bmax
if self._number_of_elements == 1:
self._bounds = (self.bmin, arg)
else:
self._bounds = ((self.bmin, arg), ) * self._number_of_elements
# Update the value to take into account the new bounds
self.value = self.value
self.trait_property_changed('bmax', old_value, arg)
@property
def _number_of_elements(self):
return self.__number_of_elements
@_number_of_elements.setter
def _number_of_elements(self, arg):
# Do nothing if the number of arguments stays the same
if self.__number_of_elements == arg:
return
if arg <= 1:
raise ValueError("Please provide an integer number equal "
"or greater to 1")
self._bounds = ((self.bmin, self.bmax), ) * arg
self.__number_of_elements = arg
if arg == 1:
self._Parameter__value = 0
else:
self._Parameter__value = (0, ) * arg
if self.component is not None:
self.component.update_number_parameters()
@property
def ext_bounded(self):
return self.__ext_bounded
@ext_bounded.setter
def ext_bounded(self, arg):
if arg is not self.__ext_bounded:
self.__ext_bounded = arg
# Update the value to take into account the new bounds
self.value = self.value
@property
def ext_force_positive(self):
return self.__ext_force_positive
@ext_force_positive.setter
def ext_force_positive(self, arg):
if arg is not self.__ext_force_positive:
self.__ext_force_positive = arg
# Update the value to take into account the new bounds
self.value = self.value
def store_current_value_in_array(self):
"""Store the value and std attributes.
See also
--------
fetch, assign_current_value_to_all
"""
indices = self._axes_manager.indices[::-1]
# If it is a single spectrum indices is ()
if not indices:
indices = (0, )
self.map['values'][indices] = self.value
self.map['is_set'][indices] = True
if self.std is not None:
self.map['std'][indices] = self.std
def fetch(self):
"""Fetch the stored value and std attributes.
See Also
--------
store_current_value_in_array, assign_current_value_to_all
"""
indices = self._axes_manager.indices[::-1]
# If it is a single spectrum indices is ()
if not indices:
indices = (0, )
if self.map['is_set'][indices]:
self.value = self.map['values'][indices]
self.std = self.map['std'][indices]
def assign_current_value_to_all(self, mask=None):
"""Assign the current value attribute to all the indices
Parameters
----------
mask: {None, boolean numpy array}
Set only the indices that are not masked i.e. where
mask is False.
See Also
--------
store_current_value_in_array, fetch
"""
if mask is None:
mask = np.zeros(self.map.shape, dtype='bool')
self.map['values'][mask == False] = self.value
self.map['is_set'][mask == False] = True
def _create_array(self):
"""Create the map array to store the information in
multidimensional datasets.
"""
shape = self._axes_manager._navigation_shape_in_array
if not shape:
shape = [
1,
]
dtype_ = np.dtype([('values', 'float', self._number_of_elements),
('std', 'float', self._number_of_elements),
('is_set', 'bool', 1)])
if (self.map is None or self.map.shape != shape
or self.map.dtype != dtype_):
self.map = np.zeros(shape, dtype_)
self.map['std'].fill(np.nan)
# TODO: in the future this class should have access to
# axes manager and should be able to fetch its own
# values. Until then, the next line is necessary to avoid
# erros when self.std is defined and the shape is different
# from the newly defined arrays
self.std = None
def as_signal(self, field='values'):
"""Get a parameter map as a signal object.
Please note that this method only works when the navigation
dimension is greater than 0.
Parameters
----------
field : {'values', 'std', 'is_set'}
Raises
------
NavigationDimensionError : if the navigation dimension is 0
"""
from hyperspy.signal import BaseSignal
s = BaseSignal(data=self.map[field],
axes=self._axes_manager._get_navigation_axes_dicts())
if self.component is not None and \
self.component.active_is_multidimensional:
s.data[np.logical_not(self.component._active_array)] = np.nan
s.metadata.General.title = ("%s parameter" %
self.name if self.component is None else
"%s parameter of %s component" %
(self.name, self.component.name))
for axis in s.axes_manager._axes:
axis.navigate = False
if self._number_of_elements > 1:
s.axes_manager._append_axis(size=self._number_of_elements,
name=self.name,
navigate=True)
s._assign_subclass()
if field == "values":
# Add the variance if available
std = self.as_signal(field="std")
if not np.isnan(std.data).all():
std.data = std.data**2
std.metadata.General.title = "Variance"
s.metadata.set_item("Signal.Noise_properties.variance", std)
return s
def plot(self, **kwargs):
"""Plot parameter signal.
Parameters
----------
**kwargs
Any extra keyword arguments are passed to the signal plot.
Example
-------
>>> parameter.plot() #doctest: +SKIP
Set the minimum and maximum displayed values
>>> parameter.plot(vmin=0, vmax=1) #doctest: +SKIP
"""
self.as_signal().plot(**kwargs)
def export(self, folder=None, name=None, format="hspy", save_std=False):
"""Save the data to a file.
All the arguments are optional.
Parameters
----------
folder : str or None
The path to the folder where the file will be saved.
If `None` the current folder is used by default.
name : str or None
The name of the file. If `None` the Components name followed
by the Parameter `name` attributes will be used by default.
If a file with the same name exists the name will be
modified by appending a number to the file path.
save_std : bool
If True, also the standard deviation will be saved
format: str
The extension of any file format supported by HyperSpy, default hspy
"""
if format is None:
format = "hspy"
if name is None:
name = self.component.name + '_' + self.name
filename = incremental_filename(slugify(name) + '.' + format)
if folder is not None:
filename = os.path.join(folder, filename)
self.as_signal().save(filename)
if save_std is True:
self.as_signal(field='std').save(append2pathname(filename, '_std'))
def as_dictionary(self, fullcopy=True):
"""Returns parameter as a dictionary, saving all attributes from
self._whitelist.keys() For more information see
:meth:`hyperspy.misc.export_dictionary.export_to_dictionary`
Parameters
----------
fullcopy : Bool (optional, False)
Copies of objects are stored, not references. If any found,
functions will be pickled and signals converted to dictionaries
Returns
-------
dic : dictionary with the following keys:
_id_name : string
_id_name of the original parameter, used to create the
dictionary. Has to match with the self._id_name
_twins : list
a list of ids of the twins of the parameter
_whitelist : dictionary
a dictionary, which keys are used as keywords to match with the
parameter attributes. For more information see
:meth:`hyperspy.misc.export_dictionary.export_to_dictionary`
* any field from _whitelist.keys() *
"""
dic = {'_twins': [id(t) for t in self._twins]}
export_to_dictionary(self, self._whitelist, dic, fullcopy)
return dic
def default_traits_view(self):
# As mentioned above, the default editor for
# value = t.Property(t.Either([t.CFloat(0), Array()]))
# gives a ValueError. We therefore implement default_traits_view so
# that configure/edit_traits will still work straight out of the box.
# A whitelist controls which traits to include in this view.
from traitsui.api import RangeEditor, View, Item
whitelist = ['bmax', 'bmin', 'free', 'name', 'std', 'units', 'value']
editable_traits = [
trait for trait in self.editable_traits() if trait in whitelist
]
if 'value' in editable_traits:
i = editable_traits.index('value')
v = editable_traits.pop(i)
editable_traits.insert(
i,
Item(v, editor=RangeEditor(low_name='bmin', high_name='bmax')))
view = View(editable_traits, buttons=['OK', 'Cancel'])
return view
class config(HasTraits):
uuid = traits.Str(desc="UUID")
# Directories
working_dir = Directory(mandatory=True,
desc="Location of the Nipype working directory")
base_dir = Directory(
os.path.abspath('.'),
mandatory=True,
desc='Base directory of data. (Should be subject-independent)')
sink_dir = Directory(mandatory=True,
desc="Location where the BIP will store the results")
crash_dir = Directory(mandatory=False,
desc="Location to store crash files")
save_script_only = traits.Bool(False)
# Execution
run_using_plugin = Bool(
False,
usedefault=True,
desc="True to run pipeline with plugin, False to run serially")
plugin = traits.Enum("PBS",
"MultiProc",
"SGE",
"Condor",
usedefault=True,
desc="plugin to use, if run_using_plugin=True")
plugin_args = traits.Dict({"qsub_args": "-q many"},
usedefault=True,
desc='Plugin arguments.')
test_mode = Bool(
False,
mandatory=False,
usedefault=True,
desc='Affects whether where and if the workflow keeps its \
intermediary files. True to keep intermediary files. '
)
timeout = traits.Float(14.0)
# Subjects
#subjects = traits.List(traits.Str, mandatory=True, usedefault=True,
# desc="Subject id's. Note: These MUST match the subject id's in the \
# Freesurfer directory. For simplicity, the subject id's should \
# also match with the location of individual functional files.")
#fwhm=traits.List(traits.Float())
#copes_template = traits.String('%s/preproc/output/fwhm_%s/cope*.nii.gz')
#varcopes_template = traits.String('%s/preproc/output/fwhm_%s/varcope*.nii.gz')
#contrasts = traits.List(traits.Str,desc="contrasts")
datagrabber = traits.Instance(Data, ())
# Regression
design_csv = traits.File(desc="design .csv file")
reg_contrasts = traits.Code(
desc=
"function named reg_contrasts which takes in 0 args and returns contrasts"
)
run_mode = traits.Enum("flame1", "ols", "flame12")
#Normalization
norm_template = traits.File(desc='Template of files')
use_mask = traits.Bool(False)
mask_file = traits.File()
#Correction:
run_correction = traits.Bool(False)
p_threshold = traits.Float(0.05)
z_threshold = traits.Float(2.3)
connectivity = traits.Int(26)
do_randomize = traits.Bool(False)
num_iterations = traits.Int(5000)
# Advanced Options
use_advanced_options = traits.Bool()
advanced_script = traits.Code()
# Buttons
check_func_datagrabber = Button("Check")
class General(t.HasTraits):
title = t.Str(t.Undefined)
original_filename = t.File(t.Undefined)
signal_kind = t.Str(t.Undefined)
record_by = t.Enum('spectrum', 'image', default=t.Undefined)
class config(HasTraits):
uuid = traits.Str(desc="UUID")
desc = traits.Str(desc='Workflow description')
# Directories
working_dir = Directory(mandatory=True, desc="Location of the Nipype working directory")
base_dir = Directory(os.path.abspath('.'),mandatory=True, desc='Base directory of data. (Should be subject-independent)')
sink_dir = Directory(os.path.abspath('.'),mandatory=True, desc="Location where the BIP will store the results")
field_dir = Directory(desc="Base directory of field-map data (Should be subject-independent) \
Set this value to None if you don't want fieldmap distortion correction")
crash_dir = Directory(mandatory=False, desc="Location to store crash files")
surf_dir = Directory(mandatory=True, desc= "Freesurfer subjects directory")
# Execution
run_using_plugin = Bool(False, usedefault=True, desc="True to run pipeline with plugin, False to run serially")
plugin = traits.Enum("PBS", "PBSGraph","MultiProc", "SGE", "Condor",
usedefault=True,
desc="plugin to use, if run_using_plugin=True")
plugin_args = traits.Dict({"qsub_args": "-q many"},
usedefault=True, desc='Plugin arguments.')
test_mode = Bool(False, mandatory=False, usedefault=True,
desc='Affects whether where and if the workflow keeps its \
intermediary files. True to keep intermediary files. ')
# Subjects
subjects= traits.List(traits.Str, mandatory=True, usedefault=True,
desc="Subject id's. Note: These MUST match the subject id's in the \
Freesurfer directory. For simplicity, the subject id's should \
also match with the location of individual functional files.")
dwi_template = traits.String('%s/functional.nii.gz')
bval_template = traits.String('%s/bval')
bvec_template = traits.String('%s/fbvec')
run_datagrabber_without_submitting = traits.Bool(desc="Run the datagrabber without \
submitting to the cluster")
timepoints_to_remove = traits.Int(0,usedefault=True)
# Fieldmap
use_fieldmap = Bool(False, mandatory=False, usedefault=True,
desc='True to include fieldmap distortion correction. Note: field_dir \
must be specified')
magnitude_template = traits.String('%s/magnitude.nii.gz')
phase_template = traits.String('%s/phase.nii.gz')
TE_diff = traits.Float(desc='difference in B0 field map TEs')
sigma = traits.Int(2, desc='2D spatial gaussing smoothing stdev (default = 2mm)')
echospacing = traits.Float(desc="EPI echo spacing")
# Bvecs
do_rotate_bvecs = traits.Bool(True, usedefault=True)
# Advanced Options
use_advanced_options = traits.Bool()
advanced_script = traits.Code()
# Buttons
check_func_datagrabber = Button("Check")
check_field_datagrabber = Button("Check")
def _check_func_datagrabber_fired(self):
subs = self.subjects
for s in subs:
if not os.path.exists(os.path.join(self.base_dir,self.dwi_template % s)):
print "ERROR", os.path.join(self.base_dir,self.dwi_template % s), "does NOT exist!"
break
else:
print os.path.join(self.base_dir,self.dwi_template % s), "exists!"
def _check_field_datagrabber_fired(self):
subs = self.subjects
for s in subs:
if not os.path.exists(os.path.join(self.field_dir,self.magnitude_template % s)):
print "ERROR:", os.path.join(self.field_dir,self.magnitude_template % s), "does NOT exist!"
break
else:
print os.path.join(self.base_dir,self.magnitude_template % s), "exists!"
if not os.path.exists(os.path.join(self.field_dir,self.phase_template % s)):
print "ERROR:", os.path.join(self.field_dir,self.phase_template % s), "does NOT exist!"
break
else:
print os.path.join(self.base_dir,self.phase_template % s), "exists!"
class LineInSignal1D(t.HasTraits):
"""Adds a vertical draggable line to a spectrum that reports its
position to the position attribute of the class.
Attributes:
-----------
position : float
The position of the vertical line in the one dimensional signal. Moving
the line changes the position but the reverse is not true.
on : bool
Turns on and off the line
color : wx.Colour
The color of the line. It automatically redraws the line.
"""
position = t.Float()
is_ok = t.Bool(False)
on = t.Bool(False)
# The following is disabled because as of traits 4.6 the Color trait
# imports traitsui (!)
# try:
# color = t.Color("black")
# except ModuleNotFoundError: # traitsui is not installed
# pass
color_str = t.Str("black")
def __init__(self, signal):
if signal.axes_manager.signal_dimension != 1:
raise SignalDimensionError(signal.axes_manager.signal_dimension, 1)
self.signal = signal
self.signal.plot()
axis_dict = signal.axes_manager.signal_axes[0].get_axis_dictionary()
am = AxesManager([
axis_dict,
])
am._axes[0].navigate = True
# Set the position of the line in the middle of the spectral
# range by default
am._axes[0].index = int(round(am._axes[0].size / 2))
self.axes_manager = am
self.axes_manager.events.indices_changed.connect(
self.update_position, [])
self.on_trait_change(self.switch_on_off, 'on')
def draw(self):
self.signal._plot.signal_plot.figure.canvas.draw_idle()
def switch_on_off(self, obj, trait_name, old, new):
if not self.signal._plot.is_active:
return
if new is True and old is False:
self._line = VerticalLineWidget(self.axes_manager)
self._line.set_mpl_ax(self.signal._plot.signal_plot.ax)
self._line.patch.set_linewidth(2)
self._color_changed("black", "black")
# There is not need to call draw because setting the
# color calls it.
elif new is False and old is True:
self._line.close()
self._line = None
self.draw()
def update_position(self, *args, **kwargs):
if not self.signal._plot.is_active:
return
self.position = self.axes_manager.coordinates[0]
def _color_changed(self, old, new):
if self.on is False:
return
self._line.patch.set_color((
self.color.Red() / 255.,
self.color.Green() / 255.,
self.color.Blue() / 255.,
))
self.draw()
class config(BaseWorkflowConfig):
uuid = traits.Str(desc="UUID")
desc = traits.Str(desc='Workflow description')
# Directories
base_dir = Directory(
os.path.abspath('.'),
mandatory=True,
desc='Base directory of data. (Should be subject-independent)')
sink_dir = Directory(os.path.abspath('.'),
mandatory=True,
desc="Location where the BIP will store the results")
field_dir = Directory(
desc="Base directory of field-map data (Should be subject-independent) \
Set this value to None if you don't want fieldmap distortion correction"
)
surf_dir = Directory(mandatory=True, desc="Freesurfer subjects directory")
# Subjects
subjects = traits.List(
traits.Str,
mandatory=True,
usedefault=True,
desc="Subject id's. Note: These MUST match the subject id's in the \
Freesurfer directory. For simplicity, the subject id's should \
also match with the location of individual functional files."
)
dwi_template = traits.String('%s/functional.nii.gz')
bval_template = traits.String('%s/bval')
bvec_template = traits.String('%s/fbvec')
run_datagrabber_without_submitting = traits.Bool(
desc="Run the datagrabber without \
submitting to the cluster")
timepoints_to_remove = traits.Int(0, usedefault=True)
# Fieldmap
use_fieldmap = Bool(
False,
mandatory=False,
usedefault=True,
desc='True to include fieldmap distortion correction. Note: field_dir \
must be specified')
magnitude_template = traits.String('%s/magnitude.nii.gz')
phase_template = traits.String('%s/phase.nii.gz')
TE_diff = traits.Float(desc='difference in B0 field map TEs')
sigma = traits.Int(
2, desc='2D spatial gaussing smoothing stdev (default = 2mm)')
echospacing = traits.Float(desc="EPI echo spacing")
# Bvecs
do_rotate_bvecs = traits.Bool(True, usedefault=True)
# Advanced Options
use_advanced_options = traits.Bool()
advanced_script = traits.Code()
# Buttons
check_func_datagrabber = Button("Check")
check_field_datagrabber = Button("Check")
def _check_func_datagrabber_fired(self):
subs = self.subjects
for s in subs:
if not os.path.exists(
os.path.join(self.base_dir, self.dwi_template % s)):
print "ERROR", os.path.join(self.base_dir, self.dwi_template %
s), "does NOT exist!"
break
else:
print os.path.join(self.base_dir,
self.dwi_template % s), "exists!"
def _check_field_datagrabber_fired(self):
subs = self.subjects
for s in subs:
if not os.path.exists(
os.path.join(self.field_dir, self.magnitude_template % s)):
print "ERROR:", os.path.join(self.field_dir,
self.magnitude_template %
s), "does NOT exist!"
break
else:
print os.path.join(self.base_dir,
self.magnitude_template % s), "exists!"
if not os.path.exists(
os.path.join(self.field_dir, self.phase_template % s)):
print "ERROR:", os.path.join(
self.field_dir, self.phase_template % s), "does NOT exist!"
break
else:
print os.path.join(self.base_dir,
self.phase_template % s), "exists!"
