class config(HasTraits): uuid = traits.Str(desc="UUID") # Directories working_dir = Directory(mandatory=True, desc="Location of the Nipype working directory") base_dir = Directory( os.path.abspath('.'), mandatory=True, desc='Base directory of data. (Should be subject-independent)') sink_dir = Directory(mandatory=True, desc="Location where the BIP will store the results") crash_dir = Directory(mandatory=False, desc="Location to store crash files") surf_dir = Directory( desc="freesurfer directory. subject id's should be the same") save_script_only = traits.Bool(False) # Execution run_using_plugin = Bool( False, usedefault=True, desc="True to run pipeline with plugin, False to run serially") plugin = traits.Enum("PBS", "MultiProc", "SGE", "Condor", usedefault=True, desc="plugin to use, if run_using_plugin=True") plugin_args = traits.Dict({"qsub_args": "-q many"}, usedefault=True, desc='Plugin arguments.') test_mode = Bool( False, mandatory=False, usedefault=True, desc='Affects whether where and if the workflow keeps its \ intermediary files. True to keep intermediary files. ' ) # Data datagrabber = traits.Instance(Data, ()) #subject_id = traits.String() #contrast = traits.File() #mask_contrast = traits.File() use_contrast_mask = traits.Bool(True) #reg_file = traits.File() #mean_image = traits.File() background_thresh = traits.Float(0.5) hemi = traits.List(['lh', 'rh']) roi = traits.List( ['superiortemporal', 'bankssts'], traits.Enum('superiortemporal', 'bankssts', 'caudalanteriorcingulate', 'caudalmiddlefrontal', 'corpuscallosum', 'cuneus', 'entorhinal', 'fusiform', 'inferiorparietal', 'inferiortemporal', 'isthmuscingulate', 'lateraloccipital', 'lateralorbitofrontal', 'lingual', 'medialorbitofrontal', 'middletemporal', 'parahippocampal', 'paracentral', 'parsopercularis', 'parsorbitalis', 'parstriangularis', 'pericalcarine', 'postcentral', 'posteriorcingulate', 'precentral', 'precuneus', 'rostralanteriorcingulate', 'rostralmiddlefrontal', 'superiorfrontal', 'superiorparietal', 'supramarginal', 'frontalpole', 'temporalpole', 'transversetemporal', 'insula'), usedefault=True) #35 freesurfer regions, thresh = traits.Float(1.5)
class MomentCurvature(tr.HasStrictTraits): r'''Class returning the moment curvature relationship. ''' b_z = tr.Any get_b_z = tr.Property @tr.cached_property def _get_get_b_z(self): return sp.lambdify(z, self.b_z, 'numpy') h = tr.Float model_params = tr.Dict({ E_ct: 24000, E_cc: 25000, eps_cr: 0.001, eps_cy: -0.003, eps_cu: -0.01, mu: 0.33, eps_tu: 0.003 }) # Number of material points along the height of the cross section n_m = tr.Int(100) # Reinforcement z_j = tr.Array(np.float_, value=[10]) A_j = tr.Array(np.float_, value=[[np.pi * (16 / 2.)**2]]) E_j = tr.Array(np.float_, value=[[210000]]) eps_sy_j = tr.Array(np.float_, value=[[500. / 210000.]]) z_m = tr.Property(depends_on='n_m, h') @tr.cached_property def _get_z_m(self): return np.linspace(0, self.h, self.n_m) kappa_range = tr.Tuple(-0.001, 0.001, 101) kappa_t = tr.Property(tr.Array(np.float_), depends_on='kappa_range') @tr.cached_property def _get_kappa_t(self): return np.linspace(*self.kappa_range) get_eps_z = tr.Property(depends_on='model_params_items') @tr.cached_property def _get_get_eps_z(self): return sp.lambdify( (kappa, eps_bot, z), eps_z.subs(subs_eps), 'numpy' ) get_sig_c_z = tr.Property(depends_on='model_params_items') @tr.cached_property def _get_get_sig_c_z(self): return sp.lambdify( (kappa, eps_bot, z), sig_c_z_lin.subs(self.model_params), 'numpy' ) get_sig_s_eps = tr.Property(depends_on='model_params_items') @tr.cached_property def _get_get_sig_s_eps(self): return sp.lambdify((eps, E_s, eps_sy), sig_s_eps, 'numpy') # Normal force def get_N_s_tj(self, kappa_t, eps_bot_t): eps_z_tj = self.get_eps_z( kappa_t[:, np.newaxis], eps_bot_t[:, np.newaxis], self.z_j[np.newaxis, :] ) sig_s_tj = self.get_sig_s_eps(eps_z_tj, self.E_j, self.eps_sy_j) return np.einsum('j,tj->tj', self.A_j, sig_s_tj) def get_N_c_t(self, kappa_t, eps_bot_t): z_tm = self.z_m[np.newaxis, :] b_z_m = self.get_b_z(z_tm) # self.get_b_z(self.z_m) also OK N_z_tm = b_z_m * self.get_sig_c_z( kappa_t[:, np.newaxis], eps_bot_t[:, np.newaxis], z_tm ) return np.trapz(N_z_tm, x=z_tm, axis=-1) def get_N_t(self, kappa_t, eps_bot_t): N_s_t = np.sum(self.get_N_s_tj(kappa_t, eps_bot_t), axis=-1) return self.get_N_c_t(kappa_t, eps_bot_t) + N_s_t # SOLVER: Get eps_bot to render zero force eps_bot_t = tr.Property() r'''Resolve the tensile strain to get zero normal force for the prescribed curvature ''' def _get_eps_bot_t(self): res = root(lambda eps_bot_t: self.get_N_t(self.kappa_t, eps_bot_t), 0.0000001 + np.zeros_like(self.kappa_t), tol=1e-6) return res.x # POSTPROCESSING eps_cr = tr.Property() def _get_eps_cr(self): return np.array([self.model_params[eps_cr]], dtype=np.float_) kappa_cr = tr.Property() def _get_kappa_cr(self): res = root(lambda kappa: self.get_N_t(kappa, self.eps_cr), 0.0000001 + np.zeros_like(self.eps_cr), tol=1e-10) return res.x # Bending moment M_s_t = tr.Property() def _get_M_s_t(self): eps_z_tj = self.get_eps_z( self.kappa_t[:, np.newaxis], self.eps_bot_t[:, np.newaxis], self.z_j[np.newaxis, :] ) sig_z_tj = self.get_sig_s_eps( eps_z_tj, self.E_j, self.eps_sy_j) return -np.einsum('j,tj,j->t', self.A_j, sig_z_tj, self.z_j) M_c_t = tr.Property() def _get_M_c_t(self): z_tm = self.z_m[np.newaxis, :] b_z_m = self.get_b_z(z_tm) N_z_tm = b_z_m * self.get_sig_c_z( self.kappa_t[:, np.newaxis], self.eps_bot_t[:, np.newaxis], z_tm ) return -np.trapz(N_z_tm * z_tm, x=z_tm, axis=-1) M_t = tr.Property() def _get_M_t(self): return self.M_c_t + self.M_s_t N_s_tj = tr.Property() def _get_N_s_tj(self): return self.get_N_s_tj(self.kappa_t, self.eps_bot_t) eps_tm = tr.Property() def _get_eps_tm(self): return self.get_eps_z( self.kappa_t[:, np.newaxis], self.eps_bot_t[:, np.newaxis], self.z_m[np.newaxis, :], ) sig_tm = tr.Property() def _get_sig_tm(self): return self.get_sig_c_z( self.kappa_t[:, np.newaxis], self.eps_bot_t[:, np.newaxis], self.z_m[np.newaxis, :], ) idx = tr.Int(0) M_norm = tr.Property() def _get_M_norm(self): # Section modulus @TODO optimize W for var b W = (self.b * self.h**2) / 6 sig_cr = self.model_params[E_ct] * self.model_params[eps_cr] return W * sig_cr kappa_norm = tr.Property() def _get_kappa_norm(self): return self.kappa_cr def plot_norm(self, ax1, ax2): idx = self.idx ax1.plot(self.kappa_t / self.kappa_norm, self.M_t / self.M_norm) ax1.plot(self.kappa_t[idx] / self.kappa_norm, self.M_t[idx] / self.M_norm, marker='o') ax2.barh(self.z_j, self.N_s_tj[idx, :], height=2, color='red', align='center') #ax2.fill_between(eps_z_arr[idx,:], z_arr, 0, alpha=0.1); ax3 = ax2.twiny() # ax3.plot(self.eps_tm[idx, :], self.z_m, color='k', linewidth=0.8) ax3.plot(self.sig_tm[idx, :], self.z_m) ax3.axvline(0, linewidth=0.8, color='k') ax3.fill_betweenx(self.z_m, self.sig_tm[idx, :], 0, alpha=0.1) self._align_xaxis(ax2, ax3) def plot(self, ax1, ax2): idx = self.idx ax1.plot(self.kappa_t, self.M_t / (1e6)) ax1.set_ylabel('Moment [kN.m]') ax1.set_xlabel('Curvature [$m^{-1}$]') ax1.plot(self.kappa_t[idx], self.M_t[idx] / (1e6), marker='o') ax2.barh(self.z_j, self.N_s_tj[idx, :], height=6, color='red', align='center') #ax2.plot(self.N_s_tj[idx, :], self.z_j, color='red') #print('Z', self.z_j) #print(self.N_s_tj[idx, :]) #ax2.fill_between(eps_z_arr[idx,:], z_arr, 0, alpha=0.1); ax3 = ax2.twiny() # ax3.plot(self.eps_tm[idx, :], self.z_m, color='k', linewidth=0.8) ax3.plot(self.sig_tm[idx, :], self.z_m) ax3.axvline(0, linewidth=0.8, color='k') ax3.fill_betweenx(self.z_m, self.sig_tm[idx, :], 0, alpha=0.1) self._align_xaxis(ax2, ax3) def _align_xaxis(self, ax1, ax2): """Align zeros of the two axes, zooming them out by same ratio""" axes = (ax1, ax2) extrema = [ax.get_xlim() for ax in axes] tops = [extr[1] / (extr[1] - extr[0]) for extr in extrema] # Ensure that plots (intervals) are ordered bottom to top: if tops[0] > tops[1]: axes, extrema, tops = [list(reversed(l)) for l in (axes, extrema, tops)] # How much would the plot overflow if we kept current zoom levels? tot_span = tops[1] + 1 - tops[0] b_new_t = extrema[0][0] + tot_span * (extrema[0][1] - extrema[0][0]) t_new_b = extrema[1][1] - tot_span * (extrema[1][1] - extrema[1][0]) axes[0].set_xlim(extrema[0][0], b_new_t) axes[1].set_xlim(t_new_b, extrema[1][1])
class config(HasTraits): uuid = traits.Str(desc="UUID") # Directories working_dir = Directory(mandatory=True, desc="Location of the Nipype working directory") base_dir = Directory( os.path.abspath('.'), mandatory=True, desc='Base directory of data. (Should be subject-independent)') sink_dir = Directory(mandatory=True, desc="Location where the BIP will store the results") crash_dir = Directory(mandatory=False, desc="Location to store crash files") surf_dir = Directory( desc="freesurfer directory. subject id's should be the same") save_script_only = traits.Bool(False) # Execution run_using_plugin = Bool( False, usedefault=True, desc="True to run pipeline with plugin, False to run serially") plugin = traits.Enum("PBS", "MultiProc", "SGE", "Condor", usedefault=True, desc="plugin to use, if run_using_plugin=True") plugin_args = traits.Dict({"qsub_args": "-q many"}, usedefault=True, desc='Plugin arguments.') test_mode = Bool( False, mandatory=False, usedefault=True, desc='Affects whether where and if the workflow keeps its \ intermediary files. True to keep intermediary files. ' ) timeout = traits.Float(30.0) # DataGrabber datagrabber = traits.Instance(Data, ()) # segstats use_reg = traits.Bool(True) inverse_reg = traits.Bool(True) use_standard_label = traits.Bool( False, desc="use same label file for all subjects") label_file = traits.File() use_annotation = traits.Bool( False, desc= "use same annotation file for all subjects (will warp to subject space" ) use_subject_annotation = traits.Bool( False, desc="you need to change datragrabber to\ have outputs lh_annotation and rh_annotation" ) annot_space = traits.String("fsaverage5", desc="subject space of annot file") lh_annotation = traits.File() rh_annotation = traits.File() color_table_file = traits.Enum("Default", "Color_Table", "GCA_color_table", "None") color_file = traits.File() proj = traits.BaseTuple(("frac", 0, 1, 0.1), traits.Enum("abs", "frac"), traits.Float(), traits.Float(), traits.Float()) statname = traits.Str('segstats1', desc="description of the segstat")
class MATS2DScalarDamage(MATS2DEval): r'''Isotropic damage model. ''' name = 'isotropic damage model' node_name = 'isotropic damage model' tree = ['omega_fn','strain_norm'] omega_fn = EitherType( options=[('exp-slope', ExpSlopeDamageFn), ('linear', LinearDamageFn), ('abaqus', AbaqusDamageFn), ('fracture-energy', GfDamageFn), ('weibull-CDF', WeibullDamageFn), ], MAT=True, on_option_change='link_omega_to_mats' ) D_alg = Float(0) r'''Selector of the stiffness calculation. ''' eps_max = Float(0.03, ALG=True) # upon change of the type attribute set the link to the material model def link_omega_to_mats(self): self.omega_fn_.trait_set(mats=self, E_name='E', x_max_name='eps_max') #========================================================================= # Material model #========================================================================= strain_norm = EitherType( options=[('Rankine', SN2DRankine), ('Masars', SN2DMasars), ('Energy', SN2DEnergy)], on_option_change='link_strain_norm_to_mats' ) def link_strain_norm_to_mats(self): self.strain_norm_.trait_set(mats=self) state_var_shapes = {'kappa': (), 'omega': ()} r''' Shapes of the state variables to be stored in the global array at the level of the domain. ''' def get_corr_pred(self, eps_ab_n1, tn1, kappa, omega): r''' Corrector predictor computation. @param eps_app_eng input variable - engineering strain ''' eps_eq = self.strain_norm_.get_eps_eq(eps_ab_n1, kappa) I = self.omega_fn_.get_f_trial(eps_eq, kappa) eps_eq_I = eps_eq[I] kappa[I] = eps_eq_I omega[I] = self._omega(eps_eq_I) phi = (1.0 - omega) D_abcd = np.einsum( '...,abcd->...abcd', phi, self.D_abcd ) sig_ab = np.einsum( '...abcd,...cd->...ab', D_abcd, eps_ab_n1 ) if self.D_alg > 0: domega_ds_I = self._omega_derivative(eps_eq_I) deps_eq_I = self.strain_norm_.get_deps_eq(eps_ab_n1[I]) D_red_I = np.einsum('...,...ef,cdef,...ef->...cdef', domega_ds_I, deps_eq_I, self.D_abcd, eps_ab_n1[I]) * self.D_alg D_abcd[I] -= D_red_I return sig_ab, D_abcd def _omega(self, kappa): return self.omega_fn_(kappa) def _omega_derivative(self, kappa): return self.omega_fn_.diff(kappa) ipw_view = View( Item('E'), Item('nu'), Item('strain_norm'), Item('omega_fn'), Item('stress_state'), Item('D_alg', latex=r'\theta_\mathrm{alg. stiff.}', editor=FloatRangeEditor(low=0,high=1)), Item('eps_max'), Item('G_f', latex=r'G_\mathrm{f}^{\mathrm{estimate}}', readonly=True), ) G_f = tr.Property(Float, depends_on='state_changed') @tr.cached_property def _get_G_f(self): eps_max = self.eps_max n_eps = 1000 eps11_range = np.linspace(1e-9,eps_max,n_eps) eps_range = np.zeros((len(eps11_range), 2, 2)) eps_range[:,1,1] = eps11_range state_vars = { var : np.zeros( (len(eps11_range),) + shape ) for var, shape in self.state_var_shapes.items() } sig_range, D = self.get_corr_pred(eps_range, 1, **state_vars) sig11_range = sig_range[:,1,1] return np.trapz(sig11_range, eps11_range) def subplots(self, fig): ax_sig = fig.subplots(1,1) ax_d_sig = ax_sig.twinx() return ax_sig, ax_d_sig def update_plot(self, axes): ax_sig, ax_d_sig = axes eps_max = self.eps_max n_eps = 100 eps11_range = np.linspace(1e-9,eps_max,n_eps) eps_range = np.zeros((n_eps, 2, 2)) eps_range[:,0,0] = eps11_range state_vars = { var : np.zeros( (n_eps,) + shape ) for var, shape in self.state_var_shapes.items() } sig_range, D_range = self.get_corr_pred(eps_range, 1, **state_vars) sig11_range = sig_range[:,0,0] ax_sig.plot(eps11_range, sig11_range,color='blue') d_sig1111_range = D_range[...,0,0,0,0] ax_d_sig.plot(eps11_range, d_sig1111_range, linestyle='dashed', color='gray') ax_sig.set_xlabel(r'$\varepsilon_{11}$ [-]') ax_sig.set_ylabel(r'$\sigma_{11}$ [MPa]') ax_d_sig.set_ylabel(r'$\mathrm{d} \sigma_{11} / \mathrm{d} \varepsilon_{11}$ [MPa]') ax_d_sig.plot(eps11_range[:-1], (sig11_range[:-1]-sig11_range[1:])/(eps11_range[:-1]-eps11_range[1:]), color='orange', linestyle='dashed') def get_omega(self, eps_ab, tn1, **Eps): return Eps['omega'] var_dict = tr.Property(tr.Dict(tr.Str, tr.Callable)) '''Dictionary of response variables ''' @tr.cached_property def _get_var_dict(self): var_dict = dict(omega=self.get_omega) var_dict.update(super()._get_var_dict()) return var_dict
class DOTSGrid(BMCSLeafNode): '''Domain time steppsr on a grid mesh ''' x_0 = tr.Tuple(0., 0., input=True) L_x = tr.Float(200, input=True, MESH=True) L_y = tr.Float(100, input=True, MESH=True) n_x = tr.Int(100, input=True, MESH=True) n_y = tr.Int(30, input=True, MESH=True) integ_factor = tr.Float(1.0, input=True, MESH=True) fets = tr.Instance(IFETSEval, input=True, MESH=True) D1_abcd = tr.Array(np.float_, input=True) '''Symmetric operator distributing the derivatives of the shape functions into the tensor field ''' def _D1_abcd_default(self): delta = np.identity(2) # symmetrization operator D1_abcd = 0.5 * (np.einsum('ac,bd->abcd', delta, delta) + np.einsum('ad,bc->abcd', delta, delta)) return D1_abcd mesh = tr.Property(tr.Instance(FEGrid), depends_on='+input') @tr.cached_property def _get_mesh(self): return FEGrid(coord_min=self.x_0, coord_max=(self.x_0[0] + self.L_x, self.x_0[1] + self.L_y), shape=(self.n_x, self.n_y), fets_eval=self.fets) cached_grid_values = tr.Property(tr.Tuple, depends_on='+input') @tr.cached_property def _get_cached_grid_values(self): x_Ia = self.mesh.X_Id n_I, n_a = x_Ia.shape dof_Ia = np.arange(n_I * n_a, dtype=np.int_).reshape(n_I, -1) I_Ei = self.mesh.I_Ei x_Eia = x_Ia[I_Ei, :] dof_Eia = dof_Ia[I_Ei] x_Ema = np.einsum('im,Eia->Ema', self.fets.N_im, x_Eia) J_Emar = np.einsum('imr,Eia->Emar', self.fets.dN_imr, x_Eia) J_Enar = np.einsum('inr,Eia->Enar', self.fets.dN_inr, x_Eia) det_J_Em = np.linalg.det(J_Emar) inv_J_Emar = np.linalg.inv(J_Emar) inv_J_Enar = np.linalg.inv(J_Enar) B_Eimabc = np.einsum('abcd,imr,Eidr->Eimabc', self.D1_abcd, self.fets.dN_imr, inv_J_Emar) B_Einabc = np.einsum('abcd,inr,Eidr->Einabc', self.D1_abcd, self.fets.dN_inr, inv_J_Enar) BB_Emicjdabef = np.einsum('Eimabc,Ejmefd, Em, m->Emicjdabef', B_Eimabc, B_Eimabc, det_J_Em, self.fets.w_m) return (BB_Emicjdabef, B_Eimabc, dof_Eia, x_Eia, dof_Ia, I_Ei, B_Einabc, det_J_Em) BB_Emicjdabef = tr.Property() '''Quadratic form of the kinematic mapping. ''' def _get_BB_Emicjdabef(self): return self.cached_grid_values[0] B_Eimabc = tr.Property() '''Kinematic mapping between displacements and strains in every integration point. ''' def _get_B_Eimabc(self): return self.cached_grid_values[1] B_Einabc = tr.Property() '''Kinematic mapping between displacement and strain in every visualization point ''' def _get_B_Einabc(self): return self.cached_grid_values[6] dof_Eia = tr.Property() '''Mapping [element, node, direction] -> degree of freedom. ''' def _get_dof_Eia(self): return self.cached_grid_values[2] x_Eia = tr.Property() '''Mapping [element, node, direction] -> value of coordinate. ''' def _get_x_Eia(self): return self.cached_grid_values[3] dof_Ia = tr.Property() '''[global node, direction] -> degree of freedom ''' def _get_dof_Ia(self): return self.cached_grid_values[4] I_Ei = tr.Property() '''[element, node] -> global node ''' def _get_I_Ei(self): return self.cached_grid_values[5] det_J_Em = tr.Property() '''Jacobi determinant in every element and integration point. ''' def _get_det_J_Em(self): return self.cached_grid_values[7] state_arrays = tr.Property(tr.Dict(tr.Str, tr.Array), depends_on='fets, mats') '''Dictionary of state arrays. The entry names and shapes are defined by the material model. ''' @tr.cached_property def _get_state_arrays(self): return { name: np.zeros(( self.mesh.n_active_elems, self.fets.n_m, ) + mats_sa_shape, dtype=np.float_) for name, mats_sa_shape in list( self.mats.state_array_shapes.items()) } def get_corr_pred(self, U, dU, t_n, t_n1, update_state, algorithmic): '''Get the corrector and predictor for the given increment of unknown . ''' n_c = self.fets.n_nodal_dofs U_Ia = U.reshape(-1, n_c) U_Eia = U_Ia[self.I_Ei] eps_Emab = np.einsum('Eimabc,Eic->Emab', self.B_Eimabc, U_Eia) dU_Ia = dU.reshape(-1, n_c) dU_Eia = dU_Ia[self.I_Ei] deps_Emab = np.einsum('Eimabc,Eic->Emab', self.B_Eimabc, dU_Eia) D_Emabef, sig_Emab = self.mats.get_corr_pred(eps_Emab, deps_Emab, t_n, t_n1, update_state, algorithmic, **self.state_arrays) K_Eicjd = self.integ_factor * np.einsum('Emicjdabef,Emabef->Eicjd', self.BB_Emicjdabef, D_Emabef) n_E, n_i, n_c, n_j, n_d = K_Eicjd.shape K_E = K_Eicjd.reshape(-1, n_i * n_c, n_j * n_d) dof_E = self.dof_Eia.reshape(-1, n_i * n_c) K_subdomain = SysMtxArray(mtx_arr=K_E, dof_map_arr=dof_E) f_Eic = self.integ_factor * np.einsum( 'm,Eimabc,Emab,Em->Eic', self.fets.w_m, self.B_Eimabc, sig_Emab, self.det_J_Em) f_Ei = f_Eic.reshape(-1, n_i * n_c) F_dof = np.bincount(dof_E.flatten(), weights=f_Ei.flatten()) F_int = F_dof norm_F_int = np.linalg.norm(F_int) return K_subdomain, F_int, norm_F_int
########################################################################## # Apply monkeypatch here _Undefined.__len__ = _length ########################################################################## Undefined = _Undefined() class Str(Unicode): """Replaces the default traits.Str based in bytes.""" # Monkeypatch Str and DictStrStr for Python 2 compatibility traits.Str = Str DictStrStr = traits.Dict((bytes, str), (bytes, str)) traits.DictStrStr = DictStrStr class BasePath(TraitType): """Defines a trait whose value must be a valid filesystem path.""" # A description of the type of value this trait accepts: exists = False resolve = False _is_file = False _is_dir = False @property def info_text(self): """Create the trait's general description."""
class Trainer(t.HasStrictTraits): model: models.BaseNet = t.Instance(torch.nn.Module, transient=True) def _model_default(self): # Merge 'base config' (if requested) and any overrides in 'model_config' if self.base_config: model_config = get_ref_arch(self.base_config) else: model_config = {} if self.model_config: model_config.update(self.model_config) if self.data_spec: model_config.update( { "input_channels": self.data_spec["input_channels"], "num_output_classes": [ s["num_classes"] for s in self.data_spec["output_spec"] ], } ) # create model accordingly model_class = getattr(models, self.model_class) return model_class(**model_config) base_config: str = t.Str() model_config: dict = t.Dict() model_class: str = t.Enum("FilterNet", "DeepConvLSTM") lr_exp: float = t.Float(-3.0) batch_size: int = t.Int() win_len: int = t.Int(512) n_samples_per_batch: int = t.Int(5000) train_step: int = t.Int(16) seed: int = t.Int() decimation: int = t.Int(1) optim_type: str = t.Enum(["Adam", "SGD, RMSprop"]) loss_func: str = t.Enum(["cross_entropy", "binary_cross_entropy"]) patience: int = t.Int(10) lr_decay: float = t.Float(0.95) weight_decay: float = t.Float(1e-4) alpha: float = t.Float(0.99) momentum: float = t.Float(0.25) validation_fold: int = t.Int() epoch_size: float = t.Float(2.0) y_cols: str = t.Str() sensor_subset: str = t.Str() has_null_class: bool = t.Bool() def _has_null_class_default(self): return self.data_spec["output_spec"][0]["classes"][0] in ("", "Null") predict_null_class: bool = t.Bool(True) _class_weights: torch.Tensor = t.Instance(torch.Tensor) def __class_weights_default(self): # Not weights for now because didn't seem to increase things significantly and # added yet another hyper-parameter. Using zero didn't seem to work well. if False and self.has_null_class and not self.predict_null_class: cw = torch.ones(self.model.num_output_classes, device=self.device) cw[0] = 0.01 cw /= cw.sum() return cw return None dataset: str = t.Enum( ["opportunity", "smartphone_hapt", "har", "intention_recognition"] ) name: str = t.Str() def _name_default(self): import time modelstr = self.model.__class__.__name__ timestr = time.strftime("%Y%m%d-%H%M%S") return f"{modelstr}_{timestr}" model_path: str = t.Str() def _model_path_default(self): return f"saved_models/{self.name}/" data_spec: dict = t.Any() epoch_iters: int = t.Int(0) train_state: TrainState = t.Instance(TrainState, ()) cp_iter: int = t.Int() cuda: bool = t.Bool(transient=True) def _cuda_default(self): return torch.cuda.is_available() device: str = t.Str(transient=True) def _device_default(self): return "cuda" if self.cuda else "cpu" dl_train: DataLoader = t.Instance(DataLoader, transient=True) def _dl_train_default(self): return self._get_dl("train") dl_val: DataLoader = t.Instance(DataLoader, transient=True) def _dl_val_default(self): return self._get_dl("val") dl_test: DataLoader = t.Instance(DataLoader, transient=True) def _dl_test_default(self): return self._get_dl("test") def _get_dl(self, s): if self.dataset == "opportunity": from filternet.datasets.opportunity import get_x_y_contig elif self.dataset == "smartphone_hapt": from filternet.datasets.smartphone_hapt import get_x_y_contig elif self.dataset == "har": from filternet.datasets.har import get_x_y_contig elif self.dataset == "intention_recognition": from filternet.datasets.intention_recognition import get_x_y_contig else: raise ValueError(f"Unknown dataset {self.dataset}") kwargs = {} if self.y_cols: kwargs["y_cols"] = self.y_cols if self.sensor_subset: kwargs["sensor_subset"] = self.sensor_subset Xc, ycs, data_spec = get_x_y_contig(s, **kwargs) if s == "train": # Training shuffles, and we set epoch size to length of the dataset. We can set train_step as # small as we want to get more windows; we'll only run len(Sc)/win_len of them in each training # epoch. self.epoch_iters = int(len(Xc) / self.decimation) X, ys = sliding_window_x_y( Xc, ycs, win_len=self.win_len, step=self.train_step, shuffle=False ) # Set the overall data spec using the training set, # and modify later if more info is needed. self.data_spec = data_spec else: # Val and test data are not shuffled. # Each point is inferred ~twice b/c step = win_len/2 X, ys = sliding_window_x_y( Xc, ycs, win_len=self.win_len, step=int(self.win_len / 2), shuffle=False, # Cannot be true with windows ) dl = DataLoader( TensorDataset(torch.Tensor(X), *[torch.Tensor(y).long() for y in ys]), batch_size=self.batch_size, shuffle=True if s == "train" else False, ) return dl def _batch_size_default(self): batch_size = int(self.n_samples_per_batch / self.win_len) print(f"Batch size: {batch_size}") return batch_size optimizer = t.Any(transient=True) def _optimizer_default(self): if self.optim_type == "SGD": optimizer = torch.optim.SGD( self.model.parameters(), lr=10 ** (self.lr_exp), momentum=self.momentum, weight_decay=self.weight_decay, ) elif self.optim_type == "Adam": optimizer = torch.optim.Adam( self.model.parameters(), lr=10 ** (self.lr_exp), weight_decay=self.weight_decay, amsgrad=True, ) elif self.optim_type == "RMSprop": optimizer = torch.optim.RMSprop( self.model.parameters(), lr=10 ** (self.lr_exp), alpha=self.alpha, weight_decay=self.weight_decay, momentum=self.momentum, ) else: raise NotImplementedError(self.optim_type) return optimizer iteration: int = t.Property(t.Int) def _get_iteration(self): return len(self.train_state.epoch_records) + 1 lr_scheduler = t.Any(transient=True) def _lr_scheduler_default(self): lr_scheduler = torch.optim.lr_scheduler.ExponentialLR( self.optimizer, self.lr_decay # , last_epoch=self._iteration ) # If this is being re-instantiated in mid-training, then we must # iterate scheduler forward to match the training step. for i in range(self.iteration): if self.lr_decay != 1: lr_scheduler.step() return lr_scheduler ##### # Training Methods ## def _train_batch(self, data, targets): self.optimizer.zero_grad() loss, output, _targets, _ = self._run_model_on_batch(data, targets) loss.backward() self.optimizer.step() # if self.max_lr: # self.lr_scheduler.step() return loss, output, _targets def _run_model_on_batch(self, data, targets): targets = torch.stack(targets) if self.cuda: data, targets = data.cuda(), targets.cuda() output = self.model(data) _targets = self.model.transform_targets(targets, one_hot=False) if self.loss_func == "cross_entropy": _losses = [ F.cross_entropy(o, t, weight=self._class_weights) for o, t in zip(output, _targets) ] loss = sum(_losses) elif self.loss_func == "binary_cross_entropy": _targets_onehot = self.model.transform_targets(targets, one_hot=True) _losses = [ F.binary_cross_entropy_with_logits(o, t, weight=self._class_weights) for o, t in zip(output, _targets_onehot) ] loss = sum(_losses) else: raise NotImplementedError(self.loss) # Assume only 1 output: return loss, output[0], _targets[0], _losses[0] def _calc_validation_loss(self): running_loss = 0 self.model.eval() with torch.no_grad(): for batch_idx, (data, *targets) in enumerate(self.dl_val): loss, _, _, _ = self._run_model_on_batch(data, targets) running_loss += loss.item() * data.size(0) return running_loss / len(self.dl_val.dataset) def _train_epoch(self): self.model.train() train_losses = [] train_accs = [] for batch_idx, (data, *targets) in enumerate(self.dl_train): if ( batch_idx * data.shape[0] * data.shape[2] > self.epoch_iters * self.epoch_size ): # we've effectively finished one epoch worth of data; break! break batch_loss, batch_output, batch_targets = self._train_batch(data, targets) train_losses.append(batch_loss.detach().cpu().item()) batch_preds = torch.argmax(batch_output, 1, False) train_accs.append( (batch_preds == batch_targets).detach().cpu().float().mean().item() ) if self.lr_decay != 1: self.lr_scheduler.step() return EpochMetrics(loss=np.mean(train_losses), acc=np.mean(train_accs)) def _val_epoch(self): return self._eval_epoch(self.dl_val) def _eval_epoch(self, data_loader): # Validation self.model.eval() losses = [] outputs = [] targets = [] with torch.no_grad(): for batch_idx, (data, *target) in enumerate(data_loader): ( batch_loss, batch_output, batch_targets, train_losses, ) = self._run_model_on_batch(data, target) losses.append(batch_loss.detach().cpu().item()) outputs.append( torch.argmax(batch_output, 1, False) .detach() .cpu() .data.numpy() .flatten() ) targets.append(batch_targets.detach().cpu().data.numpy().flatten()) targets = np.hstack(targets) outputs = np.hstack(outputs) acc = sklearn.metrics.accuracy_score(targets, outputs) f1 = sklearn.metrics.f1_score(targets, outputs, average="weighted") return EpochMetrics(loss=np.mean(losses), acc=acc, f1=f1) def init_data(self): # Initiate loading of datasets, model _, _, _ = self.dl_train, self.dl_val, self.dl_test _ = self.model def init_train(self): # initialization if self.seed: torch.manual_seed(self.seed) if self.cuda: if self.seed: torch.cuda.manual_seed(self.seed) self.model.to(self.device) def train_one_epoch(self, verbose=True) -> EpochRecord: """ traing a single epoch -- method tailored to the Ray.tune methodology.""" epoch_record = EpochRecord(epoch=len(self.train_state.epoch_records)) self.train_state.epoch_records.append(epoch_record) with Timer("Train Epoch", log_output=verbose) as t: epoch_record.train = self._train_epoch() epoch_record.iter_s_cpu = t.interval_cpu epoch_record.iter_s_wall = t.interval_wall epoch_record.lr = self.optimizer.param_groups[0]["lr"] with Timer("Val Epoch", log_output=verbose): epoch_record.val = self._val_epoch() df = self.train_state.to_df() # Early stopping / checkpointing implementation df["raw_metric"] = df.val_loss / df.val_f1 df["ewma_smoothed_loss"] = ( df["raw_metric"].ewm(ignore_na=False, halflife=3).mean() ) df["instability_penalty"] = ( df["raw_metric"].rolling(5, min_periods=3).std().fillna(0.75) ) stopping_metric = df["stopping_metric"] = ( df["ewma_smoothed_loss"] + df["instability_penalty"] ) epoch_record.stopping_metric = df["stopping_metric"].iloc[-1] idx_this_iter = stopping_metric.index.max() idx_best_yet = stopping_metric.idxmin() self.train_state.best_sm = df.loc[idx_best_yet, "stopping_metric"] self.train_state.best_loss = df.loc[idx_best_yet, "val_loss"] self.train_state.best_f1 = df.loc[idx_best_yet, "val_f1"] if idx_best_yet == idx_this_iter: # Best yet! Checkpoint. epoch_record.should_checkpoint = True self.cp_iter = epoch_record.epoch else: if self.patience is not None: patience_counter = idx_this_iter - idx_best_yet assert patience_counter >= 0 if patience_counter > self.patience: if verbose: print( f"Early stop! Out of patience ( {patience_counter} > {self.patience} )" ) epoch_record.done = True if verbose: self.print_train_summary() return epoch_record def train(self, max_epochs=50, verbose=True): """ A pretty standard training loop, constrained to stop in `max_epochs` but may stop early if our custom stopping metric does not improve for `self.patience` epochs. Always checkpoints when a new best stopping_metric is achieved. An alternative to using ray.tune for training.""" self.init_data() self.init_train() while True: epoch_record = self.train_one_epoch(verbose=verbose) if epoch_record.should_checkpoint: last_cp = self._save() if verbose: print(f"<<<< Checkpointed ({last_cp}) >>>") if epoch_record.done: break if epoch_record.epoch >= max_epochs: break # Save trainer state, but not model" self._save(save_model=False) if verbose: print(self.model_path) def print_train_summary(self): df = self.train_state.to_df() with pd.option_context( "display.max_rows", 100, "display.max_columns", 100, "display.precision", 3, "display.width", 180, ): print(df.drop(["done"], axis=1, errors="ignore")) def _save(self, checkpoint_dir=None, save_model=True, save_trainer=True): """ Saves/checkpoints model state and training state to disk. """ if checkpoint_dir is None: checkpoint_dir = self.model_path else: self.model_path = checkpoint_dir os.makedirs(checkpoint_dir, exist_ok=True) # save model params model_path = os.path.join(checkpoint_dir, "model.pth") trainer_path = os.path.join(checkpoint_dir, "trainer.pth") if save_model: torch.save(self.model.state_dict(), model_path) if save_trainer: with open(trainer_path, "wb") as f: pickle.dump(self, f) return checkpoint_dir def _restore(self, checkpoint_dir=None): """ Restores model state and training state from disk. """ if checkpoint_dir is None: checkpoint_dir = self.model_path model_path = os.path.join(checkpoint_dir, "model.pth") trainer_path = os.path.join(checkpoint_dir, "trainer.pth") # Reconstitute old trainer and copy state to this trainer. with open(trainer_path, "rb") as f: other_trainer = pickle.load(f) self.__setstate__(other_trainer.__getstate__()) # Load model (after loading state in case we need to re-initialize model from config) self.model.load_state_dict(torch.load(model_path, map_location=self.device)) # Be careful to reinitialize optimizer and lr scheduler self.optimizer = self._optimizer_default() self.lr_scheduler = self._lr_scheduler_default()
class DataSet(_traits.HasTraits): '''Data set for holding one matrix and associated metadata The reccomended access methods as properties for the DataSet object: * id: An identity string that is unique for each object * display_name: An human friendly name that for the datataset * kind: The data set type * missing_data: Boolean value indicatin if the data set have "holes" * n_vars: Number of variables * n_objs: Number of objects * var_n: List containing variable names * obj_n: List containing object names * values: The matrix values as an 2D Numpy array * mat: The matrix as an Pandas DataFrame ''' mat = _traits.Instance(_pd.DataFrame, ()) _id = _traits.Int() id = _traits.Property() new_id = _itr.count(start=101).next display_name = _traits.Unicode('Unnamed data set') kind = _traits.Enum(DS_TYPES) # FIXME: Only color style = _traits.Instance('VisualStyle', ()) # Example: {'species': [SubSet, SubSet], 'location': [SubSet, SubSet]} # Column subsets subs = _traits.Dict(_traits.Unicode, _traits.List) # Row subsets rsubs = _traits.Dict(_traits.Unicode, _traits.List) row_factors = _traits.List() col_factors = _traits.List() # FIXME: This data set has missing data # do you want to do somthing about it? # * throw rows/cols with missing data # * do imputation missing_data = _traits.Property(_traits.Bool) n_vars = _traits.Property() n_objs = _traits.Property() var_n = _traits.Property() obj_n = _traits.Property() values = _traits.Property() # FIXME: This is a dubious solution matcat = _traits.Instance(_pd.DataFrame) valuescat = _traits.Property() def _get_values(self): if self.missing_data: return _np.ma.masked_invalid(self.mat.values) else: return self.mat.values def _make_matcat(self): matcat = self.mat.copy() for cn, ssl in self.subs.items(): cs = _pd.Series(index=self.mat.index) for ss in ssl: cs[list(ss.row_selector)] = ss.id matcat[cn] = cs for cn, ssl in self.rsubs.items(): cs = _pd.Series(index=self.mat.index) for ss in ssl: cs[list(ss.row_selector)] = ss.id matcat[cn] = cs self.matcat = matcat def _get_valuescat(self): try: return self.matcat.values except AttributeError: # self._make_matcat() return self.matcat.values def _get_n_vars(self): return self.mat.shape[1] def _get_n_objs(self): return self.mat.shape[0] def _get_var_n(self): return list(self.mat.columns) def _get_obj_n(self): return list(self.mat.index) def __id_default(self): return DataSet.new_id() def __eq__(self, other): return self.id == other def __ne__(self, other): return self.id != other def _get_id(self): return str(self._id) def _get_missing_data(self): # FIXME: I must look more into this try: return _np.any(_np.isnan(self.mat.values)) except TypeError: return False def get_subset_groups(self): return self.subs.keys() def get_subsets(self, group): return self.subs[group] def get_subset_rows(self, subset): ''' Return a subset from given subset id''' return self.mat.loc[list(subset.row_selector)] def copy(self, transpose=False): new = self.clone_traits(traits=['display_name', 'kind', 'style', 'subs']) if transpose: tmp = self.mat.copy() new.mat = tmp.transpose() else: new.mat = self.mat.copy() return new
class ClassWardMetrics(t.HasStrictTraits): segment_twoset_results: dict = t.Dict() event_detailed_scores: dict = t.Dict() event_standard_scores: dict = t.Dict()
class EvalModel(t.HasStrictTraits): trainer: Trainer = t.Any() model: mo.BaseNet = t.DelegatesTo("trainer") dl_test: DataLoader = t.DelegatesTo("trainer") data_spec: dict = t.DelegatesTo("trainer") cuda: bool = t.DelegatesTo("trainer") device: str = t.DelegatesTo("trainer") loss_func: str = t.DelegatesTo("trainer") model_path: str = t.DelegatesTo("trainer") has_null_class: bool = t.DelegatesTo("trainer") predict_null_class: bool = t.DelegatesTo("trainer") # 'prediction' mode employs overlap and reconstructs signal # as a contiguous timeseries w/ optional windowing. # It aims for best accuracy/f1 by using overlap, and will # typically outperform 'training' mode. # 'training' mode does not average repeated point and does # not window; it should product acc/loss/f1 similar to # training mode. run_mode: str = t.Enum(["prediction", "training"]) window: str = t.Enum(["hanning", "boxcar"]) eval_batch_size: int = t.Int(100) target_names: ty.List[str] = t.ListStr() def _target_names_default(self): target_names = self.data_spec["output_spec"][0]["classes"] if self.has_null_class: assert target_names[0] in ("", "Null") if not self.predict_null_class: target_names = target_names[1:] return target_names def _run_model_on_batch(self, data, targets): targets = torch.stack(targets) if self.cuda: data, targets = data.cuda(), targets.cuda() output = self.model(data) _targets = self.model.transform_targets(targets, one_hot=False) if self.loss_func == "cross_entropy": _losses = [F.cross_entropy(o, t) for o, t in zip(output, _targets)] loss = sum(_losses) elif self.loss_func == "binary_cross_entropy": _targets_onehot = self.model.transform_targets(targets, one_hot=True) _losses = [ F.binary_cross_entropy_with_logits(o, t) for o, t in zip(output, _targets_onehot) ] loss = sum(_losses) else: raise NotImplementedError(self.loss) # Assume only 1 output: return loss, output[0], _targets[0], _losses[0] def run_test_set(self, dl=None): """ Runs `self.model` on `self.dl_test` (or a provided dl) and stores results for subsequent evaluation. """ if dl is None: dl = self.dl_test if self.cuda: self.model.cuda() self.model.eval() if self.eval_batch_size: dl = DataLoader(dl.dataset, batch_size=self.eval_batch_size, shuffle=False) # # # Xc, yc = data.get_x_y_contig('test') X, *ys = dl.dataset.tensors # X: [N, input_chans, win_len] step = int(X.shape[2] / 2) assert torch.equal(X[0, :, step], X[1, :, 0]) losses = [] outputsraw = [] outputs = [] targets = [] with Timer("run", log_output=False) as tr: with Timer("infer", log_output=False) as ti: for batch_idx, (data, *target) in enumerate(dl): ( batch_loss, batch_output, batch_targets, train_losses, ) = self._run_model_on_batch(data, target) losses.append(batch_loss.detach().cpu().item()) outputsraw.append(batch_output.detach().cpu().data.numpy()) outputs.append( torch.argmax(batch_output, 1, False).detach().cpu().data.numpy()) targets.append(batch_targets.detach().cpu().data.numpy()) self.infer_time_s_cpu = ti.interval_cpu self.infer_time_s_wall = ti.interval_wall self.loss = np.mean(losses) targets = np.concatenate(targets, axis=0) # [N, out_win_len] outputsraw = np.concatenate( outputsraw, axis=0) # [N, n_out_classes, out_win_len] outputs = np.concatenate(outputs, axis=0) # [N, n_out_classes, out_win_len] # win_len = toutputsraw[0].shape[-1] if (self.model.output_type == "many_to_one_takelast" or self.run_mode == "training"): self.targets = np.concatenate(targets, axis=-1) # [N,] self.outputsraw = np.concatenate( outputsraw, axis=-1) # [n_out_classes, N,] self.outputs = np.concatenate(outputs, axis=-1) # [N,] elif self.run_mode == "prediction": n_segments, n_classes, out_win_len = outputsraw.shape output_step = int(out_win_len / 2) if self.window == "hanning": EPS = 0.001 # prevents divide-by-zero arr_window = (1 - EPS) * np.hanning(out_win_len) + EPS elif self.window == "boxcar": arr_window = np.ones((out_win_len, )) else: raise ValueError() # Allocate space for merged predictions if self.has_null_class and not self.predict_null_class: outputsraw2 = np.zeros( (n_segments + 1, n_classes - 1, output_step, 2)) window2 = np.zeros( (n_segments + 1, n_classes - 1, output_step, 2)) # [N+1, out_win_len/2, 2] # Drop in outputs/window vals in the two layers outputsraw = outputsraw[:, 1:, :] else: outputsraw2 = np.zeros( (n_segments + 1, n_classes, output_step, 2)) window2 = np.zeros((n_segments + 1, n_classes, output_step, 2)) # [N+1, out_win_len/2, 2] # Drop in outputs/window vals in the two layers outputsraw2[:-1, :, :, 0] = outputsraw[:, :, :output_step] outputsraw2[1:, :, :, 1] = outputsraw[:, :, output_step:output_step * 2] window2[:-1, :, :, 0] = arr_window[:output_step] window2[1:, :, :, 1] = arr_window[output_step:output_step * 2] merged_outputsraw = (outputsraw2 * window2).sum( axis=3) / (window2).sum(axis=3) softmaxed_merged_outputsraw = softmax(merged_outputsraw, axis=1) merged_outputs = np.argmax(softmaxed_merged_outputsraw, 1) self.outputsraw = np.concatenate(merged_outputsraw, axis=-1) self.outputs = np.concatenate(merged_outputs, axis=-1) self.targets = np.concatenate( np.concatenate( [ targets[:, :output_step], targets[[-1], output_step:output_step * 2], ], axis=0, ), axis=-1, ) else: raise ValueError() if self.has_null_class and not self.predict_null_class: not_null_mask = self.targets > 0 self.outputsraw = self.outputsraw[..., not_null_mask] self.outputs = self.outputs[not_null_mask] self.targets = self.targets[not_null_mask] self.targets -= 1 self.n_samples_in = np.prod(dl.dataset.tensors[1].shape) self.n_samples_out = len(self.outputs) self.infer_samples_per_s = self.n_samples_in / self.infer_time_s_wall self.run_time_s_cpu = tr.interval_cpu self.run_time_s_wall = tr.interval_wall loss: float = t.Float() targets: np.ndarray = t.Array() outputsraw: np.ndarray = t.Array() outputs: np.ndarray = t.Array() n_samples_in: int = t.Int() n_samples_out: int = t.Int() infer_samples_per_s: float = t.Float() infer_time_s_cpu: float = t.Float() infer_time_s_wall: float = t.Float() run_time_s_cpu: float = t.Float() run_time_s_wall: float = t.Float() extra: dict = t.Dict({}) acc: float = t.Float() f1: float = t.Float() f1_mean: float = t.Float() event_f1: float = t.Float() classification_report_txt: str = t.Str() classification_report_dict: dict = t.Dict() classification_report_df: pd.DataFrame = t.Property( t.Instance(pd.DataFrame)) confusion_matrix: np.ndarray = t.Array() nonull_acc: float = t.Float() nonull_f1: float = t.Float() nonull_f1_mean: float = t.Float() nonull_classification_report_txt: str = t.Str() nonull_classification_report_dict: dict = t.Dict() nonull_classification_report_df: pd.DataFrame = t.Property( t.Instance(pd.DataFrame)) nonull_confusion_matrix: np.ndarray = t.Array() def calc_metrics(self): self.acc = sklearn.metrics.accuracy_score(self.targets, self.outputs) self.f1 = sklearn.metrics.f1_score(self.targets, self.outputs, average="weighted") self.f1_mean = sklearn.metrics.f1_score(self.targets, self.outputs, average="macro") self.classification_report_txt = sklearn.metrics.classification_report( self.targets, self.outputs, digits=3, labels=np.arange(len(self.target_names)), target_names=self.target_names, ) self.classification_report_dict = sklearn.metrics.classification_report( self.targets, self.outputs, digits=3, output_dict=True, labels=np.arange(len(self.target_names)), target_names=self.target_names, ) self.confusion_matrix = sklearn.metrics.confusion_matrix( self.targets, self.outputs) # Now, ignoring the null/none class: if self.has_null_class and self.predict_null_class: # assume null class comes fistnonull_mask = self.targets > 0 nonull_mask = self.targets > 0 nonull_targets = self.targets[nonull_mask] # nonull_outputs = self.outputs[nonull_mask] nonull_outputs = self.outputsraw[1:, :].argmax( axis=0)[nonull_mask] + 1 self.nonull_acc = sklearn.metrics.accuracy_score( nonull_targets, nonull_outputs) self.nonull_f1 = sklearn.metrics.f1_score(nonull_targets, nonull_outputs, average="weighted") self.nonull_f1_mean = sklearn.metrics.f1_score(nonull_targets, nonull_outputs, average="macro") self.nonull_classification_report_txt = sklearn.metrics.classification_report( nonull_targets, nonull_outputs, digits=3, labels=np.arange(len(self.target_names)), target_names=self.target_names, ) self.nonull_classification_report_dict = sklearn.metrics.classification_report( nonull_targets, nonull_outputs, digits=3, output_dict=True, labels=np.arange(len(self.target_names)), target_names=self.target_names, ) self.nonull_confusion_matrix = sklearn.metrics.confusion_matrix( nonull_targets, nonull_outputs) else: self.nonull_acc = self.acc self.nonull_f1 = self.f1 self.nonull_f1_mean = self.f1_mean self.nonull_classification_report_txt = self.classification_report_txt self.nonull_classification_report_dict = self.classification_report_dict self.nonull_confusion_matrix = self.confusion_matrix ward_metrics: WardMetrics = t.Instance(WardMetrics) def calc_ward_metrics(self): """ Do event-wise metrics, using the `wardmetrics` package which implements metrics from: [1] J. A. Ward, P. Lukowicz, and H. W. Gellersen, “Performance metrics for activity recognition,” ACM Trans. Intell. Syst. Technol., vol. 2, no. 1, pp. 1–23, Jan. 2011. """ import wardmetrics # Must be in prediction mode -- otherwise, data is not contiguous, ward metrics will be bogus assert self.run_mode == "prediction" targets = self.targets predictions = self.outputs wmetrics = WardMetrics() targets_events = wardmetrics.frame_results_to_events(targets) preds_events = wardmetrics.frame_results_to_events(predictions) for i, class_name in enumerate(self.target_names): class_wmetrics = ClassWardMetrics() t = targets_events.get(str(i), []) p = preds_events.get(str(i), []) # class_wmetrics['t'] = t # class_wmetrics['p'] = p try: assert len(t) and len(p) ( twoset_results, segments_with_scores, segment_counts, normed_segment_counts, ) = wardmetrics.eval_segments(t, p) class_wmetrics.segment_twoset_results = twoset_results ( gt_event_scores, det_event_scores, detailed_scores, standard_scores, ) = wardmetrics.eval_events(t, p) class_wmetrics.event_detailed_scores = detailed_scores class_wmetrics.event_standard_scores = standard_scores except (AssertionError, ZeroDivisionError) as e: class_wmetrics.segment_twoset_results = {} class_wmetrics.event_detailed_scores = {} class_wmetrics.event_standard_scores = {} # print("Empty Results or targets for a class.") # raise ValueError("Empty Results or targets for a class.") wmetrics.class_ward_metrics.append(class_wmetrics) tt = [] pp = [] for i, class_name in enumerate(self.target_names): # skip null class for combined eventing: if class_name in ("", "Null"): continue if len(tt) or len(pp): offset = np.max(tt + pp) + 2 else: offset = 0 [(a + offset, b + offset) for (a, b) in t] t = targets_events.get(str(i), []) p = preds_events.get(str(i), []) tt += [(a + offset, b + offset) for (a, b) in t] pp += [(a + offset, b + offset) for (a, b) in p] t = tt p = pp class_wmetrics = ClassWardMetrics() assert len(t) and len(p) ( twoset_results, segments_with_scores, segment_counts, normed_segment_counts, ) = wardmetrics.eval_segments(t, p) class_wmetrics.segment_twoset_results = twoset_results ( gt_event_scores, det_event_scores, detailed_scores, standard_scores, ) = wardmetrics.eval_events(t, p) class_wmetrics.event_detailed_scores = detailed_scores class_wmetrics.event_standard_scores = standard_scores # Reformat as dataframe for easier calculations df = pd.DataFrame( [cm.event_standard_scores for cm in wmetrics.class_ward_metrics], index=self.target_names, ) df.loc["all_nonull"] = class_wmetrics.event_standard_scores # Calculate F1's to summarize recall/precision for each class df["f1"] = (2 * (df["precision"] * df["recall"]) / (df["precision"] + df["recall"])) df["f1 (weighted)"] = ( 2 * (df["precision (weighted)"] * df["recall (weighted)"]) / (df["precision (weighted)"] + df["recall (weighted)"])) # Load dataframes into dictionary output wmetrics.df_event_scores = df wmetrics.df_event_detailed_scores = pd.DataFrame( [cm.event_detailed_scores for cm in wmetrics.class_ward_metrics], index=self.target_names, ) wmetrics.df_segment_2set_results = pd.DataFrame( [cm.segment_twoset_results for cm in wmetrics.class_ward_metrics], index=self.target_names, ) wmetrics.overall_ward_metrics = class_wmetrics self.ward_metrics = wmetrics self.event_f1 = self.ward_metrics.df_event_scores.loc["all_nonull", "f1"] def _get_classification_report_df(self): df = pd.DataFrame(self.classification_report_dict).T # Include Ward-metrics-derived "Event F1 (unweighted by length)" if self.ward_metrics: df["event_f1"] = self.ward_metrics.df_event_scores["f1"] else: df["event_f1"] = np.nan # Calculate various summary averages df.loc["macro avg", "event_f1"] = df["event_f1"].iloc[:-3].mean() df.loc["weighted avg", "event_f1"] = ( df["event_f1"].iloc[:-3] * df["support"].iloc[:-3]).sum() / df["support"].iloc[:-3].sum() df["support"] = df["support"].astype(int) return df def _get_nonull_classification_report_df(self): target_names = self.target_names if not (target_names[0] in ("", "Null")): return None df = pd.DataFrame(self.nonull_classification_report_dict).T df["support"] = df["support"].astype(int) return df def _save(self, checkpoint_dir=None): """ Saves/checkpoints model state and training state to disk. """ if checkpoint_dir is None: checkpoint_dir = self.model_path os.makedirs(checkpoint_dir, exist_ok=True) # save model params evalmodel_path = os.path.join(checkpoint_dir, "evalmodel.pth") with open(evalmodel_path, "wb") as f: pickle.dump(self, f) return checkpoint_dir def _restore(self, checkpoint_dir=None): """ Restores model state and training state from disk. """ if checkpoint_dir is None: checkpoint_dir = self.model_path evalmodel_path = os.path.join(checkpoint_dir, "evalmodel.pth") # Reconstitute old trainer and copy state to this trainer. with open(evalmodel_path, "rb") as f: other_evalmodel = pickle.load(f) self.__setstate__(other_evalmodel.__getstate__()) self.trainer._restore(checkpoint_dir)
class EnsembleTrainer(t.HasStrictTraits): def __init__(self, config={}, **kwargs): trainer_template = Trainer(**config) super().__init__(trainer_template=trainer_template, config=config, **kwargs) config: dict = t.Dict() trainer_template: Trainer = t.Instance(Trainer) trainers: ty.List[Trainer] = t.List(t.Instance(Trainer)) n_folds = t.Int(5) dl_test: DataLoader = t.DelegatesTo("trainer_template") data_spec: dict = t.DelegatesTo("trainer_template") cuda: bool = t.DelegatesTo("trainer_template") device: str = t.DelegatesTo("trainer_template") loss_func: str = t.DelegatesTo("trainer_template") batch_size: int = t.DelegatesTo("trainer_template") win_len: int = t.DelegatesTo("trainer_template") has_null_class: bool = t.DelegatesTo("trainer_template") predict_null_class: bool = t.DelegatesTo("trainer_template") name: str = t.Str() def _name_default(self): import time modelstr = "Ensemble" timestr = time.strftime("%Y%m%d-%H%M%S") return f"{modelstr}_{timestr}" X_folds = t.Tuple(transient=True) ys_folds = t.Tuple(transient=True) def _trainers_default(self): # Temp trainer for grabbing datasets, etc tt = self.trainer_template tt.init_data() # Combine official train & val sets X = torch.cat( [tt.dl_train.dataset.tensors[0], tt.dl_val.dataset.tensors[0]]) ys = [ torch.cat([yt, yv]) for yt, yv in zip( tt.dl_train.dataset.tensors[1:], tt.dl_val.dataset.tensors[1:]) ] # make folds fold_len = int(np.ceil(len(X) / self.n_folds)) self.X_folds = torch.split(X, fold_len) self.ys_folds = [torch.split(y, fold_len) for y in ys] trainers = [] for i_val_fold in range(self.n_folds): trainer = Trainer( validation_fold=i_val_fold, name=f"{self.name}/{i_val_fold}", **self.config, ) trainer.dl_test = tt.dl_test trainers.append(trainer) return trainers model: models.BaseNet = t.Instance(torch.nn.Module, transient=True) def _model_default(self): model = models.FilterNetEnsemble() model.set_models([trainer.model for trainer in self.trainers]) return model model_path: str = t.Str() def _model_path_default(self): return f"saved_models/{self.name}/" def init_data(self): # Initiate loading of datasets, model pass # for trainer in self.trainers: # trainer.init_data() def init_train(self): pass # for trainer in self.trainers: # trainer.init_train() def train(self, max_epochs=50): """ A pretty standard training loop, constrained to stop in `max_epochs` but may stop early if our custom stopping metric does not improve for `self.patience` epochs. Always checkpoints when a new best stopping_metric is achieved. An alternative to using ray.tune for training.""" for trainer in self.trainers: # Add data to trainer X_train = torch.cat([ arr for i, arr in enumerate(self.X_folds) if i != trainer.validation_fold ]) ys_train = [ torch.cat([ arr for i, arr in enumerate(y) if i != trainer.validation_fold ]) for y in self.ys_folds ] X_val = torch.cat([ arr for i, arr in enumerate(self.X_folds) if i == trainer.validation_fold ]) ys_val = [ torch.cat([ arr for i, arr in enumerate(y) if i == trainer.validation_fold ]) for y in self.ys_folds ] trainer.dl_train = DataLoader( TensorDataset(torch.Tensor(X_train), *ys_train), batch_size=trainer.batch_size, shuffle=True, ) trainer.data_spec = self.trainer_template.data_spec trainer.epoch_iters = self.trainer_template.epoch_iters trainer.dl_val = DataLoader( TensorDataset(torch.Tensor(X_val), *ys_val), batch_size=trainer.batch_size, shuffle=False, ) # Now clear local vars to save ranm X_train = ys_train = X_val = ys_val = None trainer.init_data() trainer.init_train() trainer.train(max_epochs=max_epochs) # Clear trainer train and val datasets to save ram trainer.dl_train = t.Undefined trainer.dl_val = t.Undefined print(f"RESTORING TO best model") trainer._restore() trainer._save() trainer.print_train_summary() em = EvalModel(trainer=trainer) em.run_test_set() em.calc_metrics() em.calc_ward_metrics() print(em.classification_report_df.to_string(float_format="%.3f")) em._save() def print_train_summary(self): for trainer in self.trainers: trainer.print_train_summary() def _save(self, checkpoint_dir=None, save_model=True, save_trainer=True): """ Saves/checkpoints model state and training state to disk. """ if checkpoint_dir is None: checkpoint_dir = self.model_path else: self.model_path = checkpoint_dir os.makedirs(checkpoint_dir, exist_ok=True) # save model params model_path = os.path.join(checkpoint_dir, "model.pth") trainer_path = os.path.join(checkpoint_dir, "trainer.pth") if save_model: torch.save(self.model.state_dict(), model_path) if save_trainer: with open(trainer_path, "wb") as f: pickle.dump(self, f) return checkpoint_dir def _restore(self, checkpoint_dir=None): """ Restores model state and training state from disk. """ if checkpoint_dir is None: checkpoint_dir = self.model_path model_path = os.path.join(checkpoint_dir, "model.pth") trainer_path = os.path.join(checkpoint_dir, "trainer.pth") # Reconstitute old trainer and copy state to this trainer. with open(trainer_path, "rb") as f: other_trainer = pickle.load(f) self.__setstate__(other_trainer.__getstate__()) # Load sub-models for trainer in self.trainers: trainer._restore() # Load model (after loading state in case we need to re-initialize model from config) self.model.load_state_dict( torch.load(model_path, map_location=self.device))
class ExperimentSnake(traits.HasTraits): """Main Experiment Snake GUI that sends arbitrary actions based on the experiment runner sequence and actions that have been set up.""" #mainLog = utilities.TextDisplay() mainLog = outputStream.OutputStream() statusString = traits.String("Press Start Snake to begin...") isRunning = traits.Bool(False) # true when the snake is running sequenceStarted = traits.Bool( False) # flashes true for ~1ms when sequence starts queue = traits.Int(0) variables = traits.Dict( key_trait=traits.Str, value_trait=traits.Float ) #dictionary mapping variable names in Exp control to their values in this sequence timingEdges = traits.Dict( key_trait=traits.Str, value_trait=traits.Float ) #dictionary mapping timing Edge names in Exp control to their values in this sequence statusList = [ ] #eventually will contain the information gathered from experiment Runner each time we poll startAction = traitsui.Action(name='start', action='_startSnake', image=pyface.image_resource.ImageResource( os.path.join('icons', 'start.png'))) stopAction = traitsui.Action(name='stop', action='_stopSnakeToolbar', image=pyface.image_resource.ImageResource( os.path.join('icons', 'stop.png'))) reloadHWAsAction = traitsui.Action( name='reload', action='_reloadHWAsToolbar', image=pyface.image_resource.ImageResource( os.path.join('icons', 'reload.png'))) connectionTimer = traits.Instance( Timer ) # polls the experiment runner and starts off callbacks at appropriate times statusStringTimer = traits.Instance( Timer) #updates status bar at regular times (less freque) getCurrentTimer = traits.Instance( Timer ) #waits for get current to return which marks the beginning of a sequence getCurrentThread = traits.Instance(SocketThread) connectionPollFrequency = traits.Float( 1000.0) #milliseconds defines accuracy you will perform callbacks at statusStringFrequency = traits.Float(2000.0) #milliseconds getCurrentFrequency = traits.Float( 1000.0) #milliseconds should be shorter than the sequence timeRunning = traits.Float(0.0) #how long the sequence has been running totalTime = traits.Float(0.0) # total length of sequence runnerHalted = traits.Bool(True) # true if runner is halted haltedCount = 0 progress = traits.Float(0.0) # % of cycle complete #progressBar = ProgressDialog() hardwareActions = traits.List(hardwareAction.hardwareAction.HardwareAction) examineVariablesDictionary = traits.Instance( variableDictionary.ExamineVariablesDictionary) xmlString = "" # STRING that will contain entire XML File menubar = traitsui.MenuBar( traitsui.Menu( traitsui.Action(name='Start Snake', action='_startSnake'), traitsui.Action(name='Stop Snake', action='_stopSnake'), traitsui.Action(name='Reload', action='_reloadHWAs'), traitsui.Menu(traitsui.Action(name='DEBUG', action='_changeLoggingLevelDebug'), traitsui.Action(name='INFO', action='_changeLoggingLevelInfo'), traitsui.Action(name='WARNING', action='_changeLoggingLevelWarning'), traitsui.Action(name='ERROR', action='_changeLoggingLevelError'), name="Log Level"), name='Menu')) toolbar = traitsui.ToolBar(startAction, stopAction, reloadHWAsAction) mainSnakeGroup = traitsui.VGroup( traitsui.Item('statusString', show_label=False, style='readonly'), traitsui.Item('mainLog', show_label=False, springy=True, style='custom', editor=traitsui.InstanceEditor())) hardwareActionsGroup = traitsui.Group(traitsui.Item( 'hardwareActions', show_label=False, style='custom', editor=traitsui.ListEditor(style="custom")), label="Hardware Actions", show_border=True) variableExaminerGroup = traitsui.Group(traitsui.Item( "examineVariablesDictionary", editor=traitsui.InstanceEditor(), style="custom", show_label=False), label="Variable Examiner") sidePanelGroup = traitsui.VSplit(hardwareActionsGroup, variableExaminerGroup) traits_view = traitsui.View(traitsui.HSplit(sidePanelGroup, mainSnakeGroup, show_labels=True), resizable=True, menubar=menubar, toolbar=toolbar, width=0.5, height=0.75, title="Experiment Snake", icon=pyface.image_resource.ImageResource( os.path.join('icons', 'snakeIcon.ico'))) def __init__(self, **traits): """ takes no arguments to construct the snake. Everything is done through GUI. Snake construction makes a ExperimentSnakeConnection object and writes to the main log window""" super(ExperimentSnake, self).__init__(**traits) self.connection = experimentRunnerConnection.Connection( ) #can override default ports and IP self.hardwareActions = [ hardwareAction.sequenceLoggerHWA.SequenceLogger( 0.0, snakeReference=self), hardwareAction.experimentTablesHWA.ExperimentTables( 0.0, snakeReference=self, enabled=False), hardwareAction.dlicEvapHWA.EvaporationRamp(1.0, snakeReference=self), #hardwareAction.dlicRFSweepHWA.DLICRFSweep(1.0, snakeReference = self,enabled=False), hardwareAction.dlicRFSweepLZHWA.DLICRFSweep(1.0, snakeReference=self, enabled=False), hardwareAction.dlicRFSweepLZWithPowerCtrlHWA.DLICRFSweep( 1.0, snakeReference=self, enabled=False), hardwareAction.dlicRFSweepLZWithPowerCtrl13PreparationHWA. DLICRFSweep(1.0, snakeReference=self, enabled=True), hardwareAction.dlicPiPulseHWA.DLICPiPulse(1.0, snakeReference=self, enabled=False), hardwareAction.evapAttenuationHWA.EvapAttenuation( 1.0, snakeReference=self), hardwareAction.greyMollassesOffsetFreqHWA.GreyMollassesOffset( 1.0, snakeReference=self, enabled=False), hardwareAction.evapAttenuation2HWA.EvapAttenuation( "EvapSnakeAttenuationTimeFinal", snakeReference=self, enabled=False), hardwareAction.picomotorPlugHWA.PicomotorPlug(1.0, snakeReference=self, enabled=False), hardwareAction.windFreakOffsetLockHWA.WindFreak( 0.0, snakeReference=self, enabled=False), hardwareAction.windFreakOffsetLockHighFieldImagingHWA.WindFreak( 0.0, snakeReference=self, enabled=True), hardwareAction.windFreakOffsetLock6ImagingHWA.WindFreak( 2.0, snakeReference=self, enabled=False), hardwareAction.windFreak6To1HWA.WindFreak(2.0, snakeReference=self, enabled=False), hardwareAction.windFreakOffsetLockLaser3.WindFreak( 3.0, snakeReference=self, enabled=False), hardwareAction.AOMChannelHWAs.AOMChannelNaZSFreq( 1.0, snakeReference=self, enabled=False), hardwareAction.AOMChannelHWAs.AOMChannelNaZSAtten( 1.0, snakeReference=self, enabled=False), hardwareAction.AOMChannelHWAs.AOMChannelNaZSEOMFreq( 1.0, snakeReference=self, enabled=False), hardwareAction.AOMChannelHWAs.AOMChannelNaZSEOMAtten( 1.0, snakeReference=self, enabled=False), hardwareAction.AOMChannelHWAs.AOMChannelNaSpecFreq( 1.0, snakeReference=self, enabled=False), hardwareAction.AOMChannelHWAs.AOMChannelLiImaging( 1.0, snakeReference=self, enabled=False), hardwareAction.AOMChannelHWAs.AOMChannelLiImagingDetuning( 1.0, snakeReference=self, enabled=False), hardwareAction.AOMChannelHWAs.AOMChannelLiPushPulseAttenuation( 1.0, snakeReference=self, enabled=False), hardwareAction.AOMChannelHWAs.AOMChannelLiPushPulseDetuning( 1.0, snakeReference=self, enabled=False), hardwareAction.AOMChannelHWAs.AOMChannelNaDarkSpotAOMFreq( 1.0, snakeReference=self, enabled=False), hardwareAction.AOMChannelHWAs.AOMChannelNaDarkSpotAOMAtten( 1.0, snakeReference=self, enabled=False), hardwareAction.AOMChannelHWAs.AOMChannelNaMOTFreq( 1.0, snakeReference=self, enabled=False), hardwareAction.AOMChannelHWAs.AOMChannelNaMOTAtten( 1.0, snakeReference=self, enabled=False), hardwareAction.AOMChannelHWAs.AOMChannelNaMOTEOMAtten( 1.0, snakeReference=self, enabled=False), hardwareAction.AOMChannelHWAs.AOMChannelNaImagingDP( 1.0, snakeReference=self, enabled=False), hardwareAction.AOMChannelHWAs.AOMChannelLiMOTRep( 1.0, snakeReference=self, enabled=False), hardwareAction.AOMChannelHWAs.AOMChannelLiMOTCool( 1.0, snakeReference=self, enabled=False), hardwareAction.AOMChannelHWAs.AOMChannelLiOpticalPump( 1.0, snakeReference=self, enabled=False), hardwareAction.AOMChannelHWAs.AOMChannelNa2to2OpticalPumpingFreq( 1.0, snakeReference=self, enabled=False), hardwareAction.AOMChannelHWAs.AOMChannelNa2to2OpticalPumpingAtt( 1.0, snakeReference=self, enabled=False), hardwareAction.AOMChannelHWAs.AOMChannelNaHighFieldImagingFreq( 1.0, snakeReference=self, enabled=False), hardwareAction.AOMChannelHWAs.AOMChannelNaHighFieldImagingAtt( 1.0, snakeReference=self, enabled=False), hardwareAction.digitalMultimeterCurrentMeasureHWA. DigitalMultimeterMeasurement(1.0, snakeReference=self, enabled=True), hardwareAction.MXGPiPulseHWA.PiPulse(1.0, snakeReference=self, enabled=False), hardwareAction.variableExplorerHWA.VariableExplorer( 2.0, snakeReference=self, enabled=False), hardwareAction.jds6600HWA.JDS6600HWA(1.0, snakeReference=self, enabled=False), hardwareAction.watchdogHWA.WatchdogHWA(18.0, snakeReference=self, enabled=True) ] introString = """Welcome to experiment snake.""" self.mainLog.addLine(introString, 1) def initialiseHardwareActions(self): for hdwAct in self.hardwareActions: if hdwAct.enabled: returnString = hdwAct.init() hdwAct.variablesReference = self.variables self.mainLog.addLine(returnString) def closeHardwareActions(self): """ this function is called when the user presses stop key. it should cleanly close or shutdown all hardware. user must appropriately implement the hardware action close function""" for hdwAct in self.hardwareActions: if hdwAct.initialised: returnString = hdwAct.close() self.mainLog.addLine(returnString) def _startSnake(self): """action call back from menu or toolbar. Simply starts the timer that polls the runner and makes the isRunning bool true """ self.mainLog.addLine("Experiment Snake Started", 1) self.isRunning = True self.getCurrentBlocking() self.initialiseHardwareActions() self.startTimers() def newSequenceStarted(self): """called by GetCurrent Thread at the beginning of every sequence """ if self.isRunning: #otherwise we have already stopped before new sequence began again self.getStatusUpdate() self.mainLog.addLine("New cycle started: %s" % self.statusList[0], 1) self.refreshExamineVariablesDictionary( ) # update the examine variables dictionary to reflect the latest values self.refreshVariableDependentCallbackTimes( ) # if a callback time is a timing edge name or variable name we must pull the value here else: self.mainLog.addLine("final connection closed") for hdwAct in self.hardwareActions: hdwAct.awaitingCallback = True def _stopSnakeToolbar(self): """if snake is stopped, addLine to main log and then run stopSnake """ self.mainLog.addLine( "Experiment Snake Stopped (you should still wait till the end of this sequence before continuing)", 1) self._stopSnake() def _reloadHWAsToolbar(self): """if snake is stopped, addLine to main log and then run stopSnake """ self.mainLog.addLine( "Experiment Snake Stopped (you should still wait till the end of this sequence before continuing)", 1) self._reloadHWAs() def _reloadHWAs(self): """if snake is stopped, addLine to main log and then run stopSnake """ self.mainLog.addLine("Reloading hardware actions (advanced feature)", 3) reload(hardwareAction.hardwareAction) reload(hardwareAction.sequenceLoggerHWA) reload(hardwareAction.dlicEvapHWA) reload(hardwareAction.dlicRFSweepHWA) reload(hardwareAction.dlicRFSweepHWA) reload(hardwareAction.evapAttenuationHWA) reload(hardwareAction.evapAttenuation2HWA) reload(hardwareAction.picomotorPlugHWA) reload(hardwareAction.windFreakOffsetLockHWA) #reload( hardwareAction.AOMChannelHWAs)#CAUSES REFERENCING PROBLEMS! reload(hardwareAction.experimentTablesHWA) reload(hardwareAction.windFreakOffsetLockHighFieldImagingHWA) reload(hardwareAction.greyMollassesOffsetFreqHWA) reload(hardwareAction.dlicRFSweepLZHWA) reload(hardwareAction.digitalMultimeterCurrentMeasureHWA) self.__init__() def stopTimers(self): """stops all timers with error catching """ try: #stop any previous timer, should only have 1 timer at a time if self.connectionTimer is not None: self.connectionTimer.stop() except Exception as e: logger.error( "couldn't stop current timer before starting new one: %s" % e.message) try: #stop any previous timer, should only have 1 timer at a time if self.statusStringTimer is not None: self.statusStringTimer.stop() except Exception as e: logger.error( "couldn't stop current timer before starting new one: %s" % e.message) try: #stop any previous timer, should only have 1 timer at a time if self.getCurrentTimer is not None: self.getCurrentTimer.stop() except Exception as e: logger.error( "couldn't stop current timer before starting new one: %s" % e.message) def _stopSnake(self): """Simply stops the timers, shuts down hardware and sets isRunning bool false """ self.stopTimers() self.closeHardwareActions() self.isRunning = False def startTimers(self): """This timer object polls the experiment runner regularly polling at any time""" #stop any previous timers self.stopTimers() #start timer self.connectionTimer = Timer(self.connectionPollFrequency, self.getStatus) time.sleep(0.1) self.statusStringTimer = Timer(self.statusStringFrequency, self.updateStatusString) time.sleep(0.1) self.getCurrentTimer = Timer(self.getCurrentFrequency, self.getCurrent) """Menu action function to change logger level """ logger.info("timers started") def getStatus(self): """calls the connection objects get status function and updates the statusList """ logger.debug("starting getStatus") try: self.getStatusUpdate() self.checkForCallback() except Exception as e: logger.error("error in getStatus Function") logger.error("error: %s " % e.message) self.mainLog.addLine( "error in getStatus Function. Error: %s" % e.message, 4) def getStatusUpdate(self): """Calls get status and updates times """ try: statusString = self.connection.getStatus() except socket.error as e: logger.error( "failed to get status . message=%s . errno=%s . errstring=%s " % (e.message, e.errno, e.strerror)) self.mainLog.addLine( "Failed to get status from Experiment Runner. message=%s . errno=%s . errstring=%s" % (e.message, e.errno, e.strerror), 3) self.mainLog.addLine( "Cannot update timeRunning - callbacks could be wrong this sequence!", 4) return self.statusList = statusString.split("\n") timeFormat = '%d/%m/%Y %H:%M:%S' timeBegin = datetime.datetime.strptime(self.statusList[2], timeFormat) timeCurrent = datetime.datetime.strptime(self.statusList[3], timeFormat) self.timeRunning = (timeCurrent - timeBegin).total_seconds() logger.debug("time Running = %s " % self.timeRunning) def checkForCallback(self): """if we've received a sequence, we check through all callback times and send off a callback on a hardware action if appropriate""" try: for hdwAct in [ hdwA for hdwA in self.hardwareActions if hdwA.enabled ]: #only iterate through enable hardware actions if hdwAct.awaitingCallback and self.timeRunning >= hdwAct.callbackTime: #callback should be started! try: logger.debug("attempting to callback %s " % hdwAct.hardwareActionName) hdwAct.setVariablesDictionary(self.variables) logger.debug("vars dictionary set to %s " % self.variables) callbackReturnString = hdwAct.callback() self.mainLog.addLine( "%s @ %s secs : %s" % (hdwAct.hardwareActionName, self.timeRunning, callbackReturnString), 2) hdwAct.awaitingCallback = False hdwAct.callbackCounter += 1 except Exception as e: logger.error( "error while performing callback on %s. see error message below" % (hdwAct.hardwareActionName)) logger.error("error: %s " % e.message) self.mainLog.addLine( "error while performing callback on %s. Error: %s" % (hdwAct.hardwareActionName, e.message), 4) except Exception as e: logger.error("error in checkForCallbackFunction") logger.error("error: %s " % e.message) self.mainLog.addLine( "error in checkForCallbackFunction. Error: %s" % e.message, 4) def getCurrent(self): """calls the connection objects get status function and updates the variables dictionary """ if self.getCurrentThread and self.getCurrentThread.isAlive(): #logger.debug( "getCurrent - already waiting - will not start new thread") #removed the above as it fills the log without any useful information self.sequenceStarted = False return else: logger.info("starting getCurrent Thread") self.getCurrentThread = SocketThread() self.getCurrentThread.snakeReference = self # for calling functions of the snake self.getCurrentThread.start() def getCurrentBlocking(self): """calls getCurrent and won't return until XML parsed. unlike above threaded function This is useful when starting up the snake so that we don't start looking for hardware events until a sequence has started and we have received XML""" self.mainLog.addLine("Waiting for next sequence to start") self.xmlString = self.connection.getCurrent( ) # only returns at the beginning of a sequence! Experiment runner then returns the entire XML file logger.debug("length of xml string = %s " % len(self.xmlString)) logger.debug("end of xml file is like [-30:]= %s" % self.xmlString[-30:]) try: root = ET.fromstring(self.xmlString) variables = root.find("variables") self.variables = { child[0].text: float(child[1].text) for child in variables } #timing edges dictionary : name--> value self.timingEdges = { timingEdge.find("name").text: float(timingEdge.find("value").text) for timingEdge in root.find("timing") } self.newSequenceStarted() except ET.ParseError as e: self.mainLog.addLine("Error. Could not parse XML: %s" % e.message, 3) self.mainLog.addLine( "Possible cause is that buffer is full. is XML length %s>= limit %s ????" % (len(self.xmlString), self.connection.BUFFER_SIZE_XML), 3) logger.error("could not parse XML: %s " % self.xmlString) logger.error(e.message) def updateStatusString(self): """update the status string with first element of return of GETSTATUS. similiar to experiment control and camera control. It also does the analysis of progress that doesn't need to be as accurate (e.g. progress bar)""" logger.info("starting update status string") self.statusString = self.statusList[ 0] + "- Time Running = %s " % self.timeRunning self.queue = int(self.statusList[1]) timeFormat = '%d/%m/%Y %H:%M:%S' timeBegin = datetime.datetime.strptime(self.statusList[2], timeFormat) timeEnd = datetime.datetime.strptime(self.statusList[4], timeFormat) self.timeTotal = (timeEnd - timeBegin).total_seconds() if self.timeRunning > self.timeTotal: self.haltedCount += 1 self.runnerHalted = True if self.haltedCount == 0: logger.critical("runner was stopped.") self.mainLog.addLine("Runner stopped!", 3) self.closeHardwareActions() else: if self.haltedCount > 0: self.initialiseHardwareActions() self.haltedCount = 0 self.runnerHalted = False self.progress = 100.0 * self.timeRunning / self.timeTotal def _examineVariablesDictionary_default(self): if len(self.hardwareActions) > 0: logger.debug( "returning first hardware action %s for examineVariablesDictionary default" % self.hardwareActions[0].hardwareActionName) return variableDictionary.ExamineVariablesDictionary( hdwA=self.hardwareActions[0] ) #default is the first in the list else: logger.warning( "hardwareActions list was empty. how should I populate variable examiner...?!." ) return None def updateExamineVariablesDictionary(self, hdwA): """Populates the examineVariablesDictionary Pane appropriately. It is passed the hdwA so that it can find the necessary variables""" self.examineVariablesDictionary.hdwA = hdwA self.examineVariablesDictionary.hardwareActionName = hdwA.hardwareActionName self.examineVariablesDictionary.updateDisplayList() logger.critical("examineVariablesDictionary changed") def refreshExamineVariablesDictionary(self): """calls the updateDisplayList function of examineVariables Dictionary this updates the values in the display list to the latest values in variables dictionary. useful for refereshing at the beginning of a sequence""" self.examineVariablesDictionary.updateDisplayList() logger.info("refreshed examine variables dictionary") def refreshVariableDependentCallbackTimes(self): """if a HWA is variable dependent call back time, we refresh the value using this function. THis should be called in each sequence""" [ hdwA.parseCallbackTime() for hdwA in self.hardwareActions if hdwA.callbackTimeVariableDependent ] def _changeLoggingLevelDebug(self): """Menu action function to change logger level """ logger.setLevel(logging.DEBUG) def _changeLoggingLevelInfo(self): """Menu action function to change logger level """ logger.setLevel(logging.INFO) def _changeLoggingLevelWarning(self): """Menu action function to change logger level """ logger.setLevel(logging.WARNING) def _changeLoggingLevelError(self): """Menu action function to change logger level """ logger.setLevel(logging.ERROR)
class MATSXDMicroplaneDamageODF(MATSXDEval): epsilon_0 = Float(59e-6, label="a", desc="Lateral pressure coefficient", enter_set=True, auto_set=False) epsilon_f = Float(250e-6, label="a", desc="Lateral pressure coefficient", enter_set=True, auto_set=False) c_T = Float(0.00, label="a", desc="Lateral pressure coefficient", enter_set=True, auto_set=False) zeta_G = Float(1.0, label="zeta_G", desc="anisotropy parameter", enter_set=True, auto_set=False) state_var_shapes = tr.Property(tr.Dict(), depends_on='n_mp') '''Dictionary of state variable entries with their array shapes. ''' @cached_property def _get_state_var_shapes(self): return {'kappa': (self.n_mp, ), 'omega': (self.n_mp, )} #------------------------------------------------------------------------- # MICROPLANE-Kinematic constraints #------------------------------------------------------------------------- # get the dyadic product of the microplane normals _MPNN = Property(depends_on='n_mp') @cached_property def _get__MPNN(self): # dyadic product of the microplane normals MPNN_nij = einsum('ni,nj->nij', self._MPN, self._MPN) return MPNN_nij # get the third order tangential tensor (operator) for each microplane _MPTT = Property(depends_on='n_mp') @cached_property def _get__MPTT(self): # Third order tangential tensor for each microplane delta = identity(2) MPTT_nijr = 0.5 * ( einsum('ni,jr -> nijr', self._MPN, delta) + einsum('nj,ir -> njir', self._MPN, delta) - 2.0 * einsum('ni,nj,nr -> nijr', self._MPN, self._MPN, self._MPN)) return MPTT_nijr def _get_e_Emna(self, eps_Emab): # Projection of apparent strain onto the individual microplanes e_ni = einsum('nb,Emba->Emna', self._MPN, eps_Emab) return e_ni def _get_e_N_Emn(self, e_Emna): # get the normal strain array for each microplane e_N_Emn = einsum('Emna, na->Emn', e_Emna, self._MPN) return e_N_Emn def _get_e_equiv_Emn(self, e_Emna): ''' Returns a list of the microplane equivalent strains based on the list of microplane strain vectors ''' # magnitude of the normal strain vector for each microplane e_N_Emn = self._get_e_N_Emn(e_Emna) # positive part of the normal strain magnitude for each microplane e_N_pos_Emn = (np.abs(e_N_Emn) + e_N_Emn) / 2 # normal strain vector for each microplane e_N_Emna = einsum('Emn,ni -> Emni', e_N_Emn, self._MPN) # tangent strain ratio c_T = self.c_T # tangential strain vector for each microplane e_T_Emna = e_Emna - e_N_Emna # squared tangential strain vector for each microplane e_TT_Emn = einsum('Emni,Emni -> Emn', e_T_Emna, e_T_Emna) # equivalent strain for each microplane e_equiv_Emn = sqrt(e_N_pos_Emn * e_N_pos_Emn + c_T * e_TT_Emn) return e_equiv_Emn def update_state_variables(self, eps_Emab, kappa, omega): e_Emna = self._get_e_Emna(eps_Emab) eps_eq_Emn = self._get_e_equiv_Emn(e_Emna) f_trial_Emn = eps_eq_Emn - self.epsilon_0 I = np.where(f_trial_Emn > 0) kappa[I] = eps_eq_Emn[I] omega[I] = self._get_omega(eps_eq_Emn[I]) return I def _get_omega(self, kappa_Emn): ''' Return new value of damage parameter @param kappa: ''' omega_Emn = np.zeros_like(kappa_Emn) epsilon_0 = self.epsilon_0 epsilon_f = self.epsilon_f kappa_idx = np.where(kappa_Emn >= epsilon_0) omega_Emn[kappa_idx] = (1.0 - (epsilon_0 / kappa_Emn[kappa_idx] * np.exp(-1.0 * (kappa_Emn[kappa_idx] - epsilon_0) / (epsilon_f - epsilon_0)))) return omega_Emn def _get_phi_Emab(self, kappa_Emn): # Returns the 2nd order damage tensor 'phi_mtx' # scalar integrity factor for each microplane phi_Emn = 1.0 - self._get_omega(kappa_Emn) # integration terms for each microplanes phi_Emab = einsum('Emn,n,nab->Emab', phi_Emn, self._MPW, self._MPNN) return phi_Emab # def _get_beta_Emabcd(self, phi_Emab): # ''' # Returns the 4th order damage tensor 'beta4' using sum-type symmetrization # (cf. [Jir99], Eq.(21)) # ''' # delta = identity(2) # beta_Emijkl = 0.25 * (einsum('Emik,jl->Emijkl', phi_Emab, delta) + # einsum('Emil,jk->Emijkl', phi_Emab, delta) + # einsum('Emjk,il->Emijkl', phi_Emab, delta) + # einsum('Emjl,ik->Emijkl', phi_Emab, delta)) # # return beta_Emijkl #---------------------------------------------------------------- # the fourth order volumetric-identity tensor #---------------------------------------------------------------- def _get_I_vol_abcd(self): delta = identity(2) I_vol_abcd = (1.0 / 3.0) * einsum('ab,cd -> abcd', delta, delta) return I_vol_abcd #---------------------------------------------------------------- # Returns the fourth order deviatoric-identity tensor #---------------------------------------------------------------- def _get_I_dev_abcd(self): delta = identity(2) I_dev_abcd = 0.5 * (einsum('ac,bd -> abcd', delta, delta) + einsum('ad,bc -> abcd', delta, delta)) \ - (1. / 3.0) * einsum('ab,cd -> abcd', delta, delta) return I_dev_abcd #---------------------------------------------------------------- # Returns the fourth order tensor P_vol [Wu.2009] #---------------------------------------------------------------- def _get_P_vol_ab(self): delta = identity(2) P_vol_ab = (1.0 / 3.0) * delta return P_vol_ab #---------------------------------------------------------------- # Returns the fourth order tensor P_dev [Wu.2009] #---------------------------------------------------------------- def _get_P_dev_nabc(self): delta = identity(2) P_dev_nabc = 0.5 * einsum('nd,da,bc -> nabc', self._MPN, delta, delta) return P_dev_nabc #---------------------------------------------------------------- # Returns the outer product of P_vol [Wu.2009] #---------------------------------------------------------------- def _get_PP_vol_abcd(self): delta = identity(2) PP_vol_abcd = (1.0 / 9.0) * einsum('ab,cd -> abcd', delta, delta) return PP_vol_abcd #---------------------------------------------------------------- # Returns the inner product of P_dev #---------------------------------------------------------------- def _get_PP_dev_nabcd(self): delta = identity(2) PP_dev_nabcd = 0.5 * (0.5 * (einsum('na,nc,bd -> nabcd', self._MPN, self._MPN, delta) + einsum('na,nd,bc -> nabcd', self._MPN, self._MPN, delta)) + 0.5 * (einsum('ac,nb,nd -> nabcd', delta, self._MPN, self._MPN) + einsum('ad,nb,nc -> nabcd', delta, self._MPN, self._MPN))) - \ (1.0 / 3.0) * (einsum('na,nb,cd -> nabcd', self._MPN, self._MPN, delta) + einsum('ab,nc,nd -> nabcd', delta, self._MPN, self._MPN)) + \ (1.0 / 9.0) * einsum('ab,cd -> abcd', delta, delta) return PP_dev_nabcd #-------------------------------------------------------------------------- # Returns the fourth order secant stiffness tensor (cf. [Wu.2009], Eq.(29)) #-------------------------------------------------------------------------- def _get_S_1_Emabcd(self, eps_Emab, kappa, omega): K0 = self.E / (1.0 - 2.0 * self.nu) G0 = self.E / (1.0 + self.nu) phi_Emn = 1.0 - self._get_omega(kappa) PP_vol_abcd = self._get_PP_vol_abcd() PP_dev_nabcd = self._get_PP_dev_nabcd() I_dev_abcd = self._get_I_dev_abcd() S_1_Emabcd = K0 * einsum('Emn, n, abcd-> Emabcd', phi_Emn, self._MPW, PP_vol_abcd) + \ G0 * 2.0 * self.zeta_G * einsum('Emn, n, nabcd-> Emabcd', phi_Emn, self._MPW, PP_dev_nabcd) - (1.0 / 3.0) * ( 2.0 * self.zeta_G - 1.0) * G0 * einsum('Emn, n, abcd-> Emabcd', phi_Emn, self._MPW, I_dev_abcd) return S_1_Emabcd # #------------------------------------------ # # scalar damage factor for each microplane # #------------------------------------------ # def _get_d_Em(self, s_Emng, eps_Emab): # # d_Emn = 1.0 - self.get_state_variables(s_Emng, eps_Emab)[0] # # d_Em = (1.0 / 3.0) * einsum('Emn,n-> Em', d_Emn, self._MPW) # # return d_Em # # #------------------------------------------ # # The 4th order volumetric damage tensor # #------------------------------------------ # def _get_M_vol_abcd(self, sctx, eps_app_eng, sigma_kk): # # d = self._get_Em( s_Emng, eps_Emab) # delta = identity(2) # # I_4th_abcd = 0.5 * (einsum('ac,bd -> ijkl', delta, delta) + # einsum('il,jk -> ijkl', delta, delta)) # # # print 'M_vol', (1 - d) * I_4th_ijkl # # return (1 - d) * I_4th_ijkl # # #------------------------------------------ # # The 4th order deviatoric damage tensor # #------------------------------------------ # def _get_M_dev_tns(self, phi_mtx): # # delta = identity(3) # I_4th_ijkl = 0.5 * (einsum('ik,jl -> ijkl', delta, delta) + # einsum('il,jk -> ijkl', delta, delta)) # tr_phi_mtx = np.trace(phi_mtx) # # M_dev_ijkl = self.zeta_G * (0.5 * (einsum('ik,jl->ijkl', delta, phi_mtx) + # einsum('il,jk->ijkl', delta, phi_mtx)) + # 0.5 * (einsum('ik,jl->ijkl', phi_mtx, delta) + # einsum('il,jk->ijkl', phi_mtx, delta))) \ # - (2. * self.zeta_G - 1.) * (tr_phi_mtx / 3.) * I_4th_ijkl # # return M_dev_ijkl # # #-------------------------------------------------------------------------- # # Returns the fourth order secant stiffness tensor (cf. [Wu.2009], Eq.(31)) # #-------------------------------------------------------------------------- # def _get_S_2_Emabcd(self, sctx, eps_app_eng, sigma_kk): # # K0 = self.E / (1. - 2. * self.nu) # G0 = self.E / (1. + self.nu) # # I_vol_ijkl = self._get_I_vol_4() # I_dev_ijkl = self._get_I_dev_4() # phi_mtx = self._get_phi_mtx(sctx, eps_app_eng, sigma_kk) # M_vol_ijkl = self._get_M_vol_tns(sctx, eps_app_eng, sigma_kk) # M_dev_ijkl = self._get_M_dev_tns(phi_mtx) # # S_2_ijkl = K0 * einsum('ijmn,mnrs,rskl -> ijkl', I_vol_ijkl, M_vol_ijkl, I_vol_ijkl ) \ # + G0 * einsum('ijmn,mnrs,rskl -> ijkl', I_dev_ijkl, M_dev_ijkl, I_dev_ijkl)\ # # return S_2_ijkl # # #-------------------------------------------------------------------------- # # Returns the fourth order secant stiffness tensor (cf. [Wu.2009], Eq.(34)) # #-------------------------------------------------------------------------- # def _get_S_3_Emabcd(self, sctx, eps_app_eng, sigma_kk): # # K0 = self.E / (1. - 2. * self.nu) # G0 = self.E / (1. + self.nu) # # I_vol_ijkl = self._get_I_vol_4() # I_dev_ijkl = self._get_I_dev_4() # # # The fourth order elastic stiffness tensor # S_0_ijkl = K0 * I_vol_ijkl + G0 * I_dev_ijkl # # d_n = self._get_state_variables(sctx, eps_app_eng, sigma_kk)[:, 5] # # PP_vol_4 = self._get_PP_vol_4() # PP_dev_4 = self._get_PP_dev_4() # # delta = identity(3) # I_4th_ijkl = einsum('ik,jl -> ijkl', delta, delta) # # D_ijkl = einsum('n,n,ijkl->ijkl', d_n, self._MPW, PP_vol_4) + \ # 2 * self.zeta_G * einsum('n,n,nijkl->ijkl', d_n, self._MPW, PP_dev_4) - ( # 1 / 3.) * (2 * self.zeta_G - 1) * einsum('n,n,ijkl->ijkl', d_n, self._MPW, I_dev_ijkl) # # phi_ijkl = (I_4th_ijkl - D_ijkl) # # S_ijkl = einsum('ijmn,mnkl', phi_ijkl, S_0_ijkl) # # return S_ijkl # #------------------------------------------------------------------------- # Returns the fourth order secant stiffness tensor using (double orthotropic) assumption #------------------------------------------------------------------------- def _get_S_4_Emabcd(self, eps_Emab, kappa, omega): K0 = self.E / (1.0 - 2.0 * self.nu) G0 = self.E / (1.0 + self.nu) I_vol_abcd = self._get_I_vol_abcd() I_dev_abcd = self._get_I_dev_abcd() delta = identity(2) phi_Emab = self._get_phi_Emab(kappa) D_Emab = delta - phi_Emab d_Em = (1.0 / 3.0) * np.trace(D_Emab) D_bar_Emab = self.zeta_G * (D_Emab - d_Em * delta) S_4_Emabcd = (1.0 - d_Em) * K0 * I_vol_abcd + (1.0 - d_Em) * G0 * I_dev_abcd + (2.0 / 3.0) * (G0 - K0) * \ (einsum('ij,Emkl -> Emijkl', delta, D_bar_Emab) + einsum('Emij,kl -> Emijkl', D_bar_Emab, delta)) + 0.5 * (-K0 + 2.0 * G0) * \ (0.5 * (einsum('ik,Emjl -> Emijkl', delta, D_bar_Emab) + einsum('Emil,jk -> Emijkl', D_bar_Emab, delta)) + 0.5 * (einsum('Emil,jk -> Emijkl', D_bar_Emab, delta) + einsum('ik,Emjl -> Emijkl', delta, D_bar_Emab))) return S_4_Emabcd def _get_S_5_Emabcd(self, eps_Emab, kappa, omega): #---------------------------------------------------------------------- # Returns the fourth order secant stiffness tensor (restrctive orthotropic) #---------------------------------------------------------------------- K0 = self.E / (1. - 2. * self.nu) G0 = self.E / (1. + self.nu) delta = identity(2) phi_Emab = self._get_phi_Emab(kappa) # damaged stiffness without simplification S_5_Emabcd = (1.0 / 3.0) * (K0 - G0) * 0.5 * ((einsum('ij,Emkl -> Emijkl', delta, phi_Emab) + einsum('Emij,kl -> Emijkl', phi_Emab, delta))) + \ G0 * 0.5 * ((0.5 * (einsum('ik,Emjl -> Emijkl', delta, phi_Emab) + einsum('Emil,jk -> Emijkl', phi_Emab, delta)) + 0.5 * (einsum('Emik,jl -> ijkl', phi_Emab, delta) + einsum('il,Emjk -> Emijkl', delta, phi_Emab)))) # print 'S_5_Emabcd', S_5_Emabcd return S_5_Emabcd # # #------------------------------------------------------------------------- # # Returns the fourth order secant stiffness tensor (double orthotropic N-T split) # #------------------------------------------------------------------------- # def _get_S_5_Emabcd(self, sctx, eps_app_eng, sigma_kk): # # E_N = self.E / (3.0 - 2.0 * (1.0 + self.nu)) # E_T = self.E / (1. + self.nu) # # I_vol_ijkl = self._get_I_vol_4() # I_dev_ijkl = self._get_I_dev_4() # delta = identity(3) # phi_mtx = self._get_phi_mtx(sctx, eps_app_eng, sigma_kk) # D_ij = delta - phi_mtx # d = (1. / 3.) * np.trace(D_ij) # D_bar_ij = self.zeta_G * (D_ij - d * delta) # # S_5_ijkl = (1 - d) * E_N * I_vol_ijkl + (1 - d) * E_T * I_dev_ijkl + (2 / 3.) * (E_T - E_N) * \ # (einsum('ij,kl -> ijkl', delta, D_bar_ij) + # einsum('ij,kl -> ijkl', D_bar_ij, delta)) + 0.5 * (2 * E_T - E_N) *\ # (0.5 * (einsum('ik,jl -> ijkl', delta, D_bar_ij) + einsum('il,jk -> ijkl', D_bar_ij, delta)) + # 0.5 * (einsum('il,jk -> ijkl', D_bar_ij, delta) + einsum('ik,jl -> ijkl', delta, D_bar_ij))) # # return S_5_ijkl #------------------------------------------------------------------------- # Evaluation - get the corrector and predictor #------------------------------------------------------------------------- def get_corr_pred(self, eps_Emab_n1, tn1, kappa, omega): I = self.update_variables(eps_Emab_n1, kappa, omega) #---------------------------------------------------------------------- # if the regularization using the crack-band concept is on calculate the # effective element length in the direction of principle strains #---------------------------------------------------------------------- # if self.regularization: # h = self.get_regularizing_length(sctx, eps_app_eng) # self.phi_fn.h = h # #------------------------------------------------------------------ # # Damage tensor (2th order): # #------------------------------------------------------------------ # phi_Emab = self._get_phi_Emab(kappa_Emn) # # #------------------------------------------------------------------ # # Damage tensor (4th order) using product- or sum-type symmetrization: # #------------------------------------------------------------------ # beta_Emabcd = self._get_beta_Emabcd(phi_Emab) # # #------------------------------------------------------------------ # # Damaged stiffness tensor calculated based on the damage tensor beta4: # #------------------------------------------------------------------ # D_Emabcd, = einsum( # 'Emijab, abef, Emabef -> Emijab', beta_Emabcd, self.D_abef, beta_Emabcd) D_Emabcd = self._get_S_1_Emabcd(eps_Emab_n1, kappa, omega) sig_Emab = einsum('Emabcd,Emcd -> Emab', D_Emabcd, eps_Emab_n1) return D_Emabcd, sig_Emab
class PipelineConfiguration(traits.HasTraits): # project settings project_dir = traits.Directory( exists=False, desc="data path to where the project is stored") # project metadata (for connectome file) project_metadata = traits.Dict( desc="project metadata to be stored in the connectome file") # DEPRECATED: this field is deprecated after version >1.0.2 generator = traits.Str() # parcellation scheme parcellation_scheme = traits.Enum("NativeFreesurfer", ["Lausanne2008", "NativeFreesurfer"], desc="used parcellation scheme") # choose between 'L' (linear) and 'N' (non-linear) and 'B' (bbregister) registration_mode = traits.Enum( "Linear", ["Linear", "Nonlinear", "BBregister"], desc="registration mode: linear or non-linear or bbregister") diffusion_imaging_model = traits.Enum("DSI", ["DSI", "DTI", "QBALL"]) # DSI nr_of_gradient_directions = traits.Str('515') nr_of_sampling_directions = traits.Str('181') odf_recon_param = traits.Str('-b0 1 -dsi -p 4 -sn 0') hardi_recon_param = traits.Str('-b0 1 -p 3 -sn 0') # DTI gradient_table_file = traits.File(exists=False) gradient_table = traits.Enum('siemens_64', [ 'custom', 'mgh_dti_006', 'mgh_dti_018', 'mgh_dti_030', 'mgh_dti_042', 'mgh_dti_060', 'mgh_dti_072', 'mgh_dti_090', 'mgh_dti_120', 'mgh_dti_144', 'siemens_06', 'siemens_12', 'siemens_20', 'siemens_256', 'siemens_30', 'siemens_64' ]) nr_of_b0 = traits.Str('1') max_b0_val = traits.Str('1000') dti_recon_param = traits.Str('') dtb_dtk2dir_param = traits.Str('') # tractography streamline_param = traits.Str('--angle 60 --seeds 32') # registration lin_reg_param = traits.Str('-usesqform -nosearch -dof 6 -cost mutualinfo') nlin_reg_bet_T2_param = traits.Str('-f 0.35 -g 0.15') nlin_reg_bet_b0_param = traits.Str('-f 0.2 -g 0.2') nlin_reg_fnirt_param = traits.Str( '--subsamp=8,4,2,2 --miter==5,5,5,5 --lambda=240,120,90,30 --splineorder=3 --applyinmask=0,0,1,1 --applyrefmask=0,0,1,1' ) bb_reg_param = traits.Str('--init-header --dti') # dicom converter do_convert_diffusion = traits.Bool(True) do_convert_T1 = traits.Bool(True) do_convert_T2 = traits.Bool(False) do_convert_fMRI = traits.Bool(False) # rsfmri # choose between 'L' (linear) and 'B' (bbregister) rsfmri_registration_mode = traits.Enum( "Linear", ["Linear", "BBregister"], desc="registration mode: linear or bbregister") rsfmri_lin_reg_param = traits.Str( '-usesqform -nosearch -dof 6 -cost mutualinfo') rsfmri_bb_reg_param = traits.Str('--init-header --dti') do_save_mat = traits.Bool(True) # rsfmri PREPROCESSING STEPS rsfmri_slice_timing = traits.Enum("none", [ "none", "bottom-top interleaved", "top-bottom interleaved", "bottom-top", "top-bottom" ], desc="time slicing mode") rsfmri_smoothing = traits.Str('0') rsfmri_discard = traits.Str('5') rsfmri_nuisance_global = traits.Bool(False) rsfmri_nuisance_WM = traits.Bool(True) rsfmri_nuisance_CSF = traits.Bool(True) rsfmri_nuisance_motion = traits.Bool(True) rsfmri_detrending = traits.Bool(True) rsfmri_lowpass = traits.Str('1') rsfmri_scrubbing_parameters = traits.Bool(True) rsfmri_scrubbing_apply = traits.Bool(True) rsfmri_scrubbing_FD = traits.Str('0.5') rsfmri_scrubbing_DVARS = traits.Str('5') # DEPRECATED: subject_raw_glob_diffusion = traits.Str("*.*") subject_raw_glob_T1 = traits.Str("*.*") subject_raw_glob_T2 = traits.Str("*.*") extract_diffusion_metadata = traits.Bool(False) # subject subject_name = traits.Str() subject_timepoint = traits.Str() subject_workingdir = traits.Directory() subject_logger = None subject_metadata = [ KeyValue(key='description', value=''), KeyValue(key='', value=''), KeyValue(key='', value=''), KeyValue(key='', value=''), KeyValue(key='', value=''), KeyValue(key='', value=''), ] active_createfolder = traits.Bool(True) active_dicomconverter = traits.Bool(False) active_registration = traits.Bool(False) active_segmentation = traits.Bool(False) active_parcellation = traits.Bool(False) active_applyregistration = traits.Bool(False) active_reconstruction = traits.Bool(False) active_tractography = traits.Bool(False) active_fiberfilter = traits.Bool(False) active_connectome = traits.Bool(False) active_statistics = traits.Bool(False) active_cffconverter = traits.Bool(False) active_rsfmri_registration = traits.Bool(False) active_rsfmri_preprocessing = traits.Bool(False) active_rsfmri_connectionmatrix = traits.Bool(False) skip_completed_stages = traits.Bool(False) # metadata creator = traits.Str() email = traits.Str() publisher = traits.Str() created = traits.Date() modified = traits.Date() license = traits.Str() # rights = traits.Str() reference = traits.Str() # relation = traits.Str() species = traits.Str('H**o sapiens') description = traits.Str() # segmentation recon_all_param = traits.Str('-all -no-isrunning') # parcellation custompar_nrroi = traits.Int() custompar_nodeinfo = traits.File() custompar_volumeparcell = traits.File() # fiber filtering apply_splinefilter = traits.Bool( True, desc='apply the spline filtering from diffusion toolkit') apply_fiberlength = traits.Bool(True, desc='apply cutoff to fiber lengths') fiber_cutoff_lower = traits.Float( 20.0, desc='cut fibers that are shorter in length than given length in mm') fiber_cutoff_upper = traits.Float( 500.0, desc='cut fibers that are longer in length than given length in mm') # measures connection_P0 = traits.Bool(False) connection_gfa = traits.Bool(False) connection_kurtosis = traits.Bool(False) connection_skewness = traits.Bool(False) connection_adc = traits.Bool(False) connection_fa = traits.Bool(False) # cff converter cff_fullnetworkpickle = traits.Bool( True, desc='stores the full network pickle generated by connectome creation') cff_cmatpickle = traits.Bool(True) cff_originalfibers = traits.Bool(True, desc='stores original fibers') cff_filteredfibers = traits.Bool(True, desc='stores filtered fibers') cff_finalfiberlabels = traits.Bool( True, desc='stores final fibers and their labelarrays') cff_fiberarr = traits.Bool(True) cff_rawdiffusion = traits.Bool(True) cff_scalars = traits.Bool(True) cff_rawT1 = traits.Bool(True) cff_rawT2 = traits.Bool(True) cff_roisegmentation = traits.Bool( True, desc='stores multi-resolution parcellation volumes') cff_surfaces = traits.Bool(True, desc='stores individually genertated surfaces') cff_surfacelabels = traits.Bool( True, desc='stores individually genertated surfaces') # do you want to do manual white matter mask correction? wm_handling = traits.Enum( 1, [1, 2, 3], desc="in what state should the freesurfer step be processed") # custom parcellation parcellation = traits.Dict( desc="provide the dictionary with your parcellation.") # start up fslview inspect_registration = traits.Bool( False, desc='start fslview to inspect the the registration results') fsloutputtype = traits.Enum('NIFTI', ['NIFTI']) # connectome creation compute_curvature = traits.Bool(False) # email notification, needs a local smtp server # sudo apt-get install postfix emailnotify = traits.ListStr( [], desc='the email address to send stage completion status message') freesurfer_home = traits.Directory(exists=False, desc="path to Freesurfer") fsl_home = traits.Directory(exists=False, desc="path to FSL") dtk_home = traits.Directory(exists=False, desc="path to diffusion toolkit") # This file stores descriptions of the inputs/outputs to each stage of the # CMP pipeline. It can be queried using the PipelineStatus python object pipeline_status_file = traits.Str("cmp.status") # Pipeline status object pipeline_status = pipeline_status.PipelineStatus() def _get_lausanne_parcellation(self, parcel="NativeFreesurfer"): if parcel == "Lausanne2008": return { 'scale33': { 'number_of_regions': 83, # contains name, url, color, freesurfer_label, etc. used for connection matrix 'node_information_graphml': op.join( self.get_lausanne_parcellation_path('resolution83'), 'resolution83.graphml'), # scalar node values on fsaverage? or atlas? 'surface_parcellation': None, # scalar node values in fsaverage volume? 'volume_parcellation': None, # the subdirectory name from where to copy parcellations, with hemispheric wildcard 'fs_label_subdir_name': 'regenerated_%s_36', # should we subtract the cortical rois for the white matter mask? 'subtract_from_wm_mask': 1, }, 'scale60': { 'number_of_regions': 129, 'node_information_graphml': op.join( self.get_lausanne_parcellation_path('resolution150'), 'resolution150.graphml'), 'surface_parcellation': None, 'volume_parcellation': None, 'fs_label_subdir_name': 'regenerated_%s_60', 'subtract_from_wm_mask': 1, }, 'scale125': { 'number_of_regions': 234, 'node_information_graphml': op.join( self.get_lausanne_parcellation_path('resolution258'), 'resolution258.graphml'), 'surface_parcellation': None, 'volume_parcellation': None, 'fs_label_subdir_name': 'regenerated_%s_125', 'subtract_from_wm_mask': 1, }, 'scale250': { 'number_of_regions': 463, 'node_information_graphml': op.join( self.get_lausanne_parcellation_path('resolution500'), 'resolution500.graphml'), 'surface_parcellation': None, 'volume_parcellation': None, 'fs_label_subdir_name': 'regenerated_%s_250', 'subtract_from_wm_mask': 1, }, 'scale500': { 'number_of_regions': 1015, 'node_information_graphml': op.join( self.get_lausanne_parcellation_path('resolution1015'), 'resolution1015.graphml'), 'surface_parcellation': None, 'volume_parcellation': None, 'fs_label_subdir_name': 'regenerated_%s_500', 'subtract_from_wm_mask': 1, }, } else: return { 'freesurferaparc': { 'number_of_regions': 83, # contains name, url, color, freesurfer_label, etc. used for connection matrix 'node_information_graphml': op.join( self.get_lausanne_parcellation_path('freesurferaparc'), 'resolution83.graphml'), # scalar node values on fsaverage? or atlas? 'surface_parcellation': None, # scalar node values in fsaverage volume? 'volume_parcellation': None, } } def __init__(self, **kwargs): # NOTE: In python 2.6, object.__init__ no longer accepts input # arguments. HasTraits does not define an __init__ and # therefore these args were being ignored. super(PipelineConfiguration, self).__init__(**kwargs) # the default parcellation provided self.parcellation = self._get_lausanne_parcellation( parcel="NativeFreesurfer") self.can_use_dipy = dipy_here # no email notify self.emailnotify = [] # default gradient table for DTI self.gradient_table_file = self.get_cmp_gradient_table('siemens_64') # try to discover paths from environment variables try: self.freesurfer_home = op.join(os.environ['FREESURFER_HOME']) self.fsl_home = op.join(os.environ['FSLDIR']) self.dtk_home = os.environ['DTDIR'] self.dtk_matrices = op.join(self.dtk_home, 'matrices') except KeyError: pass self.fsloutputtype = 'NIFTI' os.environ['FSLOUTPUTTYPE'] = self.fsloutputtype os.environ['FSLOUTPUTTYPE'] = 'NIFTI' def consistency_check(self): """ Provides a checking facility for configuration objects """ # project name not empty if not op.exists(self.project_dir): msg = 'Your project directory does not exist!' raise Exception(msg) # check metadata if self.creator == '': raise Exception('You need to enter creator metadata!') if self.publisher == '': raise Exception('You need to enter publisher metadata!') if self.email == '': raise Exception('You need to enter email of a contact person!') # check if software paths exists pas = { 'configuration.freesurfer_home': self.freesurfer_home, 'configuration.fsl_home': self.fsl_home, 'configuration.dtk_home': self.dtk_home, 'configuration.dtk_matrices': self.dtk_matrices } for k, p in pas.items(): if not op.exists(p): msg = 'Required software path for %s does not exists: %s' % (k, p) raise Exception(msg) if self.subject_workingdir == '': msg = 'No working directory defined for subject' raise Exception(msg) # else: # wdir = self.get_subj_dir() # if not op.exists(wdir): # msg = 'Working directory %s does not exists for subject' % (wdir) # raise Exception(msg) # else: # wdiff = op.join(self.get_raw_diffusion()) # print wdiff # if not op.exists(wdiff): # msg = 'Diffusion MRI subdirectory %s does not exists for the subject' % wdiff # raise Exception(msg) # wt1 = op.join(self.get_rawt1()) # if not op.exists(wt1): # msg = 'Structural MRI subdirectory %s T1 does not exist in RAWDATA' % wt1 # raise Exception(msg) def get_cmp_home(self): """ Return the cmp home path """ return op.dirname(__file__) def get_rawdata(self): """ Return raw data path for the subject """ return op.join(self.get_subj_dir(), 'RAWDATA') def get_log(self): """ Get subject log dir """ return op.join(self.get_subj_dir(), 'LOG') def get_logname(self, suffix='.log'): """ Get a generic name for the log and pickle files """ a = dt.datetime.now() return 'pipeline-%s-%02i%02i-%s-%s%s' % ( a.date().isoformat(), a.time().hour, a.time().minute, self.subject_name, self.subject_timepoint, suffix) def get_logger(self): """ Get the logger instance created """ if self.subject_logger is None: # setup logger for the subject self.subject_logger = \ getLog(os.path.join(self.get_log(), self.get_logname())) return self.subject_logger else: return self.subject_logger def get_rawglob(self, modality): """ DEPRECATED: Get the file name endings for modality """ if modality == 'diffusion': if not self.subject_raw_glob_diffusion == '': return self.subject_raw_glob_diffusion else: raise Exception('No raw_glob_diffusion defined for subject') elif modality == 'T1': if not self.subject_raw_glob_T1 == '': return self.subject_raw_glob_T1 else: raise Exception('No raw_glob_T1 defined for subject') elif modality == 'T2': if not self.subject_raw_glob_T2 == '': return self.subject_raw_glob_T2 else: raise Exception('No raw_glob_T2 defined for subject') def get_dicomfiles(self, modality): """ Get a list of dicom files for the requested modality. Tries to discover them automatically """ from glob import glob if modality == 'diffusion': pat = self.get_raw_diffusion() elif modality == 'T1': pat = self.get_rawt1() elif modality == 'T2': pat = self.get_rawt2() elif modality == 'fMRI': pat = self.get_rawrsfmri() # discover files with *.* and * difiles = sorted(glob(op.join(pat, '*.*')) + glob(op.join(pat, '*'))) # exclude potential .nii and .nii.gz files difiles = [ e for e in difiles if not e.endswith('.nii') and not e.endswith('.nii.gz') ] # check if no files and throw exception if len(difiles) == 0: raise Exception('Could not find any DICOM files in folder %s' % pat) return difiles def get_rawrsfmri(self): """ Get raw functional MRI path for subject """ return op.join(self.get_rawdata(), 'fMRI') def get_rawt1(self): """ Get raw structural MRI T1 path for subject """ return op.join(self.get_rawdata(), 'T1') def get_rawt2(self): """ Get raw structural MRI T2 path for subject """ return op.join(self.get_rawdata(), 'T2') def get_subj_dir(self): return self.subject_workingdir def get_raw_diffusion(self): """ Get the raw diffusion path for subject """ if self.diffusion_imaging_model == 'DSI': return op.join(self.get_subj_dir(), 'RAWDATA', 'DSI') elif self.diffusion_imaging_model == 'DTI': return op.join(self.get_subj_dir(), 'RAWDATA', 'DTI') elif self.diffusion_imaging_model == 'QBALL': return op.join(self.get_subj_dir(), 'RAWDATA', 'QBALL') def get_fs(self): """ Returns the subject root folder path for freesurfer files """ return op.join(self.get_subj_dir(), 'FREESURFER') def get_stats(self): """ Return statistic output path """ return op.join(self.get_subj_dir(), 'STATS') def get_cffdir(self): """ Returns path to store connectome file """ return op.join(self.get_cmp(), 'cff') def get_nifti(self): """ Returns the subject root folder path for nifti files """ return op.join(self.get_subj_dir(), 'NIFTI') def get_nifti_trafo(self): """ Returns the path to the subjects transformation / registration matrices """ return op.join(self.get_nifti(), 'transformations') def get_nifti_bbregister(self): """ Returns the path to the subjects transformation / registration matrices, bbregister mode """ return op.join(self.get_nifti(), 'bbregister') def get_diffusion_metadata(self): """ Diffusion metadata, i.e. where gradient_table.txt is stored """ return op.join(self.get_nifti(), 'diffusion_metadata') def get_nifti_wm_correction(self): """ Returns the path to the subjects wm_correction path """ return op.join(self.get_nifti(), 'wm_correction') def get_cmp(self): return op.join(self.get_subj_dir(), 'CMP') def get_cmp_rawdiff(self, ): return op.join(self.get_cmp(), 'raw_diffusion') def get_cmp_rawdiff_reconout(self): """ Returns the output path for diffusion reconstruction without prefix""" if self.diffusion_imaging_model == 'DSI': return op.join(self.get_cmp(), 'raw_diffusion', 'odf_0') elif self.diffusion_imaging_model == 'DTI': return op.join(self.get_cmp(), 'raw_diffusion', 'dti_0') elif self.diffusion_imaging_model == 'QBALL': return op.join(self.get_cmp(), 'raw_diffusion', 'qball_0') def get_cmp_rawdiff_resampled(self): return op.join(self.get_cmp_rawdiff(), '2x2x2') def get_cmp_fsout(self): return op.join(self.get_cmp(), 'fs_output') def get_cmp_fibers(self): return op.join(self.get_cmp(), 'fibers') def get_cmp_scalars(self): return op.join(self.get_cmp(), 'scalars') def get_cmp_matrices(self): return op.join(self.get_cmp_fibers(), 'matrices') def get_cmp_fmri(self): return op.join(self.get_cmp(), 'fMRI') def get_cmp_fmri_preproc(self): return op.join(self.get_cmp_fmri(), 'preprocessing') def get_cmp_fmri_matrices(self): return op.join(self.get_cmp_fmri(), 'matrices') def get_cmp_fmri_timeseries(self): return op.join(self.get_cmp_fmri(), 'timeseries') def get_cmp_tracto_mask(self): return op.join(self.get_cmp_fsout(), 'HR') def get_cmp_tracto_mask_tob0(self): return op.join(self.get_cmp_fsout(), 'HR__registered-TO-b0') def get_custom_gradient_table(self): """ Returns the absolute path to the custom gradient table with optional b-values in the 4th row """ return self.gradient_table_file def get_cmp_gradient_table(self, name): """ Return default gradient tables shipped with CMP. These are mainly derived from Diffusion Toolkit """ cmp_path = op.dirname(__file__) return op.join(cmp_path, 'data', 'diffusion', 'gradient_tables', name + '.txt') def get_dtb_streamline_vecs_file(self, as_text=False): """ Returns the odf directions file used for DTB_streamline """ cmp_path = op.dirname(__file__) if as_text: return op.join(cmp_path, 'data', 'diffusion', 'odf_directions', '181_vecs.txt') else: return op.join(cmp_path, 'data', 'diffusion', 'odf_directions', '181_vecs.dat') # XXX def get_cmp_scalarfields(self): """ Returns a list with tuples with the scalar field name and the absolute path to its nifti file """ ret = [] if self.diffusion_imaging_model == 'DSI': # add gfa per default ret.append(('gfa', op.join(self.get_cmp_scalars(), 'dsi_gfa.nii.gz'))) # XXX: add adc per default elif self.diffusion_imaging_model == 'DTI': # nothing to add yet for DTI pass return ret def get_dtk_dsi_matrix(self): """ Returns the DSI matrix from Diffusion Toolkit The parameters have to be set in the configuration object with keys: 1. number of gradient directions : 'nr_of_gradient_directions' 2. number of sampling directions : 'nr_of_sampling_directions' Example ------- confobj.nr_of_gradient_directions = 515 confobj.nr_of_sampling_directions = 181 Returns matrix including absolute path to DSI_matrix_515x181.dat """ grad = self.nr_of_gradient_directions samp = self.nr_of_sampling_directions fpath = op.join(self.dtk_matrices, "DSI_matrix_%sx%s.dat" % (grad, samp)) if not op.exists(fpath): msg = "DSI matrix does not exists: %s" % fpath raise Exception(msg) return fpath def get_lausanne_atlas(self, name=None): """ Return the absolute path to the lausanne parcellation atlas for the resolution name """ cmp_path = op.dirname(__file__) provided_atlases = [ 'myatlas_36_rh.gcs', 'myatlasP1_16_rh.gcs', 'myatlasP17_28_rh.gcs', 'myatlasP29_36_rh.gcs', 'myatlas_60_rh.gcs', 'myatlas_125_rh.gcs', 'myatlas_250_rh.gcs', 'myatlas_36_lh.gcs', 'myatlasP1_16_lh.gcs', 'myatlasP17_28_lh.gcs', 'myatlasP29_36_lh.gcs', 'myatlas_60_lh.gcs', 'myatlas_125_lh.gcs', 'myatlas_250_lh.gcs' ] if name in provided_atlases: return op.join(cmp_path, 'data', 'colortable_and_gcs', 'my_atlas_gcs', name) else: msg = "Atlas %s does not exists" % name raise Exception(msg) def get_freeview_lut(self, name): """ Returns the Look-Up-Table as text file for a given parcellation scheme in a dictionary """ cmp_path = op.dirname(__file__) if name == "NativeFreesurfer": return { 'freesurferaparc': op.join(cmp_path, 'data', 'parcellation', 'nativefreesurfer', 'freesurferaparc', 'FreeSurferColorLUT_adapted.txt') } else: return "" def get_lausanne_parcellation_path(self, parcellationname): cmp_path = op.dirname(__file__) if self.parcellation_scheme == "Lausanne2008": allowed_default_parcel = [ 'resolution83', 'resolution150', 'resolution258', 'resolution500', 'resolution1015' ] if parcellationname in allowed_default_parcel: return op.join(cmp_path, 'data', 'parcellation', 'lausanne2008', parcellationname) else: msg = "Not a valid default parcellation name for the lausanne2008 parcellation scheme" raise Exception(msg) else: allowed_default_parcel = ['freesurferaparc'] if parcellationname in allowed_default_parcel: return op.join(cmp_path, 'data', 'parcellation', 'nativefreesurfer', parcellationname) else: msg = "Not a valid default parcellation name for the NativeFreesurfer parcellation scheme" raise Exception(msg) def get_cmp_binary_path(self): """ Returns the path to the binary files for the current platform and architecture """ if sys.platform == 'linux2': import platform as pf if '32' in pf.architecture()[0]: return op.join(op.dirname(__file__), "binary", "linux2", "bit32") elif '64' in pf.architecture()[0]: return op.join(op.dirname(__file__), "binary", "linux2", "bit64") else: raise ('No binary files compiled for your platform!') def get_pipeline_status_file(self): """Returns the absolute path of the pipeline status file""" return op.join(self.get_subj_dir(), self.pipeline_status_file) def init_pipeline_status(self): """Create the 'cmp.status'. The 'cmp.status' file contains information about the inputs/outputs of each pipeline stage""" status_file = op.join(self.get_subj_dir(), self.pipeline_status_file) self.pipeline_status.Pipeline.name = "cmp" self.pipeline_status.SaveToFile(status_file) def update_pipeline_status(self): """Update the pipeline status on disk with the current status in memory""" status_file = op.join(self.get_subj_dir(), self.pipeline_status_file) self.pipeline_status.SaveToFile(status_file)
class config(HasTraits): uuid = traits.Str(desc="UUID") # Directories working_dir = Directory(mandatory=True, desc="Location of the Nipype working directory") base_dir = Directory(os.path.abspath('.'),exists=True, desc='Base directory of data. (Should be subject-independent)') sink_dir = Directory(mandatory=True, desc="Location where the BIP will store the results") crash_dir = Directory(mandatory=False, desc="Location to store crash files") surf_dir = Directory(os.path.abspath('.'),mandatory=True, desc="Freesurfer subjects directory") # Execution run_using_plugin = Bool(False, usedefault=True, desc="True to run pipeline with plugin, False to run serially") plugin = traits.Enum("PBS", "MultiProc", "SGE", "Condor", usedefault=True, desc="plugin to use, if run_using_plugin=True") plugin_args = traits.Dict({"qsub_args": "-q many"}, usedefault=True, desc='Plugin arguments.') test_mode = Bool(False, mandatory=False, usedefault=True, desc='Affects whether where and if the workflow keeps its \ intermediary files. True to keep intermediary files. ') # Subjects datagrabber = traits.Instance(Data, ()) subjects = traits.List(traits.Str, mandatory=True, usedefault=True, desc="Subject id's. Note: These MUST match the subject id's in the \ Freesurfer directory. For simplicity, the subject id's should \ also match with the location of individual functional files.") func_template = traits.String('%s/cleaned_resting.nii.gz') reg_template = traits.String('%s/cleaned_resting_reg.dat') ref_template = traits.String('%s/cleaned_resting_ref.nii.gz') combine_surfaces = traits.Bool() # Target surface target_surf = traits.Enum('fsaverage4', 'fsaverage3', 'fsaverage5', 'fsaverage6', 'fsaverage', 'subject', desc='which average surface to map to') surface_fwhm = traits.List([5], traits.Float(), mandatory=True, usedefault=True, desc="How much to smooth on target surface") projection_stem = traits.Str('-projfrac-avg 0 1 0.1', desc='how to project data onto the surface') combine_surfaces = traits.Bool(desc=('compute correlation matrix across' 'both left and right surfaces')) # Saving output out_type = traits.Enum('mat', 'hdf5', desc='mat or hdf5') hdf5_package = traits.Enum('h5py', 'pytables', desc='which hdf5 package to use') # Advanced Options use_advanced_options = traits.Bool() advanced_script = traits.Code() save_script_only = traits.Bool(False) # Atlas mapping surface_atlas = traits.Str('None', desc='Name of parcellation atlas') # Buttons check_func_datagrabber = Button("Check") def _check_func_datagrabber_fired(self): subs = self.subjects for s in subs: for template in [self.func_template, self.ref_template, self.reg_template]: check_path(os.path.join(self.base_dir, template % s)) check_path(os.path.join(self.surf_dir, s))
class Factor(_traits.HasTraits): '''Represent different factors as independent variables FIXME: is the order of levels important, the uniqueness? looks more like a mapping type than a sequences type ''' name = _traits.Unicode() levels = _traits.Dict(_traits.Unicode, Level) # Can be used to check if this can be used for a given dataset size = _traits.Property() _size = _traits.Int() default_ds_axis = _traits.Enum(("row", "col")) def __init__(self, name, size, *levels, **kwargs): '''Create a factor object name: The factor name levels: none, one or several level objects check_idx: check index strategy; no, toss_overlaping ''' check_idx = kwargs.pop('check_idx', "no") super(Factor, self).__init__(name=name, size=size, **kwargs) for level in levels: self.add_level(level, check_idx) def add_level(self, level, check_idx="no"): '''Add level level: a level object check_idx: "no", "toss_overlaping", "excpet_overlapping", "return_overlapping" ''' self.name_check(level.name) self.bound_check(level.selector) if check_idx == "no": self.levels[level.name] = level elif check_idx == "toss_overlaping": self.toss_overlaping(level) self.levels[level.name] = level else: msg = "Not valid value for check_idx: {0}".format(check_idx) raise ValueError(msg) def name_check(self, new_name): if new_name in self.levels: msg = "Level name collision. Name: {0} already exist in factor: {1}".format( new_name, self.name) raise ValueError(msg) def bound_check(self, selector): if self.size <= max(selector): msg = "Index in level out of bounds" raise IndexError(msg) def toss_overlaping(self, level): ''' FIXME: This will mainpulate an existing object. Is that unproblematic? ''' existing_idx = [] for lv in self.levels.itervalues(): existing_idx.extend(lv.selector) exist = set(existing_idx) unique = set(level.selector).difference(exist) level.selector = list(unique) def get_values(self, dataset, name, axis=None): '''Return numpy subarray ''' idx = self.levels[name].selector if axis is not None: if axis == 0: return dataset.mat.values[idx,:] elif axis == 1: return dataset.mat.values[:,idx] raise ValueError('Illegale value for axis') else: if self.default_ds_axis == "row": return dataset.mat.values[idx,:] else: return dataset.mat.values[:,idx] def get_labels(self, dataset, name, axis=None): '''Return list of selected labels ''' idx = self.levels[name].selector if axis is not None: if axis == 0: return dataset.mat.index[idx] elif axis == 1: return dataset.mat.columns[idx] raise ValueError('Illegale value for axis') else: if self.default_ds_axis == "row": return dataset.mat.index[idx] else: return dataset.mat.columns[idx] def _get_all_leveled(self): '''The index combined from all the levels ''' return list(_itr.chain.from_iterable([lv.selector for lv in self.levels.itervalues()])) def get_combined_levels_subset(self, dataset, axis=None): '''Return numpy subarray ''' sub_ds = dataset.copy() idx = self._get_all_leveled() if axis is not None: if axis == 0: sub_ds.mat = dataset.mat.iloc[idx,:] return sub_ds elif axis == 1: sub_ds.mat = dataset.mat.iloc[:,idx] return sub_ds raise ValueError('Illegale value for axis') else: if self.default_ds_axis == "row": sub_ds.mat = dataset.mat[idx,:] return sub_ds else: sub_ds.mat = dataset.mat[:,idx] return sub_ds def get_combined_levels_values(self, dataset, axis=None): '''Return numpy subarray ''' idx = self._get_all_leveled() if axis is not None: if axis == 0: return dataset.mat.values[idx,:] elif axis == 1: return dataset.mat.values[:,idx] raise ValueError('Illegale value for axis') else: if self.default_ds_axis == "row": return dataset.mat.values[idx,:] else: return dataset.mat.values[:,idx] def get_combined_levels_labels(self, dataset, axis=None): '''Return list of all selected labels ''' idx = self._get_all_leveled() if axis is not None: if axis == 0: return dataset.mat.index[idx] elif axis == 1: return dataset.mat.columns[idx] raise ValueError('Illegale value for axis') else: if self.default_ds_axis == "row": return dataset.mat.index[idx] else: return dataset.mat.columns[idx] def _get_nonleveled(self, dataset, axis): '''The indexes for a dataset that is not in a level ''' lvs = self._get_all_leveled() return list(set(range(dataset.mat.shape[axis])).difference(set(lvs))) def get_rest_values(self, dataset, axis=None): '''Return numpy subarray ''' if axis is not None: if axis == 0: idx = self._get_nonleveled(dataset, 0) return dataset.mat.values[idx,:] elif axis == 1: idx = self._get_nonleveled(dataset, 1) return dataset.mat.values[:,idx] raise ValueError('Illegale value for axis') else: if self.default_ds_axis == "row": idx = self._get_nonleveled(dataset, 0) return dataset.mat.values[idx,:] else: idx = self._get_nonleveled(dataset, 1) return dataset.mat.values[:,idx] def get_rest_labels(self, dataset, axis=None): '''Return list of not selected labels ''' if axis is not None: if axis == 0: idx = self._get_nonleveled(dataset, 0) return dataset.mat.index[idx] elif axis == 1: idx = self._get_nonleveled(dataset, 1) return dataset.mat.columns[idx] raise ValueError('Illegale value for axis') else: if self.default_ds_axis == "row": idx = self._get_nonleveled(dataset, 0) return dataset.mat.index[idx] else: idx = self._get_nonleveled(dataset, 1) return dataset.mat.columns[idx] def __len__(self): return len(self.levels) # def __getitem__(self, key): # pass # def __setitem__(self, key, value): # pass # def __iter__(self): # pass def _set_size(self, sz): self._size = sz def _get_size(self): return self._size
class config(HasTraits): uuid = traits.Str(desc="UUID") # Directories working_dir = Directory(mandatory=True, desc="Location of the Nipype working directory") base_dir = Directory( os.path.abspath('.'), mandatory=True, desc='Base directory of data. (Should be subject-independent)') sink_dir = Directory(mandatory=True, desc="Location where the BIP will store the results") crash_dir = Directory(mandatory=False, desc="Location to store crash files") save_script_only = traits.Bool(False) # Execution run_using_plugin = Bool( False, usedefault=True, desc="True to run pipeline with plugin, False to run serially") plugin = traits.Enum("PBS", "MultiProc", "SGE", "Condor", usedefault=True, desc="plugin to use, if run_using_plugin=True") plugin_args = traits.Dict({"qsub_args": "-q many"}, usedefault=True, desc='Plugin arguments.') test_mode = Bool( False, mandatory=False, usedefault=True, desc='Affects whether where and if the workflow keeps its \ intermediary files. True to keep intermediary files. ' ) timeout = traits.Float(14.0) # Subjects #subjects = traits.List(traits.Str, mandatory=True, usedefault=True, # desc="Subject id's. Note: These MUST match the subject id's in the \ # Freesurfer directory. For simplicity, the subject id's should \ # also match with the location of individual functional files.") #fwhm=traits.List(traits.Float()) #copes_template = traits.String('%s/preproc/output/fwhm_%s/cope*.nii.gz') #varcopes_template = traits.String('%s/preproc/output/fwhm_%s/varcope*.nii.gz') #contrasts = traits.List(traits.Str,desc="contrasts") datagrabber = traits.Instance(Data, ()) # Regression design_csv = traits.File(desc="design .csv file") reg_contrasts = traits.Code( desc= "function named reg_contrasts which takes in 0 args and returns contrasts" ) run_mode = traits.Enum("flame1", "ols", "flame12") #Normalization norm_template = traits.File(desc='Template of files') use_mask = traits.Bool(False) mask_file = traits.File() #Correction: run_correction = traits.Bool(False) p_threshold = traits.Float(0.05) z_threshold = traits.Float(2.3) connectivity = traits.Int(26) do_randomize = traits.Bool(False) num_iterations = traits.Int(5000) # Advanced Options use_advanced_options = traits.Bool() advanced_script = traits.Code() # Buttons check_func_datagrabber = Button("Check")
class MATS3DMplDamageEEQ(MATS3DEval): # To use the model directly in the simulator specify the # time stepping classes epsilon_0 = Float(59.0e-6, label="a", desc="Lateral pressure coefficient", enter_set=True, auto_set=False) epsilon_f = Float(250.0e-6, label="a", desc="Lateral pressure coefficient", enter_set=True, auto_set=False) c_T = Float(0.01, label="a", desc="Lateral pressure coefficient", enter_set=True, auto_set=False) #========================================================================= # Configurational parameters #========================================================================= state_var_shapes = tr.Property(tr.Dict(), depends_on='n_mp') r''' Shapes of the state variables to be stored in the global array at the level of the domain. ''' @cached_property def _get_state_var_shapes(self): return {'kappa_n': (self.n_mp, ), 'omega_n': (self.n_mp, )} U_var_shape = (6, ) '''Shape of the primary variable required by the TStepState. ''' node_name = 'Desmorat model' tree_node_list = tr.List([]) #========================================================================= # Evaluation - get the corrector and predictor #========================================================================= def get_corr_pred(self, eps_ab, tn1, kappa_n, omega_n): self._update_state_variables(eps_ab, kappa_n, omega_n) #---------------------------------------------------------------------- # if the regularization using the crack-band concept is on calculate the # effective element length in the direction of principle strains #---------------------------------------------------------------------- # if self.regularization: # h = self.get_regularizing_length(sctx, eps_app_eng) # self.phi_fn.h = h #------------------------------------------------------------------ # Damage tensor (2th order): #------------------------------------------------------------------ phi_ab = self._get_phi_ab(kappa_n) #------------------------------------------------------------------ # Damage tensor (4th order) using product- or sum-type symmetrization: #------------------------------------------------------------------ beta_abcd = self._get_beta_abcd(phi_ab) #------------------------------------------------------------------ # Damaged stiffness tensor calculated based on the damage tensor beta4: #------------------------------------------------------------------ D_ijab = einsum('...ijab, abef, ...cdef -> ...ijcd', beta_abcd, self.D_abef, beta_abcd) sig_ab = einsum('...abef,...ef -> ...ab', D_ijab, eps_ab) return sig_ab, D_ijab #========================================================================= # MICROPLANE-Kinematic constraints #========================================================================= _MPNN = Property(depends_on='n_mp') r'''Get the dyadic product of the microplane normals ''' @cached_property def _get__MPNN(self): # dyadic product of the microplane normals MPNN_nij = einsum('ni,nj->nij', self._MPN, self._MPN) return MPNN_nij _MPTT = Property(depends_on='n_mp') r'''Get the third order tangential tensor (operator) for each microplane ''' @cached_property def _get__MPTT(self): # Third order tangential tensor for each microplane delta = identity(3) MPTT_nijr = 0.5 * ( einsum('ni,jr -> nijr', self._MPN, delta) + einsum('nj,ir -> njir', self._MPN, delta) - 2.0 * einsum('ni,nj,nr -> nijr', self._MPN, self._MPN, self._MPN)) return MPTT_nijr def _get_e_na(self, eps_ab): r''' Projection of apparent strain onto the individual microplanes ''' e_ni = einsum('nb,...ba->...na', self._MPN, eps_ab) return e_ni def _get_e_N_n(self, e_na): r''' Get the normal strain array for each microplane ''' e_N_n = einsum('...na, na->...n', e_na, self._MPN) return e_N_n def _get_e_equiv_n(self, e_na): r''' Returns a list of the microplane equivalent strains based on the list of microplane strain vectors ''' # magnitude of the normal strain vector for each microplane e_N_n = self._get_e_N_n(e_na) # positive part of the normal strain magnitude for each microplane e_N_pos_n = (np.abs(e_N_n) + e_N_n) / 2.0 # normal strain vector for each microplane e_N_na = einsum('...n,ni -> ...ni', e_N_n, self._MPN) # tangent strain ratio c_T = self.c_T # tangential strain vector for each microplane e_T_na = e_na - e_N_na # squared tangential strain vector for each microplane e_TT_n = einsum('...ni,...ni -> ...n', e_T_na, e_T_na) # equivalent strain for each microplane e_equiv_n = sqrt(e_N_pos_n * e_N_pos_n + c_T * e_TT_n) return e_equiv_n def _update_state_variables(self, eps_ab, kappa_n, omega_n): e_na = self._get_e_na(eps_ab) eps_eq_n = self._get_e_equiv_n(e_na) f_trial_n = eps_eq_n - self.epsilon_0 I = np.where(f_trial_n > 0) k_n = np.max(np.array([kappa_n[I], eps_eq_n[I]]), axis=0) kappa_n[I] = k_n omega_n[I] = self._get_omega(k_n) def _get_omega(self, kappa_n): ''' Return new value of damage parameter @param kappa: ''' omega_n = np.zeros_like(kappa_n) epsilon_0 = self.epsilon_0 epsilon_f = self.epsilon_f I = np.where(kappa_n >= epsilon_0) omega_n[I] = ( 1.0 - (epsilon_0 / kappa_n[I] * np.exp(-1.0 * (kappa_n[I] - epsilon_0) / (epsilon_f - epsilon_0)))) return omega_n def _get_phi_ab(self, kappa_n): # Returns the 2nd order damage tensor 'phi_mtx' # scalar integrity factor for each microplane phi_n = np.sqrt(1.0 - self._get_omega(kappa_n)) # print 'phi_Emn', phi_Emn[:, -1, :] # integration terms for each microplanes phi_ab = einsum('...n,n,nab->...ab', phi_n, self._MPW, self._MPNN) return phi_ab def _get_beta_abcd(self, phi_ab): ''' Returns the 4th order damage tensor 'beta4' using sum-type symmetrization (cf. [Jir99], Eq.(21)) ''' delta = identity(3) beta_ijkl = 0.25 * (einsum('...ik,jl->...ijkl', phi_ab, delta) + einsum('...il,jk->...ijkl', phi_ab, delta) + einsum('...jk,il->...ijkl', phi_ab, delta) + einsum('...jl,ik->...ijkl', phi_ab, delta)) return beta_ijkl #----------------------------------------------- # number of microplanes - currently fixed for 3D #----------------------------------------------- n_mp = Constant(28) #----------------------------------------------- # get the normal vectors of the microplanes #----------------------------------------------- _MPN = Property(depends_on='n_mp') @cached_property def _get__MPN(self): return array([[.577350259, .577350259, .577350259], [.577350259, .577350259, -.577350259], [.577350259, -.577350259, .577350259], [.577350259, -.577350259, -.577350259], [.935113132, .250562787, .250562787], [.935113132, .250562787, -.250562787], [.935113132, -.250562787, .250562787], [.935113132, -.250562787, -.250562787], [.250562787, .935113132, .250562787], [.250562787, .935113132, -.250562787], [.250562787, -.935113132, .250562787], [.250562787, -.935113132, -.250562787], [.250562787, .250562787, .935113132], [.250562787, .250562787, -.935113132], [.250562787, -.250562787, .935113132], [.250562787, -.250562787, -.935113132], [.186156720, .694746614, .694746614], [.186156720, .694746614, -.694746614], [.186156720, -.694746614, .694746614], [.186156720, -.694746614, -.694746614], [.694746614, .186156720, .694746614], [.694746614, .186156720, -.694746614], [.694746614, -.186156720, .694746614], [.694746614, -.186156720, -.694746614], [.694746614, .694746614, .186156720], [.694746614, .694746614, -.186156720], [.694746614, -.694746614, .186156720], [.694746614, -.694746614, -.186156720]]) #------------------------------------- # get the weights of the microplanes #------------------------------------- _MPW = Property(depends_on='n_mp') @cached_property def _get__MPW(self): return array([ .0160714276, .0160714276, .0160714276, .0160714276, .0204744730, .0204744730, .0204744730, .0204744730, .0204744730, .0204744730, .0204744730, .0204744730, .0204744730, .0204744730, .0204744730, .0204744730, .0158350505, .0158350505, .0158350505, .0158350505, .0158350505, .0158350505, .0158350505, .0158350505, .0158350505, .0158350505, .0158350505, .0158350505 ]) * 6.0 def _get_lame_params(self): la = self.E * self.nu / ((1. + self.nu) * (1. - 2. * self.nu)) # second Lame parameter (shear modulus) mu = self.E / (2. + 2. * self.nu) return la, mu D_abef = tr.Property(tr.Array, depends_on='+input') @tr.cached_property def _get_D_abef(self): la = self._get_lame_params()[0] mu = self._get_lame_params()[1] delta = identity(3) D_abef = (einsum(',ij,kl->ijkl', la, delta, delta) + einsum(',ik,jl->ijkl', mu, delta, delta) + einsum(',il,jk->ijkl', mu, delta, delta)) return D_abef #----------------------------------------------------------- # Response variables #----------------------------------------------------------- def get_phi_ab(self, eps_ab, tn1, kappa_n, omega_n): return self._get_phi_ab(kappa_n) def get_omega_ab(self, eps_ab, tn1, kappa_n, omega_n): return np.identity(3) - self._get_phi_ab(kappa_n) def get_max_omega(self, eps_Emab, t_n1, kappa_n, omega_n): return np.max(omega_n, axis=-1) def _get_var_dict(self): var_dict = super(MATS3DMplDamageEEQ, self)._get_var_dict() var_dict.update(max_omega=self.get_max_omega, phi_ab=self.get_phi_ab, omega_ab=self.get_omega_ab) return var_dict
class config(HasTraits): uuid = traits.Str(desc="UUID") desc = traits.Str(desc='Workflow description') # Directories working_dir = Directory(mandatory=True, desc="Location of the Nipype working directory") base_dir = Directory(os.path.abspath('.'),mandatory=True, desc='Base directory of data. (Should be subject-independent)') sink_dir = Directory(os.path.abspath('.'),mandatory=True, desc="Location where the BIP will store the results") field_dir = Directory(desc="Base directory of field-map data (Should be subject-independent) \ Set this value to None if you don't want fieldmap distortion correction") crash_dir = Directory(mandatory=False, desc="Location to store crash files") surf_dir = Directory(mandatory=True, desc= "Freesurfer subjects directory") # Execution run_using_plugin = Bool(False, usedefault=True, desc="True to run pipeline with plugin, False to run serially") plugin = traits.Enum("PBS", "PBSGraph","MultiProc", "SGE", "Condor", usedefault=True, desc="plugin to use, if run_using_plugin=True") plugin_args = traits.Dict({"qsub_args": "-q many"}, usedefault=True, desc='Plugin arguments.') test_mode = Bool(False, mandatory=False, usedefault=True, desc='Affects whether where and if the workflow keeps its \ intermediary files. True to keep intermediary files. ') # Subjects subjects= traits.List(traits.Str, mandatory=True, usedefault=True, desc="Subject id's. Note: These MUST match the subject id's in the \ Freesurfer directory. For simplicity, the subject id's should \ also match with the location of individual functional files.") dwi_template = traits.String('%s/functional.nii.gz') bval_template = traits.String('%s/bval') bvec_template = traits.String('%s/fbvec') run_datagrabber_without_submitting = traits.Bool(desc="Run the datagrabber without \ submitting to the cluster") timepoints_to_remove = traits.Int(0,usedefault=True) # Fieldmap use_fieldmap = Bool(False, mandatory=False, usedefault=True, desc='True to include fieldmap distortion correction. Note: field_dir \ must be specified') magnitude_template = traits.String('%s/magnitude.nii.gz') phase_template = traits.String('%s/phase.nii.gz') TE_diff = traits.Float(desc='difference in B0 field map TEs') sigma = traits.Int(2, desc='2D spatial gaussing smoothing stdev (default = 2mm)') echospacing = traits.Float(desc="EPI echo spacing") # Bvecs do_rotate_bvecs = traits.Bool(True, usedefault=True) # Advanced Options use_advanced_options = traits.Bool() advanced_script = traits.Code() # Buttons check_func_datagrabber = Button("Check") check_field_datagrabber = Button("Check") def _check_func_datagrabber_fired(self): subs = self.subjects for s in subs: if not os.path.exists(os.path.join(self.base_dir,self.dwi_template % s)): print "ERROR", os.path.join(self.base_dir,self.dwi_template % s), "does NOT exist!" break else: print os.path.join(self.base_dir,self.dwi_template % s), "exists!" def _check_field_datagrabber_fired(self): subs = self.subjects for s in subs: if not os.path.exists(os.path.join(self.field_dir,self.magnitude_template % s)): print "ERROR:", os.path.join(self.field_dir,self.magnitude_template % s), "does NOT exist!" break else: print os.path.join(self.base_dir,self.magnitude_template % s), "exists!" if not os.path.exists(os.path.join(self.field_dir,self.phase_template % s)): print "ERROR:", os.path.join(self.field_dir,self.phase_template % s), "does NOT exist!" break else: print os.path.join(self.base_dir,self.phase_template % s), "exists!"
class Data(HasTraits): fields = traits.List(traits.Instance(DataBase, ())) base_directory = Directory(os.path.abspath('.')) template = traits.Str('*') template_args = traits.Dict({"a": "b"}, usedefault=True) field_template = traits.Dict({"key": ["hi"]}, usedefault=True) sort = traits.Bool(True) if use_view: check = traits.Button("Check") view = get_view() def __init__(self, outfields=None): if outfields: d_ft = {} d_ta = {} for out in outfields: d_ft[out] = '%s' d_ta[out] = [['name']] self.field_template = d_ft self.template_args = d_ta self.outfields = outfields def _get_infields(self): infields = [] for f in self.fields: infields.append(f.name) return infields def _add_iterable(self, field): import nipype.interfaces.utility as niu import nipype.pipeline.engine as pe it = pe.Node(niu.IdentityInterface(fields=[field.name]), name=field.name + "_iterable") it.iterables = (field.name, field.values) return it def _set_inputs(self): self._node_added = False set_dict = {} for f in self.fields: if not f.iterable: set_dict[f.name] = f.values else: it = self._add_iterable(f) self._node_added = True self._wk.connect(it, f.name, self._dg, f.name) self._dg.inputs.trait_set(**set_dict) def create_dataflow(self): import nipype.interfaces.io as nio import nipype.pipeline.engine as pe self._wk = pe.Workflow(name='custom_datagrabber') self._dg = pe.Node(nio.DataGrabber(outfields=self.outfields, infields=self._get_infields(), sort_filelist=self.sort), name='datagrabber') self._set_inputs() self._dg.inputs.base_directory = self.base_directory self._dg.inputs.field_template = self.field_template self._dg.inputs.template_args = self.template_args self._dg.inputs.template = self.template if not self._node_added: self._wk.add_nodes([self._dg]) return self._wk def get_fields(self): foo = self.get() d = {} for key, item in foo.iteritems(): if not key.startswith('_'): if isinstance(item, list): d[key] = [] for it in item: if isinstance(it, DataBase): d[key].append(it.get()) else: d[key].append(it) else: d[key] = item return d def set_fields(self, d): for key, item in d.iteritems(): if not key == "fields": self.set(**{key: item}) else: foo = [] for f in item: tmp = DataBase() tmp.set(**f) foo.append(tmp) self.set(**{key: foo}) def _check_fired(self): dg = self.create_dataflow() dg.run()
class config(HasTraits): uuid = traits.Str(desc="UUID") desc = traits.Str(desc="Workflow Description") # Directories working_dir = Directory(mandatory=True, desc="Location of the Nipype working directory") sink_dir = Directory(os.path.abspath('.'), mandatory=True, desc="Location where the BIP will store the results") crash_dir = Directory(mandatory=False, desc="Location to store crash files") json_sink = Directory(mandatory=False, desc="Location to store json_files") surf_dir = Directory(mandatory=True, desc="Freesurfer subjects directory") save_script_only = traits.Bool(False) # Execution run_using_plugin = Bool( False, usedefault=True, desc="True to run pipeline with plugin, False to run serially") plugin = traits.Enum("PBS", "MultiProc", "SGE", "Condor", usedefault=True, desc="plugin to use, if run_using_plugin=True") plugin_args = traits.Dict({"qsub_args": "-q many"}, usedefault=True, desc='Plugin arguments.') test_mode = Bool( False, mandatory=False, usedefault=True, desc='Affects whether where and if the workflow keeps its \ intermediary files. True to keep intermediary files. ' ) timeout = traits.Float(14.0) # Subjects #subjects= traits.List(traits.Str, mandatory=True, usedefault=True, # desc="Subject id's. Note: These MUST match the subject id's in the \ # Freesurfer directory. For simplicity, the subject id's should \ # also match with the location of individual functional files.") datagrabber = traits.Instance(Data, ()) # First Level interscan_interval = traits.Float() film_threshold = traits.Float() input_units = traits.Enum('scans', 'secs') is_sparse = traits.Bool(False) model_hrf = traits.Bool(True) stimuli_as_impulses = traits.Bool(True) use_temporal_deriv = traits.Bool(True) volumes_in_cluster = traits.Int(1) ta = traits.Float() tr = traits.Float() hpcutoff = traits.Float() scan_onset = traits.Int(0) scale_regressors = traits.Bool(True) #bases = traits.Dict({'dgamma':{'derivs': False}},use_default=True) bases = traits.Dict( {'dgamma': { 'derivs': False }}, use_default=True ) #traits.Enum('dgamma','gamma','none'), traits.Enum(traits.Dict(traits.Enum('derivs',None), traits.Bool),None), desc="name of basis function and options e.g., {'dgamma': {'derivs': True}}") # preprocessing info preproc_config = traits.File(desc="preproc config file") use_compcor = traits.Bool(desc="use noise components from CompCor") #advanced_options use_advanced_options = Bool(False) advanced_options = traits.Code()
class config(HasTraits): uuid = traits.Str(desc="UUID") # Directories working_dir = Directory(mandatory=True, desc="Location of the Nipype working directory") base_dir = Directory( os.path.abspath('.'), mandatory=True, desc='Base directory of data. (Should be subject-independent)') sink_dir = Directory(mandatory=True, desc="Location where the BIP will store the results") crash_dir = Directory(mandatory=False, desc="Location to store crash files") save_script_only = traits.Bool(False) # Execution run_using_plugin = Bool( False, usedefault=True, desc="True to run pipeline with plugin, False to run serially") plugin = traits.Enum("PBS", "MultiProc", "SGE", "Condor", usedefault=True, desc="plugin to use, if run_using_plugin=True") plugin_args = traits.Dict({"qsub_args": "-q many"}, usedefault=True, desc='Plugin arguments.') test_mode = Bool( False, mandatory=False, usedefault=True, desc='Affects whether where and if the workflow keeps its \ intermediary files. True to keep intermediary files. ' ) timeout = traits.Float(14.0) datagrabber = traits.Instance(Data, ()) # Regression run_one_sample_T_test = traits.Bool(True) run_regression = traits.Bool() design_csv = traits.File(desc="design .csv file") reg_contrasts = traits.Code( desc= "function named reg_contrasts which takes in 0 args and returns contrasts" ) use_regressors = traits.Bool() estimation_method = traits.Enum('Classical', 'Bayesian', 'Bayesian2') include_intercept = traits.Bool(True) #Normalization norm_template = traits.File(desc='Template of files') use_mask = traits.Bool(False) mask_file = traits.File(desc='already binarized mask file to use') #Correction: p_threshold = traits.Float(0.05) height_threshold = traits.Float(0.05) min_cluster_size = traits.Int(25) # Advanced Options use_advanced_options = traits.Bool() advanced_script = traits.Code() # Buttons check_func_datagrabber = Button("Check")
class config(HasTraits): uuid = traits.Str(desc="UUID") # Directories working_dir = Directory(mandatory=True, desc="Location of the Nipype working directory") base_dir = Directory( os.path.abspath('.'), mandatory=True, desc='Base directory of data. (Should be subject-independent)') sink_dir = Directory(mandatory=True, desc="Location where the BIP will store the results") crash_dir = Directory(mandatory=False, desc="Location to store crash files") # Execution run_using_plugin = Bool( False, usedefault=True, desc="True to run pipeline with plugin, False to run serially") plugin = traits.Enum("PBS", "MultiProc", "SGE", "Condor", usedefault=True, desc="plugin to use, if run_using_plugin=True") plugin_args = traits.Dict({"qsub_args": "-q many"}, usedefault=True, desc='Plugin arguments.') test_mode = Bool( False, mandatory=False, usedefault=True, desc='Affects whether where and if the workflow keeps its \ intermediary files. True to keep intermediary files. ' ) # Subjects subjects = traits.List( traits.Str, mandatory=True, usedefault=True, desc="Subject id's. Note: These MUST match the subject id's in the \ Freesurfer directory. For simplicity, the subject id's should \ also match with the location of individual functional files." ) fwhm = traits.List(traits.Float()) inputs_template = traits.String('%s/preproc/output/fwhm_%s/*.nii.gz') meanfunc_template = traits.String('%s/preproc/mean/*_mean.nii.gz') fsl_mat_template = traits.String('%s/preproc/bbreg/*.mat') unwarped_brain_template = traits.String('%s/smri/unwarped_brain/*.nii*') affine_transformation_template = traits.String( '%s/smri/affine_transformation/*.nii*') warp_field_template = traits.String('%s/smri/warped_field/*.nii*') #Normalization standard_transform_template = traits.File( mandatory=True, desc='Standard template to warp to') standard_warp_field_template = traits.String() standard_affine_transformation_template = traits.String() standard_norm_template = traits.File() standard_warp_field_template = traits.File() standard_affine_transformation_template = traits.File() # Advanced Options use_advanced_options = traits.Bool() advanced_script = traits.Code() # Buttons check_func_datagrabber = Button("Check")