class StripeClass(remote_traits.MaybeRemoteHasTraits):
"""base class of worker subclass, and also runs in GUI process"""
experiment_file = traits.File()
start_experiment = traits.Button(label='start experiment')
traits_view = View(
Group((
Item(name='start_experiment', show_label=False),
Item(name='experiment_file'),
)), )
def __init__(self, fname=""):
self.data = eta.Dict()
self.fname = eta.File()
self.plotting = eta.String()
self.open_VERT = eta.Button("Open VERT")
self.only_I = False
self.only_dIdV = False
self.pointers = []
self.fname = os.getcwdu()
self.plotting = "V"
class QuadMeshSmoothingOutputSpec(TraitedSpec):
outputSurface = traits.File( argstr = "----outputSurface %s")
class PipelineConfiguration(traits.HasTraits):
# project settings
project_dir = traits.Directory(
exists=False, desc="data path to where the project is stored")
# project metadata (for connectome file)
project_metadata = traits.Dict(
desc="project metadata to be stored in the connectome file")
# DEPRECATED: this field is deprecated after version >1.0.2
generator = traits.Str()
# parcellation scheme
parcellation_scheme = traits.Enum("NativeFreesurfer",
["Lausanne2008", "NativeFreesurfer"],
desc="used parcellation scheme")
# choose between 'L' (linear) and 'N' (non-linear) and 'B' (bbregister)
registration_mode = traits.Enum(
"Linear", ["Linear", "Nonlinear", "BBregister"],
desc="registration mode: linear or non-linear or bbregister")
# choose between 'L' (linear) and 'B' (bbregister)
rsfmri_registration_mode = traits.Enum(
"Linear", ["Linear", "BBregister"],
desc="registration mode: linear or bbregister")
diffusion_imaging_model = traits.Enum("DSI", ["DSI", "DTI", "QBALL"])
# DSI
nr_of_gradient_directions = traits.Str('515')
nr_of_sampling_directions = traits.Str('181')
odf_recon_param = traits.Str('-b0 1 -dsi -p 4 -sn 0')
hardi_recon_param = traits.Str('-b0 1 -p 3 -sn 0')
# DTI
gradient_table_file = traits.File(exists=False)
gradient_table = traits.Enum('siemens_64', [
'custom', 'mgh_dti_006', 'mgh_dti_018', 'mgh_dti_030', 'mgh_dti_042',
'mgh_dti_060', 'mgh_dti_072', 'mgh_dti_090', 'mgh_dti_120',
'mgh_dti_144', 'siemens_06', 'siemens_12', 'siemens_20', 'siemens_256',
'siemens_30', 'siemens_64'
])
nr_of_b0 = traits.Str('1')
max_b0_val = traits.Str('1000')
dti_recon_param = traits.Str('')
dtb_dtk2dir_param = traits.Str('')
# tractography
streamline_param = traits.Str('--angle 60 --seeds 32')
# registration
lin_reg_param = traits.Str('-usesqform -nosearch -dof 6 -cost mutualinfo')
nlin_reg_bet_T2_param = traits.Str('-f 0.35 -g 0.15')
nlin_reg_bet_b0_param = traits.Str('-f 0.2 -g 0.2')
nlin_reg_fnirt_param = traits.Str(
'--subsamp=8,4,2,2 --miter==5,5,5,5 --lambda=240,120,90,30 --splineorder=3 --applyinmask=0,0,1,1 --applyrefmask=0,0,1,1'
)
bb_reg_param = traits.Str('--init-header --dti')
# dicom converter
do_convert_diffusion = traits.Bool(True)
do_convert_T1 = traits.Bool(True)
do_convert_T2 = traits.Bool(False)
do_convert_fMRI = traits.Bool(False)
# rsfmri
rsfmri_lin_reg_param = traits.Str(
'-usesqform -nosearch -dof 6 -cost mutualinfo')
rsfmri_bb_reg_param = traits.Str('--init-header --dti')
do_save_mat = traits.Bool(True)
# DEPRECATED:
subject_raw_glob_diffusion = traits.Str("*.*")
subject_raw_glob_T1 = traits.Str("*.*")
subject_raw_glob_T2 = traits.Str("*.*")
extract_diffusion_metadata = traits.Bool(False)
# subject
subject_name = traits.Str()
subject_timepoint = traits.Str()
subject_workingdir = traits.Directory()
subject_logger = None
subject_metadata = [
KeyValue(key='description', value=''),
KeyValue(key='', value=''),
KeyValue(key='', value=''),
KeyValue(key='', value=''),
KeyValue(key='', value=''),
KeyValue(key='', value=''),
]
active_createfolder = traits.Bool(True)
active_dicomconverter = traits.Bool(False)
active_registration = traits.Bool(False)
active_segmentation = traits.Bool(False)
active_parcellation = traits.Bool(False)
active_applyregistration = traits.Bool(False)
active_reconstruction = traits.Bool(False)
active_tractography = traits.Bool(False)
active_fiberfilter = traits.Bool(False)
active_connectome = traits.Bool(False)
active_statistics = traits.Bool(False)
active_rsfmri = traits.Bool(False)
active_cffconverter = traits.Bool(False)
skip_completed_stages = traits.Bool(False)
# metadata
creator = traits.Str()
email = traits.Str()
publisher = traits.Str()
created = traits.Date()
modified = traits.Date()
license = traits.Str()
# rights = traits.Str()
reference = traits.Str()
# relation = traits.Str()
species = traits.Str('H**o sapiens')
description = traits.Str()
# segmentation
recon_all_param = traits.Str('-all -no-isrunning')
# parcellation
custompar_nrroi = traits.Int()
custompar_nodeinfo = traits.File()
custompar_volumeparcell = traits.File()
# fiber filtering
apply_splinefilter = traits.Bool(
True, desc='apply the spline filtering from diffusion toolkit')
apply_fiberlength = traits.Bool(True, desc='apply cutoff to fiber lengths')
fiber_cutoff_lower = traits.Float(
20.0,
desc='cut fibers that are shorter in length than given length in mm')
fiber_cutoff_upper = traits.Float(
500.0,
desc='cut fibers that are longer in length than given length in mm')
# measures
connection_P0 = traits.Bool(False)
connection_gfa = traits.Bool(False)
connection_kurtosis = traits.Bool(False)
connection_skewness = traits.Bool(False)
connection_adc = traits.Bool(False)
connection_fa = traits.Bool(False)
# cff converter
cff_fullnetworkpickle = traits.Bool(
True,
desc='stores the full network pickle generated by connectome creation')
cff_cmatpickle = traits.Bool(True)
cff_originalfibers = traits.Bool(True, desc='stores original fibers')
cff_filteredfibers = traits.Bool(True, desc='stores filtered fibers')
cff_finalfiberlabels = traits.Bool(
True, desc='stores final fibers and their labelarrays')
cff_fiberarr = traits.Bool(True)
cff_rawdiffusion = traits.Bool(True)
cff_scalars = traits.Bool(True)
cff_rawT1 = traits.Bool(True)
cff_rawT2 = traits.Bool(True)
cff_roisegmentation = traits.Bool(
True, desc='stores multi-resolution parcellation volumes')
cff_surfaces = traits.Bool(True,
desc='stores individually genertated surfaces')
cff_surfacelabels = traits.Bool(
True, desc='stores individually genertated surfaces')
# do you want to do manual white matter mask correction?
wm_handling = traits.Enum(
1, [1, 2, 3],
desc="in what state should the freesurfer step be processed")
# custom parcellation
parcellation = traits.Dict(
desc="provide the dictionary with your parcellation.")
# start up fslview
inspect_registration = traits.Bool(
False, desc='start fslview to inspect the the registration results')
fsloutputtype = traits.Enum('NIFTI', ['NIFTI'])
# connectome creation
compute_curvature = traits.Bool(False)
# email notification, needs a local smtp server
# sudo apt-get install postfix
emailnotify = traits.ListStr(
[], desc='the email address to send stage completion status message')
freesurfer_home = traits.Directory(exists=False, desc="path to Freesurfer")
fsl_home = traits.Directory(exists=False, desc="path to FSL")
dtk_home = traits.Directory(exists=False, desc="path to diffusion toolkit")
# This file stores descriptions of the inputs/outputs to each stage of the
# CMP pipeline. It can be queried using the PipelineStatus python object
pipeline_status_file = traits.Str("cmp.status")
# Pipeline status object
pipeline_status = pipeline_status.PipelineStatus()
def _get_lausanne_parcellation(self, parcel="NativeFreesurfer"):
if parcel == "Lausanne2008":
return {
'scale33': {
'number_of_regions':
83,
# contains name, url, color, freesurfer_label, etc. used for connection matrix
'node_information_graphml':
op.join(
self.get_lausanne_parcellation_path('resolution83'),
'resolution83.graphml'),
# scalar node values on fsaverage? or atlas?
'surface_parcellation':
None,
# scalar node values in fsaverage volume?
'volume_parcellation':
None,
# the subdirectory name from where to copy parcellations, with hemispheric wildcard
'fs_label_subdir_name':
'regenerated_%s_36',
# should we subtract the cortical rois for the white matter mask?
'subtract_from_wm_mask':
1,
},
'scale60': {
'number_of_regions':
129,
'node_information_graphml':
op.join(
self.get_lausanne_parcellation_path('resolution150'),
'resolution150.graphml'),
'surface_parcellation':
None,
'volume_parcellation':
None,
'fs_label_subdir_name':
'regenerated_%s_60',
'subtract_from_wm_mask':
1,
},
'scale125': {
'number_of_regions':
234,
'node_information_graphml':
op.join(
self.get_lausanne_parcellation_path('resolution258'),
'resolution258.graphml'),
'surface_parcellation':
None,
'volume_parcellation':
None,
'fs_label_subdir_name':
'regenerated_%s_125',
'subtract_from_wm_mask':
1,
},
'scale250': {
'number_of_regions':
463,
'node_information_graphml':
op.join(
self.get_lausanne_parcellation_path('resolution500'),
'resolution500.graphml'),
'surface_parcellation':
None,
'volume_parcellation':
None,
'fs_label_subdir_name':
'regenerated_%s_250',
'subtract_from_wm_mask':
1,
},
'scale500': {
'number_of_regions':
1015,
'node_information_graphml':
op.join(
self.get_lausanne_parcellation_path('resolution1015'),
'resolution1015.graphml'),
'surface_parcellation':
None,
'volume_parcellation':
None,
'fs_label_subdir_name':
'regenerated_%s_500',
'subtract_from_wm_mask':
1,
},
}
else:
return {
'freesurferaparc': {
'number_of_regions':
83,
# contains name, url, color, freesurfer_label, etc. used for connection matrix
'node_information_graphml':
op.join(
self.get_lausanne_parcellation_path('freesurferaparc'),
'resolution83.graphml'),
# scalar node values on fsaverage? or atlas?
'surface_parcellation':
None,
# scalar node values in fsaverage volume?
'volume_parcellation':
None,
}
}
def __init__(self, **kwargs):
# NOTE: In python 2.6, object.__init__ no longer accepts input
# arguments. HasTraits does not define an __init__ and
# therefore these args were being ignored.
super(PipelineConfiguration, self).__init__(**kwargs)
# the default parcellation provided
self.parcellation = self._get_lausanne_parcellation(
parcel="NativeFreesurfer")
self.can_use_dipy = dipy_here
# no email notify
self.emailnotify = []
# default gradient table for DTI
self.gradient_table_file = self.get_cmp_gradient_table('siemens_64')
# try to discover paths from environment variables
try:
self.freesurfer_home = op.join(os.environ['FREESURFER_HOME'])
self.fsl_home = op.join(os.environ['FSLDIR'])
self.dtk_home = os.environ['DTDIR']
self.dtk_matrices = op.join(self.dtk_home, 'matrices')
except KeyError:
pass
self.fsloutputtype = 'NIFTI'
os.environ['FSLOUTPUTTYPE'] = self.fsloutputtype
os.environ['FSLOUTPUTTYPE'] = 'NIFTI'
def consistency_check(self):
""" Provides a checking facility for configuration objects """
# project name not empty
if not op.exists(self.project_dir):
msg = 'Your project directory does not exist!'
raise Exception(msg)
# check metadata
if self.creator == '':
raise Exception('You need to enter creator metadata!')
if self.publisher == '':
raise Exception('You need to enter publisher metadata!')
if self.email == '':
raise Exception('You need to enter email of a contact person!')
# check if software paths exists
pas = {
'configuration.freesurfer_home': self.freesurfer_home,
'configuration.fsl_home': self.fsl_home,
'configuration.dtk_home': self.dtk_home,
'configuration.dtk_matrices': self.dtk_matrices
}
for k, p in pas.items():
if not op.exists(p):
msg = 'Required software path for %s does not exists: %s' % (k,
p)
raise Exception(msg)
if self.subject_workingdir == '':
msg = 'No working directory defined for subject'
raise Exception(msg)
# else:
# wdir = self.get_subj_dir()
# if not op.exists(wdir):
# msg = 'Working directory %s does not exists for subject' % (wdir)
# raise Exception(msg)
# else:
# wdiff = op.join(self.get_raw_diffusion())
# print wdiff
# if not op.exists(wdiff):
# msg = 'Diffusion MRI subdirectory %s does not exists for the subject' % wdiff
# raise Exception(msg)
# wt1 = op.join(self.get_rawt1())
# if not op.exists(wt1):
# msg = 'Structural MRI subdirectory %s T1 does not exist in RAWDATA' % wt1
# raise Exception(msg)
def get_cmp_home(self):
""" Return the cmp home path """
return op.dirname(__file__)
def get_rawdata(self):
""" Return raw data path for the subject """
return op.join(self.get_subj_dir(), 'RAWDATA')
def get_log(self):
""" Get subject log dir """
return op.join(self.get_subj_dir(), 'LOG')
def get_logname(self, suffix='.log'):
""" Get a generic name for the log and pickle files """
a = dt.datetime.now()
return 'pipeline-%s-%02i%02i-%s-%s%s' % (
a.date().isoformat(), a.time().hour, a.time().minute,
self.subject_name, self.subject_timepoint, suffix)
def get_logger(self):
""" Get the logger instance created """
if self.subject_logger is None:
# setup logger for the subject
self.subject_logger = \
getLog(os.path.join(self.get_log(), self.get_logname()))
return self.subject_logger
else:
return self.subject_logger
def get_rawglob(self, modality):
""" DEPRECATED: Get the file name endings for modality """
if modality == 'diffusion':
if not self.subject_raw_glob_diffusion == '':
return self.subject_raw_glob_diffusion
else:
raise Exception('No raw_glob_diffusion defined for subject')
elif modality == 'T1':
if not self.subject_raw_glob_T1 == '':
return self.subject_raw_glob_T1
else:
raise Exception('No raw_glob_T1 defined for subject')
elif modality == 'T2':
if not self.subject_raw_glob_T2 == '':
return self.subject_raw_glob_T2
else:
raise Exception('No raw_glob_T2 defined for subject')
def get_dicomfiles(self, modality):
""" Get a list of dicom files for the requested modality. Tries to
discover them automatically
"""
from glob import glob
if modality == 'diffusion':
pat = self.get_raw_diffusion()
elif modality == 'T1':
pat = self.get_rawt1()
elif modality == 'T2':
pat = self.get_rawt2()
elif modality == 'fMRI':
pat = self.get_rawrsfmri()
# discover files with *.* and *
difiles = sorted(glob(op.join(pat, '*.*')) + glob(op.join(pat, '*')))
# exclude potential .nii and .nii.gz files
difiles = [
e for e in difiles
if not e.endswith('.nii') and not e.endswith('.nii.gz')
]
# check if no files and throw exception
if len(difiles) == 0:
raise Exception('Could not find any DICOM files in folder %s' %
pat)
return difiles
def get_rawrsfmri(self):
""" Get raw functional MRI path for subject """
return op.join(self.get_rawdata(), 'fMRI')
def get_rawt1(self):
""" Get raw structural MRI T1 path for subject """
return op.join(self.get_rawdata(), 'T1')
def get_rawt2(self):
""" Get raw structural MRI T2 path for subject """
return op.join(self.get_rawdata(), 'T2')
def get_subj_dir(self):
return self.subject_workingdir
def get_raw_diffusion(self):
""" Get the raw diffusion path for subject """
if self.diffusion_imaging_model == 'DSI':
return op.join(self.get_subj_dir(), 'RAWDATA', 'DSI')
elif self.diffusion_imaging_model == 'DTI':
return op.join(self.get_subj_dir(), 'RAWDATA', 'DTI')
elif self.diffusion_imaging_model == 'QBALL':
return op.join(self.get_subj_dir(), 'RAWDATA', 'QBALL')
def get_fs(self):
""" Returns the subject root folder path for freesurfer files """
return op.join(self.get_subj_dir(), 'FREESURFER')
def get_stats(self):
""" Return statistic output path """
return op.join(self.get_subj_dir(), 'STATS')
def get_cffdir(self):
""" Returns path to store connectome file """
return op.join(self.get_cmp(), 'cff')
def get_nifti(self):
""" Returns the subject root folder path for nifti files """
return op.join(self.get_subj_dir(), 'NIFTI')
def get_nifti_trafo(self):
""" Returns the path to the subjects transformation / registration matrices """
return op.join(self.get_nifti(), 'transformations')
def get_nifti_bbregister(self):
""" Returns the path to the subjects transformation / registration matrices, bbregister mode """
return op.join(self.get_nifti(), 'bbregister')
def get_diffusion_metadata(self):
""" Diffusion metadata, i.e. where gradient_table.txt is stored """
return op.join(self.get_nifti(), 'diffusion_metadata')
def get_nifti_wm_correction(self):
""" Returns the path to the subjects wm_correction path """
return op.join(self.get_nifti(), 'wm_correction')
def get_cmp(self):
return op.join(self.get_subj_dir(), 'CMP')
def get_cmp_rawdiff(self, ):
return op.join(self.get_cmp(), 'raw_diffusion')
def get_cmp_rawdiff_reconout(self):
""" Returns the output path for diffusion reconstruction without prefix"""
if self.diffusion_imaging_model == 'DSI':
return op.join(self.get_cmp(), 'raw_diffusion', 'odf_0')
elif self.diffusion_imaging_model == 'DTI':
return op.join(self.get_cmp(), 'raw_diffusion', 'dti_0')
elif self.diffusion_imaging_model == 'QBALL':
return op.join(self.get_cmp(), 'raw_diffusion', 'qball_0')
def get_cmp_rawdiff_resampled(self):
return op.join(self.get_cmp_rawdiff(), '2x2x2')
def get_cmp_fsout(self):
return op.join(self.get_cmp(), 'fs_output')
def get_cmp_fibers(self):
return op.join(self.get_cmp(), 'fibers')
def get_cmp_scalars(self):
return op.join(self.get_cmp(), 'scalars')
def get_cmp_matrices(self):
return op.join(self.get_cmp_fibers(), 'matrices')
def get_cmp_fmri(self):
return op.join(self.get_cmp(), 'fMRI')
def get_cmp_tracto_mask(self):
return op.join(self.get_cmp_fsout(), 'HR')
def get_cmp_tracto_mask_tob0(self):
return op.join(self.get_cmp_fsout(), 'HR__registered-TO-b0')
def get_custom_gradient_table(self):
""" Returns the absolute path to the custom gradient table
with optional b-values in the 4th row """
return self.gradient_table_file
def get_cmp_gradient_table(self, name):
""" Return default gradient tables shipped with CMP. These are mainly derived from
Diffusion Toolkit """
cmp_path = op.dirname(__file__)
return op.join(cmp_path, 'data', 'diffusion', 'gradient_tables',
name + '.txt')
def get_dtb_streamline_vecs_file(self, as_text=False):
""" Returns the odf directions file used for DTB_streamline """
cmp_path = op.dirname(__file__)
if as_text:
return op.join(cmp_path, 'data', 'diffusion', 'odf_directions',
'181_vecs.txt')
else:
return op.join(cmp_path, 'data', 'diffusion', 'odf_directions',
'181_vecs.dat')
# XXX
def get_cmp_scalarfields(self):
""" Returns a list with tuples with the scalar field name and the
absolute path to its nifti file """
ret = []
if self.diffusion_imaging_model == 'DSI':
# add gfa per default
ret.append(('gfa', op.join(self.get_cmp_scalars(),
'dsi_gfa.nii.gz')))
# XXX: add adc per default
elif self.diffusion_imaging_model == 'DTI':
# nothing to add yet for DTI
pass
return ret
def get_dtk_dsi_matrix(self):
""" Returns the DSI matrix from Diffusion Toolkit
The parameters have to be set in the configuration object with keys:
1. number of gradient directions : 'nr_of_gradient_directions'
2. number of sampling directions : 'nr_of_sampling_directions'
Example
-------
confobj.nr_of_gradient_directions = 515
confobj.nr_of_sampling_directions = 181
Returns matrix including absolute path to DSI_matrix_515x181.dat
"""
grad = self.nr_of_gradient_directions
samp = self.nr_of_sampling_directions
fpath = op.join(self.dtk_matrices,
"DSI_matrix_%sx%s.dat" % (grad, samp))
if not op.exists(fpath):
msg = "DSI matrix does not exists: %s" % fpath
raise Exception(msg)
return fpath
def get_lausanne_atlas(self, name=None):
""" Return the absolute path to the lausanne parcellation atlas
for the resolution name """
cmp_path = op.dirname(__file__)
provided_atlases = [
'myatlas_36_rh.gcs', 'myatlasP1_16_rh.gcs', 'myatlasP17_28_rh.gcs',
'myatlasP29_36_rh.gcs', 'myatlas_60_rh.gcs', 'myatlas_125_rh.gcs',
'myatlas_250_rh.gcs', 'myatlas_36_lh.gcs', 'myatlasP1_16_lh.gcs',
'myatlasP17_28_lh.gcs', 'myatlasP29_36_lh.gcs',
'myatlas_60_lh.gcs', 'myatlas_125_lh.gcs', 'myatlas_250_lh.gcs'
]
if name in provided_atlases:
return op.join(cmp_path, 'data', 'colortable_and_gcs',
'my_atlas_gcs', name)
else:
msg = "Atlas %s does not exists" % name
raise Exception(msg)
def get_freeview_lut(self, name):
""" Returns the Look-Up-Table as text file for a given parcellation scheme
in a dictionary """
cmp_path = op.dirname(__file__)
if name == "NativeFreesurfer":
return {
'freesurferaparc':
op.join(cmp_path, 'data', 'parcellation', 'nativefreesurfer',
'freesurferaparc', 'FreeSurferColorLUT_adapted.txt')
}
else:
return ""
def get_lausanne_parcellation_path(self, parcellationname):
cmp_path = op.dirname(__file__)
if self.parcellation_scheme == "Lausanne2008":
allowed_default_parcel = [
'resolution83', 'resolution150', 'resolution258',
'resolution500', 'resolution1015'
]
if parcellationname in allowed_default_parcel:
return op.join(cmp_path, 'data', 'parcellation',
'lausanne2008', parcellationname)
else:
msg = "Not a valid default parcellation name for the lausanne2008 parcellation scheme"
raise Exception(msg)
else:
allowed_default_parcel = ['freesurferaparc']
if parcellationname in allowed_default_parcel:
return op.join(cmp_path, 'data', 'parcellation',
'nativefreesurfer', parcellationname)
else:
msg = "Not a valid default parcellation name for the NativeFreesurfer parcellation scheme"
raise Exception(msg)
def get_cmp_binary_path(self):
""" Returns the path to the binary files for the current platform
and architecture """
if sys.platform == 'linux2':
import platform as pf
if '32' in pf.architecture()[0]:
return op.join(op.dirname(__file__), "binary", "linux2",
"bit32")
elif '64' in pf.architecture()[0]:
return op.join(op.dirname(__file__), "binary", "linux2",
"bit64")
else:
raise ('No binary files compiled for your platform!')
def get_pipeline_status_file(self):
"""Returns the absolute path of the pipeline status file"""
return op.join(self.get_subj_dir(), self.pipeline_status_file)
def init_pipeline_status(self):
"""Create the 'cmp.status'. The 'cmp.status' file contains information
about the inputs/outputs of each pipeline stage"""
status_file = op.join(self.get_subj_dir(), self.pipeline_status_file)
self.pipeline_status.Pipeline.name = "cmp"
self.pipeline_status.SaveToFile(status_file)
def update_pipeline_status(self):
"""Update the pipeline status on disk with the current status in memory"""
status_file = op.join(self.get_subj_dir(), self.pipeline_status_file)
self.pipeline_status.SaveToFile(status_file)
class Fit(traits.HasTraits):
name = traits.Str(desc="name of fit")
function = traits.Str(desc="function we are fitting with all parameters")
variablesList = traits.List(FitVariable)
calculatedParametersList = traits.List(CalculatedParameter)
xs = None # will be a scipy array
ys = None # will be a scipy array
zs = None # will be a scipy array
performFitButton = traits.Button("Perform Fit")
getInitialParametersButton = traits.Button("Guess Initial Values")
drawRequestButton = traits.Button("Draw Fit")
autoFitBool = traits.Bool(
False,
desc=
"Automatically perform this Fit with current settings whenever a new image is loaded"
)
autoGuessBool = traits.Bool(
False,
desc=
"Whenever a fit is completed replace the guess values with the calculated values (useful for increasing speed of the next fit)"
)
autoDrawBool = traits.Bool(
False,
desc=
"Once a fit is complete update the drawing of the fit or draw the fit for the first time"
)
logBool = traits.Bool(
False, desc="Log the calculated and fitted values with a timestamp")
logFile = traits.File(desc="file path of logFile")
imageInspectorReference = None #will be a reference to the image inspector
fitting = traits.Bool(False) #true when performing fit
fitted = traits.Bool(
False) #true when current data displayed has been fitted
fitSubSpace = traits.Bool(
False) #true when current data displayed has been fitted
startX = traits.Int
startY = traits.Int
endX = traits.Int
endY = traits.Int
fittingStatus = traits.Str()
fitThread = None
physics = traits.Instance(physicsProperties.PhysicsProperties)
#status strings
notFittedForCurrentStatus = "Not Fitted for Current Image"
fittedForCurrentImageStatus = "Fit Complete for Current Image"
currentlyFittingStatus = "Currently Fitting..."
failedFitStatus = "Failed to finish fit. See logger"
fitSubSpaceGroup = traitsui.VGroup(
traitsui.Item("fitSubSpace", label="Fit Sub Space"),
traitsui.VGroup(traitsui.HGroup(traitsui.Item("startX"),
traitsui.Item("startY")),
traitsui.HGroup(traitsui.Item("endX"),
traitsui.Item("endY")),
visible_when="fitSubSpace"),
label="Fit Sub Space",
show_border=True)
generalGroup = traitsui.VGroup(traitsui.Item("name",
label="Fit Name",
style="readonly",
resizable=True),
traitsui.Item("function",
label="Fit Function",
style="readonly",
resizable=True),
fitSubSpaceGroup,
label="Fit",
show_border=True)
variablesGroup = traitsui.VGroup(traitsui.Item(
"variablesList",
editor=traitsui.ListEditor(style="custom"),
show_label=False,
resizable=True),
show_border=True,
label="parameters")
derivedGroup = traitsui.VGroup(traitsui.Item(
"calculatedParametersList",
editor=traitsui.ListEditor(style="custom"),
show_label=False,
resizable=True),
show_border=True,
label="derived values")
buttons = traitsui.VGroup(
traitsui.HGroup(traitsui.Item("autoFitBool"),
traitsui.Item("performFitButton")),
traitsui.HGroup(traitsui.Item("autoGuessBool"),
traitsui.Item("getInitialParametersButton")),
traitsui.HGroup(traitsui.Item("autoDrawBool"),
traitsui.Item("drawRequestButton")))
logGroup = traitsui.HGroup(traitsui.Item("logBool"),
traitsui.Item("logFile",
visible_when="logBool"),
label="Logging",
show_border=True)
actionsGroup = traitsui.VGroup(traitsui.Item("fittingStatus",
style="readonly"),
logGroup,
buttons,
label="Fit Actions",
show_border=True)
traits_view = traitsui.View(
traitsui.VGroup(generalGroup, variablesGroup, derivedGroup,
actionsGroup))
def __init__(self, **traitsDict):
super(Fit, self).__init__(**traitsDict)
self.startX = 0
self.startY = 0
def _set_xs(self, xs):
self.xs = xs
def _set_ys(self, ys):
self.ys = ys
def _set_zs(self, zs):
self.zs = zs
def _fittingStatus_default(self):
return self.notFittedForCurrentStatus
def _getInitialValues(self):
"""returns ordered list of initial values from variables List """
return [_.initialValue for _ in self.variablesList]
def _getCalculatedValues(self):
"""returns ordered list of initial values from variables List """
return [_.calculatedValue for _ in self.variablesList]
def _log_fit(self):
if self.logFile == "":
logger.warning("no log file defined. Will not log")
return
if not os.path.exists(self.logFile):
variables = [_.name for _ in self.variablesList]
calculated = [_.name for _ in self.calculatedParametersList]
times = ["datetime", "epoch seconds"]
info = ["img file name"]
columnNames = times + info + variables + calculated
with open(self.logFile, 'a+') as logFile:
writer = csv.writer(logFile)
writer.writerow(columnNames)
#column names already exist so...
variables = [_.calculatedValue for _ in self.variablesList]
calculated = [_.value for _ in self.calculatedParametersList]
now = time.time() #epoch seconds
timeTuple = time.localtime(now)
date = time.strftime("%Y-%m-%dT%H:%M:%S", timeTuple)
times = [date, now]
info = [self.imageInspectorReference.selectedFile]
data = times + info + variables + calculated
with open(self.logFile, 'a+') as logFile:
writer = csv.writer(logFile)
writer.writerow(data)
def _intelligentInitialValues(self):
"""If possible we can auto set the initial parameters to intelligent guesses user can always overwrite them """
self._setInitialValues(self._getIntelligentInitialValues())
def _get_subSpaceArrays(self):
"""returns the arrays of the selected sub space. If subspace is not
activated then returns the full arrays"""
if self.fitSubSpace:
xs = self.xs[self.startX:self.endX]
ys = self.ys[self.startY:self.endY]
logger.debug("xs array sliced length %s " % (xs.shape))
logger.debug("ys array sliced length %s " % (ys.shape))
zs = self.zs[self.startY:self.endY, self.startX:self.endX]
print zs
print zs.shape
logger.debug("zs sub space array %s,%s " % (zs.shape))
return xs, ys, zs
else:
return self.xs, self.ys, self.zs
def _getIntelligentInitialValues(self):
"""If possible we can auto set the initial parameters to intelligent guesses user can always overwrite them """
logger.debug("Dummy function should not be called directly")
return
def fitFunc(self, data, *p):
"""Function that we are trying to fit to. """
logger.error("Dummy function should not be called directly")
return
def _setCalculatedValues(self, calculated):
"""updates calculated values with calculated argument """
c = 0
for variable in self.variablesList:
variable.calculatedValue = calculated[c]
c += 1
def _setCalculatedValuesErrors(self, covarianceMatrix):
"""given the covariance matrix returned by scipy optimize fit
convert this into stdeviation errors for parameters list and updated
the stdevError attribute of variables"""
logger.debug("covariance matrix -> %s " % covarianceMatrix)
parameterErrors = scipy.sqrt(scipy.diag(covarianceMatrix))
logger.debug("parameterErrors -> %s " % parameterErrors)
c = 0
for variable in self.variablesList:
variable.stdevError = parameterErrors[c]
c += 1
def _setInitialValues(self, guesses):
"""updates calculated values with calculated argument """
c = 0
for variable in self.variablesList:
variable.initialValue = guesses[c]
c += 1
def deriveCalculatedParameters(self):
"""Wrapper for subclass definition of deriving calculated parameters
can put more general calls in here"""
if self.fitted:
self._deriveCalculatedParameters()
def _deriveCalculatedParameters(self):
"""Should be implemented by subclass. should update all variables in calculate parameters list"""
logger.error("Should only be called by subclass")
return
def _fit_routine(self):
"""This function performs the fit in an appropriate thread and
updates necessary values when the fit has been performed"""
self.fitting = True
if self.fitThread and self.fitThread.isAlive():
logger.warning(
"Fitting is already running cannot kick off a new fit until it has finished!"
)
return
else:
self.fitThread = FitThread()
self.fitThread.fitReference = self
self.fitThread.start()
self.fittingStatus = self.currentlyFittingStatus
def _perform_fit(self):
"""Perform the fit using scipy optimise curve fit.
We must supply x and y as one argument and zs as anothger. in the form
xs: 0 1 2 0 1 2 0
ys: 0 0 0 1 1 1 2
zs: 1 5 6 1 9 8 2
Hence the use of repeat and tile in positions and unravel for zs
initially xs,ys is a linspace array and zs is a 2d image array
"""
if self.xs is None or self.ys is None or self.zs is None:
logger.warning(
"attempted to fit data but had no data inside the Fit object. set xs,ys,zs first"
)
return ([], [])
p0 = self._getInitialValues()
if self.fitSubSpace: #fit only the sub space
#create xs, ys and zs which are appropriate slices of the arrays
xs, ys, zs = self._get_subSpaceArrays()
positions = [scipy.tile(xs, len(ys)),
scipy.repeat(ys, len(xs))
] #for creating data necessary for gauss2D function
params2D, cov2D = scipy.optimize.curve_fit(self.fitFunc,
positions,
scipy.ravel(zs),
p0=p0)
chi2 = scipy.sum(
(scipy.ravel(zs) - self.fitFunc(positions, *params2D))**2 /
self.fitFunc(positions, *params2D))
logger.debug("TEMPORARY ::: CHI^2 = %s " % chi2)
else: #fit the whole array of data (slower)
positions = [
scipy.tile(self.xs, len(self.ys)),
scipy.repeat(self.ys, len(self.xs))
] #for creating data necessary for gauss2D function
#note that it is necessary to ravel zs as curve_fit expects a flattened array
params2D, cov2D = scipy.optimize.curve_fit(self.fitFunc,
positions,
scipy.ravel(self.zs),
p0=p0)
return params2D, cov2D
def _performFitButton_fired(self):
self._fit_routine()
def _getInitialParametersButton_fired(self):
self._intelligentInitialValues()
def _drawRequestButton_fired(self):
"""tells the imageInspector to try and draw this fit as an overlay contour plot"""
self.imageInspectorReference.addFitPlot(self)
def _getFitFuncData(self):
"""if data has been fitted, this returns the zs data for the ideal
fitted function using the calculated paramters"""
positions = [
scipy.tile(self.xs, len(self.ys)),
scipy.repeat(self.ys, len(self.xs))
] #for creating data necessary for gauss2D function
zsravelled = self.fitFunc(positions, *self._getCalculatedValues())
return zsravelled.reshape(self.zs.shape)
class QuadMeshDecimationOutputSpec(TraitedSpec):
outputSurface = traits.File(argstr="--outputSurface %s")
class Fit(traits.HasTraits):
name = traits.Str(desc="name of fit")
function = traits.Str(desc="function we are fitting with all parameters")
variablesList = traits.List(Parameter)
calculatedParametersList = traits.List(CalculatedParameter)
xs = None # will be a scipy array
ys = None # will be a scipy array
zs = None # will be a scipy array
performFitButton = traits.Button("Perform Fit")
getInitialParametersButton = traits.Button("Guess Initial Values")
usePreviousFitValuesButton = traits.Button("Use Previous Fit")
drawRequestButton = traits.Button("Draw Fit")
setSizeButton = traits.Button("Set Initial Size")
chooseVariablesButtons = traits.Button("choose logged variables")
logLibrarianButton = traits.Button("librarian")
logLastFitButton = traits.Button("log current fit")
removeLastFitButton = traits.Button("remove last fit")
autoFitBool = traits.Bool(
False,
desc=
"Automatically perform this Fit with current settings whenever a new image is loaded"
)
autoGuessBool = traits.Bool(
False,
desc=
"Whenever a fit is completed replace the guess values with the calculated values (useful for increasing speed of the next fit)"
)
autoDrawBool = traits.Bool(
False,
desc=
"Once a fit is complete update the drawing of the fit or draw the fit for the first time"
)
autoSizeBool = traits.Bool(
False,
desc=
"If TOF variable is read from latest XML and is equal to 0.11ms (or time set in Physics) then it will automatically update the physics sizex and sizey with the Sigma x and sigma y from the gaussian fit"
)
logBool = traits.Bool(
False, desc="Log the calculated and fitted values with a timestamp")
logName = traits.String(
desc="name of the scan - will be used in the folder name")
logDirectory = os.path.join("\\\\ursa", "AQOGroupFolder",
"Experiment Humphry", "Data", "eagleLogs")
latestSequence = os.path.join("\\\\ursa", "AQOGroupFolder",
"Experiment Humphry",
"Experiment Control And Software",
"currentSequence", "latestSequence.xml")
logFile = traits.File(desc="file path of logFile")
logAnalyserBool = traits.Bool(
False, desc="only use log analyser script when True")
logAnalysers = [
] #list containing full paths to each logAnalyser file to run
logAnalyserDisplayString = traits.String(
desc=
"comma separated read only string that is a list of all logAnalyser python scripts to run. Use button to choose files"
)
logAnalyserSelectButton = traits.Button("sel. analyser",
image='@icons:function_node',
style="toolbar")
xmlLogVariables = []
imageInspectorReference = None #will be a reference to the image inspector
fitting = traits.Bool(False) #true when performing fit
fitted = traits.Bool(
False) #true when current data displayed has been fitted
fitSubSpace = traits.Bool(
False) #true when current data displayed has been fitted
startX = traits.Int
startY = traits.Int
endX = traits.Int
endY = traits.Int
fittingStatus = traits.Str()
fitThread = None
fitTimeLimit = traits.Float(
10.0,
desc=
"Time limit in seconds for fitting function. Only has an effect when fitTimeLimitBool is True"
)
fitTimeLimitBool = traits.Bool(
True,
desc=
"If True then fitting functions will be limited to time limit defined by fitTimeLimit "
)
physics = traits.Instance(
physicsProperties.physicsProperties.PhysicsProperties)
#status strings
notFittedForCurrentStatus = "Not Fitted for Current Image"
fittedForCurrentImageStatus = "Fit Complete for Current Image"
currentlyFittingStatus = "Currently Fitting..."
failedFitStatus = "Failed to finish fit. See logger"
timeExceededStatus = "Fit exceeded user time limit"
lmfitModel = traits.Instance(
lmfit.Model
) #reference to the lmfit model must be initialised in subclass
mostRecentModelResult = None # updated to the most recent ModelResult object from lmfit when a fit thread is performed
fitSubSpaceGroup = traitsui.VGroup(
traitsui.Item("fitSubSpace", label="Fit Sub Space", resizable=True),
traitsui.VGroup(traitsui.HGroup(
traitsui.Item("startX", resizable=True),
traitsui.Item("startY", resizable=True)),
traitsui.HGroup(traitsui.Item("endX", resizable=True),
traitsui.Item("endY", resizable=True)),
visible_when="fitSubSpace"),
label="Fit Sub Space",
show_border=True)
generalGroup = traitsui.VGroup(traitsui.Item("name",
label="Fit Name",
style="readonly",
resizable=True),
traitsui.Item("function",
label="Fit Function",
style="readonly",
resizable=True),
fitSubSpaceGroup,
label="Fit",
show_border=True)
variablesGroup = traitsui.VGroup(traitsui.Item(
"variablesList",
editor=traitsui.ListEditor(style="custom"),
show_label=False,
resizable=True),
show_border=True,
label="parameters")
derivedGroup = traitsui.VGroup(traitsui.Item(
"calculatedParametersList",
editor=traitsui.ListEditor(style="custom"),
show_label=False,
resizable=True),
show_border=True,
label="derived values")
buttons = traitsui.VGroup(
traitsui.HGroup(
traitsui.Item("autoFitBool", label="Auto fit?", resizable=True),
traitsui.Item("performFitButton", show_label=False,
resizable=True)),
traitsui.HGroup(
traitsui.Item("autoGuessBool", label="Auto guess?",
resizable=True),
traitsui.Item("getInitialParametersButton",
show_label=False,
resizable=True)),
traitsui.HGroup(
traitsui.Item("autoDrawBool", label="Auto draw?", resizable=True),
traitsui.Item("drawRequestButton",
show_label=False,
resizable=True)),
traitsui.HGroup(
traitsui.Item("autoSizeBool", label="Auto size?", resizable=True),
traitsui.Item("setSizeButton", show_label=False, resizable=True)),
traitsui.HGroup(
traitsui.Item("usePreviousFitValuesButton",
show_label=False,
resizable=True)))
logGroup = traitsui.VGroup(traitsui.HGroup(
traitsui.Item("logBool", resizable=True),
traitsui.Item("chooseVariablesButtons",
show_label=False,
resizable=True)),
traitsui.HGroup(
traitsui.Item("logName", resizable=True)),
traitsui.HGroup(
traitsui.Item("removeLastFitButton",
show_label=False,
resizable=True),
traitsui.Item("logLastFitButton",
show_label=False,
resizable=True)),
traitsui.HGroup(
traitsui.Item("logAnalyserBool",
label="analyser?",
resizable=True),
traitsui.Item("logAnalyserDisplayString",
show_label=False,
style="readonly",
resizable=True),
traitsui.Item("logAnalyserSelectButton",
show_label=False,
resizable=True)),
label="Logging",
show_border=True)
actionsGroup = traitsui.VGroup(traitsui.Item("fittingStatus",
style="readonly",
resizable=True),
logGroup,
buttons,
label="Fit Actions",
show_border=True)
traits_view = traitsui.View(traitsui.VGroup(generalGroup, variablesGroup,
derivedGroup, actionsGroup),
kind="subpanel")
def __init__(self, **traitsDict):
super(Fit, self).__init__(**traitsDict)
self.startX = 0
self.startY = 0
self.lmfitModel = lmfit.Model(self.fitFunc)
def _set_xs(self, xs):
self.xs = xs
def _set_ys(self, ys):
self.ys = ys
def _set_zs(self, zs):
self.zs = zs
def _fittingStatus_default(self):
return self.notFittedForCurrentStatus
def _getInitialValues(self):
"""returns ordered list of initial values from variables List """
return [_.initialValue for _ in self.variablesList]
def _getParameters(self):
"""creates an lmfit parameters object based on the user input in variablesList """
return lmfit.Parameters(
{_.name: _.parameter
for _ in self.variablesList})
def _getCalculatedValues(self):
"""returns ordered list of fitted values from variables List """
return [_.calculatedValue for _ in self.variablesList]
def _intelligentInitialValues(self):
"""If possible we can auto set the initial parameters to intelligent guesses user can always overwrite them """
self._setInitialValues(self._getIntelligentInitialValues())
def _get_subSpaceArrays(self):
"""returns the arrays of the selected sub space. If subspace is not
activated then returns the full arrays"""
if self.fitSubSpace:
xs = self.xs[self.startX:self.endX]
ys = self.ys[self.startY:self.endY]
logger.info("xs array sliced length %s " % (xs.shape))
logger.info("ys array sliced length %s " % (ys.shape))
zs = self.zs[self.startY:self.endY, self.startX:self.endX]
logger.info("zs sub space array %s,%s " % (zs.shape))
return xs, ys, zs
else:
return self.xs, self.ys, self.zs
def _getIntelligentInitialValues(self):
"""If possible we can auto set the initial parameters to intelligent guesses user can always overwrite them """
logger.debug("Dummy function should not be called directly")
return
#in python this should be a pass statement. I.e. user has to overwrite this
def fitFunc(self, data, *p):
"""Function that we are trying to fit to. """
logger.error("Dummy function should not be called directly")
return
#in python this should be a pass statement. I.e. user has to overwrite this
def _setCalculatedValues(self, modelFitResult):
"""updates calculated values with calculated argument """
parametersResult = modelFitResult.params
for variable in self.variablesList:
variable.calculatedValue = parametersResult[variable.name].value
def _setCalculatedValuesErrors(self, modelFitResult):
"""given the covariance matrix returned by scipy optimize fit
convert this into stdeviation errors for parameters list and updated
the stdevError attribute of variables"""
parametersResult = modelFitResult.params
for variable in self.variablesList:
variable.stdevError = parametersResult[variable.name].stderr
def _setInitialValues(self, guesses):
"""updates calculated values with calculated argument """
c = 0
for variable in self.variablesList:
variable.initialValue = guesses[c]
c += 1
def deriveCalculatedParameters(self):
"""Wrapper for subclass definition of deriving calculated parameters
can put more general calls in here"""
if self.fitted:
self._deriveCalculatedParameters()
def _deriveCalculatedParameters(self):
"""Should be implemented by subclass. should update all variables in calculate parameters list"""
logger.error("Should only be called by subclass")
return
def _fit_routine(self):
"""This function performs the fit in an appropriate thread and
updates necessary values when the fit has been performed"""
self.fitting = True
if self.fitThread and self.fitThread.isAlive():
logger.warning(
"Fitting is already running. You should wait till this fit has timed out before a new thread is started...."
)
#logger.warning("I will start a new fitting thread but your previous thread may finish at some undetermined time. you probably had bad starting conditions :( !")
return
self.fitThread = FitThread() #new fitting thread
self.fitThread.fitReference = self
self.fitThread.isCurrentFitThread = True # user can create multiple fit threads on a particular fit but only the latest one will have an effect in the GUI
self.fitThread.start()
self.fittingStatus = self.currentlyFittingStatus
def _perform_fit(self):
"""Perform the fit using scipy optimise curve fit.
We must supply x and y as one argument and zs as anothger. in the form
xs: 0 1 2 0 1 2 0
ys: 0 0 0 1 1 1 2
zs: 1 5 6 1 9 8 2
Hence the use of repeat and tile in positions and unravel for zs
initially xs,ys is a linspace array and zs is a 2d image array
"""
if self.xs is None or self.ys is None or self.zs is None:
logger.warning(
"attempted to fit data but had no data inside the Fit object. set xs,ys,zs first"
)
return ([], [])
params = self._getParameters()
if self.fitSubSpace: #fit only the sub space
#create xs, ys and zs which are appropriate slices of the arrays
xs, ys, zs = self._get_subSpaceArrays()
else: #fit the whole array of data (slower)
xs, ys, zs = self.xs, self.ys, self.zs
positions = scipy.array([
scipy.tile(xs, len(ys)),
scipy.repeat(ys, len(xs))
]) #for creating data necessary for gauss2D function
if self.fitTimeLimitBool:
modelFitResult = self.lmfitModel.fit(scipy.ravel(zs),
positions=positions,
params=params,
iter_cb=self.getFitCallback(
time.time()))
else: #no iter callback
modelFitResult = self.lmfitModel.fit(scipy.ravel(zs),
positions=positions,
params=params)
return modelFitResult
def getFitCallback(self, startTime):
"""returns the callback function that is called at every iteration of fit to check if it
has been running too long"""
def fitCallback(params, iter, resid, *args, **kws):
"""check the time and compare to start time """
if time.time() - startTime > self.fitTimeLimit:
raise FitException("Fit time exceeded user limit")
return fitCallback
def _performFitButton_fired(self):
self._fit_routine()
def _getInitialParametersButton_fired(self):
self._intelligentInitialValues()
def _drawRequestButton_fired(self):
"""tells the imageInspector to try and draw this fit as an overlay contour plot"""
self.imageInspectorReference.addFitPlot(self)
def _setSizeButton_fired(self):
"""use the sigmaX and sigmaY from the current fit to overwrite the
inTrapSizeX and inTrapSizeY parameters in the Physics Instance"""
self.physics.inTrapSizeX = abs(self.sigmax.calculatedValue)
self.physics.inTrapSizeY = abs(self.sigmay.calculatedValue)
def _getFitFuncData(self):
"""if data has been fitted, this returns the zs data for the ideal
fitted function using the calculated paramters"""
positions = [
scipy.tile(self.xs, len(self.ys)),
scipy.repeat(self.ys, len(self.xs))
] #for creating data necessary for gauss2D function
zsravelled = self.fitFunc(positions, *self._getCalculatedValues())
return zsravelled.reshape(self.zs.shape)
def _logAnalyserSelectButton_fired(self):
"""open a fast file editor for selecting many files """
fileDialog = FileDialog(action="open files")
fileDialog.open()
if fileDialog.return_code == pyface.constant.OK:
self.logAnalysers = fileDialog.paths
logger.info("selected log analysers: %s " % self.logAnalysers)
self.logAnalyserDisplayString = str(
[os.path.split(path)[1] for path in self.logAnalysers])
def runSingleAnalyser(self, module):
"""runs the logAnalyser module calling the run function and returns the
columnNames and values as a list"""
exec("import logAnalysers.%s as currentAnalyser" % module)
reload(
currentAnalyser
) #in case it has changed..#could make this only when user requests
#now the array also contains the raw image as this may be different to zs if you are using a processor
if hasattr(self.imageInspectorReference, "rawImage"):
rawImage = self.imageInspectorReference.rawImage
else:
rawImage = None
return currentAnalyser.run([self.xs, self.ys, self.zs, rawImage],
self.physics.variables, self.variablesList,
self.calculatedParametersList)
def runAnalyser(self):
""" if logAnalyserBool is true we perform runAnalyser at the end of _log_fit
runAnalyser checks that logAnalyser exists and is a python script with a valid run()function
it then performs the run method and passes to the run function:
-the image data as a numpy array
-the xml variables dictionary
-the fitted paramaters
-the derived values"""
for logAnalyser in self.logAnalysers:
if not os.path.isfile(logAnalyser):
logger.error(
"attempted to runAnalyser but could not find the logAnalyser File: %s"
% logAnalyser)
return
#these will contain the final column names and values
finalColumns = []
finalValues = []
#iterate over each selected logAnalyser get the column names and values and add them to the master lists
for logAnalyser in self.logAnalysers:
directory, module = os.path.split(logAnalyser)
module, ext = os.path.splitext(module)
if ext != ".py":
logger.error("file was not a python module. %s" % logAnalyser)
else:
columns, values = self.runSingleAnalyser(module)
finalColumns.extend(columns)
finalValues.extend(values)
return finalColumns, finalValues
def mostRecentModelFitReport(self):
"""returns the lmfit fit report of the most recent
lmfit model results object"""
if self.mostRecentModelResult is not None:
return lmfit.fit_report(self.mostRecentModelResult) + "\n\n"
else:
return "No fit performed"
def getCalculatedParameters(self):
"""useful for print returns tuple list of calculated parameter name and value """
return [(_.name, _.value) for _ in self.calculatedParametersList]
def _log_fit(self):
if self.logName == "":
logger.warning("no log file defined. Will not log")
return
#generate folders if they don't exist
logFolder = os.path.join(self.logDirectory, self.logName)
if not os.path.isdir(logFolder):
logger.info("creating a new log folder %s" % logFolder)
os.mkdir(logFolder)
imagesFolder = os.path.join(logFolder, "images")
if not os.path.isdir(imagesFolder):
logger.info("creating a new images Folder %s" % imagesFolder)
os.mkdir(imagesFolder)
commentsFile = os.path.join(logFolder, "comments.txt")
if not os.path.exists(commentsFile):
logger.info("creating a comments file %s" % commentsFile)
open(commentsFile,
"a+").close() #create a comments file in every folder!
firstSequenceCopy = os.path.join(logFolder,
"copyOfInitialSequence.ctr")
if not os.path.exists(firstSequenceCopy):
logger.info("creating a copy of the first sequence %s -> %s" %
(self.latestSequence, firstSequenceCopy))
shutil.copy(self.latestSequence, firstSequenceCopy)
if self.imageInspectorReference.model.imageMode == "process raw image": #if we are using a processor, save the details of the processor used to the log folder
processorParamtersFile = os.path.join(logFolder,
"processorOptions.txt")
processorPythonScript = os.path.join(logFolder,
"usedProcessor.py") #TODO!
if not os.path.exists(processorParamtersFile):
with open(processorParamtersFile, "a+") as processorParamsFile:
string = str(self.imageInspectorReference.model.
chosenProcessor) + "\n"
string += str(self.imageInspectorReference.model.processor.
optionsDict)
processorParamsFile.write(string)
logger.debug("finished all checks on log folder")
#copy current image
try:
shutil.copy(self.imageInspectorReference.selectedFile,
imagesFolder)
except IOError as e:
logger.error("Could not copy image. Got IOError: %s " % e.message)
except Exception as e:
logger.error("Could not copy image. Got %s: %s " %
(type(e), e.message))
raise e
logger.info("copying current image")
self.logFile = os.path.join(logFolder, self.logName + ".csv")
#analyser logic
if self.logAnalyserBool: #run the analyser script as requested
logger.info(
"log analyser bool enabled... will attempt to run analyser script"
)
analyserResult = self.runAnalyser()
logger.info("analyser result = %s " % list(analyserResult))
if analyserResult is None:
analyserColumnNames = []
analyserValues = []
#analyser failed. continue as if nothing happened
else:
analyserColumnNames, analyserValues = analyserResult
else: #no analyser enabled
analyserColumnNames = []
analyserValues = []
if not os.path.exists(self.logFile):
variables = [_.name for _ in self.variablesList]
calculated = [_.name for _ in self.calculatedParametersList]
times = ["datetime", "epoch seconds"]
info = ["img file name"]
xmlVariables = self.xmlLogVariables
columnNames = times + info + variables + calculated + xmlVariables + analyserColumnNames
with open(
self.logFile, 'ab+'
) as logFile: # note use of binary file so that windows doesn't write too many /r
writer = csv.writer(logFile)
writer.writerow(columnNames)
#column names already exist so...
logger.debug("copying current image")
variables = [_.calculatedValue for _ in self.variablesList]
calculated = [_.value for _ in self.calculatedParametersList]
now = time.time() #epoch seconds
timeTuple = time.localtime(now)
date = time.strftime("%Y-%m-%dT%H:%M:%S", timeTuple)
times = [date, now]
info = [self.imageInspectorReference.selectedFile]
xmlVariables = [
self.physics.variables[varName] for varName in self.xmlLogVariables
]
data = times + info + variables + calculated + xmlVariables + analyserValues
with open(self.logFile, 'ab+') as logFile:
writer = csv.writer(logFile)
writer.writerow(data)
def _logLastFitButton_fired(self):
"""logs the fit. User can use this for non automated logging. i.e. log
particular fits"""
self._log_fit()
def _removeLastFitButton_fired(self):
"""removes the last line in the log file """
logFolder = os.path.join(self.logDirectory, self.logName)
self.logFile = os.path.join(logFolder, self.logName + ".csv")
if self.logFile == "":
logger.warning("no log file defined. Will not log")
return
if not os.path.exists(self.logFile):
logger.error(
"cant remove a line from a log file that doesn't exist")
with open(self.logFile, 'r') as logFile:
lines = logFile.readlines()
with open(self.logFile, 'wb') as logFile:
logFile.writelines(lines[:-1])
def saveLastFit(self):
"""saves result of last fit to a txt/csv file. This can be useful for live analysis
or for generating sequences based on result of last fit"""
try:
with open(
self.imageInspectorReference.cameraModel + "-" +
self.physics.species + "-" + "lastFit.csv",
"wb") as lastFitFile:
writer = csv.writer(lastFitFile)
writer.writerow(["time", time.time()])
for variable in self.variablesList:
writer.writerow([variable.name, variable.calculatedValue])
for variable in self.calculatedParametersList:
writer.writerow([variable.name, variable.value])
except Exception as e:
logger.error("failed to save last fit to text file. message %s " %
e.message)
def _chooseVariablesButtons_fired(self):
self.xmlLogVariables = self.chooseVariables()
def _usePreviousFitValuesButton_fired(self):
"""update the guess initial values with the value from the last fit """
logger.info(
"use previous fit values button fired. loading previous initial values"
)
self._setInitialValues(self._getCalculatedValues())
def chooseVariables(self):
"""Opens a dialog asking user to select columns from a data File that has
been selected. THese are then returned as a string suitable for Y cols input"""
columns = self.physics.variables.keys()
columns.sort()
values = zip(range(0, len(columns)), columns)
checklist_group = traitsui.Group(
'10', # insert vertical space
traitsui.Label('Select the additional variables you wish to log'),
traitsui.UItem('columns',
style='custom',
editor=traitsui.CheckListEditor(values=values,
cols=6)),
traitsui.UItem('selectAllButton'))
traits_view = traitsui.View(checklist_group,
title='CheckListEditor',
buttons=['OK'],
resizable=True,
kind='livemodal')
col = ColumnEditor(numberOfColumns=len(columns))
try:
col.columns = [
columns.index(varName) for varName in self.xmlLogVariables
]
except Exception as e:
logger.error(
"couldn't selected correct variable names. Returning empty selection"
)
logger.error("%s " % e.message)
col.columns = []
col.edit_traits(view=traits_view)
logger.debug("value of columns selected = %s ", col.columns)
logger.debug("value of columns selected = %s ",
[columns[i] for i in col.columns])
return [columns[i] for i in col.columns]
def _logLibrarianButton_fired(self):
"""opens log librarian for current folder in logName box. """
logFolder = os.path.join(self.logDirectory, self.logName)
if not os.path.isdir(logFolder):
logger.error(
"cant open librarian on a log that doesn't exist.... Could not find %s"
% logFolder)
return
librarian = plotObjects.logLibrarian.Librarian(logFolder=logFolder)
librarian.edit_traits()