class ProcessRunPanel(ProcessPanel):
def __init__(self, parent):
super(ProcessRunPanel, self).__init__(parent)
self.loadController()
self.db = DBI(self.controller.configfile)
self.db.getconn()
# self.controller = parent.controller
# Bind timer event
# self.Bind(wx.EVT_TIMER, self.progressfunc, self.controller.timer)
# processes = [p['caption'] for p in self.controller.processes]
# self.m_checkListProcess.AppendItems(processes)
# Set up event handler for any worker thread results
EVT_RESULT(self, self.progressfunc)
# EVT_CANCEL(self, self.stopfunc)
# Set timer handler
self.start = {}
def loadController(self):
self.controller = self.Parent.controller
processes = [self.controller.processes[p]['caption'] for p in self.controller.processes]
self.m_checkListProcess.Clear()
self.m_checkListProcess.AppendItems(processes)
def OnShowDescription(self, event):
print(event.String)
desc = [self.controller.processes[p]['description'] for p in self.controller.processes if
self.controller.processes[p]['caption'] == event.String]
filesIn = [self.controller.processes[p]['filesin'] for p in self.controller.processes if
self.controller.processes[p]['caption'] == event.String]
filesOut = [self.controller.processes[p]['filesout'] for p in self.controller.processes if
self.controller.processes[p]['caption'] == event.String]
# Load from Config
filesIn = [self.controller.db.getConfigByName(self.controller.currentconfig,f) for f in filesIn[0].split(", ")]
filesOut = [self.controller.db.getConfigByName(self.controller.currentconfig,f) for f in filesOut[0].split(", ")]
# Load to GUI
self.m_stTitle.SetLabelText(event.String)
self.m_stDescription.SetLabelText(desc[0])
self.m_stFilesin.SetLabelText(", ".join(filesIn))
self.m_stFilesout.SetLabelText(", ".join(filesOut))
self.Layout()
def progressfunc(self, msg):
"""
Update progress bars in table - multithreaded
:param count:
:param row:
:param col:
:return:
"""
(count, row, i, total, process) = msg.data
print("\nProgress updated: ", time.ctime())
print('count = ', count)
status = "%d of %d files " % (i, total)
if count == 0:
self.m_dataViewListCtrlRunning.AppendItem([process, count, "Pending"])
self.start[process] = time.time()
elif count < 0:
self.m_dataViewListCtrlRunning.SetValue("ERROR in process - see log file", row=row, col=2)
self.m_btnRunProcess.Enable()
elif count < 100:
self.m_dataViewListCtrlRunning.SetValue(count, row=row, col=1)
self.m_dataViewListCtrlRunning.SetValue("Running " + status, row=row, col=2)
self.m_stOutputlog.SetLabelText("Running: %s ...please wait" % process)
else:
if process in self.start:
endtime = time.time() - self.start[process]
status = "%s (%d secs)" % (status, endtime)
print(status)
self.m_dataViewListCtrlRunning.SetValue(count, row=row, col=1)
self.m_dataViewListCtrlRunning.SetValue("Done " + status, row=row, col=2)
self.m_btnRunProcess.Enable()
self.m_stOutputlog.SetLabelText("Completed process %s" % process)
def getFilePanel(self):
"""
Get access to filepanel
:return:
"""
filepanel = None
for fp in self.Parent.Children:
if isinstance(fp, FileSelectPanel):
filepanel = fp
break
return filepanel
def OnCancelScripts(self, event):
"""
Find a way to stop processes
:param event:
:return:
"""
self.controller.shutdown()
print("Cancel multiprocessor")
event.Skip()
def OnRunScripts(self, event):
"""
Run selected scripts sequentially - updating progress bars
:param e:
:return:
"""
# Clear processing window
self.m_dataViewListCtrlRunning.DeleteAllItems()
# Disable Run button
# self.m_btnRunProcess.Disable()
btn = event.GetEventObject()
btn.Disable()
# Get selected processes
selections = self.m_checkListProcess.GetCheckedStrings()
print("Processes selected: ", len(selections))
showplots = self.m_cbShowplots.GetValue()
# Get data from other panels
filepanel = self.getFilePanel()
filenames = {'all': []}
groups = filepanel.m_cbGroups.GetItems()
for g in groups:
filenames[g]=[]
num_files = filepanel.m_dataViewListCtrl1.GetItemCount()
outputdir = filepanel.txtOutputdir.GetValue() # for batch processes
print('All Files:', num_files)
try:
self.db.getconn()
if len(selections) > 0 and num_files > 0:
# Get selected files and sort into groups
for i in range(0, num_files):
if filepanel.m_dataViewListCtrl1.GetToggleValue(i, 0):
fname = filepanel.m_dataViewListCtrl1.GetValue(i, 2)
group = filepanel.m_dataViewListCtrl1.GetValue(i, 1)
if not isdir(fname):
filenames['all'].append(fname)
if len(group) > 0:
filenames[group].append(fname)
print('Selected Files:', len(filenames['all']))
if len(filenames) <= 0:
raise ValueError("No files selected in Files Panel")
row = 0
# For each process
for pcaption in selections:
for p in self.controller.processes.keys():
if self.controller.processes[p]['caption']==pcaption:
break
print("processname =", p)
self.controller.RunProcess(self, p, outputdir,filenames, row, showplots)
row = row + 1
else:
if len(selections) <= 0:
raise ValueError("No processes selected")
else:
raise ValueError("No files selected - please go to Files Panel and add to list")
except ValueError as e:
self.Parent.Warn(e.args[0])
# Enable Run button
self.m_btnRunProcess.Enable()
finally:
if self.db is not None:
self.db.closeconn()
def OnShowLog(self, event):
"""
Load logfile into viewer
:param event:
:return:
"""
dlg = dlgLogViewer(self)
logfile = self.controller.logfile
dlg.tcLog.LoadFile(logfile)
dlg.ShowModal()
dlg.Destroy()
def OnClearWindow(self, event):
self.m_dataViewListCtrlRunning.DeleteAllItems()
class ProcessThread(threading.Thread):
"""Multi Worker Thread Class."""
# ----------------------------------------------------------------------
def __init__(self, controller, wxObject, modules, outputdir, filenames,
row, processname, showplots):
"""Init Worker Thread Class."""
threading.Thread.__init__(self)
self.controller = controller
self.config = self.controller.db.getConfig(
self.controller.currentconfig)
self.db = DBI(self.controller.configfile)
self.wxObject = wxObject
self.filenames = filenames
self.output = outputdir
self.row = row
self.showplots = showplots
self.processname = processname
(self.module_name, self.class_name) = modules
logger = logging.getLogger(processname)
# self.start() # start the thread
# ----------------------------------------------------------------------
def run(self):
i = 0
total_files = 0
try:
event.set()
lock.acquire(True)
q = dict()
# connect to db
self.db.getconn()
if isinstance(self.filenames, dict):
batch = True
total_files = len(self.filenames) - 1
i = 0
for group in self.filenames.keys():
if group == 'all' or len(self.filenames[group]) <= 0:
continue
count = (i / total_files) * 100
msg = "PROCESS THREAD (batch):%s run: count=%d of %d (%d percent)" % (
self.processname, i, total_files, count)
print(msg)
logger.info(msg)
#TODO fix this
wx.PostEvent(
self.wxObject,
ResultEvent((count, self.row, i + 1, total_files,
self.processname)))
self.processBatch(self.filenames[group], q, group)
i += 1
else:
batch = False
files = self.filenames
total_files = len(files)
for i in range(total_files):
count = (i + 1 / total_files) * 100
msg = "PROCESS THREAD: %s run: count=%d of %d (%d percent)" % (
self.processname, i + 1, total_files, count)
print(msg)
logger.info(msg)
wx.PostEvent(
self.wxObject,
ResultEvent((count, self.row, i + 1, total_files,
self.processname)))
self.processData(files[i], q)
wx.PostEvent(
self.wxObject,
ResultEvent((100, self.row, total_files, total_files,
self.processname)))
except Exception as e:
wx.PostEvent(
self.wxObject,
ResultEvent(
(-1, self.row, i + 1, total_files, self.processname)))
logging.error(e)
finally:
logger.info('Finished ProcessThread')
# self.terminate()
lock.release()
event.clear()
# connect to db
self.db.closeconn()
# ----------------------------------------------------------------------
def processData(self, filename, q):
"""
Run module here - can modify according to class if needed
:param filename: data file to process
:param q: queue for results
:return:
"""
logger.info("Process Data with file: %s", filename)
# create local subdir for output
if self.output == 'local':
inputdir = dirname(filename)
if 'processed' in inputdir:
outputdir = inputdir
else:
outputdir = join(inputdir, 'processed')
if not exists(outputdir):
mkdir(outputdir)
else:
outputdir = self.output
# Instantiate module
module = importlib.import_module(self.module_name)
class_ = getattr(module, self.class_name)
mod = class_(filename, outputdir, showplots=self.showplots)
# Load all params required for module - get list from module
cfg = mod.getConfigurables()
for c in cfg.keys():
cfg[c] = self.db.getConfigByName(self.controller.currentconfig, c)
msg = "Process Data: config set: %s=%s" % (c, str(cfg[c]))
print(msg)
logger.debug(msg)
mod.setConfigurables(cfg)
if mod.data is not None:
q[filename] = mod.run()
else:
q[filename] = None
def processBatch(self, filelist, q, group=None):
"""
Run module here - can modify according to class if needed
:param filename: data file to process
:param q: queue for results
:return:
"""
logger.info("Process Batch with filelist: %d", len(filelist))
outputdir = self.output
# Instantiate module
module = importlib.import_module(self.module_name)
class_ = getattr(module, self.class_name)
mod = class_(filelist, outputdir, showplots=self.showplots)
# Load all params required for module - get list from module
cfg = mod.getConfigurables()
for c in cfg.keys():
cfg[c] = self.db.getConfigByName(self.controller.currentconfig, c)
msg = "Process Batch: config set: %s=%s" % (c, str(cfg[c]))
logger.debug(msg)
mod.setConfigurables(cfg)
if group is not None:
mod.prefix = group
q[group] = mod.run()
else:
q[mod.base] = mod.run()
# ----------------------------------------------------------------------
def terminate(self):
logger.info("Terminating Filter Thread")
self.terminate()
class Config(ConfigPanel):
def __init__(self, parent):
super(Config, self).__init__(parent)
self.parent = parent
self.configdb = join(self.parent.resourcesdir, 'autoconfig.db')
self.dbi = DBI(self.configdb)
self.loadController()
def loadController(self):
self.controller = self.parent.controller
self.currentconfig = self.parent.controller.currentconfig
self.OnLoadData()
def OnLoadData(self):
self.dbi.getconn()
#load config values
cids = self.dbi.getConfigIds()
if len(cids)>0:
self.cboConfigid.Set(cids)
#configid = self.cboConfigid.GetStringSelection()
rownum =0
if self.currentconfig in cids:
selection = self.cboConfigid.FindString(self.currentconfig)
self.cboConfigid.SetSelection(selection)
conf = self.dbi.getConfig(self.currentconfig)
else:
selection = self.cboConfigid.FindString(cids[0])
self.cboConfigid.SetSelection(selection)
conf = self.dbi.getConfig(cids[0])
if conf is not None:
for k in conf.keys():
self.m_grid1.SetCellValue(rownum, 0, k)
self.m_grid1.SetCellValue(rownum, 1, conf[k])
rownum += 1
self.m_grid1.AutoSizeColumns()
self.m_grid1.AutoSize()
self.dbi.closeconn()
def OnLoadConfig(self,event):
self.dbi.getconn()
#load config values
configid = self.cboConfigid.GetValue()
try:
rownum =0
conf = self.dbi.getConfig(configid)
if conf is not None:
self.m_grid1.ClearGrid()
for k in conf.keys():
self.m_grid1.SetCellValue(rownum, 0, k)
self.m_grid1.SetCellValue(rownum, 1, conf[k])
rownum += 1
self.m_grid1.AutoSizeColumns()
self.m_grid1.AutoSize()
#Notify other controllers
self.parent.controller.currentconfig = configid
# reload other panels
for fp in self.parent.Children:
if isinstance(fp, wx.Panel) and self.__class__ != fp.__class__:
fp.loadController()
# notification
msg = "Config loaded: %s" % configid
self.Parent.Warn(msg)
except Exception as e:
self.Parent.Warn(e.args[0])
finally:
self.dbi.closeconn()
def OnSaveConfig(self, event):
self.dbi.getconn()
configid = self.cboConfigid.GetValue()
configlist=[]
data = self.m_grid1.GetTable()
for rownum in range(0, data.GetRowsCount()):
if not data.IsEmptyCell(rownum, 0):
configlist.append((self.m_grid1.GetCellValue(rownum, 0),self.m_grid1.GetCellValue(rownum, 1),configid))
#print('Saved config:', configlist)
# Save to DB
cnt = self.dbi.addConfig(configid,configlist)
# Load as options
cids = self.dbi.getConfigIds()
if len(cids) > 0:
self.cboConfigid.Set(cids)
selection = self.cboConfigid.FindString(configid)
self.cboConfigid.SetSelection(selection)
# Load controller
self.parent.controller.currentconfig = configid
#reload other panels
for fp in self.parent.Children:
if isinstance(fp, wx.Panel) and self.__class__ != fp.__class__:
fp.loadController()
#notification
msg = "Config saved: %s" % configid
self.Parent.Warn(msg)
self.dbi.closeconn()
def OnAddRow(self, event):
self.m_grid1.AppendRows(1, True)
class TestController(unittest.TestCase):
def setUp(self):
self.resourcesdir = join('..', 'resources')
self.processfile = join(self.resourcesdir, 'processes_test.yaml')
self.configfile = join(self.resourcesdir, 'autoconfig_test.db')
self.dbi = DBI(self.configfile)
self.dbi.getconn()
self.currentconfig = 'test'
self.controller = Controller(self.configfile, self.currentconfig,
self.processfile)
# TEST DATA
self.datafile = "D:\\Data\\Csv\\input\\control\\Brain10_Image.csv"
self.outputdir = "D:\\Data\\Csv\\output"
self.testcolumn = 'Count_ColocalizedGAD_DAPI_Objects'
def tearDown(self):
self.dbi.closeconn()
def test_ProcessInfo(self):
self.assertNotEqual(self.controller, None)
for p in self.controller.processes:
print("Process: ", p, ":", self.controller.processes[p]['caption'])
self.assertEqual('1. Filter Data',
self.controller.processes['process1']['caption'])
def test_Modules(self):
self.assertNotEqual(self.controller, None)
for p in self.controller.cmodules.keys():
print(p)
print("Module:", self.controller.cmodules[p][0])
print("Class:", self.controller.cmodules[p][1])
self.assertEqual('AutoFilter',
self.controller.processes['process1']['classname'])
def test_ModuleClass(self):
testprocess = 'process1'
module_name = self.controller.processes[testprocess]['modulename']
class_name = self.controller.processes[testprocess]['classname']
module = importlib.import_module(module_name)
class_ = getattr(module, class_name)
mod = class_(self.datafile, self.outputdir)
self.assertTrue(isinstance(mod, class_)) #class instantiated
self.assertGreater(len(mod.data), 0) #data loaded
def test_RunModuleClass_Simple(self):
testprocess = 'process1'
module_name = self.controller.processes[testprocess]['modulename']
class_name = self.controller.processes[testprocess]['classname']
module = importlib.import_module(module_name)
class_ = getattr(module, class_name)
#Instantiate FilterClass
mod = class_(self.datafile,
self.outputdir,
sheet=0,
skiprows=0,
headers=0)
# set vars manually
mod.column = self.testcolumn
mod.outputallcolumns = True
mod.minlimit = 20
mod.maxlimit = 80
#Run process
mod.run()
def test_RunModuleClass_Complex(self):
testprocess = 'process1'
module_name = self.controller.processes[testprocess]['modulename']
class_name = self.controller.processes[testprocess]['classname']
module = importlib.import_module(module_name)
class_ = getattr(module, class_name)
#Instantiate FilterClass
mod = class_(self.datafile, self.outputdir)
cfg = mod.getConfigurables()
for c in cfg.keys():
print("config: ", c)
cfg[c] = self.dbi.getConfigByName(self.currentconfig, c)
print("config set: ", cfg[c])
mod.setConfigurables(cfg)
self.assertEqual(mod.column, self.testcolumn)
mod.run()
def test_runProcessThread(self):
testprocess = 'process1'
type = self.controller.processes[testprocess]['href']
self.assertEqual(type, 'filter')
output = self.controller.processes[testprocess]['output']
processname = self.controller.processes[testprocess]['caption']
module_name = self.controller.processes[testprocess]['modulename']
class_name = self.controller.processes[testprocess]['classname']
config = self.controller.db.getConfig(self.currentconfig)
# Run Thread
t = TestThread(self.controller, self.datafile, self.outputdir, output,
processname, module_name, class_name, config)
t.start()
print("Running Thread - loaded: ", type)