def __init__(self, configfile, configID, processfile):
self.logger = self.loadLogger()
self.processfile = processfile
self.cmodules = self.loadProcesses()
self.configfile = configfile
self.currentconfig = configID #multiple configs possible
# connect to db
self.db = DBI(configfile)
self.db.getconn()
def setUp(self):
self.resourcesdir = join('..', 'resources')
self.processfile = join(self.resourcesdir, 'processes_test.yaml')
self.configfile = join(self.resourcesdir, 'autoconfig_test.db')
self.dbi = DBI(self.configfile)
self.dbi.getconn()
self.currentconfig = 'test'
self.controller = Controller(self.configfile, self.currentconfig,
self.processfile)
# TEST DATA
self.datafile = "D:\\Data\\Csv\\input\\control\\Brain10_Image.csv"
self.outputdir = "D:\\Data\\Csv\\output"
self.testcolumn = 'Count_ColocalizedGAD_DAPI_Objects'
def __init__(self, controller, wxObject, modules, outputdir, filenames, row, processname, showplots):
"""Init Worker Thread Class."""
threading.Thread.__init__(self)
self.controller = controller
self.config = self.controller.db.getConfig(self.controller.currentconfig)
self.db = DBI(self.controller.configfile)
self.wxObject = wxObject
self.filenames = filenames
self.output = outputdir
self.row = row
self.showplots = showplots
self.processname = processname
(self.module_name,self.class_name) = modules
logger = logging.getLogger(processname)
class TestDBquery(unittest.TestCase):
def setUp(self):
configdb = join('..','resources', 'autoconfig_test.db')
self.dbi = DBI(configdb)
self.dbi.getconn()
def tearDown(self):
self.dbi.conn.close()
def test_getConfig(self):
configid = 'general'
data = self.dbi.getConfig(configid)
expected = 0
self.assertGreater(len(data),expected)
def test_getConfigByName(self):
group = 'general'
test = 'BINWIDTH'
expected = 10
data = self.dbi.getConfigByName(group,test)
self.assertEqual(int(data),expected)
def test_getConfigByName_None(self):
group = 'general'
test = 'BINW'
expected = None
data = self.dbi.getConfigByName(group,test)
self.assertEqual(data,expected)
def test_getConfigIds(self):
data = self.dbi.getConfigIds()
print('IDS:', data)
self.assertGreater(len(data),0)
def test_updateConfig(self):
configid='general'
configlist = [('BINWIDTH',10,'general'),('COLUMN','TestData','general'),('MINRANGE',0,'general'),('MAXRANGE',100,'general')]
cnt = self.dbi.addConfig(configid,configlist)
expected = len(configlist)
self.assertEqual(expected,cnt)
def test_updateConfig_Secondset(self):
configid = 'test'
configlist = [('BINWIDTH', 10, configid), ('COLUMN', 'TestData', configid), ('MINRANGE', 0, configid),('MAXRANGE', 100, configid)]
cnt = self.dbi.addConfig(configid, configlist)
expected = len(configlist)
self.assertEqual(expected, cnt)
def test_updateConfig_duplicate(self):
configid='general'
configlist = [('BINWIDTH',20,'general'),('COLUMN','TestData2','general'),('MINRANGE',0,'general'),('MAXRANGE',100,'general')]
cnt = self.dbi.addConfig(configid,configlist)
expected = len(configlist)
self.assertEqual(expected,cnt)
def __init__(self, parent):
super(ProcessRunPanel, self).__init__(parent)
self.loadController()
self.db = DBI(self.controller.configfile)
self.db.getconn()
# self.controller = parent.controller
# Bind timer event
# self.Bind(wx.EVT_TIMER, self.progressfunc, self.controller.timer)
# processes = [p['caption'] for p in self.controller.processes]
# self.m_checkListProcess.AppendItems(processes)
# Set up event handler for any worker thread results
EVT_RESULT(self, self.progressfunc)
# EVT_CANCEL(self, self.stopfunc)
# Set timer handler
self.start = {}
class ProcessRunPanel(ProcessPanel):
def __init__(self, parent):
super(ProcessRunPanel, self).__init__(parent)
self.loadController()
self.db = DBI(self.controller.configfile)
self.db.getconn()
# self.controller = parent.controller
# Bind timer event
# self.Bind(wx.EVT_TIMER, self.progressfunc, self.controller.timer)
# processes = [p['caption'] for p in self.controller.processes]
# self.m_checkListProcess.AppendItems(processes)
# Set up event handler for any worker thread results
EVT_RESULT(self, self.progressfunc)
# EVT_CANCEL(self, self.stopfunc)
# Set timer handler
self.start = {}
def loadController(self):
self.controller = self.Parent.controller
processes = [self.controller.processes[p]['caption'] for p in self.controller.processes]
self.m_checkListProcess.Clear()
self.m_checkListProcess.AppendItems(processes)
def OnShowDescription(self, event):
print(event.String)
desc = [self.controller.processes[p]['description'] for p in self.controller.processes if
self.controller.processes[p]['caption'] == event.String]
filesIn = [self.controller.processes[p]['filesin'] for p in self.controller.processes if
self.controller.processes[p]['caption'] == event.String]
filesOut = [self.controller.processes[p]['filesout'] for p in self.controller.processes if
self.controller.processes[p]['caption'] == event.String]
# Load from Config
filesIn = [self.controller.db.getConfigByName(self.controller.currentconfig,f) for f in filesIn[0].split(", ")]
filesOut = [self.controller.db.getConfigByName(self.controller.currentconfig,f) for f in filesOut[0].split(", ")]
# Load to GUI
self.m_stTitle.SetLabelText(event.String)
self.m_stDescription.SetLabelText(desc[0])
self.m_stFilesin.SetLabelText(", ".join(filesIn))
self.m_stFilesout.SetLabelText(", ".join(filesOut))
self.Layout()
def progressfunc(self, msg):
"""
Update progress bars in table - multithreaded
:param count:
:param row:
:param col:
:return:
"""
(count, row, i, total, process) = msg.data
print("\nProgress updated: ", time.ctime())
print('count = ', count)
status = "%d of %d files " % (i, total)
if count == 0:
self.m_dataViewListCtrlRunning.AppendItem([process, count, "Pending"])
self.start[process] = time.time()
elif count < 0:
self.m_dataViewListCtrlRunning.SetValue("ERROR in process - see log file", row=row, col=2)
self.m_btnRunProcess.Enable()
elif count < 100:
self.m_dataViewListCtrlRunning.SetValue(count, row=row, col=1)
self.m_dataViewListCtrlRunning.SetValue("Running " + status, row=row, col=2)
self.m_stOutputlog.SetLabelText("Running: %s ...please wait" % process)
else:
if process in self.start:
endtime = time.time() - self.start[process]
status = "%s (%d secs)" % (status, endtime)
print(status)
self.m_dataViewListCtrlRunning.SetValue(count, row=row, col=1)
self.m_dataViewListCtrlRunning.SetValue("Done " + status, row=row, col=2)
self.m_btnRunProcess.Enable()
self.m_stOutputlog.SetLabelText("Completed process %s" % process)
def getFilePanel(self):
"""
Get access to filepanel
:return:
"""
filepanel = None
for fp in self.Parent.Children:
if isinstance(fp, FileSelectPanel):
filepanel = fp
break
return filepanel
def OnCancelScripts(self, event):
"""
Find a way to stop processes
:param event:
:return:
"""
self.controller.shutdown()
print("Cancel multiprocessor")
event.Skip()
def OnRunScripts(self, event):
"""
Run selected scripts sequentially - updating progress bars
:param e:
:return:
"""
# Clear processing window
self.m_dataViewListCtrlRunning.DeleteAllItems()
# Disable Run button
# self.m_btnRunProcess.Disable()
btn = event.GetEventObject()
btn.Disable()
# Get selected processes
selections = self.m_checkListProcess.GetCheckedStrings()
print("Processes selected: ", len(selections))
showplots = self.m_cbShowplots.GetValue()
# Get data from other panels
filepanel = self.getFilePanel()
filenames = {'all': []}
groups = filepanel.m_cbGroups.GetItems()
for g in groups:
filenames[g]=[]
num_files = filepanel.m_dataViewListCtrl1.GetItemCount()
outputdir = filepanel.txtOutputdir.GetValue() # for batch processes
print('All Files:', num_files)
try:
self.db.getconn()
if len(selections) > 0 and num_files > 0:
# Get selected files and sort into groups
for i in range(0, num_files):
if filepanel.m_dataViewListCtrl1.GetToggleValue(i, 0):
fname = filepanel.m_dataViewListCtrl1.GetValue(i, 2)
group = filepanel.m_dataViewListCtrl1.GetValue(i, 1)
if not isdir(fname):
filenames['all'].append(fname)
if len(group) > 0:
filenames[group].append(fname)
print('Selected Files:', len(filenames['all']))
if len(filenames) <= 0:
raise ValueError("No files selected in Files Panel")
row = 0
# For each process
for pcaption in selections:
for p in self.controller.processes.keys():
if self.controller.processes[p]['caption']==pcaption:
break
print("processname =", p)
self.controller.RunProcess(self, p, outputdir,filenames, row, showplots)
row = row + 1
else:
if len(selections) <= 0:
raise ValueError("No processes selected")
else:
raise ValueError("No files selected - please go to Files Panel and add to list")
except ValueError as e:
self.Parent.Warn(e.args[0])
# Enable Run button
self.m_btnRunProcess.Enable()
finally:
if self.db is not None:
self.db.closeconn()
def OnShowLog(self, event):
"""
Load logfile into viewer
:param event:
:return:
"""
dlg = dlgLogViewer(self)
logfile = self.controller.logfile
dlg.tcLog.LoadFile(logfile)
dlg.ShowModal()
dlg.Destroy()
def OnClearWindow(self, event):
self.m_dataViewListCtrlRunning.DeleteAllItems()
def __init__(self, parent):
super(Config, self).__init__(parent)
self.parent = parent
self.configdb = join(self.parent.resourcesdir, 'autoconfig.db')
self.dbi = DBI(self.configdb)
self.loadController()
class Config(ConfigPanel):
def __init__(self, parent):
super(Config, self).__init__(parent)
self.parent = parent
self.configdb = join(self.parent.resourcesdir, 'autoconfig.db')
self.dbi = DBI(self.configdb)
self.loadController()
def loadController(self):
self.controller = self.parent.controller
self.currentconfig = self.parent.controller.currentconfig
self.OnLoadData()
def OnLoadData(self):
self.dbi.getconn()
#load config values
cids = self.dbi.getConfigIds()
if len(cids)>0:
self.cboConfigid.Set(cids)
#configid = self.cboConfigid.GetStringSelection()
rownum =0
if self.currentconfig in cids:
selection = self.cboConfigid.FindString(self.currentconfig)
self.cboConfigid.SetSelection(selection)
conf = self.dbi.getConfig(self.currentconfig)
else:
selection = self.cboConfigid.FindString(cids[0])
self.cboConfigid.SetSelection(selection)
conf = self.dbi.getConfig(cids[0])
if conf is not None:
for k in conf.keys():
self.m_grid1.SetCellValue(rownum, 0, k)
self.m_grid1.SetCellValue(rownum, 1, conf[k])
rownum += 1
self.m_grid1.AutoSizeColumns()
self.m_grid1.AutoSize()
self.dbi.closeconn()
def OnLoadConfig(self,event):
self.dbi.getconn()
#load config values
configid = self.cboConfigid.GetValue()
try:
rownum =0
conf = self.dbi.getConfig(configid)
if conf is not None:
self.m_grid1.ClearGrid()
for k in conf.keys():
self.m_grid1.SetCellValue(rownum, 0, k)
self.m_grid1.SetCellValue(rownum, 1, conf[k])
rownum += 1
self.m_grid1.AutoSizeColumns()
self.m_grid1.AutoSize()
#Notify other controllers
self.parent.controller.currentconfig = configid
# reload other panels
for fp in self.parent.Children:
if isinstance(fp, wx.Panel) and self.__class__ != fp.__class__:
fp.loadController()
# notification
msg = "Config loaded: %s" % configid
self.Parent.Warn(msg)
except Exception as e:
self.Parent.Warn(e.args[0])
finally:
self.dbi.closeconn()
def OnSaveConfig(self, event):
self.dbi.getconn()
configid = self.cboConfigid.GetValue()
configlist=[]
data = self.m_grid1.GetTable()
for rownum in range(0, data.GetRowsCount()):
if not data.IsEmptyCell(rownum, 0):
configlist.append((self.m_grid1.GetCellValue(rownum, 0),self.m_grid1.GetCellValue(rownum, 1),configid))
#print('Saved config:', configlist)
# Save to DB
cnt = self.dbi.addConfig(configid,configlist)
# Load as options
cids = self.dbi.getConfigIds()
if len(cids) > 0:
self.cboConfigid.Set(cids)
selection = self.cboConfigid.FindString(configid)
self.cboConfigid.SetSelection(selection)
# Load controller
self.parent.controller.currentconfig = configid
#reload other panels
for fp in self.parent.Children:
if isinstance(fp, wx.Panel) and self.__class__ != fp.__class__:
fp.loadController()
#notification
msg = "Config saved: %s" % configid
self.Parent.Warn(msg)
self.dbi.closeconn()
def OnAddRow(self, event):
self.m_grid1.AppendRows(1, True)
def __init__(self, parent):
super(Config, self).__init__(parent)
self.parent = parent
self.configdb = self.parent.configfile
self.dbi = DBI(self.configdb)
self.loadController()
def setUp(self):
configdb = join('..', 'resources', 'autoconfig_test.db')
self.dbi = DBI(configdb)
self.dbi.getconn()
class TestDBquery(unittest.TestCase):
def setUp(self):
configdb = join('..', 'resources', 'autoconfig_test.db')
self.dbi = DBI(configdb)
self.dbi.getconn()
def tearDown(self):
self.dbi.conn.close()
def test_getConfig(self):
configid = 'test'
data = self.dbi.getConfig(configid)
expected = 0
self.assertGreater(len(data), expected)
def test_getConfigALL(self):
configid = 'test'
data = self.dbi.getConfigALL(configid)
expected = 0
self.assertGreater(len(data), expected)
def test_getConfigByName(self):
group = 'test'
test = 'BINWIDTH'
expected = 20
data = self.dbi.getConfigByName(group, test)
self.assertEqual(int(data), expected)
def test_getConfigByName_None(self):
group = 'test'
test = 'BINW'
expected = None
data = self.dbi.getConfigByName(group, test)
self.assertEqual(data, expected)
def test_getConfigIds(self):
data = self.dbi.getConfigIds()
print('IDS:', data)
self.assertGreater(len(data), 0)
def test_updateConfig(self):
configid = 'test'
configlist = [
('BINWIDTH', 10, 'test', "Bin width for histograms"),
('COLUMN', 'TestData', 'test', "Column name for histograms"),
('MINRANGE', 0, 'test', "Minimum range of histograms bins"),
('MAXRANGE', 100, 'test', "Maximum range of histograms bins")
]
cnt = self.dbi.addConfig(configid, configlist)
expected = len(configlist)
self.assertEqual(expected, cnt)
def test_updateConfig_Secondset(self):
configid = 'test'
configlist = [
('BINWIDTH', 10, configid, "Bin width for histograms"),
('COLUMN', 'TestData', configid, "Column name for histograms"),
('MINRANGE', 0, configid, "Minimum range of histograms bins"),
('MAXRANGE', 100, configid, "Maximum range of histograms bins")
]
cnt = self.dbi.addConfig(configid, configlist)
expected = len(configlist)
self.assertEqual(expected, cnt)
def test_updateConfig_duplicate(self):
configid = 'test'
configlist = [
('BINWIDTH', 20, 'test', "Bin width for histograms"),
('COLUMN', 'TestData2', 'test', "Column name for histograms"),
('MINRANGE', 5, 'test', "Minimum range of histograms bins"),
('MAXRANGE', 50, 'test', "Maximum range of histograms bins")
]
cnt = self.dbi.addConfig(configid, configlist)
expected = len(configlist)
self.assertEqual(expected, cnt)
class Controller():
def __init__(self, configfile, configID, processfile):
self.logger = self.loadLogger()
self.processfile = processfile
self.cmodules = self.loadProcesses()
self.configfile = configfile
self.currentconfig = configID #multiple configs possible
# connect to db
self.db = DBI(configfile)
self.db.getconn()
def loadProcesses(self):
pf = None
try:
pf = open(self.processfile, 'rb')
self.processes = yaml.load(pf)
cmodules = {}
for p in self.processes:
msg = "Controller:LoadProcessors: loading %s=%s" % (
p, self.processes[p]['caption'])
logging.debug(msg)
module_name = self.processes[p]['modulename']
class_name = self.processes[p]['classname']
cmodules[p] = (module_name, class_name)
return cmodules
except Exception as e:
raise e
finally:
if pf is not None:
pf.close()
def loadLogger(self, outputdir=None, expt=''):
#### LoggingConfig
logger.setLevel(logging.INFO)
homedir = expanduser("~")
if outputdir is not None and access(outputdir, R_OK):
homedir = outputdir
if len(expt) > 0:
expt = expt + "_"
if not access(join(homedir, "logs"), R_OK):
mkdir(join(homedir, "logs"))
self.logfile = join(homedir, "logs", expt + 'analysis.log')
handler = RotatingFileHandler(filename=self.logfile,
maxBytes=10000000,
backupCount=10)
formatter = logging.Formatter(
'[ %(asctime)s %(levelname)-4s ] %(filename)s %(lineno)d : (%(threadName)-9s) %(message)s'
)
handler.setFormatter(formatter)
logger.addHandler(handler)
return logger
# ----------------------------------------------------------------------
# def loadConfig(self, config=None):
# """
# Load from config file or config object
# :param config:
# :return:
# """
# try:
# if config is not None and isinstance(config, ConfigObj):
# logger.info("Loading config obj:%s", config.filename)
# elif isinstance(config, str) and access(config, R_OK):
# logger.info("Loading config from file:%s", config)
# config = ConfigObj(config)
# else:
# logger.warning('No config file found')
# config = ConfigObj()
#
# except IOError as e:
# logging.error(e)
# except ValueError as e:
# logging.error(e)
#
# return config
# ----------------------------------------------------------------------
def RunCompare(self, wxGui, indirs, outputdir, prefixes, searchtext):
"""
Comparison of groups
:param wxGui:
:param indirs:
:param outputdir:
:param prefixes:
:param searchtext:
:return:
"""
pass
# ----------------------------------------------------------------------
def RunProcess(self,
wxGui,
process,
outputdir,
filenames,
row,
showplots=False):
"""
Instantiate Thread with type for Process
:param wxGui:
:param filenames:
:param type:
:param row:
:return:
"""
type = self.processes[process]['href']
processname = self.processes[process]['caption']
filesIn = []
for f in self.processes[process]['filesin'].split(", "):
fin = self.db.getConfigByName(self.currentconfig, f)
if fin is not None:
filesIn.append(fin)
else:
filesIn.append(f)
filenames = CheckFilenames(filenames, filesIn)
# filesout = self.processes[process]['filesout'] #TODO link up with module config?
# suffix = self.db.getConfigByName(self.currentconfig,filesout)
if self.processes[process]['output'] == 'local':
outputdir = self.processes[process]['output']
filenames = filenames['all']
if len(filenames) > 0:
logger.info("Load Process Threads: %s [row: %d]", type, row)
wx.PostEvent(wxGui,
ResultEvent((0, row, 0, len(filenames), processname)))
t = ProcessThread(self, wxGui, self.cmodules[process], outputdir,
filenames, row, processname, showplots)
t.start()
logger.info("Running Thread: %s", type)
else:
logger.error("No files to process")
raise ValueError("No matched files to process")
# ----------------------------------------------------------------------
def shutdown(self):
logger.info('Close extra thread')
t = threading.current_thread()
#print("Thread counter:", threading.main_thread())
if t != threading.main_thread() and t.is_alive():
logger.info('Shutdown: closing %s', t.getName())
t.terminate()
class ProcessThread(threading.Thread):
"""Multi Worker Thread Class."""
# ----------------------------------------------------------------------
def __init__(self, controller, wxObject, modules, outputdir, filenames,
row, processname, showplots):
"""Init Worker Thread Class."""
threading.Thread.__init__(self)
self.controller = controller
self.config = self.controller.db.getConfig(
self.controller.currentconfig)
self.db = DBI(self.controller.configfile)
self.wxObject = wxObject
self.filenames = filenames
self.output = outputdir
self.row = row
self.showplots = showplots
self.processname = processname
(self.module_name, self.class_name) = modules
logger = logging.getLogger(processname)
# self.start() # start the thread
# ----------------------------------------------------------------------
def run(self):
i = 0
total_files = 0
try:
event.set()
lock.acquire(True)
q = dict()
# connect to db
self.db.getconn()
if isinstance(self.filenames, dict):
batch = True
total_files = len(self.filenames) - 1
i = 0
for group in self.filenames.keys():
if group == 'all' or len(self.filenames[group]) <= 0:
continue
count = (i / total_files) * 100
msg = "PROCESS THREAD (batch):%s run: count=%d of %d (%d percent)" % (
self.processname, i, total_files, count)
print(msg)
logger.info(msg)
#TODO fix this
wx.PostEvent(
self.wxObject,
ResultEvent((count, self.row, i + 1, total_files,
self.processname)))
self.processBatch(self.filenames[group], q, group)
i += 1
else:
batch = False
files = self.filenames
total_files = len(files)
for i in range(total_files):
count = (i + 1 / total_files) * 100
msg = "PROCESS THREAD: %s run: count=%d of %d (%d percent)" % (
self.processname, i + 1, total_files, count)
print(msg)
logger.info(msg)
wx.PostEvent(
self.wxObject,
ResultEvent((count, self.row, i + 1, total_files,
self.processname)))
self.processData(files[i], q)
wx.PostEvent(
self.wxObject,
ResultEvent((100, self.row, total_files, total_files,
self.processname)))
except Exception as e:
wx.PostEvent(
self.wxObject,
ResultEvent(
(-1, self.row, i + 1, total_files, self.processname)))
logging.error(e)
finally:
logger.info('Finished ProcessThread')
# self.terminate()
lock.release()
event.clear()
# connect to db
self.db.closeconn()
# ----------------------------------------------------------------------
def processData(self, filename, q):
"""
Run module here - can modify according to class if needed
:param filename: data file to process
:param q: queue for results
:return:
"""
logger.info("Process Data with file: %s", filename)
# create local subdir for output
if self.output == 'local':
inputdir = dirname(filename)
if 'processed' in inputdir:
outputdir = inputdir
else:
outputdir = join(inputdir, 'processed')
if not exists(outputdir):
mkdir(outputdir)
else:
outputdir = self.output
# Instantiate module
module = importlib.import_module(self.module_name)
class_ = getattr(module, self.class_name)
mod = class_(filename, outputdir, showplots=self.showplots)
# Load all params required for module - get list from module
cfg = mod.getConfigurables()
for c in cfg.keys():
cfg[c] = self.db.getConfigByName(self.controller.currentconfig, c)
msg = "Process Data: config set: %s=%s" % (c, str(cfg[c]))
print(msg)
logger.debug(msg)
mod.setConfigurables(cfg)
if mod.data is not None:
q[filename] = mod.run()
else:
q[filename] = None
def processBatch(self, filelist, q, group=None):
"""
Run module here - can modify according to class if needed
:param filename: data file to process
:param q: queue for results
:return:
"""
logger.info("Process Batch with filelist: %d", len(filelist))
outputdir = self.output
# Instantiate module
module = importlib.import_module(self.module_name)
class_ = getattr(module, self.class_name)
mod = class_(filelist, outputdir, showplots=self.showplots)
# Load all params required for module - get list from module
cfg = mod.getConfigurables()
for c in cfg.keys():
cfg[c] = self.db.getConfigByName(self.controller.currentconfig, c)
msg = "Process Batch: config set: %s=%s" % (c, str(cfg[c]))
logger.debug(msg)
mod.setConfigurables(cfg)
if group is not None:
mod.prefix = group
q[group] = mod.run()
else:
q[mod.base] = mod.run()
# ----------------------------------------------------------------------
def terminate(self):
logger.info("Terminating Filter Thread")
self.terminate()
class TestController(unittest.TestCase):
def setUp(self):
self.resourcesdir = join('..', 'resources')
self.processfile = join(self.resourcesdir, 'processes_test.yaml')
self.configfile = join(self.resourcesdir, 'autoconfig_test.db')
self.dbi = DBI(self.configfile)
self.dbi.getconn()
self.currentconfig = 'test'
self.controller = Controller(self.configfile, self.currentconfig,
self.processfile)
# TEST DATA
self.datafile = "D:\\Data\\Csv\\input\\control\\Brain10_Image.csv"
self.outputdir = "D:\\Data\\Csv\\output"
self.testcolumn = 'Count_ColocalizedGAD_DAPI_Objects'
def tearDown(self):
self.dbi.closeconn()
def test_ProcessInfo(self):
self.assertNotEqual(self.controller, None)
for p in self.controller.processes:
print("Process: ", p, ":", self.controller.processes[p]['caption'])
self.assertEqual('1. Filter Data',
self.controller.processes['process1']['caption'])
def test_Modules(self):
self.assertNotEqual(self.controller, None)
for p in self.controller.cmodules.keys():
print(p)
print("Module:", self.controller.cmodules[p][0])
print("Class:", self.controller.cmodules[p][1])
self.assertEqual('AutoFilter',
self.controller.processes['process1']['classname'])
def test_ModuleClass(self):
testprocess = 'process1'
module_name = self.controller.processes[testprocess]['modulename']
class_name = self.controller.processes[testprocess]['classname']
module = importlib.import_module(module_name)
class_ = getattr(module, class_name)
mod = class_(self.datafile, self.outputdir)
self.assertTrue(isinstance(mod, class_)) #class instantiated
self.assertGreater(len(mod.data), 0) #data loaded
def test_RunModuleClass_Simple(self):
testprocess = 'process1'
module_name = self.controller.processes[testprocess]['modulename']
class_name = self.controller.processes[testprocess]['classname']
module = importlib.import_module(module_name)
class_ = getattr(module, class_name)
#Instantiate FilterClass
mod = class_(self.datafile,
self.outputdir,
sheet=0,
skiprows=0,
headers=0)
# set vars manually
mod.column = self.testcolumn
mod.outputallcolumns = True
mod.minlimit = 20
mod.maxlimit = 80
#Run process
mod.run()
def test_RunModuleClass_Complex(self):
testprocess = 'process1'
module_name = self.controller.processes[testprocess]['modulename']
class_name = self.controller.processes[testprocess]['classname']
module = importlib.import_module(module_name)
class_ = getattr(module, class_name)
#Instantiate FilterClass
mod = class_(self.datafile, self.outputdir)
cfg = mod.getConfigurables()
for c in cfg.keys():
print("config: ", c)
cfg[c] = self.dbi.getConfigByName(self.currentconfig, c)
print("config set: ", cfg[c])
mod.setConfigurables(cfg)
self.assertEqual(mod.column, self.testcolumn)
mod.run()
def test_runProcessThread(self):
testprocess = 'process1'
type = self.controller.processes[testprocess]['href']
self.assertEqual(type, 'filter')
output = self.controller.processes[testprocess]['output']
processname = self.controller.processes[testprocess]['caption']
module_name = self.controller.processes[testprocess]['modulename']
class_name = self.controller.processes[testprocess]['classname']
config = self.controller.db.getConfig(self.currentconfig)
# Run Thread
t = TestThread(self.controller, self.datafile, self.outputdir, output,
processname, module_name, class_name, config)
t.start()
print("Running Thread - loaded: ", type)