def __init__(self):
self.currentconfig = 'default' # maybe future multiple configs possible
# connect to db
self.db = DBI()
self.db.connect()
self.logfile = None
self.logger = self.loadLogger(self.db.userconfigdir)
self.threadlist=[]
self._stopevent = threading.Event()
def setUp(self):
self.resourcesdir = join('..', 'resources')
self.processfile = join(self.resourcesdir, 'processes_test.yaml')
self.configfile = join(self.resourcesdir, 'autoconfig_test.db')
self.dbi = DBI(self.configfile)
self.dbi.getconn()
self.currentconfig = 'test'
self.controller = Controller(self.configfile, self.currentconfig,
self.processfile)
# TEST DATA
self.datafile = "D:\\Data\\Csv\\input\\control\\Brain10_Image.csv"
self.outputdir = "D:\\Data\\Csv\\output"
self.testcolumn = 'Count_ColocalizedGAD_DAPI_Objects'
def __init__(self, wxObject, configid, processname, processmodule, processclass, outputdir, filename, row):
"""Init Worker Thread Class."""
threading.Thread.__init__(self)
self.configID = configid
self.db = DBI()
self.max_mem = float(self.db.getConfigByName(self.configID, 'MAX_MEMORY'))
if self.max_mem is None:
self.max_mem = 80
self.wxObject = wxObject
self.filename = filename
self.output = outputdir
# Use outputfile directory rather than filename for progress bar output
self.outfile = join(self.output, splitext(basename(self.filename))[0])
self.row = row
self.processname = processname
logger = logging.getLogger(processname)
# Instantiate module
module = importlib.import_module(processmodule)
self.class_ = getattr(module, processclass)
self.mod = self.class_(filename, outputdir)
class ProcessThread(threading.Thread):
"""Multi Worker Thread Class."""
# ----------------------------------------------------------------------
def __init__(self, wxObject, configid, processname, processmodule, processclass, outputdir, filename, row):
"""Init Worker Thread Class."""
threading.Thread.__init__(self)
self.configID = configid
self.db = DBI()
self.max_mem = float(self.db.getConfigByName(self.configID, 'MAX_MEMORY'))
if self.max_mem is None:
self.max_mem = 80
self.wxObject = wxObject
self.filename = filename
self.output = outputdir
# Use outputfile directory rather than filename for progress bar output
self.outfile = join(self.output, splitext(basename(self.filename))[0])
self.row = row
self.processname = processname
logger = logging.getLogger(processname)
# Instantiate module
module = importlib.import_module(processmodule)
self.class_ = getattr(module, processclass)
self.mod = self.class_(filename, outputdir)
def configure(self):
try:
# Load all configurable params required for module - get list from module
cfg = self.mod.getConfigurables()
self.db.connect()
for c in cfg.keys():
cfg[c] = self.db.getConfigByName(self.configID, c)
self.db.closeconn()
self.mod.setConfigurables(cfg)
except Exception as e:
raise ValueError("Configure:", str(e))
# ----------------------------------------------------------------------
def run(self):
"""
Run module in thread
:return:
"""
try:
# event.set()
# lock.acquire(True)
q = dict()
# Configure parameters
self.configure()
logging.info('Configuration set')
# Run thread processing
msg = 'Running: %s' % self.filename
print(msg)
logging.info(msg)
if self.mod.imgfile is not None:
q[self.filename] = self.mod.run()
else:
msg = "ERROR: No data loaded for process: %s file: %s" % (self.class_, self.filename)
print(msg)
logging.error(msg)
q[self.filename] = None
raise ValueError(msg)
msg = 'Complete: %s' % self.filename
print(msg)
logging.info(msg)
except Exception as e:
print(e)
logging.error(e)
wx.PostEvent(self.wxObject, ResultEvent((-1, self.row, self.processname, self.outfile,e.args[0])))
finally:
msg ='Finished ProcessThread'
print(msg)
logging.info(msg)
class Controller():
def __init__(self):
self.currentconfig = 'default' # maybe future multiple configs possible
# connect to db
self.db = DBI()
self.db.connect()
self.logfile = None
self.logger = self.loadLogger(self.db.userconfigdir)
self.threadlist=[]
self._stopevent = threading.Event()
def loadLogger(self, outputdir=None):
#### LoggingConfig
dbug=self.db.getConfigByName(self.currentconfig,'DEBUG')
if dbug is not None and dbug =='1':
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
# Check dirs exist and are writable
if outputdir is not None and access(outputdir, R_OK):
homedir = outputdir
else:
userdir = expanduser("~")
homedir = join(userdir, '.qbi_apps', 'batchcrop')
if not access(homedir, W_OK):
if not access(join(userdir, '.qbi_apps'), W_OK):
mkdir(join(userdir, '.qbi_apps'))
mkdir(homedir)
logdir = join(homedir, "logs")
if not access(logdir,W_OK):
mkdir(logdir)
self.logfile = join(logdir, 'batchcrop.log')
handler = RotatingFileHandler(filename=self.logfile, maxBytes=10000000, backupCount=10)
formatter = logging.Formatter(FORMAT)
handler.setFormatter(formatter)
logger.addHandler(handler)
return logger
def poll(self,t,row,processname,outfile,wxObject,max_mem):
# Poll thread to update progress bar
ctr = 0
increment = 5
tcount = 90
while t.is_alive():
time.sleep(increment)
# Check CPU usage
percent = psutil.virtual_memory().percent
msg = "Controller:RunProcess (t.alive): [%s] %s (%s)" % (
processname, outfile, "{0}% memory".format(percent))
print(msg)
logger.debug(msg)
if percent > max_mem:
msg = 'Low memory - stop processing: %d' % percent
logging.warning(msg)
print(msg)
self._stopevent.set()
raise OSError(msg)
ctr += increment
# reset ??
if ctr == tcount:
ctr = increment
# count, row, process, filename
wx.PostEvent(wxObject, ResultEvent((ctr, row, processname, outfile, "{0}% memory".format(percent))))
wx.YieldIfNeeded()
# ----------------------------------------------------------------------
def RunProcess(self, wxGui, processref, outputdir, filenames,prow):
"""
:param wxGui:
:param processref:
:param outputdir: either set by user or defaults to subdir in input dir (from App.py)
:param filenames:
:param row:
:return:
"""
self.db.connect()
processname = self.db.getCaption(processref)
processmodule = self.db.getProcessModule(processref)
processclass = self.db.getProcessClass(processref)
self.db.closeconn()
# Check users available RAM and CPU
msg = "CPU count: %d" % psutil.cpu_count(False)
print(msg)
logging.info(msg)
max_mem = float(self.db.getConfigByName(self.currentconfig,'MAX_MEMORY'))
if max_mem is None:
max_mem = 80
avail_mem = psutil.virtual_memory().available
swap = psutil.swap_memory().free
percent = psutil.virtual_memory().percent #used/total
msg = "MEMORY: \n\tAvailable: %d \t Swap: %d \t Percent: %d \n" % (avail_mem,swap,percent)
print(msg)
logging.info(msg)
row = prow
if len(filenames) > 0:
# Remove any previous stop
self._stopevent.clear()
# One thread per file
for filename in filenames:
if not self._stopevent.isSet():
msg = "Load Process Threads: %s %s [row: %d]" % (processname, filename, row)
print(msg)
logger.info(msg)
# Outputfile directory rather than filename for progress bar output
outfolder = join(outputdir, splitext(basename(filename))[0])
# Initial entry
wx.PostEvent(wxGui, ResultEvent((0, row, processname, outfolder,'')))
wx.YieldIfNeeded()
t = ProcessThread(wxGui, self.currentconfig, processname, processmodule, processclass, outputdir,
filename, row)
self.threadlist.append(t)
t.start()
self.poll(t,row,processname,outfolder,wxGui,max_mem)
wx.PostEvent(wxGui, ResultEvent((100, row, processname, outfolder,'')))
# New row
row += 1
else:
msg ='Stop event detected - ending all processing'
logging.warning(msg)
print(msg)
break
else:
logger.error("No files to process")
raise ValueError("No matched files to process")
return row
# ----------------------------------------------------------------------
def shutdown(self):
self._stopevent.set()
msg="Controller: stop event set"
logging.warning(msg)
print(msg)
# Loop through threads to finish?
for t in self.threadlist:
if t != threading.main_thread() and t.is_alive():
msg = 'Stop processing called: closing {0}'.format(t.getName())
logging.info(msg)
print(msg)
t.join(timeout=5)
class TestController(unittest.TestCase): #TODO: REWRITE FOR SLIDE CROPPER
def setUp(self):
self.resourcesdir = join('..', 'resources')
self.processfile = join(self.resourcesdir, 'processes_test.yaml')
self.configfile = join(self.resourcesdir, 'autoconfig_test.db')
self.dbi = DBI(self.configfile)
self.dbi.getconn()
self.currentconfig = 'test'
self.controller = Controller(self.configfile, self.currentconfig,
self.processfile)
# TEST DATA
self.datafile = "D:\\Data\\Csv\\input\\control\\Brain10_Image.csv"
self.outputdir = "D:\\Data\\Csv\\output"
self.testcolumn = 'Count_ColocalizedGAD_DAPI_Objects'
def tearDown(self):
self.dbi.closeconn()
def test_ProcessInfo(self):
self.assertNotEqual(self.controller, None)
for p in self.controller.processes:
print("Process: ", p, ":", self.controller.processes[p]['caption'])
self.assertEqual('1. Filter Data',
self.controller.processes['process1']['caption'])
def test_Modules(self):
self.assertNotEqual(self.controller, None)
for p in self.controller.cmodules.keys():
print(p)
print("Module:", self.controller.cmodules[p][0])
print("Class:", self.controller.cmodules[p][1])
self.assertEqual('AutoFilter',
self.controller.processes['process1']['classname'])
def test_ModuleClass(self):
testprocess = 'process1'
module_name = self.controller.processes[testprocess]['modulename']
class_name = self.controller.processes[testprocess]['classname']
module = importlib.import_module(module_name)
class_ = getattr(module, class_name)
mod = class_(self.datafile, self.outputdir)
self.assertTrue(isinstance(mod, class_)) #class instantiated
self.assertGreater(len(mod.data), 0) #data loaded
def test_RunModuleClass_Simple(self):
testprocess = 'process1'
module_name = self.controller.processes[testprocess]['modulename']
class_name = self.controller.processes[testprocess]['classname']
module = importlib.import_module(module_name)
class_ = getattr(module, class_name)
#Instantiate FilterClass
mod = class_(self.datafile,
self.outputdir,
sheet=0,
skiprows=0,
headers=0)
# set vars manually
mod.column = self.testcolumn
mod.outputallcolumns = True
mod.minlimit = 20
mod.maxlimit = 80
#Run process
mod.run()
def test_RunModuleClass_Complex(self):
testprocess = 'process1'
module_name = self.controller.processes[testprocess]['modulename']
class_name = self.controller.processes[testprocess]['classname']
module = importlib.import_module(module_name)
class_ = getattr(module, class_name)
#Instantiate FilterClass
mod = class_(self.datafile, self.outputdir)
cfg = mod.getConfigurables()
for c in cfg.keys():
print("config: ", c)
cfg[c] = self.dbi.getConfigByName(self.currentconfig, c)
print("config set: ", cfg[c])
mod.setConfigurables(cfg)
self.assertEqual(mod.column, self.testcolumn)
mod.run()
def test_runProcessThread(self):
testprocess = 'process1'
type = self.controller.processes[testprocess]['href']
self.assertEqual(type, 'filter')
output = self.controller.processes[testprocess]['output']
processname = self.controller.processes[testprocess]['caption']
module_name = self.controller.processes[testprocess]['modulename']
class_name = self.controller.processes[testprocess]['classname']
config = self.controller.db.getConfig(self.currentconfig)
# Run Thread
t = TestThread(self.controller, self.datafile, self.outputdir, output,
processname, module_name, class_name, config)
t.start()
print("Running Thread - loaded: ", type)
def setUp(self):
self.dbi = DBI(test=True)
self.dbi.getconn()
class TestDBquery(unittest.TestCase):
def setUp(self):
self.dbi = DBI(test=True)
self.dbi.getconn()
def tearDown(self):
self.dbi.conn.close()
def test_getConfig(self):
configid = 'test'
data = self.dbi.getConfig(configid)
expected = 0
self.assertGreater(len(data),expected)
def test_getConfigALL(self):
configid = 'test'
data = self.dbi.getConfigALL(configid)
expected = 0
self.assertGreater(len(data),expected)
def test_getConfigByName(self):
group = 'test'
test = 'BINWIDTH'
expected = 20
data = self.dbi.getConfigByName(group,test)
self.assertEqual(int(data),expected)
def test_getConfigByName_None(self):
group = 'test'
test = 'BINW'
expected = None
data = self.dbi.getConfigByName(group,test)
self.assertEqual(data,expected)
def test_getConfigIds(self):
data = self.dbi.getConfigIds()
print('IDS:', data)
self.assertGreater(len(data),0)
def test_updateConfig(self):
configid='test'
configlist = [('BINWIDTH',10,'test',"Bin width for histograms"),
('COLUMN','TestData','test',"Column name for histograms"),
('MINRANGE',0,'test',"Minimum range of histograms bins"),
('MAXRANGE',100,'test',"Maximum range of histograms bins")]
cnt = self.dbi.addConfig(configid,configlist)
expected = len(configlist)
self.assertEqual(expected,cnt)
def test_updateConfig_Secondset(self):
configid = 'test'
configlist = [('BINWIDTH', 10, configid,"Bin width for histograms"),
('COLUMN', 'TestData', configid,"Column name for histograms"),
('MINRANGE', 0, configid,"Minimum range of histograms bins"),
('MAXRANGE', 100, configid,"Maximum range of histograms bins")]
cnt = self.dbi.addConfig(configid, configlist)
expected = len(configlist)
self.assertEqual(expected, cnt)
def test_updateConfig_duplicate(self):
configid='test'
configlist = [('BINWIDTH', 20, 'test', "Bin width for histograms"),
('COLUMN', 'TestData2', 'test', "Column name for histograms"),
('MINRANGE', 5, 'test', "Minimum range of histograms bins"),
('MAXRANGE', 50, 'test', "Maximum range of histograms bins")]
cnt = self.dbi.addConfig(configid,configlist)
expected = len(configlist)
self.assertEqual(expected,cnt)
def test_getCaptions(self):
data = self.dbi.getCaptions()
expected = 0
print('Captions: ', data)
self.assertIsNotNone(data)
self.assertGreater(len(data), expected)
def test_getRefs(self):
data = self.dbi.getRefs()
expected = 0
print('Refs: ', data)
self.assertIsNotNone(data)
self.assertGreater(len(data), expected)
def test_getDescription(self):
ref = 'Crop Slide into Images'
data = self.dbi.getDescription(ref)
expected = 0
print('Description: ', data)
self.assertIsNotNone(data)
self.assertGreater(len(data), expected)
def test_getCaption(self):
ref = 'crop'
data = self.dbi.getCaption(ref)
expected = 0
print('Caption: ', data)
self.assertIsNotNone(data)
self.assertGreater(len(data), expected)
def test_getProcessModule(self):
ref = 'crop'
data = self.dbi.getProcessModule(ref)
expected = 0
print('Module: ', data)
self.assertIsNotNone(data)
self.assertGreater(len(data), expected)
def test_getProcessClass(self):
ref = 'crop'
data = self.dbi.getProcessClass(ref)
expected = 0
print('Class: ', data)
self.assertIsNotNone(data)
self.assertGreater(len(data), expected)