def __init__(self, process=None):
self.client = docker.from_env()
db = DBI()
db.connect()
# DEFAULTS
self.CONTAINER_NAME = db.getServerConfigByName('DOCKER_CONTAINER')
self.INPUT_TARGET = db.getServerConfigByName('DOCKER_INPUTDIR')
self.OUTPUT_TARGET = db.getServerConfigByName('DOCKER_OUTPUTDIR')
self.OUTPUT = db.getServerConfigByName('DOCKER_OUTPUTFILE')
# Load specific process configs if set
self.process = process
if process is not None:
container = db.getServerConfigByName(
db.getProcessField('container', process))
if container is not None:
self.CONTAINER_NAME = container
input = db.getServerConfigByName(
db.getProcessField('containerinputdir', process))
if input is not None:
self.INPUT_TARGET = input
outputd = db.getServerConfigByName(
db.getProcessField('containeroutputdir', process))
if outputd is not None:
self.OUTPUT_TARGET = outputd
ofile = db.getServerConfigByName(
db.getProcessField('filename', process))
if ofile is not None:
self.OUTPUT = ofile
db.closeconn()
class ProcessThread(threading.Thread):
"""Multi Worker Thread Class."""
# wxGui, processname, self.cmodules[processref], targetdir, uuid, server, filenames, row, containername
# ----------------------------------------------------------------------
def __init__(self, wxObject, processname, inputdir, uuid, server, row):
"""Init Worker Thread Class."""
threading.Thread.__init__(self)
self.wxObject = wxObject
self.processname = processname
self.inputdir = inputdir
self.uuid = uuid
self.server = server
self.row = row
self.db = DBI()
self.db.connect()
if self.db.conn is not None:
# Dynamic process module
pref = self.db.getProcessField('ref', processname)
self.outputasfile = self.db.getProcessField(
'outputfile', processname) # file=1 or folder=0
self.module_name = self.db.getProcessModule(pref)
self.class_name = self.db.getProcessClass(pref)
# Instantiate module
module = importlib.import_module(self.module_name)
class_ = getattr(module, self.class_name)
# Docker Class
self.dcc = class_(self.processname)
# Record configs to log
if hasattr(self.dcc, 'CONTAINER_NAME'):
msg = "Running Container: %s [input=%s output=%s]" % (
self.dcc.CONTAINER_NAME, self.dcc.INPUT_TARGET,
join(self.dcc.OUTPUT_TARGET, self.dcc.OUTPUT))
print(msg)
logging.info(msg)
else:
raise Exception('Cannot access Database')
# ----------------------------------------------------------------------
def run(self):
print('Starting thread run')
msg = ''
ctr = 0
try:
event.set()
lock.acquire(True)
# Convert IMA files to MNC via Docker image
print('Running Docker image')
containerId = self.dcc.startDocker(join(self.inputdir, self.uuid))
if containerId is None:
raise Exception("ERROR: Unable to initialize Docker")
else:
print('Container ID:', containerId)
while (not self.dcc.checkIfDone(containerId)):
time.sleep(1)
wx.PostEvent(
self.wxObject,
ResultEvent((self.row, ctr, self.uuid, self.processname,
'Converting')))
ctr += 1
# restart for long running
if ctr == 100:
ctr = 1
# Check that everything ran ok (0 = success)
if self.dcc.getExitStatus(containerId):
raise Exception(
"ERROR: Docker unable to anonymize the dataset")
# Get the resulting mnc file back to the original directory
outputfile = self.dcc.finalizeJob(containerId, self.inputdir,
self.uuid, self.outputasfile)
print('Output:', outputfile)
if self.server.lower() != 'none':
msg = self.uploadCloud(outputfile)
else:
msg = 'Done: %s' % outputfile
ctr = 100
print(msg)
logger.info(msg)
except Exception as e:
msg = e.args[0]
print("ERROR:", msg)
ctr = -1
finally:
if lock.locked():
lock.release()
if event.is_set():
event.clear()
wx.PostEvent(
self.wxObject,
ResultEvent((self.row, ctr, self.uuid, self.processname, msg)))
def uploadCloud(self, mncfile):
"""
Send file to cloud for processing
:param mncfile:
:return:
"""
uploaderClass = get_class(self.server)
if uploaderClass is not None:
uploader = uploaderClass(self.uuid)
uploader.upload(mncfile, self.processname)
msg = 'Uploading to server[%s]: %s' % (self.server, mncfile)
else:
msg = 'No Uploader class available'
return msg