class Controller(): def __init__(self): self.logger = self.__loadLogger() self.db = DBI() if self.db.c is None: self.db.connect() def __loadLogger(self, outputdir=None): #### LoggingConfig logger.setLevel(logging.INFO) homedir = expanduser("~") if outputdir is not None and access(outputdir, R_OK): homedir = outputdir if not access(join(homedir, "logs"), R_OK): mkdir(join(homedir, "logs")) self.logfile = join(homedir, "logs", 'd2c.log') handler = RotatingFileHandler(filename=self.logfile, maxBytes=10000000, backupCount=10) formatter = logging.Formatter( '[ %(asctime)s %(levelname)-4s ] %(filename)s %(lineno)d : (%(threadName)-9s) %(message)s' ) handler.setFormatter(formatter) logger.addHandler(handler) return logger # ---------------------------------------------------------------------- def RunProcess(self, wxGui, inputdir, uuid, processname, server, row): """ Run processing in a thread :param wxGui: Process Panel ref :param inputdir: Directory with DICOM files :param uuid: unique ID from series number :param processname: Process eg QSM :param server: Cloud server to use eg AWS :param row: Row number in progress table in Process Panel :return: """ # Get info from database processref = self.db.getRef(processname) # containername = self.db.getProcessField('container',processname) # mountdir = self.db.getProcessField('containerinputdir',processname) # output = join(self.db.getProcessField('containeroutputdir',processname),self.db.getProcessField('outputfile', processname)) filenames = self.db.getFiles(uuid) try: if len(filenames) > 0: msg = "Load Process Threads: %s [row: %d]" % (processname, row) print(msg) # Tar DICOM files to load to container for processing tarfilename = join(inputdir, uuid + '.tar') with tarfile.open(tarfilename, "w") as tar: for f in filenames: tar.add(f) tar.close() # Run thread t = ProcessThread(wxGui, processname, tarfilename, uuid, server, row) t.start() msg = "Running Thread: %s" % processname print(msg) # Load to database for remote monitoring self.db.setSeriesProcess(uuid, self.db.getProcessId(processname), server, 1, datetime.datetime.now(), inputdir) else: msg = "No files to process" logger.error(msg) raise ValueError(msg) except ValueError as e: raise e except Exception as e: raise e def removeInputFiles(self, uuid, inputdir): """ Remove temporary files in outputdir (assumes tar file created and uploaded to Docker) Remove file entries? :return: """ files = iglob(join(inputdir, '*.IMA')) for f in files: os.remove(f) # remove database entry - dicomdata and dicomfiles self.db.deleteSeriesData(uuid) def checkRemote(self): # Check if cloud processing is done and update database seriesprocesses = self.db.getActiveProcesses() for series in seriesprocesses: seriesid = series[0] server = series[2].lower() outputdir = series[6] if outputdir is None or len(outputdir) <= 0: files = self.db.getFiles(seriesid) outputdir = dirname(files[0]) # Get uploader class and query uploaderClass = get_class(server) uploader = uploaderClass(seriesid) if uploader.isDone(): downloadfile = join(outputdir, seriesid, 'download.tar') uploader.download(downloadfile) msg = 'Series: %s \n\tSTATUS: Complete (%s)\n' % (seriesid, downloadfile) print(msg) self.db.setSeriesProcessFinished(seriesid) # Remove files in database self.db.deleteSeriesData(seriesid) else: # Still in progress self.db.setSeriesProcessInprogress(seriesid) def parseDicom(self, wxObject, filelist): ''' Read DICOM headers for filelist and load series info to db :param filelist: :return: ''' t = DicomThread(wxObject, filelist) t.start()
class ProcessThread(threading.Thread): """Multi Worker Thread Class.""" # wxGui, processname, self.cmodules[processref], targetdir, uuid, server, filenames, row, containername # ---------------------------------------------------------------------- def __init__(self, wxObject, processname, tarfile, uuid, server, row): """Init Worker Thread Class.""" threading.Thread.__init__(self) self.wxObject = wxObject self.processname = processname self.tarfile = tarfile self.uuid = uuid self.server = server self.row = row self.db = DBI() self.db.connect() if self.db.conn is not None: # Dynamic process module pref = self.db.getProcessField('ref', processname) self.module_name = self.db.getProcessModule(pref) self.class_name = self.db.getProcessClass(pref) # Instantiate module module = importlib.import_module(self.module_name) class_ = getattr(module, self.class_name) # Docker details self.containername = self.db.getProcessField( 'container', processname) self.mountdir = self.db.getProcessField('containerinputdir', processname) self.outputfile = join( self.db.getProcessField('containeroutputdir', processname), self.db.getProcessField('outputfile', processname)) # Docker Class self.dcc = class_(self.containername, self.mountdir, self.outputfile) else: raise Exception('Cannot access Database') # ---------------------------------------------------------------------- def run(self): print('Starting thread run') try: event.set() lock.acquire(True) # Convert IMA files to MNC via Docker image (container, timeout) = self.dcc.startDocker(self.tarfile) if container is None: raise Exception("Unable to initialize Docker") ctr = 0 while (not self.dcc.checkIfDone(container, timeout)): time.sleep(1) wx.PostEvent( self.wxObject, ResultEvent((self.row, ctr, self.uuid, self.processname, 'Converting'))) ctr += 10 # Check that everything ran ok if not self.dcc.getStatus(container): raise Exception( "There was an error while anonomizing the dataset.") # Get the resulting mnc file back to the original directory self.dcc.finalizeJob(container, dirname(self.tarfile)) # Upload MNC to server mncfile = join(dirname(self.tarfile), self.uuid + '.mnc') if access(mncfile, R_OK): uploaderClass = get_class(self.server) uploader = uploaderClass(self.uuid) uploader.upload(mncfile, self.processname) wx.PostEvent( self.wxObject, ResultEvent((self.row, ctr, self.uuid, self.processname, 'Uploading'))) msg = "Uploaded file: %s to %s" % (mncfile, self.server) logging.info(msg) except Exception as e: print("ERROR:", e.args[0]) wx.PostEvent( self.wxObject, ResultEvent( (self.row, -1, self.uuid, self.processname, e.args[0]))) finally: wx.PostEvent( self.wxObject, ResultEvent( (self.row, 100, self.uuid, self.processname, 'Done'))) msg = 'Finished ProcessThread' logger.info(msg) print(msg) lock.release() event.clear()