class Processing: def __init__(self): self.DBClient = DbUtils() self.sqlBuilder = SQLBuilder() def getObjectFromTuple(self, tuple): valuesDict = dict(record_id=tuple[0], study=tuple[1], rid=tuple[2], modality=tuple[3], scan_date=tuple[4].strftime("%Y-%m-%d"), scan_time=str(tuple[5]), s_identifier=tuple[6], i_identifier=tuple[7], root_folder=tuple[8], converted_folder=tuple[9], version=tuple[10], processed=tuple[12]) return ProcessingObject(valuesDict) def insertToTable(self, objList): for obj in objList: self.DBClient.executeNoResult( self.sqlBuilder.getSQL_AddNewEntryToProcessingTable(obj.sqlInsert())) def insertFromConvertionObj(self, convertionObj): convertionValues = convertionObj.getValuesDict() convertionValues['modality'] = sc.ProcessingModalityAndPipelineTypePerStudy[convertionObj.study][convertionObj.scan_type] convertionValues['root_folder'] = '/'.join(convertionObj.converted_folder.split('/')[0:-2]) # Keeping only the three last elements self.insertToTable([ProcessingObject(convertionValues)]) def getToProcessListPerStudy(self, study): toProcessList = self.DBClient.executeAllResults( self.sqlBuilder.getSQL_getToBeProcessedFromProcessingTable(study)) return [self.getObjectFromTuple(t) for t in toProcessList]
class Sorting: def __init__(self): self.tableName = 'Sorting' self.DBClient = DbUtils() self.sqlBuilder = SQLBuilder() def getObjectFromTuple(self, tuple): valuesDict = dict(record_id=tuple[0], study=tuple[1], rid=tuple[2], scan_type=tuple[3], scan_date=tuple[4].strftime("%Y-%m-%d"), scan_time=str(tuple[5]), s_identifier=tuple[6], i_identifier=tuple[7], file_type=tuple[8], download_folder=tuple[9], raw_folder=tuple[10], moved=tuple[11]) return SortingObject(valuesDict) def insertToTable(self, objList): for obj in objList: self.DBClient.executeNoResult( self.sqlBuilder.getSQL_AddNewEntryToSortingTable(obj.sqlInsert())) def getUnmovedFilesPerStudy(self, study): unmovedList = self.DBClient.executeAllResults( self.sqlBuilder.getSQL_getUnmovedFilesFromSortingTable(study, tuple(sc.ProcessingModalityAndPipelineTypePerStudy[study].keys()))) return [self.getObjectFromTuple(t) for t in unmovedList] def setMovedTrue(self, sortingObj): sortingObj.moved = 1 self.saveObj(sortingObj) def saveObj(self, sortingObj): self.DBClient.executeNoResult(self.sqlBuilder.getSQL_saveObjSortingTable(sortingObj))
class Conversion: def __init__(self): self.tableName = 'Conversion' self.DBClient = DbUtils() self.sqlBuilder = SQLBuilder() def getObjectFromTuple(self, tuple): valuesDict = dict(record_id=tuple[0], study=tuple[1], rid=tuple[2], scan_type=tuple[3], scan_date=tuple[4].strftime("%Y-%m-%d"), scan_time=str(tuple[5]), s_identifier=tuple[6], i_identifier=tuple[7], file_type=tuple[8], raw_folder=tuple[9], converted_folder=tuple[10], version=tuple[11], converted=tuple[12]) return ConversionObject(valuesDict) def insertToTable(self, objList): for obj in objList: self.DBClient.executeNoResult( self.sqlBuilder.getSQL_AddNewEntryToConversionTable(obj.sqlInsert())) def get_version(self, sortingObj, versionDict): if sortingObj.study == 'ADNI': dl_path = sortingObj.download_folder if 'Uniform' in dl_path: return 'V2' else: return versionDict[sc.ProcessingModalityAndPipelineTypePerStudy[sortingObj.study][sortingObj.scan_type]] if sc.ProcessingModalityAndPipelineTypePerStudy[sortingObj.study][sortingObj.scan_type] in versionDict else 'V1' else: return versionDict[sc.ProcessingModalityAndPipelineTypePerStudy[sortingObj.study][sortingObj.scan_type]] if sc.ProcessingModalityAndPipelineTypePerStudy[sortingObj.study][sortingObj.scan_type] in versionDict else 'V1' def insertFromSortingObj(self, sortingObj, versionDict): sortingValues = sortingObj.getValuesDict() version = self.get_version(sortingObj, versionDict) sortingValues['converted_folder'] = '{0}/{1}/{2}/{3}/{4}_{5}_{6}/{7}/converted/final'.format(sc.studyDatabaseRootDict[sortingObj.study], sortingObj.study, sortingObj.scan_type, sortingObj.rid, sortingObj.scan_date, sortingObj.s_identifier, sortingObj.i_identifier, version) sortingValues['version'] = version sortingValues['converted'] = 0 self.insertToTable([ConversionObject(sortingValues)]) def gettoBeConvertedPerStudy(self, study): toConvertList = self.DBClient.executeAllResults( self.sqlBuilder.getSQL_getToBeConvertedFileFromConversionTable(study)) return [self.getObjectFromTuple(t) for t in toConvertList] def setConvertedTrue(self, convertionObj): convertionObj.converted = 1 self.saveObj(convertionObj) def setConvertedFailed(self, convertionObj): convertionObj.skip = 1 self.saveObj(convertionObj) def saveObj(self, convertionObj): self.DBClient.executeNoResult(self.sqlBuilder.getSQL_saveObjConversionTable(convertionObj)) def getConvertedListPerStudy(self, study): convertedList = self.DBClient.executeAllResults(self.sqlBuilder.getSQL_getAllConvertedFromConvertionTable(study)) return [self.getObjectFromTuple(t) for t in convertedList]
def __init__(self, studyList, version): self.DBClient = DbUtils() self.studyList = [i.upper() for i in studyList] self.version = version self.recursorList = [] self._getRecursorList(studyList) self.sortingDataList = [] self.sqlBuilder = SQLBuilder() self.moveSortingObjListDict = {} self.toConvertObjListDict = {} self.sortingTable = Sorting() self.conversionTable = Conversion() self.raw2mincConverter = Raw2MINCConverter() self.pool = Pool() self.qsubJobHandler = QSubJobHandler() self.qsubJobHandler.start() self.convertedListDict = {} self.processingTable = Processing() self.toProcessListDict = {} self.pipelineHanlder = PipelineHandler() self.QCHandler = QCHandler() self.MongoManger = MongoDBManager() self.MongoXMLManager = MongoScanXMLManager() self.MongoXMLManager.processXMLs()
class Processing: def __init__(self): self.DBClient = DbUtils() self.sqlBuilder = SQLBuilder() def getObjectFromTuple(self, tuple): valuesDict = dict(record_id=tuple[0], study=tuple[1], rid=tuple[2], modality=tuple[3], scan_date=tuple[4].strftime("%Y-%m-%d"), scan_time=str(tuple[5]), s_identifier=tuple[6], i_identifier=tuple[7], root_folder=tuple[8], converted_folder=tuple[9], version=tuple[10], processed=tuple[12]) return ProcessingObject(valuesDict) def insertToTable(self, objList): for obj in objList: self.DBClient.executeNoResult( self.sqlBuilder.getSQL_AddNewEntryToProcessingTable( obj.sqlInsert())) def insertFromConvertionObj(self, convertionObj): convertionValues = convertionObj.getValuesDict() convertionValues[ 'modality'] = sc.ProcessingModalityAndPipelineTypePerStudy[ convertionObj.study][convertionObj.scan_type] convertionValues['root_folder'] = '/'.join( convertionObj.converted_folder.split('/') [0:-2]) # Keeping only the three last elements self.insertToTable([ProcessingObject(convertionValues)]) def getToProcessListPerStudy(self, study): toProcessList = self.DBClient.executeAllResults( self.sqlBuilder.getSQL_getToBeProcessedFromProcessingTable(study)) return [self.getObjectFromTuple(t) for t in toProcessList]
def __init__(self): self.DBClient = DbUtils() self.sqlBuilder = SQLBuilder()
def __init__(self): self.tableName = 'Sorting' self.DBClient = DbUtils() self.sqlBuilder = SQLBuilder()
def __init__(self): self.tableName = 'Conversion' self.DBClient = DbUtils() self.sqlBuilder = SQLBuilder()
class Conversion: def __init__(self): self.tableName = 'Conversion' self.DBClient = DbUtils() self.sqlBuilder = SQLBuilder() def getObjectFromTuple(self, tuple): valuesDict = dict(record_id=tuple[0], study=tuple[1], rid=tuple[2], scan_type=tuple[3], scan_date=tuple[4].strftime("%Y-%m-%d"), scan_time=str(tuple[5]), s_identifier=tuple[6], i_identifier=tuple[7], file_type=tuple[8], raw_folder=tuple[9], converted_folder=tuple[10], version=tuple[11], converted=tuple[12]) return ConversionObject(valuesDict) def insertToTable(self, objList): for obj in objList: self.DBClient.executeNoResult( self.sqlBuilder.getSQL_AddNewEntryToConversionTable( obj.sqlInsert())) def get_version(self, sortingObj, versionDict): if sortingObj.study == 'ADNI': dl_path = sortingObj.download_folder if 'Uniform' in dl_path: return 'V2' else: return versionDict[ sc.ProcessingModalityAndPipelineTypePerStudy[ sortingObj.study] [sortingObj. scan_type]] if sc.ProcessingModalityAndPipelineTypePerStudy[ sortingObj.study][ sortingObj.scan_type] in versionDict else 'V1' else: return versionDict[ sc.ProcessingModalityAndPipelineTypePerStudy[sortingObj.study] [sortingObj. scan_type]] if sc.ProcessingModalityAndPipelineTypePerStudy[ sortingObj.study][ sortingObj.scan_type] in versionDict else 'V1' def insertFromSortingObj(self, sortingObj, versionDict): sortingValues = sortingObj.getValuesDict() version = self.get_version(sortingObj, versionDict) sortingValues[ 'converted_folder'] = '{0}/{1}/{2}/{3}/{4}_{5}_{6}/{7}/converted/final'.format( sc.studyDatabaseRootDict[sortingObj.study], sortingObj.study, sortingObj.scan_type, sortingObj.rid, sortingObj.scan_date, sortingObj.s_identifier, sortingObj.i_identifier, version) sortingValues['version'] = version sortingValues['converted'] = 0 self.insertToTable([ConversionObject(sortingValues)]) def gettoBeConvertedPerStudy(self, study): toConvertList = self.DBClient.executeAllResults( self.sqlBuilder.getSQL_getToBeConvertedFileFromConversionTable( study)) return [self.getObjectFromTuple(t) for t in toConvertList] def setConvertedTrue(self, convertionObj): convertionObj.converted = 1 self.saveObj(convertionObj) def setConvertedFailed(self, convertionObj): convertionObj.skip = 1 self.saveObj(convertionObj) def saveObj(self, convertionObj): self.DBClient.executeNoResult( self.sqlBuilder.getSQL_saveObjConversionTable(convertionObj)) def getConvertedListPerStudy(self, study): convertedList = self.DBClient.executeAllResults( self.sqlBuilder.getSQL_getAllConvertedFromConvertionTable(study)) return [self.getObjectFromTuple(t) for t in convertedList]