class Processing:
def __init__(self):
self.DBClient = DbUtils()
self.sqlBuilder = SQLBuilder()
def getObjectFromTuple(self, tuple):
valuesDict = dict(record_id=tuple[0], study=tuple[1], rid=tuple[2], modality=tuple[3],
scan_date=tuple[4].strftime("%Y-%m-%d"), scan_time=str(tuple[5]),
s_identifier=tuple[6], i_identifier=tuple[7], root_folder=tuple[8], converted_folder=tuple[9], version=tuple[10],
processed=tuple[12])
return ProcessingObject(valuesDict)
def insertToTable(self, objList):
for obj in objList:
self.DBClient.executeNoResult(
self.sqlBuilder.getSQL_AddNewEntryToProcessingTable(obj.sqlInsert()))
def insertFromConvertionObj(self, convertionObj):
convertionValues = convertionObj.getValuesDict()
convertionValues['modality'] = sc.ProcessingModalityAndPipelineTypePerStudy[convertionObj.study][convertionObj.scan_type]
convertionValues['root_folder'] = '/'.join(convertionObj.converted_folder.split('/')[0:-2]) # Keeping only the three last elements
self.insertToTable([ProcessingObject(convertionValues)])
def getToProcessListPerStudy(self, study):
toProcessList = self.DBClient.executeAllResults(
self.sqlBuilder.getSQL_getToBeProcessedFromProcessingTable(study))
return [self.getObjectFromTuple(t) for t in toProcessList]
class Sorting:
def __init__(self):
self.tableName = 'Sorting'
self.DBClient = DbUtils()
self.sqlBuilder = SQLBuilder()
def getObjectFromTuple(self, tuple):
valuesDict = dict(record_id=tuple[0], study=tuple[1], rid=tuple[2], scan_type=tuple[3],
scan_date=tuple[4].strftime("%Y-%m-%d"), scan_time=str(tuple[5]),
s_identifier=tuple[6], i_identifier=tuple[7], file_type=tuple[8], download_folder=tuple[9],
raw_folder=tuple[10], moved=tuple[11])
return SortingObject(valuesDict)
def insertToTable(self, objList):
for obj in objList:
self.DBClient.executeNoResult(
self.sqlBuilder.getSQL_AddNewEntryToSortingTable(obj.sqlInsert()))
def getUnmovedFilesPerStudy(self, study):
unmovedList = self.DBClient.executeAllResults(
self.sqlBuilder.getSQL_getUnmovedFilesFromSortingTable(study, tuple(sc.ProcessingModalityAndPipelineTypePerStudy[study].keys())))
return [self.getObjectFromTuple(t) for t in unmovedList]
def setMovedTrue(self, sortingObj):
sortingObj.moved = 1
self.saveObj(sortingObj)
def saveObj(self, sortingObj):
self.DBClient.executeNoResult(self.sqlBuilder.getSQL_saveObjSortingTable(sortingObj))
class Conversion:
def __init__(self):
self.tableName = 'Conversion'
self.DBClient = DbUtils()
self.sqlBuilder = SQLBuilder()
def getObjectFromTuple(self, tuple):
valuesDict = dict(record_id=tuple[0], study=tuple[1], rid=tuple[2], scan_type=tuple[3],
scan_date=tuple[4].strftime("%Y-%m-%d"), scan_time=str(tuple[5]),
s_identifier=tuple[6], i_identifier=tuple[7], file_type=tuple[8], raw_folder=tuple[9],
converted_folder=tuple[10], version=tuple[11], converted=tuple[12])
return ConversionObject(valuesDict)
def insertToTable(self, objList):
for obj in objList:
self.DBClient.executeNoResult(
self.sqlBuilder.getSQL_AddNewEntryToConversionTable(obj.sqlInsert()))
def get_version(self, sortingObj, versionDict):
if sortingObj.study == 'ADNI':
dl_path = sortingObj.download_folder
if 'Uniform' in dl_path:
return 'V2'
else:
return versionDict[sc.ProcessingModalityAndPipelineTypePerStudy[sortingObj.study][sortingObj.scan_type]] if sc.ProcessingModalityAndPipelineTypePerStudy[sortingObj.study][sortingObj.scan_type] in versionDict else 'V1'
else:
return versionDict[sc.ProcessingModalityAndPipelineTypePerStudy[sortingObj.study][sortingObj.scan_type]] if sc.ProcessingModalityAndPipelineTypePerStudy[sortingObj.study][sortingObj.scan_type] in versionDict else 'V1'
def insertFromSortingObj(self, sortingObj, versionDict):
sortingValues = sortingObj.getValuesDict()
version = self.get_version(sortingObj, versionDict)
sortingValues['converted_folder'] = '{0}/{1}/{2}/{3}/{4}_{5}_{6}/{7}/converted/final'.format(sc.studyDatabaseRootDict[sortingObj.study],
sortingObj.study, sortingObj.scan_type, sortingObj.rid,
sortingObj.scan_date, sortingObj.s_identifier, sortingObj.i_identifier, version)
sortingValues['version'] = version
sortingValues['converted'] = 0
self.insertToTable([ConversionObject(sortingValues)])
def gettoBeConvertedPerStudy(self, study):
toConvertList = self.DBClient.executeAllResults(
self.sqlBuilder.getSQL_getToBeConvertedFileFromConversionTable(study))
return [self.getObjectFromTuple(t) for t in toConvertList]
def setConvertedTrue(self, convertionObj):
convertionObj.converted = 1
self.saveObj(convertionObj)
def setConvertedFailed(self, convertionObj):
convertionObj.skip = 1
self.saveObj(convertionObj)
def saveObj(self, convertionObj):
self.DBClient.executeNoResult(self.sqlBuilder.getSQL_saveObjConversionTable(convertionObj))
def getConvertedListPerStudy(self, study):
convertedList = self.DBClient.executeAllResults(self.sqlBuilder.getSQL_getAllConvertedFromConvertionTable(study))
return [self.getObjectFromTuple(t) for t in convertedList]
def __init__(self, studyList, version):
self.DBClient = DbUtils()
self.studyList = [i.upper() for i in studyList]
self.version = version
self.recursorList = []
self._getRecursorList(studyList)
self.sortingDataList = []
self.sqlBuilder = SQLBuilder()
self.moveSortingObjListDict = {}
self.toConvertObjListDict = {}
self.sortingTable = Sorting()
self.conversionTable = Conversion()
self.raw2mincConverter = Raw2MINCConverter()
self.pool = Pool()
self.qsubJobHandler = QSubJobHandler()
self.qsubJobHandler.start()
self.convertedListDict = {}
self.processingTable = Processing()
self.toProcessListDict = {}
self.pipelineHanlder = PipelineHandler()
self.QCHandler = QCHandler()
self.MongoManger = MongoDBManager()
self.MongoXMLManager = MongoScanXMLManager()
self.MongoXMLManager.processXMLs()
class Processing:
def __init__(self):
self.DBClient = DbUtils()
self.sqlBuilder = SQLBuilder()
def getObjectFromTuple(self, tuple):
valuesDict = dict(record_id=tuple[0],
study=tuple[1],
rid=tuple[2],
modality=tuple[3],
scan_date=tuple[4].strftime("%Y-%m-%d"),
scan_time=str(tuple[5]),
s_identifier=tuple[6],
i_identifier=tuple[7],
root_folder=tuple[8],
converted_folder=tuple[9],
version=tuple[10],
processed=tuple[12])
return ProcessingObject(valuesDict)
def insertToTable(self, objList):
for obj in objList:
self.DBClient.executeNoResult(
self.sqlBuilder.getSQL_AddNewEntryToProcessingTable(
obj.sqlInsert()))
def insertFromConvertionObj(self, convertionObj):
convertionValues = convertionObj.getValuesDict()
convertionValues[
'modality'] = sc.ProcessingModalityAndPipelineTypePerStudy[
convertionObj.study][convertionObj.scan_type]
convertionValues['root_folder'] = '/'.join(
convertionObj.converted_folder.split('/')
[0:-2]) # Keeping only the three last elements
self.insertToTable([ProcessingObject(convertionValues)])
def getToProcessListPerStudy(self, study):
toProcessList = self.DBClient.executeAllResults(
self.sqlBuilder.getSQL_getToBeProcessedFromProcessingTable(study))
return [self.getObjectFromTuple(t) for t in toProcessList]
def __init__(self):
self.DBClient = DbUtils()
self.sqlBuilder = SQLBuilder()
def __init__(self):
self.tableName = 'Sorting'
self.DBClient = DbUtils()
self.sqlBuilder = SQLBuilder()
def __init__(self):
self.tableName = 'Conversion'
self.DBClient = DbUtils()
self.sqlBuilder = SQLBuilder()
def __init__(self):
self.tableName = 'Conversion'
self.DBClient = DbUtils()
self.sqlBuilder = SQLBuilder()
class Conversion:
def __init__(self):
self.tableName = 'Conversion'
self.DBClient = DbUtils()
self.sqlBuilder = SQLBuilder()
def getObjectFromTuple(self, tuple):
valuesDict = dict(record_id=tuple[0],
study=tuple[1],
rid=tuple[2],
scan_type=tuple[3],
scan_date=tuple[4].strftime("%Y-%m-%d"),
scan_time=str(tuple[5]),
s_identifier=tuple[6],
i_identifier=tuple[7],
file_type=tuple[8],
raw_folder=tuple[9],
converted_folder=tuple[10],
version=tuple[11],
converted=tuple[12])
return ConversionObject(valuesDict)
def insertToTable(self, objList):
for obj in objList:
self.DBClient.executeNoResult(
self.sqlBuilder.getSQL_AddNewEntryToConversionTable(
obj.sqlInsert()))
def get_version(self, sortingObj, versionDict):
if sortingObj.study == 'ADNI':
dl_path = sortingObj.download_folder
if 'Uniform' in dl_path:
return 'V2'
else:
return versionDict[
sc.ProcessingModalityAndPipelineTypePerStudy[
sortingObj.study]
[sortingObj.
scan_type]] if sc.ProcessingModalityAndPipelineTypePerStudy[
sortingObj.study][
sortingObj.scan_type] in versionDict else 'V1'
else:
return versionDict[
sc.ProcessingModalityAndPipelineTypePerStudy[sortingObj.study]
[sortingObj.
scan_type]] if sc.ProcessingModalityAndPipelineTypePerStudy[
sortingObj.study][
sortingObj.scan_type] in versionDict else 'V1'
def insertFromSortingObj(self, sortingObj, versionDict):
sortingValues = sortingObj.getValuesDict()
version = self.get_version(sortingObj, versionDict)
sortingValues[
'converted_folder'] = '{0}/{1}/{2}/{3}/{4}_{5}_{6}/{7}/converted/final'.format(
sc.studyDatabaseRootDict[sortingObj.study], sortingObj.study,
sortingObj.scan_type, sortingObj.rid, sortingObj.scan_date,
sortingObj.s_identifier, sortingObj.i_identifier, version)
sortingValues['version'] = version
sortingValues['converted'] = 0
self.insertToTable([ConversionObject(sortingValues)])
def gettoBeConvertedPerStudy(self, study):
toConvertList = self.DBClient.executeAllResults(
self.sqlBuilder.getSQL_getToBeConvertedFileFromConversionTable(
study))
return [self.getObjectFromTuple(t) for t in toConvertList]
def setConvertedTrue(self, convertionObj):
convertionObj.converted = 1
self.saveObj(convertionObj)
def setConvertedFailed(self, convertionObj):
convertionObj.skip = 1
self.saveObj(convertionObj)
def saveObj(self, convertionObj):
self.DBClient.executeNoResult(
self.sqlBuilder.getSQL_saveObjConversionTable(convertionObj))
def getConvertedListPerStudy(self, study):
convertedList = self.DBClient.executeAllResults(
self.sqlBuilder.getSQL_getAllConvertedFromConvertionTable(study))
return [self.getObjectFromTuple(t) for t in convertedList]
def __init__(self):
self.DBClient = DbUtils()
self.sqlBuilder = SQLBuilder()
