def __init__(self, studyList, version):
self.DBClient = DbUtils()
self.studyList = [i.upper() for i in studyList]
self.version = version
self.recursorList = []
self._getRecursorList(studyList)
self.sortingDataList = []
self.sqlBuilder = SQLBuilder()
self.moveSortingObjListDict = {}
self.toConvertObjListDict = {}
self.sortingTable = Sorting()
self.conversionTable = Conversion()
self.raw2mincConverter = Raw2MINCConverter()
self.pool = Pool()
self.qsubJobHandler = QSubJobHandler()
self.qsubJobHandler.start()
self.convertedListDict = {}
self.processingTable = Processing()
self.toProcessListDict = {}
self.pipelineHanlder = PipelineHandler()
self.QCHandler = QCHandler()
self.MongoManger = MongoDBManager()
self.MongoXMLManager = MongoScanXMLManager()
self.MongoXMLManager.processXMLs()
def __init__(self):
self.processingPPDict = {
'ADNI': {
'V1': {
'T1': ADNI_V1_T1(),
'FMRI': ADNI_V1_FMRI(),
'AV45': ADNI_V1_AV45(),
'FDG': ADNI_V1_FDG(),
'AV1451': ADNI_V1_AV1451()
},
'V2': {
'T1': ADNI_V1_T1(),
'FMRI': ADNI_V1_FMRI(),
'AV45': ADNI_V2_AV45(),
'FDG': ADNI_V2_FDG(),
'AV1451': ADNI_V2_AV1451()
}
},
'DIAN': {
'V1': {
'T1': DIAN_V1_T1(),
'FDG': DIAN_V1_FDG(),
'PIB': DIAN_V1_PIB()
}
}
}
self.DBClient = DbUtils()
self.QCH = QCHandler()
def __init__(self, inputFolder, database_location):
# Initiate Database Client
self.DbClient = DbUtils(database=database_location)
# For each csv file, import it into the SQL database
for inputFile in glob.glob(inputFolder + '/*.csv'):
if inputFile in CSVconfig.AdniIgnored:
continue
sqlLocation = os.path.basename(inputFile).replace('.csv', '')
with open(inputFile, 'r') as csvFile:
csvToDatabase(self.DbClient, csvFile, sqlLocation)
# Close the connection to the database
self.DbClient.close()
__author__ = 'sulantha'
import datetime
from Utils.DbUtils import DbUtils
DBClient = DbUtils()
with open('/data/data03/sulantha/Downloads/fdg_list.csv', 'r') as file:
next(file)
for line in file:
row = line.split(',')
rid = row[0]
date = row[1].strip()
dateT = datetime.datetime.strptime(date, '%Y-%m-%d')
dateS = dateT.strftime('%Y-%m-%d')
findSQL = "SELECT * FROM Processing WHERE RID = {0} AND MODALITY = 'FDG' AND SCAN_DATE = '{1}'".format(
rid, dateS)
res = DBClient.executeAllResults(findSQL)
print('{0}-{1} {2}'.format(
rid, len(res), '############'
if len(res) is 0 else '')) if len(res) is 0 else None
processingSQL = "UPDATE Processing SET SKIP = 0 WHERE RID = {0} AND MODALITY = 'FDG' AND SCAN_DATE = '{1}'".format(
rid, dateS)
DBClient.executeNoResult(processingSQL)
def __init__(self):
self.DBClient = DbUtils()
def __init__(self):
self.DBClient = DbUtils()
self.MatchDBClient = DbUtils(database=pc.ADNI_dataMatchDBName)
self.PETHelper = PETHelper()
__author__ = 'sulantha'
from Utils.DbUtils import DbUtils
CSVFile = '/data/data03/sulantha/Downloads/missing_list_preprocessed.csv'
dbc = DbUtils()
with open(CSVFile, 'rU') as csv_file:
for line in csv_file:
lin = line.strip()
rid = lin.split('/')[6]
print(rid)
s_id = lin.split('/')[7].split('_')[-2]
i_id = lin.split('/')[7].split('_')[-1]
sql = "UPDATE ADNI_AV45_Pipeline SET SKIP = 0, QC = 0, FINISHED = 0, PROC_Failed = NULL, MANUAL_XFM = 'Req_man_reg' WHERE PROCESSING_TID IN (SELECT RECORD_ID FROM Processing WHERE RID = '{0}' AND MODALITY = 'AV45' AND VERSION = 'V2' AND S_IDENTIFIER = '{1}' )".format(
rid, s_id)
dbc.executeNoResult(sql)
__author__ = 'sulantha'
maskList = '/home/sulantha/Desktop/ManualMaskMatch.csv'
outputpath = '/data/data03/MANUAL_MASK'
from Utils.DbUtils import DbUtils
import shutil
Dbclient = DbUtils()
with open(maskList, 'r') as inf:
for line in inf:
row = line.split(',')
if row[0].strip() == 'None' or row[1].strip(
) == 'None' or row[2].strip() == 'None':
pass
else:
study = row[1].split('/')[-1].split('_')[0].upper()
rid = row[2].split('_')[3]
t1sid = row[2].split('.')[0].split('_')[-2]
t1iid = row[2].split('.')[0].split('_')[-1]
uid = 'SKULLMASK_{0}_{1}_{2}_{3}'.format(study, rid, t1sid, t1iid)
path = '{0}/{1}.mnc'.format(outputpath, uid)
print(study, rid, uid, sep=', ')
try:
shutil.copyfile(row[0], path)
Dbclient.executeNoResult(
"INSERT IGNORE INTO MANUAL_MASK VALUES (Null, '{0}', '{1}', '{2}', '{3}')"
.format(study, rid, uid, path))
sql2 = "UPDATE ADNI_T1_Pipeline SET MANUAL_MASK = 1 WHERE PROCESSING_TID IN (SELECT RECORD_ID FROM Processing WHERE RID = {0} AND S_IDENTIFIER = {1} AND I_IDENTIFIER = {2})".format(
def __init__(self):
self.DBClient = DbUtils()
self.MatchDBClient = DbUtils(database=pc.DIAN_dataMatchDBName)
def __init__(self):
self.tableName = 'Conversion'
self.DBClient = DbUtils()
self.sqlBuilder = SQLBuilder()
def __init__(self):
self.DBClient = DbUtils()
self.QCHandler = QCHandler()
import csv
from Utils.DbUtils import DbUtils
inputFile = '/home/sulantha/reRUNAv45.csv'
DBC = DbUtils()
with open(inputFile, 'r') as inputFile:
csvFile = csv.reader(inputFile)
for line in csvFile:
RID = line[0].split('/')[6]
IID = line[0].split('/')[7].split('_')[-1]
sql = "UPDATE ADNI_AV45_Pipeline SET FINISHED = 0, SKIP = 0 WHERE PROCESSING_TID IN (SELECT RECORD_ID FROM Processing WHERE RID = {0} AND I_IDENTIFIER = '{1}')".format(
RID, IID)
DBC.executeNoResult(sql)
__author__ = 'sulantha'
import datetime
from Utils.DbUtils import DbUtils
csvFile = '/data/data03/sulantha/Downloads/av45_list.csv'
MatchDBClient = DbUtils(database='Study_Data.ADNI')
DBClient = DbUtils()
with open(csvFile, 'r') as csv:
next(csv)
for line in csv:
row = line.split(',')
rid = row[0].strip()
date = row[1].strip()
dateT = datetime.datetime.strptime(date, '%m/%d/%Y')
#dateT = datetime.datetime.strptime(date, '%Y-%m-%d')
dateS = dateT.strftime('%Y-%m-%d')
sql = "SELECT DISTINCT subject, visit, seriesid, imageid FROM PET_META_LIST WHERE subject like '%_%_{0}' and scandate = '{1}' and origproc = 'Original'".format(
rid, dateS)
result = MatchDBClient.executeAllResults(sql)
checkDBSQL = "SELECT * FROM Conversion WHERE RID = '{0}' AND S_IDENTIFIER = '{1}' AND I_IDENTIFIER = '{2}'".format(
rid, 'S{0}'.format(result[0][2]), 'I{0}'.format(result[0][3]))
#print(checkDBSQL)
resultN = DBClient.executeAllResults(checkDBSQL)
if len(resultN) == 0:
print('########################### Not in DB - {0} - {1}'.format(
rid, date))
else:
pass
def __init__(self):
self.DBClient = DbUtils()
self.sqlBuilder = SQLBuilder()
os.environ['PERL5LIB'] = ':'.join(libpath.PERL5LIB)
os.environ['MNI_DATAPATH'] = ':'.join(libpath.MNI_DATAPATH)
os.environ['ROOT'] = ';'.join(libpath.ROOT)
os.environ['MINC_TOOLKIT_VERSION'] = libpath.MINC_TOOLKIT_VERSION
os.environ['MINC_COMPRESS'] = libpath.MINC_COMPRESS
os.environ['MINC_FORCE_V2'] = libpath.MINC_FORCE_V2
if __name__ == '__main__':
processingPPDict = {
'ADNI': {
'V1': {
'FMRI': ADNI_V1_FMRI()
},
'V2': {
'FMRI': ADNI_V1_FMRI()
}
}
}
study = 'ADNI'
modality = 'FMRI'
setEnvironmentVariables()
toProcessinModalityPerStudy = DbUtils().executeAllResults(
"SELECT * FROM Processing INNER JOIN (SELECT * FROM {0}_{1}_Pipeline WHERE NOT (FINISHED OR SKIP)) as TMP ON Processing.RECORD_ID=TMP.PROCESSING_TID"
.format(study, modality))
for processingItem in toProcessinModalityPerStudy:
version = processingItem[10]
# Calling on the process .section of given studies and modalities
processingPPDict[study][version][modality].process(processingItem)
def __init__(self):
self.DBClient = DbUtils()
self.CoregHand = CoregHandler()
self.client = MongoClient('localhost', 27017)
self.db = self.client.ADNI_Database
self.XML_collection = self.db.Scan_XML_Collection