def __init__(self, studyList, version): self.DBClient = DbUtils() self.studyList = [i.upper() for i in studyList] self.version = version self.recursorList = [] self._getRecursorList(studyList) self.sortingDataList = [] self.sqlBuilder = SQLBuilder() self.moveSortingObjListDict = {} self.toConvertObjListDict = {} self.sortingTable = Sorting() self.conversionTable = Conversion() self.raw2mincConverter = Raw2MINCConverter() self.pool = Pool() self.qsubJobHandler = QSubJobHandler() self.qsubJobHandler.start() self.convertedListDict = {} self.processingTable = Processing() self.toProcessListDict = {} self.pipelineHanlder = PipelineHandler() self.QCHandler = QCHandler() self.MongoManger = MongoDBManager() self.MongoXMLManager = MongoScanXMLManager() self.MongoXMLManager.processXMLs()
def __init__(self): self.processingPPDict = { 'ADNI': { 'V1': { 'T1': ADNI_V1_T1(), 'FMRI': ADNI_V1_FMRI(), 'AV45': ADNI_V1_AV45(), 'FDG': ADNI_V1_FDG(), 'AV1451': ADNI_V1_AV1451() }, 'V2': { 'T1': ADNI_V1_T1(), 'FMRI': ADNI_V1_FMRI(), 'AV45': ADNI_V2_AV45(), 'FDG': ADNI_V2_FDG(), 'AV1451': ADNI_V2_AV1451() } }, 'DIAN': { 'V1': { 'T1': DIAN_V1_T1(), 'FDG': DIAN_V1_FDG(), 'PIB': DIAN_V1_PIB() } } } self.DBClient = DbUtils() self.QCH = QCHandler()
def __init__(self, inputFolder, database_location): # Initiate Database Client self.DbClient = DbUtils(database=database_location) # For each csv file, import it into the SQL database for inputFile in glob.glob(inputFolder + '/*.csv'): if inputFile in CSVconfig.AdniIgnored: continue sqlLocation = os.path.basename(inputFile).replace('.csv', '') with open(inputFile, 'r') as csvFile: csvToDatabase(self.DbClient, csvFile, sqlLocation) # Close the connection to the database self.DbClient.close()
__author__ = 'sulantha' import datetime from Utils.DbUtils import DbUtils DBClient = DbUtils() with open('/data/data03/sulantha/Downloads/fdg_list.csv', 'r') as file: next(file) for line in file: row = line.split(',') rid = row[0] date = row[1].strip() dateT = datetime.datetime.strptime(date, '%Y-%m-%d') dateS = dateT.strftime('%Y-%m-%d') findSQL = "SELECT * FROM Processing WHERE RID = {0} AND MODALITY = 'FDG' AND SCAN_DATE = '{1}'".format( rid, dateS) res = DBClient.executeAllResults(findSQL) print('{0}-{1} {2}'.format( rid, len(res), '############' if len(res) is 0 else '')) if len(res) is 0 else None processingSQL = "UPDATE Processing SET SKIP = 0 WHERE RID = {0} AND MODALITY = 'FDG' AND SCAN_DATE = '{1}'".format( rid, dateS) DBClient.executeNoResult(processingSQL)
def __init__(self): self.DBClient = DbUtils()
def __init__(self): self.DBClient = DbUtils() self.MatchDBClient = DbUtils(database=pc.ADNI_dataMatchDBName) self.PETHelper = PETHelper()
__author__ = 'sulantha' from Utils.DbUtils import DbUtils CSVFile = '/data/data03/sulantha/Downloads/missing_list_preprocessed.csv' dbc = DbUtils() with open(CSVFile, 'rU') as csv_file: for line in csv_file: lin = line.strip() rid = lin.split('/')[6] print(rid) s_id = lin.split('/')[7].split('_')[-2] i_id = lin.split('/')[7].split('_')[-1] sql = "UPDATE ADNI_AV45_Pipeline SET SKIP = 0, QC = 0, FINISHED = 0, PROC_Failed = NULL, MANUAL_XFM = 'Req_man_reg' WHERE PROCESSING_TID IN (SELECT RECORD_ID FROM Processing WHERE RID = '{0}' AND MODALITY = 'AV45' AND VERSION = 'V2' AND S_IDENTIFIER = '{1}' )".format( rid, s_id) dbc.executeNoResult(sql)
__author__ = 'sulantha' maskList = '/home/sulantha/Desktop/ManualMaskMatch.csv' outputpath = '/data/data03/MANUAL_MASK' from Utils.DbUtils import DbUtils import shutil Dbclient = DbUtils() with open(maskList, 'r') as inf: for line in inf: row = line.split(',') if row[0].strip() == 'None' or row[1].strip( ) == 'None' or row[2].strip() == 'None': pass else: study = row[1].split('/')[-1].split('_')[0].upper() rid = row[2].split('_')[3] t1sid = row[2].split('.')[0].split('_')[-2] t1iid = row[2].split('.')[0].split('_')[-1] uid = 'SKULLMASK_{0}_{1}_{2}_{3}'.format(study, rid, t1sid, t1iid) path = '{0}/{1}.mnc'.format(outputpath, uid) print(study, rid, uid, sep=', ') try: shutil.copyfile(row[0], path) Dbclient.executeNoResult( "INSERT IGNORE INTO MANUAL_MASK VALUES (Null, '{0}', '{1}', '{2}', '{3}')" .format(study, rid, uid, path)) sql2 = "UPDATE ADNI_T1_Pipeline SET MANUAL_MASK = 1 WHERE PROCESSING_TID IN (SELECT RECORD_ID FROM Processing WHERE RID = {0} AND S_IDENTIFIER = {1} AND I_IDENTIFIER = {2})".format(
def __init__(self): self.DBClient = DbUtils() self.MatchDBClient = DbUtils(database=pc.DIAN_dataMatchDBName)
def __init__(self): self.tableName = 'Conversion' self.DBClient = DbUtils() self.sqlBuilder = SQLBuilder()
def __init__(self): self.DBClient = DbUtils() self.QCHandler = QCHandler()
import csv from Utils.DbUtils import DbUtils inputFile = '/home/sulantha/reRUNAv45.csv' DBC = DbUtils() with open(inputFile, 'r') as inputFile: csvFile = csv.reader(inputFile) for line in csvFile: RID = line[0].split('/')[6] IID = line[0].split('/')[7].split('_')[-1] sql = "UPDATE ADNI_AV45_Pipeline SET FINISHED = 0, SKIP = 0 WHERE PROCESSING_TID IN (SELECT RECORD_ID FROM Processing WHERE RID = {0} AND I_IDENTIFIER = '{1}')".format( RID, IID) DBC.executeNoResult(sql)
__author__ = 'sulantha' import datetime from Utils.DbUtils import DbUtils csvFile = '/data/data03/sulantha/Downloads/av45_list.csv' MatchDBClient = DbUtils(database='Study_Data.ADNI') DBClient = DbUtils() with open(csvFile, 'r') as csv: next(csv) for line in csv: row = line.split(',') rid = row[0].strip() date = row[1].strip() dateT = datetime.datetime.strptime(date, '%m/%d/%Y') #dateT = datetime.datetime.strptime(date, '%Y-%m-%d') dateS = dateT.strftime('%Y-%m-%d') sql = "SELECT DISTINCT subject, visit, seriesid, imageid FROM PET_META_LIST WHERE subject like '%_%_{0}' and scandate = '{1}' and origproc = 'Original'".format( rid, dateS) result = MatchDBClient.executeAllResults(sql) checkDBSQL = "SELECT * FROM Conversion WHERE RID = '{0}' AND S_IDENTIFIER = '{1}' AND I_IDENTIFIER = '{2}'".format( rid, 'S{0}'.format(result[0][2]), 'I{0}'.format(result[0][3])) #print(checkDBSQL) resultN = DBClient.executeAllResults(checkDBSQL) if len(resultN) == 0: print('########################### Not in DB - {0} - {1}'.format( rid, date)) else: pass
def __init__(self): self.DBClient = DbUtils() self.sqlBuilder = SQLBuilder()
os.environ['PERL5LIB'] = ':'.join(libpath.PERL5LIB) os.environ['MNI_DATAPATH'] = ':'.join(libpath.MNI_DATAPATH) os.environ['ROOT'] = ';'.join(libpath.ROOT) os.environ['MINC_TOOLKIT_VERSION'] = libpath.MINC_TOOLKIT_VERSION os.environ['MINC_COMPRESS'] = libpath.MINC_COMPRESS os.environ['MINC_FORCE_V2'] = libpath.MINC_FORCE_V2 if __name__ == '__main__': processingPPDict = { 'ADNI': { 'V1': { 'FMRI': ADNI_V1_FMRI() }, 'V2': { 'FMRI': ADNI_V1_FMRI() } } } study = 'ADNI' modality = 'FMRI' setEnvironmentVariables() toProcessinModalityPerStudy = DbUtils().executeAllResults( "SELECT * FROM Processing INNER JOIN (SELECT * FROM {0}_{1}_Pipeline WHERE NOT (FINISHED OR SKIP)) as TMP ON Processing.RECORD_ID=TMP.PROCESSING_TID" .format(study, modality)) for processingItem in toProcessinModalityPerStudy: version = processingItem[10] # Calling on the process .section of given studies and modalities processingPPDict[study][version][modality].process(processingItem)
def __init__(self): self.DBClient = DbUtils() self.CoregHand = CoregHandler() self.client = MongoClient('localhost', 27017) self.db = self.client.ADNI_Database self.XML_collection = self.db.Scan_XML_Collection