def update(self, lockfile_prefix): print('\n-------------- Update --------------') success = self.lock_update(lockfile_prefix) if not success: print('ERROR:failed to get lock on new update') exit(1) try: print('Connecting to XNAT at '+self.xnat_host) xnat = Interface(self.xnat_host, self.xnat_user, self.xnat_pass) #Priority: if self.priority_project: project_list=self.get_project_list(list(set(self.project_process_dict.keys() + self.project_modules_dict.keys()))) else: project_list = sorted(set(self.project_process_dict.keys() + self.project_modules_dict.keys())) # Update projects for project_id in project_list: print('===== PROJECT:'+project_id+' =====') self.update_project(xnat, project_id, lockfile_prefix) finally: self.unlock_update(lockfile_prefix) xnat.disconnect() print('Connection to XNAT closed')
def lastupdated_subj2sess(self): print('DEBUG:check for up to date subjects, apply timestamp to session last_updated') try: print('Connecting to XNAT at '+self.xnat_host) xnat = Interface(self.xnat_host, self.xnat_user, self.xnat_pass) project_list = sorted(set(self.project_process_dict.keys() + self.project_modules_dict.keys())) # Update projects for project_id in project_list: print('===== PROJECT:'+project_id+' =====') for subj_info in XnatUtils.list_subjects(xnat, project_id): last_mod = datetime.strptime(subj_info['last_modified'][0:19], '%Y-%m-%d %H:%M:%S') last_up = self.get_lastupdated(subj_info) if (last_up != None and last_mod < last_up): print(' +Subject:'+subj_info['label']+', subject up to date:last_mod='+str(last_mod)+',last_up='+str(last_up)) for sess_info in XnatUtils.list_sessions(xnat, subj_info['project'], subj_info['ID']): if sess_info['last_updated'] == '': print(' +Session:'+sess_info['label']+': subject up to date, setting update time to now') self.set_session_lastupdated(xnat, sess_info) else: print(' +Subject:'+subj_info['label']+', skipping:last_mod='+str(last_mod)+',last_up='+str(last_up)) finally: xnat.disconnect() print('Connection to XNAT closed')
def upload_XNAT_files(download_info, path, xnat_folder): url = "/project/" path = os.path.expanduser("~") + "/Documents/.central.cfg" for j in range(0, len(download_info)): for experiment_dir in download_info[j]["Scan Dirs"]: # print("Scan dir è {}",format(experiment_dir)) list_files = os.listdir(download_info[j]["Subject Folder"] + "/" + download_info[j]["Session ID"] + "/" + download_info[j]["Scan ID"]) folder2xnat = (download_info[j]["Subject Folder"] + "/" + download_info[j]["Session ID"] + "/" + download_info[j]["Scan ID"]) print('folder2xnat is', folder2xnat) # file_folder=(download_info[j]["Subject Folder"]+'/'+scan_dirs) # tmp=file_folder.split('PROCESSED/') # tmp=tmp[1].split('/') # tmp=tmp[1].split('/') # scan_id=tmp[0].split('-')[0] # print(list_files) # print(file_folder) for files in list_files: #file_path=download_info[j]["Subject Folder"]+'/'+scan_dirs+'/'+files #print(file_path) central = Interface(config=path) try: serverurl = (url + download_info[j]["Project"] + "/subject/" + download_info[j]["Subject ID"] + "/experiments/" + download_info[j]["Session ID"] + "/") experiment = central.select(serverurl) os.chdir(folder2xnat) experiment.resource(xnat_folder).file(files).insert(files) finally: central.disconnect()
def download_project_scans_of_type(build_dir, output_dir, scan_type): # PARSE Project ID. splitted = build_dir.split("build/") splitted = splitted[1].split("/") prj_ID = splitted[0] # Select the data with specific constraints path = os.path.expanduser("~") + "/Documents/.central.cfg" central = Interface(config=path) try: constraints = [("xnat:mrSessionData/project", "=", prj_ID)] data = central.select( "xnat:mrSessionData", [ "xnat:mrSessionData/MR_SCAN_COUNT_AGG", "xnat:mrSessionData/SESSION_ID", "xnat:mrSessionData/SUBJECT_ID", "xnat:mrSessionData/SUBJECT_LABEL", ], ).where(constraints) # Load the useful data in a python dictioanry dictlist = [dict()] j = 0 base_dir = output_dir ### Download info rinominalo come scan_payload o something similar. reg_expr = [] array_scan_types = [] for elem in scan_type: pattern = "(?i)" + elem + "[^\(\)]*" #match each substring containing: scan_type followed by any character excluding '(' and ')' zero or more times. reg_expr.append(re.compile(pattern)) #central.array.experiments(project_id=prj_ID,experiment_type='xnat:mrSessionData',constraints={'xnat:mrSessionData/MR_SCAN_COUNT_AGG':}) # range esclude il secondo estremo for i in range(0, len(data)): array_scan_types = [] for regular_expression in reg_expr: scan_type_retrieved = regular_expression.findall( data[i]["mr_scan_count_agg"]) if len(scan_type_retrieved) != 0: array_scan_types += scan_type_retrieved dictlist[j]["Scan Type"] = array_scan_types dictlist[j]["Project"] = prj_ID dictlist[j]["Subject ID"] = data[i]["subject_id"] dictlist[j]["Session ID"] = data[i]["session_id"] dictlist[j]["Subject Label"] = data[i]["subject_label"] dictlist[j]["Subject Folder"] = (base_dir + "/" + dictlist[j]["Subject Label"]) j = j + 1 dictlist.insert(j, dict()) dictlist.pop(j) # pop the last empty element return dictlist except Exception as err: print(err) finally: central.disconnect()
def update_open_tasks(self, lockfile_prefix): task_queue = [] print('\n-------------- Open Tasks Update --------------') success = self.lock_open_tasks(lockfile_prefix) if not success: print('ERROR:failed to get lock on open tasks update') exit(1) try: print('Connecting to XNAT at '+self.xnat_host) xnat = Interface(self.xnat_host, self.xnat_user, self.xnat_pass) print('Getting task list...') task_list = self.get_open_tasks(xnat) print(str(len(task_list))+' open jobs found') print('Updating tasks...') for cur_task in task_list: print(' Updating task:'+cur_task.assessor_label) task_status = cur_task.update_status() if task_status == task.NEED_TO_RUN: task_queue.append(cur_task) print(str(len(task_queue))+' jobs ready to be launched') #===== Sort the task queue as desired - random? breadth-first? depth-first? #task_queue.sort() # Launch jobs self.launch_jobs(task_queue) finally: self.unlock_open_tasks(lockfile_prefix) xnat.disconnect() print('Connection to XNAT closed')
def upload(direct): #Subject, Experiment y Scan se llaman como el nombre entero de la secuencia #Si un metadata se encuentra vacío [], no se guarda ese field por defecto #Desde el XML en XNAT se pueden ver los fields #BW se ha cambiado a bw, XNAT no emplea bien las mayúsculas parece ser #Cada archivo le cuesta 26 segundos para subirse #Conexión XNAT session = xnat.connect('http://localhost', user='******', password='******') interface = Interface(server='http://localhost', user='******', password='******') #Para SNAPSHOTS print("Connected to XNAT with API") #PROYECTO tiene que existir en XNAT project = session.projects['physio'] project_snapshot = interface.select.project( 'physio').insert() #Para SNAPSHOTS #Seteamos el directorio en la carpeta de datos os.chdir(direct) #1-Accedemos a los files y guardamos sus nombres files = os.listdir() nii = 0 mat = 0 for i in files: if i.find('nii') != -1: nii = i elif i.find('mat') != -1: mat = i #Si solo hay .mat, se sube solo .mat if nii == 0 and mat != 0: print("Only .mat file detected") #Eliminamos .nii y .mat de los nombres filename = mat.split('.mat')[0] filename_date = mat.split('.mat')[0] filename = filename.replace(".", "-") s_label = filename #2-subject subject = session.classes.SubjectData(parent=project, label=s_label) #3-experiment experiment = session.classes.MrSessionData(parent=subject, label=filename) #Si queremos añadir FECHA array_date = filename_date.split('.')[1:4] date = array_date[0] + "-" + array_date[1] + "-" + array_date[2] experiment.date = date #4-scan scan = session.classes.MrScanData(parent=experiment, id=filename, type='MRI') print("Subject, Experiment and Scan created") #6-Resource y subir .mat try: #XNATResponseError (status 409) resource_MAT = session.classes.ResourceCatalog(parent=scan, label='MAT', type='mat') resource_MAT.upload(mat, mat) print(".mat uploaded") except: resource_MAT = scan.resources['MAT'] resource_MAT.upload(mat, mat) print(".mat already exists") #7-METADATA metadata = scipy.io.loadmat(mat) if 'BW' in metadata.keys(): metadata["bw"] = metadata.pop("BW") del metadata["image3D"] del metadata["kSpace3D"] del metadata["dataFull"] del metadata["imgFull"] del metadata["sampled"] del metadata["__header__"] del metadata["__version__"] del metadata["__globals__"] del metadata["phGradients"] # del metadata["average"] # del metadata["rawdata"] for key in metadata.keys(): experiment.fields[key] = metadata.get(key) session.disconnect() interface.disconnect() elif nii != 0 and mat != 0: print(".mat and .nii files detected") #Eliminamos .nii y .mat de los nombres filename = nii.split('.nii')[0] filename_date = nii.split('.nii')[0] filename = filename.replace(".", "-") s_label = filename.split('-')[0] exp_label = filename.split('-')[1] #2-subject subject = session.classes.SubjectData(parent=project, label=s_label) #3-experiment experiment = session.classes.MrSessionData(parent=subject, label=exp_label) #Si queremos añadir FECHA array_date = filename_date.split('.')[2:5] date = array_date[0] + "-" + array_date[1] + "-" + array_date[2] print(date) experiment.date = date #4-scan scan = session.classes.MrScanData(parent=experiment, id=filename, type='MRI') print("Subject, Experiment and Scan created") #5-Resource y subir NIFTI #Actualizamos el NIFTI si ya existe try: #XNATResponseError (status 409) resource_NIFTI = session.classes.ResourceCatalog(parent=scan, label='NIFTI', type='nii') resource_NIFTI.upload(nii, nii) print("NIFTI uploaded") except: resource_NIFTI = scan.resources['NIFTI'] resource_NIFTI.upload(nii, nii) print("NIFTI already exists") #6-Resource y subir .mat try: #XNATResponseError (status 409) resource_MAT = session.classes.ResourceCatalog(parent=scan, label='MAT', type='mat') resource_MAT.upload(mat, mat) print(".mat uploaded") except: resource_MAT = scan.resources['MAT'] resource_MAT.upload(mat, mat) print(".mat already exists") #7-METADATA metadata = scipy.io.loadmat(mat) if 'BW' in metadata.keys(): metadata["bw"] = metadata.pop("BW") del metadata["image3D"] del metadata["kSpace3D"] del metadata["dataFull"] del metadata["imgFull"] del metadata["sampled"] del metadata["__header__"] del metadata["__version__"] del metadata["__globals__"] del metadata["phGradients"] for key in metadata.keys(): experiment.fields[key] = metadata.get(key) print("metadata uploaded") # ============================================================================= # SNAPSHOTS # ============================================================================= NIFTI_img = nib.load(nii) data = NIFTI_img.get_fdata() #Creamos directorio para guardar imágenes dentro de cada carpeta y será destruido #Comprobamos si existe el directorio por error y se borra para crearlo de nuevo if os.path.isdir("SNAPSHOTS") == False: os.mkdir("SNAPSHOTS") os.chdir("SNAPSHOTS") else: shutil.rmtree("SNAPSHOTS") os.mkdir("SNAPSHOTS") os.chdir("SNAPSHOTS") if data.ndim != 3: print( "Se espera un número de 3 dimensiones" ) #################################MESSAGE FUNCTION##################################### else: #Dimensión donde se encuentra el min shape = np.array(data.shape) for i in list(range(data.ndim)): if shape[i] == min(shape): dim_10 = i images = [] for i in list(range(min(shape))): if dim_10 == 0: data_1 = np.array(data[i, :, :]) elif dim_10 == 1: data_1 = np.array(data[:, i, :]) elif dim_10 == 2: data_1 = np.array(data[:, :, i]) imsave(str(i) + '.gif', data_1) images.append(imageio.imread(str(i) + '.gif')) imageio.mimsave(filename + '.gif', images) #Subir snapshot subject_snapshot = project_snapshot.subject(s_label) experiment = subject_snapshot.experiment(exp_label) experiment.scan(filename).resource('SNAPSHOTS').file( filename + '.gif').insert(filename + '.gif', format="GIF", content="ORIGINAL", tags=None) experiment.scan(filename).resource('SNAPSHOTS').file( filename + '_t.gif').insert(filename + '.gif', format="GIF", content="THUMBNAIL", tags=None) os.chdir("../") shutil.rmtree("SNAPSHOTS") print("Snapshots uploaded") # ============================================================================= # FIN SNAPSHOTS # ============================================================================= session.disconnect() interface.disconnect()
# currently avail. data types that may be useful are: # 'xnat:projectData' # 'xnat:subjectData' # 'xnat:mrSessionData' # 'xnat:pVisitData' # 'val:protocolData' # 'xnat:investigatorData' # 'xnat:qcManualAssessorData' # 'xnat:qcAssessmentData' # currently avail. subjectData sub-types that may be useful are: # ID, GENDER_TEXT, PROJECTS, PROJECT, XNAT_COL_SUBJECTDATALABEL # currently avail. mrSessionData sub-types that may be useful are: # PROJECT, SUBJECT_ID, SUBJECT_LABEL, SESSION_ID, DATE, TYPE, SCANNER_CSV, DTI_COUNT, MR_SCAN_COUNT_AGG, LAST_MODIFIED # currently avail. pVisitData sub-types that may be useful are: # SUBJECT_ID, EXPT_ID, DATE, PROJECT, PROTOCOLVERSION, PROTOCOLID # print(radcxnat.inspect.datatypes('xnat:mrSessionData', 'SUBJECT*')) # radcxnat.inspect.set_autolearn('True') # delete project/subject(id, not label!)/experiment (use cautiously!) subj = radcxnat.select('/project/{}/subject/{}/'.format(project, subject)) subj.delete() # print('/project/{}/subject/{} deleted'.format(project, subject)) radcxnat.disconnect()
finally: #Sent an email if (warning_list or flag_files_list) and emailaddress!='' : if warning_list: TEXT='\nThe following assessor already exists and the Spider try to upload files on existing files :\n' for warning in warning_list: TEXT+=' - '+warning+'\n' TEXT+='\nYou should :\n\t-remove the assessor if you want to upload the data \n\t-set the status of the assessor to "uploading" \n\t-remove the data from the upload folder if you do not want to upload this data.\n' SUBJECT='ERROR/WARNING: XNAT Process Upload' if flag_files_list: if not TEXT: TEXT='\nCheck the following processes if they are still running:\n' else: TEXT+='Check the following process if they are still running:\n' for files in flag_files_list: TEXT+=' - '+files+'\n' sendMail(VUEMAIL_ADDR,VUEMAIL_PWS,emailaddress,SUBJECT,TEXT,'smtp.gmail.com') #disconnect xnat.disconnect() print 'INFO: Connection to Xnat closed' #Stop the process before the end or end of the script, remove the flagfile for the spider running finally: #remove flagfile os.remove(os.path.join(UploadDir,'FlagFiles','Process_Upload_running.txt')) print '===================================================================\n' else: print 'WARNING: Upload already running.'
or flag_files_list) and emailaddress != '': if warning_list: TEXT = '\nThe following assessor already exists and the Spider try to upload files on existing files :\n' for warning in warning_list: TEXT += ' - ' + warning + '\n' TEXT += '\nYou should :\n\t-remove the assessor if you want to upload the data \n\t-set the status of the assessor to "uploading" \n\t-remove the data from the upload folder if you do not want to upload this data.\n' SUBJECT = 'ERROR/WARNING: XNAT Process Upload' if flag_files_list: if not TEXT: TEXT = '\nCheck the following processes if they are still running:\n' else: TEXT += 'Check the following process if they are still running:\n' for files in flag_files_list: TEXT += ' - ' + files + '\n' sendMail(VUEMAIL_ADDR, VUEMAIL_PWS, emailaddress, SUBJECT, TEXT, 'smtp.gmail.com') #disconnect xnat.disconnect() print 'INFO: Connection to Xnat closed' #Stop the process before the end or end of the script, remove the flagfile for the spider running finally: #remove flagfile os.remove( os.path.join(UploadDir, 'FlagFiles', 'Process_Upload_running.txt')) print '===================================================================\n' else: print 'WARNING: Upload already running.'
def main(args): # ===================================================================== # INFERENCE # - Find the ID of each scan as well as the corresponding DICOMS # - get model paths # - Predict chest view # - Predict COVID19 probability on frontal chest scans # ===================================================================== scan_ids, scan_dicoms = os.listdir('/input/SCANS'), [] for scan_id in scan_ids: base = '/input/SCANS/{}/DICOM/'.format(scan_id) current_scan_dicoms = [ os.path.join(base, x) for x in os.listdir(base) if x.endswith('.dcm')] if len(current_scan_dicoms) > 1: current_scan_dicoms = current_scan_dicoms[0] scan_dicoms += current_scan_dicoms covid_probs = [] for scan in scan_dicoms: im, _ = load_image(scan) covid_prob = infer_session_simple(im)[0] covid_probs.append(covid_prob) print('DEBUG:', scan, 'PROB:', covid_prob) # ===================================================================== # RESULTS # - Write note on XNAT scan with the results # ===================================================================== session = Interface(server=os.environ['XNAT_HOST'], user=os.environ['XNAT_USER'], password=os.environ['XNAT_PASS']) try: for scan_id, dicom_filepath, covid_prob in zip(scan_ids, scan_dicoms, covid_probs): print('--------------------------------------------------') print('DICOM PATH', dicom_filepath) print("SCAN ID:", scan_id) print("predicted probability:", covid_prob) print('--------------------------------------------------') if covid_prob is not -1: scan = session.select( '/projects/{}/subjects/{}/experiments/{}/scans/{}'.format( args.project, args.subject, args.experiment, scan_id)) scan.attrs.set( 'xnat:imageScanData/note', 'p(COVID): {0:.2f}%'.format( covid_prob)) except Exception as e: session.disconnect() raise RuntimeError('Error while writting results to XNAT', str(e)) # =================================================================== # EMAIL COMMUNICATION # send email to user. # - Get the preamble # - update the probabilities # - send email # =================================================================== body = get_preamble() for scan_id, dicom_filepath, covid_prob in zip(scan_ids, scan_dicoms, covid_probs): if covid_prob is not -1: body.append('Diagnostic results:') body.append('-------------------------------------------------------------------------') body.append('Subject ID: {}'.format(args.subject)) # body.append('SUBJECT ID: {}'.format(args.subject)) # body.append('DICOM Path: {}'.format(dicom_filepath)) body.append('Predicted Probability for COVID-19: {:.2f}%'.format(covid_prob)) body.append('-------------------------------------------------------------------------') # send email send_email(session, args, body)