def update(self, lockfile_prefix):
print('\n-------------- Update --------------')
success = self.lock_update(lockfile_prefix)
if not success:
print('ERROR:failed to get lock on new update')
exit(1)
try:
print('Connecting to XNAT at '+self.xnat_host)
xnat = Interface(self.xnat_host, self.xnat_user, self.xnat_pass)
#Priority:
if self.priority_project:
project_list=self.get_project_list(list(set(self.project_process_dict.keys() + self.project_modules_dict.keys())))
else:
project_list = sorted(set(self.project_process_dict.keys() + self.project_modules_dict.keys()))
# Update projects
for project_id in project_list:
print('===== PROJECT:'+project_id+' =====')
self.update_project(xnat, project_id, lockfile_prefix)
finally:
self.unlock_update(lockfile_prefix)
xnat.disconnect()
print('Connection to XNAT closed')
def lastupdated_subj2sess(self):
print('DEBUG:check for up to date subjects, apply timestamp to session last_updated')
try:
print('Connecting to XNAT at '+self.xnat_host)
xnat = Interface(self.xnat_host, self.xnat_user, self.xnat_pass)
project_list = sorted(set(self.project_process_dict.keys() + self.project_modules_dict.keys()))
# Update projects
for project_id in project_list:
print('===== PROJECT:'+project_id+' =====')
for subj_info in XnatUtils.list_subjects(xnat, project_id):
last_mod = datetime.strptime(subj_info['last_modified'][0:19], '%Y-%m-%d %H:%M:%S')
last_up = self.get_lastupdated(subj_info)
if (last_up != None and last_mod < last_up):
print(' +Subject:'+subj_info['label']+', subject up to date:last_mod='+str(last_mod)+',last_up='+str(last_up))
for sess_info in XnatUtils.list_sessions(xnat, subj_info['project'], subj_info['ID']):
if sess_info['last_updated'] == '':
print(' +Session:'+sess_info['label']+': subject up to date, setting update time to now')
self.set_session_lastupdated(xnat, sess_info)
else:
print(' +Subject:'+subj_info['label']+', skipping:last_mod='+str(last_mod)+',last_up='+str(last_up))
finally:
xnat.disconnect()
print('Connection to XNAT closed')
def upload_XNAT_files(download_info, path, xnat_folder):
url = "/project/"
path = os.path.expanduser("~") + "/Documents/.central.cfg"
for j in range(0, len(download_info)):
for experiment_dir in download_info[j]["Scan Dirs"]:
# print("Scan dir è {}",format(experiment_dir))
list_files = os.listdir(download_info[j]["Subject Folder"] + "/" +
download_info[j]["Session ID"] + "/" +
download_info[j]["Scan ID"])
folder2xnat = (download_info[j]["Subject Folder"] + "/" +
download_info[j]["Session ID"] + "/" +
download_info[j]["Scan ID"])
print('folder2xnat is', folder2xnat)
# file_folder=(download_info[j]["Subject Folder"]+'/'+scan_dirs)
# tmp=file_folder.split('PROCESSED/')
# tmp=tmp[1].split('/')
# tmp=tmp[1].split('/')
# scan_id=tmp[0].split('-')[0]
# print(list_files)
# print(file_folder)
for files in list_files:
#file_path=download_info[j]["Subject Folder"]+'/'+scan_dirs+'/'+files
#print(file_path)
central = Interface(config=path)
try:
serverurl = (url + download_info[j]["Project"] +
"/subject/" + download_info[j]["Subject ID"] +
"/experiments/" +
download_info[j]["Session ID"] + "/")
experiment = central.select(serverurl)
os.chdir(folder2xnat)
experiment.resource(xnat_folder).file(files).insert(files)
finally:
central.disconnect()
def download_project_scans_of_type(build_dir, output_dir, scan_type):
# PARSE Project ID.
splitted = build_dir.split("build/")
splitted = splitted[1].split("/")
prj_ID = splitted[0]
# Select the data with specific constraints
path = os.path.expanduser("~") + "/Documents/.central.cfg"
central = Interface(config=path)
try:
constraints = [("xnat:mrSessionData/project", "=", prj_ID)]
data = central.select(
"xnat:mrSessionData",
[
"xnat:mrSessionData/MR_SCAN_COUNT_AGG",
"xnat:mrSessionData/SESSION_ID",
"xnat:mrSessionData/SUBJECT_ID",
"xnat:mrSessionData/SUBJECT_LABEL",
],
).where(constraints)
# Load the useful data in a python dictioanry
dictlist = [dict()]
j = 0
base_dir = output_dir
### Download info rinominalo come scan_payload o something similar.
reg_expr = []
array_scan_types = []
for elem in scan_type:
pattern = "(?i)" + elem + "[^\(\)]*" #match each substring containing: scan_type followed by any character excluding '(' and ')' zero or more times.
reg_expr.append(re.compile(pattern))
#central.array.experiments(project_id=prj_ID,experiment_type='xnat:mrSessionData',constraints={'xnat:mrSessionData/MR_SCAN_COUNT_AGG':})
# range esclude il secondo estremo
for i in range(0, len(data)):
array_scan_types = []
for regular_expression in reg_expr:
scan_type_retrieved = regular_expression.findall(
data[i]["mr_scan_count_agg"])
if len(scan_type_retrieved) != 0:
array_scan_types += scan_type_retrieved
dictlist[j]["Scan Type"] = array_scan_types
dictlist[j]["Project"] = prj_ID
dictlist[j]["Subject ID"] = data[i]["subject_id"]
dictlist[j]["Session ID"] = data[i]["session_id"]
dictlist[j]["Subject Label"] = data[i]["subject_label"]
dictlist[j]["Subject Folder"] = (base_dir + "/" +
dictlist[j]["Subject Label"])
j = j + 1
dictlist.insert(j, dict())
dictlist.pop(j) # pop the last empty element
return dictlist
except Exception as err:
print(err)
finally:
central.disconnect()
def update_open_tasks(self, lockfile_prefix):
task_queue = []
print('\n-------------- Open Tasks Update --------------')
success = self.lock_open_tasks(lockfile_prefix)
if not success:
print('ERROR:failed to get lock on open tasks update')
exit(1)
try:
print('Connecting to XNAT at '+self.xnat_host)
xnat = Interface(self.xnat_host, self.xnat_user, self.xnat_pass)
print('Getting task list...')
task_list = self.get_open_tasks(xnat)
print(str(len(task_list))+' open jobs found')
print('Updating tasks...')
for cur_task in task_list:
print(' Updating task:'+cur_task.assessor_label)
task_status = cur_task.update_status()
if task_status == task.NEED_TO_RUN:
task_queue.append(cur_task)
print(str(len(task_queue))+' jobs ready to be launched')
#===== Sort the task queue as desired - random? breadth-first? depth-first?
#task_queue.sort()
# Launch jobs
self.launch_jobs(task_queue)
finally:
self.unlock_open_tasks(lockfile_prefix)
xnat.disconnect()
print('Connection to XNAT closed')
def upload(direct):
#Subject, Experiment y Scan se llaman como el nombre entero de la secuencia
#Si un metadata se encuentra vacío [], no se guarda ese field por defecto
#Desde el XML en XNAT se pueden ver los fields
#BW se ha cambiado a bw, XNAT no emplea bien las mayúsculas parece ser
#Cada archivo le cuesta 26 segundos para subirse
#Conexión XNAT
session = xnat.connect('http://localhost', user='******', password='******')
interface = Interface(server='http://localhost',
user='******',
password='******') #Para SNAPSHOTS
print("Connected to XNAT with API")
#PROYECTO tiene que existir en XNAT
project = session.projects['physio']
project_snapshot = interface.select.project(
'physio').insert() #Para SNAPSHOTS
#Seteamos el directorio en la carpeta de datos
os.chdir(direct)
#1-Accedemos a los files y guardamos sus nombres
files = os.listdir()
nii = 0
mat = 0
for i in files:
if i.find('nii') != -1:
nii = i
elif i.find('mat') != -1:
mat = i
#Si solo hay .mat, se sube solo .mat
if nii == 0 and mat != 0:
print("Only .mat file detected")
#Eliminamos .nii y .mat de los nombres
filename = mat.split('.mat')[0]
filename_date = mat.split('.mat')[0]
filename = filename.replace(".", "-")
s_label = filename
#2-subject
subject = session.classes.SubjectData(parent=project, label=s_label)
#3-experiment
experiment = session.classes.MrSessionData(parent=subject,
label=filename)
#Si queremos añadir FECHA
array_date = filename_date.split('.')[1:4]
date = array_date[0] + "-" + array_date[1] + "-" + array_date[2]
experiment.date = date
#4-scan
scan = session.classes.MrScanData(parent=experiment,
id=filename,
type='MRI')
print("Subject, Experiment and Scan created")
#6-Resource y subir .mat
try:
#XNATResponseError (status 409)
resource_MAT = session.classes.ResourceCatalog(parent=scan,
label='MAT',
type='mat')
resource_MAT.upload(mat, mat)
print(".mat uploaded")
except:
resource_MAT = scan.resources['MAT']
resource_MAT.upload(mat, mat)
print(".mat already exists")
#7-METADATA
metadata = scipy.io.loadmat(mat)
if 'BW' in metadata.keys():
metadata["bw"] = metadata.pop("BW")
del metadata["image3D"]
del metadata["kSpace3D"]
del metadata["dataFull"]
del metadata["imgFull"]
del metadata["sampled"]
del metadata["__header__"]
del metadata["__version__"]
del metadata["__globals__"]
del metadata["phGradients"]
# del metadata["average"]
# del metadata["rawdata"]
for key in metadata.keys():
experiment.fields[key] = metadata.get(key)
session.disconnect()
interface.disconnect()
elif nii != 0 and mat != 0:
print(".mat and .nii files detected")
#Eliminamos .nii y .mat de los nombres
filename = nii.split('.nii')[0]
filename_date = nii.split('.nii')[0]
filename = filename.replace(".", "-")
s_label = filename.split('-')[0]
exp_label = filename.split('-')[1]
#2-subject
subject = session.classes.SubjectData(parent=project, label=s_label)
#3-experiment
experiment = session.classes.MrSessionData(parent=subject,
label=exp_label)
#Si queremos añadir FECHA
array_date = filename_date.split('.')[2:5]
date = array_date[0] + "-" + array_date[1] + "-" + array_date[2]
print(date)
experiment.date = date
#4-scan
scan = session.classes.MrScanData(parent=experiment,
id=filename,
type='MRI')
print("Subject, Experiment and Scan created")
#5-Resource y subir NIFTI
#Actualizamos el NIFTI si ya existe
try:
#XNATResponseError (status 409)
resource_NIFTI = session.classes.ResourceCatalog(parent=scan,
label='NIFTI',
type='nii')
resource_NIFTI.upload(nii, nii)
print("NIFTI uploaded")
except:
resource_NIFTI = scan.resources['NIFTI']
resource_NIFTI.upload(nii, nii)
print("NIFTI already exists")
#6-Resource y subir .mat
try:
#XNATResponseError (status 409)
resource_MAT = session.classes.ResourceCatalog(parent=scan,
label='MAT',
type='mat')
resource_MAT.upload(mat, mat)
print(".mat uploaded")
except:
resource_MAT = scan.resources['MAT']
resource_MAT.upload(mat, mat)
print(".mat already exists")
#7-METADATA
metadata = scipy.io.loadmat(mat)
if 'BW' in metadata.keys():
metadata["bw"] = metadata.pop("BW")
del metadata["image3D"]
del metadata["kSpace3D"]
del metadata["dataFull"]
del metadata["imgFull"]
del metadata["sampled"]
del metadata["__header__"]
del metadata["__version__"]
del metadata["__globals__"]
del metadata["phGradients"]
for key in metadata.keys():
experiment.fields[key] = metadata.get(key)
print("metadata uploaded")
# =============================================================================
# SNAPSHOTS
# =============================================================================
NIFTI_img = nib.load(nii)
data = NIFTI_img.get_fdata()
#Creamos directorio para guardar imágenes dentro de cada carpeta y será destruido
#Comprobamos si existe el directorio por error y se borra para crearlo de nuevo
if os.path.isdir("SNAPSHOTS") == False:
os.mkdir("SNAPSHOTS")
os.chdir("SNAPSHOTS")
else:
shutil.rmtree("SNAPSHOTS")
os.mkdir("SNAPSHOTS")
os.chdir("SNAPSHOTS")
if data.ndim != 3:
print(
"Se espera un número de 3 dimensiones"
) #################################MESSAGE FUNCTION#####################################
else:
#Dimensión donde se encuentra el min
shape = np.array(data.shape)
for i in list(range(data.ndim)):
if shape[i] == min(shape):
dim_10 = i
images = []
for i in list(range(min(shape))):
if dim_10 == 0:
data_1 = np.array(data[i, :, :])
elif dim_10 == 1:
data_1 = np.array(data[:, i, :])
elif dim_10 == 2:
data_1 = np.array(data[:, :, i])
imsave(str(i) + '.gif', data_1)
images.append(imageio.imread(str(i) + '.gif'))
imageio.mimsave(filename + '.gif', images)
#Subir snapshot
subject_snapshot = project_snapshot.subject(s_label)
experiment = subject_snapshot.experiment(exp_label)
experiment.scan(filename).resource('SNAPSHOTS').file(
filename + '.gif').insert(filename + '.gif',
format="GIF",
content="ORIGINAL",
tags=None)
experiment.scan(filename).resource('SNAPSHOTS').file(
filename + '_t.gif').insert(filename + '.gif',
format="GIF",
content="THUMBNAIL",
tags=None)
os.chdir("../")
shutil.rmtree("SNAPSHOTS")
print("Snapshots uploaded")
# =============================================================================
# FIN SNAPSHOTS
# =============================================================================
session.disconnect()
interface.disconnect()
# currently avail. data types that may be useful are:
# 'xnat:projectData'
# 'xnat:subjectData'
# 'xnat:mrSessionData'
# 'xnat:pVisitData'
# 'val:protocolData'
# 'xnat:investigatorData'
# 'xnat:qcManualAssessorData'
# 'xnat:qcAssessmentData'
# currently avail. subjectData sub-types that may be useful are:
# ID, GENDER_TEXT, PROJECTS, PROJECT, XNAT_COL_SUBJECTDATALABEL
# currently avail. mrSessionData sub-types that may be useful are:
# PROJECT, SUBJECT_ID, SUBJECT_LABEL, SESSION_ID, DATE, TYPE, SCANNER_CSV, DTI_COUNT, MR_SCAN_COUNT_AGG, LAST_MODIFIED
# currently avail. pVisitData sub-types that may be useful are:
# SUBJECT_ID, EXPT_ID, DATE, PROJECT, PROTOCOLVERSION, PROTOCOLID
# print(radcxnat.inspect.datatypes('xnat:mrSessionData', 'SUBJECT*'))
# radcxnat.inspect.set_autolearn('True')
# delete project/subject(id, not label!)/experiment (use cautiously!)
subj = radcxnat.select('/project/{}/subject/{}/'.format(project, subject))
subj.delete()
# print('/project/{}/subject/{} deleted'.format(project, subject))
radcxnat.disconnect()
finally:
#Sent an email
if (warning_list or flag_files_list) and emailaddress!='' :
if warning_list:
TEXT='\nThe following assessor already exists and the Spider try to upload files on existing files :\n'
for warning in warning_list:
TEXT+=' - '+warning+'\n'
TEXT+='\nYou should :\n\t-remove the assessor if you want to upload the data \n\t-set the status of the assessor to "uploading" \n\t-remove the data from the upload folder if you do not want to upload this data.\n'
SUBJECT='ERROR/WARNING: XNAT Process Upload'
if flag_files_list:
if not TEXT:
TEXT='\nCheck the following processes if they are still running:\n'
else:
TEXT+='Check the following process if they are still running:\n'
for files in flag_files_list:
TEXT+=' - '+files+'\n'
sendMail(VUEMAIL_ADDR,VUEMAIL_PWS,emailaddress,SUBJECT,TEXT,'smtp.gmail.com')
#disconnect
xnat.disconnect()
print 'INFO: Connection to Xnat closed'
#Stop the process before the end or end of the script, remove the flagfile for the spider running
finally:
#remove flagfile
os.remove(os.path.join(UploadDir,'FlagFiles','Process_Upload_running.txt'))
print '===================================================================\n'
else:
print 'WARNING: Upload already running.'
or flag_files_list) and emailaddress != '':
if warning_list:
TEXT = '\nThe following assessor already exists and the Spider try to upload files on existing files :\n'
for warning in warning_list:
TEXT += ' - ' + warning + '\n'
TEXT += '\nYou should :\n\t-remove the assessor if you want to upload the data \n\t-set the status of the assessor to "uploading" \n\t-remove the data from the upload folder if you do not want to upload this data.\n'
SUBJECT = 'ERROR/WARNING: XNAT Process Upload'
if flag_files_list:
if not TEXT:
TEXT = '\nCheck the following processes if they are still running:\n'
else:
TEXT += 'Check the following process if they are still running:\n'
for files in flag_files_list:
TEXT += ' - ' + files + '\n'
sendMail(VUEMAIL_ADDR, VUEMAIL_PWS, emailaddress,
SUBJECT, TEXT, 'smtp.gmail.com')
#disconnect
xnat.disconnect()
print 'INFO: Connection to Xnat closed'
#Stop the process before the end or end of the script, remove the flagfile for the spider running
finally:
#remove flagfile
os.remove(
os.path.join(UploadDir, 'FlagFiles',
'Process_Upload_running.txt'))
print '===================================================================\n'
else:
print 'WARNING: Upload already running.'
def main(args):
# =====================================================================
# INFERENCE
# - Find the ID of each scan as well as the corresponding DICOMS
# - get model paths
# - Predict chest view
# - Predict COVID19 probability on frontal chest scans
# =====================================================================
scan_ids, scan_dicoms = os.listdir('/input/SCANS'), []
for scan_id in scan_ids:
base = '/input/SCANS/{}/DICOM/'.format(scan_id)
current_scan_dicoms = [
os.path.join(base, x) for x in os.listdir(base)
if x.endswith('.dcm')]
if len(current_scan_dicoms) > 1:
current_scan_dicoms = current_scan_dicoms[0]
scan_dicoms += current_scan_dicoms
covid_probs = []
for scan in scan_dicoms:
im, _ = load_image(scan)
covid_prob = infer_session_simple(im)[0]
covid_probs.append(covid_prob)
print('DEBUG:', scan, 'PROB:', covid_prob)
# =====================================================================
# RESULTS
# - Write note on XNAT scan with the results
# =====================================================================
session = Interface(server=os.environ['XNAT_HOST'],
user=os.environ['XNAT_USER'],
password=os.environ['XNAT_PASS'])
try:
for scan_id, dicom_filepath, covid_prob in zip(scan_ids,
scan_dicoms,
covid_probs):
print('--------------------------------------------------')
print('DICOM PATH', dicom_filepath)
print("SCAN ID:", scan_id)
print("predicted probability:", covid_prob)
print('--------------------------------------------------')
if covid_prob is not -1:
scan = session.select(
'/projects/{}/subjects/{}/experiments/{}/scans/{}'.format(
args.project, args.subject, args.experiment, scan_id))
scan.attrs.set(
'xnat:imageScanData/note', 'p(COVID): {0:.2f}%'.format(
covid_prob))
except Exception as e:
session.disconnect()
raise RuntimeError('Error while writting results to XNAT', str(e))
# ===================================================================
# EMAIL COMMUNICATION
# send email to user.
# - Get the preamble
# - update the probabilities
# - send email
# ===================================================================
body = get_preamble()
for scan_id, dicom_filepath, covid_prob in zip(scan_ids,
scan_dicoms,
covid_probs):
if covid_prob is not -1:
body.append('Diagnostic results:')
body.append('-------------------------------------------------------------------------')
body.append('Subject ID: {}'.format(args.subject))
# body.append('SUBJECT ID: {}'.format(args.subject))
# body.append('DICOM Path: {}'.format(dicom_filepath))
body.append('Predicted Probability for COVID-19: {:.2f}%'.format(covid_prob))
body.append('-------------------------------------------------------------------------')
# send email
send_email(session, args, body)
