def upload_XNAT_files(download_info, path, xnat_folder):
url = "/project/"
path = os.path.expanduser("~") + "/Documents/.central.cfg"
for j in range(0, len(download_info)):
for experiment_dir in download_info[j]["Scan Dirs"]:
# print("Scan dir è {}",format(experiment_dir))
list_files = os.listdir(download_info[j]["Subject Folder"] + "/" +
download_info[j]["Session ID"] + "/" +
download_info[j]["Scan ID"])
folder2xnat = (download_info[j]["Subject Folder"] + "/" +
download_info[j]["Session ID"] + "/" +
download_info[j]["Scan ID"])
print('folder2xnat is', folder2xnat)
# file_folder=(download_info[j]["Subject Folder"]+'/'+scan_dirs)
# tmp=file_folder.split('PROCESSED/')
# tmp=tmp[1].split('/')
# tmp=tmp[1].split('/')
# scan_id=tmp[0].split('-')[0]
# print(list_files)
# print(file_folder)
for files in list_files:
#file_path=download_info[j]["Subject Folder"]+'/'+scan_dirs+'/'+files
#print(file_path)
central = Interface(config=path)
try:
serverurl = (url + download_info[j]["Project"] +
"/subject/" + download_info[j]["Subject ID"] +
"/experiments/" +
download_info[j]["Session ID"] + "/")
experiment = central.select(serverurl)
os.chdir(folder2xnat)
experiment.resource(xnat_folder).file(files).insert(files)
finally:
central.disconnect()
def lastupdated_subj2sess(self):
print('DEBUG:check for up to date subjects, apply timestamp to session last_updated')
try:
print('Connecting to XNAT at '+self.xnat_host)
xnat = Interface(self.xnat_host, self.xnat_user, self.xnat_pass)
project_list = sorted(set(self.project_process_dict.keys() + self.project_modules_dict.keys()))
# Update projects
for project_id in project_list:
print('===== PROJECT:'+project_id+' =====')
for subj_info in XnatUtils.list_subjects(xnat, project_id):
last_mod = datetime.strptime(subj_info['last_modified'][0:19], '%Y-%m-%d %H:%M:%S')
last_up = self.get_lastupdated(subj_info)
if (last_up != None and last_mod < last_up):
print(' +Subject:'+subj_info['label']+', subject up to date:last_mod='+str(last_mod)+',last_up='+str(last_up))
for sess_info in XnatUtils.list_sessions(xnat, subj_info['project'], subj_info['ID']):
if sess_info['last_updated'] == '':
print(' +Session:'+sess_info['label']+': subject up to date, setting update time to now')
self.set_session_lastupdated(xnat, sess_info)
else:
print(' +Subject:'+subj_info['label']+', skipping:last_mod='+str(last_mod)+',last_up='+str(last_up))
finally:
xnat.disconnect()
print('Connection to XNAT closed')
def update(self, lockfile_prefix):
print('\n-------------- Update --------------')
success = self.lock_update(lockfile_prefix)
if not success:
print('ERROR:failed to get lock on new update')
exit(1)
try:
print('Connecting to XNAT at '+self.xnat_host)
xnat = Interface(self.xnat_host, self.xnat_user, self.xnat_pass)
#Priority:
if self.priority_project:
project_list=self.get_project_list(list(set(self.project_process_dict.keys() + self.project_modules_dict.keys())))
else:
project_list = sorted(set(self.project_process_dict.keys() + self.project_modules_dict.keys()))
# Update projects
for project_id in project_list:
print('===== PROJECT:'+project_id+' =====')
self.update_project(xnat, project_id, lockfile_prefix)
finally:
self.unlock_update(lockfile_prefix)
xnat.disconnect()
print('Connection to XNAT closed')
def download_project_scans_of_type(build_dir, output_dir, scan_type):
# PARSE Project ID.
splitted = build_dir.split("build/")
splitted = splitted[1].split("/")
prj_ID = splitted[0]
# Select the data with specific constraints
path = os.path.expanduser("~") + "/Documents/.central.cfg"
central = Interface(config=path)
try:
constraints = [("xnat:mrSessionData/project", "=", prj_ID)]
data = central.select(
"xnat:mrSessionData",
[
"xnat:mrSessionData/MR_SCAN_COUNT_AGG",
"xnat:mrSessionData/SESSION_ID",
"xnat:mrSessionData/SUBJECT_ID",
"xnat:mrSessionData/SUBJECT_LABEL",
],
).where(constraints)
# Load the useful data in a python dictioanry
dictlist = [dict()]
j = 0
base_dir = output_dir
### Download info rinominalo come scan_payload o something similar.
reg_expr = []
array_scan_types = []
for elem in scan_type:
pattern = "(?i)" + elem + "[^\(\)]*" #match each substring containing: scan_type followed by any character excluding '(' and ')' zero or more times.
reg_expr.append(re.compile(pattern))
#central.array.experiments(project_id=prj_ID,experiment_type='xnat:mrSessionData',constraints={'xnat:mrSessionData/MR_SCAN_COUNT_AGG':})
# range esclude il secondo estremo
for i in range(0, len(data)):
array_scan_types = []
for regular_expression in reg_expr:
scan_type_retrieved = regular_expression.findall(
data[i]["mr_scan_count_agg"])
if len(scan_type_retrieved) != 0:
array_scan_types += scan_type_retrieved
dictlist[j]["Scan Type"] = array_scan_types
dictlist[j]["Project"] = prj_ID
dictlist[j]["Subject ID"] = data[i]["subject_id"]
dictlist[j]["Session ID"] = data[i]["session_id"]
dictlist[j]["Subject Label"] = data[i]["subject_label"]
dictlist[j]["Subject Folder"] = (base_dir + "/" +
dictlist[j]["Subject Label"])
j = j + 1
dictlist.insert(j, dict())
dictlist.pop(j) # pop the last empty element
return dictlist
except Exception as err:
print(err)
finally:
central.disconnect()
def generator(self): # Function for creating the configuration file
central = Interface(server='https://central.xnat.org',
user=self.name,
password=self.password)
central.save_config('ConfigFile/central.cfg')
print("Configuration file Created...........")
def get_scandata(session_id):
central = Interface(config=cfg)
constraint = [('xnat:mrScanData/IMAGE_SESSION_ID', '=', session_id)]
f = central.select('xnat:mrScanData', [
'xnat:mrScanData/SERIES_DESCRIPTION', 'xnat:mrScanData/ID',
'xnat:mrScanData/QUALITY', 'xnat:mrSessionData/DATE'
]).where(constraint)
return f.items()
def check_and_raise():
x = Interface(config='.xnat.cfg')
try:
x.head('')
print('Docker instance found.')
except ConnectionError:
print('Skipping it.')
raise SkipTest('Docker-based XNAT instance unavailable')
def check_and_raise():
x = Interface(config='.xnat.cfg')
try:
x.head('')
list(x.select.projects())
print('Docker instance found.')
except (ConnectionError, KeyError):
print('Skipping it.')
raise SkipTest('Docker-based XNAT instance unavailable')
def xnat(cfg=os.path.join(os.path.expanduser('~'), '.xnat.cfg'), server_url=None,
username=None, password=None, cache=os.path.join(os.path.expanduser('~'), '.xnat.cache')):
"""Initialize and test xnat connection from a previously-stored cfg file
or passed credentials
Parameters
----------
cfg: str
Path to a stored interface configuration
This is not from Interface.save_config but rather looks like this:
[xnat]
user: [your user name]
password: [your password]
server: [your xnat server]
cache: [dir]
server_url: str
url to your xnat install
username: str
user name
password: str
user password
cache: str
Cache directory
Returns
-------
A valid Interface object to your XNAT system.
This may throw an error from pyxnat.core.errors
"""
if not (cfg or (server_url and username and upass)):
raise ValueError("Must pass cfg file or server/username/password")
if (server_url and user and upass):
server = server_url
user = username
pword = password
else:
cp = ConfigParser()
with open(cfg) as f:
cp.readfp(f)
user = cp.get('xnat', 'user')
pword = cp.get('xnat', 'password')
server = cp.get('xnat', 'server')
cachedir = cp.get('xnat', 'cache')
if '~' in cachedir:
cachedir = os.path.expanduser(cachedir)
xnat = Interface(server=server,
user=user,
password=pword,
cachedir=cachedir)
# Because the constructor doesn't test the connection, make sure 'admin' is
# in the list of users. Any errors are passed to the caller.
if user not in xnat.manage.users():
raise ValueError('This XNAT is weird.')
xnat._memtimeout = 0.001
return xnat
def main():
quit = "n"
username = os.environ["XNAT_USER"]
password = os.environ["XNAT_PASS"]
central = Interface(server="https://xnat.imaging-genetics.camh.ca",
user=username,
password=password)
while (quit != "y"):
study = raw_input("Which study do you want to track scans for? ")
con = CON.config()
try:
projects = set(con.get_xnat_projects(study))
except ValueError:
print "Study does not exist"
return 0
tracking_table = dict()
for project in projects:
constraints = [('xnat:mrSessionData/PROJECT', '=', project)]
table = central.select('xnat:mrSessionData', [
'xnat:mrSessionData/SUBJECT_LABEL', 'xnat:mrSessionData/DATE',
'xnat:mrSessionData/INSERT_DATE'
]).where(constraints)
sort = sorted(table.items(), key=operator.itemgetter(2))
for item in sort:
#print(item)
site_name = scanid.parse(item[0]).site
if scanid.is_phantom(item[0]):
site_name += "_PHA"
if "FBN" in item[0]:
site_name += "_FBN"
elif "ADN" in item[0]:
site_name += "_ADN"
site_dict = tracking_table.setdefault(site_name, dict())
last_update = site_dict.setdefault(uploaddate, datetime.min)
current_update = datetime.strptime(item[2], datetimeformat)
if last_update < current_update:
site_dict[date] = item[1]
site_dict[uploaddate] = current_update
if last_update == datetime.min:
site_dict[uploaddiff] = "No Other Uploads"
else:
site_dict[uploaddiff] = dttostr(current_update -
last_update)
#break
printdict(tracking_table)
quit = raw_input("Quit? y/n ")
def display_project_structure(url, username=None, password=None):
if username is None:
xnat = Interface(url, anonymous=True)
else:
xnat = Interface(url, username, password)
xnat.cache.clear()
xnat.manage.schemas.add('xnat.xsd')
xnat.manage.schemas.add('fs.xsd')
for p in xnat.select.projects():
_display_project(p)
def __connect(self):
if self.connection == None:
self.connection = Interface(server=self.server,
user=self.user,
password=self.password,
cachedir='/dev/null')
return self.connection
def __init__(self, uri=None, user=None, password=None):
if uri is not None:
self._uri = uri
elif 'XNAT_URI' in os.environ:
self._uri = os.environ['XNAT_URI']
else:
raise ValueError('no URI specified')
if user is not None:
self._user = user
elif 'XNAT_USER' in os.environ:
self._user = os.environ['XNAT_USER']
else:
raise ValueError('no user specified')
if password is not None:
pass
elif 'XNAT_PASSWORD' in os.environ:
user = os.environ['XNAT_PASSWORD']
else:
raise ValueError('no password specified')
self._pyxnat_interface = Interface(server=self._uri,
user=self._user,
password=password)
self._pyxnat_interface._get_entry_point()
self._projects = []
self._accessible_projects = []
return
class PyXNAT(XNATCommunicator):
def setup(self):
#=======================================================================
# CLEAR THE PYXNAT CACHE
#=======================================================================
self.utils.removeFilesInDir(self.utils.pyXNATCache)
import pyxnat
#pyxnat.core.cache.CacheManager(self.XNAT).clear()
from pyxnat import Interface
#=======================================================================
# INITIALIZE PARENT
#=======================================================================
self.XNAT = Interface(server=self.server,
user=self.user,
password=self.password,
cachedir=self.utils.pyXNATCache)
def getFile(self, srcDstMap, withProgressBar = True):
return self.getFiles_URL(srcDstMap, fileOrFolder = "file")
def getFiles(self, srcDstMap, withProgressBar = True):
return self.getFiles_URL(srcDstMap, fileOrFolder = "folder")
def fileExists(self, XNATsrc):
#print self.utils.lf() + "Seeing if '%s' exists..."%(XNATsrc)
return self.XNAT.select(self.cleanSelectString(XNATsrc)).exists()
def makeDir(self, XNATPath):
#print (self.utils.lf() + " MAING XNAT DIR: " + XNATPath)
p = self.XNAT.select(self.cleanSelectString(XNATPath))
try:
if not p.exists():
p.insert()
return True
else:
return False
except Exception, e:
#print (self.utils.lf() + "ERROR CAUGHT: %s"%(str(e)))
#print (self.utils.lf() + "PATH EXISTS! %s"%(XNATPath))
return False
def test_add_remove_user():
x = Interface(config='.xnat.cfg')
x.select.project('nosetests3').remove_user('admin')
x.select.project('nosetests3').add_user('admin', 'collaborator')
assert 'admin' in x.select.project('nosetests3').collaborators()
x.select.project('nosetests3').remove_user('admin')
assert 'admin' not in x.select.project('nosetests3').collaborators()
x.select.project('nosetests3').add_user('admin', 'owner')
def _init_xnat_connection(self):
self.xnat = Interface(os.environ["XNAT_URL"], os.environ["XNAT_USER"],
os.environ["XNAT_PASSWORD"])
self.xnat.cache.clear()
self.xnat.manage.schemas.add("xnat.xsd")
self.xnat.manage.schemas.add("fs.xsd")
return self.xnat
def test_project_accessibility():
x = Interface(config='.xnat.cfg')
assert x.select.project('nosetests3').accessibility() in \
[b'public', b'protected', b'private']
x.select.project('nosetests3').set_accessibility('private')
assert x.select.project('nosetests3').accessibility() == b'private'
x.select.project('nosetests3').set_accessibility('protected')
print(x.select.project('nosetests3').accessibility())
assert x.select.project('nosetests3').accessibility() == b'protected'
def test_connection_xnat(host, user, pwd):
"""
Method to check connection to XNAT using host, user, pwd.
:param host: Host for XNAT
:param user: User for XNAT
:param pwd: Password for XNAT
:return: True if succeeded, False otherwise.
"""
from pyxnat.core.errors import DatabaseError
from pyxnat import Interface
try:
xnat = Interface(host, user, pwd)
# try deleting SESSION connection
xnat._exec('/data/JSESSION', method='DELETE')
print(' --> Good login.')
return True
except DatabaseError:
print(' --> error: Wrong login.')
return False
def test_create_xml():
x = Interface(config='.xnat.cfg')
_modulepath = op.dirname(op.abspath(__file__))
p = x.select.project('nosetests')
s = p.subject('10001')
e = s.experiment('10001_MR')
fp = op.join(_modulepath, 'sess.xml')
assert (op.isfile(fp))
e.create(xml=fp)
xml = e.get().decode()
assert ('Alomar' in xml)
def get_subjects(req,username,password, instance,project_name):
# setup the connection
xnat = Interface(server=instance,
user=username,
password=password,
cachedir=os.path.join(os.path.expanduser('~'),'XNATVIEW/.store'))
json_obj = []
for sid in xnat.select.project(project_name).subjects().get('label'):
json_obj.append({'subject_label':sid})
return json.dumps(json_obj)
def get_subject_list(project_id, url, user, pwd):
interface = Interface(server=url,
user=user,
password=pwd,
proxy='www-int2:80')
xnat_subjects = interface.select.project(project_id).subjects().get()
xnat_sub_labels = [
interface.select.project(project_id).subject(x).label()
for x in xnat_subjects
]
return xnat_sub_labels
def update_open_tasks(self, lockfile_prefix):
task_queue = []
print('\n-------------- Open Tasks Update --------------')
success = self.lock_open_tasks(lockfile_prefix)
if not success:
print('ERROR:failed to get lock on open tasks update')
exit(1)
try:
print('Connecting to XNAT at '+self.xnat_host)
xnat = Interface(self.xnat_host, self.xnat_user, self.xnat_pass)
print('Getting task list...')
task_list = self.get_open_tasks(xnat)
print(str(len(task_list))+' open jobs found')
print('Updating tasks...')
for cur_task in task_list:
print(' Updating task:'+cur_task.assessor_label)
task_status = cur_task.update_status()
if task_status == task.NEED_TO_RUN:
task_queue.append(cur_task)
print(str(len(task_queue))+' jobs ready to be launched')
#===== Sort the task queue as desired - random? breadth-first? depth-first?
#task_queue.sort()
# Launch jobs
self.launch_jobs(task_queue)
finally:
self.unlock_open_tasks(lockfile_prefix)
xnat.disconnect()
print('Connection to XNAT closed')
def xnat_search(self, session_label, series_label, xnat_config,
series_offset=0):
from pyxnat import Interface
central = Interface(server=xnat_config['server'],
user=xnat_config['user'],
password=xnat_config['pwd'])
if self.inputs.sanitize_wildcard:
session_pattern = '%%%s%%' % session_label.replace('*', '%')
series_pattern = '%%%s%%' % series_label.replace('*', '%')
else:
session_pattern = session_label
series_pattern = series_label
results = central.select(
'xnat:mrscandata',
['xnat:mrSessionData/SUBJECT_ID', 'xnat:mrSessionData/SESSION_ID',
'xnat:mrSessionData/LABEL',
'xnat:mrscandata/ID', 'xnat:mrscandata/QUALITY']
).where([
('xnat:mrscandata/SERIES_DESCRIPTION', 'LIKE', series_pattern),
('xnat:mrSessionData/LABEL', 'LIKE', session_pattern),
('xnat:mrScanData/QUALITY', '=', 'usable'),
'AND'])
"""[('S04272', 'E01333', 'M87115017', '14', 'usable'),
('S04272', 'E01333', 'M87115017', '15', 'usable')]
"""
if len(results.items()) == 0:
raise LookupError("No scans found using %s and %s" % (
series_pattern, session_pattern))
# Cast each scan to an int since the search returns them as strings.
scans = [int(result[3]) + series_offset for result in results.items()]
subject = results.data[0]['xnat_mrsessiondata_subject_id']
exam = results.data[0]['xnat_mrsessiondata_session_id']
return scans, subject, exam
def get_projects(req,username,password,instance):
# setup the connection
#instance = 'http://xnat.cci.emory.edu:8080/xnat'
xnat = Interface(server=instance,
user=username,
password=password,
cachedir=os.path.join(os.path.expanduser('~'),'XNATVIEW/.store'))
try:
project_list = xnat.select.projects().get()
json_obj = []
for project in project_list:
json_obj.append({"project_name":project, "status":"success"})
except:
json_obj = [{"status":"fail"}]
return json.dumps(json_obj)
def setup(self):
#=======================================================================
# CLEAR THE PYXNAT CACHE
#=======================================================================
self.utils.removeFilesInDir(self.utils.pyXNATCache)
import pyxnat
#pyxnat.core.cache.CacheManager(self.XNAT).clear()
from pyxnat import Interface
#=======================================================================
# INITIALIZE PARENT
#=======================================================================
self.XNAT = Interface(server=self.server,
user=self.user,
password=self.password,
cachedir=self.utils.pyXNATCache)
def get_scans(req,username,password,instance, project_name,subject_label,experiment_label):
# setup the connection
xnat = Interface(server=instance,
user=username,
password=password,
cachedir=os.path.join(os.path.expanduser('~'),'XNATVIEW/.store'))
json_obj = []
for scan in xnat.select.project(project_name).subject(subject_label) \
.experiment(experiment_label).scans():
try:
attr = scan.attrs.mget(['type','frames','xnat:mrScanData/parameters/tr','xnat:mrScanData/parameters/te','xnat:mrscandata/parameters/voxelres/x','xnat:mrscandata/parameters/voxelres/y','xnat:mrscandata/parameters/voxelres/z'])
except:
attr = ["","","","","","",""]
json_obj.append({'scan_id':scan.id(),"scan_type":attr[0],"scan_frames":attr[1],"scan_tr":attr[2], "scan_te":attr[3], "x_res":attr[4], "y_res":attr[5], 'z_res':attr[6]})
return json.dumps(json_obj)
def get_dicom_scans(req,username,password,instance ,project_name,subject_label,experiment_label,scan_id):
# setup the connection
#instance = 'http://xnat.cci.emory.edu:8080/xnat'
xnat = Interface(server=instance,
user=username,
password=password,
cachedir=os.path.join(os.path.expanduser('~'),'XNATVIEW/.store'))
instance = hashlib.sha1(instance).hexdigest()
## PATHS ON THE FS
png_path = '/var/www/XNATVIEW/IMAGE_CACHE/PNG_CACHE/'+instance+'/'
thumbnail_path = '/var/www/XNATVIEW/IMAGE_CACHE/THUMBNAIL_CACHE/'+instance+'/'
xml_path = '/var/www/XNATVIEW/IMAGE_CACHE/XML_FILES/'+instance+'/'
if not os.path.exists(thumbnail_path):
os.system("mkdir "+thumbnail_path)
if not os.path.exists(png_path):
os.system("mkdir "+png_path)
if not os.path.exists(xml_path):
os.system("mkdir "+xml_path)
folder_id = project_name+'-'+subject_label+'-'+experiment_label+'-SCAN_'+scan_id
png_folder = png_path+folder_id+'/'
thumbnail_folder = thumbnail_path+folder_id+'/'
xml_folder = xml_path+folder_id+'/'
thumbnail_loc = 'http://cerebro.cci.emory.edu/XNATVIEW/IMAGE_CACHE/THUMBNAIL_CACHE/'+instance+'/'+folder_id+'/'
png_loc = 'http://cerebro.cci.emory.edu/XNATVIEW/IMAGE_CACHE/PNG_CACHE/'+instance+'/'+folder_id+'/'
##
if (os.path.exists(thumbnail_folder) and os.path.exists(png_folder) and os.path.exists(xml_folder)):
f = open(xml_folder+folder_id+'.xml','r+')
xml_doc = f.read()
f.close()
else:
scan_object = xnat.select.project(project_name).subject(subject_label) \
.experiment(experiment_label).scan(scan_id)
dicom_map = downloadDicomScan(scan_object, png_folder, thumbnail_folder)
xml_doc = make_xml(dicom_map, folder_id, thumbnail_loc, png_loc, xml_folder)
# compatibility with new format from xnatview_backend.py
xml_doc = xml_doc % {'host': 'http://cerebro.cci.emory.edu/XNATVIEW/'}
return xml_doc
def get_experiments(req,username,instance, password,project_name,subject_label):
# setup the connection
xnat = Interface(server=instance,
user=username,
password=password,
cachedir=os.path.join(os.path.expanduser('~'),'XNATVIEW/.store'))
json_obj = []
if len( xnat.select.project(project_name).subject(subject_label) \
.experiments().get()) == 0:
json_obj.append({'experiment_label':"None"})
else:
for label in xnat.select.project(project_name).subject(subject_label).experiments().get('label'):
json_obj.append({'experiment_label':label})
return json.dumps(json_obj)
class Fetcher:
try: # Checking if the configuration file is created
SELECTOR = Interface(config='ConfigFile/central.cfg')
except:
print("Please create the configuration file first")
exit(1)
def get_projects(self):
# Returns a json table with all visible project details or public projects to user
try:
print("Processing............")
output = self.SELECTOR.select('xnat:projectData').all()
return output
except:
print("ERROR : Unable to connect to the database")
return None
def get_subjects(self):
# Returns a json table with all visible subjects details or public subjects to the user
try:
print("Processing............")
output = self.SELECTOR.select('xnat:subjectData').all()
return output
except:
print("ERROR : Unable to connect to the database")
return None
def get_experiments(self):
# Returns a json table with all visible project details or public projects to user
try:
print("Processing............")
output = self.SELECTOR.select('xnat:mrSessionData').all()
return output
except:
print("ERROR : Unable to connect to the database")
return None
def user_exists(username, password, server, ssl):
"""Checks whether user exist on XNAT instance.
Args:
username (str): Name of user on XNAT instance
password (str): Password of user on XNAT instance
server (str): Server url of XNAT instance
ssl (str): whether to verify remote host
Returns:
int/list: If user exist returns an integer else an empty list.
"""
verify = ssl == 1
exists = Interface(
user=username, password=password, server=server, verify=verify)\
.select.projects().get()
# If user exists then exists lengths will be more than 0
return len(exists) if len(exists) > 0 else []
#Check if the folder is not empty
UploadDirList=os.listdir(UploadDir)
if len(UploadDirList)==0:
print 'WARNING: No data need to be upload.\n'
else:
#Get the assessor label from the directory :
assessor_label_in_dir_list=get_assessor_name_from_folder()
#Get the list of OUTLOG which need to be upload:
outlog_list=get_outlog_from_folder()
#Get the list of OUTLOG which need to be upload:
pbs_list=get_pbs_from_folder()
#Start the process to upload
try:
print 'INFO: Connecting to XNAT to start uploading processes at '+VUIISxnat_host
xnat = Interface(VUIISxnat_host, VUIISxnat_user, VUIISxnat_pwd)
################# 1) Upload the assessor data ###############
#For each assessor label that need to be upload :
number_of_processes=len(assessor_label_in_dir_list)
for index,assessor_label in enumerate(assessor_label_in_dir_list):
assessor_path=UploadDir+'/'+assessor_label
if os.path.isdir(assessor_path):
sys.stdout.flush()
sys.stdout.write(" *Process: "+str(index+1)+"/"+str(number_of_processes)+' -- label: '+assessor_label+' / time: '+str(datetime.now())+'\n')
#Get the Project Name, the subject label, the experiment label and the assessor label from the folder name :
labels=assessor_label.split('-x-')
ProjectName=labels[0]
Subject=labels[1]
Experiment=labels[2]
#The Process name is the last labels
def get_mrsessionid(subject_id):
central = Interface(config=cfg)
constraint = [('xnat:mrSessionData/SUBJECT_ID', '=', subject_id)]
f = central.select('xnat:mrSessionData',
['xnat:mrSessionData/SESSION_ID']).where(constraint)
return f.items()[0][0]
def get_mrsubjectid(r):
central = Interface(config=cfg)
constraint = [('xnat:SubjectData/SUBJECT_LABEL', '=', r)]
f = central.select('xnat:SubjectData',
['xnat:SubjectData/SUBJECT_ID']).where(constraint)
return f.items()[0][0]
def get_rs(project):
central = Interface(config=cfg)
constraint = [('xnat:SubjectData/PROJECT', '=', project)]
f = central.select('xnat:SubjectData',
['xnat:SubjectData/SUBJECT_LABEL']).where(constraint)
return f.items()
'T1_weighted_FreeSurfer_BRAVO_IR_ARC2_1mm_Cor': 't1',
'DTI-Ax-35DIR-2.5mm_TR11.5_TE93_b900_PA': 'dti_pa',
'RestingState-200Volumes-36slices-TR1.76sec-Asset2': 'rsfmri',
'3D CUBE Eyes Loc': 't2',
'PU:T1_weighted_FreeSurfer_BRAVO_IR_ARC2_1mm_Cor': 't1_pure',
'SIRP-verb-WkMem-TR1.76sec-Asset2': 'sirp_verb_wm',
'SIRP_visspac-WkMem-TR1.76sec-Asset2': 'sirp_vispat_wm',
'T1_weighted_FreeSurfer_BRAVO_IR-SPGR_ASSET2_1mm_Cor': 't1_asset'
}
afniBrik = ('RestingState-200Volumes-36slices-TR1.76sec-Asset2')
#set to true to overwrite existing nifti images...
overwriteExisting = False
#connect to the db
central = Interface(config=pyxnat_cfg)
#set up some functions to re-use later
def get_scanData(session_id):
#get the scan data you're interested in. In this case, it's the series description (scan name) and the series ID number (the number on the disk)
constraint = [('xnat:mrScanData/IMAGE_SESSION_ID', '=', session_id)]
f = central.select('xnat:mrScanData', [
'xnat:mrScanData/SERIES_DESCRIPTION', 'xnat:mrScanData/ID',
'xnat:mrScanData/QUALITY'
]).where(constraint)
return f.items()
def eval_scans(scans):
#function to determine whether or not a scan should be selected
import numpy as np
import nipype.interfaces.io as nio # Data i/o
import nipype.interfaces.fsl as fsl # fsl
import nipype.interfaces.utility as util # utility
import nipype.pipeline.engine as pe # pypeline engine
import nipype.algorithms.modelgen as model # model generation
from nipeep import Memory
from pyxnat import Interface
mem = Memory('.')
central = Interface('https://central.xnat.org', 'schwarty', 'plopplop',
'/havoc/store/xnat')
fsl.FSLCommand.set_default_output_type('NIFTI_GZ')
input_files = [central.select('/project/Volatile/resources/123150742/files/000075760021s002a1001.nii.gz').get()]
# where put files arguments?
# extract_ref = partial(mem.cache(fsl.ExtractROI), t_min=42, t_size=1)
nosestrip = partial(mem.cache(fsl.BET), frac=0.3)
skullstrip = partial(mem.cache(fsl.BET), mask=True)
refskullstrip = partial(mem.cache(fsl.BET), mask=True)
# coregister = mem.cache(fsl.FLIRT)(dof=6)
# extract_ref_exec = [
# extract_ref(in_file=in_file) for in_file in input_files]
parser = argparse.ArgumentParser(description='Performs xnat audit: xnat_audit_v1.py -configfile ~/.xnat.cfg -project EONS_810366 -scan mprage -scan idemo -scan frac2back');
group = parser.add_argument_group('Required')
group.add_argument('-configfile', action="store", dest='configfile', required=True, help='Path to .xnat.cfg')
group.add_argument('-project', action="store", dest='project', required=True, help='Project name as in xnat')
group.add_argument('-scan', action="append", dest='scan', required=True, help='Single scan to be evaluated')
parser.add_argument('-version', action='version', version='%(prog)s 3.0')
inputArguments = parser.parse_args()
configfile = inputArguments.configfile
project = inputArguments.project
scans = inputArguments.scan
central = Interface(config=configfile)
#define function to perform final quality check#
################################################
def checkquality(scanid,scantype,array):
quality = 0
matching = []
matching = [a for a in array if scantype in str(a.get('qlux_qluxname')) and scanid == str(a.get('session_id'))]
for i in matching:
if qual_dict[str(i.get('session_id')),str(i.get('imagescan_id'))] == 'usable' : quality = 1
return quality
#define function for checking if log file uploaded#
###################################################
def checklog(scanid,dictionary):
taskname=0
class Connection(object):
'''
A pyxnat connection to an XNAT server
Parameters
----------
uri: str
user: str
password: str
'''
def __init__(self, uri=None, user=None, password=None):
if uri is not None:
self._uri = uri
elif 'XNAT_URI' in os.environ:
self._uri = os.environ['XNAT_URI']
else:
raise ValueError('no URI specified')
if user is not None:
self._user = user
elif 'XNAT_USER' in os.environ:
self._user = os.environ['XNAT_USER']
else:
raise ValueError('no user specified')
if password is not None:
pass
elif 'XNAT_PASSWORD' in os.environ:
user = os.environ['XNAT_PASSWORD']
else:
raise ValueError('no password specified')
self._pyxnat_interface = Interface(server=self._uri,
user=self._user,
password=password)
self._pyxnat_interface._get_entry_point()
self._projects = []
self._accessible_projects = []
return
def is_connected(self):
if self._pyxnat_interface:
return True
return False
def close(self):
uri = self._pyxnat_interface._get_entry_point() + '/JSESSION'
self._pyxnat_interface._exec(uri, 'DELETE')
self._pyxnat_interface = None
return
def set_projects(self):
for project in self._pyxnat_interface.select.projects():
#print project.id()
#self._projects[project.id()] = Project(self, project.id())
if self._user in project.users():
self._accessible_projects.append(project.id())
#self._projects[project.id()] = project.user_role(self._user)
self._projects.append(project.id())
@property
def accessible_projects(self):
if not self._accessible_projects:
self.set_projects()
return self._accessible_projects
@property
def projects(self):
if not self._projects:
self.set_projects()
return self._projects
@property
def pyxnat_interface(self):
return self._pyxnat_interface
)
group.add_argument("-comments", action="store", dest="comments", required=True, help="comments by uploader")
group.add_argument("-configfile", action="store", dest="configfile", required=True, help="configfile of uploader")
inputArguments = parser.parse_args()
scanid = inputArguments.scanid
bblid = inputArguments.bblid
dotest = inputArguments.dotest
logfile = inputArguments.logfile
test = inputArguments.test
siteid = inputArguments.siteid
study_xnat_name = inputArguments.study_xnat_name
comments = inputArguments.comments
configfile = inputArguments.configfile
central = Interface(config=configfile)
# imagescan_id='1234'
# logfile='/import/monstrum/Users/hopsonr/07001-Jolo_1.00.log'
###define functions###
######################
# get shortened lowercase filename from path
def short(filename):
index = filename.rfind("/") + 1
last = len(filename)
filename = filename[index:last]
filename = filename.lower()
return filename
def upload(direct):
#Subject, Experiment y Scan se llaman como el nombre entero de la secuencia
#Si un metadata se encuentra vacío [], no se guarda ese field por defecto
#Desde el XML en XNAT se pueden ver los fields
#BW se ha cambiado a bw, XNAT no emplea bien las mayúsculas parece ser
#Cada archivo le cuesta 26 segundos para subirse
#Conexión XNAT
session = xnat.connect('http://localhost', user='******', password='******')
interface = Interface(server='http://localhost',
user='******',
password='******') #Para SNAPSHOTS
print("Connected to XNAT with API")
#PROYECTO tiene que existir en XNAT
project = session.projects['physio']
project_snapshot = interface.select.project(
'physio').insert() #Para SNAPSHOTS
#Seteamos el directorio en la carpeta de datos
os.chdir(direct)
#1-Accedemos a los files y guardamos sus nombres
files = os.listdir()
nii = 0
mat = 0
for i in files:
if i.find('nii') != -1:
nii = i
elif i.find('mat') != -1:
mat = i
#Si solo hay .mat, se sube solo .mat
if nii == 0 and mat != 0:
print("Only .mat file detected")
#Eliminamos .nii y .mat de los nombres
filename = mat.split('.mat')[0]
filename_date = mat.split('.mat')[0]
filename = filename.replace(".", "-")
s_label = filename
#2-subject
subject = session.classes.SubjectData(parent=project, label=s_label)
#3-experiment
experiment = session.classes.MrSessionData(parent=subject,
label=filename)
#Si queremos añadir FECHA
array_date = filename_date.split('.')[1:4]
date = array_date[0] + "-" + array_date[1] + "-" + array_date[2]
experiment.date = date
#4-scan
scan = session.classes.MrScanData(parent=experiment,
id=filename,
type='MRI')
print("Subject, Experiment and Scan created")
#6-Resource y subir .mat
try:
#XNATResponseError (status 409)
resource_MAT = session.classes.ResourceCatalog(parent=scan,
label='MAT',
type='mat')
resource_MAT.upload(mat, mat)
print(".mat uploaded")
except:
resource_MAT = scan.resources['MAT']
resource_MAT.upload(mat, mat)
print(".mat already exists")
#7-METADATA
metadata = scipy.io.loadmat(mat)
if 'BW' in metadata.keys():
metadata["bw"] = metadata.pop("BW")
del metadata["image3D"]
del metadata["kSpace3D"]
del metadata["dataFull"]
del metadata["imgFull"]
del metadata["sampled"]
del metadata["__header__"]
del metadata["__version__"]
del metadata["__globals__"]
del metadata["phGradients"]
# del metadata["average"]
# del metadata["rawdata"]
for key in metadata.keys():
experiment.fields[key] = metadata.get(key)
session.disconnect()
interface.disconnect()
elif nii != 0 and mat != 0:
print(".mat and .nii files detected")
#Eliminamos .nii y .mat de los nombres
filename = nii.split('.nii')[0]
filename_date = nii.split('.nii')[0]
filename = filename.replace(".", "-")
s_label = filename.split('-')[0]
exp_label = filename.split('-')[1]
#2-subject
subject = session.classes.SubjectData(parent=project, label=s_label)
#3-experiment
experiment = session.classes.MrSessionData(parent=subject,
label=exp_label)
#Si queremos añadir FECHA
array_date = filename_date.split('.')[2:5]
date = array_date[0] + "-" + array_date[1] + "-" + array_date[2]
print(date)
experiment.date = date
#4-scan
scan = session.classes.MrScanData(parent=experiment,
id=filename,
type='MRI')
print("Subject, Experiment and Scan created")
#5-Resource y subir NIFTI
#Actualizamos el NIFTI si ya existe
try:
#XNATResponseError (status 409)
resource_NIFTI = session.classes.ResourceCatalog(parent=scan,
label='NIFTI',
type='nii')
resource_NIFTI.upload(nii, nii)
print("NIFTI uploaded")
except:
resource_NIFTI = scan.resources['NIFTI']
resource_NIFTI.upload(nii, nii)
print("NIFTI already exists")
#6-Resource y subir .mat
try:
#XNATResponseError (status 409)
resource_MAT = session.classes.ResourceCatalog(parent=scan,
label='MAT',
type='mat')
resource_MAT.upload(mat, mat)
print(".mat uploaded")
except:
resource_MAT = scan.resources['MAT']
resource_MAT.upload(mat, mat)
print(".mat already exists")
#7-METADATA
metadata = scipy.io.loadmat(mat)
if 'BW' in metadata.keys():
metadata["bw"] = metadata.pop("BW")
del metadata["image3D"]
del metadata["kSpace3D"]
del metadata["dataFull"]
del metadata["imgFull"]
del metadata["sampled"]
del metadata["__header__"]
del metadata["__version__"]
del metadata["__globals__"]
del metadata["phGradients"]
for key in metadata.keys():
experiment.fields[key] = metadata.get(key)
print("metadata uploaded")
# =============================================================================
# SNAPSHOTS
# =============================================================================
NIFTI_img = nib.load(nii)
data = NIFTI_img.get_fdata()
#Creamos directorio para guardar imágenes dentro de cada carpeta y será destruido
#Comprobamos si existe el directorio por error y se borra para crearlo de nuevo
if os.path.isdir("SNAPSHOTS") == False:
os.mkdir("SNAPSHOTS")
os.chdir("SNAPSHOTS")
else:
shutil.rmtree("SNAPSHOTS")
os.mkdir("SNAPSHOTS")
os.chdir("SNAPSHOTS")
if data.ndim != 3:
print(
"Se espera un número de 3 dimensiones"
) #################################MESSAGE FUNCTION#####################################
else:
#Dimensión donde se encuentra el min
shape = np.array(data.shape)
for i in list(range(data.ndim)):
if shape[i] == min(shape):
dim_10 = i
images = []
for i in list(range(min(shape))):
if dim_10 == 0:
data_1 = np.array(data[i, :, :])
elif dim_10 == 1:
data_1 = np.array(data[:, i, :])
elif dim_10 == 2:
data_1 = np.array(data[:, :, i])
imsave(str(i) + '.gif', data_1)
images.append(imageio.imread(str(i) + '.gif'))
imageio.mimsave(filename + '.gif', images)
#Subir snapshot
subject_snapshot = project_snapshot.subject(s_label)
experiment = subject_snapshot.experiment(exp_label)
experiment.scan(filename).resource('SNAPSHOTS').file(
filename + '.gif').insert(filename + '.gif',
format="GIF",
content="ORIGINAL",
tags=None)
experiment.scan(filename).resource('SNAPSHOTS').file(
filename + '_t.gif').insert(filename + '.gif',
format="GIF",
content="THUMBNAIL",
tags=None)
os.chdir("../")
shutil.rmtree("SNAPSHOTS")
print("Snapshots uploaded")
# =============================================================================
# FIN SNAPSHOTS
# =============================================================================
session.disconnect()
interface.disconnect()
import os.path as op
from uuid import uuid1
import time
from . import skip_if_no_network
from pyxnat import Interface
from pyxnat.core import interfaces
_modulepath = op.dirname(op.abspath(__file__))
fp = op.join(op.dirname(op.abspath(__file__)), 'central.cfg')
central = Interface(config=fp)
interfaces.STUBBORN = True
sid = uuid1().hex
eid = uuid1().hex
subject = central.select.project('nosetests5').subject(sid)
experiment = subject.experiment(eid)
@skip_if_no_network
def test_01_fancy_resource_create():
field_data = {
'experiment': 'xnat:mrSessionData',
'ID': 'TEST_%s' % eid,
'xnat:mrSessionData/age': '42',
'xnat:subjectData/investigator/lastname': 'doe',
'xnat:subjectData/investigator/firstname': 'john',
'xnat:subjectData/ID': 'TEST_%s' % sid,
}
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
from pyxnat import Interface
# Change this for different projects
P = '2096'
# Passing no arguments, this will ask for values at the commandline
xnat = Interface()
# Or, you can pass installation URL, username, password
# xnat = Interface('http://masi.vuse.vanderbilt.edu', 'sburns', MYPASSWORD)
"""
xnat.select is the most useful function
You can select by full paths:
project = xnat.select('/project/MYPROJ')
subject = xnat.select('/project/MYPROJ/subject/MYSUBJ')
all_subjects = xnat.select('/project/MYPROJ/subject/*')
"""
project = xnat.select.project(P)
all_subjects = xnat.select('/project/%s/subjects/*' % P)
# clean up
projdir = DEST_BASE
projdir += '/'
projdir += pss[-1]
shutil.rmtree(projdir)
os.remove(outvol)
passFile.write(nr + '\n')
else:
print 'conversion of ', nr, ' does not match'
errFile.write(nr + '\n')
except KeyboardInterrupt:
print 'Keyboard Interrupt'
pass
except:
print ''
print 'Error during processing'
passFile.close()
errFile.close()
failedConversions.close()
if __name__ == '__main__':
xnatUrl = 'https://www.predict-hd.net/xnat'
XNAT = Interface(server=xnatUrl, user='******', cachedir=CACHE_DIR)
if not os.path.isfile(ALL_SCANS):
write_nrrds_to_file(ALL_SCANS)
nrrds = get_all_nrrds_from_file(ALL_SCANS)
main(XNAT, nrrds)
pass
def setup_xnat_connection(configfile, run_by_pipeline):
if configfile=='X' and run_by_pipeline == 1:
configfile='/import/monstrum/Users/xnat/.xnat-localhost.cfg'
central = Interface(config=configfile)
return central
elif os.getenv('HOSTNAME') == 'xnat.uphs.upenn.edu' and run_by_pipeline == 0:
homedir=os.getenv("HOME")
if os.path.isfile(homedir+'/.xnat-localhost.cfg'):
print "Found ~/.xnat-localhost.cfg on xnat"
central = Interface(config=homedir+'/.xnat-localhost.cfg')
return central
else:
print "Login using your XNAT username and password, this will be saved in a configuration file for next time."
try:
central = Interface(config=configfile)
except AttributeError:
central = Interface(server='http://xnat.uphs.upenn.edu:8080/xnat')
central.save_config(homedir+'/.xnat-localhost.cfg')
return central
elif configfile=='X' and run_by_pipeline == 0:
homedir=os.getenv("HOME")
if os.path.isfile(homedir+'/.xnat.cfg'):
print "Found ~/.xnat.cfg"
central = Interface(config=homedir+'/.xnat.cfg')
return central
else:
print "Login using your XNAT username and password, this will be saved in a configuration file for next time."
try:
central = Interface(config=configfile)
except AttributeError:
central = Interface(server='https://xnat.uphs.upenn.edu/xnat')
central.save_config(homedir+'/.xnat.cfg')
return central
else:
try:
central = Interface(config=configfile)
except AttributeError, e:
print "Error with the configfile you specified: " + str(e)
sys.exit(1)
from uuid import uuid1
from pyxnat import manage
from pyxnat import Interface
import os.path as op
x = Interface(config=op.join(op.dirname(op.abspath(__file__)), 'central.cfg'))
def test_project_manager():
pm = manage.ProjectManager('nosetests', x)
import sys
from pyxnat import Interface
import argparse, getpass
parser = argparse.ArgumentParser(description='Returns project specific qa_epi10 module data: get_xnat_epi10.py -configfile ~/.xnat.cfg -project EONS_810366');
group = parser.add_argument_group('Required')
group.add_argument('-configfile', action="store", dest='configfile', required=True, help='Path to .xnat.cfg')
group.add_argument('-project', action="store", dest='project', required=True, help='Project name as in xnat')
parser.add_argument('-version', action='version', version='%(prog)s 3.0')
inputArguments = parser.parse_args()
configfile = inputArguments.configfile
project = inputArguments.project
central = Interface(config=configfile)
X = central.select('bbl:epi10').where([('bbl:epi10/PROJECT','=',project),'AND'])
epi10_fields=['subject_id','scan_date','tr','func_seqname','imagescan_id','session_id','project','meanrelrms','maxrelrms','tsnr2','spikerate2','spikepts2','gsnr3','gmean2','drift2','nclip','bbl_epi10_externalid']
Y = central.select('bbl:Sequence').where([('bbl:Sequence/PROJECT','=',project),'AND'])
scan_info={}
for i in Y:
scan_info[str(i.get('session_id')+'_'+i.get('imagescan_id'))]=i.get('qlux_qluxname')
D=""
for j in epi10_fields:
D=D+str(j)+','
D=D+str('qlux_qluxname')
print D
