def post(self):
import pluricent as pl
import numpy as np
import json
from sqlalchemy import distinct
structure = self.get_argument('id')
print structure
args = {}
p = pl.Pluricent(pl.global_settings()['database'])
structures = [
str(e[0]) for e in p.session.query(
distinct(pl.models.Measurement.structure)).all()
]
measurements = dict(p.session.query(pl.models.Subject.identifier, pl.models.Measurement.value)\
.join(pl.models.T1Image).join(pl.models.Measurement).filter(pl.models.Measurement.structure == structure).all())
args['data'], args['labels'] = np.histogram(measurements.values())
args = dict([(k, json.dumps([int(e) for e in v.tolist()]))
for k, v in args.items()])
args['structure'] = structure
args['structures'] = structures
args['measurements'] = measurements
res = json.dumps(args)
self.write(res)
return None
def get(self):
username = self.current_user[1:-1]
import pluricent as pl
import os.path as osp
import json
warning = ''
default = True
# Retrieving studies from the database
p = pl.Pluricent(pl.global_settings()['database'])
studies = p.studies()
if 'study' in self.request.arguments:
# Study selected: displaying more info
study = self.get_argument('study')
if study in studies:
images = p.t1images(study)
print study, images
d = []
for each in images:
d.append({'subject': each.subject.identifier,
'path': each.path,
'id': each.id})
self.render("html/explore_study.html", username = username, study_name = study, warning = warning, images=d)
return
else:
warning = 'invalid study'
default = True
if default:
# Welcome page for study selection
studies = json.dumps(studies)
self.render("html/explore.html", username = username, studies = studies, warning = warning)
def post(self):
import pluricent as pl
from sqlalchemy import func, distinct
import numpy as np
import json
args = {}
p = pl.Pluricent(pl.global_settings()['database'])
datatypes = [e[0] for e in p.session.query(distinct(pl.models.Processing.datatype)).all()]
table = []
headers = ['subject', 't1image']
q = []
q.append(dict(p.session.query(pl.models.Subject.identifier, func.count(pl.models.T1Image.path)).filter(pl.models.T1Image.subject_id == pl.models.Subject.id).group_by(pl.models.Subject.identifier).all()))
for each in datatypes:
headers.append(each)
res = p.session.query(pl.models.Subject.identifier, func.count(pl.models.Processing.path)).join(pl.models.T1Image).filter(pl.models.Processing.input_id == pl.models.T1Image.id).filter(pl.models.T1Image.subject_id == pl.models.Subject.id).filter(pl.models.Processing.datatype==each).group_by(pl.models.Subject.identifier).all()
q.append(dict(res))
subjects = [e[0] for e in p.session.query(pl.models.Subject.identifier).all()]
table.append(headers)
for s in subjects:
table.append([s])
table[-1].extend([each.get(s, 0) for each in q])
#print t1images
args['images'] = table
res = json.dumps(args)
self.write(res)
return None
def test_database_exists():
''' Returns False if the database as defined in pluricent.settings
is missing'''
import os.path as osp
db = global_settings()['database']
print db
return osp.isfile(db)
def test_login_logout():
''' Performs 3 tests in a row :
- login with wrong credentials
- login with correct credentials
- logout
Returns True if the 3 tests are successful'''
import requests
print 'Check the server is running'
baseurl = global_settings()['baseurl']
url = baseurl + 'auth/login/'
data = {'username': '******', 'password':'******'}
try:
r = requests.post(url, data=data)
res = r.url == baseurl + 'auth/login/?error=Login+incorrect'
print res
data = credentials
r = requests.post(url, data=data)
res = res and '<a id="logout"' in r.text
print r.text, res
url = baseurl + 'auth/logout/'
r = requests.get(url)
res = res and baseurl + 'auth/login/' in r.url
print res
except Exception as e:
res = False
print e
return res
def test_database_exists():
''' Returns False if the database as defined in pluricent.settings
is missing'''
import os.path as osp
db = global_settings()['database']
print db
return osp.isfile(db)
def test_login_logout():
''' Performs 3 tests in a row :
- login with wrong credentials
- login with correct credentials
- logout
Returns True if the 3 tests are successful'''
import requests
print 'Check the server is running'
baseurl = global_settings()['baseurl']
url = baseurl + 'auth/login/'
data = {'username': '******', 'password': '******'}
try:
r = requests.post(url, data=data)
res = r.url == baseurl + 'auth/login/?error=Login+incorrect'
print res
data = credentials
r = requests.post(url, data=data)
res = res and '<a id="logout"' in r.text
print r.text, res
url = baseurl + 'auth/logout/'
r = requests.get(url)
res = res and baseurl + 'auth/login/' in r.url
print res
except Exception as e:
res = False
print e
return res
def get(self):
username = self.current_user[1:-1]
import pluricent as pl
from sqlalchemy import distinct
args = {}
p = pl.Pluricent(pl.global_settings()['database'])
structures = [str(e[0]) for e in p.session.query(distinct(pl.models.Measurement.structure)).all()]
args['structures'] = structures
self.render("html/analyze.html", username = username, **args)
def test_datasource_exists():
''' Returns False if the datasource as defined in the database
is missing'''
import os.path as osp
db = global_settings()['database']
print db
p = pl.Pluricent(db)
ds = osp.dirname(db)
return osp.isdir(ds)
def test_datasource_exists():
''' Returns False if the datasource as defined in the database
is missing'''
import os.path as osp
db = global_settings()['database']
print db
p = pl.Pluricent(db)
ds = osp.dirname(db)
return osp.isdir(ds)
def test_valid_study(): #FIXME import requests baseurl = global_settings()['baseurl'] url = baseurl + 'auth/login/' r = requests.post(url, data=credentials) url = baseurl + 'explore/?study=study01' r = requests.get(url) res = 'invalid study' in r.text return res
def test_valid_study():
#FIXME
import requests
baseurl = global_settings()['baseurl']
url = baseurl + 'auth/login/'
r = requests.post(url, data=credentials)
url = baseurl + 'explore/?study=study01'
r = requests.get(url)
res = 'invalid study' in r.text
return res
def test_studies():
''' Compares studies between the repository and the database '''
import os, os.path as osp
db = global_settings()['database']
p = pl.Pluricent(db)
ds = p.datasource()
studies_db = list(set([p.study_dir(e) for e in p.studies()]))
studies_ds = list(set([e for e in os.listdir(ds) \
if osp.isdir(osp.join(ds,e)) and not e in ['.','..']]))
print 'studies in database:', studies_db, '- studies in repo:', studies_ds
return studies_db == studies_ds
def test_studies():
''' Compares studies between the repository and the database '''
import os, os.path as osp
db = global_settings()['database']
p = pl.Pluricent(db)
ds = p.datasource()
studies_db = list(set([p.study_dir(e) for e in p.studies()]))
studies_ds = list(set([e for e in os.listdir(ds) \
if osp.isdir(osp.join(ds,e)) and not e in ['.','..']]))
print 'studies in database:', studies_db, '- studies in repo:', studies_ds
return studies_db == studies_ds
def get(self):
username = self.current_user[1:-1]
import pluricent as pl
from sqlalchemy import distinct
args = {}
p = pl.Pluricent(pl.global_settings()['database'])
structures = [
str(e[0]) for e in p.session.query(
distinct(pl.models.Measurement.structure)).all()
]
args['structures'] = structures
self.render("html/analyze.html", username=username, **args)
def get(self):
username = self.current_user[1:-1]
import pluricent as pl
import os.path as osp
fn = osp.abspath(pl.global_settings()['database'])
ds = osp.dirname(fn)
args = {'danger': '', 'datasource': '', 'database': fn}
if osp.isfile(fn):
args.update({'datasource': ds, 'database': fn})
else:
args['danger'] = 'The database %s is missing' % fn
self.render("html/index.html", username=username, **args)
def get(self):
username = self.current_user[1:-1]
import pluricent as pl
import os.path as osp
fn = osp.abspath(pl.global_settings()['database'])
ds = osp.dirname(fn)
args = {'danger':'', 'datasource':'', 'database':fn}
if osp.isfile(fn):
args.update({'datasource': ds, 'database': fn})
else:
args['danger'] = 'The database %s is missing'%fn
self.render("html/index.html", username = username, **args)
def test_respect_hierarchy():
''' Checks that every file/folder in the repository is identified by the hierarchy
Returns True if the unknown list is empty'''
from pluricent import checkbase as cb
import os.path as osp
db = global_settings()['database']
print 'db', db
p = pl.Pluricent(db)
destdir = osp.dirname(db)
from pluricent import tests as t
return t.test_respect_hierarchy(destdir)
def test_unique_subjects():
''' Checks that subjects identifiers are unique in every study '''
import os, os.path as osp
db = global_settings()['database']
p = pl.Pluricent(db)
studies = p.studies()
unique = True
for each in studies:
subjects = p.subjects(each)
if not len(subjects) == len(set(subjects)):
unique = False
print each
return unique
def test_respect_hierarchy():
''' Checks that every file/folder in the repository is identified by the hierarchy
Returns True if the unknown list is empty'''
from pluricent import checkbase as cb
import os.path as osp
db = global_settings()['database']
print 'db', db
p = pl.Pluricent(db)
destdir = osp.dirname(db)
from pluricent import tests as t
return t.test_respect_hierarchy(destdir)
def test_unique_subjects():
''' Checks that subjects identifiers are unique in every study '''
import os, os.path as osp
db = global_settings()['database']
p = pl.Pluricent(db)
studies = p.studies()
unique = True
for each in studies:
subjects = p.subjects(each)
if not len(subjects) == len(set(subjects)):
unique = False
print each
return unique
def test_invalid_study(): ''' Logs in and acts as if just asked for a non-existing study. Returns True if successfully redirected with an information message''' import requests baseurl = global_settings()['baseurl'] url = baseurl + 'auth/login/' s = requests.Session() r = s.post(url, data=credentials) url = baseurl + 'explore/?study=toto' print url r = s.get(url) print r.text res = 'invalid study' in r.text return res
def test_invalid_study():
''' Logs in and acts as if just asked for a non-existing study.
Returns True if successfully redirected with an information message'''
import requests
baseurl = global_settings()['baseurl']
url = baseurl + 'auth/login/'
s = requests.Session()
r = s.post(url, data=credentials)
url = baseurl + 'explore/?study=toto'
print url
r = s.get(url)
print r.text
res = 'invalid study' in r.text
return res
def test_each_entry_has_action():
''' Checks every entry in tables Subject, T1Image, Study
has an associated action in Action.
The opposite way is not verified.'''
import pluricent as pl
import json
p = pl.Pluricent(pl.global_settings()['database'])
studies = p.studies()
actions = [json.loads(each.action) for each in p.actions()]
each_has_action = True
# sorting actions
recognized = ['add_study', 'add_subject', 'add_image']
sorted_actions = {}
for a in actions:
if a[0] in recognized:
sorted_actions.setdefault(a[0], []).append(a)
for s in studies:
found = 0
for each in sorted_actions['add_study']:
action_type, params = each
if s == params['name']:
found += 1
if found != 1:
each_has_action = False
print s, 'found %s times' % found
for subject in p.subjects(s):
found = 0
for each in sorted_actions['add_subject']:
action_type, params = each
if s == params['study'] and subject in params['subjects']:
found += 1
if found != 1:
each_has_action = False
print subject, 'in', s, 'found %s times' % found
for image in p.t1images(s, subject):
found = 0
for each in sorted_actions['add_image']:
action_type, params = each
if s == params['study'] and subject == params[
'subject'] and image.path == params['path']:
found += 1
if found != 1:
each_has_action = False
print image, 'from', subject, 'in', s, 'found %s times' % found
#TODO Verify every Action has associated entry in appropriate tables
return each_has_action
def post(self):
import pluricent as pl
from sqlalchemy import func, distinct
import numpy as np
import json
args = {}
p = pl.Pluricent(pl.global_settings()['database'])
datatypes = [
e[0] for e in p.session.query(
distinct(pl.models.Processing.datatype)).all()
]
table = []
headers = ['subject', 't1image']
q = []
q.append(
dict(
p.session.query(pl.models.Subject.identifier,
func.count(pl.models.T1Image.path)).filter(
pl.models.T1Image.subject_id ==
pl.models.Subject.id).group_by(
pl.models.Subject.identifier).all()))
for each in datatypes:
headers.append(each)
res = p.session.query(
pl.models.Subject.identifier,
func.count(pl.models.Processing.path)
).join(pl.models.T1Image).filter(
pl.models.Processing.input_id == pl.models.T1Image.id).filter(
pl.models.T1Image.subject_id == pl.models.Subject.id
).filter(pl.models.Processing.datatype == each).group_by(
pl.models.Subject.identifier).all()
q.append(dict(res))
subjects = [
e[0] for e in p.session.query(pl.models.Subject.identifier).all()
]
table.append(headers)
for s in subjects:
table.append([s])
table[-1].extend([each.get(s, 0) for each in q])
#print t1images
args['images'] = table
res = json.dumps(args)
self.write(res)
return None
def test_units():
''' Checks that same variables have same units'''
p = pl.Pluricent(pl.global_settings()['database'])
are_units_valid = True
units = {}
for m in p.measurements():
if m.measurement in units.keys():
if m.unit != units[m.measurement]:
are_units_valid = False
print '%s has unit %s and %s'%(m.measurement, m.unit, units[m.measurement])
break
else:
units[m.measurement] = m.unit
return are_units_valid
def test_each_entry_has_action():
''' Checks every entry in tables Subject, T1Image, Study
has an associated action in Action.
The opposite way is not verified.'''
import pluricent as pl
import json
p = pl.Pluricent(pl.global_settings()['database'])
studies = p.studies()
actions = [json.loads(each.action) for each in p.actions()]
each_has_action = True
# sorting actions
recognized = ['add_study', 'add_subject', 'add_image']
sorted_actions = {}
for a in actions:
if a[0] in recognized:
sorted_actions.setdefault(a[0], []).append(a)
for s in studies:
found = 0
for each in sorted_actions['add_study']:
action_type, params = each
if s == params['name']:
found += 1
if found != 1:
each_has_action = False
print s, 'found %s times'%found
for subject in p.subjects(s):
found = 0
for each in sorted_actions['add_subject']:
action_type, params = each
if s == params['study'] and subject in params['subjects']:
found += 1
if found != 1:
each_has_action = False
print subject, 'in', s, 'found %s times'%found
for image in p.t1images(s, subject):
found = 0
for each in sorted_actions['add_image']:
action_type, params = each
if s == params['study'] and subject == params['subject'] and image.path == params['path']:
found += 1
if found != 1:
each_has_action = False
print image, 'from', subject, 'in', s, 'found %s times'%found
#TODO Verify every Action has associated entry in appropriate tables
return each_has_action
def test_actions():
''' Checks that every action starts with a recognized code
e.g. add_study, add_subject, add_image...'''
import pluricent as pl
import json
p = pl.Pluricent(pl.global_settings()['database'])
actions = [json.loads(each.action) for each in p.actions()]
authorized = True
recognized = ['add_study', 'add_subject', 'add_image']
for each in actions:
if not each[0] in recognized:
authorized = False
print each, 'not recognized'
break
return authorized
def test_actions():
''' Checks that every action starts with a recognized code
e.g. add_study, add_subject, add_image...'''
import pluricent as pl
import json
p = pl.Pluricent(pl.global_settings()['database'])
actions = [json.loads(each.action) for each in p.actions()]
authorized = True
recognized = ['add_study', 'add_subject', 'add_image']
for each in actions:
if not each[0] in recognized:
authorized = False
print each, 'not recognized'
break
return authorized
def test_units():
''' Checks that same variables have same units'''
p = pl.Pluricent(pl.global_settings()['database'])
are_units_valid = True
units = {}
for m in p.measurements():
if m.measurement in units.keys():
if m.unit != units[m.measurement]:
are_units_valid = False
print '%s has unit %s and %s' % (m.measurement, m.unit,
units[m.measurement])
break
else:
units[m.measurement] = m.unit
return are_units_valid
def test_measurements_format():
''' Checks headers in measurements files '''
have_good_format = True
import os.path as osp
from pluricent import checkbase as cb
cl = cb.CloudyCheckbase(osp.dirname(pl.global_settings()['database']))
f = cl.get_files_of_type('measurements')
print f
from pluricent import tests
for each in f:
has_good_format = tests.test_measurements_format(each)
if not has_good_format:
print each, 'has a wrong format'
have_good_format = False
break
return have_good_format
def test_measurements_format():
''' Checks headers in measurements files '''
have_good_format = True
import os.path as osp
from pluricent import checkbase as cb
cl = cb.CloudyCheckbase(osp.dirname(pl.global_settings()['database']))
f = cl.get_files_of_type('measurements')
print f
from pluricent import tests
for each in f:
has_good_format = tests.test_measurements_format(each)
if not has_good_format:
print each, 'has a wrong format'
have_good_format = False
break
return have_good_format
def test_matching_t1images():
''' Checks if T1 images entries in the database are matching with
existing files in the repository'''
from pluricent import checkbase as cb
import os.path as osp
db = global_settings()['database']
p = pl.Pluricent(db)
destdir = osp.dirname(db)
cl = cb.CloudyCheckbase(destdir)
import os
import os.path as osp
unknown = []
scanned = 0
print destdir
raw_files = []
for root, dirs, files in os.walk(destdir):
for f in files:
scanned += 1
fp = osp.join(root, f)
res = cb.parsefilepath(fp, cl.patterns)
if not res is None:
datatype, att = res
if datatype == 'raw':
raw_files.append(fp[len(destdir)+1:])
raw_entries = [e.path for e in p.t1images()]
# comparing raw_files and raw_entries
matching = True
for f in raw_files:
if not f in raw_entries:
print f, 'missing from raw_entries'
matching = False
for f in raw_entries:
if not f in raw_files:
print f, 'missing from raw_files'
matching = False
print 'items in %s :'%destdir, scanned
print 'entries in db:', len(raw_entries)
return matching
def test_matching_t1images():
''' Checks if T1 images entries in the database are matching with
existing files in the repository'''
from pluricent import checkbase as cb
import os.path as osp
db = global_settings()['database']
p = pl.Pluricent(db)
destdir = osp.dirname(db)
cl = cb.CloudyCheckbase(destdir)
import os
import os.path as osp
unknown = []
scanned = 0
print destdir
raw_files = []
for root, dirs, files in os.walk(destdir):
for f in files:
scanned += 1
fp = osp.join(root, f)
res = cb.parsefilepath(fp, cl.patterns)
if not res is None:
datatype, att = res
if datatype == 'raw':
raw_files.append(fp[len(destdir) + 1:])
raw_entries = [e.path for e in p.t1images()]
# comparing raw_files and raw_entries
matching = True
for f in raw_files:
if not f in raw_entries:
print f, 'missing from raw_entries'
matching = False
for f in raw_entries:
if not f in raw_files:
print f, 'missing from raw_files'
matching = False
print 'items in %s :' % destdir, scanned
print 'entries in db:', len(raw_entries)
return matching
def get(self):
username = self.current_user[1:-1]
import pluricent as pl
import os.path as osp
import json
warning = ''
default = True
# Retrieving studies from the database
p = pl.Pluricent(pl.global_settings()['database'])
studies = p.studies()
if 'study' in self.request.arguments:
# Study selected: displaying more info
study = self.get_argument('study')
if study in studies:
images = p.t1images(study)
print study, images
d = []
for each in images:
d.append({
'subject': each.subject.identifier,
'path': each.path,
'id': each.id
})
self.render("html/explore_study.html",
username=username,
study_name=study,
warning=warning,
images=d)
return
else:
warning = 'invalid study'
default = True
if default:
# Welcome page for study selection
studies = json.dumps(studies)
self.render("html/explore.html",
username=username,
studies=studies,
warning=warning)
def post(self):
import pluricent as pl
import numpy as np
import json
from sqlalchemy import distinct
structure = self.get_argument('id')
print structure
args = {}
p = pl.Pluricent(pl.global_settings()['database'])
structures = [str(e[0]) for e in p.session.query(distinct(pl.models.Measurement.structure)).all()]
measurements = dict(p.session.query(pl.models.Subject.identifier, pl.models.Measurement.value)\
.join(pl.models.T1Image).join(pl.models.Measurement).filter(pl.models.Measurement.structure == structure).all())
args['data'], args['labels'] = np.histogram(measurements.values())
args = dict([(k, json.dumps([int(e) for e in v.tolist()])) for k,v in args.items()])
args['structure'] = structure
args['structures'] = structures
args['measurements'] = measurements
res = json.dumps(args)
self.write(res)
return None
parser.add_argument("-t", dest='test_name', type=str, help="Run specific test (in debug mode)", required=False)
args = parser.parse_args()
# ==== run specific test ? ====
if args.test_name:
if not args.test_name in results.keys():
raise Exception('%s should refer to an existing test (%s)'%(args.test_name, results.keys()))
res = run_test(args.test_name)
print '=== %s : %s'%(args.test_name, res)
import sys
sys.exit(0)
# ==== run tests ====
from datetime import datetime
results['last_checked'] = datetime.now().isoformat()
results = run_tests(results, args.debug)
import json
import os.path as osp
print '===================================================================================================='
fp = osp.abspath(osp.join(osp.dirname(pl.__file__), '..', '..', 'web', 'json', 'sysdiag.json'))
print 'Writing json... %s'%fp
json.dump(results, open(fp, 'w'), indent=2)
print '====================================================================='
print 'baseurl:', global_settings()['baseurl']
print 'database:', global_settings()['database']
print '====================================================================='
args = parser.parse_args()
# ==== run specific test ? ====
if args.test_name:
if not args.test_name in results.keys():
raise Exception('%s should refer to an existing test (%s)' %
(args.test_name, results.keys()))
res = run_test(args.test_name)
print '=== %s : %s' % (args.test_name, res)
import sys
sys.exit(0)
# ==== run tests ====
from datetime import datetime
results['last_checked'] = datetime.now().isoformat()
results = run_tests(results, args.debug)
import json
import os.path as osp
print '===================================================================================================='
fp = osp.abspath(
osp.join(osp.dirname(pl.__file__), '..', '..', 'web', 'json',
'sysdiag.json'))
print 'Writing json... %s' % fp
json.dump(results, open(fp, 'w'), indent=2)
print '====================================================================='
print 'baseurl:', global_settings()['baseurl']
print 'database:', global_settings()['database']
print '====================================================================='
