def post(self): import pluricent as pl import numpy as np import json from sqlalchemy import distinct structure = self.get_argument('id') print structure args = {} p = pl.Pluricent(pl.global_settings()['database']) structures = [ str(e[0]) for e in p.session.query( distinct(pl.models.Measurement.structure)).all() ] measurements = dict(p.session.query(pl.models.Subject.identifier, pl.models.Measurement.value)\ .join(pl.models.T1Image).join(pl.models.Measurement).filter(pl.models.Measurement.structure == structure).all()) args['data'], args['labels'] = np.histogram(measurements.values()) args = dict([(k, json.dumps([int(e) for e in v.tolist()])) for k, v in args.items()]) args['structure'] = structure args['structures'] = structures args['measurements'] = measurements res = json.dumps(args) self.write(res) return None
def get(self): username = self.current_user[1:-1] import pluricent as pl import os.path as osp import json warning = '' default = True # Retrieving studies from the database p = pl.Pluricent(pl.global_settings()['database']) studies = p.studies() if 'study' in self.request.arguments: # Study selected: displaying more info study = self.get_argument('study') if study in studies: images = p.t1images(study) print study, images d = [] for each in images: d.append({'subject': each.subject.identifier, 'path': each.path, 'id': each.id}) self.render("html/explore_study.html", username = username, study_name = study, warning = warning, images=d) return else: warning = 'invalid study' default = True if default: # Welcome page for study selection studies = json.dumps(studies) self.render("html/explore.html", username = username, studies = studies, warning = warning)
def post(self): import pluricent as pl from sqlalchemy import func, distinct import numpy as np import json args = {} p = pl.Pluricent(pl.global_settings()['database']) datatypes = [e[0] for e in p.session.query(distinct(pl.models.Processing.datatype)).all()] table = [] headers = ['subject', 't1image'] q = [] q.append(dict(p.session.query(pl.models.Subject.identifier, func.count(pl.models.T1Image.path)).filter(pl.models.T1Image.subject_id == pl.models.Subject.id).group_by(pl.models.Subject.identifier).all())) for each in datatypes: headers.append(each) res = p.session.query(pl.models.Subject.identifier, func.count(pl.models.Processing.path)).join(pl.models.T1Image).filter(pl.models.Processing.input_id == pl.models.T1Image.id).filter(pl.models.T1Image.subject_id == pl.models.Subject.id).filter(pl.models.Processing.datatype==each).group_by(pl.models.Subject.identifier).all() q.append(dict(res)) subjects = [e[0] for e in p.session.query(pl.models.Subject.identifier).all()] table.append(headers) for s in subjects: table.append([s]) table[-1].extend([each.get(s, 0) for each in q]) #print t1images args['images'] = table res = json.dumps(args) self.write(res) return None
def test_database_exists(): ''' Returns False if the database as defined in pluricent.settings is missing''' import os.path as osp db = global_settings()['database'] print db return osp.isfile(db)
def test_login_logout(): ''' Performs 3 tests in a row : - login with wrong credentials - login with correct credentials - logout Returns True if the 3 tests are successful''' import requests print 'Check the server is running' baseurl = global_settings()['baseurl'] url = baseurl + 'auth/login/' data = {'username': '******', 'password':'******'} try: r = requests.post(url, data=data) res = r.url == baseurl + 'auth/login/?error=Login+incorrect' print res data = credentials r = requests.post(url, data=data) res = res and '<a id="logout"' in r.text print r.text, res url = baseurl + 'auth/logout/' r = requests.get(url) res = res and baseurl + 'auth/login/' in r.url print res except Exception as e: res = False print e return res
def test_login_logout(): ''' Performs 3 tests in a row : - login with wrong credentials - login with correct credentials - logout Returns True if the 3 tests are successful''' import requests print 'Check the server is running' baseurl = global_settings()['baseurl'] url = baseurl + 'auth/login/' data = {'username': '******', 'password': '******'} try: r = requests.post(url, data=data) res = r.url == baseurl + 'auth/login/?error=Login+incorrect' print res data = credentials r = requests.post(url, data=data) res = res and '<a id="logout"' in r.text print r.text, res url = baseurl + 'auth/logout/' r = requests.get(url) res = res and baseurl + 'auth/login/' in r.url print res except Exception as e: res = False print e return res
def get(self): username = self.current_user[1:-1] import pluricent as pl from sqlalchemy import distinct args = {} p = pl.Pluricent(pl.global_settings()['database']) structures = [str(e[0]) for e in p.session.query(distinct(pl.models.Measurement.structure)).all()] args['structures'] = structures self.render("html/analyze.html", username = username, **args)
def test_datasource_exists(): ''' Returns False if the datasource as defined in the database is missing''' import os.path as osp db = global_settings()['database'] print db p = pl.Pluricent(db) ds = osp.dirname(db) return osp.isdir(ds)
def test_valid_study(): #FIXME import requests baseurl = global_settings()['baseurl'] url = baseurl + 'auth/login/' r = requests.post(url, data=credentials) url = baseurl + 'explore/?study=study01' r = requests.get(url) res = 'invalid study' in r.text return res
def test_studies(): ''' Compares studies between the repository and the database ''' import os, os.path as osp db = global_settings()['database'] p = pl.Pluricent(db) ds = p.datasource() studies_db = list(set([p.study_dir(e) for e in p.studies()])) studies_ds = list(set([e for e in os.listdir(ds) \ if osp.isdir(osp.join(ds,e)) and not e in ['.','..']])) print 'studies in database:', studies_db, '- studies in repo:', studies_ds return studies_db == studies_ds
def get(self): username = self.current_user[1:-1] import pluricent as pl from sqlalchemy import distinct args = {} p = pl.Pluricent(pl.global_settings()['database']) structures = [ str(e[0]) for e in p.session.query( distinct(pl.models.Measurement.structure)).all() ] args['structures'] = structures self.render("html/analyze.html", username=username, **args)
def get(self): username = self.current_user[1:-1] import pluricent as pl import os.path as osp fn = osp.abspath(pl.global_settings()['database']) ds = osp.dirname(fn) args = {'danger': '', 'datasource': '', 'database': fn} if osp.isfile(fn): args.update({'datasource': ds, 'database': fn}) else: args['danger'] = 'The database %s is missing' % fn self.render("html/index.html", username=username, **args)
def get(self): username = self.current_user[1:-1] import pluricent as pl import os.path as osp fn = osp.abspath(pl.global_settings()['database']) ds = osp.dirname(fn) args = {'danger':'', 'datasource':'', 'database':fn} if osp.isfile(fn): args.update({'datasource': ds, 'database': fn}) else: args['danger'] = 'The database %s is missing'%fn self.render("html/index.html", username = username, **args)
def test_respect_hierarchy(): ''' Checks that every file/folder in the repository is identified by the hierarchy Returns True if the unknown list is empty''' from pluricent import checkbase as cb import os.path as osp db = global_settings()['database'] print 'db', db p = pl.Pluricent(db) destdir = osp.dirname(db) from pluricent import tests as t return t.test_respect_hierarchy(destdir)
def test_unique_subjects(): ''' Checks that subjects identifiers are unique in every study ''' import os, os.path as osp db = global_settings()['database'] p = pl.Pluricent(db) studies = p.studies() unique = True for each in studies: subjects = p.subjects(each) if not len(subjects) == len(set(subjects)): unique = False print each return unique
def test_invalid_study(): ''' Logs in and acts as if just asked for a non-existing study. Returns True if successfully redirected with an information message''' import requests baseurl = global_settings()['baseurl'] url = baseurl + 'auth/login/' s = requests.Session() r = s.post(url, data=credentials) url = baseurl + 'explore/?study=toto' print url r = s.get(url) print r.text res = 'invalid study' in r.text return res
def test_each_entry_has_action(): ''' Checks every entry in tables Subject, T1Image, Study has an associated action in Action. The opposite way is not verified.''' import pluricent as pl import json p = pl.Pluricent(pl.global_settings()['database']) studies = p.studies() actions = [json.loads(each.action) for each in p.actions()] each_has_action = True # sorting actions recognized = ['add_study', 'add_subject', 'add_image'] sorted_actions = {} for a in actions: if a[0] in recognized: sorted_actions.setdefault(a[0], []).append(a) for s in studies: found = 0 for each in sorted_actions['add_study']: action_type, params = each if s == params['name']: found += 1 if found != 1: each_has_action = False print s, 'found %s times' % found for subject in p.subjects(s): found = 0 for each in sorted_actions['add_subject']: action_type, params = each if s == params['study'] and subject in params['subjects']: found += 1 if found != 1: each_has_action = False print subject, 'in', s, 'found %s times' % found for image in p.t1images(s, subject): found = 0 for each in sorted_actions['add_image']: action_type, params = each if s == params['study'] and subject == params[ 'subject'] and image.path == params['path']: found += 1 if found != 1: each_has_action = False print image, 'from', subject, 'in', s, 'found %s times' % found #TODO Verify every Action has associated entry in appropriate tables return each_has_action
def post(self): import pluricent as pl from sqlalchemy import func, distinct import numpy as np import json args = {} p = pl.Pluricent(pl.global_settings()['database']) datatypes = [ e[0] for e in p.session.query( distinct(pl.models.Processing.datatype)).all() ] table = [] headers = ['subject', 't1image'] q = [] q.append( dict( p.session.query(pl.models.Subject.identifier, func.count(pl.models.T1Image.path)).filter( pl.models.T1Image.subject_id == pl.models.Subject.id).group_by( pl.models.Subject.identifier).all())) for each in datatypes: headers.append(each) res = p.session.query( pl.models.Subject.identifier, func.count(pl.models.Processing.path) ).join(pl.models.T1Image).filter( pl.models.Processing.input_id == pl.models.T1Image.id).filter( pl.models.T1Image.subject_id == pl.models.Subject.id ).filter(pl.models.Processing.datatype == each).group_by( pl.models.Subject.identifier).all() q.append(dict(res)) subjects = [ e[0] for e in p.session.query(pl.models.Subject.identifier).all() ] table.append(headers) for s in subjects: table.append([s]) table[-1].extend([each.get(s, 0) for each in q]) #print t1images args['images'] = table res = json.dumps(args) self.write(res) return None
def test_units(): ''' Checks that same variables have same units''' p = pl.Pluricent(pl.global_settings()['database']) are_units_valid = True units = {} for m in p.measurements(): if m.measurement in units.keys(): if m.unit != units[m.measurement]: are_units_valid = False print '%s has unit %s and %s'%(m.measurement, m.unit, units[m.measurement]) break else: units[m.measurement] = m.unit return are_units_valid
def test_each_entry_has_action(): ''' Checks every entry in tables Subject, T1Image, Study has an associated action in Action. The opposite way is not verified.''' import pluricent as pl import json p = pl.Pluricent(pl.global_settings()['database']) studies = p.studies() actions = [json.loads(each.action) for each in p.actions()] each_has_action = True # sorting actions recognized = ['add_study', 'add_subject', 'add_image'] sorted_actions = {} for a in actions: if a[0] in recognized: sorted_actions.setdefault(a[0], []).append(a) for s in studies: found = 0 for each in sorted_actions['add_study']: action_type, params = each if s == params['name']: found += 1 if found != 1: each_has_action = False print s, 'found %s times'%found for subject in p.subjects(s): found = 0 for each in sorted_actions['add_subject']: action_type, params = each if s == params['study'] and subject in params['subjects']: found += 1 if found != 1: each_has_action = False print subject, 'in', s, 'found %s times'%found for image in p.t1images(s, subject): found = 0 for each in sorted_actions['add_image']: action_type, params = each if s == params['study'] and subject == params['subject'] and image.path == params['path']: found += 1 if found != 1: each_has_action = False print image, 'from', subject, 'in', s, 'found %s times'%found #TODO Verify every Action has associated entry in appropriate tables return each_has_action
def test_actions(): ''' Checks that every action starts with a recognized code e.g. add_study, add_subject, add_image...''' import pluricent as pl import json p = pl.Pluricent(pl.global_settings()['database']) actions = [json.loads(each.action) for each in p.actions()] authorized = True recognized = ['add_study', 'add_subject', 'add_image'] for each in actions: if not each[0] in recognized: authorized = False print each, 'not recognized' break return authorized
def test_units(): ''' Checks that same variables have same units''' p = pl.Pluricent(pl.global_settings()['database']) are_units_valid = True units = {} for m in p.measurements(): if m.measurement in units.keys(): if m.unit != units[m.measurement]: are_units_valid = False print '%s has unit %s and %s' % (m.measurement, m.unit, units[m.measurement]) break else: units[m.measurement] = m.unit return are_units_valid
def test_measurements_format(): ''' Checks headers in measurements files ''' have_good_format = True import os.path as osp from pluricent import checkbase as cb cl = cb.CloudyCheckbase(osp.dirname(pl.global_settings()['database'])) f = cl.get_files_of_type('measurements') print f from pluricent import tests for each in f: has_good_format = tests.test_measurements_format(each) if not has_good_format: print each, 'has a wrong format' have_good_format = False break return have_good_format
def test_matching_t1images(): ''' Checks if T1 images entries in the database are matching with existing files in the repository''' from pluricent import checkbase as cb import os.path as osp db = global_settings()['database'] p = pl.Pluricent(db) destdir = osp.dirname(db) cl = cb.CloudyCheckbase(destdir) import os import os.path as osp unknown = [] scanned = 0 print destdir raw_files = [] for root, dirs, files in os.walk(destdir): for f in files: scanned += 1 fp = osp.join(root, f) res = cb.parsefilepath(fp, cl.patterns) if not res is None: datatype, att = res if datatype == 'raw': raw_files.append(fp[len(destdir)+1:]) raw_entries = [e.path for e in p.t1images()] # comparing raw_files and raw_entries matching = True for f in raw_files: if not f in raw_entries: print f, 'missing from raw_entries' matching = False for f in raw_entries: if not f in raw_files: print f, 'missing from raw_files' matching = False print 'items in %s :'%destdir, scanned print 'entries in db:', len(raw_entries) return matching
def test_matching_t1images(): ''' Checks if T1 images entries in the database are matching with existing files in the repository''' from pluricent import checkbase as cb import os.path as osp db = global_settings()['database'] p = pl.Pluricent(db) destdir = osp.dirname(db) cl = cb.CloudyCheckbase(destdir) import os import os.path as osp unknown = [] scanned = 0 print destdir raw_files = [] for root, dirs, files in os.walk(destdir): for f in files: scanned += 1 fp = osp.join(root, f) res = cb.parsefilepath(fp, cl.patterns) if not res is None: datatype, att = res if datatype == 'raw': raw_files.append(fp[len(destdir) + 1:]) raw_entries = [e.path for e in p.t1images()] # comparing raw_files and raw_entries matching = True for f in raw_files: if not f in raw_entries: print f, 'missing from raw_entries' matching = False for f in raw_entries: if not f in raw_files: print f, 'missing from raw_files' matching = False print 'items in %s :' % destdir, scanned print 'entries in db:', len(raw_entries) return matching
def get(self): username = self.current_user[1:-1] import pluricent as pl import os.path as osp import json warning = '' default = True # Retrieving studies from the database p = pl.Pluricent(pl.global_settings()['database']) studies = p.studies() if 'study' in self.request.arguments: # Study selected: displaying more info study = self.get_argument('study') if study in studies: images = p.t1images(study) print study, images d = [] for each in images: d.append({ 'subject': each.subject.identifier, 'path': each.path, 'id': each.id }) self.render("html/explore_study.html", username=username, study_name=study, warning=warning, images=d) return else: warning = 'invalid study' default = True if default: # Welcome page for study selection studies = json.dumps(studies) self.render("html/explore.html", username=username, studies=studies, warning=warning)
def post(self): import pluricent as pl import numpy as np import json from sqlalchemy import distinct structure = self.get_argument('id') print structure args = {} p = pl.Pluricent(pl.global_settings()['database']) structures = [str(e[0]) for e in p.session.query(distinct(pl.models.Measurement.structure)).all()] measurements = dict(p.session.query(pl.models.Subject.identifier, pl.models.Measurement.value)\ .join(pl.models.T1Image).join(pl.models.Measurement).filter(pl.models.Measurement.structure == structure).all()) args['data'], args['labels'] = np.histogram(measurements.values()) args = dict([(k, json.dumps([int(e) for e in v.tolist()])) for k,v in args.items()]) args['structure'] = structure args['structures'] = structures args['measurements'] = measurements res = json.dumps(args) self.write(res) return None
parser.add_argument("-t", dest='test_name', type=str, help="Run specific test (in debug mode)", required=False) args = parser.parse_args() # ==== run specific test ? ==== if args.test_name: if not args.test_name in results.keys(): raise Exception('%s should refer to an existing test (%s)'%(args.test_name, results.keys())) res = run_test(args.test_name) print '=== %s : %s'%(args.test_name, res) import sys sys.exit(0) # ==== run tests ==== from datetime import datetime results['last_checked'] = datetime.now().isoformat() results = run_tests(results, args.debug) import json import os.path as osp print '====================================================================================================' fp = osp.abspath(osp.join(osp.dirname(pl.__file__), '..', '..', 'web', 'json', 'sysdiag.json')) print 'Writing json... %s'%fp json.dump(results, open(fp, 'w'), indent=2) print '=====================================================================' print 'baseurl:', global_settings()['baseurl'] print 'database:', global_settings()['database'] print '====================================================================='
args = parser.parse_args() # ==== run specific test ? ==== if args.test_name: if not args.test_name in results.keys(): raise Exception('%s should refer to an existing test (%s)' % (args.test_name, results.keys())) res = run_test(args.test_name) print '=== %s : %s' % (args.test_name, res) import sys sys.exit(0) # ==== run tests ==== from datetime import datetime results['last_checked'] = datetime.now().isoformat() results = run_tests(results, args.debug) import json import os.path as osp print '====================================================================================================' fp = osp.abspath( osp.join(osp.dirname(pl.__file__), '..', '..', 'web', 'json', 'sysdiag.json')) print 'Writing json... %s' % fp json.dump(results, open(fp, 'w'), indent=2) print '=====================================================================' print 'baseurl:', global_settings()['baseurl'] print 'database:', global_settings()['database'] print '====================================================================='