def run_freesurfer_hippocampal(project_id, subject_id, mri_type = "T1-weighted"): ''' Run FreeSurfer for ENIGMA Hippocampal pipeline''' # Query data query_txt = """query { mri_exam(project_id: "%s", scan_type: "%s", with_path_to:{type:"case", submitter_id:"%s"}){ mri_images{ file_name id } } }""" % (project_id, mri_type, subject_id) data = query_api(query_txt) # Get file from S3 filename = data['data']['mri_exam'][0]['mri_images'][0]['file_name'] fileid = data['data']['mri_exam'][0]['mri_images'][0]['file_name'] localpath = utils.download_file(auth, api_url, fileid, filename) # Run freesurfer datetime_start = datetime.datetime.now() print("%s: Running FreeSurfer for %s subject..." % (str(datetime_start),subject_id)) local_output = './freesurfer/' + subject_id if not os.path.exists(local_output): cmd = ['/bin/bash', './scripts/run_freesurfer_hippocampal.sh', subject_id, localpath] output = utils.run_command(cmd) datetime_end = datetime.datetime.now() print("%s: Hippocampal FreeSurfer FINISHED (Total time: %s)." % (str(datetime_end), str(datetime_end-datetime_start))) else: print("%s: Hippocampal FreeSurfer results were already found for %s subject." % (str(datetime_start), subject_id))
def download_glove_vectors( remote_path='http://nlp.stanford.edu/data/glove.6B.zip'): if os.path.exists('./vendor/glove/glove.6B'): return './vendor/glove/glove.6B/glove.6B.50d.txt' downloaded_filepath = download_file(remote_path, './vendor/glove') call(['unzip', downloaded_filepath, '-d', './vendor/glove/glove.6B']) return './vendor/glove/glove.6B/glove.6B.50d.txt'
def get_external_surface_qc(project_id, subject_id): ''' Get external surface results from data model''' # Query data query_txt = """query { case(submitter_id: "%s"){ mri_exams(first:0){ mri_images(first:0){ mri_analysis_workflows(first:0){ mri_results(first:0, data_category: "MRI Derived Image", data_type: "Cortical Thickness"){ file_name id } } } } } }""" % (subject_id) data = query_api(query_txt) # Display external surface fig = plt.figure(figsize=(15, 15)) fig.suptitle('EXTERNAL SURFACE SEGMENTATION', fontsize=20, fontweight='bold') for file in data['data']['case'][0]['mri_exams'][0]['mri_images'][0][ 'mri_analysis_workflows'][0]['mri_results']: # Download image files if not in local image = file['file_name'] if not os.path.exists(image): image = utils.download_file(auth, api_url, file['id'], image) # Plot images print(image) if 'lh.lat.' in image: title_view = 'Lateral View - Left Hemisphere' pos = 1 elif 'lh.med.' in image: title_view = 'Medial View - Left Hemisphere' pos = 2 elif 'rh.lat.' in image: title_view = 'Lateral View - Right Hemisphere' pos = 3 elif 'rh.med.' in image: title_view = 'Medial View - Right Hemisphere' pos = 4 a = fig.add_subplot(2, 2, pos) a.axis('off') img = mpimg.imread(image) imgplot = plt.imshow(img) a.set_title(title_view, fontsize=18)
def get_cortical_measure_by_disease_when_dataAvailableInCommons( project_id, measure): ''' Query metrics from cortical pipeline''' out_dir = './results/' + project_id + '/downloadedFiles/' query_txt = """query { subject(first:0, project_id: "%s"){ condition(first:0){ primary_diagnosis } imaging_mri_exams(first:0){ imaging_files(first:0){ imaging_analysis_workflows(first:0){ imaging_report_files(first:0, data_category: "Derived Measures", data_type: "Cortical Thickness"){ file_name id } } } } } }""" % (project_id) data = query_api(query_txt) values = {} for case in data['data']['case']: if case['diagnoses']: diagnosis = case['diagnoses'][0]['primary_diagnosis'] values.setdefault(diagnosis, []) if case['imaging_mri_exams'] \ and case['imaging_mri_exams'][0]['imaging_files'] \ and case['imaging_mri_exams'][0]['imaging_files'][0]['imaging_analysis_workflows'] \ and case['imaging_mri_exams'][0]['imaging_files'][0]['imaging_analysis_workflows'][0]['imaging_report_files']: filename = case['imaging_mri_exams'][0]['imaging_files'][0][ 'imaging_analysis_workflows'][0]['imaging_report_files'][ 0]['file_name'] fileID = case['imaging_mri_exams'][0]['imaging_files'][0][ 'imaging_analysis_workflows'][0]['imaging_report_files'][ 0]['id'] file_path = out_dir + filename resFile = utils.download_file(auth, api_url, fileID, file_path) results = read_result_file(file_path) for subj in results: if measure in results[subj]: values[diagnosis].append(float(results[subj][measure])) pvalues = run_statistical_test(values) plot_measure(values, 'CORTICAL MEASUREMENTS', measure, pvalues) return values
def get_cortical_measure_by_disease(project_id, measure): ''' Query metrics from cortical pipeline''' query_txt = """query { case(first:0, project_id: "%s"){ diagnoses(first:0){ primary_diagnosis } mri_exams(first:0){ mri_images(first:0){ mri_analysis_workflows(first:0){ mri_results(first:0, data_category: "MRI Derived Measures", data_type: "Cortical Thickness"){ file_name id } } } } } }""" % (project_id) data = query_api(query_txt) values = {} for case in data['data']['case']: if case['diagnoses']: diagnosis = case['diagnoses'][0]['primary_diagnosis'] values.setdefault(diagnosis, []) if case['mri_exams'] \ and case['mri_exams'][0]['mri_images'] \ and case['mri_exams'][0]['mri_images'][0]['mri_analysis_workflows'] \ and case['mri_exams'][0]['mri_images'][0]['mri_analysis_workflows'][0]['mri_results']: filename = case['mri_exams'][0]['mri_images'][0][ 'mri_analysis_workflows'][0]['mri_results'][0]['file_name'] fileID = case['mri_exams'][0]['mri_images'][0][ 'mri_analysis_workflows'][0]['mri_results'][0]['id'] resFile = utils.download_file(auth, api_url, fileID, filename) results = read_result_file(resFile) for subj in results: if measure in results[subj]: values[diagnosis].append(float(results[subj][measure])) pvalues = run_statistical_test(values) plot_measure(values, 'CORTICAL MEASUREMENTS', measure, pvalues) return values
def run_freesurfer_test(project_id, subject_id, outDir, mri_type="T1-weighted"): ''' Run FreeSurfer for ENIGMA cortical pipeline''' # Query data #query_txt = """query { # case(project_id: "%s", submitter_id: "%s"){ # imaging_mri_exams(imaging_subtype: "%s"){ # imaging_files(quick_search:"subT1"){ # id # file_name # } # } # } # }""" % (project_id, subject_id, mri_type) query_txt = """query { imaging_file(project_id: "%s",submitter_id: "%s"){ object_id file_name } }""" % (project_id, subject_id) data = query_api(query_txt) # Get file from S3 filename = data['data']['imaging_file'][0]['file_name'] fileid = data['data']['imaging_file'][0]['object_id'] #localpath = utils.get_file_from_s3(filename, project_id) localpath = outDir + '/' + project_id + '/' + filename if not os.path.exists(outDir): os.mkdir(outDir) os.mkdir(outDir + '/' + project_id) elif os.path.exists(outDir) and not os.path.exists(outDir + '/' + project_id): os.mkdir(outDir + '/' + project_id) response = utils.download_file(auth, api_url, fileid, localpath) inputPath = (localpath[2:]).encode("ascii") return (inputPath)
def _download_yolo_weights(): remote_path = 'https://pjreddie.com/media/files/darknet19_448.conv.23' local_dir = './vendor/lightnet' return download_file(remote_path, local_dir)
def get_mri_subfield_by_genotype(project_id, genotype_var, variable, baseline='e3/e3', threshold=0.05): ''' Query a specific MRI subfield by phenotype''' if variable in mri_measures: measure = mri_measures[variable] else: measure = variable # Prepare and run query query_txt = """query { case(first:0, project_id: "%s"){ biomarkers{ %s } followups(first:0){ submitter_id mri_exams(first:0){ mri_results(first:0, data_category: "MRI Derived Measures"){ file_name id } } } } }""" % (project_id, genotype_var) data = query_api(query_txt) # Format data from query output values = {} for case in data['data']['case']: if case['biomarkers']: genotype = case['biomarkers'][0][genotype_var] if genotype == None: continue values.setdefault(genotype, []) if case['followups']: for fu in case['followups']: subjid = fu['submitter_id'] if fu['mri_exams']: for exam in fu['mri_exams']: if exam['mri_results']: for r in exam['mri_results']: filename = r['file_name'] fileid = r['id'] # Download file from commons if not existing if not os.path.exists(filename): localpath = utils.download_file( auth, api_url, fileid, filename) # Read file and prepare data results = read_result_file(filename) if subjid in results and results[subjid][ measure] != '': values[genotype].append( float(results[subjid][measure])) # Run statistical analysis and show boxplots pvalues = run_statistical_test(values, baseline) plot_measure(values, 'MRI Metrics', variable, pvalues, threshold) return values
datasets_path = hparams.data.datasets_path dataset_file_url = \ f'https://open-speech-data.oss-cn-hangzhou.aliyuncs.com/{hparams.data.dataset_dir}.tar.bz2' dataset_file_name = osp.basename(dataset_file_url) dataset_dir = dataset_file_name[:-8] dataset_path = osp.join(datasets_path, dataset_dir) wavfile_path = osp.join(dataset_path, "wavs") melspec_path = osp.join(dataset_path, "mels") if osp.isdir(melspec_path) and False: print("%s dataset folder already exists" % dataset_dir) sys.exit(0) else: dataset_file_path = osp.join(datasets_path, dataset_file_name) if not osp.isfile(dataset_file_path): download_file(dataset_file_url, dataset_file_path) else: print("'%s' already exists" % dataset_file_name) if not osp.isdir(wavfile_path): print("extracting '%s'..." % dataset_file_name) os.system('cd %s; tar xvjf %s' % (datasets_path, dataset_file_name)) else: print("'%s' already exists" % wavfile_path) dataset_root = osp.join(hparams.data.datasets_path, hparams.data.dataset_dir) dataset = SpeechDataset([], dataset_root, hparams.text) processor = Processor(hparams=hparams.audio) # pre process/compute
def download_vim_plugged(self): local = Path.home().joinpath(".local", "share", "nvim", "site", "autoload") os.makedirs(local, exist_ok=True) download_file(_VIM_PLUGGED_SRC, str(local.joinpath("plug.vim")))