def run_freesurfer_hippocampal(project_id, subject_id, mri_type = "T1-weighted"):
''' Run FreeSurfer for ENIGMA Hippocampal pipeline'''
# Query data
query_txt = """query {
mri_exam(project_id: "%s", scan_type: "%s", with_path_to:{type:"case", submitter_id:"%s"}){
mri_images{
file_name
id
}
}
}""" % (project_id, mri_type, subject_id)
data = query_api(query_txt)
# Get file from S3
filename = data['data']['mri_exam'][0]['mri_images'][0]['file_name']
fileid = data['data']['mri_exam'][0]['mri_images'][0]['file_name']
localpath = utils.download_file(auth, api_url, fileid, filename)
# Run freesurfer
datetime_start = datetime.datetime.now()
print("%s: Running FreeSurfer for %s subject..." % (str(datetime_start),subject_id))
local_output = './freesurfer/' + subject_id
if not os.path.exists(local_output):
cmd = ['/bin/bash', './scripts/run_freesurfer_hippocampal.sh', subject_id, localpath]
output = utils.run_command(cmd)
datetime_end = datetime.datetime.now()
print("%s: Hippocampal FreeSurfer FINISHED (Total time: %s)." % (str(datetime_end), str(datetime_end-datetime_start)))
else:
print("%s: Hippocampal FreeSurfer results were already found for %s subject." % (str(datetime_start), subject_id))
def download_glove_vectors(
remote_path='http://nlp.stanford.edu/data/glove.6B.zip'):
if os.path.exists('./vendor/glove/glove.6B'):
return './vendor/glove/glove.6B/glove.6B.50d.txt'
downloaded_filepath = download_file(remote_path, './vendor/glove')
call(['unzip', downloaded_filepath, '-d', './vendor/glove/glove.6B'])
return './vendor/glove/glove.6B/glove.6B.50d.txt'
def get_external_surface_qc(project_id, subject_id):
''' Get external surface results from data model'''
# Query data
query_txt = """query {
case(submitter_id: "%s"){
mri_exams(first:0){
mri_images(first:0){
mri_analysis_workflows(first:0){
mri_results(first:0, data_category: "MRI Derived Image", data_type: "Cortical Thickness"){
file_name
id
}
}
}
}
}
}""" % (subject_id)
data = query_api(query_txt)
# Display external surface
fig = plt.figure(figsize=(15, 15))
fig.suptitle('EXTERNAL SURFACE SEGMENTATION',
fontsize=20,
fontweight='bold')
for file in data['data']['case'][0]['mri_exams'][0]['mri_images'][0][
'mri_analysis_workflows'][0]['mri_results']:
# Download image files if not in local
image = file['file_name']
if not os.path.exists(image):
image = utils.download_file(auth, api_url, file['id'], image)
# Plot images
print(image)
if 'lh.lat.' in image:
title_view = 'Lateral View - Left Hemisphere'
pos = 1
elif 'lh.med.' in image:
title_view = 'Medial View - Left Hemisphere'
pos = 2
elif 'rh.lat.' in image:
title_view = 'Lateral View - Right Hemisphere'
pos = 3
elif 'rh.med.' in image:
title_view = 'Medial View - Right Hemisphere'
pos = 4
a = fig.add_subplot(2, 2, pos)
a.axis('off')
img = mpimg.imread(image)
imgplot = plt.imshow(img)
a.set_title(title_view, fontsize=18)
def get_cortical_measure_by_disease_when_dataAvailableInCommons(
project_id, measure):
''' Query metrics from cortical pipeline'''
out_dir = './results/' + project_id + '/downloadedFiles/'
query_txt = """query {
subject(first:0, project_id: "%s"){
condition(first:0){
primary_diagnosis
}
imaging_mri_exams(first:0){
imaging_files(first:0){
imaging_analysis_workflows(first:0){
imaging_report_files(first:0, data_category: "Derived Measures", data_type: "Cortical Thickness"){
file_name
id
}
}
}
}
}
}""" % (project_id)
data = query_api(query_txt)
values = {}
for case in data['data']['case']:
if case['diagnoses']:
diagnosis = case['diagnoses'][0]['primary_diagnosis']
values.setdefault(diagnosis, [])
if case['imaging_mri_exams'] \
and case['imaging_mri_exams'][0]['imaging_files'] \
and case['imaging_mri_exams'][0]['imaging_files'][0]['imaging_analysis_workflows'] \
and case['imaging_mri_exams'][0]['imaging_files'][0]['imaging_analysis_workflows'][0]['imaging_report_files']:
filename = case['imaging_mri_exams'][0]['imaging_files'][0][
'imaging_analysis_workflows'][0]['imaging_report_files'][
0]['file_name']
fileID = case['imaging_mri_exams'][0]['imaging_files'][0][
'imaging_analysis_workflows'][0]['imaging_report_files'][
0]['id']
file_path = out_dir + filename
resFile = utils.download_file(auth, api_url, fileID, file_path)
results = read_result_file(file_path)
for subj in results:
if measure in results[subj]:
values[diagnosis].append(float(results[subj][measure]))
pvalues = run_statistical_test(values)
plot_measure(values, 'CORTICAL MEASUREMENTS', measure, pvalues)
return values
def get_cortical_measure_by_disease(project_id, measure):
''' Query metrics from cortical pipeline'''
query_txt = """query {
case(first:0, project_id: "%s"){
diagnoses(first:0){
primary_diagnosis
}
mri_exams(first:0){
mri_images(first:0){
mri_analysis_workflows(first:0){
mri_results(first:0, data_category: "MRI Derived Measures", data_type: "Cortical Thickness"){
file_name
id
}
}
}
}
}
}""" % (project_id)
data = query_api(query_txt)
values = {}
for case in data['data']['case']:
if case['diagnoses']:
diagnosis = case['diagnoses'][0]['primary_diagnosis']
values.setdefault(diagnosis, [])
if case['mri_exams'] \
and case['mri_exams'][0]['mri_images'] \
and case['mri_exams'][0]['mri_images'][0]['mri_analysis_workflows'] \
and case['mri_exams'][0]['mri_images'][0]['mri_analysis_workflows'][0]['mri_results']:
filename = case['mri_exams'][0]['mri_images'][0][
'mri_analysis_workflows'][0]['mri_results'][0]['file_name']
fileID = case['mri_exams'][0]['mri_images'][0][
'mri_analysis_workflows'][0]['mri_results'][0]['id']
resFile = utils.download_file(auth, api_url, fileID, filename)
results = read_result_file(resFile)
for subj in results:
if measure in results[subj]:
values[diagnosis].append(float(results[subj][measure]))
pvalues = run_statistical_test(values)
plot_measure(values, 'CORTICAL MEASUREMENTS', measure, pvalues)
return values
def run_freesurfer_test(project_id,
subject_id,
outDir,
mri_type="T1-weighted"):
''' Run FreeSurfer for ENIGMA cortical pipeline'''
# Query data
#query_txt = """query {
# case(project_id: "%s", submitter_id: "%s"){
# imaging_mri_exams(imaging_subtype: "%s"){
# imaging_files(quick_search:"subT1"){
# id
# file_name
# }
# }
# }
# }""" % (project_id, subject_id, mri_type)
query_txt = """query {
imaging_file(project_id: "%s",submitter_id: "%s"){
object_id
file_name
}
}""" % (project_id, subject_id)
data = query_api(query_txt)
# Get file from S3
filename = data['data']['imaging_file'][0]['file_name']
fileid = data['data']['imaging_file'][0]['object_id']
#localpath = utils.get_file_from_s3(filename, project_id)
localpath = outDir + '/' + project_id + '/' + filename
if not os.path.exists(outDir):
os.mkdir(outDir)
os.mkdir(outDir + '/' + project_id)
elif os.path.exists(outDir) and not os.path.exists(outDir + '/' +
project_id):
os.mkdir(outDir + '/' + project_id)
response = utils.download_file(auth, api_url, fileid, localpath)
inputPath = (localpath[2:]).encode("ascii")
return (inputPath)
def _download_yolo_weights():
remote_path = 'https://pjreddie.com/media/files/darknet19_448.conv.23'
local_dir = './vendor/lightnet'
return download_file(remote_path, local_dir)
def get_mri_subfield_by_genotype(project_id,
genotype_var,
variable,
baseline='e3/e3',
threshold=0.05):
''' Query a specific MRI subfield by phenotype'''
if variable in mri_measures:
measure = mri_measures[variable]
else:
measure = variable
# Prepare and run query
query_txt = """query {
case(first:0, project_id: "%s"){
biomarkers{
%s
}
followups(first:0){
submitter_id
mri_exams(first:0){
mri_results(first:0, data_category: "MRI Derived Measures"){
file_name
id
}
}
}
}
}""" % (project_id, genotype_var)
data = query_api(query_txt)
# Format data from query output
values = {}
for case in data['data']['case']:
if case['biomarkers']:
genotype = case['biomarkers'][0][genotype_var]
if genotype == None:
continue
values.setdefault(genotype, [])
if case['followups']:
for fu in case['followups']:
subjid = fu['submitter_id']
if fu['mri_exams']:
for exam in fu['mri_exams']:
if exam['mri_results']:
for r in exam['mri_results']:
filename = r['file_name']
fileid = r['id']
# Download file from commons if not existing
if not os.path.exists(filename):
localpath = utils.download_file(
auth, api_url, fileid, filename)
# Read file and prepare data
results = read_result_file(filename)
if subjid in results and results[subjid][
measure] != '':
values[genotype].append(
float(results[subjid][measure]))
# Run statistical analysis and show boxplots
pvalues = run_statistical_test(values, baseline)
plot_measure(values, 'MRI Metrics', variable, pvalues, threshold)
return values
datasets_path = hparams.data.datasets_path
dataset_file_url = \
f'https://open-speech-data.oss-cn-hangzhou.aliyuncs.com/{hparams.data.dataset_dir}.tar.bz2'
dataset_file_name = osp.basename(dataset_file_url)
dataset_dir = dataset_file_name[:-8]
dataset_path = osp.join(datasets_path, dataset_dir)
wavfile_path = osp.join(dataset_path, "wavs")
melspec_path = osp.join(dataset_path, "mels")
if osp.isdir(melspec_path) and False:
print("%s dataset folder already exists" % dataset_dir)
sys.exit(0)
else:
dataset_file_path = osp.join(datasets_path, dataset_file_name)
if not osp.isfile(dataset_file_path):
download_file(dataset_file_url, dataset_file_path)
else:
print("'%s' already exists" % dataset_file_name)
if not osp.isdir(wavfile_path):
print("extracting '%s'..." % dataset_file_name)
os.system('cd %s; tar xvjf %s' % (datasets_path, dataset_file_name))
else:
print("'%s' already exists" % wavfile_path)
dataset_root = osp.join(hparams.data.datasets_path,
hparams.data.dataset_dir)
dataset = SpeechDataset([], dataset_root, hparams.text)
processor = Processor(hparams=hparams.audio)
# pre process/compute
def download_vim_plugged(self):
local = Path.home().joinpath(".local", "share", "nvim", "site",
"autoload")
os.makedirs(local, exist_ok=True)
download_file(_VIM_PLUGGED_SRC, str(local.joinpath("plug.vim")))
