step_params = ' ' + measure_fname + ' ' + mdir absolute_str = '' if absval: absolute_str = ' ' + au.abs_str() step_params += absolute_str leave_str = '' if leave > -1: leave_str = ' excluding subject ' + str(leave) step_params += leave_str #checking if this measure has already been done endstep = au.measure_str() + step_params stepdone = au.is_done(chkf, endstep) #add pluses to output dir if it already exists if stepdone: while os.path.exists(mdir): mdir += '+' else: #work in the last folder used plus = False while os.path.exists(mdir): mdir += '+' plus = True if plus: mdir = mdir[0:-1] #setting statsdir pluses = mdir.count('+') for i in np.arange(pluses):
step_params = ' ' + measure_fname + ' ' + mdir absolute_str = '' if absval: absolute_str = ' ' + au.abs_str() step_params += absolute_str leave_str = '' if leave > -1: leave_str = ' excluding subject ' + str(leave) step_params += leave_str #checking if this measure has already been done endstep = au.measure_str() + step_params stepdone = au.is_done(chkf, endstep) #add pluses to output dir if it already exists if stepdone: while os.path.exists (mdir): mdir += '+' else: #work in the last folder used plus = False while os.path.exists (mdir): mdir += '+' plus = True if plus: mdir = mdir[0:-1] #setting statsdir pluses = mdir.count('+') for i in np.arange(pluses): statsdir += '+'
def slice_and_merge (subsfname, labelsfname, checkfname, outdir='', maskfname='', outbase=''): check_data (subsfname, labelsfname, outdir, maskfname, outbase) nsubs = au.file_len(subsfname) nlabs = au.file_len(labelsfname) if not outdir: outdir = os.getcwd() slicesdir = outdir + os.path.sep + au.slices_str() tempdir = outdir + os.path.sep + au.temp_str() tmpdirlst = os.listdir(tempdir) fpath = '' olddir = os.getcwd() if not au.is_done (checkfname, au.preslicingdata_str()): #slicing subjects au.log.info ('Slicing all subjects: takes a while') subsfile = open(subsfname, 'r') for line in subsfile: fpath = line.strip() isfile = os.path.basename(fpath) osfile = tempdir + os.path.sep + isfile #test if file has been sliced fdim3 = au.fslval(fpath, 'dim3') regex = au.remove_ext(isfile) + '*' nslices = au.count_match (tmpdirlst, regex) if fdim3 != nslices: #if not, then slice it in tempdir au.log.debug('Slicing ' + isfile) os.chdir (tempdir) shutil.copy (fpath, tempdir) au.fslslice (osfile) os.remove(osfile) os.chdir (olddir) else: au.log.debug(isfile + ' previously sliced') subsfile.close() au.checklist_add (checkfname, au.preslicingdata_str()) if not au.is_done (checkfname, au.premergingdata_str()): #merging each slice if not outbase: outbase = au.data_str() if not fpath: subsfile = open(subsfname, 'r') for line in subsfile: fpath = line.strip() break au.log.info ('Merging all subject slices: takes a while') nslices = int(au.fslval(fpath,'dim3')) for slice in range(nslices): slicezp = au.zeropad(slice) mergeout = outbase + '_slice_' + slicezp outdata = slicesdir + os.path.sep + mergeout #check if data file exists if not au.imtest(outdata): #if not, then create it, merging all the corresponding slices au.log.debug('Merging slice ' + slicezp) imglob = '' subsfile = open(subsfname, 'r') for line in subsfile: fpath = line.strip() isfile = au.remove_ext(os.path.basename(fpath)).strip() isfile = tempdir + os.path.sep + isfile + '_slice_' + slicezp imglob += isfile + ' ' os.system('fslmerge -t ' + outdata + ' ' + imglob) for f in imglob.split(): au.imrm(f) else: au.log.debug('Slice ' + slicezp + ' previously done') au.checklist_add (checkfname, au.premergingdata_str()) #slicing mask if maskfname: if os.path.exists(maskfname) and not au.is_done (checkfname, au.preslicingmask_str()): au.log.info('Slicing mask ' + maskfname) au.imcp (maskfname, slicesdir + os.path.sep + au.mask_str()) os.chdir (slicesdir) au.fslslice (au.mask_str()) au.imrm (au.mask_str()) au.checklist_add (checkfname, au.preslicingmask_str()) os.chdir (olddir) au.log.debug('Done preprocessing')
def group_distance (measure_function, datadir, groups, groupsizes, chkf, absolute=False, outdir='', foldno='', expname='', exclude_idx=-1, exclude_subj='', exclude_subjclass=''): olddir = os.getcwd() if not outdir: outdir = datadir ngroups = len(groups) #matrix of strings of 2000 characters maximum, to save filepaths gfnames = np.zeros ([ngroups,3], dtype=np.dtype('a2000')) subject_excluded = False for g1 in range(ngroups): g1name = groups[g1] #mean1fname gfnames[g1,0] = datadir + os.path.sep + g1name + '_' + au.mean_str() #var1fname gfnames[g1,1] = datadir + os.path.sep + g1name + '_' + au.var_str() #std1fname gfnames[g1,2] = datadir + os.path.sep + g1name + '_' + au.std_str() for g2 in range(g1+1, ngroups): g2name = groups[g2] gfnames[g2,0] = datadir + os.path.sep + g2name + '_' + au.mean_str() gfnames[g2,1] = datadir + os.path.sep + g2name + '_' + au.var_str() gfnames[g2,2] = datadir + os.path.sep + g2name + '_' + au.std_str() experiment = g1name + '_vs_' + g2name #check if exclude_subjclass is any of both current groups eg = -1 if exclude_idx > -1: if exclude_subjclass == g1name: eg = g2 elif exclude_subjclass == g2name: eg = g1 step = au.measure_str() + ' ' + measure_function.func_name + ' ' + experiment + ' ' + datadir #remove subject from stats if eg > -1: exclude_str = '_' + au.excluded_str() + str(exclude_idx) step += exclude_str experiment += exclude_str if not au.is_done(chkf, step): if not subject_excluded: newmeanfname = gfnames[eg,0] + exclude_str newvarfname = gfnames[eg,1] + exclude_str newstdfname = gfnames[eg,2] + exclude_str rstep = au.remove_str() + ' ' + au.subject_str() + ' ' + str(exclude_subj) + ' ' + au.fromstats_str() + ' ' + datadir if not au.is_done(chkf, rstep): #(meanfname, varfname, samplesize, subjvolfname, newmeanfname, newvarfname, newstdfname='') remove_subject_from_stats (gfnames[eg,0], gfnames[eg,1], groupsizes[eg][1], exclude_subj, newmeanfname, newvarfname, newstdfname) au.checklist_add (chkf, rstep) gfnames[eg,0] += exclude_str gfnames[eg,1] += exclude_str gfnames[eg,2] += exclude_str groupsizes[eg][1] -= 1 subject_excluded = True #calculating distance if not au.is_done(chkf, step): mean1fname = au.add_extension_if_needed (gfnames[g1,0], au.ext_str()) mean2fname = au.add_extension_if_needed (gfnames[g2,0], au.ext_str()) var1fname = au.add_extension_if_needed (gfnames[g1,1], au.ext_str()) var2fname = au.add_extension_if_needed (gfnames[g2,1], au.ext_str()) std1fname = au.add_extension_if_needed (gfnames[g1,2], au.ext_str()) std2fname = au.add_extension_if_needed (gfnames[g2,2], au.ext_str()) outfname = measure_function (mean1fname, mean2fname, var1fname, var2fname, std1fname, std2fname, groupsizes[g1][1], groupsizes[g2][1], experiment, outdir, exclude_idx) if absolute: change_to_absolute_values (outfname) au.checklist_add (chkf, step) return outfname
def group_distance(measure_function, datadir, groups, groupsizes, chkf, absolute=False, outdir='', foldno='', expname='', exclude_idx=-1, exclude_subj='', exclude_subjclass=''): olddir = os.getcwd() if not outdir: outdir = datadir ngroups = len(groups) #matrix of strings of 2000 characters maximum, to save filepaths gfnames = np.zeros([ngroups, 3], dtype=np.dtype('a2000')) subject_excluded = False for g1 in range(ngroups): g1name = groups[g1] #mean1fname gfnames[g1, 0] = datadir + os.path.sep + g1name + '_' + au.mean_str() #var1fname gfnames[g1, 1] = datadir + os.path.sep + g1name + '_' + au.var_str() #std1fname gfnames[g1, 2] = datadir + os.path.sep + g1name + '_' + au.std_str() for g2 in range(g1 + 1, ngroups): g2name = groups[g2] gfnames[g2, 0] = datadir + os.path.sep + g2name + '_' + au.mean_str() gfnames[g2, 1] = datadir + os.path.sep + g2name + '_' + au.var_str() gfnames[g2, 2] = datadir + os.path.sep + g2name + '_' + au.std_str() experiment = g1name + '_vs_' + g2name #check if exclude_subjclass is any of both current groups eg = -1 if exclude_idx > -1: if exclude_subjclass == g1name: eg = g2 elif exclude_subjclass == g2name: eg = g1 step = au.measure_str( ) + ' ' + measure_function.func_name + ' ' + experiment + ' ' + datadir #remove subject from stats if eg > -1: exclude_str = '_' + au.excluded_str() + str(exclude_idx) step += exclude_str experiment += exclude_str if not au.is_done(chkf, step): if not subject_excluded: newmeanfname = gfnames[eg, 0] + exclude_str newvarfname = gfnames[eg, 1] + exclude_str newstdfname = gfnames[eg, 2] + exclude_str rstep = au.remove_str() + ' ' + au.subject_str( ) + ' ' + str(exclude_subj) + ' ' + au.fromstats_str( ) + ' ' + datadir if not au.is_done(chkf, rstep): #(meanfname, varfname, samplesize, subjvolfname, newmeanfname, newvarfname, newstdfname='') remove_subject_from_stats(gfnames[eg, 0], gfnames[eg, 1], groupsizes[eg][1], exclude_subj, newmeanfname, newvarfname, newstdfname) au.checklist_add(chkf, rstep) gfnames[eg, 0] += exclude_str gfnames[eg, 1] += exclude_str gfnames[eg, 2] += exclude_str groupsizes[eg][1] -= 1 subject_excluded = True #calculating distance if not au.is_done(chkf, step): mean1fname = au.add_extension_if_needed( gfnames[g1, 0], au.ext_str()) mean2fname = au.add_extension_if_needed( gfnames[g2, 0], au.ext_str()) var1fname = au.add_extension_if_needed(gfnames[g1, 1], au.ext_str()) var2fname = au.add_extension_if_needed(gfnames[g2, 1], au.ext_str()) std1fname = au.add_extension_if_needed(gfnames[g1, 2], au.ext_str()) std2fname = au.add_extension_if_needed(gfnames[g2, 2], au.ext_str()) outfname = measure_function(mean1fname, mean2fname, var1fname, var2fname, std1fname, std2fname, groupsizes[g1][1], groupsizes[g2][1], experiment, outdir, exclude_idx) if absolute: change_to_absolute_values(outfname) au.checklist_add(chkf, step) return outfname