def submit_all(): from pymeg import parallel # fmt: off subjects = [ 1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 17, 18, 19, ] # fmt: on for session in [5, 6, 7]: for subject in subjects: parallel.pmap( preprocess, [(subject, session)], walltime="02:30:00", tasks=3, memory=30, )
def submit(sub, env='Hummel'): name_list = [ "The", "owls", "are", "not", "what", "they", "seem", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER" ] if env == 'Hummel': def keys(sub): keys = [] for ses in [2, 3]: keys.append((sub, ses, env)) return keys slu.pmap(par_execute, keys(sub), walltime='2:00:00', memory=40, nodes=1, tasks=2, name='SubjectLevel') elif env == 'Climag': for ses in [2, 3]: pbs.pmap(execute, [(sub, ses, env)], walltime='4:00:00', memory=40, nodes=1, tasks=2, name=name_list[sub])
def submit(): fits = pd.read_csv('/home/khagena/FLEXRULE/behavior/summary_stan_fits.csv') subjects = fits.loc[fits.vmode < 2.5].subject.unique() out_dir = join( '/home/khagena/FLEXRULE/behavior/kernel_simulation/KernelSimulation_{}' .format(datetime.datetime.now().strftime("%Y-%m-%d"))) slu.mkdir_p(out_dir) ''' for subject in subjects: V = fits.loc[fits.subject == subject].vmode.mean() H = fits.loc[fits.subject == subject].hmode.mean() for C in [1, 1e8]: for n in [8, 12]: pbs.pmap(simulate_regression, [(100000, H, V, C, n, out_dir, subject)], walltime='1:00:00', memory=15, nodes=1, tasks=1, name='kernels')\ ''' for H in [1 / 70, 0.08]: for gen_sigma in [0.75, 1, 1.25]: for n in [12]: pbs.pmap(simulate_regression, [(12000, H, 1, 1, n, out_dir, gen_sigma)], walltime='4:00:00', memory=15, nodes=1, tasks=1, name='kernels')
def submit(): from pymeg import parallel for i in range(len(recordings)): parallel.pmap( wrap_preprocess, [(i,)], walltime='15:00:00', memory=50, nodes=1, tasks=4, name='PREP_'+recordings[i].filename, ssh_to=None, env='mne')
def submit(sub, env='Climag'): for ses in [2, 3]: pbs.pmap(execute, [(sub, ses, env)], walltime='4:00:00', memory=40, nodes=1, tasks=2, name='subvert_sub-{}'.format(sub))
def climag_submit(subrange): flex_dir = '/home/khagena/FLEXRULE' for sub in subrange: pbs.pmap(execute, [(sub, flex_dir)], walltime='4:00:00', memory=15, nodes=1, tasks=1, name='decode_{0}'.format(sub))
def climag_submit(): flex_dir = '/home/khagena/FLEXRULE' for sub in range(1, 23): for task in ['inference', 'instructed']: pbs.pmap(extract_brainstem, [(sub, flex_dir, task)], walltime='1:00:00', memory=15, nodes=1, tasks=1, name='bs_coefs_{}'.format(sub))
def climag_submit(GLM_run, subrange): flex_dir = '/home/khagena/FLEXRULE' for sub in subrange: for task in ['inference', 'instructed']: pbs.pmap(extract_brainstem, [(sub, flex_dir, GLM_run, task)], walltime='1:00:00', memory=15, nodes=1, tasks=1, name='coefs_{0}_{1}'.format(sub, task))
def submit(recs=None): if recs is None: recs = np.arange(len(recordings)) from pymeg import parallel for i in recs: parallel.pmap( wrap_preprocess, [(i,)], walltime='15:00:00', memory=50, nodes=1, tasks=4, name='PREP_'+recordings[i].filename, ssh_to=None, env='py36')
def submit_aggregates(cluster='uke'): from pymeg import parallel for subject, session in subjects.items(): for sessnum in range(1, session + 1): for datatype in ['F']: #,'BB']: parallel.pmap(aggregate, [(subject, sessnum, datatype)], name='agg' + str(sessnum) + str(subject) + datatype, tasks=6, memory=60, walltime='12:00:00', env="mne")
def submit(): from pymeg import parallel for subject, tasks in subjects.items(): for session, recording in tasks: for signal in ['BB', 'HF', 'LF']: parallel.pmap(extract, [(subject, session, recording, signal)], walltime='15:00:00', memory=50, nodes=1, tasks=4, name='SR' + str(subject) + '_' + str(session) + str(recording), ssh_to=None, env='mne')
def submit(older_than="201911010000", only_glasser=False): from pymeg import parallel from itertools import product cnt = 1 older_than = datetime.datetime.strptime(older_than, "%Y%m%d%H%M") cnt = 1 #for subject, session, epoch, signal in [ # [3, 0, "stimulus", "F"], # [9, 2, "stimulus", "F"], # [10, 0, "response", "F"], #]: for subject, session, epoch, signal in product(range(1, 16), range(4), ["response"], ["F", "LF"]): mod_time = [ modification_date(x) for x in lcmvfilename(subject, session, signal, epoch, chunk="all", only_glasser=only_glasser) ] # if(any([x > older_than for x in mod_time])): # print("Skipping %i %i %s %s because existing output is newer than requested date" % ( # subject, session, epoch, signal)) # continue # def extract( # subject, # session, # epoch_type="stimulus", # signal_type="BB", # only_glasser=False, print("Submitting %i %i %s %s" % (subject, session, epoch, signal)) parallel.pmap( extract, [(subject, session, epoch, signal, only_glasser)], walltime="10:00:00", memory=40, nodes=1, tasks=5, env="py36", name="SR" + str(subject) + "_" + str(session) + epoch, ssh_to=None, ) # if np.mod(cnt, 15) == 0: # import time # time.sleep(60 * 30) cnt += 1
def submit_aggregates(cluster="uke", only_glasser=False): from pymeg import parallel import time for subject, epoch, session in product( [1, 2, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15, 3, 9], ["stimulus"], range(4)): parallel.pmap( aggregate, [(subject, session, epoch, only_glasser)], name="agg" + str(session) + epoch + str(subject), tasks=8, memory=60, walltime="12:00:00", )
def submit(): out_dir = join('/home/khagena/FLEXRULE/behavior/kernels_psi-{}-3'.format( datetime.datetime.now().strftime("%Y-%m-%d"))) slu.mkdir_p(out_dir) C = 1 n = 12 run = samples['psi4'] for sub in range(1, 23): for psi in [False, True]: for mode in ['normative']: pbs.pmap(regress, [(n, run, C, out_dir, mode, sub, psi)], walltime='4:00:00', memory=15, nodes=1, tasks=1, name='kernels_{0}'.format(n))
def _eval(func, args, collect=False, **kw): """ Intermediate helper to toggle cluster vs non cluster """ if not collect: if not func.in_store(*args): print("Submitting %s to %s for parallel execution" % (len(args), func)) print(args) parallel.pmap(func, [args], **kw) else: if func.in_store(*args): print("Submitting %s to %s for collection" % (str(args), func)) df = func(*args) return df else: raise RuntimeError("Result not available.")
def single(): out_dir = join( '/home/khagena/FLEXRULE/behavior/kernel_simulation/simulate_murphy_task' ) slu.mkdir_p(out_dir) for H in [1 / 70]: for gen_sigma in [0.75]: for n in [12]: for C in [1]: pbs.pmap(simulate_regression, [(50, H, 1, C, n, out_dir, gen_sigma)], walltime='1:00:00', memory=15, nodes=1, tasks=1, name='kernels')
def submit(): out_dir = join( '/home/khagena/FLEXRULE/behavior/kernel_simulation/simulate_murphy_task' ) slu.mkdir_p(out_dir) for H in [0.001, 0.01, 1 / 70, 0.08, 0.2, 0.3]: for gen_sigma in [1, 0.5, 0.75, 1.25, 1.5]: for n in [12]: for C in [1, 1e8]: pbs.pmap(simulate_regression, [(5000, H, 1, C, n, out_dir, gen_sigma)], walltime='4:00:00', memory=15, nodes=1, tasks=1, name='kernels')
def submit(only_glasser=False): from pymeg import parallel from itertools import product for area in list(areas_to_labels.keys()): for subject in range(1, 16): print("Submitting S%i -> %s" % (subject, area)) parallel.pmap( decode, [(subject, area)], walltime="12:00:00", memory=50, nodes=1, tasks=6, env="py36", name="iDCD" + str(subject) + area, ssh_to=None, )
def submit(recordings=None): if recordings is None: recordings = range(ps.recordings) from pymeg import parallel for recording in recordings: for epoch in ['stimulus', 'response']: for signal in ["HF", "LF"]: parallel.pmap( extract, [(recording, epoch, signal)], walltime="15:00:00", memory=50, nodes=1, tasks=4, name="SR" + str(subject) + "_" + str(session) + str(recording), ssh_to=None, env="mne", )
def submit_aggregates(subjects=range(1, 16), epochs=['stimulus', 'response', 'confidence'], cluster="uke", only_glasser=True): from pymeg import parallel import time for subject, epoch, session in product(subjects, epochs, range(2)): parallel.pmap( aggregate, [( subject, session, epoch, )], name="agg" + str(session) + epoch + str(subject), tasks=8, memory=60, walltime="12:00:00", )
def single(): fits = pd.read_csv('/home/khagena/FLEXRULE/behavior/summary_stan_fits.csv') subjects = fits.loc[fits.vmode < 2.5].subject.unique() out_dir = join( '/home/khagena/FLEXRULE/behavior/kernel_simulation/KernelSimulation_{}-2' .format(datetime.datetime.now().strftime("%Y-%m-%d"))) slu.mkdir_p(out_dir) for subject in subjects[0:2]: V = fits.loc[fits.subject == subject].vmode.mean() H = fits.loc[fits.subject == subject].hmode.mean() gen_sigma = 0.75 for C in [1]: for n in [8]: print(subject, n, C, V, H, out_dir) pbs.pmap(simulate_regression, [(100000, H, V, C, n, out_dir, gen_sigma)], walltime='1:00:00', memory=15, nodes=1, tasks=1, name='kernels')
def submit(sub, env='Climag'): if env == 'Hummel': def keys(sub): keys = [] for ses in [2, 3]: keys.append((sub, ses, env)) return keys slu.pmap(par_execute, keys(sub), walltime='2:00:00', memory=40, nodes=1, tasks=2, name='SubjectLevel') elif env == 'Climag': for ses in [2, 3]: pbs.pmap(execute, [(sub, ses, env)], walltime='20:00:00', memory=40, nodes=1, tasks=2, name='subvert_sub-{}'.format(sub))
def submit_aggregates(collect=False): import numpy as np import time tasks = [] subjects = [1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 17, 18, 19] for subject in subjects: for session in [5,6,7,8]: for epoch in ['stimulus', 'response']: tasks.append([(subject, session, epoch)]) res = [] for cnt, task in enumerate(tasks): try: r = parallel.pmap( get_aggregate, task, walltime="05:30:00", tasks=4, memory=40, ) res.append(r) except RuntimeError: print("Task", task, " not available yet") return res
def submit_surface_data(glm_run): run_dir = join('/home/khagena/FLEXRULE/Workflow', glm_run) for task in ['inference', 'instructed']: pbs.pmap(surface_data, [(run_dir, task)], walltime='1:00:00', memory=15, nodes=1, tasks=1, name='surface_data_{0}'.format(task))