def submit_all():
from pymeg import parallel
# fmt: off
subjects = [
1,
4,
5,
6,
7,
8,
9,
10,
11,
12,
14,
15,
16,
17,
18,
19,
]
# fmt: on
for session in [5, 6, 7]:
for subject in subjects:
parallel.pmap(
preprocess,
[(subject, session)],
walltime="02:30:00",
tasks=3,
memory=30,
)
def submit(sub, env='Hummel'):
name_list = [
"The", "owls", "are", "not", "what", "they", "seem", "COOPER",
"COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER",
"COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER",
"COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER",
"COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER", "COOPER",
"COOPER", "COOPER", "COOPER", "COOPER", "COOPER"
]
if env == 'Hummel':
def keys(sub):
keys = []
for ses in [2, 3]:
keys.append((sub, ses, env))
return keys
slu.pmap(par_execute,
keys(sub),
walltime='2:00:00',
memory=40,
nodes=1,
tasks=2,
name='SubjectLevel')
elif env == 'Climag':
for ses in [2, 3]:
pbs.pmap(execute, [(sub, ses, env)],
walltime='4:00:00',
memory=40,
nodes=1,
tasks=2,
name=name_list[sub])
def submit():
fits = pd.read_csv('/home/khagena/FLEXRULE/behavior/summary_stan_fits.csv')
subjects = fits.loc[fits.vmode < 2.5].subject.unique()
out_dir = join(
'/home/khagena/FLEXRULE/behavior/kernel_simulation/KernelSimulation_{}'
.format(datetime.datetime.now().strftime("%Y-%m-%d")))
slu.mkdir_p(out_dir)
'''
for subject in subjects:
V = fits.loc[fits.subject == subject].vmode.mean()
H = fits.loc[fits.subject == subject].hmode.mean()
for C in [1, 1e8]:
for n in [8, 12]:
pbs.pmap(simulate_regression, [(100000, H, V, C, n, out_dir, subject)],
walltime='1:00:00', memory=15, nodes=1, tasks=1,
name='kernels')\
'''
for H in [1 / 70, 0.08]:
for gen_sigma in [0.75, 1, 1.25]:
for n in [12]:
pbs.pmap(simulate_regression,
[(12000, H, 1, 1, n, out_dir, gen_sigma)],
walltime='4:00:00',
memory=15,
nodes=1,
tasks=1,
name='kernels')
def submit():
from pymeg import parallel
for i in range(len(recordings)):
parallel.pmap(
wrap_preprocess, [(i,)],
walltime='15:00:00', memory=50, nodes=1, tasks=4,
name='PREP_'+recordings[i].filename,
ssh_to=None, env='mne')
def submit(sub, env='Climag'):
for ses in [2, 3]:
pbs.pmap(execute, [(sub, ses, env)],
walltime='4:00:00',
memory=40,
nodes=1,
tasks=2,
name='subvert_sub-{}'.format(sub))
def climag_submit(subrange):
flex_dir = '/home/khagena/FLEXRULE'
for sub in subrange:
pbs.pmap(execute, [(sub, flex_dir)],
walltime='4:00:00',
memory=15,
nodes=1,
tasks=1,
name='decode_{0}'.format(sub))
def climag_submit():
flex_dir = '/home/khagena/FLEXRULE'
for sub in range(1, 23):
for task in ['inference', 'instructed']:
pbs.pmap(extract_brainstem, [(sub, flex_dir, task)],
walltime='1:00:00',
memory=15,
nodes=1,
tasks=1,
name='bs_coefs_{}'.format(sub))
def climag_submit(GLM_run, subrange):
flex_dir = '/home/khagena/FLEXRULE'
for sub in subrange:
for task in ['inference', 'instructed']:
pbs.pmap(extract_brainstem, [(sub, flex_dir, GLM_run, task)],
walltime='1:00:00',
memory=15,
nodes=1,
tasks=1,
name='coefs_{0}_{1}'.format(sub, task))
def submit(recs=None):
if recs is None:
recs = np.arange(len(recordings))
from pymeg import parallel
for i in recs:
parallel.pmap(
wrap_preprocess, [(i,)],
walltime='15:00:00', memory=50, nodes=1, tasks=4,
name='PREP_'+recordings[i].filename,
ssh_to=None, env='py36')
def submit_aggregates(cluster='uke'):
from pymeg import parallel
for subject, session in subjects.items():
for sessnum in range(1, session + 1):
for datatype in ['F']: #,'BB']:
parallel.pmap(aggregate, [(subject, sessnum, datatype)],
name='agg' + str(sessnum) + str(subject) +
datatype,
tasks=6,
memory=60,
walltime='12:00:00',
env="mne")
def submit():
from pymeg import parallel
for subject, tasks in subjects.items():
for session, recording in tasks:
for signal in ['BB', 'HF', 'LF']:
parallel.pmap(extract, [(subject, session, recording, signal)],
walltime='15:00:00',
memory=50,
nodes=1,
tasks=4,
name='SR' + str(subject) + '_' + str(session) +
str(recording),
ssh_to=None,
env='mne')
def submit(older_than="201911010000", only_glasser=False):
from pymeg import parallel
from itertools import product
cnt = 1
older_than = datetime.datetime.strptime(older_than, "%Y%m%d%H%M")
cnt = 1
#for subject, session, epoch, signal in [
# [3, 0, "stimulus", "F"],
# [9, 2, "stimulus", "F"],
# [10, 0, "response", "F"],
#]:
for subject, session, epoch, signal in product(range(1, 16), range(4),
["response"], ["F", "LF"]):
mod_time = [
modification_date(x)
for x in lcmvfilename(subject,
session,
signal,
epoch,
chunk="all",
only_glasser=only_glasser)
]
# if(any([x > older_than for x in mod_time])):
# print("Skipping %i %i %s %s because existing output is newer than requested date" % (
# subject, session, epoch, signal))
# continue
# def extract(
# subject,
# session,
# epoch_type="stimulus",
# signal_type="BB",
# only_glasser=False,
print("Submitting %i %i %s %s" % (subject, session, epoch, signal))
parallel.pmap(
extract,
[(subject, session, epoch, signal, only_glasser)],
walltime="10:00:00",
memory=40,
nodes=1,
tasks=5,
env="py36",
name="SR" + str(subject) + "_" + str(session) + epoch,
ssh_to=None,
)
# if np.mod(cnt, 15) == 0:
# import time
# time.sleep(60 * 30)
cnt += 1
def submit_aggregates(cluster="uke", only_glasser=False):
from pymeg import parallel
import time
for subject, epoch, session in product(
[1, 2, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15, 3, 9], ["stimulus"],
range(4)):
parallel.pmap(
aggregate,
[(subject, session, epoch, only_glasser)],
name="agg" + str(session) + epoch + str(subject),
tasks=8,
memory=60,
walltime="12:00:00",
)
def submit():
out_dir = join('/home/khagena/FLEXRULE/behavior/kernels_psi-{}-3'.format(
datetime.datetime.now().strftime("%Y-%m-%d")))
slu.mkdir_p(out_dir)
C = 1
n = 12
run = samples['psi4']
for sub in range(1, 23):
for psi in [False, True]:
for mode in ['normative']:
pbs.pmap(regress, [(n, run, C, out_dir, mode, sub, psi)],
walltime='4:00:00',
memory=15,
nodes=1,
tasks=1,
name='kernels_{0}'.format(n))
def _eval(func, args, collect=False, **kw):
"""
Intermediate helper to toggle cluster vs non cluster
"""
if not collect:
if not func.in_store(*args):
print("Submitting %s to %s for parallel execution" % (len(args), func))
print(args)
parallel.pmap(func, [args], **kw)
else:
if func.in_store(*args):
print("Submitting %s to %s for collection" % (str(args), func))
df = func(*args)
return df
else:
raise RuntimeError("Result not available.")
def single():
out_dir = join(
'/home/khagena/FLEXRULE/behavior/kernel_simulation/simulate_murphy_task'
)
slu.mkdir_p(out_dir)
for H in [1 / 70]:
for gen_sigma in [0.75]:
for n in [12]:
for C in [1]:
pbs.pmap(simulate_regression,
[(50, H, 1, C, n, out_dir, gen_sigma)],
walltime='1:00:00',
memory=15,
nodes=1,
tasks=1,
name='kernels')
def submit():
out_dir = join(
'/home/khagena/FLEXRULE/behavior/kernel_simulation/simulate_murphy_task'
)
slu.mkdir_p(out_dir)
for H in [0.001, 0.01, 1 / 70, 0.08, 0.2, 0.3]:
for gen_sigma in [1, 0.5, 0.75, 1.25, 1.5]:
for n in [12]:
for C in [1, 1e8]:
pbs.pmap(simulate_regression,
[(5000, H, 1, C, n, out_dir, gen_sigma)],
walltime='4:00:00',
memory=15,
nodes=1,
tasks=1,
name='kernels')
def submit(only_glasser=False):
from pymeg import parallel
from itertools import product
for area in list(areas_to_labels.keys()):
for subject in range(1, 16):
print("Submitting S%i -> %s" % (subject, area))
parallel.pmap(
decode,
[(subject, area)],
walltime="12:00:00",
memory=50,
nodes=1,
tasks=6,
env="py36",
name="iDCD" + str(subject) + area,
ssh_to=None,
)
def submit(recordings=None):
if recordings is None:
recordings = range(ps.recordings)
from pymeg import parallel
for recording in recordings:
for epoch in ['stimulus', 'response']:
for signal in ["HF", "LF"]:
parallel.pmap(
extract,
[(recording, epoch, signal)],
walltime="15:00:00",
memory=50,
nodes=1,
tasks=4,
name="SR" + str(subject) + "_" + str(session) +
str(recording),
ssh_to=None,
env="mne",
)
def submit_aggregates(subjects=range(1, 16),
epochs=['stimulus', 'response', 'confidence'],
cluster="uke",
only_glasser=True):
from pymeg import parallel
import time
for subject, epoch, session in product(subjects, epochs, range(2)):
parallel.pmap(
aggregate,
[(
subject,
session,
epoch,
)],
name="agg" + str(session) + epoch + str(subject),
tasks=8,
memory=60,
walltime="12:00:00",
)
def single():
fits = pd.read_csv('/home/khagena/FLEXRULE/behavior/summary_stan_fits.csv')
subjects = fits.loc[fits.vmode < 2.5].subject.unique()
out_dir = join(
'/home/khagena/FLEXRULE/behavior/kernel_simulation/KernelSimulation_{}-2'
.format(datetime.datetime.now().strftime("%Y-%m-%d")))
slu.mkdir_p(out_dir)
for subject in subjects[0:2]:
V = fits.loc[fits.subject == subject].vmode.mean()
H = fits.loc[fits.subject == subject].hmode.mean()
gen_sigma = 0.75
for C in [1]:
for n in [8]:
print(subject, n, C, V, H, out_dir)
pbs.pmap(simulate_regression,
[(100000, H, V, C, n, out_dir, gen_sigma)],
walltime='1:00:00',
memory=15,
nodes=1,
tasks=1,
name='kernels')
def submit(sub, env='Climag'):
if env == 'Hummel':
def keys(sub):
keys = []
for ses in [2, 3]:
keys.append((sub, ses, env))
return keys
slu.pmap(par_execute,
keys(sub),
walltime='2:00:00',
memory=40,
nodes=1,
tasks=2,
name='SubjectLevel')
elif env == 'Climag':
for ses in [2, 3]:
pbs.pmap(execute, [(sub, ses, env)],
walltime='20:00:00',
memory=40,
nodes=1,
tasks=2,
name='subvert_sub-{}'.format(sub))
def submit_aggregates(collect=False):
import numpy as np
import time
tasks = []
subjects = [1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 17, 18, 19]
for subject in subjects:
for session in [5,6,7,8]:
for epoch in ['stimulus', 'response']:
tasks.append([(subject, session, epoch)])
res = []
for cnt, task in enumerate(tasks):
try:
r = parallel.pmap(
get_aggregate,
task,
walltime="05:30:00",
tasks=4,
memory=40,
)
res.append(r)
except RuntimeError:
print("Task", task, " not available yet")
return res
def submit_surface_data(glm_run):
run_dir = join('/home/khagena/FLEXRULE/Workflow', glm_run)
for task in ['inference', 'instructed']:
pbs.pmap(surface_data, [(run_dir, task)],
walltime='1:00:00', memory=15, nodes=1, tasks=1,
name='surface_data_{0}'.format(task))
