try:
output_path = str(sys.argv[2])
except:
output_path = "/home/mszul/git/DANC_learning_beh/data"
print(subjects_path, op.exists(subjects_path))
print(output_path, op.exists(output_path))
subs = files.get_folders_files(subjects_path)[0]
subs.sort()
csv_files = []
for sub in subs:
print(sub)
csvs = files.get_files(sub, "sub", "-beh.csv")[2]
csvs.sort
csv_files.extend(csvs)
all_data = []
for i in csv_files:
sub_id = i.split(sep)[-2]
file_data = pd.read_csv(i)
file_data.subject_id = sub_id
all_data.append(file_data)
all_data = pd.concat(all_data)
all_data = all_data.sort_values(["subject_id", "block", "trial_in_block"])
all_data.reset_index(inplace=True)
all_data["reach_sub_perturb"] = all_data.reach_target - all_data.perturb_cat
import time
import sys
import os.path as op
import json
from joblib import Parallel, delayed
try:
path = str(sys.argv[1])
except:
print("incorrect path")
sys.exit()
print(path)
files_npy = files.get_files(path, "", ".npy")[2]
files_npy.sort()
files_json = files.get_files(path, "", ".json")[2]
files_json.sort()
# files_npy_json = list(zip(files_npy, files_json))
# print(files_npy_json)
def convert(file):
filename = file.split("/")[-1].split(".")[0]
raw = np.load(file, allow_pickle=True)
f_size = (1280, 1024)
fourcc = cv2.VideoWriter_fourcc(*'MJPG')
vid = cv2.VideoWriter(
"{}.avi".format(filename),
files.make_folder(der_path)
proc_path = op.join(der_path, "processed")
files.make_folder(proc_path)
subjects = files.get_folders_files(sub_path)[0]
subjects.sort()
for subject in subjects:
subject_id = subject.split("/")[-1]
meg_path = op.join(subject, "ses-01", "meg")
if len(files.get_folders_files(meg_path)[0]) > 0:
continue
else:
sub_path = op.join(proc_path, subject_id)
files.make_folder(sub_path)
zip_file = files.get_files(meg_path, "MEG", ".zip")[2][0]
print(files.get_files(meg_path, "MEG", ".zip")[1])
sp.call(["unzip", zip_file, "-d", meg_path])
unzip_path_part = op.join(meg_path, "scans", "MEGSCAN_CTF")
unzip_path = files.get_folders_files(unzip_path_part)[0][0]
print(unzip_path, op.exists(unzip_path))
path_to_move = "mv -v " + unzip_path + "/* " + meg_path
print(path_to_move)
sp.call(path_to_move, shell=True)
sp.call(["rm", "-rf", op.join(meg_path, "scans")])
print(subject_id, "DONE")
files.make_folder(der_path)
proc_path = op.join(der_path, "processed")
files.make_folder(proc_path)
subjects = files.get_folders_files(proc_path)[0]
subjects.sort()
subject = subjects[index]
subject_id = subject.split("/")[-1]
sub_path = op.join(proc_path, subject_id)
files.make_folder(sub_path)
qc_folder = op.join(sub_path, "QC")
files.make_folder(qc_folder)
epo_paths = files.get_files(sub_path, "sub", "-epo.fif")[2]
epo_paths.sort()
beh_paths = files.get_files(sub_path, "sub", "-beh.csv")[2]
eve_paths = files.get_files(sub_path, "sub", "-eve.fif")[2]
cmap = colors.ListedColormap(["#FFFFFF", "#CFEEFA", "#FFDE00", "#FF9900", "#FF0000", "#000000"])
boundaries = [-0.9, -0.1, 1.1, 10, 100, 1000, 10000]
norm = colors.BoundaryNorm(boundaries, cmap.N, clip=True)
# for epo in epo_paths:
for epo in epo_paths[1:]:
numero = epo.split(sep)[-1].split("-")[2]
epochs = read_epochs(epo, verbose=False)
epochs = epochs.decimate(3)
print("INPUT FILE:", epo)
beh_path = [i for i in beh_paths if numero in i][0]
files.make_folder(proc_path)
subjects = files.get_folders_files(proc_path)[0]
subjects.sort()
subject = subjects[index]
subject_id = subject.split("/")[-1]
meg_path = op.join(subject, "ses-01", "meg")
sub_path = op.join(proc_path, subject_id)
files.make_folder(sub_path)
qc_folder = op.join(sub_path, "QC")
files.make_folder(qc_folder)
raw_paths = files.get_files(sub_path, subject_id, "-raw.fif")[2]
raw_paths.sort()
event_paths = files.get_files(sub_path, subject_id, "-eve.fif")[2]
event_paths.sort()
raw_eve_list = list(zip(raw_paths, event_paths))
# for raw_path, eve_path in [raw_eve_list[0]]:
for raw_path, eve_path in raw_eve_list:
print("INPUT RAW FILE:", raw_path)
print("EVE_RAW MATCH:", raw_path.split("-")[-2] == eve_path.split("-")[-2])
numero = str(raw_path.split("-")[-2]).zfill(3)
raw = mne.io.read_raw_fif(raw_path, verbose=False)
# raw = raw.apply_gradient_compensation(2, verbose=True)
raw = raw.pick_types(meg=True, eeg=False, ref_meg=True)
files.make_folder(proc_path)
subjects = files.get_folders_files(proc_path)[0]
subjects.sort()
subject = subjects[index]
subject_id = subject.split("/")[-1]
sub_path = op.join(proc_path, subject_id)
files.make_folder(sub_path)
all_folders = files.get_folders_files(sub_path)[0]
all_folders = [i for i in all_folders if "-epo" in i]
all_folders.sort()
all_visual = [i for i in all_folders if "visual" in i]
all_motor = [i for i in all_folders if "motor" in i]
mot_size = 658038528
vis_size = 789558528
for folder in all_visual:
all_files = files.get_files(folder, "", ".npy")[2]
for file in all_files:
if op.getsize(file) < vis_size:
print(file, op.getsize(file))
for folder in all_motor:
all_files = files.get_files(folder, "", ".npy")[2]
for file in all_files:
if op.getsize(file) < mot_size:
print(file, op.getsize(file))
resamp = interp1d(x, y, kind='slinear', fill_value='extrapolate')
new_data = resamp(new_x)
return new_data
path = parameters["dataset_path"]
sfreq = parameters["downsample_head"]
sub_path = op.join(path, "data")
subjects = files.get_folders_files(sub_path)[0]
subject = subjects[index]
subject_id = subject.split("/")[-1]
beh_path = op.join(subject, "ses-01", "behaviour")
der_path = op.join(path, "derivatives")
head_path = op.join(der_path, "head_motion", subject_id)
beh_files = files.get_files(beh_path, "block", ".mat")[2]
beh_files.sort()
head_csvs = files.get_files(head_path, subject_id, ".csv")[2]
head_csvs.sort()
head_evnts = files.get_files(head_path, subject_id, "-eve.fif")[2]
head_evnts.sort()
b_h_e = list(zip(beh_files, head_csvs, head_evnts))
for (beh_f, hpi_f, ev_f) in b_h_e[1:]:
print(beh_f)
print(hpi_f)
print(ev_f)
begin = mne.read_events(ev_f, include=[50])
files.make_folder(proc_path)
subjects = files.get_folders_files(proc_path)[0]
subjects.sort()
subject = subjects[index]
subject_id = subject.split("/")[-1]
meg_path = op.join(subject, "ses-01", "meg")
sub_path = op.join(proc_path, subject_id)
files.make_folder(sub_path)
qc_folder = op.join(sub_path, "QC")
files.make_folder(qc_folder)
raw_paths = files.get_files(sub_path, "zapline-" + subject_id, "-raw.fif")[2]
raw_paths.sort()
raw_path = raw_paths[file_index]
ica_paths = files.get_files(sub_path, subject_id, "-ica.fif")[2]
ica_paths.sort()
ica_path = ica_paths[file_index]
ica_json_file = op.join(sub_path, "{}-ICA_to_reject.json".format(subject_id))
print("SUBJ: {}".format(subject_id), index, file_index)
print("INPUT RAW FILE:", raw_path.split(os.sep)[-1])
print("INPUT ICA FILE:", ica_path.split(os.sep)[-1])
print("INPUT JSON FILE", ica_json_file.split(os.sep)[-1])
raw = mne.io.read_raw_fif(raw_path, preload=True, verbose=False)
files.make_folder(der_path)
proc_path = op.join(der_path, "processed")
files.make_folder(proc_path)
subjects = files.get_folders_files(proc_path)[0]
subjects.sort()
subject = subjects[index]
subject_id = subject.split(sep)[-1]
sub_path = op.join(proc_path, subject_id)
files.make_folder(sub_path)
qc_folder = op.join(sub_path, "QC")
files.make_folder(qc_folder)
epo_paths = files.get_files(sub_path, "sub", "-epo.fif")[2]
epo_paths.sort()
for epo_path in epo_paths:
filename = epo_path.split(sep)[-1].split(".")[0]
npz_path = op.join(subject, "{}.npz".format(filename))
output_folder = op.join(subject, filename)
files.make_folder(output_folder)
if not op.exists(npz_path):
print(strftime("%a, %d %b %Y %H:%M:%S", gmtime()), "STARTED",
output_folder)
epochs = read_epochs(epo_path)
epochs = epochs.pick_types(meg=True, ref_meg=False)
for ix, epoch in enumerate(epochs.__iter__()):
fname = "{}-{}.npy".format(str(ix).zfill(3), filename)
output_array = superlets_do_single_epoch(epoch,
files.make_folder(der_path)
proc_path = op.join(der_path, "processed")
files.make_folder(proc_path)
subjects = files.get_folders_files(proc_path)[0]
subjects.sort()
subject = subjects[index]
subject_id = subject.split("/")[-1]
sub_path = op.join(proc_path, subject_id)
files.make_folder(sub_path)
#setting the paths and extracting files
slt_mot_paths = [i for i in files.get_folders_files(sub_path)[0] if "motor" in i]
slt_vis_paths = [i for i in files.get_folders_files(sub_path)[0] if "visual" in i]
epo_mot_paths = files.get_files(sub_path, "sub", "motor-epo.fif")[2]
epo_vis_paths = files.get_files(sub_path, "sub", "visual-epo.fif")[2]
beh_match_path = files.get_files(sub_path, "sub", "beh-match.json")[2][0]
with open(beh_match_path) as f:
beh_match = json.load(f)
slt_mot_paths.sort()
slt_vis_paths.sort()
epo_mot_paths.sort()
epo_vis_paths.sort()
epo_slt_mot_vis = list(zip(epo_mot_paths, epo_vis_paths, slt_mot_paths, slt_vis_paths))
info = read_epochs(epo_mot_paths[0], verbose=False)
info.pick_types(meg=True, ref_meg=False, misc=False)
info = info.info
freqs = np.linspace(1,120, num=400)
search_range = np.where((freqs >= 10) & (freqs <= 33))[0]
der_path = op.join(path, "derivatives")
files.make_folder(der_path)
proc_path = op.join(der_path, "processed")
files.make_folder(proc_path)
subjects = files.get_folders_files(sub_path)[0]
subjects.sort()
subject = subjects[index]
subject_id = subject.split("/")[-1]
print("ID:", subject_id)
beh_path = op.join(subject, "ses-01", "behaviour")
beh_files = files.get_files(beh_path, "block", ".mat")[2]
beh_files.sort()
sub_path = op.join(proc_path, subject_id)
files.make_folder(sub_path)
qc_folder = op.join(sub_path, "QC")
files.make_folder(qc_folder)
# for beh_file in [beh_files[4]]:
for beh_file in beh_files:
numero = beh_file.split(os.sep)[-1].split("_")[-1].split(".")[0].zfill(3)
# try:
# if numero == "001":
# raise Exception
# opening a json file
with open(json_file) as pipeline_file:
parameters = json.load(pipeline_file)
path = parameters["dataset_path"]
sfreq = parameters["sfreq"]
sub_path = op.join(path, "data")
subjects = files.get_folders_files(sub_path)[0]
subject = subjects[index]
subject_id = subject.split("/")[-1]
der_path = op.join(path, "derivatives")
head_path = op.join(der_path, "head_motion", subject_id)
head_csvs = files.get_files(head_path, subject_id, ".csv")[2]
head_csvs.sort()
head_evnts = files.get_files(head_path, subject_id, "-eve.fif")[2]
head_evnts.sort()
h_e = list(zip(head_csvs, head_evnts))
thyme = np.arange(-0.6, 1.2, 0.008)
params = {
"pitch": (0, 0, "#ffca44"),
"roll": (0, 1, "#f344ff"),
"yaw": (0, 2, "#445dff"),
"X": (1, 0, "#ff2600"),
"Y": (1, 1, "#8c3fff"),
"Z": (1, 2, "#33ff00")
path = parameters["dataset_path"]
sfreq = parameters["downsample_dataset"]
der_path = op.join(path, "derivatives")
files.make_folder(der_path)
proc_path = op.join(der_path, "processed")
files.make_folder(proc_path)
subjects = files.get_folders_files(proc_path)[0]
subjects.sort()
subject = subjects[index]
subject_id = subject.split("/")[-1]
beh_path = op.join(path, "data", subject_id, "ses-01", "behaviour")
edf_paths = files.get_files(beh_path, "", ".edf")[2]
edf_paths.sort()
edf_paths = [i for i in edf_paths if "0.edf" not in i]
sub_path = op.join(proc_path, subject_id)
files.make_folder(sub_path)
qc_folder = op.join(sub_path, "QC")
files.make_folder(qc_folder)
raw_paths = files.get_files(sub_path, "zapline-" + subject_id, "-raw.fif")[2]
raw_paths.sort()
ica_json_file = op.join(sub_path, "{}-ICA_to_reject.json".format(subject_id))
with open(ica_json_file) as ica_file:
files.make_folder(der_path)
proc_path = op.join(der_path, "processed")
files.make_folder(proc_path)
subjects = files.get_folders_files(proc_path)[0]
subjects.sort()
subject = subjects[index]
subject_id = subject.split("/")[-1]
print(subject)
raw_meg_dir = op.join(path, "data")
raw_meg_path = op.join(raw_meg_dir, subject_id, "ses-01", "meg")
ds_paths = files.get_folders_files(raw_meg_path)[0]
ds_paths = [i for i in ds_paths if "misc" not in i]
ds_paths.sort()
res4_paths = [files.get_files(i, "", ".res4")[2][0] for i in ds_paths]
res4_paths.sort()
fs_folder = op.join(der_path, "freesurfer", subject_id)
surface_file = op.join(fs_folder, "pial.ds.gii")
mri_file = op.join(fs_folder, "T1_headcast.nii")
src_folder = op.join(der_path, "source")
files.make_folder(src_folder)
#### MODIFY THE FIF SEARCH PATHS ####
fif_paths = files.get_files(subject, "sub", "motor-epo.fif")[2]
fif_paths.sort()
subject_data = files.get_files(raw_meg_dir, "", ".tsv")[2][0]
print(subject_data)
misc_value = -5
score = 0
# misc
score_text = visual.TextStim(win, text='score', height=1, color='black', pos=[0, h/2+2.5])
cursor = visual.Circle(win, radius=1.5, lineColor='white', autoDraw=True)
info_text = visual.TextStim(win, text='score', height=1, color='black', pos=[0, 0])
stopwatch = core.Clock()
ITI = core.StaticPeriod(win=win)
counter = 0
# stimuli
healthy = files.get_files('icons/healthy', '', 'png', wp=True)[0]
junk = files.get_files('icons/junk', '', 'png', wp=True)[0]
misc = files.get_files('icons/misc', '', 'png', wp=True)[0]
# init objects of generator
healthy_images = list(np.random.choice(healthy, healthy_am))
junk_images = list(np.random.choice(junk, junk_am))
misc_images = list(np.random.choice(misc, misc_am))
all_images = healthy_images + junk_images + misc_images
image_order = range(grid_size)
np.random.shuffle(image_order)
files.make_folder(der_path)
proc_path = op.join(der_path, "processed")
files.make_folder(proc_path)
subjects = files.get_folders_files(proc_path)[0]
subjects.sort()
subject = subjects[index]
subject_id = subject.split("/")[-1]
sub_path = op.join(proc_path, subject_id)
files.make_folder(sub_path)
qc_folder = op.join(sub_path, "QC")
files.make_folder(qc_folder)
mot_epo_paths = files.get_files(sub_path, "sub", "motor-epo.fif")[2]
mot_epo_paths.sort()
vis_epo_paths = files.get_files(sub_path, "sub", "visual-epo.fif")[2]
vis_epo_paths.sort()
beh_paths = files.get_files(sub_path, "sub", "-beh.csv")[2]
beh_paths.sort()
eve_paths = files.get_files(sub_path, "sub", "-eve.fif")[2]
eve_paths.sort()
match_out_dict = {}
# for epo in epo_paths:
for beh_path in beh_paths[1:]:
numero = beh_path.split(sep)[-1].split("-")[2]
mot_epo_path = [i for i in mot_epo_paths if numero in i][0]
vis_epo_path = [i for i in vis_epo_paths if numero in i][0]
eve_path = [i for i in eve_paths if numero in i][0]