def clabel_table_create(
common_acts, n_parts=12, data_lp="/data2/users/stepeter/files_nwb/downloads/000055/"
):
"""Create table of coarse label durations across participants.
Labels to include in the table are specified by common_acts."""
with DandiAPIClient() as client:
paths = []
for file in client.get_dandiset("000055", "draft").get_assets_with_path_prefix(""):
paths.append(file.path)
paths = natsort.natsorted(paths)
vals_all = np.zeros([n_parts, len(common_acts) + 1])
for part_ind in tqdm(range(n_parts)):
fids = [val for val in paths if "sub-" + str(part_ind + 1).zfill(2) in val]
for fid in fids:
with DandiAPIClient() as client:
asset = client.get_dandiset("000055", "draft").get_asset_by_path(fid)
s3_path = asset.get_content_url(follow_redirects=1, strip_query=True)
with NWBHDF5IO(s3_path, mode="r", driver="ros3") as io:
nwb = io.read()
curr_labels = nwb.intervals["epochs"].to_dataframe()
durations = (
curr_labels.loc[:, "stop_time"].values
- curr_labels.loc[:, "start_time"].values
)
# Add up durations of each label
for s, curr_act in enumerate(common_acts):
for i, curr_label in enumerate(curr_labels["labels"].tolist()):
if curr_act in curr_label.split(", "):
vals_all[part_ind, s] += durations[i] / 3600
# Add up total durations of selected labels (avoid double counting)
for i, curr_label in enumerate(curr_labels["labels"].tolist()):
in_lab_grp = False
for sub_lab in curr_label.split(", "):
if sub_lab in common_acts:
in_lab_grp = True
vals_all[part_ind, -1] += durations[i] / 3600 if in_lab_grp else 0
del nwb, io
# Make final table/dataframe
common_acts_col = [val.lstrip("Blocklist (").rstrip(")") for val in common_acts]
df_all = pd.DataFrame(
vals_all.round(1),
index=["P" + str(val + 1).zfill(2) for val in range(n_parts)],
columns=common_acts_col + ["Total"],
)
return df_all
def main():
g_client = GirderCli("http://3.19.164.171")
a_client = DandiAPIClient("https://api.dandiarchive.org/api")
with a_client.session():
g_client.dandi_authenticate()
# gather all dandisets known to girder: hardcoded _id for "drafts" collection
g_dandisets = list(
g_client.listFolder("5e59bb0af19e820ab6ea6c62", "collection"))
for dandiset, girder_id in [(x["name"], x["_id"])
for x in g_dandisets]:
if dandiset != "000026":
continue
print(f"DANDI:{dandiset}", end="\t")
g_meta, g_assets_ = g_client.get_dandiset_and_assets(
girder_id, "folder")
g_assets = list(g_assets_)
# harmonize and get only what we care about ATM - path and size,
# or otherwise we would need to query each asset for metadata
g_assets_h = set(
(a["path"].lstrip("/"), a["size"]) for a in g_assets)
# Yarik trusts nobody. Two identical bugs are less likely!
g_assets_adhoc = set(adhoc_list_girder(girder_id, g_client))
if g_assets_h != g_assets_adhoc:
print("ad-hoc and dandi listing of girder differs!")
import pdb
pdb.set_trace()
a_meta, a_assets_ = a_client.get_dandiset_and_assets(
dandiset, "draft")
a_assets = list(a_assets_)
a_assets_h = set(
(a["path"].lstrip("/"), a["size"]) for a in a_assets)
if a_assets_h != g_assets_h:
print("differs")
import pdb
pdb.set_trace()
else:
print(f"{len(a_assets)} assets the same")
def main(api_url, token, dandiset_path, delete_extant, only_metadata):
client = DandiAPIClient(api_url=api_url, token=token)
with client.session():
for dpath in dandiset_path:
dandiset = APIDandiset(dpath)
if delete_extant:
try:
client.get_dandiset(dandiset.identifier, "draft")
except requests.HTTPError as e:
if e.response.status_code != 404:
raise
else:
print("Dandiset", dandiset.identifier, "already exists; deleting")
client.delete(f"/dandisets/{dandiset.identifier}/")
if only_metadata:
print("Setting metadata for Dandiset", dandiset.identifier)
client.set_dandiset_metadata(
dandiset.identifier, metadata=dandiset.metadata
)
else:
print("Creating Dandiset", dandiset.identifier)
client.create_dandiset(
name=dandiset.metadata.get("name", ""), metadata=dandiset.metadata
)
from dandi.dandiapi import DandiAPIClient
from .spec_utils import project_power, proj_mat_compute
# Set parameters
sp = '' # save path
win_spec_len = 30 # sec
large_win = 30 * 60 # sec
fs = 500 # Hz
freq_range = [3, 125] # Hz
hgrid_fid = "headGrid.mat"
aal_fid = "aal_rois.mat"
n_parts = 12 # number of participants
# Determine all file paths
with DandiAPIClient() as client:
paths = []
for file in client.get_dandiset("000055",
"draft").get_assets_under_path(""):
paths.append(file.path)
paths = natsort.natsorted(paths)
# Create ROI projection matrices
elec_dens_thresh = 3 # threshold for dipole density
proj_mats = []
for s in range(n_parts):
fid = [val for val in paths if "sub-" + str(s + 1).zfill(2) in val][0]
with DandiAPIClient() as client:
asset = client.get_dandiset("000055", "draft").get_asset_by_path(fid)
s3_path = asset.get_content_url(follow_redirects=1, strip_query=True)
def _get_wrist_trajs(
base_start=-1.5, base_end=-1, before=3, after=3, fs_video=30, n_parts=12
):
"""Load in wrist trajectories around move onset events."""
with DandiAPIClient() as client:
paths = []
for file in client.get_dandiset("000055", "draft").get_assets_with_path_prefix(""):
paths.append(file.path)
paths = natsort.natsorted(paths)
displ_lst, part_lst, time_lst, pose_lst = [], [], [], []
for pat in range(n_parts):
fids = [val for val in paths if "sub-" + str(pat + 1).zfill(2) in val]
for i, fid in enumerate(fids):
with DandiAPIClient() as client:
asset = client.get_dandiset("000055", "draft").get_asset_by_path(fid)
s3_path = asset.get_content_url(follow_redirects=1, strip_query=True)
with NWBHDF5IO(
s3_path, mode="r", driver="ros3"
) as io:
nwb_file = io.read()
# Segment data
events = nwb_file.processing["behavior"].data_interfaces["ReachEvents"]
times = events.timestamps[:]
starts = times - before
stops = times + after
# Get event hand label
contra_arm = events.description
contra_arm = map(lambda x: x.capitalize(), contra_arm.split("_"))
contra_arm = list(contra_arm)
contra_arm = "_".join(contra_arm)
ipsi_arm = (
"R" + contra_arm[1:]
if contra_arm[0] == "L"
else "L" + contra_arm[1:]
)
reach_lab = ["contra", "ipsi"]
for k, reach_arm in enumerate([contra_arm, ipsi_arm]):
spatial_series = nwb_file.processing["behavior"].data_interfaces[
"Position"
][reach_arm]
ep_dat = align_by_times(spatial_series, starts, stops)
ep_dat_mag = np.sqrt(
np.square(ep_dat[..., 0]) + np.square(ep_dat[..., 1])
)
# Interpolate and median filter
for j in range(ep_dat_mag.shape[0]):
df_mag = pd.DataFrame(ep_dat_mag[j, :])
df_mag = df_mag.interpolate(method="pad")
tmp_val = (
df_mag.values.copy().flatten()
) # medfilt(df_mag.values, kernel_size=31)
df_mag = pd.DataFrame(tmp_val[::-1])
df_mag = df_mag.interpolate(method="pad")
ep_dat_mag[j, :] = medfilt(
df_mag.values.copy().flatten()[::-1], kernel_size=31
)
zero_ind = timeseries_time_to_ind(spatial_series, before)
base_start_ind = timeseries_time_to_ind(
spatial_series, base_start + before
)
base_end_ind = timeseries_time_to_ind(
spatial_series, base_end + before
)
n_tpoints = ep_dat_mag.shape[1]
t_vals = np.arange(n_tpoints) / fs_video - before
# Subtract baseline from position data
for j in range(ep_dat_mag.shape[0]):
curr_magnitude = ep_dat_mag[j, :]
curr_magnitude = np.abs(
curr_magnitude
- np.mean(curr_magnitude[base_start_ind:base_end_ind])
)
curr_magnitude[np.isnan(curr_magnitude)] = 0
displ_lst.extend(curr_magnitude.tolist())
part_lst.extend(["P" + str(pat + 1).zfill(2)] * n_tpoints)
time_lst.extend(t_vals.tolist())
pose_lst.extend([reach_lab[k]] * n_tpoints)
del nwb_file, io
df_pose = pd.DataFrame(
{"Displ": displ_lst, "Sbj": part_lst, "Time": time_lst, "Contra": pose_lst}
)
return df_pose, part_lst
def plot_ecog_descript(
n_elecs_tot,
n_elecs_good,
part_ids,
nparts=12,
allLH=False,
nrows=3,
chan_labels="all",
width=7,
height=3,
):
"""Plot ECoG electrode positions and identified noisy
electrodes side by side."""
with DandiAPIClient() as client:
paths = []
for file in client.get_dandiset("000055", "draft").get_assets_with_path_prefix(""):
paths.append(file.path)
paths = natsort.natsorted(paths)
fig = plt.figure(figsize=(width * 3, height * 3), dpi=150)
# First subplot: electrode locations
ncols = nparts // nrows
gs = gridspec.GridSpec(
nrows=nrows,
ncols=ncols, # +2,
figure=fig,
width_ratios=[width / ncols]
* ncols, # [width/ncols/2]*ncols+[width/10, 4*width/10],
height_ratios=[height / nrows] * nrows,
wspace=0,
hspace=-0.5,
)
ax = [None] * (nparts) # +1)
for part_ind in tqdm(range(nparts)):
# Load NWB data file
fids = [val for val in paths if "sub-" + str(part_ind + 1).zfill(2) in val]
with DandiAPIClient() as client:
asset = client.get_dandiset("000055", "draft").get_asset_by_path(fids[0])
s3_path = asset.get_content_url(follow_redirects=1, strip_query=True)
with NWBHDF5IO(s3_path, mode="r", driver="ros3") as io:
nwb = io.read()
# Determine hemisphere to display
if allLH:
sides_2_display = "l"
else:
average_xpos_sign = np.nanmean(nwb.electrodes["x"][:])
sides_2_display = "r" if average_xpos_sign > 0 else "l"
# Run electrode plotting function
ax[part_ind] = fig.add_subplot(gs[part_ind // ncols, part_ind % ncols])
plot_ecog_electrodes_mni_from_nwb_file(
nwb,
chan_labels,
num_grid_chans=64,
node_size=50,
colors="silver",
alpha=0.9,
sides_2_display=sides_2_display,
node_edge_colors="k",
edge_linewidths=1.5,
ax_in=ax[part_ind],
allLH=allLH,
)
del nwb, io
# ax[part_ind].text(-0.2,0.1,'P'+str(part_ind+1).zfill(2), fontsize=8)
# fig.text(0.1, 0.91, '(a) ECoG electrode positions', fontsize=10)
# Second subplot: noisy electrodes per participant
# ax[-1] = fig.add_subplot(gs[:, -1])
# ax[-1].bar(part_ids,n_elecs_tot,color='lightgrey')
# ax[-1].bar(part_ids,n_elecs_good,color='dimgrey')
# ax[-1].spines['right'].set_visible(False)
# ax[-1].spines['top'].set_visible(False)
# ax[-1].set_xticklabels(part_ids, rotation=45)
# ax[-1].legend(['Total','Good'], frameon=False, fontsize=8)
# ax[-1].tick_params(labelsize=9)
# ax[-1].set_ylabel('Number of electrodes', fontsize=9, labelpad=0)
# ax[-1].set_title('(b) Total/good electrodes per participant',
# fontsize=10)
plt.show()
return fig
def load_data_characteristics(nparts=12):
"""Load data characteristics including the number of
good and total ECoG electrodes, hemisphere implanted,
and number of recording days for each participant."""
with DandiAPIClient() as client:
paths = []
for file in client.get_dandiset("000055", "draft").get_assets_with_path_prefix(""):
paths.append(file.path)
paths = natsort.natsorted(paths)
n_elecs_tot, n_elecs_good = [], []
rec_days, hemis, n_elecs_surf_tot, n_elecs_depth_tot = [], [], [], []
n_elecs_surf_good, n_elecs_depth_good = [], []
for part_ind in tqdm(range(nparts)):
fids = [val for val in paths if "sub-" + str(part_ind + 1).zfill(2) in val]
rec_days.append(len(fids))
for fid in fids[:1]:
with DandiAPIClient() as client:
asset = client.get_dandiset("000055", "draft").get_asset_by_path(fid)
s3_path = asset.get_content_url(follow_redirects=1, strip_query=True)
with NWBHDF5IO(s3_path, mode="r", driver="ros3") as io:
nwb = io.read()
# Determine good/total electrodes
n_elecs_good.append(np.sum(nwb.electrodes["good"][:]))
n_elecs_tot.append(len(nwb.electrodes["good"][:]))
# Determine implanted hemisphere
c_wrist = (
nwb.processing["behavior"].data_interfaces["ReachEvents"].description[0]
)
hemis.append("L" if c_wrist == "r" else "R")
# Determine surface vs. depth electrode count
is_surf = identify_elecs(nwb.electrodes["group_name"][:])
n_elecs_surf_tot.append(np.sum(is_surf))
n_elecs_depth_tot.append(np.sum(1 - is_surf))
n_elecs_surf_good.append(
np.sum(nwb.electrodes["good"][is_surf.nonzero()[0]])
)
n_elecs_depth_good.append(
np.sum(nwb.electrodes["good"][(1 - is_surf).nonzero()[0]])
)
del nwb, io
part_nums = [val + 1 for val in range(nparts)]
part_ids = ["P" + str(val).zfill(2) for val in part_nums]
return [
rec_days,
hemis,
n_elecs_surf_tot,
n_elecs_surf_good,
n_elecs_depth_tot,
n_elecs_depth_good,
part_nums,
part_ids,
n_elecs_good,
n_elecs_tot,
]
def __init__(self, datasets_path: Path):
self.datasets_path = datasets_path
self.dandi_client = DandiAPIClient("https://api.dandiarchive.org/api")
self.s3client = boto3.client("s3",
config=Config(signature_version=UNSIGNED))