예제 #1
0
def download_all_dlc():

    eids = get_repeated_sites()
    one = ONE()
    dataset_types = ['camera.dlc', 'camera.times']

    for eid in eids:
        try:

            a = one.list(eid, 'dataset-types')
            # for newer iblib version do [x['dataset_type'] for x in a]
            if not all([x['dataset_type'] for x in a]):
                print('not all data available')
                continue

            one.load(eid, dataset_types=dataset_types)
        except:
            continue
예제 #2
0
def get_example_images(eid):

    eids = get_repeated_sites()
    #    eid = eids[23]
    #    video_type = 'body'

    #eids = ['15f742e1-1043-45c9-9504-f1e8a53c1744']
    eids = ['4a45c8ba-db6f-4f11-9403-56e06a33dfa4']
    frts = {'body': 30, 'left': 60, 'right': 150}

    one = ONE()

    #for eid in eids:
    for video_type in frts:

        frame_idx = [20 * 60 * frts[video_type]]
        try:

            r = one.list(eid, 'dataset_types')
            recs = [
                x for x in r if f'{video_type}Camera.raw.mp4' in x['name']
            ][0]['file_records']
            video_path = [
                x['data_url'] for x in recs if x['data_url'] is not None
            ][0]

            frames = get_video_frames_preload(video_path,
                                              frame_idx,
                                              mask=np.s_[:, :, 0])
            np.save(
                '/home/mic/reproducible_dlc/example_images/'
                f'{eid}_{video_type}.npy', frames)
            print(eid, video_type, 'done')
        except:
            print(eid, video_type, 'error')
            continue
예제 #3
0
def get_ME(eid, video_type):

    #video_type = 'left'
    one = ONE()
    dataset_types = ['camera.ROIMotionEnergy', 'camera.times']
    a = one.list(eid, 'dataset-types')
    # for newer iblib version do [x['dataset_type'] for x in a]
    #    if not all([(u in [x['dataset_type'] for x in a]) for u in dataset_types]):
    #        print('not all data available')
    #        return

    one.load(eid, dataset_types=dataset_types)
    local_path = one.path_from_eid(eid)
    alf_path = local_path / 'alf'

    cam0 = alf.io.load_object(alf_path,
                              '%sCamera' % video_type,
                              namespace='ibl')

    ME = np.load(alf_path / f'{video_type}Camera.ROIMotionEnergy.npy')

    Times = cam0['times']

    return Times, ME
예제 #4
0
def get_dlc_XYs(eid, video_type):

    #video_type = 'left'
    one = ONE()
    dataset_types = ['camera.dlc', 'camera.times']
    a = one.list(eid, 'dataset-types')
    # for newer iblib version do [x['dataset_type'] for x in a]
    #    if not all([(u in [x['dataset_type'] for x in a]) for u in dataset_types]):
    #        print('not all data available')
    #        return

    one.load(eid, dataset_types=dataset_types)  #clobber=True # force download
    local_path = one.path_from_eid(eid)
    alf_path = local_path / 'alf'

    cam0 = alf.io.load_object(alf_path,
                              '%sCamera' % video_type,
                              namespace='ibl')

    Times = cam0['times']

    cam = cam0['dlc']
    points = np.unique(['_'.join(x.split('_')[:-1]) for x in cam.keys()])

    # Set values to nan if likelyhood is too low # for pqt: .to_numpy()
    XYs = {}
    for point in points:
        x = np.ma.masked_where(cam[point + '_likelihood'] < 0.9,
                               cam[point + '_x'])
        x = x.filled(np.nan)
        y = np.ma.masked_where(cam[point + '_likelihood'] < 0.9,
                               cam[point + '_y'])
        y = y.filled(np.nan)
        XYs[point] = np.array([x, y])

    return Times, XYs
예제 #5
0
    try:
        _ = alf.io.load_object(session_path.joinpath('alf'), 'probes')
    except FileNotFoundError:
        print(session_path.joinpath('alf'))
        print("no probes")
        continue

    probe_path = session_path.joinpath('alf', probe)
    try:
        metrics = alf.io.load_object(probe_path, object='clusters.metrics')
    except FileNotFoundError:
        print(probe_path)
        print("one probe missing")
        continue

    labs.append(one.list(eid, 'labs'))
    metric_list.append(metrics.metrics)
    ss.append(s)


def split(d, l):
    dizzle = {}
    for data, label, in zip(d, l):
        if label not in dizzle:
            dizzle[label] = [data]
        else:
            dizzle[label].append(data)
    return list(dizzle.values())

for i, (metric, metric_name) in enumerate(metric_funcs):
예제 #6
0

from oneibl.one import ONE
from plot import gen_figures

# if you could also try just adding 'ibllib - brainbox', 'iblscripts - certification', and 'analysis - cert_master_fn' repositories (on those branches) to your python path
import sys
sys.path.append('~/Documents/code/ibllib')
sys.path.append('~/Documents/code/iblscripts')

one = ONE()
eid = one.search(subject='ZM_2104', date='2019-09-19', number=1)[0]
one.load(eid, dataset_types=one.list(), clobber=False, download_only=True)
gen_figures(eid, probe='probe_right', cluster_ids_summary=1)
예제 #7
0
    "probe00", "probe00", "probe00", "probe00", "probe01", "probe01",
    "probe00", "probe01", "probe00", "probe01", "probe00", "probe01",
    "probe00", "probe00", "probe00", "probe00", "probe00", "probe00", "probe00"
]
one = ONE()

assert len(eids) == len(probes)
metrics = {}
for _, metric_name in metric_funcs:
    metrics[metric_name] = []

for i, (eid, probe) in enumerate(zip(eids, probes)):
    print(eid)
    if eid in bad_eids: continue
    print("{} from {}".format(i, len(eids)))
    print(one.list(eid, 'subjects'))
    coords = one.load(eid, dataset_types=['probes.trajectory'])
    for c in coords[0]:
        if c['label'] == probe:
            print("{}, x: {}, y: {}, z: {}".format(c['label'], c['x'], c['y'],
                                                   c['z']))

    continue
    spikes, _ = load_spike_sorting(eid, one=one)
    spikes = spikes[0]

    if spikes[probe]['times'] is None:
        print('empty times skip')
        continue

    fr = calc_fr(spikes[probe]['times'], spikes[probe]['clusters'])
import matplotlib.pyplot as plt
import ibllib.io.extractors

from oneibl.one import ONE
import alf.io
import ibllib.plots as iblplots

one = ONE()
eid = one.search(subject='KS005', date_range='2019-08-30', number=1)[0]
# eid = one.search(subject='CSHL_020', date_range='2019-12-04', number=5)[0]

one.alyx.rest('sessions', 'read', id=eid)['task_protocol']

one.list(eid)
dtypes = [
    '_spikeglx_sync.channels',
    '_spikeglx_sync.polarities',
    '_spikeglx_sync.times',
    '_iblrig_taskSettings.raw',
    '_iblrig_taskData.raw',
    '_iblrig_encoderEvents.raw',
    '_iblrig_encoderPositions.raw',
    '_iblrig_encoderTrialInfo.raw',
]

files = one.load(eid, dataset_types=dtypes, download_only=True)
sess_path = alf.io.get_session_path(files[0])
# TODO here we will have to deal with 3A versions by looking up the master probe
chmap = ibllib.io.extractors.ephys_fpga.CHMAPS['3B']['nidq']
# chmap = ibllib.io.extractors.ephys_fpga.CHMAPS['3A']['ap']
"""get the sync pulses"""
예제 #9
0
need_load = bool(0)
probe = 'probe_00'  # must set probe name

if need_load:
    # This is only useful if you do not have the data accessible on the machine

    # -- GET EID FOR SPECIFIC SESSION
    # Check that each dataset type needed is present
    dataset_types = ['ephysData.raw.ap', 'spikes.times', 'clusters.depths']
    eid, sinfo = one.search(datasets=dataset_types,
                            subjects='ZM_2407',
                            date_range='2019-11-05',
                            details=True)
    assert (len(eid) > 0), 'No EID found with those search terms'
    dtypes_session = one.list(eid)[0]
    if not set(dataset_types).issubset(set(dtypes_session)):
        missing_dtypes = [
            dt for dt in dataset_types if dt not in dtypes_session
        ]
        raise ValueError('Missing datasets: ' + ','.join(missing_dtypes))

    # In case the above did not run,
    # you can find all sessions containing raw data by looking into sinfos:
    # eids, sinfos = one.search(
    #    datasets=['ephysData.raw.ap', 'spikes.times'], task_protocol='certification', details=True)

    # Set important directories from `eid`
    spikes_path = one.load(eid,
                           dataset_types='spikes.amps',
                           clobber=False,
def stream_save_labeled_frames(eid, video_type):

    startTime = time.time()
    '''
    For a given eid and camera type, stream
    sample frames, print DLC labels on them
    and save
    '''

    # eid = '5522ac4b-0e41-4c53-836a-aaa17e82b9eb'
    # video_type = 'left'

    n_frames = 5  # sample 5 random frames

    save_images_folder = '/home/mic/DLC_QC/example_frames/'
    one = ONE()
    info = '_'.join(
        np.array(str(one.path_from_eid(eid)).split('/'))[[5, 7, 8]])
    print(info, video_type)

    r = one.list(eid, 'dataset_types')

    dtypes_DLC = [
        '_ibl_rightCamera.times.npy', '_ibl_leftCamera.times.npy',
        '_ibl_bodyCamera.times.npy', '_iblrig_leftCamera.raw.mp4',
        '_iblrig_rightCamera.raw.mp4', '_iblrig_bodyCamera.raw.mp4',
        '_ibl_leftCamera.dlc.pqt', '_ibl_rightCamera.dlc.pqt',
        '_ibl_bodyCamera.dlc.pqt'
    ]

    dtype_names = [x['name'] for x in r]

    assert all([i in dtype_names
                for i in dtypes_DLC]), 'For this eid, not all data available'

    D = one.load(eid,
                 dataset_types=['camera.times', 'camera.dlc'],
                 dclass_output=True)
    alf_path = Path(D.local_path[0]).parent.parent / 'alf'

    cam0 = alf.io.load_object(alf_path,
                              '%sCamera' % video_type,
                              namespace='ibl')

    Times = cam0['times']

    cam = cam0['dlc']
    points = np.unique(['_'.join(x.split('_')[:-1]) for x in cam.keys()])

    XYs = {}
    for point in points:
        x = np.ma.masked_where(cam[point + '_likelihood'] < 0.9,
                               cam[point + '_x'])
        x = x.filled(np.nan)
        y = np.ma.masked_where(cam[point + '_likelihood'] < 0.9,
                               cam[point + '_y'])
        y = y.filled(np.nan)
        XYs[point] = np.array([x, y])

    if video_type != 'body':
        d = list(points)
        d.remove('tube_top')
        d.remove('tube_bottom')
        points = np.array(d)

    # stream frames
    recs = [x for x in r
            if f'{video_type}Camera.raw.mp4' in x['name']][0]['file_records']
    video_path = [x['data_url'] for x in recs if x['data_url'] is not None][0]
    vid_meta = get_video_meta(video_path)

    frame_idx = sample(range(vid_meta['length']), n_frames)
    print('frame indices:', frame_idx)
    frames = get_video_frames_preload(video_path,
                                      frame_idx,
                                      mask=np.s_[:, :, 0])
    size = [vid_meta['width'], vid_meta['height']]
    #return XYs, frames

    x0 = 0
    x1 = size[0]
    y0 = 0
    y1 = size[1]
    if video_type == 'left':
        dot_s = 10  # [px] for painting DLC dots
    else:
        dot_s = 5

    # writing stuff on frames
    font = cv2.FONT_HERSHEY_SIMPLEX

    if video_type == 'left':
        bottomLeftCornerOfText = (20, 1000)
        fontScale = 4
    else:
        bottomLeftCornerOfText = (10, 500)
        fontScale = 2

    lineType = 2

    # assign a color to each DLC point (now: all points red)
    cmap = matplotlib.cm.get_cmap('Set1')
    CR = np.arange(len(points)) / len(points)

    block = np.ones((2 * dot_s, 2 * dot_s, 3))

    k = 0
    for frame in frames:

        gray = cv2.cvtColor(frame, cv2.COLOR_GRAY2RGB)

        # print session info
        fontColor = (255, 255, 255)
        cv2.putText(gray, info, bottomLeftCornerOfText, font, fontScale / 4,
                    fontColor, lineType)

        # print time
        Time = round(Times[frame_idx[k]], 3)
        a, b = bottomLeftCornerOfText
        bottomLeftCornerOfText0 = (int(a * 10 + b / 2), b)

        a, b = bottomLeftCornerOfText
        bottomLeftCornerOfText0 = (int(a * 10 + b / 2), b)
        cv2.putText(gray, '  time: ' + str(Time), bottomLeftCornerOfText0,
                    font, fontScale / 2, fontColor, lineType)

        # print DLC dots
        ll = 0
        for point in points:

            # Put point color legend
            fontColor = (np.array([cmap(CR[ll])]) * 255)[0][:3]
            a, b = bottomLeftCornerOfText
            if video_type == 'right':
                bottomLeftCornerOfText2 = (a, a * 2 * (1 + ll))
            else:
                bottomLeftCornerOfText2 = (b, a * 2 * (1 + ll))
            fontScale2 = fontScale / 4
            cv2.putText(gray, point, bottomLeftCornerOfText2, font, fontScale2,
                        fontColor, lineType)

            X0 = XYs[point][0][frame_idx[k]]
            Y0 = XYs[point][1][frame_idx[k]]

            X = Y0
            Y = X0

            #print(point,X,Y)
            if not np.isnan(X) and not np.isnan(Y):
                try:
                    col = (np.array([cmap(CR[ll])]) * 255)[0][:3]
                    # col = np.array([0, 0, 255]) # all points red
                    X = X.astype(int)
                    Y = Y.astype(int)

                    uu = block * col
                    gray[X - dot_s:X + dot_s, Y - dot_s:Y + dot_s] = uu

                except Exception as e:
                    print('frame', frame_idx[k])
                    print(e)
            ll += 1

        gray = gray[y0:y1, x0:x1]
        # cv2.imshow('frame', gray)
        cv2.imwrite(f'{save_images_folder}{eid}_frame_{frame_idx[k]}.png',
                    gray)
        cv2.waitKey(1)
        k += 1

    print(f'{n_frames} frames done in', np.round(time.time() - startTime))
예제 #11
0
## Init
from oneibl.one import ONE
one = ONE()  # need to instantiate the class to have the API.

## Info about a session
eid = '86e27228-8708-48d8-96ed-9aa61ab951db'
list_of_datasets = one.list(eid)

## More Info about a session
d = one.session_data_info(eid)
print(d)

## Load #1
dataset_types = [
    'clusters.templateWaveforms', 'clusters.probes', 'clusters.depths'
]
eid = '86e27228-8708-48d8-96ed-9aa61ab951db'
wf, pr, d = one.load(eid, dataset_types=dataset_types)

## Load #2
my_data = one.load(eid, dataset_types=dataset_types, dclass_output=True)
from ibllib.misc import pprint
pprint(my_data.local_path)
pprint(my_data.dataset_type)

## Load everything
eid, ses_info = one.search(subject='flowers')
my_data = one.load(eid[0])
pprint(my_data.dataset_type)

## Load
예제 #12
0
import matplotlib.pyplot as plt
import pandas as pd

from oneibl.one import ONE
from ibllib.time import isostr2date

# import sys
# sys.path.extend('/home/owinter/PycharmProjects/WGs/BehaviourAnaysis/python')
from load_mouse_data import get_behavior
from behavior_plots import plot_psychometric

one = ONE()
# https://alyx.internationalbrainlab.org/admin/actions/session/e752b02d-b54d-4373-b51e-0b31be5f8ee5/change/
# first get the subject information
subject_details = one.alyx.rest('subjects', 'read', 'IBL_14')

# plot the weight curve
# https://alyx.internationalbrainlab.org/admin-actions/water-history/37c8f897-cbcc-4743-bad6-764ccbbfb190
wei = pd.DataFrame(subject_details['weighings'])
wei['date_time'].apply(isostr2date)
wei.sort_values('date_time', inplace=True)
plt.plot(wei.date_time, wei.weight)

# now let's get some session information
ses_ids = one.search(subjects='IBL_14', date_range='2018-11-27')
print(one.list(ses_ids[0]))
df = get_behavior('IBL_14', date_range='2018-11-27')
plt.figure()
plot_psychometric(df, ax=plt.axes(), color="orange")
예제 #13
0
def Viewer(eid, video_type, trial_range, save_video=True, eye_zoom=False):
    '''
    eid: session id, e.g. '3663d82b-f197-4e8b-b299-7b803a155b84'
    video_type: one of 'left', 'right', 'body'
    trial_range: first and last trial number of range to be shown, e.g. [5,7]
    save_video: video is displayed and saved in local folder

    Example usage to view and save labeled video with wheel angle:
    Viewer('3663d82b-f197-4e8b-b299-7b803a155b84', 'left', [5,7])
    3D example: 'cb2ad999-a6cb-42ff-bf71-1774c57e5308', [5,7]
    '''

    save_vids_here = '/home/mic/'
    if save_vids_here[-1] != '/':
        return 'Last character of save_vids_here must be slash'

    one = ONE()
    dataset_types = [
        'camera.times', 'wheel.position', 'wheel.timestamps',
        'trials.intervals', 'camera.dlc'
    ]

    a = one.list(eid, 'dataset-types')

    assert all([i in a for i in dataset_types
                ]), 'For this eid, not all data available'

    D = one.load(eid, dataset_types=dataset_types, dclass_output=True)
    alf_path = Path(D.local_path[0]).parent.parent / 'alf'

    # Download a single video
    video_data = alf_path.parent / 'raw_video_data'
    download_raw_video(eid, cameras=[video_type])
    video_path = list(video_data.rglob('_iblrig_%sCamera.raw.*' %
                                       video_type))[0]
    print(video_path)

    # that gives cam time stamps and DLC output (change to alf_path eventually)

    cam = alf.io.load_object(alf_path,
                             '%sCamera' % video_type,
                             namespace='ibl')

    # just to read in times for newer data (which has DLC results in pqt format
    # cam = alf.io.load_object(alf_path, '_ibl_%sCamera' % video_type)

    # set where to read and save video and get video info
    cap = cv2.VideoCapture(video_path.as_uri())
    length = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
    fps = cap.get(cv2.CAP_PROP_FPS)
    size = (int(cap.get(3)), int(cap.get(4)))

    assert length < len(cam['times']), '#frames > #stamps'
    print(eid, ', ', video_type, ', fsp:', fps, ', #frames:', length,
          ', #stamps:', len(cam['times']), ', #frames - #stamps = ',
          length - len(cam['times']))

    # pick trial range for which to display stuff
    trials = alf.io.load_object(alf_path, 'trials', namespace='ibl')
    num_trials = len(trials['intervals'])
    if trial_range[-1] > num_trials - 1:
        print('There are only %s trials' % num_trials)

    frame_start = find_nearest(cam['times'],
                               [trials['intervals'][trial_range[0]][0]])
    frame_stop = find_nearest(cam['times'],
                              [trials['intervals'][trial_range[-1]][1]])
    '''
    wheel related stuff
    '''

    wheel = alf.io.load_object(alf_path, 'wheel', namespace='ibl')
    import brainbox.behavior.wheel as wh
    try:
        pos, t = wh.interpolate_position(wheel['timestamps'],
                                         wheel['position'],
                                         freq=1000)
    except BaseException:
        pos, t = wh.interpolate_position(wheel['times'],
                                         wheel['position'],
                                         freq=1000)

    w_start = find_nearest(t, trials['intervals'][trial_range[0]][0])
    w_stop = find_nearest(t, trials['intervals'][trial_range[-1]][1])

    # confine to interval
    pos_int = pos[w_start:w_stop]
    t_int = t[w_start:w_stop]

    # alignment of cam stamps and interpolated wheel stamps
    wheel_pos = []
    kk = 0
    for wt in cam['times'][frame_start:frame_stop]:
        wheel_pos.append(pos_int[find_nearest(t_int, wt)])
        kk += 1
        if kk % 3000 == 0:
            print('iteration', kk)
    '''
    DLC related stuff
    '''
    Times = cam['times'][frame_start:frame_stop]
    del cam['times']

    # some exception for inconsisitent data formats
    try:
        dlc_name = '_ibl_%sCamera.dlc.pqt' % video_type
        dlc_path = alf_path / dlc_name
        cam = pd.read_parquet(dlc_path, engine="fastparquet")
        print('it is pqt')
    except BaseException:
        raw_vid_path = alf_path.parent / 'raw_video_data'
        cam = alf.io.load_object(raw_vid_path,
                                 '%sCamera' % video_type,
                                 namespace='ibl')

    points = np.unique(['_'.join(x.split('_')[:-1]) for x in cam.keys()])
    if len(points) == 1:
        cam = cam['dlc']
        points = np.unique(['_'.join(x.split('_')[:-1]) for x in cam.keys()])

    if video_type != 'body':
        d = list(points)
        d.remove('tube_top')
        d.remove('tube_bottom')
        points = np.array(d)

    # Set values to nan if likelyhood is too low # for pqt: .to_numpy()
    XYs = {}
    for point in points:
        x = np.ma.masked_where(cam[point + '_likelihood'] < 0.9,
                               cam[point + '_x'])
        x = x.filled(np.nan)
        y = np.ma.masked_where(cam[point + '_likelihood'] < 0.9,
                               cam[point + '_y'])
        y = y.filled(np.nan)
        XYs[point] = np.array(
            [x[frame_start:frame_stop], y[frame_start:frame_stop]])

    # Just for 3D testing
    # return XYs

    # Zoom at eye
    if eye_zoom:
        pivot = np.nanmean(XYs['pupil_top_r'], axis=1)
        x0 = int(pivot[0]) - 33
        x1 = int(pivot[0]) + 33
        y0 = int(pivot[1]) - 28
        y1 = int(pivot[1]) + 38
        size = (66, 66)
        dot_s = 1  # [px] for painting DLC dots

    else:
        x0 = 0
        x1 = size[0]
        y0 = 0
        y1 = size[1]
        if video_type == 'left':
            dot_s = 10  # [px] for painting DLC dots
        else:
            dot_s = 5

    if save_video:
        loc = save_vids_here + '%s_trials_%s_%s_%s.mp4' % (
            eid, trial_range[0], trial_range[-1], video_type)
        out = cv2.VideoWriter(loc, cv2.VideoWriter_fourcc(*'mp4v'), fps,
                              size)  # put , 0 if grey scale

    # writing stuff on frames
    font = cv2.FONT_HERSHEY_SIMPLEX

    if video_type == 'left':
        bottomLeftCornerOfText = (20, 1000)
        fontScale = 4
    else:
        bottomLeftCornerOfText = (10, 500)
        fontScale = 2

    lineType = 2

    # assign a color to each DLC point (now: all points red)
    cmap = matplotlib.cm.get_cmap('Spectral')
    CR = np.arange(len(points)) / len(points)

    block = np.ones((2 * dot_s, 2 * dot_s, 3))

    # set start frame
    cap.set(1, frame_start)

    k = 0
    while (cap.isOpened()):
        ret, frame = cap.read()
        gray = frame

        # print wheel angle
        fontColor = (255, 255, 255)
        Angle = round(wheel_pos[k], 2)
        Time = round(Times[k], 3)
        cv2.putText(gray, 'Wheel angle: ' + str(Angle), bottomLeftCornerOfText,
                    font, fontScale / 2, fontColor, lineType)

        a, b = bottomLeftCornerOfText
        bottomLeftCornerOfText0 = (int(a * 10 + b / 2), b)
        cv2.putText(gray, '  time: ' + str(Time), bottomLeftCornerOfText0,
                    font, fontScale / 2, fontColor, lineType)

        # print DLC dots
        ll = 0
        for point in points:

            # Put point color legend
            fontColor = (np.array([cmap(CR[ll])]) * 255)[0][:3]
            a, b = bottomLeftCornerOfText
            if video_type == 'right':
                bottomLeftCornerOfText2 = (a, a * 2 * (1 + ll))
            else:
                bottomLeftCornerOfText2 = (b, a * 2 * (1 + ll))
            fontScale2 = fontScale / 4
            cv2.putText(gray, point, bottomLeftCornerOfText2, font, fontScale2,
                        fontColor, lineType)

            X0 = XYs[point][0][k]
            Y0 = XYs[point][1][k]
            # transform for opencv?
            X = Y0
            Y = X0

            if not np.isnan(X) and not np.isnan(Y):
                col = (np.array([cmap(CR[ll])]) * 255)[0][:3]
                # col = np.array([0, 0, 255]) # all points red
                X = X.astype(int)
                Y = Y.astype(int)
                gray[X - dot_s:X + dot_s, Y - dot_s:Y + dot_s] = block * col
            ll += 1

        gray = gray[y0:y1, x0:x1]
        if save_video:
            out.write(gray)
        cv2.imshow('frame', gray)
        cv2.waitKey(1)
        k += 1
        if k == (frame_stop - frame_start) - 1:
            break

    if save_video:
        out.release()
    cap.release()
    cv2.destroyAllWindows()
예제 #14
0
def GetXYs(eid, video_type, trial_range):
    '''
    INPUT:
        eid: session id, e.g. '3663d82b-f197-4e8b-b299-7b803a155b84'
        video_type: one of 'left', 'right', 'body'
        trial_range: first and last trial number
            of range to be accessed, e.g. [5,7]
    OUTPUT:
        XYs: dictionary with DLC-tracked points as keys,
            x,y coordinates as entries, set to nan for low
            likelihood
        Times: corresponding timestamps
    '''

    one = ONE()
    dataset_types = ['camera.times', 'trials.intervals', 'camera.dlc']

    a = one.list(eid, 'dataset-types')

    assert all([i in a for i in dataset_types
                ]), 'For this eid, not all data available'

    D = one.load(eid, dataset_types=dataset_types, dclass_output=True)
    alf_path = Path(D.local_path[0]).parent.parent / 'alf'
    cam = alf.io.load_object(alf_path,
                             '%sCamera' % video_type,
                             namespace='ibl')

    # pick trial range
    trials = alf.io.load_object(alf_path, 'trials', namespace='ibl')
    num_trials = len(trials['intervals'])
    if trial_range[-1] > num_trials - 1:
        print('There are only %s trials' % num_trials)

    frame_start = find_nearest(cam['times'],
                               [trials['intervals'][trial_range[0]][0]])
    frame_stop = find_nearest(cam['times'],
                              [trials['intervals'][trial_range[-1]][1]])

    last_time_stamp = trials['intervals'][-1][-1]
    last_stamp_idx = find_nearest(cam['times'], last_time_stamp)

    print('Last trial ends at time %s, which is stamp index %s' %
          (last_time_stamp, last_stamp_idx))

    Times = cam['times'][frame_start:frame_stop]
    n_stamps = len(cam['times'])  #

    cam = cam['dlc']
    points = np.unique(['_'.join(x.split('_')[:-1]) for x in cam.keys()])

    if video_type != 'body':
        d = list(points)
        d.remove('tube_top')
        d.remove('tube_bottom')
        points = np.array(d)

    # Set values to nan if likelyhood is too low # for pqt: .to_numpy()
    XYs = {}
    for point in points:
        print(point, len(cam[point + '_x']), n_stamps)
        assert len(cam[point + '_x']) <= n_stamps, 'n_stamps > dlc'
        x = np.ma.masked_where(cam[point + '_likelihood'] < 0.9,
                               cam[point + '_x'])
        x = x.filled(np.nan)
        y = np.ma.masked_where(cam[point + '_likelihood'] < 0.9,
                               cam[point + '_y'])
        y = y.filled(np.nan)
        XYs[point] = np.array(
            [x[frame_start:frame_stop], y[frame_start:frame_stop]])

    return XYs, Times
예제 #15
0
## Init
from oneibl.one import ONE
from ibllib.misc import pprint
one = ONE(base_url='https://test.alyx.internationalbrainlab.org',
          username='******',
          password='******')

## Find an experiment
eid = one.search(users='olivier', date_range=['2018-08-24', '2018-08-24'])
pprint(eid)
one.search_terms()

## List dataset types for a session
eid = 'cf264653-2deb-44cb-aa84-89b82507028a'
one.list(eid)
## More Info about a session
d = one.list(eid, 'All')

## Get More Info about datasets
d = one.list(eid, details=True)
print(d)

print(d)
## List #1
one.list(None, 'dataset-types')
one.list(None, 'users')
one.list(None, 'subjects')

## Load #1
dataset_types = [
    'clusters.templateWaveforms', 'clusters.probes', 'clusters.depths'
예제 #16
0
class Alyx2NWBMetadata:
    def __init__(self, eid=None, one_obj=None, **one_search_kwargs):
        """
        Query the sessions, subject, lab tables of the Alyx database using the ONE api.
        Retrieve a mice experiment's metadata as well as the various data types (ONE format)
        created during the experiment like: Trials, Behavior, Electrophysiology(raw, spike sorted),
        Stimulus, Probes used.
        Parameters
        ----------
        eid: str
            uuid of IBL experiment
        one_obj: ONE()
            one object created after user authenticated connection to ALyx servers
        one_search_kwargs: dict
            various search terms to retrieve an eid of interest using the ONE api to query Alyx.
        """
        if one_obj is None:
            self.one_obj = ONE()
        elif not isinstance(one_obj, OneAbstract):
            raise Exception('one_obj is not of ONE class')
        else:
            self.one_obj = one_obj
        if eid is None:
            eid = self.one_obj.search(**one_search_kwargs)
            if len(eid) > 1:
                print(f'nos of EIDs found for your search query: {len(eid)}, '
                      f'generating metadata from the first')
                if input('continue? y/n') == 'y':
                    pass
                else:
                    exit()
            self.eid = eid[0]
        elif isinstance(eid, list):
            self.eid = eid[0]
        else:
            self.eid = eid
        self.one_search_kwargs = one_search_kwargs
        self.schema = nwb_schema
        self.dataset_description_list = self._get_dataset_details()
        self.eid_session_info = self._retrieve_eid_endpoint()
        self.dataset_type_list = self._list_eid_metadata('dataset_type')
        self.users_list = self._list_eid_metadata('users')
        self.subjects_list = self._list_eid_metadata('subjects')
        self.labs_list = self._list_eid_metadata('labs')
        self.dataset_details, self.dataset_simple = self._dataset_type_parse()
        self._get_lab_table()
        self._get_subject_table()

    def _get_datetime(self, dtstr, format='%Y-%m-%dT%X'):
        if '.' in dtstr:
            dtstr = dtstr.split('.')[0]
        if len(dtstr) > 19:
            dtstr = dtstr[:19]
        if len(dtstr) == 10:
            format = '%Y-%m-%d'
        elif len(dtstr) == 19:
            if 'T' in dtstr:
                format = '%Y-%m-%dT%X'
            else:
                format = '%Y-%m-%d %X'
        try:
            return datetime.strptime(dtstr, format)
        except:
            raise Exception('could not convert to datetime')

    def _get_dataset_details(self):
        """
        Retrieves all datasets in the alyx database currently.
        Retrieves a list of dicts with keys like id, name, created_by,description etc. Uses only name and description.
        Returns
        -------
        list
            List of dicts:
            {<dataset-name> : <dataset_description>
        """
        data_url_resp = self.one_obj.alyx.rest('dataset-types', 'list')
        return {i['name']: i['description'] for i in data_url_resp}

    def _list_eid_metadata(self, list_type):
        """
        Uses one's list method to get the types of <list_type> data from the given eid.
        Parameters
        ----------
        list_type: str
            one of strings from
            >>> ONE().search_terms()
        Returns
        -------
        list
        """
        return self.one_obj.list(self.eid, list_type)

    def _retrieve_eid_endpoint(self):
        """
        To get the current sessions url response. Contains all the session metadata as well as the current datasets etc.
        Returns
        -------
        list
            list of server responses.
        """
        return self.one_obj.alyx.rest('sessions/' + self.eid, 'list')

    def _get_lab_table(self):
        self.lab_table = self.one_obj.alyx.rest('labs', 'list')

    def _get_subject_table(self):
        self.subject_table = self.one_obj.alyx.rest(
            'subjects/' + self.eid_session_info['subject'], 'list')

    def _dataset_type_parse(self):
        """

        Returns
        -------
        list
            list of dicts:
            {<dataset object name>: (eg. spikes, clusters etc)
                [
                    {name: objects attribute type (eg. times, intervals etc
                    description: attributes description}
                    {name: objects attribute type (eg. times, intervals etc
                    description: attributes description}
                ]
            }
        """
        split_list_objects = [i.split('.')[0] for i in self.dataset_type_list]
        split_list_attributes = [
            '.'.join(i.split('.')[1:]) for i in self.dataset_type_list
        ]
        dataset_description = [
            self.dataset_description_list[i] for i in self.dataset_type_list
        ]
        split_list_objects_dict_details = dict()
        split_list_objects_dict = dict()
        for obj in set(split_list_objects):
            split_list_objects_dict_details[obj] = []
            split_list_objects_dict[obj] = []
        for att_idx, attrs in enumerate(split_list_attributes):
            append_dict = {
                'name': attrs,
                'description': dataset_description[att_idx]
            }
            # 'extension': dataset_extension[att_idx] }
            split_list_objects_dict_details[
                split_list_objects[att_idx]].extend([append_dict])
            split_list_objects_dict[split_list_objects[att_idx]].extend(
                [attrs])
        dataset_type_list = split_list_objects_dict_details
        dataset_type_list_simple = split_list_objects_dict
        return dataset_type_list, dataset_type_list_simple

    @staticmethod
    def _unpack_dataset_details(dataset_details,
                                object_name,
                                custom_attrs=None,
                                match_str=' '):
        """
        Unpacks the dataset_details into:
        Parameters
        ----------
        dataset_details: dict
            self.dataset_details
        object_name: str
            eg: spikes, clusters, Ecephys
        custom_attrs: list
            attrs to unpack
        match_str: regex
            match string: attrs to exclude (like .times/.intervals etc)
        Returns
        -------
        datafiles: str
            ex: 'face.motionEnergy'
        datanames: str
            ex: 'motionEnergy'
        datadesc: str
            ex: <description string for motionEnergy>
        """
        cond = lambda x: re.match(match_str, x)
        datafiles_all = [
            object_name + '.' + ii['name']
            for ii in dataset_details[object_name] if not cond(ii['name'])
        ]
        datafiles_names_all = [
            ii['name'] for ii in dataset_details[object_name]
            if not cond(ii['name'])
        ]
        datafiles_desc_all = [
            ii['description'] for ii in dataset_details[object_name]
            if not cond(ii['name'])
        ]
        if custom_attrs:
            datafiles_inc = []
            datafiles_names_inc = []
            datafiles_desc_inc = []
            for attrs in custom_attrs:
                datafiles_inc.extend([
                    i for i in datafiles_all if i in object_name + '.' + attrs
                ])
                datafiles_names_inc.extend([
                    datafiles_names_all[j] for j, i in enumerate(datafiles_all)
                    if i in object_name + '.' + attrs
                ])
                datafiles_desc_inc.extend([
                    datafiles_desc_all[j] for j, i in enumerate(datafiles_all)
                    if i in object_name + '.' + attrs
                ])
        else:
            datafiles_inc = datafiles_all
            datafiles_names_inc = datafiles_names_all
            datafiles_desc_inc = datafiles_desc_all
        return datafiles_inc, datafiles_names_inc, datafiles_desc_inc

    def _initialize_container_dict(self, name=None, default_value=None):
        if default_value is None:
            default_value = dict()
        if name:
            return dict({name: default_value.copy()})
        else:
            return None

    def _get_all_object_names(self):
        return sorted(
            list(set([i.split('.')[0] for i in self.dataset_type_list])))

    def _get_current_object_names(self, obj_list):
        loop_list = []
        for j, k in enumerate(obj_list):
            loop_list.extend(
                [i for i in self._get_all_object_names() if k == i])
        return loop_list

    def _get_timeseries_object(self,
                               dataset_details,
                               object_name,
                               ts_name,
                               custom_attrs=None,
                               drop_attrs=None,
                               **kwargs):
        """

        Parameters
        ----------
        dataset_details: dict
            self.dataset_details
        object_name: str
            name of hte object_name in the IBL datatype
        ts_name: str
            the key name for the timeseries list
        custom_attrs: list
            Attributes to consider
        drop_attrs: list
            Attributes to drop
        kwargs
            additional keys/values to add to the default timeseries. For derivatives of TimeSEries

        Returns
        -------
        dict()
            {
                "time_series": [
                    {
                      "name": "face_motionEnergy",
                      "data": "face.motionEnergy",
                      "timestamps": "face.timestamps",
                      "description": "Features extracted from the video of the frontal aspect of the subject, including the subject\\'s face and forearms."
                    },
                    {
                      "name": "_ibl_lickPiezo_times",
                      "data": "_ibl_lickPiezo.raw",
                      "timestamps": "_ibl_lickPiezo.timestamps",
                      "description": "Voltage values from a thin-film piezo connected to the lick spout, so that values are proportional to deflection of the spout and licks can be detected as peaks of the signal."
                    }
                ]
            }
        """
        matchstr = r'.*time.*|.*interval.*'
        timeattr_name = [
            i['name'] for i in dataset_details[object_name]
            if re.match(matchstr, i['name'])
        ]
        dataset_details[object_name], _ = self._drop_attrs(
            dataset_details[object_name].copy(), drop_attrs)
        datafiles, datafiles_names, datafiles_desc = \
            self._unpack_dataset_details(dataset_details.copy(), object_name, custom_attrs, match_str=matchstr)
        if timeattr_name:
            datafiles_timedata, datafiles_time_name, datafiles_time_desc = \
                self._unpack_dataset_details(dataset_details.copy(), object_name, timeattr_name)
        elif not kwargs:  # if no timestamps info, then let this fields be data
            return {ts_name: []}
        else:
            datafiles_timedata, datafiles_time_name, datafiles_time_desc = \
                datafiles, datafiles_names, datafiles_desc
        if not datafiles:
            if not kwargs:
                return {ts_name: []}
            # datafiles_names = datafiles_time_name
            # datafiles_desc = datafiles_time_desc
            # datafiles = ['None']
        timeseries_dict = {ts_name: [None] * len(datafiles)}
        for i, j in enumerate(datafiles):
            original = {
                'name': datafiles_names[i],
                'description': datafiles_desc[i],
                'timestamps': datafiles_timedata[0],
                'data': datafiles[i]
            }
            original.update(**kwargs)
            timeseries_dict[ts_name][i] = {
                k: v
                for k, v in original.items() if v is not None
            }
        return timeseries_dict

    @staticmethod
    def _attrnames_align(attrs_dict, custom_names):
        """
        the attributes that receive the custom names are reordered to be first in the list
        Parameters. This assigns description:'no_description' to those that are not found. This will
        later be used(nwb_converter) as an identifier for non-existent data for the given eid.
        ----------
        attrs_dict:list
            list of dict(attr_name:'',attr_description:'')
        custom_names
            same as 'default_colnames_dict' in self._get_dynamictable_object
        Returns
        -------
        dict()
        """
        attrs_list = [i['name'] for i in attrs_dict]
        list_id_func_exclude = \
            lambda val, comp_list, comp_bool: [i for i, j in enumerate(comp_list) if comp_bool & (j == val)]
        cleanup = lambda x: [i[0] for i in x if i]
        if custom_names:
            custom_names_list = [i for i in list(custom_names.values())]
            custom_names_dict = []
            for i in range(len(custom_names_list)):
                custom_names_dict.extend([{
                    'name': custom_names_list[i],
                    'description': 'no_description'
                }])
            attr_list_include_idx = cleanup([
                list_id_func_exclude(i, attrs_list, True)
                for i in custom_names_list
            ])
            attr_list_exclude_idx = set(range(len(attrs_list))).difference(
                set(attr_list_include_idx))
            custom_names_list_include_idx = [
                i for i, j in enumerate(custom_names_list)
                if list_id_func_exclude(j, attrs_list, True)
            ]
            for ii, jj in enumerate(custom_names_list_include_idx):
                custom_names_dict[custom_names_list_include_idx[
                    ii]] = attrs_dict[attr_list_include_idx[ii]]
                custom_names_list[custom_names_list_include_idx[
                    ii]] = attrs_list[attr_list_include_idx[ii]]
            extend_dict = [attrs_dict[i] for i in attr_list_exclude_idx]
            extend_list = [attrs_list[i] for i in attr_list_exclude_idx]
            custom_names_dict.extend(extend_dict)
            custom_names_list.extend(extend_list)
            return custom_names_dict, custom_names_list
        else:
            out_dict = attrs_dict
            out_list = attrs_list
            return out_dict, out_list

    @staticmethod
    def _drop_attrs(dataset_details, drop_attrs, default_colnames_dict=None):
        """
        Used to remove given attributes of the IBL dataset.
        Parameters
        ----------
        dataset_details: list
            self.dataset_details['clusters']
            [
                {
                    'name': 'amps',
                    'description': description
                },
                {
                    'name': 'channels',
                    'description': description
                }
            ]
        drop_attrs: list
            list of str: attribute names to drop of the self.dataset_details dict
        default_colnames_dict
        Returns
        -------
        dataset_details: list
            list without dictionaries with 'name' as in drop_attrs

        """
        attrs_list = [i['name'] for i in dataset_details]
        if default_colnames_dict is not None:
            default_colnames_dict_copy = default_colnames_dict.copy()
            for i, j in default_colnames_dict.items():
                if j not in attrs_list:
                    default_colnames_dict_copy.pop(i)
        else:
            default_colnames_dict_copy = default_colnames_dict
        if drop_attrs is None:
            return dataset_details, default_colnames_dict_copy
        elif default_colnames_dict is not None:
            for i, j in default_colnames_dict.items():
                if j in drop_attrs and j in attrs_list:
                    default_colnames_dict_copy.pop(i)
        dataset_details_return = [
            dataset_details[i] for i, j in enumerate(attrs_list)
            if j not in drop_attrs
        ]
        return dataset_details_return, default_colnames_dict_copy

    @staticmethod
    def _get_dynamictable_array(**kwargs):
        """
        Helper to dynamictable object method
        Parameters
        ----------
        kwargs
            keys and values that define the dictionary,
            both keys and values are lists where each index would slice all the keys/values and create a dict out of that

        Returns
        -------
        list
            list of dictionaries each with the keys and values from kwargs

        """
        custom_keys = list(kwargs.keys())
        custom_data = list(kwargs.values())
        out_list = [None] * len(custom_data[0])
        for ii, jj in enumerate(custom_data[0]):
            out_list[ii] = dict().copy()
            for i, j in enumerate(custom_keys):
                out_list[ii][j] = custom_data[i][ii]
        return out_list

    def _get_dynamictable_object(self,
                                 dataset_details,
                                 object_name,
                                 dt_name,
                                 default_colnames_dict=None,
                                 custom_attrs=None,
                                 drop_attrs=None):
        """

        Parameters
        ----------
        dataset_details
            self.dataset_details for each eid
        object_name:str
            object from the IBL data types from which to create this table.
        dt_name:str
            custom name for the dynamic table. Its the key with the value being dynamictable_array
        default_colnames_dict:dict()
            keys are the custom names of the columns, corresponding values are the attributes which have to be renamed.
        custom_attrs:list
            list of attributes for the given IBL object in object_name to be considered, all others are ignored

        Returns
        -------
        outdict:dict()
            example output below:
            {'Trials':
                    [
                        {
                          "name": "column1 name",
                          "data": "column data uri (string)",
                          "description": "col1 description"
                        },
                        {
                           "name": "column2 name",
                          "data": "column data uri (string)",
                          "description": "col2 description"
                        }
                    ]
                }
        """
        dataset_details[object_name], default_colnames_dict = self._drop_attrs(
            dataset_details[object_name].copy(), drop_attrs,
            default_colnames_dict)
        dataset_details[object_name], _ = self._attrnames_align(
            dataset_details[object_name].copy(), default_colnames_dict)
        if not default_colnames_dict:
            default_colnames = []
        else:
            default_colnames = list(default_colnames_dict.keys())
        custom_columns_datafilename, custom_columns_name, custom_columns_description = \
            self._unpack_dataset_details(dataset_details.copy(), object_name, custom_attrs)
        custom_columns_name[:len(default_colnames)] = default_colnames
        in_list = self._get_dynamictable_array(
            name=custom_columns_name,
            data=custom_columns_datafilename,
            description=custom_columns_description)
        outdict = {dt_name: in_list}
        return outdict

    @property
    def eid_metadata(self):
        return dict(eid=self.eid)

    @property
    def probe_metadata(self):
        probes_metadata_dict = self._initialize_container_dict(
            'Probes', default_value=[])
        probe_list = self.eid_session_info['probe_insertion']
        probe_dict_keys = ['id', 'model', 'name', 'trajectory_estimate']
        input_dict = dict()
        for key in probe_dict_keys:
            if key == 'trajectory_estimate':
                input_dict.update({
                    key:
                    [[json.dumps(l) for l in probe_list[i].get(key, ["None"])]
                     for i in range(len(probe_list))]
                })
            else:
                input_dict.update({
                    key: [
                        probe_list[i].get(key, "None")
                        for i in range(len(probe_list))
                    ]
                })
        probes_metadata_dict['Probes'].extend(
            self._get_dynamictable_array(**input_dict))
        return probes_metadata_dict

    @property
    def nwbfile_metadata(self):
        nwbfile_metadata_dict = self._initialize_container_dict('NWBFile')
        nwbfile_metadata_dict['NWBFile'].update(
            session_start_time=self._get_datetime(
                self.eid_session_info['start_time']),
            keywords=[
                ','.join(self.eid_session_info['users']),
                self.eid_session_info['lab'], 'IBL'
            ],
            experiment_description=self.eid_session_info['project'],
            session_id=self.eid,
            experimenter=self.eid_session_info['users'],
            identifier=self.eid,
            institution=[
                i['institution'] for i in self.lab_table
                if i['name'] == [self.eid_session_info['lab']][0]
            ][0],
            lab=self.eid_session_info['lab'],
            protocol=self.eid_session_info['task_protocol'],
            surgery='none',
            notes=', '.join([
                f"User:{i['user']}{i['text']}"
                for i in self.eid_session_info['notes']
            ]),
            session_description=','.join(self.eid_session_info['procedures']))
        return nwbfile_metadata_dict

    @property
    def sessions_metadata(self):
        sessions_metadata_dict = self._initialize_container_dict(
            'IBLSessionsData')
        custom_fields = [
            'location', 'project', 'type', 'number', 'end_time',
            'parent_session', 'url', 'qc'
        ]
        sessions_metadata_dict['IBLSessionsData'] = {
            i: str(self.eid_session_info[i])
            if i not in ['procedures', 'number'] else self.eid_session_info[i]
            for i in custom_fields
        }
        sessions_metadata_dict['IBLSessionsData']['extended_qc'] = json.dumps(
            self.eid_session_info['extended_qc'])
        sessions_metadata_dict['IBLSessionsData']['json'] = json.dumps(
            self.eid_session_info['json'])
        sessions_metadata_dict['IBLSessionsData']['wateradmin_session_related'] = \
            [json.dumps(i) for i in self.eid_session_info['wateradmin_session_related']]\
                if len(self.eid_session_info['wateradmin_session_related']) > 0 else ['None']
        sessions_metadata_dict['IBLSessionsData']['notes'] = \
            [json.dumps(i) for i in self.eid_session_info['notes']]\
                if len(self.eid_session_info['notes']) > 0 else ['None']
        return sessions_metadata_dict

    @property
    def subject_metadata(self):
        subject_metadata_dict = self._initialize_container_dict('IBLSubject')
        sub_table_dict = deepcopy(self.subject_table)
        if sub_table_dict:
            subject_metadata_dict['IBLSubject'] = dict(
                age=f'P{sub_table_dict.pop("age_weeks")}W',
                subject_id=sub_table_dict.pop('id'),
                description=sub_table_dict.pop('description'),
                genotype=','.join(sub_table_dict.pop('genotype')),
                sex=sub_table_dict.pop('sex'),
                species=sub_table_dict.pop('species'),
                weight=str(sub_table_dict.pop('reference_weight')),
                date_of_birth=self._get_datetime(
                    sub_table_dict.pop('birth_date')),
                **sub_table_dict)
            water_admin_data = [json.dumps(i) for i in subject_metadata_dict['IBLSubject']['water_administrations']]\
                    if len(subject_metadata_dict['IBLSubject']['water_administrations']) > 0 else ['None']
            subject_metadata_dict['IBLSubject'].update(
                weighings=[
                    json.dumps(i)
                    for i in subject_metadata_dict['IBLSubject']['weighings']
                ],
                water_administrations=water_admin_data)
            temp_metadatadict = deepcopy(subject_metadata_dict['IBLSubject'])
            for key, val in temp_metadatadict.items():
                if isinstance(val, list) and len(val) == 0:
                    _ = subject_metadata_dict['IBLSubject'].pop(key)
        return subject_metadata_dict

    @property
    def surgery_metadata(self):  # currently not exposed by api
        return dict()

    @property
    def behavior_metadata(self):
        behavior_metadata_dict = self._initialize_container_dict('Behavior')
        behavior_objects = [
            'wheel', 'wheelMoves', 'licks', 'lickPiezo', 'face', 'eye',
            'camera'
        ]
        current_behavior_objects = self._get_current_object_names(
            behavior_objects)
        for object_name in current_behavior_objects:
            if 'wheel' == object_name:
                behavior_metadata_dict['Behavior']['BehavioralTimeSeries'] = \
                    self._get_timeseries_object(self.dataset_details.copy(), object_name, 'time_series')
            if 'wheelMoves' in object_name:
                behavior_metadata_dict['Behavior']['BehavioralEpochs'] = \
                    self._get_timeseries_object(self.dataset_details.copy(), object_name, 'time_intervals')
            if 'lickPiezo' in object_name:
                behavior_metadata_dict['Behavior']['BehavioralTimeSeries'][
                    'time_series'].extend(
                        self._get_timeseries_object(
                            self.dataset_details.copy(), object_name,
                            'time_series')['time_series'])
            if 'licks' in object_name:
                behavior_metadata_dict['Behavior']['BehavioralEvents'] = \
                    self._get_timeseries_object(self.dataset_details.copy(), object_name, 'time_series')
            if 'face' in object_name:
                behavior_metadata_dict['Behavior']['BehavioralTimeSeries'][
                    'time_series'].extend(
                        self._get_timeseries_object(
                            self.dataset_details.copy(), object_name,
                            'time_series')['time_series'])
            if 'eye' in object_name:
                behavior_metadata_dict['Behavior']['PupilTracking'] = \
                    self._get_timeseries_object(self.dataset_details.copy(), object_name, 'time_series')
            if 'camera' in object_name:
                behavior_metadata_dict['Behavior']['Position'] = \
                    self._get_timeseries_object(self.dataset_details.copy(), object_name, 'spatial_series', name='camera_dlc')
                if len(behavior_metadata_dict['Behavior']['Position']['spatial_series']) > 0 and \
                        behavior_metadata_dict['Behavior']['Position']['spatial_series'][0]['data']== \
                        behavior_metadata_dict['Behavior']['Position']['spatial_series'][0]['timestamps']:
                    behavior_metadata_dict['Behavior']['Position'][
                        'spatial_series'][0][
                            'timestamps'] = '_iblrig_Camera.timestamps'
        return behavior_metadata_dict

    @property
    def trials_metadata(self):
        trials_metadata_dict = self._initialize_container_dict('Trials')
        trials_objects = ['trials']
        current_trial_objects = self._get_current_object_names(trials_objects)
        for object_name in current_trial_objects:
            if 'trial' in object_name:
                trials_metadata_dict = self._get_dynamictable_object(
                    self.dataset_details.copy(),
                    'trials',
                    'Trials',
                    default_colnames_dict=dict(start_time='intervals',
                                               stop_time='intervals'))
        return trials_metadata_dict

    @property
    def stimulus_metadata(self):
        stimulus_objects = [
            'sparseNoise', 'passiveBeeps', 'passiveValveClick',
            'passiveVisual', 'passiveWhiteNoise'
        ]
        stimulus_metadata_dict = self._initialize_container_dict('Stimulus')
        current_stimulus_objects = self._get_current_object_names(
            stimulus_objects)
        for object_name in current_stimulus_objects:
            if 'sparseNoise' in object_name:
                stimulus_metadata_dict['Stimulus'] = \
                    self._get_timeseries_object(self.dataset_details.copy(), object_name, 'time_series')
            if 'passiveBeeps' in object_name:
                stimulus_metadata_dict['Stimulus']['time_series'].extend(
                    self._get_timeseries_object(self.dataset_details.copy(),
                                                object_name,
                                                'time_series')['time_series'])
            if 'passiveValveClick' in object_name:
                stimulus_metadata_dict['Stimulus']['time_series'].extend(
                    self._get_timeseries_object(self.dataset_details.copy(),
                                                object_name,
                                                'time_series')['time_series'])
            if 'passiveVisual' in object_name:
                stimulus_metadata_dict['Stimulus']['time_series'].extend(
                    self._get_timeseries_object(self.dataset_details.copy(),
                                                object_name,
                                                'time_series')['time_series'])
            if 'passiveWhiteNoise' in object_name:
                stimulus_metadata_dict['Stimulus']['time_series'].extend(
                    self._get_timeseries_object(self.dataset_details.copy(),
                                                object_name,
                                                'time_series')['time_series'])
        return stimulus_metadata_dict

    @property
    def device_metadata(self):
        device_metadata_dict = self._initialize_container_dict(
            'Device', default_value=[])
        device_metadata_dict['Device'].extend(
            self._get_dynamictable_array(name=['NeuroPixels probe'],
                                         description=['NeuroPixels probe']))
        return device_metadata_dict

    @property
    def units_metadata(self):
        units_objects = ['clusters', 'spikes']
        metrics_columns = [
            'cluster_id', 'cluster_id.1', 'num_spikes', 'firing_rate',
            'presence_ratio', 'presence_ratio_std', 'isi_viol',
            'amplitude_cutoff', 'amplitude_std', 'epoch_name',
            'ks2_contamination_pct', 'ks2_label'
        ]

        units_metadata_dict = self._initialize_container_dict(
            'Units', default_value=list())
        current_units_objects = self._get_current_object_names(units_objects)
        for object_name in current_units_objects:
            if 'clusters' in object_name:
                units_metadata_dict = \
                    self._get_dynamictable_object(self.dataset_details.copy(), 'clusters', 'Units',
                                                  default_colnames_dict=dict(location='brainAcronyms',
                                                                             waveform_mean='waveforms',
                                                                             electrodes='channels',
                                                                             electrode_group='probes',
                                                                             ),
                                                  drop_attrs=['uuids', 'metrics'])
                units_metadata_dict['Units'].extend(
                    self._get_dynamictable_array(
                        name=['obs_intervals', 'spike_times'],
                        data=[
                            'trials.intervals', 'spikes.clusters,spikes.times'
                        ],
                        description=[
                            'time intervals of each cluster',
                            'spike times of cluster'
                        ]))
                units_metadata_dict['Units'].extend(
                    self._get_dynamictable_array(
                        name=metrics_columns,
                        data=['clusters.metrics'] * len(metrics_columns),
                        description=['metrics_table columns data'] *
                        len(metrics_columns)))
        return units_metadata_dict

    @property
    def electrodegroup_metadata(self):
        electrodes_group_metadata_dict = self._initialize_container_dict(
            'ElectrodeGroup', default_value=[])
        for ii in range(len(self.probe_metadata['Probes'])):
            try:
                location_str = self.probe_metadata['Probes'][ii][
                    'trajectory_estimate'][0]['coordinate_system']
            except:
                location_str = 'None'
            electrodes_group_metadata_dict['ElectrodeGroup'].extend(
                self._get_dynamictable_array(
                    name=[self.probe_metadata['Probes'][ii]['name']],
                    description=[
                        'model {}'.format(
                            self.probe_metadata['Probes'][ii]['model'])
                    ],
                    device=[self.device_metadata['Device'][0]['name']],
                    location=[
                        'Mouse CoordinateSystem:{}'.format(location_str)
                    ]))
        return electrodes_group_metadata_dict

    @property
    def electrodetable_metadata(self):
        electrodes_objects = ['channels']
        electrodes_table_metadata_dict = self._initialize_container_dict(
            'ElectrodeTable')
        current_electrodes_objects = self._get_current_object_names(
            electrodes_objects)
        for i in current_electrodes_objects:
            electrodes_table_metadata_dict = self._get_dynamictable_object(
                self.dataset_details.copy(),
                'channels',
                'ElectrodeTable',
                default_colnames_dict=dict(group='probes',
                                           x='localCoordinates',
                                           y='localCoordinates'))
        return electrodes_table_metadata_dict

    @property
    def ecephys_metadata(self):
        ecephys_objects = [
            'templates', '_iblqc_ephysTimeRms', '_iblqc_ephysSpectralDensity'
        ]
        container_object_names = [
            'SpikeEventSeries', 'ElectricalSeries', 'Spectrum'
        ]
        custom_attrs_objects = [['waveforms'], ['rms'], ['power']]
        ecephys_container = self._initialize_container_dict('Ecephys')
        kwargs = dict()
        for i, j, k in zip(ecephys_objects, container_object_names,
                           custom_attrs_objects):
            current_ecephys_objects = self._get_current_object_names([i])
            if current_ecephys_objects:
                if j == 'Spectrum':
                    kwargs = dict(
                        name=i,
                        power='_iblqc_ephysSpectralDensity.power',
                        frequencies='_iblqc_ephysSpectralDensity.freqs',
                        timestamps=None)
                ecephys_container['Ecephys'].update(
                    self._get_timeseries_object(self.dataset_details.copy(),
                                                i,
                                                j,
                                                custom_attrs=k,
                                                **kwargs))
            else:
                warnings.warn(f'could not find {i} data in eid {self.eid}')
        return ecephys_container

    @property
    def acquisition_metadata(self):
        acquisition_objects = [
            'ephysData', '_iblrig_Camera', '_iblmic_audioSpectrogram'
        ]
        container_name_objects = [
            'ElectricalSeries', 'ImageSeries', 'DecompositionSeries'
        ]
        custom_attrs_objects = [['raw.nidq', 'raw.ap', 'raw.lf'], ['raw'],
                                ['power']]
        acquisition_container = self._initialize_container_dict('Acquisition')
        current_acquisition_objects = self._get_current_object_names(
            acquisition_objects)
        idx = [
            no for no, i in enumerate(acquisition_objects)
            if i in current_acquisition_objects
        ]
        current_container_name_objects = [
            container_name_objects[i] for i in idx
        ]
        current_custom_attrs_objects = [custom_attrs_objects[i] for i in idx]
        kwargs = dict()
        for i, j, k in zip(current_acquisition_objects,
                           current_container_name_objects,
                           current_custom_attrs_objects):
            if j == 'DecompositionSeries':
                kwargs = dict(name=i,
                              metric='power',
                              bands='_iblmic_audioSpectrogram.frequencies')
            acquisition_container['Acquisition'].update(
                self._get_timeseries_object(self.dataset_details.copy(),
                                            i,
                                            j,
                                            custom_attrs=k,
                                            **kwargs))
        return acquisition_container

    @property
    def ophys_metadata(self):
        raise NotImplementedError

    @property
    def icephys_metadata(self):
        raise NotImplementedError

    @property
    def scratch_metadata(self):
        # this can be used to add further details about subject, lab,
        raise NotImplementedError

    @property
    def complete_metadata(self):
        metafile_dict = {
            **self.eid_metadata,
            **self.probe_metadata,
            **self.nwbfile_metadata,
            **self.sessions_metadata,
            **self.subject_metadata,
            **self.behavior_metadata,
            **self.trials_metadata,
            **self.stimulus_metadata,
            **self.units_metadata,
            **self.electrodetable_metadata, 'Ecephys': {
                **self.ecephys_metadata,
                **self.device_metadata,
                **self.electrodegroup_metadata,
            },
            'Ophys': dict(),
            'Icephys': dict(),
            **self.acquisition_metadata
        }
        return metafile_dict

    def write_metadata(self, fileloc, savetype=None):
        if savetype is not None:
            if Path(fileloc).suffix != savetype:
                raise ValueError(f'{fileloc} should of of type {savetype}')
        else:
            savetype = Path(fileloc).suffix
        full_metadata = self.complete_metadata
        if savetype == '.json':
            full_metadata['NWBFile']['session_start_time'] = datetime.strftime(
                full_metadata['NWBFile']['session_start_time'], '%Y-%m-%dT%X')
            full_metadata['IBLSubject']['date_of_birth'] = datetime.strftime(
                full_metadata['IBLSubject']['date_of_birth'], '%Y-%m-%dT%X')
            with open(fileloc, 'w') as f:
                json.dump(full_metadata, f, indent=2)
        elif savetype in ['.yaml', '.yml']:
            with open(fileloc, 'w') as f:
                yaml.dump(full_metadata, f, default_flow_style=False)
        print(f'data written in {fileloc}')
예제 #17
0
    a, b = bb.singlecell.calculate_peths(spikes, clusters,
                                         quality.index[quality], times)
    print(time.time() - start)

    depths = depths[quality]
    """vals, indizes = np.unique(clusters, return_index=True)
    clusts = [clusters[i] for i in sorted(indizes)]
    depths = depths[np.argsort(np.flip(clusts))]  # interesting results, weirdly enough"""

    perps = [5, 8]
    np.random.seed(4)
    for p in perps:
        neurons_embedded = TSNE(perplexity=p).fit_transform(a.means)
        plt.scatter(neurons_embedded[:, 0], neurons_embedded[:, 1], c=depths)
        title = "good sort Mouse {} Perplexity {}".format(
            one.list(eid, 'subject'), p)
        plt.title(title)
        plt.savefig('../../figures/' + title + '.png')
        plt.close()

quit()


def split(d, l):
    dizzle = {}
    for data, label, in zip(d, l):
        if label not in dizzle:
            dizzle[label] = [data]
        else:
            dizzle[label].append(data)
    return list(dizzle.values())
예제 #18
0
class Patcher(abc.ABC):
    def __init__(self, one=None):
        # one object
        if one is None:
            self.one = ONE()
        else:
            self.one = one

    def patch_dataset(self, path, dset_id=None, dry=False):
        """
        Uploads a dataset from an arbitrary location to FlatIron.
        :param path:
        :param dset_id:
        :param dry:
        :return:
        """
        status = self._patch_dataset(path, dset_id=dset_id, dry=dry)
        if not dry and status == 0:
            self.one.alyx.rest('datasets', 'partial_update', id=dset_id,
                               data={'hash': md5(path),
                                     'file_size': path.stat().st_size,
                                     'version': version.ibllib()}
                               )

    def _patch_dataset(self, path, dset_id=None, dry=False, ftp=False):
        """
        Private method that skips
        """
        path = Path(path)
        if dset_id is None:
            dset_id = path.name.split('.')[-2]
            if not alf.io.is_uuid_string(dset_id):
                dset_id = None
        assert dset_id
        assert alf.io.is_uuid_string(dset_id)
        assert path.exists()
        dset = self.one.alyx.rest('datasets', "read", id=dset_id)
        fr = next(fr for fr in dset['file_records'] if 'flatiron' in fr['data_repository'])
        remote_path = Path(fr['data_repository_path']).joinpath(fr['relative_path'])
        remote_path = alf.io.add_uuid_string(remote_path, dset_id).as_posix()
        if remote_path.startswith('/'):
            full_remote_path = PurePosixPath(FLATIRON_MOUNT + remote_path)
        else:
            full_remote_path = PurePosixPath(FLATIRON_MOUNT, remote_path)
        if isinstance(path, WindowsPath) and not ftp:
            # On Windows replace drive map with Globus uri, e.g. C:/ -> /~/C/
            path = '/~/' + path.as_posix().replace(':', '')
        status = self._scp(path, full_remote_path, dry=dry)[0]
        return status

    def register_dataset(self, file_list, **kwargs):
        """
        Registers a set of files belonging to a session only on the server
        :param file_list: (list of pathlib.Path)
        :param created_by: (string) name of user in Alyx (defaults to 'root')
        :param repository: optional: (string) name of the server repository in Alyx
        :param versions: optional (list of strings): versions tags (defaults to ibllib version)
        :param dry: (bool) False by default
        :return:
        """
        return register_dataset(file_list, one=self.one, server_only=True, **kwargs)

    def create_dataset(self, file_list, repository=None, created_by='root', dry=False, ftp=False):
        """
        Creates a new dataset on FlatIron and uploads it from arbitrary location.
        Rules for creation/patching are the same that apply for registration via Alyx
        as this uses the registration endpoint to get the dataset.
        An existing file (same session and path relative to session) will be patched.
        :param path: full file path. Must be whithin an ALF session folder (subject/date/number)
        can also be a list of full file pathes belonging to the same session.
        :param server_repository: Alyx server repository name
        :param created_by: alyx username for the dataset (optional, defaults to root)
        :param ftp: flag for case when using ftppatcher. Don't adjust windows path in
        _patch_dataset when ftp=True
        :return: the registrations response, a list of dataset records

        """
        # first register the file
        if not isinstance(file_list, list):
            file_list = [Path(file_list)]
        assert len(set([alf.io.get_session_path(f) for f in file_list])) == 1
        assert all([Path(f).exists() for f in file_list])
        response = self.register_dataset(file_list, created_by=created_by,
                                         repository=repository, dry=dry)
        if dry:
            return
        # from the dataset info, set flatIron flag to exists=True
        for p, d in zip(file_list, response):
            self._patch_dataset(p, dset_id=d['id'], dry=dry, ftp=ftp)
        return response

    def delete_dataset(self, dset_id, dry=False):
        """
        Deletes a single dataset from the Flatiron and Alyx database.
        This does not remove the dataset from local servers.
        :param dset_id:
        :param dry:
        :return:
        """
        if isinstance(dset_id, dict):
            dset = dset_id
            dset_id = dset['url'][-36:]
        else:
            dset = self.one.alyx.rest('datasets', "read", id=dset_id)
        assert dset
        for fr in dset['file_records']:
            if 'flatiron' in fr['data_repository']:
                flatiron_path = Path(FLATIRON_MOUNT).joinpath(fr['data_repository_path'][1:],
                                                              fr['relative_path'])
                flatiron_path = alf.io.add_uuid_string(flatiron_path, dset_id)
                status = self._rm(flatiron_path, dry=dry)[0]
                if status == 0 and not dry:
                    self.one.alyx.rest('datasets', 'delete', id=dset_id)

    def delete_session_datasets(self, eid, dry=True):
        """
            Deletes all datasets attached to the session from database and flatiron but leaves
            the session on the database.
            Useful for a full re-extraction
        """
        ses_details = self.one.alyx.rest('sessions', 'read', id=eid)
        raise NotImplementedError("Code below only removes existing files. Need to search"
                                  "for datasets in a better way")
        # first delete attached datasets from the database
        dataset_details = self.one.list(eid, details=True)
        for n in range(len(dataset_details.dataset_id)):
            print(dataset_details.dataset_id[n])
            if dry:
                continue
            try:
                self.one.alyx.rest('datasets', 'delete', id=dataset_details.dataset_id[n])
            except Exception as e:
                print(e)

        # then delete the session folder from flatiron
        flatiron_path = Path(FLATIRON_MOUNT).joinpath(ses_details['lab'],
                                                      'public',
                                                      'projects',
                                                      ses_details['project'],
                                                      'ALF',
                                                      ses_details['subject'],
                                                      ses_details['start_time'][:10],
                                                      str(ses_details['number']).zfill(3))
        """flatiron_path = Path(FLATIRON_MOUNT).joinpath(ses_details['lab'],
                                                      'Subjects',
                                                      ses_details['subject'],
                                                      ses_details['start_time'][:10],
                                                      str(ses_details['number']).zfill(3))"""

        cmd = f"ssh -p {FLATIRON_PORT} {FLATIRON_USER}@{FLATIRON_HOST} rm -fR {flatiron_path}"
        print(cmd)

    @abc.abstractmethod
    def _scp(self, *args, **kwargs):
        pass

    @abc.abstractmethod
    def _rm(self, *args, **kwargs):
        pass
예제 #19
0
    '38d95489-2e82-412a-8c1a-c5377b5f1555',
    '4153bd83-2168-4bd4-a15c-f7e82f3f73fb',
    '614e1937-4b24-4ad3-9055-c8253d089919'
]
names_n_counts = []
names = []
all_data = []

for i, (eid, probe) in enumerate(zip(eids, probes)):
    if eid not in good_eids:
        print('skipped')
        continue
    if eid == '614e1937-4b24-4ad3-9055-c8253d089919':
        probe = 'probe00'
    print(eid)
    print(one.list(eid, 'subjects'))
    print(probe)
    channels = load_channel_locations(eid, one=one)
    # spikes, clusters = load_spike_sorting(eids[0], one=one)
    all_data.append(channels[probe])
    names_n_counts.append(channels[probe].acronym)
    names.append(one.list(eid, 'subjects'))

    print(len(channels[probe].acronym))

fs = 18

for i, d in enumerate(names_n_counts):
    df = pd.DataFrame(data=d, columns=["acronym"])
    ax = plt.subplot(2, 3, i + 1)
    plt.title(names[i], fontsize=fs)