def test_create_delete_project_and_childs():
module_contents = {'species': {'value': 'rat'}}
project = expipe.core.require_project(pytest.PROJECT_ID)
action = project.require_action(pytest.ACTION_ID)
action_module = action.create_module(pytest.MODULE_ID,
contents=module_contents,
overwrite=False)
project_module = project.create_module(pytest.MODULE_ID,
contents=module_contents,
overwrite=False)
expipe.delete_project(pytest.PROJECT_ID, remove_all_childs=True)
with pytest.raises(KeyError):
expipe.get_project(pytest.PROJECT_ID)
# remake project, then the "old" action and project_module should be deleted
project = expipe.require_project(pytest.PROJECT_ID)
with pytest.raises(KeyError):
project.actions[pytest.ACTION_ID]
project.modules[pytest.MODULE_ID]
# remake action, then the "old" action_module should be deleted
action = project.require_action(pytest.ACTION_ID)
with pytest.raises(KeyError):
action.modules[pytest.MODULE_ID]
def display(project_path=None):
project_path = project_path or PAR.PROJECT_ROOT
assert project_path is not None
project = expipe.get_project(project_path)
# register tab
register_tab_tab_titles = [
'OpenEphys', 'Axona', 'Adjustment', 'Entity', 'Surgery', 'Perfusion'
]
register_tab = ipywidgets.Tab()
register_tab.children = [
openephys_view(project),
axona_view(project),
adjustment_view(project),
entity_view(project),
surgery_view(project),
perfuse_view(project),
]
for i, title in enumerate(register_tab_tab_titles):
register_tab.set_title(i, title)
process_tab_tab_titles = [
'OpenEphys',
]
process_tab = ipywidgets.Tab()
process_tab.children = [process_view(project)]
for i, title in enumerate(process_tab_tab_titles):
process_tab.set_title(i, title)
tab_titles = ['Register', 'Process']
tab = ipywidgets.Tab()
tab.children = [register_tab, process_tab]
for i, title in enumerate(tab_titles):
tab.set_title(i, title)
ipd.display(tab)
def list_plugins():
cwd = pathlib.Path.cwd()
try:
project = expipe_module.get_project(path=pathlib.Path.cwd())
config = project.config
except KeyError as e:
config = expipe_module.settings
return config.get('plugins')
def list_stuff(object_type):
"""Print project objects."""
try:
project = expipe_module.get_project(path=pathlib.Path.cwd())
except KeyError as e:
print(str(e))
return
for object in getattr(project, object_type):
print(object)
def status():
"""Print project status."""
try:
project = expipe_module.get_project(path=pathlib.Path.cwd())
except KeyError as e:
print(str(e))
return
for k, v in project.config.items():
print('{}: {}'.format(k, v))
def display(project_path):
project = expipe.get_project(project_path)
# register tab
register_tab_tab_titles = [
'OpenEphys',
'Intan',
'Axona',
'Adjustment',
'Entity',
'Surgery',
'Perfusion',
'Annotate']
register_tab = ipywidgets.Tab()
register_tab.children = [
register_openephys_view(project),
register_intan_view(project),
axona_view(project),
adjustment_view(project),
entity_view(project),
surgery_view(project),
perfuse_view(project),
annotate_view(project)
]
for i, title in enumerate(register_tab_tab_titles):
register_tab.set_title(i, title)
process_tab_tab_titles = [
'OpenEphys', 'Intan', 'Tracking', 'Psychopy', 'Curation']
process_tab = ipywidgets.Tab()
process_tab.children = [
process_openephys_view(project),
process_intan_view(project),
process_tracking_view(project),
process_psychopy_view(project),
process_curation_view(project)
]
for i, title in enumerate(process_tab_tab_titles):
process_tab.set_title(i, title)
tab_titles = ['Register', 'Process']
tab = ipywidgets.Tab()
tab.children = [
register_tab,
process_tab
]
for i, title in enumerate(tab_titles):
tab.set_title(i, title)
ipd.display(tab)
def __init__(self, stim_mask=False, baseline_duration=None, **kwargs):
self.project_path = project_path()
self.params = kwargs
self.project = expipe.get_project(self.project_path)
self.actions = self.project.actions
self._spike_trains = {}
self._templates = {}
self._stim_times = {}
self._unit_names = {}
self._tracking = {}
self._head_direction = {}
self._lfp = {}
self._occupancy = {}
self._rate_maps = {}
self._prob_dist = {}
self._spatial_bins = None
self.stim_mask = stim_mask
self.baseline_duration = baseline_duration
def __init__(
self, project_path, action_ids, group_ids, position_sampling_rate,
position_low_pass_frequency, box_size):
project = expipe.get_project(project_path)
actions = project.actions
self._data = {}
self.group_ids = group_ids
self.action_ids = action_ids
for action_id in action_ids:
action = actions[action_id]
self._data[action_id] = {}
data_path = dp.get_data_path(action)
epochs = dp.load_epochs(data_path)
x, y, t, speed = dp.load_tracking(
data_path, position_sampling_rate, position_low_pass_frequency, box_size=box_size)
a, at = dp.load_head_direction(
data_path, position_sampling_rate, position_low_pass_frequency, box_size=box_size)
self._data[action_id] = {
'x': x,
'y': y,
't': t,
'speed': speed,
'a': a,
'at': at,
'epochs': epochs,
'groups': {}
}
for group_id in group_ids:
try:
anas = dp.load_lfp(data_path, group_id) # TODO remove artifacts
except:
print(group_id)
raise
spike_trains = dp.load_spiketrains(data_path, group_id, load_waveforms=True)
self._data[action_id]['groups'][group_id] = {
'spike_trains': {_name(s): s for s in dp.sort_by_cluster_id(spike_trains)},
'anas': anas
}
@lazy_import
def expipe():
import expipe
return expipe
@lazy_import
def pathlib():
import pathlib
return pathlib
local_root, _ = expipe.config._load_local_config(pathlib.Path.cwd())
if local_root is not None:
project = expipe.get_project(path=local_root)
else:
class P:
config = {}
project = P
@lazy_import
def pd():
import pandas as pd
return pd
@lazy_import
def experiment_plot(project_path,
action_id,
n_channel=8,
rem_channel="all",
skip_channels=None,
raster_start=-0.5,
raster_stop=1):
"""
Plot raster, isi-mean-median, tuning (polar and linear) from visual data acquired through open-ephys and psychopy
Parameters
----------
project_path: os.path, or equivallent
Path to project directory
action_id: string
The experiment/action id
n_channel: default 8; int
Number of channel groups in recordings
rem_channel: default "all"; "all" or int
Signify what channels are of interest(=rem)
skip_channels: default None; int or (list, array, tupule)
ID of channel groups to skip
raster_start: default -0.5; int or float
When the rasters start on the x-axis relative to trial start, which is 0
raster_stop: default 1; int or float
When the rasters stop on the x-axis relative to trial start, which is 0
Returns
-------
savesfigures into exdir directory: main.exdir/figures
"""
if not (rem_channel == "all" or
(isinstance(rem_channel, int) and rem_channel < n_channel)):
msg = "rem_channel must be either 'all' or integer between 0 and n_channel ({}); not {}".format(
n_channel, rem_channel)
raise AttributeError(msg)
# Define project tree
project = expipe.get_project(project_path)
action = project.actions[action_id]
data_path = er.get_data_path(action)
epochs = er.load_epochs(data_path)
# Get data of interest (orients vs rates vs channel)
oe_epoch = epochs[0] # openephys
assert (oe_epoch.annotations['provenance'] == 'open-ephys')
ps_epoch = epochs[1] # psychopy
assert (ps_epoch.annotations['provenance'] == 'psychopy')
# Create directory for figures
exdir_file = exdir.File(data_path, plugins=exdir.plugins.quantities)
figures_group = exdir_file.require_group('figures')
raster_start = raster_start * pq.s
raster_stop = raster_stop * pq.s
orients = ps_epoch.labels # the labels are orrientations (135, 90, ...)
def plot(channel_num, channel_path, spiketrains):
# Create figures from spiketrains
for spiketrain in spiketrains:
try:
if spiketrain.annotations["cluster_group"] == "noise":
continue
except KeyError:
msg = "Cluster/channel group {} seems to not have been sorted in phys".format(
channel_num)
raise KeyError(msg)
figure_id = "{}_{}_".format(channel_num,
spiketrain.annotations['cluster_id'])
sns.set()
sns.set_style("white")
# Raster plot processing
trials = er.make_spiketrain_trials(spiketrain,
oe_epoch,
t_start=raster_start,
t_stop=raster_stop)
er.add_orientation_to_trials(trials, orients)
orf_path = os.path.join(channel_path,
figure_id + "orrient_raster.png")
orient_raster_fig = orient_raster_plots(trials)
orient_raster_fig.savefig(orf_path)
# Orrientation vs spikefrequency plot (tuning curves) processing
trials = er.make_spiketrain_trials(spiketrain, oe_epoch)
er.add_orientation_to_trials(trials, orients)
tf_path = os.path.join(channel_path, figure_id + "tuning.png")
tuning_fig = plot_tuning_overview(trials, spiketrain)
tuning_fig.savefig(tf_path)
# Reset before next loop to save memory
plt.close(fig="all")
if rem_channel == "all":
channels = range(n_channel)
if isinstance(skip_channels, int):
channels = [x for x in channels]
del channels[skip_channels]
elif isinstance(skip_channels, (list, tuple, type(empty(0)))):
channels = [x for x in channels]
for channel in skip_channels:
del channels[channel]
for channel in channels:
channel_name = "channel_{}".format(channel)
channel_group = figures_group.require_group(channel_name)
channel_path = os.path.join(str(data_path),
"figures\\" + channel_name)
spiketrains = er.load_spiketrains(str(data_path), channel)
plot(channel, channel_path, spiketrains)
elif isinstance(rem_channel, int):
channel_name = "channel_{}".format(rem_channel)
channel_group = figures_group.require_group(channel_name)
channel_path = os.path.join(str(data_path), "figures\\" + channel_name)
spiketrains = er.load_spiketrains(str(data_path), rem_channel)
plot(rem_channel, channel_path, spiketrains)
def load_data_frames(queries=None,
labels=None,
colors=None,
stim_location='stim_location=="ms"'):
cell_types = [
'ns_inhibited', 'ns_not_inhibited', 'gridcell', 'bs_not_gridcell'
]
import expipe
import septum_mec.analysis.data_processing as dp
if queries is None:
queries, labels, colors, gridcell_query = get_queries(stim_location)
else:
_, _, _, gridcell_query = get_queries()
label_nums = list(range(len(labels)))
project_path = dp.project_path()
project = expipe.get_project(project_path)
actions = project.actions
identification_action = actions['identify-neurons']
sessions = pd.read_csv(identification_action.data_path('sessions'))
units = pd.read_csv(identification_action.data_path('units'))
session_units = pd.merge(sessions, units, on='action')
statistics_action = actions['calculate-statistics']
statistics_results = pd.read_csv(statistics_action.data_path('results'))
statistics = pd.merge(session_units, statistics_results, how='left')
statistics['unit_day'] = statistics.apply(
lambda x: str(x.unit_idnum) + '_' + x.action.split('-')[1], axis=1)
# statistics_action_extra = actions['calculate-statistics-extra']
# statistics_action_extra = pd.read_csv(statistics_action_extra.data_path('results'))
# statistics = pd.merge(statistics, statistics_action_extra, how='left')
stim_response_action = actions['stimulus-response']
stim_response_results = pd.read_csv(
stim_response_action.data_path('results'))
statistics = pd.merge(statistics, stim_response_results, how='left')
shuffling = actions['shuffling']
quantiles_95 = pd.read_csv(shuffling.data_path('quantiles_95'))
quantiles_95.head()
action_columns = ['action', 'channel_group', 'unit_name']
data = pd.merge(statistics,
quantiles_95,
on=action_columns,
suffixes=("", "_threshold"))
data['specificity'] = np.log10(data['in_field_mean_rate'] /
data['out_field_mean_rate'])
# waveform
waveform_action = actions['waveform-analysis']
waveform_results = pd.read_csv(waveform_action.data_path('results')).drop(
'template', axis=1)
data = data.merge(waveform_results, how='left')
data.bs = data.bs.astype(bool)
data.loc[data.eval('t_i_peak == t_i_peak and not bs'),
'ns_inhibited'] = True
data.ns_inhibited.fillna(False, inplace=True)
data.loc[data.eval('t_i_peak != t_i_peak and not bs'),
'ns_not_inhibited'] = True
data.ns_not_inhibited.fillna(False, inplace=True)
# if a neuron is significantly inhibited once, we count it as a ns_inhibited
data.loc[data.unit_id.isin(data.query('ns_inhibited').unit_id.values),
'ns_inhibited'] = True
# we alsochange label from not inhibted to inhibited
data.loc[data.eval('ns_inhibited'), 'ns_not_inhibited'] = False
data.loc[data.eval('ns_inhibited'), 'bs'] = False
# data.loc[data.unit_id.isin(data.query('ns_not_inhibited').unit_id.values), 'ns_not_inhibited'] = True
# gridcells
sessions_above_threshold = data.query(gridcell_query)
print("Number of sessions above threshold", len(sessions_above_threshold))
print("Number of animals",
len(sessions_above_threshold.groupby(['entity'])))
gridcell_sessions = data[data.unit_day.isin(
sessions_above_threshold.unit_day.values)]
print("Number of individual gridcells",
gridcell_sessions.unit_idnum.nunique())
print("Number of gridcell recordings", len(gridcell_sessions))
data.loc[:, 'gridcell'] = np.nan
data['gridcell'] = data.isin(gridcell_sessions)
data.loc[data.eval('not gridcell and bs'), 'bs_not_gridcell'] = True
data.bs_not_gridcell.fillna(False, inplace=True)
for i, query in enumerate(queries):
data.loc[data.eval(query), 'label'] = labels[i]
data.loc[data.eval(query), 'label_num'] = label_nums[i]
data.loc[data.eval(query), 'query'] = query
data.loc[data.eval(query), 'color'] = colors[i]
data['cell_type'] = np.nan
for cell_type in cell_types:
data.loc[data.eval(cell_type), 'cell_type'] = cell_type
return data, labels, colors, queries
command_line = ' '.join(command_line)
else:
raise TypeError('str or list')
logger.info(command_line)
command_line_process = subprocess.Popen(
command_line_args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT
)
try:
stdout, stderr = command_line_process.communicate()
for line in stdout.decode().split('\n'):
logger.info(line)
if 'Error' in stdout.decode():
print('Error occured.')
except Exception as e:
logger.exception('Exception: ')
project = expipe.get_project(PAR.PROJECT_ID)
# your code
for action in project.actions:
if action.type != 'Recording':
continue
# if 'no' in action.tags:
# continue
print('Evaluating ', action.id)
run_shell_command(['expipe', 'analyse', action.id, '-a',
'spatial', '--skip'])
expipe.core.refresh_token()