def abs2rel_paths(folders: List[str]) -> None:
for folder in folders:
chdir(folder)
# noinspection PyArgumentList
for sub_folder in listdir():
if path.isdir(sub_folder) and path.exists(path.join(sub_folder, 'attributes.json')):
data = File(sub_folder, 'w+')
data.attrs['img_path'] = path.join('..', path.split(data.attrs['img_path'])[1])
def convert(folder_paths: List[str]):
for folder in folder_paths:
data = File(folder, 'w+')
if 'spike' not in data: # not deconvolved
print("starting file: \n\t{0}\nat frame rate {1}".format(
split(folder)[1], data.attrs['frame_rate']))
spike_train, time_axis, sample_rate = deconvolve(
data['measurement'], data.attrs['frame_rate'])
data['spike'] = {
"data": spike_train,
'x': time_axis,
'y': data['measurement']['y']
}
data.attrs['spike_resolution'] = sample_rate
def save_roi(self, roi_list: List[Roi]) -> List[Dict[str, np.ndarray]]: # type: ignore
result: List[Dict[str, np.ndarray]] = self.measure_roi(roi_list)
save_file = File(self.target_path, 'w+')
self.measuremnt = result
save_file['measurement-chan1'] = result[0]
save_file['measurement-chan2'] = result[1]
return result
def get_thresholds(data_file: File, overwrite: bool = False):
linkage, mask = get_linkage(data_file, motion_params)
if overwrite or ('hierarchy_threshold' not in data_file.attrs):
threshold = get_threshold(linkage)
data_file.attrs['hierarchy_threshold'] = threshold
data_file.attrs.flush()
threshold = data_file.attrs['hierarchy_threshold']
clusters = get_cluster_labels(linkage, threshold)
return clusters, mask
def load(cls,
data_file: Union[str, File],
data_target='spike') -> "Recording":
if isinstance(data_file, str):
data_file = File(data_file)
sample_rate = data_file.attrs.get('spike_resolution',
data_file.attrs['frame_rate'])
return cls(DataFrame.load(data_file[data_target]),
data_file['stimulus'], sample_rate)
def convert_data(data_path: str):
data = File(data_path, 'w+')
for x in data:
if isinstance(data[x], np.lib.npyio.NpzFile):
array = data[x]
if 'x' in array:
continue
if 'column' in array:
if 'z' in array:
data[x] = {'x': array['row'], 'y': array['column'], 'data': array['data'], 'z': array['z']}
else:
data[x] = {'x': array['row'], 'y': array['column'], 'data': array['data']}
def _decode_pheno_id(folder: str) -> IDs:
result = defaultdict(
list) # type: Dict[Tuple[int, int], List[Tuple[datetime, str]]]
for file_name in listdir(folder):
if not isdir(join(folder, file_name)):
continue
ids = File(join(folder, file_name)).attrs['id']
for id_str in ids:
date = datetime.strptime(file_name, '%Y%m%d')
result[(id_str // 1000, id_str % 1000)].append(
(date, join(folder, file_name)))
return result
def save(self, target_path: str = None, output_shape: Point = (512, 512),
draw_limit: bool = False):
target_path = target_path if target_path else self.target_path
self.target_path = target_path
output = File(target_path, 'w+')
for x in self._fields:
if hasattr(self, x):
output.attrs[x] = getattr(self, x)
output.attrs['img_path'] = path.relpath(self.img_path, target_path)
if hasattr(self, "displacement"):
output['displacement'] = self.displacement
if hasattr(self, "_template"):
output['template'] = self.template
for name in ('mean_frame', 'std_frame'):
if getattr(self, name, None) is not None:
frame = getattr(self, name)
if output_shape is not None:
frame = zoom(full_contrast(frame), np.divide(output_shape, frame.shape))
else:
frame = full_contrast(frame)
frame = self.draw_limit(frame) if draw_limit else frame
save_tiff(frame, path.join(target_path, name + '.tif'))
"quiet_var": 0.001,
"window_size": 1000,
"event_thres": 0.3,
"pre_time": 0.1,
"post_time": 1.4
}
with open(join(project_folder, 'data', 'recording.toml')) as fp:
mice = {
group_str: [{
'group': group_str,
**x
} for x in group]
for group_str, group in toml.load(fp).items()
}
files = map_tree(lambda x: (File(join(project_folder, "data", x["path"]))),
mice)
COLORS = [
"#dc322fff", "#268bd2ff", "#d33682ff", "#2aa198ff", "#859900ff",
"#b58900ff", "#50D0B8FF"
]
# Interactively set threshold and save in file attrs
def get_thresholds(data_file: File, overwrite: bool = False):
linkage, mask = get_linkage(data_file, motion_params)
if overwrite or ('hierarchy_threshold' not in data_file.attrs):
threshold = get_threshold(linkage)
data_file.attrs['hierarchy_threshold'] = threshold
data_file.attrs.flush()
threshold = data_file.attrs['hierarchy_threshold']
def save_roi(self, roi_list: List[Roi]) -> Dict[str, np.ndarray]:
result = self.measure_roi(roi_list)
save_file = File(self.target_path, 'w+')
self.measurement = result[0]
save_file['measurement'] = result[0]
return self.measurement
from noformat import File
from algorithm.array import DataFrame
from algorithm.stats import perm_test
from algorithm.utils import map_tree, zip_tree
from algorithm.time_series import fold_by
from lever.filter import devibrate_trials
from lever.utils import MotionParams, get_trials, filter_empty_trial_sets
from mplplot.importer import Figure
motion_params = {"quiet_var": 0.001, "window_size": 1000, "event_thres": 0.3, "pre_time": 0.1, "post_time": 0.9}
project_folder = expanduser("~/Sync/project/2017-leverpush")
img_folder = join(project_folder, "report", "img")
res_folder = join(project_folder, "report", "measure")
COLORS = ["#dc322fff", "#268bd2ff", "#d33682ff", "#2aa198ff", "#859900ff", "#b58900ff"]
mice = toml.load(join(project_folder, "data/recording.toml"))
files = {x['session']: File(join(project_folder, 'data', x['path'])) for x in mice['wt']['0304']['3']}
##
def neuron_lever_unsampled(data_file: File, params: Dict[str, float]) -> Tuple[np.ndarray, np.ndarray]:
"""
Returns:
neural_activity, lever
"""
lever = get_trials(data_file)
neurons = fold_by(DataFrame.load(data_file["spike"]), lever, data_file.attrs['frame_rate'], True)
neurons, lever = filter_empty_trial_sets(neurons.values, lever.values[0])
mask, filtered = devibrate_trials(lever, params["pre_time"])
return neurons[:, mask, :], lever[mask, :]
def neural_vs_motor_reliability(record_file: File, params: MotionParams, neural_no: int = 10, trial_no: int = 10,
repeat: int = 1000) -> np.ndarray:
"""Bootstrap the distribution to """
def data():
data_file = resource_filename(Requirement.parse('algorithm'),
join(TEST_DATA_FOLDER, 'contra-test'))
return File(data_file)
from os.path import expanduser, join
import toml
import numpy as np
from scipy.stats import ttest_ind, ks_2samp
from noformat import File
from algorithm.utils import map_tree
from algorithm.stats import combine_test, perm_test
from lever.utils import MotionParams, get_trials
from lever.plot import plot_scatter
from lever.filter import devibrate_rec
motion_params = {"quiet_var": 0.001, "window_size": 1000, "event_thres": 0.3, "pre_time": 0.3, "post_time": 0.7}
proj_folder = expanduser("~/Sync/project/2018-leverpush-chloe")
res_folder = join(proj_folder, "report", "measure")
mice = toml.load(join(proj_folder, 'data', 'recording.toml'))
files = map_tree(lambda x: File(join(proj_folder, 'data', x['path'])), mice)
COLORS = ["#dc322fff", "#268bd2ff", "#d33682ff", "#2aa198ff", "#859900ff", "#b58900ff"]
#
def reliability(data: np.ndarray) -> float:
t = data.shape[0]
coef = 2 / (t ** 2 - t)
return np.corrcoef(data, rowvar=True)[np.triu_indices(t, 1)].sum() * coef
def get_initial(data_file: File, params: MotionParams):
lever = devibrate_rec(get_trials(data_file, params))
pre_value = lever.values[:, :lever._pre // 2].mean(axis=1, keepdims=True)
lever_off = int(np.median(np.argmax(lever.values[:, lever._pre:] <= pre_value, axis=1))) + lever._pre
return reliability(lever.values[:, lever._pre // 2: lever_off])
def get_rise(data_file: File, params: MotionParams):
lever = devibrate_rec(get_trials(data_file, params))
from mplplot import Figure
project_folder = expanduser("~/Sync/project/2018-leverpush-chloe")
# project_folder = expanduser("~/Sync/project/2017-leverpush")
img_folder = join(project_folder, 'report', 'img')
res_folder = join(project_folder, 'report', 'measure')
motion_params = {
"quiet_var": 0.001,
"window_size": 1000,
"event_thres": 0.3,
"pre_time": 0.1,
"post_time": 1.4
}
mice = toml.load(join(project_folder, 'data', 'recording.toml'))
files = map_tree(lambda x: (File(join(project_folder, "data", x["path"]))),
mice)
COLORS = [
"#dc322fff", "#268bd2ff", "#d33682ff", "#2aa198ff", "#859900ff",
"#b58900ff"
]
## Amplitude
def get_amp(data_file: File) -> float:
mask, filtered = devibrate_trials(
get_trials(data_file, motion_params).values, motion_params['pre_time'])
return np.quantile(
filtered[mask, 25:64].max(axis=1) - filtered[mask, 0:15].mean(axis=1),
0.75)
from algorithm.array import DataFrame
from algorithm.stats import combine_test, perm_test
from algorithm.time_series import take_segment
from lever.filter import devibrate_trials
from lever.utils import get_trials
from lever.decoding.validate import cross_predict
from lever.plot import plot_scatter
project_folder = expanduser("~/Sync/project/2018-leverpush-chloe")
res_folder = join(project_folder, "report", "measure")
COLORS = [
"#dc322fff", "#268bd2ff", "#d33682ff", "#2aa198ff", "#859900ff",
"#b58900ff"
]
mice = toml.load(join(project_folder, "data/recording.toml"))
files = map_tree(lambda x: File(join(project_folder, 'data', x['path'])), mice)
motion_params = {
"quiet_var": 0.001,
"window_size": 1000,
"event_thres": 0.3,
"pre_time": 0.1,
"post_time": 0.9
}
## actual running
def run_amp_power(data_file: File) -> Tuple[float, float, float, float]:
"""Try to decode the max lever trajectory amplitude of each trial.
Returns:
pre_amp_power: mutual info between predicted (from neuron activity before motor onset)
and real amplitude of trials in one session