def test_extract_dataframe():
# Vanilla test
nrns = nm.load_neurons([Path(SWC_PATH, name)
for name in ['Neuron.swc', 'simple.swc']])
actual = ms.extract_dataframe(nrns, REF_CONFIG)
expected = pd.read_csv(Path(DATA_PATH, 'extracted-stats.csv'), header=[0, 1], index_col=0)
assert_frame_equal(actual, expected)
# Test with a single neuron in the population
nrns = nm.load_neurons(Path(SWC_PATH, 'Neuron.swc'))
actual = ms.extract_dataframe(nrns, REF_CONFIG)
assert_frame_equal(actual, expected.iloc[[0]], check_dtype=False)
# Test with a config without the 'neuron' key
nrns = nm.load_neurons([Path(SWC_PATH, name)
for name in ['Neuron.swc', 'simple.swc']])
config = {'neurite': {'section_lengths': ['total']},
'neurite_type': ['AXON', 'APICAL_DENDRITE', 'BASAL_DENDRITE', 'ALL']}
actual = ms.extract_dataframe(nrns, config)
idx = pd.IndexSlice
expected = expected.loc[:, idx[:, ['name', 'total_section_length']]]
assert_frame_equal(actual, expected)
# Test with a FstNeuron argument
nrn = nm.load_neuron(Path(SWC_PATH, 'Neuron.swc'))
actual = ms.extract_dataframe(nrn, config)
assert_frame_equal(actual, expected.iloc[[0]], check_dtype=False)
# Test with a List[FstNeuron] argument
nrns = [nm.load_neuron(Path(SWC_PATH, name))
for name in ['Neuron.swc', 'simple.swc']]
actual = ms.extract_dataframe(nrns, config)
assert_frame_equal(actual, expected)
# Test with a List[Path] argument
nrns = [Path(SWC_PATH, name) for name in ['Neuron.swc', 'simple.swc']]
actual = ms.extract_dataframe(nrns, config)
assert_frame_equal(actual, expected)
# Test without any neurite_type keys, it should pick the defaults
config = {'neurite': {'total_length_per_neurite': ['total']}}
actual = ms.extract_dataframe(nrns, config)
expected_columns = pd.MultiIndex.from_tuples(
[('property', 'name'),
('axon', 'total_total_length_per_neurite'),
('basal_dendrite', 'total_total_length_per_neurite'),
('apical_dendrite', 'total_total_length_per_neurite'),
('all', 'total_total_length_per_neurite')])
expected = pd.DataFrame(
columns=expected_columns,
data=[['Neuron', 207.87975221, 418.43241644, 214.37304578, 840.68521442],
['simple', 15., 16., 0., 31., ]])
assert_frame_equal(actual, expected)
def test_extract_dataframe():
# Vanilla test
nrns = nm.load_neurons([Path(SWC_PATH, name)
for name in ['Neuron.swc', 'simple.swc']])
actual = ms.extract_dataframe(nrns, REF_CONFIG)
expected = pd.read_csv(Path(DATA_PATH, 'extracted-stats.csv'), index_col=0)
assert_frame_equal(actual, expected)
# Test with a single neuron in the population
nrns = nm.load_neurons(Path(SWC_PATH, 'Neuron.swc'))
actual = ms.extract_dataframe(nrns, REF_CONFIG)
assert_frame_equal(actual, expected[expected.name == 'Neuron'], check_dtype=False)
# Test with a config without the 'neuron' key
nrns = nm.load_neurons([Path(SWC_PATH, name)
for name in ['Neuron.swc', 'simple.swc']])
config = {'neurite': {'section_lengths': ['total']},
'neurite_type': ['AXON', 'APICAL_DENDRITE', 'BASAL_DENDRITE', 'ALL']}
actual = ms.extract_dataframe(nrns, config)
expected = expected[['name', 'neurite_type', 'total_section_length']]
assert_frame_equal(actual, expected)
# Test with a FstNeuron argument
nrn = nm.load_neuron(Path(SWC_PATH, 'Neuron.swc'))
actual = ms.extract_dataframe(nrn, config)
assert_frame_equal(actual, expected[expected.name == 'Neuron'], check_dtype=False)
# Test with a List[FstNeuron] argument
nrns = [nm.load_neuron(Path(SWC_PATH, name))
for name in ['Neuron.swc', 'simple.swc']]
actual = ms.extract_dataframe(nrns, config)
assert_frame_equal(actual, expected)
# Test with a List[Path] argument
nrns = [Path(SWC_PATH, name) for name in ['Neuron.swc', 'simple.swc']]
actual = ms.extract_dataframe(nrns, config)
assert_frame_equal(actual, expected)
# Test without any neurite_type keys, it should pick the defaults
config = {'neurite': {'total_length_per_neurite': ['total']}}
actual = ms.extract_dataframe(nrns, config)
expected = pd.DataFrame(
columns=['name', 'neurite_type', 'total_total_length_per_neurite'],
data=[['Neuron', 'axon', 207.879752],
['Neuron', 'basal_dendrite', 418.432416],
['Neuron', 'apical_dendrite', 214.373046],
['Neuron', 'all', 840.685214],
['simple', 'axon', 15.000000],
['simple', 'basal_dendrite', 16.000000],
['simple', 'apical_dendrite', 0.000000],
['simple', 'all', 31.000000]])
assert_frame_equal(actual, expected)
def transform_package(mtype, files, components):
'''Put together header and list of data into one json output.
feature_list contains all the information about the data to be extracted:
features, feature_names, feature_components, feature_min, feature_max
'''
data_dict = transform_header(mtype)
neurons = load_neurons(files)
for comp in components:
params = PARAM_MAP[comp.name]
for feature in comp.features:
result = stats.fit_results_to_dict(
extract_data(neurons, feature.name, params),
feature.limits.min, feature.limits.max
)
# When the distribution is normal with sigma = 0
# it will be replaced with constant
if result['type'] == 'normal' and result['sigma'] == 0.0:
replace_result = OrderedDict((('type', 'constant'),
('val', result['mu'])))
result = replace_result
data_dict["components"][comp.name].update({feature.name: result})
return data_dict
def transform_package(mtype, files, components):
'''Put together header and list of data into one json output.
feature_list contains all the information about the data to be extracted:
features, feature_names, feature_components, feature_min, feature_max
'''
data_dict = transform_header(mtype)
neurons = load_neurons(files)
for comp in components:
params = PARAM_MAP[comp.name]
for feature in comp.features:
result = stats.fit_results_to_dict(
extract_data(neurons, feature.name, params),
feature.limits.min, feature.limits.max)
# When the distribution is normal with sigma = 0
# it will be replaced with constant
if result['type'] == 'normal' and result['sigma'] == 0.0:
replace_result = OrderedDict(
(('type', 'constant'), ('val', result['mu'])))
result = replace_result
data_dict["components"][comp.name].update({feature.name: result})
return data_dict
def __init__(self, path_root=""):
"""Create morphological analysis processor.
path_root: set path root for neuron morphology.
Default path_root is empty, use default path_root."""
if not (path_root):
self.path_root = {
"swc": "../../../reconstruction/traces/",
"imaris": "../../../reconstruction/stat/",
}
else:
self.path_root = path_root
self.neurons = nm.load_neurons(self.path_root["swc"])
self.files = {}
self.files["swc"], _ = getAllFiles(self.path_root["swc"], ends="swc")
self.files["imaris"] = {"stat": [], "soma": []}
shollFiles, _ = getAllFiles(self.path_root["imaris"])
for f in shollFiles:
if "Detailed.csv" in f:
# filter Detailed labled files
if "soma" in f:
self.files["imaris"]["soma"].append(f)
else:
self.files["imaris"]["stat"].append(f)
# Cache imaris stat data
self.imarisData = None
# Cache depth parameter stat data computed
self.depthData = None
# Cache different dormain sholl intersection
self.sholl_part_data = None
# Cache computed result of sholl_prat_data
self.sholl_part_stat = None
# Cache all morphological parameters of neurons
self.morphoData = dict(imaris=None, python=None, all=None)
def load_neurite_features(filepath):
"""Unpack relevant data into megadict"""
stuff = defaultdict(lambda: defaultdict(list))
nrns = nm.load_neurons(filepath)
# unpack data into arrays
for nrn in nrns:
for t in NEURITES_:
for feat in FEATURES:
stuff[feat][str(t).split(".")[1]].extend(nm.get(feat, nrn, neurite_type=t))
return stuff
def load_neurite_features(filepath):
'''Unpack relevant data into megadict'''
stuff = defaultdict(lambda: defaultdict(list))
nrns = nm.load_neurons(filepath)
# unpack data into arrays
for nrn in nrns:
for t in NEURITES_:
for feat in FEATURES:
stuff[feat][str(t).split('.')[1]].extend(
nm.get(feat, nrn, neurite_type=t))
return stuff
def extract_data(data_path, feature):
'''Loads a list of neurons, extracts feature
and transforms the fitted distribution in the correct format.
Returns the optimal distribution, corresponding parameters,
minimun and maximum values.
'''
population = nm.load_neurons(data_path)
feature_data = [nm.get(feature, n) for n in population]
feature_data = list(chain(*feature_data))
return stats.optimal_distribution(feature_data)
def extract_data(data_path, feature):
'''Loads a list of neurons, extracts feature
and transforms the fitted distribution in the correct format.
Returns the optimal distribution, corresponding parameters,
minimun and maximum values.
'''
population = nm.load_neurons(data_path)
feature_data = [nm.get(feature, n) for n in population]
feature_data = list(chain(*feature_data))
return stats.optimal_distribution(feature_data)
def main(datapath, config, output_file, is_full_config, as_population,
ignored_exceptions):
"""Main function that get statistics for morphologies.
Args:
datapath (str|Path): path to a morphology file or folder
config (str|Path): path to a statistics config file
output_file (str|Path): path to output the resulted statistics file
is_full_config (bool): should be statistics made over all possible features, modes, neurites
as_population (bool): treat ``datapath`` as directory of morphologies population
ignored_exceptions (list|tuple|None): exceptions to ignore when loading a morphology
"""
if is_full_config:
config = full_config()
else:
try:
config = get_config(config, EXAMPLE_CONFIG)
config = sanitize_config(config)
except ConfigError as e:
L.error(e)
raise
if ignored_exceptions is None:
ignored_exceptions = ()
ignored_exceptions = tuple(IGNORABLE_EXCEPTIONS[k]
for k in ignored_exceptions)
neurons = nm.load_neurons(get_files_by_path(datapath),
ignored_exceptions=ignored_exceptions)
results = {}
if as_population:
results[datapath] = extract_stats(neurons, config)
else:
for neuron in tqdm(neurons):
results[neuron.name] = extract_stats(neuron, config)
if not output_file:
print(
json.dumps(results,
indent=2,
separators=(',', ':'),
cls=NeuromJSON))
elif output_file.endswith('.json'):
with open(output_file, 'w') as f:
json.dump(results, f, cls=NeuromJSON)
else:
with open(output_file, 'w') as f:
csvwriter = csv.writer(f)
header = get_header(results)
csvwriter.writerow(header)
for line in generate_flattened_dict(header, dict(results)):
csvwriter.writerow(line)
def test_extract_dataframe_multiproc():
nrns = nm.load_neurons([Path(SWC_PATH, name)
for name in ['Neuron.swc', 'simple.swc']])
with warnings.catch_warnings(record=True) as w:
actual = ms.extract_dataframe(nrns, REF_CONFIG, n_workers=2)
expected = pd.read_csv(Path(DATA_PATH, 'extracted-stats.csv'), index_col=0, header=[0, 1])
assert_frame_equal(actual, expected)
with warnings.catch_warnings(record=True) as w:
actual = ms.extract_dataframe(nrns, REF_CONFIG, n_workers=os.cpu_count() + 1)
assert_equal(len(w), 1, "Warning not emitted")
assert_frame_equal(actual, expected)
def test_multiple_distr(filepath):
'''Runs the distribution fit for multiple distributions and returns
the optimal distribution along with the corresponding parameters.
'''
# load a neuron from an SWC file
population = nm.load_neurons(filepath)
# Create a list of basic distributions
distr_to_check = ('norm', 'expon', 'uniform')
# Get the soma radii of a population of neurons
soma_size = nm.get('soma_radii', population)
# Find the best fit distribution
return st.optimal_distribution(soma_size, distr_to_check)
def test_multiple_distr(filepath):
'''Runs the distribution fit for multiple distributions and returns
the optimal distribution along with the corresponding parameters.
'''
# load a neuron from an SWC file
population = nm.load_neurons(filepath)
# Create a list of basic distributions
distr_to_check = ('norm', 'expon', 'uniform')
# Get the soma radii of a population of neurons
soma_size = nm.get('soma_radii', population)
# Find the best fit distribution
return st.optimal_distribution(soma_size, distr_to_check)
def test_extract_dataframe_multiproc():
nrns = nm.load_neurons([Path(SWC_PATH, name)
for name in ['Neuron.swc', 'simple.swc']])
with warnings.catch_warnings(record=True) as w:
actual = ms.extract_dataframe(nrns, REF_CONFIG, n_workers=2)
expected = pd.read_csv(Path(DATA_PATH, 'extracted-stats.csv'), index_col=0)
# Compare sorted DataFrame since Pool.imap_unordered disrupted the order
assert_frame_equal(actual.sort_values(by=['name']).reset_index(drop=True),
expected.sort_values(by=['name']).reset_index(drop=True))
with warnings.catch_warnings(record=True) as w:
actual = ms.extract_dataframe(nrns, REF_CONFIG, n_workers=os.cpu_count() + 1)
assert_equal(len(w), 1, "Warning not emitted")
assert_frame_equal(actual.sort_values(by=['name']).reset_index(drop=True),
expected.sort_values(by=['name']).reset_index(drop=True))
from neurom.core.population import Population
from neurom import (core, load_neurons, iter_neurites, iter_sections,
load_neuron, fst)
from neurom.fst import get as fst_get
from neurom.fst import NEURITEFEATURES
from neurom.fst import _neuritefunc as nf
from neurom.core.types import tree_type_checker as _is_type
from neurom.exceptions import NeuroMError
_PWD = os.path.dirname(os.path.abspath(__file__))
DATA_PATH = os.path.join(_PWD, '../../../test_data')
NRN_FILES = [os.path.join(DATA_PATH, 'h5/v1', f)
for f in ('Neuron.h5', 'Neuron_2_branch.h5', 'bio_neuron-001.h5')]
NRNS = load_neurons(NRN_FILES)
NRN = NRNS[0]
POP = Population(NRNS)
NEURITES = (NeuriteType.axon,
NeuriteType.apical_dendrite,
NeuriteType.basal_dendrite,
NeuriteType.all)
def assert_items_equal(a, b):
nt.eq_(sorted(a), sorted(b))
def assert_features_for_neurite(feat, neurons, expected, exact=True):
for neurite_type, expected_values in expected.items():
sample_points, scipy.spatial.ConvexHull(node_data[:, :3]))
sample_points = sample_points[mask]
dstep = 1.
results = list(data_generator(node_data, edges, sample_points, dstep))
return numpy.asarray(results)
if __name__ == '__main__':
dirpath = sys.argv[1]
population = neurom.load_neurons(dirpath)
results_list = neurocover.parallel.multiprocessing_map(
apply_to_morphology, population)
f, ax = matplotlib.pylab.subplots(1, 2)
for i, morphology in enumerate(population):
name = morphology.name
coeffs, percs = results_list[i].T
ax[0].plot(coeffs, percs, label=name)
ax[0].legend()
dstep = 1.
from neurom.core.population import Population
from neurom import (core, load_neurons, iter_neurites, iter_sections,
load_neuron, fst)
from neurom.fst import get as fst_get
from neurom.fst import NEURITEFEATURES
from neurom.fst import _neuritefunc as nf
from neurom.core.types import tree_type_checker as _is_type
from neurom.exceptions import NeuroMError
_PWD = os.path.dirname(os.path.abspath(__file__))
DATA_PATH = os.path.join(_PWD, '../../../test_data')
NRN_FILES = [os.path.join(DATA_PATH, 'h5/v1', f)
for f in ('Neuron.h5', 'Neuron_2_branch.h5', 'bio_neuron-001.h5')]
NRNS = load_neurons(NRN_FILES)
NRN = NRNS[0]
POP = Population(NRNS)
NEURITES = (NeuriteType.axon,
NeuriteType.apical_dendrite,
NeuriteType.basal_dendrite,
NeuriteType.all)
def assert_items_equal(a, b):
nt.eq_(sorted(a), sorted(b))
def assert_features_for_neurite(feat, neurons, expected, exact=True):
for neurite_type, expected_values in expected.items():