def test_zero_frame(self):
try:
ex = multiworm.Experiment(DATA_DIR / 'bad_framezero')
except multiworm.core.MWTDataError:
pass
else:
self.fail("Didn't raise error on malformed data with a frame 0")
def test_missing_summary(self):
try:
ex = multiworm.Experiment(DATA_DIR / 'bad_empty')
except multiworm.core.MWTDataError as e:
pass
else:
self.fail("Didn't raise error despite no summary file")
def test_dupe_summary(self):
try:
ex = multiworm.Experiment(DATA_DIR / 'bad_twosummary')
except multiworm.core.MWTSummaryError as e:
pass
else:
self.fail("Didn't raise error with ambiguous summary file")
def test_check_is_dir(self):
try:
ex = multiworm.Experiment(SYNTH1 / 'test_blobsfile.png')
except multiworm.core.MWTDataError:
self.fail('Overly specific error raised')
except IOError as e:
self.assertIn('directory', str(e))
else:
self.fail("Didn't even mention the folder isn't there")
def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(description='Get basic information '
'about a particular blob.')
parser.add_argument('data_set', help='The location of the data set.')
parser.add_argument(
'time',
type=float,
help='The time to display. '
'Coerced to the nearest available image if no exact match.')
args = parser.parse_args(argv[1:])
#args.data_set = where.where(args.data_set)
experiment = multiworm.Experiment(experiment_id=args.data_set)
experiment.load_summary()
# find the closest still to the given time
time = find_nearest(list(experiment.image_files.keys()), args.time)
print("- Found image at {0:.2f} s ({1:+.2f} s relative to specified "
"time)".format(time, time - args.time))
image_file = experiment.image_files[time]
print(image_file)
img = mpimg.imread(str(image_file))
# find the closest frame to the derived still time
frame = find_nearest_index(experiment.frame_times, time) + 1
frame_time = experiment.frame_times[frame - 1]
print("- Nearest frame at {0:.2f} s ({1:+.2f} s "
"relative to image)".format(frame_time, frame_time - time))
bids = experiment.blobs_in_frame(frame)
print("- {0} blobs tracked on frame {1}".format(len(bids), frame))
outlines = []
parser = frame_parser_spec(frame)
for bid in bids:
blob = experiment.parse_blob(bid, parser)
if blob['contour_encode_len'][0]:
outline = blob_reader.decode_outline(
blob['contour_start'][0],
blob['contour_encode_len'][0],
blob['contour_encoded'][0],
)
outlines.append(outline)
f, ax = plt.subplots()
ax.imshow(img.T, cmap=plt.cm.Greys_r)
for outline in outlines:
ax.plot(*outline.T)
plt.show()
return
def test_nonexistent_folder(self):
try:
ex = multiworm.Experiment(DATA_DIR /
'guaranteedtohopefullynotbethere')
except multiworm.core.MWTDataError:
self.fail('Overly specific error raised')
except IOError as e:
self.assertIn('exist', str(e))
else:
self.fail("Didn't even mention the folder isn't there")
def test_callback(self):
class StateThing(object):
def __init__(self):
self.progress = -1
def __call__(self, progress):
assert progress >= self.progress
self.progress = progress
ex = multiworm.Experiment(SYNTH1, callback=StateThing())
def test_empty_fail(self):
try:
multiworm.Experiment()
except Exception as e:
if not isinstance(e, ValueError):
self.fail('raised some unexpected error')
if not all(x in str(e)
for x in ['experiment_id', 'must', 'provided']):
self.fail('error message unexpected')
else:
self.fail('experiment constructor worked with no arguments')
def main():
parser = argparse.ArgumentParser(description='')
parser.add_argument('test_set')
args = parser.parse_args()
plate = multiworm.Experiment(TEST_DATA_SETS[args.test_set])
plate.add_summary_filter(multiworm.filters.summary_lifetime_minimum(120))
plate.load_summary()
plate.add_filter(multiworm.filters.relative_move_minimum(2))
plate.load_blobs()
print(len(plate.blobs_data))
def test_dataroot_only_fail(self):
try:
multiworm.Experiment(data_root=DATA_DIR)
except Exception as e:
if not isinstance(e, ValueError):
self.fail('raised some unexpected error')
if not all(x in str(e)
for x in ['experiment_id', 'must', 'provided']):
self.fail('error message unexpected')
else:
self.fail(
'experiment constructor allowed data-root only without erroring'
)
def main():
plate = multiworm.Experiment('/home/projects/worm_movement/Data/MWT_RawData/20130702_135704')
plate.add_summary_filter(multiworm.filters.summary_lifetime_minimum(int(sys.argv[1])))
plate.load_summary()
crude_set = set(k for k in six.iterkeys(plate.bs_mapping))
print('Crude blobs from filtered summary :', len(crude_set))
print('Known good blobs (from Mongo) :', len(known_good))
print('Known good blobs & found blobs :', len(known_good & crude_set))
missing = known_good - crude_set
print('Blobs missing :', len(missing))
print(known_good - crude_set)
def main():
parser = argparse.ArgumentParser(description='')
parser.add_argument('test_set')
args = parser.parse_args()
plate = multiworm.Experiment(TEST_DATA_SETS[args.test_set])
plate.add_summary_filter(multiworm.filters.summary_lifetime_minimum(120))
plate.load_summary()
ids = []
t = time.time()
for blob in itertools.islice(plate.good_blobs(), 50):
now = time.time()
t, tdelta = now, now - t
bid, bdata = blob
print(memprint(),
'ID: {0:5d}, t: {1:7.1f} ms'.format(bid, tdelta * 1000))
ids.append(bid)
blob, bdata = None, None
print(ids)
def test_pathlib(self):
ex = multiworm.Experiment(SYNTH1)
def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(description='Get basic information '
'about a particular blob.')
parser.add_argument('data_set', help='The location of the data set.')
parser.add_argument('blob_id',
type=int,
help='The blob ID in the '
'data set to summarize.')
parser.add_argument('-ht', '--head-and-tail', action='store_true')
parser.add_argument('--xy',
action='store_true',
help='Plot X and Y '
'coordinates for the blob')
parser.add_argument(
'--smooth',
nargs='+',
help='Smooth the '
'X-Y values. Must provide method (e.g. "sgolay"), and the '
'appropriate number of parameters for the filter.')
parser.add_argument('--spec', action='store_true', help='Spectogram')
#parser.add_argument('--show', action='store_true', help='Try to show the blob using images')
parser.add_argument('--dist',
action='store_true',
help='Distribution of '
'steps')
parser.add_argument('--speeds',
action='store_true',
help='Distribution '
'of speeds (requires --smooth ...)')
parser.add_argument('--frames',
type=int,
nargs=2,
help='Start/stop frames')
parser.add_argument('--subsample',
type=int,
default=1,
help='Subsample speed by this many frames')
parser.add_argument('--waldo',
action='store_true',
help='Pull data from '
'Waldo-processed files')
parser.add_argument('--noshow',
action='store_true',
help="Don't show "
"the plot")
args = parser.parse_args()
#args.data_set = where.where(args.data_set)
experiment = multiworm.Experiment(experiment_id=args.data_set)
experiment.load_summary()
if args.blob_id not in experiment.bs_mapping:
print('Blob ID {0} not found.'.format(args.blob_id), file=sys.stderr)
sys.exit(1)
ALL_METHODS = list(six.iterkeys(SMOOTH_METHODS)) + STOCK_METHODS
if args.smooth and args.smooth[0] not in ALL_METHODS:
print('Smoothing method "{}" not valid. Must be one of: {}'.format(
args.smooth[0], ', '.join(ALL_METHODS)),
file=sys.stderr)
sys.exit(1)
file_no, offset = experiment.summary[[
'file_no', 'offset'
]][experiment.bs_mapping[args.blob_id]]
if args.head_and_tail:
for line in experiment.parse_blob(args.blob_id, head_and_tail):
print(line, end='')
return
blob = experiment.parse_blob(args.blob_id)
if blob is None:
print("Blob ID {} exists, but has no data.".format(args.blob_id),
file=sys.stderr)
return
print('Data in blobs file number {0}, starting at byte {1}'.format(
file_no, offset))
print('Path: {0}'.format(experiment.blobs_files[file_no]))
print(' {0:^25s} | {1:^30s} '.format('Field', 'Data'))
print(' ' + '-' * 65)
life_s = blob['time'][-1] - blob['time'][0]
life_f = blob['frame'][-1] - blob['frame'][0]
fld('Lifetime (s, frames)', life_s, life_f)
fld('Time Range (s)', blob['time'][0], blob['time'][-1], joiner=' - ')
fld('Frame Range', blob['frame'][0], blob['frame'][-1], joiner=' - ')
fld('Found at', *blob['centroid'][0])
fld('Lost at', *blob['centroid'][-1])
if args.xy or args.spec or args.dist or args.smooth or args.waldo:
import matplotlib.pyplot as plt
if args.waldo:
times, centroid = waldo_pull(os.path.basename(args.data_set),
args.blob_id)
if centroid is None:
print("Blob ID {} exists, but has no Waldo data.".format(
args.blob_id),
file=sys.stderr)
return
else:
centroid = excise_frames(
blob, *args.frames) if args.frames else blob['centroid']
if args.spec:
spectrogram(centroid)
elif args.dist:
step_distribution(centroid)
elif args.smooth and args.speeds:
f = plt.figure()
ax_x = plt.subplot2grid((3, 2), (0, 0))
ax_y = plt.subplot2grid((3, 2), (1, 0), sharex=ax_x)
ax_speed = plt.subplot2grid((3, 2), (2, 0), sharex=ax_x)
ax_distspeed = plt.subplot2grid((3, 2), (0, 1), rowspan=3)
smooth_method, smooth_params = args.smooth[0], args.smooth[1:]
xy = list(zip(*centroid))
print(xy)
xy_smoothed = [
smooth(smooth_method, c, *smooth_params) for c in xy
]
for ax, c, c_smoothed in zip([ax_x, ax_y], xy, xy_smoothed):
ax.plot(c, color='blue', alpha=0.5)
ax.plot(c_smoothed, lw=2, color='green')
dxy = np.diff(np.array(xy_smoothed)[..., ::args.subsample], axis=1)
print(len(dxy), len(xy_smoothed))
#import pdb;pdb.set_trace()
ds = np.linalg.norm(dxy, axis=0)
ax_speed.plot(ds)
#bins = np.ceil(2 * len(ds)**(1/3)) # Rice's Rule
bins = np.ceil(np.ptp(ds) * len(ds)**(1 / 3) /
(2 * IQR(ds))) # Freedman–Diaconis' choice
ax_distspeed.hist(ds,
bins,
histtype='stepfilled',
alpha=0.5,
normed=True)
decades = range(10, 100, 10)
deciles = np.percentile(ds, decades)
print("\n{:>7s} | {:<s}".format('%ile', 'Speed (px/frame)'))
print(" -------------------")
for pct, pctile in zip(decades, deciles):
print("{:>7.0f} | {:6.3f}".format(pct, pctile))
#ax_distspeed.set_yscale('log')
elif args.smooth:
f, axs = plt.subplots(2, sharex=True)
for ax, data in zip(axs, zip(*centroid)):
smooth_method, smooth_params = args.smooth[0], args.smooth[1:]
data_smoothed = smooth(smooth_method, data, *smooth_params)
ax.plot(data, color='blue', alpha=0.5)
ax.plot(data_smoothed, lw=2, color='green')
else:
f, axs = plt.subplots(2, sharex=True)
for ax, data in zip(axs, zip(*centroid)):
ax.plot(data, color='blue')
if not args.noshow:
plt.show()
def setUp(self):
self.ex = multiworm.Experiment(SYNTH1)
def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(description='Get basic information '
'about an experiment.')
parser.add_argument(
'data_set',
help='The location of the data set.' #' If names specified in a lookup '
#'file can be selected with a prefix of {0}.'
# .format(where.named_location).replace('%', '%%')
)
parser.add_argument(
'-n',
'--noise',
action='store_true',
help='Estimate the amount of noise present in the centroid data. '
'Plot available.')
parser.add_argument('-l',
'--longest',
type=int,
default=10,
help='Display the longest-lived blobs')
parser.add_argument('-p',
'--plot',
action='store_true',
help='Show a plot (if supported by another command)')
parser.add_argument('-w',
'--waldo',
action='store_true',
help='Get some Waldo data')
args = parser.parse_args()
#args.data_set = where.where(args.data_set)
if args.waldo:
waldo_data(os.path.basename(args.data_set))
else:
experiment = multiworm.Experiment(experiment_id=args.data_set)
experiment.load_summary()
print('Experiment summary file: {}'.format(experiment.summary_file))
print('Number of blobs : {}'.format(len(experiment.summary)))
if args.noise:
mean_xy, std_dev_xy = assess_noise(experiment, plot=args.plot)
x_stats, y_stats = zip(mean_xy, std_dev_xy)
print('X mean of means/stddevs: {:0.3e} \u00B1 {:0.3e}'.format(
*x_stats))
print('Y mean of means/stddevs: {:0.3e} \u00B1 {:0.3e}'.format(
*y_stats))
if args.longest:
print(' {:>5s} | {}'.format('ID', 'Life (s)'))
print(' {:->7s}+{:-<10s}'.format('', ''))
for bid, lifespan in longest_10(experiment, args.longest):
print(' {:5d} | {:7.2f}'.format(bid, lifespan))
if args.plot:
import matplotlib.pyplot as plt
plt.show()
def setUp(self):
self.ex = multiworm.Experiment(SYNTH1)
self.blob = self.ex[1]
def test_strpath(self):
ex = multiworm.Experiment(str(SYNTH1))
def main():
parser = argparse.ArgumentParser(description='Get basic information '
'about an experiment.')
parser.add_argument(
'data_set',
help='The location of the data set. If '
'names specified in a lookup file can be selected with a prefix of '
'{0}.'.format(where.named_location).replace('%', '%%'))
parser.add_argument('-l',
'--limit',
type=int,
help='Limit the number '
'of blobs to process')
parser.add_argument('-p',
'--plot',
action='store_true',
help='Show a '
'plot (if supported by another command)')
parser.add_argument('-j',
'--json',
action='store_true',
help='Dump '
'data to a JSON.')
args = parser.parse_args()
args.data_set = where.where(args.data_set)
experiment = multiworm.Experiment(args.data_set)
experiment.load_summary()
experiment.add_summary_filter(
multiworm.filters.summary_lifetime_minimum(120))
experiment.add_filter(multiworm.filters.relative_move_minimum(2))
if args.limit:
blob_gen = itertools.islice(experiment.good_blobs(), args.limit)
else:
blob_gen = experiment.good_blobs()
speed_config = {
'smoothing': ('sgolay', 71, 5),
'percentiles': [10, 20, 50, 80, 90],
}
noise_est = multiworm.analytics.NoiseEstimator()
speed_est = multiworm.analytics.SpeedEstimator(**speed_config)
analyzer = multiworm.analytics.ExperimentAnalyzer()
analyzer.add_analysis_method(noise_est)
analyzer.add_analysis_method(speed_est)
analyzer.analyze(blob_gen)
data = analyzer.results()
if args.json:
print(json.dumps(data, indent=4))
else:
print('Mean X/Y : ', data['noise']['mean_xy'])
print('Standard Deviation X/Y: ', data['noise']['std_dev_xy'])
if args.plot:
import matplotlib.pyplot as plt
f, axs = plt.subplots(ncols=2)
f.suptitle(
'Normal summary statistics for frame-by-frame centroid steps')
for ax, data, title in zip(
axs, [data['noise']['means'], data['noise']['std_devs']],
['X/Y means', 'X/Y stddev']):
ax.scatter(*zip(*data))
ax.set_title(title)
ax.axvline()
ax.axhline()
ax.axis('equal')
axs[1].set_xlim(left=0)
axs[1].set_ylim(bottom=0)
plt.show()
def test_strroot_and_id(self):
ex = multiworm.Experiment(
data_root=str(DATA_DIR),
experiment_id='synth1',
)
def test_custom_id(self):
my_id = 'peterspeppers'
ex = multiworm.Experiment(fullpath=SYNTH1, experiment_id=my_id)
self.assertEquals(ex.id, my_id)
