def __init__(self, data_central, id_robot, id_agent, num_episodes, cumulative=True, interval_print=5): self.data_central = data_central self.id_robot = id_robot self.cumulative = cumulative if self.cumulative: log_index = data_central.get_log_index() self.done_before = log_index.get_episodes_for_robot(id_robot, id_agent) self.num_episodes_done_before = len(self.done_before) self.num_episodes_todo = (num_episodes - self.num_episodes_done_before) logger.info('Preparing to do %d episodes (already done %d).' % (self.num_episodes_todo, self.num_episodes_done_before)) else: self.num_episodes_todo = num_episodes logger.info('Preparing to do %d episodes.' % self.num_episodes_todo) self.num_episodes_done = 0 self.num_observations = 0 self.num_observations_episode = 0 self.observations_per_episode = [] self.interval_print = interval_print self.tracker = InAWhile(interval_print) self.id_episodes = set() try: from compmake import progress progress('Simulating episodes', (0, self.num_episodes_todo)) except ImportError: pass
def test_hierarchy_flat(self): """ Testing basic case. """ init_progress_tracking(lambda _: None) self.assert_stack_len(0) progress('A', (0, 2)) self.assert_stack_len(1) progress('A', (1, 2)) self.assert_stack_len(1)
def test_hierarchy_flat(self): ''' Testing basic case. ''' init_progress_tracking(lambda stack:None) #@UnusedVariable self.assert_stack_len(0) progress('A', (0, 2)) self.assert_stack_len(1) progress('A', (1, 2)) self.assert_stack_len(1)
def mylongfunction(): N = 4 for i in range(N): progress('Task A', (i, N)) time.sleep(wait) for i in range(N): progress('Task B', (i, N)) time.sleep(wait)
def episode_done(self): self.num_episodes_done += 1 self.observations_per_episode.append(self.num_observations_episode) self.num_observations_episode = 0 try: from compmake import progress progress("Simulating episodes", (self.num_episodes_done, self.num_episodes_todo)) except ImportError: pass
def compute_saccade_stats(flydra_db_directory, samples, image, conditions): ''' Computes the stats of an image for the saccades that respect a set of conditions. db: FlydraDB directory samples: list of IDs condition: saccade table -> true/false selector ''' with safe_flydra_db_open(flydra_db_directory) as db: progress('Computing stats', (0, 2), 'First pass') # first compute the mean group_mean = Expectation() group_min = None group_max = None iter = saccades_iterate_image('computing mean', db, samples, image, conditions) for sample, sample_saccades, values in iter: #@UnusedVariable sample_mean = numpy.mean(values, axis=0) sample_min = numpy.min(values, axis=0) sample_max = numpy.max(values, axis=0) if group_min is None: group_min = sample_min group_max = sample_max else: group_min = numpy.minimum(sample_min, group_min) group_max = numpy.maximum(sample_max, group_max) group_mean.update(sample_mean, len(values)) num_samples = group_mean.num_samples group_mean = group_mean.get_value() group_var = Expectation() progress('Computing stats', (1, 2), 'Second pass') iter = saccades_iterate_image('computing var', db, samples, image, conditions) for sample, sample_saccades, values in iter: #@UnusedVariable err = values - group_mean sample_var = numpy.mean(numpy.square(err), axis=0) group_var.update(sample_var, len(values)) group_var = group_var.get_value() result = Stats(mean=group_mean, var=group_var, min=group_min, max=group_max, nsamples=num_samples) return result
def data_pass(name): for i, id in enumerate(samples): progress(name, (i, len(samples)), "Sample %s" % id) if not (db.has_sample(id) and db.has_table(id, image)): raise ValueError('No table "%s" for id %s' % (image, id)) rows = db.get_rows(id) data = db.get_table(id, image) select = rows[:]['linear_velocity_modulus'] > 0.1 #select = condition(rows) values = data[select]['value'] yield id, values db.release_table(data) db.release_table(rows)
def saccades_iterate_image(name, db, samples, image, conditions): ''' Iterates over the values of an image corresponding to the saccades that respect a given condition. yields sample, saccades, image_values ''' num_saccades = 0 num_selected = 0 for i, id in enumerate(samples): progress(name, (i, len(samples)), "Sample %s" % id) if not db.has_saccades(id): raise ValueError('No saccades for %s' % id) if not (db.has_sample(id) and db.has_table(id, image)): raise ValueError('No table "%s" for id %s' % (image, id)) saccades_table = db.get_saccades(id) saccades = add_position_information(saccades_table) data = db.get_table(id, image) # computes and of all conditions select = reduce(numpy.logical_and, map(lambda c:c(saccades), conditions)) values = data[select]['value'] if len(values) == 0: print ("No saccades selected for %s (out of %d)." % (id, len(saccades))) else: yield id, saccades[select], values num_saccades += len(select) num_selected += (select * 1).sum() db.release_table(data) db.release_table(saccades_table) ratio = 100.0 / num_saccades * num_selected print "Selected %.2f %% of saccades" % ratio
def test_hierarchy_flat2(self): data = {} def mystack(x): data['stack'] = x init_progress_tracking(mystack) self.assert_stack_len(0) progress('A', (0, 2)) self.assert_stack_len(1) progress('B', (0, 2)) self.assert_stack_len(2) progress('B', (1, 2)) self.assert_stack_len(2) progress('A', (1, 2)) self.assert_stack_len(1)
def compute_mean_generic(db, samples, image, operator): ''' db: FlydraDB directory samples: list of IDs ''' db = FlydraDB(db, False) results = { 'samples': {} } ex = Expectation() for i, id in enumerate(samples): progress('Computing mean %s' % image, (i, len(samples)), "Sample %s" % id) if not (db.has_sample(id) and db.has_table(id, image)): raise ValueError('No table "%s" for id %s' % (image, id)) data = db.get_table(id, image) values = data[:]['value'] this = operator(values) # print "id: %s len: %d %d" % (id, len(data), len(values)) ex.update(this, len(data)) results['samples'][id] = this db.release_table(data) results['all'] = ex.get_value() db.close() return results
def mylongfunction(): directories = ['a', 'b', 'c', 'd', 'e'] n = len(directories) for i, d in enumerate(directories): progress('Processing directories (first)', (i, n), 'Directory %s' % d) N = 3 for k in range(N): progress('Processing files (a)', (k, N), 'file #%d' % k) time.sleep(wait) for i, d in enumerate(directories): progress('Processing directories (second)', (i, n), 'Directory %s' % d) N = 3 for k in range(N): progress('Processing files (b)', (k, N), 'file #%d' % k) time.sleep(wait)
def mylongfunction(): directories = ['a', 'b', 'c', 'd', 'e'] n = len(directories) for i, d in enumerate(directories): progress('Processing directories (first)', (i, n), 'Directory %s' % d) N = 3 for k in range(N): progress('Processing files (a)', (k, N), 'file #%d' % k) time.sleep(1) for i, d in enumerate(directories): progress('Processing directories (second)', (i, n), 'Directory %s' % d) N = 3 for k in range(N): progress('Processing files (b)', (k, N), 'file #%d' % k) time.sleep(1)
def compute_environment_autocorrelation(db, samples, image, maxlag=50): nsensors = 1398 results = numpy.ndarray(shape=(nsensors, 2 * maxlag + 1)) db = FlydraDB(db, create=False) block_size = 50 num_blocks = int(numpy.ceil(nsensors * 1.0 / block_size)) for b in range(num_blocks): start = block_size * b stop = min(start + block_size, nsensors) progress('Computing autocorrelation', (b, num_blocks)) data = [[] for i in range(nsensors)] for k, sample in enumerate(samples): progress('getting data', (k, len(samples)), sample) table = db.get_table(sample, image) chunk = (table[:]['value'][:, start:stop]).copy() for j, i in enumerate(range(start, stop)): data[i].append(chunk[:, j]) db.release_table(table) for j, i in enumerate(range(start, stop)): progress('Computing correlation', (j, stop - start)) x = numpy.concatenate(data[i]) corr, lags = xcorr(x, maxlag=maxlag) assert(len(lags) == 2 * maxlag + 1) results[i, :] = corr db.close() data = { 'results': results, 'lags': lags } return data
def enumerate_data(db, samples, interval_function, image, signal, signal_component, signal_op, what='enumerate_data'): for k, id in enumerate(samples): progress(what, (k, len(samples)), "Sample %s" % id) if not db.has_rows(id): logger.warning('Could not find rows table for %s; skipping.' % (id)) continue if not db.has_table(id, image): logger.warning('Could not find table "%s" for %s; skipping.' % (image, id)) continue rows_table = db.get_rows(id) image_table = db.get_table(id, image) image_values = image_table[:]['value'] try: interval = interval_function(db, id, rows_table) except Exception as e: logger.warning('Cannot compute interval for sample %s: %s '\ % (id, e)) db.release_table(rows_table) continue if numpy.logical_not(interval).all(): logger.warning('Sample %s with interval function "%s" ' 'gives empty subset; skipping. ' % (id, interval_function.__name__)) db.release_table(rows_table) continue if False: percentage = numpy.mean(interval * 1.0) * 100 logger.info('Sample %s: function "%s" selects %.1f%% of data.' % (id, interval_function.__name__, percentage)) # subset everything image_values = image_values[interval] rows = rows_table[interval] # get the action vector actions = extract_signal(rows, signal, signal_component) # remove the tails actions, removed_percentage = cut_tails(actions, percent=0.3) if removed_percentage > 1: logger.warning('Too much tail removed (%.3f%%) for %s/%s,' % (removed_percentage, id, signal)) actions = signal_op(actions) yield id, actions, image_values db.release_table(rows_table) db.release_table(image_table)
def compute_stats(flydra_db_directory, samples, image): ''' Computes the stats of an image. *db* FlydraDB directory *samples* list of IDs image: name of a table ''' with safe_flydra_db_open(flydra_db_directory) as db: def data_pass(name): for i, id in enumerate(samples): progress(name, (i, len(samples)), "Sample %s" % id) if not (db.has_sample(id) and db.has_table(id, image)): raise ValueError('No table "%s" for id %s' % (image, id)) rows = db.get_rows(id) data = db.get_table(id, image) select = rows[:]['linear_velocity_modulus'] > 0.1 #select = condition(rows) values = data[select]['value'] yield id, values db.release_table(data) db.release_table(rows) progress('Computing stats', (0, 2), 'First pass') # first compute the mean group_mean = Expectation() group_min = None group_max = None for sample, values in data_pass('computing mean'): #@UnusedVariable sample_mean = numpy.mean(values, axis=0) sample_min = numpy.min(values, axis=0) sample_max = numpy.max(values, axis=0) if group_min is None: group_min = sample_min group_max = sample_max else: group_min = numpy.minimum(sample_min, group_min) group_max = numpy.maximum(sample_max, group_max) group_mean.update(sample_mean, len(values)) num_samples = group_mean.num_samples group_mean = group_mean.get_value() group_var = Expectation() progress('Computing stats', (1, 2), 'Second pass') for sample, values in data_pass('computing var'): #@UnusedVariable err = values - group_mean sample_var = numpy.mean(numpy.square(err), axis=0) group_var.update(sample_var, len(values)) group_var = group_var.get_value() return Stats(mean=group_mean, var=group_var, min=group_min, max=group_max, nsamples=num_samples)
def bet_on_flies(flydra_db_directory, samples, image, saccades_set): kernels = {} for degrees in [15, 30, 45, 60, 90, 180]: # kernels['mf%d' % degrees] = create_matched_filter(degrees, [-45, 0], False) kernels['mf%d' % degrees] = create_matched_filter(degrees, [-20, 20], False) #kernels['pf%d' % degrees] = create_matched_filter(degrees, True) results = {} with safe_flydra_db_open(flydra_db_directory) as db: conditions = [saccades_set.args] # looking for the optimal dividing plane ex = {-1: Expectation(), +1: Expectation()} for sample, sample_saccades, image_values in saccades_iterate_image(#@UnusedVariable 'computing optimal kernel', db, samples, image, conditions): for s in [-1, +1]: values = image_values[sample_saccades['sign'] == s] if len(values) > 0: ex[s].update(values.mean(axis=0), len(values)) kernels['optimal'] = ex[+1].get_value() - ex[-1].get_value() dir = '${SNPENV_DATA}/flydra_db/out/saccade_view_joint_analysis/lasvegas/images/' others = { 'center': 'lasvegas_contrast_w_posts_center:optimal:kernel.pickle', 'border': 'lasvegas_contrast_w_posts_border:optimal:kernel.pickle', 'allsac': 'lasvegas_contrast_w_posts_allsac:optimal:kernel.pickle' } for name, f in others.items(): filename = os.path.expandvars(os.path.join(dir, f)) kernel = cPickle.load(open(filename, 'rb')) #mask_deg = create_matched_filter(75, [-60, 60],True) mask_deg = create_matched_filter(75, [-90, 90], True) kernel = kernel * numpy.abs(mask_deg) kernels['common_%s' % name] = kernel for i, kernel_name in enumerate(kernels.keys()): progress('betting', (i, len(kernels)), kernel_name) kernel = kernels[kernel_name] signs = [] response = [] overlap = [] for sample, sample_saccades, image_values in saccades_iterate_image(#@UnusedVariable 'computing response', db, samples, image, conditions): s = sample_saccades[:]['sign'] signs.extend(s) for i in range(len(image_values)): response_i = (image_values[i, :] * kernel).sum() response.append(response_i) overlap_i = (image_values[i, :] * numpy.abs(kernel)).sum() overlap.append(overlap_i) signs = numpy.array(signs) response = numpy.array(response) results[kernel_name] = { 'signs': signs, 'response': response, 'kernel': kernel, 'overlap': overlap} return results