def get_error_logs(behavior_session_id): conn = db.Database('visual_behavior_data') res = conn['sdk_validation']['error_logs'].find( {'behavior_session_id': behavior_session_id}) error_logs = pd.DataFrame(list(res)) conn.close() return error_logs
def get_session_stats(container_df, ophys_container_id, session_number): vb = db.Database('visual_behavior_data') behavior_session_uuid = get_uuid(container_df, ophys_container_id, session_number) stats = vb.behavior_data['summary'].find_one( {'behavior_session_uuid': behavior_session_uuid}) vb.close() return stats
def get_sessions_to_load(): vb = db.Database('visual_behavior_data') ms = db.Database('mouseseeks') vb_summary = vb.query('behavior_data', 'summary') ms_sessions = ms.query('db', 'behavior_session_log') merged = ms_sessions[['foraging_id' ]].merge(vb_summary[['behavior_session_uuid']], left_on='foraging_id', right_on='behavior_session_uuid', how='outer') merged['to_load'] = (pd.isnull( merged['behavior_session_uuid'])) & (~pd.isnull(merged['foraging_id'])) vb.close() ms.close() return merged.query('to_load == True')['foraging_id'].drop_duplicates()
def log_validation_results_to_mongo(behavior_session_id, validation_results): validation_results.update({'behavior_session_id': behavior_session_id}) validation_results.update( {'is_ophys': data_access_utilities.is_ophys(behavior_session_id)}) validation_results.update({'timestamp': str(datetime.datetime.now())}) conn = db.Database('visual_behavior_data') collection = conn['sdk_validation']['validation_results'] document = validation_results keys_to_check = ['behavior_session_id'] db.update_or_create(collection, document, keys_to_check, force_write=False) conn.close()
def log_summary_to_db(df_to_log): conn = db.Database('visual_behavior_data') for idx, row in df_to_log.iterrows(): entry = row.to_dict() db.update_or_create( conn['ophys_analysis']['event_triggered_responses'], db.clean_and_timestamp(entry), keys_to_check=[ 'ophys_experiment_id', 'cell_specimen_id', 'event_type', 'engagement_state' ]) conn.close()
def rewrite_record(uuid): pkl_path = db.get_pkl_path(uuid) data = pd.read_pickle(pkl_path) core_data = data_to_change_detection_core(data) trials = create_extended_dataframe(**core_data).drop( columns=['date', 'LDT_mode']) summary = summarize.session_level_summary(trials).iloc[0].to_dict() summary.update({'error_on_load': 0}) vb = db.Database('visual_behavior_data') db.update_or_create(vb['behavior_data']['summary'], db.simplify_entry(summary), ['behavior_session_uuid'], force_write=False) vb.close()
def log_error_to_mongo(behavior_session_id, failed_attribute, error_class, traceback): conn = db.Database('visual_behavior_data') entry = { 'timestamp': str(datetime.datetime.now()), 'sdk_version': allensdk.__version__, 'python_version': platform.python_version(), 'platform': platform.platform(), 'behavior_session_id': behavior_session_id, 'failed_attribute': failed_attribute, 'error_class': str(error_class), 'traceback': traceback } conn['sdk_validation']['error_logs'].insert_one(entry) conn.close()
def get_validation_results(behavior_session_id=None): conn = db.Database('visual_behavior_data') if behavior_session_id is None: res = conn['sdk_validation']['validation_results'].find({}) else: res = conn['sdk_validation']['validation_results'].find( {'behavior_session_id': behavior_session_id}) if res.count() > 0: ans = pd.DataFrame( list(res)).drop(columns=['_id']).set_index('behavior_session_id') conn.close() return ans else: conn.close() return pd.DataFrame()
def error_query(behavior_session_id, attribute): ''' query mongo for the most recent error log for a given session/attribute ''' query = { "behavior_session_id": int(behavior_session_id), "failed_attribute": attribute } conn = db.Database('visual_behavior_data') matching_errors = pd.DataFrame( list(conn['sdk_validation']['error_logs'].find(query))).drop( columns='_id') conn.close() return matching_errors.sort_values(by='timestamp').drop_duplicates( subset=['behavior_session_id'], keep='last')
def load_flashwise_summary(behavior_session_id=None): conn = db.Database('visual_behavior_data') collection = conn['behavior_analysis']['annotated_stimulus_presentations'] if behavior_session_id is None: # load all df = pd.DataFrame(list(collection.find({}))) else: # load data from one behavior session df = pd.DataFrame( list( collection.find( {'behavior_session_id': int(behavior_session_id)}))) conn.close() return df.sort_values(by=['behavior_session_id', 'flash_index'])
def main(bsid): running_data = load_running_df(bsid) df_sample = running_data.query('time > 300 and time < 600') print('generating spline fits for a range of n_knot_factors') n_knot_factors = [3,10,20,50,100] for n_knot_factor in n_knot_factors: print('n_knot_factor = {}'.format(n_knot_factor)) df_sample = utilities.apply_spline_regression(df_sample, n_knot_factor) fig,ax = utilities.make_visualization(df_sample.query('time >= 310 and time <=330'), n_knot_factors) plt.subplots_adjust(top=0.9) fig.suptitle('bsid = {}'.format(bsid)) fig.savefig( '/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/running_smoothing/sample_data_figures/bsid={}.png'.format(bsid), dpi=150 ) print('calculating cost function') optimization_results = utilities.calculate_cost_function(df_sample) optimal_f = utilities.calculate_optimal_knot_factor(optimization_results) document = {} document['behavior_session_id'] = bsid document['optimal_f'] = optimal_f # document['data_sample'] = df_sample.to_dict('records') document['optimization_results'] = optimization_results.to_dict('records') print('logging to db') conn = db.Database('visual_behavior_data') collection = conn['behavior_analysis']['running_smoothing'] db.update_or_create( collection, document, keys_to_check = ['behavior_session_id'], force_write=False ) print('done')
def to_mongo(data_to_log, display_level): '''log data to mongo''' conn = db.Database('visual_behavior_data') collection = conn['ophys_qc']['{}_qc_records'.format(display_level)] collection.insert_one(db.clean_and_timestamp(data_to_log)) conn.close()