def get_error_logs(behavior_session_id):
conn = db.Database('visual_behavior_data')
res = conn['sdk_validation']['error_logs'].find(
{'behavior_session_id': behavior_session_id})
error_logs = pd.DataFrame(list(res))
conn.close()
return error_logs
def get_session_stats(container_df, ophys_container_id, session_number):
vb = db.Database('visual_behavior_data')
behavior_session_uuid = get_uuid(container_df, ophys_container_id,
session_number)
stats = vb.behavior_data['summary'].find_one(
{'behavior_session_uuid': behavior_session_uuid})
vb.close()
return stats
def get_sessions_to_load():
vb = db.Database('visual_behavior_data')
ms = db.Database('mouseseeks')
vb_summary = vb.query('behavior_data', 'summary')
ms_sessions = ms.query('db', 'behavior_session_log')
merged = ms_sessions[['foraging_id'
]].merge(vb_summary[['behavior_session_uuid']],
left_on='foraging_id',
right_on='behavior_session_uuid',
how='outer')
merged['to_load'] = (pd.isnull(
merged['behavior_session_uuid'])) & (~pd.isnull(merged['foraging_id']))
vb.close()
ms.close()
return merged.query('to_load == True')['foraging_id'].drop_duplicates()
def log_validation_results_to_mongo(behavior_session_id, validation_results):
validation_results.update({'behavior_session_id': behavior_session_id})
validation_results.update(
{'is_ophys': data_access_utilities.is_ophys(behavior_session_id)})
validation_results.update({'timestamp': str(datetime.datetime.now())})
conn = db.Database('visual_behavior_data')
collection = conn['sdk_validation']['validation_results']
document = validation_results
keys_to_check = ['behavior_session_id']
db.update_or_create(collection, document, keys_to_check, force_write=False)
conn.close()
def log_summary_to_db(df_to_log):
conn = db.Database('visual_behavior_data')
for idx, row in df_to_log.iterrows():
entry = row.to_dict()
db.update_or_create(
conn['ophys_analysis']['event_triggered_responses'],
db.clean_and_timestamp(entry),
keys_to_check=[
'ophys_experiment_id', 'cell_specimen_id', 'event_type',
'engagement_state'
])
conn.close()
def rewrite_record(uuid):
pkl_path = db.get_pkl_path(uuid)
data = pd.read_pickle(pkl_path)
core_data = data_to_change_detection_core(data)
trials = create_extended_dataframe(**core_data).drop(
columns=['date', 'LDT_mode'])
summary = summarize.session_level_summary(trials).iloc[0].to_dict()
summary.update({'error_on_load': 0})
vb = db.Database('visual_behavior_data')
db.update_or_create(vb['behavior_data']['summary'],
db.simplify_entry(summary), ['behavior_session_uuid'],
force_write=False)
vb.close()
def log_error_to_mongo(behavior_session_id, failed_attribute, error_class,
traceback):
conn = db.Database('visual_behavior_data')
entry = {
'timestamp': str(datetime.datetime.now()),
'sdk_version': allensdk.__version__,
'python_version': platform.python_version(),
'platform': platform.platform(),
'behavior_session_id': behavior_session_id,
'failed_attribute': failed_attribute,
'error_class': str(error_class),
'traceback': traceback
}
conn['sdk_validation']['error_logs'].insert_one(entry)
conn.close()
def get_validation_results(behavior_session_id=None):
conn = db.Database('visual_behavior_data')
if behavior_session_id is None:
res = conn['sdk_validation']['validation_results'].find({})
else:
res = conn['sdk_validation']['validation_results'].find(
{'behavior_session_id': behavior_session_id})
if res.count() > 0:
ans = pd.DataFrame(
list(res)).drop(columns=['_id']).set_index('behavior_session_id')
conn.close()
return ans
else:
conn.close()
return pd.DataFrame()
def error_query(behavior_session_id, attribute):
'''
query mongo for the most recent error log for a given session/attribute
'''
query = {
"behavior_session_id": int(behavior_session_id),
"failed_attribute": attribute
}
conn = db.Database('visual_behavior_data')
matching_errors = pd.DataFrame(
list(conn['sdk_validation']['error_logs'].find(query))).drop(
columns='_id')
conn.close()
return matching_errors.sort_values(by='timestamp').drop_duplicates(
subset=['behavior_session_id'], keep='last')
def load_flashwise_summary(behavior_session_id=None):
conn = db.Database('visual_behavior_data')
collection = conn['behavior_analysis']['annotated_stimulus_presentations']
if behavior_session_id is None:
# load all
df = pd.DataFrame(list(collection.find({})))
else:
# load data from one behavior session
df = pd.DataFrame(
list(
collection.find(
{'behavior_session_id': int(behavior_session_id)})))
conn.close()
return df.sort_values(by=['behavior_session_id', 'flash_index'])
def main(bsid):
running_data = load_running_df(bsid)
df_sample = running_data.query('time > 300 and time < 600')
print('generating spline fits for a range of n_knot_factors')
n_knot_factors = [3,10,20,50,100]
for n_knot_factor in n_knot_factors:
print('n_knot_factor = {}'.format(n_knot_factor))
df_sample = utilities.apply_spline_regression(df_sample, n_knot_factor)
fig,ax = utilities.make_visualization(df_sample.query('time >= 310 and time <=330'), n_knot_factors)
plt.subplots_adjust(top=0.9)
fig.suptitle('bsid = {}'.format(bsid))
fig.savefig(
'/allen/programs/braintv/workgroups/nc-ophys/visual_behavior/running_smoothing/sample_data_figures/bsid={}.png'.format(bsid),
dpi=150
)
print('calculating cost function')
optimization_results = utilities.calculate_cost_function(df_sample)
optimal_f = utilities.calculate_optimal_knot_factor(optimization_results)
document = {}
document['behavior_session_id'] = bsid
document['optimal_f'] = optimal_f
# document['data_sample'] = df_sample.to_dict('records')
document['optimization_results'] = optimization_results.to_dict('records')
print('logging to db')
conn = db.Database('visual_behavior_data')
collection = conn['behavior_analysis']['running_smoothing']
db.update_or_create(
collection,
document,
keys_to_check = ['behavior_session_id'],
force_write=False
)
print('done')
def to_mongo(data_to_log, display_level):
'''log data to mongo'''
conn = db.Database('visual_behavior_data')
collection = conn['ophys_qc']['{}_qc_records'.format(display_level)]
collection.insert_one(db.clean_and_timestamp(data_to_log))
conn.close()