def test_normalize_nedm_val_ahp_amp(self):
with open('tests/test_html_data_tables/example_html_table_exp_facts.html', mode='rb') as f:
exp_fact_table_text = f.read()
article_ob = m.Article.objects.create(title='asdf', pmid='456')
data_table_ob = m.DataTable.objects.create(table_html=exp_fact_table_text, article=article_ob)
data_source_ob = m.DataSource.objects.create(data_table=data_table_ob)
# create ephys concept maps
ephys_unit = m.Unit.objects.create(name=u'V', prefix = 'm')
ahp_amp_ephys_ob = m.EphysProp.objects.create(name='AHP amplitude', units = ephys_unit,
min_range = 0, max_range = 50)
ecm = m.EphysConceptMap.objects.create(dt_id='td-68', source=data_source_ob, ephys_prop=ahp_amp_ephys_ob, ref_text = u'blah (mV)')
neuron_ob = m.Neuron.objects.get_or_create(name='Other')[0]
ncm = m.NeuronConceptMap.objects.create(dt_id='th-2', source=data_source_ob, neuron=neuron_ob,
neuron_long_name='thalamus parafascicular nucleus')
input_value = -9
error_value = 1.0
nedm = m.NeuronEphysDataMap(neuron_concept_map = ncm, ephys_concept_map = ecm, dt_id = 'td-3',
source = data_source_ob, val = input_value, err = error_value)
normalized_dict = normalize_nedm_val(nedm)
expected_value = 9
expected_error = 1.0
self.assertEqual(normalized_dict['value'], expected_value)
self.assertEqual(normalized_dict['error'], expected_error)
def normalize_all_nedms():
"""Iterate through all neuroelectro NeuronEphysDataMap objects and normalize for differences in units"""
nedms = m.NeuronEphysDataMap.objects.filter(neuron_concept_map__times_validated__gt = 0)
nedms = nedms.exclude(source__data_table__irrelevant_flag = True)
nedm_count = nedms.count()
for i,nedm in enumerate(nedms):
prog(i, nedm_count)
norm_dict = normalize_nedm_val(nedm)
# print norm_dict
# print nedm.val_norm
if nedm.val_norm is None:
# if no existing normalized value for nedm in DB
nedm.val_norm = norm_dict['value']
nedm.err_norm = norm_dict['error']
nedm.save()
elif np.isclose(nedm.val, nedm.val_norm):
# if there is a normalized value, but it's the same as the unnormalized value
# so it may need to be updated
norm_value = norm_dict['value']
if norm_value is None:
# can't normalize value but there's an existing one in the database
# so keep it as long as it's in an appropriate range
if check_data_val_range(nedm.val_norm, nedm.ephys_concept_map.ephys_prop) is False:
print 'deleting pre-normalized value of %s because out of appropriate range for %s ' % (nedm.val_norm, nedm.ephys_concept_map.ephys_prop )
nedm.val_norm = None
nedm.err_norm = None
nedm.save()
pass
elif np.isclose(norm_value, nedm.val_norm):
# new normalized value same as old normalized value, so do nothing
pass
# nedm.err_norm = norm_dict['error']
# nedm.save()
else:
# save nedm value
nedm.val_norm = norm_value
nedm.err_norm = norm_dict['error']
nedm.save()
# normalizing basically failed for some reason
else:
# there's a normalized value but it's different from what the algorithm suggests, so it's likely manually added
# if existing normalized value is out of range
if check_data_val_range(nedm.val_norm, nedm.ephys_concept_map.ephys_prop) is False:
print 'deleting pre-normalized value of %s because out of appropriate range for %s ' % (nedm.val_norm, nedm.ephys_concept_map.ephys_prop )
nedm.val_norm = None
nedm.err_norm = None
nedm.save()
else:
nedm.err_norm = norm_dict['error']
nedm.save()
# after all the checks above, do a final check for if algorithmically normalzied value is in correct range
annotate_misnormalized_nedm(nedm)
def annotate_misnormalized_nedm(nedm):
''' if can't algorithmically normalize nedm value to something appropriate, and raw value is out of range,
leave a note in corresponding ecm in table'''
norm_dict = normalize_nedm_val(nedm)
if norm_dict['value'] is None and check_data_val_range(nedm.val, nedm.ephys_concept_map.ephys_prop) is False:
ecm = nedm.ephys_concept_map
normalizing_failed_note = 'Parsing failed to normalize ephys data'
if not ecm.note:
ecm.note = normalizing_failed_note
ecm.changed_by = m.get_robot_user()
ecm.save()
print 'adding failed normalizing note to %s with data table id %d' % (ecm.ephys_prop, ecm.source.data_table.pk)
def normalizeNedms():
nedm_list = m.NeuronEphysDataMap.objects.all()
for nedm in nedm_list:
if nedm.ephys_concept_map.ephys_prop.id in [8, 1, 13, 27]:
pass
elif nedm.val_norm:
continue
normalized_dict = normalize_nedm_val(nedm)
if normalized_dict["value"]:
nedm.val_norm = normalized_dict["value"]
nedm.err_norm = normalized_dict["error"]
# print [nedm.ephys_concept_map.ephys_prop, nedm.ephys_concept_map.ref_text, nedm.val, nedm.val_norm]
nedm.save()
def test_normalize_nedm_val_ir(self):
with open('tests/test_html_data_tables/example_html_table_exp_facts.html', mode='rb') as f:
exp_fact_table_text = f.read()
article_ob = m.Article.objects.create(title='asdf', pmid='456')
data_table_ob = m.DataTable.objects.create(table_html=exp_fact_table_text, article=article_ob)
data_source_ob = m.DataSource.objects.create(data_table=data_table_ob)
# create ephys concept maps
ephys_unit = m.Unit.objects.create(name=u'Ω', prefix = 'M')
ir_ephys_ob = m.EphysProp.objects.create(name='input resistance', units = ephys_unit)
ecm = m.EphysConceptMap.objects.create(dt_id='td-68', source=data_source_ob, ephys_prop=ir_ephys_ob, ref_text = u'blah (GΩ)')
neuron_ob = m.Neuron.objects.get_or_create(name='Other')[0]
ncm = m.NeuronConceptMap.objects.create(dt_id='th-2', source=data_source_ob, neuron=neuron_ob,
neuron_long_name='thalamus parafascicular nucleus')
input_value = .2
nedm = m.NeuronEphysDataMap(neuron_concept_map = ncm, ephys_concept_map = ecm, dt_id = 'td-3', source = data_source_ob, val = input_value)
normalized_dict = normalize_nedm_val(nedm)
expected_value = 200
self.assertEqual(normalized_dict['value'], expected_value)
