コード例 #1
0
def add_old_full_texts():
    #article_list = m.Article.objects.filter(datatable__datasource__neuronconceptmap__isnull = False).distinct()
    article_list = m.Article.objects.filter(datatable__datasource__neuronephysdatamap__isnull = False).distinct()
    path = "K:\Full_text_dir\Neuro_full_texts"
    publisher = "Highwire"
    os.chdir(path)
    file_name_list = []
    for a in article_list:
        file_name_list.append(make_html_filename(a.title, a.pmid))
#    file_name_list = [f for f in glob.glob("*.html")]
#    file_name_list_act
    if os.path.isfile('analyzed_files.txt'):
        # read files
        f = open('analyzed_files.txt')
        lines = f.readlines()
        f.close()
        # need to remove newline from lines
        new_lines = []
        for l in lines:
            new_lines.append(l[:-1])
        file_name_list = list(set(file_name_list).difference(set(new_lines)))

    num_files = len(file_name_list)
    print 'adding %s files...' % num_files
    for i,f in enumerate(file_name_list):
        prog(i, num_files)
        add_article_full_text_from_file_deprecated(f, path)
コード例 #2
0
def apply_article_metadata(article = None):
    if article:
        artObs = [article]
        num_arts = 1
    else:
    #    artObs = m.Article.objects.filter(metadata__isnull = True, articlefulltext__isnull = False).distinct()
        artObs = m.Article.objects.filter(articlefulltext__isnull = False).distinct()
    #    artObs = artObs.exclude(articlefulltext__articlefulltextstat__metadata_processed = True)


    #    artObs = m.Article.objects.filter(articlefulltext__isnull = False, articlefulltext__articlefulltextstat__methods_tag_found = True).distinct()
    #    artObs = artObs.exclude(articlefulltext__articlefulltextstat__metadata_processed = True)
    #    artObs = artObs.exclude(articlefulltext__articlefulltextstat__metadata_human_assigned = True)
        num_arts = artObs.count()
        print 'annotating %s articles for metadata...' % num_arts
    for i,art in enumerate(artObs):
        if not article:
            prog(i,num_arts)
        assign_metadata.assign_species(art)
        assign_metadata.assign_electrode_type(art)
        assign_metadata.assign_strain(art)
        assign_metadata.assign_rec_temp(art)
        assign_metadata.assign_prep_type(art)
        assign_metadata.assign_animal_age(art)
        assign_metadata.assign_jxn_potential(art)
        assign_metadata.assign_solution_concs(art)
        aft_ob = art.get_full_text()
        aftStatOb = m.ArticleFullTextStat.objects.get_or_create(article_full_text = aft_ob)[0]
        aftStatOb.metadata_processed = True
        aftStatOb.save()
コード例 #3
0
def apply_article_metadata_jxn_potential():
    articles = m.Article.objects.filter(Q(datatable__datasource__neuronconceptmap__times_validated__gte = 1) |
    Q(usersubmission__datasource__neuronconceptmap__times_validated__gte = 1)).distinct()
    articles = articles.filter(articlefulltext__articlefulltextstat__metadata_human_assigned = True ).distinct()
    num_arts = articles.count()
    print 'annotating %s articles for metadata...' % num_arts
    for i,art in enumerate(articles):
        prog(i,num_arts)
        assign_metadata.assign_jxn_potential(art)
コード例 #4
0
def ephys_table_identify():
    artObs = m.Article.objects.filter(datatable__isnull = False, articlefulltext__isnull = False).distinct()
    artObs = artObs.exclude(articlefulltext__articlefulltextstat__data_table_ephys_processed = True)
    dataTableObs = m.DataTable.objects.filter(article__in = artObs).distinct()
    num_tables = dataTableObs.count()
    print 'analyzing %s tables' % num_tables
    for i,dt in enumerate(dataTableObs):
        prog(i, num_tables)
        assocDataTableEphysVal(dt)
        art = dt.article
        aft_ob = art.get_full_text()
        if aft_ob is not None:
            aftStatOb = m.ArticleFullTextStat.objects.get_or_create(article_full_text = aft_ob)[0]
            aftStatOb.data_table_ephys_processed = True
            aftStatOb.save()
コード例 #5
0
def add_full_texts_from_directory(dir_path):
    base_dir = dir_path
    os.chdir(base_dir)
    file_name_list = [f for f in glob.glob("*.html")]
    shuffle(file_name_list)

    #file_name_list = file_name_list[0:3]

    os.chdir(settings.PROJECT_BASE_DIRECTORY)

    print len(file_name_list)
    for i, fn in enumerate(file_name_list):
        prog(i, len(file_name_list))
        file_name = base_dir + fn

        pmid_str = re.search('\d+', file_name).group()
        add_single_full_text(file_name, pmid_str)
コード例 #6
0
def record_solution_concs():
#     articles = m.Article.objects.all()
    
    articles = m.Article.objects.filter(Q(datatable__datasource__neuronconceptmap__times_validated__gte = 1,
                                            datatable__datasource__neuronephysdatamap__isnull = False) | 
                                            Q(usersubmission__datasource__neuronconceptmap__times_validated__gte = 1,
                                              usersubmission__datasource__neuronephysdatamap__isnull = False)).distinct()
    
    robot_user = m.get_robot_user()
    
    solution_names = {"external": 'ExternalSolution', 
                      "internal": 'InternalSolution'}
    
    len_articles = articles.count()
    
    for i, article in enumerate(articles):
        prog(i, len_articles)
        for soln, soln_name in solution_names.iteritems():
            solution_ob = m.ArticleMetaDataMap.objects.filter(article = article, metadata__name = soln_name)
            if solution_ob and solution_ob[0].ref_text:
                record_compounds(article, None, solution_ob[0].ref_text.text, ["", "", "", ""], "%s_0" % soln, robot_user)
コード例 #7
0
def add_multiple_full_texts_all(path):
    os.chdir(path)
    file_name_list_xml = [f for f in glob.glob("*.xml")]
    file_name_list_html = [f for f in glob.glob("*.html")]
    file_name_list = file_name_list_xml
    file_name_list.extend(file_name_list_html)
    if os.path.isfile('analyzed_files.txt'):
        # read files
        f = open('analyzed_files.txt')
        lines = f.readlines()
        f.close()
        # need to remove newline from lines
        new_lines = []
        for l in lines:
            new_lines.append(l[:-1])
        file_name_list = list(set(file_name_list).difference(set(new_lines)))

    num_files = len(file_name_list)
    print 'adding %s files...' % num_files
    for i,f in enumerate(file_name_list):
        prog(i, num_files)
        add_article_full_text_from_file_deprecated(f, path)
コード例 #8
0
def add_full_texts_from_directory(dir_path):
    base_dir = dir_path
    os.chdir(base_dir)
    file_name_list = [f for f in glob.glob("*.html")]

    # shuffle(file_name_list)
    file_name_list.sort(reverse=True)

    os.chdir(settings.PROJECT_BASE_DIRECTORY)

    print len(file_name_list)
    for i, fn in enumerate(file_name_list):
        prog(i, len(file_name_list))
        file_name = base_dir + fn

        pmid_str = re.search("\d+", file_name).group()

        add_single_full_text(file_name, pmid_str)

        # close database connection to deal with MySQL server gone away issue
        if i % 100 == 0:
            connection.close()
コード例 #9
0
def export_db_to_data_frame():
    """Returns a nicely formatted pandas data frame of the ephys data and metadata for each stored article"""

    ncms = m.NeuronConceptMap.objects.all()#.order_by('-history__latest__history_date') # gets human-validated neuron mappings
    ncms = ncms.exclude(Q(source__data_table__irrelevant_flag = True) | Q(source__data_table__needs_expert = True)) # exclude
    ncm_count = ncms.count()
    ephys_props = m.EphysProp.objects.all().order_by('-ephyspropsummary__num_neurons')
    ephys_names = []
    for e in ephys_props:
        ephys_names.append(e.short_name)
        ephys_names.append(e.short_name + '_err')
        ephys_names.append(e.short_name + '_n')
        ephys_names.append(e.short_name + '_sd')
        ephys_names.append(e.short_name + '_note')
    #ephys_names = [e.name for e in ephys_props]
    #ncms = ncms.sort('-changed_on')
    dict_list = []
    for kk, ncm in enumerate(ncms):
        prog(kk, ncm_count)

    # TODO: need to check whether nedms under the same ncm have different experimental factor concept maps
    #     # check if any nedms have any experimental factors assoc with them
    #     efcms = ne_db.ExpFactConceptMap.objects.filter(neuronephysdatamap__in = nedms)
    #     for efcm in efcms:
    #         nedms = ne_db.NeuronEphysDataMap.objects.filter(neuron_concept_map = ncm, exp_fact_concept_map = ).distinct()

        nedms = m.NeuronEphysDataMap.objects.filter(neuron_concept_map = ncm, expert_validated = True).distinct()
        if nedms.count() == 0:
            continue

        sd_errors = identify_stdev(nedms)

        temp_dict = dict()
        temp_metadata_list = []
        for nedm in nedms:
            e = nedm.ephys_concept_map.ephys_prop
            # check data integrity - value MUST be in appropriate range for property
            data_val =  nedm.val_norm
            err_val = nedm.err_norm
            n_val = nedm.n
            note_val = nedm.ephys_concept_map.note
            if check_data_val_range(data_val, e):
                output_ephys_name = e.short_name
                output_ephys_err_name = '%s_err' % output_ephys_name
                output_ephys_sd_name = '%s_sd' % output_ephys_name
                output_ephys_n_name = '%s_n' % output_ephys_name
                output_ephys_note_name = '%s_note' % output_ephys_name
                temp_dict[output_ephys_name] = data_val
                temp_dict[output_ephys_err_name] = err_val
                temp_dict[output_ephys_n_name] = n_val
                temp_dict[output_ephys_note_name] = note_val

                # do converting to standard dev from standard error if needed
                if sd_errors:
                    temp_dict[output_ephys_sd_name] = err_val
                else:
                    # need to calculate sd
                    if err_val and n_val:
                        sd_val = err_val * np.sqrt(n_val)
                        temp_dict[output_ephys_sd_name] = sd_val

            #temp_metadata_list.append(nedm.get_metadata())

        temp_dict['NeuronName'] =  ncm.neuron.name
        temp_dict['NeuronLongName'] =  ncm.neuron_long_name
        if ncm.neuron_long_name:
            temp_dict['NeuronPrefName'] = ncm.neuron_long_name
        else:
            temp_dict['NeuronPrefName'] = ncm.neuron.name
        article = ncm.get_article()

        brain_reg_dict = get_neuron_region(ncm.neuron)
        if brain_reg_dict:
            temp_dict['BrainRegion'] = brain_reg_dict['region_name']

        #article_metadata = normalize_metadata(article)

        metadata_list = nedm.get_metadata()
        out_dict = dict()
        for metadata in metadata_list:
            #print metadata.name
            if not metadata.cont_value:
                if metadata.name in out_dict:
                    out_dict[metadata.name] = '%s, %s' % (out_dict[metadata.name], metadata.value)
                else:
                    out_dict[metadata.name] = metadata.value
            elif metadata.cont_value and 'Solution' in metadata.name:
                article = nedm.get_article()
                amdm = m.ArticleMetaDataMap.objects.filter(article = article, metadata__name = metadata.name)[0]
                ref_text = amdm.ref_text
                out_dict[metadata.name] = ref_text.text.encode('utf8', "replace")
                out_dict[metadata.name + '_conf'] = metadata.cont_value.mean
            elif metadata.cont_value and 'AnimalAge' in metadata.name:
                # return geometric mean of age ranges, not arithmetic mean
                if metadata.cont_value.min_range and metadata.cont_value.max_range:
                    min_range = metadata.cont_value.min_range
                    max_range = metadata.cont_value.max_range
                    if min_range <= 0:
                        min_range = 1
                    geom_mean = np.sqrt(min_range * max_range)
                    out_dict[metadata.name] = geom_mean
                else:
                    out_dict[metadata.name] = metadata.cont_value.mean
            else:
                out_dict[metadata.name] = metadata.cont_value.mean

        # has article metadata been curated by a human?
        afts = article.get_full_text_stat()
        if afts and afts.metadata_human_assigned:
            metadata_curated = True
            metadata_curation_note = afts.metadata_curation_note
        else:
            metadata_curated = False
            metadata_curation_note = None

        if ncm.source.data_table:
            data_table_note = ncm.source.data_table.note
        else:
            data_table_note = None

        temp_dict2 = temp_dict.copy()
        temp_dict2.update(out_dict)
        temp_dict = temp_dict2
        temp_dict['Title'] = article.title
        temp_dict['Pmid'] = article.pmid
        temp_dict['PubYear'] = article.pub_year
        temp_dict['LastAuthor'] = unicode(get_article_last_author(article))
        temp_dict['TableID'] = ncm.source.data_table_id
        temp_dict['TableNote'] = data_table_note
        temp_dict['ArticleID'] = article.pk
        temp_dict['MetadataCurated'] = metadata_curated
        temp_dict['MetadataNote'] = metadata_curation_note
        #print temp_dict
        dict_list.append(temp_dict)

    base_names = ['Title', 'Pmid', 'PubYear', 'LastAuthor', 'ArticleID', 'TableID',
                  'NeuronName', 'NeuronLongName', 'NeuronPrefName', 'BrainRegion']
    nom_vars = ['MetadataCurated', 'Species', 'Strain', 'ElectrodeType', 'PrepType', 'JxnPotential']
    cont_vars  = ['JxnOffset', 'RecTemp', 'AnimalAge', 'AnimalWeight', 'FlagSoln']
    annot_notes = ['MetadataNote', 'TableNote']

    for i in range(0, 1):
        cont_vars.extend([ 'ExternalSolution', 'ExternalSolution_conf', 'external_%s_Mg' % i, 'external_%s_Ca' % i, 'external_%s_Na' % i, 'external_%s_Cl' % i, 'external_%s_K' % i, 'external_%s_pH' % i, 'InternalSolution', 'InternalSolution_conf', 'internal_%s_Mg' % i, 'internal_%s_Ca' % i, 'internal_%s_Na' % i, 'internal_%s_Cl' % i, 'internal_%s_K' % i, 'internal_%s_pH' % i])
        #cont_var_headers.extend(['External_%s_Mg' % i, 'External_%s_Ca' % i, 'External_%s_Na' % i, 'External_%s_Cl' % i, 'External_%s_K' % i, 'External_%s_pH' % i, 'External_%s_text' % i, 'Internal_%s_Mg' % i, 'Internal_%s_Ca' % i, 'Internal_%s_Na' % i, 'Internal_%s_Cl' % i, 'Internal_%s_K' % i, 'Internal_%s_pH' % i, 'Internal_%s_text' % i])

    col_names = base_names + nom_vars + cont_vars + annot_notes + ephys_names

    # set up pandas data frame for export
    df = pd.DataFrame(dict_list, columns = col_names)

    # perform collapsing of rows about same neuron types but potentially across different tables
    cleaned_df = df
    # need to generate a random int for coercing NaN's to something - required for pandas grouping
    rand_int = -abs(np.random.randint(20000))
    cleaned_df.loc[:, 'Pmid':'FlagSoln'] = df.loc[:, 'Pmid':'FlagSoln'].fillna(rand_int)
    grouping_fields = base_names + nom_vars + cont_vars
    grouping_fields.remove('TableID')
    cleaned_df.groupby(by = grouping_fields).mean()
    cleaned_df.replace(to_replace = rand_int, value = np.nan, inplace=True)
    cleaned_df.reset_index(inplace=True)
    cleaned_df.sort_values(by = ['PubYear', 'Pmid', 'NeuronName'], ascending=[False, True, True], inplace=True)
    cleaned_df.index.name = "Index"

    # add in extra ephys data from columns based on known relationships, e.g., AP amp from AP peak and AP thr
    cleaned_df = add_ephys_props_by_conversion(cleaned_df)

    return cleaned_df
コード例 #10
0
def export_db_to_data_frame():
    """Returns a nicely formatted pandas data frame of the ephys data and metadata for each stored article"""

    ncms = (
        m.NeuronConceptMap.objects.all()
    )  # .order_by('-history__latest__history_date') # gets human-validated neuron mappings
    # ncms = ncms.exclude(Q(source__data_table__irrelevant_flag = True) | Q(source__data_table__needs_expert = True)) # exclude
    ncms = ncms.exclude(Q(source__data_table__irrelevant_flag=True))  # exclude

    ncm_count = ncms.count()
    ephys_props = m.EphysProp.objects.all().order_by("-ephyspropsummary__num_neurons")
    ephys_names = []
    for e in ephys_props:
        ephys_names.append(e.short_name)
        ephys_names.append(e.short_name + "_raw")
        ephys_names.append(e.short_name + "_err")
        ephys_names.append(e.short_name + "_n")
        ephys_names.append(e.short_name + "_sd")
        ephys_names.append(e.short_name + "_note")
    # ephys_names = [e.name for e in ephys_props]
    # ncms = ncms.sort('-changed_on')
    dict_list = []
    for kk, ncm in enumerate(ncms):
        prog(kk, ncm_count)

        # TODO: need to check whether nedms under the same ncm have different experimental factor concept maps
        #     # check if any nedms have any experimental factors assoc with them
        #     efcms = ne_db.ExpFactConceptMap.objects.filter(neuronephysdatamap__in = nedms)
        #     for efcm in efcms:
        #         nedms = ne_db.NeuronEphysDataMap.objects.filter(neuron_concept_map = ncm, exp_fact_concept_map = ).distinct()

        # only check whether ncms have been expertly validated, not the nedm itself
        nedms = m.NeuronEphysDataMap.objects.filter(
            neuron_concept_map=ncm, neuron_concept_map__expert_validated=True
        ).distinct()
        if nedms.count() == 0:
            continue

        temp_dict = dict()
        temp_metadata_list = []
        for nedm in nedms:
            e = nedm.ephys_concept_map.ephys_prop

            # get error type for nedm by db lookup
            error_type = nedm.get_error_type()

            # check data integrity - value MUST be in appropriate range for property
            data_val = nedm.val_norm
            data_raw_val = nedm.val
            err_val = nedm.err_norm
            n_val = nedm.n
            note_val = nedm.ephys_concept_map.note
            output_ephys_name = e.short_name
            output_ephys_raw_name = "%s_raw" % output_ephys_name
            output_ephys_err_name = "%s_err" % output_ephys_name
            output_ephys_sem_name = "%s_sem" % output_ephys_name
            output_ephys_sd_name = "%s_sd" % output_ephys_name
            output_ephys_n_name = "%s_n" % output_ephys_name
            output_ephys_note_name = "%s_note" % output_ephys_name

            # output raw vals and notes for all props
            temp_dict[output_ephys_raw_name] = data_raw_val
            temp_dict[output_ephys_note_name] = note_val

            if check_data_val_range(data_val, e):

                temp_dict[output_ephys_name] = data_val
                temp_dict[output_ephys_err_name] = err_val
                temp_dict[output_ephys_n_name] = n_val

                # do converting to standard dev from standard error if needed
                if error_type == "sd":
                    temp_dict[output_ephys_sd_name] = err_val
                else:
                    # need to calculate sd
                    if err_val and n_val:
                        sd_val = err_val * np.sqrt(n_val)
                        temp_dict[output_ephys_sd_name] = sd_val

            # temp_metadata_list.append(nedm.get_metadata())

        temp_dict["NeuronName"] = ncm.neuron.name
        temp_dict["NeuronLongName"] = ncm.neuron_long_name
        if ncm.neuron_long_name:
            temp_dict["NeuronPrefName"] = ncm.neuron_long_name
        else:
            temp_dict["NeuronPrefName"] = ncm.neuron.name
        temp_dict["NeuroNERAnnots"] = ncm.get_neuroner()
        article = ncm.get_article()

        brain_reg_dict = get_neuron_region(ncm.neuron)
        if brain_reg_dict:
            temp_dict["BrainRegion"] = brain_reg_dict["region_name"]

        # article_metadata = normalize_metadata(article)

        metadata_list = nedm.get_metadata()
        out_dict = dict()
        for metadata in metadata_list:
            # print metadata.name
            if not metadata.cont_value:
                if metadata.name in out_dict:
                    out_dict[metadata.name] = "%s, %s" % (out_dict[metadata.name], metadata.value)
                else:
                    out_dict[metadata.name] = metadata.value
                if metadata.name == "Strain":
                    out_dict["StrainNote"] = metadata.note
                if metadata.name == "Species":
                    out_dict["SpeciesNote"] = metadata.note
            elif metadata.cont_value and "Solution" in metadata.name:
                article = nedm.get_article()
                if metadata.ref_text:
                    ref_text = metadata.ref_text
                else:
                    amdm = m.ArticleMetaDataMap.objects.filter(article=article, metadata__name=metadata.name)[0]
                    ref_text = amdm.ref_text
                out_dict[metadata.name] = ref_text.text.encode("utf8", "replace")
                out_dict[metadata.name + "_conf"] = metadata.cont_value.mean
            elif metadata.cont_value and "AnimalAge" in metadata.name:
                # return geometric mean of age ranges, not arithmetic mean
                if metadata.cont_value.min_range and metadata.cont_value.max_range:
                    min_range = metadata.cont_value.min_range
                    max_range = metadata.cont_value.max_range
                    if min_range <= 0:
                        min_range = 1
                    geom_mean = np.sqrt(min_range * max_range)
                    out_dict[metadata.name] = geom_mean
                else:
                    out_dict[metadata.name] = metadata.cont_value.mean
            else:
                out_dict[metadata.name] = metadata.cont_value.mean

        # has article metadata been curated by a human?
        afts = article.get_full_text_stat()
        if afts and afts.metadata_human_assigned:
            metadata_curated = True
            metadata_curation_note = afts.metadata_curation_note
        else:
            metadata_curated = False
            metadata_curation_note = None

        if ncm.source.data_table:
            data_table_note = ncm.source.data_table.note
        else:
            data_table_note = None

        temp_dict2 = temp_dict.copy()
        temp_dict2.update(out_dict)
        temp_dict = temp_dict2
        temp_dict["Title"] = article.title
        temp_dict["Pmid"] = article.pmid
        temp_dict["PubYear"] = article.pub_year
        temp_dict["LastAuthor"] = unicode(get_article_last_author(article))
        temp_dict["FirstAuthor"] = unicode(get_article_author(article, 0))
        temp_dict["TableID"] = ncm.source.data_table_id
        temp_dict["TableNote"] = data_table_note
        temp_dict["ArticleID"] = article.pk
        temp_dict["MetadataCurated"] = metadata_curated
        temp_dict["MetadataNote"] = metadata_curation_note
        # print temp_dict
        dict_list.append(temp_dict)

    base_names = [
        "Title",
        "Pmid",
        "PubYear",
        "FirstAuthor",
        "LastAuthor",
        "ArticleID",
        "TableID",
        "NeuronName",
        "NeuronLongName",
        "NeuronPrefName",
        "NeuroNERAnnots",
        "BrainRegion",
    ]
    nom_vars = [
        "MetadataCurated",
        "Species",
        "SpeciesNote",
        "Strain",
        "StrainNote",
        "ElectrodeType",
        "PrepType",
        "JxnPotential",
    ]
    cont_vars = ["JxnOffset", "RecTemp", "AnimalAge", "AnimalWeight", "FlagSoln"]
    annot_notes = ["MetadataNote", "TableNote"]

    grouping_fields = base_names + nom_vars + cont_vars

    for i in range(0, 1):
        cont_vars.extend(
            [
                "ExternalSolution",
                "ExternalSolution_conf",
                "external_%s_Mg" % i,
                "external_%s_Ca" % i,
                "external_%s_Na" % i,
                "external_%s_Cl" % i,
                "external_%s_K" % i,
                "external_%s_pH" % i,
                "external_%s_Cs" % i,
                "external_%s_glucose" % i,
                "external_%s_HEPES" % i,
                "external_%s_EDTA" % i,
                "external_%s_EGTA" % i,
                "external_%s_BAPTA" % i,
                "external_%s_ATP" % i,
                "external_%s_GTP" % i,
                "external_%s_CNQX" % i,
                "external_%s_DNQX" % i,
                "external_%s_NBQX" % i,
                "external_%s_MK801" % i,
                "external_%s_DAPV" % i,
                "external_%s_CPP" % i,
                "external_%s_kynur" % i,
                "external_%s_BIC" % i,
                "external_%s_picro" % i,
                "external_%s_gabazine" % i,
                "external_%s_CGP" % i,
                "external_%s_strychnine" % i,
                "InternalSolution",
                "InternalSolution_conf",
                "internal_%s_Mg" % i,
                "internal_%s_Ca" % i,
                "internal_%s_Na" % i,
                "internal_%s_Cl" % i,
                "internal_%s_K" % i,
                "internal_%s_pH" % i,
                "internal_%s_Cs" % i,
                "internal_%s_glucose" % i,
                "internal_%s_HEPES" % i,
                "internal_%s_EDTA" % i,
                "internal_%s_EGTA" % i,
                "internal_%s_BAPTA" % i,
                "internal_%s_ATP" % i,
                "internal_%s_GTP" % i,
            ]
        )

    col_names = base_names + nom_vars + cont_vars + annot_notes + ephys_names

    # not sure why but writing and reading data frame seems to fix a problem with ephys property pooling fxn
    df = pd.DataFrame(dict_list, columns=col_names)
    df.to_csv("temp.csv", sep="\t", encoding="utf-8")
    df = pd.read_csv("temp.csv", sep="\t", index_col=0, header=0)

    # perform collapsing of rows about same neuron types but potentially across different tables
    # this should be optional if the goal is ephys recuration, not ephys reanalysis
    grouping_fields.remove("TableID")
    grouping_fields.remove("NeuroNERAnnots")
    cleaned_df = pool_ephys_props_across_tables(df, grouping_fields)

    # add in extra ephys data from columns based on known relationships, e.g., AP amp from AP peak and AP thr
    cleaned_df = add_ephys_props_by_conversion(cleaned_df)

    # returning 2 data frames, 1 with properties pooled and calculated based on algebra, 1 not
    return cleaned_df, df