def assign_stat_object_to_data_tables(): """go through each data table object and add info about who curated and when""" dts = m.DataTable.objects.filter(datasource__ephysconceptmap__isnull=False) num_dts = dts.count() for i, dt in enumerate(dts): print "updating data tables with summary objects" prog(i, num_dts) update_data_table_stat(dt)
def add_table_ob_to_article(table_html, article_ob, text_mine = True, uploading_user = None): if uploading_user: user_uploaded = True else: user_uploaded = False table_soup = BeautifulSoup(table_html, 'lxml') table_html_cleaned = str(table_soup) table_html_cleaned = add_id_tags_to_table(table_html_cleaned) table_text = table_soup.get_text() table_text = table_text[0:min(9999,len(table_text))] data_table_ob = m.DataTable.objects.get_or_create(article = article_ob, table_html = table_html_cleaned, table_text = table_text, uploading_user = uploading_user, user_uploaded = user_uploaded )[0] data_table_ob = remove_spurious_table_headers(data_table_ob) # takes care of weird header thing for elsevier xml tables ds = m.DataSource.objects.get_or_create(data_table=data_table_ob)[0] # apply initial text mining of ephys concepts to table if text_mine: assocDataTableEphysVal(data_table_ob) # creates data table stat object, relevance is to count and store num of unique ecms that were TMed data_table_stat = update_data_table_stat(data_table_ob) return data_table_ob