def update_articles(): print 'Updating articles' all_arts = m.Article.objects.all() pmid_list = [a.pmid for a in all_arts] pmid_list_len = len(pmid_list) for i,pmid in enumerate(pmid_list): prog(i, pmid_list_len) a = m.Article.objects.filter(pmid=pmid)[0] if a.author_list_str is None: add_single_article_full(pmid)
def add_article_full_text_from_file(abs_path, pmid, html_table_list, overwrite_existing = False): a = add_single_article_full(int(pmid), overwrite_existing) if a is None: return None # does article already have full text assoc with it? if m.ArticleFullText.objects.filter(article__pmid = pmid).count() > 0: aft = m.ArticleFullText.objects.get(article = a) if len(aft.get_content()) > 0: print "Article %s full text already in db, skipping..." % pmid return None try: print 'adding article %s' % (pmid) f = open(unicode(abs_path), 'r') file_ob = File(f) os.chdir(settings.PROJECT_BASE_DIRECTORY) aft = m.ArticleFullText.objects.get_or_create(article = a)[0] aft.full_text_file.save(pmid, file_ob) file_ob.close() for table in html_table_list: add_table_ob_to_article(table, a, text_mine = True) # text mine article level metadata apply_article_metadata(a) except Exception, e: # with open('failed_files.txt', 'a') as f: # f.write('%s\\%s' % (file_name, e)) print e print pmid
def add_ephys_nedm(ephys_name, ephys_value, pmid, neuron_type, user, overwrite=True): if ephys_value is '': return ephys_value_list = resolve.resolve_data_float(ephys_value) if not 'error' in ephys_value_list: ephys_value_list['error'] = None if not 'num_obs' in ephys_value_list: ephys_value_list['num_obs'] = None a = pubmed_functions.add_single_article_full(pmid) n = m.Neuron.objects.filter(name = neuron_type)[0] us_ob = m.UserSubmission.objects.get_or_create(user = user, article = a)[0] ds_ob = m.DataSource.objects.get_or_create(user_submission = us_ob)[0] ncm_ob = m.NeuronConceptMap.objects.get_or_create(source = ds_ob, added_by = user, neuron = n, times_validated = 1)[0] ephys_prop_ob = m.EphysProp.objects.get(name = ephys_name) ecm_ob = m.EphysConceptMap.objects.get_or_create(ephys_prop = ephys_prop_ob, source = ds_ob, added_by = user, times_validated = 1)[0] ds_ob.save() try: nedm = m.NeuronEphysDataMap.objects.get(source = ds_ob, added_by = user, neuron_concept_map = ncm_ob, ephys_concept_map = ecm_ob) if overwrite is True: nedm.delete() except ObjectDoesNotExist: pass # if overwrite is false, just make a new nedm, otherwise find the old nedm (if it exists) and then overwrite it m.NeuronEphysDataMap.objects.get_or_create(source = ds_ob, added_by = user, neuron_concept_map = ncm_ob, ephys_concept_map = ecm_ob, val = ephys_value_list['value'], val_norm = ephys_value_list['value'], err = ephys_value_list['error'], times_validated = 1, n = ephys_value_list['numCells'], )[0]
def get_journal_from_file_path(file_path, pmid): """returns a neuroelectro Journal object given a standardized file path and pmid as input adds journal to database """ journal_short_name = file_path.split('/')[-2] try: journal_ob = m.Journal.objects.get(short_title = journal_short_name) except ObjectDoesNotExist: # journal short_title not found from path, try getting from pubmed directly journal_title = get_journal(pmid) try: journal_ob = m.Journal.objects.get(short_title = journal_short_name) except ObjectDoesNotExist: article = add_single_article_full(pmid, overwrite_existing = False) # check if returned article is None if article is None: return None journal_ob = article.journal return journal_ob
def update_articles(): print 'Updating articles' all_arts = m.Article.objects.all() pmid_list = [a.pmid for a in all_arts] pmid_list_len = len(pmid_list) for i,pmid in enumerate(pmid_list): prog(i, pmid_list_len) a = m.Article.objects.filter(pmid=pmid)[0] if a.author_list_str is None: print "updating article because of null author list str" add_single_article_full(pmid) elif a.full_text_link is None: print "updating article %s because of null full_text_link " % (a.pk) add_single_article_full(pmid) elif a.pub_year is None: print "updating article because of null pub_year" add_single_article_full(pmid)
return None journal_name = get_journal(pmid_str) # is journal one among list of full text journals? if not isFullTextJournal(journal_name): # with open("analyzed_files.txt", "a") as af: # write_str = '%s\n' % file_name # af.write(write_str) print "Journal %s is not a full text journal, skipping..." % journal_name return None # does journal already have full text assoc with it? if m.ArticleFullText.objects.filter(article__pmid = pmid_str).count() > 0: print "Article %s already in db, skipping..." % pmid_str return None print 'adding article with pmid: %s' % pmid_str a = add_single_article_full(int(pmid_str)) if a is None: # f.close() return None try: print 'adding %s as %s' % (file_name, a.pmid) f = open(unicode(file_name), 'r') file_ob = File(f) os.chdir(settings.PROJECT_BASE_DIRECTORY) aft = m.ArticleFullText.objects.get_or_create(article = a)[0] aft.full_text_file.save(file_name, file_ob) file_ob.close() file_ext = os.path.splitext(file_name) # first check if any tables