def update_articles():
print 'Updating articles'
all_arts = m.Article.objects.all()
pmid_list = [a.pmid for a in all_arts]
pmid_list_len = len(pmid_list)
for i,pmid in enumerate(pmid_list):
prog(i, pmid_list_len)
a = m.Article.objects.filter(pmid=pmid)[0]
if a.author_list_str is None:
add_single_article_full(pmid)
def add_article_full_text_from_file(abs_path, pmid, html_table_list, overwrite_existing = False):
a = add_single_article_full(int(pmid), overwrite_existing)
if a is None:
return None
# does article already have full text assoc with it?
if m.ArticleFullText.objects.filter(article__pmid = pmid).count() > 0:
aft = m.ArticleFullText.objects.get(article = a)
if len(aft.get_content()) > 0:
print "Article %s full text already in db, skipping..." % pmid
return None
try:
print 'adding article %s' % (pmid)
f = open(unicode(abs_path), 'r')
file_ob = File(f)
os.chdir(settings.PROJECT_BASE_DIRECTORY)
aft = m.ArticleFullText.objects.get_or_create(article = a)[0]
aft.full_text_file.save(pmid, file_ob)
file_ob.close()
for table in html_table_list:
add_table_ob_to_article(table, a, text_mine = True)
# text mine article level metadata
apply_article_metadata(a)
except Exception, e:
# with open('failed_files.txt', 'a') as f:
# f.write('%s\\%s' % (file_name, e))
print e
print pmid
def add_ephys_nedm(ephys_name, ephys_value, pmid, neuron_type, user, overwrite=True):
if ephys_value is '':
return
ephys_value_list = resolve.resolve_data_float(ephys_value)
if not 'error' in ephys_value_list:
ephys_value_list['error'] = None
if not 'num_obs' in ephys_value_list:
ephys_value_list['num_obs'] = None
a = pubmed_functions.add_single_article_full(pmid)
n = m.Neuron.objects.filter(name = neuron_type)[0]
us_ob = m.UserSubmission.objects.get_or_create(user = user, article = a)[0]
ds_ob = m.DataSource.objects.get_or_create(user_submission = us_ob)[0]
ncm_ob = m.NeuronConceptMap.objects.get_or_create(source = ds_ob, added_by = user, neuron = n,
times_validated = 1)[0]
ephys_prop_ob = m.EphysProp.objects.get(name = ephys_name)
ecm_ob = m.EphysConceptMap.objects.get_or_create(ephys_prop = ephys_prop_ob, source = ds_ob, added_by = user,
times_validated = 1)[0]
ds_ob.save()
try:
nedm = m.NeuronEphysDataMap.objects.get(source = ds_ob,
added_by = user,
neuron_concept_map = ncm_ob,
ephys_concept_map = ecm_ob)
if overwrite is True:
nedm.delete()
except ObjectDoesNotExist:
pass
# if overwrite is false, just make a new nedm, otherwise find the old nedm (if it exists) and then overwrite it
m.NeuronEphysDataMap.objects.get_or_create(source = ds_ob,
added_by = user,
neuron_concept_map = ncm_ob,
ephys_concept_map = ecm_ob,
val = ephys_value_list['value'],
val_norm = ephys_value_list['value'],
err = ephys_value_list['error'],
times_validated = 1,
n = ephys_value_list['numCells'],
)[0]
def get_journal_from_file_path(file_path, pmid):
"""returns a neuroelectro Journal object given a standardized file path and pmid as input
adds journal to database """
journal_short_name = file_path.split('/')[-2]
try:
journal_ob = m.Journal.objects.get(short_title = journal_short_name)
except ObjectDoesNotExist:
# journal short_title not found from path, try getting from pubmed directly
journal_title = get_journal(pmid)
try:
journal_ob = m.Journal.objects.get(short_title = journal_short_name)
except ObjectDoesNotExist:
article = add_single_article_full(pmid, overwrite_existing = False)
# check if returned article is None
if article is None:
return None
journal_ob = article.journal
return journal_ob
def update_articles():
print 'Updating articles'
all_arts = m.Article.objects.all()
pmid_list = [a.pmid for a in all_arts]
pmid_list_len = len(pmid_list)
for i,pmid in enumerate(pmid_list):
prog(i, pmid_list_len)
a = m.Article.objects.filter(pmid=pmid)[0]
if a.author_list_str is None:
print "updating article because of null author list str"
add_single_article_full(pmid)
elif a.full_text_link is None:
print "updating article %s because of null full_text_link " % (a.pk)
add_single_article_full(pmid)
elif a.pub_year is None:
print "updating article because of null pub_year"
add_single_article_full(pmid)
return None
journal_name = get_journal(pmid_str)
# is journal one among list of full text journals?
if not isFullTextJournal(journal_name):
# with open("analyzed_files.txt", "a") as af:
# write_str = '%s\n' % file_name
# af.write(write_str)
print "Journal %s is not a full text journal, skipping..." % journal_name
return None
# does journal already have full text assoc with it?
if m.ArticleFullText.objects.filter(article__pmid = pmid_str).count() > 0:
print "Article %s already in db, skipping..." % pmid_str
return None
print 'adding article with pmid: %s' % pmid_str
a = add_single_article_full(int(pmid_str))
if a is None:
# f.close()
return None
try:
print 'adding %s as %s' % (file_name, a.pmid)
f = open(unicode(file_name), 'r')
file_ob = File(f)
os.chdir(settings.PROJECT_BASE_DIRECTORY)
aft = m.ArticleFullText.objects.get_or_create(article = a)[0]
aft.full_text_file.save(file_name, file_ob)
file_ob.close()
file_ext = os.path.splitext(file_name)
# first check if any tables
