def test_callable(self):
"Regression for #10349: A callable can be provided as the initial value for an m2m field"
# Set up a callable initial value
def formfield_for_dbfield(db_field, **kwargs):
if db_field.name == 'publications':
kwargs['initial'] = lambda: Publication.objects.all().order_by(
'date_published')[:2]
return db_field.formfield(**kwargs)
# Set up some Publications to use as data
Publication(title="First Book", date_published=date(2007, 1, 1)).save()
Publication(title="Second Book", date_published=date(2008, 1,
1)).save()
Publication(title="Third Book", date_published=date(2009, 1, 1)).save()
# Create a ModelForm, instantiate it, and check that the output is as expected
ModelForm = modelform_factory(Article,
formfield_callback=formfield_for_dbfield)
form = ModelForm()
self.assertEquals(
form.as_ul(),
u"""<li><label for="id_headline">Headline:</label> <input id="id_headline" type="text" name="headline" maxlength="100" /></li>
<li><label for="id_publications">Publications:</label> <select multiple="multiple" name="publications" id="id_publications">
<option value="1" selected="selected">First Book</option>
<option value="2" selected="selected">Second Book</option>
<option value="3">Third Book</option>
</select> Hold down "Control", or "Command" on a Mac, to select more than one.</li>"""
)
def create():
form = RegistrationForm()
if form.validate() is False:
return render_template('news/new.html', form=form)
else:
publication = Publication()
publication.title = form.title.data
subject_query = PublicationSubject.query.filter_by(
name=form.subject.data)
if (subject_query.first()):
publication.subject_id = subject_query.first().id
else:
subject = PublicationSubject()
subject.name = form.subject.data
db.session.add(subject)
db.session.commit()
publication.subject_id = subject.id
publication.text_content = form.text_content.data
publication.text_call = form.text_call.data
publication.last_modification = datetime.now().strftime(
'%Y-%m-%d %H:%M:%S')
publication.publish_date = form.publish_date.data.strftime('%Y-%m-%d')
publication.show_home = form.show_home.data
publication.thumb = form.thumb.data
publication.active = 0
publication.author = form.author.data
db.session.add(publication)
db.session.commit()
message = u'Muito obrigado! Sua notícia foi submetida com sucesso!'
flash(message, 'success')
return redirect(url_for('news.admin'))
def publish_book(body):
user = select(user for user in User
if user.name == body['username']).first()
new_book = Publication(title=body['title'],
author=user,
content=body['content'],
status=2)
print(new_book.title, new_book.content)
def create():
form = RegistrationForm()
if form.validate() is False:
form.set_choices()
return render_template('news/new.html', form=form)
else:
publication = Publication()
publication.title = form.title.data
publication.text_call = form.text_call.data
publication.last_modification = datetime.now().strftime(
'%Y-%m-%d %H:%M:%S')
publication.publish_date = form.publish_date.data.strftime('%Y-%m-%d')
publication.show_home = form.show_home.data
publication.active = 0
publication.author = form.author.data
publication.language = form.language.data
publication.add_subjects(form.subject.data, form.language.data)
if form.thumb_src.data:
publication.thumb_src = form.thumb_src.data
db.session.add(publication)
db.session.flush()
text_content = upload_images_to_s3(form.text_content.data, mod.name,
publication.id)
Publication.query.get(publication.id).text_content = text_content
clean_s3_folder(text_content, mod.name, publication.id)
if len(form.thumb.data.split(',')) > 1:
upload_folder = os.path.join(app.config['UPLOAD_FOLDER'], mod.name,
str(publication.id), 'images')
publication.thumb = save_b64_image(
form.thumb.data.split(',')[1], upload_folder, 'thumb')
db.session.commit()
log_operation(module=mod.name,
operation='create',
user=(g.user.id, g.user.email),
objs=[(publication.id, publication.title)])
message = u'Muito obrigado! Sua notícia foi submetida com sucesso!'
flash(message, 'success')
return redirect(url_for('news.admin'))
def publications_form(id=None):
if id:
publication = Publication.get(id)
else:
publication = Publication()
publication.user_created = session["id"]
if request.method == 'POST':
form = PublicationForm(request.form,
obj=publication) if id else PublicationForm(
request.form)
if form.validate():
form.populate_obj(publication)
publication.save()
flash('Your publication has been saved')
return redirect(url_for('publication'))
else:
form = PublicationForm(obj=publication) if id else PublicationForm()
return render_template('publications/form.html',
form=form,
publication=publication)
def index(request):
"""
Index view
It can be accessed via GET: when we first access the web page
It can also be accessed via POST: when the form is executed
The view behaviour depends on the http method making the call: get or post.
Let's take a look at the implementation!
"""
# if it is called through post method, register the publication will be needed
if request.method == 'POST':
# getting the arguments via POST
username = request.POST.get('username', '')
email = request.POST.get('email', '')
note = request.POST.get('note', '')
# check whether or not the fields have a real value
# it also has to be checked here taking into account the fact of how easy avoid the front-end security through the browser console is
if username.strip() and email.strip():
# if the user has not filled the note out, a generic message is added
if not note.strip():
note = 'This user did not want to write a note!'
# this is the Django ORM method used to insert elements into the database
Publication(username=username, email=email, note=note).save()
# redirect to the publications' page
return redirect("/test/publications")
# applications context: in this case it only contains the form
context = {'form': PublicationForm()}
# if it is called via get, the landing page will be shown normally
return render(request, 'index.html', context)
def create_publications():
connect(config.database['db_name'], host=config.database['host'], port=config.database['port'])
print('Verificando publicaciones')
for pub_dict in config.publications:
pub_list = Publication.objects(name=pub_dict.get('name'))
if len(pub_list) == 0:
new_pub = Publication(name=pub_dict.get('name'), url=pub_dict.get('url'), location=pub_dict.get('location'),
fetch_method=pub_dict.get('fetch_method'), api_url=pub_dict.get('api_url'))
new_pub.save()
print(f'Publication creada: {pub_dict.get("name")}')
else:
exitent_pub = pub_list.get()
exitent_pub.name = pub_dict.get('name')
exitent_pub.url = pub_dict.get('url')
exitent_pub.location = pub_dict.get('location')
exitent_pub.fetch_method = pub_dict.get('fetch_method')
exitent_pub.api_url = pub_dict.get('api_url')
exitent_pub.save()
print(f'Publication Modificada: {pub_dict.get("name")}')
pubs = Publication.objects()
print(f'Total de publicaciones en la db: {len(pubs)}')
for p in pubs:
print(f'- {p.name}')
print('#################################')
def post(self, request, *args, **kwargs):
self.form = self.form_class(request.POST)
# self.SetFormUser (request)
# context = {'form': self.form}
# ib = request.session['requst_POST']
# aaaaaaaaaa = ffffffffff
if self.form.is_valid():
cd = self.form.cleaned_data
self.db_error = False
try:
Publication(date=datetime.now(),
text=cd['message'],
author=request.user).save()
# form = MsgForm()
except DatabaseError:
self.form = self.form_class(request.POST)
self.db_error = exc_info()[1].message
return super(BlogMainView, self).get(
request
) # self.render_to_response(self.get_context_data(context)) #
return HttpResponseRedirect(reverse('blogclass'))
return super(BlogMainView, self).get(
request
) # self.render_to_response(self.get_context_data(context))
def crawl_publication():
"""
Crawls Google Scholar in order to retrieve information about a publication.
"""
# The ID of the publication in Google Scholar.
scholar_id = request.form['scholar_id']
print 'Crawl publication ' + scholar_id + '.'
url = 'https://scholar.google.com/citations'
publication = Publication.query.filter_by(scholar_id=scholar_id).first()
if publication is None:
publication = Publication()
cookie_jar = CookieJar()
opener = build_opener(HTTPCookieProcessor(cookie_jar))
install_opener(opener)
url = 'https://scholar.google.com/citations'
params = urlencode({
'hl': 'en',
'view_op': 'view_citation',
'citation_for_view': scholar_id
})
req = Request(url + '?' + params)
opener.open(req)
res = opener.open(req)
doc = html.parse(res)
publication.scholar_id = scholar_id
ntitle = doc.find('.//a[@class="gsc_title_link"]')
if ntitle is not None:
# The title of the publication.
publication.title = ntitle.text_content()
ntype = doc.find('.//div[@class="gs_scl"][3]//div[@class="gsc_field"]')
if ntype is not None:
# The type of the publication.
publication.type = ntype.text_content()
if publication.type == 'Description':
publication.type = 'Other'
nyear = doc.xpath(
'.//div[text()="Publication date"]/ancestor::div[@class="gs_scl"]//div[@class="gsc_value"]'
)
if nyear is not None and len(nyear):
# The year of the publication.
publication.year_of_publication = int(nyear[0].text.split('/')[0])
ncitations = doc.xpath(
'.//div[text()="Total citations"]/ancestor::div[@class="gs_scl"]//div[@class="gsc_value"]//a'
)
if ncitations is not None and len(ncitations):
# The total citations for the publication.
publication.total_citations = ncitations[0].text.split(' ')[-1]
nauthors = doc.xpath(
'.//div[text()="Authors"]/ancestor::div[@class="gs_scl"]//div[@class="gsc_value"]'
)
if nauthors is not None and len(nauthors):
# The authors of the publication.
publication.author_names = nauthors[0].text
# The citations per year for the publication.
publication_citations_per_year = []
nhistogram = doc.find('.//div[@id="gsc_graph_bars"]')
if nhistogram is not None:
years = [x.text for x in nhistogram.xpath('.//span[@class="gsc_g_t"]')]
for a in nhistogram.xpath('.//a[@class="gsc_g_a"]'):
i = int(a.get('style').split('z-index:')[1])
year = int(years[-i])
citations_per_year = PublicationCitationsPerYear.query.filter_by(
publication_id=publication.id, year=year).first()
if citations_per_year is None:
citations_per_year = PublicationCitationsPerYear()
citations_per_year.year = int(years[-i])
citations_per_year.citations = int(
a.xpath('./span[@class="gsc_g_al"]')[0].text)
publication_citations_per_year.append(citations_per_year)
publication.citations_per_year = publication_citations_per_year
# When information about the author was retrieved from Google Scholar.
publication.retrieved_at = datetime.datetime.now()
db.session.add(publication)
db.session.commit()
print 'Crawled publication ' + scholar_id + '.'
return 'Done.'
def crawl_author():
"""
Crawls Google Scholar in order to retrieve information about an author.
"""
# The ID of the author in Google Scholar.
scholar_id = request.form['scholar_id']
print 'Crawl author ' + scholar_id + '.'
# Retrieve the author with that ID (if any).
author = Author.query.filter_by(scholar_id=scholar_id).first()
if author is None:
author = Author()
cookie_jar = CookieJar()
opener = build_opener(HTTPCookieProcessor(cookie_jar))
install_opener(opener)
url = 'https://scholar.google.com/citations'
params = urlencode({
'hl': 'en',
'view_op': 'list_works',
'sortby': 'pubdate',
'user': scholar_id,
'cstart': 0,
'pagesize': 20
})
req = Request(url + '?' + params)
opener.open(req)
res = opener.open(req)
doc = html.parse(res)
no_content = doc.xpath(
'.//div[contains(text(), "Sorry, no content found for this URL")]')
if len(no_content):
print 'Author ' + scholar_id + ' not found.'
return 'Done.'
author.scholar_id = scholar_id
nname = doc.find('.//div[@id="gsc_prf_in"]')
if nname is not None:
# The name of the author.
author.name = nname.text_content()
nemaildomain = doc.find('.//div[@id="gsc_prf_ivh"]')
if nemaildomain is not None:
# The domain where the author has an email.
author.email_domain = nemaildomain.text_content().split(
" - ")[0].split()[-1]
ncitations = doc.find('.//table[@id="gsc_rsb_st"]')
if ncitations is not None:
# The total citations for the author.
author.total_citations = ncitations.xpath('.//tr[2]/td')[1].text
# The h-index for the author.
author.h_index = ncitations.xpath('.//tr[3]/td')[1].text
# The i10-index for the author.
author.i10_index = ncitations.xpath('.//tr[4]/td')[1].text
params = urlencode({
'hl': 'en',
'view_op': 'citations_histogram',
'user': scholar_id
})
req = Request(url + '?' + params)
opener.open(req)
res = opener.open(req)
doc = html.parse(res)
# The citations per year for the author.
author_citations_per_year = []
nhistogram = doc.find('.//div[@id="gsc_md_hist_b"]')
if nhistogram is not None:
years = [x.text for x in nhistogram.xpath('.//span[@class="gsc_g_t"]')]
for a in nhistogram.xpath('.//a[@class="gsc_g_a"]'):
i = int(a.get('style').split('z-index:')[1])
year = int(years[-i])
citations_per_year = AuthorCitationsPerYear.query.filter_by(
author_id=author.id, year=year).first()
if citations_per_year is None:
citations_per_year = AuthorCitationsPerYear()
citations_per_year.year = int(years[-i])
citations_per_year.citations = int(
a.xpath('./span[@class="gsc_g_al"]')[0].text)
author_citations_per_year.append(citations_per_year)
author.citations_per_year = author_citations_per_year
params = urlencode({
'hl': 'en',
'view_op': 'list_colleagues',
'user': scholar_id
})
req = Request(url + '?' + params)
opener.open(req)
res = opener.open(req)
doc = html.parse(res)
# The co-authors of the author.
author_coauthors = []
for a in doc.xpath('.//h3[@class="gsc_1usr_name"]//a'):
co_scholar_id = a.get('href').split('user='******'&hl')[0]
coauthor = Author.query.filter_by(scholar_id=co_scholar_id).first()
if coauthor is None:
coauthor = Author()
coauthor.scholar_id = co_scholar_id
author_coauthors.append(coauthor)
author.coauthors = author_coauthors
# The publications.
author_publications = []
cstart = 0
pagesize = 100
while True:
params = urlencode({
'hl': 'en',
'view_op': 'list_works',
'sortby': 'pubdate',
'user': scholar_id,
'cstart': cstart,
'pagesize': pagesize
})
req = Request(url + '?' + params)
opener.open(req)
res = opener.open(req)
doc = html.parse(res)
for tr in doc.xpath('.//tr[@class="gsc_a_tr"]'):
a = tr.find('.//td[@class="gsc_a_t"]//a')
# NOTE: When there are no publications, there is a single tr.
# <tr class="gsc_a_tr"><td class="gsc_a_e" colspan="3">There are no articles in this profile.</td></tr>
if a is None:
continue
purl = a.get('href')
# The ID of the publication in Google Scholar.
pub_scholar_id = purl.split('citation_for_view=')[1]
# Retrieve the publication with that ID (if any).
publication = Publication.query.filter_by(
scholar_id=pub_scholar_id).first()
if publication is None:
publication = Publication()
publication.scholar_id = pub_scholar_id
# The title of the publication.
publication.title = a.text_content()
pub_nyear = tr.find('.//td[@class="gsc_a_y"]//span')
if pub_nyear is not None:
year_of_publication = pub_nyear.text_content().strip()
if year_of_publication:
# The year of the publication.
publication.year_of_publication = int(year_of_publication)
pub_ncitations = tr.find('.//a[@class="gsc_a_ac"]')
if pub_ncitations is not None:
total_citations = pub_ncitations.text_content().strip()
if total_citations:
# The total citations for the publication.
publication.total_citations = int(total_citations)
author_publications.append(publication)
if doc.xpath('.//button[@id="gsc_bpf_next"]')[0].get("disabled"):
break
cstart += 100
author.publications = author_publications
# When information about the author was retrieved from Google Scholar.
author.retrieved_at = datetime.datetime.now()
db.session.add(author)
db.session.commit()
print 'Crawled author ' + scholar_id + '.'
return 'Done.'
