def update_num_comment(action, boker_key, current_user):
""" Update number of comment per Boker """
boker = Boker.get(boker_key)
if boker:
if action=='inc_comment':
boker.num_comment += 1
boker.put()
# Notify Boker owner
notify_user = True
if current_user:
if current_user['id'] == boker.user.id:
notify_user = False
if notify_user and not settings.DEBUG:
try:
boker_url = encode_url("/boker/%s" % boker.key().id())
graph = facebook.GraphAPI(settings.FACEBOOK_APP_ACCESS_TOKEN)
graph.request('%s/notifications' % boker.user.id,
post_args={
'href': '?to='+boker_url,
'template': 'Ada komentar baru untuk boker anda.',
})
except Exception as e:
print e
else:
logging.info('Runtask: notify new comment...')
if action=='dec_comment':
if boker.num_comment > 0:
boker.num_comment -= 1
boker.put()
def institution_detail(iid):
institution_cursor = g.conn.execute(text("""
SELECT *
FROM Institutions I
WHERE I.iid = :iid
"""), iid=iid)
details = institution_cursor.fetchone()
authors_cursor = g.conn.execute(text("""
SELECT A.aid, A.first_name, A.last_name
FROM Authors A
INNER JOIN Works_At WA ON WA.aid = A.aid
INNER JOIN Institutions I ON I.iid = WA.iid
WHERE I.iid = :iid
ORDER BY A.last_name
"""), iid=iid)
authors = authors_cursor.fetchall()
p_cursor = g.conn.execute(text("""
SELECT DISTINCT P.title, P.purl, P.number_of_citations
FROM Papers P
INNER JOIN Published_By PB ON PB.purl = P.purl
INNER JOIN Authors A ON PB.aid = A.aid
INNER JOIN Works_At WA ON WA.aid = A.aid
INNER JOIN Institutions I ON I.iid = WA.iid
WHERE I.iid = :iid
ORDER BY P.number_of_citations DESC
"""), iid=iid)
papers = []
for p in p_cursor:
papers.append({'title': p.title, 'purl': utils.encode_url(p.purl), 'citations': p.number_of_citations})
context = {'details': details, 'authors': authors, 'papers': papers}
institution_cursor.close()
authors_cursor.close()
p_cursor.close()
return render_template('institution_details.html', **context)
def search_term(search):
string_match = '%%' + search.upper().replace(' ', '%%') + '%%'
cursor = g.conn.execute(text("""WITH FullTable AS
(SELECT P.purl, P.title, P.model, K.keyword, A.first_name,
A.last_name, I.type, I.name, I.country, I.city
FROM Papers P
LEFT OUTER JOIN Is_Related_To IRT ON P.purl = IRT.purl
LEFT OUTER JOIN Keywords K ON IRT.keyword = K.keyword
LEFT OUTER JOIN Published_By PB ON P.purl = PB.purl
LEFT OUTER JOIN Authors A ON PB.aid = A.aid
LEFT OUTER JOIN Works_At WA ON WA.aid = A.aid
LEFT OUTER JOIN Institutions I ON I.iid = WA.iid)
SELECT DISTINCT FT.title, FT.purl
FROM FullTable FT
WHERE
upper(FT.title) LIKE :string_match OR
upper(FT.model) LIKE :string_match OR
upper(FT.keyword) LIKE :string_match OR
upper(FT.first_name) LIKE :string_match OR
upper(FT.last_name) LIKE :string_match OR
upper(FT.name) LIKE :string_match OR
upper(FT.type) LIKE :string_match OR
upper(FT.country) LIKE :string_match OR
upper(FT.city) LIKE :string_match;"""), string_match=string_match)
results = []
for r in cursor:
results.append({'title': r.title, 'purl': utils.encode_url(r.purl)})
return results
def advanced():
search_form = AdvancedSearchForm()
if search_form.validate_on_submit():
if search_form.title:
cursor = g.conn.execute(text("""
WITH FullTable AS
(SELECT P.purl, P.title, P.model, P.number_of_citations, R.programming_language, K.keyword, A.first_name,
A.last_name, I.type, I.name, I.country, I.city, I.street, I.street_number, I.zip, R.rdate_published, P.date_published
FROM Papers P
LEFT OUTER JOIN Published_On PO ON P.purl = PO.purl
LEFT OUTER JOIN Repositories R ON PO.url = R.url
LEFT OUTER JOIN Is_Related_To IRT ON PO.purl = IRT.purl
LEFT OUTER JOIN Keywords K ON IRT.keyword = K.keyword
LEFT OUTER JOIN Published_By PB ON P.purl = PB.purl
LEFT OUTER JOIN Authors A ON PB.aid = A.aid
LEFT OUTER JOIN Works_At WA ON WA.aid = A.aid
LEFT OUTER JOIN Institutions I ON I.iid = WA.iid)
SELECT DISTINCT FT.title, FT.purl, FT.programming_language, FT.rdate_published
FROM FullTable FT
WHERE
upper(FT.title) LIKE '%%' || :title || '%%' AND
upper(FT.model) LIKE '%%' || :model || '%%' AND
FT.date_published >= :pdate AND
FT.number_of_citations >= :citations AND
upper(FT.first_name) LIKE '%%' || :first || '%%' AND
upper(FT.last_name) LIKE '%%' || :last || '%%' AND
upper(FT.name) LIKE '%%' || :institution || '%%' AND
FT.type IN :insttype AND
upper(FT.country) LIKE '%%' || :instcountry || '%%' AND
upper(FT.city) LIKE '%%' || :instcity || '%%' AND
upper(FT.zip) LIKE '%%' || :instzip || '%%' AND
upper(FT.street) LIKE '%%' || :inststreet || '%%' AND
upper(FT.street_number) LIKE '%%' || :instno || '%%';
"""), title=search_form.title.data.upper(), model=search_form.model.data.upper(),
pdate=str(
search_form.published_year.data if search_form.published_year.data else 1900) + '01' + '01',
citations=search_form.minimum_citations.data if search_form.minimum_citations.data else 0,
prog=search_form.repo_programming_language.data.upper(),
rdate=str(
search_form.repo_published_year.data if search_form.repo_published_year.data else 1900) + '01' + '01',
first=search_form.author_first_name.data.upper(),
last=search_form.author_last_name.data.upper(), institution=search_form.inst_name.data.upper(),
insttype=tuple(search_form.inst_type.data.split(' ')),
instcountry=search_form.inst_country.data.upper(), instcity=search_form.inst_city.data.upper(),
instzip=search_form.inst_zip.data.upper(),
inststreet=search_form.inst_street.data.upper(), instno=search_form.inst_street_no.data.upper())
results = []
for r in cursor:
if (search_form.repo_programming_language.data != '' and
(not r.programming_language or
r.programming_language.upper() != search_form.repo_programming_language.data.upper())):
continue
if (search_form.repo_published_year.data is not None and
(not r.rdate_published or
r.rdate_published.year < search_form.repo_published_year.data)):
continue
results.append({'title': r.title, 'purl': utils.encode_url(r.purl)})
return render_template('advancedsearch.html', results=results)
return render_template('advanced.html', form=search_form)
async def search(query, obj_type='track', limit=5):
if not var.spotify_token or time() > var.spotify_token_expires:
await authorize()
data = {'type': obj_type, 'limit': limit, 'q': query}
headers = {'Authorization': f'Bearer {var.spotify_token}'}
r = await request_get(encode_url(
'https://api.spotify.com/v1/search', data=data), headers=headers)
json = await r.json()
return [AttrDict(track) for track in json['tracks']['items']]
def index(request):
context = RequestContext(request)
category_list = Category.objects.order_by('-likes')[:5]
pages_list = Page.objects.order_by('-views')[:5]
context_dict = {'categories':category_list,
'pages':pages_list}
for category in category_list:
category.url = encode_url(category.name)
return render_to_response('rango/index.html', context_dict, context)
async def search(obj='track', q=''):
encoded_url = utils.encode_url(f'{api_url}/search/{obj}/', {'q': q, 'limit': 50})
results = await (await var.session.get(encoded_url)).json()
try:
if obj == 'artist':
result = [Artist(result) for result in results['data']]
elif obj == 'album':
result = [Album(result) for result in results['data']]
else:
result = [AttrDict(result) for result in results['data']]
return result or []
except KeyError:
print(obj)
async def search(self, query, obj_type='track', limit=5):
if self.expires_in < time():
self.restart()
data = {'type': obj_type, 'limit': limit, 'q': query}
headers = {'Authorization': f'Bearer {self.token}'}
r = await var.session.get(encode_url(
'https://api.spotify.com/v1/search', data=data), headers=headers)
json = await r.json(content_type=None)
result = []
if json['tracks']['total'] != 0:
for item in json['tracks']['items']:
result.append(AttrDict(item))
return result
def create_html_attachment_index(attachments):
""" Creates a HTML list for a list of attachments.
:param attachments: List of attachments.
:returns: Attachment list as HTML.
"""
html_content = '\n\n<h2>Attachments</h2>'
if len(attachments) > 0:
html_content += '<ul>\n'
for attachment in attachments:
relative_file_path = '/'.join(attachment['file_path'].split('/')[2:])
relative_file_path = utils.encode_url(relative_file_path)
html_content += '\t<li><a href="%s">%s</a></li>\n' % (relative_file_path, attachment['file_name'])
html_content += '</ul>\n'
return html_content
def create_html_index(index_content):
""" Creates an HTML index (mainly to navigate through the exported pages).
:param index_content: Dictionary which contains file paths, page titles and their children recursively.
:returns: Content index as HTML.
"""
file_path = utils.encode_url(index_content['file_path'])
page_title = index_content['page_title']
page_children = index_content['child_pages']
html_content = '<a href="%s">%s</a>' % (utils.sanitize_for_filename(file_path), page_title)
if len(page_children) > 0:
html_content += '<ul>\n'
for child in page_children:
html_content += '\t<li>%s</li>\n' % create_html_index(child)
html_content += '</ul>\n'
return html_content
def index():
global USER
search_form = SearchForm()
recommendations = recommender()
models = g.conn.execute("""
SELECT DISTINCT P.model FROM Papers P;
""").fetchall()
applications = ['security', 'finance', 'medicine', 'recognition']
context = {
'title': 'Home',
'search_form': search_form,
'user': USER['first_name'] if USER else '',
'recommendations': recommendations,
'models': models,
'applications': applications,
}
if search_form.validate_on_submit():
return redirect('/results/' + utils.encode_url(search_form.searchTerms.data))
return render_template('index.html', **context)
def like_boker(user_key, boker_key, explicitly_shared=False):
"""Like a boker"""
user = User.get_by_key_name(user_key)
boker = Boker.get(boker_key)
boker_owner = boker.user
if user and boker and not Like.already_like(user, boker):
# Create like
like = Like(user=user, boker=boker)
like.put()
# Update like numbers
boker.num_like += 1
boker.put()
if not settings.DEBUG:
# Post to FB
try:
boker_url = "%s/boker/%s" % (settings.APP_DOMAIN, boker.key().id())
graph = facebook.GraphAPI(user.access_token)
graph.request('me/og.likes',
post_args={'object': boker_url, 'fb:explicitly_shared': str(explicitly_shared).lower()})
except Exception as e:
print e
# Notify Boker owner
if user != boker_owner:
try:
boker_url = encode_url("/boker/%s" % boker.key().id())
graph = facebook.GraphAPI(settings.FACEBOOK_APP_ACCESS_TOKEN)
graph.request('%s/notifications' % boker_owner.id,
post_args={
'href': '?to=' + boker_url,
'template': '@[%s] menyukai boker anda.' % user.id,
})
except Exception as e:
print e
else:
logging.info('Runtask: post_like_story...')
def author_detail(aid):
cursor = g.conn.execute(text("""
SELECT A.first_name, A.last_name, I.name as inst_name, I.iid
FROM Authors A INNER JOIN Works_At WA ON WA.aid = A.aid
INNER JOIN Institutions I ON I.iid = WA.iid
WHERE A.aid = :aid
"""), aid=aid)
details = cursor.fetchone()
cursor = g.conn.execute(text("""
SELECT DISTINCT P.purl, P.title, P.number_of_citations
FROM Authors A INNER JOIN Published_By PB ON A.aid = PB.aid
INNER JOIN Papers P ON P.purl = PB.purl
WHERE A.aid = :aid
ORDER BY P.number_of_citations DESC
"""), aid=aid)
papers = []
for p in cursor:
new_p = {'purl': utils.encode_url(p.purl), 'title': p.title, 'citations': p.number_of_citations}
papers.append(new_p)
context = {'author': details, 'papers': papers}
return render_template('author_details.html', **context)
def recommender():
global USER
if not USER:
return []
s = text("""
SELECT P1.title, P1.purl
FROM Papers P1 NATURAL JOIN Is_Related_To I
WHERE P1.purl = I.purl AND I.keyword IN (SELECT IR.keyword FROM Is_Related_To IR WHERE IR.purl IN (SELECT P.purl
FROM Papers P NATURAL JOIN Have_Read HR
WHERE P.purl = HR.purl AND HR.user_name = :user_name))
EXCEPT
SELECT SUB.title, SUB.purl
FROM (SELECT * FROM Papers P2 NATURAL JOIN Have_Read HR2
WHERE P2.purl = HR2.purl AND HR2.user_name = :user_name
ORDER BY HR2.date) AS SUB
LIMIT 3;
""")
cursor = g.conn.execute(s, user_name=USER['user_name'])
recommendations = []
for r in cursor:
recommendations.append({'title': r.title, 'purl': utils.encode_url(r.purl)})
return recommendations
def my_account():
global USER
if not USER:
return redirect('/login')
user_cursor = g.conn.execute(text("""
SELECT *
FROM Users U
WHERE U.user_name = :username;
"""), username=USER['user_name'])
user = user_cursor.fetchone()
history_cursor = g.conn.execute(text("""
SELECT P.purl, P.title, HR.date
FROM Have_Read HR INNER JOIN Papers P ON P.purl = HR.purl
WHERE HR.user_name = :username
ORDER BY HR.date DESC
"""), username=USER['user_name'])
recommendations = recommender()
history = []
for h in history_cursor:
h = {'title': h.title, 'purl': utils.encode_url(h.purl), 'date': h.date}
history.append(h)
context = {'user': user, 'history': history, 'recommendations': recommendations}
return render_template("my_account.html", **context)
def post(self):
to = self.request.get('to', encode_url('/'))
# self.redirect(self.uri_for('onfb') + '?next=' + to)
self.redirect(decode_url(to))
def handle_html_references(html_content,
page_duplicate_file_names,
page_file_matching,
depth=0):
""" Repairs links in the page contents with local links.
:param html_content: Confluence HTML content.
:param page_duplicate_file_names: A dict in the structure {'<sanitized filename>': amount of duplicates}
:param page_file_matching: A dict in the structure {'<page title>': '<used offline filename>'}
:param depth: (optional) Hierarchy depth of the handled Confluence page.
:returns: Fixed HTML content.
"""
try:
html_tree = html.fromstring(html_content)
except ParserError:
print('page is empty')
return html_content
except XMLSyntaxError:
print(
'%sWARNING: Could not parse HTML content of last page. Original content will be downloaded as it is.'
% ('\t' * (depth + 1)))
return html_content
# Fix links to other Confluence pages
# Example: /display/TES/pictest1
# => pictest1.html
# TODO: This code does not work for "Recent space activity" areas in space pages because of a different url format.
xpath_expr = '//a[contains(@href, "/display/")]'
for link_element in html_tree.xpath(xpath_expr):
if not link_element.get('class'):
page_title = link_element.attrib['href'].split('/')[3]
page_title = page_title.replace('+', ' ')
decoded_page_title = utils.decode_url(page_title)
offline_link = provide_unique_file_name(
page_duplicate_file_names,
page_file_matching,
decoded_page_title,
explicit_file_extension='html')
link_element.attrib['href'] = utils.encode_url(offline_link)
# Fix links to other Confluence pages when page ids are used
xpath_expr = '//a[contains(@href, "/pages/viewpage.action?pageId=")]'
for link_element in html_tree.xpath(xpath_expr):
if not link_element.get('class'):
page_id = link_element.attrib['href'].split(
'/pages/viewpage.action?pageId=')[1]
offline_link = '%s.html' % utils.sanitize_for_filename(page_id)
link_element.attrib['href'] = utils.encode_url(offline_link)
# Fix attachment links
xpath_expr = '//a[contains(@class, "confluence-embedded-file")]'
for link_element in html_tree.xpath(xpath_expr):
file_url = link_element.attrib['href']
file_name = derive_downloaded_file_name(file_url)
relative_file_path = '%s/%s' % (settings.DOWNLOAD_SUB_FOLDER,
file_name)
#link_element.attrib['href'] = utils.encode_url(relative_file_path)
link_element.attrib['href'] = relative_file_path
# Fix file paths for img tags
# TODO: Handle non-<img> tags as well if necessary.
# TODO: Support files with different versions as well if necessary.
possible_image_xpaths = [
'//img[contains(@src, "/download/")]',
'//img[contains(@src, "/rest/documentConversion/latest/conversion/thumbnail/")]'
]
xpath_expr = '|'.join(possible_image_xpaths)
for img_element in html_tree.xpath(xpath_expr):
# Replace file path
file_url = img_element.attrib['src']
file_name = derive_downloaded_file_name(file_url)
relative_file_path = '%s/%s' % (settings.DOWNLOAD_SUB_FOLDER,
file_name)
img_element.attrib['src'] = relative_file_path
# Add alt attribute if it does not exist yet
if not 'alt' in img_element.attrib.keys():
img_element.attrib['alt'] = relative_file_path
return html.tostring(html_tree)
def fetch_page_recursively(page_id,
folder_path,
download_folder,
html_template,
depth=0,
page_duplicate_file_names=None,
page_file_matching=None,
attachment_duplicate_file_names=None,
attachment_file_matching=None):
""" Fetches a Confluence page and its child pages (with referenced downloads).
:param page_id: Confluence page id.
:param folder_path: Folder to place downloaded pages in.
:param download_folder: Folder to place downloaded files in.
:param html_template: HTML template used to export Confluence pages.
:param depth: (optional) Hierarchy depth of the handled Confluence page.
:param page_duplicate_file_names: A dict in the structure {'<sanitized page filename>': amount of duplicates}
:param page_file_matching: A dict in the structure {'<page title>': '<used offline filename>'}
:param attachment_duplicate_file_names: A dict in the structure {'<sanitized attachment filename>': amount of \
duplicates}
:param attachment_file_matching: A dict in the structure {'<attachment title>': '<used offline filename>'}
:returns: Information about downloaded files (pages, attachments, images, ...) as a dict (None for exceptions)
"""
if not page_duplicate_file_names:
page_duplicate_file_names = {}
if not page_file_matching:
page_file_matching = {}
if not attachment_duplicate_file_names:
attachment_duplicate_file_names = {}
if not attachment_file_matching:
attachment_file_matching = {}
page_url = '%s/rest/api/content/%s?expand=children.page,children.attachment,body.view.value' \
% (settings.CONFLUENCE_BASE_URL, page_id)
try:
response = utils.http_get(
page_url,
auth=settings.HTTP_AUTHENTICATION,
headers=settings.HTTP_CUSTOM_HEADERS,
verify_peer_certificate=settings.VERIFY_PEER_CERTIFICATE,
proxies=settings.HTTP_PROXIES)
page_content = response['body']['view']['value']
page_title = response['title']
print('%sPAGE: %s (%s)' % ('\t' * (depth + 1), page_title, page_id))
# Construct unique file name
file_name = provide_unique_file_name(page_duplicate_file_names,
page_file_matching,
str(page_id),
explicit_file_extension='html')
# Remember this file and all children
path_collection = {
'file_path': file_name,
'page_title': page_title,
'child_pages': [],
'child_attachments': []
}
# Download attachments of this page
# TODO: Outsource/Abstract the following two while loops because of much duplicate code.
page_url = '%s/rest/api/content/%s/child/attachment?limit=25' % (
settings.CONFLUENCE_BASE_URL, page_id)
counter = 0
while page_url:
response = utils.http_get(
page_url,
auth=settings.HTTP_AUTHENTICATION,
headers=settings.HTTP_CUSTOM_HEADERS,
verify_peer_certificate=settings.VERIFY_PEER_CERTIFICATE,
proxies=settings.HTTP_PROXIES)
counter += len(response['results'])
for attachment in response['results']:
download_url = attachment['_links']['download']
attachment_id = attachment['id'][3:]
attachment_info = download_attachment(
download_url,
download_folder,
attachment_id,
attachment_duplicate_file_names,
attachment_file_matching,
depth=depth + 1)
path_collection['child_attachments'].append(attachment_info)
if 'next' in response['_links'].keys():
page_url = response['_links']['next']
page_url = '%s%s' % (settings.CONFLUENCE_BASE_URL, page_url)
else:
page_url = None
# Export HTML file
page_content = handle_html_references(page_content,
page_duplicate_file_names,
page_file_matching,
depth=depth + 1)
file_path = '%s/%s' % (folder_path, file_name)
page_content += create_html_attachment_index(
path_collection['child_attachments'])
utils.write_html_2_file(file_path, page_title, page_content,
html_template)
# Save another file with page id which forwards to the original one
id_file_path = '%s/%s.html' % (folder_path, page_id)
id_file_page_title = 'Forward to page %s' % page_title
original_file_link = utils.encode_url(
utils.sanitize_for_filename(file_name))
# id_file_page_content = settings.HTML_FORWARD_MESSAGE % (original_file_link, page_title)
# id_file_forward_header = '<meta http-equiv="refresh" content="0; url=%s" />' % original_file_link
# utils.write_html_2_file(id_file_path, id_file_page_title, id_file_page_content, html_template,
# additional_headers=[id_file_forward_header])
# Iterate through all child pages
page_url = '%s/rest/api/content/%s/child/page?limit=25' % (
settings.CONFLUENCE_BASE_URL, page_id)
counter = 0
while page_url:
response = utils.http_get(
page_url,
auth=settings.HTTP_AUTHENTICATION,
headers=settings.HTTP_CUSTOM_HEADERS,
verify_peer_certificate=settings.VERIFY_PEER_CERTIFICATE,
proxies=settings.HTTP_PROXIES)
counter += len(response['results'])
for child_page in response['results']:
paths = fetch_page_recursively(
child_page['id'],
folder_path,
download_folder,
html_template,
depth=depth + 1,
page_duplicate_file_names=page_duplicate_file_names,
page_file_matching=page_file_matching)
if paths:
path_collection['child_pages'].append(paths)
if 'next' in response['_links'].keys():
page_url = response['_links']['next']
page_url = '%s%s' % (settings.CONFLUENCE_BASE_URL, page_url)
else:
page_url = None
return path_collection
except utils.ConfluenceException as e:
error_print('%sERROR: %s' % ('\t' * (depth + 1), e))
return None
