def get_content_list(per_page=20):
"""
Create a query over ``Content`` objects using query string parameters.
:param per_page: number of items to return per page
:returns: ``QueryResult`` object
"""
search = request.params.getunicode('q', '').strip()
status = request.params.get('status')
license = request.params.get('license')
votes = request.params.get('votes')
page = int(request.params.get('p', '1'))
q = Content.query()
if search:
keywords = Content.get_keywords(search)
if len(keywords) > 1:
q = q.filter(ndb.AND(*[Content.keywords == kw for kw in keywords]))
if len(keywords) == 1:
q = q.filter(Content.keywords == keywords[0])
if status:
q = q.filter(Content.status == status)
if license == 'free':
q = q.filter(Content.is_free == True)
elif license == 'nonfree':
q = q.filter(Content.is_free == False)
elif license == 'unknown':
q = q.filter(Content.license == None)
if votes == 'asc':
q = q.order(+Content.votes)
elif votes == 'desc':
q = q.order(-Content.votes)
q = q.order(-Content.updated)
count = q.count()
if not count:
return QueryResult([], count, 1, 1)
npages = int(math.ceil(count / per_page))
if page * per_page > count:
page = npages
offset = int(per_page * (page - 1))
return QueryResult(q.fetch(per_page, offset=offset), count, page, npages)
def get_content_list(per_page=20):
"""
Create a query over ``Content`` objects using query string parameters.
:param per_page: number of items to return per page
:returns: ``QueryResult`` object
"""
search = request.params.getunicode('q', '').strip()
status = request.params.get('status')
license = request.params.get('license')
votes = request.params.get('votes')
page = int(request.params.get('p', '1'))
q = Content.query()
if search:
keywords = Content.get_keywords(search)
if len(keywords) > 1:
q = q.filter(ndb.AND(*[Content.keywords == kw for kw in keywords]))
if len(keywords) == 1:
q = q.filter(Content.keywords == keywords[0])
if status:
q = q.filter(Content.status == status)
if license == 'free':
q = q.filter(Content.is_free == True)
elif license == 'nonfree':
q = q.filter(Content.is_free == False)
elif license == 'unknown':
q = q.filter(Content.license == None)
if votes == 'asc':
q = q.order(+Content.votes)
elif votes == 'desc':
q = q.order(-Content.votes)
q = q.order(-Content.updated)
count = q.count()
if not count:
return QueryResult([], count, 1, 1)
npages = int(math.ceil(count / per_page))
if page * per_page > count:
page = npages
offset = int(per_page * (page - 1))
return QueryResult(q.fetch(per_page, offset=offset), count, page, npages)
def create_post():
try:
post = Content(title=request.form['title'],
content=request.form['content'],
author=get_current_user().username,
create_at=get_timestamp())
post.save()
return jsonify({'success': 1, 'post_id': str(post._id)})
except ValidationError as e:
logger.warning('Failed to create new post. Exception: {}'.format(
e.message))
return jsonify({
'success':
0,
'msg':
'{} : {}'.format(error_msg.ILLEGAL_ARGUMENT,
','.join(list(e.message.keys())))
}), 400
def get_content_or_404(id):
"""
Return content entity or abort with HTTP 404 if none is found.
:param id: id of the content
:returns: ``Content`` object
"""
content = Content.get_cached(id)
if not content:
abort(404)
return content
def get_content_or_404(id):
"""
Return content entity or abort with HTTP 404 if none is found.
:param id: id of the content
:returns: ``Content`` object
"""
content = Content.get_cached(id)
if not content:
abort(404)
return content
def get_common_context(extra_context={}):
"""
Return base context for handlers in this module
"""
sel = request.params.get('select', '0') == '1'
ctx = dict(per_page=PER_PAGE_CHOICES,
votes=VOTE_CHOICES,
notes=NOTES_CHOICES,
licenses=LICENSE_CHOICES,
content=get_content_list(),
vals=request.params,
sel=sel,
css='admin',
total_count=Content.query().count())
ctx.update(extra_context)
return ctx
def handle_manual_add():
url = request.params.get('url', '').strip()
title = request.params.getunicode('title', '').strip()
license = request.params.get('license') or None
archive = request.params.get('archive') or None
errors = {}
if not url:
# Translators, used as error message on failure to submit content
errors['url'] = _('Please type in a valid URL')
if not errors:
try:
content = Content.create(url=url,
license=license,
title=title,
archive=archive)
logging.info("Created content for '%s' (real url: '%s')", url,
content.url)
response.flash(_('Content has been added'))
redirect(i18n_path(PREFIX + '/'))
except Content.InvalidURLError as err:
logging.debug("URL error while parsing '%s': %s", url, err)
# Translators, used as error message on failure submit suggestion
errors['url'] = _('This URL is invalid')
except Content.FetchError as err:
logging.debug("Fetch error while parsing '%s': %s (%s)", url, err,
err.error)
# Translators, used as error message on failure submit suggestion
errors['url'] = _('The page at specified URL does not exist')
except Content.NotAllowedError as err:
logging.debug("Access error while parsing '%s': %s", url, err)
# Translators, used as error message on failure submit suggestion
errors['url'] = _('The page must be accessible to robots')
except Content.ContentError as err:
logging.debug("Content error while parsing '%s': %s (%s)", url,
err, err.error)
# Translators, used as error message on failure submit suggestion
errors['url'] = _('The content on the page could not be '
'understood, please provide and URL to a valid '
'web page')
except Content.BotError as err:
logging.exception("Error while fetching '%s': %s", url, err)
# Translators, used as error message on failure submit suggestion
errors['url'] = _('There was an unknown error with the URL')
return get_common_context(dict(vals=request.forms, errors=errors))
def bulk_create(rows, check=False):
content = []
logging.info('Enqueued bulk import with %s rows', len(rows))
for url, title, license, archive, partner, timestamp, replaces, notes in rows:
ts = datetime.strptime(timestamp, '%Y-%m-%d %H:%M:%S %Z')
try:
content.append(
Content.create(
url=url,
title=title,
license=license or None,
archive=archive or None,
created=ts,
replaces=replaces or None,
notes=notes or None,
partner=partner or None,
is_partner=partner != '',
check_url=check and not url.startswith('outernet://'),
override_timestamp=ts,
))
except Exception as err:
logging.exception("Error while adding content with URL '%s'", url)
logging.debug('Successfully importent following:\n\n%s',
'\n'.join([c.key.id() for c in content]))
def get_content_list():
"""
Create a query over ``Content`` objects using query string parameters.
:param per_page: number of items to return per page
:returns: ``QueryResult`` object
"""
search = request.params.getunicode('q', '').strip()
archive = request.params.get('archives')
license = request.params.get('license')
votes = request.params.get('votes')
notes = request.params.get('notes')
try:
page = int(request.params.get('p', '1'))
except ValueError:
page = 1
try:
per_page = int(request.params.get('pp', '20'))
except ValueError:
per_page = 10
if per_page not in ALLOWED_PER_PAGE:
per_page = 10
q = Content.query()
if search:
keywords = Content.get_keywords(search)
q = q.filter(ndb.AND(*[Content.keywords == kw for kw in keywords]))
if archive:
q = q.filter(Content.archive == archive)
if license:
if license == 'NONE':
q = q.filter(Content.license == None)
elif license == 'FREE':
q = q.filter(Content.license.IN(Content.FREE_LICENSES))
elif license == 'NONFREE':
q = q.filter(Content.license.IN(Content.NONFREE_LICENSES))
else:
q = q.filter(Content.license == license)
if votes == 'asc':
q = q.order(+Content.votes)
elif votes == 'desc':
q = q.order(-Content.votes)
elif votes == 'cont':
q = q.filter(
ndb.AND(Content.votes_ratio <= 1.2, Content.votes_ratio >= 0.8))
q = q.order(-Content.votes_ratio)
if notes == '1':
q = q.filter(Content.has_notes == True)
elif notes == '0':
q = q.filter(Content.has_notes == False)
q = q.order(-Content.updated)
count = q.count()
if not count:
return QueryResult([], count, 1, 1)
npages = int(math.ceil(count / per_page))
if page * per_page > count:
page = npages
offset = int(per_page * (page - 1))
items = q.fetch(per_page, offset=offset)
return QueryResult(items, count, per_page, page)
def get_content_or_404(urlid):
content = Content.get_cached(urlid)
if not content:
abort(404)
return content
def add_content_suggestion():
"""
Handle a content suggestion request.
"""
# TODO: Handle Unicode URLs
url = Content.validate_url(request.forms.get('url', ''))
license = request.forms.get('license') or None
errors = {}
if not url:
# Translators, used as error message on failure submit suggestion
errors['url'] = _('This URL is invalid')
if license:
license = license.strip().upper()
if license not in Content.LICENSE_CHOICES:
# Translators, used as error message on failure to submit
# suggestion
errors['license'] = _('Please select a license from provided '
'choices')
if not url:
# Translators, used as error message on failure to submit suggestion
errors['url'] = _('Please type in a valid URL')
if not errors:
try:
content = Content.create(url=url, license=license)
logging.info("Created content for '%s' (real url: '%s')", url,
content.url)
response.flash(_('Your suggestion has been added'))
redirect(i18n_path(content.path))
except Content.InvalidURLError as err:
logging.debug("URL error while parsing '%s': %s", url, err)
# Translators, used as error message on failure submit suggestion
errors['url'] = _('This URL is invalid')
except Content.FetchError as err:
logging.debug("Fetch error while parsing '%s': %s (%s)", url, err,
err.error)
# Translators, used as error message on failure submit suggestion
errors['url'] = _('The page at specified URL does not exist or '
'the domain cannot be reached.')
except Content.NotAllowedError as err:
logging.debug("Access error while parsing '%s': %s", url, err)
# Translators, used as error message on failure submit suggestion
errors['url'] = _('The page must be accessible to robots')
except Content.ContentError as err:
logging.debug("Content error while parsing '%s': %s (%s)", url,
err, err.error)
# Translators, used as error message on failure submit suggestion
errors['url'] = _('The content on the page could not be '
'understood, please provide and URL to a valid '
'web page')
except Exception as err:
logging.debug("Unknown error fetching '%s': %s", url, err)
# Translators, used as error message on failure submit suggestion
errors['url'] = _('There was an unknown error with the URL')
return dict(vals=request.forms,
errors=errors,
Content=Content,
content=get_content_list())
def add_content_suggestion():
"""
Handle a content suggestion request.
"""
# TODO: Handle Unicode URLs
url = Content.validate_url(request.forms.get('url', ''))
license = request.forms.get('license') or None
errors = {}
if not url:
# Translators, used as error message on failure submit suggestion
errors['url'] = _('This URL is invalid')
if license:
license = license.strip().upper()
if license not in Content.LICENSE_CHOICES:
# Translators, used as error message on failure to submit
# suggestion
errors['license'] = _('Please select a license from provided '
'choices')
if not url:
# Translators, used as error message on failure to submit suggestion
errors['url'] = _('Please type in a valid URL')
if not errors:
try:
content = Content.create(url=url, license=license)
logging.info("Created content for '%s' (real url: '%s')", url,
content.url)
response.flash(_('Your suggestion has been added'))
redirect(i18n_path(content.path))
except Content.InvalidURLError as err:
logging.debug("URL error while parsing '%s': %s", url, err)
# Translators, used as error message on failure submit suggestion
errors['url'] = _('This URL is invalid')
except Content.FetchError as err:
logging.debug("Fetch error while parsing '%s': %s (%s)",
url, err, err.error)
# Translators, used as error message on failure submit suggestion
errors['url'] = _('The page at specified URL does not exist or '
'the domain cannot be reached.')
except Content.NotAllowedError as err:
logging.debug("Access error while parsing '%s': %s", url, err)
# Translators, used as error message on failure submit suggestion
errors['url'] = _('The page must be accessible to robots')
except Content.ContentError as err:
logging.debug("Content error while parsing '%s': %s (%s)", url,
err, err.error)
# Translators, used as error message on failure submit suggestion
errors['url'] = _('The content on the page could not be '
'understood, please provide and URL to a valid '
'web page')
except Exception as err:
logging.debug("Unknown error fetching '%s': %s", url, err)
# Translators, used as error message on failure submit suggestion
errors['url'] = _('There was an unknown error with the URL')
return dict(vals=request.forms, errors=errors, Content=Content,
content=get_content_list())
