def setUp(self):
self.testbed = testbed.Testbed()
self.testbed.activate()
self.policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=0)
self.testbed.init_datastore_v3_stub(consistency_policy=self.policy)
self.testbed.init_memcache_stub()
# Silences the logging messages during the tests
ndb.add_flow_exception(ValueError)
ndb.add_flow_exception(IndexError)
def setUp(self):
self.testbed = testbed.Testbed()
self.testbed.activate()
self.policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy(
probability=0)
self.testbed.init_datastore_v3_stub(consistency_policy=self.policy)
self.testbed.init_memcache_stub()
# Silences the logging messages during the tests
ndb.add_flow_exception(ValueError)
ndb.add_flow_exception(IndexError)
from urlparse import urlparse, parse_qs
import json
from flask import url_for
from forms import FeedUpdate, FeedCreate, FeedPreview, InstagramFeedCreate, NoOpForm
from fetcher import fetch_parsed_feed_for_url, fetch_parsed_feed_for_feed, fetch_url
from constants import (ENTRY_STATE, FEED_STATE, FORMAT_MODE, UPDATE_INTERVAL, PERIOD_SCHEDULE, OVERFLOW_REASON,
DEFAULT_PERIOD_SCHEDULE, MAX_STORIES_PER_PERIOD, FEED_TYPE)
from poster import build_html_from_post, format_for_adn, prepare_entry_from_item, instagram_format_for_adn
from utils import get_language, guid_for_item, find_feed_url, fit_to_box
logger = logging.getLogger(__name__)
# Don't complain about this
ndb.add_flow_exception(urlfetch.DeadlineExceededError)
def format_date(dt):
logger.info('dt: %s', dt)
return dt.strftime('%a %b %d %I:%M %p')
class User(ndb.Model):
access_token = ndb.StringProperty()
@classmethod
def key_from_adn_user(cls, adn_user):
return 'adn_user_id=%d' % int(adn_user.id)
class Entry(ndb.Model):
guid = ndb.StringProperty(required=True)
'private_key',
# Service account key fingerprint, an unique identifier of this key.
'private_key_id',
])
class AccessTokenError(Exception):
"""Raised by get_access_token() on fatal or transient errors."""
def __init__(self, msg, transient=False):
super(AccessTokenError, self).__init__(msg)
self.transient = transient
# Do not log AccessTokenError exception raised from a tasklet.
ndb.add_flow_exception(AccessTokenError)
@ndb.tasklet
def get_access_token_async(
scopes, service_account_key=None, act_as=None, min_lifetime_sec=5*60):
"""Returns an OAuth2 access token for a service account.
If 'service_account_key' is specified, will use it to generate access token
for corresponding @*iam.gserviceaccount.com account. Otherwise will invoke
app_identity.get_access_token(...) to use app's @appspot.gserviceaccount.com
account.
If 'act_as' is specified, will return an access token for this account with
given scopes, generating it through a call to signJwt IAM API, using
IAM-scoped access token of a primary service account (an appspot one, or the
def __init__(self, msg, status_code, response):
super(Error, self).__init__(msg)
self.status_code = status_code
self.response = response
class NotFoundError(Error):
"""Raised if endpoint returns 404."""
class AuthError(Error):
"""Raised if endpoint returns 401 or 403."""
# Do not log Error exception raised from a tasklet, it is expected to happen.
ndb.add_flow_exception(Error)
def urlfetch_async(**kwargs):
"""To be mocked in tests."""
return ndb.get_context().urlfetch(**kwargs)
@ndb.tasklet
def request_async(
url,
method='GET',
payload=None,
params=None,
headers=None,
scopes=None,
def __init__(self, msg, status_code, response):
super(Error, self).__init__(msg)
self.status_code = status_code
self.response = response
class NotFoundError(Error):
"""Raised if endpoint returns 404."""
class AuthError(Error):
"""Raised if endpoint returns 401 or 403."""
# Do not log Error exception raised from a tasklet, it is expected to happen.
ndb.add_flow_exception(Error)
def urlfetch_async(**kwargs):
"""To be mocked in tests."""
return ndb.get_context().urlfetch(**kwargs)
def is_transient_error(response, url):
"""Returns True to retry the request."""
if response.status_code >= 500 or response.status_code == 408:
return True
# Retry 404 iff it is a Cloud Endpoints API call *and* the
# result is not JSON. This assumes that we only use JSON encoding.
if response.status_code == 404:
content_type = response.headers.get('Content-Type', '')
from google.appengine.api import urlfetch
from constants import VALID_STATUS
from utils import find_feed_url
logger = logging.getLogger(__name__)
# Monkeypatch feedparser
feedparser._HTMLSanitizer.acceptable_elements = set(list(feedparser._HTMLSanitizer.acceptable_elements) + ["object", "embed", "iframe", "param"])
class FetchException(Exception):
pass
# Don't complain about this
ndb.add_flow_exception(FetchException)
@ndb.tasklet
def fetch_url(url, etag=None, user_agent=None):
# Handle network issues here, handle other exceptions where this is called from
# GAE's built in urlfetch doesn't expose what HTTP Status caused a request to follow
# a redirect. Which is important in this case because on 301 we are suppose to update the
# feed URL in our database. So, we have to write our own follow redirect path here.
max_redirects = 5
redirects = 0
was_permanente_redirect = False
ctx = ndb.get_context()
try:
while redirects < max_redirects:
from google.appengine.api import datastore_errors
from google.appengine.ext import ndb
from components import auth
from components import utils
class WarmupHandler(webapp2.RequestHandler):
def get(self):
auth.warmup()
self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
self.response.write('ok')
assert utils.is_local_dev_server()
auth.disable_process_cache()
# See components/auth/change_log.py, is_changle_log_indexed.
ndb.add_flow_exception(datastore_errors.NeedIndexError)
# Add a fake admin for local dev server.
if not auth.is_replica():
auth.bootstrap_group(
auth.ADMIN_GROUP,
[auth.Identity(auth.IDENTITY_USER, '*****@*****.**')],
'Users that can manage groups')
# /_ah/warmup is used by the smoke test to detect that app is alive.
app = webapp2.WSGIApplication([webapp2.Route(r'/_ah/warmup', WarmupHandler)],
debug=True)
from urlparse import urlparse
import json
from flask import url_for
from application.forms import FeedUpdate, FeedCreate, FeedPreview, InstagramFeedCreate, NoOpForm
from application.fetcher import fetch_url
from application.constants import (FEED_STATE, FORMAT_MODE, UPDATE_INTERVAL,
PERIOD_SCHEDULE, OVERFLOW_REASON, FEED_TYPE,
INBOUND_EMAIL_VERSION)
from application.poster import format_for_adn, instagram_format_for_adn, broadcast_format_for_adn
from entry import Entry
logger = logging.getLogger(__name__)
# Don't complain about this
ndb.add_flow_exception(urlfetch.DeadlineExceededError)
class InstagramFeed(ndb.Model):
"""
Feed URL can just be the API call that we make
https://api.instagram.com/v1/users/3/media/recent/
"""
access_token = ndb.StringProperty()
user_id = ndb.IntegerProperty()
username = ndb.StringProperty()
description = ndb.StringProperty()
added = ndb.DateTimeProperty(auto_now_add=True)
# Posting Schedule By Default will be auto controlled
from utils import find_feed_url
logger = logging.getLogger(__name__)
# Monkeypatch feedparser
feedparser._HTMLSanitizer.acceptable_elements = set(
list(feedparser._HTMLSanitizer.acceptable_elements) +
["object", "embed", "iframe", "param"])
class FetchException(Exception):
pass
# Don't complain about this
ndb.add_flow_exception(FetchException)
@ndb.tasklet
def fetch_url(url, etag=None, user_agent=None):
# Handle network issues here, handle other exceptions where this is called from
# GAE's built in urlfetch doesn't expose what HTTP Status caused a request to follow
# a redirect. Which is important in this case because on 301 we are suppose to update the
# feed URL in our database. So, we have to write our own follow redirect path here.
max_redirects = 5
redirects = 0
was_permanente_redirect = False
ctx = ndb.get_context()
try:
while redirects < max_redirects:
from google.appengine.api import datastore_errors
from google.appengine.ext import ndb
from components import auth
from components import utils
class WarmupHandler(webapp2.RequestHandler):
def get(self):
auth.warmup()
self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
self.response.write('ok')
assert utils.is_local_dev_server()
auth.disable_process_cache()
# See components/auth/change_log.py, is_changle_log_indexed.
ndb.add_flow_exception(datastore_errors.NeedIndexError)
# Add a fake admin for local dev server.
if not auth.is_replica():
auth.bootstrap_group(
auth.ADMIN_GROUP,
[auth.Identity(auth.IDENTITY_USER, '*****@*****.**')],
'Users that can manage groups')
# /_ah/warmup is used by the smoke test to detect that app is alive.
app = webapp2.WSGIApplication(
[webapp2.Route(r'/_ah/warmup', WarmupHandler)], debug=True)
