def run_deferred_tasks(self, queue='default'):
tasks = self.taskqueue_stub.GetTasks(queue)
while tasks:
self.taskqueue_stub.FlushQueue(queue)
for task in tasks:
deferred.run(base64.b64decode(task['body']))
tasks = self.taskqueue_stub.GetTasks(queue)
def deferred(request):
from google.appengine.ext.deferred.deferred import (
run,
SingularTaskFailure,
PermanentTaskFailure
)
response = HttpResponse()
if 'HTTP_X_APPENGINE_TASKEXECUTIONCOUNT' in request.META:
logger.debug("[DEFERRED] Retry %s of deferred task", request.META['HTTP_X_APPENGINE_TASKEXECUTIONCOUNT'])
if 'HTTP_X_APPENGINE_TASKNAME' not in request.META:
logger.critical('Detected an attempted XSRF attack. The header "X-AppEngine-Taskname" was not set.')
response.status_code = 403
return response
in_prod = environment.is_production_environment()
if in_prod and os.environ.get("REMOTE_ADDR") != "0.1.0.2":
logger.critical('Detected an attempted XSRF attack. This request did not originate from Task Queue.')
response.status_code = 403
return response
try:
run(request.body)
except SingularTaskFailure:
logger.debug("Failure executing task, task retry forced")
response.status_code = 408
except PermanentTaskFailure:
logger.exception("Permanent failure attempting to execute task")
return response
def deferred(request):
from google.appengine.ext.deferred.deferred import (
run,
SingularTaskFailure,
PermanentTaskFailure
)
response = HttpResponse()
if 'HTTP_X_APPENGINE_TASKEXECUTIONCOUNT' in request.META:
logger.debug("[DEFERRED] Retry %s of deferred task", request.META['HTTP_X_APPENGINE_TASKEXECUTIONCOUNT'])
if 'HTTP_X_APPENGINE_TASKNAME' not in request.META:
logger.critical('Detected an attempted XSRF attack. The header "X-AppEngine-Taskname" was not set.')
response.status_code = 403
return response
in_prod = environment.is_production_environment()
if in_prod and os.environ.get("REMOTE_ADDR") != "0.1.0.2":
logger.critical('Detected an attempted XSRF attack. This request did not originate from Task Queue.')
response.status_code = 403
return response
try:
run(request.body)
except SingularTaskFailure:
logger.debug("Failure executing task, task retry forced")
response.status_code = 408
except PermanentTaskFailure:
logger.exception("Permanent failure attempting to execute task")
return response
def process_request(self, req):
"""
Process the request
"""
if req.method == 'POST':
headers = ["%s:%s" % (k, v) for k, v in req._inheaders
if k.lower().startswith("x-appengine-")]
logging.info(", ".join(headers))
try:
run(req.read())
except PermanentTaskFailure, e:
logging.exception("Permanent failure attempting to execute task")
def __exit__(self, type, value, traceback):
from google.appengine.ext.deferred import deferred
queues = (queue['name'] for queue in self.stub.GetQueues())
for queue in queues:
tasks = self.stub.get_filtered_tasks(queue_names=[queue])
tasks = (task for task in tasks if 'deferred' in task.url)
for task in tasks:
deferred.run(task.payload)
self.stub.DeleteTask(queue, task.name)
def test_invalid_update_campaign(self):
self.policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=0)
self.testbed.init_datastore_v3_stub(consistency_policy=self.policy)
# create new campaign
response = self.admin_app.post("/api/admin/campaign", params=json.dumps(self.CAMPAIGN_SAMPLE),
headers=self.ADMIN_HEADERS)
campaign = json.loads(response.body)
campaign_id = campaign["id"]
campaign_url = response.headers["Location"]
# simulate a user click
self.tracker_app.get('/api/campaign/%d/platform/android' % campaign_id)
# run the background task to store the click in Datastore
[deferred.run(task.payload) for task in self.taskqueue_stub.get_filtered_tasks()]
def check_missing_parameter(parameter):
campaign_dict = deepcopy(self.CAMPAIGN_SAMPLE)
del campaign_dict[parameter]
response = self.admin_app.put(campaign_url, params=json.dumps(campaign_dict),
headers=self.ADMIN_HEADERS)
self.assertEqual(response.status_int, 200)
map(check_missing_parameter, ["name", "link", "platforms"])
campaign_update = {"platforms": ["android"]}
response = self.admin_app.put(campaign_url, params=json.dumps(campaign_update),
headers=self.ADMIN_HEADERS)
campaign_updated = json.loads(response.body)
self.assertEqual(response.status_int, 200)
self.assertEqual({"id", "name", "link", "create_date", "update_date", "platform_counters"},
set(campaign_updated.keys()))
self.assertEqual(["android"], campaign_updated["platform_counters"].keys())
def test_update_campaign(self):
self.policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=0)
self.testbed.init_datastore_v3_stub(consistency_policy=self.policy)
# create new campaign
response = self.admin_app.post("/api/admin/campaign", params=json.dumps(self.CAMPAIGN_SAMPLE),
headers=self.ADMIN_HEADERS)
campaign = json.loads(response.body)
campaign_id = campaign["id"]
campaign_url = response.headers["Location"]
# simulate a user click
response = self.tracker_app.get('/api/campaign/%d/platform/android' % campaign_id)
# run the background task to store the click in Datastore
[deferred.run(task.payload) for task in self.taskqueue_stub.get_filtered_tasks()]
campaign_new = deepcopy(self.CAMPAIGN_SAMPLE)
campaign_new["name"] = "new name"
response = self.admin_app.put(campaign_url, params=json.dumps(self.CAMPAIGN_SAMPLE),
headers=self.ADMIN_HEADERS)
campaign_updated = json.loads(response.body)
# check if update resets counter (it should not)
self.assertEqual(campaign_updated["platform_counters"]["android"], 1)
# check if update_date is not null any more
self.assertIsNotNone(campaign_updated["update_date"])
def test_platform_clicks(self):
# generate some campaigns
campaign_ids = []
for i in range(10):
response = self.admin_app.post("/api/admin/campaign", params=json.dumps(self.CAMPAIGN_SAMPLE),
headers=self.ADMIN_HEADERS)
campaign = json.loads(response.body)
campaign_ids.append(campaign["id"])
# make 10 clicks to android platform and randomly selected campaigns
for i in range(10):
self.tracker_app.get('/api/campaign/%d/platform/android' % random.sample(campaign_ids, 1)[0])
# run the background task to store the clicks in Datastore
[deferred.run(task.payload) for task in self.taskqueue_stub.get_filtered_tasks()]
response = self.admin_app.get("/api/admin/platform/android/clicks", headers=self.ADMIN_HEADERS)
results = json.loads(response.body)
self.assertEqual(results, 10)
def test_create_campaign_and_track_click(self):
# create new campaign
response = self.admin_app.post("/api/admin/campaign", params=json.dumps(self.CAMPAIGN_SAMPLE),
headers=self.ADMIN_HEADERS)
self.assertEqual(response.status_int, 201)
self.assertIn("Location", response.headers)
campaign_url = response.headers["Location"]
campaign = json.loads(response.body)
# check if response contains all necessary data
self.assertEqual({"id", "name", "link", "create_date", "update_date", "platform_counters"},
set(campaign.keys()))
# check if counters are 0
for platform in campaign["platform_counters"]:
self.assertEqual(campaign["platform_counters"][platform], 0)
self.assertIsNone(campaign["update_date"])
# check if campaign is available through the campaign list
response = self.admin_app.get("/api/admin/campaign", headers=self.ADMIN_HEADERS)
campaigns = json.loads(response.body)
self.assertEqual(len(campaigns), 1)
# get new campaign details
response = self.admin_app.get(campaign_url, headers=self.ADMIN_HEADERS)
self.assertEqual(response.status_int, 200)
self.assertEqual(response.content_type, "application/json")
campaign = json.loads(response.body)
# check if counters are 0
for platform in campaign["platform_counters"]:
self.assertEqual(campaign["platform_counters"][platform], 0)
# generate a user click
response = self.tracker_app.get('/api/campaign/%d/platform/android' % campaign["id"])
self.assertEqual(response.status_int, 302)
self.assertIn("Location", response.headers)
self.assertNotEqual(response.headers["Location"], "http://outfit7.com")
# run the background task to store the click in Datastore
[deferred.run(task.payload) for task in self.taskqueue_stub.get_filtered_tasks()]
# check if click was stored properly
response = self.admin_app.get("/api/admin/campaign/%d/platform/android" % campaign["id"],
headers=self.ADMIN_HEADERS)
data = json.loads(response.body)
self.assertEqual(data["counter"], 1)