def run_deferred_tasks(self, queue='default'): tasks = self.taskqueue_stub.GetTasks(queue) while tasks: self.taskqueue_stub.FlushQueue(queue) for task in tasks: deferred.run(base64.b64decode(task['body'])) tasks = self.taskqueue_stub.GetTasks(queue)
def deferred(request): from google.appengine.ext.deferred.deferred import ( run, SingularTaskFailure, PermanentTaskFailure ) response = HttpResponse() if 'HTTP_X_APPENGINE_TASKEXECUTIONCOUNT' in request.META: logger.debug("[DEFERRED] Retry %s of deferred task", request.META['HTTP_X_APPENGINE_TASKEXECUTIONCOUNT']) if 'HTTP_X_APPENGINE_TASKNAME' not in request.META: logger.critical('Detected an attempted XSRF attack. The header "X-AppEngine-Taskname" was not set.') response.status_code = 403 return response in_prod = environment.is_production_environment() if in_prod and os.environ.get("REMOTE_ADDR") != "0.1.0.2": logger.critical('Detected an attempted XSRF attack. This request did not originate from Task Queue.') response.status_code = 403 return response try: run(request.body) except SingularTaskFailure: logger.debug("Failure executing task, task retry forced") response.status_code = 408 except PermanentTaskFailure: logger.exception("Permanent failure attempting to execute task") return response
def process_request(self, req): """ Process the request """ if req.method == 'POST': headers = ["%s:%s" % (k, v) for k, v in req._inheaders if k.lower().startswith("x-appengine-")] logging.info(", ".join(headers)) try: run(req.read()) except PermanentTaskFailure, e: logging.exception("Permanent failure attempting to execute task")
def __exit__(self, type, value, traceback): from google.appengine.ext.deferred import deferred queues = (queue['name'] for queue in self.stub.GetQueues()) for queue in queues: tasks = self.stub.get_filtered_tasks(queue_names=[queue]) tasks = (task for task in tasks if 'deferred' in task.url) for task in tasks: deferred.run(task.payload) self.stub.DeleteTask(queue, task.name)
def test_invalid_update_campaign(self): self.policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=0) self.testbed.init_datastore_v3_stub(consistency_policy=self.policy) # create new campaign response = self.admin_app.post("/api/admin/campaign", params=json.dumps(self.CAMPAIGN_SAMPLE), headers=self.ADMIN_HEADERS) campaign = json.loads(response.body) campaign_id = campaign["id"] campaign_url = response.headers["Location"] # simulate a user click self.tracker_app.get('/api/campaign/%d/platform/android' % campaign_id) # run the background task to store the click in Datastore [deferred.run(task.payload) for task in self.taskqueue_stub.get_filtered_tasks()] def check_missing_parameter(parameter): campaign_dict = deepcopy(self.CAMPAIGN_SAMPLE) del campaign_dict[parameter] response = self.admin_app.put(campaign_url, params=json.dumps(campaign_dict), headers=self.ADMIN_HEADERS) self.assertEqual(response.status_int, 200) map(check_missing_parameter, ["name", "link", "platforms"]) campaign_update = {"platforms": ["android"]} response = self.admin_app.put(campaign_url, params=json.dumps(campaign_update), headers=self.ADMIN_HEADERS) campaign_updated = json.loads(response.body) self.assertEqual(response.status_int, 200) self.assertEqual({"id", "name", "link", "create_date", "update_date", "platform_counters"}, set(campaign_updated.keys())) self.assertEqual(["android"], campaign_updated["platform_counters"].keys())
def test_update_campaign(self): self.policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=0) self.testbed.init_datastore_v3_stub(consistency_policy=self.policy) # create new campaign response = self.admin_app.post("/api/admin/campaign", params=json.dumps(self.CAMPAIGN_SAMPLE), headers=self.ADMIN_HEADERS) campaign = json.loads(response.body) campaign_id = campaign["id"] campaign_url = response.headers["Location"] # simulate a user click response = self.tracker_app.get('/api/campaign/%d/platform/android' % campaign_id) # run the background task to store the click in Datastore [deferred.run(task.payload) for task in self.taskqueue_stub.get_filtered_tasks()] campaign_new = deepcopy(self.CAMPAIGN_SAMPLE) campaign_new["name"] = "new name" response = self.admin_app.put(campaign_url, params=json.dumps(self.CAMPAIGN_SAMPLE), headers=self.ADMIN_HEADERS) campaign_updated = json.loads(response.body) # check if update resets counter (it should not) self.assertEqual(campaign_updated["platform_counters"]["android"], 1) # check if update_date is not null any more self.assertIsNotNone(campaign_updated["update_date"])
def test_platform_clicks(self): # generate some campaigns campaign_ids = [] for i in range(10): response = self.admin_app.post("/api/admin/campaign", params=json.dumps(self.CAMPAIGN_SAMPLE), headers=self.ADMIN_HEADERS) campaign = json.loads(response.body) campaign_ids.append(campaign["id"]) # make 10 clicks to android platform and randomly selected campaigns for i in range(10): self.tracker_app.get('/api/campaign/%d/platform/android' % random.sample(campaign_ids, 1)[0]) # run the background task to store the clicks in Datastore [deferred.run(task.payload) for task in self.taskqueue_stub.get_filtered_tasks()] response = self.admin_app.get("/api/admin/platform/android/clicks", headers=self.ADMIN_HEADERS) results = json.loads(response.body) self.assertEqual(results, 10)
def test_create_campaign_and_track_click(self): # create new campaign response = self.admin_app.post("/api/admin/campaign", params=json.dumps(self.CAMPAIGN_SAMPLE), headers=self.ADMIN_HEADERS) self.assertEqual(response.status_int, 201) self.assertIn("Location", response.headers) campaign_url = response.headers["Location"] campaign = json.loads(response.body) # check if response contains all necessary data self.assertEqual({"id", "name", "link", "create_date", "update_date", "platform_counters"}, set(campaign.keys())) # check if counters are 0 for platform in campaign["platform_counters"]: self.assertEqual(campaign["platform_counters"][platform], 0) self.assertIsNone(campaign["update_date"]) # check if campaign is available through the campaign list response = self.admin_app.get("/api/admin/campaign", headers=self.ADMIN_HEADERS) campaigns = json.loads(response.body) self.assertEqual(len(campaigns), 1) # get new campaign details response = self.admin_app.get(campaign_url, headers=self.ADMIN_HEADERS) self.assertEqual(response.status_int, 200) self.assertEqual(response.content_type, "application/json") campaign = json.loads(response.body) # check if counters are 0 for platform in campaign["platform_counters"]: self.assertEqual(campaign["platform_counters"][platform], 0) # generate a user click response = self.tracker_app.get('/api/campaign/%d/platform/android' % campaign["id"]) self.assertEqual(response.status_int, 302) self.assertIn("Location", response.headers) self.assertNotEqual(response.headers["Location"], "http://outfit7.com") # run the background task to store the click in Datastore [deferred.run(task.payload) for task in self.taskqueue_stub.get_filtered_tasks()] # check if click was stored properly response = self.admin_app.get("/api/admin/campaign/%d/platform/android" % campaign["id"], headers=self.ADMIN_HEADERS) data = json.loads(response.body) self.assertEqual(data["counter"], 1)