def set_cookie(res, key, value='', max_age=None, path='/', domain=None, secure=None, httponly=False, version=None, comment=None): """ Set (add) a cookie for the response """ fn_name = "set_cookie(): " cookies = Cookie.SimpleCookie() cookies[key] = value for var_name, var_value in [ ('max-age', max_age), ('path', path), ('domain', domain), ('secure', secure), ('HttpOnly', httponly), ('version', version), ('comment', comment), ]: if var_value is not None and var_value is not False: cookies[key][var_name] = str(var_value) if max_age is not None: cookies[key]['expires'] = max_age header_value = cookies[key].output(header='').lstrip() res.headers.add_header("Set-Cookie", header_value) logging.debug(fn_name + "Writing cookie: '" + str(key) + "' = '" + str(value) + "', max age = '" + str(max_age) + "'") logservice.flush()
def post(self): line_bytes = int(self.request.get('line_bytes')) line_count = int(self.request.get('line_count')) flush_every = int(self.request.get('flush_count')) entity_count = int(self.request.get('entity_count')) logging.info("Test Specs: %s", { 'line_bytes': line_bytes, 'line_count': line_count, 'flush_every': flush_every, 'entity_count': entity_count, }) #logservice.flush() repeat_count = max(1, line_bytes / len(string.ascii_letters)) logging_data = list( itertools.repeat(string.ascii_letters, repeat_count)) for line in xrange(line_count): random.shuffle(logging_data) logging.info("%s: %s", line, ''.join(logging_data)) if flush_every and not line % flush_every: logservice.flush() logging.info("Writing entities") db.put(Entity(keyname=i) for i in xrange(100)) logging.info("Done writing entities")
def post(self): """ Delete a selection of Blobstores (selected in form, and posted """ fn_name = "BulkDeleteBlobstoreHandler.post(): " logging.debug(fn_name + "<Start>") logservice.flush() try: self.response.out.write('<html><body>') blobstores_to_delete = self.request.get_all('blob_key') del_count = 0 for blob_key in blobstores_to_delete: blob_info = blobstore.BlobInfo.get(blob_key) if blob_info: try: blob_info.delete() del_count = del_count + 1 except Exception, e: logging.exception(fn_name + "Exception deleting blobstore [" + str(del_count) + "] " + str(blob_key)) self.response.out.write("""<div>Error deleting blobstore %s</div>%s""" % (blob_key, shared.get_exception_msg(e))) else: self.response.out.write("""<div>Blobstore %s doesn't exist</div>""" % blob_key) self.response.out.write('Deleted ' + str(del_count) + ' blobstores') self.response.out.write('<br /><br /><a href="' + settings.ADMIN_MANAGE_BLOBSTORE_URL + '">Back to Blobstore Management</a><br /><br />') self.response.out.write("""<br /><br /><a href=""" + settings.MAIN_PAGE_URL + """>Home page</a><br /><br />""") self.response.out.write('</body></html>') logging.debug(fn_name + "<End>") logservice.flush()
def _process_analytics(self, cls): try: c = 0 now = datetime.now() # Check if this is the first time running last_ana = cls.get_last() if last_ana is None: # This is the first time running start_time = cls.__base__.get_first_ts() log.info('first: %s', start_time) if not start_time: # Check if there are any Analytics Stats log.warn('No Analytics Stats') return # There are no analyticstats to process start_time = cls.get_start_time(start_time=start_time) end_time = cls.get_end_time(start_time) parents = [] log.debug('Loop Times s: %s e: %s', start_time, end_time) while end_time < now: log.info('Loop Count %s: Start Time: %s End Time: %s', c, start_time, end_time) c += 1 if hasattr(cls, 'entity'): parents += self._process_time_frame_with_entity( start_time, end_time, cls) else: parents += self._process_time_frame( start_time, end_time, cls) start_time = end_time end_time = cls.get_end_time(start_time) logservice.flush() self._finallize_anastats(parents) ndb.put_multi(parents) else: # We have already processed ConceptAnalyticsDailyStat before start_time = last_ana.end_time end_time = cls.get_end_time(start_time) log.debug('Times s: %s e: %s', start_time, end_time) while end_time < datetime.now(): if hasattr(cls, 'entity'): ndb.put_multi( self._finallize_anastats( self._process_time_frame_with_entity( start_time, end_time, cls))) else: ndb.put_multi( self._finallize_anastats( self._process_time_frame( start_time, end_time, cls))) start_time = end_time end_time = cls.get_end_time(start_time) except Exception: raise
def set_RFC3339_timestamp(task_row_data, field_name, formats): """ Parse timestamp field value and replace with an RFC-3339 datetime string. task_row_data A row of data, as a dictionary object field_name The name of the field to be set formats A list of possible datetime string formats, as used by strptime() Parses the specified field and replaces the value with an RFC-3339 datetime string as required by the Google server. CAUTION: Attempting to create a task with a zero 'due' value results in strange behaviour; insert() returns a task object with an 'id', however attemping to get() that 'id' returns "404 Not found" If there is a major problem, such as the field no being able to be parsed at all, we delete the field from the task_row_data. """ fn_name = "set_RFC3339_timestamp: " try: if field_name in task_row_data: if task_row_data[field_name] == None: # If we find a None value, we restore the "Zero" RFC-3339 date value here. # GTB stores '0000-01-01T00:00:00.000Z' datetime values (aka constants.ZERO_RFC3339_DATETIME_STRING) # as None, because there is no way to store dates earlier than 1900 in a datetime object logging.debug(fn_name + "DEBUG: '" + field_name + "' value is None; setting value to '" + constants.ZERO_RFC3339_DATETIME_STRING + "'") task_row_data[field_name] = constants.ZERO_RFC3339_DATETIME_STRING return # Field exists, so convert the value to an RFC-3339 datetime string datetime_str = task_row_data[field_name].strip() rfc_datetime_str = convert_datetime_string_to_RFC3339(datetime_str, field_name, formats) if rfc_datetime_str: task_row_data[field_name] = rfc_datetime_str else: # Unable to parse the datetime value, so delete the field which has the un-parseable value. # The server's behaviour for a missing field depends on the field and task state; # If the 'due' field is missing, the created task will not have a due date # If the 'completed' field is missing; # If the 'status' is 'completed', the server uses the current date and time # If the 'status' is 'needsAction', the created task will not have a completed date try: del(task_row_data[field_name]) except Exception, e: logging.error(fn_name + "Unable to delete '" + field_name + "': " + get_exception_msg(e)) logservice.flush() except Exception, e: logging.exception(fn_name + "Error attempting to set '" + field_name + "' datetime field value, so deleting field") # Delete the field which caused the exception try: del(task_row_data[field_name]) except Exception, e: logging.error(fn_name + "Unable to delete '" + field_name + "': " + get_exception_msg(e))
def timer_loop(): while True: now = time.time() then = now - (now % 180) + 180 seconds = then - now logger.debug("Next calculation in " + str(seconds) + " seconds") logservice.flush() time.sleep(seconds) compute_leaderboard()
def serve_outer_exception_message(self, e): """ Display an Oops message when something goes very wrong. This is called from the outer exception handler of major methods (such as get/post handlers) """ fn_name = "serve_outer_exception_message: " self.response.out.write("""Oops! Something went terribly wrong.<br />%s<br /><br />This system is in beta, and is being activeley developed.<br />Please report any errors to <a href="http://%s">%s</a> so that they can be fixed. Thank you.""" % ( get_exception_msg(e), settings.url_issues_page, settings.url_issues_page)) logging.error(fn_name + get_exception_msg(e)) logservice.flush()
def AppWrapper(env, start_response): """Wrapper for <app>.""" status_list = [] headers_list = [] exc_info_list = [] write_buffer_list = [] def MyWrite(buf, s): if s is not None: buf.append(s) def DeferredStartResponse(status, headers, exc_info=None): status_list.append(status) headers_list.append(headers) exc_info_list.append(exc_info) buf = [] write_buffer_list.append(buf) return functools.partial(MyWrite, buf) retval = [] result = app(env, DeferredStartResponse) if result is not None: for value in result: retval.append(value) if logservice.log_buffer_bytes(): logservice.flush() flush_count = str(getattr( request_environment.current_request, 'flush_count', -1)) if headers_list: headers_list[0].append((LOG_FLUSH_COUNTER_HEADER, flush_count)) for status, headers, exc_info, write_buffer in zip( status_list, headers_list, exc_info_list, write_buffer_list): write_fn = start_response(status, headers, exc_info) if write_buffer: write_fn(''.join(write_buffer)) write_buffer = [] return retval
def _save_gcs_image(self, img, mime_type, image_size): size_in_pixels = img.size[0] * img.size[1] import logging from google.appengine.api import runtime from google.appengine.api import logservice logservice.AUTOFLUSH_ENABLED = False logging.debug("image size=%s" % size_in_pixels) logservice.flush() if size_in_pixels > 12 * (10**6): del img raise ValueError("Image too large (%s pixels)", size_in_pixels) if mime_type.lower().startswith("image/"): format_ = mime_type[6:].upper() else: raise ValueError("Unexpected MIME type %s" % mime_type) settings = COMPRESSION_SETTINGS[image_size] ImageFile.MAXBLOCK = 2**20 img_io = StringIO() if settings.get('thumbnail_dimensions'): img = img.copy() img.thumbnail(settings['thumbnail_dimensions'], PIL_Image.ANTIALIAS) img = img.convert('RGB') img.save( img_io, format_, quality=settings['quality'], optimize=settings['quality'], progressive=settings['progressive'] ) img_data = img_io.getvalue() _, saved_width, saved_height = _get_image_info(img_data) img_io.close() file_name = "/images/%s/%s.jpg" % ( self._allocated_image_key.id(), settings['file_shortname'] ) blob_key = _save_gcs_object( img_data, file_name, content_type=mime_type, options={'x-goog-acl': 'public-read'} ) return blob_key, img_data, [saved_width, saved_height]
def check_task_params_exist(tasklists_svc, tasks_svc, tasklist_id, parent_id, sibling_id): fn_name = "check_task_params_exist: " logging.debug(fn_name + "DEBUG: Checking tasklist [" + str(tasklist_id) + "], parent [" + str(parent_id) + "], sibling [" + str(sibling_id) + "]") result = [] # Check if tasklist exists if tasklist_id: if not tasklist_exists(tasklists_svc, tasklist_id): # logging.error(fn_name + "ERROR: Tasklist [" + str(tasklist_id) + "] doesn't exist") # logservice.flush() result.append("Tasklist [" + str(tasklist_id) + "] doesn't exist") else: # logging.error(fn_name + "ERROR: No tasklist ID!") # logservice.flush() result.append("No tasklist ID") # Check if parent task exists (if it was specified) if parent_id: if not task_exists(tasks_svc, tasklist_id, parent_id): # logging.error(fn_name + "ERROR: Parent task [" + str(parent_id) + "] doesn't exist") # logservice.flush() result.append("Parent task [" + str(parent_id) + "] doesn't exist") # else: # logging.debug(fn_name + "DEBUG: No parent ID") # logservice.flush() # Check if sibling task exists (if it was specified) if sibling_id: if not task_exists(tasks_svc, tasklist_id, sibling_id): # logging.error(fn_name + "ERROR: Sibling task [" + str(sibling_id) + "] doesn't exist") # logservice.flush() result.append("Sibling task [" + str(sibling_id) + "] doesn't exist") # else: # logging.debug(fn_name + "DEBUG: No sibling ID") # logservice.flush() if len(result) > 0: logging.error(fn_name + "ERROR: " + ', '.join(result)) else: logging.debug(fn_name + "DEBUG: All task params exist") logservice.flush()
def dispatch(self, *args, **kwargs): self.session_store = sessions.get_store(request=self.request) self.enforce_ssl() self.fetch_user() logservice.flush() self.response.headers.add_header('X-Frame-Options', 'DENY') self.response.headers.add_header('Content-Type', 'text/html; charset=UTF-8') self.response.headers.add_header('X-Content-Type-Options', 'nosniff') self.response.headers.add_header('X-XSS-Protection', '1; mode=block') try: webapp2.RequestHandler.dispatch(self, *args, **kwargs) except Exception: raise finally: self.session_store.save_sessions(self.response)
def reject_non_test_user(self): fn_name = "reject_non_test_user: "******"Rejecting non-test user on limited access server") logservice.flush() # self.response.out.write("<html><body><h2>This is a test server. Access is limited to test users.</h2>" + # "<br /><br /><div>Please use the production server at <href='http://tasks-backup.appspot.com'>tasks-backup.appspot.com</a></body></html>") # logging.debug(fn_name + "<End> (restricted access)" ) serve_message_page(self, "This is a test server. Access is limited to test users.", "Please click the button to go to the production server at tasks-backup.appspot.com", show_custom_button=True, custom_button_text='Go to live server', custom_button_url='http://tasks-backup.appspot.com', show_heading_messages=False)
def delete_blobstore(blob_info): fn_name = "delete_blobstore(): " # logging.debug(fn_name + "<Start>") # logservice.flush() if blob_info: # ------------------------------------- # Delete the Blobstore item # ------------------------------------- try: blob_info.delete() logging.debug(fn_name + "Blobstore deleted") logservice.flush() except Exception, e: logging.exception(fn_name + "Exception deleting %s, key = %s" % (blob_info.filename, blob_info.key())) logservice.flush()
def _log_progress(self, prefix_msg=""): fn_name = "_log_progress: " if prefix_msg: logging.debug(fn_name + prefix_msg + " - Job status = '" + str(self.process_tasks_job.status) + "', progress: " + str(self.process_tasks_job.total_progress)) else: logging.debug(fn_name + "Job status = '" + str(self.process_tasks_job.status) + "', progress: " + "', progress: " + str(self.process_tasks_job.total_progress)) if self.process_tasks_job.message: logging.debug(fn_name + "Message = " + str(self.process_tasks_job.message)) if self.process_tasks_job.error_message: logging.debug(fn_name + "Error message = " + str(self.process_tasks_job.error_message)) logservice.flush()
def movie_listings(max_pages=1, subreddits=SUBREDDITS, newest_only=True): '''wrapper for bulk fetching movie listings from reddit. if newest_only is True, walks forward through results using the cached before_cursor (otherwise it walks backwards) until either max_pages is reached or results are exhausted''' for subreddit in subreddits: after_cursor = None before_cursor = fetch_before_cursor(subreddit) if newest_only else None for _ in range(max_pages): movies, before_cursor, after_cursor = fetch_and_parse_raw_movie_listings(subreddit, before_cursor=before_cursor, after_cursor=after_cursor) if not len(movies): break for movie in movies: yield movie logservice.flush() if before_cursor is None: # i.e we've run out of listings break if newest_only is True: after_cursor = None else: before_cursor = None
def logThis(nLogLevel, strMessage): if ARE_WE_PYTH_APP == True: print strMessage return if nLogLevel == AEL_LEVEL_DEBUG: logging.debug(WE_APP_NAME + strMessage) return if nLogLevel == AEL_LEVEL_INFO: logging.info(WE_APP_NAME + strMessage) return if nLogLevel == AEL_LEVEL_WARNING: logging.warning(WE_APP_NAME + strMessage) return if nLogLevel == AEL_LEVEL_ERROR: logging.error(WE_APP_NAME + strMessage) return if nLogLevel == AEL_LEVEL_CRITICAL: logging.critical(WE_APP_NAME + strMessage) return logservice.flush()
def index_project(self, ): count = 1 num = len(self.project_keys) log.info('Starting indexing') logservice.flush() for pro_key in self.project_keys: pro = pro_key.get() log.info('Index %s of %s projects', str(count), str(num)) logservice.flush() indexes = pro.get_put_index() pro.index(indexes) docs = ndb.get_multi(pro.documents) docs.append(pro.distilled_document.get()) for doc in docs: doc.index(indexes) children = ndb.get_multi(pro.children) for child in children: if child: child.index_phrasings(indexes, index_children=True) count += 1 log.info('Finished indexing') logservice.flush()
def AppWrapper(env, start_response): """Wrapper for <app>.""" status_list = [] headers_list = [] exc_info_list = [] write_buffer_list = [] def MyWrite(buf, s): if s is not None: buf.append(s) def DeferredStartResponse(status, headers, exc_info=None): status_list.append(status) headers_list.append(headers) exc_info_list.append(exc_info) buf = [] write_buffer_list.append(buf) return functools.partial(MyWrite, buf) retval = [] result = app(env, DeferredStartResponse) if result is not None: for value in result: retval.append(value) if logservice.log_buffer_bytes(): logservice.flush() flush_count = str( getattr(request_environment.current_request, 'flush_count', -1)) if headers_list: headers_list[0].append((LOG_FLUSH_COUNTER_HEADER, flush_count)) for status, headers, exc_info, write_buffer in zip( status_list, headers_list, exc_info_list, write_buffer_list): write_fn = start_response(status, headers, exc_info) if write_buffer: write_fn(''.join(write_buffer)) write_buffer = [] return retval
def _run(self): log.info('Processing Aanlytic Daily Stats') logservice.flush() self._process_analytics(AnalyticsDailyStat) # log.info('Processing Project Aanlytic Daily Stats') # self._process_analytics(ProjectAnalyticsDailyStat) log.info('Processing Concept Aanlytic Daily Stats') logservice.flush() self._process_analytics(ConceptAnalyticsDailyStat) log.info('Processing Organization Aanlytic Daily Stats') logservice.flush() self._process_analytics(OrgAnalyticsDailyStat) log.info('Processing User Aanlytic Daily Stats') logservice.flush() self._process_analytics(UserAnalyticsDailyStat) log.info('Finished') logservice.flush()
def get(self): hours = int(self.request.get('hours', 24)) log.info('Hours: %s', hours) logservice.flush() an_action_count = {} for action in ANALYTIC_ACTIONS: an_action_count[action] = int(self.request.get(action, 2)) start = datetime.now().replace(minute=0, second=0, microsecond=0) - timedelta(hours=hours) log.info('Start Time: %s', start) logservice.flush() total_per_hour = 0 for count in an_action_count.itervalues(): total_per_hour += count log.info('total per hours: %s', total_per_hour) logservice.flush() ana_obj = [] for h in xrange(0, hours): log.info('Current Hour: %s', h) logservice.flush() start += timedelta(hours=1) for action, count in an_action_count.iteritems(): for i in xrange(0, count): ana_obj.append( Analytics.new( ts=start + timedelta(minutes=random.randint(0, 59)), artifact=ndb.Key( 'Concept', '043155e8eb4311e3bc89534d9485b528'), artifact_owners=[ndb.Key('User', 'amiller')], project=ndb.Key( 'Project', '03b0826beb4311e3a269534d9485b528'), action=action, )) ndb.put_multi(ana_obj) ana_obj[:] = [] self.response.write(Analytics.query().count())
def set_timestamp(task, field_name, date_only=False): """ Parse timestamp field value and replace with a datetime object. task A task dictionary object, as returned from the Google server field_name The name of the field to be set date_only If True, store a date object instead of datetime Parses the specified field and stores a date or datetime object so that methods which access the task (including Django templates) can format the displayed date. If there is a major problem, such as the field no being able to be parsed at all, we delete the field from the task dictionary. """ fn_name = "set_timestamp: " try: if field_name in task: # Field exists, so try to parse its value try: datetime_str = task[field_name] task[field_name] = convert_RFC3339_string_to_datetime(datetime_str, field_name, date_only) except Exception, e: try: logging.error(fn_name + "Unable to parse '" + field_name + "' datetime field value '" + str(datetime_str) + "', so deleting field: " + get_exception_msg(e)) except Exception, e: # In case logging the value causes an exception, log without value logging.error(fn_name + "Unable to parse '" + field_name + "' datetime field value, and unable to log field value, so deleting field: " + get_exception_msg(e)) # Delete the field which has the un-parseable value try: del(task[field_name]) except Exception, e: logging.error(fn_name + "Unable to delete '" + field_name + "': " + get_exception_msg(e)) logservice.flush()
def _process_hour_block(self, anas): log.info('Processing %s anlytics', len(anas)) logservice.flush() anastat = AnalyticsHourlyStat.new() anastat.start_time = self.start_time anastat.end_time = self.end_time for ana in anas: if ana.artifact_organization: aid = ana.artifact_organization.id( ) + '-' + self.end_time.strftime('%Y-%m-%d-%H-%M') if aid in self.org_ana_stats: self._record_ana_action(self.org_ana_stats[aid], ana) else: org_ana_stat = OrgAnalyticsHourlyStat.new(id=aid) org_ana_stat.start_time = self.start_time org_ana_stat.end_time = self.end_time org_ana_stat.entity = ana.artifact_organization org_ana_stat.entity_organization = ana.artifact_organization self.org_ana_stats[org_ana_stat.key.id()] = org_ana_stat self._record_ana_action(org_ana_stat, ana) # if ana.artifact and ana.artifact.kind() == 'Project': # self._record_project_ana_action(ana) if ana.artifact and ana.artifact.kind() == 'Concept': self._record_concept_ana_action(ana) self._record_user_ana_action(ana) self._record_ana_action(anastat, ana) anastats = [anastat] + self.art_ana_stats.values( ) + self.org_ana_stats.values() + self.user_ana_stats.values() self._finallize_anastats(anastats) ndb.put_multi(anastats)
def movie_listings(max_pages=1, subreddits=SUBREDDITS, newest_only=True): '''wrapper for bulk fetching movie listings from reddit. if newest_only is True, walks forward through results using the cached before_cursor (otherwise it walks backwards) until either max_pages is reached or results are exhausted''' for subreddit in subreddits: after_cursor = None before_cursor = fetch_before_cursor(subreddit) if newest_only else None for _ in range(max_pages): movies, before_cursor, after_cursor = fetch_and_parse_raw_movie_listings( subreddit, before_cursor=before_cursor, after_cursor=after_cursor) if not len(movies): break for movie in movies: yield movie logservice.flush() if before_cursor is None: # i.e we've run out of listings break if newest_only is True: after_cursor = None else: before_cursor = None
def process(self): logging.info('Backend running') server_name = backends.get_backend() logging.info(server_name) servers = memcache.get('servers') if servers is None: servers = dict() servers[server_name] = 1 else: if server_name not in servers: servers[server_name] = 1 else: servers[server_name] = servers[server_name] + 1 memcache.set('servers', servers) logservice.flush() runtime.set_shutdown_hook(self.shutdown) self.response.set_status(200) return
def get(self): l1 = logging.getLogger("test"); l2 = logging.getLogger("test.test"); l1.critical("nohandler: test nohandler") l1.info("nohandler: test nohandler") l1.warn("nohandler: test nohandler") l1.critical("nohandler: test nohandler") l2.critical("stdouthandler: test handler") logging.getLogger(__name__).debug(__name__ + ": debug") logging.getLogger(__name__).critical(__name__ + ": critical") l1.critical("nohandler: test nohandler") l2.critical("stdouthandler: test handler") logservice.flush() logservice.AUTOFLUSH_ENABLED = True logservice.AUTOFLUSH_EVERY_SECONDS = None logservice.AUTOFLUSH_EVERY_BYTES = None logservice.AUTOFLUSH_EVERY_LINES = 1 time.sleep(5) l1.critical("post sleep") l2.critical("post sleep") open("etc", "w") #t = [0] * 268435456 l1.critical("post big alloc") l2.critical("post big alloc") self.response.headers['Content-Type'] = 'text/plain' self.response.write(self.request)
def serve_message_page(self, msg1, msg2 = None, msg3 = None, show_back_button=False, back_button_text="Back to previous page", show_custom_button=False, custom_button_text='Try again', custom_button_url=settings.MAIN_PAGE_URL, show_heading_messages=True): """ Serve message.html page to user with message, with an optional button (Back, or custom URL) msg1, msg2, msg3 Text to be displayed.msg2 and msg3 are option. Each msg is displayed in a separate div show_back_button If True, a [Back] button is displayed, to return to previous page show_custom_button If True, display button to jump to any URL. title set by custom_button_text custom_button_text Text label for custom button custom_button_url URL to go to when custom button is pressed show_heading_messages If True, display app_title and (optional) host_msg """ fn_name = "serve_message_page: " logging.debug(fn_name + "<Start>") logservice.flush() try: client_id, client_secret, user_agent, app_title, product_name, host_msg = get_settings(self.request.host) if msg1: logging.debug(fn_name + "Msg1: " + msg1) if msg2: logging.debug(fn_name + "Msg2: " + msg2) if msg3: logging.debug(fn_name + "Msg3: " + msg3) path = os.path.join(os.path.dirname(__file__), constants.PATH_TO_TEMPLATES, "message.html") template_values = {'app_title' : app_title, 'host_msg' : host_msg, 'msg1': msg1, 'msg2': msg2, 'msg3': msg3, 'show_heading_messages' : show_heading_messages, 'show_back_button' : show_back_button, 'back_button_text' : back_button_text, 'show_custom_button' : show_custom_button, 'custom_button_text' : custom_button_text, 'custom_button_url' : custom_button_url, 'product_name' : product_name, 'url_discussion_group' : settings.url_discussion_group, 'email_discussion_group' : settings.email_discussion_group, 'url_issues_page' : settings.url_issues_page, 'url_source_code' : settings.url_source_code, 'app_version' : appversion.version, 'upload_timestamp' : appversion.upload_timestamp} self.response.out.write(template.render(path, template_values)) logging.debug(fn_name + "<End>" ) logservice.flush() except Exception, e: logging.exception(fn_name + "Caught top-level exception") serve_outer_exception_message(self, e) logging.debug(fn_name + "<End> due to exception" ) logservice.flush()
def get(self): fn_name = "DisplayStatsHandler.get(): " logging.debug(fn_name + "<Start> (app version %s)" % appversion.version ) logservice.flush() stats_query = model.UsageStats.all() # stats_query.order('start_time') # stats_query.order('user_hash') stats = stats_query.run() try: stats_filename = "stats_" + get_application_id() + "_" + datetime.datetime.now().strftime("%Y-%m-%d") + ".csv" template_values = {'stats' : stats} self.response.headers["Content-Type"] = "text/csv" self.response.headers.add_header( "Content-Disposition", "attachment; filename=%s" % stats_filename) path = os.path.join(os.path.dirname(__file__), constants.PATH_TO_TEMPLATES, "stats.csv") self.response.out.write(template.render(path, template_values)) logging.debug(fn_name + "<End>" ) logservice.flush() except Exception, e: logging.exception(fn_name + "Caught top-level exception") self.response.headers["Content-Type"] = "text/html; charset=utf-8" try: # Clear "Content-Disposition" so user will see error in browser. # If not removed, output goes to file (if error generated after "Content-Disposition" was set), # and user would not see the error message! del self.response.headers["Content-Disposition"] except Exception, e: logging.debug(fn_name + "Unable to delete 'Content-Disposition' from headers: " + shared.get_exception_msg(e))
def handle_auth_callback(self): fn_name = "handle_auth_callback() " logging.debug(fn_name + "<Start>") logservice.flush() try: if not self.request.get("code"): logging.debug(fn_name + "No 'code', so redirecting to " + str(settings.WELCOME_PAGE_URL)) logservice.flush() self.redirect(settings.WELCOME_PAGE_URL) logging.debug(fn_name + "<End> (no code)") logservice.flush() return user = users.get_current_user() logging.debug(fn_name + "Retrieving flow for " + str(user.user_id())) flow = pickle.loads(memcache.get(user.user_id())) if flow: logging.debug(fn_name + "Got flow. Retrieving credentials") error = False retry_count = settings.NUM_API_TRIES while retry_count > 0: try: credentials = flow.step2_exchange(self.request.params) # Success! error = False if isTestUser(user.email()): logging.debug(fn_name + "Retrieved credentials for " + str(user.email()) + ", expires " + str(credentials.token_expiry) + " UTC") else: logging.debug(fn_name + "Retrieved credentials, expires " + str(credentials.token_expiry) + " UTC") break except client.FlowExchangeError, e: logging.warning(fn_name + "FlowExchangeError: Giving up - " + get_exception_msg(e)) error = True credentials = None break except Exception, e: logging.warning(fn_name + "Exception: " + get_exception_msg(e)) error = True credentials = None
class DownloadStatsHandler(webapp.RequestHandler): """Display statistics""" def get(self): fn_name = "DisplayStatsHandler.get(): " logging.debug(fn_name + "<Start> (app version %s)" % appversion.version ) logservice.flush() stats_query = model.UsageStats.all() # stats_query.order('start_time') # stats_query.order('user_hash') stats = stats_query.run() try: stats_filename = "stats_" + get_application_id() + "_" + datetime.datetime.now().strftime("%Y-%m-%d") + ".csv" template_values = {'stats' : stats} self.response.headers["Content-Type"] = "text/csv" self.response.headers.add_header( "Content-Disposition", "attachment; filename=%s" % stats_filename) path = os.path.join(os.path.dirname(__file__), constants.PATH_TO_TEMPLATES, "stats.csv") self.response.out.write(template.render(path, template_values)) logging.debug(fn_name + "<End>" ) logservice.flush() except Exception, e: logging.exception(fn_name + "Caught top-level exception") self.response.headers["Content-Type"] = "text/html; charset=utf-8" try: # Clear "Content-Disposition" so user will see error in browser. # If not removed, output goes to file (if error generated after "Content-Disposition" was set), # and user would not see the error message! del self.response.headers["Content-Disposition"] except Exception, e: logging.debug(fn_name + "Unable to delete 'Content-Disposition' from headers: " + shared.get_exception_msg(e)) self.response.clear() self.response.out.write("""Oops! Something went terribly wrong.<br />%s<br />Please report this error to <a href="http://code.google.com/p/tasks-backup/issues/list">code.google.com/p/tasks-backup/issues/list</a>""" % shared.get_exception_msg(e)) logging.debug(fn_name + "<End> due to exception" ) logservice.flush()
def _process_hourly_analytics(self): log.info('Hourly Analytics Processer Started') logservice.flush() last_analyticstat = AnalyticsHourlyStat.get_last() if not last_analyticstat: # If no last analytic than this is the first time. log.info( 'Analytics have not been process yet, starting from the beginning' ) logservice.flush() first_ana = Analytics.get_first() last_ana = Analytics.get_last() if first_ana: self.now = datetime.now() self.start_time = first_ana.ts.replace(minute=0, second=0, microsecond=0) self.end_time = self.start_time + timedelta(hours=1) self.hour_count = 0 while True: log.info('Hour: %s Start Time: %s End Time: %s', self.hour_count, self.start_time, self.end_time) logservice.flush() anas = Analytics.get_time_frame(self.start_time, self.end_time) self._process_hour_block(anas) if last_ana in anas or self.end_time > self.now: break self.hour_count += 1 self.start_time = self.end_time self.end_time = self.start_time + timedelta(hours=1) else: log.info('Found no Analytics to process') else: self.start_time = last_analyticstat.end_time self.end_time = self.start_time + timedelta(hours=1) while self.end_time < datetime.now(): anas = Analytics.get_time_frame(self.start_time, self.end_time) self._process_hour_block(anas) self.start_time = self.end_time self.end_time = self.end_time + timedelta(hours=1) log.info('Hourly Analytics Processer Finished')
def redirect_for_auth(self, user, redirect_url=None): """Redirects the webapp response to authenticate the user with OAuth2. Args: redirect_url [OPTIONAL] The URL to return to once authorised. Usually unused (left as None), so that the URL of the calling page is used Uses the 'state' parameter to store redirect_url. The handler for /oauth2callback can therefore redirect the user back to the page they were on when get_credentials() failed (or to the specified redirect_url). """ fn_name = "redirect_for_auth(): " try: client_id, client_secret, user_agent, app_title, product_name, host_msg = get_settings( self.request.host) # Check how many times this has been called (without credentials having been successfully retrieved) if self.request.cookies.has_key('auth_retry_count'): auth_retry_count = int(self.request.cookies['auth_retry_count']) logging.debug(fn_name + "auth_retry_count = " + str(auth_retry_count)) if auth_retry_count > settings.MAX_NUM_AUTH_RETRIES: # Exceeded maximum number of retries, so don't try again. # Redirect user to invalid credentials page logging.warning( fn_name + "Not re-authorising, because there have already been " + str(auth_retry_count) + " attempts. Redirecting to " + settings.INVALID_CREDENTIALS_URL) self.redirect(settings.INVALID_CREDENTIALS_URL + "?rc=NC&nr=" + str(auth_retry_count)) return else: logging.debug( fn_name + "No auth_retry_count cookie found. Probably means last authorisation was more than " + str(settings.AUTH_RETRY_COUNT_COOKIE_EXPIRATION_TIME) + " seconds ago") auth_retry_count = 0 if not redirect_url: # By default, return to the same page redirect_url = self.request.path_qs # According to https://developers.google.com/accounts/docs/OAuth_ref#RequestToken # xoauth_displayname is optional. # (optional) String identifying the application. # This string is displayed to end users on Google's authorization confirmation page. # For registered applications, the value of this parameter overrides the name set during registration and # also triggers a message to the user that the identity can't be verified. # For unregistered applications, this parameter enables them to specify an application name, # In the case of unregistered applications, if this parameter is not set, Google identifies the application # using the URL value of oauth_callback; if neither parameter is set, Google uses the string "anonymous". # It seems preferable to NOT supply xoauth_displayname, so that Google doesn't display "identity can't be verified" msg. flow = client.OAuth2WebServerFlow( client_id=client_id, client_secret=client_secret, scope="https://www.googleapis.com/auth/tasks", user_agent=user_agent, state=redirect_url) callback = self.request.relative_url("/oauth2callback") authorize_url = flow.step1_get_authorize_url(callback) memcache.set(user.user_id(), pickle.dumps(flow)) # Keep track of how many times we've called the authorise URL if self.request.cookies.has_key('auth_retry_count'): auth_retry_count = int(self.request.cookies['auth_retry_count']) else: auth_retry_count = 0 __store_auth_retry_count(self, auth_retry_count + 1) logging.debug(fn_name + "Redirecting to " + str(authorize_url)) logservice.flush() self.redirect(authorize_url) except Exception, e: logging.exception(fn_name + "Caught top-level exception") serve_outer_exception_message(self, e) logging.debug(fn_name + "<End> due to exception") logservice.flush()
credentials = None result = False else: # No credentials fail_msg = "No credentials" fail_reason = "Unable to retrieve credentials for user" #logging.debug(fn_name + fail_msg) result = False if result: # TODO: Successfuly retrieved credentials, so reset auth_retry_count to 0 __reset_auth_retry_count(self) else: logging.debug(fn_name + fail_msg) logservice.flush() except Exception, e: logging.exception(fn_name + "Caught top-level exception") logging.debug(fn_name + "<End> due to exception") raise e if fail_reason == "Daily limit exceeded": # Will be caught in calling method's outer try-except raise DailyLimitExceededError() return result, user, credentials, fail_msg, fail_reason def serve_message_page(self, msg1,
def _do_migration(): log.info('Starting Datastore Migration') logservice.flush() projects = {} concept_count = Concept.query().count() concepts = Concept.query() index = 1 logservice.flush() for concept in concepts.iter(): if index % 50 == 0: log.info('%s concepts finished of %s' % (index, concept_count)) logservice.flush() project = projects.get(concept.project.id(), None) if not project: project = concept.project.get() if not project: if len(concept.phrasings) > 0: perms = [] phrasings = ndb.get_multi(concept.phrasings) for phrase in phrasings: if phrase: perms.append(phrase.permissions) ndb.delete_multi(concept.phrasings) ndb.delete_multi(perms) if len(concept.selected_phrasings) > 0: ndb.delete_multi(concept.selected_phrasings) if len(concept.attributes) > 0: ndb.delete_multi(concept.attributes) if len(concept.crawlcontext) > 0: ndb.delete_multi(concept.crawlcontext) if len(concept.linkes) > 0: links = ndb.get_multi(concept.linkes) for link in links: if link: link.delete(None, False, True) if concept.media_blob is not None: concept.delete_media() concept.distilled_phrasing.delete() concept.permissions.delete() concept.key.delete() continue projects[project.key.id()] = project perm = project.permissions.get() if not perm.project: perm.project = project.key perm.put() for doc in project.documents: d = doc.get() perm = d.permissions.get() if not perm.project: perm.project = project.key perm.put() d = project.distilled_document.get() perm = d.permissions.get() if not perm.project: perm.project = project.key perm.put() _fix_artifacts(concept, project) index += 1 log.info('Finished Datastore Migration') logservice.flush()
def dump_obj(obj): for attr in dir(obj): logging.debug(" obj.%s = %s" % (attr, getattr(obj, attr))) logservice.flush()
def get_credentials(self): """ Retrieve credentials for the user Returns: result True if we have valid credentials for the user user User object for current user credentials Credentials object for current user. None if no credentials. fail_msg If result==False, message suitabale for displaying to user fail_reason If result==False, cause of the failure. Can be one of "User not logged on" "No credentials" "Invalid credentials" "Credential use error" (Unspecified error when attempting to use credentials) "Credential use HTTP error" (Returned an HTTP error when attempting to use credentials) If no credentials, or credentials are invalid, the calling method can call redirect_for_auth(self, user), which sets the redirect URL back to the calling page. That is, user is redirected to calling page after authorising. """ fn_name = "get_credentials(): " user = None fail_msg = '' fail_reason = '' credentials = None result = False try: user = users.get_current_user() if user is None: # User is not logged in, so there can be no credentials. fail_msg = "User is not logged in" fail_reason = "User not logged on" logging.debug(fn_name + fail_msg) logservice.flush() return False, None, None, fail_msg, fail_reason credentials = appengine.StorageByKeyName( model.Credentials, user.user_id(), "credentials").get() result = False if credentials: if credentials.invalid: # We have credentials, but they are invalid fail_msg = "Invalid credentials for this user" fail_reason = "Invalid credentials" result = False else: #logging.debug(fn_name + "Calling tasklists service to confirm valid credentials") # so it turns out that the method that checks if the credentials are okay # doesn't give the correct answer unless you try to refresh it. So we do that # here in order to make sure that the credentials are valid before being # passed to a worker. Obviously if the user revokes the credentials after # this point we will continue to get an error, but we can't stop that. # Credentials are possibly valid, but need to be confirmed by refreshing # Try multiple times, just in case call to server fails due to external probs (e.g., timeout) # retry_count = settings.NUM_API_TRIES # while retry_count > 0: try: http = httplib2.Http() http = credentials.authorize(http) service = discovery.build("tasks", "v1", http) tasklists_svc = service.tasklists() tasklists_list = tasklists_svc.list().execute() # Successfully used credentials, everything is OK, so break out of while loop fail_msg = '' fail_reason = '' result = True # break except apiclient_errors.HttpError, e: #logging.info(fn_name + "HttpError using credentials: " + get_exception_msg(e)) if e._get_reason().lower() == "daily limit exceeded": fail_reason = "Daily limit exceeded" fail_msg = "HttpError: Daily limit exceeded using credentials." else: fail_reason = "Credential use HTTP error" fail_msg = "Error accessing tasks service: " + e._get_reason() result = False credentials = None result = False except Exception, e: #logging.info(fn_name + "Exception using credentials: " + get_exception_msg(e)) fail_reason = "Credential use error" fail_msg = "Exception using credentials: " + get_exception_msg(e) credentials = None result = False
def post(self): fn_name = "ProcessTasksWorker.post(): " logging.debug(fn_name + "<start> (app version %s)" %appversion.version) logservice.flush() client_id, client_secret, user_agent, app_title, project_name, host_msg = shared.get_settings(self.request.host) self.user_email = self.request.get(settings.TASKS_QUEUE_KEY_NAME) self.is_test_user = shared.isTestUser(self.user_email) if self.user_email: # Retrieve the DB record for this user self.process_tasks_job = model.ProcessTasksJob.get_by_key_name(self.user_email) if self.process_tasks_job is None: logging.error(fn_name + "No DB record for " + self.user_email) logservice.flush() logging.debug(fn_name + "<End> No DB record") # TODO: Find some way of notifying the user????? # Could use memcache to relay a message which is displayed in ProgressHandler return else: logging.debug(fn_name + "Retrieved process tasks job for " + str(self.user_email)) logservice.flush() self.process_tasks_job.status = constants.ExportJobStatus.INITIALISING self.process_tasks_job.job_progress_timestamp = datetime.datetime.now() self.process_tasks_job.message = "Validating background job ..." logging.debug(fn_name + "Initialising - Job status: '" + str(self.process_tasks_job.status) + "', progress: " + str(self.process_tasks_job.total_progress) + ", msg: '" + str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'") logservice.flush() self.process_tasks_job.put() user = self.process_tasks_job.user if not user: logging.error(fn_name + "No user object in DB record for " + str(self.user_email)) logservice.flush() self.process_tasks_job.status = constants.ExportJobStatus.ERROR self.process_tasks_job.message = '' self.process_tasks_job.error_message = "Problem with user details. Please restart." self.process_tasks_job.job_progress_timestamp = datetime.datetime.now() logging.debug(fn_name + "No user - Job status: '" + str(self.process_tasks_job.status) + "', progress: " + str(self.process_tasks_job.total_progress) + ", msg: '" + str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'") logservice.flush() self.process_tasks_job.put() logging.debug(fn_name + "<End> No user object") return self.credentials = self.process_tasks_job.credentials if not self.credentials: logging.error(fn_name + "No credentials in DB record for " + str(self.user_email)) logservice.flush() self.process_tasks_job.status = constants.ExportJobStatus.ERROR self.process_tasks_job.message = '' self.process_tasks_job.error_message = "Problem with user self.credentials. Please restart." self.process_tasks_job.job_progress_timestamp = datetime.datetime.now() logging.debug(fn_name + "Job status: '" + str(self.process_tasks_job.status) + "', progress: " + str(self.process_tasks_job.total_progress) + ", msg: '" + str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'") logservice.flush() self.process_tasks_job.put() logging.debug(fn_name + "<End> No self.credentials") return if self.credentials.invalid: logging.error(fn_name + "Invalid credentials in DB record for " + str(self.user_email)) logservice.flush() self.process_tasks_job.status = constants.ExportJobStatus.ERROR self.process_tasks_job.message = '' self.process_tasks_job.error_message = "Invalid self.credentials. Please restart and re-authenticate." self.process_tasks_job.job_progress_timestamp = datetime.datetime.now() logging.debug(fn_name + "Credentials invalid - Job status: '" + str(self.process_tasks_job.status) + "', progress: " + str(self.process_tasks_job.total_progress) + ", msg: '" + str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'") logservice.flush() self.process_tasks_job.put() logging.debug(fn_name + "<End> Invalid self.credentials") return if self.is_test_user: logging.debug(fn_name + "User is test user %s" % self.user_email) logservice.flush() http = httplib2.Http() http = self.credentials.authorize(http) service = discovery.build("tasks", "v1", http) self.tasklists_svc = service.tasklists() self.tasks_svc = service.tasks() self.export_tasks() # logging.debug(fn_name + "Finished processing. Total progress = " + # str(self.process_tasks_job.total_progress) + " for " + str(self.user_email)) else: logging.error(fn_name + "No processing, as there was no user_email key") logservice.flush() logging.debug(fn_name + "<End>, user = " + str(self.user_email)) logservice.flush()
def convert_RFC3339_string_to_datetime(datetime_str, field_name, date_only=False): """ Attempt to convert the RFC 3339 datetime string to a valid datetime object. If the field value cannot be parsed, return a default '1900-01-01 00:00:00' value. If the string is '0001-01-01T00:00:00.000Z', return None. datetime_str String to be converted field_name The name of the field being parsed. Only used for logging date_only If True, return a date object instead of datetime strptime() can only parse dates after Jan 1900, but sometimes the server returns dates outside the valid range. Invalid values will be converted to 1900-01-01 so that they can be parsed by strptime() or formatted by strftime() Parse the yyyy-mm-ddThh:mm:ss.dddZ return a date or datetime object. There have been occassional strange values in the 'completed' property, such as "-1701567-04-26T07:12:55.000Z" According to http://docs.python.org/library/datetime.html "The exact range of years for which strftime() works also varies across platforms. Regardless of platform, years before 1900 cannot be used." so if any date/timestamp value is invalid, set the property to '1900-01-01 00:00:00' NOTE: Sometimes a task has a completion date of '0000-01-01T00:00:00.000Z', which cannot be converted to datetime, because the earliest allowable datetime year is 0001, so that is returned as None Raises an exception if datetime_str is empty, or cannot be handled at all. In this case, it is recommended that the calling method delete the associated field in the task object. """ fn_name = "convert_RFC3339_string_to_datetime: " try: if not datetime_str: # Nothing to parse, so raise an Exception, so that the calling method can deal with it raise ValueError("No datetime string - nothing to parse") if datetime_str == constants.ZERO_RFC3339_DATETIME_STRING: # Zero timestamp is represented by None. # Methods seeing None for a datetime object should replace it with a string representing # the zero date (e.g., '0000-01-01 00:00:00') return None d = None try: # Parse the RFC 3339 datetime string d = datetime.datetime.strptime(datetime_str, "%Y-%m-%dT%H:%M:%S.000Z") # Test the resultant datetime to ensure that it can be displayed by strftime() # This prevents exceptions later when other modules try to display a datetime # that is outside the valid range for strftime(), such as dates before 1900. try: test_date_str = d.strftime('%H:%M:%S %a, %d %b %Y') except Exception, e: d = datetime.datetime(1900,1,1,0,0,0) logging.warning(fn_name + "Unable to convert '" + field_name + "' string '" + str(datetime_str) + "' to a datetime that can be displayed by strftime, so using " + str(d) + ": " + get_exception_msg(e)) logservice.flush() except ValueError, e: # Minimum datestamp that can be parsed by strptime or displayed by strftime is 1900-01-01 d = datetime.datetime(1900,1,1,0,0,0) logging.warning(fn_name + "Invalid '" + field_name + "' timestamp (" + str(datetime_str) + "), so using " + str(d) + ": " + get_exception_msg(e)) logservice.flush() except Exception, e: logging.exception(fn_name + "Unable to parse '" + field_name + "' value '" + str(datetime_str) + "' as a datetime") logservice.flush() raise e
def serve_message_page(self, msg1, msg2 = None, msg3 = None, show_back_button=False, back_button_text="Back to previous page", show_custom_button=False, custom_button_text='Try again', custom_button_url = settings.MAIN_PAGE_URL, show_heading_messages=True, template_file="message.html", extra_template_values=None): """ Serve message.html page to user with message, with an optional button (Back, or custom URL) self A webapp.RequestHandler or similar msg1, msg2, msg3 Text to be displayed.msg2 and msg3 are option. Each msg is displayed in a separate div show_back_button If True, a [Back] button is displayed, to return to previous page show_custom_button If True, display button to jump to any URL. title set by custom_button_text custom_button_text Text label for custom button custom_button_url URL to go to when custom button is pressed. Should be an absolute URL show_heading_messages If True, display app_title and (optional) host_msg template_file Specify an alternate HTML template file extra_template_values A dictionary containing values that will be merged with the existing template values They may be additional parameters, or overwrite existing parameters. These new values will be available to the HTML template. All args except self and msg1 are optional. """ fn_name = "serve_message_page: " logging.debug(fn_name + "<Start>") logservice.flush() try: if custom_button_url == settings.MAIN_PAGE_URL: # Relative URLs sometimes fail on Firefox, so convert the default relative URL to an absolute URL custom_button_url = urljoin("https://" + self.request.host, settings.MAIN_PAGE_URL) if msg1: logging.debug(fn_name + "Msg1: " + msg1) if msg2: logging.debug(fn_name + "Msg2: " + msg2) if msg3: logging.debug(fn_name + "Msg3: " + msg3) path = os.path.join(os.path.dirname(__file__), constants.PATH_TO_TEMPLATES, template_file) template_values = {'app_title' : host_settings.APP_TITLE, 'host_msg' : host_settings.HOST_MSG, 'msg1': msg1, 'msg2': msg2, 'msg3': msg3, 'show_heading_messages' : show_heading_messages, 'show_back_button' : show_back_button, 'back_button_text' : back_button_text, 'show_custom_button' : show_custom_button, 'custom_button_text' : custom_button_text, 'custom_button_url' : custom_button_url, 'product_name' : host_settings.PRODUCT_NAME, 'url_discussion_group' : settings.url_discussion_group, 'email_discussion_group' : settings.email_discussion_group, 'url_issues_page' : settings.url_issues_page, 'url_source_code' : settings.url_source_code, 'app_version' : appversion.version, 'upload_timestamp' : appversion.upload_timestamp} if extra_template_values: # Add/update template values logging.debug(fn_name + "DEBUG: Updating template values ==>") logging.debug(extra_template_values) logservice.flush() template_values.update(extra_template_values) self.response.out.write(template.render(path, template_values)) logging.debug(fn_name + "<End>" ) logservice.flush() except Exception, e: logging.exception(fn_name + "Caught top-level exception") serve_outer_exception_message(self, e) logging.debug(fn_name + "<End> due to exception" ) logservice.flush()
def process(self): logging.info(modules.get_current_module_name() + ' started') logservice.flush() self.response.set_status(200) return
def get_credentials(self): """ Retrieve credentials for the user Returns: result True if we have valid credentials for the user user User object for current user credentials Credentials object for current user. None if no credentials. fail_msg If result==False, message suitabale for displaying to user fail_reason If result==False, cause of the failure. Can be one of "User not logged on" "No credentials" "Invalid credentials" "Credential use error" (Unspecified error when attempting to use credentials) "Credential use HTTP error" (Returned an HTTP error when attempting to use credentials) If no credentials, or credentials are invalid, the calling method can call redirect_for_auth(self, user), which sets the redirect URL back to the calling page. That is, user is redirected to calling page after authorising. """ fn_name = "get_credentials(): " user = None fail_msg = '' fail_reason = '' credentials = None result = False try: user = users.get_current_user() if user is None: # User is not logged in, so there can be no credentials. fail_msg = "User is not logged in" fail_reason = "User not logged on" logging.debug(fn_name + fail_msg) logservice.flush() return False, None, None, fail_msg, fail_reason credentials = appengine.StorageByKeyName(model.Credentials, user.user_id(), "credentials").get() result = False if credentials: if credentials.invalid: # We have credentials, but they are invalid fail_msg = "Invalid credentials for this user" fail_reason = "Invalid credentials" result = False else: #logging.debug(fn_name + "Calling tasklists service to confirm valid credentials") # so it turns out that the method that checks if the credentials are okay # doesn't give the correct answer unless you try to refresh it. So we do that # here in order to make sure that the credentials are valid before being # passed to a worker. Obviously if the user revokes the credentials after # this point we will continue to get an error, but we can't stop that. # Credentials are possibly valid, but need to be confirmed by refreshing # Try multiple times, just in case call to server fails due to external probs (e.g., timeout) # retry_count = settings.NUM_API_TRIES # while retry_count > 0: try: http = httplib2.Http() http = credentials.authorize(http) service = discovery.build("tasks", "v1", http) tasklists_svc = service.tasklists() tasklists_list = tasklists_svc.list().execute() # Successfully used credentials, everything is OK, so break out of while loop fail_msg = '' fail_reason = '' result = True # break except apiclient_errors.HttpError, e: #logging.info(fn_name + "HttpError using credentials: " + get_exception_msg(e)) if e._get_reason().lower() == "daily limit exceeded": fail_reason = "Daily limit exceeded" fail_msg = "HttpError: Daily limit exceeded using credentials." else: fail_reason = "Credential use HTTP error" fail_msg = "Error accessing tasks service: " + e._get_reason( ) result = False credentials = None result = False except Exception, e: #logging.info(fn_name + "Exception using credentials: " + get_exception_msg(e)) fail_reason = "Credential use error" fail_msg = "Exception using credentials: " + get_exception_msg( e) credentials = None result = False
def _export_tasks(self): fn_name = "_export_tasks: " logging.debug(fn_name + "<Start>") logservice.flush() start_time = datetime.datetime.now() include_hidden = self.process_tasks_job.include_hidden include_completed = self.process_tasks_job.include_completed include_deleted = self.process_tasks_job.include_deleted summary_msg = '' # Retrieve all tasks for the user try: logging.debug(fn_name + "include_completed = " + str(include_completed) + ", include_hidden = " + str(include_hidden) + ", include_deleted = " + str(include_deleted)) logservice.flush() # ############################################## # FLOW # ---------------------------------------------- # For each page of taskslists # For each tasklist # For each page of tasks # For each task # Fix date format # Add tasks to tasklist collection # Add tasklist to tasklists collection # Use tasklists collection to return tasks backup to user self.process_tasks_job.status = constants.ExportJobStatus.BUILDING self.process_tasks_job.job_progress_timestamp = datetime.datetime.now() self.process_tasks_job.message = 'Retrieving tasks from server ...' self._log_progress("Building") self.process_tasks_job.put() # This list will contain zero or more tasklist dictionaries, which each contain tasks tasklists = [] total_num_tasklists = 0 total_num_tasks = 0 tasks_per_list = [] # --------------------------------------- # Retrieve all the tasklists for the user # --------------------------------------- logging.debug(fn_name + "Retrieve all the tasklists for the user") logservice.flush() next_tasklists_page_token = None more_tasklists_data_to_retrieve = True while more_tasklists_data_to_retrieve: if self.is_test_user: logging.debug(fn_name + "calling tasklists.list().execute() to create tasklists list") logservice.flush() retry_count = settings.NUM_API_TRIES while retry_count > 0: try: if next_tasklists_page_token: tasklists_data = self.tasklists_svc.list(pageToken=next_tasklists_page_token).execute() else: tasklists_data = self.tasklists_svc.list().execute() # Successfully retrieved data, so break out of retry loop break except Exception, e: retry_count = retry_count - 1 if retry_count > 0: if isinstance(e, AccessTokenRefreshError): # Log first 'n' AccessTokenRefreshError as Info, because they are reasonably common, # and the system usually continues normally after the 2nd call to # "new_request: Refreshing due to a 401" # Occassionally, the system seems to need a 3rd attempt # (i.e., success after waiting 45 seconds) logging.info(fn_name + "Access Token Refresh Error whilst retrieving list of tasklists (1st time, not yet an error). " + str(retry_count) + " attempts remaining: " + shared.get_exception_msg(e)) else: logging.warning(fn_name + "Error retrieving list of tasklists. " + str(retry_count) + " attempts remaining: " + shared.get_exception_msg(e)) logservice.flush() if retry_count <= 2: logging.debug(fn_name + "Giving server an extra chance; Sleeping for " + str(settings.WORKER_API_RETRY_SLEEP_DURATION) + " seconds before retrying") logservice.flush() # Update job_progress_timestamp so that job doesn't time out self.process_tasks_job.job_progress_timestamp = datetime.datetime.now() self.process_tasks_job.put() time.sleep(settings.WORKER_API_RETRY_SLEEP_DURATION) else: logging.exception(fn_name + "Still error retrieving list of tasklists after " + str(settings.NUM_API_TRIES) + " attempts. Giving up") logservice.flush() raise e if self.is_test_user and settings.DUMP_DATA: logging.debug(fn_name + "tasklists_data ==>") logging.debug(tasklists_data) logservice.flush() if tasklists_data.has_key(u'items'): tasklists_list = tasklists_data[u'items'] else: # If there are no tasklists, then there will be no 'items' element. This could happen if # the user has deleted all their tasklists. Not sure if this is even possible, but # checking anyway, since it is possible to have a tasklist without 'items' (see issue #9) logging.debug(fn_name + "User has no tasklists.") logservice.flush() tasklists_list = [] # tasklists_list is a list containing the details of the user's tasklists. # We are only interested in the title # if self.is_test_user and settings.DUMP_DATA: # logging.debug(fn_name + "tasklists_list ==>") # logging.debug(tasklists_list) # --------------------------------------- # Process all the tasklists for this user # --------------------------------------- for tasklist_data in tasklists_list: total_num_tasklists = total_num_tasklists + 1 if self.is_test_user and settings.DUMP_DATA: logging.debug(fn_name + "tasklist_data ==>") logging.debug(tasklist_data) logservice.flush() """ Example of a tasklist entry; u'id': u'MDAxNTkzNzU0MzA0NTY0ODMyNjI6MDow', u'kind': u'tasks#taskList', u'selfLink': u'https://www.googleapis.com/tasks/v1/users/@me/lists/MDAxNTkzNzU0MzA0NTY0ODMyNjI6MDow', u'title': u'Default List', u'updated': u'2012-01-28T07:30:18.000Z'}, """ tasklist_title = tasklist_data[u'title'] tasklist_id = tasklist_data[u'id'] if self.is_test_user and settings.DUMP_DATA: logging.debug(fn_name + "Process all the tasks in " + str(tasklist_title)) logservice.flush() # ===================================================== # Process all the tasks in this task list # ===================================================== tasklist_dict, num_tasks = self._get_tasks_in_tasklist(tasklist_title, tasklist_id, include_hidden, include_completed, include_deleted) # Track number of tasks per tasklist tasks_per_list.append(num_tasks) total_num_tasks = total_num_tasks + num_tasks self.process_tasks_job.total_progress = total_num_tasks self.process_tasks_job.tasklist_progress = 0 # Because total_progress now includes num_tasks for current tasklist self.process_tasks_job.job_progress_timestamp = datetime.datetime.now() self.process_tasks_job.message = '' self._log_progress("Processed tasklist") self.process_tasks_job.put() # if self.is_test_user: # logging.debug(fn_name + "Adding %d tasks to tasklist" % len(tasklist_dict[u'tasks'])) # Add the data for this tasklist (including all the tasks) into the collection of tasklists tasklists.append(tasklist_dict) # Check if there is another page of tasklists to be retrieved if tasklists_data.has_key('nextPageToken'): # There is another page of tasklists to be retrieved for this user, # which we'll retrieve next time around the while loop. # This happens if there is more than 1 page of tasklists. # It seems that each page contains 20 tasklists. more_tasklists_data_to_retrieve = True # Go around while loop again next_tasklists_page_token = tasklists_data['nextPageToken'] # if self.is_test_user: # logging.debug(fn_name + "There is (at least) one more page of tasklists to be retrieved") else: # This is the last (or only) page of results (list of tasklists) more_tasklists_data_to_retrieve = False next_tasklists_page_token = None # *** end while more_tasks_data_to_retrieve *** # ------------------------------------------------------ # Store the data, so we can return it to the user # ------------------------------------------------------ """ tasklists is a list of tasklist structures structure of tasklist { "title" : tasklist.title, # Name of this tasklist "tasks" : [ task ] # List of task items in this tasklist } structure of task { "title" : title, # Free text "status" : status, # "completed" | "needsAction" "id" : id, # Used when determining parent-child relationships "parent" : parent, # OPT: ID of the parent of this task (only if this is a sub-task) "notes" : notes, # OPT: Free text "due" : due, # OPT: Date due, e.g. 2012-01-30T00:00:00.000Z NOTE time = 0 "updated" : updated, # Timestamp, e.g., 2012-01-26T07:47:18.000Z "completed" : completed # Timestamp, e.g., 2012-01-27T10:38:56.000Z } """ # Delete existing backup data records tasklist_data_records = model.TasklistsData.gql("WHERE ANCESTOR IS :1", db.Key.from_path(settings.DB_KEY_TASKS_BACKUP_DATA, self.user_email)) num_records = tasklist_data_records.count() logging.debug(fn_name + "Deleting " + str(num_records) + " old blobs") logservice.flush() for tasklists_data_record in tasklist_data_records: tasklists_data_record.delete() # logging.debug(fn_name + "Pickling tasks data ...") pickled_tasklists = pickle.dumps(tasklists) # logging.debug(fn_name + "Pickled data size = " + str(len(pickled_tasklists))) data_len = len(pickled_tasklists) # Multiply by 1.0 float value so that we can use ceiling to find number of Blobs required num_of_blobs = int(math.ceil(data_len * 1.0 / constants.MAX_BLOB_SIZE)) logging.debug(fn_name + "Calculated " + str(num_of_blobs) + " blobs required to store " + str(data_len) + " bytes") logservice.flush() try: for i in range(num_of_blobs): # Write backup data records tasklist_rec = model.TasklistsData(db.Key.from_path(settings.DB_KEY_TASKS_BACKUP_DATA, self.user_email)) slice_start = int(i*constants.MAX_BLOB_SIZE) slice_end = int((i+1)*constants.MAX_BLOB_SIZE) # logging.debug(fn_name + "Creating part " + str(i+1) + " of " + str(num_of_blobs) + # " using slice " + str(slice_start) + " to " + str(slice_end)) pkl_part = pickled_tasklists[slice_start : slice_end] tasklist_rec.pickled_tasks_data = pkl_part tasklist_rec.idx = i tasklist_rec.put() # logging.debug(fn_name + "Marking backup job complete") end_time = datetime.datetime.now() process_time = end_time - start_time proc_time_str = str(process_time.seconds) + "." + str(process_time.microseconds)[:3] + " seconds" # Mark backup completed summary_msg = "Retrieved %d tasks from %d tasklists" % (total_num_tasks, total_num_tasklists) breakdown_msg = "Tasks per list: " + str(tasks_per_list) self.process_tasks_job.status = constants.ExportJobStatus.EXPORT_COMPLETED self.process_tasks_job.job_progress_timestamp = datetime.datetime.now() # self.process_tasks_job.message = summary_msg + " in " + proc_time_str self.process_tasks_job.message = summary_msg + " at " + \ start_time.strftime("%H:%M UTC, %a %d %b %Y") logging.info(fn_name + "COMPLETED: " + summary_msg + " for " + self.user_email + " in " + proc_time_str) logservice.flush() self.process_tasks_job.put() try: end_time = datetime.datetime.now() process_time = end_time - start_time processing_time = process_time.days * 3600*24 + process_time.seconds + process_time.microseconds / 1000000.0 included_options_str = "Includes: Completed = %s, Deleted = %s, Hidden = %s" % (str(include_completed), str(include_deleted), str(include_hidden)) logging.debug(fn_name + "STATS: Job started at " + str(self.process_tasks_job.job_start_timestamp) + "\n Worker started at " + str(start_time) + "\n " + summary_msg + "\n " + breakdown_msg + "\n " + proc_time_str + "\n " + included_options_str) logservice.flush() usage_stats = model.UsageStats( user_hash = hash(self.user_email), number_of_tasks = self.process_tasks_job.total_progress, number_of_tasklists = total_num_tasklists, tasks_per_tasklist = tasks_per_list, include_completed = include_completed, include_deleted = include_deleted, include_hidden = include_hidden, start_time = start_time, processing_time = processing_time) usage_stats.put() logging.debug(fn_name + "Saved stats") logservice.flush() except Exception, e: logging.exception("Error saving stats") logservice.flush() # Don't bother doing anything else, because stats aren't critical except apiproxy_errors.RequestTooLargeError, e: logging.exception(fn_name + "Error putting results in DB - Request too large") logservice.flush() self.process_tasks_job.status = constants.ExportJobStatus.ERROR self.process_tasks_job.message = '' self.process_tasks_job.error_message = "Tasklists data is too large - Unable to store tasklists in DB: " + shared.get_exception_msg(e) self.process_tasks_job.job_progress_timestamp = datetime.datetime.now() self._log_progress("apiproxy_errors.RequestTooLargeError") self.process_tasks_job.put()
def post(self): fn_name = "ProcessTasksWorker.post(): " logging.debug(fn_name + "<start> (app version %s)" %appversion.version) logservice.flush() self.user_email = self.request.get(settings.TASKS_QUEUE_KEY_NAME) self.is_test_user = shared.is_test_user(self.user_email) if self.user_email: # Retrieve the DB record for this user self.process_tasks_job = model.ProcessTasksJob.get_by_key_name(self.user_email) if self.process_tasks_job is None: logging.error(fn_name + "No DB record for " + self.user_email) logservice.flush() logging.debug(fn_name + "<End> No DB record") # TODO: Find some way of notifying the user????? # Could use memcache to relay a message which is displayed in ProgressHandler return logging.debug(fn_name + "Retrieved process tasks job for " + str(self.user_email)) logging.debug(fn_name + "Job was requested at " + str(self.process_tasks_job.job_created_timestamp)) logservice.flush() if self.process_tasks_job.status != constants.ExportJobStatus.TO_BE_STARTED: # Very occassionally, GAE starts a 2nd instance of the worker, so we check for that here. # Check when job status was last updated. If it was less than settings.MAX_JOB_PROGRESS_INTERVAL # seconds ago, assume that another instance is already running, log error and exit time_since_last_update = datetime.datetime.now() - self.process_tasks_job.job_progress_timestamp if time_since_last_update.seconds < settings.MAX_JOB_PROGRESS_INTERVAL: logging.error(fn_name + "It appears that worker was called whilst another job is already running for " + str(self.user_email)) logging.error(fn_name + "Previous job requested at " + str(self.process_tasks_job.job_created_timestamp) + " UTC is still running.") logging.error(fn_name + "Previous worker started at " + str(self.process_tasks_job.job_start_timestamp) + " UTC and last job progress update was " + str(time_since_last_update.seconds) + " seconds ago, with status " + str(self.process_tasks_job.status) ) logging.warning(fn_name + "<End> (Another worker is already running)") logservice.flush() return else: # A previous job hasn't completed, and hasn't updated progress for more than # settings.MAX_JOB_PROGRESS_INTERVAL secons, so assume that previous worker # for this job has died. logging.error(fn_name + "It appears that a previous job requested by " + str(self.user_email) + " at " + str(self.process_tasks_job.job_created_timestamp) + " UTC has stalled.") logging.error(fn_name + "Previous worker started at " + str(self.process_tasks_job.job_start_timestamp) + " UTC and last job progress update was " + str(time_since_last_update.seconds) + " seconds ago, with status " + str(self.process_tasks_job.status) + ", progress = ") if self.process_tasks_job.number_of_job_starts > settings.MAX_NUM_JOB_STARTS: logging.error(fn_name + "This job has already been started " + str(self.process_tasks_job.number_of_job_starts) + " times. Giving up") logging.warning(fn_name + "<End> (Multiple job restart failures)") logservice.flush() return else: logging.info(fn_name + "Attempting to restart backup job. Attempt number " + str(self.process_tasks_job.number_of_job_starts + 1)) logservice.flush() self.process_tasks_job.status = constants.ExportJobStatus.INITIALISING self.process_tasks_job.number_of_job_starts = self.process_tasks_job.number_of_job_starts + 1 self.process_tasks_job.job_progress_timestamp = datetime.datetime.now() self.process_tasks_job.job_start_timestamp = datetime.datetime.now() self.process_tasks_job.message = "Validating background job ..." self._log_progress("Initialising") self.process_tasks_job.put() time_since_job_request = datetime.datetime.now() - self.process_tasks_job.job_created_timestamp logging.debug(fn_name + "Starting job that was requested " + str(time_since_job_request.seconds) + " seconds ago at " + str(self.process_tasks_job.job_created_timestamp) + " UTC") user = self.process_tasks_job.user if not user: logging.error(fn_name + "No user object in DB record for " + str(self.user_email)) logservice.flush() self.process_tasks_job.status = constants.ExportJobStatus.ERROR self.process_tasks_job.message = '' self.process_tasks_job.error_message = "Problem with user details. Please restart." self.process_tasks_job.job_progress_timestamp = datetime.datetime.now() self._log_progress("No user") self.process_tasks_job.put() logging.debug(fn_name + "<End> No user object") return # self.credentials = self.process_tasks_job.credentials # DEBUG: 2012-09-16; Trying a different method of retrieving credentials, to see if it # allows retrieveal of credentials for TAFE account self.credentials = StorageByKeyName(CredentialsModel, user.user_id(), 'credentials').get() if not self.credentials: logging.error(fn_name + "No credentials in DB record for " + str(self.user_email)) logservice.flush() self.process_tasks_job.status = constants.ExportJobStatus.ERROR self.process_tasks_job.message = '' self.process_tasks_job.error_message = "Problem with user credentials. Please restart." self.process_tasks_job.job_progress_timestamp = datetime.datetime.now() self._log_progress("No credentials") self.process_tasks_job.put() logging.debug(fn_name + "<End> No credentials") return if self.credentials.invalid: logging.error(fn_name + "Invalid credentials in DB record for " + str(self.user_email)) logservice.flush() self.process_tasks_job.status = constants.ExportJobStatus.ERROR self.process_tasks_job.message = '' self.process_tasks_job.error_message = "Invalid credentials. Please restart and re-authenticate." self.process_tasks_job.job_progress_timestamp = datetime.datetime.now() self._log_progress("Credentials invalid") logservice.flush() self.process_tasks_job.put() logging.debug(fn_name + "<End> Invalid credentials") return if self.is_test_user: logging.debug(fn_name + "User is test user %s" % self.user_email) logservice.flush() http = httplib2.Http() http = self.credentials.authorize(http) service = discovery.build("tasks", "v1", http=http) self.tasklists_svc = service.tasklists() self.tasks_svc = service.tasks() # ========================================= # Retrieve tasks from the Google server # ========================================= self._export_tasks() else: logging.error(fn_name + "No processing, as there was no user_email key") logservice.flush() logging.debug(fn_name + "<End>") logservice.flush()
class ProcessTasksWorker(webapp.RequestHandler): """ Process tasks according to data in the ProcessTasksJob entity """ credentials = None user_email = None is_test_user = False process_tasks_job = None tasks_svc = None tasklists_svc = None def post(self): fn_name = "ProcessTasksWorker.post(): " logging.debug(fn_name + "<start> (app version %s)" %appversion.version) logservice.flush() client_id, client_secret, user_agent, app_title, project_name, host_msg = shared.get_settings(self.request.host) self.user_email = self.request.get(settings.TASKS_QUEUE_KEY_NAME) self.is_test_user = shared.isTestUser(self.user_email) if self.user_email: # Retrieve the DB record for this user self.process_tasks_job = model.ProcessTasksJob.get_by_key_name(self.user_email) if self.process_tasks_job is None: logging.error(fn_name + "No DB record for " + self.user_email) logservice.flush() logging.debug(fn_name + "<End> No DB record") # TODO: Find some way of notifying the user????? # Could use memcache to relay a message which is displayed in ProgressHandler return else: logging.debug(fn_name + "Retrieved process tasks job for " + str(self.user_email)) logservice.flush() self.process_tasks_job.status = constants.ExportJobStatus.INITIALISING self.process_tasks_job.job_progress_timestamp = datetime.datetime.now() self.process_tasks_job.message = "Validating background job ..." logging.debug(fn_name + "Initialising - Job status: '" + str(self.process_tasks_job.status) + "', progress: " + str(self.process_tasks_job.total_progress) + ", msg: '" + str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'") logservice.flush() self.process_tasks_job.put() user = self.process_tasks_job.user if not user: logging.error(fn_name + "No user object in DB record for " + str(self.user_email)) logservice.flush() self.process_tasks_job.status = constants.ExportJobStatus.ERROR self.process_tasks_job.message = '' self.process_tasks_job.error_message = "Problem with user details. Please restart." self.process_tasks_job.job_progress_timestamp = datetime.datetime.now() logging.debug(fn_name + "No user - Job status: '" + str(self.process_tasks_job.status) + "', progress: " + str(self.process_tasks_job.total_progress) + ", msg: '" + str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'") logservice.flush() self.process_tasks_job.put() logging.debug(fn_name + "<End> No user object") return self.credentials = self.process_tasks_job.credentials if not self.credentials: logging.error(fn_name + "No credentials in DB record for " + str(self.user_email)) logservice.flush() self.process_tasks_job.status = constants.ExportJobStatus.ERROR self.process_tasks_job.message = '' self.process_tasks_job.error_message = "Problem with user self.credentials. Please restart." self.process_tasks_job.job_progress_timestamp = datetime.datetime.now() logging.debug(fn_name + "Job status: '" + str(self.process_tasks_job.status) + "', progress: " + str(self.process_tasks_job.total_progress) + ", msg: '" + str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'") logservice.flush() self.process_tasks_job.put() logging.debug(fn_name + "<End> No self.credentials") return if self.credentials.invalid: logging.error(fn_name + "Invalid credentials in DB record for " + str(self.user_email)) logservice.flush() self.process_tasks_job.status = constants.ExportJobStatus.ERROR self.process_tasks_job.message = '' self.process_tasks_job.error_message = "Invalid self.credentials. Please restart and re-authenticate." self.process_tasks_job.job_progress_timestamp = datetime.datetime.now() logging.debug(fn_name + "Credentials invalid - Job status: '" + str(self.process_tasks_job.status) + "', progress: " + str(self.process_tasks_job.total_progress) + ", msg: '" + str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'") logservice.flush() self.process_tasks_job.put() logging.debug(fn_name + "<End> Invalid self.credentials") return if self.is_test_user: logging.debug(fn_name + "User is test user %s" % self.user_email) logservice.flush() http = httplib2.Http() http = self.credentials.authorize(http) service = discovery.build("tasks", "v1", http) self.tasklists_svc = service.tasklists() self.tasks_svc = service.tasks() self.export_tasks() # logging.debug(fn_name + "Finished processing. Total progress = " + # str(self.process_tasks_job.total_progress) + " for " + str(self.user_email)) else: logging.error(fn_name + "No processing, as there was no user_email key") logservice.flush() logging.debug(fn_name + "<End>, user = "******"export_tasks: " logging.debug(fn_name + "<Start>") logservice.flush() start_time = datetime.datetime.now() include_hidden = self.process_tasks_job.include_hidden include_completed = self.process_tasks_job.include_completed include_deleted = self.process_tasks_job.include_deleted summary_msg = '' # Retrieve all tasks for the user try: logging.debug(fn_name + "include_hidden = " + str(include_hidden) + ", include_completed = " + str(include_completed) + ", include_deleted = " + str(include_deleted)) logservice.flush() # ############################################## # FLOW # ---------------------------------------------- # For each page of taskslists # For each tasklist # For each page of tasks # For each task # Fix date format # Add tasks to tasklist collection # Add tasklist to tasklists collection # Use tasklists collection to return tasks backup to user self.process_tasks_job.status = constants.ExportJobStatus.BUILDING self.process_tasks_job.job_progress_timestamp = datetime.datetime.now() self.process_tasks_job.message = 'Retrieving tasks from server ...' logging.debug(fn_name + "Building - Job status: '" + str(self.process_tasks_job.status) + "', progress: " + str(self.process_tasks_job.total_progress) + ", msg: '" + str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'") logservice.flush() self.process_tasks_job.put() # This list will contain zero or more tasklist dictionaries, which each contain tasks tasklists = [] total_num_tasklists = 0 total_num_tasks = 0 tasks_per_list = [] # --------------------------------------- # Retrieve all the tasklists for the user # --------------------------------------- logging.debug(fn_name + "Retrieve all the tasklists for the user") logservice.flush() next_tasklists_page_token = None more_tasklists_data_to_retrieve = True while more_tasklists_data_to_retrieve: if self.is_test_user: logging.debug(fn_name + "calling tasklists.list().execute() to create tasklists list") logservice.flush() retry_count = constants.NUM_API_RETRIES while retry_count > 0: try: if next_tasklists_page_token: tasklists_data = self.tasklists_svc.list(pageToken=next_tasklists_page_token).execute() else: tasklists_data = self.tasklists_svc.list().execute() # Successfully retrieved data, so break out of retry loop break except Exception, e: retry_count = retry_count - 1 if retry_count > 0: logging.warning(fn_name + "Error retrieving list of tasklists. " + str(retry_count) + " retries remaining") logservice.flush() if retry_count <= 2: logging.debug(fn_name + "Sleeping for " + str(settings.WORKER_API_RETRY_SLEEP_DURATION) + " seconds before retrying") logservice.flush() time.sleep(settings.WORKER_API_RETRY_SLEEP_DURATION) else: logging.exception(fn_name + "Still error retrieving list of tasklists after " + str(constants.NUM_API_RETRIES) + " retries. Giving up") logservice.flush() raise e if self.is_test_user and settings.DUMP_DATA: logging.debug(fn_name + "tasklists_data ==>") logging.debug(tasklists_data) logservice.flush() if tasklists_data.has_key(u'items'): tasklists_list = tasklists_data[u'items'] else: # If there are no tasklists, then there will be no 'items' element. This could happen if # the user has deleted all their tasklists. Not sure if this is even possible, but # checking anyway, since it is possible to have a tasklist without 'items' (see issue #9) logging.debug(fn_name + "User has no tasklists.") logservice.flush() tasklists_list = [] # tasklists_list is a list containing the details of the user's tasklists. # We are only interested in the title # if self.is_test_user and settings.DUMP_DATA: # logging.debug(fn_name + "tasklists_list ==>") # logging.debug(tasklists_list) # --------------------------------------- # Process all the tasklists for this user # --------------------------------------- for tasklist_data in tasklists_list: total_num_tasklists = total_num_tasklists + 1 if self.is_test_user and settings.DUMP_DATA: logging.debug(fn_name + "tasklist_data ==>") logging.debug(tasklist_data) logservice.flush() """ Example of a tasklist entry; u'id': u'MDAxNTkzNzU0MzA0NTY0ODMyNjI6MDow', u'kind': u'tasks#taskList', u'selfLink': u'https://www.googleapis.com/tasks/v1/users/@me/lists/MDAxNTkzNzU0MzA0NTY0ODMyNjI6MDow', u'title': u'Default List', u'updated': u'2012-01-28T07:30:18.000Z'}, """ tasklist_title = tasklist_data[u'title'] tasklist_id = tasklist_data[u'id'] if self.is_test_user and settings.DUMP_DATA: logging.debug(fn_name + "Process all the tasks in " + str(tasklist_title)) logservice.flush() # ===================================================== # Process all the tasks in this task list # ===================================================== tasklist_dict, num_tasks = self.get_tasks_in_tasklist(tasklist_title, tasklist_id, include_hidden, include_completed, include_deleted) # Track number of tasks per tasklist tasks_per_list.append(num_tasks) total_num_tasks = total_num_tasks + num_tasks self.process_tasks_job.total_progress = total_num_tasks self.process_tasks_job.tasklist_progress = 0 # Because total_progress now includes num_tasks for current tasklist self.process_tasks_job.job_progress_timestamp = datetime.datetime.now() self.process_tasks_job.message = '' logging.debug(fn_name + "Processed tasklist. Updated job status: '" + str(self.process_tasks_job.status) + "', progress: " + str(self.process_tasks_job.total_progress) + ", msg: '" + str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'") logservice.flush() self.process_tasks_job.put() # if self.is_test_user: # logging.debug(fn_name + "Adding %d tasks to tasklist" % len(tasklist_dict[u'tasks'])) # Add the data for this tasklist (including all the tasks) into the collection of tasklists tasklists.append(tasklist_dict) # Check if there is another page of tasklists to be retrieved if tasklists_data.has_key('nextPageToken'): # There is another page of tasklists to be retrieved for this user, # which we'll retrieve next time around the while loop. # This happens if there is more than 1 page of tasklists. # It seems that each page contains 20 tasklists. more_tasklists_data_to_retrieve = True # Go around while loop again next_tasklists_page_token = tasklists_data['nextPageToken'] # if self.is_test_user: # logging.debug(fn_name + "There is (at least) one more page of tasklists to be retrieved") else: # This is the last (or only) page of results (list of tasklists) more_tasklists_data_to_retrieve = False next_tasklists_page_token = None # *** end while more_tasks_data_to_retrieve *** # ------------------------------------------------------ # Store the data, so we can return it to the user # ------------------------------------------------------ """ tasklists is a list of tasklist structures structure of tasklist { "title" : tasklist.title, # Name of this tasklist "tasks" : [ task ] # List of task items in this tasklist } structure of task { "title" : title, # Free text "status" : status, # "completed" | "needsAction" "id" : id, # Used when determining parent-child relationships "parent" : parent, # OPT: ID of the parent of this task (only if this is a sub-task) "notes" : notes, # OPT: Free text "due" : due, # OPT: Date due, e.g. 2012-01-30T00:00:00.000Z NOTE time = 0 "updated" : updated, # Timestamp, e.g., 2012-01-26T07:47:18.000Z "completed" : completed # Timestamp, e.g., 2012-01-27T10:38:56.000Z } """ # Delete existing backup data records tasklist_data_records = model.TasklistsData.gql("WHERE ANCESTOR IS :1", db.Key.from_path(settings.DB_KEY_TASKS_BACKUP_DATA, self.user_email)) num_records = tasklist_data_records.count() logging.debug(fn_name + "Deleting " + str(num_records) + " old blobs") logservice.flush() for tasklists_data_record in tasklist_data_records: tasklists_data_record.delete() # logging.debug(fn_name + "Pickling tasks data ...") pickled_tasklists = pickle.dumps(tasklists) # logging.debug(fn_name + "Pickled data size = " + str(len(pickled_tasklists))) data_len = len(pickled_tasklists) # Multiply by 1.0 float value so that we can use ceiling to find number of Blobs required num_of_blobs = int(math.ceil(data_len * 1.0 / constants.MAX_BLOB_SIZE)) logging.debug(fn_name + "Calculated " + str(num_of_blobs) + " blobs required to store " + str(data_len) + " bytes") logservice.flush() try: for i in range(num_of_blobs): # Write backup data records tasklist_rec = model.TasklistsData(db.Key.from_path(settings.DB_KEY_TASKS_BACKUP_DATA, self.user_email)) slice_start = int(i*constants.MAX_BLOB_SIZE) slice_end = int((i+1)*constants.MAX_BLOB_SIZE) # logging.debug(fn_name + "Creating part " + str(i+1) + " of " + str(num_of_blobs) + # " using slice " + str(slice_start) + " to " + str(slice_end)) pkl_part = pickled_tasklists[slice_start : slice_end] tasklist_rec.pickled_tasks_data = pkl_part tasklist_rec.idx = i tasklist_rec.put() # logging.debug(fn_name + "Marking backup job complete") end_time = datetime.datetime.now() process_time = end_time - start_time proc_time_str = str(process_time.seconds) + "." + str(process_time.microseconds)[:3] + " seconds" # Mark backup completed summary_msg = "Retrieved %d tasks from %d tasklists" % (total_num_tasks, total_num_tasklists) breakdown_msg = "Tasks per list: " + str(tasks_per_list) self.process_tasks_job.status = constants.ExportJobStatus.EXPORT_COMPLETED self.process_tasks_job.job_progress_timestamp = datetime.datetime.now() # self.process_tasks_job.message = summary_msg + " in " + proc_time_str self.process_tasks_job.message = summary_msg + " at " + \ self.process_tasks_job.job_start_timestamp.strftime("%H:%M UTC, %a %d %b %Y") # logging.debug(fn_name + "COMPLETED: Export complete - Job status: '" + str(self.process_tasks_job.status) + "', progress: " + # str(self.process_tasks_job.total_progress) + ", msg: '" + # str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'") logging.info(fn_name + "COMPLETED: " + summary_msg + " for " + self.user_email + " in " + proc_time_str) logservice.flush() self.process_tasks_job.put() try: end_time = datetime.datetime.now() process_time = end_time - start_time processing_time = process_time.days * 3600*24 + process_time.seconds + process_time.microseconds / 1000000.0 included_options_str = "Includes: Completed = %s, Deleted = %s, Hidden = %s" % (str(include_completed), str(include_deleted), str(include_hidden)) logging.debug(fn_name + "STATS: Started at " + str(start_time) + "\n " + summary_msg + "\n " + breakdown_msg + "\n " + proc_time_str + "\n " + included_options_str) logservice.flush() usage_stats = model.UsageStats( user_hash = hash(self.user_email), number_of_tasks = self.process_tasks_job.total_progress, number_of_tasklists = total_num_tasklists, tasks_per_tasklist = tasks_per_list, include_completed = include_completed, include_deleted = include_deleted, include_hidden = include_hidden, start_time = start_time, processing_time = processing_time) usage_stats.put() logging.debug(fn_name + "Saved stats") logservice.flush() except Exception, e: logging.exception("Error saving stats") logservice.flush() # Don't bother doing anything else, because stats aren't critical except apiproxy_errors.RequestTooLargeError, e: logging.exception(fn_name + "Error putting results in DB - Request too large") logservice.flush() self.process_tasks_job.status = constants.ExportJobStatus.ERROR self.process_tasks_job.message = '' self.process_tasks_job.error_message = "Tasklists data is too large - Unable to store tasklists in DB: " + shared.get_exception_msg(e) self.process_tasks_job.job_progress_timestamp = datetime.datetime.now() logging.debug(fn_name + "apiproxy_errors.RequestTooLargeError - Job status: '" + str(self.process_tasks_job.status) + "', progress: " + str(self.process_tasks_job.total_progress) + ", msg: '" + str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'") logservice.flush() self.process_tasks_job.put() except Exception, e: logging.exception(fn_name + "Error putting results in DB") logservice.flush() self.process_tasks_job.status = constants.ExportJobStatus.ERROR self.process_tasks_job.message = '' self.process_tasks_job.error_message = "Unable to store tasklists in DB: " + shared.get_exception_msg(e) self.process_tasks_job.job_progress_timestamp = datetime.datetime.now() logging.debug(fn_name + "Exception - Job status: '" + str(self.process_tasks_job.status) + "', progress: " + str(self.process_tasks_job.total_progress) + ", msg: '" + str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'") logservice.flush() self.process_tasks_job.put()
def flush_logs(): try: logservice.flush() except apiproxy_errors.CancelledError: pass
if memcache_payload_md5 == current_payload_md5: logging.info('Twitter Search/Tweets API payload is identical to last time; ending Task') else: logging.info('Twitter Search/Tweets API payload has changed; running MessageQueue') for status in statuses: message = status.get('text') message_id = status.get('id') taskqueue.add( queue_name='messagequeue', url=urls.taskMessageQueue, params={ 'user_id':status.get('user').get('screen_name'), 'message_id':message_id, 'message':message } ) memcache.set(payload_key, value=current_payload_md5, time=1*60*60*24) else: logging.exception(content) else: logging.exception('No twitter_bearer_token found in Memcache, or available from the API call') except Exception, e: logging.exception(e) finally: logservice.flush() return def get(self): self.get_tweets() def post(self): self.get_tweets()
def get_tasks_in_tasklist(self, tasklist_title, tasklist_id, include_hidden, include_completed, include_deleted): """ Returns all the tasks in the tasklist arguments: tasklist_title -- Name of the tasklist tasklist_id -- ID used to retrieve tasks from this tasklist MUST match the ID returned in the tasklist data include_hidden -- If true, include hidden tasks in the backup include_completed -- If true, include completed tasks in the backup include_deleted -- If true, include deleted tasks in the backup returns a tuple; two-element dictionary; 'title' is a string, the name of the tasklist 'tasks' is a list. Each element in the list is dictionary representing 1 task number of tasks """ fn_name = "CreateBackupHandler.get_tasks_in_tasklist(): " tasklist_dict = {} # Blank dictionary for this tasklist tasklist_dict[u'title'] = tasklist_title # Store the tasklist name in the dictionary tasklist_dict[u'id'] = tasklist_id # Store the tasklist ID in the dictionary num_tasks = 0 more_tasks_data_to_retrieve = True next_tasks_page_token = None # Keep track of when last updated, to prevent excessive DB access which could exceed quota prev_progress_timestamp = datetime.datetime.now() if self.is_test_user and settings.DUMP_DATA: logging.debug(fn_name + "include_hidden = " + str(include_hidden) + ", include_completed = " + str(include_completed) + ", include_deleted = " + str(include_deleted)) logservice.flush() # --------------------------------------------------------------------------- # Retrieve the tasks in this tasklist, and store as "tasks" in the dictionary # --------------------------------------------------------------------------- while more_tasks_data_to_retrieve: retry_count = constants.NUM_API_RETRIES while retry_count > 0: try: # Retrieve a page of (up to 100) tasks if next_tasks_page_token: # Get the next page of results # This happens if there are more than 100 tasks in the list # See http://code.google.com/apis/tasks/v1/using.html#api_params # "Maximum allowable value: maxResults=100" tasks_data = self.tasks_svc.list(tasklist = tasklist_id, pageToken=next_tasks_page_token, showHidden=include_hidden, showCompleted=include_completed, showDeleted=include_deleted).execute() else: # Get the first (or only) page of results for this tasklist tasks_data = self.tasks_svc.list(tasklist = tasklist_id, showHidden=include_hidden, showCompleted=include_completed, showDeleted=include_deleted).execute() # Succeeded, so continue break except Exception, e: retry_count = retry_count - 1 if retry_count > 0: logging.warning(fn_name + "Error retrieving tasks, " + str(retry_count) + " retries remaining") logservice.flush() # Last chances - sleep to give the server some extra time before re-requesting if retry_count <= 2: logging.debug(fn_name + "Sleeping for " + str(settings.WORKER_API_RETRY_SLEEP_DURATION) + " seconds before retrying") logservice.flush() time.sleep(settings.WORKER_API_RETRY_SLEEP_DURATION) else: logging.exception(fn_name + "Still error retrieving tasks for tasklist after " + str(constants.NUM_API_RETRIES) + " retries. Giving up") logservice.flush() raise e if self.is_test_user and settings.DUMP_DATA: logging.debug(fn_name + "tasks_data ==>") logging.debug(tasks_data) if not tasks_data.has_key(u'items'): # When using the Google Tasks webpage at https://mail.google.com/tasks/canvas, there will always # be at least one task in any tasklist, because when deleting the last task, a new blank task is # automatically created. # However, a third-party app (e.g., Calengoo on Android) CAN delete all the tasks in a task list, # which results in a tasklist without an 'items' element. logging.debug(fn_name + "No tasks in tasklist") logservice.flush() else: try: tasks = tasks_data[u'items'] # Store all the tasks (List of Dict) except Exception, e: logging.exception(fn_name, "Exception extracting items from tasks_data.") #logging.error(tasks_data) logservice.flush() raise e # if self.is_test_user and settings.DUMP_DATA: # logging.debug(fn_name + "tasks ==>") # logging.debug(tasks) # logservice.flush() # ------------------------------------------------------------------------------------------------ # Fix date/time format for each task, so that the date/time values can be used in Django templates # Convert the yyyy-mm-ddThh:mm:ss.dddZ format to a datetime object, and store that. # There have been occassional format errors in the 'completed' property, # due to 'completed' value such as "-1701567-04-26T07:12:55.000Z" # According to http://docs.python.org/library/datetime.html # "The exact range of years for which strftime() works also varies across platforms. # Regardless of platform, years before 1900 cannot be used." # so if any date/timestamp value is invalid, set the property to '1900-01-01 00:00:00' # NOTE: Sometimes a task has a completion date of '0000-01-01T00:00:00.000Z', which also cannot # be converted to datetime, because the earliest allowable datetime year is 0001 # ------------------------------------------------------------------------------------------------ for t in tasks: num_tasks = num_tasks + 1 date_due = t.get(u'due') if date_due: try: new_due_date = datetime.datetime.strptime(date_due, "%Y-%m-%dT00:00:00.000Z").date() except ValueError, e: new_due_date = datetime.date(1900, 1, 1) logging.warning(fn_name + "Invalid 'due' timestamp (" + str(date_due) + "), so using " + str(new_due_date) + ": " + shared.get_exception_msg(e)) logservice.flush() t[u'due'] = new_due_date datetime_updated = t.get(u'updated') if datetime_updated: try: new_datetime_updated = datetime.datetime.strptime(datetime_updated, "%Y-%m-%dT%H:%M:%S.000Z") except ValueError, e: new_datetime_updated = datetime.datetime(1900, 1, 1, 0, 0, 0) logging.warning(fn_name + "Invalid 'updated' timestamp (" + str(datetime_updated) + "), so using " + str(new_datetime_updated) + ": " + shared.get_exception_msg(e)) logservice.flush() t[u'updated'] = new_datetime_updated
def _get_tasks_in_tasklist(self, tasklist_title, tasklist_id, include_hidden, include_completed, include_deleted): """ Returns all the tasks in the tasklist arguments: tasklist_title -- Name of the tasklist tasklist_id -- ID used to retrieve tasks from this tasklist MUST match the ID returned in the tasklist data include_hidden -- If true, include hidden tasks in the backup include_completed -- If true, include completed tasks in the backup include_deleted -- If true, include deleted tasks in the backup returns a tuple; two-element dictionary; 'title' is a string, the name of the tasklist 'tasks' is a list. Each element in the list is dictionary representing 1 task number of tasks """ fn_name = "CreateBackupHandler._get_tasks_in_tasklist(): " tasklist_dict = {} # Blank dictionary for this tasklist tasklist_dict[u'title'] = tasklist_title # Store the tasklist name in the dictionary tasklist_dict[u'id'] = tasklist_id # Store the tasklist ID in the dictionary num_tasks = 0 more_tasks_data_to_retrieve = True next_tasks_page_token = None # Keep track of when last updated, to prevent excessive DB access which could exceed quota prev_progress_timestamp = datetime.datetime.now() if self.is_test_user and settings.DUMP_DATA: logging.debug(fn_name + "TEST: include_completed = " + str(include_completed) + ", include_hidden = " + str(include_hidden) + ", include_deleted = " + str(include_deleted)) logservice.flush() # --------------------------------------------------------------------------- # Retrieve the tasks in this tasklist, and store as "tasks" in the dictionary # --------------------------------------------------------------------------- while more_tasks_data_to_retrieve: retry_count = settings.NUM_API_TRIES while retry_count > 0: try: # Retrieve a page of (up to 100) tasks if next_tasks_page_token: # Get the next page of results # This happens if there are more than 100 tasks in the list # See http://code.google.com/apis/tasks/v1/using.html#api_params # "Maximum allowable value: maxResults=100" tasks_data = self.tasks_svc.list(tasklist = tasklist_id, pageToken=next_tasks_page_token, showHidden=include_hidden, showCompleted=include_completed, showDeleted=include_deleted).execute() else: # Get the first (or only) page of results for this tasklist tasks_data = self.tasks_svc.list(tasklist = tasklist_id, showHidden=include_hidden, showCompleted=include_completed, showDeleted=include_deleted).execute() # Succeeded, so continue break except Exception, e: retry_count = retry_count - 1 if retry_count > 0: logging.warning(fn_name + "Error retrieving tasks, " + str(retry_count) + " attempts remaining: " + shared.get_exception_msg(e)) logservice.flush() # Last chances - sleep to give the server some extra time before re-requesting if retry_count <= 2: logging.debug(fn_name + "Giving server an extra chance; Sleeping for " + str(settings.WORKER_API_RETRY_SLEEP_DURATION) + " seconds before retrying") logservice.flush() # Update job_progress_timestamp so that job doesn't time out self.process_tasks_job.job_progress_timestamp = datetime.datetime.now() self.process_tasks_job.put() time.sleep(settings.WORKER_API_RETRY_SLEEP_DURATION) else: logging.exception(fn_name + "Still error retrieving tasks for tasklist after " + str(settings.NUM_API_TRIES) + " attempts. Giving up") logservice.flush() raise e if self.is_test_user and settings.DUMP_DATA: logging.debug(fn_name + "tasks_data ==>") logging.debug(tasks_data) if not tasks_data.has_key(u'items'): # When using the Google Tasks webpage at https://mail.google.com/tasks/canvas, there will always # be at least one task in any tasklist, because when deleting the last task, a new blank task is # automatically created. # However, a third-party app (e.g., Calengoo on Android) CAN delete all the tasks in a task list, # which results in a tasklist without an 'items' element. logging.debug(fn_name + "No tasks in tasklist") logservice.flush() else: try: tasks = tasks_data[u'items'] # Store all the tasks (List of Dict) except Exception, e: logging.exception(fn_name, "Exception extracting items from tasks_data: " + shared.get_exception_msg(e)) #logging.error(tasks_data) logservice.flush() raise e # if self.is_test_user and settings.DUMP_DATA: # logging.debug(fn_name + "tasks ==>") # logging.debug(tasks) # logservice.flush() for t in tasks: num_tasks = num_tasks + 1 # TODO: Investigate if including this will cause memory to be exceeded for very large tasks list # Store original RFC-3339 timestamps (used for raw2 export format) if t.has_key('due'): t['due_RFC3339'] = t['due'] if t.has_key('updated'): t['updated_RFC3339'] = t['updated'] if t.has_key('completed'): t['completed_RFC3339'] = t['completed'] # Converts the RFC-3339 string returned by the server to a date or datetime object # so that other methods (such as Django templates) can display a custom formatted date shared.set_timestamp(t, u'due', date_only=True) shared.set_timestamp(t, u'updated') shared.set_timestamp(t, u'completed') if tasklist_dict.has_key(u'tasks'): # This is the n'th page of task data for this tasklist, so extend the existing list of tasks tasklist_dict[u'tasks'].extend(tasks) else: # This is the first (or only) list of task for this tasklist tasklist_dict[u'tasks'] = tasks
def redirect_for_auth(self, user, redirect_url=None): """Redirects the webapp response to authenticate the user with OAuth2. Args: redirect_url [OPTIONAL] The URL to return to once authorised. Usually unused (left as None), so that the URL of the calling page is used Uses the 'state' parameter to store redirect_url. The handler for /oauth2callback can therefore redirect the user back to the page they were on when get_credentials() failed (or to the specified redirect_url). """ fn_name = "redirect_for_auth(): " try: client_id, client_secret, user_agent, app_title, product_name, host_msg = get_settings(self.request.host) # Check how many times this has been called (without credentials having been successfully retrieved) if self.request.cookies.has_key('auth_retry_count'): auth_retry_count = int(self.request.cookies['auth_retry_count']) logging.debug(fn_name + "auth_retry_count = " + str(auth_retry_count)) if auth_retry_count > settings.MAX_NUM_AUTH_RETRIES: # Exceeded maximum number of retries, so don't try again. # Redirect user to invalid credentials page logging.warning(fn_name + "Not re-authorising, because there have already been " + str(auth_retry_count) + " attempts. Redirecting to " + settings.INVALID_CREDENTIALS_URL) self.redirect(settings.INVALID_CREDENTIALS_URL + "?rc=NC&nr=" + str(auth_retry_count)) return else: logging.debug(fn_name + "No auth_retry_count cookie found. Probably means last authorisation was more than " + str(settings.AUTH_RETRY_COUNT_COOKIE_EXPIRATION_TIME) + " seconds ago") auth_retry_count = 0 if not redirect_url: # By default, return to the same page redirect_url = self.request.path_qs # According to https://developers.google.com/accounts/docs/OAuth_ref#RequestToken # xoauth_displayname is optional. # (optional) String identifying the application. # This string is displayed to end users on Google's authorization confirmation page. # For registered applications, the value of this parameter overrides the name set during registration and # also triggers a message to the user that the identity can't be verified. # For unregistered applications, this parameter enables them to specify an application name, # In the case of unregistered applications, if this parameter is not set, Google identifies the application # using the URL value of oauth_callback; if neither parameter is set, Google uses the string "anonymous". # It seems preferable to NOT supply xoauth_displayname, so that Google doesn't display "identity can't be verified" msg. flow = client.OAuth2WebServerFlow( client_id=client_id, client_secret=client_secret, scope="https://www.googleapis.com/auth/tasks", user_agent=user_agent, state=redirect_url) callback = self.request.relative_url("/oauth2callback") authorize_url = flow.step1_get_authorize_url(callback) memcache.set(user.user_id(), pickle.dumps(flow)) # Keep track of how many times we've called the authorise URL if self.request.cookies.has_key('auth_retry_count'): auth_retry_count = int(self.request.cookies['auth_retry_count']) else: auth_retry_count = 0 __store_auth_retry_count(self, auth_retry_count + 1) logging.debug(fn_name + "Redirecting to " + str(authorize_url)) logservice.flush() self.redirect(authorize_url) except Exception, e: logging.exception(fn_name + "Caught top-level exception") serve_outer_exception_message(self, e) logging.debug(fn_name + "<End> due to exception" ) logservice.flush()