def set_cookie(res, key, value='', max_age=None,
path='/', domain=None, secure=None, httponly=False,
version=None, comment=None):
"""
Set (add) a cookie for the response
"""
fn_name = "set_cookie(): "
cookies = Cookie.SimpleCookie()
cookies[key] = value
for var_name, var_value in [
('max-age', max_age),
('path', path),
('domain', domain),
('secure', secure),
('HttpOnly', httponly),
('version', version),
('comment', comment),
]:
if var_value is not None and var_value is not False:
cookies[key][var_name] = str(var_value)
if max_age is not None:
cookies[key]['expires'] = max_age
header_value = cookies[key].output(header='').lstrip()
res.headers.add_header("Set-Cookie", header_value)
logging.debug(fn_name + "Writing cookie: '" + str(key) + "' = '" + str(value) + "', max age = '" + str(max_age) + "'")
logservice.flush()
def set_cookie(res, key, value='', max_age=None,
path='/', domain=None, secure=None, httponly=False,
version=None, comment=None):
"""
Set (add) a cookie for the response
"""
fn_name = "set_cookie(): "
cookies = Cookie.SimpleCookie()
cookies[key] = value
for var_name, var_value in [
('max-age', max_age),
('path', path),
('domain', domain),
('secure', secure),
('HttpOnly', httponly),
('version', version),
('comment', comment),
]:
if var_value is not None and var_value is not False:
cookies[key][var_name] = str(var_value)
if max_age is not None:
cookies[key]['expires'] = max_age
header_value = cookies[key].output(header='').lstrip()
res.headers.add_header("Set-Cookie", header_value)
logging.debug(fn_name + "Writing cookie: '" + str(key) + "' = '" + str(value) + "', max age = '" + str(max_age) + "'")
logservice.flush()
def post(self):
line_bytes = int(self.request.get('line_bytes'))
line_count = int(self.request.get('line_count'))
flush_every = int(self.request.get('flush_count'))
entity_count = int(self.request.get('entity_count'))
logging.info("Test Specs: %s", {
'line_bytes': line_bytes,
'line_count': line_count,
'flush_every': flush_every,
'entity_count': entity_count,
})
#logservice.flush()
repeat_count = max(1, line_bytes / len(string.ascii_letters))
logging_data = list(
itertools.repeat(string.ascii_letters, repeat_count))
for line in xrange(line_count):
random.shuffle(logging_data)
logging.info("%s: %s", line, ''.join(logging_data))
if flush_every and not line % flush_every:
logservice.flush()
logging.info("Writing entities")
db.put(Entity(keyname=i) for i in xrange(100))
logging.info("Done writing entities")
def post(self):
""" Delete a selection of Blobstores (selected in form, and posted """
fn_name = "BulkDeleteBlobstoreHandler.post(): "
logging.debug(fn_name + "<Start>")
logservice.flush()
try:
self.response.out.write('<html><body>')
blobstores_to_delete = self.request.get_all('blob_key')
del_count = 0
for blob_key in blobstores_to_delete:
blob_info = blobstore.BlobInfo.get(blob_key)
if blob_info:
try:
blob_info.delete()
del_count = del_count + 1
except Exception, e:
logging.exception(fn_name + "Exception deleting blobstore [" + str(del_count) + "] " + str(blob_key))
self.response.out.write("""<div>Error deleting blobstore %s</div>%s""" % (blob_key, shared.get_exception_msg(e)))
else:
self.response.out.write("""<div>Blobstore %s doesn't exist</div>""" % blob_key)
self.response.out.write('Deleted ' + str(del_count) + ' blobstores')
self.response.out.write('<br /><br /><a href="' + settings.ADMIN_MANAGE_BLOBSTORE_URL + '">Back to Blobstore Management</a><br /><br />')
self.response.out.write("""<br /><br /><a href=""" + settings.MAIN_PAGE_URL + """>Home page</a><br /><br />""")
self.response.out.write('</body></html>')
logging.debug(fn_name + "<End>")
logservice.flush()
def _process_analytics(self, cls):
try:
c = 0
now = datetime.now()
# Check if this is the first time running
last_ana = cls.get_last()
if last_ana is None: # This is the first time running
start_time = cls.__base__.get_first_ts()
log.info('first: %s', start_time)
if not start_time: # Check if there are any Analytics Stats
log.warn('No Analytics Stats')
return # There are no analyticstats to process
start_time = cls.get_start_time(start_time=start_time)
end_time = cls.get_end_time(start_time)
parents = []
log.debug('Loop Times s: %s e: %s', start_time, end_time)
while end_time < now:
log.info('Loop Count %s: Start Time: %s End Time: %s', c,
start_time, end_time)
c += 1
if hasattr(cls, 'entity'):
parents += self._process_time_frame_with_entity(
start_time, end_time, cls)
else:
parents += self._process_time_frame(
start_time, end_time, cls)
start_time = end_time
end_time = cls.get_end_time(start_time)
logservice.flush()
self._finallize_anastats(parents)
ndb.put_multi(parents)
else: # We have already processed ConceptAnalyticsDailyStat before
start_time = last_ana.end_time
end_time = cls.get_end_time(start_time)
log.debug('Times s: %s e: %s', start_time, end_time)
while end_time < datetime.now():
if hasattr(cls, 'entity'):
ndb.put_multi(
self._finallize_anastats(
self._process_time_frame_with_entity(
start_time, end_time, cls)))
else:
ndb.put_multi(
self._finallize_anastats(
self._process_time_frame(
start_time, end_time, cls)))
start_time = end_time
end_time = cls.get_end_time(start_time)
except Exception:
raise
def set_RFC3339_timestamp(task_row_data, field_name, formats):
""" Parse timestamp field value and replace with an RFC-3339 datetime string.
task_row_data A row of data, as a dictionary object
field_name The name of the field to be set
formats A list of possible datetime string formats, as used by strptime()
Parses the specified field and replaces the value with an RFC-3339 datetime string as required
by the Google server.
CAUTION: Attempting to create a task with a zero 'due' value results in strange behaviour;
insert() returns a task object with an 'id', however attemping to get() that 'id' returns
"404 Not found"
If there is a major problem, such as the field no being able to be parsed at all, we delete
the field from the task_row_data.
"""
fn_name = "set_RFC3339_timestamp: "
try:
if field_name in task_row_data:
if task_row_data[field_name] == None:
# If we find a None value, we restore the "Zero" RFC-3339 date value here.
# GTB stores '0000-01-01T00:00:00.000Z' datetime values (aka constants.ZERO_RFC3339_DATETIME_STRING)
# as None, because there is no way to store dates earlier than 1900 in a datetime object
logging.debug(fn_name + "DEBUG: '" + field_name + "' value is None; setting value to '" +
constants.ZERO_RFC3339_DATETIME_STRING + "'")
task_row_data[field_name] = constants.ZERO_RFC3339_DATETIME_STRING
return
# Field exists, so convert the value to an RFC-3339 datetime string
datetime_str = task_row_data[field_name].strip()
rfc_datetime_str = convert_datetime_string_to_RFC3339(datetime_str, field_name, formats)
if rfc_datetime_str:
task_row_data[field_name] = rfc_datetime_str
else:
# Unable to parse the datetime value, so delete the field which has the un-parseable value.
# The server's behaviour for a missing field depends on the field and task state;
# If the 'due' field is missing, the created task will not have a due date
# If the 'completed' field is missing;
# If the 'status' is 'completed', the server uses the current date and time
# If the 'status' is 'needsAction', the created task will not have a completed date
try:
del(task_row_data[field_name])
except Exception, e:
logging.error(fn_name + "Unable to delete '" + field_name + "': " + get_exception_msg(e))
logservice.flush()
except Exception, e:
logging.exception(fn_name + "Error attempting to set '" + field_name + "' datetime field value, so deleting field")
# Delete the field which caused the exception
try:
del(task_row_data[field_name])
except Exception, e:
logging.error(fn_name + "Unable to delete '" + field_name + "': " + get_exception_msg(e))
def timer_loop():
while True:
now = time.time()
then = now - (now % 180) + 180
seconds = then - now
logger.debug("Next calculation in " + str(seconds) + " seconds")
logservice.flush()
time.sleep(seconds)
compute_leaderboard()
def serve_outer_exception_message(self, e):
""" Display an Oops message when something goes very wrong.
This is called from the outer exception handler of major methods (such as get/post handlers)
"""
fn_name = "serve_outer_exception_message: "
self.response.out.write("""Oops! Something went terribly wrong.<br />%s<br /><br />This system is in beta, and is being activeley developed.<br />Please report any errors to <a href="http://%s">%s</a> so that they can be fixed. Thank you.""" %
( get_exception_msg(e), settings.url_issues_page, settings.url_issues_page))
logging.error(fn_name + get_exception_msg(e))
logservice.flush()
def AppWrapper(env, start_response):
"""Wrapper for <app>."""
status_list = []
headers_list = []
exc_info_list = []
write_buffer_list = []
def MyWrite(buf, s):
if s is not None:
buf.append(s)
def DeferredStartResponse(status, headers, exc_info=None):
status_list.append(status)
headers_list.append(headers)
exc_info_list.append(exc_info)
buf = []
write_buffer_list.append(buf)
return functools.partial(MyWrite, buf)
retval = []
result = app(env, DeferredStartResponse)
if result is not None:
for value in result:
retval.append(value)
if logservice.log_buffer_bytes():
logservice.flush()
flush_count = str(getattr(
request_environment.current_request, 'flush_count', -1))
if headers_list:
headers_list[0].append((LOG_FLUSH_COUNTER_HEADER, flush_count))
for status, headers, exc_info, write_buffer in zip(
status_list, headers_list, exc_info_list, write_buffer_list):
write_fn = start_response(status, headers, exc_info)
if write_buffer:
write_fn(''.join(write_buffer))
write_buffer = []
return retval
def _save_gcs_image(self, img, mime_type, image_size):
size_in_pixels = img.size[0] * img.size[1]
import logging
from google.appengine.api import runtime
from google.appengine.api import logservice
logservice.AUTOFLUSH_ENABLED = False
logging.debug("image size=%s" % size_in_pixels)
logservice.flush()
if size_in_pixels > 12 * (10**6):
del img
raise ValueError("Image too large (%s pixels)", size_in_pixels)
if mime_type.lower().startswith("image/"):
format_ = mime_type[6:].upper()
else:
raise ValueError("Unexpected MIME type %s" % mime_type)
settings = COMPRESSION_SETTINGS[image_size]
ImageFile.MAXBLOCK = 2**20
img_io = StringIO()
if settings.get('thumbnail_dimensions'):
img = img.copy()
img.thumbnail(settings['thumbnail_dimensions'], PIL_Image.ANTIALIAS)
img = img.convert('RGB')
img.save(
img_io,
format_,
quality=settings['quality'],
optimize=settings['quality'],
progressive=settings['progressive']
)
img_data = img_io.getvalue()
_, saved_width, saved_height = _get_image_info(img_data)
img_io.close()
file_name = "/images/%s/%s.jpg" % (
self._allocated_image_key.id(),
settings['file_shortname']
)
blob_key = _save_gcs_object(
img_data,
file_name,
content_type=mime_type,
options={'x-goog-acl': 'public-read'}
)
return blob_key, img_data, [saved_width, saved_height]
def check_task_params_exist(tasklists_svc, tasks_svc, tasklist_id, parent_id, sibling_id):
fn_name = "check_task_params_exist: "
logging.debug(fn_name + "DEBUG: Checking tasklist [" + str(tasklist_id) +
"], parent [" + str(parent_id) +
"], sibling [" + str(sibling_id) + "]")
result = []
# Check if tasklist exists
if tasklist_id:
if not tasklist_exists(tasklists_svc, tasklist_id):
# logging.error(fn_name + "ERROR: Tasklist [" + str(tasklist_id) + "] doesn't exist")
# logservice.flush()
result.append("Tasklist [" + str(tasklist_id) + "] doesn't exist")
else:
# logging.error(fn_name + "ERROR: No tasklist ID!")
# logservice.flush()
result.append("No tasklist ID")
# Check if parent task exists (if it was specified)
if parent_id:
if not task_exists(tasks_svc, tasklist_id, parent_id):
# logging.error(fn_name + "ERROR: Parent task [" + str(parent_id) + "] doesn't exist")
# logservice.flush()
result.append("Parent task [" + str(parent_id) + "] doesn't exist")
# else:
# logging.debug(fn_name + "DEBUG: No parent ID")
# logservice.flush()
# Check if sibling task exists (if it was specified)
if sibling_id:
if not task_exists(tasks_svc, tasklist_id, sibling_id):
# logging.error(fn_name + "ERROR: Sibling task [" + str(sibling_id) + "] doesn't exist")
# logservice.flush()
result.append("Sibling task [" + str(sibling_id) + "] doesn't exist")
# else:
# logging.debug(fn_name + "DEBUG: No sibling ID")
# logservice.flush()
if len(result) > 0:
logging.error(fn_name + "ERROR: " + ', '.join(result))
else:
logging.debug(fn_name + "DEBUG: All task params exist")
logservice.flush()
def dispatch(self, *args, **kwargs):
self.session_store = sessions.get_store(request=self.request)
self.enforce_ssl()
self.fetch_user()
logservice.flush()
self.response.headers.add_header('X-Frame-Options', 'DENY')
self.response.headers.add_header('Content-Type',
'text/html; charset=UTF-8')
self.response.headers.add_header('X-Content-Type-Options', 'nosniff')
self.response.headers.add_header('X-XSS-Protection', '1; mode=block')
try:
webapp2.RequestHandler.dispatch(self, *args, **kwargs)
except Exception:
raise
finally:
self.session_store.save_sessions(self.response)
def reject_non_test_user(self):
fn_name = "reject_non_test_user: "******"Rejecting non-test user on limited access server")
logservice.flush()
# self.response.out.write("<html><body><h2>This is a test server. Access is limited to test users.</h2>" +
# "<br /><br /><div>Please use the production server at <href='http://tasks-backup.appspot.com'>tasks-backup.appspot.com</a></body></html>")
# logging.debug(fn_name + "<End> (restricted access)" )
serve_message_page(self,
"This is a test server. Access is limited to test users.",
"Please click the button to go to the production server at tasks-backup.appspot.com",
show_custom_button=True,
custom_button_text='Go to live server',
custom_button_url='http://tasks-backup.appspot.com',
show_heading_messages=False)
def delete_blobstore(blob_info):
fn_name = "delete_blobstore(): "
# logging.debug(fn_name + "<Start>")
# logservice.flush()
if blob_info:
# -------------------------------------
# Delete the Blobstore item
# -------------------------------------
try:
blob_info.delete()
logging.debug(fn_name + "Blobstore deleted")
logservice.flush()
except Exception, e:
logging.exception(fn_name + "Exception deleting %s, key = %s" % (blob_info.filename, blob_info.key()))
logservice.flush()
def _log_progress(self, prefix_msg=""):
fn_name = "_log_progress: "
if prefix_msg:
logging.debug(fn_name + prefix_msg + " - Job status = '" + str(self.process_tasks_job.status) +
"', progress: " + str(self.process_tasks_job.total_progress))
else:
logging.debug(fn_name + "Job status = '" + str(self.process_tasks_job.status) + "', progress: " +
"', progress: " + str(self.process_tasks_job.total_progress))
if self.process_tasks_job.message:
logging.debug(fn_name + "Message = " + str(self.process_tasks_job.message))
if self.process_tasks_job.error_message:
logging.debug(fn_name + "Error message = " + str(self.process_tasks_job.error_message))
logservice.flush()
def movie_listings(max_pages=1, subreddits=SUBREDDITS, newest_only=True):
'''wrapper for bulk fetching movie listings from reddit.
if newest_only is True, walks forward through results using the cached before_cursor
(otherwise it walks backwards) until either max_pages is reached or results are exhausted'''
for subreddit in subreddits:
after_cursor = None
before_cursor = fetch_before_cursor(subreddit) if newest_only else None
for _ in range(max_pages):
movies, before_cursor, after_cursor = fetch_and_parse_raw_movie_listings(subreddit, before_cursor=before_cursor, after_cursor=after_cursor)
if not len(movies):
break
for movie in movies:
yield movie
logservice.flush()
if before_cursor is None: # i.e we've run out of listings
break
if newest_only is True:
after_cursor = None
else:
before_cursor = None
def logThis(nLogLevel, strMessage):
if ARE_WE_PYTH_APP == True:
print strMessage
return
if nLogLevel == AEL_LEVEL_DEBUG:
logging.debug(WE_APP_NAME + strMessage)
return
if nLogLevel == AEL_LEVEL_INFO:
logging.info(WE_APP_NAME + strMessage)
return
if nLogLevel == AEL_LEVEL_WARNING:
logging.warning(WE_APP_NAME + strMessage)
return
if nLogLevel == AEL_LEVEL_ERROR:
logging.error(WE_APP_NAME + strMessage)
return
if nLogLevel == AEL_LEVEL_CRITICAL:
logging.critical(WE_APP_NAME + strMessage)
return
logservice.flush()
def logThis(nLogLevel, strMessage):
if ARE_WE_PYTH_APP == True:
print strMessage
return
if nLogLevel == AEL_LEVEL_DEBUG:
logging.debug(WE_APP_NAME + strMessage)
return
if nLogLevel == AEL_LEVEL_INFO:
logging.info(WE_APP_NAME + strMessage)
return
if nLogLevel == AEL_LEVEL_WARNING:
logging.warning(WE_APP_NAME + strMessage)
return
if nLogLevel == AEL_LEVEL_ERROR:
logging.error(WE_APP_NAME + strMessage)
return
if nLogLevel == AEL_LEVEL_CRITICAL:
logging.critical(WE_APP_NAME + strMessage)
return
logservice.flush()
def index_project(self, ):
count = 1
num = len(self.project_keys)
log.info('Starting indexing')
logservice.flush()
for pro_key in self.project_keys:
pro = pro_key.get()
log.info('Index %s of %s projects', str(count), str(num))
logservice.flush()
indexes = pro.get_put_index()
pro.index(indexes)
docs = ndb.get_multi(pro.documents)
docs.append(pro.distilled_document.get())
for doc in docs:
doc.index(indexes)
children = ndb.get_multi(pro.children)
for child in children:
if child:
child.index_phrasings(indexes, index_children=True)
count += 1
log.info('Finished indexing')
logservice.flush()
def AppWrapper(env, start_response):
"""Wrapper for <app>."""
status_list = []
headers_list = []
exc_info_list = []
write_buffer_list = []
def MyWrite(buf, s):
if s is not None:
buf.append(s)
def DeferredStartResponse(status, headers, exc_info=None):
status_list.append(status)
headers_list.append(headers)
exc_info_list.append(exc_info)
buf = []
write_buffer_list.append(buf)
return functools.partial(MyWrite, buf)
retval = []
result = app(env, DeferredStartResponse)
if result is not None:
for value in result:
retval.append(value)
if logservice.log_buffer_bytes():
logservice.flush()
flush_count = str(
getattr(request_environment.current_request, 'flush_count', -1))
if headers_list:
headers_list[0].append((LOG_FLUSH_COUNTER_HEADER, flush_count))
for status, headers, exc_info, write_buffer in zip(
status_list, headers_list, exc_info_list, write_buffer_list):
write_fn = start_response(status, headers, exc_info)
if write_buffer:
write_fn(''.join(write_buffer))
write_buffer = []
return retval
def _run(self):
log.info('Processing Aanlytic Daily Stats')
logservice.flush()
self._process_analytics(AnalyticsDailyStat)
# log.info('Processing Project Aanlytic Daily Stats')
# self._process_analytics(ProjectAnalyticsDailyStat)
log.info('Processing Concept Aanlytic Daily Stats')
logservice.flush()
self._process_analytics(ConceptAnalyticsDailyStat)
log.info('Processing Organization Aanlytic Daily Stats')
logservice.flush()
self._process_analytics(OrgAnalyticsDailyStat)
log.info('Processing User Aanlytic Daily Stats')
logservice.flush()
self._process_analytics(UserAnalyticsDailyStat)
log.info('Finished')
logservice.flush()
def get(self):
hours = int(self.request.get('hours', 24))
log.info('Hours: %s', hours)
logservice.flush()
an_action_count = {}
for action in ANALYTIC_ACTIONS:
an_action_count[action] = int(self.request.get(action, 2))
start = datetime.now().replace(minute=0, second=0,
microsecond=0) - timedelta(hours=hours)
log.info('Start Time: %s', start)
logservice.flush()
total_per_hour = 0
for count in an_action_count.itervalues():
total_per_hour += count
log.info('total per hours: %s', total_per_hour)
logservice.flush()
ana_obj = []
for h in xrange(0, hours):
log.info('Current Hour: %s', h)
logservice.flush()
start += timedelta(hours=1)
for action, count in an_action_count.iteritems():
for i in xrange(0, count):
ana_obj.append(
Analytics.new(
ts=start +
timedelta(minutes=random.randint(0, 59)),
artifact=ndb.Key(
'Concept', '043155e8eb4311e3bc89534d9485b528'),
artifact_owners=[ndb.Key('User', 'amiller')],
project=ndb.Key(
'Project', '03b0826beb4311e3a269534d9485b528'),
action=action,
))
ndb.put_multi(ana_obj)
ana_obj[:] = []
self.response.write(Analytics.query().count())
def set_timestamp(task, field_name, date_only=False):
""" Parse timestamp field value and replace with a datetime object.
task A task dictionary object, as returned from the Google server
field_name The name of the field to be set
date_only If True, store a date object instead of datetime
Parses the specified field and stores a date or datetime object so that methods which access
the task (including Django templates) can format the displayed date.
If there is a major problem, such as the field no being able to be parsed at all, we delete
the field from the task dictionary.
"""
fn_name = "set_timestamp: "
try:
if field_name in task:
# Field exists, so try to parse its value
try:
datetime_str = task[field_name]
task[field_name] = convert_RFC3339_string_to_datetime(datetime_str, field_name, date_only)
except Exception, e:
try:
logging.error(fn_name + "Unable to parse '" + field_name +
"' datetime field value '" + str(datetime_str) + "', so deleting field: " +
get_exception_msg(e))
except Exception, e:
# In case logging the value causes an exception, log without value
logging.error(fn_name + "Unable to parse '" + field_name +
"' datetime field value, and unable to log field value, so deleting field: " +
get_exception_msg(e))
# Delete the field which has the un-parseable value
try:
del(task[field_name])
except Exception, e:
logging.error(fn_name + "Unable to delete '" + field_name + "': " + get_exception_msg(e))
logservice.flush()
def _process_hour_block(self, anas):
log.info('Processing %s anlytics', len(anas))
logservice.flush()
anastat = AnalyticsHourlyStat.new()
anastat.start_time = self.start_time
anastat.end_time = self.end_time
for ana in anas:
if ana.artifact_organization:
aid = ana.artifact_organization.id(
) + '-' + self.end_time.strftime('%Y-%m-%d-%H-%M')
if aid in self.org_ana_stats:
self._record_ana_action(self.org_ana_stats[aid], ana)
else:
org_ana_stat = OrgAnalyticsHourlyStat.new(id=aid)
org_ana_stat.start_time = self.start_time
org_ana_stat.end_time = self.end_time
org_ana_stat.entity = ana.artifact_organization
org_ana_stat.entity_organization = ana.artifact_organization
self.org_ana_stats[org_ana_stat.key.id()] = org_ana_stat
self._record_ana_action(org_ana_stat, ana)
# if ana.artifact and ana.artifact.kind() == 'Project':
# self._record_project_ana_action(ana)
if ana.artifact and ana.artifact.kind() == 'Concept':
self._record_concept_ana_action(ana)
self._record_user_ana_action(ana)
self._record_ana_action(anastat, ana)
anastats = [anastat] + self.art_ana_stats.values(
) + self.org_ana_stats.values() + self.user_ana_stats.values()
self._finallize_anastats(anastats)
ndb.put_multi(anastats)
def movie_listings(max_pages=1, subreddits=SUBREDDITS, newest_only=True):
'''wrapper for bulk fetching movie listings from reddit.
if newest_only is True, walks forward through results using the cached before_cursor
(otherwise it walks backwards) until either max_pages is reached or results are exhausted'''
for subreddit in subreddits:
after_cursor = None
before_cursor = fetch_before_cursor(subreddit) if newest_only else None
for _ in range(max_pages):
movies, before_cursor, after_cursor = fetch_and_parse_raw_movie_listings(
subreddit,
before_cursor=before_cursor,
after_cursor=after_cursor)
if not len(movies):
break
for movie in movies:
yield movie
logservice.flush()
if before_cursor is None: # i.e we've run out of listings
break
if newest_only is True:
after_cursor = None
else:
before_cursor = None
def process(self):
logging.info('Backend running')
server_name = backends.get_backend()
logging.info(server_name)
servers = memcache.get('servers')
if servers is None:
servers = dict()
servers[server_name] = 1
else:
if server_name not in servers:
servers[server_name] = 1
else:
servers[server_name] = servers[server_name] + 1
memcache.set('servers', servers)
logservice.flush()
runtime.set_shutdown_hook(self.shutdown)
self.response.set_status(200)
return
def get(self):
l1 = logging.getLogger("test");
l2 = logging.getLogger("test.test");
l1.critical("nohandler: test nohandler")
l1.info("nohandler: test nohandler")
l1.warn("nohandler: test nohandler")
l1.critical("nohandler: test nohandler")
l2.critical("stdouthandler: test handler")
logging.getLogger(__name__).debug(__name__ + ": debug")
logging.getLogger(__name__).critical(__name__ + ": critical")
l1.critical("nohandler: test nohandler")
l2.critical("stdouthandler: test handler")
logservice.flush()
logservice.AUTOFLUSH_ENABLED = True
logservice.AUTOFLUSH_EVERY_SECONDS = None
logservice.AUTOFLUSH_EVERY_BYTES = None
logservice.AUTOFLUSH_EVERY_LINES = 1
time.sleep(5)
l1.critical("post sleep")
l2.critical("post sleep")
open("etc", "w")
#t = [0] * 268435456
l1.critical("post big alloc")
l2.critical("post big alloc")
self.response.headers['Content-Type'] = 'text/plain'
self.response.write(self.request)
def serve_message_page(self, msg1, msg2 = None, msg3 = None,
show_back_button=False,
back_button_text="Back to previous page",
show_custom_button=False, custom_button_text='Try again', custom_button_url=settings.MAIN_PAGE_URL,
show_heading_messages=True):
""" Serve message.html page to user with message, with an optional button (Back, or custom URL)
msg1, msg2, msg3 Text to be displayed.msg2 and msg3 are option. Each msg is displayed in a separate div
show_back_button If True, a [Back] button is displayed, to return to previous page
show_custom_button If True, display button to jump to any URL. title set by custom_button_text
custom_button_text Text label for custom button
custom_button_url URL to go to when custom button is pressed
show_heading_messages If True, display app_title and (optional) host_msg
"""
fn_name = "serve_message_page: "
logging.debug(fn_name + "<Start>")
logservice.flush()
try:
client_id, client_secret, user_agent, app_title, product_name, host_msg = get_settings(self.request.host)
if msg1:
logging.debug(fn_name + "Msg1: " + msg1)
if msg2:
logging.debug(fn_name + "Msg2: " + msg2)
if msg3:
logging.debug(fn_name + "Msg3: " + msg3)
path = os.path.join(os.path.dirname(__file__), constants.PATH_TO_TEMPLATES, "message.html")
template_values = {'app_title' : app_title,
'host_msg' : host_msg,
'msg1': msg1,
'msg2': msg2,
'msg3': msg3,
'show_heading_messages' : show_heading_messages,
'show_back_button' : show_back_button,
'back_button_text' : back_button_text,
'show_custom_button' : show_custom_button,
'custom_button_text' : custom_button_text,
'custom_button_url' : custom_button_url,
'product_name' : product_name,
'url_discussion_group' : settings.url_discussion_group,
'email_discussion_group' : settings.email_discussion_group,
'url_issues_page' : settings.url_issues_page,
'url_source_code' : settings.url_source_code,
'app_version' : appversion.version,
'upload_timestamp' : appversion.upload_timestamp}
self.response.out.write(template.render(path, template_values))
logging.debug(fn_name + "<End>" )
logservice.flush()
except Exception, e:
logging.exception(fn_name + "Caught top-level exception")
serve_outer_exception_message(self, e)
logging.debug(fn_name + "<End> due to exception" )
logservice.flush()
def serve_message_page(self, msg1, msg2 = None, msg3 = None,
show_back_button=False,
back_button_text="Back to previous page",
show_custom_button=False, custom_button_text='Try again', custom_button_url=settings.MAIN_PAGE_URL,
show_heading_messages=True):
""" Serve message.html page to user with message, with an optional button (Back, or custom URL)
msg1, msg2, msg3 Text to be displayed.msg2 and msg3 are option. Each msg is displayed in a separate div
show_back_button If True, a [Back] button is displayed, to return to previous page
show_custom_button If True, display button to jump to any URL. title set by custom_button_text
custom_button_text Text label for custom button
custom_button_url URL to go to when custom button is pressed
show_heading_messages If True, display app_title and (optional) host_msg
"""
fn_name = "serve_message_page: "
logging.debug(fn_name + "<Start>")
logservice.flush()
try:
client_id, client_secret, user_agent, app_title, product_name, host_msg = get_settings(self.request.host)
if msg1:
logging.debug(fn_name + "Msg1: " + msg1)
if msg2:
logging.debug(fn_name + "Msg2: " + msg2)
if msg3:
logging.debug(fn_name + "Msg3: " + msg3)
path = os.path.join(os.path.dirname(__file__), constants.PATH_TO_TEMPLATES, "message.html")
template_values = {'app_title' : app_title,
'host_msg' : host_msg,
'msg1': msg1,
'msg2': msg2,
'msg3': msg3,
'show_heading_messages' : show_heading_messages,
'show_back_button' : show_back_button,
'back_button_text' : back_button_text,
'show_custom_button' : show_custom_button,
'custom_button_text' : custom_button_text,
'custom_button_url' : custom_button_url,
'product_name' : product_name,
'url_discussion_group' : settings.url_discussion_group,
'email_discussion_group' : settings.email_discussion_group,
'url_issues_page' : settings.url_issues_page,
'url_source_code' : settings.url_source_code,
'app_version' : appversion.version,
'upload_timestamp' : appversion.upload_timestamp}
self.response.out.write(template.render(path, template_values))
logging.debug(fn_name + "<End>" )
logservice.flush()
except Exception, e:
logging.exception(fn_name + "Caught top-level exception")
serve_outer_exception_message(self, e)
logging.debug(fn_name + "<End> due to exception" )
logservice.flush()
def get(self):
fn_name = "DisplayStatsHandler.get(): "
logging.debug(fn_name + "<Start> (app version %s)" % appversion.version )
logservice.flush()
stats_query = model.UsageStats.all()
# stats_query.order('start_time')
# stats_query.order('user_hash')
stats = stats_query.run()
try:
stats_filename = "stats_" + get_application_id() + "_" + datetime.datetime.now().strftime("%Y-%m-%d") + ".csv"
template_values = {'stats' : stats}
self.response.headers["Content-Type"] = "text/csv"
self.response.headers.add_header(
"Content-Disposition", "attachment; filename=%s" % stats_filename)
path = os.path.join(os.path.dirname(__file__), constants.PATH_TO_TEMPLATES, "stats.csv")
self.response.out.write(template.render(path, template_values))
logging.debug(fn_name + "<End>" )
logservice.flush()
except Exception, e:
logging.exception(fn_name + "Caught top-level exception")
self.response.headers["Content-Type"] = "text/html; charset=utf-8"
try:
# Clear "Content-Disposition" so user will see error in browser.
# If not removed, output goes to file (if error generated after "Content-Disposition" was set),
# and user would not see the error message!
del self.response.headers["Content-Disposition"]
except Exception, e:
logging.debug(fn_name + "Unable to delete 'Content-Disposition' from headers: " + shared.get_exception_msg(e))
def handle_auth_callback(self):
fn_name = "handle_auth_callback() "
logging.debug(fn_name + "<Start>")
logservice.flush()
try:
if not self.request.get("code"):
logging.debug(fn_name + "No 'code', so redirecting to " +
str(settings.WELCOME_PAGE_URL))
logservice.flush()
self.redirect(settings.WELCOME_PAGE_URL)
logging.debug(fn_name + "<End> (no code)")
logservice.flush()
return
user = users.get_current_user()
logging.debug(fn_name + "Retrieving flow for " + str(user.user_id()))
flow = pickle.loads(memcache.get(user.user_id()))
if flow:
logging.debug(fn_name + "Got flow. Retrieving credentials")
error = False
retry_count = settings.NUM_API_TRIES
while retry_count > 0:
try:
credentials = flow.step2_exchange(self.request.params)
# Success!
error = False
if isTestUser(user.email()):
logging.debug(fn_name + "Retrieved credentials for " +
str(user.email()) + ", expires " +
str(credentials.token_expiry) + " UTC")
else:
logging.debug(fn_name +
"Retrieved credentials, expires " +
str(credentials.token_expiry) + " UTC")
break
except client.FlowExchangeError, e:
logging.warning(fn_name +
"FlowExchangeError: Giving up - " +
get_exception_msg(e))
error = True
credentials = None
break
except Exception, e:
logging.warning(fn_name + "Exception: " +
get_exception_msg(e))
error = True
credentials = None
class DownloadStatsHandler(webapp.RequestHandler):
"""Display statistics"""
def get(self):
fn_name = "DisplayStatsHandler.get(): "
logging.debug(fn_name + "<Start> (app version %s)" % appversion.version )
logservice.flush()
stats_query = model.UsageStats.all()
# stats_query.order('start_time')
# stats_query.order('user_hash')
stats = stats_query.run()
try:
stats_filename = "stats_" + get_application_id() + "_" + datetime.datetime.now().strftime("%Y-%m-%d") + ".csv"
template_values = {'stats' : stats}
self.response.headers["Content-Type"] = "text/csv"
self.response.headers.add_header(
"Content-Disposition", "attachment; filename=%s" % stats_filename)
path = os.path.join(os.path.dirname(__file__), constants.PATH_TO_TEMPLATES, "stats.csv")
self.response.out.write(template.render(path, template_values))
logging.debug(fn_name + "<End>" )
logservice.flush()
except Exception, e:
logging.exception(fn_name + "Caught top-level exception")
self.response.headers["Content-Type"] = "text/html; charset=utf-8"
try:
# Clear "Content-Disposition" so user will see error in browser.
# If not removed, output goes to file (if error generated after "Content-Disposition" was set),
# and user would not see the error message!
del self.response.headers["Content-Disposition"]
except Exception, e:
logging.debug(fn_name + "Unable to delete 'Content-Disposition' from headers: " + shared.get_exception_msg(e))
self.response.clear()
self.response.out.write("""Oops! Something went terribly wrong.<br />%s<br />Please report this error to <a href="http://code.google.com/p/tasks-backup/issues/list">code.google.com/p/tasks-backup/issues/list</a>""" % shared.get_exception_msg(e))
logging.debug(fn_name + "<End> due to exception" )
logservice.flush()
def _process_hourly_analytics(self):
log.info('Hourly Analytics Processer Started')
logservice.flush()
last_analyticstat = AnalyticsHourlyStat.get_last()
if not last_analyticstat: # If no last analytic than this is the first time.
log.info(
'Analytics have not been process yet, starting from the beginning'
)
logservice.flush()
first_ana = Analytics.get_first()
last_ana = Analytics.get_last()
if first_ana:
self.now = datetime.now()
self.start_time = first_ana.ts.replace(minute=0,
second=0,
microsecond=0)
self.end_time = self.start_time + timedelta(hours=1)
self.hour_count = 0
while True:
log.info('Hour: %s Start Time: %s End Time: %s',
self.hour_count, self.start_time, self.end_time)
logservice.flush()
anas = Analytics.get_time_frame(self.start_time,
self.end_time)
self._process_hour_block(anas)
if last_ana in anas or self.end_time > self.now:
break
self.hour_count += 1
self.start_time = self.end_time
self.end_time = self.start_time + timedelta(hours=1)
else:
log.info('Found no Analytics to process')
else:
self.start_time = last_analyticstat.end_time
self.end_time = self.start_time + timedelta(hours=1)
while self.end_time < datetime.now():
anas = Analytics.get_time_frame(self.start_time, self.end_time)
self._process_hour_block(anas)
self.start_time = self.end_time
self.end_time = self.end_time + timedelta(hours=1)
log.info('Hourly Analytics Processer Finished')
def handle_auth_callback(self):
fn_name = "handle_auth_callback() "
logging.debug(fn_name + "<Start>")
logservice.flush()
try:
if not self.request.get("code"):
logging.debug(fn_name + "No 'code', so redirecting to " + str(settings.WELCOME_PAGE_URL))
logservice.flush()
self.redirect(settings.WELCOME_PAGE_URL)
logging.debug(fn_name + "<End> (no code)")
logservice.flush()
return
user = users.get_current_user()
logging.debug(fn_name + "Retrieving flow for " + str(user.user_id()))
flow = pickle.loads(memcache.get(user.user_id()))
if flow:
logging.debug(fn_name + "Got flow. Retrieving credentials")
error = False
retry_count = settings.NUM_API_TRIES
while retry_count > 0:
try:
credentials = flow.step2_exchange(self.request.params)
# Success!
error = False
if isTestUser(user.email()):
logging.debug(fn_name + "Retrieved credentials for " + str(user.email()) + ", expires " +
str(credentials.token_expiry) + " UTC")
else:
logging.debug(fn_name + "Retrieved credentials, expires " + str(credentials.token_expiry) + " UTC")
break
except client.FlowExchangeError, e:
logging.warning(fn_name + "FlowExchangeError: Giving up - " + get_exception_msg(e))
error = True
credentials = None
break
except Exception, e:
logging.warning(fn_name + "Exception: " + get_exception_msg(e))
error = True
credentials = None
def redirect_for_auth(self, user, redirect_url=None):
"""Redirects the webapp response to authenticate the user with OAuth2.
Args:
redirect_url [OPTIONAL] The URL to return to once authorised.
Usually unused (left as None), so that the URL of the calling page is used
Uses the 'state' parameter to store redirect_url.
The handler for /oauth2callback can therefore redirect the user back to the page they
were on when get_credentials() failed (or to the specified redirect_url).
"""
fn_name = "redirect_for_auth(): "
try:
client_id, client_secret, user_agent, app_title, product_name, host_msg = get_settings(
self.request.host)
# Check how many times this has been called (without credentials having been successfully retrieved)
if self.request.cookies.has_key('auth_retry_count'):
auth_retry_count = int(self.request.cookies['auth_retry_count'])
logging.debug(fn_name + "auth_retry_count = " +
str(auth_retry_count))
if auth_retry_count > settings.MAX_NUM_AUTH_RETRIES:
# Exceeded maximum number of retries, so don't try again.
# Redirect user to invalid credentials page
logging.warning(
fn_name +
"Not re-authorising, because there have already been " +
str(auth_retry_count) + " attempts. Redirecting to " +
settings.INVALID_CREDENTIALS_URL)
self.redirect(settings.INVALID_CREDENTIALS_URL + "?rc=NC&nr=" +
str(auth_retry_count))
return
else:
logging.debug(
fn_name +
"No auth_retry_count cookie found. Probably means last authorisation was more than "
+ str(settings.AUTH_RETRY_COUNT_COOKIE_EXPIRATION_TIME) +
" seconds ago")
auth_retry_count = 0
if not redirect_url:
# By default, return to the same page
redirect_url = self.request.path_qs
# According to https://developers.google.com/accounts/docs/OAuth_ref#RequestToken
# xoauth_displayname is optional.
# (optional) String identifying the application.
# This string is displayed to end users on Google's authorization confirmation page.
# For registered applications, the value of this parameter overrides the name set during registration and
# also triggers a message to the user that the identity can't be verified.
# For unregistered applications, this parameter enables them to specify an application name,
# In the case of unregistered applications, if this parameter is not set, Google identifies the application
# using the URL value of oauth_callback; if neither parameter is set, Google uses the string "anonymous".
# It seems preferable to NOT supply xoauth_displayname, so that Google doesn't display "identity can't be verified" msg.
flow = client.OAuth2WebServerFlow(
client_id=client_id,
client_secret=client_secret,
scope="https://www.googleapis.com/auth/tasks",
user_agent=user_agent,
state=redirect_url)
callback = self.request.relative_url("/oauth2callback")
authorize_url = flow.step1_get_authorize_url(callback)
memcache.set(user.user_id(), pickle.dumps(flow))
# Keep track of how many times we've called the authorise URL
if self.request.cookies.has_key('auth_retry_count'):
auth_retry_count = int(self.request.cookies['auth_retry_count'])
else:
auth_retry_count = 0
__store_auth_retry_count(self, auth_retry_count + 1)
logging.debug(fn_name + "Redirecting to " + str(authorize_url))
logservice.flush()
self.redirect(authorize_url)
except Exception, e:
logging.exception(fn_name + "Caught top-level exception")
serve_outer_exception_message(self, e)
logging.debug(fn_name + "<End> due to exception")
logservice.flush()
credentials = None
result = False
else:
# No credentials
fail_msg = "No credentials"
fail_reason = "Unable to retrieve credentials for user"
#logging.debug(fn_name + fail_msg)
result = False
if result:
# TODO: Successfuly retrieved credentials, so reset auth_retry_count to 0
__reset_auth_retry_count(self)
else:
logging.debug(fn_name + fail_msg)
logservice.flush()
except Exception, e:
logging.exception(fn_name + "Caught top-level exception")
logging.debug(fn_name + "<End> due to exception")
raise e
if fail_reason == "Daily limit exceeded":
# Will be caught in calling method's outer try-except
raise DailyLimitExceededError()
return result, user, credentials, fail_msg, fail_reason
def serve_message_page(self,
msg1,
def _do_migration():
log.info('Starting Datastore Migration')
logservice.flush()
projects = {}
concept_count = Concept.query().count()
concepts = Concept.query()
index = 1
logservice.flush()
for concept in concepts.iter():
if index % 50 == 0:
log.info('%s concepts finished of %s' % (index, concept_count))
logservice.flush()
project = projects.get(concept.project.id(), None)
if not project:
project = concept.project.get()
if not project:
if len(concept.phrasings) > 0:
perms = []
phrasings = ndb.get_multi(concept.phrasings)
for phrase in phrasings:
if phrase:
perms.append(phrase.permissions)
ndb.delete_multi(concept.phrasings)
ndb.delete_multi(perms)
if len(concept.selected_phrasings) > 0:
ndb.delete_multi(concept.selected_phrasings)
if len(concept.attributes) > 0:
ndb.delete_multi(concept.attributes)
if len(concept.crawlcontext) > 0:
ndb.delete_multi(concept.crawlcontext)
if len(concept.linkes) > 0:
links = ndb.get_multi(concept.linkes)
for link in links:
if link:
link.delete(None, False, True)
if concept.media_blob is not None:
concept.delete_media()
concept.distilled_phrasing.delete()
concept.permissions.delete()
concept.key.delete()
continue
projects[project.key.id()] = project
perm = project.permissions.get()
if not perm.project:
perm.project = project.key
perm.put()
for doc in project.documents:
d = doc.get()
perm = d.permissions.get()
if not perm.project:
perm.project = project.key
perm.put()
d = project.distilled_document.get()
perm = d.permissions.get()
if not perm.project:
perm.project = project.key
perm.put()
_fix_artifacts(concept, project)
index += 1
log.info('Finished Datastore Migration')
logservice.flush()
def dump_obj(obj):
for attr in dir(obj):
logging.debug(" obj.%s = %s" % (attr, getattr(obj, attr)))
logservice.flush()
def get_credentials(self):
""" Retrieve credentials for the user
Returns:
result True if we have valid credentials for the user
user User object for current user
credentials Credentials object for current user. None if no credentials.
fail_msg If result==False, message suitabale for displaying to user
fail_reason If result==False, cause of the failure. Can be one of
"User not logged on"
"No credentials"
"Invalid credentials"
"Credential use error" (Unspecified error when attempting to use credentials)
"Credential use HTTP error" (Returned an HTTP error when attempting to use credentials)
If no credentials, or credentials are invalid, the calling method can call redirect_for_auth(self, user),
which sets the redirect URL back to the calling page. That is, user is redirected to calling page after authorising.
"""
fn_name = "get_credentials(): "
user = None
fail_msg = ''
fail_reason = ''
credentials = None
result = False
try:
user = users.get_current_user()
if user is None:
# User is not logged in, so there can be no credentials.
fail_msg = "User is not logged in"
fail_reason = "User not logged on"
logging.debug(fn_name + fail_msg)
logservice.flush()
return False, None, None, fail_msg, fail_reason
credentials = appengine.StorageByKeyName(
model.Credentials, user.user_id(), "credentials").get()
result = False
if credentials:
if credentials.invalid:
# We have credentials, but they are invalid
fail_msg = "Invalid credentials for this user"
fail_reason = "Invalid credentials"
result = False
else:
#logging.debug(fn_name + "Calling tasklists service to confirm valid credentials")
# so it turns out that the method that checks if the credentials are okay
# doesn't give the correct answer unless you try to refresh it. So we do that
# here in order to make sure that the credentials are valid before being
# passed to a worker. Obviously if the user revokes the credentials after
# this point we will continue to get an error, but we can't stop that.
# Credentials are possibly valid, but need to be confirmed by refreshing
# Try multiple times, just in case call to server fails due to external probs (e.g., timeout)
# retry_count = settings.NUM_API_TRIES
# while retry_count > 0:
try:
http = httplib2.Http()
http = credentials.authorize(http)
service = discovery.build("tasks", "v1", http)
tasklists_svc = service.tasklists()
tasklists_list = tasklists_svc.list().execute()
# Successfully used credentials, everything is OK, so break out of while loop
fail_msg = ''
fail_reason = ''
result = True
# break
except apiclient_errors.HttpError, e:
#logging.info(fn_name + "HttpError using credentials: " + get_exception_msg(e))
if e._get_reason().lower() == "daily limit exceeded":
fail_reason = "Daily limit exceeded"
fail_msg = "HttpError: Daily limit exceeded using credentials."
else:
fail_reason = "Credential use HTTP error"
fail_msg = "Error accessing tasks service: " + e._get_reason()
result = False
credentials = None
result = False
except Exception, e:
#logging.info(fn_name + "Exception using credentials: " + get_exception_msg(e))
fail_reason = "Credential use error"
fail_msg = "Exception using credentials: " + get_exception_msg(e)
credentials = None
result = False
def post(self):
fn_name = "ProcessTasksWorker.post(): "
logging.debug(fn_name + "<start> (app version %s)" %appversion.version)
logservice.flush()
client_id, client_secret, user_agent, app_title, project_name, host_msg = shared.get_settings(self.request.host)
self.user_email = self.request.get(settings.TASKS_QUEUE_KEY_NAME)
self.is_test_user = shared.isTestUser(self.user_email)
if self.user_email:
# Retrieve the DB record for this user
self.process_tasks_job = model.ProcessTasksJob.get_by_key_name(self.user_email)
if self.process_tasks_job is None:
logging.error(fn_name + "No DB record for " + self.user_email)
logservice.flush()
logging.debug(fn_name + "<End> No DB record")
# TODO: Find some way of notifying the user?????
# Could use memcache to relay a message which is displayed in ProgressHandler
return
else:
logging.debug(fn_name + "Retrieved process tasks job for " + str(self.user_email))
logservice.flush()
self.process_tasks_job.status = constants.ExportJobStatus.INITIALISING
self.process_tasks_job.job_progress_timestamp = datetime.datetime.now()
self.process_tasks_job.message = "Validating background job ..."
logging.debug(fn_name + "Initialising - Job status: '" + str(self.process_tasks_job.status) + "', progress: " +
str(self.process_tasks_job.total_progress) + ", msg: '" +
str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'")
logservice.flush()
self.process_tasks_job.put()
user = self.process_tasks_job.user
if not user:
logging.error(fn_name + "No user object in DB record for " + str(self.user_email))
logservice.flush()
self.process_tasks_job.status = constants.ExportJobStatus.ERROR
self.process_tasks_job.message = ''
self.process_tasks_job.error_message = "Problem with user details. Please restart."
self.process_tasks_job.job_progress_timestamp = datetime.datetime.now()
logging.debug(fn_name + "No user - Job status: '" + str(self.process_tasks_job.status) + "', progress: " +
str(self.process_tasks_job.total_progress) + ", msg: '" +
str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'")
logservice.flush()
self.process_tasks_job.put()
logging.debug(fn_name + "<End> No user object")
return
self.credentials = self.process_tasks_job.credentials
if not self.credentials:
logging.error(fn_name + "No credentials in DB record for " + str(self.user_email))
logservice.flush()
self.process_tasks_job.status = constants.ExportJobStatus.ERROR
self.process_tasks_job.message = ''
self.process_tasks_job.error_message = "Problem with user self.credentials. Please restart."
self.process_tasks_job.job_progress_timestamp = datetime.datetime.now()
logging.debug(fn_name + "Job status: '" + str(self.process_tasks_job.status) + "', progress: " +
str(self.process_tasks_job.total_progress) + ", msg: '" +
str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'")
logservice.flush()
self.process_tasks_job.put()
logging.debug(fn_name + "<End> No self.credentials")
return
if self.credentials.invalid:
logging.error(fn_name + "Invalid credentials in DB record for " + str(self.user_email))
logservice.flush()
self.process_tasks_job.status = constants.ExportJobStatus.ERROR
self.process_tasks_job.message = ''
self.process_tasks_job.error_message = "Invalid self.credentials. Please restart and re-authenticate."
self.process_tasks_job.job_progress_timestamp = datetime.datetime.now()
logging.debug(fn_name + "Credentials invalid - Job status: '" + str(self.process_tasks_job.status) + "', progress: " +
str(self.process_tasks_job.total_progress) + ", msg: '" +
str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'")
logservice.flush()
self.process_tasks_job.put()
logging.debug(fn_name + "<End> Invalid self.credentials")
return
if self.is_test_user:
logging.debug(fn_name + "User is test user %s" % self.user_email)
logservice.flush()
http = httplib2.Http()
http = self.credentials.authorize(http)
service = discovery.build("tasks", "v1", http)
self.tasklists_svc = service.tasklists()
self.tasks_svc = service.tasks()
self.export_tasks()
# logging.debug(fn_name + "Finished processing. Total progress = " +
# str(self.process_tasks_job.total_progress) + " for " + str(self.user_email))
else:
logging.error(fn_name + "No processing, as there was no user_email key")
logservice.flush()
logging.debug(fn_name + "<End>, user = " + str(self.user_email))
logservice.flush()
def convert_RFC3339_string_to_datetime(datetime_str, field_name, date_only=False):
""" Attempt to convert the RFC 3339 datetime string to a valid datetime object.
If the field value cannot be parsed, return a default '1900-01-01 00:00:00' value.
If the string is '0001-01-01T00:00:00.000Z', return None.
datetime_str String to be converted
field_name The name of the field being parsed. Only used for logging
date_only If True, return a date object instead of datetime
strptime() can only parse dates after Jan 1900, but sometimes the server returns
dates outside the valid range. Invalid values will be converted to 1900-01-01
so that they can be parsed by strptime() or formatted by strftime()
Parse the yyyy-mm-ddThh:mm:ss.dddZ return a date or datetime object.
There have been occassional strange values in the 'completed' property, such as
"-1701567-04-26T07:12:55.000Z"
According to http://docs.python.org/library/datetime.html
"The exact range of years for which strftime() works also varies across platforms.
Regardless of platform, years before 1900 cannot be used."
so if any date/timestamp value is invalid, set the property to '1900-01-01 00:00:00'
NOTE: Sometimes a task has a completion date of '0000-01-01T00:00:00.000Z', which cannot
be converted to datetime, because the earliest allowable datetime year is 0001, so that is
returned as None
Raises an exception if datetime_str is empty, or cannot be handled at all. In this case, it
is recommended that the calling method delete the associated field in the task object.
"""
fn_name = "convert_RFC3339_string_to_datetime: "
try:
if not datetime_str:
# Nothing to parse, so raise an Exception, so that the calling method can deal with it
raise ValueError("No datetime string - nothing to parse")
if datetime_str == constants.ZERO_RFC3339_DATETIME_STRING:
# Zero timestamp is represented by None.
# Methods seeing None for a datetime object should replace it with a string representing
# the zero date (e.g., '0000-01-01 00:00:00')
return None
d = None
try:
# Parse the RFC 3339 datetime string
d = datetime.datetime.strptime(datetime_str, "%Y-%m-%dT%H:%M:%S.000Z")
# Test the resultant datetime to ensure that it can be displayed by strftime()
# This prevents exceptions later when other modules try to display a datetime
# that is outside the valid range for strftime(), such as dates before 1900.
try:
test_date_str = d.strftime('%H:%M:%S %a, %d %b %Y')
except Exception, e:
d = datetime.datetime(1900,1,1,0,0,0)
logging.warning(fn_name + "Unable to convert '" + field_name + "' string '" + str(datetime_str) +
"' to a datetime that can be displayed by strftime, so using " + str(d) +
": " + get_exception_msg(e))
logservice.flush()
except ValueError, e:
# Minimum datestamp that can be parsed by strptime or displayed by strftime is 1900-01-01
d = datetime.datetime(1900,1,1,0,0,0)
logging.warning(fn_name + "Invalid '" + field_name + "' timestamp (" + str(datetime_str) +
"), so using " + str(d) +
": " + get_exception_msg(e))
logservice.flush()
except Exception, e:
logging.exception(fn_name + "Unable to parse '" + field_name + "' value '" + str(datetime_str) +
"' as a datetime")
logservice.flush()
raise e
def serve_message_page(self,
msg1, msg2 = None, msg3 = None,
show_back_button=False,
back_button_text="Back to previous page",
show_custom_button=False, custom_button_text='Try again',
custom_button_url = settings.MAIN_PAGE_URL,
show_heading_messages=True,
template_file="message.html",
extra_template_values=None):
""" Serve message.html page to user with message, with an optional button (Back, or custom URL)
self A webapp.RequestHandler or similar
msg1, msg2, msg3 Text to be displayed.msg2 and msg3 are option. Each msg is displayed in a separate div
show_back_button If True, a [Back] button is displayed, to return to previous page
show_custom_button If True, display button to jump to any URL. title set by custom_button_text
custom_button_text Text label for custom button
custom_button_url URL to go to when custom button is pressed. Should be an absolute URL
show_heading_messages If True, display app_title and (optional) host_msg
template_file Specify an alternate HTML template file
extra_template_values A dictionary containing values that will be merged with the existing template values
They may be additional parameters, or overwrite existing parameters.
These new values will be available to the HTML template.
All args except self and msg1 are optional.
"""
fn_name = "serve_message_page: "
logging.debug(fn_name + "<Start>")
logservice.flush()
try:
if custom_button_url == settings.MAIN_PAGE_URL:
# Relative URLs sometimes fail on Firefox, so convert the default relative URL to an absolute URL
custom_button_url = urljoin("https://" + self.request.host, settings.MAIN_PAGE_URL)
if msg1:
logging.debug(fn_name + "Msg1: " + msg1)
if msg2:
logging.debug(fn_name + "Msg2: " + msg2)
if msg3:
logging.debug(fn_name + "Msg3: " + msg3)
path = os.path.join(os.path.dirname(__file__), constants.PATH_TO_TEMPLATES, template_file)
template_values = {'app_title' : host_settings.APP_TITLE,
'host_msg' : host_settings.HOST_MSG,
'msg1': msg1,
'msg2': msg2,
'msg3': msg3,
'show_heading_messages' : show_heading_messages,
'show_back_button' : show_back_button,
'back_button_text' : back_button_text,
'show_custom_button' : show_custom_button,
'custom_button_text' : custom_button_text,
'custom_button_url' : custom_button_url,
'product_name' : host_settings.PRODUCT_NAME,
'url_discussion_group' : settings.url_discussion_group,
'email_discussion_group' : settings.email_discussion_group,
'url_issues_page' : settings.url_issues_page,
'url_source_code' : settings.url_source_code,
'app_version' : appversion.version,
'upload_timestamp' : appversion.upload_timestamp}
if extra_template_values:
# Add/update template values
logging.debug(fn_name + "DEBUG: Updating template values ==>")
logging.debug(extra_template_values)
logservice.flush()
template_values.update(extra_template_values)
self.response.out.write(template.render(path, template_values))
logging.debug(fn_name + "<End>" )
logservice.flush()
except Exception, e:
logging.exception(fn_name + "Caught top-level exception")
serve_outer_exception_message(self, e)
logging.debug(fn_name + "<End> due to exception" )
logservice.flush()
def process(self): logging.info(modules.get_current_module_name() + ' started') logservice.flush() self.response.set_status(200) return
def get_credentials(self):
""" Retrieve credentials for the user
Returns:
result True if we have valid credentials for the user
user User object for current user
credentials Credentials object for current user. None if no credentials.
fail_msg If result==False, message suitabale for displaying to user
fail_reason If result==False, cause of the failure. Can be one of
"User not logged on"
"No credentials"
"Invalid credentials"
"Credential use error" (Unspecified error when attempting to use credentials)
"Credential use HTTP error" (Returned an HTTP error when attempting to use credentials)
If no credentials, or credentials are invalid, the calling method can call redirect_for_auth(self, user),
which sets the redirect URL back to the calling page. That is, user is redirected to calling page after authorising.
"""
fn_name = "get_credentials(): "
user = None
fail_msg = ''
fail_reason = ''
credentials = None
result = False
try:
user = users.get_current_user()
if user is None:
# User is not logged in, so there can be no credentials.
fail_msg = "User is not logged in"
fail_reason = "User not logged on"
logging.debug(fn_name + fail_msg)
logservice.flush()
return False, None, None, fail_msg, fail_reason
credentials = appengine.StorageByKeyName(model.Credentials,
user.user_id(),
"credentials").get()
result = False
if credentials:
if credentials.invalid:
# We have credentials, but they are invalid
fail_msg = "Invalid credentials for this user"
fail_reason = "Invalid credentials"
result = False
else:
#logging.debug(fn_name + "Calling tasklists service to confirm valid credentials")
# so it turns out that the method that checks if the credentials are okay
# doesn't give the correct answer unless you try to refresh it. So we do that
# here in order to make sure that the credentials are valid before being
# passed to a worker. Obviously if the user revokes the credentials after
# this point we will continue to get an error, but we can't stop that.
# Credentials are possibly valid, but need to be confirmed by refreshing
# Try multiple times, just in case call to server fails due to external probs (e.g., timeout)
# retry_count = settings.NUM_API_TRIES
# while retry_count > 0:
try:
http = httplib2.Http()
http = credentials.authorize(http)
service = discovery.build("tasks", "v1", http)
tasklists_svc = service.tasklists()
tasklists_list = tasklists_svc.list().execute()
# Successfully used credentials, everything is OK, so break out of while loop
fail_msg = ''
fail_reason = ''
result = True
# break
except apiclient_errors.HttpError, e:
#logging.info(fn_name + "HttpError using credentials: " + get_exception_msg(e))
if e._get_reason().lower() == "daily limit exceeded":
fail_reason = "Daily limit exceeded"
fail_msg = "HttpError: Daily limit exceeded using credentials."
else:
fail_reason = "Credential use HTTP error"
fail_msg = "Error accessing tasks service: " + e._get_reason(
)
result = False
credentials = None
result = False
except Exception, e:
#logging.info(fn_name + "Exception using credentials: " + get_exception_msg(e))
fail_reason = "Credential use error"
fail_msg = "Exception using credentials: " + get_exception_msg(
e)
credentials = None
result = False
def _export_tasks(self):
fn_name = "_export_tasks: "
logging.debug(fn_name + "<Start>")
logservice.flush()
start_time = datetime.datetime.now()
include_hidden = self.process_tasks_job.include_hidden
include_completed = self.process_tasks_job.include_completed
include_deleted = self.process_tasks_job.include_deleted
summary_msg = ''
# Retrieve all tasks for the user
try:
logging.debug(fn_name +
"include_completed = " + str(include_completed) +
", include_hidden = " + str(include_hidden) +
", include_deleted = " + str(include_deleted))
logservice.flush()
# ##############################################
# FLOW
# ----------------------------------------------
# For each page of taskslists
# For each tasklist
# For each page of tasks
# For each task
# Fix date format
# Add tasks to tasklist collection
# Add tasklist to tasklists collection
# Use tasklists collection to return tasks backup to user
self.process_tasks_job.status = constants.ExportJobStatus.BUILDING
self.process_tasks_job.job_progress_timestamp = datetime.datetime.now()
self.process_tasks_job.message = 'Retrieving tasks from server ...'
self._log_progress("Building")
self.process_tasks_job.put()
# This list will contain zero or more tasklist dictionaries, which each contain tasks
tasklists = []
total_num_tasklists = 0
total_num_tasks = 0
tasks_per_list = []
# ---------------------------------------
# Retrieve all the tasklists for the user
# ---------------------------------------
logging.debug(fn_name + "Retrieve all the tasklists for the user")
logservice.flush()
next_tasklists_page_token = None
more_tasklists_data_to_retrieve = True
while more_tasklists_data_to_retrieve:
if self.is_test_user:
logging.debug(fn_name + "calling tasklists.list().execute() to create tasklists list")
logservice.flush()
retry_count = settings.NUM_API_TRIES
while retry_count > 0:
try:
if next_tasklists_page_token:
tasklists_data = self.tasklists_svc.list(pageToken=next_tasklists_page_token).execute()
else:
tasklists_data = self.tasklists_svc.list().execute()
# Successfully retrieved data, so break out of retry loop
break
except Exception, e:
retry_count = retry_count - 1
if retry_count > 0:
if isinstance(e, AccessTokenRefreshError):
# Log first 'n' AccessTokenRefreshError as Info, because they are reasonably common,
# and the system usually continues normally after the 2nd call to
# "new_request: Refreshing due to a 401"
# Occassionally, the system seems to need a 3rd attempt
# (i.e., success after waiting 45 seconds)
logging.info(fn_name +
"Access Token Refresh Error whilst retrieving list of tasklists (1st time, not yet an error). " +
str(retry_count) + " attempts remaining: " + shared.get_exception_msg(e))
else:
logging.warning(fn_name + "Error retrieving list of tasklists. " +
str(retry_count) + " attempts remaining: " + shared.get_exception_msg(e))
logservice.flush()
if retry_count <= 2:
logging.debug(fn_name + "Giving server an extra chance; Sleeping for " +
str(settings.WORKER_API_RETRY_SLEEP_DURATION) +
" seconds before retrying")
logservice.flush()
# Update job_progress_timestamp so that job doesn't time out
self.process_tasks_job.job_progress_timestamp = datetime.datetime.now()
self.process_tasks_job.put()
time.sleep(settings.WORKER_API_RETRY_SLEEP_DURATION)
else:
logging.exception(fn_name + "Still error retrieving list of tasklists after " +
str(settings.NUM_API_TRIES) + " attempts. Giving up")
logservice.flush()
raise e
if self.is_test_user and settings.DUMP_DATA:
logging.debug(fn_name + "tasklists_data ==>")
logging.debug(tasklists_data)
logservice.flush()
if tasklists_data.has_key(u'items'):
tasklists_list = tasklists_data[u'items']
else:
# If there are no tasklists, then there will be no 'items' element. This could happen if
# the user has deleted all their tasklists. Not sure if this is even possible, but
# checking anyway, since it is possible to have a tasklist without 'items' (see issue #9)
logging.debug(fn_name + "User has no tasklists.")
logservice.flush()
tasklists_list = []
# tasklists_list is a list containing the details of the user's tasklists.
# We are only interested in the title
# if self.is_test_user and settings.DUMP_DATA:
# logging.debug(fn_name + "tasklists_list ==>")
# logging.debug(tasklists_list)
# ---------------------------------------
# Process all the tasklists for this user
# ---------------------------------------
for tasklist_data in tasklists_list:
total_num_tasklists = total_num_tasklists + 1
if self.is_test_user and settings.DUMP_DATA:
logging.debug(fn_name + "tasklist_data ==>")
logging.debug(tasklist_data)
logservice.flush()
"""
Example of a tasklist entry;
u'id': u'MDAxNTkzNzU0MzA0NTY0ODMyNjI6MDow',
u'kind': u'tasks#taskList',
u'selfLink': u'https://www.googleapis.com/tasks/v1/users/@me/lists/MDAxNTkzNzU0MzA0NTY0ODMyNjI6MDow',
u'title': u'Default List',
u'updated': u'2012-01-28T07:30:18.000Z'},
"""
tasklist_title = tasklist_data[u'title']
tasklist_id = tasklist_data[u'id']
if self.is_test_user and settings.DUMP_DATA:
logging.debug(fn_name + "Process all the tasks in " + str(tasklist_title))
logservice.flush()
# =====================================================
# Process all the tasks in this task list
# =====================================================
tasklist_dict, num_tasks = self._get_tasks_in_tasklist(tasklist_title, tasklist_id,
include_hidden, include_completed, include_deleted)
# Track number of tasks per tasklist
tasks_per_list.append(num_tasks)
total_num_tasks = total_num_tasks + num_tasks
self.process_tasks_job.total_progress = total_num_tasks
self.process_tasks_job.tasklist_progress = 0 # Because total_progress now includes num_tasks for current tasklist
self.process_tasks_job.job_progress_timestamp = datetime.datetime.now()
self.process_tasks_job.message = ''
self._log_progress("Processed tasklist")
self.process_tasks_job.put()
# if self.is_test_user:
# logging.debug(fn_name + "Adding %d tasks to tasklist" % len(tasklist_dict[u'tasks']))
# Add the data for this tasklist (including all the tasks) into the collection of tasklists
tasklists.append(tasklist_dict)
# Check if there is another page of tasklists to be retrieved
if tasklists_data.has_key('nextPageToken'):
# There is another page of tasklists to be retrieved for this user,
# which we'll retrieve next time around the while loop.
# This happens if there is more than 1 page of tasklists.
# It seems that each page contains 20 tasklists.
more_tasklists_data_to_retrieve = True # Go around while loop again
next_tasklists_page_token = tasklists_data['nextPageToken']
# if self.is_test_user:
# logging.debug(fn_name + "There is (at least) one more page of tasklists to be retrieved")
else:
# This is the last (or only) page of results (list of tasklists)
more_tasklists_data_to_retrieve = False
next_tasklists_page_token = None
# *** end while more_tasks_data_to_retrieve ***
# ------------------------------------------------------
# Store the data, so we can return it to the user
# ------------------------------------------------------
"""
tasklists is a list of tasklist structures
structure of tasklist
{
"title" : tasklist.title, # Name of this tasklist
"tasks" : [ task ] # List of task items in this tasklist
}
structure of task
{
"title" : title, # Free text
"status" : status, # "completed" | "needsAction"
"id" : id, # Used when determining parent-child relationships
"parent" : parent, # OPT: ID of the parent of this task (only if this is a sub-task)
"notes" : notes, # OPT: Free text
"due" : due, # OPT: Date due, e.g. 2012-01-30T00:00:00.000Z NOTE time = 0
"updated" : updated, # Timestamp, e.g., 2012-01-26T07:47:18.000Z
"completed" : completed # Timestamp, e.g., 2012-01-27T10:38:56.000Z
}
"""
# Delete existing backup data records
tasklist_data_records = model.TasklistsData.gql("WHERE ANCESTOR IS :1",
db.Key.from_path(settings.DB_KEY_TASKS_BACKUP_DATA, self.user_email))
num_records = tasklist_data_records.count()
logging.debug(fn_name + "Deleting " + str(num_records) + " old blobs")
logservice.flush()
for tasklists_data_record in tasklist_data_records:
tasklists_data_record.delete()
# logging.debug(fn_name + "Pickling tasks data ...")
pickled_tasklists = pickle.dumps(tasklists)
# logging.debug(fn_name + "Pickled data size = " + str(len(pickled_tasklists)))
data_len = len(pickled_tasklists)
# Multiply by 1.0 float value so that we can use ceiling to find number of Blobs required
num_of_blobs = int(math.ceil(data_len * 1.0 / constants.MAX_BLOB_SIZE))
logging.debug(fn_name + "Calculated " + str(num_of_blobs) + " blobs required to store " +
str(data_len) + " bytes")
logservice.flush()
try:
for i in range(num_of_blobs):
# Write backup data records
tasklist_rec = model.TasklistsData(db.Key.from_path(settings.DB_KEY_TASKS_BACKUP_DATA, self.user_email))
slice_start = int(i*constants.MAX_BLOB_SIZE)
slice_end = int((i+1)*constants.MAX_BLOB_SIZE)
# logging.debug(fn_name + "Creating part " + str(i+1) + " of " + str(num_of_blobs) +
# " using slice " + str(slice_start) + " to " + str(slice_end))
pkl_part = pickled_tasklists[slice_start : slice_end]
tasklist_rec.pickled_tasks_data = pkl_part
tasklist_rec.idx = i
tasklist_rec.put()
# logging.debug(fn_name + "Marking backup job complete")
end_time = datetime.datetime.now()
process_time = end_time - start_time
proc_time_str = str(process_time.seconds) + "." + str(process_time.microseconds)[:3] + " seconds"
# Mark backup completed
summary_msg = "Retrieved %d tasks from %d tasklists" % (total_num_tasks, total_num_tasklists)
breakdown_msg = "Tasks per list: " + str(tasks_per_list)
self.process_tasks_job.status = constants.ExportJobStatus.EXPORT_COMPLETED
self.process_tasks_job.job_progress_timestamp = datetime.datetime.now()
# self.process_tasks_job.message = summary_msg + " in " + proc_time_str
self.process_tasks_job.message = summary_msg + " at " + \
start_time.strftime("%H:%M UTC, %a %d %b %Y")
logging.info(fn_name + "COMPLETED: " + summary_msg + " for " + self.user_email + " in " + proc_time_str)
logservice.flush()
self.process_tasks_job.put()
try:
end_time = datetime.datetime.now()
process_time = end_time - start_time
processing_time = process_time.days * 3600*24 + process_time.seconds + process_time.microseconds / 1000000.0
included_options_str = "Includes: Completed = %s, Deleted = %s, Hidden = %s" % (str(include_completed), str(include_deleted), str(include_hidden))
logging.debug(fn_name + "STATS: Job started at " + str(self.process_tasks_job.job_start_timestamp) +
"\n Worker started at " + str(start_time) +
"\n " + summary_msg +
"\n " + breakdown_msg +
"\n " + proc_time_str +
"\n " + included_options_str)
logservice.flush()
usage_stats = model.UsageStats(
user_hash = hash(self.user_email),
number_of_tasks = self.process_tasks_job.total_progress,
number_of_tasklists = total_num_tasklists,
tasks_per_tasklist = tasks_per_list,
include_completed = include_completed,
include_deleted = include_deleted,
include_hidden = include_hidden,
start_time = start_time,
processing_time = processing_time)
usage_stats.put()
logging.debug(fn_name + "Saved stats")
logservice.flush()
except Exception, e:
logging.exception("Error saving stats")
logservice.flush()
# Don't bother doing anything else, because stats aren't critical
except apiproxy_errors.RequestTooLargeError, e:
logging.exception(fn_name + "Error putting results in DB - Request too large")
logservice.flush()
self.process_tasks_job.status = constants.ExportJobStatus.ERROR
self.process_tasks_job.message = ''
self.process_tasks_job.error_message = "Tasklists data is too large - Unable to store tasklists in DB: " + shared.get_exception_msg(e)
self.process_tasks_job.job_progress_timestamp = datetime.datetime.now()
self._log_progress("apiproxy_errors.RequestTooLargeError")
self.process_tasks_job.put()
def post(self):
fn_name = "ProcessTasksWorker.post(): "
logging.debug(fn_name + "<start> (app version %s)" %appversion.version)
logservice.flush()
self.user_email = self.request.get(settings.TASKS_QUEUE_KEY_NAME)
self.is_test_user = shared.is_test_user(self.user_email)
if self.user_email:
# Retrieve the DB record for this user
self.process_tasks_job = model.ProcessTasksJob.get_by_key_name(self.user_email)
if self.process_tasks_job is None:
logging.error(fn_name + "No DB record for " + self.user_email)
logservice.flush()
logging.debug(fn_name + "<End> No DB record")
# TODO: Find some way of notifying the user?????
# Could use memcache to relay a message which is displayed in ProgressHandler
return
logging.debug(fn_name + "Retrieved process tasks job for " + str(self.user_email))
logging.debug(fn_name + "Job was requested at " + str(self.process_tasks_job.job_created_timestamp))
logservice.flush()
if self.process_tasks_job.status != constants.ExportJobStatus.TO_BE_STARTED:
# Very occassionally, GAE starts a 2nd instance of the worker, so we check for that here.
# Check when job status was last updated. If it was less than settings.MAX_JOB_PROGRESS_INTERVAL
# seconds ago, assume that another instance is already running, log error and exit
time_since_last_update = datetime.datetime.now() - self.process_tasks_job.job_progress_timestamp
if time_since_last_update.seconds < settings.MAX_JOB_PROGRESS_INTERVAL:
logging.error(fn_name + "It appears that worker was called whilst another job is already running for " + str(self.user_email))
logging.error(fn_name + "Previous job requested at " + str(self.process_tasks_job.job_created_timestamp) + " UTC is still running.")
logging.error(fn_name + "Previous worker started at " + str(self.process_tasks_job.job_start_timestamp) + " UTC and last job progress update was " + str(time_since_last_update.seconds) + " seconds ago, with status " + str(self.process_tasks_job.status) )
logging.warning(fn_name + "<End> (Another worker is already running)")
logservice.flush()
return
else:
# A previous job hasn't completed, and hasn't updated progress for more than
# settings.MAX_JOB_PROGRESS_INTERVAL secons, so assume that previous worker
# for this job has died.
logging.error(fn_name + "It appears that a previous job requested by " + str(self.user_email) + " at " + str(self.process_tasks_job.job_created_timestamp) + " UTC has stalled.")
logging.error(fn_name + "Previous worker started at " + str(self.process_tasks_job.job_start_timestamp) + " UTC and last job progress update was " + str(time_since_last_update.seconds) + " seconds ago, with status " + str(self.process_tasks_job.status) + ", progress = ")
if self.process_tasks_job.number_of_job_starts > settings.MAX_NUM_JOB_STARTS:
logging.error(fn_name + "This job has already been started " + str(self.process_tasks_job.number_of_job_starts) + " times. Giving up")
logging.warning(fn_name + "<End> (Multiple job restart failures)")
logservice.flush()
return
else:
logging.info(fn_name + "Attempting to restart backup job. Attempt number " +
str(self.process_tasks_job.number_of_job_starts + 1))
logservice.flush()
self.process_tasks_job.status = constants.ExportJobStatus.INITIALISING
self.process_tasks_job.number_of_job_starts = self.process_tasks_job.number_of_job_starts + 1
self.process_tasks_job.job_progress_timestamp = datetime.datetime.now()
self.process_tasks_job.job_start_timestamp = datetime.datetime.now()
self.process_tasks_job.message = "Validating background job ..."
self._log_progress("Initialising")
self.process_tasks_job.put()
time_since_job_request = datetime.datetime.now() - self.process_tasks_job.job_created_timestamp
logging.debug(fn_name + "Starting job that was requested " + str(time_since_job_request.seconds) +
" seconds ago at " + str(self.process_tasks_job.job_created_timestamp) + " UTC")
user = self.process_tasks_job.user
if not user:
logging.error(fn_name + "No user object in DB record for " + str(self.user_email))
logservice.flush()
self.process_tasks_job.status = constants.ExportJobStatus.ERROR
self.process_tasks_job.message = ''
self.process_tasks_job.error_message = "Problem with user details. Please restart."
self.process_tasks_job.job_progress_timestamp = datetime.datetime.now()
self._log_progress("No user")
self.process_tasks_job.put()
logging.debug(fn_name + "<End> No user object")
return
# self.credentials = self.process_tasks_job.credentials
# DEBUG: 2012-09-16; Trying a different method of retrieving credentials, to see if it
# allows retrieveal of credentials for TAFE account
self.credentials = StorageByKeyName(CredentialsModel, user.user_id(), 'credentials').get()
if not self.credentials:
logging.error(fn_name + "No credentials in DB record for " + str(self.user_email))
logservice.flush()
self.process_tasks_job.status = constants.ExportJobStatus.ERROR
self.process_tasks_job.message = ''
self.process_tasks_job.error_message = "Problem with user credentials. Please restart."
self.process_tasks_job.job_progress_timestamp = datetime.datetime.now()
self._log_progress("No credentials")
self.process_tasks_job.put()
logging.debug(fn_name + "<End> No credentials")
return
if self.credentials.invalid:
logging.error(fn_name + "Invalid credentials in DB record for " + str(self.user_email))
logservice.flush()
self.process_tasks_job.status = constants.ExportJobStatus.ERROR
self.process_tasks_job.message = ''
self.process_tasks_job.error_message = "Invalid credentials. Please restart and re-authenticate."
self.process_tasks_job.job_progress_timestamp = datetime.datetime.now()
self._log_progress("Credentials invalid")
logservice.flush()
self.process_tasks_job.put()
logging.debug(fn_name + "<End> Invalid credentials")
return
if self.is_test_user:
logging.debug(fn_name + "User is test user %s" % self.user_email)
logservice.flush()
http = httplib2.Http()
http = self.credentials.authorize(http)
service = discovery.build("tasks", "v1", http=http)
self.tasklists_svc = service.tasklists()
self.tasks_svc = service.tasks()
# =========================================
# Retrieve tasks from the Google server
# =========================================
self._export_tasks()
else:
logging.error(fn_name + "No processing, as there was no user_email key")
logservice.flush()
logging.debug(fn_name + "<End>")
logservice.flush()
class ProcessTasksWorker(webapp.RequestHandler):
""" Process tasks according to data in the ProcessTasksJob entity """
credentials = None
user_email = None
is_test_user = False
process_tasks_job = None
tasks_svc = None
tasklists_svc = None
def post(self):
fn_name = "ProcessTasksWorker.post(): "
logging.debug(fn_name + "<start> (app version %s)" %appversion.version)
logservice.flush()
client_id, client_secret, user_agent, app_title, project_name, host_msg = shared.get_settings(self.request.host)
self.user_email = self.request.get(settings.TASKS_QUEUE_KEY_NAME)
self.is_test_user = shared.isTestUser(self.user_email)
if self.user_email:
# Retrieve the DB record for this user
self.process_tasks_job = model.ProcessTasksJob.get_by_key_name(self.user_email)
if self.process_tasks_job is None:
logging.error(fn_name + "No DB record for " + self.user_email)
logservice.flush()
logging.debug(fn_name + "<End> No DB record")
# TODO: Find some way of notifying the user?????
# Could use memcache to relay a message which is displayed in ProgressHandler
return
else:
logging.debug(fn_name + "Retrieved process tasks job for " + str(self.user_email))
logservice.flush()
self.process_tasks_job.status = constants.ExportJobStatus.INITIALISING
self.process_tasks_job.job_progress_timestamp = datetime.datetime.now()
self.process_tasks_job.message = "Validating background job ..."
logging.debug(fn_name + "Initialising - Job status: '" + str(self.process_tasks_job.status) + "', progress: " +
str(self.process_tasks_job.total_progress) + ", msg: '" +
str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'")
logservice.flush()
self.process_tasks_job.put()
user = self.process_tasks_job.user
if not user:
logging.error(fn_name + "No user object in DB record for " + str(self.user_email))
logservice.flush()
self.process_tasks_job.status = constants.ExportJobStatus.ERROR
self.process_tasks_job.message = ''
self.process_tasks_job.error_message = "Problem with user details. Please restart."
self.process_tasks_job.job_progress_timestamp = datetime.datetime.now()
logging.debug(fn_name + "No user - Job status: '" + str(self.process_tasks_job.status) + "', progress: " +
str(self.process_tasks_job.total_progress) + ", msg: '" +
str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'")
logservice.flush()
self.process_tasks_job.put()
logging.debug(fn_name + "<End> No user object")
return
self.credentials = self.process_tasks_job.credentials
if not self.credentials:
logging.error(fn_name + "No credentials in DB record for " + str(self.user_email))
logservice.flush()
self.process_tasks_job.status = constants.ExportJobStatus.ERROR
self.process_tasks_job.message = ''
self.process_tasks_job.error_message = "Problem with user self.credentials. Please restart."
self.process_tasks_job.job_progress_timestamp = datetime.datetime.now()
logging.debug(fn_name + "Job status: '" + str(self.process_tasks_job.status) + "', progress: " +
str(self.process_tasks_job.total_progress) + ", msg: '" +
str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'")
logservice.flush()
self.process_tasks_job.put()
logging.debug(fn_name + "<End> No self.credentials")
return
if self.credentials.invalid:
logging.error(fn_name + "Invalid credentials in DB record for " + str(self.user_email))
logservice.flush()
self.process_tasks_job.status = constants.ExportJobStatus.ERROR
self.process_tasks_job.message = ''
self.process_tasks_job.error_message = "Invalid self.credentials. Please restart and re-authenticate."
self.process_tasks_job.job_progress_timestamp = datetime.datetime.now()
logging.debug(fn_name + "Credentials invalid - Job status: '" + str(self.process_tasks_job.status) + "', progress: " +
str(self.process_tasks_job.total_progress) + ", msg: '" +
str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'")
logservice.flush()
self.process_tasks_job.put()
logging.debug(fn_name + "<End> Invalid self.credentials")
return
if self.is_test_user:
logging.debug(fn_name + "User is test user %s" % self.user_email)
logservice.flush()
http = httplib2.Http()
http = self.credentials.authorize(http)
service = discovery.build("tasks", "v1", http)
self.tasklists_svc = service.tasklists()
self.tasks_svc = service.tasks()
self.export_tasks()
# logging.debug(fn_name + "Finished processing. Total progress = " +
# str(self.process_tasks_job.total_progress) + " for " + str(self.user_email))
else:
logging.error(fn_name + "No processing, as there was no user_email key")
logservice.flush()
logging.debug(fn_name + "<End>, user = "******"export_tasks: "
logging.debug(fn_name + "<Start>")
logservice.flush()
start_time = datetime.datetime.now()
include_hidden = self.process_tasks_job.include_hidden
include_completed = self.process_tasks_job.include_completed
include_deleted = self.process_tasks_job.include_deleted
summary_msg = ''
# Retrieve all tasks for the user
try:
logging.debug(fn_name + "include_hidden = " + str(include_hidden) +
", include_completed = " + str(include_completed) +
", include_deleted = " + str(include_deleted))
logservice.flush()
# ##############################################
# FLOW
# ----------------------------------------------
# For each page of taskslists
# For each tasklist
# For each page of tasks
# For each task
# Fix date format
# Add tasks to tasklist collection
# Add tasklist to tasklists collection
# Use tasklists collection to return tasks backup to user
self.process_tasks_job.status = constants.ExportJobStatus.BUILDING
self.process_tasks_job.job_progress_timestamp = datetime.datetime.now()
self.process_tasks_job.message = 'Retrieving tasks from server ...'
logging.debug(fn_name + "Building - Job status: '" + str(self.process_tasks_job.status) + "', progress: " +
str(self.process_tasks_job.total_progress) + ", msg: '" +
str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'")
logservice.flush()
self.process_tasks_job.put()
# This list will contain zero or more tasklist dictionaries, which each contain tasks
tasklists = []
total_num_tasklists = 0
total_num_tasks = 0
tasks_per_list = []
# ---------------------------------------
# Retrieve all the tasklists for the user
# ---------------------------------------
logging.debug(fn_name + "Retrieve all the tasklists for the user")
logservice.flush()
next_tasklists_page_token = None
more_tasklists_data_to_retrieve = True
while more_tasklists_data_to_retrieve:
if self.is_test_user:
logging.debug(fn_name + "calling tasklists.list().execute() to create tasklists list")
logservice.flush()
retry_count = constants.NUM_API_RETRIES
while retry_count > 0:
try:
if next_tasklists_page_token:
tasklists_data = self.tasklists_svc.list(pageToken=next_tasklists_page_token).execute()
else:
tasklists_data = self.tasklists_svc.list().execute()
# Successfully retrieved data, so break out of retry loop
break
except Exception, e:
retry_count = retry_count - 1
if retry_count > 0:
logging.warning(fn_name + "Error retrieving list of tasklists. " +
str(retry_count) + " retries remaining")
logservice.flush()
if retry_count <= 2:
logging.debug(fn_name + "Sleeping for " + str(settings.WORKER_API_RETRY_SLEEP_DURATION) +
" seconds before retrying")
logservice.flush()
time.sleep(settings.WORKER_API_RETRY_SLEEP_DURATION)
else:
logging.exception(fn_name + "Still error retrieving list of tasklists after " + str(constants.NUM_API_RETRIES) + " retries. Giving up")
logservice.flush()
raise e
if self.is_test_user and settings.DUMP_DATA:
logging.debug(fn_name + "tasklists_data ==>")
logging.debug(tasklists_data)
logservice.flush()
if tasklists_data.has_key(u'items'):
tasklists_list = tasklists_data[u'items']
else:
# If there are no tasklists, then there will be no 'items' element. This could happen if
# the user has deleted all their tasklists. Not sure if this is even possible, but
# checking anyway, since it is possible to have a tasklist without 'items' (see issue #9)
logging.debug(fn_name + "User has no tasklists.")
logservice.flush()
tasklists_list = []
# tasklists_list is a list containing the details of the user's tasklists.
# We are only interested in the title
# if self.is_test_user and settings.DUMP_DATA:
# logging.debug(fn_name + "tasklists_list ==>")
# logging.debug(tasklists_list)
# ---------------------------------------
# Process all the tasklists for this user
# ---------------------------------------
for tasklist_data in tasklists_list:
total_num_tasklists = total_num_tasklists + 1
if self.is_test_user and settings.DUMP_DATA:
logging.debug(fn_name + "tasklist_data ==>")
logging.debug(tasklist_data)
logservice.flush()
"""
Example of a tasklist entry;
u'id': u'MDAxNTkzNzU0MzA0NTY0ODMyNjI6MDow',
u'kind': u'tasks#taskList',
u'selfLink': u'https://www.googleapis.com/tasks/v1/users/@me/lists/MDAxNTkzNzU0MzA0NTY0ODMyNjI6MDow',
u'title': u'Default List',
u'updated': u'2012-01-28T07:30:18.000Z'},
"""
tasklist_title = tasklist_data[u'title']
tasklist_id = tasklist_data[u'id']
if self.is_test_user and settings.DUMP_DATA:
logging.debug(fn_name + "Process all the tasks in " + str(tasklist_title))
logservice.flush()
# =====================================================
# Process all the tasks in this task list
# =====================================================
tasklist_dict, num_tasks = self.get_tasks_in_tasklist(tasklist_title, tasklist_id,
include_hidden, include_completed, include_deleted)
# Track number of tasks per tasklist
tasks_per_list.append(num_tasks)
total_num_tasks = total_num_tasks + num_tasks
self.process_tasks_job.total_progress = total_num_tasks
self.process_tasks_job.tasklist_progress = 0 # Because total_progress now includes num_tasks for current tasklist
self.process_tasks_job.job_progress_timestamp = datetime.datetime.now()
self.process_tasks_job.message = ''
logging.debug(fn_name + "Processed tasklist. Updated job status: '" + str(self.process_tasks_job.status) + "', progress: " +
str(self.process_tasks_job.total_progress) + ", msg: '" +
str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'")
logservice.flush()
self.process_tasks_job.put()
# if self.is_test_user:
# logging.debug(fn_name + "Adding %d tasks to tasklist" % len(tasklist_dict[u'tasks']))
# Add the data for this tasklist (including all the tasks) into the collection of tasklists
tasklists.append(tasklist_dict)
# Check if there is another page of tasklists to be retrieved
if tasklists_data.has_key('nextPageToken'):
# There is another page of tasklists to be retrieved for this user,
# which we'll retrieve next time around the while loop.
# This happens if there is more than 1 page of tasklists.
# It seems that each page contains 20 tasklists.
more_tasklists_data_to_retrieve = True # Go around while loop again
next_tasklists_page_token = tasklists_data['nextPageToken']
# if self.is_test_user:
# logging.debug(fn_name + "There is (at least) one more page of tasklists to be retrieved")
else:
# This is the last (or only) page of results (list of tasklists)
more_tasklists_data_to_retrieve = False
next_tasklists_page_token = None
# *** end while more_tasks_data_to_retrieve ***
# ------------------------------------------------------
# Store the data, so we can return it to the user
# ------------------------------------------------------
"""
tasklists is a list of tasklist structures
structure of tasklist
{
"title" : tasklist.title, # Name of this tasklist
"tasks" : [ task ] # List of task items in this tasklist
}
structure of task
{
"title" : title, # Free text
"status" : status, # "completed" | "needsAction"
"id" : id, # Used when determining parent-child relationships
"parent" : parent, # OPT: ID of the parent of this task (only if this is a sub-task)
"notes" : notes, # OPT: Free text
"due" : due, # OPT: Date due, e.g. 2012-01-30T00:00:00.000Z NOTE time = 0
"updated" : updated, # Timestamp, e.g., 2012-01-26T07:47:18.000Z
"completed" : completed # Timestamp, e.g., 2012-01-27T10:38:56.000Z
}
"""
# Delete existing backup data records
tasklist_data_records = model.TasklistsData.gql("WHERE ANCESTOR IS :1",
db.Key.from_path(settings.DB_KEY_TASKS_BACKUP_DATA, self.user_email))
num_records = tasklist_data_records.count()
logging.debug(fn_name + "Deleting " + str(num_records) + " old blobs")
logservice.flush()
for tasklists_data_record in tasklist_data_records:
tasklists_data_record.delete()
# logging.debug(fn_name + "Pickling tasks data ...")
pickled_tasklists = pickle.dumps(tasklists)
# logging.debug(fn_name + "Pickled data size = " + str(len(pickled_tasklists)))
data_len = len(pickled_tasklists)
# Multiply by 1.0 float value so that we can use ceiling to find number of Blobs required
num_of_blobs = int(math.ceil(data_len * 1.0 / constants.MAX_BLOB_SIZE))
logging.debug(fn_name + "Calculated " + str(num_of_blobs) + " blobs required to store " + str(data_len) + " bytes")
logservice.flush()
try:
for i in range(num_of_blobs):
# Write backup data records
tasklist_rec = model.TasklistsData(db.Key.from_path(settings.DB_KEY_TASKS_BACKUP_DATA, self.user_email))
slice_start = int(i*constants.MAX_BLOB_SIZE)
slice_end = int((i+1)*constants.MAX_BLOB_SIZE)
# logging.debug(fn_name + "Creating part " + str(i+1) + " of " + str(num_of_blobs) +
# " using slice " + str(slice_start) + " to " + str(slice_end))
pkl_part = pickled_tasklists[slice_start : slice_end]
tasklist_rec.pickled_tasks_data = pkl_part
tasklist_rec.idx = i
tasklist_rec.put()
# logging.debug(fn_name + "Marking backup job complete")
end_time = datetime.datetime.now()
process_time = end_time - start_time
proc_time_str = str(process_time.seconds) + "." + str(process_time.microseconds)[:3] + " seconds"
# Mark backup completed
summary_msg = "Retrieved %d tasks from %d tasklists" % (total_num_tasks, total_num_tasklists)
breakdown_msg = "Tasks per list: " + str(tasks_per_list)
self.process_tasks_job.status = constants.ExportJobStatus.EXPORT_COMPLETED
self.process_tasks_job.job_progress_timestamp = datetime.datetime.now()
# self.process_tasks_job.message = summary_msg + " in " + proc_time_str
self.process_tasks_job.message = summary_msg + " at " + \
self.process_tasks_job.job_start_timestamp.strftime("%H:%M UTC, %a %d %b %Y")
# logging.debug(fn_name + "COMPLETED: Export complete - Job status: '" + str(self.process_tasks_job.status) + "', progress: " +
# str(self.process_tasks_job.total_progress) + ", msg: '" +
# str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'")
logging.info(fn_name + "COMPLETED: " + summary_msg + " for " + self.user_email + " in " + proc_time_str)
logservice.flush()
self.process_tasks_job.put()
try:
end_time = datetime.datetime.now()
process_time = end_time - start_time
processing_time = process_time.days * 3600*24 + process_time.seconds + process_time.microseconds / 1000000.0
included_options_str = "Includes: Completed = %s, Deleted = %s, Hidden = %s" % (str(include_completed), str(include_deleted), str(include_hidden))
logging.debug(fn_name + "STATS: Started at " + str(start_time) +
"\n " + summary_msg +
"\n " + breakdown_msg +
"\n " + proc_time_str +
"\n " + included_options_str)
logservice.flush()
usage_stats = model.UsageStats(
user_hash = hash(self.user_email),
number_of_tasks = self.process_tasks_job.total_progress,
number_of_tasklists = total_num_tasklists,
tasks_per_tasklist = tasks_per_list,
include_completed = include_completed,
include_deleted = include_deleted,
include_hidden = include_hidden,
start_time = start_time,
processing_time = processing_time)
usage_stats.put()
logging.debug(fn_name + "Saved stats")
logservice.flush()
except Exception, e:
logging.exception("Error saving stats")
logservice.flush()
# Don't bother doing anything else, because stats aren't critical
except apiproxy_errors.RequestTooLargeError, e:
logging.exception(fn_name + "Error putting results in DB - Request too large")
logservice.flush()
self.process_tasks_job.status = constants.ExportJobStatus.ERROR
self.process_tasks_job.message = ''
self.process_tasks_job.error_message = "Tasklists data is too large - Unable to store tasklists in DB: " + shared.get_exception_msg(e)
self.process_tasks_job.job_progress_timestamp = datetime.datetime.now()
logging.debug(fn_name + "apiproxy_errors.RequestTooLargeError - Job status: '" + str(self.process_tasks_job.status) + "', progress: " +
str(self.process_tasks_job.total_progress) + ", msg: '" +
str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'")
logservice.flush()
self.process_tasks_job.put()
except Exception, e:
logging.exception(fn_name + "Error putting results in DB")
logservice.flush()
self.process_tasks_job.status = constants.ExportJobStatus.ERROR
self.process_tasks_job.message = ''
self.process_tasks_job.error_message = "Unable to store tasklists in DB: " + shared.get_exception_msg(e)
self.process_tasks_job.job_progress_timestamp = datetime.datetime.now()
logging.debug(fn_name + "Exception - Job status: '" + str(self.process_tasks_job.status) + "', progress: " +
str(self.process_tasks_job.total_progress) + ", msg: '" +
str(self.process_tasks_job.message) + "', err msg: '" + str(self.process_tasks_job.error_message) + "'")
logservice.flush()
self.process_tasks_job.put()
def dump_obj(obj):
for attr in dir(obj):
logging.debug(" obj.%s = %s" % (attr, getattr(obj, attr)))
logservice.flush()
def flush_logs():
try:
logservice.flush()
except apiproxy_errors.CancelledError:
pass
if memcache_payload_md5 == current_payload_md5:
logging.info('Twitter Search/Tweets API payload is identical to last time; ending Task')
else:
logging.info('Twitter Search/Tweets API payload has changed; running MessageQueue')
for status in statuses:
message = status.get('text')
message_id = status.get('id')
taskqueue.add(
queue_name='messagequeue',
url=urls.taskMessageQueue,
params={
'user_id':status.get('user').get('screen_name'),
'message_id':message_id,
'message':message
}
)
memcache.set(payload_key, value=current_payload_md5, time=1*60*60*24)
else:
logging.exception(content)
else:
logging.exception('No twitter_bearer_token found in Memcache, or available from the API call')
except Exception, e:
logging.exception(e)
finally:
logservice.flush()
return
def get(self):
self.get_tweets()
def post(self):
self.get_tweets()
def get_tasks_in_tasklist(self, tasklist_title, tasklist_id, include_hidden, include_completed, include_deleted):
""" Returns all the tasks in the tasklist
arguments:
tasklist_title -- Name of the tasklist
tasklist_id -- ID used to retrieve tasks from this tasklist
MUST match the ID returned in the tasklist data
include_hidden -- If true, include hidden tasks in the backup
include_completed -- If true, include completed tasks in the backup
include_deleted -- If true, include deleted tasks in the backup
returns a tuple;
two-element dictionary;
'title' is a string, the name of the tasklist
'tasks' is a list. Each element in the list is dictionary representing 1 task
number of tasks
"""
fn_name = "CreateBackupHandler.get_tasks_in_tasklist(): "
tasklist_dict = {} # Blank dictionary for this tasklist
tasklist_dict[u'title'] = tasklist_title # Store the tasklist name in the dictionary
tasklist_dict[u'id'] = tasklist_id # Store the tasklist ID in the dictionary
num_tasks = 0
more_tasks_data_to_retrieve = True
next_tasks_page_token = None
# Keep track of when last updated, to prevent excessive DB access which could exceed quota
prev_progress_timestamp = datetime.datetime.now()
if self.is_test_user and settings.DUMP_DATA:
logging.debug(fn_name + "include_hidden = " + str(include_hidden) +
", include_completed = " + str(include_completed) +
", include_deleted = " + str(include_deleted))
logservice.flush()
# ---------------------------------------------------------------------------
# Retrieve the tasks in this tasklist, and store as "tasks" in the dictionary
# ---------------------------------------------------------------------------
while more_tasks_data_to_retrieve:
retry_count = constants.NUM_API_RETRIES
while retry_count > 0:
try:
# Retrieve a page of (up to 100) tasks
if next_tasks_page_token:
# Get the next page of results
# This happens if there are more than 100 tasks in the list
# See http://code.google.com/apis/tasks/v1/using.html#api_params
# "Maximum allowable value: maxResults=100"
tasks_data = self.tasks_svc.list(tasklist = tasklist_id, pageToken=next_tasks_page_token,
showHidden=include_hidden, showCompleted=include_completed, showDeleted=include_deleted).execute()
else:
# Get the first (or only) page of results for this tasklist
tasks_data = self.tasks_svc.list(tasklist = tasklist_id,
showHidden=include_hidden, showCompleted=include_completed, showDeleted=include_deleted).execute()
# Succeeded, so continue
break
except Exception, e:
retry_count = retry_count - 1
if retry_count > 0:
logging.warning(fn_name + "Error retrieving tasks, " +
str(retry_count) + " retries remaining")
logservice.flush()
# Last chances - sleep to give the server some extra time before re-requesting
if retry_count <= 2:
logging.debug(fn_name + "Sleeping for " + str(settings.WORKER_API_RETRY_SLEEP_DURATION) +
" seconds before retrying")
logservice.flush()
time.sleep(settings.WORKER_API_RETRY_SLEEP_DURATION)
else:
logging.exception(fn_name + "Still error retrieving tasks for tasklist after " + str(constants.NUM_API_RETRIES) + " retries. Giving up")
logservice.flush()
raise e
if self.is_test_user and settings.DUMP_DATA:
logging.debug(fn_name + "tasks_data ==>")
logging.debug(tasks_data)
if not tasks_data.has_key(u'items'):
# When using the Google Tasks webpage at https://mail.google.com/tasks/canvas, there will always
# be at least one task in any tasklist, because when deleting the last task, a new blank task is
# automatically created.
# However, a third-party app (e.g., Calengoo on Android) CAN delete all the tasks in a task list,
# which results in a tasklist without an 'items' element.
logging.debug(fn_name + "No tasks in tasklist")
logservice.flush()
else:
try:
tasks = tasks_data[u'items'] # Store all the tasks (List of Dict)
except Exception, e:
logging.exception(fn_name, "Exception extracting items from tasks_data.")
#logging.error(tasks_data)
logservice.flush()
raise e
# if self.is_test_user and settings.DUMP_DATA:
# logging.debug(fn_name + "tasks ==>")
# logging.debug(tasks)
# logservice.flush()
# ------------------------------------------------------------------------------------------------
# Fix date/time format for each task, so that the date/time values can be used in Django templates
# Convert the yyyy-mm-ddThh:mm:ss.dddZ format to a datetime object, and store that.
# There have been occassional format errors in the 'completed' property,
# due to 'completed' value such as "-1701567-04-26T07:12:55.000Z"
# According to http://docs.python.org/library/datetime.html
# "The exact range of years for which strftime() works also varies across platforms.
# Regardless of platform, years before 1900 cannot be used."
# so if any date/timestamp value is invalid, set the property to '1900-01-01 00:00:00'
# NOTE: Sometimes a task has a completion date of '0000-01-01T00:00:00.000Z', which also cannot
# be converted to datetime, because the earliest allowable datetime year is 0001
# ------------------------------------------------------------------------------------------------
for t in tasks:
num_tasks = num_tasks + 1
date_due = t.get(u'due')
if date_due:
try:
new_due_date = datetime.datetime.strptime(date_due, "%Y-%m-%dT00:00:00.000Z").date()
except ValueError, e:
new_due_date = datetime.date(1900, 1, 1)
logging.warning(fn_name + "Invalid 'due' timestamp (" + str(date_due) + "), so using " + str(new_due_date) +
": " + shared.get_exception_msg(e))
logservice.flush()
t[u'due'] = new_due_date
datetime_updated = t.get(u'updated')
if datetime_updated:
try:
new_datetime_updated = datetime.datetime.strptime(datetime_updated, "%Y-%m-%dT%H:%M:%S.000Z")
except ValueError, e:
new_datetime_updated = datetime.datetime(1900, 1, 1, 0, 0, 0)
logging.warning(fn_name + "Invalid 'updated' timestamp (" + str(datetime_updated) + "), so using " + str(new_datetime_updated) +
": " + shared.get_exception_msg(e))
logservice.flush()
t[u'updated'] = new_datetime_updated
def _get_tasks_in_tasklist(self, tasklist_title, tasklist_id, include_hidden, include_completed, include_deleted):
""" Returns all the tasks in the tasklist
arguments:
tasklist_title -- Name of the tasklist
tasklist_id -- ID used to retrieve tasks from this tasklist
MUST match the ID returned in the tasklist data
include_hidden -- If true, include hidden tasks in the backup
include_completed -- If true, include completed tasks in the backup
include_deleted -- If true, include deleted tasks in the backup
returns a tuple;
two-element dictionary;
'title' is a string, the name of the tasklist
'tasks' is a list. Each element in the list is dictionary representing 1 task
number of tasks
"""
fn_name = "CreateBackupHandler._get_tasks_in_tasklist(): "
tasklist_dict = {} # Blank dictionary for this tasklist
tasklist_dict[u'title'] = tasklist_title # Store the tasklist name in the dictionary
tasklist_dict[u'id'] = tasklist_id # Store the tasklist ID in the dictionary
num_tasks = 0
more_tasks_data_to_retrieve = True
next_tasks_page_token = None
# Keep track of when last updated, to prevent excessive DB access which could exceed quota
prev_progress_timestamp = datetime.datetime.now()
if self.is_test_user and settings.DUMP_DATA:
logging.debug(fn_name +
"TEST: include_completed = " + str(include_completed) +
", include_hidden = " + str(include_hidden) +
", include_deleted = " + str(include_deleted))
logservice.flush()
# ---------------------------------------------------------------------------
# Retrieve the tasks in this tasklist, and store as "tasks" in the dictionary
# ---------------------------------------------------------------------------
while more_tasks_data_to_retrieve:
retry_count = settings.NUM_API_TRIES
while retry_count > 0:
try:
# Retrieve a page of (up to 100) tasks
if next_tasks_page_token:
# Get the next page of results
# This happens if there are more than 100 tasks in the list
# See http://code.google.com/apis/tasks/v1/using.html#api_params
# "Maximum allowable value: maxResults=100"
tasks_data = self.tasks_svc.list(tasklist = tasklist_id, pageToken=next_tasks_page_token,
showHidden=include_hidden, showCompleted=include_completed, showDeleted=include_deleted).execute()
else:
# Get the first (or only) page of results for this tasklist
tasks_data = self.tasks_svc.list(tasklist = tasklist_id,
showHidden=include_hidden, showCompleted=include_completed, showDeleted=include_deleted).execute()
# Succeeded, so continue
break
except Exception, e:
retry_count = retry_count - 1
if retry_count > 0:
logging.warning(fn_name + "Error retrieving tasks, " +
str(retry_count) + " attempts remaining: " + shared.get_exception_msg(e))
logservice.flush()
# Last chances - sleep to give the server some extra time before re-requesting
if retry_count <= 2:
logging.debug(fn_name + "Giving server an extra chance; Sleeping for " +
str(settings.WORKER_API_RETRY_SLEEP_DURATION) +
" seconds before retrying")
logservice.flush()
# Update job_progress_timestamp so that job doesn't time out
self.process_tasks_job.job_progress_timestamp = datetime.datetime.now()
self.process_tasks_job.put()
time.sleep(settings.WORKER_API_RETRY_SLEEP_DURATION)
else:
logging.exception(fn_name + "Still error retrieving tasks for tasklist after " +
str(settings.NUM_API_TRIES) + " attempts. Giving up")
logservice.flush()
raise e
if self.is_test_user and settings.DUMP_DATA:
logging.debug(fn_name + "tasks_data ==>")
logging.debug(tasks_data)
if not tasks_data.has_key(u'items'):
# When using the Google Tasks webpage at https://mail.google.com/tasks/canvas, there will always
# be at least one task in any tasklist, because when deleting the last task, a new blank task is
# automatically created.
# However, a third-party app (e.g., Calengoo on Android) CAN delete all the tasks in a task list,
# which results in a tasklist without an 'items' element.
logging.debug(fn_name + "No tasks in tasklist")
logservice.flush()
else:
try:
tasks = tasks_data[u'items'] # Store all the tasks (List of Dict)
except Exception, e:
logging.exception(fn_name, "Exception extracting items from tasks_data: " + shared.get_exception_msg(e))
#logging.error(tasks_data)
logservice.flush()
raise e
# if self.is_test_user and settings.DUMP_DATA:
# logging.debug(fn_name + "tasks ==>")
# logging.debug(tasks)
# logservice.flush()
for t in tasks:
num_tasks = num_tasks + 1
# TODO: Investigate if including this will cause memory to be exceeded for very large tasks list
# Store original RFC-3339 timestamps (used for raw2 export format)
if t.has_key('due'):
t['due_RFC3339'] = t['due']
if t.has_key('updated'):
t['updated_RFC3339'] = t['updated']
if t.has_key('completed'):
t['completed_RFC3339'] = t['completed']
# Converts the RFC-3339 string returned by the server to a date or datetime object
# so that other methods (such as Django templates) can display a custom formatted date
shared.set_timestamp(t, u'due', date_only=True)
shared.set_timestamp(t, u'updated')
shared.set_timestamp(t, u'completed')
if tasklist_dict.has_key(u'tasks'):
# This is the n'th page of task data for this tasklist, so extend the existing list of tasks
tasklist_dict[u'tasks'].extend(tasks)
else:
# This is the first (or only) list of task for this tasklist
tasklist_dict[u'tasks'] = tasks
def redirect_for_auth(self, user, redirect_url=None):
"""Redirects the webapp response to authenticate the user with OAuth2.
Args:
redirect_url [OPTIONAL] The URL to return to once authorised.
Usually unused (left as None), so that the URL of the calling page is used
Uses the 'state' parameter to store redirect_url.
The handler for /oauth2callback can therefore redirect the user back to the page they
were on when get_credentials() failed (or to the specified redirect_url).
"""
fn_name = "redirect_for_auth(): "
try:
client_id, client_secret, user_agent, app_title, product_name, host_msg = get_settings(self.request.host)
# Check how many times this has been called (without credentials having been successfully retrieved)
if self.request.cookies.has_key('auth_retry_count'):
auth_retry_count = int(self.request.cookies['auth_retry_count'])
logging.debug(fn_name + "auth_retry_count = " + str(auth_retry_count))
if auth_retry_count > settings.MAX_NUM_AUTH_RETRIES:
# Exceeded maximum number of retries, so don't try again.
# Redirect user to invalid credentials page
logging.warning(fn_name + "Not re-authorising, because there have already been " + str(auth_retry_count) +
" attempts. Redirecting to " + settings.INVALID_CREDENTIALS_URL)
self.redirect(settings.INVALID_CREDENTIALS_URL + "?rc=NC&nr=" + str(auth_retry_count))
return
else:
logging.debug(fn_name + "No auth_retry_count cookie found. Probably means last authorisation was more than " +
str(settings.AUTH_RETRY_COUNT_COOKIE_EXPIRATION_TIME) + " seconds ago")
auth_retry_count = 0
if not redirect_url:
# By default, return to the same page
redirect_url = self.request.path_qs
# According to https://developers.google.com/accounts/docs/OAuth_ref#RequestToken
# xoauth_displayname is optional.
# (optional) String identifying the application.
# This string is displayed to end users on Google's authorization confirmation page.
# For registered applications, the value of this parameter overrides the name set during registration and
# also triggers a message to the user that the identity can't be verified.
# For unregistered applications, this parameter enables them to specify an application name,
# In the case of unregistered applications, if this parameter is not set, Google identifies the application
# using the URL value of oauth_callback; if neither parameter is set, Google uses the string "anonymous".
# It seems preferable to NOT supply xoauth_displayname, so that Google doesn't display "identity can't be verified" msg.
flow = client.OAuth2WebServerFlow(
client_id=client_id,
client_secret=client_secret,
scope="https://www.googleapis.com/auth/tasks",
user_agent=user_agent,
state=redirect_url)
callback = self.request.relative_url("/oauth2callback")
authorize_url = flow.step1_get_authorize_url(callback)
memcache.set(user.user_id(), pickle.dumps(flow))
# Keep track of how many times we've called the authorise URL
if self.request.cookies.has_key('auth_retry_count'):
auth_retry_count = int(self.request.cookies['auth_retry_count'])
else:
auth_retry_count = 0
__store_auth_retry_count(self, auth_retry_count + 1)
logging.debug(fn_name + "Redirecting to " + str(authorize_url))
logservice.flush()
self.redirect(authorize_url)
except Exception, e:
logging.exception(fn_name + "Caught top-level exception")
serve_outer_exception_message(self, e)
logging.debug(fn_name + "<End> due to exception" )
logservice.flush()
