def post(self):
entityType = self.request.get('entityType')
email = self.request.get('email')
if entityType == 'EstimatedTime':
user = User.get_by_key_name(key_names = email)
if not user:
self.response.out.write("null")
else:
hexGmailThreadIdList = self.request.get('hexGmailThreadIdList')
if hexGmailThreadIdList:
user_ctx_id = user.user_ctx_id
tid_list = json.loads(hexGmailThreadIdList)
results = []
for thread_id in tid_list:
if thread_id:
thread = Thread.get_or_insert(key_name = thread_id, thread_id = thread_id, user_email = email)
if thread.is_processed:
results.append(str(thread.estimated_reading_time))
else:
if not thread.is_queued:
thread.is_queued = True
thread.put()
processReadingThread(user_ctx_id, email, [thread_id])
#FIXME
#deferred.defer(processReadingThread, user_ctx_id, email, [thread_id], _queue='reading-queue', _target='crawler')
results.append("-1")
else:
results.append(None)
self.response.out.write(json.dumps(results))
else:
self.response.out.write("null")
elif entityType == 'User':
user = User.get_by_key_name(key_names = email)
if not user:
self.response.out.write('null')
else:
result = {
"userId": user.user_ctx_id,
"email": email,
"orgKey": "",
"creationTimestamp": "1359721200347",
"lastUpdatedTimestamp": "1359878999598",
"lastSeenTimestamp": "1359878400000",
"lastSeenClientVersion": "5.10",
"lastSeenExtVersion": "5.0",
"lastSeenBrowser": "Chrome",
"userSettings": {
"value": "{}"
},
"isOauthComplete": user.is_oauth_complete,
"userKey": "",
"displayName": email,
"key": "",
"experiments": {}
}
self.response.out.write(json.dumps(result))
else:
self.response.out.write("[]")
def get(self):
email = self.request.get('email')
thread_ids = json.loads(self.request.get('thread_ids'))
user = User.get_by_key_name(key_names = email)
for index, thread_id in enumerate(thread_ids):
thread = Thread.get_or_insert(key_name = thread_id, thread_id = thread_id, user_email = email)
thread.is_queued = True
thread.is_processed = False
thread.put()
deferred.defer(processReadingThread, user.user_ctx_id, email, thread_ids, _queue='reading-queue', _target='crawler')
def get(self):
try:
email = self.request.get('email')
thread_ids = json.loads(self.request.get('thread_ids'))
user = User.get_by_key_name(key_names = email)
if not user:
raise Exception('NO such users')
for index, thread_id in enumerate(thread_ids):
thread = Thread.get_or_insert(key_name = thread_id, thread_id = thread_id, user_email = email)
thread.is_queued = True
thread.is_processed = False
thread.put()
processReadingThread(user.user_ctx_id, email, thread_ids)
for index, thread_id in enumerate(thread_ids):
thread = Thread.get_or_insert(key_name = thread_id, thread_id = thread_id, user_email = email)
logging.info(str(thread.estimated_reading_time))
print "%s:%s" % (thread_id, str(thread.estimated_reading_time))
except Exception as e:
logging.error(traceback.format_exc(e))
def post(self):
entityType = self.request.get('entityType')
email = self.request.get('email')
if entityType == 'ReadingTime':
thread_id = self.request.get('threadId')
elapsed_time = self.request.get('elapsedTime')
if not thread_id or not elapsed_time:
return
try:
elapsed_time = int(elapsed_time)
except:
return
thread = Thread.get_or_insert(key_name = thread_id, thread_id = thread_id, user_email = email)
thread.reading_time += elapsed_time
thread.put()
logging.info('ReadingTime of thread %s is updated to %s' % (thread_id, thread.reading_time))
def processReadingThread(user_ctx_id, user_email, thread_ids):
ctxio = ContextIOConn.getInstance().ctxio
account = contextIO2.Account(ctxio, {'id' : user_ctx_id})
index = 0
try:
for index, thread_id in enumerate(thread_ids):
thread = Thread.get_or_insert(key_name = thread_id, thread_id = thread_id, user_email = user_email)
if thread.is_processed:
continue
if not thread.is_queued:
continue
thread_data = account.get_message_thread('gm-' + thread_id, include_body=True)
thread.last_date = 0
thread.estimated_reading_time = 0
if not thread_data or not 'messages' in thread_data:
thread.is_queued = False
thread.estimated_reading_time = 1000
thread.put()
return
for message_data in thread_data['messages']:
message_id = message_data['gmail_message_id']
message = Message.get_or_insert(key_name = message_id, message_id = message_id, thread_id=thread_id, user_email=user_email)
if not message.is_processed:
message.is_html, plain_text = getPlainText(message_data['body'])
message.word_count = getWordCount(stripPlainTextSignature(plain_text))
message.addresses = json.dumps(message_data['addresses'])
message.date = message_data['date']
message.is_sent = '\\Sent' in message_data['folders']
message.has_unsubscribe = 'unsubscribe' in plain_text.lower()
message.is_processed = True
message.put()
if message.date > thread.last_date:
thread.last_date = message.date
thread.last_message_id = message_id
thread.estimated_reading_time += estimateReadingTime(message)
thread.is_queued = False
thread.is_processed = True
thread.put()
except DeadlineExceededError:
deferred.defer(processReadingThread, user_ctx_id, user_email, thread_ids[index:], _queue='reading-queue', _target='crawler')
except Exception as e:
logging.error('thread_id: %s user_email:%s %s' % (thread_id, user_email, traceback.print_exc()))
raise e
