def post(self):
if self.checkIfAuthentificatedUserIsAdmin():
name = strip_ml_tags(self.request.get('name'))
if name != '':
thread = Thread()
thread.name = name
thread.position = int(self.request.get('position'))
thread.put()
self.redirect('/?mode=admin')
def post(self):
user = self.getAuthentificatedUser()
if not user:
return
id = self.request.get('id')
try:
thread = Thread.get(db.Key.from_path('Thread', int(id)))
except:
return
name = strip_ml_tags(self.request.get('name'))
if name == '':
template_values = {
'topics' : self.topics,
'name' : name,
}
else:
topic = Topic() #parent=thread
topic.thread = thread
topic.name = name
if users.get_current_user():
topic.author = users.get_current_user()
topic.put()
mode = self.request.get('mode')
self.redirect('/view?id=' + str(topic.key().id()))
return
template_values = {
'topics' : self.topics,
'name' : '',
}
path = os.path.join(os.path.dirname(__file__), os.path.join('templates', 'addTopic.htm'))
self.response.out.write(template.render(path, template_values))
def get(self):
id = int(self.request.get('id'))
thread = Thread.get(db.Key.from_path('Thread', int(id)))
template_values = {
'thread' : thread,
}
path = os.path.join(os.path.dirname(__file__), os.path.join('templates', 'modifyThread.htm'))
self.response.out.write(template.render(path, template_values))
def get(self):
self.response.out.write('UID\tTID\tMId\tRTime\tERTime\twC\tunsub\tedu\tsent\tis_html\n')
for thread in Thread.gql('WHERE is_processed = TRUE AND is_queued = FALSE AND estimated_reading_time > 0'):
message = Message.get_by_key_name(thread.last_message_id, None)
if message != None:
self.response.out.write('%s\t%s\t%s\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n' % (message.user_email,
message.thread_id, message.message_id, thread.reading_time, thread.estimated_reading_time,
message.word_count, message.has_unsubscribe, 'edu' in message.addresses, message.is_sent, message.is_html))
def post(self):
entityType = self.request.get('entityType')
email = self.request.get('email')
if entityType == 'EstimatedTime':
user = User.get_by_key_name(key_names = email)
if not user:
self.response.out.write("null")
else:
hexGmailThreadIdList = self.request.get('hexGmailThreadIdList')
if hexGmailThreadIdList:
user_ctx_id = user.user_ctx_id
tid_list = json.loads(hexGmailThreadIdList)
results = []
for thread_id in tid_list:
if thread_id:
thread = Thread.get_or_insert(key_name = thread_id, thread_id = thread_id, user_email = email)
if thread.is_processed:
results.append(str(thread.estimated_reading_time))
else:
if not thread.is_queued:
thread.is_queued = True
thread.put()
processReadingThread(user_ctx_id, email, [thread_id])
#FIXME
#deferred.defer(processReadingThread, user_ctx_id, email, [thread_id], _queue='reading-queue', _target='crawler')
results.append("-1")
else:
results.append(None)
self.response.out.write(json.dumps(results))
else:
self.response.out.write("null")
elif entityType == 'User':
user = User.get_by_key_name(key_names = email)
if not user:
self.response.out.write('null')
else:
result = {
"userId": user.user_ctx_id,
"email": email,
"orgKey": "",
"creationTimestamp": "1359721200347",
"lastUpdatedTimestamp": "1359878999598",
"lastSeenTimestamp": "1359878400000",
"lastSeenClientVersion": "5.10",
"lastSeenExtVersion": "5.0",
"lastSeenBrowser": "Chrome",
"userSettings": {
"value": "{}"
},
"isOauthComplete": user.is_oauth_complete,
"userKey": "",
"displayName": email,
"key": "",
"experiments": {}
}
self.response.out.write(json.dumps(result))
else:
self.response.out.write("[]")
def test_thread_stat_parsing(self):
thread = Thread(thread_id=1234)
parse_stat(thread, self.example)
self.assertEqual('sqlitebrowser', thread.comm)
self.assertEqual(46582, thread.minor_faults)
self.assertEqual(70, thread.major_faults)
self.assertEqual(6605, thread.user_time)
self.assertEqual(429, thread.system_time)
self.assertEqual(8244322, thread.start_time)
def get(self):
if not self.checkIfAuthentificatedUserIsAdmin():
return
try:
id = int(self.request.get('id'))
thread = Thread.get(db.Key.from_path('Thread', id))
thread.delete()
except:
pass
self.redirect('/?mode=admin')
def post(self):
if self.checkIfAuthentificatedUserIsAdmin():
for key, value in self.request.POST.items():
if key[0:7] == "thread_":
try:
thread = Thread.get(db.Key.from_path('Thread', int(key[7:])))
thread.position = int(value)
thread.put()
except:
pass
self.redirect('/?mode=admin')
def get(self):
email = self.request.get('email')
thread_ids = json.loads(self.request.get('thread_ids'))
user = User.get_by_key_name(key_names = email)
for index, thread_id in enumerate(thread_ids):
thread = Thread.get_or_insert(key_name = thread_id, thread_id = thread_id, user_email = email)
thread.is_queued = True
thread.is_processed = False
thread.put()
deferred.defer(processReadingThread, user.user_ctx_id, email, thread_ids, _queue='reading-queue', _target='crawler')
def post(self):
if self.checkIfAuthentificatedUserIsAdmin():
id = int(self.request.get('id'))
try:
thread = Thread.get(db.Key.from_path('Thread', int(id)))
name = strip_ml_tags(self.request.get('name'))
if name != '':
thread.name = name
thread.put()
except:
pass
self.redirect('/?mode=admin')
def get(self):
try:
email = self.request.get('email')
thread_ids = json.loads(self.request.get('thread_ids'))
user = User.get_by_key_name(key_names = email)
if not user:
raise Exception('NO such users')
for index, thread_id in enumerate(thread_ids):
thread = Thread.get_or_insert(key_name = thread_id, thread_id = thread_id, user_email = email)
thread.is_queued = True
thread.is_processed = False
thread.put()
processReadingThread(user.user_ctx_id, email, thread_ids)
for index, thread_id in enumerate(thread_ids):
thread = Thread.get_or_insert(key_name = thread_id, thread_id = thread_id, user_email = email)
logging.info(str(thread.estimated_reading_time))
print "%s:%s" % (thread_id, str(thread.estimated_reading_time))
except Exception as e:
logging.error(traceback.format_exc(e))
def get(self):
entityType = self.request.get('entityType')
email = self.request.get('email')
if entityType == 'User':
logging.info('Remove User %s' % email);
user = User.get_by_key_name(key_names = email)
if not user:
logging.error('no such User %s' % email);
return
user.delete()
for m in Message.gql('WHERE user_email=:1',email):
m.delete()
for t in Thread.gql('WHERE user_email=:1',thread):
t.delete()
StorageByKeyName(CredentialsModel, user_email, 'credentials').locked_delete()
logging.info('Remove User successfully')
def post(self):
entityType = self.request.get('entityType')
email = self.request.get('email')
if entityType == 'ReadingTime':
thread_id = self.request.get('threadId')
elapsed_time = self.request.get('elapsedTime')
if not thread_id or not elapsed_time:
return
try:
elapsed_time = int(elapsed_time)
except:
return
thread = Thread.get_or_insert(key_name = thread_id, thread_id = thread_id, user_email = email)
thread.reading_time += elapsed_time
thread.put()
logging.info('ReadingTime of thread %s is updated to %s' % (thread_id, thread.reading_time))
def processReadingThread(user_ctx_id, user_email, thread_ids):
ctxio = ContextIOConn.getInstance().ctxio
account = contextIO2.Account(ctxio, {'id' : user_ctx_id})
index = 0
try:
for index, thread_id in enumerate(thread_ids):
thread = Thread.get_or_insert(key_name = thread_id, thread_id = thread_id, user_email = user_email)
if thread.is_processed:
continue
if not thread.is_queued:
continue
thread_data = account.get_message_thread('gm-' + thread_id, include_body=True)
thread.last_date = 0
thread.estimated_reading_time = 0
if not thread_data or not 'messages' in thread_data:
thread.is_queued = False
thread.estimated_reading_time = 1000
thread.put()
return
for message_data in thread_data['messages']:
message_id = message_data['gmail_message_id']
message = Message.get_or_insert(key_name = message_id, message_id = message_id, thread_id=thread_id, user_email=user_email)
if not message.is_processed:
message.is_html, plain_text = getPlainText(message_data['body'])
message.word_count = getWordCount(stripPlainTextSignature(plain_text))
message.addresses = json.dumps(message_data['addresses'])
message.date = message_data['date']
message.is_sent = '\\Sent' in message_data['folders']
message.has_unsubscribe = 'unsubscribe' in plain_text.lower()
message.is_processed = True
message.put()
if message.date > thread.last_date:
thread.last_date = message.date
thread.last_message_id = message_id
thread.estimated_reading_time += estimateReadingTime(message)
thread.is_queued = False
thread.is_processed = True
thread.put()
except DeadlineExceededError:
deferred.defer(processReadingThread, user_ctx_id, user_email, thread_ids[index:], _queue='reading-queue', _target='crawler')
except Exception as e:
logging.error('thread_id: %s user_email:%s %s' % (thread_id, user_email, traceback.print_exc()))
raise e
def get(self):
for thread in Thread.gql('WHERE is_processed = FALSE AND is_queued = TRUE'):
thread.is_queued = False
thread.put()
self.response.out.write('%s reset + \n' % thread.thread_id)
