def process_mail():
"""Send some mails"""
last_process = memcache.get('last_process_mail')
if last_process and util.td_seconds(last_process) < PROCESS_MAIL_INTERVAL:
return
logging.debug('Processing mail')
memcache.set('last_process_mail', util.now())
now = util.now()
mail_before = now - datetime.timedelta(seconds=user.MAIL_INTERVAL)
update_after = now - datetime.timedelta(seconds=user.UPDATE_INTERVAL)
# Queue those needs get updated, which have email address
#q = user.User.gql("WHERE email > '' AND last_updated >= :1 AND last_mailed < :2", update_after, mail_before)
# TODO only few users have email in datastore, should querying for email > '' first,
# then check the last_mailed and the last_updated
q = user.User.gql("WHERE last_mailed < :1", mail_before)
count = 0
offset = 0
while count < MAILS_PER_PROCESS:
users = q.fetch(50, offset * 50)
if not users:
break
for u in users:
if u.email and u.last_updated and u.last_updated >= update_after:
user.try_mail(u)
count += 1
if count >= MAILS_PER_PROCESS:
break
offset += 1
def process_mail():
"""Send some mails"""
last_process = memcache.get('last_process_mail')
if last_process and util.td_seconds(last_process) < PROCESS_MAIL_INTERVAL:
return
logging.debug('Processing mail')
memcache.set('last_process_mail', util.now())
now = util.now()
mail_before = now - datetime.timedelta(seconds=user.MAIL_INTERVAL)
update_after = now - datetime.timedelta(seconds=user.UPDATE_INTERVAL)
# Queue those needs get updated, which have email address
#q = user.User.gql("WHERE email > '' AND last_updated >= :1 AND last_mailed < :2", update_after, mail_before)
# TODO only few users have email in datastore, should querying for email > '' first,
# then check the last_mailed and the last_updated
q = user.User.gql("WHERE last_mailed < :1", mail_before)
count = 0
offset = 0
while count < MAILS_PER_PROCESS:
users = q.fetch(50, offset * 50)
if not users:
break
for u in users:
if u.email and u.last_updated and u.last_updated >= update_after:
user.try_mail(u)
count += 1
if count >= MAILS_PER_PROCESS:
break
offset += 1
def lock_one_username():
q = memcache.get('q') or []
for username in q:
logging.debug('Retrieving %s from db' % username)
u = user.get(username)
if not u._need_update:
remove(u)
continue
logging.debug('Checking lock')
locked_time = memcache.get('qlock_' + username)
logging.debug('locked_time: %s' % locked_time)
if locked_time:
logging.debug('Locked, timeout?')
# I don't think this would be used...
if util.td_seconds(locked_time) >= FETCH_TIMEOUT:
logging.debug('Yes, timed out')
# Force to acquire, if locked more than FETCH_TIMEOUT ago
memcache.set('qlock_' + username, util.now())
return username
else:
continue
else:
logging.debug(
'locked_time does not have a value, force to acquire')
memcache.set('qlock_' + username, util.now())
return username
return False
def lock_one_username():
q = memcache.get('q') or []
for username in q:
logging.debug('Retrieving %s from db' % username)
u = user.get(username)
if not u._need_update:
remove(u)
continue
logging.debug('Checking lock')
locked_time = memcache.get('qlock_' + username)
logging.debug('locked_time: %s' % locked_time)
if locked_time:
logging.debug('Locked, timeout?')
# I don't think this would be used...
if util.td_seconds(locked_time) >= FETCH_TIMEOUT:
logging.debug('Yes, timed out')
# Force to acquire, if locked more than FETCH_TIMEOUT ago
memcache.set('qlock_' + username, util.now())
return username
else:
continue
else:
logging.debug('locked_time does not have a value, force to acquire')
memcache.set('qlock_' + username, util.now())
return username
return False
def process_auto_queue():
"""Find people need to be updated"""
last_process = memcache.get('last_process_auto_queue')
if last_process and util.td_seconds(last_process) < PROCESS_AUTO_QUEUE_INTERVAL:
return
logging.debug('Processing auto queue')
memcache.set('last_process_auto_queue', util.now())
now = util.now()
update_after = now - datetime.timedelta(seconds=user.UPDATE_INTERVAL)
# Queue those needs get updated, which have email address
#q = user.User.gql("WHERE email > '' AND last_updated < :1 AND last_updated > ''", update_after)
# TODO only few users have email in datastore, should querying for email > '' first,
# then check the last_updated
q = user.User.gql("WHERE last_updated < :1 AND last_updated > DATE(1,1,1)", update_after)
count = 0
offset = 0
while count < 50:
users = q.fetch(50, offset * 50)
if not users:
break
for u in users:
if u._need_mail:
add(u)
count += 1
if count >= 50:
break
offset += 1
# Queue those never get updated
q = user.User.gql("WHERE last_updated < DATE(1,1,1)")
for u in q.fetch(50):
add(u)
def _need_update(self):
# Need update when
# a) never updated
# b) longer than UPDATE_INTERVAL since last updated
if not self.last_updated or util.td_seconds(self.last_updated) > UPDATE_INTERVAL:
return True
return False
def lock(key, timeout=None, force=False, wait_interval=0.1):
"""Lock helper
timeout in second, None is never timeout
force
"""
logging.debug('Trying to acquire lock %s' % key)
start = util.now()
while memcache.get(key):
logging.debug('already has lock %s in memcache' % key)
# There already is 'qlock' in memcache
if timeout is not None or util.td_seconds(start) >= timeout:
if not force:
# Can't acquire lock
logging.debug('cannot acquire lock %s' % key)
return False
# Force to acquire lock
logging.debug('forcibly acquire lock %s' % key)
break
time.sleep(wait_interval)
logging.debug('acquire lock %s' % key)
memcache.set(key, util.now())
return True
def lock(key, timeout=None, force=False, wait_interval=0.1):
"""Lock helper
timeout in second, None is never timeout
force
"""
logging.debug('Trying to acquire lock %s' % key)
start = util.now()
while memcache.get(key):
logging.debug('already has lock %s in memcache' % key)
# There already is 'qlock' in memcache
if timeout is not None or util.td_seconds(start) >= timeout:
if not force:
# Can't acquire lock
logging.debug('cannot acquire lock %s' % key)
return False
# Force to acquire lock
logging.debug('forcibly acquire lock %s' % key)
break
time.sleep(wait_interval)
logging.debug('acquire lock %s' % key)
memcache.set(key, util.now());
return True
def process_auto_queue():
"""Find people need to be updated"""
last_process = memcache.get('last_process_auto_queue')
if last_process and util.td_seconds(
last_process) < PROCESS_AUTO_QUEUE_INTERVAL:
return
logging.debug('Processing auto queue')
memcache.set('last_process_auto_queue', util.now())
now = util.now()
update_after = now - datetime.timedelta(seconds=user.UPDATE_INTERVAL)
# Queue those needs get updated, which have email address
#q = user.User.gql("WHERE email > '' AND last_updated < :1 AND last_updated > ''", update_after)
# TODO only few users have email in datastore, should querying for email > '' first,
# then check the last_updated
q = user.User.gql("WHERE last_updated < :1 AND last_updated > DATE(1,1,1)",
update_after)
count = 0
offset = 0
while count < 50:
users = q.fetch(50, offset * 50)
if not users:
break
for u in users:
if u._need_mail:
add(u)
count += 1
if count >= 50:
break
offset += 1
# Queue those never get updated
q = user.User.gql("WHERE last_updated < DATE(1,1,1)")
for u in q.fetch(50):
add(u)
def try_mail(u):
if u._need_update or not u._need_mail:
return
# FIXME don't hard-coded the sender
# Filter tweets
tweets = []
if u.tweets:
for tweet in u._tweets_:
if tweet['published'] and util.td_seconds(tweet['published']) < MAIL_INTERVAL:
continue
tweets.append(tweet)
if len(tweets):
template_values = {
'username': u.username,
'last_updated': u.last_updated,
'tweets': tweets,
}
path = os.path.join(os.path.dirname(__file__), 'mail.txt')
body = template.render(path, template_values)
if 'appspot.com' in os.environ.get('SERVER_NAME', ''):
sender = "*****@*****.**"
to = "%s <%s>" % (u.username, u.email)
else:
sender = "*****@*****.**"
to = "%s" % u.email
mail.send_mail(
subject="Last Tweets Subscription",
sender=sender,
to=to,
body=body)
# Update mailed date
db.run_in_transaction(transaction_mailed, u.username)
def process_queue():
"""Process a bit"""
# Check if it's time to process
last_process = memcache.get('last_process_queue')
if last_process and util.td_seconds(last_process) < PROCESS_QUEUE_INTERVAL:
return
memcache.set('last_process_queue', util.now())
username = lock_one_username()
if not username:
logging.debug('No item in queue, skipped')
return
curr = memcache.get('q_' + username)
if curr is None:
u = user.get(username)
if not u:
logging.debug('No such user %s in db' % username)
remove(username)
return
# Retrieve the friends list
friends = u._friends
# TODO Should drop protected friends?
curr = (u.username, friends, [])
memcache.set('q_' + username, curr)
# Start to process a bit
# In case this user do have friends
if curr[1]:
curr_f = curr[1].popitem()
client = twitter_client.service.TwitterService(
application_name='LasTweet/0')
gdata.alt.appengine.run_on_appengine(client)
search = client.NewSearch()
search.keywords = ['from:' + curr[0], 'to:' + curr_f[0]]
search.rpp = 1
new_tweet = {
'username': curr_f[0],
'msg': '',
'msg_id': 0,
'published': None,
'profile_image': curr_f[1],
'maggie': None,
}
result = search.Search()
if len(result.entry) == 1:
entry = result.entry[0]
# Process the message
# Get the unicode string
msg = entry.title.text.decode('utf-8')
# Remove the @reply
msg = message_body_pattern.match(msg).group(1)
# Truncate
if len(msg) > 50:
msg = msg[:47] + '...'
else:
msg = msg[:50]
new_tweet['msg'] = msg
new_tweet['msg_id'] = int(entry.GetMessageID())
new_tweet['published'] = entry.published.Get()
# Search for maggie ads
search.keywords = ['from:' + curr_f[0], '#magpie']
search.rpp = 10
result = search.Search()
for entry in result.entry:
# If #magpie at beginning, then this is a possiblity; and a link as well
if entry.title.text.find(
'#magpie') == 0 and entry.title.text.find('http://') >= 0:
msg = entry.title.text.decode('utf-8')
if len(msg) > 50:
msg = msg[:47] + '...'
else:
msg = msg[:50]
new_tweet['magpie'] = {
'msg': msg,
'msg_id': int(entry.GetMessageID()),
'published': entry.published.Get(),
}
# Only store the last
break
curr[2].append(new_tweet)
# If there is no more in curr[1]
if not curr[1]:
u = db.run_in_transaction(user.transaction_update_tweets, curr[0],
sort_messages(curr[2]))
user.try_mail(u)
# End of updating for this user
remove(u)
else:
memcache.set('q_' + username, curr)
unlock('qlock_' + username)
def process_queue():
"""Process a bit"""
# Check if it's time to process
last_process = memcache.get('last_process_queue')
if last_process and util.td_seconds(last_process) < PROCESS_QUEUE_INTERVAL:
return
memcache.set('last_process_queue', util.now())
username = lock_one_username()
if not username:
logging.debug('No item in queue, skipped')
return
curr = memcache.get('q_' + username)
if curr is None:
u = user.get(username)
if not u:
logging.debug('No such user %s in db' % username)
remove(username)
return
# Retrieve the friends list
friends = u._friends
# TODO Should drop protected friends?
curr = (u.username, friends, [])
memcache.set('q_' + username, curr)
# Start to process a bit
# In case this user do have friends
if curr[1]:
curr_f = curr[1].popitem()
client = twitter_client.service.TwitterService(application_name='LasTweet/0')
gdata.alt.appengine.run_on_appengine(client)
search = client.NewSearch()
search.keywords = ['from:' + curr[0], 'to:' + curr_f[0]]
search.rpp = 1
new_tweet = {
'username': curr_f[0],
'msg': '',
'msg_id': 0,
'published': None,
'profile_image': curr_f[1],
'maggie': None,
}
result = search.Search()
if len(result.entry) == 1:
entry = result.entry[0]
# Process the message
# Get the unicode string
msg = entry.title.text.decode('utf-8')
# Remove the @reply
msg = message_body_pattern.match(msg).group(1)
# Truncate
if len(msg) > 50:
msg = msg[:47] + '...'
else:
msg = msg[:50]
new_tweet['msg'] = msg
new_tweet['msg_id'] = int(entry.GetMessageID())
new_tweet['published'] = entry.published.Get()
# Search for maggie ads
search.keywords = ['from:' + curr_f[0], '#magpie']
search.rpp = 10
result = search.Search()
for entry in result.entry:
# If #magpie at beginning, then this is a possiblity; and a link as well
if entry.title.text.find('#magpie') == 0 and entry.title.text.find('http://') >= 0:
msg = entry.title.text.decode('utf-8')
if len(msg) > 50:
msg = msg[:47] + '...'
else:
msg = msg[:50]
new_tweet['magpie'] = {
'msg': msg,
'msg_id': int(entry.GetMessageID()),
'published': entry.published.Get(),
}
# Only store the last
break
curr[2].append(new_tweet)
# If there is no more in curr[1]
if not curr[1]:
u = db.run_in_transaction(user.transaction_update_tweets, curr[0],
sort_messages(curr[2]))
user.try_mail(u)
# End of updating for this user
remove(u)
else:
memcache.set('q_' + username, curr)
unlock('qlock_' + username)
def get(self, username):
logging.debug('%s asked' % username)
# Check if this username in db
u = user.get(username)
if u is not None:
logging.debug('%s retrieved from db' % u.username)
if u._need_update:
# More than 24 hours or haven't updated
queue.add(u)
template_values = {
'username': u.username,
'profile_image': u.profile_image,
'last_updated': u.last_updated,
'messages': [],
}
pos = u._queued
if pos:
template_values['messages'].append(['message', 'This user is in queue #%d' % pos])
if u.tweets:
template_values['tweets'] = u._tweets_
else:
# the weets has been updated within 24 hours
template_values = {
'username': u.username,
'profile_image': u.profile_image,
'email': u.email,
'last_mailed': u.last_mailed,
'last_updated': u.last_updated,
'messages': [],
'tweets': u._tweets_,
}
else:
# This username isn't in db, trying to add
u = user.add(username)
if isinstance(u, user.User):
# Show page
template_values = {
'username': u.username,
'profile_image': u.profile_image,
'email': u.email,
'last_mailed': u.last_mailed,
'messages': [['message', 'Put in queue #%d' % u._queued]],
}
elif u == 403:
# Reject protected twitter user, can retrieve correct screen name and image from
# friends list, but no need to waste a request to Twitter
template_values = {
'username': username,
'profile_image': 'http://static.twitter.com/images/default_profile_normal.png',
'messages': [['error', "This Twitter's tweets are protected."]],
}
elif u == 404:
template_values = {
'username': username,
'profile_image': 'http://static.twitter.com/images/default_profile_normal.png',
'messages': [['error', 'No such Twitter user']],
}
else:
# Unknown error
# FIXME use a real error page with status code for errors
template_values = {
'username': '******',
'profile_image': 'http://static.twitter.com/images/default_profile_normal.png',
'messages': [['error', 'Twitter responses with %d' % u]],
}
# Check if pinging system offline
last_process = memcache.get('last_process_queue')
if not last_process or util.td_seconds(last_process) > 600:
template_values['messages'].append(['message', 'Pinging system is temporarily offine, you request will be processed in a few hours.'])
path = os.path.join(os.path.dirname(__file__), 'template/user.html')
self.response.out.write(template.render(path, template_values))
def _need_mail(self):
if not self.email:
return False
if not self.last_mailed or util.td_seconds(self.last_mailed) > MAIL_INTERVAL:
return True
return False
def get(self, username):
logging.debug('%s asked' % username)
# Check if this username in db
u = user.get(username)
if u is not None:
logging.debug('%s retrieved from db' % u.username)
if u._need_update:
# More than 24 hours or haven't updated
queue.add(u)
template_values = {
'username': u.username,
'profile_image': u.profile_image,
'last_updated': u.last_updated,
'messages': [],
}
pos = u._queued
if pos:
template_values['messages'].append(
['message',
'This user is in queue #%d' % pos])
if u.tweets:
template_values['tweets'] = u._tweets_
else:
# the weets has been updated within 24 hours
template_values = {
'username': u.username,
'profile_image': u.profile_image,
'email': u.email,
'last_mailed': u.last_mailed,
'last_updated': u.last_updated,
'messages': [],
'tweets': u._tweets_,
}
else:
# This username isn't in db, trying to add
u = user.add(username)
if isinstance(u, user.User):
# Show page
template_values = {
'username': u.username,
'profile_image': u.profile_image,
'email': u.email,
'last_mailed': u.last_mailed,
'messages': [['message',
'Put in queue #%d' % u._queued]],
}
elif u == 403:
# Reject protected twitter user, can retrieve correct screen name and image from
# friends list, but no need to waste a request to Twitter
template_values = {
'username':
username,
'profile_image':
'http://static.twitter.com/images/default_profile_normal.png',
'messages':
[['error', "This Twitter's tweets are protected."]],
}
elif u == 404:
template_values = {
'username': username,
'profile_image':
'http://static.twitter.com/images/default_profile_normal.png',
'messages': [['error', 'No such Twitter user']],
}
else:
# Unknown error
# FIXME use a real error page with status code for errors
template_values = {
'username': '******',
'profile_image':
'http://static.twitter.com/images/default_profile_normal.png',
'messages': [['error',
'Twitter responses with %d' % u]],
}
# Check if pinging system offline
last_process = memcache.get('last_process_queue')
if not last_process or util.td_seconds(last_process) > 600:
template_values['messages'].append([
'message',
'Pinging system is temporarily offine, you request will be processed in a few hours.'
])
path = os.path.join(os.path.dirname(__file__), 'template/user.html')
self.response.out.write(template.render(path, template_values))