def process_mail(): """Send some mails""" last_process = memcache.get('last_process_mail') if last_process and util.td_seconds(last_process) < PROCESS_MAIL_INTERVAL: return logging.debug('Processing mail') memcache.set('last_process_mail', util.now()) now = util.now() mail_before = now - datetime.timedelta(seconds=user.MAIL_INTERVAL) update_after = now - datetime.timedelta(seconds=user.UPDATE_INTERVAL) # Queue those needs get updated, which have email address #q = user.User.gql("WHERE email > '' AND last_updated >= :1 AND last_mailed < :2", update_after, mail_before) # TODO only few users have email in datastore, should querying for email > '' first, # then check the last_mailed and the last_updated q = user.User.gql("WHERE last_mailed < :1", mail_before) count = 0 offset = 0 while count < MAILS_PER_PROCESS: users = q.fetch(50, offset * 50) if not users: break for u in users: if u.email and u.last_updated and u.last_updated >= update_after: user.try_mail(u) count += 1 if count >= MAILS_PER_PROCESS: break offset += 1
def lock_one_username(): q = memcache.get('q') or [] for username in q: logging.debug('Retrieving %s from db' % username) u = user.get(username) if not u._need_update: remove(u) continue logging.debug('Checking lock') locked_time = memcache.get('qlock_' + username) logging.debug('locked_time: %s' % locked_time) if locked_time: logging.debug('Locked, timeout?') # I don't think this would be used... if util.td_seconds(locked_time) >= FETCH_TIMEOUT: logging.debug('Yes, timed out') # Force to acquire, if locked more than FETCH_TIMEOUT ago memcache.set('qlock_' + username, util.now()) return username else: continue else: logging.debug( 'locked_time does not have a value, force to acquire') memcache.set('qlock_' + username, util.now()) return username return False
def lock_one_username(): q = memcache.get('q') or [] for username in q: logging.debug('Retrieving %s from db' % username) u = user.get(username) if not u._need_update: remove(u) continue logging.debug('Checking lock') locked_time = memcache.get('qlock_' + username) logging.debug('locked_time: %s' % locked_time) if locked_time: logging.debug('Locked, timeout?') # I don't think this would be used... if util.td_seconds(locked_time) >= FETCH_TIMEOUT: logging.debug('Yes, timed out') # Force to acquire, if locked more than FETCH_TIMEOUT ago memcache.set('qlock_' + username, util.now()) return username else: continue else: logging.debug('locked_time does not have a value, force to acquire') memcache.set('qlock_' + username, util.now()) return username return False
def process_auto_queue(): """Find people need to be updated""" last_process = memcache.get('last_process_auto_queue') if last_process and util.td_seconds(last_process) < PROCESS_AUTO_QUEUE_INTERVAL: return logging.debug('Processing auto queue') memcache.set('last_process_auto_queue', util.now()) now = util.now() update_after = now - datetime.timedelta(seconds=user.UPDATE_INTERVAL) # Queue those needs get updated, which have email address #q = user.User.gql("WHERE email > '' AND last_updated < :1 AND last_updated > ''", update_after) # TODO only few users have email in datastore, should querying for email > '' first, # then check the last_updated q = user.User.gql("WHERE last_updated < :1 AND last_updated > DATE(1,1,1)", update_after) count = 0 offset = 0 while count < 50: users = q.fetch(50, offset * 50) if not users: break for u in users: if u._need_mail: add(u) count += 1 if count >= 50: break offset += 1 # Queue those never get updated q = user.User.gql("WHERE last_updated < DATE(1,1,1)") for u in q.fetch(50): add(u)
def _need_update(self): # Need update when # a) never updated # b) longer than UPDATE_INTERVAL since last updated if not self.last_updated or util.td_seconds(self.last_updated) > UPDATE_INTERVAL: return True return False
def lock(key, timeout=None, force=False, wait_interval=0.1): """Lock helper timeout in second, None is never timeout force """ logging.debug('Trying to acquire lock %s' % key) start = util.now() while memcache.get(key): logging.debug('already has lock %s in memcache' % key) # There already is 'qlock' in memcache if timeout is not None or util.td_seconds(start) >= timeout: if not force: # Can't acquire lock logging.debug('cannot acquire lock %s' % key) return False # Force to acquire lock logging.debug('forcibly acquire lock %s' % key) break time.sleep(wait_interval) logging.debug('acquire lock %s' % key) memcache.set(key, util.now()) return True
def lock(key, timeout=None, force=False, wait_interval=0.1): """Lock helper timeout in second, None is never timeout force """ logging.debug('Trying to acquire lock %s' % key) start = util.now() while memcache.get(key): logging.debug('already has lock %s in memcache' % key) # There already is 'qlock' in memcache if timeout is not None or util.td_seconds(start) >= timeout: if not force: # Can't acquire lock logging.debug('cannot acquire lock %s' % key) return False # Force to acquire lock logging.debug('forcibly acquire lock %s' % key) break time.sleep(wait_interval) logging.debug('acquire lock %s' % key) memcache.set(key, util.now()); return True
def process_auto_queue(): """Find people need to be updated""" last_process = memcache.get('last_process_auto_queue') if last_process and util.td_seconds( last_process) < PROCESS_AUTO_QUEUE_INTERVAL: return logging.debug('Processing auto queue') memcache.set('last_process_auto_queue', util.now()) now = util.now() update_after = now - datetime.timedelta(seconds=user.UPDATE_INTERVAL) # Queue those needs get updated, which have email address #q = user.User.gql("WHERE email > '' AND last_updated < :1 AND last_updated > ''", update_after) # TODO only few users have email in datastore, should querying for email > '' first, # then check the last_updated q = user.User.gql("WHERE last_updated < :1 AND last_updated > DATE(1,1,1)", update_after) count = 0 offset = 0 while count < 50: users = q.fetch(50, offset * 50) if not users: break for u in users: if u._need_mail: add(u) count += 1 if count >= 50: break offset += 1 # Queue those never get updated q = user.User.gql("WHERE last_updated < DATE(1,1,1)") for u in q.fetch(50): add(u)
def try_mail(u): if u._need_update or not u._need_mail: return # FIXME don't hard-coded the sender # Filter tweets tweets = [] if u.tweets: for tweet in u._tweets_: if tweet['published'] and util.td_seconds(tweet['published']) < MAIL_INTERVAL: continue tweets.append(tweet) if len(tweets): template_values = { 'username': u.username, 'last_updated': u.last_updated, 'tweets': tweets, } path = os.path.join(os.path.dirname(__file__), 'mail.txt') body = template.render(path, template_values) if 'appspot.com' in os.environ.get('SERVER_NAME', ''): sender = "*****@*****.**" to = "%s <%s>" % (u.username, u.email) else: sender = "*****@*****.**" to = "%s" % u.email mail.send_mail( subject="Last Tweets Subscription", sender=sender, to=to, body=body) # Update mailed date db.run_in_transaction(transaction_mailed, u.username)
def process_queue(): """Process a bit""" # Check if it's time to process last_process = memcache.get('last_process_queue') if last_process and util.td_seconds(last_process) < PROCESS_QUEUE_INTERVAL: return memcache.set('last_process_queue', util.now()) username = lock_one_username() if not username: logging.debug('No item in queue, skipped') return curr = memcache.get('q_' + username) if curr is None: u = user.get(username) if not u: logging.debug('No such user %s in db' % username) remove(username) return # Retrieve the friends list friends = u._friends # TODO Should drop protected friends? curr = (u.username, friends, []) memcache.set('q_' + username, curr) # Start to process a bit # In case this user do have friends if curr[1]: curr_f = curr[1].popitem() client = twitter_client.service.TwitterService( application_name='LasTweet/0') gdata.alt.appengine.run_on_appengine(client) search = client.NewSearch() search.keywords = ['from:' + curr[0], 'to:' + curr_f[0]] search.rpp = 1 new_tweet = { 'username': curr_f[0], 'msg': '', 'msg_id': 0, 'published': None, 'profile_image': curr_f[1], 'maggie': None, } result = search.Search() if len(result.entry) == 1: entry = result.entry[0] # Process the message # Get the unicode string msg = entry.title.text.decode('utf-8') # Remove the @reply msg = message_body_pattern.match(msg).group(1) # Truncate if len(msg) > 50: msg = msg[:47] + '...' else: msg = msg[:50] new_tweet['msg'] = msg new_tweet['msg_id'] = int(entry.GetMessageID()) new_tweet['published'] = entry.published.Get() # Search for maggie ads search.keywords = ['from:' + curr_f[0], '#magpie'] search.rpp = 10 result = search.Search() for entry in result.entry: # If #magpie at beginning, then this is a possiblity; and a link as well if entry.title.text.find( '#magpie') == 0 and entry.title.text.find('http://') >= 0: msg = entry.title.text.decode('utf-8') if len(msg) > 50: msg = msg[:47] + '...' else: msg = msg[:50] new_tweet['magpie'] = { 'msg': msg, 'msg_id': int(entry.GetMessageID()), 'published': entry.published.Get(), } # Only store the last break curr[2].append(new_tweet) # If there is no more in curr[1] if not curr[1]: u = db.run_in_transaction(user.transaction_update_tweets, curr[0], sort_messages(curr[2])) user.try_mail(u) # End of updating for this user remove(u) else: memcache.set('q_' + username, curr) unlock('qlock_' + username)
def process_queue(): """Process a bit""" # Check if it's time to process last_process = memcache.get('last_process_queue') if last_process and util.td_seconds(last_process) < PROCESS_QUEUE_INTERVAL: return memcache.set('last_process_queue', util.now()) username = lock_one_username() if not username: logging.debug('No item in queue, skipped') return curr = memcache.get('q_' + username) if curr is None: u = user.get(username) if not u: logging.debug('No such user %s in db' % username) remove(username) return # Retrieve the friends list friends = u._friends # TODO Should drop protected friends? curr = (u.username, friends, []) memcache.set('q_' + username, curr) # Start to process a bit # In case this user do have friends if curr[1]: curr_f = curr[1].popitem() client = twitter_client.service.TwitterService(application_name='LasTweet/0') gdata.alt.appengine.run_on_appengine(client) search = client.NewSearch() search.keywords = ['from:' + curr[0], 'to:' + curr_f[0]] search.rpp = 1 new_tweet = { 'username': curr_f[0], 'msg': '', 'msg_id': 0, 'published': None, 'profile_image': curr_f[1], 'maggie': None, } result = search.Search() if len(result.entry) == 1: entry = result.entry[0] # Process the message # Get the unicode string msg = entry.title.text.decode('utf-8') # Remove the @reply msg = message_body_pattern.match(msg).group(1) # Truncate if len(msg) > 50: msg = msg[:47] + '...' else: msg = msg[:50] new_tweet['msg'] = msg new_tweet['msg_id'] = int(entry.GetMessageID()) new_tweet['published'] = entry.published.Get() # Search for maggie ads search.keywords = ['from:' + curr_f[0], '#magpie'] search.rpp = 10 result = search.Search() for entry in result.entry: # If #magpie at beginning, then this is a possiblity; and a link as well if entry.title.text.find('#magpie') == 0 and entry.title.text.find('http://') >= 0: msg = entry.title.text.decode('utf-8') if len(msg) > 50: msg = msg[:47] + '...' else: msg = msg[:50] new_tweet['magpie'] = { 'msg': msg, 'msg_id': int(entry.GetMessageID()), 'published': entry.published.Get(), } # Only store the last break curr[2].append(new_tweet) # If there is no more in curr[1] if not curr[1]: u = db.run_in_transaction(user.transaction_update_tweets, curr[0], sort_messages(curr[2])) user.try_mail(u) # End of updating for this user remove(u) else: memcache.set('q_' + username, curr) unlock('qlock_' + username)
def get(self, username): logging.debug('%s asked' % username) # Check if this username in db u = user.get(username) if u is not None: logging.debug('%s retrieved from db' % u.username) if u._need_update: # More than 24 hours or haven't updated queue.add(u) template_values = { 'username': u.username, 'profile_image': u.profile_image, 'last_updated': u.last_updated, 'messages': [], } pos = u._queued if pos: template_values['messages'].append(['message', 'This user is in queue #%d' % pos]) if u.tweets: template_values['tweets'] = u._tweets_ else: # the weets has been updated within 24 hours template_values = { 'username': u.username, 'profile_image': u.profile_image, 'email': u.email, 'last_mailed': u.last_mailed, 'last_updated': u.last_updated, 'messages': [], 'tweets': u._tweets_, } else: # This username isn't in db, trying to add u = user.add(username) if isinstance(u, user.User): # Show page template_values = { 'username': u.username, 'profile_image': u.profile_image, 'email': u.email, 'last_mailed': u.last_mailed, 'messages': [['message', 'Put in queue #%d' % u._queued]], } elif u == 403: # Reject protected twitter user, can retrieve correct screen name and image from # friends list, but no need to waste a request to Twitter template_values = { 'username': username, 'profile_image': 'http://static.twitter.com/images/default_profile_normal.png', 'messages': [['error', "This Twitter's tweets are protected."]], } elif u == 404: template_values = { 'username': username, 'profile_image': 'http://static.twitter.com/images/default_profile_normal.png', 'messages': [['error', 'No such Twitter user']], } else: # Unknown error # FIXME use a real error page with status code for errors template_values = { 'username': '******', 'profile_image': 'http://static.twitter.com/images/default_profile_normal.png', 'messages': [['error', 'Twitter responses with %d' % u]], } # Check if pinging system offline last_process = memcache.get('last_process_queue') if not last_process or util.td_seconds(last_process) > 600: template_values['messages'].append(['message', 'Pinging system is temporarily offine, you request will be processed in a few hours.']) path = os.path.join(os.path.dirname(__file__), 'template/user.html') self.response.out.write(template.render(path, template_values))
def _need_mail(self): if not self.email: return False if not self.last_mailed or util.td_seconds(self.last_mailed) > MAIL_INTERVAL: return True return False
def get(self, username): logging.debug('%s asked' % username) # Check if this username in db u = user.get(username) if u is not None: logging.debug('%s retrieved from db' % u.username) if u._need_update: # More than 24 hours or haven't updated queue.add(u) template_values = { 'username': u.username, 'profile_image': u.profile_image, 'last_updated': u.last_updated, 'messages': [], } pos = u._queued if pos: template_values['messages'].append( ['message', 'This user is in queue #%d' % pos]) if u.tweets: template_values['tweets'] = u._tweets_ else: # the weets has been updated within 24 hours template_values = { 'username': u.username, 'profile_image': u.profile_image, 'email': u.email, 'last_mailed': u.last_mailed, 'last_updated': u.last_updated, 'messages': [], 'tweets': u._tweets_, } else: # This username isn't in db, trying to add u = user.add(username) if isinstance(u, user.User): # Show page template_values = { 'username': u.username, 'profile_image': u.profile_image, 'email': u.email, 'last_mailed': u.last_mailed, 'messages': [['message', 'Put in queue #%d' % u._queued]], } elif u == 403: # Reject protected twitter user, can retrieve correct screen name and image from # friends list, but no need to waste a request to Twitter template_values = { 'username': username, 'profile_image': 'http://static.twitter.com/images/default_profile_normal.png', 'messages': [['error', "This Twitter's tweets are protected."]], } elif u == 404: template_values = { 'username': username, 'profile_image': 'http://static.twitter.com/images/default_profile_normal.png', 'messages': [['error', 'No such Twitter user']], } else: # Unknown error # FIXME use a real error page with status code for errors template_values = { 'username': '******', 'profile_image': 'http://static.twitter.com/images/default_profile_normal.png', 'messages': [['error', 'Twitter responses with %d' % u]], } # Check if pinging system offline last_process = memcache.get('last_process_queue') if not last_process or util.td_seconds(last_process) > 600: template_values['messages'].append([ 'message', 'Pinging system is temporarily offine, you request will be processed in a few hours.' ]) path = os.path.join(os.path.dirname(__file__), 'template/user.html') self.response.out.write(template.render(path, template_values))