def post(self, slug): random_key = self.request.get('key') pipe_web_address = self.request.get('pipe_web_address') logging.debug('start add feed for %s', pipe_web_address) entries_list = memcache.get(random_key) if not entries_list: return entries = entries_list['entries'] fetch_index = entries_list['fetch_index'] put_index = entries_list['put_index'] logging.debug('start add %d entries, fetch_index is %d, put_index is %d.', len(entries), fetch_index, put_index) while fetch_index < len(entries): try: entries[fetch_index].content = fetchpage.fetch_page(entries[fetch_index].link, '<!content_tag txt>', '<!/content_tag txt>') #logging.debug('the link is %s, the key_name is %s, the fetch_index= is %d', entries[fetch_index].link, entries[fetch_index].key().id_or_name(), fetch_index) fetch_index += 1 except runtime.DeadlineExceededError, e: logging.exception(e) logging.debug('we have fetched %d contents', fetch_index - entries_list['fetch_index']) entries_list['fetch_index'] = fetch_index entries_list['entries'] = entries memcache.set(random_key, entries_list, 120) add_feed_queue.add(taskqueue.Task(url="/addFeed", params={'key': random_key, 'pipe_web_address': pipe_web_address})) return except Exception, e : logging.exception(e) logging.debug('fetch Exception entries[fetch_index].link is %s, current fetch_index is %d', entries[fetch_index].link, fetch_index) #skip it fetch_index += 1
def post(self, slug): random_key = self.request.get('key') pipe_web_address = self.request.get('pipe_web_address') logging.debug('start add feed for %s', pipe_web_address) entries_list = memcache.get(random_key) if not entries_list: return entries = entries_list['entries'] fetch_index = entries_list['fetch_index'] put_index = entries_list['put_index'] logging.debug( 'start add %d entries, fetch_index is %d, put_index is %d.', len(entries), fetch_index, put_index) while fetch_index < len(entries): try: entries[fetch_index].content = fetchpage.fetch_page( entries[fetch_index].link, '<!content_tag txt>', '<!/content_tag txt>') #logging.debug('the link is %s, the key_name is %s, the fetch_index= is %d', entries[fetch_index].link, entries[fetch_index].key().id_or_name(), fetch_index) fetch_index += 1 except runtime.DeadlineExceededError, e: logging.exception(e) logging.debug('we have fetched %d contents', fetch_index - entries_list['fetch_index']) entries_list['fetch_index'] = fetch_index entries_list['entries'] = entries memcache.set(random_key, entries_list, 120) add_feed_queue.add( taskqueue.Task(url="/addFeed", params={ 'key': random_key, 'pipe_web_address': pipe_web_address })) return except Exception, e: logging.exception(e) logging.debug( 'fetch Exception entries[fetch_index].link is %s, current fetch_index is %d', entries[fetch_index].link, fetch_index) #skip it fetch_index += 1
"WHERE pipe = :1 AND updated_time >=:2 ORDER BY updated_time DESC", pipe, oldest_update_time).fetch(200) logging.debug('query finished. get %d entries', len(db_entries)) pass elif type == 'random': logging.debug('the str is %s', get_random_str()) elif type == 'getentrycontent': #pipe = get_pipe(slug) key_name = self.request.get('name') if key_name: feed = model.FeedEntry.get_by_key_name(key_name) self.response.out.write(feed.content) elif type == 'fetchpage': link = self.request.get('link') logging.debug('link is %s', link) content = fetchpage.fetch_page(link, '<!content_tag txt>', '<!/content_tag txt>') self.response.out.write(content) elif type == 'fetchandput': link = self.request.get('link') logging.debug('link is %s', link) if link: content = fetchpage.fetch_page(link, '<!content_tag txt>', '<!/content_tag txt>') self.response.out.write(content) db_entries = model.FeedEntry.gql('WHERE link = :1', link).fetch(20) for e in db_entries: e.content = content db.put(db_entries) pass
logging.debug('the oldest_update_time is %s', str(oldest_update_time)) db_entries = model.FeedEntry.gql("WHERE pipe = :1 AND updated_time >=:2 ORDER BY updated_time DESC", pipe, oldest_update_time).fetch(200) logging.debug('query finished. get %d entries', len(db_entries)) pass elif type == 'random': logging.debug('the str is %s' , get_random_str()) elif type == 'getentrycontent': #pipe = get_pipe(slug) key_name = self.request.get('name') if key_name: feed = model.FeedEntry.get_by_key_name(key_name) self.response.out.write(feed.content) elif type == 'fetchpage': link = self.request.get('link') logging.debug('link is %s', link) content = fetchpage.fetch_page(link, '<!content_tag txt>', '<!/content_tag txt>') self.response.out.write(content) elif type == 'fetchandput': link = self.request.get('link') logging.debug('link is %s', link) if link: content = fetchpage.fetch_page(link, '<!content_tag txt>', '<!/content_tag txt>') self.response.out.write(content) db_entries = model.FeedEntry.gql('WHERE link = :1', link).fetch(20) for e in db_entries: e.content = content db.put(db_entries) pass def get_xml(self, pipe): feed_url = pipe.feed_url feed_xml = zrqutil.fetch_content(feed_url)