def post(self, slug):
random_key = self.request.get('key')
pipe_web_address = self.request.get('pipe_web_address')
logging.debug('start add feed for %s', pipe_web_address)
entries_list = memcache.get(random_key)
if not entries_list:
return
entries = entries_list['entries']
fetch_index = entries_list['fetch_index']
put_index = entries_list['put_index']
logging.debug('start add %d entries, fetch_index is %d, put_index is %d.', len(entries), fetch_index, put_index)
while fetch_index < len(entries):
try:
entries[fetch_index].content = fetchpage.fetch_page(entries[fetch_index].link, '<!content_tag txt>', '<!/content_tag txt>')
#logging.debug('the link is %s, the key_name is %s, the fetch_index= is %d', entries[fetch_index].link, entries[fetch_index].key().id_or_name(), fetch_index)
fetch_index += 1
except runtime.DeadlineExceededError, e:
logging.exception(e)
logging.debug('we have fetched %d contents', fetch_index - entries_list['fetch_index'])
entries_list['fetch_index'] = fetch_index
entries_list['entries'] = entries
memcache.set(random_key, entries_list, 120)
add_feed_queue.add(taskqueue.Task(url="/addFeed", params={'key': random_key, 'pipe_web_address': pipe_web_address}))
return
except Exception, e :
logging.exception(e)
logging.debug('fetch Exception entries[fetch_index].link is %s, current fetch_index is %d', entries[fetch_index].link, fetch_index)
#skip it
fetch_index += 1
def post(self, slug):
random_key = self.request.get('key')
pipe_web_address = self.request.get('pipe_web_address')
logging.debug('start add feed for %s', pipe_web_address)
entries_list = memcache.get(random_key)
if not entries_list:
return
entries = entries_list['entries']
fetch_index = entries_list['fetch_index']
put_index = entries_list['put_index']
logging.debug(
'start add %d entries, fetch_index is %d, put_index is %d.',
len(entries), fetch_index, put_index)
while fetch_index < len(entries):
try:
entries[fetch_index].content = fetchpage.fetch_page(
entries[fetch_index].link, '<!content_tag txt>',
'<!/content_tag txt>')
#logging.debug('the link is %s, the key_name is %s, the fetch_index= is %d', entries[fetch_index].link, entries[fetch_index].key().id_or_name(), fetch_index)
fetch_index += 1
except runtime.DeadlineExceededError, e:
logging.exception(e)
logging.debug('we have fetched %d contents',
fetch_index - entries_list['fetch_index'])
entries_list['fetch_index'] = fetch_index
entries_list['entries'] = entries
memcache.set(random_key, entries_list, 120)
add_feed_queue.add(
taskqueue.Task(url="/addFeed",
params={
'key': random_key,
'pipe_web_address': pipe_web_address
}))
return
except Exception, e:
logging.exception(e)
logging.debug(
'fetch Exception entries[fetch_index].link is %s, current fetch_index is %d',
entries[fetch_index].link, fetch_index)
#skip it
fetch_index += 1
"WHERE pipe = :1 AND updated_time >=:2 ORDER BY updated_time DESC",
pipe, oldest_update_time).fetch(200)
logging.debug('query finished. get %d entries', len(db_entries))
pass
elif type == 'random':
logging.debug('the str is %s', get_random_str())
elif type == 'getentrycontent':
#pipe = get_pipe(slug)
key_name = self.request.get('name')
if key_name:
feed = model.FeedEntry.get_by_key_name(key_name)
self.response.out.write(feed.content)
elif type == 'fetchpage':
link = self.request.get('link')
logging.debug('link is %s', link)
content = fetchpage.fetch_page(link, '<!content_tag txt>',
'<!/content_tag txt>')
self.response.out.write(content)
elif type == 'fetchandput':
link = self.request.get('link')
logging.debug('link is %s', link)
if link:
content = fetchpage.fetch_page(link, '<!content_tag txt>',
'<!/content_tag txt>')
self.response.out.write(content)
db_entries = model.FeedEntry.gql('WHERE link = :1',
link).fetch(20)
for e in db_entries:
e.content = content
db.put(db_entries)
pass
logging.debug('the oldest_update_time is %s', str(oldest_update_time))
db_entries = model.FeedEntry.gql("WHERE pipe = :1 AND updated_time >=:2 ORDER BY updated_time DESC", pipe, oldest_update_time).fetch(200)
logging.debug('query finished. get %d entries', len(db_entries))
pass
elif type == 'random':
logging.debug('the str is %s' , get_random_str())
elif type == 'getentrycontent':
#pipe = get_pipe(slug)
key_name = self.request.get('name')
if key_name:
feed = model.FeedEntry.get_by_key_name(key_name)
self.response.out.write(feed.content)
elif type == 'fetchpage':
link = self.request.get('link')
logging.debug('link is %s', link)
content = fetchpage.fetch_page(link, '<!content_tag txt>', '<!/content_tag txt>')
self.response.out.write(content)
elif type == 'fetchandput':
link = self.request.get('link')
logging.debug('link is %s', link)
if link:
content = fetchpage.fetch_page(link, '<!content_tag txt>', '<!/content_tag txt>')
self.response.out.write(content)
db_entries = model.FeedEntry.gql('WHERE link = :1', link).fetch(20)
for e in db_entries:
e.content = content
db.put(db_entries)
pass
def get_xml(self, pipe):
feed_url = pipe.feed_url
feed_xml = zrqutil.fetch_content(feed_url)
