def get(self, profile_name): '''Processes the request. We don't use VKontakte API because it requires authorization and gives tokens with expiration time which is not suitable for RSS generator. ''' headers = self.__get_headers() user_agent = headers.get('user-agent', '').strip() if user_agent and ( # Google Reader bot still crawls the Web. Reject it to save # bandwidth. user_agent.startswith('Feedfetcher-Google;') or # FeedNotifier updates feeds every minute user_agent.startswith('FeedNotifier/') or # YandexBlogs bot sends a lot of requests (2/minute) for some # feeds. The support doesn't respond adequately. 'YandexBlogs' in user_agent): self.error(httplib.FORBIDDEN) return user_error = None http_status = httplib.OK unknown_user_error = False try: show_photo = (self.request.get('show_photo', '1') != '0') foreign_posts = (self.request.get('foreign_posts', '0') != '0') hash_tag_title = (self.request.get('hash_tag_title', '0') != '0') text_title = (self.request.get('text_title', '0') != '0') big_photos = (self.request.get('big_photos', '0') != '0') LOG.info( 'Requested feed for "%s" (foreign_posts = %s, show_photo = %s, hash_tag_title = %s, text_title = %s, big_photos = %s).', profile_name, foreign_posts, show_photo, hash_tag_title, text_title, big_photos) use_api = True if_modified_since = None if use_api: # Use VKontakte API from vkfeed.tools import wall_reader cur_time = int(time.time()) latency = constants.MINUTE_SECONDS min_timestamp = cur_time - constants.WEEK_SECONDS ## This confuses Google Reader users because it always requests ## feeds with 'Cache-Control: max-age=3600' when adding ## subscriptions and users often gen an empty feed. #for cache_control in headers.get('cache-control', '').split(','): # cache_control = cache_control.strip() # if cache_control.startswith('max-age='): # LOG.info('Applying Cache-Control: %s...', cache_control) # try: # cache_max_age = int(cache_control[len('max-age='):]) # except ValueError: # LOG.error('Invalid header: Cache-Control = %s.', cache_control) # else: # if cache_max_age: # min_timestamp = max(min_timestamp, cur_time - cache_max_age - latency) if 'if-modified-since' in headers and headers[ 'if-modified-since'] != '0': LOG.info('Applying If-Modified-Since: %s...', headers['if-modified-since']) try: if_modified_since = vkfeed.utils.http_timestamp( headers['if-modified-since']) except Exception as e: LOG.error('Invalid header: If-Modified-Since = %s.', headers['if-modified-since']) else: min_timestamp = max(min_timestamp, if_modified_since - latency) max_age = cur_time - min_timestamp if max_age > constants.DAY_SECONDS: max_posts_num = 10 else: max_posts_num = 50 if user_agent and vkfeed.utils.zero_subscribers(user_agent): max_posts_num /= 2 LOG.info( 'Applying the following limits: max_age=%s, max_posts_num=%s', max_age, max_posts_num) try: data = wall_reader.read(profile_name, min_timestamp, max_posts_num, foreign_posts, show_photo, hash_tag_title, text_title, big_photos) except wall_reader.ConnectionError as e: http_status = httplib.BAD_GATEWAY user_error = 'Ошибка соединения с сервером <a href="{0}" target="_blank">{0}</a>.'.format( constants.API_URL) raise except wall_reader.ServerError as e: http_status = httplib.NOT_FOUND user_error = unicode(e) raise else: # Parse HTML from site from vkfeed.tools.wall_parser import WallPageParser, ParseError, PrivateGroupError, ProfileNotAvailableError, ServerError url = constants.VK_URL + cgi.escape(profile_name) url_html = '<a href="{0}" target="_blank">{0}</a>'.format(url) if profile_name == 'feed': http_status = httplib.NOT_FOUND user_error = 'Страница {0} не является профилем пользователя или группы.'.format( url_html) raise Error('Unsupported page.') try: profile_page = vkfeed.utils.fetch_url(url) except vkfeed.utils.HTTPNotFoundError: http_status = httplib.NOT_FOUND user_error = 'Пользователя или группы {0} не существует.'.format( url_html) raise except Error: http_status = httplib.BAD_GATEWAY user_error = 'Не удалось загрузить страницу {0}.'.format( url_html) unknown_user_error = True raise try: data = WallPageParser().parse(profile_page) except PrivateGroupError as e: http_status = httplib.NOT_FOUND user_error = 'Группа {0} является закрытой группой.'.format( url_html) raise except ProfileNotAvailableError as e: http_status = httplib.NOT_FOUND user_error = 'Страница пользователя {0} удалена или доступна только авторизованным пользователям.'.format( url_html) raise except ServerError as e: LOG.debug('Page contents:\n%s', profile_page) http_status = httplib.BAD_GATEWAY user_error = 'Сервер {0} вернул ошибку{1}'.format( url_html, ':<br />' + e.server_error if e.server_error else '.') unknown_user_error = True raise except ParseError as e: LOG.debug('Page contents:\n%s', profile_page) http_status = httplib.NOT_FOUND user_error = 'Сервер вернул страницу, на которой не удалось найти стену с сообщениями пользователя.' unknown_user_error = True raise data['url'] = url if 'user_photo' not in data: data[ 'user_photo'] = constants.APP_URL + 'images/vk-rss-logo.png' LOG.info('Return %s items.', len(data['posts'])) if if_modified_since is not None and not data['posts']: http_status = httplib.NOT_MODIFIED else: feed = self.__generate_feed(data) except Exception as e: if isinstance(e, Error): if user_error and not unknown_user_error: log_function = LOG.warning else: log_function = LOG.error else: log_function = LOG.exception log_function('Unable to generate a feed for "%s": %s', profile_name, e) if user_error: self.error(http_status) error = '<p>Ошибка при генерации RSS-ленты:</p><p>{0}</p>'.format( user_error) if unknown_user_error: error += '''<p> Пожалуйста, убедитесь, что вы правильно указали профиль пользователя или группы, и что данный профиль является общедоступным. Если все указано верно, и ошибка повторяется, пожалуйста, свяжитесь с <a href="mailto:{0}">администратором</a>. </p>'''.format( cgi.escape(constants.ADMIN_EMAIL, quote=True)) else: self.error(httplib.INTERNAL_SERVER_ERROR) error = ''' При генерации RSS-ленты произошла внутренняя ошибка сервера. Если ошибка повторяется, пожалуйста, свяжитесь с <a href="mailto:{0}">администратором</a>. '''.format(cgi.escape(constants.ADMIN_EMAIL, quote=True)) self.response.headers[ b'Content-Type'] = b'text/html; charset=utf-8' self.response.out.write( vkfeed.utils.render_template('error.html', {'error': error})) else: if http_status == httplib.OK: self.response.headers[b'Content-Type'] = b'application/rss+xml' self.response.out.write(feed) else: self.error(http_status)
def setUp(self): self.__parser = WallPageParser(ignore_errors = False)
def setUp(self): self.__parser = WallPageParser(ignore_errors = False)
class TestWallParser(unittest.TestCase): '''Tests vk.com wall parser.''' def setUp(self): self.__parser = WallPageParser(ignore_errors = False) def test_invalid_page(self): '''Testing parsing of invalid page.''' self.assertRaises(ParseError, lambda: self.__parser.parse(open('wall_parser/invalid_page.html').read().decode('cp1251'))) def test_group_wall(self): '''Testing parsing of group wall''' self.__test_parsing( open('wall_parser/group_profile_page.html').read().decode('cp1251'), { 'user_name': u'Хабрахабр', 'user_photo': 'http://cs11159.vk.com/g20629724/a_ba3bb3dc.jpg', 'posts': 10, } ) def test_user_empty_wall(self): '''Testing parsing of empty wall''' self.__test_parsing( open('wall_parser/user_profile_page_with_empty_wall.html').read().decode('cp1251'), { 'user_name': u'Григорий Бакунов', 'user_photo': 'http://cs4383.vk.com/u78983895/a_912f563f.jpg', 'posts': 0, } ) def test_user_wall(self): '''Testing parsing of user wall''' self.__test_parsing( open('wall_parser/user_profile_page.html').read().decode('cp1251'), { 'user_name': u'Павел Дуров', 'user_photo': 'http://vk.com/u00001/a_a964f9a2.jpg', 'posts': 10, } ) def __test_parsing(self, html, etalon): '''Runs the test with the specified data.''' clear_run_data = self.__normalize_data(self.__parser.parse(html)) self.assertEqual(etalon, clear_run_data) dirty_run_data = self.__normalize_data(self.__parser.parse(html)) self.assertEqual(etalon, clear_run_data) self.assertEqual(etalon, dirty_run_data) def __normalize_data(self, data): '''Removes fields that don't exist in the etalon.''' if 'posts' in data: for post in data['posts']: self.assertNotEqual(post['text'].strip(), '') data['posts'] = len(data['posts']) return data
class TestWallParser(unittest.TestCase): '''Tests vk.com wall parser.''' def setUp(self): self.__parser = WallPageParser(ignore_errors = False) def test_invalid_page(self): '''Testing parsing of invalid page.''' self.assertRaises(ParseError, lambda: self.__parser.parse(open('wall_parser/invalid_page.html').read().decode('cp1251'))) def test_group_wall(self): '''Testing parsing of group wall''' self.__test_parsing( open('wall_parser/group_profile_page.html').read().decode('cp1251'), { 'user_name': 'Хабрахабр', 'user_photo': 'http://cs11159.vk.com/g20629724/a_ba3bb3dc.jpg', 'posts': 10, } ) def test_user_empty_wall(self): '''Testing parsing of empty wall''' self.__test_parsing( open('wall_parser/user_profile_page_with_empty_wall.html').read().decode('cp1251'), { 'user_name': 'Григорий Бакунов', 'user_photo': 'http://cs4383.vk.com/u78983895/a_912f563f.jpg', 'posts': 0, } ) def test_user_wall(self): '''Testing parsing of user wall''' self.__test_parsing( open('wall_parser/user_profile_page.html').read().decode('cp1251'), { 'user_name': 'Павел Дуров', 'user_photo': 'http://vk.com/u00001/a_a964f9a2.jpg', 'posts': 10, } ) def __test_parsing(self, html, etalon): '''Runs the test with the specified data.''' clear_run_data = self.__normalize_data(self.__parser.parse(html)) self.assertEqual(etalon, clear_run_data) dirty_run_data = self.__normalize_data(self.__parser.parse(html)) self.assertEqual(etalon, clear_run_data) self.assertEqual(etalon, dirty_run_data) def __normalize_data(self, data): '''Removes fields that don't exist in the etalon.''' if 'posts' in data: for post in data['posts']: self.assertNotEqual(post['text'].strip(), '') data['posts'] = len(data['posts']) return data