def parse_alltheragefaces(url=None): """ Parse all png rage faces from alltheragefaces.com and store its to the redis list. """ url = url or app.config['SITE_URL'] response = get(url) if response is None: return images = [] soup = BeautifulSoup(response.content) faces = soup.findAll('div', attrs={'class': 'face'}) finder = lambda tag: tag.name == 'a' and \ dict(tag.attrs).get('href', '').endswith('.png') for face in faces: try: download = face.findAll('div', attrs={'class': 'download'})[0] except IndexError: continue try: image = download.findAll(finder)[0] except IndexError: continue href = dict(image.attrs)['href'] images.append(build_url(url, href)) if not images: return key = app.config['DATABASE_KEY'] exists_key = app.config['DATABASE_EXISTS_KEY'] redis.setnx(exists_key, 1) for image in images: redis.lpush(key, image) try: pagination = soup.findAll('div', attrs={'class': 'pagination'})[0] except IndexError: return span = pagination.find('span') try: attrs = span.next.next.attrs except AttributeError: redis.delete(exists_key) redis.expire(key, app.config['EXPIRE_TIME']) return return parse_alltheragefaces(build_url(url, dict(attrs)['href']))
def tearDown(self): redis.delete(app.config['DATABASE_KEY']) redis.delete(app.config['DATABASE_EXISTS_KEY']) app.config['DATABASE_KEY'] = self.old_DATABASE_KEY app.config['DATABASE_EXISTS_KEY'] = self.old_DATABASE_EXISTS_KEY