def parse_alltheragefaces(url=None):
"""
Parse all png rage faces from alltheragefaces.com and store its to the
redis list.
"""
url = url or app.config['SITE_URL']
response = get(url)
if response is None:
return
images = []
soup = BeautifulSoup(response.content)
faces = soup.findAll('div', attrs={'class': 'face'})
finder = lambda tag: tag.name == 'a' and \
dict(tag.attrs).get('href', '').endswith('.png')
for face in faces:
try:
download = face.findAll('div', attrs={'class': 'download'})[0]
except IndexError:
continue
try:
image = download.findAll(finder)[0]
except IndexError:
continue
href = dict(image.attrs)['href']
images.append(build_url(url, href))
if not images:
return
key = app.config['DATABASE_KEY']
exists_key = app.config['DATABASE_EXISTS_KEY']
redis.setnx(exists_key, 1)
for image in images:
redis.lpush(key, image)
try:
pagination = soup.findAll('div', attrs={'class': 'pagination'})[0]
except IndexError:
return
span = pagination.find('span')
try:
attrs = span.next.next.attrs
except AttributeError:
redis.delete(exists_key)
redis.expire(key, app.config['EXPIRE_TIME'])
return
return parse_alltheragefaces(build_url(url, dict(attrs)['href']))
def tearDown(self):
redis.delete(app.config['DATABASE_KEY'])
redis.delete(app.config['DATABASE_EXISTS_KEY'])
app.config['DATABASE_KEY'] = self.old_DATABASE_KEY
app.config['DATABASE_EXISTS_KEY'] = self.old_DATABASE_EXISTS_KEY
