async def start_scraper():
tags = await db.execute(
Hashtag.select()
)
logging.info("[ENV] CONCURRENCY: %s, PROXY: %s",
os.getenv("CONCURRENCY"), os.getenv("PROXY"))
scraper = Scraper(
tags,
concurrency=int(os.getenv("CONCURRENCY", 50)),
proxy=os.getenv("PROXY")
)
asyncio.ensure_future(scraper.parse_all_tags())
await updater.update_statistics(db, scraper.result_queue)
async def get(self):
n = self.request.match_info.get('n')
if not n.isdigit():
return json_response({
'success': False,
'errors': 'n must be int'
},
status=400)
n = int(n)
db = self.request.app['objects']
hashtags = await db.execute(Hashtag.select().order_by(
Hashtag.total_posts.desc()).limit(n))
data, _ = HashtagSerializer().dump(hashtags, many=True)
return json_response(data)
def hashtags(self):
from models.hashtags import Hashtag
tags = [x.tag for x in Hashtag.select().where(Hashtag.photo == self)]
return list(map(lambda x: '#' + x, tags))