def run(self, screenshot_page_url): if self.user is None: logging.warning('Attempting to get screenshot for an empty user') return # First check if screenshot exists in db already parsed_url = urlparse.urlparse(screenshot_page_url) screenshot_id = urlparse.parse_qs(parsed_url.query)['id'][0] if Screenshot.all().filter('screenshot_id =', screenshot_id).get() is not None: logging.debug('Screenshot already exists in database: ' + screenshot_id) return # Then check if we got 200 from Steam page_soup = helpers.request_soup(screenshot_page_url) if page_soup is None: return logging.info('Retrieved screenshot: ' + screenshot_page_url) # TODO Smarter check for spoiler tag # # Apparently the spoiler html element is only on the desktop version # so I probably have to schedule a separate task just to search the # desktop listings for potential spoilers screenshot_src = page_soup.find('img', class_='userScreenshotImg')['src'] screenshot_desc = page_soup.find('h1', class_='captionText').string.strip() screenshot_is_spoiler = "spoiler" in screenshot_desc.lower() screenshot_is_nsfw = "nsfw" in screenshot_desc.lower() screenshot_game = page_soup.find(id='gameName').find('a', class_='itemLink').string.strip() s = Screenshot( parent = self.user, screenshot_id = screenshot_id, url = screenshot_page_url, src = screenshot_src, desc = screenshot_desc, game = screenshot_game, is_spoiler = screenshot_is_spoiler, is_nsfw = screenshot_is_nsfw ) result = s.put() if result is db.TransactionFailedError: logging.error('Failed to save screenshot for ' + self.user.steam_username + ' id:' + screenshot_id) else: logging.debug('Successfully saved screenshot for ' + self.user.steam_username + ' id:' + screenshot_id)
def post(self): self.checkChannelKey() self.parseTriggerFields() self.parseLimit() screenshots_query = Screenshot.all() screenshots_query.ancestor(self.user) screenshots_query.filter('seen_already =', False) if not self.user.steam_show_spoilers: screenshots_query.filter('is_spoiler =', False) if not self.user.steam_show_nsfw: screenshots_query.filter('is_nsfw =', False) screenshots = screenshots_query.fetch(self.limit) screenshot_data = [] for screenshot in screenshots: screenshot_data.append({ IFTTT_INGREDIENT_SCREENSHOT_PAGE_KEY: screenshot.url, IFTTT_INGREDIENT_SCREENSHOT_URL_KEY: screenshot.src, IFTTT_INGREDIENT_SCREENSHOT_CAPTION_KEY: screenshot.desc, IFTTT_INGREDIENT_SCREENSHOT_USERNAME_KEY: self.user.steam_username, IFTTT_INGREDIENT_SCREENSHOT_GAME_KEY: screenshot.game, IFTTT_INGREDIENT_META_LABEL_KEY: { IFTTT_INGREDIENT_META_ID_KEY: screenshot.screenshot_id, IFTTT_INGREDIENT_META_TIMESTAMP_KEY: int((screenshot.scraped - datetime(1970, 1, 1)).total_seconds()) } }) screenshot.seen_already = True # Save the seen_already flags db.put(screenshots) self.setResponseHeaders() self.response.out.write(json.dumps({ # Sort by timestsamp because apparently the db query isn't sorted by scraped date 'data': sorted(screenshot_data, reverse=True, key=lambda k: k[IFTTT_INGREDIENT_META_LABEL_KEY][IFTTT_INGREDIENT_META_TIMESTAMP_KEY]) }))