def make_api_call_for_site(self, site): if site not in self.queue: return self.queue_modify_lock.acquire() new_posts = self.queue.pop(site) store_bodyfetcher_queue() self.queue_modify_lock.release() new_post_ids = [int(k) for k in new_posts.keys()] if GlobalVars.flovis is not None: for post_id in new_post_ids: GlobalVars.flovis.stage('bodyfetcher/api_request', site, post_id, {'site': site, 'posts': list(new_posts.keys())}) self.queue_timing_modify_lock.acquire() post_add_times = [v for k, v in new_posts.items()] pop_time = datetime.utcnow() for add_time in post_add_times: try: seconds_in_queue = (pop_time - add_time).total_seconds() if site in self.queue_timings: self.queue_timings[site].append(seconds_in_queue) else: self.queue_timings[site] = [seconds_in_queue] except KeyError: # XXX: Any other possible exception? continue # Skip to next item if we've got invalid data or missing values. store_queue_timings() self.queue_timing_modify_lock.release() self.max_ids_modify_lock.acquire() if site in self.previous_max_ids and max(new_post_ids) > self.previous_max_ids[site]: previous_max_id = self.previous_max_ids[site] intermediate_posts = range(previous_max_id + 1, max(new_post_ids)) # We don't want to go over the 100-post API cutoff, so take the last # (100-len(new_post_ids)) from intermediate_posts intermediate_posts = intermediate_posts[(100 - len(new_post_ids)):] # new_post_ids could contain edited posts, so merge it back in combined = chain(intermediate_posts, new_post_ids) # Could be duplicates, so uniquify posts = list(set(combined)) else: posts = new_post_ids try: if max(new_post_ids) > self.previous_max_ids[site]: self.previous_max_ids[site] = max(new_post_ids) store_bodyfetcher_max_ids() except KeyError: self.previous_max_ids[site] = max(new_post_ids) store_bodyfetcher_max_ids() self.max_ids_modify_lock.release() log('debug', "New IDs / Hybrid Intermediate IDs for {}:".format(site)) if len(new_post_ids) > 30: log('debug', "{} +{} more".format(sorted(new_post_ids)[:30], len(new_post_ids) - 30)) else: log('debug', sorted(new_post_ids)) if len(new_post_ids) == len(posts): log('debug', "[ *Identical* ]") elif len(posts) > 30: log('debug', "{} +{} more".format(sorted(posts)[:30], len(posts) - 30)) else: log('debug', sorted(posts)) question_modifier = "" pagesize_modifier = "" if site == "stackoverflow.com": # Not all SO questions are shown in the realtime feed. We now # fetch all recently modified SO questions to work around that. if self.last_activity_date != 0: pagesize = "50" else: pagesize = "25" pagesize_modifier = "&pagesize={pagesize}" \ "&min={time_length}".format(pagesize=pagesize, time_length=str(self.last_activity_date)) else: question_modifier = "/{0}".format(";".join([str(post) for post in posts])) url = "https://api.stackexchange.com/2.2/questions{q_modifier}?site={site}" \ "&filter=!*xq08dCDNr)PlxxXfaN8ntivx(BPlY_8XASyXLX-J7F-)VK*Q3KTJVkvp*&key=IAkbitmze4B8KpacUfLqkw((" \ "{optional_min_query_param}".format(q_modifier=question_modifier, site=site, optional_min_query_param=pagesize_modifier) # wait to make sure API has/updates post data time.sleep(3) GlobalVars.api_request_lock.acquire() # Respect backoff, if we were given one if GlobalVars.api_backoff_time > time.time(): time.sleep(GlobalVars.api_backoff_time - time.time() + 2) try: time_request_made = datetime.now().strftime('%H:%M:%S') response = requests.get(url, timeout=20).json() except (requests.exceptions.Timeout, requests.ConnectionError, Exception): # Any failure in the request being made (timeout or otherwise) should be added back to # the queue. self.queue_modify_lock.acquire() if site in self.queue: self.queue[site].update(new_posts) else: self.queue[site] = new_posts self.queue_modify_lock.release() GlobalVars.api_request_lock.release() return self.api_data_lock.acquire() add_or_update_api_data(site) self.api_data_lock.release() message_hq = "" if "quota_remaining" in response: if response["quota_remaining"] - GlobalVars.apiquota >= 5000 and GlobalVars.apiquota >= 0: tell_rooms_with("debug", "API quota rolled over with {0} requests remaining. " "Current quota: {1}.".format(GlobalVars.apiquota, response["quota_remaining"])) sorted_calls_per_site = sorted(GlobalVars.api_calls_per_site.items(), key=itemgetter(1), reverse=True) api_quota_used_per_site = "" for site_name, quota_used in sorted_calls_per_site: sanatized_site_name = site_name.replace('.com', '').replace('.stackexchange', '') api_quota_used_per_site += sanatized_site_name + ": {0}\n".format(str(quota_used)) api_quota_used_per_site = api_quota_used_per_site.strip() tell_rooms_with("debug", api_quota_used_per_site) clear_api_data() if response["quota_remaining"] == 0: tell_rooms_with("debug", "API reports no quota left! May be a glitch.") tell_rooms_with("debug", str(response)) # No code format for now? if GlobalVars.apiquota == -1: tell_rooms_with("debug", "Restart: API quota is {quota}." .format(quota=response["quota_remaining"])) GlobalVars.apiquota = response["quota_remaining"] else: message_hq = "The quota_remaining property was not in the API response." if "error_message" in response: message_hq += " Error: {} at {} UTC.".format(response["error_message"], time_request_made) if "error_id" in response and response["error_id"] == 502: if GlobalVars.api_backoff_time < time.time() + 12: # Add a backoff of 10 + 2 seconds as a default GlobalVars.api_backoff_time = time.time() + 12 message_hq += " Backing off on requests for the next 12 seconds." message_hq += " Previous URL: `{}`".format(url) if "backoff" in response: if GlobalVars.api_backoff_time < time.time() + response["backoff"]: GlobalVars.api_backoff_time = time.time() + response["backoff"] GlobalVars.api_request_lock.release() if len(message_hq) > 0: tell_rooms_with("debug", message_hq.strip()) if "items" not in response: return if site == "stackoverflow.com": items = response["items"] if len(items) > 0 and "last_activity_date" in items[0]: self.last_activity_date = items[0]["last_activity_date"] num_scanned = 0 start_time = time.time() for post in response["items"]: pnb = copy.deepcopy(post) if 'body' in pnb: pnb['body'] = 'Present, but truncated' if 'answers' in pnb: del pnb['answers'] if "title" not in post or "body" not in post: if GlobalVars.flovis is not None and 'question_id' in post: GlobalVars.flovis.stage('bodyfetcher/api_response/no_content', site, post['question_id'], pnb) continue post['site'] = site try: post['edited'] = (post['creation_date'] != post['last_edit_date']) except KeyError: post['edited'] = False # last_edit_date not present = not edited try: post_ = Post(api_response=post) except PostParseError as err: log('error', 'Error {0} when parsing post: {1!r}'.format(err, post_)) if GlobalVars.flovis is not None and 'question_id' in post: GlobalVars.flovis.stage('bodyfetcher/api_response/error', site, post['question_id'], pnb) continue num_scanned += 1 is_spam, reason, why = check_if_spam(post_) if is_spam: try: if GlobalVars.flovis is not None and 'question_id' in post: GlobalVars.flovis.stage('bodyfetcher/api_response/spam', site, post['question_id'], {'post': pnb, 'check_if_spam': [is_spam, reason, why]}) handle_spam(post=post_, reasons=reason, why=why) except Exception as e: log('error', "Exception in handle_spam:", e) elif GlobalVars.flovis is not None and 'question_id' in post: GlobalVars.flovis.stage('bodyfetcher/api_response/not_spam', site, post['question_id'], {'post': pnb, 'check_if_spam': [is_spam, reason, why]}) try: if "answers" not in post: pass else: for answer in post["answers"]: anb = copy.deepcopy(answer) if 'body' in anb: anb['body'] = 'Present, but truncated' num_scanned += 1 answer["IsAnswer"] = True # Necesssary for Post object answer["title"] = "" # Necessary for proper Post object creation answer["site"] = site # Necessary for proper Post object creation try: answer['edited'] = (answer['creation_date'] != answer['last_edit_date']) except KeyError: answer['edited'] = False # last_edit_date not present = not edited answer_ = Post(api_response=answer, parent=post_) is_spam, reason, why = check_if_spam(answer_) if is_spam: try: if GlobalVars.flovis is not None and 'answer_id' in answer: GlobalVars.flovis.stage('bodyfetcher/api_response/spam', site, answer['answer_id'], {'post': anb, 'check_if_spam': [is_spam, reason, why]}) handle_spam(answer_, reasons=reason, why=why) except Exception as e: log('error', "Exception in handle_spam:", e) elif GlobalVars.flovis is not None and 'answer_id' in answer: GlobalVars.flovis.stage('bodyfetcher/api_response/not_spam', site, answer['answer_id'], {'post': anb, 'check_if_spam': [is_spam, reason, why]}) except Exception as e: log('error', "Exception handling answers:", e) end_time = time.time() GlobalVars.posts_scan_stats_lock.acquire() GlobalVars.num_posts_scanned += num_scanned GlobalVars.post_scan_time += end_time - start_time GlobalVars.posts_scan_stats_lock.release() return
def make_api_call_for_site(self, site): if site not in self.queue: return with self.queue_modify_lock: new_posts = self.queue.pop(site) store_bodyfetcher_queue() new_post_ids = [int(k) for k in new_posts.keys()] if GlobalVars.flovis is not None: for post_id in new_post_ids: GlobalVars.flovis.stage('bodyfetcher/api_request', site, post_id, {'site': site, 'posts': list(new_posts.keys())}) with self.queue_timing_modify_lock: post_add_times = [v for k, v in new_posts.items()] pop_time = datetime.utcnow() for add_time in post_add_times: try: seconds_in_queue = (pop_time - add_time).total_seconds() if site in self.queue_timings: self.queue_timings[site].append(seconds_in_queue) else: self.queue_timings[site] = [seconds_in_queue] except KeyError: # XXX: Any other possible exception? continue # Skip to next item if we've got invalid data or missing values. store_queue_timings() with self.max_ids_modify_lock: if site in self.previous_max_ids and max(new_post_ids) > self.previous_max_ids[site]: previous_max_id = self.previous_max_ids[site] intermediate_posts = range(previous_max_id + 1, max(new_post_ids)) # We don't want to go over the 100-post API cutoff, so take the last # (100-len(new_post_ids)) from intermediate_posts intermediate_posts = intermediate_posts[-(100 - len(new_post_ids)):] # new_post_ids could contain edited posts, so merge it back in combined = chain(intermediate_posts, new_post_ids) # Could be duplicates, so uniquify posts = list(set(combined)) else: posts = new_post_ids try: if max(new_post_ids) > self.previous_max_ids[site]: self.previous_max_ids[site] = max(new_post_ids) store_bodyfetcher_max_ids() except KeyError: self.previous_max_ids[site] = max(new_post_ids) store_bodyfetcher_max_ids() log('debug', "New IDs / Hybrid Intermediate IDs for {}:".format(site)) if len(new_post_ids) > 30: log('debug', "{} +{} more".format(sorted(new_post_ids)[:30], len(new_post_ids) - 30)) else: log('debug', sorted(new_post_ids)) if len(new_post_ids) == len(posts): log('debug', "[ *Identical* ]") elif len(posts) > 30: log('debug', "{} +{} more".format(sorted(posts)[:30], len(posts) - 30)) else: log('debug', sorted(posts)) question_modifier = "" pagesize_modifier = {} if site == "stackoverflow.com": # Not all SO questions are shown in the realtime feed. We now # fetch all recently modified SO questions to work around that. if self.last_activity_date != 0: pagesize = "50" else: pagesize = "25" pagesize_modifier = { 'pagesize': pagesize, 'min': str(self.last_activity_date) } else: question_modifier = "/{0}".format(";".join([str(post) for post in posts])) url = "https://api.stackexchange.com/2.2/questions{}".format(question_modifier) params = { 'filter': '!1rs)sUKylwB)8isvCRk.xNu71LnaxjnPS12*pX*CEOKbPFwVFdHNxiMa7GIVgzDAwMa', 'key': 'IAkbitmze4B8KpacUfLqkw((', 'site': site } params.update(pagesize_modifier) # wait to make sure API has/updates post data time.sleep(3) with GlobalVars.api_request_lock: # Respect backoff, if we were given one if GlobalVars.api_backoff_time > time.time(): time.sleep(GlobalVars.api_backoff_time - time.time() + 2) try: time_request_made = datetime.utcnow().strftime('%H:%M:%S') response = requests.get(url, params=params, timeout=20).json() except (requests.exceptions.Timeout, requests.ConnectionError, Exception): # Any failure in the request being made (timeout or otherwise) should be added back to # the queue. with self.queue_modify_lock: if site in self.queue: self.queue[site].update(new_posts) else: self.queue[site] = new_posts return with self.api_data_lock: add_or_update_api_data(site) message_hq = "" with GlobalVars.apiquota_rw_lock: if "quota_remaining" in response: if response["quota_remaining"] - GlobalVars.apiquota >= 5000 and GlobalVars.apiquota >= 0: tell_rooms_with("debug", "API quota rolled over with {0} requests remaining. " "Current quota: {1}.".format(GlobalVars.apiquota, response["quota_remaining"])) sorted_calls_per_site = sorted(GlobalVars.api_calls_per_site.items(), key=itemgetter(1), reverse=True) api_quota_used_per_site = "" for site_name, quota_used in sorted_calls_per_site: sanatized_site_name = site_name.replace('.com', '').replace('.stackexchange', '') api_quota_used_per_site += sanatized_site_name + ": {0}\n".format(str(quota_used)) api_quota_used_per_site = api_quota_used_per_site.strip() tell_rooms_with("debug", api_quota_used_per_site) clear_api_data() if response["quota_remaining"] == 0: tell_rooms_with("debug", "API reports no quota left! May be a glitch.") tell_rooms_with("debug", str(response)) # No code format for now? if GlobalVars.apiquota == -1: tell_rooms_with("debug", "Restart: API quota is {quota}." .format(quota=response["quota_remaining"])) GlobalVars.apiquota = response["quota_remaining"] else: message_hq = "The quota_remaining property was not in the API response." if "error_message" in response: message_hq += " Error: {} at {} UTC.".format(response["error_message"], time_request_made) if "error_id" in response and response["error_id"] == 502: if GlobalVars.api_backoff_time < time.time() + 12: # Add a backoff of 10 + 2 seconds as a default GlobalVars.api_backoff_time = time.time() + 12 message_hq += " Backing off on requests for the next 12 seconds." message_hq += " Previous URL: `{}`".format(url) if "backoff" in response: if GlobalVars.api_backoff_time < time.time() + response["backoff"]: GlobalVars.api_backoff_time = time.time() + response["backoff"] if len(message_hq) > 0 and "site is required" not in message_hq: message_hq = message_hq.strip() if len(message_hq) > 500: message_hq = "\n" + message_hq tell_rooms_with("debug", message_hq) if "items" not in response: return if site == "stackoverflow.com": items = response["items"] if len(items) > 0 and "last_activity_date" in items[0]: self.last_activity_date = items[0]["last_activity_date"] num_scanned = 0 start_time = time.time() for post in response["items"]: pnb = copy.deepcopy(post) if 'body' in pnb: pnb['body'] = 'Present, but truncated' if 'answers' in pnb: del pnb['answers'] if "title" not in post or "body" not in post: if GlobalVars.flovis is not None and 'question_id' in post: GlobalVars.flovis.stage('bodyfetcher/api_response/no_content', site, post['question_id'], pnb) continue post['site'] = site try: post['edited'] = (post['creation_date'] != post['last_edit_date']) except KeyError: post['edited'] = False # last_edit_date not present = not edited try: post_ = Post(api_response=post) except PostParseError as err: log('error', 'Error {0} when parsing post: {1!r}'.format(err, post_)) if GlobalVars.flovis is not None and 'question_id' in post: GlobalVars.flovis.stage('bodyfetcher/api_response/error', site, post['question_id'], pnb) continue num_scanned += 1 is_spam, reason, why = check_if_spam(post_) if is_spam: try: if GlobalVars.flovis is not None and 'question_id' in post: GlobalVars.flovis.stage('bodyfetcher/api_response/spam', site, post['question_id'], {'post': pnb, 'check_if_spam': [is_spam, reason, why]}) handle_spam(post=post_, reasons=reason, why=why) except Exception as e: log('error', "Exception in handle_spam:", e) elif GlobalVars.flovis is not None and 'question_id' in post: GlobalVars.flovis.stage('bodyfetcher/api_response/not_spam', site, post['question_id'], {'post': pnb, 'check_if_spam': [is_spam, reason, why]}) try: if "answers" not in post: pass else: for answer in post["answers"]: anb = copy.deepcopy(answer) if 'body' in anb: anb['body'] = 'Present, but truncated' num_scanned += 1 answer["IsAnswer"] = True # Necesssary for Post object answer["title"] = "" # Necessary for proper Post object creation answer["site"] = site # Necessary for proper Post object creation try: answer['edited'] = (answer['creation_date'] != answer['last_edit_date']) except KeyError: answer['edited'] = False # last_edit_date not present = not edited answer_ = Post(api_response=answer, parent=post_) is_spam, reason, why = check_if_spam(answer_) if is_spam: try: if GlobalVars.flovis is not None and 'answer_id' in answer: GlobalVars.flovis.stage('bodyfetcher/api_response/spam', site, answer['answer_id'], {'post': anb, 'check_if_spam': [is_spam, reason, why]}) handle_spam(answer_, reasons=reason, why=why) except Exception as e: log('error', "Exception in handle_spam:", e) elif GlobalVars.flovis is not None and 'answer_id' in answer: GlobalVars.flovis.stage('bodyfetcher/api_response/not_spam', site, answer['answer_id'], {'post': anb, 'check_if_spam': [is_spam, reason, why]}) except Exception as e: log('error', "Exception handling answers:", e) end_time = time.time() scan_time = end_time - start_time GlobalVars.PostScanStat.add_stat(num_scanned, scan_time) return
def make_api_call_for_site(self, site): if site not in self.queue: return self.queue_modify_lock.acquire() new_posts = self.queue.pop(site) store_bodyfetcher_queue() self.queue_modify_lock.release() new_post_ids = [int(k) for k, v in new_posts.items()] if GlobalVars.flovis is not None: for post_id in new_post_ids: GlobalVars.flovis.stage('bodyfetcher/api_request', site, post_id, {'queue': dict([[sk, [k for k, v in sq.items()]] for sk, sq in self.queue.items()]), 'site': site, 'posts': [k for k, v in new_posts.items()]}) self.queue_timing_modify_lock.acquire() post_add_times = [v for k, v in new_posts.items()] pop_time = datetime.utcnow() for add_time in post_add_times: try: seconds_in_queue = (pop_time - add_time).total_seconds() if site in self.queue_timings: self.queue_timings[site].append(seconds_in_queue) else: self.queue_timings[site] = [seconds_in_queue] except: continue # Skip to next item if we've got invalid data or missing values. store_queue_timings() self.queue_timing_modify_lock.release() self.max_ids_modify_lock.acquire() if site in self.previous_max_ids and max(new_post_ids) > self.previous_max_ids[site]: previous_max_id = self.previous_max_ids[site] intermediate_posts = range(previous_max_id + 1, max(new_post_ids)) # We don't want to go over the 100-post API cutoff, so take the last # (100-len(new_post_ids)) from intermediate_posts intermediate_posts = intermediate_posts[(100 - len(new_post_ids)):] # new_post_ids could contain edited posts, so merge it back in combined = chain(intermediate_posts, new_post_ids) # Could be duplicates, so uniquify posts = list(set(combined)) else: posts = new_post_ids try: if max(new_post_ids) > self.previous_max_ids[site]: self.previous_max_ids[site] = max(new_post_ids) store_bodyfetcher_max_ids() except KeyError: self.previous_max_ids[site] = max(new_post_ids) store_bodyfetcher_max_ids() self.max_ids_modify_lock.release() log('debug', "New IDs / Hybrid Intermediate IDs for {0}:".format(site)) log('debug', sorted(new_post_ids)) log('debug', sorted(posts)) question_modifier = "" pagesize_modifier = "" if site == "stackoverflow.com": # Not all SO questions are shown in the realtime feed. We now # fetch all recently modified SO questions to work around that. if self.last_activity_date != 0: pagesize = "50" else: pagesize = "25" pagesize_modifier = "&pagesize={pagesize}" \ "&min={time_length}".format(pagesize=pagesize, time_length=str(self.last_activity_date)) else: question_modifier = "/{0}".format(";".join(str(post) for post in posts)) url = "https://api.stackexchange.com/2.2/questions{q_modifier}?site={site}" \ "&filter=!)E0g*ODaEZ(SgULQhYvCYbu09*ss(bKFdnTrGmGUxnqPptuHP&key=IAkbitmze4B8KpacUfLqkw((" \ "{optional_min_query_param}".format(q_modifier=question_modifier, site=site, optional_min_query_param=pagesize_modifier) # wait to make sure API has/updates post data time.sleep(3) GlobalVars.api_request_lock.acquire() # Respect backoff, if we were given one if GlobalVars.api_backoff_time > time.time(): time.sleep(GlobalVars.api_backoff_time - time.time() + 2) try: time_request_made = datetime.now().strftime('%H:%M:%S') response = requests.get(url, timeout=20).json() except (requests.exceptions.Timeout, requests.ConnectionError, Exception): # Any failure in the request being made (timeout or otherwise) should be added back to # the queue. self.queue_modify_lock.acquire() if site in self.queue: self.queue[site].update(new_posts) else: self.queue[site] = new_posts self.queue_modify_lock.release() GlobalVars.api_request_lock.release() return self.api_data_lock.acquire() add_or_update_api_data(site) self.api_data_lock.release() message_hq = "" if "quota_remaining" in response: if response["quota_remaining"] - GlobalVars.apiquota >= 5000 and GlobalVars.apiquota >= 0: tell_rooms_with("debug", "API quota rolled over with {0} requests remaining. " "Current quota: {1}.".format(GlobalVars.apiquota, response["quota_remaining"])) sorted_calls_per_site = sorted(GlobalVars.api_calls_per_site.items(), key=itemgetter(1), reverse=True) api_quota_used_per_site = "" for site_name, quota_used in sorted_calls_per_site: sanatized_site_name = site_name.replace('.com', '').replace('.stackexchange', '') api_quota_used_per_site += sanatized_site_name + ": {0}\n".format(str(quota_used)) api_quota_used_per_site = api_quota_used_per_site.strip() tell_rooms_with("debug", api_quota_used_per_site) clear_api_data() if response["quota_remaining"] == 0: tell_rooms_with("debug", "API reports no quota left! May be a glitch.") tell_rooms_with("debug", str(response)) # No code format for now? if GlobalVars.apiquota == -1: tell_rooms_with("debug", "Restart: API quota is {quota}." .format(quota=response["quota_remaining"])) GlobalVars.apiquota = response["quota_remaining"] else: message_hq = "The quota_remaining property was not in the API response." if "error_message" in response: message_hq += " Error: {} at {} UTC.".format(response["error_message"], time_request_made) if "error_id" in response and response["error_id"] == 502: if GlobalVars.api_backoff_time < time.time() + 12: # Add a backoff of 10 + 2 seconds as a default GlobalVars.api_backoff_time = time.time() + 12 message_hq += " Backing off on requests for the next 12 seconds." message_hq += " Previous URL: `{}`".format(url) if "backoff" in response: if GlobalVars.api_backoff_time < time.time() + response["backoff"]: GlobalVars.api_backoff_time = time.time() + response["backoff"] GlobalVars.api_request_lock.release() if len(message_hq) > 0: tell_rooms_with("debug", message_hq.strip()) if "items" not in response: return if site == "stackoverflow.com": items = response["items"] if len(items) > 0 and "last_activity_date" in items[0]: self.last_activity_date = items[0]["last_activity_date"] num_scanned = 0 start_time = time.time() for post in response["items"]: if "title" not in post or "body" not in post: if GlobalVars.flovis is not None and 'question_id' in post: GlobalVars.flovis.stage('bodyfetcher/api_response/no_content', site, post['question_id'], post) continue post['site'] = site try: post_ = Post(api_response=post) except PostParseError as err: log('error', 'Error {0} when parsing post: {1!r}'.format(err, post_)) if GlobalVars.flovis is not None and 'question_id' in post: GlobalVars.flovis.stage('bodyfetcher/api_response/error', site, post['question_id'], post) continue num_scanned += 1 is_spam, reason, why = check_if_spam(post_) if is_spam: try: if GlobalVars.flovis is not None and 'question_id' in post: GlobalVars.flovis.stage('bodyfetcher/api_response/spam', site, post['question_id'], {'post': post, 'check_if_spam': [is_spam, reason, why]}) handle_spam(post=post_, reasons=reason, why=why) except Exception as e: log('error', "Exception in handle_spam:", e) elif GlobalVars.flovis is not None and 'question_id' in post: GlobalVars.flovis.stage('bodyfetcher/api_response/not_spam', site, post['question_id'], {'post': post, 'check_if_spam': [is_spam, reason, why]}) try: if "answers" not in post: pass else: for answer in post["answers"]: num_scanned += 1 answer["IsAnswer"] = True # Necesssary for Post object answer["title"] = "" # Necessary for proper Post object creation answer["site"] = site # Necessary for proper Post object creation answer_ = Post(api_response=answer, parent=post_) is_spam, reason, why = check_if_spam(answer_) if is_spam: try: if GlobalVars.flovis is not None and 'answer_id' in answer: GlobalVars.flovis.stage('bodyfetcher/api_response/spam', site, answer['answer_id'], {'post': answer, 'check_if_spam': [is_spam, reason, why]}) handle_spam(answer_, reasons=reason, why=why) except Exception as e: log('error', "Exception in handle_spam:", e) elif GlobalVars.flovis is not None and 'answer_id' in answer: GlobalVars.flovis.stage('bodyfetcher/api_response/not_spam', site, answer['answer_id'], {'post': answer, 'check_if_spam': [is_spam, reason, why]}) except Exception as e: log('error', "Exception handling answers:", e) end_time = time.time() GlobalVars.posts_scan_stats_lock.acquire() GlobalVars.num_posts_scanned += num_scanned GlobalVars.post_scan_time += end_time - start_time GlobalVars.posts_scan_stats_lock.release() return
def make_api_call_for_site(self, site): if site not in self.queue: return self.queue_modify_lock.acquire() new_post_ids = self.queue.pop(site) store_bodyfetcher_queue() self.queue_modify_lock.release() self.max_ids_modify_lock.acquire() if site in self.previous_max_ids and max(new_post_ids) > self.previous_max_ids[site]: previous_max_id = self.previous_max_ids[site] intermediate_posts = range(previous_max_id + 1, max(new_post_ids)) # We don't want to go over the 100-post API cutoff, so take the last # (100-len(new_post_ids)) from intermediate_posts intermediate_posts = intermediate_posts[(len(new_post_ids) - 100):] # new_post_ids could contain edited posts, so merge it back in combined = intermediate_posts + new_post_ids # Could be duplicates, so uniquify posts = list(set(combined)) else: posts = new_post_ids try: if max(new_post_ids) > self.previous_max_ids[site]: self.previous_max_ids[site] = max(new_post_ids) store_bodyfetcher_max_ids() except KeyError: self.previous_max_ids[site] = max(new_post_ids) store_bodyfetcher_max_ids() self.max_ids_modify_lock.release() print("New IDs / Hybrid Intermediate IDs for {0}:".format(site)) print(sorted(new_post_ids)) print(sorted(posts)) question_modifier = "" pagesize_modifier = "" if site == "stackoverflow.com": # Not all SO questions are shown in the realtime feed. We now # fetch all recently modified SO questions to work around that. if self.last_activity_date != 0: pagesize = "50" else: pagesize = "25" pagesize_modifier = "&pagesize={pagesize}" \ "&min={time_length}".format(pagesize=pagesize, time_length=str(self.last_activity_date)) else: question_modifier = "/{0}".format(";".join(str(post) for post in posts)) url = "https://api.stackexchange.com/2.2/questions{q_modifier}?site={site}" \ "&filter=!)E0g*ODaEZ(SgULQhYvCYbu09*ss(bKFdnTrGmGUxnqPptuHP&key=IAkbitmze4B8KpacUfLqkw((" \ "{optional_min_query_param}".format(q_modifier=question_modifier, site=site, optional_min_query_param=pagesize_modifier) # wait to make sure API has/updates post data time.sleep(3) GlobalVars.api_request_lock.acquire() # Respect backoff, if we were given one if GlobalVars.api_backoff_time > time.time(): time.sleep(GlobalVars.api_backoff_time - time.time() + 2) try: time_request_made = datetime.now().strftime('%H:%M:%S') response = requests.get(url, timeout=20).json() except (requests.exceptions.Timeout, requests.ConnectionError, Exception): # Any failure in the request being made (timeout or otherwise) should be added back to # the queue. self.queue_modify_lock.acquire() if site in self.queue: self.queue[site].extend(posts) else: self.queue[site] = posts self.queue_modify_lock.release() return self.api_data_lock.acquire() add_or_update_api_data(site) self.api_data_lock.release() message_hq = "" if "quota_remaining" in response: if response["quota_remaining"] - GlobalVars.apiquota >= 5000 and GlobalVars.apiquota >= 0: GlobalVars.charcoal_hq.send_message("API quota rolled over with {0} requests remaining. " "Current quota: {1}.".format(GlobalVars.apiquota, response["quota_remaining"])) sorted_calls_per_site = sorted(GlobalVars.api_calls_per_site.items(), key=itemgetter(1), reverse=True) api_quota_used_per_site = "" for site_name, quota_used in sorted_calls_per_site: sanatized_site_name = site_name.replace('.com', '').replace('.stackexchange', '') api_quota_used_per_site += sanatized_site_name + ": {0}\n".format(str(quota_used)) api_quota_used_per_site = api_quota_used_per_site.strip() GlobalVars.charcoal_hq.send_message(api_quota_used_per_site, False) clear_api_data() if response["quota_remaining"] == 0: GlobalVars.charcoal_hq.send_message("API reports no quota left! May be a glitch.") GlobalVars.charcoal_hq.send_message(str(response)) # No code format for now? if GlobalVars.apiquota == -1: GlobalVars.charcoal_hq.send_message("Restart: API quota is {quota}." .format(quota=response["quota_remaining"])) GlobalVars.apiquota = response["quota_remaining"] else: message_hq = "The quota_remaining property was not in the API response." if "error_message" in response: message_hq += " Error: {} at {} UTC.".format(response["error_message"], time_request_made) if "error_id" in response and response["error_id"] == 502: if GlobalVars.api_backoff_time < time.time() + 12: # Add a backoff of 10 + 2 seconds as a default GlobalVars.api_backoff_time = time.time() + 12 message_hq += " Backing off on requests for the next 12 seconds." if "backoff" in response: if GlobalVars.api_backoff_time < time.time() + response["backoff"]: GlobalVars.api_backoff_time = time.time() + response["backoff"] GlobalVars.api_request_lock.release() if len(message_hq) > 0: GlobalVars.charcoal_hq.send_message(message_hq.strip()) if "items" not in response: return if site == "stackoverflow.com": items = response["items"] if len(items) > 0 and "last_activity_date" in items[0]: self.last_activity_date = items[0]["last_activity_date"] num_scanned = 0 start_time = time.time() for post in response["items"]: if "title" not in post or "body" not in post: continue num_scanned += 1 title = GlobalVars.parser.unescape(post["title"]) body = GlobalVars.parser.unescape(post["body"]) link = post["link"] post_score = post["score"] up_vote_count = post["up_vote_count"] down_vote_count = post["down_vote_count"] try: owner_name = GlobalVars.parser.unescape(post["owner"]["display_name"]) owner_link = post["owner"]["link"] owner_rep = post["owner"]["reputation"] except: owner_name = "" owner_link = "" owner_rep = 0 q_id = str(post["question_id"]) is_spam, reason, why = check_if_spam(title=title, body=body, user_name=owner_name, user_url=owner_link, post_site=site, post_id=q_id, is_answer=False, body_is_summary=False, owner_rep=owner_rep, post_score=post_score) if is_spam: try: handle_spam(title=title, body=body, poster=owner_name, site=site, post_url=link, poster_url=owner_link, post_id=q_id, reasons=reason, is_answer=False, why=why, owner_rep=owner_rep, post_score=post_score, up_vote_count=up_vote_count, down_vote_count=down_vote_count, question_id=None) except: pass try: for answer in post["answers"]: num_scanned += 1 answer_title = "" body = answer["body"] link = answer["link"] a_id = str(answer["answer_id"]) post_score = answer["score"] up_vote_count = answer["up_vote_count"] down_vote_count = answer["down_vote_count"] try: owner_name = GlobalVars.parser.unescape(answer["owner"]["display_name"]) owner_link = answer["owner"]["link"] owner_rep = answer["owner"]["reputation"] except: owner_name = "" owner_link = "" owner_rep = 0 is_spam, reason, why = check_if_spam(title=answer_title, body=body, user_name=owner_name, user_url=owner_link, post_site=site, post_id=a_id, is_answer=True, body_is_summary=False, owner_rep=owner_rep, post_score=post_score) if is_spam: try: handle_spam(title=title, body=body, poster=owner_name, site=site, post_url=link, poster_url=owner_link, post_id=a_id, reasons=reason, is_answer=True, why=why, owner_rep=owner_rep, post_score=post_score, up_vote_count=up_vote_count, down_vote_count=down_vote_count, question_id=q_id) except: pass except: pass end_time = time.time() GlobalVars.posts_scan_stats_lock.acquire() GlobalVars.num_posts_scanned += num_scanned GlobalVars.post_scan_time += end_time - start_time GlobalVars.posts_scan_stats_lock.release() return