def send_status_ping_and_verify_scanning_if_active():
in_standby_mode = GlobalVars.standby_mode or GlobalVars.no_se_activity_scan
if not in_standby_mode:
# This is the active instance, so should be scanning. If it's not scanning, then report or go to standby.
if GlobalVars.PostScanStat.get_stat(
) == Metasmoke.scan_stat_snapshot:
# There's been no actvity since the last ping.
Metasmoke.status_pings_since_scan_activity += 1
if Metasmoke.status_pings_since_scan_activity >= NO_ACTIVITY_PINGS_TO_STANDBY:
# Assume something is very wrong. Report to debug rooms and go into standby mode.
error_message = "There's been no scan activity for {} status pings. Going into standby." \
.format(Metasmoke.status_pings_since_scan_activity)
log('error', error_message)
chatcommunicate.tell_rooms_with("debug", error_message)
GlobalVars.standby_mode = True
# Let MS know immediately, to lessen potential wait time (e.g. if we fail to reboot).
Metasmoke.send_status_ping()
time.sleep(8)
exit_mode("standby")
elif Metasmoke.status_pings_since_scan_activity >= NO_ACTIVITY_PINGS_TO_REPORT:
# Something might be wrong. Let people in debug rooms know.
status_message = "There's been no scan activity for {} status pings. There may be a problem." \
.format(Metasmoke.status_pings_since_scan_activity)
log('warning', status_message)
chatcommunicate.tell_rooms_with("debug", status_message)
else:
Metasmoke.status_pings_since_scan_activity = 0
Metasmoke.scan_stat_snapshot = GlobalVars.PostScanStat.get_stat(
)
Metasmoke.send_status_ping()
def restart_automatically():
Metasmoke.send_statistics()
chatcommunicate.tell_rooms_with(
"debug", "{}: Executing automatic scheduled reboot.".format(
GlobalVars.location))
time.sleep(6)
exit_mode("reboot")
def func(url, *args, **kwargs):
if not GlobalVars.metasmoke_host or GlobalVars.metasmoke_down:
return None
response = None # Should return None upon failure, if any
try:
response = method(GlobalVars.metasmoke_host + url, *args,
**kwargs)
except Exception:
GlobalVars.metasmoke_failures += 1
if GlobalVars.metasmoke_failures > MAX_FAILURES:
GlobalVars.metasmoke_down = True
chatcommunicate.tell_rooms_with(
'debug',
"**Warning**: {} latest connections to metasmoke have fa"
"iled. Disabling metasmoke".format(
GlobalVars.metasmoke_failures))
# No need to log here because it's re-raised
raise # Maintain minimal difference to the original get/post methods
else:
GlobalVars.metasmoke_failures -= 1
if GlobalVars.metasmoke_failures < 0:
GlobalVars.metasmoke_failures = 0
return response
def get_post_bodies_from_ms(post_url):
if not GlobalVars.metasmoke_key:
return None
payload = {
'key': GlobalVars.metasmoke_key,
'filter':
'GHOOIJGNLKHIIOIKGILKIJGHFMNKKGFJ', # posts.id, posts.body, posts.created_at
'urls': parsing.to_protocol_relative(post_url)
}
try:
response = Metasmoke.get('/api/v2.0/posts/urls',
params=payload).json()
except AttributeError:
return None
except Exception as e:
log('error', '{}: {}'.format(type(e).__name__, e))
log_exception(*sys.exc_info())
exception_only = ''.join(
traceback.format_exception_only(type(e), e)).strip()
chatcommunicate.tell_rooms_with(
"debug",
"{}: In getting MS post information, recovered from `{}`".
format(GlobalVars.location, exception_only))
return None
return response['items']
def func(url, *args, ignore_down=False, **kwargs):
if not GlobalVars.metasmoke_host or (GlobalVars.metasmoke_down and not ignore_down):
return None
if 'timeout' not in kwargs:
kwargs['timeout'] = 10.000 # Don't throttle by MS
response = None # Should return None upon failure, if any
try:
response = method(GlobalVars.metasmoke_host + url, *args, **kwargs)
except Exception:
GlobalVars.metasmoke_failures += 1
if GlobalVars.metasmoke_failures >= MAX_FAILURES and not GlobalVars.metasmoke_down:
GlobalVars.metasmoke_down = True
chatcommunicate.tell_rooms_with(
'debug', '**Warning**: {}: {} latest connections to '
'metasmoke have failed. Disabling metasmoke'.format(
GlobalVars.location, GlobalVars.metasmoke_failures))
# No need to log here because it's re-raised
raise # Maintain minimal difference to the original get/post methods
else:
GlobalVars.metasmoke_failures -= 1
if GlobalVars.metasmoke_failures < 0:
GlobalVars.metasmoke_failures = 0
return response
def func(url, *args, ignore_down=False, **kwargs):
if not GlobalVars.metasmoke_host or (GlobalVars.metasmoke_down
and not ignore_down):
return None
if 'timeout' not in kwargs:
kwargs['timeout'] = 10.000 # Don't throttle by MS
response = None # Should return None upon failure, if any
try:
response = method(GlobalVars.metasmoke_host + url, *args,
**kwargs)
except Exception:
GlobalVars.metasmoke_failures += 1
if GlobalVars.metasmoke_failures >= MAX_FAILURES and not GlobalVars.metasmoke_down:
GlobalVars.metasmoke_down = True
chatcommunicate.tell_rooms_with(
'debug', '**Warning**: {}: {} latest connections to '
'metasmoke have failed. Disabling metasmoke'.format(
GlobalVars.location,
GlobalVars.metasmoke_failures))
# No need to log here because it's re-raised
raise # Maintain minimal difference to the original get/post methods
else:
GlobalVars.metasmoke_failures -= 1
if GlobalVars.metasmoke_failures < 0:
GlobalVars.metasmoke_failures = 0
return response
def send_status_ping():
if GlobalVars.metasmoke_host is None:
log(
'info',
'Attempted to send status ping but metasmoke_host is undefined. Not sent.'
)
return
elif GlobalVars.metasmoke_down:
payload = {
"location": GlobalVars.location,
"timestamp": time.time()
}
SocketScience.send(payload)
metasmoke_key = GlobalVars.metasmoke_key
try:
payload = {
'location': GlobalVars.location,
'key': metasmoke_key,
'standby': GlobalVars.standby_mode
}
headers = {'content-type': 'application/json'}
response = Metasmoke.post("/status-update.json",
data=json.dumps(payload),
headers=headers)
try:
response = response.json()
GlobalVars.metasmoke_last_ping_time = datetime.now(
) # Otherwise the ping watcher will exit(10)
if 'failover' in response and GlobalVars.standby_mode:
if response['failover']:
GlobalVars.standby_mode = False
chatcommunicate.tell_rooms_with(
"debug",
GlobalVars.location + " received failover signal.",
notify_site="/failover")
if response['standby']:
chatcommunicate.tell_rooms_with(
"debug", GlobalVars.location +
" entering metasmoke-forced standby.")
time.sleep(2)
os._exit(7)
if 'shutdown' in response:
if response['shutdown']:
os._exit(6)
except Exception: # TODO: What could happen here?
pass
except Exception as e:
log('error', e)
def set_ms_down():
""" Switch metasmoke status to down """
ms_down_msg = ""
if GlobalVars.MSStatus.is_up():
ms_down_msg = "Metasmoke status is now set to down."
GlobalVars.MSStatus.set_down()
if ms_down_msg:
log("info", ms_down_msg)
chatcommunicate.tell_rooms_with("debug", ms_down_msg)
def reboot_or_standby(action):
error_message = "There's been no scan activity for {} status pings. Going to {}." \
.format(Metasmoke.status_pings_since_scan_activity, action)
log('error', error_message)
chatcommunicate.tell_rooms_with("debug", error_message)
if action == "standby":
GlobalVars.standby_mode = True
# Let MS know immediately, to lessen potential wait time (e.g. if we fail to reboot).
Metasmoke.send_status_ping()
time.sleep(8)
exit_mode(action)
def set_ms_up():
""" Switch metasmoke status to up """
ms_up_msg = ""
if GlobalVars.MSStatus.is_down():
ms_up_msg = "Metasmoke status is now set to up."
GlobalVars.MSStatus.set_up()
# We must first set metasmoke to up, then say that metasmoke is up, not the other way around.
if ms_up_msg:
log("info", ms_up_msg)
chatcommunicate.tell_rooms_with("debug", ms_up_msg)
def set_ms_down(tell=True):
""" Switch metasmoke status to down """
ms_msg = ""
if GlobalVars.MSStatus.is_up():
ms_msg = "Metasmoke status: set to down."
GlobalVars.MSStatus.set_down()
if ms_msg:
log("info", ms_msg)
if tell:
chatcommunicate.tell_rooms_with(
"debug", "{}: {}".format(GlobalVars.location, ms_msg))
def send_status_ping():
if GlobalVars.metasmoke_host is None:
log('info', 'Attempted to send status ping but metasmoke_host is undefined. Not sent.')
return
elif GlobalVars.metasmoke_down:
payload = {
"location": GlobalVars.location,
"timestamp": time.time()
}
SocketScience.send(payload)
metasmoke_key = GlobalVars.metasmoke_key
try:
payload = {
'location': GlobalVars.location,
'key': metasmoke_key,
'standby': GlobalVars.standby_mode
}
headers = {'content-type': 'application/json'}
response = Metasmoke.post("/status-update.json",
data=json.dumps(payload), headers=headers, ignore_down=True)
try:
response = response.json()
GlobalVars.metasmoke_last_ping_time = datetime.now() # Otherwise the ping watcher will exit(10)
if response.get('pull_update', False):
log('info', "Received pull command from MS ping response")
exit_mode("pull_update")
if 'failover' in response and GlobalVars.standby_mode:
if response['failover']:
GlobalVars.standby_mode = False
chatcommunicate.tell_rooms_with("debug", GlobalVars.location + " received failover signal.",
notify_site="/failover")
if response.get('standby', False):
chatcommunicate.tell_rooms_with("debug",
GlobalVars.location + " entering metasmoke-forced standby.")
time.sleep(2)
exit_mode("standby")
if response.get('shutdown', False):
exit_mode("shutdown")
except Exception: # TODO: What could happen here?
pass
except Exception as e:
log('error', e)
def enable_autoswitch(to_enable):
""" Enable or disable auto status switch """
switch_auto_msg = ""
with Metasmoke.AutoSwitch.rw_lock:
if Metasmoke.AutoSwitch.autoswitch_is_on is not to_enable:
# Log and post chat message only if there really is a change.
switch_auto_msg = "Metasmoke status autoswitch is now {}abled.".format(
"en" if to_enable else "dis")
Metasmoke.AutoSwitch.autoswitch_is_on = to_enable
if switch_auto_msg:
log("info", switch_auto_msg)
chatcommunicate.tell_rooms_with("debug", switch_auto_msg)
def set_ms_up(tell=True):
""" Switch metasmoke status to up """
# We must first set metasmoke to up, then say that metasmoke is up, not the other way around.
ms_msg = ""
if GlobalVars.MSStatus.is_down():
ms_msg = "Metasmoke status: set to up."
GlobalVars.MSStatus.set_up()
if ms_msg:
log("info", ms_msg)
if tell:
chatcommunicate.tell_rooms_with(
"debug", "{}: {}".format(GlobalVars.location, ms_msg))
def send_status_ping():
if GlobalVars.metasmoke_host is None:
log('info',
"Metasmoke location not defined; not sending status ping")
return
metasmoke_key = GlobalVars.metasmoke_key
try:
payload = {
'location': GlobalVars.location,
'key': metasmoke_key,
'standby': GlobalVars.standby_mode
}
headers = {'content-type': 'application/json'}
response = requests.post(GlobalVars.metasmoke_host +
"/status-update.json",
data=json.dumps(payload),
headers=headers)
try:
response = response.json()
if 'failover' in response and GlobalVars.standby_mode:
if response['failover']:
GlobalVars.standby_mode = False
GlobalVars.metasmoke_last_ping_time = datetime.now(
) # Otherwise the ping watcher will exit(10)
chatcommunicate.tell_rooms_with(
"debug",
GlobalVars.location + " received failover signal.",
notify_site="/failover")
if response['standby']:
chatcommunicate.tell_rooms_with(
"debug", GlobalVars.location +
" entering metasmoke-forced standby.")
time.sleep(2)
os._exit(7)
if 'shutdown' in response:
if response['shutdown']:
os._exit(6)
except Exception:
pass
except Exception as e:
log('error', e)
def send(payload):
encoded = msgpack.dumps(payload)
# Messages can be 500 chars, but we need to leave space for control and message ident
chunks = [encoded[i:i + 485] for i in range(0, len(encoded), 485)]
message_id = random.randint(1000, 9999)
chunks[0] = "\u0002" + str(message_id) + chunks[0]
chunks[-1] = chunks[-1] + str(message_id) + "\u0003"
for n in range(1, len(chunks) - 1):
chunks[n] = "\u0016" + str(message_id) + chunks[n]
for chunk in chunks:
chatcommunicate.tell_rooms_with("direct", chunk)
def init_se_websocket_or_reboot(max_tries, tell_debug_room_on_error=False):
for tries in range(1, 1 + max_tries, 1):
ws = setup_websocket(tries, max_tries)
if ws:
break
else:
error_message = 'SE WebSocket: Max retries exceeded. Exiting, maybe a restart will kick things.'
log('error', error_message)
if tell_debug_room_on_error:
chatcommunicate.tell_rooms_with("debug", error_message)
time.sleep(
6
) # Make it more likely the message is actually sent to the rooms prior to rebooting.
exit_mode("reboot")
return ws
def send_status_ping():
if GlobalVars.metasmoke_host is None:
log('info', 'Attempted to send status ping but metasmoke_host is undefined. Not sent.')
return
elif GlobalVars.metasmoke_down:
log('info', "Metasmoke is down, wat?")
return
metasmoke_key = GlobalVars.metasmoke_key
try:
payload = {
'location': GlobalVars.location,
'key': metasmoke_key,
'standby': GlobalVars.standby_mode
}
headers = {'content-type': 'application/json'}
response = Metasmoke.post("/status-update.json",
data=json.dumps(payload), headers=headers)
try:
response = response.json()
GlobalVars.metasmoke_last_ping_time = datetime.now() # Otherwise the ping watcher will exit(10)
if 'failover' in response and GlobalVars.standby_mode:
if response['failover']:
GlobalVars.standby_mode = False
chatcommunicate.tell_rooms_with("debug", GlobalVars.location + " received failover signal.",
notify_site="/failover")
if response['standby']:
chatcommunicate.tell_rooms_with("debug",
GlobalVars.location + " entering metasmoke-forced standby.")
time.sleep(2)
os._exit(7)
if 'shutdown' in response:
if response['shutdown']:
os._exit(6)
except Exception: # TODO: What could happen here?
pass
except Exception as e:
log('error', e)
def send_status_ping():
if GlobalVars.metasmoke_host is None:
log('info', "Metasmoke location not defined; not sending status ping")
return
metasmoke_key = GlobalVars.metasmoke_key
try:
payload = {
'location': GlobalVars.location,
'key': metasmoke_key,
'standby': GlobalVars.standby_mode
}
headers = {'content-type': 'application/json'}
response = requests.post(GlobalVars.metasmoke_host + "/status-update.json",
data=json.dumps(payload), headers=headers)
try:
response = response.json()
if 'failover' in response and GlobalVars.standby_mode:
if response['failover']:
GlobalVars.standby_mode = False
GlobalVars.metasmoke_last_ping_time = datetime.now() # Otherwise the ping watcher will exit(10)
chatcommunicate.tell_rooms_with("debug", GlobalVars.location + " received failover signal.")
if response['standby']:
chatcommunicate.tell_rooms_with("debug",
GlobalVars.location + " entering metasmoke-forced standby.")
time.sleep(2)
os._exit(7)
if 'shutdown' in response:
if response['shutdown']:
os._exit(6)
except Exception:
pass
except Exception as e:
log('error', e)
def handle(cls, content):
for k, d in content.items():
if k in cls._callbacks:
for cb in cls._callbacks[k]:
cb(d)
if "metasmoke_state" in content:
if content["metasmoke_state"] == "down":
log(
'info',
"{} says metasmoke is down, switching to active ping monitoring."
.format(content["location"]))
chatcommunicate.tell_rooms_with(
"debug",
"{} says metasmoke is down,".format(content["location"]) +
" switching to active ping monitoring.")
# This is an exception, to prevent circular import.
# Other classes should not do the same. Always use Metasmoke.ms_down(). (20 May 2020)
GlobalVars.MSStatus.set_down()
Tasks.later(SocketScience.check_recent_pings, after=90)
if content["metasmoke_state"] == "up":
log(
'info',
'{} says metasmoke is up, disabling ping monitoring.'.
format(content["location"]))
chatcommunicate.tell_rooms_with(
"debug",
"{} says metasmoke is up,".format(content["location"]) +
" disabling ping monitoring.")
# This is an exception, to prevent circular import.
# Other classes should not do the same. Always use Metasmoke.ms_up(). (20 May 2020)
GlobalVars.MSStatus.set_up()
if "ping" in content:
cls._pings.append({
"timestamp": content["ping"],
"location": content["location"]
})
if cls._switch_task is not None:
cls._switch_task.cancel()
def func(url, *args, **kwargs):
if not GlobalVars.metasmoke_host or GlobalVars.metasmoke_down:
return None
response = None # Should return None upon failure, if any
try:
response = method(GlobalVars.metasmoke_host + url, *args, **kwargs)
except Exception:
GlobalVars.metasmoke_failures += 1
if GlobalVars.metasmoke_failures > MAX_FAILURES:
GlobalVars.metasmoke_down = True
chatcommunicate.tell_rooms_with('debug', "**Warning**: {} latest connections to metasmoke have fa"
"iled. Disabling metasmoke".format(GlobalVars.metasmoke_failures))
# No need to log here because it's re-raised
raise # Maintain minimal difference to the original get/post methods
else:
GlobalVars.metasmoke_failures -= 1
if GlobalVars.metasmoke_failures < 0:
GlobalVars.metasmoke_failures = 0
return response
def init_websocket():
has_succeeded = False
failed_connection_attempts = 0
while GlobalVars.metasmoke_key and GlobalVars.metasmoke_ws_host:
try:
Metasmoke.connect_websocket()
has_succeeded = True
while True:
a = GlobalVars.metasmoke_ws.recv()
try:
data = json.loads(a)
GlobalVars.metasmoke_last_ping_time = datetime.utcnow()
Metasmoke.handle_websocket_data(data)
Metasmoke.reset_failure_count()
except ConnectionError:
raise
except Exception as e:
Metasmoke.connect_websocket()
GlobalVars.metasmoke_failures += 1
log('error', e, f=True)
traceback.print_exc()
except Exception:
GlobalVars.metasmoke_failures += 1
log('error', "Couldn't bind to MS websocket")
if not has_succeeded:
failed_connection_attempts += 1
if failed_connection_attempts > MAX_MS_WEBSOCKET_RETRIES:
chatcommunicate.tell_rooms_with(
"debug", "Cannot initiate MS websocket." +
" Manual `!!/reboot` is required once MS is up")
log(
'warning', "Cannot initiate MS websocket." +
" init_websocket() in metasmoke.py is terminating."
)
break
else:
# Wait and hopefully network issues will be solved
time.sleep(10)
else:
time.sleep(10)
def init_websocket():
has_succeeded = False
failed_connection_attempts = 0
while GlobalVars.metasmoke_key and GlobalVars.metasmoke_ws_host:
try:
Metasmoke.connect_websocket()
has_succeeded = True
while True:
a = GlobalVars.metasmoke_ws.recv()
try:
data = json.loads(a)
Metasmoke.handle_websocket_data(data)
GlobalVars.MSStatus.succeeded()
failed_connection_attempts = 0
except ConnectionError:
raise
except Exception as e:
log('error', '{}: {}'.format(type(e).__name__, e))
log_exception(*sys.exc_info())
GlobalVars.MSStatus.failed()
Metasmoke.connect_websocket()
except Exception:
GlobalVars.MSStatus.failed()
log('error', "Couldn't bind to MS websocket")
if not has_succeeded:
failed_connection_attempts += 1
if failed_connection_attempts == MAX_MS_WEBSOCKET_RETRIES_TO_LONG_INTERVAL:
chatcommunicate.tell_rooms_with(
"debug", "Cannot initiate MS websocket."
" Continuing to retry at longer intervals.")
log(
'warning', "Cannot initiate MS websocket."
" Continuing to retry at longer intervals.")
if failed_connection_attempts >= MAX_MS_WEBSOCKET_RETRIES_TO_LONG_INTERVAL:
time.sleep(MS_WEBSOCKET_LONG_INTERVAL)
else:
# Wait and hopefully network issues will be solved
time.sleep(10)
else:
time.sleep(10)
def ping_failed():
""" Indicate a metasmoke status ping connection failure """
with Metasmoke.AutoSwitch.rw_lock:
if Metasmoke.AutoSwitch.ping_failure_counter < 0:
# Consecutive counter. Switch sign.
Metasmoke.AutoSwitch.ping_failure_counter = 0
Metasmoke.AutoSwitch.ping_failure_counter += 1
current_counter = Metasmoke.AutoSwitch.ping_failure_counter
current_auto = Metasmoke.AutoSwitch.autoswitch_is_on
# MAX_FAILURES is constant so no lock.
if current_counter > Metasmoke.AutoSwitch.MAX_FAILURES and\
GlobalVars.MSStatus.is_up() and current_auto:
log(
"warning",
"Last {} connection(s) to metasmoke failed".format(
current_counter) +
" Setting metasmoke status to down.")
chatcommunicate.tell_rooms_with(
"debug", "**Warning**: {}: ".format(GlobalVars.location) +
"Last {} connection(s) to metasmoke".format(
current_counter) +
" failed. Setting metasmoke status to **down**.")
Metasmoke.set_ms_down(tell=False)
def ping_succeeded():
""" Indicate a metasmoke status ping connection success """
with Metasmoke.AutoSwitch.rw_lock:
if Metasmoke.AutoSwitch.ping_failure_counter > 0:
# Consecutive counter. Switch sign.
Metasmoke.AutoSwitch.ping_failure_counter = 0
Metasmoke.AutoSwitch.ping_failure_counter -= 1
# Negative values for success
current_counter = -Metasmoke.AutoSwitch.ping_failure_counter
current_auto = Metasmoke.AutoSwitch.autoswitch_is_on
# MAX_SUCCESSES is constant so no lock.
if current_counter > Metasmoke.AutoSwitch.MAX_SUCCESSES and\
GlobalVars.MSStatus.is_down() and current_auto:
# Why use warning? Because some action may be needed if people don't think metasmoke is up.
log(
"warning",
"Last {} connection(s) to metasmoke succeeded".format(
current_counter) + " Setting metasmoke status to up.")
chatcommunicate.tell_rooms_with(
"debug", "**Notice**: {}: ".format(GlobalVars.location) +
"Last {} connection(s) to metasmoke".format(
current_counter) +
" succeeded. Setting metasmoke status to **up**.")
Metasmoke.set_ms_up(tell=False)
def switch_to_active():
GlobalVars.standby_mode = False
chatcommunicate.tell_rooms_with("debug", GlobalVars.location + " entering autonomous failover.",
notify_site="/failover")
def handle_websocket_data(data):
if "message" not in data:
return
message = data['message']
if isinstance(message, Iterable):
if "message" in message:
chatcommunicate.tell_rooms_with("debug", message['message'])
elif "exit" in message:
os._exit(message["exit"])
elif "blacklist" in message:
datahandling.add_blacklisted_user((message['blacklist']['uid'], message['blacklist']['site']),
"metasmoke", message['blacklist']['post'])
elif "naa" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"])
datahandling.add_ignored_post(post_site_id[0:2])
elif "fp" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"])
datahandling.add_false_positive(post_site_id[0:2])
elif "report" in message:
post_data = apigetpost.api_get_post(message["report"]["post_link"])
if post_data is None or post_data is False:
return
if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \
and not datahandling.is_false_positive((post_data.post_id, post_data.site)):
return
user = parsing.get_user_from_url(post_data.owner_url)
if user is not None:
datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url)
why = u"Post manually reported by user *{}* from metasmoke.\n".format(message["report"]["user"])
postobj = classes.Post(api_response={'title': post_data.title, 'body': post_data.body,
'owner': {'display_name': post_data.owner_name,
'reputation': post_data.owner_rep,
'link': post_data.owner_url},
'site': post_data.site,
'is_answer': (post_data.post_type == "answer"),
'score': post_data.score, 'link': post_data.post_url,
'question_id': post_data.post_id,
'up_vote_count': post_data.up_vote_count,
'down_vote_count': post_data.down_vote_count})
spamhandling.handle_spam(post=postobj,
reasons=["Manually reported " + post_data.post_type],
why=why)
elif "deploy_updated" in message:
sha = message["deploy_updated"]["head_commit"]["id"]
if sha != os.popen('git log --pretty=format:"%H" -n 1').read():
if "autopull" in message["deploy_updated"]["head_commit"]["message"]:
if only_blacklists_changed(GitManager.get_remote_diff()):
commit_md = "[`{0}`](https://github.com/Charcoal-SE/SmokeDetector/commit/{0})" \
.format(sha[:7])
i = [] # Currently no issues with backlists
for bl_file in glob('bad_*.txt') + glob('blacklisted_*.txt'): # Check blacklists for issues
with open(bl_file, 'r') as lines:
seen = dict()
for lineno, line in enumerate(lines, 1):
if line.endswith('\r\n'):
i.append("DOS line ending at `{0}:{1}` in {2}".format(bl_file, lineno,
commit_md))
if not line.endswith('\n'):
i.append("No newline at end of `{0}` in {1}".format(bl_file, commit_md))
if line == '\n':
i.append("Blank line at `{0}:{1}` in {2}".format(bl_file, lineno,
commit_md))
if line in seen:
i.append("Duplicate entry of {0} at lines {1} and {2} of {3} in {4}"
.format(line.rstrip('\n'), seen[line], lineno, bl_file, commit_md))
seen[line] = lineno
if i == []: # No issues
GitManager.pull_remote()
load_blacklists()
chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists"
" reloaded.".format(commit_md))
else:
i.append("please fix before pulling.")
chatcommunicate.tell_rooms_with("debug", ", ".join(i))
elif "commit_status" in message:
c = message["commit_status"]
sha = c["commit_sha"][:7]
if c["commit_sha"] != os.popen('git log --pretty=format:"%H" -n 1').read():
if c["status"] == "success":
if "autopull" in c["commit_message"]:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
"commit/{commit_sha})"\
" succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"],
commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s)
time.sleep(2)
os._exit(3)
else:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
"commit/{commit_sha}) succeeded.".format(ci_link=c["ci_url"], commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s)
elif c["status"] == "failure":
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
"commit/{commit_sha}) failed.".format(ci_link=c["ci_url"], commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s)
elif "everything_is_broken" in message:
if message["everything_is_broken"] is True:
os._exit(6)
"at [rev " +\
GlobalVars.commit_with_author +\
"](" + GlobalVars.bot_repository + "/commit/" +\
GlobalVars.commit['id'] +\
") (running on " +\
GlobalVars.location +\
")"
GlobalVars.standby_mode = "standby" in sys.argv
chatcommunicate.init(username, password)
Tasks.periodic(Metasmoke.send_status_ping, interval=60)
Tasks.periodic(Metasmoke.check_last_pingtime, interval=30)
if GlobalVars.standby_mode:
chatcommunicate.tell_rooms_with("debug", GlobalVars.standby_message)
Metasmoke.send_status_ping()
while GlobalVars.standby_mode:
time.sleep(3)
chatcommunicate.init(username, password) # to rejoin rooms
# noinspection PyProtectedMember
def check_socket_connections():
for client in chatcommunicate._clients.values():
if client.last_activity and (datetime.utcnow() - client.last_activity).total_seconds() >= 60:
os._exit(10)
def make_api_call_for_site(self, site):
with self.queue_lock:
new_posts = self.queue.pop(site, None)
if new_posts is None:
# site was not in the queue
return
Tasks.do(store_bodyfetcher_queue)
new_post_ids = [int(k) for k in new_posts.keys()]
if GlobalVars.flovis is not None:
for post_id in new_post_ids:
GlobalVars.flovis.stage('bodyfetcher/api_request', site,
post_id, {
'site': site,
'posts': list(new_posts.keys())
})
# Add queue timing data
pop_time = datetime.utcnow()
post_add_times = [(pop_time - v).total_seconds()
for k, v in new_posts.items()]
Tasks.do(add_queue_timing_data, site, post_add_times)
store_max_ids = False
with self.max_ids_modify_lock:
if site in self.previous_max_ids and max(
new_post_ids) > self.previous_max_ids[site]:
previous_max_id = self.previous_max_ids[site]
intermediate_posts = range(previous_max_id + 1,
max(new_post_ids))
# We don't want to go over the 100-post API cutoff, so take the last
# (100-len(new_post_ids)) from intermediate_posts
intermediate_posts = intermediate_posts[-(100 -
len(new_post_ids)):]
# new_post_ids could contain edited posts, so merge it back in
combined = chain(intermediate_posts, new_post_ids)
# Could be duplicates, so uniquify
posts = list(set(combined))
else:
posts = new_post_ids
new_post_ids_max = max(new_post_ids)
if new_post_ids_max > self.previous_max_ids.get(site, 0):
self.previous_max_ids[site] = new_post_ids_max
store_max_ids = True
if store_max_ids:
schedule_store_bodyfetcher_max_ids()
log('debug', "New IDs / Hybrid Intermediate IDs for {}:".format(site))
if len(new_post_ids) > 30:
log(
'debug', "{} +{} more".format(
sorted(new_post_ids)[:30],
len(new_post_ids) - 30))
else:
log('debug', sorted(new_post_ids))
if len(new_post_ids) == len(posts):
log('debug', "[ *Identical* ]")
elif len(posts) > 30:
log('debug',
"{} +{} more".format(sorted(posts)[:30],
len(posts) - 30))
else:
log('debug', sorted(posts))
question_modifier = ""
pagesize_modifier = {}
if site == "stackoverflow.com":
# Not all SO questions are shown in the realtime feed. We now
# fetch all recently modified SO questions to work around that.
with self.last_activity_date_lock:
if self.last_activity_date != 0:
pagesize = "100"
else:
pagesize = "50"
pagesize_modifier = {
'pagesize':
pagesize,
'min':
str(self.last_activity_date -
self.ACTIVITY_DATE_EXTRA_EARLIER_MS_TO_FETCH)
}
else:
question_modifier = "/{0}".format(";".join(
[str(post) for post in posts]))
url = "https://api.stackexchange.com/2.2/questions{}".format(
question_modifier)
params = {
'filter':
'!1rs)sUKylwB)8isvCRk.xNu71LnaxjnPS12*pX*CEOKbPFwVFdHNxiMa7GIVgzDAwMa',
'key': 'IAkbitmze4B8KpacUfLqkw((',
'site': site
}
params.update(pagesize_modifier)
# wait to make sure API has/updates post data
time.sleep(3)
with GlobalVars.api_request_lock:
# Respect backoff, if we were given one
if GlobalVars.api_backoff_time > time.time():
time.sleep(GlobalVars.api_backoff_time - time.time() + 2)
try:
time_request_made = datetime.utcnow().strftime('%H:%M:%S')
response = requests.get(url, params=params, timeout=20).json()
except (requests.exceptions.Timeout, requests.ConnectionError,
Exception):
# Any failure in the request being made (timeout or otherwise) should be added back to
# the queue.
with self.queue_lock:
if site in self.queue:
self.queue[site].update(new_posts)
else:
self.queue[site] = new_posts
return
with self.api_data_lock:
add_or_update_api_data(site)
message_hq = ""
with GlobalVars.apiquota_rw_lock:
if "quota_remaining" in response:
quota_remaining = response["quota_remaining"]
if quota_remaining - GlobalVars.apiquota >= 5000 and GlobalVars.apiquota >= 0 \
and quota_remaining > 39980:
tell_rooms_with(
"debug",
"API quota rolled over with {0} requests remaining. "
"Current quota: {1}.".format(
GlobalVars.apiquota, quota_remaining))
sorted_calls_per_site = sorted(
GlobalVars.api_calls_per_site.items(),
key=itemgetter(1),
reverse=True)
api_quota_used_per_site = ""
for site_name, quota_used in sorted_calls_per_site:
sanatized_site_name = site_name.replace(
'.com', '').replace('.stackexchange', '')
api_quota_used_per_site += sanatized_site_name + ": {0}\n".format(
str(quota_used))
api_quota_used_per_site = api_quota_used_per_site.strip(
)
tell_rooms_with("debug", api_quota_used_per_site)
clear_api_data()
if quota_remaining == 0:
tell_rooms_with(
"debug",
"API reports no quota left! May be a glitch.")
tell_rooms_with(
"debug", str(response)) # No code format for now?
if GlobalVars.apiquota == -1:
tell_rooms_with(
"debug", "Restart: API quota is {quota}.".format(
quota=quota_remaining))
GlobalVars.apiquota = quota_remaining
else:
message_hq = "The quota_remaining property was not in the API response."
if "error_message" in response:
message_hq += " Error: {} at {} UTC.".format(
response["error_message"], time_request_made)
if "error_id" in response and response["error_id"] == 502:
if GlobalVars.api_backoff_time < time.time(
) + 12: # Add a backoff of 10 + 2 seconds as a default
GlobalVars.api_backoff_time = time.time() + 12
message_hq += " Backing off on requests for the next 12 seconds."
message_hq += " Previous URL: `{}`".format(url)
if "backoff" in response:
if GlobalVars.api_backoff_time < time.time(
) + response["backoff"]:
GlobalVars.api_backoff_time = time.time(
) + response["backoff"]
if len(message_hq) > 0 and "site is required" not in message_hq:
message_hq = message_hq.strip()
if len(message_hq) > 500:
message_hq = "\n" + message_hq
tell_rooms_with("debug", message_hq)
if "items" not in response:
return
if site == "stackoverflow.com":
items = response["items"]
if len(items) > 0 and "last_activity_date" in items[0]:
with self.last_activity_date_lock:
self.last_activity_date = items[0]["last_activity_date"]
num_scanned = 0
start_time = time.time()
for post in response["items"]:
if GlobalVars.flovis is not None:
pnb = copy.deepcopy(post)
if 'body' in pnb:
pnb['body'] = 'Present, but truncated'
if 'answers' in pnb:
del pnb['answers']
if "title" not in post or "body" not in post:
if GlobalVars.flovis is not None and 'question_id' in post:
GlobalVars.flovis.stage(
'bodyfetcher/api_response/no_content', site,
post['question_id'], pnb)
continue
post['site'] = site
try:
post['edited'] = (post['creation_date'] !=
post['last_edit_date'])
except KeyError:
post[
'edited'] = False # last_edit_date not present = not edited
question_doesnt_need_scan = is_post_recently_scanned_and_unchanged(
post)
add_recently_scanned_post(post)
if not question_doesnt_need_scan:
try:
post_ = Post(api_response=post)
except PostParseError as err:
log(
'error', 'Error {0} when parsing post: {1!r}'.format(
err, post_))
if GlobalVars.flovis is not None and 'question_id' in post:
GlobalVars.flovis.stage(
'bodyfetcher/api_response/error', site,
post['question_id'], pnb)
continue
num_scanned += 1
is_spam, reason, why = check_if_spam(post_)
if is_spam:
try:
if GlobalVars.flovis is not None and 'question_id' in post:
GlobalVars.flovis.stage(
'bodyfetcher/api_response/spam', site,
post['question_id'], {
'post': pnb,
'check_if_spam': [is_spam, reason, why]
})
handle_spam(post=post_, reasons=reason, why=why)
except Exception as e:
log('error', "Exception in handle_spam:", e)
elif GlobalVars.flovis is not None and 'question_id' in post:
GlobalVars.flovis.stage(
'bodyfetcher/api_response/not_spam', site,
post['question_id'], {
'post': pnb,
'check_if_spam': [is_spam, reason, why]
})
try:
if "answers" not in post:
pass
else:
for answer in post["answers"]:
if GlobalVars.flovis is not None:
anb = copy.deepcopy(answer)
if 'body' in anb:
anb['body'] = 'Present, but truncated'
num_scanned += 1
answer["IsAnswer"] = True # Necesssary for Post object
answer[
"title"] = "" # Necessary for proper Post object creation
answer[
"site"] = site # Necessary for proper Post object creation
try:
answer['edited'] = (answer['creation_date'] !=
answer['last_edit_date'])
except KeyError:
answer[
'edited'] = False # last_edit_date not present = not edited
answer_doesnt_need_scan = is_post_recently_scanned_and_unchanged(
answer)
add_recently_scanned_post(answer)
if answer_doesnt_need_scan:
continue
answer_ = Post(api_response=answer, parent=post_)
is_spam, reason, why = check_if_spam(answer_)
if is_spam:
try:
if GlobalVars.flovis is not None and 'answer_id' in answer:
GlobalVars.flovis.stage(
'bodyfetcher/api_response/spam', site,
answer['answer_id'], {
'post': anb,
'check_if_spam':
[is_spam, reason, why]
})
handle_spam(answer_, reasons=reason, why=why)
except Exception as e:
log('error', "Exception in handle_spam:", e)
elif GlobalVars.flovis is not None and 'answer_id' in answer:
GlobalVars.flovis.stage(
'bodyfetcher/api_response/not_spam', site,
answer['answer_id'], {
'post': anb,
'check_if_spam': [is_spam, reason, why]
})
except Exception as e:
log('error', "Exception handling answers:", e)
end_time = time.time()
scan_time = end_time - start_time
GlobalVars.PostScanStat.add_stat(num_scanned, scan_time)
return
if GlobalVars.flovis_host:
GlobalVars.flovis = Flovis(GlobalVars.flovis_host)
load_files()
load_ms_cache_data()
filter_auto_ignored_posts()
GlobalVars.standby_mode = "standby" in sys.argv
GlobalVars.no_se_activity_scan = 'no_se_activity_scan' in sys.argv
chatcommunicate.init(username, password)
Tasks.periodic(Metasmoke.send_status_ping_and_verify_scanning_if_active, interval=60)
if GlobalVars.standby_mode:
chatcommunicate.tell_rooms_with("debug", GlobalVars.standby_message)
Metasmoke.send_status_ping()
while GlobalVars.standby_mode:
time.sleep(3)
chatcommunicate.join_command_rooms()
# noinspection PyProtectedMember
def check_socket_connections():
for client in chatcommunicate._clients.values():
if client.last_activity and (datetime.utcnow() - client.last_activity).total_seconds() >= 60:
exit_mode("socket_failure")
def make_api_call_for_site(self, site):
if site not in self.queue:
return
self.queue_modify_lock.acquire()
new_posts = self.queue.pop(site)
store_bodyfetcher_queue()
self.queue_modify_lock.release()
new_post_ids = [int(k) for k, v in new_posts.items()]
if GlobalVars.flovis is not None:
for post_id in new_post_ids:
GlobalVars.flovis.stage('bodyfetcher/api_request', site, post_id,
{'queue':
dict([[sk, [k for k, v in sq.items()]] for sk, sq in self.queue.items()]),
'site': site, 'posts': [k for k, v in new_posts.items()]})
self.queue_timing_modify_lock.acquire()
post_add_times = [v for k, v in new_posts.items()]
pop_time = datetime.utcnow()
for add_time in post_add_times:
try:
seconds_in_queue = (pop_time - add_time).total_seconds()
if site in self.queue_timings:
self.queue_timings[site].append(seconds_in_queue)
else:
self.queue_timings[site] = [seconds_in_queue]
except:
continue # Skip to next item if we've got invalid data or missing values.
store_queue_timings()
self.queue_timing_modify_lock.release()
self.max_ids_modify_lock.acquire()
if site in self.previous_max_ids and max(new_post_ids) > self.previous_max_ids[site]:
previous_max_id = self.previous_max_ids[site]
intermediate_posts = range(previous_max_id + 1, max(new_post_ids))
# We don't want to go over the 100-post API cutoff, so take the last
# (100-len(new_post_ids)) from intermediate_posts
intermediate_posts = intermediate_posts[(100 - len(new_post_ids)):]
# new_post_ids could contain edited posts, so merge it back in
combined = chain(intermediate_posts, new_post_ids)
# Could be duplicates, so uniquify
posts = list(set(combined))
else:
posts = new_post_ids
try:
if max(new_post_ids) > self.previous_max_ids[site]:
self.previous_max_ids[site] = max(new_post_ids)
store_bodyfetcher_max_ids()
except KeyError:
self.previous_max_ids[site] = max(new_post_ids)
store_bodyfetcher_max_ids()
self.max_ids_modify_lock.release()
log('debug', "New IDs / Hybrid Intermediate IDs for {0}:".format(site))
log('debug', sorted(new_post_ids))
log('debug', sorted(posts))
question_modifier = ""
pagesize_modifier = ""
if site == "stackoverflow.com":
# Not all SO questions are shown in the realtime feed. We now
# fetch all recently modified SO questions to work around that.
if self.last_activity_date != 0:
pagesize = "50"
else:
pagesize = "25"
pagesize_modifier = "&pagesize={pagesize}" \
"&min={time_length}".format(pagesize=pagesize, time_length=str(self.last_activity_date))
else:
question_modifier = "/{0}".format(";".join(str(post) for post in posts))
url = "https://api.stackexchange.com/2.2/questions{q_modifier}?site={site}" \
"&filter=!)E0g*ODaEZ(SgULQhYvCYbu09*ss(bKFdnTrGmGUxnqPptuHP&key=IAkbitmze4B8KpacUfLqkw((" \
"{optional_min_query_param}".format(q_modifier=question_modifier, site=site,
optional_min_query_param=pagesize_modifier)
# wait to make sure API has/updates post data
time.sleep(3)
GlobalVars.api_request_lock.acquire()
# Respect backoff, if we were given one
if GlobalVars.api_backoff_time > time.time():
time.sleep(GlobalVars.api_backoff_time - time.time() + 2)
try:
time_request_made = datetime.now().strftime('%H:%M:%S')
response = requests.get(url, timeout=20).json()
except (requests.exceptions.Timeout, requests.ConnectionError, Exception):
# Any failure in the request being made (timeout or otherwise) should be added back to
# the queue.
self.queue_modify_lock.acquire()
if site in self.queue:
self.queue[site].update(new_posts)
else:
self.queue[site] = new_posts
self.queue_modify_lock.release()
GlobalVars.api_request_lock.release()
return
self.api_data_lock.acquire()
add_or_update_api_data(site)
self.api_data_lock.release()
message_hq = ""
if "quota_remaining" in response:
if response["quota_remaining"] - GlobalVars.apiquota >= 5000 and GlobalVars.apiquota >= 0:
tell_rooms_with("debug", "API quota rolled over with {0} requests remaining. "
"Current quota: {1}.".format(GlobalVars.apiquota,
response["quota_remaining"]))
sorted_calls_per_site = sorted(GlobalVars.api_calls_per_site.items(), key=itemgetter(1), reverse=True)
api_quota_used_per_site = ""
for site_name, quota_used in sorted_calls_per_site:
sanatized_site_name = site_name.replace('.com', '').replace('.stackexchange', '')
api_quota_used_per_site += sanatized_site_name + ": {0}\n".format(str(quota_used))
api_quota_used_per_site = api_quota_used_per_site.strip()
tell_rooms_with("debug", api_quota_used_per_site)
clear_api_data()
if response["quota_remaining"] == 0:
tell_rooms_with("debug", "API reports no quota left! May be a glitch.")
tell_rooms_with("debug", str(response)) # No code format for now?
if GlobalVars.apiquota == -1:
tell_rooms_with("debug", "Restart: API quota is {quota}."
.format(quota=response["quota_remaining"]))
GlobalVars.apiquota = response["quota_remaining"]
else:
message_hq = "The quota_remaining property was not in the API response."
if "error_message" in response:
message_hq += " Error: {} at {} UTC.".format(response["error_message"], time_request_made)
if "error_id" in response and response["error_id"] == 502:
if GlobalVars.api_backoff_time < time.time() + 12: # Add a backoff of 10 + 2 seconds as a default
GlobalVars.api_backoff_time = time.time() + 12
message_hq += " Backing off on requests for the next 12 seconds."
message_hq += " Previous URL: `{}`".format(url)
if "backoff" in response:
if GlobalVars.api_backoff_time < time.time() + response["backoff"]:
GlobalVars.api_backoff_time = time.time() + response["backoff"]
GlobalVars.api_request_lock.release()
if len(message_hq) > 0:
tell_rooms_with("debug", message_hq.strip())
if "items" not in response:
return
if site == "stackoverflow.com":
items = response["items"]
if len(items) > 0 and "last_activity_date" in items[0]:
self.last_activity_date = items[0]["last_activity_date"]
num_scanned = 0
start_time = time.time()
for post in response["items"]:
if "title" not in post or "body" not in post:
if GlobalVars.flovis is not None and 'question_id' in post:
GlobalVars.flovis.stage('bodyfetcher/api_response/no_content', site, post['question_id'], post)
continue
post['site'] = site
try:
post_ = Post(api_response=post)
except PostParseError as err:
log('error', 'Error {0} when parsing post: {1!r}'.format(err, post_))
if GlobalVars.flovis is not None and 'question_id' in post:
GlobalVars.flovis.stage('bodyfetcher/api_response/error', site, post['question_id'], post)
continue
num_scanned += 1
is_spam, reason, why = check_if_spam(post_)
if is_spam:
try:
if GlobalVars.flovis is not None and 'question_id' in post:
GlobalVars.flovis.stage('bodyfetcher/api_response/spam', site, post['question_id'],
{'post': post, 'check_if_spam': [is_spam, reason, why]})
handle_spam(post=post_,
reasons=reason,
why=why)
except Exception as e:
log('error', "Exception in handle_spam:", e)
elif GlobalVars.flovis is not None and 'question_id' in post:
GlobalVars.flovis.stage('bodyfetcher/api_response/not_spam', site, post['question_id'],
{'post': post, 'check_if_spam': [is_spam, reason, why]})
try:
if "answers" not in post:
pass
else:
for answer in post["answers"]:
num_scanned += 1
answer["IsAnswer"] = True # Necesssary for Post object
answer["title"] = "" # Necessary for proper Post object creation
answer["site"] = site # Necessary for proper Post object creation
answer_ = Post(api_response=answer, parent=post_)
is_spam, reason, why = check_if_spam(answer_)
if is_spam:
try:
if GlobalVars.flovis is not None and 'answer_id' in answer:
GlobalVars.flovis.stage('bodyfetcher/api_response/spam', site, answer['answer_id'],
{'post': answer, 'check_if_spam': [is_spam, reason, why]})
handle_spam(answer_,
reasons=reason,
why=why)
except Exception as e:
log('error', "Exception in handle_spam:", e)
elif GlobalVars.flovis is not None and 'answer_id' in answer:
GlobalVars.flovis.stage('bodyfetcher/api_response/not_spam', site, answer['answer_id'],
{'post': answer, 'check_if_spam': [is_spam, reason, why]})
except Exception as e:
log('error', "Exception handling answers:", e)
end_time = time.time()
GlobalVars.posts_scan_stats_lock.acquire()
GlobalVars.num_posts_scanned += num_scanned
GlobalVars.post_scan_time += end_time - start_time
GlobalVars.posts_scan_stats_lock.release()
return
def handle_websocket_data(data):
if "message" not in data:
if "type" in data and data['type'] == "reject_subscription":
log(
'error',
"MS WebSocket subscription was rejected. Check your MS key."
)
raise ConnectionError("MS WebSocket connection rejected")
return
message = data['message']
if not isinstance(message, Iterable):
return
if "message" in message:
chatcommunicate.tell_rooms_with("metasmoke", message['message'])
elif "autoflag_fp" in message:
event = message["autoflag_fp"]
chatcommunicate.tell_rooms(event["message"],
("debug", "site-" + event["site"]),
("no-site-" + event["site"], ),
notify_site="/autoflag_fp")
elif "exit" in message:
os._exit(message["exit"])
elif "blacklist" in message:
ids = (message['blacklist']['uid'], message['blacklist']['site'])
datahandling.add_blacklisted_user(ids, "metasmoke",
message['blacklist']['post'])
datahandling.last_feedbacked = (ids, time.time() + 60)
elif "unblacklist" in message:
ids = (message['unblacklist']['uid'],
message['unblacklist']['site'])
datahandling.remove_blacklisted_user(ids)
elif "naa" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(
message["naa"]["post_link"])
datahandling.add_ignored_post(post_site_id[0:2])
elif "fp" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(
message["fp"]["post_link"])
datahandling.add_false_positive(post_site_id[0:2])
elif "report" in message:
import chatcommands # Do it here
chatcommands.report_posts([message["report"]["post_link"]],
message["report"]["user"], True,
"the metasmoke API")
elif "deploy_updated" in message:
return # Disabled
sha = message["deploy_updated"]["head_commit"]["id"]
if sha != os.popen('git log -1 --pretty="%H"').read():
if "autopull" in message["deploy_updated"]["head_commit"][
"message"]:
if only_blacklists_changed(GitManager.get_remote_diff()):
commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \
.format(sha[:7], GlobalVars.bot_repo_slug)
integrity = blacklist_integrity_check()
if len(integrity) == 0: # No issues
GitManager.pull_remote()
findspam.reload_blacklists()
chatcommunicate.tell_rooms_with(
"debug",
"No code modified in {0}, only blacklists"
" reloaded.".format(commit_md))
else:
integrity.append("please fix before pulling.")
chatcommunicate.tell_rooms_with(
"debug", ", ".join(integrity))
elif "commit_status" in message:
c = message["commit_status"]
sha = c["commit_sha"][:7]
recent_commits = sp.check_output(
["git", "log", "-50",
"--pretty=%H"]).decode('utf-8').strip().split('\n')
if c["commit_sha"] in recent_commits:
return # Same rev, or earlier rev (e.g. when watching things faster than CI completes), nothing to do
if c["status"] == "success":
if "autopull" in c["commit_message"] or c["commit_message"].startswith("!") or \
c["commit_message"].startswith("Auto "):
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \
"commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format(
ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
remote_diff = GitManager.get_remote_diff()
if only_blacklists_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_branch:
# Restart if HEAD detached
log('warning',
"Pulling remote with HEAD detached, checkout deploy",
f=True)
exit_mode("checkout_deploy")
GlobalVars.reload()
findspam.FindSpam.reload_blacklists()
chatcommunicate.tell_rooms_with(
'debug', GlobalVars.s_norestart_blacklists)
elif only_modules_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_branch:
# Restart if HEAD detached
log('warning',
"Pulling remote with HEAD detached, checkout deploy",
f=True)
exit_mode("checkout_deploy")
GlobalVars.reload()
reload_modules()
chatcommunicate.tell_rooms_with(
'debug', GlobalVars.s_norestart_findspam)
else:
chatcommunicate.tell_rooms_with('debug',
s,
notify_site="/ci")
exit_mode("pull_update")
else:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug",
s,
notify_site="/ci")
elif c["status"] == "failure":
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
elif "everything_is_broken" in message:
if message["everything_is_broken"] is True:
exit_mode("shutdown")
elif "domain_whitelist" in message:
if message["domain_whitelist"] == "refresh":
metasmoke_cache.MetasmokeCache.delete('whitelisted-domains')
def send_status_ping():
if GlobalVars.metasmoke_host is None:
log(
'info',
'Attempted to send status ping but metasmoke_host is undefined. Not sent.'
)
return
elif GlobalVars.metasmoke_down:
payload = {
"location": GlobalVars.location,
"timestamp": time.time()
}
SocketScience.send(payload)
metasmoke_key = GlobalVars.metasmoke_key
try:
payload = {
'location':
GlobalVars.location,
'key':
metasmoke_key,
'standby':
GlobalVars.standby_mode or GlobalVars.no_se_activity_scan
}
headers = {'content-type': 'application/json'}
response = Metasmoke.post("/status-update.json",
data=json.dumps(payload),
headers=headers,
ignore_down=True)
try:
response = response.json()
GlobalVars.metasmoke_last_ping_time = datetime.utcnow(
) # Otherwise the ping watcher will exit(10)
if response.get('pull_update', False):
log('info', "Received pull command from MS ping response")
exit_mode("pull_update")
if ('failover' in response and GlobalVars.standby_mode
and not GlobalVars.no_se_activity_scan):
# If we're not scanning, then we don't want to become officially active due to failover.
if response['failover']:
GlobalVars.standby_mode = False
chatcommunicate.tell_rooms_with(
"debug",
GlobalVars.location + " received failover signal.",
notify_site="/failover")
if response.get('standby', False):
chatcommunicate.tell_rooms_with(
"debug", GlobalVars.location +
" entering metasmoke-forced standby.")
time.sleep(2)
exit_mode("standby")
if response.get('shutdown', False):
exit_mode("shutdown")
except Exception: # TODO: What could happen here?
pass
except Exception as e:
log('error', e)
def handle_websocket_data(data):
if "message" not in data:
return
message = data['message']
if not isinstance(message, Iterable):
return
if "message" in message:
chatcommunicate.tell_rooms_with("metasmoke", message['message'])
elif "autoflag_fp" in message:
event = message["autoflag_fp"]
chatcommunicate.tell_rooms(event["message"], ("debug", "site-" + event["site"]),
("no-site-" + event["site"],), notify_site="/autoflag_fp")
elif "exit" in message:
os._exit(message["exit"])
elif "blacklist" in message:
ids = (message['blacklist']['uid'], message['blacklist']['site'])
datahandling.add_blacklisted_user(ids, "metasmoke", message['blacklist']['post'])
datahandling.last_feedbacked = (ids, time.time() + 60)
elif "unblacklist" in message:
ids = (message['unblacklist']['uid'], message['unblacklist']['site'])
datahandling.remove_blacklisted_user(ids)
elif "naa" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"])
datahandling.add_ignored_post(post_site_id[0:2])
elif "fp" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"])
datahandling.add_false_positive(post_site_id[0:2])
elif "report" in message:
import chatcommands # Do it here
chatcommands.report_posts([message["report"]["post_link"]], message["report"]["user"],
True, "the metasmoke API")
elif "deploy_updated" in message:
return # Disabled
sha = message["deploy_updated"]["head_commit"]["id"]
if sha != os.popen('git log -1 --pretty="%H"').read():
if "autopull" in message["deploy_updated"]["head_commit"]["message"]:
if only_blacklists_changed(GitManager.get_remote_diff()):
commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \
.format(sha[:7], GlobalVars.bot_repo_slug)
integrity = blacklist_integrity_check()
if len(integrity) == 0: # No issues
GitManager.pull_remote()
findspam.reload_blacklists()
chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists"
" reloaded.".format(commit_md))
else:
integrity.append("please fix before pulling.")
chatcommunicate.tell_rooms_with("debug", ", ".join(integrity))
elif "commit_status" in message:
c = message["commit_status"]
sha = c["commit_sha"][:7]
if c["commit_sha"] == sp.check_output(["git", "log", "-1", "--pretty=%H"]).decode('utf-8').strip():
return # Same rev, nothing to do
if c["status"] == "success":
if "autopull" in c["commit_message"] or c["commit_message"].startswith("!") or \
c["commit_message"].startswith("Auto "):
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \
"commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format(
ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
remote_diff = GitManager.get_remote_diff()
if only_blacklists_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_branch:
# Restart if HEAD detached
log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True)
exit_mode("checkout_deploy")
GlobalVars.reload()
findspam.FindSpam.reload_blacklists()
chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart)
elif only_modules_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_branch:
# Restart if HEAD detached
log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True)
exit_mode("checkout_deploy")
GlobalVars.reload()
reload_modules()
chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart2)
else:
chatcommunicate.tell_rooms_with('debug', s, notify_site="/ci")
exit_mode("pull_update")
else:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
elif c["status"] == "failure":
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
elif "everything_is_broken" in message:
if message["everything_is_broken"] is True:
exit_mode("shutdown")
def handle_websocket_data(data):
if "message" not in data:
return
message = data['message']
if not isinstance(message, Iterable):
return
if "message" in message:
chatcommunicate.tell_rooms_with("metasmoke", message['message'])
elif "autoflag_fp" in message:
event = message["autoflag_fp"]
chatcommunicate.tell_rooms(event["message"],
("debug", "site-" + event["site"]),
("no-site-" + event["site"], ),
notify_site="/autoflag_fp")
elif "exit" in message:
os._exit(message["exit"])
elif "blacklist" in message:
ids = (message['blacklist']['uid'], message['blacklist']['site'])
datahandling.add_blacklisted_user(ids, "metasmoke",
message['blacklist']['post'])
datahandling.last_feedbacked = (ids, time.time() + 60)
elif "unblacklist" in message:
ids = (message['unblacklist']['uid'],
message['unblacklist']['site'])
datahandling.remove_blacklisted_user(ids)
elif "naa" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(
message["naa"]["post_link"])
datahandling.add_ignored_post(post_site_id[0:2])
elif "fp" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(
message["fp"]["post_link"])
datahandling.add_false_positive(post_site_id[0:2])
elif "report" in message:
import chatcommands # Do it here
chatcommands.report_posts([message["report"]["post_link"]],
"the metasmoke API", None,
"the metasmoke API")
return
post_data = apigetpost.api_get_post(message["report"]["post_link"])
if post_data is None or post_data is False:
return
if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \
and not datahandling.is_false_positive((post_data.post_id, post_data.site)):
return
user = parsing.get_user_from_url(post_data.owner_url)
post = classes.Post(api_response=post_data.as_dict)
scan_spam, scan_reasons, scan_why = spamhandling.check_if_spam(
post)
if scan_spam:
why_append = u"This post would have also been caught for: " + \
u", ".join(scan_reasons).capitalize() + "\n" + scan_why
else:
why_append = u"This post would not have been caught otherwise."
# Add user to blacklist *after* post is scanned
if user is not None:
datahandling.add_blacklisted_user(user, "metasmoke",
post_data.post_url)
why = u"Post manually reported by user *{}* from metasmoke.\n\n{}".format(
message["report"]["user"], why_append)
spamhandling.handle_spam(
post=post,
reasons=["Manually reported " + post_data.post_type],
why=why)
elif "deploy_updated" in message:
return # Disabled
sha = message["deploy_updated"]["head_commit"]["id"]
if sha != os.popen('git log -1 --pretty="%H"').read():
if "autopull" in message["deploy_updated"]["head_commit"][
"message"]:
if only_blacklists_changed(GitManager.get_remote_diff()):
commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \
.format(sha[:7], GlobalVars.bot_repo_slug)
integrity = blacklist_integrity_check()
if len(integrity) == 0: # No issues
GitManager.pull_remote()
findspam.reload_blacklists()
chatcommunicate.tell_rooms_with(
"debug",
"No code modified in {0}, only blacklists"
" reloaded.".format(commit_md))
else:
integrity.append("please fix before pulling.")
chatcommunicate.tell_rooms_with(
"debug", ", ".join(integrity))
elif "commit_status" in message:
c = message["commit_status"]
sha = c["commit_sha"][:7]
if c["commit_sha"] == sp.check_output(
["git", "log", "-1", "--pretty=%H"]).decode('utf-8').strip():
return
if c["status"] == "success":
if "autopull" in c["commit_message"]:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \
"commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format(
ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
remote_diff = GitManager.get_remote_diff()
if only_blacklists_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_master:
# Restart if HEAD detached
log('warning',
"Pulling remote with HEAD detached, checkout deploy",
f=True)
os._exit(8)
GlobalVars.reload()
findspam.FindSpam.reload_blacklists()
chatcommunicate.tell_rooms_with(
'debug', GlobalVars.s_norestart)
elif only_modules_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_master:
# Restart if HEAD detached
log('warning',
"Pulling remote with HEAD detached, checkout deploy",
f=True)
os._exit(8)
GlobalVars.reload()
reload_modules()
chatcommunicate.tell_rooms_with(
'debug', GlobalVars.s_norestart2)
else:
chatcommunicate.tell_rooms_with('debug',
s,
notify_site="/ci")
os._exit(3)
else:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug",
s,
notify_site="/ci")
elif c["status"] == "failure":
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
elif "everything_is_broken" in message:
if message["everything_is_broken"] is True:
os._exit(6)
"at [rev " +\
GlobalVars.commit_with_author +\
"](" + GlobalVars.bot_repository + "/commit/" +\
GlobalVars.commit['id'] +\
") (running on " +\
GlobalVars.location +\
")"
GlobalVars.standby_mode = "standby" in sys.argv
chatcommunicate.init(username, password)
Tasks.periodic(Metasmoke.send_status_ping, interval=60)
Tasks.periodic(Metasmoke.check_last_pingtime, interval=30)
if GlobalVars.standby_mode:
chatcommunicate.tell_rooms_with("debug", GlobalVars.standby_message)
Metasmoke.send_status_ping()
while GlobalVars.standby_mode:
time.sleep(3)
chatcommunicate.join_command_rooms()
# noinspection PyProtectedMember
def check_socket_connections():
for client in chatcommunicate._clients.values():
if client.last_activity and (datetime.utcnow() - client.last_activity
).total_seconds() >= 60:
os._exit(10)
def handle_websocket_data(data):
if "message" not in data:
return
message = data['message']
if isinstance(message, Iterable):
if "message" in message:
chatcommunicate.tell_rooms_with("metasmoke",
message['message'])
elif "exit" in message:
os._exit(message["exit"])
elif "blacklist" in message:
datahandling.add_blacklisted_user(
(message['blacklist']['uid'],
message['blacklist']['site']), "metasmoke",
message['blacklist']['post'])
elif "unblacklist" in message:
datahandling.remove_blacklisted_user(
message['unblacklist']['uid'])
elif "naa" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(
message["naa"]["post_link"])
datahandling.add_ignored_post(post_site_id[0:2])
elif "fp" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(
message["fp"]["post_link"])
datahandling.add_false_positive(post_site_id[0:2])
elif "report" in message:
post_data = apigetpost.api_get_post(
message["report"]["post_link"])
if post_data is None or post_data is False:
return
if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \
and not datahandling.is_false_positive((post_data.post_id, post_data.site)):
return
user = parsing.get_user_from_url(post_data.owner_url)
if user is not None:
datahandling.add_blacklisted_user(user, "metasmoke",
post_data.post_url)
why = u"Post manually reported by user *{}* from metasmoke.\n".format(
message["report"]["user"])
postobj = classes.Post(
api_response={
'title': post_data.title,
'body': post_data.body,
'owner': {
'display_name': post_data.owner_name,
'reputation': post_data.owner_rep,
'link': post_data.owner_url
},
'site': post_data.site,
'is_answer': (post_data.post_type == "answer"),
'score': post_data.score,
'link': post_data.post_url,
'question_id': post_data.post_id,
'up_vote_count': post_data.up_vote_count,
'down_vote_count': post_data.down_vote_count
})
spamhandling.handle_spam(
post=postobj,
reasons=["Manually reported " + post_data.post_type],
why=why)
elif "deploy_updated" in message:
sha = message["deploy_updated"]["head_commit"]["id"]
if sha != os.popen('git log --pretty=format:"%H" -n 1').read():
if "autopull" in message["deploy_updated"]["head_commit"][
"message"]:
if only_blacklists_changed(
GitManager.get_remote_diff()):
commit_md = "[`{0}`](https://github.com/Charcoal-SE/SmokeDetector/commit/{0})" \
.format(sha[:7])
i = [] # Currently no issues with backlists
for bl_file in glob('bad_*.txt') + glob(
'blacklisted_*.txt'
): # Check blacklists for issues
with open(bl_file, 'r') as lines:
seen = dict()
for lineno, line in enumerate(lines, 1):
if line.endswith('\r\n'):
i.append(
"DOS line ending at `{0}:{1}` in {2}"
.format(
bl_file, lineno,
commit_md))
if not line.endswith('\n'):
i.append(
"No newline at end of `{0}` in {1}"
.format(bl_file, commit_md))
if line == '\n':
i.append(
"Blank line at `{0}:{1}` in {2}"
.format(
bl_file, lineno,
commit_md))
if line in seen:
i.append(
"Duplicate entry of {0} at lines {1} and {2} of {3} in {4}"
.format(
line.rstrip('\n'),
seen[line], lineno,
bl_file, commit_md))
seen[line] = lineno
if i == []: # No issues
GitManager.pull_remote()
load_blacklists()
chatcommunicate.tell_rooms_with(
"debug",
"No code modified in {0}, only blacklists"
" reloaded.".format(commit_md))
else:
i.append("please fix before pulling.")
chatcommunicate.tell_rooms_with(
"debug", ", ".join(i))
elif "commit_status" in message:
c = message["commit_status"]
sha = c["commit_sha"][:7]
if c["commit_sha"] != os.popen(
'git log --pretty=format:"%H" -n 1').read():
if c["status"] == "success":
if "autopull" in c["commit_message"]:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
"commit/{commit_sha})"\
" succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"],
commit_sha=sha)
chatcommunicate.tell_rooms_with("debug",
s,
notify_site="/ci")
time.sleep(2)
os._exit(3)
else:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
"commit/{commit_sha}) succeeded.".format(ci_link=c["ci_url"], commit_sha=sha)
chatcommunicate.tell_rooms_with("debug",
s,
notify_site="/ci")
elif c["status"] == "failure":
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
"commit/{commit_sha}) failed.".format(ci_link=c["ci_url"], commit_sha=sha)
chatcommunicate.tell_rooms_with("debug",
s,
notify_site="/ci")
elif "everything_is_broken" in message:
if message["everything_is_broken"] is True:
os._exit(6)
def send(cls, payload, single_message=True):
encoded = base64.b64encode(msgpack.dumps(payload)).decode("utf-8")
message_id = random.randint(0, 9999)
content = ".\n\u0002{:04}{}\u0003".format(message_id, encoded)
chatcommunicate.tell_rooms_with("direct", content)
def switch_to_active():
GlobalVars.standby_mode = False
chatcommunicate.tell_rooms_with("debug",
GlobalVars.location +
" entering autonomous failover.",
notify_site="/failover")
def make_api_call_for_site(self, site):
if site not in self.queue:
return
self.queue_modify_lock.acquire()
new_posts = self.queue.pop(site)
store_bodyfetcher_queue()
self.queue_modify_lock.release()
new_post_ids = [int(k) for k in new_posts.keys()]
if GlobalVars.flovis is not None:
for post_id in new_post_ids:
GlobalVars.flovis.stage('bodyfetcher/api_request', site, post_id,
{'site': site, 'posts': list(new_posts.keys())})
self.queue_timing_modify_lock.acquire()
post_add_times = [v for k, v in new_posts.items()]
pop_time = datetime.utcnow()
for add_time in post_add_times:
try:
seconds_in_queue = (pop_time - add_time).total_seconds()
if site in self.queue_timings:
self.queue_timings[site].append(seconds_in_queue)
else:
self.queue_timings[site] = [seconds_in_queue]
except KeyError: # XXX: Any other possible exception?
continue # Skip to next item if we've got invalid data or missing values.
store_queue_timings()
self.queue_timing_modify_lock.release()
self.max_ids_modify_lock.acquire()
if site in self.previous_max_ids and max(new_post_ids) > self.previous_max_ids[site]:
previous_max_id = self.previous_max_ids[site]
intermediate_posts = range(previous_max_id + 1, max(new_post_ids))
# We don't want to go over the 100-post API cutoff, so take the last
# (100-len(new_post_ids)) from intermediate_posts
intermediate_posts = intermediate_posts[(100 - len(new_post_ids)):]
# new_post_ids could contain edited posts, so merge it back in
combined = chain(intermediate_posts, new_post_ids)
# Could be duplicates, so uniquify
posts = list(set(combined))
else:
posts = new_post_ids
try:
if max(new_post_ids) > self.previous_max_ids[site]:
self.previous_max_ids[site] = max(new_post_ids)
store_bodyfetcher_max_ids()
except KeyError:
self.previous_max_ids[site] = max(new_post_ids)
store_bodyfetcher_max_ids()
self.max_ids_modify_lock.release()
log('debug', "New IDs / Hybrid Intermediate IDs for {}:".format(site))
if len(new_post_ids) > 30:
log('debug', "{} +{} more".format(sorted(new_post_ids)[:30], len(new_post_ids) - 30))
else:
log('debug', sorted(new_post_ids))
if len(new_post_ids) == len(posts):
log('debug', "[ *Identical* ]")
elif len(posts) > 30:
log('debug', "{} +{} more".format(sorted(posts)[:30], len(posts) - 30))
else:
log('debug', sorted(posts))
question_modifier = ""
pagesize_modifier = ""
if site == "stackoverflow.com":
# Not all SO questions are shown in the realtime feed. We now
# fetch all recently modified SO questions to work around that.
if self.last_activity_date != 0:
pagesize = "50"
else:
pagesize = "25"
pagesize_modifier = "&pagesize={pagesize}" \
"&min={time_length}".format(pagesize=pagesize, time_length=str(self.last_activity_date))
else:
question_modifier = "/{0}".format(";".join([str(post) for post in posts]))
url = "https://api.stackexchange.com/2.2/questions{q_modifier}?site={site}" \
"&filter=!*xq08dCDNr)PlxxXfaN8ntivx(BPlY_8XASyXLX-J7F-)VK*Q3KTJVkvp*&key=IAkbitmze4B8KpacUfLqkw((" \
"{optional_min_query_param}".format(q_modifier=question_modifier, site=site,
optional_min_query_param=pagesize_modifier)
# wait to make sure API has/updates post data
time.sleep(3)
GlobalVars.api_request_lock.acquire()
# Respect backoff, if we were given one
if GlobalVars.api_backoff_time > time.time():
time.sleep(GlobalVars.api_backoff_time - time.time() + 2)
try:
time_request_made = datetime.now().strftime('%H:%M:%S')
response = requests.get(url, timeout=20).json()
except (requests.exceptions.Timeout, requests.ConnectionError, Exception):
# Any failure in the request being made (timeout or otherwise) should be added back to
# the queue.
self.queue_modify_lock.acquire()
if site in self.queue:
self.queue[site].update(new_posts)
else:
self.queue[site] = new_posts
self.queue_modify_lock.release()
GlobalVars.api_request_lock.release()
return
self.api_data_lock.acquire()
add_or_update_api_data(site)
self.api_data_lock.release()
message_hq = ""
if "quota_remaining" in response:
if response["quota_remaining"] - GlobalVars.apiquota >= 5000 and GlobalVars.apiquota >= 0:
tell_rooms_with("debug", "API quota rolled over with {0} requests remaining. "
"Current quota: {1}.".format(GlobalVars.apiquota,
response["quota_remaining"]))
sorted_calls_per_site = sorted(GlobalVars.api_calls_per_site.items(), key=itemgetter(1), reverse=True)
api_quota_used_per_site = ""
for site_name, quota_used in sorted_calls_per_site:
sanatized_site_name = site_name.replace('.com', '').replace('.stackexchange', '')
api_quota_used_per_site += sanatized_site_name + ": {0}\n".format(str(quota_used))
api_quota_used_per_site = api_quota_used_per_site.strip()
tell_rooms_with("debug", api_quota_used_per_site)
clear_api_data()
if response["quota_remaining"] == 0:
tell_rooms_with("debug", "API reports no quota left! May be a glitch.")
tell_rooms_with("debug", str(response)) # No code format for now?
if GlobalVars.apiquota == -1:
tell_rooms_with("debug", "Restart: API quota is {quota}."
.format(quota=response["quota_remaining"]))
GlobalVars.apiquota = response["quota_remaining"]
else:
message_hq = "The quota_remaining property was not in the API response."
if "error_message" in response:
message_hq += " Error: {} at {} UTC.".format(response["error_message"], time_request_made)
if "error_id" in response and response["error_id"] == 502:
if GlobalVars.api_backoff_time < time.time() + 12: # Add a backoff of 10 + 2 seconds as a default
GlobalVars.api_backoff_time = time.time() + 12
message_hq += " Backing off on requests for the next 12 seconds."
message_hq += " Previous URL: `{}`".format(url)
if "backoff" in response:
if GlobalVars.api_backoff_time < time.time() + response["backoff"]:
GlobalVars.api_backoff_time = time.time() + response["backoff"]
GlobalVars.api_request_lock.release()
if len(message_hq) > 0:
tell_rooms_with("debug", message_hq.strip())
if "items" not in response:
return
if site == "stackoverflow.com":
items = response["items"]
if len(items) > 0 and "last_activity_date" in items[0]:
self.last_activity_date = items[0]["last_activity_date"]
num_scanned = 0
start_time = time.time()
for post in response["items"]:
pnb = copy.deepcopy(post)
if 'body' in pnb:
pnb['body'] = 'Present, but truncated'
if 'answers' in pnb:
del pnb['answers']
if "title" not in post or "body" not in post:
if GlobalVars.flovis is not None and 'question_id' in post:
GlobalVars.flovis.stage('bodyfetcher/api_response/no_content', site, post['question_id'], pnb)
continue
post['site'] = site
try:
post['edited'] = (post['creation_date'] != post['last_edit_date'])
except KeyError:
post['edited'] = False # last_edit_date not present = not edited
try:
post_ = Post(api_response=post)
except PostParseError as err:
log('error', 'Error {0} when parsing post: {1!r}'.format(err, post_))
if GlobalVars.flovis is not None and 'question_id' in post:
GlobalVars.flovis.stage('bodyfetcher/api_response/error', site, post['question_id'], pnb)
continue
num_scanned += 1
is_spam, reason, why = check_if_spam(post_)
if is_spam:
try:
if GlobalVars.flovis is not None and 'question_id' in post:
GlobalVars.flovis.stage('bodyfetcher/api_response/spam', site, post['question_id'],
{'post': pnb, 'check_if_spam': [is_spam, reason, why]})
handle_spam(post=post_,
reasons=reason,
why=why)
except Exception as e:
log('error', "Exception in handle_spam:", e)
elif GlobalVars.flovis is not None and 'question_id' in post:
GlobalVars.flovis.stage('bodyfetcher/api_response/not_spam', site, post['question_id'],
{'post': pnb, 'check_if_spam': [is_spam, reason, why]})
try:
if "answers" not in post:
pass
else:
for answer in post["answers"]:
anb = copy.deepcopy(answer)
if 'body' in anb:
anb['body'] = 'Present, but truncated'
num_scanned += 1
answer["IsAnswer"] = True # Necesssary for Post object
answer["title"] = "" # Necessary for proper Post object creation
answer["site"] = site # Necessary for proper Post object creation
try:
answer['edited'] = (answer['creation_date'] != answer['last_edit_date'])
except KeyError:
answer['edited'] = False # last_edit_date not present = not edited
answer_ = Post(api_response=answer, parent=post_)
is_spam, reason, why = check_if_spam(answer_)
if is_spam:
try:
if GlobalVars.flovis is not None and 'answer_id' in answer:
GlobalVars.flovis.stage('bodyfetcher/api_response/spam', site, answer['answer_id'],
{'post': anb, 'check_if_spam': [is_spam, reason, why]})
handle_spam(answer_,
reasons=reason,
why=why)
except Exception as e:
log('error', "Exception in handle_spam:", e)
elif GlobalVars.flovis is not None and 'answer_id' in answer:
GlobalVars.flovis.stage('bodyfetcher/api_response/not_spam', site, answer['answer_id'],
{'post': anb, 'check_if_spam': [is_spam, reason, why]})
except Exception as e:
log('error', "Exception handling answers:", e)
end_time = time.time()
GlobalVars.posts_scan_stats_lock.acquire()
GlobalVars.num_posts_scanned += num_scanned
GlobalVars.post_scan_time += end_time - start_time
GlobalVars.posts_scan_stats_lock.release()
return
def handle_websocket_data(data):
if "message" not in data:
return
message = data['message']
if not isinstance(message, Iterable):
return
if "message" in message:
chatcommunicate.tell_rooms_with("metasmoke", message['message'])
elif "autoflag_fp" in message:
event = message["autoflag_fp"]
chatcommunicate.tell_rooms(event["message"], ("debug", "site-" + event["site"]),
("no-site-" + event["site"],), notify_site="/autoflag_fp")
elif "exit" in message:
os._exit(message["exit"])
elif "blacklist" in message:
ids = (message['blacklist']['uid'], message['blacklist']['site'])
datahandling.add_blacklisted_user(ids, "metasmoke", message['blacklist']['post'])
datahandling.last_feedbacked = (ids, time.time() + 60)
elif "unblacklist" in message:
ids = (message['unblacklist']['uid'], message['unblacklist']['site'])
datahandling.remove_blacklisted_user(ids)
elif "naa" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"])
datahandling.add_ignored_post(post_site_id[0:2])
elif "fp" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"])
datahandling.add_false_positive(post_site_id[0:2])
elif "report" in message:
import chatcommands # Do it here
chatcommands.report_posts([message["report"]["post_link"]], message["report"]["user"],
True, "the metasmoke API")
elif "deploy_updated" in message:
return # Disabled
sha = message["deploy_updated"]["head_commit"]["id"]
if sha != os.popen('git log -1 --pretty="%H"').read():
if "autopull" in message["deploy_updated"]["head_commit"]["message"]:
if only_blacklists_changed(GitManager.get_remote_diff()):
commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \
.format(sha[:7], GlobalVars.bot_repo_slug)
integrity = blacklist_integrity_check()
if len(integrity) == 0: # No issues
GitManager.pull_remote()
findspam.reload_blacklists()
chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists"
" reloaded.".format(commit_md))
else:
integrity.append("please fix before pulling.")
chatcommunicate.tell_rooms_with("debug", ", ".join(integrity))
elif "commit_status" in message:
c = message["commit_status"]
sha = c["commit_sha"][:7]
if c["commit_sha"] == sp.check_output(["git", "log", "-1", "--pretty=%H"]).decode('utf-8').strip():
return
if c["status"] == "success":
if "autopull" in c["commit_message"]:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \
"commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format(
ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
remote_diff = GitManager.get_remote_diff()
if only_blacklists_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_master:
# Restart if HEAD detached
log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True)
os._exit(8)
GlobalVars.reload()
findspam.FindSpam.reload_blacklists()
chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart)
elif only_modules_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_master:
# Restart if HEAD detached
log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True)
os._exit(8)
GlobalVars.reload()
reload_modules()
chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart2)
else:
chatcommunicate.tell_rooms_with('debug', s, notify_site="/ci")
os._exit(3)
else:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
elif c["status"] == "failure":
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
elif "everything_is_broken" in message:
if message["everything_is_broken"] is True:
os._exit(6)