def send_status_ping_and_verify_scanning_if_active():
in_standby_mode = GlobalVars.standby_mode or GlobalVars.no_se_activity_scan
if not in_standby_mode:
# This is the active instance, so should be scanning. If it's not scanning, then report or go to standby.
if GlobalVars.PostScanStat.get_stat(
) == Metasmoke.scan_stat_snapshot:
# There's been no actvity since the last ping.
Metasmoke.status_pings_since_scan_activity += 1
if Metasmoke.status_pings_since_scan_activity >= NO_ACTIVITY_PINGS_TO_STANDBY:
# Assume something is very wrong. Report to debug rooms and go into standby mode.
error_message = "There's been no scan activity for {} status pings. Going into standby." \
.format(Metasmoke.status_pings_since_scan_activity)
log('error', error_message)
chatcommunicate.tell_rooms_with("debug", error_message)
GlobalVars.standby_mode = True
# Let MS know immediately, to lessen potential wait time (e.g. if we fail to reboot).
Metasmoke.send_status_ping()
time.sleep(8)
exit_mode("standby")
elif Metasmoke.status_pings_since_scan_activity >= NO_ACTIVITY_PINGS_TO_REPORT:
# Something might be wrong. Let people in debug rooms know.
status_message = "There's been no scan activity for {} status pings. There may be a problem." \
.format(Metasmoke.status_pings_since_scan_activity)
log('warning', status_message)
chatcommunicate.tell_rooms_with("debug", status_message)
else:
Metasmoke.status_pings_since_scan_activity = 0
Metasmoke.scan_stat_snapshot = GlobalVars.PostScanStat.get_stat(
)
Metasmoke.send_status_ping()
def restart_automatically():
Metasmoke.send_statistics()
chatcommunicate.tell_rooms_with(
"debug", "{}: Executing automatic scheduled reboot.".format(
GlobalVars.location))
time.sleep(6)
exit_mode("reboot")
def uncaught_exception(exctype, value, tb):
delta = datetime.utcnow() - GlobalVars.startup_utc_date
log_exception(exctype, value, tb)
if delta.total_seconds() < 180 and exctype not in \
{KeyboardInterrupt, SystemExit, requests.ConnectionError, WebSocketConnectionClosedException}:
exit_mode("early_exception")
else:
exit_mode("restart")
def uncaught_exception(exctype, value, tb):
delta = datetime.utcnow() - GlobalVars.startup_utc_date
log_exception(exctype, value, tb)
if delta.total_seconds() < 180 and exctype not in \
{KeyboardInterrupt, SystemExit, requests.ConnectionError, WebSocketConnectionClosedException}:
exit_mode("early_exception")
else:
exit_mode("restart")
def check_socket_connections():
socket_failure = False
with chatcommunicate._clients_lock:
for client in chatcommunicate._clients.values():
if client.last_activity and (
datetime.utcnow() -
client.last_activity).total_seconds() >= 60:
socket_failure = True
if socket_failure:
exit_mode("socket_failure")
def reboot_or_standby(action):
error_message = "There's been no scan activity for {} status pings. Going to {}." \
.format(Metasmoke.status_pings_since_scan_activity, action)
log('error', error_message)
chatcommunicate.tell_rooms_with("debug", error_message)
if action == "standby":
GlobalVars.standby_mode = True
# Let MS know immediately, to lessen potential wait time (e.g. if we fail to reboot).
Metasmoke.send_status_ping()
time.sleep(8)
exit_mode(action)
def init_se_websocket_or_reboot(max_tries, tell_debug_room_on_error=False):
for tries in range(1, 1 + max_tries, 1):
ws = setup_websocket(tries, max_tries)
if ws:
break
else:
error_message = 'SE WebSocket: Max retries exceeded. Exiting, maybe a restart will kick things.'
log('error', error_message)
if tell_debug_room_on_error:
chatcommunicate.tell_rooms_with("debug", error_message)
time.sleep(
6
) # Make it more likely the message is actually sent to the rooms prior to rebooting.
exit_mode("reboot")
return ws
def send_status_ping():
if GlobalVars.metasmoke_host is None:
log('info', 'Attempted to send status ping but metasmoke_host is undefined. Not sent.')
return
elif GlobalVars.metasmoke_down:
payload = {
"location": GlobalVars.location,
"timestamp": time.time()
}
SocketScience.send(payload)
metasmoke_key = GlobalVars.metasmoke_key
try:
payload = {
'location': GlobalVars.location,
'key': metasmoke_key,
'standby': GlobalVars.standby_mode
}
headers = {'content-type': 'application/json'}
response = Metasmoke.post("/status-update.json",
data=json.dumps(payload), headers=headers, ignore_down=True)
try:
response = response.json()
GlobalVars.metasmoke_last_ping_time = datetime.now() # Otherwise the ping watcher will exit(10)
if response.get('pull_update', False):
log('info', "Received pull command from MS ping response")
exit_mode("pull_update")
if 'failover' in response and GlobalVars.standby_mode:
if response['failover']:
GlobalVars.standby_mode = False
chatcommunicate.tell_rooms_with("debug", GlobalVars.location + " received failover signal.",
notify_site="/failover")
if response.get('standby', False):
chatcommunicate.tell_rooms_with("debug",
GlobalVars.location + " entering metasmoke-forced standby.")
time.sleep(2)
exit_mode("standby")
if response.get('shutdown', False):
exit_mode("shutdown")
except Exception: # TODO: What could happen here?
pass
except Exception as e:
log('error', e)
def handle_websocket_data(data):
if "message" not in data:
if "type" in data and data['type'] == "reject_subscription":
log(
'error',
"MS WebSocket subscription was rejected. Check your MS key."
)
raise ConnectionError("MS WebSocket connection rejected")
return
message = data['message']
if not isinstance(message, Iterable):
return
if "message" in message:
chatcommunicate.tell_rooms_with("metasmoke", message['message'])
elif "autoflag_fp" in message:
event = message["autoflag_fp"]
chatcommunicate.tell_rooms(event["message"],
("debug", "site-" + event["site"]),
("no-site-" + event["site"], ),
notify_site="/autoflag_fp")
elif "exit" in message:
os._exit(message["exit"])
elif "blacklist" in message:
ids = (message['blacklist']['uid'], message['blacklist']['site'])
datahandling.add_blacklisted_user(ids, "metasmoke",
message['blacklist']['post'])
datahandling.last_feedbacked = (ids, time.time() + 60)
elif "unblacklist" in message:
ids = (message['unblacklist']['uid'],
message['unblacklist']['site'])
datahandling.remove_blacklisted_user(ids)
elif "naa" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(
message["naa"]["post_link"])
datahandling.add_ignored_post(post_site_id[0:2])
elif "fp" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(
message["fp"]["post_link"])
datahandling.add_false_positive(post_site_id[0:2])
elif "report" in message:
import chatcommands # Do it here
chatcommands.report_posts([message["report"]["post_link"]],
message["report"]["user"], True,
"the metasmoke API")
elif "deploy_updated" in message:
return # Disabled
sha = message["deploy_updated"]["head_commit"]["id"]
if sha != os.popen('git log -1 --pretty="%H"').read():
if "autopull" in message["deploy_updated"]["head_commit"][
"message"]:
if only_blacklists_changed(GitManager.get_remote_diff()):
commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \
.format(sha[:7], GlobalVars.bot_repo_slug)
integrity = blacklist_integrity_check()
if len(integrity) == 0: # No issues
GitManager.pull_remote()
findspam.reload_blacklists()
chatcommunicate.tell_rooms_with(
"debug",
"No code modified in {0}, only blacklists"
" reloaded.".format(commit_md))
else:
integrity.append("please fix before pulling.")
chatcommunicate.tell_rooms_with(
"debug", ", ".join(integrity))
elif "commit_status" in message:
c = message["commit_status"]
sha = c["commit_sha"][:7]
recent_commits = sp.check_output(
["git", "log", "-50",
"--pretty=%H"]).decode('utf-8').strip().split('\n')
if c["commit_sha"] in recent_commits:
return # Same rev, or earlier rev (e.g. when watching things faster than CI completes), nothing to do
if c["status"] == "success":
if "autopull" in c["commit_message"] or c["commit_message"].startswith("!") or \
c["commit_message"].startswith("Auto "):
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \
"commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format(
ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
remote_diff = GitManager.get_remote_diff()
if only_blacklists_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_branch:
# Restart if HEAD detached
log('warning',
"Pulling remote with HEAD detached, checkout deploy",
f=True)
exit_mode("checkout_deploy")
GlobalVars.reload()
findspam.FindSpam.reload_blacklists()
chatcommunicate.tell_rooms_with(
'debug', GlobalVars.s_norestart_blacklists)
elif only_modules_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_branch:
# Restart if HEAD detached
log('warning',
"Pulling remote with HEAD detached, checkout deploy",
f=True)
exit_mode("checkout_deploy")
GlobalVars.reload()
reload_modules()
chatcommunicate.tell_rooms_with(
'debug', GlobalVars.s_norestart_findspam)
else:
chatcommunicate.tell_rooms_with('debug',
s,
notify_site="/ci")
exit_mode("pull_update")
else:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug",
s,
notify_site="/ci")
elif c["status"] == "failure":
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
elif "everything_is_broken" in message:
if message["everything_is_broken"] is True:
exit_mode("shutdown")
elif "domain_whitelist" in message:
if message["domain_whitelist"] == "refresh":
metasmoke_cache.MetasmokeCache.delete('whitelisted-domains')
def send_status_ping():
if GlobalVars.metasmoke_host is None:
log(
'info',
'Attempted to send status ping but metasmoke_host is undefined. Not sent.'
)
return
elif GlobalVars.metasmoke_down:
payload = {
"location": GlobalVars.location,
"timestamp": time.time()
}
SocketScience.send(payload)
metasmoke_key = GlobalVars.metasmoke_key
try:
payload = {
'location':
GlobalVars.location,
'key':
metasmoke_key,
'standby':
GlobalVars.standby_mode or GlobalVars.no_se_activity_scan
}
headers = {'content-type': 'application/json'}
response = Metasmoke.post("/status-update.json",
data=json.dumps(payload),
headers=headers,
ignore_down=True)
try:
response = response.json()
GlobalVars.metasmoke_last_ping_time = datetime.utcnow(
) # Otherwise the ping watcher will exit(10)
if response.get('pull_update', False):
log('info', "Received pull command from MS ping response")
exit_mode("pull_update")
if ('failover' in response and GlobalVars.standby_mode
and not GlobalVars.no_se_activity_scan):
# If we're not scanning, then we don't want to become officially active due to failover.
if response['failover']:
GlobalVars.standby_mode = False
chatcommunicate.tell_rooms_with(
"debug",
GlobalVars.location + " received failover signal.",
notify_site="/failover")
if response.get('standby', False):
chatcommunicate.tell_rooms_with(
"debug", GlobalVars.location +
" entering metasmoke-forced standby.")
time.sleep(2)
exit_mode("standby")
if response.get('shutdown', False):
exit_mode("shutdown")
except Exception: # TODO: What could happen here?
pass
except Exception as e:
log('error', e)
def handle_websocket_data(data):
if "message" not in data:
return
message = data['message']
if not isinstance(message, Iterable):
return
if "message" in message:
chatcommunicate.tell_rooms_with("metasmoke", message['message'])
elif "autoflag_fp" in message:
event = message["autoflag_fp"]
chatcommunicate.tell_rooms(event["message"], ("debug", "site-" + event["site"]),
("no-site-" + event["site"],), notify_site="/autoflag_fp")
elif "exit" in message:
os._exit(message["exit"])
elif "blacklist" in message:
ids = (message['blacklist']['uid'], message['blacklist']['site'])
datahandling.add_blacklisted_user(ids, "metasmoke", message['blacklist']['post'])
datahandling.last_feedbacked = (ids, time.time() + 60)
elif "unblacklist" in message:
ids = (message['unblacklist']['uid'], message['unblacklist']['site'])
datahandling.remove_blacklisted_user(ids)
elif "naa" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"])
datahandling.add_ignored_post(post_site_id[0:2])
elif "fp" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"])
datahandling.add_false_positive(post_site_id[0:2])
elif "report" in message:
import chatcommands # Do it here
chatcommands.report_posts([message["report"]["post_link"]], message["report"]["user"],
True, "the metasmoke API")
elif "deploy_updated" in message:
return # Disabled
sha = message["deploy_updated"]["head_commit"]["id"]
if sha != os.popen('git log -1 --pretty="%H"').read():
if "autopull" in message["deploy_updated"]["head_commit"]["message"]:
if only_blacklists_changed(GitManager.get_remote_diff()):
commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \
.format(sha[:7], GlobalVars.bot_repo_slug)
integrity = blacklist_integrity_check()
if len(integrity) == 0: # No issues
GitManager.pull_remote()
findspam.reload_blacklists()
chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists"
" reloaded.".format(commit_md))
else:
integrity.append("please fix before pulling.")
chatcommunicate.tell_rooms_with("debug", ", ".join(integrity))
elif "commit_status" in message:
c = message["commit_status"]
sha = c["commit_sha"][:7]
if c["commit_sha"] == sp.check_output(["git", "log", "-1", "--pretty=%H"]).decode('utf-8').strip():
return # Same rev, nothing to do
if c["status"] == "success":
if "autopull" in c["commit_message"] or c["commit_message"].startswith("!") or \
c["commit_message"].startswith("Auto "):
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \
"commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format(
ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
remote_diff = GitManager.get_remote_diff()
if only_blacklists_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_branch:
# Restart if HEAD detached
log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True)
exit_mode("checkout_deploy")
GlobalVars.reload()
findspam.FindSpam.reload_blacklists()
chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart)
elif only_modules_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_branch:
# Restart if HEAD detached
log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True)
exit_mode("checkout_deploy")
GlobalVars.reload()
reload_modules()
chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart2)
else:
chatcommunicate.tell_rooms_with('debug', s, notify_site="/ci")
exit_mode("pull_update")
else:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
elif c["status"] == "failure":
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
elif "everything_is_broken" in message:
if message["everything_is_broken"] is True:
exit_mode("shutdown")
# Ran into this error in testing on Windows, best to throw a warn if we get this...
err_msg = "WARNING: Cannot verify SSL connection for TLD names update; skipping TLD names update."
else:
err_msg = strerror
log_exception(type(ioerr), ioerr, err_msg, True, level="warning")
if "ChatExchangeU" in os.environ:
log('debug', "ChatExchange username loaded from environment")
username = os.environ["ChatExchangeU"]
elif GlobalVars.chatexchange_u:
log('debug', "ChatExchange username loaded from config")
username = GlobalVars.chatexchange_u
else:
log('error', "No ChatExchange username provided. Set it in config or provide it via environment variable")
exit_mode("shutdown")
if "ChatExchangeP" in os.environ:
log('debug', "ChatExchange password loaded from environment")
password = os.environ["ChatExchangeP"]
elif GlobalVars.chatexchange_p:
log('debug', "ChatExchange password loaded from config")
password = GlobalVars.chatexchange_p
else:
log('error', "No ChatExchange password provided. Set it in config or provide it via environment variable")
exit_mode("shutdown")
# We need an instance of bodyfetcher before load_files() is called
GlobalVars.bodyfetcher = BodyFetcher()
if GlobalVars.flovis_host:
GlobalVars.flovis = Flovis(GlobalVars.flovis_host)
def restart_automatically():
Metasmoke.send_statistics()
exit_mode("reboot")
def restart_automatically():
Metasmoke.send_statistics()
exit_mode("reboot")
def check_socket_connections():
for client in chatcommunicate._clients.values():
if client.last_activity and (datetime.utcnow() - client.last_activity).total_seconds() >= 60:
exit_mode("socket_failure")
errlogs.write("WARNING: Cannot verify SSL connection for TLD names update; skipping TLD names update.")
errlogs.close()
pass
else:
raise ioerr
if "ChatExchangeU" in os.environ:
log('debug', "ChatExchange username loaded from environment")
username = os.environ["ChatExchangeU"]
elif GlobalVars.chatexchange_u:
log('debug', "ChatExchange username loaded from config")
username = GlobalVars.chatexchange_u
else:
log('error', "No ChatExchange username provided. Set it in config or provide it via environment variable")
exit_mode("shutdown")
if "ChatExchangeP" in os.environ:
log('debug', "ChatExchange password loaded from environment")
password = os.environ["ChatExchangeP"]
elif GlobalVars.chatexchange_p:
log('debug', "ChatExchange password loaded from config")
password = GlobalVars.chatexchange_p
else:
log('error', "No ChatExchange password provided. Set it in config or provide it via environment variable")
exit_mode("shutdown")
# We need an instance of bodyfetcher before load_files() is called
GlobalVars.bodyfetcher = BodyFetcher()
if GlobalVars.flovis_host:
GlobalVars.flovis = Flovis(GlobalVars.flovis_host)