def test_blacklist_integrity():
errors = blacklist_integrity_check()
if len(errors) == 1:
raise ValueError(errors[0])
elif len(errors) > 1:
raise ValueError("\n\t".join(["{} errors has occurred:".format(len(errors))] + errors))
def test_blacklist_integrity():
errors = blacklist_integrity_check()
if len(errors) == 1:
raise ValueError(errors[0])
elif len(errors) > 1:
raise ValueError("\n\t".join(["{} errors has occurred:".format(len(errors))] + errors))
def test_blacklist_integrity():
errors = blacklist_integrity_check()
if len(errors) == 1:
raise ValueError(errors[0])
elif len(errors) > 1:
raise ValueError("\n\t".join(["Multiple errors has occurred."] +
errors))
def test_blacklist_integrity():
errors = blacklist_integrity_check()
if len(errors) == 1:
pytest.fail(errors[0])
elif len(errors) > 1:
pytest.fail(
"\n\t".join(["{} errors have occurred:".format(len(errors))] +
errors))
def handle_websocket_data(data):
if "message" not in data:
if "type" in data and data['type'] == "reject_subscription":
log(
'error',
"MS WebSocket subscription was rejected. Check your MS key."
)
raise ConnectionError("MS WebSocket connection rejected")
return
message = data['message']
if not isinstance(message, Iterable):
return
if "message" in message:
chatcommunicate.tell_rooms_with("metasmoke", message['message'])
elif "autoflag_fp" in message:
event = message["autoflag_fp"]
chatcommunicate.tell_rooms(event["message"],
("debug", "site-" + event["site"]),
("no-site-" + event["site"], ),
notify_site="/autoflag_fp")
elif "exit" in message:
os._exit(message["exit"])
elif "blacklist" in message:
ids = (message['blacklist']['uid'], message['blacklist']['site'])
datahandling.add_blacklisted_user(ids, "metasmoke",
message['blacklist']['post'])
datahandling.last_feedbacked = (ids, time.time() + 60)
elif "unblacklist" in message:
ids = (message['unblacklist']['uid'],
message['unblacklist']['site'])
datahandling.remove_blacklisted_user(ids)
elif "naa" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(
message["naa"]["post_link"])
datahandling.add_ignored_post(post_site_id[0:2])
elif "fp" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(
message["fp"]["post_link"])
datahandling.add_false_positive(post_site_id[0:2])
elif "report" in message:
import chatcommands # Do it here
chatcommands.report_posts([message["report"]["post_link"]],
message["report"]["user"], True,
"the metasmoke API")
elif "deploy_updated" in message:
return # Disabled
sha = message["deploy_updated"]["head_commit"]["id"]
if sha != os.popen('git log -1 --pretty="%H"').read():
if "autopull" in message["deploy_updated"]["head_commit"][
"message"]:
if only_blacklists_changed(GitManager.get_remote_diff()):
commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \
.format(sha[:7], GlobalVars.bot_repo_slug)
integrity = blacklist_integrity_check()
if len(integrity) == 0: # No issues
GitManager.pull_remote()
findspam.reload_blacklists()
chatcommunicate.tell_rooms_with(
"debug",
"No code modified in {0}, only blacklists"
" reloaded.".format(commit_md))
else:
integrity.append("please fix before pulling.")
chatcommunicate.tell_rooms_with(
"debug", ", ".join(integrity))
elif "commit_status" in message:
c = message["commit_status"]
sha = c["commit_sha"][:7]
recent_commits = sp.check_output(
["git", "log", "-50",
"--pretty=%H"]).decode('utf-8').strip().split('\n')
if c["commit_sha"] in recent_commits:
return # Same rev, or earlier rev (e.g. when watching things faster than CI completes), nothing to do
if c["status"] == "success":
if "autopull" in c["commit_message"] or c["commit_message"].startswith("!") or \
c["commit_message"].startswith("Auto "):
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \
"commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format(
ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
remote_diff = GitManager.get_remote_diff()
if only_blacklists_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_branch:
# Restart if HEAD detached
log('warning',
"Pulling remote with HEAD detached, checkout deploy",
f=True)
exit_mode("checkout_deploy")
GlobalVars.reload()
findspam.FindSpam.reload_blacklists()
chatcommunicate.tell_rooms_with(
'debug', GlobalVars.s_norestart_blacklists)
elif only_modules_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_branch:
# Restart if HEAD detached
log('warning',
"Pulling remote with HEAD detached, checkout deploy",
f=True)
exit_mode("checkout_deploy")
GlobalVars.reload()
reload_modules()
chatcommunicate.tell_rooms_with(
'debug', GlobalVars.s_norestart_findspam)
else:
chatcommunicate.tell_rooms_with('debug',
s,
notify_site="/ci")
exit_mode("pull_update")
else:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug",
s,
notify_site="/ci")
elif c["status"] == "failure":
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
elif "everything_is_broken" in message:
if message["everything_is_broken"] is True:
exit_mode("shutdown")
elif "domain_whitelist" in message:
if message["domain_whitelist"] == "refresh":
metasmoke_cache.MetasmokeCache.delete('whitelisted-domains')
def handle_websocket_data(data):
if "message" not in data:
return
message = data['message']
if not isinstance(message, Iterable):
return
if "message" in message:
chatcommunicate.tell_rooms_with("metasmoke", message['message'])
elif "autoflag_fp" in message:
event = message["autoflag_fp"]
chatcommunicate.tell_rooms(event["message"], ("debug", "site-" + event["site"]),
("no-site-" + event["site"],), notify_site="/autoflag_fp")
elif "exit" in message:
os._exit(message["exit"])
elif "blacklist" in message:
ids = (message['blacklist']['uid'], message['blacklist']['site'])
datahandling.add_blacklisted_user(ids, "metasmoke", message['blacklist']['post'])
datahandling.last_feedbacked = (ids, time.time() + 60)
elif "unblacklist" in message:
ids = (message['unblacklist']['uid'], message['unblacklist']['site'])
datahandling.remove_blacklisted_user(ids)
elif "naa" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"])
datahandling.add_ignored_post(post_site_id[0:2])
elif "fp" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"])
datahandling.add_false_positive(post_site_id[0:2])
elif "report" in message:
import chatcommands # Do it here
chatcommands.report_posts([message["report"]["post_link"]], message["report"]["user"],
True, "the metasmoke API")
elif "deploy_updated" in message:
return # Disabled
sha = message["deploy_updated"]["head_commit"]["id"]
if sha != os.popen('git log -1 --pretty="%H"').read():
if "autopull" in message["deploy_updated"]["head_commit"]["message"]:
if only_blacklists_changed(GitManager.get_remote_diff()):
commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \
.format(sha[:7], GlobalVars.bot_repo_slug)
integrity = blacklist_integrity_check()
if len(integrity) == 0: # No issues
GitManager.pull_remote()
findspam.reload_blacklists()
chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists"
" reloaded.".format(commit_md))
else:
integrity.append("please fix before pulling.")
chatcommunicate.tell_rooms_with("debug", ", ".join(integrity))
elif "commit_status" in message:
c = message["commit_status"]
sha = c["commit_sha"][:7]
if c["commit_sha"] == sp.check_output(["git", "log", "-1", "--pretty=%H"]).decode('utf-8').strip():
return # Same rev, nothing to do
if c["status"] == "success":
if "autopull" in c["commit_message"] or c["commit_message"].startswith("!") or \
c["commit_message"].startswith("Auto "):
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \
"commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format(
ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
remote_diff = GitManager.get_remote_diff()
if only_blacklists_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_branch:
# Restart if HEAD detached
log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True)
exit_mode("checkout_deploy")
GlobalVars.reload()
findspam.FindSpam.reload_blacklists()
chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart)
elif only_modules_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_branch:
# Restart if HEAD detached
log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True)
exit_mode("checkout_deploy")
GlobalVars.reload()
reload_modules()
chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart2)
else:
chatcommunicate.tell_rooms_with('debug', s, notify_site="/ci")
exit_mode("pull_update")
else:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
elif c["status"] == "failure":
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
elif "everything_is_broken" in message:
if message["everything_is_broken"] is True:
exit_mode("shutdown")
def handle_websocket_data(data):
if "message" not in data:
return
message = data['message']
if not isinstance(message, Iterable):
return
if "message" in message:
chatcommunicate.tell_rooms_with("metasmoke", message['message'])
elif "autoflag_fp" in message:
event = message["autoflag_fp"]
chatcommunicate.tell_rooms(event["message"], ("debug", "site-" + event["site"]),
("no-site-" + event["site"],), notify_site="/autoflag_fp")
elif "exit" in message:
os._exit(message["exit"])
elif "blacklist" in message:
ids = (message['blacklist']['uid'], message['blacklist']['site'])
datahandling.add_blacklisted_user(ids, "metasmoke", message['blacklist']['post'])
datahandling.last_feedbacked = (ids, time.time() + 60)
elif "unblacklist" in message:
ids = (message['unblacklist']['uid'], message['unblacklist']['site'])
datahandling.remove_blacklisted_user(ids)
elif "naa" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"])
datahandling.add_ignored_post(post_site_id[0:2])
elif "fp" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"])
datahandling.add_false_positive(post_site_id[0:2])
elif "report" in message:
import chatcommands # Do it here
chatcommands.report_posts([message["report"]["post_link"]], message["report"]["user"],
True, "the metasmoke API")
elif "deploy_updated" in message:
return # Disabled
sha = message["deploy_updated"]["head_commit"]["id"]
if sha != os.popen('git log -1 --pretty="%H"').read():
if "autopull" in message["deploy_updated"]["head_commit"]["message"]:
if only_blacklists_changed(GitManager.get_remote_diff()):
commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \
.format(sha[:7], GlobalVars.bot_repo_slug)
integrity = blacklist_integrity_check()
if len(integrity) == 0: # No issues
GitManager.pull_remote()
findspam.reload_blacklists()
chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists"
" reloaded.".format(commit_md))
else:
integrity.append("please fix before pulling.")
chatcommunicate.tell_rooms_with("debug", ", ".join(integrity))
elif "commit_status" in message:
c = message["commit_status"]
sha = c["commit_sha"][:7]
if c["commit_sha"] == sp.check_output(["git", "log", "-1", "--pretty=%H"]).decode('utf-8').strip():
return
if c["status"] == "success":
if "autopull" in c["commit_message"]:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \
"commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format(
ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
remote_diff = GitManager.get_remote_diff()
if only_blacklists_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_master:
# Restart if HEAD detached
log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True)
os._exit(8)
GlobalVars.reload()
findspam.FindSpam.reload_blacklists()
chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart)
elif only_modules_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_master:
# Restart if HEAD detached
log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True)
os._exit(8)
GlobalVars.reload()
reload_modules()
chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart2)
else:
chatcommunicate.tell_rooms_with('debug', s, notify_site="/ci")
os._exit(3)
else:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
elif c["status"] == "failure":
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
elif "everything_is_broken" in message:
if message["everything_is_broken"] is True:
os._exit(6)
def handle_websocket_data(data):
if "message" not in data:
return
message = data['message']
if not isinstance(message, Iterable):
return
if "message" in message:
chatcommunicate.tell_rooms_with("metasmoke", message['message'])
elif "autoflag_fp" in message:
event = message["autoflag_fp"]
chatcommunicate.tell_rooms(event["message"],
("debug", "site-" + event["site"]),
("no-site-" + event["site"], ),
notify_site="/autoflag_fp")
elif "exit" in message:
os._exit(message["exit"])
elif "blacklist" in message:
ids = (message['blacklist']['uid'], message['blacklist']['site'])
datahandling.add_blacklisted_user(ids, "metasmoke",
message['blacklist']['post'])
datahandling.last_feedbacked = (ids, time.time() + 60)
elif "unblacklist" in message:
ids = (message['unblacklist']['uid'],
message['unblacklist']['site'])
datahandling.remove_blacklisted_user(ids)
elif "naa" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(
message["naa"]["post_link"])
datahandling.add_ignored_post(post_site_id[0:2])
elif "fp" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(
message["fp"]["post_link"])
datahandling.add_false_positive(post_site_id[0:2])
elif "report" in message:
import chatcommands # Do it here
chatcommands.report_posts([message["report"]["post_link"]],
"the metasmoke API", None,
"the metasmoke API")
return
post_data = apigetpost.api_get_post(message["report"]["post_link"])
if post_data is None or post_data is False:
return
if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \
and not datahandling.is_false_positive((post_data.post_id, post_data.site)):
return
user = parsing.get_user_from_url(post_data.owner_url)
post = classes.Post(api_response=post_data.as_dict)
scan_spam, scan_reasons, scan_why = spamhandling.check_if_spam(
post)
if scan_spam:
why_append = u"This post would have also been caught for: " + \
u", ".join(scan_reasons).capitalize() + "\n" + scan_why
else:
why_append = u"This post would not have been caught otherwise."
# Add user to blacklist *after* post is scanned
if user is not None:
datahandling.add_blacklisted_user(user, "metasmoke",
post_data.post_url)
why = u"Post manually reported by user *{}* from metasmoke.\n\n{}".format(
message["report"]["user"], why_append)
spamhandling.handle_spam(
post=post,
reasons=["Manually reported " + post_data.post_type],
why=why)
elif "deploy_updated" in message:
return # Disabled
sha = message["deploy_updated"]["head_commit"]["id"]
if sha != os.popen('git log -1 --pretty="%H"').read():
if "autopull" in message["deploy_updated"]["head_commit"][
"message"]:
if only_blacklists_changed(GitManager.get_remote_diff()):
commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \
.format(sha[:7], GlobalVars.bot_repo_slug)
integrity = blacklist_integrity_check()
if len(integrity) == 0: # No issues
GitManager.pull_remote()
findspam.reload_blacklists()
chatcommunicate.tell_rooms_with(
"debug",
"No code modified in {0}, only blacklists"
" reloaded.".format(commit_md))
else:
integrity.append("please fix before pulling.")
chatcommunicate.tell_rooms_with(
"debug", ", ".join(integrity))
elif "commit_status" in message:
c = message["commit_status"]
sha = c["commit_sha"][:7]
if c["commit_sha"] == sp.check_output(
["git", "log", "-1", "--pretty=%H"]).decode('utf-8').strip():
return
if c["status"] == "success":
if "autopull" in c["commit_message"]:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \
"commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format(
ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
remote_diff = GitManager.get_remote_diff()
if only_blacklists_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_master:
# Restart if HEAD detached
log('warning',
"Pulling remote with HEAD detached, checkout deploy",
f=True)
os._exit(8)
GlobalVars.reload()
findspam.FindSpam.reload_blacklists()
chatcommunicate.tell_rooms_with(
'debug', GlobalVars.s_norestart)
elif only_modules_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_master:
# Restart if HEAD detached
log('warning',
"Pulling remote with HEAD detached, checkout deploy",
f=True)
os._exit(8)
GlobalVars.reload()
reload_modules()
chatcommunicate.tell_rooms_with(
'debug', GlobalVars.s_norestart2)
else:
chatcommunicate.tell_rooms_with('debug',
s,
notify_site="/ci")
os._exit(3)
else:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug",
s,
notify_site="/ci")
elif c["status"] == "failure":
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
elif "everything_is_broken" in message:
if message["everything_is_broken"] is True:
os._exit(6)
