def find_suspicious(pv_id):
pageviews = Pageview.query.filter(Pageview.id == pv_id).all()
suspicious = []
if not pageviews:
print('ERROR: no pageview found with id {}'.format(pv_id))
elif len(pageviews) > 1:
print('ERROR: >1 pageviews found with id {}'.format(pv_id))
else:
pageview = pageviews[0]
hostname = urlparse(pageview.url).netloc
root_domain = get_root_domain(hostname)
domains = [get_root_domain(urlparse(rsc.url).netloc) for rsc in pageview.resources]
# dedup the list of domains and drop all empty values:
domains = list(set(filter(lambda x: x, domains)))
# remove any domains that are rooted in the WHITELIST or the current root_domain:
suspicious = list(filter(lambda x: not any([x.endswith(y) for y in WHITELIST + [root_domain]]), domains))
if suspicious:
print("FOUND SUSPICIOUS DOMAINS: {}".format(suspicious))
sendmail('*****@*****.**',
'Suspicious Resource Result! {}'.format(pageview.url),
render_template('email/suspicious_resource_result.html', url=pageview.url, suspicious=suspicious, domains=domains))
else:
print("Didn't find any suspicious domains")
return suspicious
def yara_retroscan_for_rule(rule_id):
rule = YaraRuleset.query.filter_by(id=rule_id).one()
sources = {rule.namespace: rule.source}
try:
yara_rule = yara.compile(sources=sources)
except:
# TODO: this should never happen, so this email should go to the site admin
# TODO: this should not catch any/all exceptions!
sendmail('*****@*****.**', 'YARA Retroscan Results (error in rule compilation!)', render_template('email/yara_error.html'))
return
os.nice(5)
matches = []
try:
for subdir in os.listdir(app.config['RESOURCE_CONTENT_FOLDER']):
try:
subdir = os.path.join(app.config['RESOURCE_CONTENT_FOLDER'], subdir)
for path in os.listdir(subdir):
with gzip.open(os.path.join(subdir, path), 'rb') as f:
if yara_rule.match(data=f.read()):
matches.append(path.split('.')[0])
if len(matches) > app.config['MAX_HASHES']:
break
except NotADirectoryError:
pass
except:
sendmail(rule.email, 'YARA Retroscan Error ({0})'.format(sys.exc_info()), render_template('email/yara_error.html'))
else:
yara_report_matches.apply_async(args=(rule.email, rule.namespace, matches))
os.nice(0)
def yara_report_matches(email, namespace, hashes):
print("got match for {0} - {1} !".format(email, namespace))
now = datetime.datetime.now()
matches = []
for hash in hashes:
record = Resource.query.filter(Resource.hash == hash).limit(
app.config['MAX_PAGES_PER_HASH']).all()
urls = [{
'webpage_url': r.pageview.url,
'webpage_hash': sha256(r.pageview.url),
'resource_url': r.url
} for r in record]
new_match = {'hash': hash, 'urls': urls}
matches.append(new_match)
rule = YaraRuleset.query.filter_by(email=email,
namespace=namespace).first()
if rule.email_tokens is None:
rule.email_tokens = 5
print("email tokens = {}".format(rule.email_tokens))
if rule.last_received_tokens is None:
rule.last_received_tokens = now
print("last rx tokens = {}".format(rule.last_received_tokens))
rule.email_tokens -= 1
rule.email_tokens = min(
10, rule.email_tokens +
(now - rule.last_received_tokens) // datetime.timedelta(minutes=1))
rule.last_received_tokens = now
print("at end, email tokens = {}".format(rule.email_tokens))
sendmail(
email, 'YARA Scan Results (success!): {}'.format(namespace),
render_template('email/yara_match.html',
matches=matches,
tokens=rule.email_tokens))
if rule.email_tokens <= 0:
print("removing rule!")
sendmail(rule.email, 'YARA Rule Removed ({})'.format(rule.namespace),
render_template('email/yara_goodbye.html', rule=rule))
db.session.delete(rule)
db.session.commit()
def yara_index():
""" yara rule submission UI view """
errors = []
if request.method == 'POST':
email = request.form['email'].strip()
if len(email) <= 0:
errors.append("You must provide an email address")
elif email not in app.config['EMAIL_WHITELIST']:
errors.append("Email address not in whitelist")
namespace = request.form['namespace'].strip()
if len(namespace) <= 0:
errors.append("You must provide a namespace")
else:
dups = YaraRuleset.query.filter_by(email=email,
namespace=namespace).count()
if dups > 0:
errors.append("The namespace you have provided has already "
"been used for that email address. Please "
"choose a new one.")
source = request.form['source'].strip()
if len(source) <= 0:
errors.append("You must provide some Yara rules")
try:
if len(errors) <= 0:
ruleset = YaraRuleset(email, namespace, source, True)
db.session.add(ruleset)
db.session.commit()
# send email
r = {
'email': email,
'content': source,
'removal_code': ruleset.removal_code
}
sendmail(email, 'New YARA Rule Added! {}'.format(namespace),
render_template('email/yara_welcome.html', rule=r))
flash("Rules successfully added!")
return redirect(url_for('yara_index'))
except yara.libyara_wrapper.YaraSyntaxError as e:
errors.append("Syntax error in yara rule: {}".format(e))
return render_template('yara/index.html', errors=errors)
def yara_index():
""" yara rule submission UI view """
errors = []
if request.method == 'POST':
email = request.form['email'].strip()
if len(email) <= 0:
errors.append("You must provide an email address")
elif email not in app.config['EMAIL_WHITELIST']:
errors.append("Email address not in whitelist")
namespace = request.form['namespace'].strip()
if len(namespace) <= 0:
errors.append("You must provide a namespace")
else:
dups = YaraRuleset.query.filter_by(email=email,
namespace=namespace).count()
if dups > 0:
errors.append("The namespace you have provided has already "
"been used for that email address. Please "
"choose a new one.")
source = request.form['source'].strip()
if len(source) <= 0:
errors.append("You must provide some Yara rules")
try:
if len(errors) <= 0:
ruleset = YaraRuleset(email, namespace, source, True)
db.session.add(ruleset)
db.session.commit()
# send email
r = {'email': email, 'content': source, 'removal_code': ruleset.removal_code}
sendmail(email,
'New YARA Rule Added! {}'.format(namespace),
render_template('email/yara_welcome.html', rule=r))
flash("Rules successfully added!")
return redirect(url_for('yara_index'))
except yara.libyara_wrapper.YaraSyntaxError as e:
errors.append("Syntax error in yara rule: {}".format(e))
return render_template('yara/index.html', errors=errors)
def yara_scan_file_for_email(email, path):
# TODO: filter for 'scan_on_upload==True' too
# TODO: store compiled rules in database to avoid re-compiling?
try:
rulesets = YaraRuleset.query.filter_by(email=email).all()
sources = {}
for r in rulesets:
sources[r.namespace] = r.source
def matchcb(data):
if data['matches']:
yara_report_matches.apply_async(
args=(email, data['namespace'],
[path.split('/')[-1].split('.')[0]]),
countdown=10)
return yara.CALLBACK_CONTINUE
# TODO: these "except" clauses should be consolidated and should catch
# specific exceptions.
try:
rules = yara.compile(sources=sources)
with gzip.open(path, 'rb') as f:
try:
rules.match(data=f.read(), callback=matchcb)
except:
# TODO: this should never happen, so this email should
# go to the site admin
sendmail(
'*****@*****.**',
'YARA Livescan Results (error while reading file / matching ruleset!)',
render_template('email/yara_error.html'))
except:
# TODO: this should never happen, so this email should go to the site admin
sendmail('*****@*****.**',
'YARA Scan Results (error in rule compilation!)',
render_template('email/yara_error.html'))
except (sqlalchemy.exc.DatabaseError,
sqlalchemy.exc.OperationalError) as exc:
print("retrying....")
def yara_report_matches(email, namespace, hashes):
print("got match for {0} - {1} !".format(email, namespace))
now = datetime.datetime.now()
matches = []
for hash in hashes:
record = Resource.query.filter(Resource.hash == hash).limit(app.config['MAX_PAGES_PER_HASH']).all()
urls = [{'webpage_url': r.pageview.url, 'webpage_hash': sha256(r.pageview.url), 'resource_url': r.url} for r in record]
new_match = {'hash': hash, 'urls': urls}
matches.append(new_match)
rule = YaraRuleset.query.filter_by(email=email, namespace=namespace).first()
if rule.email_tokens is None:
rule.email_tokens = 5
print("email tokens = {}".format(rule.email_tokens))
if rule.last_received_tokens is None:
rule.last_received_tokens = now
print("last rx tokens = {}".format(rule.last_received_tokens))
rule.email_tokens -= 1
rule.email_tokens = min(10, rule.email_tokens + (now - rule.last_received_tokens) // datetime.timedelta(minutes=1))
rule.last_received_tokens = now
print("at end, email tokens = {}".format(rule.email_tokens))
sendmail(email,
'YARA Scan Results (success!): {}'.format(namespace),
render_template('email/yara_match.html', matches=matches, tokens=rule.email_tokens))
if rule.email_tokens <= 0:
print("removing rule!")
sendmail(rule.email,
'YARA Rule Removed ({})'.format(rule.namespace),
render_template('email/yara_goodbye.html', rule=rule))
db.session.delete(rule)
db.session.commit()
def yara_scan_file_for_email(email, path):
# TODO: filter for 'scan_on_upload==True' too
# TODO: store compiled rules in database to avoid re-compiling?
try:
rulesets = YaraRuleset.query.filter_by(email=email).all()
sources = {}
for r in rulesets:
sources[r.namespace] = r.source
def matchcb(data):
if data['matches']:
yara_report_matches.apply_async(args=(email,
data['namespace'],
[path.split('/')[-1].split('.')[0]]),
countdown=10)
return yara.CALLBACK_CONTINUE
# TODO: these "except" clauses should be consolidated and should catch
# specific exceptions.
try:
rules = yara.compile(sources=sources)
with gzip.open(path, 'rb') as f:
try:
rules.match(data=f.read(), callback=matchcb)
except:
# TODO: this should never happen, so this email should
# go to the site admin
sendmail('*****@*****.**',
'YARA Livescan Results (error while reading file / matching ruleset!)',
render_template('email/yara_error.html'))
except:
# TODO: this should never happen, so this email should go to the site admin
sendmail('*****@*****.**',
'YARA Scan Results (error in rule compilation!)',
render_template('email/yara_error.html'))
except (sqlalchemy.exc.DatabaseError, sqlalchemy.exc.OperationalError) as exc:
print("retrying....")
def find_suspicious(pv_id):
pageviews = Pageview.query.filter(Pageview.id == pv_id).all()
suspicious = []
if not pageviews:
print('ERROR: no pageview found with id {}'.format(pv_id))
elif len(pageviews) > 1:
print('ERROR: >1 pageviews found with id {}'.format(pv_id))
else:
pageview = pageviews[0]
hostname = urlparse(pageview.url).netloc
root_domain = get_root_domain(hostname)
domains = [
get_root_domain(urlparse(rsc.url).netloc)
for rsc in pageview.resources
]
# dedup the list of domains and drop all empty values:
domains = list(set(filter(lambda x: x, domains)))
# remove any domains that are rooted in the WHITELIST or the current root_domain:
suspicious = list(
filter(
lambda x: not any(
[x.endswith(y) for y in WHITELIST + [root_domain]]),
domains))
if suspicious:
print("FOUND SUSPICIOUS DOMAINS: {}".format(suspicious))
sendmail(
'*****@*****.**',
'Suspicious Resource Result! {}'.format(pageview.url),
render_template('email/suspicious_resource_result.html',
url=pageview.url,
suspicious=suspicious,
domains=domains))
else:
print("Didn't find any suspicious domains")
return suspicious
def yara_retroscan_for_rule(rule_id):
rule = YaraRuleset.query.filter_by(id=rule_id).one()
sources = {rule.namespace: rule.source}
try:
yara_rule = yara.compile(sources=sources)
except:
# TODO: this should never happen, so this email should go to the site admin
# TODO: this should not catch any/all exceptions!
sendmail('*****@*****.**',
'YARA Retroscan Results (error in rule compilation!)',
render_template('email/yara_error.html'))
return
os.nice(5)
matches = []
try:
for subdir in os.listdir(app.config['RESOURCE_CONTENT_FOLDER']):
try:
subdir = os.path.join(app.config['RESOURCE_CONTENT_FOLDER'],
subdir)
for path in os.listdir(subdir):
with gzip.open(os.path.join(subdir, path), 'rb') as f:
if yara_rule.match(data=f.read()):
matches.append(path.split('.')[0])
if len(matches) > app.config['MAX_HASHES']:
break
except NotADirectoryError:
pass
except:
sendmail(rule.email,
'YARA Retroscan Error ({0})'.format(sys.exc_info()),
render_template('email/yara_error.html'))
else:
yara_report_matches.apply_async(args=(rule.email, rule.namespace,
matches))
os.nice(0)
def yara_remove():
""" yara rule removal UI view """
errors = []
if request.method == 'POST':
code = request.form['removal_code'].strip()
if len(code) <= 0:
errors.append("You must provide a removal code")
else:
rule = YaraRuleset.query.filter_by(removal_code=code).first()
if not rule:
errors.append("Could not find an active YARA rule with that "
"removal code.")
if len(errors) == 0:
sendmail(rule.email,
'YARA Rule Removed ({})'.format(rule.namespace),
render_template('email/yara_goodbye.html', rule=rule))
db.session.delete(rule)
db.session.commit()
flash("Rules successfully removed")
return render_template('yara/remove.html', errors=errors)
def yara_remove():
""" yara rule removal UI view """
errors = []
if request.method == 'POST':
code = request.form['removal_code'].strip()
if len(code) <= 0:
errors.append("You must provide a removal code")
else:
rule = YaraRuleset.query.filter_by(removal_code=code).first()
if not rule:
errors.append("Could not find an active YARA rule with that "
"removal code.")
if len(errors) == 0:
sendmail(rule.email,
'YARA Rule Removed ({})'.format(rule.namespace),
render_template('email/yara_goodbye.html', rule=rule))
db.session.delete(rule)
db.session.commit()
flash("Rules successfully removed")
return render_template('yara/remove.html', errors=errors)