def remote_whitelist_jump(target_env, target_project, ip_or_regex, ip_format, sql_mode):
"""
:param target_env: stage or prod environment
:param target_project: particular project
:param ip_or_regex: and ip address, or regex string
:param ip_format: a choice, ip or regex
:param sql_mode: insert, select
:return: similar to remote_whitelist but created to hit jump first, pull down repo, then run remote command.
this would be 2 jumps from Jenkins, not really any use case for this now
"""
id = '/home/ec2-user/.ssh/id_rsa'
# git clone to /tmp or user directory? will clone to tmp to keep things clean
with cd('/tmp'):
if repo_exists('scripts'):
print 'repo exists, just update'
with cd('scripts'):
run('git pull')
else:
run('git clone ssh://git@%s' % repo)
# run script remotely
with cd('scripts'):
# create sql file on dest host
run('python whitelist.py %s %s %s %s %s' % (target_env, target_project, ip_or_regex, ip_format, sql_mode))
# create whitelist object to return login to be used on dest host
w = Whitelist(target_env=target_env, target_project=target_project, ip_or_regex=ip_or_regex, ip_format=ip_format, sql_mode=sql_mode)
login_cmd = w.sql_login()
host = host_data[target_env][target_project]['host']
# print 'Will login to remote mysql with cmd:'
# print login_cmd
ssh_and_insert = 'ssh -o StrictHostKeyChecking=no -i %s %s %s < %s_%s_whitelist_%s.sql' % (id, host, login_cmd, target_env, target_project, sql_mode)
# need to switch to ec2 user
run_su(ssh_and_insert)
def remote_whitelist(target_env,
target_project,
whitelist_type,
sql_mode,
ip_format,
account_number,
ip_or_regex=None,
email=None):
"""
:param target_env: stage or prod environment
:param target_project: particular project
:param ip_or_regex: and ip address, or regex string
:param ip_format: a choice, ip or regex
:param sql_mode: insert, or select
:param account: account # to add
:return: create a sql query file, and then use whitelist object to create a SQL login
"""
w = Whitelist(target_env=target_env,
target_project=target_project,
whitelist_type=whitelist_type,
ip_or_regex=ip_or_regex,
ip_format=ip_format,
sql_mode=sql_mode,
email=email,
account_number=account_number)
# create sql file in working jenkins dir, then run ssh command.
# running the ssh command locally instead of remote() b/c the ' < sqlfile ' insert process worked best
w.setup()
local(w.ssh_mysql_cmd())
def __init__(self,
config_path=_DEFAULT_CONFIG_PATH,
metadata_server=_METADATA_SERVICE_ADDRESS):
self._config_path = config_path
self._metadata_server = metadata_server
self._use_iam_role_credentials = False
self.region = ''
self.endpoint = ''
self.ec2_endpoint = ''
self.host = ''
self.asg_name = 'NONE'
self.proxy_server_name = ''
self.proxy_server_port = ''
self.debug = False
self.pass_through = False
self.push_asg = False
self.push_constant = False
self.constant_dimension_value = ''
self.enable_high_resolution_metrics = False
self.flush_interval_in_seconds = ''
self._mapper = None
self._mapper_time = 0
self._load_configuration()
self.whitelist = Whitelist(
WhitelistConfigReader(self.WHITELIST_CONFIG_PATH,
self.pass_through).get_regex_list(),
self.BLOCKED_METRIC_PATH)
def remote_whitelist(target_env, target_project, whitelist_type, sql_mode, ip_format, account_number, ip_or_regex=None, email=None):
"""
:param target_env: stage or prod environment
:param target_project: particular project
:param ip_or_regex: and ip address, or regex string
:param ip_format: a choice, ip or regex
:param sql_mode: insert, or select
:param account: account # to add
:return: create a sql query file, and then use whitelist object to create a SQL login
"""
w = Whitelist(target_env=target_env, target_project=target_project, whitelist_type=whitelist_type, ip_or_regex=ip_or_regex, ip_format=ip_format, sql_mode=sql_mode, email=email, account_number=account_number)
# create sql file in working jenkins dir, then run ssh command.
# running the ssh command locally instead of remote() b/c the ' < sqlfile ' insert process worked best
w.setup()
local(w.ssh_mysql_cmd())
def __init__(self, db, config):
self._db = db
self.config = config
if self.config["DEBUG"]:
print("Initializing CallScreener")
self._blacklist = Blacklist(db, config)
self._whitelist = Whitelist(db, config)
self._nomorobo = NomoroboService()
if self.config["DEBUG"]:
print("CallScreener initialized")
def remote_whitelist_jump(target_env, target_project, ip_or_regex, ip_format,
sql_mode):
"""
:param target_env: stage or prod environment
:param target_project: particular project
:param ip_or_regex: and ip address, or regex string
:param ip_format: a choice, ip or regex
:param sql_mode: insert, select
:return: similar to remote_whitelist but created to hit jump first, pull down repo, then run remote command.
this would be 2 jumps from Jenkins, not really any use case for this now
"""
id = '/home/ec2-user/.ssh/id_rsa'
# git clone to /tmp or user directory? will clone to tmp to keep things clean
with cd('/tmp'):
if repo_exists('scripts'):
print 'repo exists, just update'
with cd('scripts'):
run('git pull')
else:
run('git clone ssh://git@%s' % repo)
# run script remotely
with cd('scripts'):
# create sql file on dest host
run('python whitelist.py %s %s %s %s %s' %
(target_env, target_project, ip_or_regex, ip_format, sql_mode))
# create whitelist object to return login to be used on dest host
w = Whitelist(target_env=target_env,
target_project=target_project,
ip_or_regex=ip_or_regex,
ip_format=ip_format,
sql_mode=sql_mode)
login_cmd = w.sql_login()
host = host_data[target_env][target_project]['host']
# print 'Will login to remote mysql with cmd:'
# print login_cmd
ssh_and_insert = 'ssh -o StrictHostKeyChecking=no -i %s %s %s < %s_%s_whitelist_%s.sql' % (
id, host, login_cmd, target_env, target_project, sql_mode)
# need to switch to ec2 user
run_su(ssh_and_insert)
def __init__(self, config_path=_DEFAULT_CONFIG_PATH, metadata_server=_METADATA_SERVICE_ADDRESS):
self._config_path = config_path
self._metadata_server = metadata_server
self._use_iam_role_credentials = False
self.region = ''
self.endpoint = ''
self.host = ''
self.proxy_server_name = ''
self.proxy_server_port = ''
self.debug = False
self.pass_through = False
self._load_configuration()
self.whitelist = Whitelist(WhitelistConfigReader(self.WHITELIST_CONFIG_PATH, self.pass_through).get_regex_list(), self.BLOCKED_METRIC_PATH)
def retrieve_whitelist(self):
""" retrieve and parse the .cvmfswhitelist file from the repository """
whitelist = self._fetcher.retrieve_raw_file(_common._WHITELIST_NAME)
return Whitelist(whitelist)
def __init__(self, db, config):
""" Initializes the CallLogger object and creates the
CallLog table if it doesn't exist
"""
self.db = db
self.config = config
if self.config["DEBUG"]:
print("Initializing CallLogger")
# Create the Call_History table if it does not exist
sql = """CREATE TABLE IF NOT EXISTS CallLog (
CallLogID INTEGER PRIMARY KEY AUTOINCREMENT,
Name TEXT,
Number TEXT,
Action TEXT,
Reason TEXT,
Date TEXT,
Time TEXT,
SystemDateTime TEXT);"""
curs = self.db.cursor()
curs.executescript(sql)
try:
# Early versions of callattendant (<= v0.3.1) do not contain
# an Action column or Reason column. This hack/code attempts
# to add the columns if they don't exit.
sql = """SELECT COUNT(*) AS CNTREC
FROM pragma_table_info('CallLog')
WHERE name='Action'"""
curs.execute(sql)
count = curs.fetchone()[0]
if count == 0:
# Dependencies: ensures tables used in the sql exist
from whitelist import Whitelist
from blacklist import Blacklist
whitelist = Whitelist(db, config)
blacklist = Blacklist(db, config)
print(">> Adding Action column to CallLog table")
sql = """ALTER TABLE CallLog ADD COLUMN Action TEXT default null"""
curs.executescript(sql)
print(">> Updating Action column in CallLog table")
sql = """UPDATE CallLog
SET `Action`=(select
CASE
WHEN b.PhoneNo is not null then 'Permitted'
WHEN c.PhoneNo is not null then 'Blocked'
ELSE 'Screened'
END actn
FROM CallLog as a
LEFT JOIN Whitelist as b ON a.Number = b.PhoneNo
LEFT JOIN Blacklist as c ON a.Number = c.PhoneNo
WHERE CallLog.CallLogID = a.CallLogID)"""
curs.executescript(sql)
print(">> Adding Reason column to CallLog table")
sql = """ALTER TABLE CallLog ADD COLUMN Reason TEXT default null"""
curs.executescript(sql)
print(">> Updating Reason column in CallLog table")
sql = """UPDATE CallLog
SET `Reason`=(select
CASE
WHEN b.PhoneNo is not null then b.Reason
WHEN c.PhoneNo is not null then c.Reason
ELSE null
END reason
FROM CallLog as a
LEFT JOIN Whitelist as b ON a.Number = b.PhoneNo
LEFT JOIN Blacklist as c ON a.Number = c.PhoneNo
WHERE CallLog.CallLogID = a.CallLogID)"""
curs.executescript(sql)
except Exception as e:
print(e)
curs.close()
self.db.commit()
if self.config["DEBUG"]:
print("CallLogger initialized")
def __init__(self, db):
self._db = db
self._blacklist = Blacklist(db)
self._whitelist = Whitelist(db)
self._nomorobo = NomoroboService()
from flask import Flask, request, jsonify
from urllib.parse import urlparse
from grega_classifier import GregaClassifier
from uci_classifier import UciClassifier
import pickle
import numpy as np
from whitelist import Whitelist
from joblib import Parallel, delayed
import functools
import sys
app = Flask(__name__)
classifiers = []
whitelist = Whitelist()
UCI_MODEL_PATH = "./models/uci/decision-uci.joblib"
GREGA_MODEL_PATH = "./models/grega/small-ensemble-knn-rf-dt.pkl"
WHITELIST_CSV_PATH = "./data/whitelist_domains.csv"
def is_valid_url_scheme(url):
""" Determine if the url has valid scheme for phishing-detection """
if url is None or len(url.strip()) == 0:
return False
parsed_url = urlparse(url)
scheme = parsed_url.scheme
return scheme in ("http", "https")
@app.route("/detect", methods=["POST"])