def get_difference(self, response_df):
"""
A method for getting changed cell values of existing records against database records
:param response_df: A dataframe generated from response
:return: None
"""
ip_table = conf.read().get('db-conf', 'ip_table')
try:
table = pd.read_sql(
"SELECT * FROM {};".format(ip_table), con=self.db.engine, chunksize=self.chunk_size
)
cols = [
'ip_address', 'reliability', 'priority', 'activity', 'sub_category', 'country', 'city', 'latitude',
'longitude', 'source', 'target', 'dest_port', 'last_online', 'first_seen', 'used_by', 'reference_link'
]
df = response_df
diff_df = []
while True:
chunk = next(table)
exist_df = df[df['ip_address'].isin(chunk['ip_address'].values)]
exist_df = exist_df.sort_values('ip_address').reset_index(drop=True)
chunk_df = chunk[chunk['ip_address'].isin(exist_df['ip_address'].values)]
chunk_df = chunk_df.sort_values('ip_address').reset_index(drop=True)
chunk_df = chunk_df[cols]
exist_df = exist_df[cols]
diff = self.diff_df(exist_df, chunk_df, how='left')
diff_df.append(diff)
if len(chunk.index) < self.chunk_size:
break
self._updated_df = pd.concat(diff_df)
return self._updated_df
except (StopIteration, TypeError, ValueError) as err:
logger.exception(err, exc_info=False)
def top_threats(count):
"""
API for getting top threats
"""
count = int(count)
if count > 10:
count = 10
url = "https://api.metadefender.com/v4/feed/infected/"
header = {"apikey": conf.read().get('Alexa', 'apikey')}
response = requests.get(url=url, headers=header)
response = json.loads(response.text)
response = json.dumps(response['top_infected'][:count])
list_response = json.loads(response)
for dic in list_response:
threat_name = "Not available"
threat_type = None
if dic['scan_results']['threat_name'] is not None:
threat_type = dic['scan_results']['threat_name'].split('/')[0]
threat_name = dic['scan_results']['threat_name'].split('/')[1]
else:
threat_type = dic['file_type_extension']
dic['threat_type'] = threat_type
dic['threat_name'] = threat_name
del dic['scan_results']
final_response = json.dumps(list_response)
return final_response
def __init__(self):
"""
Constructor
"""
self.db_uri = conf.read().get('db-conf', 'DB_URI')
self.engine = create_engine(self.db_uri)
self.conn = self.engine.connect()
def __init__(self, ip_address):
"""
Instance initialisation
:param ip_address:
"""
self.ip_address = ip_address
self.geolite_db = conf.read().get('geoip2', 'GEOLITE_DB')
if not os.path.isfile(self.geolite_db):
raise FileNotFoundError("GeoLite database is not present")
else:
self._reader = geoip2.database.Reader(self.geolite_db)
self._result = self._reader.city(self.ip_address)
self._reader.close()
def get_unique(self, response_df):
"""
A method for getting unique records from response dataframe by comparing with database.
:param response_df:
:return: A dataframe with unique ip_address records
"""
ip_table = conf.read().get('db-conf', 'ip_table')
try:
table = pd.read_sql("SELECT * FROM {};".format(ip_table), con=self.db.engine, chunksize=self.chunk_size)
while True:
chunk = next(table)
response_df = response_df[~response_df['ip_address'].isin(chunk['ip_address'].values)]
if len(chunk.index) < self.chunk_size:
break
return response_df
except (StopIteration, TypeError) as err:
logger.exception(err, exc_info=False)
def current_threats():
now = datetime.now()
curr_time = now.strftime("%Y-%m-%d %H:%M:%S")
USER = conf.read().get('SIEM-CONF', 'USER')
PASSWORD = conf.read().get('SIEM-CONF', 'PASSWORD')
HOST = conf.read().get('SIEM-CONF', 'HOST')
DB_NAME = conf.read().get('SIEM-CONF', 'DB_NAME')
if os.path.isfile('/home/threat-intelligent-hub/alexa_files/timestamp.txt'):
tmp = open('/home/threat-intelligent-hub/alexa_files/timestamp.txt', 'r+')
last_read = tmp.read()
tmp.close()
tmp = open('/home/threat-intelligent-hub/alexa_files/timestamp.txt', 'w+')
else:
tmp = open('/home/threat-intelligent-hub/alexa_files/timestamp.txt', 'w+')
last_read = None
conn = connect(user=USER, password=PASSWORD, host=HOST, database=DB_NAME)
cursor = conn.cursor()
query_first = "SELECT A.*,B.name FROM (SELECT inet6_ntoa(src_ip),inet6_ntoa(dst_ip),plugin_sid,risk,cast(timestamp as char) timestamp FROM alarm WHERE DATE(TIMESTAMP) = DATE(NOW() )) A INNER JOIN plugin_sid B ON A.plugin_sid = B.sid where A.risk >= 3 order by A.risk,A.timestamp desc limit 10;"
query_last = "SELECT A.src_ip,A.dst_ip,A.plugin_sid,A.risk,A.timestamp,B.name FROM (SELECT inet6_ntoa(src_ip) as src_ip,inet6_ntoa(dst_ip) as dst_ip,plugin_sid,risk,cast(timestamp as char) timestamp,plugin_id FROM alarm WHERE timestamp between CONVERT_TZ('{0}' , @@session.time_zone, '+00:00') and CONVERT_TZ( '{1}', @@session.time_zone, '+00:00') and risk >= 3 ) A INNER JOIN plugin_sid B ON A.plugin_sid = B.sid and B.plugin_id = A.plugin_id inner join plugin C ON C.id = B.plugin_id order by A.risk desc ,A.timestamp desc limit 10;".format(
last_read, curr_time)
if not last_read:
cursor.execute(query_first)
tmp.write(str(curr_time))
else:
cursor.execute(query_last)
tmp.write(str(curr_time))
# fields = [x[0] for x in cursor.description] # this will extract row headers
result = cursor.fetchall()
len_data = len(result)
json_dict = {}
data = []
formatted_date = ''
if last_read:
date_time_obj = datetime.strptime(last_read, '%Y-%m-%d %H:%M:%S')
formatted_date = date_time_obj.strftime("%d %B, %Y %H:%M %p")
else:
formatted_date = None
if len_data > 0:
for row in result:
alarm_dict = {}
alarm_dict["source_ip"] = row[0]
alarm_dict["destination_ip"] = row[1]
alarm_dict["plugin_sid"] = row[2]
alarm_dict["risk"] = row[3]
alarm_dict["timestamp"] = row[4]
alarm_dict["name"] = row[5]
data.append(alarm_dict)
json_dict["last_read"] = formatted_date
json_dict["count"] = len_data
json_dict["data"] = data
else:
json_dict["last_read"] = formatted_date
json_dict["count"] = 0
json_dict["data"] = []
json_obj = json.dumps(json_dict)
json_obj = json.loads(json_obj)
tmp.close()
cursor.close()
conn.close()
return jsonify(json_obj)
# imports
import os
import json
from datetime import datetime
import requests
from flask import Flask, request, jsonify
from mysql.connector import connect
# local imports
from api.models import BlockedIPsModel, MalwareURLsModel, AuthDataModel, RevisionTrackerModel, db
from api.authenticator import Authenticator
from common.config import conf
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = conf.read().get('db-conf', 'DB_URI')
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
auth = Authenticator(request)
@app.route('/', methods=['GET'])
def home():
"""
A index api view
:return:
"""
host = request.host_url
response = {
'message': 'Welcome to threat hub API service',
'api_list': {
1: host + 'api/register',
class Logger:
"""
A Threat intelligence hub logger which will take care
of logging to console and file.
"""
def __init__(self, filepath):
"""
Constructor
:param filepath:
"""
self.filepath = filepath
self.logger = logging.getLogger('THREAT-HUB')
self.logger.setLevel(logging.DEBUG)
self._formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# file handler
file_handller = logging.FileHandler(os.path.join(self.filepath))
file_handller.setLevel(logging.DEBUG)
file_handller.setFormatter(self._formatter)
self.logger.addHandler(file_handller)
# console handler
con_handler = logging.StreamHandler()
con_handler.setLevel(logging.ERROR)
con_handler.setFormatter(self._formatter)
self.logger.addHandler(con_handler)
log_file = conf.read().get('Other', 'LOG_FILE')
logger = Logger(log_file).logger