def readrows(self):
"""The readrows method reads simply 'combines' the rows of
multiple files OR gunzips the file and then reads the rows
"""
# For each file (may be just one) create a BroLogReader and use it
for self._filepath in self._files:
# Check if the file is zipped
tmp = None
if self._filepath.endswith('.gz'):
tmp = tempfile.NamedTemporaryFile(delete=False)
with gzip.open(self._filepath,
'rb') as f_in, open(tmp.name, 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
# Set the file path to the new temp file
self._filepath = tmp.name
# Create a BroLogReader
reader = bro_log_reader.BroLogReader(self._filepath)
for row in reader.readrows():
yield row
# Clean up any temp files
try:
if tmp:
os.remove(tmp.name)
print('Removed temporary file {:s}...'.format(tmp.name))
except IOError:
pass
def parseFile(self, filename, json=False):
""" Creates a pandas dataframe from given brofile
Parameters
----------
filename : string
Path to file to be parsed
Returns
-------
result : pd.DataFrame
Pandas dataframe containing bro log file
"""
df = None
if not json:
bro_log = bro_log_reader.BroLogReader(filename)
df = pd.DataFrame(bro_log.readrows())
else:
df = pd.read_json(filename, lines=True)
#df.rename(
# index=str,
# columns={
# 'client_header_names': 'header_values'},
# inplace=True)
df['header_values'] = df['header_values'].apply(
self.__parseHeaderValues__)
return df
def parseLOG(filename):
""" Generate a list of Dumont Requests from a bro log file
Parameters
----------
filename : string
path to .pcap file to parse.
Returns
-------
result : list of DumontLog()
ordered list of dumont logs.
"""
DumontRequests = []
bro_log = bro_log_reader.BroLogReader(filename)
data = pd.DataFrame(bro_log.readrows())
data['header_values'] = data['header_values'].apply(__parseHeaderValues__)
for d in data.iterrows():
if d[1]['method'] == 'GET' or d[1]['method'] == 'POST':
DumontRequests.append(DumontLog(d[1]))
return aggregateTemporalFeatures(DumontRequests)
def create_dataframe(self,
log_filename,
ts_index=True,
aggressive_category=True,
usecols=None):
""" Create a Pandas dataframe from a Bro/Zeek log file
Args:
log_fllename (string): The full path to the Zeek log
ts_index (bool): Set the index to the 'ts' field (default = True)
aggressive_category (bool): convert unknown columns to category (default = True)
"""
# Create a Zeek log reader just to read in the header for names and types
_bro_reader = bro_log_reader.BroLogReader(log_filename)
_, field_names, field_types, _ = _bro_reader._parse_bro_header(
log_filename)
header_names = field_names
# If usecols is set then we'll subset the fields and types
if usecols:
# Usecols needs to include ts
if 'ts' not in usecols:
usecols.append('ts')
field_types = [
t for t, field in zip(field_types, field_names)
if field in usecols
]
field_names = [field for field in field_names if field in usecols]
# Get the appropriate types for the Pandas Dataframe
pandas_types = self.pd_column_types(field_names, field_types,
aggressive_category)
# Now actually read the Zeek Log using Pandas read CSV
self._df = pd.read_csv(log_filename,
sep='\t',
names=header_names,
usecols=usecols,
dtype=pandas_types,
comment="#",
na_values='-')
# Now we convert 'time' and 'interval' fields to datetime and timedelta respectively
for name, bro_type in zip(field_names, field_types):
if bro_type == 'time':
self._df[name] = pd.to_datetime(self._df[name], unit='s')
if bro_type == 'interval':
self._df[name] = pd.to_timedelta(self._df[name], unit='s')
# Set the index
if ts_index and not self._df.empty:
self._df.set_index('ts', inplace=True)
return self._df
def create_dataframe(self, log_filename, fillna=True):
""" Create a Spark dataframe from a Bro/Zeek log file
Args:
log_fllename (string): The full path to the Zeek log
fillna (bool): Fill in NA/NaN values (default=True)
"""
# Create a Zeek log reader just to read in the header for names and types
_bro_reader = bro_log_reader.BroLogReader(log_filename)
_, field_names, field_types, _ = _bro_reader._parse_bro_header(log_filename)
# Get the appropriate types for the Spark Dataframe
spark_schema = self.build_spark_schema(field_names, field_types)
# Now actually read the Zeek Log using Spark read CSV
_df = self.spark.read.csv(log_filename, schema=spark_schema, sep='\t', comment="#", nullValue='-')
''' Secondary processing (cleanup)
- Fix column names with '.' in them
- Fill in Nulls (optional)
- timestamp convert
- boolean convert
'''
# Fix column names
''' Note: Yes column names with '.' in them can be escaped with backticks when selecting them BUT
many pipeline operations will FAIL internally if the column names have a '.' in them.
'''
fixed_columns = list(map(lambda x: x.replace('.', '_'), _df.columns))
_df = _df.toDF(*fixed_columns)
# Fill in NULL values
if fillna:
_df = _df.na.fill(0) # For numeric columns
_df = _df.na.fill('-') # For string columns
# Convert timestamp and boolean columns
for name, f_type in zip(field_names, field_types):
# Some field names may have '.' in them, so we create a reference name to those fields
ref_name = name.replace('.', '_')
if f_type == 'time':
_df = _df.withColumn(name, _df[ref_name].cast('timestamp'))
if f_type == 'bool':
_df = _df.withColumn(name, when(col(ref_name) == 'T', 'true').when(col(ref_name) == 'F', 'false')
.otherwise('null').cast('boolean'))
# Return the spark dataframe
return _df
def __init__(self, filepath, eps=10, max_rows=None):
"""Initialization for the LiveSimulator Class
Args:
eps (int): Events Per Second that the simulator will emit events (default = 10)
max_rows (int): The maximum number of rows to generate (default = None (go forever))
"""
# Compute EPS timer
# Logic:
# - Normal distribution centered around 1.0/eps
# - Make sure never less than 0
# - Precompute 1000 deltas and then just cycle around
self.eps_timer = itertools.cycle([
max(0, delta) for delta in np.random.normal(
1.0 / float(eps), .5 / float(eps), size=1000)
])
# Initialize the Zeek log reader
self.log_reader = bro_log_reader.BroLogReader(filepath, tail=False)
# Store max_rows
self.max_rows = max_rows
def _get_field_info(self, log_filename):
"""Internal Method: Use ZAT log reader to read header for names and types"""
_bro_reader = bro_log_reader.BroLogReader(log_filename)
_, field_names, field_types, _ = _bro_reader._parse_bro_header(log_filename)
return field_names, field_types
print('This example only works with Zeek x509.log files..')
sys.exit(1)
# File may have a tilde in it
if args.bro_log:
args.bro_log = os.path.expanduser(args.bro_log)
# Create a VirusTotal Query Class
vtq = vt_query.VTQuery()
# These domains may be spoofed with a certificate issued by 'Let's Encrypt'
spoofed_domains = set(
['paypal', 'gmail', 'google', 'apple', 'ebay', 'amazon'])
# Run the bro reader on the x509.log file looking for spoofed domains
reader = bro_log_reader.BroLogReader(args.bro_log, tail=True)
for row in reader.readrows():
# Pull out the Certificate Issuer
issuer = row['certificate.issuer']
if "Let's Encrypt" in issuer:
# Check if the certificate subject has any spoofed domains
subject = row['certificate.subject']
domains = subject[3:] # Just chopping off the 'CN=' part
if any([domain in domains for domain in spoofed_domains]):
print('\n<<< Suspicious Certificate Found >>>')
pprint(row)
# Make a Virus Total query with the spoofed domain (just for fun)
results = vtq.query_url(domain)
if __name__ == '__main__':
# Example to run the bro log reader on a given file
# Collect args from the command line
parser = argparse.ArgumentParser()
parser.add_argument('bro_log',
type=str,
help='Specify a bro log to run BroLogReader test on')
parser.add_argument('-t',
'--tail',
action='store_true',
help='Turn on log tailing')
args, commands = parser.parse_known_args()
# Check for unknown args
if commands:
print('Unrecognized args: %s' % commands)
sys.exit(1)
# File may have a tilde in it
if args.bro_log:
args.bro_log = os.path.expanduser(args.bro_log)
# Run the bro reader on a given log file
reader = bro_log_reader.BroLogReader(args.bro_log,
tail=args.tail,
strict=True)
for row in reader.readrows():
pprint(row)
except IOError:
vtq = vt_query.VTQuery(max_cache_time=60 * 24 *
7) # One week cache
# See our 'Risky Domains' Notebook for the analysis and
# statistical methods used to compute this risky set of TLDs
risky_tlds = set([
'info', 'tk', 'xyz', 'online', 'club', 'ru', 'website', 'in', 'ws',
'top', 'site', 'work', 'biz', 'name', 'tech', 'loan', 'win', 'pro'
])
# Launch long lived process with signal catcher
with signal_utils.signal_catcher(save_vtq):
# Run the bro reader on the dns.log file looking for risky TLDs
reader = bro_log_reader.BroLogReader(args.bro_log)
for row in reader.readrows():
# Pull out the TLD
query = row['query']
tld = tldextract.extract(query).suffix
# Check if the TLD is in the risky group
if tld in risky_tlds:
# Make the query with the full query
results = vtq.query_url(query)
if results.get('positives', 0) > 3: # At least four hits
print('\nRisky Domain DNS Query Found')
print('From: {:s} To: {:s} QType: {:s} RCode: {:s}'.
format(row['id.orig_h'], row['id.resp_h'],
row['qtype_name'], row['rcode_name']))