def load_config(self):
cfg = ConfigParser.SafeConfigParser()
for parent, dirnames, filenames in os.walk(conf_dir):
for filename in filenames:
split_name = filename.split('.')
if len(split_name) == 2 and split_name[1] == 'ini':
conf_file = os.path.join(parent, filename)
cfg.read(conf_file)
try:
config.parse_config(cfg)
except config.ParsingError, err:
self.center_log.error(str(err))
continue
self.center_log.info("Configuration file <%s> loaded." % conf_file)
if config.config['name'] == "Globalpot":
self.globalpot_cfg = config.config.copy()
else:
honeypot_cfg = config.config.copy()
name = honeypot_cfg['name']
if self.honeypots.has_key(name):
self.center_log.warning("Duplicate name of honeypots: %s\n", name)
else:
self.honeypots[name] = [honeypot_cfg, None]
config.config.clear()
def load_config(self):
cfg = ConfigParser.SafeConfigParser()
for parent, dirnames, filenames in os.walk(conf_dir):
for filename in filenames:
split_name = filename.split('.')
if len(split_name) == 2 and split_name[1] == 'ini':
conf_file = os.path.join(parent, filename)
cfg.read(conf_file)
try:
config.parse_config(cfg)
except config.ParsingError, err:
self.center_log.error(str(err))
continue
self.center_log.info("Configuration file <%s> loaded." %
conf_file)
if config.config['name'] == "Globalpot":
self.globalpot_cfg = config.config.copy()
else:
honeypot_cfg = config.config.copy()
name = honeypot_cfg['name']
if self.honeypots.has_key(name):
self.center_log.warning(
"Duplicate name of honeypots: %s\n", name)
else:
self.honeypots[name] = [honeypot_cfg, None]
config.config.clear()
def test_unify_separate_event_files(self):
configs = parse_config()
configs['data']['raw_path'] = 'tests/raw_data_test/'
unified_df = unify_separate_event_files(self.spark,
configs['data']['raw_path'])
self.assertTrue(unified_df, 'The dataframe should assert to True')
def main():
configs = parse_config()
spark = create_spark_session()
read_stream = start_watch_for_csvs(spark, configs)
write_stream = create_streaming_window(read_stream, configs)
write_stream.awaitTermination()
def test_calculate_jerk_from_truck_events(self):
configs = parse_config()
configs['data']['raw_path'] = 'tests/raw_data_test/'
unified_df = unify_separate_event_files(self.spark,
configs['data']['raw_path'])
self.assertTrue(unified_df,
'The unified dataframe should assert to True')
jerked_truck_events_df = calculate_jerk_from_truck_events(unified_df)
self.assertTrue(
jerked_truck_events_df,
'The jerked truck events dataframe should assert to True')
def create_timescaledb_connection(configs=None, autocommit=True):
if configs is None:
configs = parse_config()
# connect to recreate the database
conn = psycopg2.connect(
"host={} dbname={} port={} user={} password={}".format(
configs['timescaledb']['host'], configs['timescaledb']['db'],
configs['timescaledb']['port'], configs['timescaledb']['user'],
configs['timescaledb']['password']))
conn.set_session(autocommit=autocommit)
return conn
def main():
configs = parse_config()
spark = create_spark_session()
truck_events_df = unify_separate_event_files(spark,
configs['data']['raw_path'])
truck_events_df.createOrReplaceTempView("truck_events")
spark.sql(average_acceleration_query).show(truncate=False)
jerked_truck_events_df = calculate_jerk_from_truck_events(truck_events_df)
jerked_truck_events_df.createOrReplaceTempView("jerked_truck_events")
spark.sql(max_jerk_x_query).show(truncate=False)
spark.sql(max_jerk_y_query).show(truncate=False)
spark.sql(max_jerk_z_query).show(truncate=False)
spark.stop()
def test_watch_and_stream_test_files(self):
configs = parse_config()
new_db_name = configs['timescaledb']['db'] + '_test'
configs['timescaledb']['db'] = new_db_name
configs['data']['raw_path'] = 'tests/raw_data_test/'
conn = create_timescaledb_connection()
cur = conn.cursor()
cur.execute(
"select count(*) AS c from pg_catalog.pg_database where datname = '"
+ new_db_name + "'")
results = cur.fetchone()
if results[0] < 1:
cur.execute('CREATE DATABASE ' + new_db_name)
cur.close()
conn.close()
conn = create_timescaledb_connection(configs)
cur = conn.cursor()
cur.execute(sql_queries.drop_jerked_truck_events_table)
cur.execute(sql_queries.create_jerked_truck_events_table)
cur.close()
conn.close()
read_stream = start_watch_for_csvs(self.spark, configs)
self.assertTrue(read_stream, 'The read stream should assert True')
write_stream = create_streaming_window(read_stream, configs)
write_stream.processAllAvailable()
self.assertTrue(read_stream,
'The write stream also should assert True')
def test_parse_config(self):
self.assertTrue(parse_config())
def create_spark_session():
configs = parse_config()
return SparkSession.builder.appName(
configs['spark']['app_name']).getOrCreate()