def model_running_applications(self) -> None:
"""
Models running applications.
"""
logger = logging.getLogger(self.__logger_name)
modelled_apps = list()
saved_application_profile_names = AppProfileDataManager.get_saved_app_profiles_names(
)
logger.info("Starting to model running applications.")
for saved_app_profile_name in saved_application_profile_names:
app_profile = AppProfileDataManager.get_saved_profile(
saved_app_profile_name)
if ProcessHandler.is_application_recently_retrieved(app_profile):
modelled_app = Modeller.model_application_profiles(
[app_profile])
modelled_apps.extend(modelled_app)
self.__modelled_applications = modelled_apps
logger.info("Finished modelling {} application profiles.".format(
len(modelled_apps)))
# Save data
abnormal_applications = self.get_abnormal_applications()
AppProfileDataManager.save_abnormal_apps(abnormal_applications)
def __add_processes_to_application_profile_and_save(
self, application_name: str,
application_processes: List[dict]) -> None:
"""
Adds the process information to its respective application profile.
:raises TypeError if application_name is not of type 'str' or application_processes is not pf type 'List[dict]'.
:raises ValueError if at least one of the processes' names is not equal to the application_name provided.
:param application_name: The name of the application.
:type application_name: str
:param application_processes: The list of processes associated to the application.
:type application_processes: List[dict]
"""
if not isinstance(application_name, str):
raise TypeError(
expected_type_but_received_message.format(
"application_name", 'str', application_name))
if not isinstance(application_processes, list):
raise TypeError(
expected_type_but_received_message.format(
"application_processes", 'List[dict]',
application_processes))
saved_app_profile = AppProfileDataManager.get_saved_profile(
application_name)
if saved_app_profile is None:
saved_app_profile = AppProfile(application_name=application_name)
for process in application_processes:
if not isinstance(process, dict):
raise TypeError(
expected_type_but_received_message.format(
"application_processes", 'List[dict]',
application_processes))
process_name = process[ProcessAttribute.name.name]
if process_name != application_name:
raise ValueError(
expected_application_message.format(
application_name, process_name))
rss_memory = process[ProcessAttribute.memory_info.name].rss
children_count = process[ProcessAttribute.children_count.name]
users = [process[ProcessAttribute.username.name]] if process[ProcessAttribute.username.name] is not None \
else list()
open_files = process.get(ProcessAttribute.open_files.name, list())
open_files = open_files if open_files is not None else list()
cpu_percentage = process[ProcessAttribute.cpu_percent.name]
num_threads = process[ProcessAttribute.num_threads.name]
connections_num = process[ProcessAttribute.connections.name]
saved_app_profile.add_new_information(
memory_usage=rss_memory,
child_processes_count=children_count,
users=users,
open_files=open_files,
cpu_percentage=cpu_percentage,
data_retrieval_timestamp=self.__latest_retrieval_time,
threads_number=num_threads,
connections_num=connections_num)
AppProfileDataManager.save_app_profile(saved_app_profile)
def test_get_saved_application_names_with_invalid_inputs() -> None:
"""
Tests get saved application names with invalid base directory.
"""
# None base path
with pytest.raises(TypeError):
AppProfileDataManager.get_saved_app_profiles_names(None)
# Number base path
with pytest.raises(TypeError):
AppProfileDataManager.get_saved_app_profiles_names(7)
def test_get_app_profile_with_invalid_input() -> None:
"""
Test get_saved_profiles() with invalid inputs.
Checks that an exception is thrown when an invalid input is provided.
"""
# None input for app_profile_file
with pytest.raises(TypeError):
AppProfileDataManager.get_saved_profile(app_profile_name=None)
# Invalid input type for app_profile_file
with pytest.raises(TypeError):
AppProfileDataManager.get_saved_profile(app_profile_name=5)
def test_save_and_get_abnormal_apps_from_file() -> None:
"""
Test saving and getting abnormal apps from a file is in the correct format and has the right values.
"""
app_profiles = list()
app_names = [
"common_case_app", "app_with_high_risk_and_low_risk_anomaly",
"app_with_blacklisted_file"
]
for app_name in app_names:
saved_app_profile = AppProfileDataManager.get_saved_profile(
app_name, paths.SAMPLE_APP_PROF_DATA_PATH)
app_profiles.append(saved_app_profile)
ft = FrequencyTechnique()
modelled_apps = ft(app_profiles)
# Note: Some of the saved app profiles are not anomalous, however this test assumes that they are
# and verifies that they are saved in correct format.
AppProfileDataManager.save_abnormal_apps(modelled_apps)
actual_saved_abnormal_apps = AppProfileDataManager.get_saved_abnormal_apps(
)
expected_saved_abnormal_apps = dict()
for modelled_app in modelled_apps:
app_name = modelled_app.get_app_name()
abnormal_attrs = list(modelled_app.get_abnormal_attrs())
risk_level = modelled_app.get_risk_level()
error_message = modelled_app.get_error_message()
latest_retrieved_timestamp = modelled_app.get_latest_retrieved_app_details() \
[AppProfileAttribute.data_retrieval_timestamps.name][0]
if app_name not in expected_saved_abnormal_apps:
expected_saved_abnormal_apps[app_name] = list()
abnormal_app_saved_entries = {
AppSummaryAttribute.abnormal_attributes.name:
set(abnormal_attrs),
AppSummaryAttribute.risk.name:
risk_level.name,
AppSummaryAttribute.error_message.name:
error_message if error_message is not None else nan,
AppProfileAttribute.data_retrieval_timestamps.name:
latest_retrieved_timestamp
}
expected_saved_abnormal_apps[app_name].append(
abnormal_app_saved_entries)
# Changing the abnormal attribute values to set as during modelling the order of those values may change.
for saved_abnormal_app in actual_saved_abnormal_apps.values():
for abnormal_entry in saved_abnormal_app:
# noinspection PyTypeChecker
abnormal_entry[AppSummaryAttribute.abnormal_attributes.name] = \
set(abnormal_entry[AppSummaryAttribute.abnormal_attributes.name])
assert expected_saved_abnormal_apps == actual_saved_abnormal_apps
def test_save_last_retrieved_data_timestamp_with_invalid_inputs() -> None:
"""
Tests saving last retrieved_data_timestamp with invalid inputs.
"""
timestamp = datetime.now()
# None retrieval_timestamp
with pytest.raises(TypeError):
AppProfileDataManager.save_last_retrieved_data_timestamp(
retrieval_timestamp=None)
# Numeric retrieval_timestamp
with pytest.raises(TypeError):
AppProfileDataManager.save_last_retrieved_data_timestamp(
retrieval_timestamp=7)
# None retrieval_timestamp_file_path
with pytest.raises(TypeError):
AppProfileDataManager.save_last_retrieved_data_timestamp(
retrieval_timestamp=timestamp, retrieval_timestamp_file_path=None)
# Invalid retrieval_timestamp_file_path
with pytest.raises(TypeError):
AppProfileDataManager.save_last_retrieved_data_timestamp(
retrieval_timestamp=timestamp,
retrieval_timestamp_file_path="sdjs")
def collect_running_processes_information(self) -> None:
"""
Collects running process information. To get the information call get_registered_app_profiles_as_dict.
"""
logger = logging.getLogger(self.__logger_name)
logger.info("Started retrieving running processes information.")
app_name_to_processes_map = self.__collect_running_processes_and_group_by_application(
)
self.__detected_app_profile_names = set(
app_name_to_processes_map.keys())
for app_name, processes in app_name_to_processes_map.items():
self.__add_processes_to_application_profile_and_save(
application_name=app_name, application_processes=processes)
AppProfileDataManager.save_last_retrieved_data_timestamp(
self.__latest_retrieval_time)
def test_execute_frequency_modelling_with_anomalies(
modelling_test_scenario: AppModellingTestScenario) -> None:
"""
Test modelling the application profiles using the frequency technique.
:param modelling_test_scenario: The modelling test scenarios.
:type modelling_test_scenario: AppModellingTestScenario
"""
app_profile = AppProfileDataManager.get_saved_profile(
modelling_test_scenario.app_name, paths.SAMPLE_APP_PROF_DATA_PATH)
fq = FrequencyTechnique()
app_summaries = fq(data=[app_profile])
assert len(app_summaries) == 1
app_summary = app_summaries[0]
app_summary_dict = app_summary.dict_format()
if modelling_test_scenario.is_anomalous:
assert app_summary_dict[
AppSummaryAttribute.error_message.name] == anomaly_detected_message
else:
assert app_summary_dict[AppSummaryAttribute.error_message.name] is None
assert app_summary_dict[
AppSummaryAttribute.risk.name] == modelling_test_scenario.risk_level
assert app_summary_dict[AppSummaryAttribute.abnormal_attributes.
name] == modelling_test_scenario.anomalous_attrs
def test_get_saved_application_names() -> None:
"""
Tests get saved application names.
"""
ps_handler = ProcessHandler()
ps_handler.collect_running_processes_information()
application_names = AppProfileDataManager.get_saved_app_profiles_names()
assert len(application_names) > 0
assert "systemd" in application_names
def test_collect_running_processes_information() -> None:
"""
Test collecting running process information by checking that it sets the value to the registered application
properties.
"""
common_app_names = ["systemd", "kthreadd", "rcu_gp"]
process_handler = ProcessHandler(logger_name)
for common_app_name in common_app_names:
app_profile = AppProfileDataManager.get_saved_profile(common_app_name)
assert app_profile is None
process_handler.collect_running_processes_information()
for common_app_name in common_app_names:
app_profile = AppProfileDataManager.get_saved_profile(common_app_name)
assert isinstance(app_profile, AppProfile)
data_retrieval_timestamps = app_profile.get_data_retrieval_timestamps()
assert len(data_retrieval_timestamps
) >= 1 # Some apps may have more than one process
assert data_retrieval_timestamps[0] is not None
def service_run(self) -> None:
"""
Runs the listener service thread. All request send by the ui/front-end will be handled here.
"""
logger = logging.getLogger(self.__logger_name)
while True:
# noinspection PyBroadException
try:
connection, address = self.__socket.accept()
with connection:
data_raw = connection.recv(1024)
data = data_raw.decode()
logger.info("Request received {} from {}".format(
data, address))
if data == "modelled apps":
data_to_send = self.__modeller.get_modelled_application_as_json(
)
encoded_data = data_to_send.encode(
) # Defaults to utf-8
connection.sendall(encoded_data)
elif data.startswith("abnormal apps"):
if data.endswith(" --history"):
data_to_send = json.dumps(
AppProfileDataManager.get_saved_abnormal_apps(
))
else:
data_to_send = self.__modeller.get_modelled_application_as_json(
)
encoded_data = data_to_send.encode(
) # Defaults to utf-8
connection.sendall(encoded_data)
elif data == "modeller pause":
self.__stop_modelling = True
logger.info("Starting modelling")
elif data == "modeller continue":
self.__stop_modelling = False
logger.info("Stopping modelling")
elif data == "modeller status":
data_raw = [
"Modelling paused."
] if self.__stop_modelling else ["Modelling running."]
data_to_send = json.dumps(data_raw)
connection.sendall(data_to_send.encode())
else:
data_to_send = json.dumps(["Command not supported"])
connection.sendall(data_to_send.encode())
except Exception:
logger.error(traceback.format_exc())
def test_save_and_get_profile_data() -> None:
"""
Test save_app_profiles() and get_saved_profiles().
Checks that saving and retrieving the app profiles does not modify the data.
"""
process_handler = ProcessHandler(logger_name)
process_handler.collect_running_processes_information()
latest_retrieved_data_timestamp = process_handler.get_latest_retrieved_data_timestamp(
)
app_name = "systemd"
expected_app_profiles = AppProfileDataManager.get_saved_profile(app_name)
saved_retrieved_timestamp = AppProfileDataManager.get_last_retrieved_data_timestamp(
)
assert isinstance(expected_app_profiles, AppProfile)
assert latest_retrieved_data_timestamp == saved_retrieved_timestamp
def is_application_recently_retrieved(app_profile: AppProfile) -> bool:
"""
Checks if the application profile provided was recently retrieved.
:param app_profile: The application profile to check.
:type app_profile: AppProfile
:return: True if the application profile was recently retrieved, False otherwise.
:rtype: bool
"""
latest_retrieval_timestamp = AppProfileDataManager.get_last_retrieved_data_timestamp(
)
retrieval_timestamps = app_profile.get_data_retrieval_timestamps()
app_profile_last_retrieved_data_timestamp = retrieval_timestamps[-1]
return latest_retrieval_timestamp == app_profile_last_retrieved_data_timestamp
def test_get_normalized_app_profile_data() -> None:
"""
Test get_previously_retrieved_data() with sample data.
It checks that the expected value of the sample app_profile is returned by get_previously_retrieved_data().
"""
app_name = "common_case_app"
last_retrieved_timestamp = "2021-01-d 23:33:03:575118"
app_profile_dict = AppProfileDataManager.get_saved_profile_as_dict(app_name, SAMPLE_APP_PROF_DATA_PATH)
app_profile = AppProfile(application_name=app_name)
app_profile.set_value_from_dict(app_profile_dict=app_profile_dict)
normalized_retrieved_data_size = \
len(app_profile.get_data_retrieval_timestamps()) - app_profile.get_latest_retrieved_data_size()
assert normalized_retrieved_data_size == 4
# Build expected normalized app profile
expected_normalized_app_profile = {
AppProfileAttribute.app_name.name: app_name,
AppProfileAttribute.memory_infos.name:
app_profile_dict[AppProfileAttribute.memory_infos.name][:normalized_retrieved_data_size],
AppProfileAttribute.cpu_percents.name:
app_profile_dict[AppProfileAttribute.cpu_percents.name][:normalized_retrieved_data_size],
AppProfileAttribute.children_counts.name:
app_profile_dict[AppProfileAttribute.children_counts.name][:normalized_retrieved_data_size],
AppProfileAttribute.usernames.name:
app_profile_dict[AppProfileAttribute.usernames.name][:normalized_retrieved_data_size],
AppProfileAttribute.opened_files.name:
app_profile_dict[AppProfileAttribute.opened_files.name][:normalized_retrieved_data_size],
AppProfileAttribute.data_retrieval_timestamps.name:
app_profile_dict[AppProfileAttribute.data_retrieval_timestamps.name][:normalized_retrieved_data_size],
AppProfileAttribute.threads_numbers.name:
app_profile_dict[AppProfileAttribute.threads_numbers.name][:normalized_retrieved_data_size],
AppProfileAttribute.connections_numbers.name:
app_profile_dict[AppProfileAttribute.connections_numbers.name][:normalized_retrieved_data_size]
}
actual_normalized_app_profile = app_profile.get_previously_retrieved_data()
assert actual_normalized_app_profile == expected_normalized_app_profile
def test_get_app_profile_as_dict_with_valid_input() -> None:
"""
Test get_saved_profiles_as_dict().
Checks that get_saved_profiles_as_dict() can retrieve app_profiles as dict in the right format.
"""
app_name = "common_case_app"
app_profile_dict = AppProfileDataManager.get_saved_profile_as_dict(
app_name, paths.SAMPLE_APP_PROF_DATA_PATH)
app_profile_attribute_names = {enum.name for enum in AppProfileAttribute}
assert isinstance(app_profile_dict, dict)
assert len(app_profile_dict) > 0
actual_app_profile_attrs = app_profile_dict.keys()
assert actual_app_profile_attrs == app_profile_attribute_names
check_app_profile_has_the_right_format(app_name, app_profile_dict)
def test_dict_format() -> None:
"""
Test dict_format() for an application profile. It checks that the returned value is in the right format.
"""
process_handler = ProcessHandler(logger_name)
process_handler.collect_running_processes_information()
registered_app_names = process_handler.get_registered_app_profile_names()
for registered_app_name in registered_app_names:
app_profile = AppProfileDataManager.get_saved_profile(registered_app_name)
registered_app_dict = app_profile.dict_format()
check_app_profile_has_the_right_format(app_name=registered_app_name,
app_profile=registered_app_dict)
# Check that all datetime values are in string format. Will throw an error if it is not in the right format.
object_created_timestamp = registered_app_dict[AppProfileAttribute.date_created_timestamp.name]
datetime.datetime.strptime(object_created_timestamp, wades_config.datetime_format)
retrieval_timestamps = registered_app_dict[AppProfileAttribute.data_retrieval_timestamps.name]
for retrieval_timestamp in retrieval_timestamps:
datetime.datetime.strptime(retrieval_timestamp, wades_config.datetime_format)
def test_get_latest_retrieved_data() -> None:
"""
Test get_latest_retrieved_data() with sample data.
It checks that the expected value of the sample app_profile is returned by get_latest_retrieved_data().
"""
app_name = "common_case_app"
app_profile_dict = AppProfileDataManager.get_saved_profile_as_dict(app_name, SAMPLE_APP_PROF_DATA_PATH)
app_profile = AppProfile(application_name=app_name)
app_profile.set_value_from_dict(app_profile_dict=app_profile_dict)
latest_retrieved_data_size = app_profile.get_latest_retrieved_data_size()
assert latest_retrieved_data_size == 1
# Build expected latest app profile data
expected_latest_app_profile_data = {
AppProfileAttribute.app_name.name: app_name,
AppProfileAttribute.memory_infos.name:
app_profile_dict[AppProfileAttribute.memory_infos.name][-latest_retrieved_data_size:],
AppProfileAttribute.cpu_percents.name:
app_profile_dict[AppProfileAttribute.cpu_percents.name][-latest_retrieved_data_size:],
AppProfileAttribute.children_counts.name:
app_profile_dict[AppProfileAttribute.children_counts.name][-latest_retrieved_data_size:],
AppProfileAttribute.usernames.name:
app_profile_dict[AppProfileAttribute.usernames.name][-latest_retrieved_data_size:],
AppProfileAttribute.opened_files.name:
app_profile_dict[AppProfileAttribute.opened_files.name][-latest_retrieved_data_size:],
AppProfileAttribute.data_retrieval_timestamps.name:
app_profile_dict[AppProfileAttribute.data_retrieval_timestamps.name][-latest_retrieved_data_size:],
AppProfileAttribute.threads_numbers.name:
app_profile_dict[AppProfileAttribute.threads_numbers.name][-latest_retrieved_data_size:],
AppProfileAttribute.connections_numbers.name:
app_profile_dict[AppProfileAttribute.connections_numbers.name][-latest_retrieved_data_size:]
}
actual_latest_app_profile_data = app_profile.get_latest_retrieved_data()
assert actual_latest_app_profile_data == expected_latest_app_profile_data
def test_save_app_profiles_with_input_validation() -> None:
"""
Test save_app_profiles() with input validation.
Checks that an exception is thrown when an invalid input is provided.
"""
timestamp = datetime.now()
# None input for app_profiles
with pytest.raises(TypeError):
AppProfileDataManager.save_app_profiles(app_profiles=None,
retrieval_timestamp=timestamp)
# Empty string input for app_profiles
with pytest.raises(TypeError):
AppProfileDataManager.save_app_profiles(app_profiles="",
retrieval_timestamp=timestamp)
# Invalid elements in list as input for app_profiles
with pytest.raises(TypeError):
AppProfileDataManager.save_app_profiles(app_profiles=["!2", "error"],
retrieval_timestamp=timestamp)
with pytest.raises(TypeError):
AppProfileDataManager.save_app_profiles(app_profiles=[2, 7],
retrieval_timestamp=timestamp)
# None input for app_profile_file
with pytest.raises(TypeError):
AppProfileDataManager.save_app_profiles(app_profiles=list(),
retrieval_timestamp=timestamp,
app_profile_base_dir=None)
# Invalid input type for app_profile_file
with pytest.raises(TypeError):
AppProfileDataManager.save_app_profiles(app_profiles=list(),
retrieval_timestamp=timestamp,
app_profile_base_dir=5)
# None input for app_profile_file
with pytest.raises(TypeError):
AppProfileDataManager.save_app_profiles(app_profiles=list(),
retrieval_timestamp=None)
# Invalid input type for app_profile_file
with pytest.raises(TypeError):
AppProfileDataManager.save_app_profiles(app_profiles=list(),
retrieval_timestamp="wew")