def test_validate_files(self):
os.chdir(self.test_directory)
config = ConfigService.initialize_config(TEST_CONFIG_FILE_PATH)
FileService.validate_files(config)
if "*" in config.FILES:
for file in config.FILES:
assert file in glob.glob("*.csv")
def test_validate_csv_file_extensions_with_valid_extensions(self):
os.chdir(self.test_directory)
config = ConfigService.initialize_config(TEST_CONFIG_FILE_PATH)
FileService.navigate_to_working_directory(config)
FileService.validate_csv_file_extensions(config)
for file in config.FILES:
assert file[-4:] == ".csv"
def test_validate_files_exist_with_nonexistant_file(self):
with pytest.raises(FileNotFoundError):
os.chdir(self.test_directory)
config = ConfigService.initialize_config(TEST_CONFIG_FILE_PATH)
config.WORKING_DIRECTORY = "tests"
FileService.navigate_to_working_directory(config)
FileService.validate_csv_file_extensions(config)
FileService.validate_files_exist(config)
def test_validate_files_exist_with_existing_file(self):
os.chdir(self.test_directory)
config = ConfigService.initialize_config(TEST_CONFIG_FILE_PATH)
FileService.navigate_to_working_directory(config)
FileService.validate_csv_file_extensions(config)
FileService.validate_files_exist(config)
file_instance = glob.glob(config.FILES[0])
assert len(file_instance) > 0
def test_initialize_session_from_cache(self):
config = ConfigService.initialize_config(TEST_CONFIG_FILE_PATH)
authentication_service = AuthenticationService(config)
authentication_service.initialize_cache()
authentication_service.initialize_app_instance()
authentication_service.initialize_session_from_cache()
if authentication_service.app_instance.get_accounts():
pass
else:
assert authentication_service.session is None
def test_set_config(self):
config_dictionary = ConfigService.read_input(TEST_CONFIG_FILE_PATH)
ConfigService.validate_input(config_dictionary)
config = ConfigService.set_config(config_dictionary)
assert config.WORKING_DIRECTORY == config_dictionary[
ConfigKeys.
WORKING_DIRECTORY] or config.WORKING_DIRECTORY == DEFAULT_WORKING_DIRECTORY
assert config.FILES == config_dictionary[
ConfigKeys.FILES] or config.FILES == ['*']
assert config.COLUMNS_TO_HASH == config_dictionary[
ConfigKeys.
COLUMNS_TO_HASH] or config.COLUMNS_TO_HASH == DEFAULT_COLUMNS_TO_HASH
assert config.AUTHENTICATION_CLIENT_ID is not None
assert config.AUTHENTICATION_TENANT_AUTHORITY is not None
assert config.AUTHENTICATION_SCOPE is not None
assert config.START_DATE is not None
assert config.END_DATE is not None
assert config.COLUMN_TO_SEARCH == config_dictionary[
ConfigKeys.COLUMN_TO_SEARCH] or config.COLUMN_TO_SEARCH is None
class TestUploadServiceClass:
test_directory = os.path.realpath(os.getcwd())
config = ConfigService.initialize_config(TEST_CONFIG_FILE_PATH)
def test_upload_files_without_authorization(self):
with pytest.raises(RuntimeError):
os.chdir(self.test_directory)
self.config.ENCRYPTED_FILES = [TEST_DATA_FILE_PATH]
mock_sessipn = {'access_token': '6798697698'}
UploadService.upload_files(self.config, mock_sessipn)
def test_initialize_config(self):
config = ConfigService.initialize_config(TEST_CONFIG_FILE_PATH)
assert isinstance(config.WORKING_DIRECTORY, str)
assert isinstance(config.FILES, list)
assert isinstance(config.FILES[0], str)
assert isinstance(config.COLUMNS_TO_HASH, list)
assert isinstance(config.COLUMNS_TO_HASH[0], str)
assert isinstance(config.CSV_DELIMITER, str)
assert isinstance(config.AUTHENTICATION_TENANT_AUTHORITY, str)
assert isinstance(config.AUTHENTICATION_SCOPE, str)
assert isinstance(config.AUTHENTICATION_CLIENT_ID, str)
def test_encrypt_files(self):
config = ConfigService.initialize_config(TEST_CONFIG_FILE_PATH)
mock_keys = {
ApiKeys.SALT_KEYS: [{
"value": "test-salt-key"
}],
ApiKeys.ENCRYPTION_KEY:
"_hSmVyMTLi-Qo_rmISp8jrH5Aob7frHp1X-28sxQZAU="
}
FileService.validate_files(config)
HashService.hash_files(config, mock_keys)
EncryptionService.encrypt_files(config, mock_keys)
assert len(config.ENCRYPTED_FILES) > 0
os.remove(config.ENCRYPTED_FILES[0])
os.remove(config.HASHED_FILES[0])
os.chdir(self.test_directory)
class TestKeyServiceClass:
test_directory = os.path.realpath(os.getcwd())
config = ConfigService.initialize_config(TEST_CONFIG_FILE_PATH)
def test_validate_api_response_with_valid_response(self):
mock_keys = {
ApiKeys.SALT_KEYS: [{
"value": "test-salt-key"
}],
ApiKeys.ENCRYPTION_KEY:
"_hSmVyMTLi-Qo_rmISp8jrH5Aob7frHp1X-28sxQZAU="
}
KeyService.validate_api_response(mock_keys, self.config)
def test_validate_api_response_with_no_salt_keys(self):
with pytest.raises(RuntimeError):
mock_keys = {
ApiKeys.SALT_KEYS: [],
ApiKeys.ENCRYPTION_KEY:
"_hSmVyMTLi-Qo_rmISp8jrH5Aob7frHp1X-28sxQZAU="
}
KeyService.validate_api_response(mock_keys, self.config)
def test_validate_api_response_with_no_empty_keys(self):
with pytest.raises(RuntimeError):
mock_keys = {
ApiKeys.ENCRYPTION_KEY:
"_hSmVyMTLi-Qo_rmISp8jrH5Aob7frHp1X-28sxQZAU="
}
KeyService.validate_api_response(mock_keys, self.config)
def test_validate_api_response_with_no_encryption_key(self):
with pytest.raises(RuntimeError):
mock_keys = {ApiKeys.SALT_KEYS: [{"value": "test-salt-key"}]}
KeyService.validate_api_response(mock_keys, self.config)
def test_get_keys_without_authorization(self):
with pytest.raises(RuntimeError):
mock_sessipn = {'access_token': '6798697698'}
KeyService.get_keys(self.config, mock_sessipn)
def run(input_file, log_file, mode):
"""Entry point for the INOIS application."""
log_level = logging.DEBUG if log_file else logging.CRITICAL
log_location = log_file if log_file else "inois.log"
logging.basicConfig(format='%(asctime)s (%(levelname)s): %(message)s',
filename=log_location,
level=log_level)
application_mode = mode if mode == ApplicationModeKeys.SEARCH else ApplicationModeKeys.UPLOAD
logging.info(
Notifications.APPLICATION_STARTED.format(application_mode,
datetime.now()))
print("\n" + Notifications.APPLICATION_STARTED.format(
application_mode, datetime.now()))
print(Banner.TEXT)
config = ConfigService.initialize_config(input_file=input_file)
session = AuthenticationService(config).get_authorization()
FileService.validate_files(config)
keys = KeyService.get_keys(config, session)
if application_mode == ApplicationModeKeys.UPLOAD:
HashService.hash_files(config, keys)
FileService.delete_chunked_files(config)
EncryptionService.encrypt_files(config, keys)
FileService.delete_hashed_files(config)
UploadService.upload_files(config, session)
FileService.delete_encrypted_files(config)
elif application_mode == ApplicationModeKeys.SEARCH:
search_queries = HashService.hash_records_for_search(config, keys)
SearchService.search_on_all_queries(search_queries, session)
os.chdir(config.LAUNCH_DIRECTORY)
logging.info(Notifications.APPLICATION_TERMINATED.format(datetime.now()))
print("\n" + Notifications.APPLICATION_TERMINATED.format(datetime.now()))
def test_validate_input_with_invalid_file_item_value(self):
config = ConfigService.read_input(TEST_CONFIG_FILE_PATH)
config[ConfigKeys.FILES] = [TEST_CONFIG_FILE_PATH, 9]
with pytest.raises(ValueError):
ConfigService.validate_input(config)
def test_validate_input_with_invalid_column_item_value(self):
config = ConfigService.read_input(TEST_CONFIG_FILE_PATH)
config[ConfigKeys.COLUMNS_TO_HASH] = [None]
with pytest.raises(ValueError):
ConfigService.validate_input(config)
def test_validate_input_with_nonexistant_client_id_key(self):
config = ConfigService.read_input(TEST_CONFIG_FILE_PATH)
del config[ConfigKeys.AUTHENTICATION_CLIENT_ID]
with pytest.raises(ValueError):
ConfigService.validate_input(config)
def test_read_input_with_nonexistant_file(self):
with pytest.raises(FileNotFoundError):
ConfigService.read_input('invalid_file_path')
def test_validate_input_with_invalid_auth_scope_value(self):
config = ConfigService.read_input(TEST_CONFIG_FILE_PATH)
config[ConfigKeys.AUTHENTICATION_SCOPE] = 9
with pytest.raises(ValueError):
ConfigService.validate_input(config)
def test_validate_input_with_nonexistant_auth_authority_key(self):
config = ConfigService.read_input(TEST_CONFIG_FILE_PATH)
del config[ConfigKeys.AUTHENTICATION_TENANT_AUTHORITY]
with pytest.raises(ValueError):
ConfigService.validate_input(config)
def test_validate_input_with_invalid_key(self):
config = ConfigService.read_input(TEST_CONFIG_FILE_PATH)
config['INVALID_KEY'] = True
with pytest.raises(ValueError):
ConfigService.validate_input(config)
def test_validate_input_with_invalid_working_directory_value(self):
config = ConfigService.read_input(TEST_CONFIG_FILE_PATH)
config[ConfigKeys.WORKING_DIRECTORY] = 0
with pytest.raises(ValueError):
ConfigService.validate_input(config)
def test_validate_input_with_invalid_end_date(self):
config = ConfigService.read_input(TEST_CONFIG_FILE_PATH)
config[ConfigKeys.START_DATE] = '9-9-99'
with pytest.raises(ValueError):
ConfigService.validate_input(config)
def test_navgiate_to_working_directory_with_valid_directory(self):
config = ConfigService.initialize_config(TEST_CONFIG_FILE_PATH)
previous_directory = os.path.realpath(os.getcwd())
FileService.navigate_to_working_directory(config)
assert config.WORKING_DIRECTORY == os.getcwd()
os.chdir(previous_directory)
def test_read_input_with_invalid_file_format(self):
with pytest.raises(TypeError):
ConfigService.read_input('tests/utils/example_inois_data.csv')
def test_initialize_app_instance(self):
config = ConfigService.initialize_config(TEST_CONFIG_FILE_PATH)
authentication_service = AuthenticationService(config)
authentication_service.initialize_cache()
authentication_service.initialize_app_instance()
assert authentication_service.app_instance is not None
def test_navgiate_to_working_directory_with_nonexistant_directory(self):
with pytest.raises(OSError):
config = ConfigService.initialize_config(TEST_CONFIG_FILE_PATH)
config.WORKING_DIRECTORY = config.WORKING_DIRECTORY + '^&(^&(^&*('
FileService.navigate_to_working_directory(config)
def test_credentials_exist_in_config(self):
config = ConfigService.initialize_config(TEST_CONFIG_FILE_PATH)
assert config.AUTHENTICATION_TENANT_AUTHORITY is not None
assert config.AUTHENTICATION_SCOPE is not None
assert config.AUTHENTICATION_CLIENT_ID is not None
class TestHashServiceClass:
config = ConfigService.initialize_config(TEST_CONFIG_FILE_PATH)
def test_read_csv_with_valid_csv_file(self):
HashService.read_csv(self.config.FILES[0], self.config)
def test_read_csv_with_empty_csv_file(self):
with pytest.raises(ValueError):
self.config.FILES = ['empty_file.csv']
HashService.read_csv(self.config.FILES[0], self.config)
def test_read_csv_with_non_csv_file(self):
with pytest.raises(TypeError):
self.config.FILES = ['png_image_file.csv']
HashService.read_csv(self.config.FILES[0], self.config)
def test_read_csv_with_empty_csv_delimiter(self):
with pytest.raises(TypeError):
self.config.FILES = ['example_inois_data.csv']
self.config.CSV_DELIMITER = ''
HashService.read_csv(self.config.FILES[0], self.config)
def test_verify_columns_to_hash_exist_with_valid_column(self):
self.config.CSV_DELIMITER = ','
file_data = HashService.read_csv(self.config.FILES[0], self.config)
HashService.verify_columns_to_hash_exist(self.config.FILES[0],
file_data, self.config)
def test_verify_columns_to_hash_exist_with_invalid_column(self):
with pytest.raises(ValueError):
self.config.COLUMNS_TO_HASH = ["Bad-Column"]
file_data = HashService.read_csv(self.config.FILES[0], self.config)
HashService.verify_columns_to_hash_exist(self.config.FILES[0],
file_data, self.config)
def test_hash_csv_with_valid_input(self):
self.config.COLUMNS_TO_HASH = ['SSN']
mock_keys = {
ApiKeys.SALT_KEYS: [{
"value": "test-salt-key"
}],
ApiKeys.ENCRYPTION_KEY:
"_hSmVyMTLi-Qo_rmISp8jrH5Aob7frHp1X-28sxQZAU="
}
file_data = HashService.read_csv(self.config.FILES[0], self.config)
HashService.hash_csv(self.config.FILES[0], file_data, self.config,
mock_keys)
def test_hash_value(self):
value = "123456789"
identical_value = "123456789"
salt_key = "vhjkwfeho178348"
hashed_value = HashService.hash_value(value, salt_key)
hashed_identical_value = HashService.hash_value(
identical_value, salt_key)
assert value == identical_value
assert value != hashed_value
assert hashed_value == hashed_identical_value
def test_write_hashed_csv_with_valid_input(self):
file_data = HashService.read_csv(self.config.FILES[0], self.config)
mock_keys = {
ApiKeys.SALT_KEYS: [{
"value": "test-salt-key"
}],
ApiKeys.ENCRYPTION_KEY:
"_hSmVyMTLi-Qo_rmISp8jrH5Aob7frHp1X-28sxQZAU="
}
HashService.hash_csv(self.config.FILES[0], file_data, self.config,
mock_keys)
HashService.write_hashed_csv(self.config.FILES[0], file_data,
self.config)
assert os.path.exists(self.config.HASHED_FILES[0])
os.remove(self.config.HASHED_FILES[0])
def test_hash_files(self):
mock_keys = {
ApiKeys.SALT_KEYS: [{
"value": "test-salt-key"
}],
ApiKeys.ENCRYPTION_KEY:
"_hSmVyMTLi-Qo_rmISp8jrH5Aob7frHp1X-28sxQZAU="
}
HashService.hash_files(self.config, mock_keys)
input_csv = pandas.read_csv(self.config.FILES[0],
delimiter=self.config.CSV_DELIMITER,
dtype=object,
encoding=DEFAULT_CSV_ENCODING)
hashed_csv_file_name = self.config.FILES[
0][:-4] + "_chunk_1" + HASHED_FILE_EXTENSION + ".csv"
hashed_csv = pandas.read_csv(hashed_csv_file_name,
delimiter=self.config.CSV_DELIMITER,
dtype=object,
encoding=DEFAULT_CSV_ENCODING)
input_csv_columns = input_csv.columns.values.tolist()
hashed_csv_columns = hashed_csv.columns.values.tolist()
for column in input_csv_columns:
if column not in self.config.COLUMNS_TO_HASH:
assert column in hashed_csv_columns
assert input_csv[column][0] == hashed_csv[column][0]
else:
assert column + HASHED_FILE_EXTENSION in hashed_csv_columns
assert input_csv[column][0] != hashed_csv[
column + HASHED_FILE_EXTENSION][0]
def test_hash_records_for_search(self):
mock_keys = {
ApiKeys.SALT_KEYS: [{
"value": "test-salt-key"
}],
ApiKeys.ENCRYPTION_KEY:
"_hSmVyMTLi-Qo_rmISp8jrH5Aob7frHp1X-28sxQZAU="
}
search_queries = HashService.hash_records_for_search(
self.config, mock_keys)
assert search_queries.keys() is not None
for key in search_queries:
assert isinstance(search_queries[key], list)
assert isinstance(search_queries[key][0], str)
def test_read_input_with_valid_file_format(self):
config = ConfigService.read_input(TEST_CONFIG_FILE_PATH)
assert isinstance(config, dict)
def test_validate_csv_file_extensions_with_invalid_extensions(self):
with pytest.raises(TypeError):
config = ConfigService.initialize_config(TEST_CONFIG_FILE_PATH)
config.FILES = config.FILES.append("file.xls")
FileService.navigate_to_working_directory(config)
FileService.validate_csv_file_extensions(config)