def test_custom_mongo_uri_overwrites_and_fails(config_path, config_name):
""" verifies that setting a bad customURI overwrites other settings
and returns false when attempting to setup a connection
"""
# setup config object and moddify the customMongoURI setting
configObj = ConfigHandler.ConfigHandler(config_path)
configObj.config['connectionInfo']['customMongoURI'] = ""
# assert that setting up a connection with a valid uri works
ValidMongoObj = MongoHandler.MongoHandler(configObj.config)
assert ValidMongoObj.setup_connection() is True
ValidMongoObj.close_connection() # close connection from the mongo server
# moddify the custom uri to be a invalid server
configObj.config['connectionInfo'][
'customMongoURI'] = "mongo://www.fake.com"
# setup mongo handler object with config
InvalidMongoObj = MongoHandler.MongoHandler(configObj.config)
# assert that setting up a connection fails with a bad custom uri
assert InvalidMongoObj.setup_connection() is False
InvalidMongoObj.close_connection(
) # close connection from the mongo server
def __init__(self, config_file_path=""):
logging.basicConfig(format='%(asctime)s -- %(levelname)s: %(message)s',
filename='ETLactivity.log',
level=logging.DEBUG,
datefmt='%m/%d/%Y %I:%M:%S %p')
logging.info(
" ========== ETL HANDLER INITIATED for ({}) ==========".format(
os.path.basename(config_file_path)))
self.config_path = config_file_path
self.config_handler = ConfigHandler.ConfigHandler(config_file_path)
# check config handler config exists and is formatted correctly
if self.config_handler.config == None:
logging.error(
"Config json could not be loaded or is invalid, exiting ETL handler"
)
print(
"ERROR: Config json could not be loaded or is invalid, exiting"
)
self.is_config_valid = self.config_handler.verify_format()
if self.is_config_valid is False:
logging.error("Config format is invalid, exiting ETL handler")
print("ERROR: invalid config format, exiting")
else:
logging.info("Config format is valid")
self.mongo_handler = None
self.sql_handler = None
self.package_queue = [] # packages to send to destination db
# setup handlers
self.setup_handles()
def test_check_key_optional_checks():
""" test all the optional parameters in check_key for proper function """
test_dict = {"key": "value"}
# setup config object
ConfigObj = ConfigHandler.ConfigHandler()
# assert check_key passes if expected_type is found
ConfigObj.check_key("key", test_dict, expected_type=str)
assert ConfigObj.valid == True
# assert check_key fails if expected_type is not found
ConfigObj.valid = True # reset config status
ConfigObj.check_key("key", test_dict, expected_type=int)
assert ConfigObj.valid == False
# assert check_key passes if key is in valid_values
ConfigObj.valid = True # reset config status
ConfigObj.check_key("key", test_dict, valid_values=["value"])
assert ConfigObj.valid == True
# assert check_key fails if key is not in valid_values
ConfigObj.valid = True # reset config status
ConfigObj.check_key("key", test_dict, valid_values=["wrong_value"])
assert ConfigObj.valid == False
def test_verify_format_invalid(config_path, config_name):
""" verifies an error in the config returns false """
ConfigObj = ConfigHandler.ConfigHandler(config_path)
# moddify the config to be invalid
del ConfigObj.config['connectionInfo']
# perform verification
return_value = ConfigObj.verify_format()
# assert config is not valid
assert ConfigObj.valid == False and return_value == False
def test_check_data_field(config_path, config_name):
""" verify the data field is properly handled """
ConfigObj = ConfigHandler.ConfigHandler(config_path)
# ensure no key is named 'data' in config
if 'data' in ConfigObj.config:
del ConfigObj.config['data']
# assert absense of 'data' in the config, results in a new dict
ConfigObj.check_data_field()
assert 'data' in ConfigObj.config and 'last_mongo_id_pulled' in ConfigObj.config[
'data']
def test_check_key_simple():
""" test that a value in a dictionary is the correct type """
test_dict = {"key": "value"}
# setup config object
ConfigObj = ConfigHandler.ConfigHandler()
# assert check_key is fine with a valid key and object
ConfigObj.check_key("key", test_dict)
assert ConfigObj.valid == True
# assert check_key errors with a invalid key for object
ConfigObj.check_key("key", {"wrong_key": "value"})
assert ConfigObj.valid == False
def test_push_package(config_path, config_name):
""" test that a small valid package will insert to a sql database
and the function will return true if successful
"""
ConfigObj = ConfigHandler.ConfigHandler(config_path)
SQLObj = SQLHandler.SQLHandler(ConfigObj.config)
SQLObj.setup_connection()
# setup simple package with data to push to a valid table
dest = {"schema": "dbo", "db": "Sandbox1", "table": "zips.unittest"}
PkgObj = Package.Package(dest, ["c1", "c2", "c3"], [None, None, None])
PkgObj.data = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
# assert the assumed valid package inserts to sql and the function returns true
assert SQLObj.push_package(PkgObj) == True
SQLObj.cleanup()
def test_setup_connection_pos(config_path, config_name):
""" positive unit test: verify that valid information connects a returns
true from setup_connection
"""
# setup config object and mongo object
ConfigObj = ConfigHandler.ConfigHandler(config_path)
MongoObj = MongoHandler.MongoHandler(ConfigObj.config)
# setup connection and assert its valid
response = MongoObj.setup_connection()
assert response == True
# close connection
MongoObj.close_connection()
def test_setup_connection_neg(config_path, config_name):
""" negative unit test: verify that invalid information does not connect and
returns false from setup_connection
"""
# setup config object and mongo object
ConfigObj = ConfigHandler.ConfigHandler(config_path)
ConfigObj.config['connectionInfo']['mongoServerHost'] = "www.fake.com"
MongoObj = MongoHandler.MongoHandler(ConfigObj.config)
# setup connection and assert its valid
response = MongoObj.setup_connection()
assert response == False
# close connection
MongoObj.close_connection()
def test_pull_required_value_missing(config_path, config_name):
""" verify that int,str,list and dict are properly handled when casting """
# setup config object and mongo object
ConfigObj = ConfigHandler.ConfigHandler(config_path)
ConfigObj.config['mapping'][0]['sql_cols']['grade'] = {
"mongo_path": "grades[0].grade",
"required": True
}
MongoObj = MongoHandler.MongoHandler(ConfigObj.config)
# setup connection and assert its valid
MongoObj.setup_connection()
# assert a pull will fail because not all documents will have a value for zipcode
assert MongoObj.pull_collection() == []
# close connection
MongoObj.close_connection()
def test_push_packages(config_path, config_name):
""" verify that pushing multiple valid packages returns true """
# setup objects
ConfigObj = ConfigHandler.ConfigHandler(config_path)
SQLObj = SQLHandler.SQLHandler(ConfigObj.config)
SQLObj.setup_connection()
# setup simple packages with data to push to a valid table
dest = {"schema": "dbo", "db": "Sandbox1", "table": "zips.unittest"}
PkgObj1 = Package.Package(dest, ["c1", "c2", "c3"], [None, None, None])
PkgObj1.data = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
dest = {"schema": "dbo", "db": "Sandbox1", "table": "zips.unittest"}
PkgObj2 = Package.Package(dest, ["c1", "c2", "c3"], [None, None, None])
PkgObj2.data = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
pkg_list = [PkgObj1, PkgObj2]
# assert pushing multiple valid packages returns true
assert SQLObj.push_packages(pkg_list) == True
def test_generate_sql_update(config_path, config_name):
""" verify that an expected string is returned from the function """
# expected output string
expectedReturnTuple = (
"UPDATE [Sandbox1].[dbo].[unittest] SET [type] = ?, [color] = ? WHERE [id] = ? AND [number] = ?",
['Ostrich', 'Mustang brown', '54323fhwa', 54])
# sql_dest to test
dest = "[Sandbox1].[dbo].[unittest]"
primaryKeys = ['id', 'number']
columns = ['id', 'type', 'number', 'color']
values = ['54323fhwa', 'Ostrich', 54, 'Mustang brown']
# setup objects
ConfigObj = ConfigHandler.ConfigHandler(config_path)
SQLObj = SQLHandler.SQLHandler(ConfigObj.config)
# check that the expected value is returned
assert expectedReturnTuple == SQLObj.generate_sql_update(
dest, primaryKeys, columns, values)
def test_empty_config_format(config_path, config_name):
""" verifies an empty dictionary for the config is not valid """
ConfigObj = ConfigHandler.ConfigHandler(config_path)
ConfigObj.config = {}
assert ConfigObj.verify_format() == False
def test_setup_connection(config_path, config_name):
""" assert that a valid sqlhandler object is not none upon setup"""
ConfigObj = ConfigHandler.ConfigHandler(config_path)
SQLObj = SQLHandler.SQLHandler(ConfigObj.config)
SQLObj.setup_connection()
assert SQLObj != None
def test_valid_config_setup(config_path, config_name):
""" verify that a valid config file opens and is not None """
ConfigObj = ConfigHandler.ConfigHandler(config_path)
assert ConfigObj != None
def test_verify_format_simple(config_path, config_name):
""" verifies a simple config is valid """
ConfigObj = ConfigHandler.ConfigHandler(config_path)
return_value = ConfigObj.verify_format()
assert ConfigObj.valid == True and return_value == True
def test_valid_config_format(config_path, config_name):
""" verifies that a valid config passes the formatting test """
ConfigObj = ConfigHandler.ConfigHandler(config_path)
return_value = ConfigObj.verify_format()
assert return_value == True
