def test_custom_mongo_uri_overwrites_and_fails(config_path, config_name): """ verifies that setting a bad customURI overwrites other settings and returns false when attempting to setup a connection """ # setup config object and moddify the customMongoURI setting configObj = ConfigHandler.ConfigHandler(config_path) configObj.config['connectionInfo']['customMongoURI'] = "" # assert that setting up a connection with a valid uri works ValidMongoObj = MongoHandler.MongoHandler(configObj.config) assert ValidMongoObj.setup_connection() is True ValidMongoObj.close_connection() # close connection from the mongo server # moddify the custom uri to be a invalid server configObj.config['connectionInfo'][ 'customMongoURI'] = "mongo://www.fake.com" # setup mongo handler object with config InvalidMongoObj = MongoHandler.MongoHandler(configObj.config) # assert that setting up a connection fails with a bad custom uri assert InvalidMongoObj.setup_connection() is False InvalidMongoObj.close_connection( ) # close connection from the mongo server
def __init__(self, config_file_path=""): logging.basicConfig(format='%(asctime)s -- %(levelname)s: %(message)s', filename='ETLactivity.log', level=logging.DEBUG, datefmt='%m/%d/%Y %I:%M:%S %p') logging.info( " ========== ETL HANDLER INITIATED for ({}) ==========".format( os.path.basename(config_file_path))) self.config_path = config_file_path self.config_handler = ConfigHandler.ConfigHandler(config_file_path) # check config handler config exists and is formatted correctly if self.config_handler.config == None: logging.error( "Config json could not be loaded or is invalid, exiting ETL handler" ) print( "ERROR: Config json could not be loaded or is invalid, exiting" ) self.is_config_valid = self.config_handler.verify_format() if self.is_config_valid is False: logging.error("Config format is invalid, exiting ETL handler") print("ERROR: invalid config format, exiting") else: logging.info("Config format is valid") self.mongo_handler = None self.sql_handler = None self.package_queue = [] # packages to send to destination db # setup handlers self.setup_handles()
def test_check_key_optional_checks(): """ test all the optional parameters in check_key for proper function """ test_dict = {"key": "value"} # setup config object ConfigObj = ConfigHandler.ConfigHandler() # assert check_key passes if expected_type is found ConfigObj.check_key("key", test_dict, expected_type=str) assert ConfigObj.valid == True # assert check_key fails if expected_type is not found ConfigObj.valid = True # reset config status ConfigObj.check_key("key", test_dict, expected_type=int) assert ConfigObj.valid == False # assert check_key passes if key is in valid_values ConfigObj.valid = True # reset config status ConfigObj.check_key("key", test_dict, valid_values=["value"]) assert ConfigObj.valid == True # assert check_key fails if key is not in valid_values ConfigObj.valid = True # reset config status ConfigObj.check_key("key", test_dict, valid_values=["wrong_value"]) assert ConfigObj.valid == False
def test_verify_format_invalid(config_path, config_name): """ verifies an error in the config returns false """ ConfigObj = ConfigHandler.ConfigHandler(config_path) # moddify the config to be invalid del ConfigObj.config['connectionInfo'] # perform verification return_value = ConfigObj.verify_format() # assert config is not valid assert ConfigObj.valid == False and return_value == False
def test_check_data_field(config_path, config_name): """ verify the data field is properly handled """ ConfigObj = ConfigHandler.ConfigHandler(config_path) # ensure no key is named 'data' in config if 'data' in ConfigObj.config: del ConfigObj.config['data'] # assert absense of 'data' in the config, results in a new dict ConfigObj.check_data_field() assert 'data' in ConfigObj.config and 'last_mongo_id_pulled' in ConfigObj.config[ 'data']
def test_check_key_simple(): """ test that a value in a dictionary is the correct type """ test_dict = {"key": "value"} # setup config object ConfigObj = ConfigHandler.ConfigHandler() # assert check_key is fine with a valid key and object ConfigObj.check_key("key", test_dict) assert ConfigObj.valid == True # assert check_key errors with a invalid key for object ConfigObj.check_key("key", {"wrong_key": "value"}) assert ConfigObj.valid == False
def test_push_package(config_path, config_name): """ test that a small valid package will insert to a sql database and the function will return true if successful """ ConfigObj = ConfigHandler.ConfigHandler(config_path) SQLObj = SQLHandler.SQLHandler(ConfigObj.config) SQLObj.setup_connection() # setup simple package with data to push to a valid table dest = {"schema": "dbo", "db": "Sandbox1", "table": "zips.unittest"} PkgObj = Package.Package(dest, ["c1", "c2", "c3"], [None, None, None]) PkgObj.data = [[1, 2, 3], [4, 5, 6], [7, 8, 9]] # assert the assumed valid package inserts to sql and the function returns true assert SQLObj.push_package(PkgObj) == True SQLObj.cleanup()
def test_setup_connection_pos(config_path, config_name): """ positive unit test: verify that valid information connects a returns true from setup_connection """ # setup config object and mongo object ConfigObj = ConfigHandler.ConfigHandler(config_path) MongoObj = MongoHandler.MongoHandler(ConfigObj.config) # setup connection and assert its valid response = MongoObj.setup_connection() assert response == True # close connection MongoObj.close_connection()
def test_setup_connection_neg(config_path, config_name): """ negative unit test: verify that invalid information does not connect and returns false from setup_connection """ # setup config object and mongo object ConfigObj = ConfigHandler.ConfigHandler(config_path) ConfigObj.config['connectionInfo']['mongoServerHost'] = "www.fake.com" MongoObj = MongoHandler.MongoHandler(ConfigObj.config) # setup connection and assert its valid response = MongoObj.setup_connection() assert response == False # close connection MongoObj.close_connection()
def test_pull_required_value_missing(config_path, config_name): """ verify that int,str,list and dict are properly handled when casting """ # setup config object and mongo object ConfigObj = ConfigHandler.ConfigHandler(config_path) ConfigObj.config['mapping'][0]['sql_cols']['grade'] = { "mongo_path": "grades[0].grade", "required": True } MongoObj = MongoHandler.MongoHandler(ConfigObj.config) # setup connection and assert its valid MongoObj.setup_connection() # assert a pull will fail because not all documents will have a value for zipcode assert MongoObj.pull_collection() == [] # close connection MongoObj.close_connection()
def test_push_packages(config_path, config_name): """ verify that pushing multiple valid packages returns true """ # setup objects ConfigObj = ConfigHandler.ConfigHandler(config_path) SQLObj = SQLHandler.SQLHandler(ConfigObj.config) SQLObj.setup_connection() # setup simple packages with data to push to a valid table dest = {"schema": "dbo", "db": "Sandbox1", "table": "zips.unittest"} PkgObj1 = Package.Package(dest, ["c1", "c2", "c3"], [None, None, None]) PkgObj1.data = [[1, 2, 3], [4, 5, 6], [7, 8, 9]] dest = {"schema": "dbo", "db": "Sandbox1", "table": "zips.unittest"} PkgObj2 = Package.Package(dest, ["c1", "c2", "c3"], [None, None, None]) PkgObj2.data = [[1, 2, 3], [4, 5, 6], [7, 8, 9]] pkg_list = [PkgObj1, PkgObj2] # assert pushing multiple valid packages returns true assert SQLObj.push_packages(pkg_list) == True
def test_generate_sql_update(config_path, config_name): """ verify that an expected string is returned from the function """ # expected output string expectedReturnTuple = ( "UPDATE [Sandbox1].[dbo].[unittest] SET [type] = ?, [color] = ? WHERE [id] = ? AND [number] = ?", ['Ostrich', 'Mustang brown', '54323fhwa', 54]) # sql_dest to test dest = "[Sandbox1].[dbo].[unittest]" primaryKeys = ['id', 'number'] columns = ['id', 'type', 'number', 'color'] values = ['54323fhwa', 'Ostrich', 54, 'Mustang brown'] # setup objects ConfigObj = ConfigHandler.ConfigHandler(config_path) SQLObj = SQLHandler.SQLHandler(ConfigObj.config) # check that the expected value is returned assert expectedReturnTuple == SQLObj.generate_sql_update( dest, primaryKeys, columns, values)
def test_empty_config_format(config_path, config_name): """ verifies an empty dictionary for the config is not valid """ ConfigObj = ConfigHandler.ConfigHandler(config_path) ConfigObj.config = {} assert ConfigObj.verify_format() == False
def test_setup_connection(config_path, config_name): """ assert that a valid sqlhandler object is not none upon setup""" ConfigObj = ConfigHandler.ConfigHandler(config_path) SQLObj = SQLHandler.SQLHandler(ConfigObj.config) SQLObj.setup_connection() assert SQLObj != None
def test_valid_config_setup(config_path, config_name): """ verify that a valid config file opens and is not None """ ConfigObj = ConfigHandler.ConfigHandler(config_path) assert ConfigObj != None
def test_verify_format_simple(config_path, config_name): """ verifies a simple config is valid """ ConfigObj = ConfigHandler.ConfigHandler(config_path) return_value = ConfigObj.verify_format() assert ConfigObj.valid == True and return_value == True
def test_valid_config_format(config_path, config_name): """ verifies that a valid config passes the formatting test """ ConfigObj = ConfigHandler.ConfigHandler(config_path) return_value = ConfigObj.verify_format() assert return_value == True