def __init__(self):
globalVars = Globals()
self.__ROOT_DIR = globalVars.ROOT_DIR
self.__RESOURCE_DIR = "resource"
self.logger = FileLogger("logsettingreader")
config_parser = ConfigParser()
self.file_name = config_parser.get_config_value("logparser-param")
def __init__(self):
globalVars = Globals()
self.utilFunctions = Funcs()
self.lambdaFuncs = Lambdas()
self.__ROOT_DIR = globalVars.ROOT_DIR
self.__RESOURCE_DIR = "resource"
self.__VALIDATION_CONFIG_KEY = "validator-param"
self.logger = FileLogger("validationreader")
config_parser = ConfigParser()
self.file_name = config_parser.get_config_value(
self.__VALIDATION_CONFIG_KEY)
self.obj_sep = config_parser.get_config_value("validator-sep")
def __init__(self):
globalVars = Globals()
self.utilFuncs = Funcs()
self.lambdaFuncs = Lambdas()
config_parser = ConfigParser()
self.logger = FileLogger("generatorreader")
self.__ROOT_DIR, self.__RESOURCE_DIR = globalVars.ROOT_DIR, "resource"
self.__GENERATOR_CONFIG_KEY = "generator-param"
self.obj_sep = config_parser.get_config_value("generator-obj-sep")
self.file_name = config_parser.get_config_value(
self.__GENERATOR_CONFIG_KEY)
def write2file(self, job_name, file_ext, sqoop_stmt):
try:
obj_name = job_name
if "." in job_name:
obj_name = job_name.split(".")[1]
confParser = ConfigParser()
out_file_name = confParser.get_config_value(
"sqoop-out-file-pattern") % (obj_name) + "." + file_ext
out_file_path = self.set_file_path(out_file_name)
with open(out_file_path, "w") as fw:
fw.write(sqoop_stmt)
except:
raise
def __init__(self):
config_parser = ConfigParser()
generator_file_sep = config_parser.get_config_value("generator-sep")
self.fn_gen_splitter = lambda l: l.replace("\n", "").replace("\r", "").split(generator_file_sep)
validator_file_sep = config_parser.get_config_value("validator-sep")
self.fn_valid_splitter = lambda l: l.replace("\n", "").replace("\r", "").split(validator_file_sep)
bee_pattern = re.compile(r"(INFO|ERROR|WARN|DEBUG)")
self.fn_bee_filter_lines = lambda x: bee_pattern.match(x)
sqoop_pattern = re.compile(r"^\d{2}\/\d{2}\/\d{2}\s\d{2}\:\d{2}\:\d{2}\s(INFO|ERROR|WARN|DEBUG)")
self.fn_sqoop_filter_lines = lambda x: sqoop_pattern.match(x)
hive_pattern = re.compile(r"^\d{2}\/\d{2}\/\d{2}\s\d{2}\:\d{2}\:\d{2}\s\[\w+")
self.fn_hive_filter_lines = lambda x: hive_pattern.match(x)
self.fn_sqoop_splitter = lambda x: x.split(": ")
self.fn_bee_splitter = lambda x: x.split(": ")
self.fn_hive_splitter = lambda x: x.split(": ")
def fill_args(self, obj_data):
sqoop_args = dict()
confParser = ConfigParser()
sqoop_args["connstr"] = confParser.get_config_value("sqoop-conn-str")
sqoop_args["username"] = confParser.get_config_value("db-user")
sqoop_args["pwdfile"] = confParser.get_config_value("db-pass-file")
sqoop_args["outdir"] = confParser.get_config_value("sqoop-out-dir")
sqoop_args["tablename"] = self.job_name
sqoop_args["targetdir"] = confParser.get_config_value(
"hdfs-root-path") + "/" + self.job_name
if obj_data:
if len(self.utilFuncs.get_generator_value(obj_data, "HDFS")) > 0:
sqoop_args["targetdir"] = self.utilFuncs.get_generator_value(
obj_data, "HDFS")[0]
sqoop_args["exportdir"] = self.utilFuncs.get_generator_value(
obj_data, "HDFS")[0]
sqoop_args["fetchsize"] = 1000
if obj_data:
if len(self.utilFuncs.get_generator_value(obj_data,
"FETCHSIZE")) > 0:
sqoop_args["fetchsize"] = self.utilFuncs.get_generator_value(
obj_data, "FETCHSIZE")[0]
sqoop_args["nmappers"] = 20
if obj_data:
if len(self.utilFuncs.get_generator_value(obj_data, "NMAP")) > 0:
sqoop_args["nmappers"] = self.utilFuncs.get_generator_value(
obj_data, "NMAP")[0]
sqoop_args["splitcol"] = "ID"
if obj_data:
if len(self.utilFuncs.get_generator_value(obj_data,
"SPLITCOL")) > 0:
sqoop_args["splitcol"] = self.utilFuncs.get_generator_value(
obj_data, "SPLITCOL")[0]
sqoop_args["mapredname"] = "sqoop_job_" + self.job_name
return sqoop_args
def test_get_logparser_param(self):
config_parser_inst = ConfigParser()
generator_param = config_parser_inst.get_config_value("generator-param")
self.assertEqual(generator_param, "generator-settings.txt")
def test_get_logparser_param(self):
config_parser_inst = ConfigParser()
logparser_param = config_parser_inst.get_config_value("logparser-param")
self.assertEqual(logparser_param, "logparser-settings.json")
def test_get_validator_param(self):
config_parser_inst = ConfigParser()
validator_param = config_parser_inst.get_config_value("validator-param")
self.assertEqual(validator_param, "validation-settings.txt")
def test_get_hdfs_root(self):
config_parser_inst = ConfigParser()
hdfs_root_path = config_parser_inst.get_config_value("hdfs-root-path")
self.assertIsNotNone(hdfs_root_path)
def test_get_db_passfile(self):
config_parser_inst = ConfigParser()
db_pass_file = config_parser_inst.get_config_value("db-pass-file")
self.assertIsNotNone(db_pass_file)
def test_get_db_pass(self):
config_parser_inst = ConfigParser()
db_password = config_parser_inst.get_config_value("db-pass")
self.assertIsNotNone(db_password)
def test_dbuser_exists(self):
config_parser_inst = ConfigParser()
db_user_name = config_parser_inst.get_config_value("db-user")
self.assertIsNotNone(db_user_name)
def test_config_file_element_length(self):
config_parser_inst = ConfigParser()
conf_json = config_parser_inst.parse_config()
conf_json_len = len(conf_json)
self.assertEqual(conf_json_len, 7)
def test_config_file_name_invalid(self):
config_parser_inst = ConfigParser("app1.conf")
conf_json = config_parser_inst.get_file_path()
self.failureException()