def __init__(self,config_file=None):
"""
Initializes the FlowviewHandler superclass and instantiates manager classes
needed to performing load-related actions
:param topic: Dataset's Trinity topic name
:param db: Dataset's Thrive database name in Hive
:param table: Dataset's Thrive table name in Hive
:return: None
"""
super(LoadHandler,self).__init__(config_file)
self.hive_mgr = HiveManager(self.topic,self.table)
self.hdfs_dir_pending = None
self.hdfs_ptn_list = set()
self.hive_ptn_pending = None
self.hdfs_new_last_dir = None
self.hive_new_last_ptn = None
self.hdfs_proceed = False
self.hive_proceed = False
logger.info("Starting load of %s %s" %(self.topic,self.table))
def __init__(self,config_file=None):
"""
:param topic: Dataset's Trinity topic name
:param db: Dataset's Thrive database name in Hive
:table: Dataset's Thrive table name in Hive
:return:
"""
self.parser = SafeConfigParser()
self.parser.read(config_file)
self.topic = self.get_config("topic_name")
self.database = self.get_config("database_name")
self.table = self.get_config("table_name")
self.connection_info = self.get_config("connection_info")
self.metadata_mgr = MetadataManager(self.connection_info,self.table,self.topic)
self.hdfs_mgr = hdfsManager(self.topic)
self.hive_mng = HiveManager(self.database,self.table)
self.shell_exec = ShellExecutor()
self.loadts = datetime.now()
self.hdfs_topic = "idea-flowview"
