def create_hive_ts_ptn(self, partition, hive_hive_ts_path): """ Creates hive partition :param partition: Partition to create in the format of 'year=YYYY/month=MM/day=DD/hour=HH' :return: """ ptn_year, ptn_month, ptn_day, ptn_hour = utils.split_ptn(partition) create_ptn_cmd = ''' hive -e "use flowview; alter table %s_hive add partition (year = %s, month = %s, day = %s, hour = %s) location '%s/%s/%s/%s/%s'"; ''' % (self.table, ptn_year, ptn_month, ptn_day, ptn_hour, hive_hive_ts_path, ptn_year, ptn_month, ptn_day, ptn_hour) try: self.shell_exec.safe_execute(create_ptn_cmd, splitcmd=False, as_shell=True) except ShellException: logger.error( "Error in creating hive table to store hive timestamp") raise logger.info( "Created hive partition year = %s, month = %s, day = %s, hour = %s for %s" % (ptn_year, ptn_month, ptn_day, ptn_hour, self.table))
def pull_hive_ts(self, partition, hive_hive_ts_path): """ Pulls hive timestamp from the given list of partitions and write results into the corresponding directory under FlowView database :param partition: partition in the format of 'year=YYYY/month=MM/day=DD/hour=HH' :return: """ ptn_year, ptn_month, ptn_day, ptn_hour = utils.split_ptn(partition) pull_cmd = ''' hive -e "use %s; insert overwrite directory '%s/%s/%s/%s/%s' select event_id, hive_timestamp from %s where year = %s and month = %s and day = %s and hour = %s;"''' \ %(self.database,hive_hive_ts_path,ptn_year,ptn_month,ptn_day,ptn_hour, self.table, ptn_year, ptn_month,ptn_day,ptn_hour) try: self.shell_exec.safe_execute(pull_cmd, splitcmd=False, as_shell=True) except Exception: logger.error( "Error pulling data from %s hive table and writing to flowview database" % self.database) logger.info( "Wrote hive timestamp data into hive table flowview.db/%s_hive_ts" % self.table)
def create_hive_ts_ptn(self,partition,hive_hive_ts_path): """ Creates hive partition :param partition: Partition to create in the format of 'year=YYYY/month=MM/day=DD/hour=HH' :return: """ ptn_year,ptn_month,ptn_day,ptn_hour = utils.split_ptn(partition) create_ptn_cmd = ''' hive -e "use flowview; alter table %s_hive add partition (year = %s, month = %s, day = %s, hour = %s) location '%s/%s/%s/%s/%s'"; ''' %(self.table, ptn_year,ptn_month,ptn_day,ptn_hour, hive_hive_ts_path, ptn_year,ptn_month,ptn_day,ptn_hour) try: self.shell_exec.safe_execute(create_ptn_cmd,splitcmd=False,as_shell=True) except ShellException: logger.error("Error in creating hive table to store hive timestamp") raise logger.info("Created hive partition year = %s, month = %s, day = %s, hour = %s for %s" %(ptn_year,ptn_month,ptn_day,ptn_hour,self.table))
def pull_hive_ts(self,partition,hive_hive_ts_path): """ Pulls hive timestamp from the given list of partitions and write results into the corresponding directory under FlowView database :param partition: partition in the format of 'year=YYYY/month=MM/day=DD/hour=HH' :return: """ ptn_year,ptn_month,ptn_day,ptn_hour = utils.split_ptn(partition) pull_cmd = ''' hive -e "use %s; insert overwrite directory '%s/%s/%s/%s/%s' select event_id, hive_timestamp from %s where year = %s and month = %s and day = %s and hour = %s;"''' \ %(self.database,hive_hive_ts_path,ptn_year,ptn_month,ptn_day,ptn_hour, self.table, ptn_year, ptn_month,ptn_day,ptn_hour) try: self.shell_exec.safe_execute(pull_cmd,splitcmd=False,as_shell=True) except Exception: logger.error("Error pulling data from %s hive table and writing to flowview database" %self.database) logger.info("Wrote hive timestamp data into hive table flowview.db/%s_hive_ts" %self.table)