def create_hive_ts_ptn(self, partition, hive_hive_ts_path):
"""
Creates hive partition
:param partition: Partition to create in the format of 'year=YYYY/month=MM/day=DD/hour=HH'
:return:
"""
ptn_year, ptn_month, ptn_day, ptn_hour = utils.split_ptn(partition)
create_ptn_cmd = '''
hive -e "use flowview;
alter table %s_hive add partition (year = %s, month = %s, day = %s, hour = %s)
location '%s/%s/%s/%s/%s'";
''' % (self.table, ptn_year, ptn_month, ptn_day, ptn_hour,
hive_hive_ts_path, ptn_year, ptn_month, ptn_day, ptn_hour)
try:
self.shell_exec.safe_execute(create_ptn_cmd,
splitcmd=False,
as_shell=True)
except ShellException:
logger.error(
"Error in creating hive table to store hive timestamp")
raise
logger.info(
"Created hive partition year = %s, month = %s, day = %s, hour = %s for %s"
% (ptn_year, ptn_month, ptn_day, ptn_hour, self.table))
def pull_hive_ts(self, partition, hive_hive_ts_path):
"""
Pulls hive timestamp from the given list of partitions
and write results into the corresponding directory under FlowView database
:param partition: partition in the format of 'year=YYYY/month=MM/day=DD/hour=HH'
:return:
"""
ptn_year, ptn_month, ptn_day, ptn_hour = utils.split_ptn(partition)
pull_cmd = '''
hive -e "use %s;
insert overwrite directory '%s/%s/%s/%s/%s'
select event_id, hive_timestamp from %s
where year = %s and month = %s and day = %s and hour = %s;"''' \
%(self.database,hive_hive_ts_path,ptn_year,ptn_month,ptn_day,ptn_hour,
self.table, ptn_year, ptn_month,ptn_day,ptn_hour)
try:
self.shell_exec.safe_execute(pull_cmd,
splitcmd=False,
as_shell=True)
except Exception:
logger.error(
"Error pulling data from %s hive table and writing to flowview database"
% self.database)
logger.info(
"Wrote hive timestamp data into hive table flowview.db/%s_hive_ts"
% self.table)
def create_hive_ts_ptn(self,partition,hive_hive_ts_path):
"""
Creates hive partition
:param partition: Partition to create in the format of 'year=YYYY/month=MM/day=DD/hour=HH'
:return:
"""
ptn_year,ptn_month,ptn_day,ptn_hour = utils.split_ptn(partition)
create_ptn_cmd = '''
hive -e "use flowview;
alter table %s_hive add partition (year = %s, month = %s, day = %s, hour = %s)
location '%s/%s/%s/%s/%s'";
''' %(self.table, ptn_year,ptn_month,ptn_day,ptn_hour,
hive_hive_ts_path, ptn_year,ptn_month,ptn_day,ptn_hour)
try:
self.shell_exec.safe_execute(create_ptn_cmd,splitcmd=False,as_shell=True)
except ShellException:
logger.error("Error in creating hive table to store hive timestamp")
raise
logger.info("Created hive partition year = %s, month = %s, day = %s, hour = %s for %s"
%(ptn_year,ptn_month,ptn_day,ptn_hour,self.table))
def pull_hive_ts(self,partition,hive_hive_ts_path):
"""
Pulls hive timestamp from the given list of partitions
and write results into the corresponding directory under FlowView database
:param partition: partition in the format of 'year=YYYY/month=MM/day=DD/hour=HH'
:return:
"""
ptn_year,ptn_month,ptn_day,ptn_hour = utils.split_ptn(partition)
pull_cmd = '''
hive -e "use %s;
insert overwrite directory '%s/%s/%s/%s/%s'
select event_id, hive_timestamp from %s
where year = %s and month = %s and day = %s and hour = %s;"''' \
%(self.database,hive_hive_ts_path,ptn_year,ptn_month,ptn_day,ptn_hour,
self.table, ptn_year, ptn_month,ptn_day,ptn_hour)
try:
self.shell_exec.safe_execute(pull_cmd,splitcmd=False,as_shell=True)
except Exception:
logger.error("Error pulling data from %s hive table and writing to flowview database" %self.database)
logger.info("Wrote hive timestamp data into hive table flowview.db/%s_hive_ts" %self.table)
