def hive_shell_exe(self, sql, log_path):
Util.file_mkdirs(os.path.dirname(log_path))
exec_sql = 'touch %s' % log_path
os.popen(exec_sql)
exec_sql = str('$HIVE_HOME/bin/hive -S -e "%s" > %s' % (sql, log_path))
os.popen(exec_sql)
self.q.put_singal(log_path[os.path.dirname(log_path).__len__()+1:])
def hive_shell_exe(self, sql, log_path):
Util.file_mkdirs(os.path.dirname(log_path))
exec_sql = 'touch %s' % log_path
os.popen(exec_sql)
exec_sql = str('$HIVE_HOME/bin/hive -S -e "%s" > %s' % (sql, log_path))
os.popen(exec_sql)
self.q.put_singal(log_path[os.path.dirname(log_path).__len__() + 1:])
reportOracle = ReportOracle(
myQueue, conn_report_params=ReportOracle.conn_oracle_params)
#reportMysql = ReportMysql(ReportMysql.conn_space_params)
reportMysql = ReportMysql(ReportMysql.conn_formal_params)
if cmp(sys.argv[1], '-Dsave') == 0:
mydate = Util.get_yesterday()
if len(sys.argv) == 3:
mydate = sys.argv[2]
reportOracle.__setattr__("mydate", MyDate(mydate))
donateMysql.__setattr__("mydate", MyDate(mydate))
reportMysql.__setattr__("mydate", MyDate(mydate))
base_log = base_log + "/" + mydate + "/"
Util.file_mkdirs(os.path.dirname(base_log))
log = Log(base_log + "/stdout.log")
log.begin_log()
hiveQuery = HiveQuery(base_log, myQueue, dt=mydate)
jobs_size = constants.myContants.keys().__len__() + 1
myThread = MyThread(myQueue, constants, jobs_size, monitor, base_log,
reportMysql)
myThread.start()
donateMysql.query_all_donate(base_log + "MysqlDonate")
reportOracle.write_rows_file(base_log + "OracleAd")
hiveQuery.query_hive()
myThread.sub_job_join()
donateMysql = ReportMysql(ReportMysql.conn_donate_params,q=myQueue)
reportOracle = ReportOracle(myQueue,conn_report_params=ReportOracle.conn_oracle_params)
#reportMysql = ReportMysql(ReportMysql.conn_space_params)
reportMysql = ReportMysql(ReportMysql.conn_formal_params)
if cmp(sys.argv[1], '-Dsave') == 0:
mydate = Util.get_yesterday()
if len(sys.argv) == 3:
mydate = sys.argv[2]
reportOracle.__setattr__("mydate",MyDate(mydate))
donateMysql.__setattr__("mydate",MyDate(mydate))
reportMysql.__setattr__("mydate",MyDate(mydate))
base_log = base_log+ "/" + mydate + "/"
Util.file_mkdirs(os.path.dirname(base_log))
log = Log(base_log+"/stdout.log")
log.begin_log()
hiveQuery = HiveQuery(base_log,myQueue,dt=mydate)
jobs_size = constants.myContants.keys().__len__() + 1
myThread = MyThread(myQueue,constants,jobs_size,monitor,base_log,reportMysql)
myThread.start()
donateMysql.query_all_donate(base_log + "MysqlDonate")
reportOracle.write_rows_file(base_log + "OracleAd")
hiveQuery.query_hive()
myThread.sub_job_join()
fileWrite.checkpoint_save(base_log+"checkpoint.log",copy.deepcopy(monitor))
