def hive_shell_exe(self, sql, log_path): Util.file_mkdirs(os.path.dirname(log_path)) exec_sql = 'touch %s' % log_path os.popen(exec_sql) exec_sql = str('$HIVE_HOME/bin/hive -S -e "%s" > %s' % (sql, log_path)) os.popen(exec_sql) self.q.put_singal(log_path[os.path.dirname(log_path).__len__()+1:])
def hive_shell_exe(self, sql, log_path): Util.file_mkdirs(os.path.dirname(log_path)) exec_sql = 'touch %s' % log_path os.popen(exec_sql) exec_sql = str('$HIVE_HOME/bin/hive -S -e "%s" > %s' % (sql, log_path)) os.popen(exec_sql) self.q.put_singal(log_path[os.path.dirname(log_path).__len__() + 1:])
reportOracle = ReportOracle( myQueue, conn_report_params=ReportOracle.conn_oracle_params) #reportMysql = ReportMysql(ReportMysql.conn_space_params) reportMysql = ReportMysql(ReportMysql.conn_formal_params) if cmp(sys.argv[1], '-Dsave') == 0: mydate = Util.get_yesterday() if len(sys.argv) == 3: mydate = sys.argv[2] reportOracle.__setattr__("mydate", MyDate(mydate)) donateMysql.__setattr__("mydate", MyDate(mydate)) reportMysql.__setattr__("mydate", MyDate(mydate)) base_log = base_log + "/" + mydate + "/" Util.file_mkdirs(os.path.dirname(base_log)) log = Log(base_log + "/stdout.log") log.begin_log() hiveQuery = HiveQuery(base_log, myQueue, dt=mydate) jobs_size = constants.myContants.keys().__len__() + 1 myThread = MyThread(myQueue, constants, jobs_size, monitor, base_log, reportMysql) myThread.start() donateMysql.query_all_donate(base_log + "MysqlDonate") reportOracle.write_rows_file(base_log + "OracleAd") hiveQuery.query_hive() myThread.sub_job_join()
donateMysql = ReportMysql(ReportMysql.conn_donate_params,q=myQueue) reportOracle = ReportOracle(myQueue,conn_report_params=ReportOracle.conn_oracle_params) #reportMysql = ReportMysql(ReportMysql.conn_space_params) reportMysql = ReportMysql(ReportMysql.conn_formal_params) if cmp(sys.argv[1], '-Dsave') == 0: mydate = Util.get_yesterday() if len(sys.argv) == 3: mydate = sys.argv[2] reportOracle.__setattr__("mydate",MyDate(mydate)) donateMysql.__setattr__("mydate",MyDate(mydate)) reportMysql.__setattr__("mydate",MyDate(mydate)) base_log = base_log+ "/" + mydate + "/" Util.file_mkdirs(os.path.dirname(base_log)) log = Log(base_log+"/stdout.log") log.begin_log() hiveQuery = HiveQuery(base_log,myQueue,dt=mydate) jobs_size = constants.myContants.keys().__len__() + 1 myThread = MyThread(myQueue,constants,jobs_size,monitor,base_log,reportMysql) myThread.start() donateMysql.query_all_donate(base_log + "MysqlDonate") reportOracle.write_rows_file(base_log + "OracleAd") hiveQuery.query_hive() myThread.sub_job_join() fileWrite.checkpoint_save(base_log+"checkpoint.log",copy.deepcopy(monitor))