def run(self):
if self.singal == "URLRequest":
dataMonitor = DataMonitor(self.path,self.constants,self.monitor)
dataMonitor.init_monitor()
self.monitor.monitor_start(self.path+"/monitor.log")
log = Log(self.path+"/url_error.log")
log.begin_log()
ReportJsonSerial(self.monitor).json_start()
ReportJsonParallel(self.monitor).json_start()
time.sleep(5)
log.end_log()
else:
reportReadFile = ReportReadFile(self.path,self.singal,self.constants,self.monitor)
self.lock.acquire()
Util.printf("lock acquire".ljust(50,"."))
reportReadFile.read_file()
self.lock.release()
Util.printf("lock release".ljust(50,"."))
Util.printf("proccess stop".ljust(50,"."))
logdir = "/home/qs/scripts/video_report_yyyymm/" + monitorReport.mydate.get_now(
) + "/"
error_log = logdir + 'save_error.log'
retry_timeout_log = logdir + 'retry_timeout.log'
monitor_log = logdir + 'monitor.log'
checkpoint_log = logdir + 'checkpoint.log'
oracle_log = logdir + 'ORACLE_SQL.log'
uids_log = logdir + "uids.log"
Util.file_remove(error_log)
Util.file_remove(retry_timeout_log)
Util.file_remove(monitor_log)
Util.file_remove(checkpoint_log)
log = Log(error_log)
log.begin_log()
fileRead.read_file(logdir + "CDN_SQL.log")
fileRead.read_file(logdir + "VV_SQL.log")
fileRead.read_file(logdir + "INVALID_SQL.log")
fileRead.read_file(logdir + "W_SQL.log")
fileRead.read_file(logdir + "DM_SQL.log")
fileRead.read_file(logdir + "IP_SQL.log")
fileRead.read_file(logdir + "CDN_VV_DISTINCT_SQL.log")
fileRead.read_file(logdir + "ORACLE_SQL.log")
log.flush_log()
monitorReport.monitor_start(monitor_log)
jsonParallel.urls.append('http://my.tv.sohu.com/wm/u/vids.do?vid=%s')
jsonParallel.urls.append(
monitorReport.__setattr__("mydate",MyDate(mydate))
logdir = "/home/qs/scripts/video_report_yyyymm/"+monitorReport.mydate.get_now()+"/"
error_log = logdir + 'save_error.log'
retry_timeout_log = logdir + 'retry_timeout.log'
monitor_log = logdir + 'monitor.log'
checkpoint_log = logdir + 'checkpoint.log'
oracle_log = logdir + 'ORACLE_SQL.log'
uids_log = logdir + "uids.log"
Util.file_remove(error_log)
Util.file_remove(retry_timeout_log)
Util.file_remove(monitor_log)
Util.file_remove(checkpoint_log)
log = Log(error_log)
log.begin_log()
fileRead.read_file(logdir + "CDN_SQL.log")
fileRead.read_file(logdir + "VV_SQL.log")
fileRead.read_file(logdir + "INVALID_SQL.log")
fileRead.read_file(logdir + "W_SQL.log")
fileRead.read_file(logdir + "DM_SQL.log")
fileRead.read_file(logdir + "IP_SQL.log")
fileRead.read_file(logdir + "CDN_VV_DISTINCT_SQL.log")
fileRead.read_file(logdir + "ORACLE_SQL.log")
log.flush_log()
monitorReport.monitor_start(monitor_log)
jsonParallel.urls.append('http://my.tv.sohu.com/wm/u/vids.do?vid=%s')
jsonParallel.urls.append('http://api.my.tv.sohu.com/video/videoinfolist.do?vid=%s')
myQueue, conn_report_params=ReportOracle.conn_oracle_params)
#reportMysql = ReportMysql(ReportMysql.conn_space_params)
reportMysql = ReportMysql(ReportMysql.conn_formal_params)
if cmp(sys.argv[1], '-Dsave') == 0:
mydate = Util.get_yesterday()
if len(sys.argv) == 3:
mydate = sys.argv[2]
reportOracle.__setattr__("mydate", MyDate(mydate))
donateMysql.__setattr__("mydate", MyDate(mydate))
reportMysql.__setattr__("mydate", MyDate(mydate))
base_log = base_log + "/" + mydate + "/"
Util.file_mkdirs(os.path.dirname(base_log))
log = Log(base_log + "/stdout.log")
log.begin_log()
hiveQuery = HiveQuery(base_log, myQueue, dt=mydate)
jobs_size = constants.myContants.keys().__len__() + 1
myThread = MyThread(myQueue, constants, jobs_size, monitor, base_log,
reportMysql)
myThread.start()
donateMysql.query_all_donate(base_log + "MysqlDonate")
reportOracle.write_rows_file(base_log + "OracleAd")
hiveQuery.query_hive()
myThread.sub_job_join()
fileWrite.checkpoint_save(base_log + "checkpoint.log",
reportOracle = ReportOracle(myQueue,conn_report_params=ReportOracle.conn_oracle_params)
#reportMysql = ReportMysql(ReportMysql.conn_space_params)
reportMysql = ReportMysql(ReportMysql.conn_formal_params)
if cmp(sys.argv[1], '-Dsave') == 0:
mydate = Util.get_yesterday()
if len(sys.argv) == 3:
mydate = sys.argv[2]
reportOracle.__setattr__("mydate",MyDate(mydate))
donateMysql.__setattr__("mydate",MyDate(mydate))
reportMysql.__setattr__("mydate",MyDate(mydate))
base_log = base_log+ "/" + mydate + "/"
Util.file_mkdirs(os.path.dirname(base_log))
log = Log(base_log+"/stdout.log")
log.begin_log()
hiveQuery = HiveQuery(base_log,myQueue,dt=mydate)
jobs_size = constants.myContants.keys().__len__() + 1
myThread = MyThread(myQueue,constants,jobs_size,monitor,base_log,reportMysql)
myThread.start()
donateMysql.query_all_donate(base_log + "MysqlDonate")
reportOracle.write_rows_file(base_log + "OracleAd")
hiveQuery.query_hive()
myThread.sub_job_join()
fileWrite.checkpoint_save(base_log+"checkpoint.log",copy.deepcopy(monitor))
