def __init__(self,
q,
mydate=MyDate(Util.get_yesterday()),
conn_report_params=None):
OracleWrap.__init__(self, conn_report_params)
self.mydate = mydate
self.q = q
def __init__(self,
conn_params,
q=None,
mydate=MyDate(Util.get_yesterday())):
MysqlWrap.__init__(self, conn_params)
self.q = q
self.mydate = mydate
self.table_name = "video_report_%d" % int(
self.mydate.get_now(format='ym'))
def __init__(self, map_store={}):
self.mydate = MyDate()
self.map_store = map_store # storage data row
self.uids = {} # vid:uid
self.urls_parallel_total = 0 # parallel urls length
self.urls_serial_total = 0 # serial urls length
self.urls_parallel_running = [] # parallel urls (needed to be request)
self.urls_serial_running = [] # serial urls (needed to be request)
self.urls_parallel_timeout = [] # parallel request url timeout list
self.urls_serial_timeout = [] # serial request url timeout list
self.urls_parallel_timeout_retry = [
] # parallel request url timeout retry list
self.urls_serial_timeout_retry = [
] # serial request url timeout retry list
self.urls_retry_failure = [] # urls request retry failure list
self.begin_time = None # json begin request record
self.uid_share_map = {} # record uid:is_video_share
from pythonFrame.myUtils.Util import MyDate
from report.ReportEntity import ReportEntity, ReportDMEntity
from report.ReportMonitor import ReportMonitor
__author__ = 'linzhou208438'
__update__ = '2015/6/24'
class A(object):
def __init__(self, map):
print map.map_store
if __name__ == '__main__':
# reportMysql=ReportMysql(ReportMysql.conn_space_params)
# re = reportMysql.select_map_store()
#print re.__len__()
# mysqlWrap = ReportMysql(ReportMysql.conn_space_params)
# mysqlWrap.insert_update_report(map_store=re,update_fields=ReportDMEntity().__dict__)
monitorReport = ReportMonitor()
mydatea = 20150623
monitorReport.__setattr__("11", MyDate(mydatea))
print monitorReport.mydate.get_now()
print monitorReport.__dict__
print monitorReport.map_store
print monitorReport.__getattribute__("map_store")
a = A(monitorReport)
monitorReport = ReportMonitor()
fileRead = ReportReadFile(monitorReport)
fileWrite = FileWrite()
jsonParallel = ReportJsonParallel(monitorReport)
jsonSerial = ReportJsonSerial(monitorReport)
mysqlWrap = ReportMysql(ReportMysql.conn_formal_params, monitorReport)
#mysqlWrap = ReportMysql(ReportMysql.conn_space_params,monitorReport)
if cmp(sys.argv[1], '-Dsave') == 0:
mydate = Util.get_yesterday()
if len(sys.argv) == 3:
mydate = sys.argv[2]
monitorReport.__setattr__("mydate", MyDate(mydate))
logdir = "/home/qs/scripts/video_report_yyyymm/" + monitorReport.mydate.get_now(
) + "/"
error_log = logdir + 'save_error.log'
retry_timeout_log = logdir + 'retry_timeout.log'
monitor_log = logdir + 'monitor.log'
checkpoint_log = logdir + 'checkpoint.log'
oracle_log = logdir + 'ORACLE_SQL.log'
uids_log = logdir + "uids.log"
Util.file_remove(error_log)
Util.file_remove(retry_timeout_log)
Util.file_remove(monitor_log)
Util.file_remove(checkpoint_log)
myQueue = MyQueue()
monitor = ReportMonitor()
fileWrite = FileWrite()
constants = ContantsMappingColumn()
donateMysql = ReportMysql(ReportMysql.conn_donate_params, q=myQueue)
reportOracle = ReportOracle(
myQueue, conn_report_params=ReportOracle.conn_oracle_params)
#reportMysql = ReportMysql(ReportMysql.conn_space_params)
reportMysql = ReportMysql(ReportMysql.conn_formal_params)
if cmp(sys.argv[1], '-Dsave') == 0:
mydate = Util.get_yesterday()
if len(sys.argv) == 3:
mydate = sys.argv[2]
reportOracle.__setattr__("mydate", MyDate(mydate))
donateMysql.__setattr__("mydate", MyDate(mydate))
reportMysql.__setattr__("mydate", MyDate(mydate))
base_log = base_log + "/" + mydate + "/"
Util.file_mkdirs(os.path.dirname(base_log))
log = Log(base_log + "/stdout.log")
log.begin_log()
hiveQuery = HiveQuery(base_log, myQueue, dt=mydate)
jobs_size = constants.myContants.keys().__len__() + 1
myThread = MyThread(myQueue, constants, jobs_size, monitor, base_log,
reportMysql)
myThread.start()
"passwd": "linhailei"
}
def __init__(self, monitorReport, conn_report_params=None):
OracleWrap.__init__(self, conn_report_params)
self.monitorReport = monitorReport
def write_rows_file(self, path):
#sql = 'select v_id,max_adv,uv,view_all_uv,stock,occu,pos1,pos2,pos3,pos4,pos5,t_occu from dwpdata.core_mobile_pgc where sver is not null and substr(sver,0,1)>=5 and data_date=%s'
sql = "select v_id,sum(max_adv),sum(uv),sum(view_all_uv),sum(stock),sum(occu),sum(pos1),sum(pos2),sum(pos3),sum(pos4),sum(pos5),sum(t_occu) from dwpdata.core_mobile_pgc where sver is not null and sver <> 'null' and regexp_like(sver, '^\d') and substr(sver,0,1)>=5 and data_date=%s group by v_id "
Util.printf(sql % self.monitorReport.mydate.get_now())
rows = self.oracle_fetchall(sql % self.monitorReport.mydate.get_now())
self.save_rows(rows, path)
def save_rows(self, rows, path):
file_write = open(path, 'w')
val = []
for row in rows:
for i in range(12):
val.append(str(row[i]))
file_write.write(" ".join(val) + "\n")
val = []
file_write.close()
if __name__ == '__main__':
monitorReport = ReportMonitor()
monitorReport.__setattr__("mydate", MyDate(20150726))
reportMysql = ReportMysql(ReportMysql.conn_space_params, monitorReport)
reportMysql.check_create_table(26)
def __init__(self, conn_params, date_param):
MysqlWrap.__init__(self, conn_params)
self.mydate = MyDate(date_param)