예제 #1
0
 def __init__(self,
              q,
              mydate=MyDate(Util.get_yesterday()),
              conn_report_params=None):
     OracleWrap.__init__(self, conn_report_params)
     self.mydate = mydate
     self.q = q
예제 #2
0
 def __init__(self,
              conn_params,
              q=None,
              mydate=MyDate(Util.get_yesterday())):
     MysqlWrap.__init__(self, conn_params)
     self.q = q
     self.mydate = mydate
     self.table_name = "video_report_%d" % int(
         self.mydate.get_now(format='ym'))
예제 #3
0
 def __init__(self, map_store={}):
     self.mydate = MyDate()
     self.map_store = map_store  # storage data row
     self.uids = {}  # vid:uid
     self.urls_parallel_total = 0  # parallel urls length
     self.urls_serial_total = 0  # serial urls length
     self.urls_parallel_running = []  # parallel urls (needed to be request)
     self.urls_serial_running = []  # serial urls (needed to be request)
     self.urls_parallel_timeout = []  # parallel request url timeout list
     self.urls_serial_timeout = []  # serial request url timeout list
     self.urls_parallel_timeout_retry = [
     ]  # parallel request url timeout retry list
     self.urls_serial_timeout_retry = [
     ]  # serial request url timeout retry list
     self.urls_retry_failure = []  # urls request retry failure list
     self.begin_time = None  # json begin request record
     self.uid_share_map = {}  # record  uid:is_video_share
예제 #4
0
from pythonFrame.myUtils.Util import MyDate
from report.ReportEntity import ReportEntity, ReportDMEntity
from report.ReportMonitor import ReportMonitor

__author__ = 'linzhou208438'
__update__ = '2015/6/24'


class A(object):
    def __init__(self, map):
        print map.map_store


if __name__ == '__main__':

    # reportMysql=ReportMysql(ReportMysql.conn_space_params)
    # re = reportMysql.select_map_store()
    #print re.__len__()
    # mysqlWrap = ReportMysql(ReportMysql.conn_space_params)
    # mysqlWrap.insert_update_report(map_store=re,update_fields=ReportDMEntity().__dict__)
    monitorReport = ReportMonitor()

    mydatea = 20150623
    monitorReport.__setattr__("11", MyDate(mydatea))
    print monitorReport.mydate.get_now()
    print monitorReport.__dict__
    print monitorReport.map_store
    print monitorReport.__getattribute__("map_store")
    a = A(monitorReport)
예제 #5
0
    monitorReport = ReportMonitor()

    fileRead = ReportReadFile(monitorReport)
    fileWrite = FileWrite()

    jsonParallel = ReportJsonParallel(monitorReport)
    jsonSerial = ReportJsonSerial(monitorReport)

    mysqlWrap = ReportMysql(ReportMysql.conn_formal_params, monitorReport)
    #mysqlWrap = ReportMysql(ReportMysql.conn_space_params,monitorReport)

    if cmp(sys.argv[1], '-Dsave') == 0:
        mydate = Util.get_yesterday()
        if len(sys.argv) == 3:
            mydate = sys.argv[2]
        monitorReport.__setattr__("mydate", MyDate(mydate))

        logdir = "/home/qs/scripts/video_report_yyyymm/" + monitorReport.mydate.get_now(
        ) + "/"
        error_log = logdir + 'save_error.log'
        retry_timeout_log = logdir + 'retry_timeout.log'
        monitor_log = logdir + 'monitor.log'
        checkpoint_log = logdir + 'checkpoint.log'
        oracle_log = logdir + 'ORACLE_SQL.log'
        uids_log = logdir + "uids.log"

        Util.file_remove(error_log)
        Util.file_remove(retry_timeout_log)
        Util.file_remove(monitor_log)
        Util.file_remove(checkpoint_log)
예제 #6
0
    myQueue = MyQueue()
    monitor = ReportMonitor()
    fileWrite = FileWrite()
    constants = ContantsMappingColumn()
    donateMysql = ReportMysql(ReportMysql.conn_donate_params, q=myQueue)
    reportOracle = ReportOracle(
        myQueue, conn_report_params=ReportOracle.conn_oracle_params)
    #reportMysql = ReportMysql(ReportMysql.conn_space_params)
    reportMysql = ReportMysql(ReportMysql.conn_formal_params)

    if cmp(sys.argv[1], '-Dsave') == 0:
        mydate = Util.get_yesterday()
        if len(sys.argv) == 3:
            mydate = sys.argv[2]

        reportOracle.__setattr__("mydate", MyDate(mydate))
        donateMysql.__setattr__("mydate", MyDate(mydate))
        reportMysql.__setattr__("mydate", MyDate(mydate))

        base_log = base_log + "/" + mydate + "/"
        Util.file_mkdirs(os.path.dirname(base_log))
        log = Log(base_log + "/stdout.log")
        log.begin_log()

        hiveQuery = HiveQuery(base_log, myQueue, dt=mydate)
        jobs_size = constants.myContants.keys().__len__() + 1

        myThread = MyThread(myQueue, constants, jobs_size, monitor, base_log,
                            reportMysql)
        myThread.start()
예제 #7
0
        "passwd": "linhailei"
    }

    def __init__(self, monitorReport, conn_report_params=None):
        OracleWrap.__init__(self, conn_report_params)
        self.monitorReport = monitorReport

    def write_rows_file(self, path):
        #sql = 'select v_id,max_adv,uv,view_all_uv,stock,occu,pos1,pos2,pos3,pos4,pos5,t_occu from dwpdata.core_mobile_pgc where sver is not null and substr(sver,0,1)>=5 and  data_date=%s'
        sql = "select v_id,sum(max_adv),sum(uv),sum(view_all_uv),sum(stock),sum(occu),sum(pos1),sum(pos2),sum(pos3),sum(pos4),sum(pos5),sum(t_occu) from dwpdata.core_mobile_pgc where sver is not null   and sver <> 'null'  and  regexp_like(sver, '^\d')  and substr(sver,0,1)>=5  and  data_date=%s group by v_id "
        Util.printf(sql % self.monitorReport.mydate.get_now())
        rows = self.oracle_fetchall(sql % self.monitorReport.mydate.get_now())
        self.save_rows(rows, path)

    def save_rows(self, rows, path):
        file_write = open(path, 'w')
        val = []
        for row in rows:
            for i in range(12):
                val.append(str(row[i]))
            file_write.write("  ".join(val) + "\n")
            val = []
        file_write.close()


if __name__ == '__main__':
    monitorReport = ReportMonitor()
    monitorReport.__setattr__("mydate", MyDate(20150726))
    reportMysql = ReportMysql(ReportMysql.conn_space_params, monitorReport)
    reportMysql.check_create_table(26)
예제 #8
0
 def __init__(self, conn_params, date_param):
     MysqlWrap.__init__(self, conn_params)
     self.mydate = MyDate(date_param)