def insert_to_mysql_on_duplicate(self, columns_data): field_exclude = ["uid", "video_id", "is56", "dtime", "play_plat"] cmc = ContantsMappingColumn() table_field_len = 0 update_field = [] table_field = [] datalist = [] for report in columns_data.values(): report.setdefault("dtime", self.mydate.get_now()) report.setdefault("play_plat", 1) if table_field_len == 0: for k in report.keys(): table_field.append(k) if k not in field_exclude: update_field.append("%s=values(%s)" % (k, k)) table_field_len = len(table_field) table_value = [] for v in table_field: table_value.append(report.get(v, cmc.get_default_value(v))) datalist.append(tuple(table_value)) sql_head = ' insert into ' + self.table_name sql_body = ' ( ' + ','.join(table_field) + ' ) ' sql_blank = ' , %s ' * (table_field_len - 1) sql_tail = ' values ( %s ' + sql_blank + ' ) ' sql_duplicate = ' ON DUPLICATE KEY UPDATE ' + ','.join(update_field) sql = sql_head + sql_body + sql_tail + sql_duplicate Util.printf(sql) Util.printf("will update rows_num:%d" % datalist.__len__()) self.mysql_executemany(sql, datalist)
def insert_to_mysql_default(self, columns_data): field_exclude = ["uid", "video_id", "is56", "dtime", "play_plat"] datalist = [] cmc = ContantsMappingColumn() contants = cmc.get_unique_value() table_field = [k for k in contants] table_field.append("dtime") table_field.append("play_plat") table_field_len = len(table_field) set_diff = contants.difference(set(field_exclude)) update_field = [] map(lambda s: update_field.append("%s=values(%s)" % (s, s)), set_diff) for report in columns_data.values(): report.setdefault("dtime", self.mydate.get_now()) report.setdefault("play_plat", 1) table_value = [] for v in table_field: table_value.append(report.get(v, cmc.get_default_value(v))) datalist.append(tuple(table_value)) sql_head = ' insert into ' + self.table_name sql_body = ' ( ' + ','.join(table_field) + ' ) ' sql_blank = ' , %s ' * (table_field_len - 1) sql_tail = ' values ( %s ' + sql_blank + ' ) ' sql_duplicate = ' ON DUPLICATE KEY UPDATE ' + ','.join(update_field) sql = sql_head + sql_body + sql_tail + sql_duplicate Util.printf(sql) Util.printf("will update rows_num:%d" % datalist.__len__()) self.mysql_executemany(sql, datalist)
-DqueryOracle query oracle data to generate a file -DqueryDonate query donate data to generate a file -Drecovery disaster recovery if mysql no response -DcheckParent check parent son account -Dtest test python script -DsendMsg eg: .py -DsendMsg phone1,phone2 message -DupdateAll include url request ''' exit() base_log = "/opt/video_report_yyyymm/data" myQueue = MyQueue() monitor = ReportMonitor() fileWrite = FileWrite() constants = ContantsMappingColumn() donateMysql = ReportMysql(ReportMysql.conn_donate_params, q=myQueue) reportOracle = ReportOracle( myQueue, conn_report_params=ReportOracle.conn_oracle_params) #reportMysql = ReportMysql(ReportMysql.conn_space_params) reportMysql = ReportMysql(ReportMysql.conn_formal_params) if cmp(sys.argv[1], '-Dsave') == 0: mydate = Util.get_yesterday() if len(sys.argv) == 3: mydate = sys.argv[2] reportOracle.__setattr__("mydate", MyDate(mydate)) donateMysql.__setattr__("mydate", MyDate(mydate)) reportMysql.__setattr__("mydate", MyDate(mydate))
p.start() Util.printf("queque pop %s" % singal) else: jbs = [job.is_alive() for job in self.jobs.values() ] if jbs.count(False) == self.jobs_size : self.isRun = False if jbs.__len__() == self.jobs_size: self.isRun = False if jbs.__len__() > 0 and jbs.__len__() == self.jobs_size - 1: self.q.put_singal("URLRequest") jps_show = [(name,job.is_alive()) for name,job in self.jobs.items() ] Util.printf("current process is_alive:%s" % jps_show) time.sleep(2) Util.printf("thread stop".ljust(50,".")) if __name__ == "__main__": q=MyQueue(ContantsMappingColumn()) q.put_singal("HiveUv") q.put_singal("HiveVv") m = MyThread(q,None,2) m.start()