def insert_to_mysql_on_duplicate(self, columns_data):
field_exclude = ["uid", "video_id", "is56", "dtime", "play_plat"]
cmc = ContantsMappingColumn()
table_field_len = 0
update_field = []
table_field = []
datalist = []
for report in columns_data.values():
report.setdefault("dtime", self.mydate.get_now())
report.setdefault("play_plat", 1)
if table_field_len == 0:
for k in report.keys():
table_field.append(k)
if k not in field_exclude:
update_field.append("%s=values(%s)" % (k, k))
table_field_len = len(table_field)
table_value = []
for v in table_field:
table_value.append(report.get(v, cmc.get_default_value(v)))
datalist.append(tuple(table_value))
sql_head = ' insert into ' + self.table_name
sql_body = ' ( ' + ','.join(table_field) + ' ) '
sql_blank = ' , %s ' * (table_field_len - 1)
sql_tail = ' values ( %s ' + sql_blank + ' ) '
sql_duplicate = ' ON DUPLICATE KEY UPDATE ' + ','.join(update_field)
sql = sql_head + sql_body + sql_tail + sql_duplicate
Util.printf(sql)
Util.printf("will update rows_num:%d" % datalist.__len__())
self.mysql_executemany(sql, datalist)
def insert_to_mysql_default(self, columns_data):
field_exclude = ["uid", "video_id", "is56", "dtime", "play_plat"]
datalist = []
cmc = ContantsMappingColumn()
contants = cmc.get_unique_value()
table_field = [k for k in contants]
table_field.append("dtime")
table_field.append("play_plat")
table_field_len = len(table_field)
set_diff = contants.difference(set(field_exclude))
update_field = []
map(lambda s: update_field.append("%s=values(%s)" % (s, s)), set_diff)
for report in columns_data.values():
report.setdefault("dtime", self.mydate.get_now())
report.setdefault("play_plat", 1)
table_value = []
for v in table_field:
table_value.append(report.get(v, cmc.get_default_value(v)))
datalist.append(tuple(table_value))
sql_head = ' insert into ' + self.table_name
sql_body = ' ( ' + ','.join(table_field) + ' ) '
sql_blank = ' , %s ' * (table_field_len - 1)
sql_tail = ' values ( %s ' + sql_blank + ' ) '
sql_duplicate = ' ON DUPLICATE KEY UPDATE ' + ','.join(update_field)
sql = sql_head + sql_body + sql_tail + sql_duplicate
Util.printf(sql)
Util.printf("will update rows_num:%d" % datalist.__len__())
self.mysql_executemany(sql, datalist)
-DqueryOracle query oracle data to generate a file
-DqueryDonate query donate data to generate a file
-Drecovery disaster recovery if mysql no response
-DcheckParent check parent son account
-Dtest test python script
-DsendMsg eg: .py -DsendMsg phone1,phone2 message
-DupdateAll include url request
'''
exit()
base_log = "/opt/video_report_yyyymm/data"
myQueue = MyQueue()
monitor = ReportMonitor()
fileWrite = FileWrite()
constants = ContantsMappingColumn()
donateMysql = ReportMysql(ReportMysql.conn_donate_params, q=myQueue)
reportOracle = ReportOracle(
myQueue, conn_report_params=ReportOracle.conn_oracle_params)
#reportMysql = ReportMysql(ReportMysql.conn_space_params)
reportMysql = ReportMysql(ReportMysql.conn_formal_params)
if cmp(sys.argv[1], '-Dsave') == 0:
mydate = Util.get_yesterday()
if len(sys.argv) == 3:
mydate = sys.argv[2]
reportOracle.__setattr__("mydate", MyDate(mydate))
donateMysql.__setattr__("mydate", MyDate(mydate))
reportMysql.__setattr__("mydate", MyDate(mydate))
p.start()
Util.printf("queque pop %s" % singal)
else:
jbs = [job.is_alive() for job in self.jobs.values() ]
if jbs.count(False) == self.jobs_size :
self.isRun = False
if jbs.__len__() == self.jobs_size:
self.isRun = False
if jbs.__len__() > 0 and jbs.__len__() == self.jobs_size - 1:
self.q.put_singal("URLRequest")
jps_show = [(name,job.is_alive()) for name,job in self.jobs.items() ]
Util.printf("current process is_alive:%s" % jps_show)
time.sleep(2)
Util.printf("thread stop".ljust(50,"."))
if __name__ == "__main__":
q=MyQueue(ContantsMappingColumn())
q.put_singal("HiveUv")
q.put_singal("HiveVv")
m = MyThread(q,None,2)
m.start()
