-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.py
executable file
·104 lines (74 loc) · 3.6 KB
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
#coding=utf-8
import sys
import logging
import multiprocessing
import datetime
from lib.DB import DB
from lib.Config import Config
from lib.Process import Process as DataProcess
from lib.MssqlUtil import MssqlUtil
from lib.Utils import Utils
reload(sys)
sys.setdefaultencoding('utf-8')
def processData(mssqlConfig, mysqlConfig, logFile, tables):
logging.basicConfig(filename=logFile, filemode='a', level=logging.INFO, format='%(asctime)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
mysqlServer = mysqlConfig['host']
mysqlPort = mysqlConfig['port']
mysqlUsername = mysqlConfig['username']
mysqlPassword = mysqlConfig['password']
mysqlDB = mysqlConfig['db']
mysqlHandle = DB.myHandle(mysqlServer, mysqlUsername, mysqlPassword, mysqlDB, mysqlPort)
print 'mysqlServer:' + mysqlServer + ' mysqlPort:' + str(mysqlPort) + ' mysqlUserName:' + mysqlUsername
print 'mysqlPassword:' + mysqlPassword + ' mysqlDB:' + mysqlDB
mssqlDriver = mssqlConfig['driver']
mssqlServer = mssqlConfig['host']
mssqlUsername = mssqlConfig['username']
mssqlPassword = mssqlConfig['password']
mssqlDB = mssqlConfig['db']
mssqlPort = mssqlConfig['port']
mssqlDSN = 'Driver={};Server={};Database={};Uid={};Pwd={};Port={};TDS_Version=8.0;'.format(mssqlDriver, mssqlServer,
mssqlDB, mssqlUsername,
mssqlPassword, mssqlPort)
print 'mssqlDriver:' + mssqlDriver + ' mssqlServer:' + mssqlServer + ' mssqlUserName:' + mssqlUsername
print 'mssqlPassword:' + mssqlPassword + ' mssqlDB:' + mssqlDB + ' mssqlPort:' + str(mssqlPort)
mssqlHandle = DB.msHandle(mssqlDSN)
mssqlUtil = MssqlUtil()
mssqlUtil.setMssqlHandle(mssqlHandle.getCursor())
dataTypes = Utils.getDataTypes()
process = DataProcess()
process.setDBHandle(mysqlHandle, mssqlHandle)
process.setLogging(logging)
for table in tables:
msTable = table
myTable = '{}_{}'.format(mssqlServer, table)
pk = mssqlUtil.getPrimaryKey(table)
# Auto create table
createSql = mssqlUtil.genCreateTableSql(table, myTable, dataTypes)
mysqlHandle.getCursor().execute(createSql)
# Update/Insert records in destination DB that do not match source DB.
logging.info("Preparing to insert/update records.")
process.refreshData(myTable, msTable, pk)
logging.info("Insert/update records complete.")
# Delete records from destination DB that no longer exist in source DB.
logging.info("Preparing to delete records.")
process.deleteData(myTable, msTable, pk)
logging.info("Delete records complete.")
mssqlHandle.disconnect()
mysqlHandle.disconnect()
if __name__ == '__main__':
logFile = Config.getLogFile()
logFile += "." + datetime.datetime.now().strftime("%Y-%m-%d")
logging.basicConfig(filename=logFile, filemode='a', level=logging.INFO, format='%(asctime)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
logging.info("Database replication started.")
mysqlConfig = Config.getMysqlConfig()
mssqlConfigs = Config.getMssqlConfig()
# 异步多进程并发运行任务
pool = multiprocessing.Pool()
for mssqlConfig in mssqlConfigs:
mssqlDB = mssqlConfig['db']
tables = Config.getSyncTableWithDB(mssqlDB)
multiprocessing.freeze_support()
pool.apply_async(processData, (mssqlConfig, mysqlConfig, logFile, tables))
pool.close()
pool.join()
logging.info("Database replication complete.")