def setUpClass(self):
host = config().get('redis', 'host')
port = config().get('redis', 'port')
db = config().get('redis', 'db')
self.client = RedisClient(host, port, db)
self.key = '__redis_key_' + app.getUuid()
def testKafkaConnection(result):
host = config().get('kafka', 'host')
result['config']['kafka'] = {'host': host}
startTime = time.time()
try:
producer = utils.kafkaclient.KafkaProducer(host)
metadata = producer.list_topics()
endTime = time.time()
elapsedTime = int((endTime - startTime) * 1000)
topicMetadata = metadata.topics
topicName = config().get('kafka', 'topic')
relayTopicName = config().get('kafka', 'relay_topic')
if topicName in topicMetadata and relayTopicName in topicMetadata:
result['passed']['kafka'] = {'elapsed_ms': elapsedTime}
else:
result['failed']['kafka'] = {
'elapsed_ms': elapsedTime,
'message': 'topics NOT exist'
}
except Exception as e:
endTime = time.time()
elapsedTime = int((endTime - startTime) * 1000)
result['code'] = 1001
result['failed']['kafka'] = {
'elapsed_ms': elapsedTime,
'message': str(e)
}
def testRedisConnection(result):
host = config().get('redis', 'host')
port = config().get('redis', 'port')
db = config().get('redis', 'db')
result['config']['redis'] = {'host': host, 'port': port, 'db': db}
startTime = time.time()
try:
client = utils.redisclient.RedisClient(host, port, db)
pingResult = client.ping()
endTime = time.time()
elapsedTime = int((endTime - startTime) * 1000)
if pingResult:
result['passed']['redis'] = {'elapsed_ms': elapsedTime}
else:
result['code'] = 1001
result['failed']['redis'] = {'elapsed_ms': elapsedTime}
except Exception as e:
endTime = time.time()
elapsedTime = int((endTime - startTime) * 1000)
result['code'] = 1001
result['failed']['redis'] = {
'elapsed_ms': elapsedTime,
'message': str(e)
}
def testMysqlConnection(result, configKey):
host = config().get('mysql:' + configKey, 'host')
port = config().get('mysql:' + configKey, 'port')
db = config().get('mysql:' + configKey, 'database')
result['config']['mysql:' + configKey] = {
'host': host,
'port': port,
'database': db
}
startTime = time.time()
try:
connPool = ConnectinoPool()
conn = connPool.connection(configKey)
with conn.cursor() as cursor:
cursor.execute('show tables;')
data = cursor.fetchall()
endTime = time.time()
elapsedTime = int((endTime - startTime) * 1000)
result['passed']['mysql:' + configKey] = {'elapsed_ms': elapsedTime}
except Exception as e:
endTime = time.time()
elapsedTime = int((endTime - startTime) * 1000)
result['code'] = 1001
result['failed']['mysql:' + configKey] = {
'elapsed_ms': elapsedTime,
'message': str(e)
}
def getRedisClient():
global _redis_client
if _redis_client is None:
host = config().get('redis', 'host')
port = config().get('redis', 'port')
db = config().get('redis', 'db')
_redis_client = utils.redisclient.RedisClient(host, port, db)
return _redis_client
def testElasticSearchConnection(result):
host = config().get('elasticsearch', 'host')
hosts = host.split(',')
result['config']['elasticsearch'] = {'host': host}
startTime = time.time()
try:
client = Elasticsearch(hosts)
pingResult = client.ping()
endTime = time.time()
elapsedTime = int((endTime - startTime) * 1000)
if pingResult:
result['passed']['elasticsearch'] = {'elapsed_ms': elapsedTime}
else:
result['failed']['elasticsearch'] = {
'elapsed_ms': elapsedTime,
'message': 'fail to ping'
}
except Exception as e:
endTime = time.time()
elapsedTime = int((endTime - startTime) * 1000)
result['code'] = 1001
result['failed']['elasticsearch'] = {
'elapsed_ms': elapsedTime,
'message': str(e)
}
def __init__(self, taskName):
# instance of RedisStatus
self.status = RedisStatus(taskName)
self.relayTopic = config().get('kafka', 'topic')
self.statusConfig = None
self.kafkaConsumer = None
self.handler = None
self.itemCount = 0
super(SyncService, self).__init__()
def __init__(self, database):
self.topic = config().get('kafka', 'topic')
self.database = database
self._connPool = ConnectinoPool()
self._runPath = app.getPrjRoot() + "/run"
self._binlogPosFile = self._runPath + "/" + database + "_collector_position.safe"
self._kafkaProducer = self._initKafkaProducer()
self._kafkaMsgRedeliveryCount = 0
self._posStream = None
self._stream = None
def db_connect(query, parameter = None):
conn = None
try:
params = config.config()
conn = psycopg2.connect(**params)
cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
cur.execute(query, (parameter,))
row_headers=[desc[0] for desc in cur.description] # automagically get row headers
data = cur.fetchall()
return (data, row_headers)
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn:
conn.close()
print("Connection closed")
def _copyKafkaOffset(self):
"""
将新的消费者的offset设置为 latest
"""
# 首先要获取kafka topic的所有分区
topicName = config().get('kafka', 'topic')
if self.status.nextConfig:
nextStatusConfig = RedisStatusConfig(self.status.nextConfig, forceSync=True)
try:
nextConsumer = remote.getKafkaConsumer(
nextStatusConfig.kafkaGroupId,
autoCommit=False,
autoOffsetReset='latest'
)
_logger.debug('next kafka groupid is: %s', nextStatusConfig.kafkaGroupId)
clusterMetadata = nextConsumer.list_topics(topicName)
topicMetadata = clusterMetadata.topics.get(topicName, {})
partitions = topicMetadata.partitions
for pid in partitions.keys():
p = TopicPartition(topicName, pid)
nextConsumer.assign([p])
msg = nextConsumer.poll(10)
if msg:
offset = msg.offset() - 1
_logger.debug('pid[%s] topic[%s] offset[%s]', pid, topicName, offset)
if offset >= 0:
p.offset = offset
nextConsumer.commit(offsets=[p])
except Exception as e:
_logger.error('exception occurs when setting offset for new consumer: %s', Failure())
raise
finally:
if nextConsumer:
nextConsumer.close()
def setUpClass(self):
self.host = config().get('kafka', 'host')
self.groupId = '__group_id_' + app.getUuid()
def getElasticClient():
host = config().get('elasticsearch', 'host')
hosts = host.split(',')
return Elasticsearch(hosts)
def getKafkaConsumer(groupId, autoCommit=True, autoOffsetReset='earliest'):
host = config().get('kafka', 'host')
return utils.kafkaclient.KafkaConsumer(groupId,
host,
autoCommit=autoCommit,
autoOffsetReset=autoOffsetReset)
def getKafkaProducer():
host = config().get('kafka', 'host')
return utils.kafkaclient.KafkaProducer(host)
from flask import Flask from flask_sqlalchemy import SQLAlchemy from application.config import config app = Flask(__name__, static_url_path='/static') config(app) db = SQLAlchemy(app) # endpoints from application.http.index import index from application.http.index import any_root_path from application.http.get_all_users import get_all_users from application.http.login import login from application.http.register import register
def listen(self):
_logger.info("Start to listen to the binlog of %s" % self.database)
section = 'mysql:' + self.database
mysqlSetting = {
"host": config().get(section, "host"),
"port": int(config().get(section, 'port')),
"user": config().get(section, "user"),
"password": config().get(section, "password"),
}
watchedDatabases = [self.database]
# load last binlog reader position
logFile, logPos, resumeStrem = self._loadLastBinlogPos()
self._stream = BinLogStreamReader(
connection_settings=mysqlSetting,
server_id=int(config().get(section, "slaveid")),
only_events=[DeleteRowsEvent, WriteRowsEvent, UpdateRowsEvent],
blocking=True,
resume_stream=resumeStrem,
log_file=logFile,
log_pos=logPos,
)
while True:
refresh = False
try:
for binlogEvent in self._stream:
refresh = True
logFile, logPos = self._stream.log_file, self._stream.log_pos
# filter no watch database
if binlogEvent.schema not in watchedDatabases:
self._writeBinlogPos(logFile, logPos)
continue
binlog = {}
binlog['storage'] = 'mysql'
binlog['database'] = '%s' % binlogEvent.schema
binlog['table'] = '%s' % binlogEvent.table
binlog['timestamp'] = datetime.fromtimestamp(
binlogEvent.timestamp).strftime('%Y-%m-%d %H:%M:%S')
for row in binlogEvent.rows:
if isinstance(binlogEvent, DeleteRowsEvent):
binlog['values'] = row['values']
binlog['type'] = 'DELETE'
elif isinstance(binlogEvent, UpdateRowsEvent):
binlog['before'] = row['before_values']
binlog['values'] = row['after_values']
binlog['type'] = 'UPDATE'
elif isinstance(binlogEvent, WriteRowsEvent):
binlog['values'] = row['values']
binlog['type'] = 'INSERT'
binlogRow = json.dumps(binlog,
default=timeutil.dateHandler)
self._pushToKafka(binlogRow, binlog['database'],
binlog['table'])
# after pushing binlog to kafka, update the binlog position
self._writeBinlogPos(logFile, logPos)
if not refresh:
_logger.info(
"NO new input binlog, current position: [%s:%d]",
logFile if logFile is not None else "",
logPos if logPos is not None else 0)
time.sleep(0.1)
except Exception as e:
print(e)
sys.exit(1)
