def query(client, db_name, coll_name, start_time, end_time, period):
# start_time = 20210107000000
# period = 235959
# end_time = start_time + period
aggregate_sql = (
'[{"$match":{"StartTime":{"$gte":"starttime","$lte":"endtime"}}},'
'{"$group":{"_id":null,'
'"ReadXdrCount":{"$sum":"$ReadXdrCount"},'
'"DeleteOkCount":{"$sum": "$DeleteOkCount"},'
'"SkippedXdrCount":{"$sum":"$SkippedXdrCount"},'
'"WriteOkXdrCount":{"$sum":"$WriteOkXdrCount"},'
'"WriteFailXdrCount":{"$sum":"$WriteFailXdrCount"},'
'"WriteFailMongoServerErrXdrCount":{"$sum": "$WriteFailMongoServerErrXdrCount"},'
'"WriteFailMongoErrXdrCount":{"$sum":"$WriteFailMongoErrXdrCount"},'
'"WriteFailSocketErrXdrCount":{"$sum":"$WriteFailSocketErrXdrCount"},'
'"WriteFailBaseErrXdrCount":{"$sum":"$WriteFailBaseErrXdrCount"},'
'"WriteDuplicateXdrCount":{"$sum": "$WriteDuplicateXdrCount"},'
'"WriteNotRecycleCount":{"$sum":"$WriteNotRecycleCount"}}}]')
aggregate_sql = aggregate_sql.replace("starttime",
str(start_time)).replace(
"endtime", str(end_time))
logging.info(f'aggregate_sql is {aggregate_sql}')
aggregate_sql_list = json.loads(aggregate_sql)
result_list = mongo_writer.conn_aggregate(client, db_name, coll_name,
aggregate_sql_list)
logging.info(f'result_list is {result_list}')
if len(result_list) == 0:
speed = 0
else:
speed = result_list[0].get("ReadXdrCount") / period
logging.info(f"speed : {speed} 条/秒")
return speed
def query(client, db_name, coll_name, start_time, end_time, period):
start_time = datetime.datetime.strptime(
start_time, '%Y%m%d%H%M%S').strftime('%Y%m%d %H:%M:%S')
end_time = datetime.datetime.strptime(
end_time, '%Y%m%d%H%M%S').strftime('%Y%m%d %H:%M:%S')
aggregate_sql = (
'[{"$match":{"StartTime":{"$gte":"starttime","$lte":"endtime"}}},'
'{"$group":{"_id":null,'
'"QueryCount":{"$sum":1},'
'"QueryTimeCount":{"$sum":"$TotalUse"}}}]')
aggregate_sql = aggregate_sql.replace("starttime",
str(start_time)).replace(
"endtime", str(end_time))
logging.info(f'aggregate_sql is {aggregate_sql}')
aggregate_sql_list = json.loads(aggregate_sql)
result_list = mongo_writer.conn_aggregate(client, db_name, coll_name,
aggregate_sql_list)
logging.info(f'result_list is {result_list}')
if len(result_list) == 0:
logging.info("result is empty")
speed = 0
else:
try:
speed = result_list[0].get("QueryCount") / period
except Exception as e:
logging.error(e)
speed = 0
logging.info(f"speed : {speed} 次/分钟")
return speed
def query(client, db_name, coll_name, start_time, end_time, period):
# username = '******'
# password = '******'
# mongos_host = '10.19.85.33'
# mongos_port = 34000
# db_name = 'test'
# coll_name = 'stat_emit'
# client = mongo_writer.auth(username, password, mongos_host, mongos_port)
# 使用系统当前时间前时间范围进行数据查找
# end_time = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
# # period = -5
# start_time = (datetime.datetime.now()+datetime.timedelta(minutes=period)).strftime('%Y%m%d%H%M%S')
aggregate_sql = (
'[{"$match":{"StartTime":{"$gte":"starttime","$lte":"endtime"}}},'
'{"$group":{"_id":null,'
'"CorrectLines":{"$sum":"$CorrectLines"},'
'"ErrorLines":{"$sum": "$ErrorLines"},'
'"SendLines":{"$sum":"$SendLines"},'
'"NonTimeoutExceptionLines":{"$sum":"$NonTimeoutExceptionLines"},'
'"TimeoutExceptionLines":{"$sum":"$TimeoutExceptionLines"}}}]')
aggregate_sql = aggregate_sql.replace("starttime",
str(start_time)).replace(
"endtime", str(end_time))
logging.info(f'aggregate_sql is {aggregate_sql}')
aggregate_sql_list = json.loads(aggregate_sql)
result_list = mongo_writer.conn_aggregate(client, db_name, coll_name,
aggregate_sql_list)
logging.info(f'result_list is {result_list}')
if len(result_list) == 0:
speed = 0
else:
speed = result_list[0].get("SendLines") / period
logging.info(f"speed : {speed} 条/秒")
return speed
def query_excep(client, db_name, coll_name, start_time, end_time, period):
aggregate_sql = ('[{"$match":{"ExceptionTime":{"$gte":"starttime","$lte":"endtime"}}},'
'{"$group":{"_id":{"ExceptionType":"$ExceptionType","FunctionName":"$FunctionName",'
'"Ip":"$Ip"},"count":{"$sum":1}}}]')
aggregate_sql = aggregate_sql.replace("starttime", str(start_time)).replace("endtime", str(end_time))
logging.info(f'aggregate_sql is {aggregate_sql}')
aggregate_sql_list = json.loads(aggregate_sql)
result_list = mongo_writer.conn_aggregate(client, db_name, coll_name, aggregate_sql_list)
logging.info(f'result_list is {result_list}')
return result_list
def query(client, db_name, coll_name, start_time, end_time, period):
# end_time = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
# start_time = (datetime.datetime.now() + datetime.timedelta(minutes=period)).strftime('%Y%m%d%H%M%S')
aggregate_sql = (
'[{"$match":{"startTime":{"$gte":"starttime","$lte":"endtime"}}},'
'{"$group":{"_id":null,'
'"TotalLines":{"$sum":1}}}]')
aggregate_sql = aggregate_sql.replace("starttime",
str(start_time)).replace(
"endtime", str(end_time))
logging.info(f'aggregate_sql is {aggregate_sql}')
aggregate_sql_list = json.loads(aggregate_sql)
result_list = mongo_writer.conn_aggregate(client, db_name, coll_name,
aggregate_sql_list)
logging.info(f'result_list is {result_list}')
if len(result_list) == 0:
speed = 0
else:
speed = result_list[0].get("TotalLines") / period
logging.info(f"speed : {speed} 个/秒")
return speed
filemode='a',
format='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s')
# rollback stat 查询，统计各台主机rollback文件数量
# 统计InputTime为当天时间的rollback记录
if __name__ == '__main__':
username = '******'
password = '******'
mongos_host = '10.19.85.33'
mongos_port = 34000
db_name = 'test'
coll_name = 'stat_rollback'
client = mongo_writer.auth(username, password, mongos_host, mongos_port)
start_time = datetime.datetime.strptime('2021-03-08 00:00:00', '%Y-%m-%d %H:%M:%S')
period = 1
end_time = (start_time + datetime.timedelta(days=period)).strftime("%Y-%m-%d %H:%M:%S")
aggregate_sql = ('[{"$match":{"InputTime":{"$gte":"starttime","$lte":"endtime"}}},'
'{"$group":{"_id":null,'
'"rollbackfilecount":{"$sum":1}}}]').replace("starttime", str(start_time)).\
replace("endtime", str(end_time))
logging.info(f'aggregate_sql is {aggregate_sql}')
aggregate_sql_list = json.loads(aggregate_sql)
result_list = mongo_writer.conn_aggregate(client, db_name, coll_name, aggregate_sql_list)
logging.info(f'result_list is {result_list}')
if len(result_list) == 0:
rollbackfilecount = 0
else:
rollbackfilecount = result_list[0].get('rollbackfilecount')
logging.info(f"rollbackfilecount : {rollbackfilecount} 个文件")