def test_sync_model():
# 获取 mongodb 到表的连接
collection = MongoUtils("", "local", "oplog.rs").get_mongo_collection()
oplog_start = bson.timestamp.Timestamp(1552618080, 11)
oplog_end = bson.timestamp.Timestamp(1552618081, 0)
document = {"ns": "", "ts": {"$gte": oplog_start, "$lt": oplog_end}}
cursor = collection.find(document,
cursor_type=pymongo.CursorType.TAILABLE_AWAIT,
oplog_replay=True).sort([
(u'$natural', pymongo.ASCENDING)
]).limit(100)
for docu in cursor:
obj = docu["o"]["model_result"]
print obj["level"], type(obj["level"]), obj.has_key(
"level"), sync_mongo_service.get_mysql_data(
obj["level"]) is not "nan", obj["level"] is not None
def query_sort():
'''
mongo 查询
排序
limit 操作
noCursorTimeout 设置为 true
Returns:
'''
collection = MongoUtils(u'', u'acrm', u'policy_result').getMongo()
document = {u'ut': {u'$gte': 1504022400000, u'$lt': 1504540800000}}
# 按照 pymongo.ASCENDING 表示升序排序
# pymongo.DESCENDING 表示降序
cursor_ = collection.find(document, no_cursor_timeout=True).sort([(u'ut', pymongo.ASCENDING)]).limit(5)
for docu in cursor_:
ut = docu[u'ut']
date_str = ts_str(ut / 1000)
print date_str
def test():
logger.error(" logger ================= ")
email_logger.error(" email_logger ================= ")
errorSql_logger.error(" errorSql_logger ================= ")
fullSql_logger.error(" fullSql_logger ================= ")
collection = MongoUtils("", "local", "oplog.rs").get_mongo_collection()
oplog_start = bson.timestamp.Timestamp(1552298024, 0)
oplog_end = bson.timestamp.Timestamp(1552298025, 0)
document = {"ns": "jdzz_acrm.policy_result", "ts": {"$gte": oplog_start, "$lt": oplog_end}}
cursor = collection.find(document, cursor_type=pymongo.CursorType.TAILABLE_AWAIT, oplog_replay=True).sort(
[(u'$natural', pymongo.ASCENDING)]).limit(100)
for docu in cursor:
_id = docu["o2"]["_id"]
result_json = {"_id": str(_id), "ts": sync_mongo_data.get_ts("jdzz", "policy_result")}
return jsonify(result_json)
def test_insert_policy():
collection = MongoUtils("", "local", "oplog.rs").get_mongo_collection()
# 20190128 1548604800
# 20190308 1552035860
oplog_start = bson.timestamp.Timestamp(1552035860, 0)
oplog_end = bson.timestamp.Timestamp(1552035861, 0)
# 5c6e04666ffe5a3ef6a92bdc
document = {"ns": "", "ts": {"$gte": oplog_start, "$lt": oplog_end}}
cursor = collection.find(document,
cursor_type=pymongo.CursorType.TAILABLE_AWAIT,
oplog_replay=True).sort([
(u'$natural', pymongo.ASCENDING)
]).limit(100)
for docu in cursor:
try:
string = "ss" + str(docu)
print string
except Exception as e:
print e
def sync_mongo_data():
"""
同步 mongodb 数据到 mysql
Returns:
"""
try:
# 获取 post 请求的 json 数据
json_data = json.loads(request.get_data())
# mongodb 的 url
ip = json_data['ip']
org_id = json_data['org_id']
# 过滤 oplog.rs 数据
ns = json_data['ns']
# 表
table_name = ns.split(".")[1]
# 获取 mongodb 到表的连接
collection = MongoUtils(ip, "local", "oplog.rs").get_mongo_collection()
# 获取初始时间戳参数
result = get_ts(org_id, table_name)
ts = result["ts"]
inc = result["inc"]
while True:
oplog_start = bson.timestamp.Timestamp(ts, inc)
document = {"ns": ns, "ts": {"$gt": oplog_start}}
cursor = collection.find(
document,
cursor_type=pymongo.CursorType.TAILABLE_AWAIT,
oplog_replay=True).sort([(u'$natural', pymongo.ASCENDING)
]).limit(100)
# 循环遍历游标数据
for docu in cursor:
try:
# 解析游标数据
sync_mongo_service.sync_mongo(docu, org_id, table_name)
bts = docu["ts"]
# 当前时间戳
current_ts = bts.time
# 当前增量
current_inc = bts.inc
ts = current_ts
inc = current_inc
# 更新当时的 时间戳 增量
update_ts(org_id + "_" + table_name, current_ts,
current_inc)
except Exception as e:
logger.error(u" 循环遍历 cursor 的时候出现异常 %s doc : \n %s", e,
docu)
EmailUtils.send_email(u"循环遍历 cursor 的时候出现异常",
str(e) + "\n" + str(docu),
email_conf.receiver)
continue
# 让游标休息 100 ms
time.sleep(0.01)
cursor.close()
except Exception as e:
logger.error(u" 获取 mongo 端数据源报错 %s", e)
EmailUtils.send_email(u"获取 mongo 端数据源报错", str(e), email_conf.receiver)
return jsonify(result)