def is_verify_ntease_failed(ori_task, remote_addr):
authentication.verify(ori_task.get('username'), PASSWORD_KEY_NTEASE, remote_addr)
md5 = hashlib.md5()
params_key = "%s%s%s%s" % (time.strftime('%Y%m%d', time.localtime()), ori_task.get('username'), CDN_KEY_NTEASE , PASSWORD_KEY_NTEASE)
md5.update(params_key.encode('utf-8'))
logger.debug("base_key:%s verify:%s" % (md5.hexdigest(), ori_task.get('md5')))
return md5.hexdigest() != ori_task.get('md5')
def get_tencent_task(ori_task, remote_addr):
try:
ori_task = json.loads(ori_task)
refresh_task = {"username": USERNAME_TENCENT, "urls": ori_task.get('delete_urls', []), "update_urls":ori_task.get('update_urls', [])}
if is_verify_request_timeout(ori_task.get('request_time')):
return refresh_task, False, '{"ret": -102, "msg": "请求超时"}'
if is_verify_tencent_failed(ori_task):
raise Forbidden("verify failed.")
authentication.verify(USERNAME_TENCENT, PASSWORD_KEY_TENCENT, remote_addr)
return refresh_task, True, '{"ret": 0, "msg": ""}'
except JSONDecodeError:
raise BadRequest("%s 格式不正确。" % ori_task)
except Forbidden:
return refresh_task, False, '{"ret": -101, "msg": "未授权的上报IP:%s"}' % remote_addr
def search_by_condition():
ticket = authentication.verify(
request.args.get('username'), request.args.get('password'), request.remote_addr)
condition = json.loads(request.args.get('condition', {}))
if ticket.get('isSub', False):
query = {'username': ticket.get("name"), "created_time": {}}
else:
query = {'parent': ticket.get("parent"), "created_time": {}}
if condition.get('begin'):
query["created_time"]["$gte"] = datetime.strptime(
condition.get('begin'), '%Y%m%d%H%M%S')
if condition.get('end'):
query["created_time"]["$lte"] = datetime.strptime(
condition.get('end'), '%Y%m%d%H%M%S')
result = []
for refresh_request in query_db_session.request.find(query).limit(100):
totalTime = (refresh_request.get('finish_time', datetime.now())
- refresh_request.get('created_time')).seconds
urlStatus = [get_urlStatus(u) for u in query_db_session.url.find(
{"url": {"$regex": condition.get('url', '')}, "r_id": refresh_request.get('_id')})]
if not urlStatus:
continue
successCount = len([u for u in urlStatus if u.get('code') == 200])
result.append({
'r_id': str(refresh_request.get('_id')),
'status': 'UNKNOWN' if refresh_request.get('status') == 'PROGRESS' else 'SUCCESS',
'createdTime': str(refresh_request.get('created_time')),
'finishedTime': str(refresh_request.get('finish_time')) if refresh_request.get('finish_time') else None,
'successRate': 1 if successCount == len(urlStatus) else float(successCount) / len(urlStatus),
'totalTime': totalTime,
'username': request.args.get('username', ''),
'urlStatus': urlStatus
})
return make_response(json.dumps(result))
def search_by_request():
ticket = authentication.verify(
request.args.get('username'), request.args.get('password'), request.remote_addr)
result = []
if ticket.get('isSub', False):
query = {'username': ticket.get("name")}
else:
query = {'parent': ticket.get("parent")}
for refresh_request in query_db_session.request.find(query).sort('created_time', pymongo.DESCENDING).limit(20):
totalTime = (refresh_request.get('finish_time', datetime.now())
- refresh_request.get('created_time')).seconds
urlStatus = [get_urlStatus(u)
for u in query_db_session.url.find({"r_id": refresh_request.get('_id')})]
successCount = len([u for u in urlStatus if u.get('code') == 200])
result.append({
'r_id': str(refresh_request.get('_id')),
'status': 'UNKNOWN' if refresh_request.get('status') == 'PROGRESS' else 'SUCCESS',
'createdTime': str(refresh_request.get('created_time')),
'finishedTime': str(refresh_request.get('finish_time')) if refresh_request.get('finish_time') else None,
'successRate': 1 if successCount == len(urlStatus) else float(successCount) / len(urlStatus),
'totalTime': totalTime,
'username': request.args.get('username', ''),
'urlStatus': urlStatus
})
return make_response(json.dumps(result))
def refresh_query(rid):
username = request.args.get('username', '')
password = request.args.get('password', '')
ticket = authentication.verify(username, password, request.remote_addr)
results = get_search_result_by_rid(rid, username)
res = make_response(json.dumps(results))
res.headers['Content-Type'] = 'application/json'
return res
def search_autodesk(rid):
username = request.args.get('username', '')
password = request.args.get('password', '')
logger.debug('query autodesk rid:%s, request.args:%s' %
(rid, request.args))
ticket = authentication.verify(username, password, request.remote_addr)
results = get_search_result_by_rid_autodesk_new(rid, username)
return make_response(json.dumps(results))
def search(rid):
username = request.args.get('username', '')
password = request.args.get('password', '')
ticket = authentication.verify(username, password, request.remote_addr)
# try:
# user_list = eval(config.get('task_forward', 'usernames'))
# except Exception, e:
# logger.debug('splitter_new submit error:%s' % traceback.format_exc())
# user_list = []
logger.debug(
"/content/refresh/ search rid:%s, username:%s, remote_addr:%s" %
(rid, username, request.remote_addr))
results = get_search_result_by_rid(rid, username)
return make_response(json.dumps(results))
def snda_search(rid):
'''
盛大查询
'''
username = request.form.get('username', '')
password = request.form.get('password', '')
time = request.form.get('time', '')
p_password = adapters.verify_snda(username, password, time)
if not p_password:
return jsonify({"success": False, "message": "WRONG_PASSWORD"}), 403
#TODO 测试
#p_password = '******'
ticket = authentication.verify(username, p_password, request.remote_addr)
results = get_search_result_by_rid(rid, username)
return make_response(json.dumps(results))
def content_query_failed():
"""
receive data type json:
{
'username': xxxx,
'password':xxxxx,
'start_time':'2017-07-02 18:30:41',
'end_time': '2012-07-02 18:50:51'
}
require
start_time > end_time
end_time - start_time <= 3600 seconds
Returns:
json:
{
'msg': 'ok',
'result': [
{"username": xxxx, 'created_time': xxxxx, "url":xxxx, "isdir": True or False}
]
}
"""
try:
task = json.loads(request.data)
except Exception:
logger.info('content query_failed error:%s, remote_addr' %
(traceback.format_exc(), request.remote_addr))
return json.dumps({'msg': 'error', 'info': 'data is not json'})
logger.debug(
'/content/query_failed username:%s, start_time:%s, end_time:%s' %
(task.get('username'), task.get('start_time'), task.get('end_time')))
ticket = authentication.verify(task.get('username'), task.get('password'),
request.remote_addr)
try:
start_time = task.get('start_time')
end_time = task.get('end_time')
if not start_time:
return json.dumps({'msg': 'error', 'info': 'not have start_time'})
if not end_time:
return json.dumps({'msg': 'error', 'info': 'not have end_time'})
start_time = datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S')
end_time = datetime.strptime(end_time, '%Y-%m-%d %H:%M:%S')
# start_timestamp = time.mktime(start_time.timetuple())
# end_timestamp = time.mktime(end_time.timetuple())
# if (end_timestamp - start_timestamp > 3600):
# logger.info('the interval is more than an hour')
# return json.dumps({'msg': 'error', 'info': 'the interval is more than an hour'})
except Exception:
logger.info('content query_failed get start_time end_time error:%s' %
traceback.format_exc())
return json.dumps({
'msg': 'error',
'info': 'start_time or end_time format error'
})
try:
dev_failed_rate = float(
config.get('failed_query_rate', 'dev_failed_rate'))
except Exception:
logger.debug('content query_failed error:%s' % traceback.format_exc())
dev_failed_rate = 0.05
try:
failed_rate = float(config.get('failed_query_rate', 'failed_rate'))
except Exception:
logger.debug('content query_failed error:%s' % traceback.format_exc())
failed_rate = 0.9
# task_new = {'username': task.get('username'), 'start_time': start_time, 'end_time': end_time,
# 'dev_failed_rate': dev_failed_rate, 'failed_rate': failed_rate}
try:
failed_task_result = query_db_session.failed_task.find(
{
"username": task.get('username'),
'created_time': {
'$gte': start_time,
'$lt': end_time
}
}, {
'username': 1,
'created_time': 1,
'url': 1,
'isdir': 1,
'failed_rate': 1
}).batch_size(50)
except Exception:
logger.debug('query failed_task error:%s' % traceback.format_exc())
return json.dumps({'msg': 'error', 'info': 'get data error'})
list_result = []
for result in failed_task_result:
if result.get('failed_rate') > dev_failed_rate:
del result['_id']
del result['failed_rate']
result['created_time'] = result['created_time'].strftime(
'%Y-%m-%d %H:%M:%S')
list_result.append(result)
list_result = list_result[:int(len(list_result) * failed_rate)]
# result = get_failed_result(task_new)
return json.dumps({"msg": 'ok', 'result': list_result})
{
'msg': 'ok',
'result': [
{"username": xxxx, 'created_time': xxxxx, "url":xxxx, "isdir": True or False}
]
}
"""
try:
task = json.loads(request.data)
except Exception, e:
logger.info('content query_failed error:%s, remote_addr' % (traceback.format_exc(e), request.remote_addr))
return json.dumps({'msg': 'error', 'info':'data is not json'})
logger.debug('/content/query_failed username:%s, start_time:%s, end_time:%s' % (task.get('username'),
task.get('start_time'), task.get('end_time')))
ticket = authentication.verify(task.get('username'), task.get('password'), request.remote_addr)
try:
start_time = task.get('start_time')
end_time = task.get('end_time')
if not start_time:
return json.dumps({'msg': 'error', 'info': 'not have start_time'})
if not end_time:
return json.dumps({'msg': 'error', 'info': 'not have end_time'})
start_time = datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S')
end_time = datetime.strptime(end_time, '%Y-%m-%d %H:%M:%S')
# start_timestamp = time.mktime(start_time.timetuple())
# end_timestamp = time.mktime(end_time.timetuple())
# if (end_timestamp - start_timestamp > 3600):
# logger.info('the interval is more than an hour')
# return json.dumps({'msg': 'error', 'info': 'the interval is more than an hour'})
except Exception, e: