def delete_specific_kind_pods():
if not request.json or 'service' not in request.json:
abort(400)
mq_control = RabbitMq.control()
dto = {'service': request.json['service'], 'open': mq_control}
print dto
return jsonify(FaultInjector.delete_all_pods_for_service(dto))
def chaos_inject_random():
if not request.json or 'host' not in request.json or 'timeout' not in request.json:
abort(400)
mq_control = RabbitMq.control()
dto = {
'host': request.json['host'],
'timeout': request.json['timeout'],
'open': mq_control
}
return jsonify(FaultInjector.chaos_inject_random(dto))
def chaos_inject_cpu_with_time():
if not request.json or 'host' not in request.json or 'second' not in request.json:
abort(400)
mq_control = RabbitMq.control()
dto_time = {'time': request.json['second']}
dto = {'host': request.json['host'], 'open': mq_control}
scheduler = BackgroundScheduler()
now = datetime.now()
delta = timedelta(
seconds=int(dto_time['time'].encode('raw_unicode_escape')))
scheduler.add_job(func=lambda: FaultInjector.chaos_inject_random(dto),
trigger='date',
next_run_time=(now + delta))
scheduler.start()
time.sleep(delta.total_seconds() + 1)
return "success"
def stop_all_inject_on_all_nodes():
mq_control = RabbitMq.control()
mq_control = {'open': mq_control}
return jsonify(
FaultInjector.stop_all_chaos_inject_on_all_nodes(mq_control))
def stop_specific_inject():
if not request.json or 'tag' not in request.json:
abort(400)
mq_control = RabbitMq.control()
dto = {'tag': request.json['tag'], 'open': mq_control}
return jsonify(FaultInjector.stop_specific_chaos_inject(dto))
def stop_all_inject():
if not request.json or 'host' not in request.json:
abort(400)
mq_control = RabbitMq.control()
dto = {'host': request.json['host'], 'open': mq_control}
return jsonify(FaultInjector.stop_all_on_specific_node(dto))
def get_message():
return jsonify(RabbitMq.consumer())
def mq_delete():
if not request.json or 'queue' not in request.json:
abort(400)
return jsonify(RabbitMq.clear_all_messages(request.json['queue']))
def mq_control():
if not request.json or 'open' not in request.json:
abort(400)
return jsonify(RabbitMq.mq_control(request.json['open']))
def handle_inject_result(inject_type, target_host, target_inject, result,
method_name, mq_control):
"""
处理返回结果
:param inject_type: 注入类型
:param target_host: 目标主机地址
:param target_inject: 注入指令
:param result: 注入结果
:return: 注入结果
:param method_name: 调用者的方法名
"""
if len(result["success"]) > 0:
transform_ip = result["success"].keys()[0]
stdout = result["success"][transform_ip]["stdout"]
result_ = stdout.split(',', 3)[2]
tag_result = result_.split(':', 2)[1].replace('"', '').replace('}', '')
the_has_injected = {
"host":
target_host.encode('unicode_escape').decode('string_escape'),
"inject_type": inject_type,
"tag": tag_result.encode('unicode_escape').decode('string_escape')
}
if inject_type != "destroy":
has_injected.append(the_has_injected)
the_inject_info = {
"position":
inject_type,
"ip":
target_host,
"start_time":
result["success"][transform_ip]["start"],
"cmd":
result["success"][transform_ip]["cmd"],
"cmd_id":
json.loads(result["success"][transform_ip]["stdout"].encode(
'unicode-escape').decode('string_escape'))["result"].encode(
'unicode-escape').decode('string_escape')
}
if inject_type != "destroy":
inject_info.append(the_inject_info)
Logger.log(
'info', 'SUCCESS - Method : ' + method_name + "() - - " +
str(the_inject_info))
if mq_control == 'true':
RabbitMq.connect(the_inject_info)
return result
else:
if len(result["unreachable"]) > 0:
transform_ip = result["unreachable"].keys()[0]
message = result["unreachable"][transform_ip]["msg"]
flag = "UNREACHABLE"
else:
transform_ip = result["failed"].keys()[0]
message = result["failed"][transform_ip]["msg"]
flag = "FAILED"
the_inject_info = {
"ip": target_host,
"position": inject_type,
"cmd": target_inject,
"message": message
}
Logger.log(
"error", flag + " - Method : " + method_name + "() - - " +
str(the_inject_info))
return result
def stop_specific_chaos_inject(dto):
stop_id = dto['tag']
target_host = ''
find = 0
key = 0
for i in range(0, len(inject_info)):
if inject_info[i]['cmd_id'] == stop_id:
target_host = inject_info[i]['ip']
find = 1
key = i
for i in range(0, len(has_injected)):
if target_host == has_injected[i][
'host'] and stop_id == has_injected[i]['tag']:
has_injected.pop(i)
break
if find == 1:
r = Runner()
r.run_ad_hoc(hosts=Spare_hosts[inject_info[key]['ip']],
module='shell',
args='./blade destroy ' + stop_id)
result = r.get_adhoc_result()
if len(result["success"]) > 0:
transform_ip = result["success"].keys()[0]
the_stop_info = {
"position": inject_info[key]["position"],
"ip": Spare_hosts[inject_info[key]["ip"]],
"start_time": result["success"][transform_ip]["start"],
"cmd": result["success"][transform_ip]["cmd"],
}
inject_info.pop(key)
Logger.log(
'info',
'SUCCESS - Method : ' + sys._getframe().f_code.co_name +
"() - - " + str(the_stop_info))
if dto['open'] == 'true':
RabbitMq.connect(the_stop_info)
return result
else:
if len(result["unreachable"]) > 0:
transform_ip = result["unreachable"].keys()[0]
message = result["unreachable"][transform_ip]["msg"]
flag = "UNREACHABLE"
else:
transform_ip = result["failed"].keys()[0]
message = result["failed"][transform_ip]["msg"]
flag = "FAILED"
the_stop_info = {
"position": inject_info[key]["position"],
"ip": Spare_hosts[inject_info[key]["ip"]],
"cmd": "./blade destroy " + stop_id,
"message": message
}
Logger.log(
"error",
flag + " - Method : " + sys._getframe().f_code.co_name +
"() - - " + str(the_stop_info))
return result
else:
the_stop_info = {
"cmd": "./blade destroy " + stop_id,
}
Logger.log(
'error',
'UID NOT FOUND - Method : ' + sys._getframe().f_code.co_name +
"() - - " + str(the_stop_info))
return 'Inject not found'