def verify_volumes(service_name, volume_id, app_name=''): print "===========get service detail to verify volumes============" time1 = time() (code, service_detail) = apicall_claas_service('get_service', service_name, app_name=app_name) if code != 200: return { "success": False, "total": "get service failed, error code is {}.error message is ".format( code, service_detail) } #判断服务的存储卷类型 if volume_id != json.loads(service_detail)['volumes'][0]['volume_id']: return { "success": False, "total": "service volume_id is {},inspect {}".format( json.loads(service_detail)['volumes'][0]['volume_id'], volume_id) } time2 = time() return {"success": True, "total": time2 - time1}
def start_app(name, num): logger.info(" start service") print "start_app()" time1 = time() (code, text) = apicall_claas_service('start_service', name) logger.info(" apicall returns (%d, %s)" % (code, text)) if code < 200 or code >= 300: msg = "Error in calling start API {}:{}".format(code, text) return {"success": False, "total": msg} target = {"status": "Running", "num": num} if not get_events(name, "start"): return {"success": False, "total": "this action do not have events"} isdeploying = is_deploying(name, target, 180, settings.CLAAS_NAMESPACE) time2 = time() if isdeploying: (code, text, obj) = get_service_info(name, 0, settings.CLAAS_NAMESPACE) if 'current_status' in obj: if obj['current_status'].find('Error') >= 0: return { "success": False, "total": "restart {} service status is Error".format(name) } return { "success": False, "total": "Timeout in starting {}".format(name) } logger.info(" ==> Starting service uses {}s".format(time2 - time1)) obj = {"success": True, "total": time2 - time1} return obj
def check_node(service_name, nodetag): print "===========get service detail to check node============" time1 = time() (code, service_detail) = apicall_claas_service('get_service', service_name) if code != 200: return { "success": False, "total": "get service failed, error code is {}.error message is ".format( code, service_detail) } # 判断容器部署在哪台机器 print "service detail is {}".format(service_detail) instances = json.loads(service_detail)['instances'] for i in range(0, len(instances)): if (instances[i]['host_ip'].replace('.', '-') not in nodetag): return { "success": False, "total": "instance in host:{}, nodetag ip:{}".format( instances[i]['host_ip'], nodetag) } time2 = time() return {"success": True, "total": time2 - time1}
def create_service(name, payload, service_detail='Hello', region='STAGING'): delete_app(name, region) print "create claas service" logger.info(region + " Start creating service") time1 = time() print payload (code, text) = apicall_claas_service('create_service', name, payload) logger.info("apicall returns (%d, %s)" % (code, text)) print("apicall returns (%d, %s)" % (code, text)) if code < 200 or code >= 300: msg = "Error in calling create Claas API {}:{}".format(code, text) return {"success": False, "total": msg} if not get_events(name, "create"): return {"success": False, "total": "this action do not have events"} # test if at least 1 instance is running target = {"status": "Running", "text": service_detail} isdeploying = is_deploying(name, target, 200, settings.CLAAS_NAMESPACE) time2 = time() logger.info(region + " is_deploying check use {}s".format(time2 - time1)) if isdeploying: (code, text, obj) = get_service_info(name, 0, settings.CLAAS_NAMESPACE) if 'current_status' in obj: if obj['current_status'].find('Error') >= 0: return { "success": False, "total": "create service {} status is Error".format(name) } if "instance_ports" in json.loads(text) and len( json.loads(text)['instance_ports']): if "endpoint_type" in json.loads(text)['instance_ports'][ 0] and json.loads(text)['instance_ports'][0][ 'endpoint_type'] == "internal-endpoint": (code, html) = get_service_result( json.loads(text)["instance_ports"][0] ["default_domain"]) if code > 300 or code < 200: return {"success": True, "total": time2 - time1} else: return { "success": False, "total": "internal haproxy service should not can visit" } flag = can_visit(target, text) if not flag and obj['current_status'] == 'Running': return { "success": False, "total": "create service {} can not visit".format(name) } msg = "Timeout in creating {} service".format(name) return {"success": False, "total": msg} logger.info(region + " create use {}s".format(time2 - time1)) return {"success": True, "total": time2 - time1}
def get_metrics(name, app_name=''): logger.info(" start get service metrics") print "get_metrics()" time1 = time() (code, service_detail) = apicall_claas_service('get_service', name, app_name=app_name) if code < 200 or code > 300: msg = "Error in get_instances API {}:{}".format(code, service_detail) return {"success": False, "total": msg} cnt = 0 isdeploying = True service_id = json.loads(service_detail).get("uuid", None) text = '' while cnt < 200 and isdeploying: cnt = cnt + 1 tm_end = int(time()) tm_start = tm_end - 1800 isdeploying = False url = settings.API_URL.replace("v1", "v2") + "monitor/" + settings.CLAAS_NAMESPACE + \ "/metrics/query/?q=avg:service.mem.utilization{service_id="+ service_id + "}by{instance_id}&start=" \ + str(tm_start) + "&end=" + str(tm_end) + "&namespace=" + settings.CLAAS_NAMESPACE r = requests.get(url, headers=headers) logger.info(" get service metrics {} {} '{}'".format( name, r.status_code, r.text)) if r.status_code >= 300 or json.loads(r.text) == []: isdeploying = True sleep(2) continue obj = json.loads(r.text) print "metrics is {}".format(obj) if obj[0].get('dps', {}) == {}: isdeploying = True sleep(2) continue if r.status_code >= 300: msg = "Failed in getting metrics , return code:{}, text:{}".format( r.status_code, r.text) return {"success": False, "total": msg} if json.loads(r.text) == []: msg = "Failed in getting metrics , text is null" return {"success": False, "total": msg} if json.loads(r.text)[0]['dps'] == {}: msg = "Failed in getting metrics , text is {}" return {"success": False, "total": msg} time2 = time() logger.info(" ==> Getting metrics uses {}s".format(time2 - time1)) obj = {"success": True, "total": time2 - time1} return obj
def update_service(name, num, size, service_detail='Hello'): print "scale_app()" logger.info(" Start scaling service {}".format(name)) time1 = time() payload = { # "image_tag": "latest", "target_num_instances": num, "instance_size": size } (code, text) = apicall_claas_service('modify_service', name, payload) logger.info(" apicall returns (%d, %s)" % (code, text)) if code < 200 or code >= 300: msg = "Error in calling scale API {}:{}".format(code, text) return {"success": False, "total": msg} if not get_events(name, "update"): return {"success": False, "total": "this action do not have events"} target = { "status": "Running", "num": num, "size": size, "text": service_detail } isdeploying = is_deploying(name, target, 240, settings.CLAAS_NAMESPACE) time2 = time() if isdeploying: (code, text, obj) = get_service_info(name, 0, settings.CLAAS_NAMESPACE) if 'current_status' in obj: if obj['current_status'].find('Error') >= 0: return { "success": False, "total": "update service {} status is Error".format(name) } flag = can_visit(target, text) if not flag and obj['current_status'] == 'Running': return { "success": False, "total": "update service {} can not visit".format(name) } msg = "Timeout in scaling {}".format(name) return {"success": False, "total": msg} logger.info(" ==> Scaling uses {}s".format(time2 - time1)) return {"success": True, "total": time2 - time1}
def delete_app(name, region_name='STAGING'): logger.info(region_name + " start deleting {}".format(name)) time1 = time() (code, text) = apicall_claas_service('delete_service', name) logger.info(region_name + " apicall returns (%d, %s)" % (code, text)) if code < 200 or code >= 300: msg = "Error in calling delete API {}:{}".format(code, text) return {"success": False, "total": msg} time2 = time() logger.info(region_name + " total deletion uses %f seconds" % (time2 - time1)) sleep(10) if not get_events(name, "destroy"): return {"success": False, "total": "this action do not have events"} obj = {"success": True, "total": time2 - time1} return obj
def stop_app(name): logger.info(" Stopping service") print "stop_app()" time1 = time() (code, text) = apicall_claas_service('stop_service', name) logger.info(" apicall returns (%d, %s)" % (code, text)) if code < 200 or code >= 300: msg = "Error in calling stop API {}:{}".format(code, text) return {"success": False, "total": msg} if not get_events(name, "stop"): return {"success": False, "total": "this action do not have events"} target = {"status": "Stopped", "num": 0} isdeploying = is_deploying(name, target, 120, settings.CLAAS_NAMESPACE) time2 = time() if isdeploying: return { "success": False, "total": "Timeout in stopping {}".format(name) } logger.info(" ==> Stopping service uses {}s".format(time2 - time1)) obj = {"success": True, "total": time2 - time1} return obj