def terminate(service_instance_id):
LOG.debug("[lifecycle.operations] [terminate] service_instance_id=" +
service_instance_id)
try:
# 1. get service_instance object
service_instance = data_adapter.get_service_instance(
service_instance_id)
if service_instance is None or service_instance == -1:
return common.gen_response(
500, 'Error getting service instance object',
'service_instance_id', service_instance_id)
# 2. updates service instance status
service_instance['status'] = STATUS_TERMINATING
data_adapter.update_service_instance(service_instance_id,
service_instance)
# 3.1 service instance status = Stopped => stop and terminate
if service_instance['status'] != "Stopped":
return operation_service(service_instance_id,
OPERATION_STOP_TERMINATE)
# 3.2 terminate
else:
return operation_service(service_instance_id, OPERATION_TERMINATE)
except:
LOG.exception('[lifecycle.operations] [terminate] Exception')
return common.gen_response(500, 'Exception', 'service_instance_id',
service_instance_id)
def deploy_service_agent(service, service_instance, agent):
LOG.debug(
"[lifecycle.modules.apps.docker.adapter] [deploy_service_agent] " +
str(service) + ", " + str(agent))
try:
# docker-compose
if service['exec_type'] == SERVICE_DOCKER_COMPOSE:
return __deploy_docker_compose(service, agent)
# docker
elif service['exec_type'] == SERVICE_DOCKER:
return __deploy_docker_image(service, agent)
# compss (docker)
elif service['exec_type'] == SERVICE_COMPSS:
return __deploy_docker_compss(service, service_instance, agent)
# not defined
else:
LOG.warning(
"[lifecycle.modules.apps.docker.adapter] [deploy_service_agent] ["
+ service['exec_type'] + "] not defined")
return common.gen_response(
500, 'Exception: type not defined: deploy_service_agent()',
'agent', str(agent), 'service', str(service))
except:
LOG.exception(
'[lifecycle.modules.apps.docker.adapter] [deploy_service_agent] Exception'
)
return common.gen_response(500, 'Exception: deploy_service_agent()',
'agent', str(agent), 'service',
str(service))
def getServiceInstance(service_instance_id):
if service_instance_id == "all":
LOG.debug("[lifecycle.app_funcs] [getServiceInstance] Call to 'get all' ")
try:
obj_response_cimi = common.ResponseCIMI()
service_instances = data_adapter.get_all_service_instances(obj_response_cimi)
if not service_instances is None:
return common.gen_response_ok('Service instances content', 'service_instances', service_instances, "Msg", obj_response_cimi.msj)
else:
return common.gen_response(500, "Error in 'get_all' function", "Error_Msg", obj_response_cimi.msj)
except:
LOG.exception("[lifecycle.app_funcs] [getServiceInstance] Exception. Returning error 500 ...")
return common.gen_response(500, 'Exception', 'get_all', "-")
else:
LOG.debug("[lifecycle.app_funcs] [getServiceInstance] " + service_instance_id)
try:
obj_response_cimi = common.ResponseCIMI()
service_instance = data_adapter.get_service_instance(service_instance_id, obj_response_cimi)
if not service_instance is None and service_instance != -1:
return common.gen_response_ok('Service instance content', 'service_instance_id', service_instance_id, 'service_instance', service_instance)
elif service_instance == -1:
return common.gen_response_ok('Service instance not found', 'service_instance_id', service_instance_id, 'service_instance', {})
else:
return common.gen_response(500, "Error in 'get' function", "service_instance_id", service_instance_id, "Error_Msg", obj_response_cimi.msj)
except:
LOG.exception('[lifecycle.app_funcs] [getServiceInstance] Exception. Returning error 500 ...')
return common.gen_response(500, 'Exception', 'service_instance_id', service_instance_id)
def getServiceInstanceReport(service_instance_id):
LOG.debug("[lifecycle.app_funcs] [getServiceInstanceReport] " + service_instance_id)
try:
service_instance_report = data_adapter.get_service_instance_report(service_instance_id)
if not service_instance_report is None and service_instance_report != -1:
return common.gen_response_ok('Service Operation Report content', 'service_instance_id', service_instance_id, 'report', service_instance_report)
else:
LOG.warning("[lifecycle.app_funcs] [getServiceInstanceReport] service_instance_report is None")
return common.gen_response(500, "service_instance_report is None", "service_instance_id", service_instance_id)
except:
LOG.exception('[lifecycle.app_funcs] [getServiceInstanceReport] Exception')
return common.gen_response(500, 'Exception', 'service_instance_id', service_instance_id)
def __deploy_docker_compss(service, service_instance, agent):
LOG.debug(
"[lifecycle.modules.apps.docker.adapter] [__deploy_docker_compss] " +
str(service) + ", " + str(agent))
try:
# service image / location. Examples: "mf2c/compss-agent:latest", "mf2c/compss-mf2c:1.0"
service_image = service['exec']
# port(s); COMPSs exposes port 8080
ports = agent['ports']
# ip
ip = agent['url']
# ip_leader
ip_leader = service_instance[
'device_ip'] # TODO create a 'exec_device_ip'
container1, agents_ports = docker_client.create_docker_compss_container(
service_image, ip, ports, ip_leader)
if container1 is not None:
db.SERVICE_INSTANCES_LIST.append({
"type": SERVICE_COMPSS,
"container_main": container1['Id'],
"container_2": "-"
})
LOG.debug(
"[lifecycle.modules.apps.docker.adapter] [__deploy_docker_compss] container: "
+ str(container1))
# update agent properties
agent['container_id'] = container1['Id']
agent['agent_param'] = "-"
agent['ports'] = agents_ports
agent['status'] = STATUS_WAITING
return common.gen_response_ok('Deploy service in agent', 'agent',
str(agent), 'service', str(service))
else:
LOG.error(
"[lifecycle.modules.apps.docker.adapter] [__deploy_docker_compss] Could not connect to DOCKER API"
)
agent['status'] = STATUS_ERROR
return common.gen_response(500,
'Error when connecting to DOCKER API',
'agent', str(agent), 'service',
str(service))
except:
LOG.exception(
'[lifecycle.modules.apps.docker.adapter] [__deploy_docker_compss] Exception'
)
return common.gen_response(500, 'Exception: __deploy_docker_compss()',
'agent', str(agent), 'service',
str(service))
def __deploy_docker_image(service, agent):
LOG.debug(
"[lifecycle.modules.apps.docker.adapter] [__deploy_docker_image] " +
str(service) + ", " + str(agent))
try:
# service image / location. Examples: "yeasy/simple-web"
service_image = service['exec']
# service_name examples: "simple-web-test"
service_name = service['name'] + "-" + str(uuid.uuid4())
# command. Docker examples: "/bin/sh -c 'python index.py'"
service_command = ""
# port(s)
ports = agent['ports']
container1 = docker_client.create_docker_container(
service_image, service_name, service_command, ports)
if container1 is not None:
db.SERVICE_INSTANCES_LIST.append({
"type": SERVICE_DOCKER,
"container_main": container1['Id'],
"container_2": "-"
})
LOG.debug(
"[lifecycle.modules.apps.docker.adapter] [__deploy_docker_image] container: "
+ str(container1))
# update agent properties
agent['container_id'] = container1['Id']
agent['agent_param'] = "-"
agent['status'] = STATUS_WAITING
return common.gen_response_ok('Deploy service in agent', 'agent',
str(agent), 'service', str(service))
else:
LOG.error(
"[lifecycle.modules.apps.docker.adapter] [__deploy_docker_image] Could not connect to DOCKER API"
)
agent['status'] = STATUS_ERROR
return common.gen_response(500,
'Error when connecting to DOCKER API',
'agent', str(agent), 'service',
str(service))
except:
LOG.exception(
'[lifecycle.modules.apps.docker.adapter] [__deploy_docker_image] Exception'
)
return common.gen_response(500, 'Exception: __deploy_docker_image()',
'agent', str(agent), 'service',
str(service))
def terminate_all():
if data_adapter.del_all_service_instances():
# TODO stop and terminate containers
return common.gen_response_ok('Terminate all services', 'result',
'True')
else:
return common.gen_response(500, 'Exception', 'result', 'False')
def handle_qos_notification(notification):
LOG.info(
"########################################################################################"
)
LOG.info("######## QoS ENFORCEMENT")
try:
global QoS_SERVICE_INSTANCES_LIST
LOG.info(
"[lifecycle.events.handler_qos] [handle_qos_notification] service_instance_id: notification: "
+ str(notification))
if not 'num_agents' in notification or not 'service_instance_id' in notification:
LOG.error(
"[lifecycle.events.handler_qos] [handle_qos_notification] 'num_agents' / 'service_instance_id' parameters not found in notification!"
)
return common.gen_response(
406, 'Error',
'parameter num_agents / service_instance_id not found in qos notification',
str(notification))
# handle notification
if __check_service_instance_id(notification['service_instance_id']):
LOG.info(
"[lifecycle.events.handler_qos] [handle_qos_notification] Processing request..."
)
QoS_SERVICE_INSTANCES_LIST.append(
notification['service_instance_id'])
t = threading.Thread(target=thr, args=(notification, ))
t.start()
return common.gen_response_ok(
'QoS Notification is being processed...', 'notification',
str(notification))
LOG.info(
"[lifecycle.events.handler_qos] [handle_qos_notification] Request not processed."
)
return common.gen_response_ok(
"QoS Notification was not processed: List of current service instances being processed: "
+ str(QoS_SERVICE_INSTANCES_LIST), "notification",
str(notification))
except:
LOG.exception(
'[lifecycle.events.handler_qos] [handle_qos_notification] Exception'
)
return common.gen_response(500, 'Exception', 'notification',
str(notification))
def stop(service, agent):
LOG.info("[lifecycle.int_operations] [stop] " + str(service) +
", agent: " + str(agent))
try:
status = apps_adapter.stop_service_agent(service, agent)
return common.gen_response_ok('Stop service', 'agent', str(agent),
'status', status)
except:
LOG.exception('[lifecycle.int_operations] [stop] Exception')
return common.gen_response(500, 'Exception', 'agent', str(agent))
def terminate(service, agent):
LOG.info("[lifecycle.int_operations] [terminate] " + str(service) +
", agent: " + str(agent))
try:
status = apps_adapter.terminate_service_agent(service, agent)
return common.gen_response_ok('Terminate service', 'agent', str(agent),
'status', status)
except:
LOG.exception('[lifecycle.int_operations] [terminate] Exception')
return common.gen_response(500, 'Exception', 'agent', str(agent))
def operation_service(service_instance_id, operation):
LOG.debug("[lifecycle.operations] [operation_service] operation=" +
operation + ", service_instance_id=" + service_instance_id)
try:
# 1. get service_instance object
service_instance = data_adapter.get_service_instance(
service_instance_id)
if service_instance is None or service_instance == -1:
return common.gen_response(
500, 'Error getting service instance object',
'service_instance_id', service_instance_id)
if operation == OPERATION_STOP_TERMINATE:
service_instance['status'] = STATUS_TERMINATING
# execute thread
t = threading.Thread(target=__thr_stop_terminate_service,
args=(service_instance, ))
t.start()
else:
# submit operation thread
if operation == OPERATION_START:
service_instance['status'] = STATUS_STARTING
elif operation == OPERATION_STOP:
service_instance['status'] = STATUS_STOPPING
elif operation == OPERATION_TERMINATE:
service_instance['status'] = STATUS_TERMINATING
# execute thread
t = threading.Thread(target=__thr_operation_service,
args=(
service_instance,
operation,
))
t.start()
# response
return common.gen_response_ok(
"Service " + operation + " operation is being processed ...",
"service_instance", service_instance)
except:
LOG.exception('[lifecycle.operations] [operation_service] Exception')
return common.gen_response(500, 'Exception', 'service_instance_id',
service_instance_id)
def start_job(body, service_instance_id):
LOG.info(
"########################################################################################"
)
LOG.info("######## JOBS: START JOB")
LOG.debug("[lifecycle.operations] [start_job] body=" + str(body))
LOG.debug("[lifecycle.operations] [start_job] service_instance_id=" +
service_instance_id)
try:
# service instance
service_instance = data_adapter.get_service_instance(
service_instance_id)
if service_instance is None or service_instance == -1:
return common.gen_response(
500, 'Error getting service instance object',
'service_instance_id', service_instance_id)
LOG.debug("[lifecycle.operations] [start_job] service_instance=" +
str(service_instance))
# start job in agent(s)
if len(service_instance['agents']) == 1:
res = apps_adapter.start_job_compss(service_instance, body)
elif len(service_instance['agents']) >= 2:
res = apps_adapter.start_job_compss_multiple_agents(
service_instance, body)
else:
LOG.warning(
"[lifecycle.operations] [start_job] Execution supported in only 1 or more agents! agents size="
+ str(len(service_instance['agents'])))
res = None
if res:
return common.gen_response_ok('Start job', 'service_id',
service_instance_id, 'res', res)
else:
return common.gen_response(500, 'Error when starting job',
'service_instance',
str(service_instance))
except:
LOG.exception('[lifecycle.operations] [start_job] Exception')
return common.gen_response(500, 'Exception', 'data', str(body))
def handle_warning(warning):
try:
LOG.info("[lifecycle.events.handler_um] [handle_warning] warning: " + str(warning))
# handle notification
t = threading.Thread(target=thr, args=(warning,))
t.start()
return common.gen_response_ok('UM Warning is being processed...', 'warning', str(warning))
except:
LOG.exception('[lifecycle.events.handler_um] [handle_warning] Exception')
return common.gen_response(500, 'Exception', 'warning', str(warning))
def stop_service_agent(service, agent):
LOG.debug("[lifecycle.modules.apps.swarm.compose.adapter] [stop_service_agent] " + str(service) + ", " + str(agent))
try:
# 1. Remove stack: docker stack rm <STACK_NAME>
stack_id = service['name'].strip().lower() # + str(uuid.uuid4())
LOG.debug("[lifecycle.modules.apps.swarm.compose.adapter] [stop_service_agent] 'cd " + config.dic['WORKING_DIR_VOLUME'] + "'")
resOs = os.system("cd " + config.dic['WORKING_DIR_VOLUME'])
LOG.debug("[lifecycle.modules.apps.swarm.compose.adapter] [stop_service_agent] resOs: " + str(resOs))
LOG.debug("[lifecycle.modules.apps.swarm.compose.adapter] [stop_service_agent] 'docker stack rm " + stack_id + "'")
resOs = os.system("docker stack rm " + stack_id)
LOG.debug("[lifecycle.modules.apps.swarm.compose.adapter] [stop_service_agent] resOs: " + str(resOs))
# 3.
if service['name'].strip().lower() is not None:
db.SERVICE_INSTANCES_LIST.append({
"type": SERVICE_DOCKER_COMPOSE_SWARM,
"container_main": stack_id,
"container_2": "-"
})
LOG.debug("[lifecycle.modules.apps.swarm.compose.adapter] [stop_service_agent] container: " + stack_id)
# update agent properties
agent['container_id'] = stack_id
agent['agent_param'] = "-"
agent['status'] = STATUS_STOPPING
return common.gen_response_ok('Deploy docker-compose-swarm service in agent', 'agent', str(agent),
'service', str(service))
else:
LOG.error("[lifecycle.modules.apps.swarm.compose.adapter] [stop_service_agent] Could not connect to DOCKER API")
agent['status'] = STATUS_ERROR
return common.gen_response(500, 'Error when connecting to DOCKER API', 'agent', str(agent), 'service',
str(service))
except:
LOG.exception('[lifecycle.modules.apps.swarm.compose.adapter] [stop_service_agent] Exception')
return common.gen_response(500, 'Exception: deploy_docker_compose()', 'agent', str(agent), 'service',
str(service))
def handle_dinamicity(notification):
try:
LOG.info(
"[lifecycle.events.handler_dinamicity] [handle_dinamicity] notification: "
+ str(notification))
# handle notification
t = threading.Thread(target=thr, args=(notification, ))
t.start()
return common.gen_response_ok('UM notification is being processed...',
'notification', str(notification))
except:
LOG.exception(
'[lifecycle.events.handler_dinamicity] [handle_dinamicity] Exception'
)
return common.gen_response(500, 'Exception', 'notification',
str(notification))
def handle_sla_notification(notification):
try:
LOG.info(
"[lifecycle.events.handler_sla] [handle_sla_notification] service_instance_id: notification: "
+ str(notification))
# handle notification
t = threading.Thread(target=thr, args=(notification, ))
t.start()
return common.gen_response_ok('SLA Notification has been processed',
'notification', str(notification))
except:
LOG.error(
'[lifecycle.events.handler_sla] [handle_sla_notification] Exception'
)
return common.gen_response(500, 'Exception', 'notification',
str(notification))
def deploy_service(service, agent):
LOG.debug("[lifecycle.modules.apps.kubernetes.adapter] [deploy_service] " +
str(service) + ", " + str(agent))
try:
'''
:oauth-token (config/get-openshift-oauth-token)
:insecure? true
:content-type :json
:accept :json
requests.post:
:param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
requests.get('https://api.github.com/user', auth=HTTPBasicAuth('user', 'pass'))
'''
headers = {'content-type': 'application/json'}
# deployment
r = requests.post(config.dic['K8S_PROXY'] +
"/apis/apps/v1/namespaces/" +
config.dic['K8S_NAMESPACE'] + "/deployments",
headers=headers,
json=shemas.genDeploymentDict(service['name']),
verify=config.dic['VERIFY_SSL'])
# service
r = requests.post(config.dic['K8S_PROXY'] + "/api/v1/namespaces/" +
config.dic['K8S_NAMESPACE'] + "/services",
headers=headers,
json=shemas.genServiceDict(service['name']),
verify=config.dic['VERIFY_SSL'])
# app name is stored in 'container_id' field
agent['container_id'] = service['name']
agent['status'] = STATUS_WAITING
except:
LOG.exception(
'[lifecycle.modules.apps.kubernetes.adapter] [deploy_service] Exception'
)
return common.gen_response(500, 'Exception: deploy_service()', 'agent',
str(agent), 'service', str(service))
def create_docker_service(service_image, service_name, service_command, prts,
replicas, service, agent):
LOG.debug(
"[lifecycle.modules.apps.swarm.adapter] [create_docker_service] [service_name="
+ service_name + "], "
"[service_command=" + service_command + "], [service_image=" +
service_image + "], [ports=" + str(prts) + "]")
# connect to docker api
lclient = get_client_agent_docker()
try:
if lclient:
# check if image already exists in agent
l_images = lclient.images(name=service_image)
# if not, download image
if not l_images or len(l_images) == 0:
LOG.debug(
"[lifecycle.modules.apps.swarm.adapter] [create_docker_service] call to 'import_image' ["
+ service_image + "] ...")
lclient.import_image(
image=service_image
) # (tag="latest", image="ubuntu") # (tag="latest", image="ubuntu")
LOG.debug(
"[lifecycle.modules.apps.swarm.adapter] [create_docker_service] Creating service ..."
)
prts_list = list(prts)
ports_list = create_ports_dict(prts)
LOG.debug(
"[lifecycle.modules.apps.swarm.adapter] [create_docker_service] ports_list: "
+ str(ports_list))
LOG.debug(
"[lifecycle.modules.apps.swarm.adapter] [create_docker_service] prts: "
+ str(prts))
LOG.debug(
"[lifecycle.modules.apps.swarm.adapter] [create_docker_service] prts_list: "
+ str(prts_list))
# ContainerSpec:
# (self, image, command=None, args=None, hostname=None, env=None,
# workdir=None, user=None, labels=None, mounts=None,
# stop_grace_period=None, secrets=None, tty=None, groups=None,
# open_stdin=None, read_only=None, stop_signal=None,
# healthcheck=None, hosts=None, dns_config=None, configs=None,
# privileges=None, isolation=None)
container_spec = docker.types.ContainerSpec(
image=service_image, tty=True, command=service_command)
LOG.debug(
"[lifecycle.modules.apps.swarm.adapter] [create_docker_service] container_spec: "
+ str(container_spec))
# TaskTemplate:
# (self, container_spec, resources=None, restart_policy=None,
# placement=None, log_driver=None, networks=None,
# force_update=None)
task_tmpl = docker.types.TaskTemplate(container_spec,
restart_policy=None)
LOG.debug(
"[lifecycle.modules.apps.swarm.adapter] [create_docker_service] task_tmpl: "
+ str(task_tmpl))
# ServiceMode:
# (self, mode, replicas=None)
serv_mode = docker.types.ServiceMode(mode="replicated",
replicas=replicas)
LOG.debug(
"[lifecycle.modules.apps.swarm.adapter] [create_docker_service] serv_mode: "
+ str(serv_mode))
# create a new service (DOCKER SWARM):
# create_service(task_template, name=None, labels=None, mode=None, update_config=None, networks=None,
# endpoint_config=None, endpoint_spec=None, rollback_config=None)
res = lclient.create_service(
task_tmpl,
name=service_name,
mode=serv_mode,
endpoint_spec={
'Ports': ports_list
#[{'Protocol': 'tcp', 'PublishedPort': 8013, 'TargetPort': 80}]
}) # published_port: target_port
LOG.debug(
"[lifecycle.modules.apps.swarm.adapter] [create_docker_service] res: "
+ str(res))
# update agent properties
# service ID is stored in 'container_id' field
agent['container_id'] = res['ID'] #container1['Id']
agent['status'] = STATUS_WAITING
return common.gen_response_ok(
'Deploy service in agent (Docker Swarm)', 'agent', str(agent),
'service', str(service))
else:
LOG.error(
"[lifecycle.modules.apps.swarm.adapter] [create_docker_service] Could not connect to DOCKER API"
)
agent['status'] = STATUS_ERROR
return common.gen_response(500,
'Error when connecting to DOCKER API',
'agent', str(agent), 'service',
str(service))
except:
LOG.exception(
'[lifecycle.modules.apps.swarm.adapter] [create_docker_service] Exception'
)
return common.gen_response(500,
'Exception: deploy_docker_image()', 'agent',
str(agent), 'service', str(service))
def isUMAlive():
if not connector.user_management_check_avialability() is None:
return Response(json.dumps({'res': 'ok'}), status=200, mimetype='application/json')
else:
return common.gen_response(500, 'Exception', 'isUMAlive', "user_management_check_avialability returned None")
def __forward_submit_request_to_leader(service, user_id, sla_template_id,
service_instance_id):
LOG.debug(
"[lifecycle.deployment] [forward_submit_request_to_leader] Forwarding service ["
+ service['name'] + "] deployment to leader" + " (user_id: " +
user_id + ", sla_template_id: " + sla_template_id +
", service_instance_id: " + service_instance_id + ") ...")
# leader IP and current agent IP
leader_ip = data_adapter.get_leader_ip()
my_ip = data_adapter.get_my_ip()
LOG.debug(
"[lifecycle.deployment] [forward_submit_request_to_leader] leader_ip="
+ str(leader_ip) + ", my_ip=" + str(my_ip))
# if 'my_ip' == 'leader_ip', cannot forward => cancel operation
if leader_ip is not None and my_ip is not None and my_ip == leader_ip:
LOG.warning(
"[lifecycle.deployment] [forward_submit_request_to_leader] LM cannot forward request to leader. Reason: 'my_ip' == 'leader_ip'"
)
# delete service_instance
if service_instance_id is not None:
LOG.debug(
"[lifecycle.deployment] [forward_submit_request_to_leader] Deleting service instance ["
+ service_instance_id + "] ...")
service_instance_id = service_instance_id.replace(
'service-instance/', '')
operations.terminate(service_instance_id)
# send error
return common.gen_response(
500,
"Not enough resources found in current cluster => Error when forwarding request to Leader: my_ip == leader_ip !",
"service_instance_id", service_instance_id, "actions",
"1) Service instance deleted, 2) Request not completed")
# if leader_ip =/= current agent IP, forward request to leader
elif leader_ip is not None and connector.lifecycle_parent_deploy(
leader_ip, service['id'], user_id, sla_template_id,
service_instance_id):
LOG.debug(
"[lifecycle.deployment] [forward_submit_request_to_leader] Checking service_instance before forwarding request to leader..."
)
# check if service instance was created
service_instance = None
if not service_instance_id:
service_instance = data_adapter.create_service_instance(
service, [], user_id, "")
service_instance_id = service_instance['id']
LOG.debug(
"[lifecycle.deployment] [forward_submit_request_to_leader] Request forwarded to leader."
)
return common.gen_response_ok(
"Not enough resources found in current cluster => Request forwarded to Leader. Service deployment operation is being processed...",
"service_instance_id", service_instance_id, "service_instance",
service_instance)
# error
else:
LOG.error(
"[lifecycle.deployment] [forward_submit_request_to_leader] LM could not forward request to leader: 'leader_ip' is None / error connecting to leader agent"
)
# delete service_instance
if service_instance_id is not None:
LOG.debug(
"[lifecycle.deployment] [forward_submit_request_to_leader] Deleting service instance ["
+ service_instance_id + "] ...")
service_instance_id = service_instance_id.replace(
'service-instance/', '')
operations.terminate(service_instance_id)
# send error
return common.gen_response(
500,
"Not enough resources found in current cluster => Error when forwarding request to Leader!",
"service_instance_id", service_instance_id, "actions",
"1) Service instance deleted, 2) Request not completed")
def deploy_service_agent(service, agent):
LOG.debug("[lifecycle.modules.apps.swarm.compose.adapter] [deploy_service_agent] " + str(service) + ", " + str(agent))
try:
# 1. Download docker-compose.yml file
location = service['exec']
LOG.debug("[lifecycle.modules.apps.swarm.compose.adapter] [deploy_service_agent] Getting docker-compose.yml from " + location + " ...")
# remove previous files
LOG.debug("[lifecycle.modules.apps.swarm.compose.adapter] [deploy_service_agent] Cleaning folder [" + config.dic['WORKING_DIR_VOLUME'] + "] ...")
try:
LOG.debug("[lifecycle.modules.apps.swarm.compose.adapter] [deploy_service_agent] Checking files from [" + config.dic['WORKING_DIR_VOLUME'] + "] ...")
if os.path.exists(config.dic['WORKING_DIR_VOLUME'] + "/docker-compose.yml"):
os.remove(config.dic['WORKING_DIR_VOLUME'] + "/docker-compose.yml")
LOG.debug("[lifecycle.modules.apps.swarm.compose.adapter] [deploy_service_agent] 'docker-compose.yml' file removed!")
else:
LOG.debug("[lifecycle.modules.apps.swarm.compose.adapter] [deploy_service_agent] Folder has no 'docker-compose.yml' file")
except:
LOG.warning("[lifecycle.modules.apps.docker.adapter] [__deploy_docker_compose] Error when removing file: " + config.dic['WORKING_DIR_VOLUME'] + "/docker-compose.yml")
# copy / download docker-compose.yml
LOG.debug("[lifecycle.modules.apps.swarm.compose.adapter] [deploy_service_agent] Copying [deployment] 'docker-compose.yml' to [" + config.dic['WORKING_DIR_VOLUME'] + "] ...")
try:
res, _ = urequest.urlretrieve(location, config.dic['WORKING_DIR_VOLUME'] + "/docker-compose.yml")
LOG.debug("[lifecycle.modules.apps.swarm.compose.adapter] [deploy_service_agent] res: " + str(res))
except:
LOG.exception("[lifecycle.modules.apps.swarm.compose.adapter] [deploy_service_agent] Error when copying file to: " + config.dic['WORKING_DIR_VOLUME'] + "/docker-compose.yml")
return common.gen_response(500,
"Exception: deploy_docker_compose(): Error when copying file to WORKING_DIR_VOLUME",
"agent", str(agent),
"WORKING_DIR_VOLUME", config.dic['WORKING_DIR_VOLUME'])
# 2. Deploy container: docker stack deploy -c <DOCKER_COMPOSE_FILE_PATH> <STACK_NAME>
stack_id = service['name'].strip().lower() # + str(uuid.uuid4())
LOG.debug("[lifecycle.modules.apps.swarm.compose.adapter] [deploy_service_agent] 'cd " + config.dic['WORKING_DIR_VOLUME'] + "'")
resOs = os.system("cd " + config.dic['WORKING_DIR_VOLUME'])
LOG.debug("[lifecycle.modules.apps.swarm.compose.adapter] [deploy_service_agent] resOs: " + str(resOs))
LOG.debug("[lifecycle.modules.apps.swarm.compose.adapter] [deploy_service_agent] 'docker stack deploy -c " + config.dic['WORKING_DIR_VOLUME'] + "/docker-compose.yml " + stack_id + "'")
resOs = os.system("docker stack deploy -c " + config.dic['WORKING_DIR_VOLUME'] + "/docker-compose.yml " + stack_id)
LOG.debug("[lifecycle.modules.apps.swarm.compose.adapter] [deploy_service_agent] resOs: " + str(resOs))
# 3.
if service['name'].strip().lower() is not None:
db.SERVICE_INSTANCES_LIST.append({
"type": SERVICE_DOCKER_COMPOSE_SWARM,
"container_main": stack_id,
"container_2": "-"
})
LOG.debug("[lifecycle.modules.apps.swarm.compose.adapter] [deploy_service_agent] container: " + stack_id)
# update agent properties
agent['container_id'] = stack_id
agent['agent_param'] = "-"
agent['status'] = STATUS_WAITING
return common.gen_response_ok('Deploy docker-compose-swarm service in agent', 'agent', str(agent), 'service', str(service))
else:
LOG.error("[lifecycle.modules.apps.swarm.compose.adapter] [deploy_service_agent] Could not connect to DOCKER API")
agent['status'] = STATUS_ERROR
return common.gen_response(500, 'Error when connecting to DOCKER API', 'agent', str(agent), 'service',
str(service))
except:
LOG.exception('[lifecycle.modules.apps.swarm.compose.adapter] [deploy_service_agent] Exception')
return common.gen_response(500, 'Exception: deploy_docker_compose()', 'agent', str(agent), 'service', str(service))
def submit_service_in_agents(service,
user_id,
service_instance_id,
sla_template_id,
agents_list,
check_service=False):
if service_instance_id is not None:
LOG.debug(
"[lifecycle.deployment] [submit_service_in_agents] Deploying service [name="
+ service['name'] + ", service_instance_id=" +
service_instance_id + ", user_id=" + user_id +
", sla_template_id=" + sla_template_id + ", agents_list=" +
str(agents_list) + "] in agents ...")
else:
LOG.debug(
"[lifecycle.deployment] [submit_service_in_agents] Deploying service [name="
+ service['name'] + ", user_id=" + user_id + ", sla_template_id=" +
sla_template_id + ", agents_list=" + str(agents_list) +
"] in agents ...")
try:
# 1. check parameters content
if check_service and not check_service_content(service):
return common.gen_response(
500, 'field(s) category/exec/exec_type not found', 'service',
str(service))
# 2.1 FORWARD REQUEST
if service_instance_id is not None and service_instance_id != "":
LOG.info(
"[lifecycle.deployment] [submit_service_in_agents] 'FORWARD REQUEST' ..."
)
LOG.info(
"[lifecycle.deployment] [submit_service_in_agents] Getting service instance ["
+ service_instance_id + "] from cimi ... ")
service_instance = data_adapter.get_service_instance(
service_instance_id)
LOG.info(
"[lifecycle.deployment] [submit_service_in_agents] service_instance="
+ str(service_instance))
# try to get the service-instance created in child agent (x3)
if service_instance is None or service_instance == -1:
LOG.info(
"[lifecycle.deployment] [submit_service_in_agents] Waiting 20s (for synchronization) to try again ..."
)
time.sleep(20)
service_instance = data_adapter.get_service_instance(
service_instance_id)
LOG.info(
"[lifecycle.deployment] [submit_service_in_agents] service_instance="
+ str(service_instance))
if service_instance is None or service_instance == -1:
LOG.info(
"[lifecycle.deployment] [submit_service_in_agents] Waiting 40s (for synchronization) to try again ..."
)
time.sleep(40)
service_instance = data_adapter.get_service_instance(
service_instance_id)
LOG.info(
"[lifecycle.deployment] [submit_service_in_agents] service_instance="
+ str(service_instance))
if service_instance is None or service_instance == -1:
LOG.error(
"[lifecycle.deployment] [submit_service_in_agents] error getting service_instance"
)
return common.gen_response(
500, 'error getting service_instance', 'service',
str(service))
# update service-instance's agents list
LOG.info(
"[lifecycle.deployment] [submit_service_in_agents] Updating service instance's agents list ... "
)
service_instance = data_adapter.serv_instance_replace_service_instance_agents(
service_instance, service, user_id, sla_template_id,
agents_list)
# 2.2 create new service instance
else:
LOG.debug(
"[lifecycle.deployment] [submit_service_in_agents] Creating service instance ... "
)
service_instance = data_adapter.create_service_instance(
service, agents_list, user_id, "DEFAULT-VALUE")
if not service_instance or 'id' not in service_instance:
LOG.error(
"[lifecycle.deployment] [submit_service_in_agents] error creating service_instance"
)
return common.gen_response(500,
'error creating service_instance',
'service', str(service))
# 3. select from agents list
if 'num_agents' not in service or service['num_agents'] == 0:
num_agents = -1
else:
num_agents = service['num_agents']
LOG.info(
"[lifecycle.deployment] [submit_service_in_agents] Total agents needed to run the service ['num_agents']: "
+ str(num_agents))
LOG.info(
"[lifecycle.deployment] [submit_service_in_agents] Selecting agents ... "
)
# get agent type
if 'agent_type' not in service:
agent_type = "normal"
else:
agent_type = service['agent_type']
r, m = agent_decision.select_agents(service['exec_type'], agent_type,
num_agents, service_instance)
if m == "error" or r is None:
LOG.error(
"[lifecycle.deployment] [submit_service_in_agents] error when selecting agents. Forwarding to Leader..."
)
# forward to parent
return __forward_submit_request_to_leader(service, user_id,
sla_template_id,
service_instance['id'])
elif m == "not-enough-resources-found" or len(r['agents']) == 0:
LOG.warning(
"[lifecycle.deployment] [submit_service_in_agents] Not enough resources (number of agents) found. Forwarding to Leader..."
)
# forward to parent
return __forward_submit_request_to_leader(service, user_id,
sla_template_id,
service_instance['id'])
else:
LOG.info(
"######## DEPLOYMENT ###########################################################################"
)
service_instance = r
LOG.debug(
"[lifecycle.deployment] [submit_service_in_agents] service_instance: "
+ str(service_instance))
# submit service thread
service_instance['status'] = STATUS_DEPLOYING
t = threading.Thread(target=thr_submit_service_in_agents,
args=(
service,
service_instance,
sla_template_id,
user_id,
))
t.start()
return common.gen_response_ok(
"Service deployment operation is being processed [" +
service_instance['id'] + "]...", "service_instance",
service_instance)
except:
LOG.exception(
'[lifecycle.deployment] [submit_service_in_agents] Exception')
return common.gen_response(500, 'Exception', 'service', str(service))
def submit(service, user_id, service_instance_id, sla_template_id):
LOG.debug("[lifecycle.deployment] [submit] Deploying service [" +
service['name'] + "] in mF2C ...")
try:
# 1. check parameters content
if not check_service_content(service):
return common.gen_response(
500, 'field(s) category/exec/exec_type not found', 'service',
str(service))
# 2. get list of available agents / resources / VMs. Example: [{"agent_ip": "192.168.252.41"}, {...}]
# Call to landscaper / recommender
available_agents_list = agent_decision.get_available_agents_resources(
service)
LOG.debug(
"[lifecycle.deployment] [submit] Checking total of agents found by recommender ..."
)
if not available_agents_list or len(available_agents_list) == 0:
# warning
LOG.warning(
"[lifecycle.deployment] [submit] available_agents_list is None or is empty. Forwarding to Leader..."
)
# create empty service instance
LOG.debug(
"[lifecycle.deployment] [submit] Creating new empty service instance ..."
)
service_instance = data_adapter.create_service_instance(
service, [], user_id, sla_template_id)
if not service_instance or 'id' not in service_instance:
LOG.error(
"[lifecycle.deployment] [submit] error creating empty service_instance"
)
return common.gen_response(
500, 'error creating empty service_instance', 'service',
str(service))
# forward to parent
return __forward_submit_request_to_leader(service, user_id,
sla_template_id,
service_instance['id'])
elif available_agents_list is not None and 'num_agents' in service and service[
'num_agents'] > len(available_agents_list):
# warning
LOG.warning(
"[lifecycle.deployment] [submit] List of available agents contains not enough resources. Forwarding to Leader..."
)
# create empty service instance
LOG.debug(
"[lifecycle.deployment] [submit] Creating new empty service instance ..."
)
service_instance = data_adapter.create_service_instance(
service, [], user_id, sla_template_id)
if not service_instance or 'id' not in service_instance:
LOG.error(
"[lifecycle.deployment] [submit] error creating empty service_instance"
)
return common.gen_response(
500, 'error creating empty service_instance', 'service',
str(service))
# forward to parent
return __forward_submit_request_to_leader(service, user_id,
sla_template_id,
service_instance['id'])
else:
LOG.debug(
"[lifecycle.deployment] [submit] Agents found by recommender = "
+ str(len(available_agents_list)))
# 3. Create new service instance & allocate service / call other agents when needed
return submit_service_in_agents(service, user_id,
service_instance_id,
sla_template_id,
available_agents_list)
except:
LOG.exception('[lifecycle.deployment] [submit] Exception')
return common.gen_response(500, 'Exception', 'service', str(service))
def __deploy_docker_compose(service, agent):
LOG.debug(
"[lifecycle.modules.apps.docker.adapter] [__deploy_docker_compose] " +
str(service) + ", " + str(agent))
try:
# 1. Download docker-compose.yml file
location = service['exec']
LOG.debug(
"[lifecycle.modules.apps.docker.adapter] [__deploy_docker_compose] Getting docker-compose.yml from "
+ location + " ...")
# remove previous files
try:
os.remove(config.dic['WORKING_DIR_VOLUME'] + "/docker-compose.yml")
except:
LOG.warning(
"[lifecycle.modules.apps.docker.adapter] [__deploy_docker_compose] Error when removing file: "
+ config.dic['WORKING_DIR_VOLUME'] + "/docker-compose.yml")
# download docker-compose.yml
try:
res, _ = urequest.urlretrieve(
location,
config.dic['WORKING_DIR_VOLUME'] + "/docker-compose.yml")
LOG.debug(
"[lifecycle.modules.apps.docker.adapter] [__deploy_docker_compose] > download result: "
+ str(res))
except:
LOG.exception(
"[lifecycle.modules.apps.docker.adapter] [__deploy_docker_compose] Error when downloading file to: "
+ config.dic['WORKING_DIR_VOLUME'] + "/docker-compose.yml")
return common.gen_response(
500,
"Exception: deploy_docker_compose(): Error when downloading file to WORKING_DIR_VOLUME",
"agent", str(agent), "WORKING_DIR_VOLUME",
config.dic['WORKING_DIR_VOLUME'])
# 2. Deploy container
service_name = service['name'] + "-" + str(
uuid.uuid4()) # service_name
service_command = "up" # command
# container 1 => command 'up'
container1 = docker_client.create_docker_compose_container(
service_name, service_command)
if container1 is not None:
LOG.debug(
"[lifecycle.modules.apps.docker.adapter] [__deploy_docker_compose] container1: "
+ str(container1))
# container 2 => command 'down'
container2 = docker_client.create_docker_compose_container(
service_name + "-" + str(uuid.uuid4()), "down")
if container2 is not None:
db.SERVICE_INSTANCES_LIST.append({
"type":
SERVICE_DOCKER_COMPOSE,
"container_main":
container1['Id'],
"container_2":
container2['Id']
})
LOG.debug(
"[lifecycle.modules.apps.docker.adapter] [__deploy_docker_compose] container2: "
+ str(container2))
LOG.debug(
"[lifecycle.modules.apps.docker.adapter] [__deploy_docker_compose] container '1' & '2' created"
)
agent['agent_param'] = container2['Id']
else:
db.SERVICE_INSTANCES_LIST.append({
"type":
SERVICE_DOCKER_COMPOSE,
"container_main":
container1['Id'],
"container_2":
'error'
})
LOG.error(
"[lifecycle.modules.apps.docker.adapter] [__deploy_docker_compose] container '2' not created"
)
agent['agent_param'] = "-"
# update agent properties
agent['container_id'] = container1['Id']
agent['status'] = STATUS_WAITING
return common.gen_response_ok('Deploy service in agent', 'agent',
str(agent), 'service', str(service))
else:
LOG.error(
"[lifecycle.modules.apps.docker.adapter] [__deploy_docker_compose] Could not connect to DOCKER API"
)
agent['status'] = STATUS_ERROR
return common.gen_response(500,
'Error when connecting to DOCKER API',
'agent', str(agent), 'service',
str(service))
except:
LOG.exception(
'[lifecycle.modules.apps.docker.adapter] [__deploy_docker_compose] Exception'
)
return common.gen_response(500, 'Exception: __deploy_docker_compose()',
'agent', str(agent), 'service',
str(service))
