def deploy_service_agent(service, service_instance, agent): LOG.debug( "[lifecycle.modules.apps.docker.adapter] [deploy_service_agent] " + str(service) + ", " + str(agent)) try: # docker-compose if service['exec_type'] == SERVICE_DOCKER_COMPOSE: return __deploy_docker_compose(service, agent) # docker elif service['exec_type'] == SERVICE_DOCKER: return __deploy_docker_image(service, agent) # compss (docker) elif service['exec_type'] == SERVICE_COMPSS: return __deploy_docker_compss(service, service_instance, agent) # not defined else: LOG.warning( "[lifecycle.modules.apps.docker.adapter] [deploy_service_agent] [" + service['exec_type'] + "] not defined") return common.gen_response( 500, 'Exception: type not defined: deploy_service_agent()', 'agent', str(agent), 'service', str(service)) except: LOG.exception( '[lifecycle.modules.apps.docker.adapter] [deploy_service_agent] Exception' ) return common.gen_response(500, 'Exception: deploy_service_agent()', 'agent', str(agent), 'service', str(service))
def new_service_instance(service, agents_list, user_id, agreement_id): LOG.debug( "[lifecycle.data.standalone.service_instance] [new_service_instance] " + service['name'] + ", " + str(agents_list) + ", " + str(user_id) + ", " + str(agreement_id)) # create list of agents list_of_agents = [] # ports: ports_l = [] try: ports_l = service['exec_ports'] except: LOG.warning( "[lifecycle.data.standalone.service_instance] [new_service_instance] No ports values found in service definition" ) # AGENTs: i = 0 if len(agents_list) > 0: for agent in agents_list: if len(agents_list) == 1 or i == 0: master_compss = True else: master_compss = False # Add new AGENT to list list_of_agents.append({ "device_id": "-", "app_type": service['exec_type'], "ports": ports_l, "url": agent['agent_ip'], "status": STATUS_CREATED_NOT_INITIALIZED, "compss_app_id": "-", "allow": True, "container_id": "-", "master_compss": master_compss, # master_compss... selected / final master "agent_param": "not-defined" }) i += 1 # service instance new_service_instance = { "service": service, "agreement": agreement_id, "user": user_id, "device_id": "not-defined", "device_ip": "not-defined", "parent_device_id": "not-defined", "parent_device_ip": "not-defined", "agents": list_of_agents, "service_type": service['exec_type'], "status": STATUS_CREATED_NOT_INITIALIZED } LOG.debug( "[lifecycle.data.standalone.service_instance] [new_service_instance] service_intance=" + str(new_service_instance)) return new_service_instance
def operation_service_agent(agent, operation): LOG.debug("LIFECYCLE: Docker Swarm: operation_service_agent [" + operation + "]: " + str(agent)) try: # connect to docker api / check existing connection if docker_client.get_client_agent_docker() is not None: if operation == OPERATION_STOP or operation == OPERATION_TERMINATE: # service ID is stored in 'container_id' field delete_docker_service(agent['container_id']) agent['status'] = STATUS_TERMINATED else: LOG.warning( "[lifecycle.modules.apps.swarm.adapter] [operation_service_agent] [" + operation + "]: " + str(agent) + ": operation not supported") # if error when connecting to agent... else: LOG.error( "[lifecycle.modules.apps.swarm.adapter] [operation_service_agent] Could not connect to DOCKER API" ) agent['status'] = STATUS_UNKNOWN # return status return agent['status'] except: agent['status'] = STATUS_ERROR LOG.exception( '[lifecycle.modules.apps.swarm.adapter] [operation_service_agent] Exception. Returning STATUS_ERROR ...' ) return STATUS_ERROR
def __get_current_device_id(): LOG.info( "[lifecycle.data.mf2c.data_interface] [__get_current_device_id] Getting 'my' device ID ..." ) agent = cimi.get_agent_info() LOG.debug( "[lifecycle.data.mf2c.data_interface] [__get_current_device_id] agent=" + str(agent)) if not agent is None and agent != -1: LOG.info( "[lifecycle.data.mf2c.data_interface] [__get_current_device_id] Getting device 'id' by 'deviceID'=" + agent['device_id']) id = cimi.get_id_from_device(agent['device_id']) if not id is None and id != -1: LOG.info( "[lifecycle.data.mf2c.data_interface] [__get_current_device_id] Returning 'my' device ID = " + id) return id else: LOG.warning( "[lifecycle.data.mf2c.data_interface] [__get_current_device_id] Device information not found. Returning -1 ..." ) return -1 else: LOG.warning( "[lifecycle.data.mf2c.data_interface] [__get_current_device_id] Agent information not found. Returning -1 ..." ) return -1
def serv_instance_replace_service_instance_agents(self, service_instance, service, user_id, sla_template_id, agents_list): LOG.warning( "[lifecycle.data.standalone_data_adapter] [serv_instance_replace_service_instance_agents] not implemented" ) return None
def __check_service_instance_id(service_instance_id): for sid in QoS_SERVICE_INSTANCES_LIST: if sid == service_instance_id: LOG.warning( "[lifecycle.events.handler_qos] [__check_notification] service instance [" + service_instance_id + "] is being processed") return False LOG.debug( "[lifecycle.events.handler_qos] [__check_notification] service instance [" + service_instance_id + "] is NOT being processed") return True
def tryPort(port): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) result = False try: sock.bind(("0.0.0.0", port)) #sock.connect_ex(('127.0.0.1', 80)) result = True except: LOG.warning("[lifecycle.modules.apps.ports_mngr] [tryPort] Port (" + str(port) + ") is in use") sock.close() return result
def getServiceInstanceReport(service_instance_id): LOG.debug("[lifecycle.app_funcs] [getServiceInstanceReport] " + service_instance_id) try: service_instance_report = data_adapter.get_service_instance_report(service_instance_id) if not service_instance_report is None and service_instance_report != -1: return common.gen_response_ok('Service Operation Report content', 'service_instance_id', service_instance_id, 'report', service_instance_report) else: LOG.warning("[lifecycle.app_funcs] [getServiceInstanceReport] service_instance_report is None") return common.gen_response(500, "service_instance_report is None", "service_instance_id", service_instance_id) except: LOG.exception('[lifecycle.app_funcs] [getServiceInstanceReport] Exception') return common.gen_response(500, 'Exception', 'service_instance_id', service_instance_id)
def db_si_getrecord(id): try: records = [r for r in DB_LM_SERVICE_INSTANCES if r['id'] == id] LOG.debug("[lifecycle.data.app.lm_db] [db_si_get] records: " + str(records)) if len(records) >= 1: return records[0] else: LOG.warning( '[lifecycle.data.app.lm_db] [db_si_get] No records found') except: LOG.exception('[lifecycle.data.app.lm_db] [db_si_get] Exception') return None
def get_all_service_instances(): try: res = requests.get(config.dic['CIMI_URL'] + '/' + RSRC_SERVICE_INSTANCE, headers=CIMI_HEADER, verify=False) LOG.debug("[lifecycle.data.mf2c.cimi] [get_all_service_instances] response: " + str(res) + ", " + str(res.json())) if res.status_code == 200: return res.json()['serviceInstances'] LOG.warning("[lifecycle.data.mf2c.cimi] [get_all_service_instances] No service instances retrieved; Returning None ...") except: LOG.exception("[lifecycle.data.mf2c.cimi] [get_all_service_instances] Exception; Returning None ...") return None
def get_id_from_device(deviceID): try: res = requests.get(config.dic['CIMI_URL'] + "/device?$filter=deviceID=\"" + deviceID + "\"", headers=CIMI_HEADER, verify=False) LOG.log(TRACE, "[lifecycle.data.mf2c.cimi] [get_id_from_device] response: " + str(res) + ", " + str(res.json())) if res.status_code == 200 and len(res.json()['devices']) > 0: return res.json()['devices'][0]['id'] else: LOG.warning("[lifecycle.data.mf2c.cimi] [get_id_from_device] No device found; Returning -1 ...") return -1 except: LOG.exception("[lifecycle.data.mf2c.cimi] [get_id_from_device] Exception; Returning None ...") return None
def del_service_instance_by_id(resource_id): try: resource_id = resource_id.replace(RSRC_SERVICE_INSTANCE + '/', '') res = requests.delete(config.dic['CIMI_URL'] + '/' + RSRC_SERVICE_INSTANCE + '/' + resource_id, headers=CIMI_HEADER, verify=False) LOG.log(TRACE, "[lifecycle.data.mf2c.cimi] [del_service_instance_by_id] [" + resource_id + "] response: " + str(res) + ", " + str(res.json())) if res.status_code == 200: return res.json() LOG.warning("[lifecycle.data.mf2c.cimi] [del_service_instance_by_id] Service instance not deleted. Id=" + resource_id + "; Returning None ...") except: LOG.exception("[lifecycle.data.mf2c.cimi] [del_service_instance_by_id] Exception (parameters: resource_id=" + resource_id + "); Returning None ...") return None
def get_sharing_model_by_device(device_id): try: device_id = device_id.replace('device/', '') res = requests.get(config.dic['CIMI_URL'] + "/sharing-model?$filter=device_id=\"device/" + device_id + "\"", headers=CIMI_HEADER, verify=False) LOG.log(TRACE, "[lifecycle.data.mf2c.cimi] [get_sharing_model_by_device] [" + device_id + "] response: " + str(res) + ", " + str(res.json())) if res.status_code == 200 and len(res.json()['sharingModels']) > 0: return res.json()['sharingModels'][0] else: LOG.warning("[lifecycle.data.mf2c.cimi] [get_sharing_model_by_device] Sharing-model not found [device_id=" + device_id + "]; Returning -1 ...") return -1 except: LOG.exception("[lifecycle.data.mf2c.cimi] [get_sharing_model_by_device] Exception (parameters: device_id=" + device_id + "); Returning None ...") return None
def get_service_instance_by_id(id): try: id = id.replace(RSRC_SERVICE_INSTANCE + '/', '') res = requests.get(config.dic['CIMI_URL'] + '/' + RSRC_SERVICE_INSTANCE + '/' + id, headers=CIMI_HEADER, verify=False) LOG.log(TRACE, "[lifecycle.data.mf2c.cimi] [get_service_instance_by_id] [" + id + "] response: " + str(res) + ", " + str(res.json())) if res.status_code == 200 and 'id' in res.json(): return res.json() elif res.status_code == 200: LOG.warning("[lifecycle.data.mf2c.cimi] [get_service_instance_by_id] No service instance retrieved. Id=" + id + "; Returning -1 ...") return -1 except: LOG.exception("[lifecycle.data.mf2c.cimi] [get_service_instance_by_id] Exception (parameters: id=" + id + "); Returning None ...") return None
def is_port_free(port): try: if port < 10000: LOG.warning( "[lifecycle.modules.apps.ports_mngr] [is_port_free] Ports (" + str(port) + ") under 10000 are nor allowed") return False if data_adapter.db_get_port_mapped(port) is None and tryPort(port): LOG.debug( "[lifecycle.modules.apps.ports_mngr] [is_port_free] Port (" + str(port) + ") is free") return True except: LOG.exception("[lifecycle.modules.apps.ports_mngr] [is_port_free] [" + str(port) + "]: Exception") return False
def db_si_getall(): try: records = DB_LM_SERVICE_INSTANCES() LOG.debug( "[lifecycle.data.app.lm_db] [db_si_getall] Getting all records ... total=" + str(len(records))) if len(records) >= 1: list_of_records = [] for r in records: list_of_records.append(r['service_instance']) return list_of_records else: LOG.warning( '[lifecycle.data.app.lm_db] [db_si_getall] No records found') except: LOG.exception('[lifecycle.data.app.lm_db] [db_si_getall] Exception') return None
def get_agent(): try: res = requests.get(config.dic['CIMI_URL'] + "/agent", headers=CIMI_HEADER, verify=False) LOG.log(TRACE, "[lifecycle.data.mf2c.cimi] [get_agent] response: " + str(res) + ", " + str(res.json())) if res.status_code == 200 and res.json()['count'] == 0: LOG.warning("[lifecycle.data.mf2c.cimi] [get_agent] 'agent' not found") return -1 elif res.status_code == 200: return res.json()['agents'][0] LOG.warning("[lifecycle.data.mf2c.cimi] [get_agent] 'agent' not found; Returning -1 ...") return -1 except: LOG.error("[lifecycle.data.mf2c.cimi] [get_agent] Exception; Returning None ...") return None
def db_si_del(id): try: record = db_si_getrecord(id) if record is not None: LOG.debug( "[lifecycle.data.app.lm_db] [db_si_del] deleted records: " + str(DB_LM_SERVICE_INSTANCES.delete(record))) # save changes on disk DB_LM_SERVICE_INSTANCES.commit() return True else: LOG.warning( '[lifecycle.data.app.lm_db] [db_si_del] Service Instance was not found in DB' ) return False except: LOG.exception('[lifecycle.data.app.lm_db] [db_si_del] Exception') return False
def add_service_instance(content, user): try: content.update(common_new_map_fields_user(user)) #common_new_map_fields()) # complete map and update resource LOG.log(TRACE, "[lifecycle.data.mf2c.cimi] [add_service_instance] [content=" + str(content) + "] [user="******"] ... ") res = requests.post(config.dic['CIMI_URL'] + '/' + RSRC_SERVICE_INSTANCE, headers=CIMI_HEADER, verify=False, json=content) LOG.log(TRACE, "[lifecycle.data.mf2c.cimi] [add_service_instance] response: " + str(res) + ", " + str(res.json())) if res.status_code == 201: id = res.json()['resource-id'].replace(RSRC_SERVICE_INSTANCE + '/', '') return get_service_instance_by_id(id) LOG.warning("[lifecycle.data.mf2c.cimi] [add_service_instance] Service instance not added. content=" + str(content) + "; Returning None ...") except: LOG.exception("[lifecycle.data.mf2c.cimi] [add_service_instance] Exception while adding new resource to service instances!; Returning None ...") return None
def get_service_instance_report(id): try: res = requests.get(config.dic['CIMI_URL'] + "/" + RSRC_SERVICE_INSTANCE_REPORT + "?$filter=operation_id='" + id + "'", headers=CIMI_HEADER, verify=False) LOG.log(TRACE, "[lifecycle.data.mf2c.cimi] [get_service_instance_report] [" + id + "] response: " + str(res) + ", " + str(res.json())) if res.status_code == 200 and len(res.json()['serviceOperationReports']) > 0: return res.json()['serviceOperationReports'][0] elif res.status_code == 200: LOG.warning("[lifecycle.data.mf2c.cimi] [get_service_instance_report] Report not found [service-instance=" + id + "]; Returning empty dict ...") return {} LOG.warning("[lifecycle.data.mf2c.cimi] [get_service_instance_report] No report retrieved. id=" + id + "; Returning None ...") except: LOG.exception("[lifecycle.data.mf2c.cimi] [get_service_instance_report] Exception (parameters: id=" + id + "); Returning None ...") return None
def del_from_DB_DOCKER_PORTS(port): try: record = get_from_DB_DOCKER_PORTS(port) if record is not None: LOG.debug( "[lifecycle.data.app.db] [del_from_DB_DOCKER_PORTS] deleted records: " + str(DB_DOCKER_PORTS.delete(record))) # save changes on disk DB_DOCKER_PORTS.commit() return True else: LOG.warning( '[lifecycle.data.app.db] [del_from_DB_DOCKER_PORTS] Port was not found in DB' ) return False except: LOG.exception( '[lifecycle.data.app.db] [del_from_DB_DOCKER_PORTS] Exception') return False
def start_job(body, service_instance_id): LOG.info( "########################################################################################" ) LOG.info("######## JOBS: START JOB") LOG.debug("[lifecycle.operations] [start_job] body=" + str(body)) LOG.debug("[lifecycle.operations] [start_job] service_instance_id=" + service_instance_id) try: # service instance service_instance = data_adapter.get_service_instance( service_instance_id) if service_instance is None or service_instance == -1: return common.gen_response( 500, 'Error getting service instance object', 'service_instance_id', service_instance_id) LOG.debug("[lifecycle.operations] [start_job] service_instance=" + str(service_instance)) # start job in agent(s) if len(service_instance['agents']) == 1: res = apps_adapter.start_job_compss(service_instance, body) elif len(service_instance['agents']) >= 2: res = apps_adapter.start_job_compss_multiple_agents( service_instance, body) else: LOG.warning( "[lifecycle.operations] [start_job] Execution supported in only 1 or more agents! agents size=" + str(len(service_instance['agents']))) res = None if res: return common.gen_response_ok('Start job', 'service_id', service_instance_id, 'res', res) else: return common.gen_response(500, 'Error when starting job', 'service_instance', str(service_instance)) except: LOG.exception('[lifecycle.operations] [start_job] Exception') return common.gen_response(500, 'Exception', 'data', str(body))
def __select_agents_service_compss(service_instance, num_agents): # compss ==> deploy in all agents if len(service_instance['agents']) > 0 and num_agents == -1: LOG.debug( "[lifecycle.modules.agent_decision] [__select_agents_service_compss] [SERVICE_COMPSS] service will be deployed in all selected agents" ) # compss ==> deploy in 1 agent elif num_agents == 1 and __get_local_agent(service_instance) is not None: LOG.debug( "[lifecycle.modules.agent_decision] [__select_agents_service_compss] [SERVICE_COMPSS] service will be deployed in 1 agent" ) service_instance['agents'] = [__get_local_agent(service_instance)] # compss ==> deploy in 'num_agents' elif len(service_instance['agents']) > 0: if len(service_instance['agents']) >= num_agents: LOG.debug( "[lifecycle.modules.agent_decision] [__select_agents_service_compss] [SERVICE_COMPSS] service will be deployed in " + str(num_agents) + " agents") list_of_agents = [] i = 0 while i < num_agents: list_of_agents.append(service_instance['agents'][i]) i += 1 service_instance['agents'] = list_of_agents else: LOG.warning( "[lifecycle.modules.agent_decision] [__select_agents_service_compss] [SERVICE_COMPSS] service should be deployed in " + str(num_agents) + " agents, but only " + str(len(service_instance['agents'])) + " are available") return service_instance, "not-enough-resources-found" # compss: not-enough-resources-found else: return service_instance, "not-enough-resources-found" LOG.debug( "[lifecycle.modules.agent_decision] [__select_agents_service_compss] agents FINAL list: " + str(service_instance['agents'])) return service_instance, "ok"
def update_service_instance(resource_id, content): try: resource_id = resource_id.replace(RSRC_SERVICE_INSTANCE + '/', '') content.update(common_update_map_fields_user(content['user']))#common_update_map_fields()) # complete map and update resource #content.update(patch_update_map_fields()) LOG.log(TRACE, "[lifecycle.data.mf2c.cimi] [update_service_instance] [content=" + str(content) + "] ... ") res = requests.put(config.dic['CIMI_URL'] + '/' + RSRC_SERVICE_INSTANCE + '/' + resource_id, headers=CIMI_HEADER, verify=False, json=content) LOG.log(TRACE, "[lifecycle.data.mf2c.cimi] [update_service_instance] response: " + str(res) + ", " + str(res.json())) if res.status_code == 200: return get_service_instance_by_id(resource_id) LOG.warning("[lifecycle.data.mf2c.cimi] [update_service_instance] Service instance not updated. resource_id=" + resource_id + ", content=" + str(content) + "; Returning None ...") except: LOG.exception("[lifecycle.data.mf2c.cimi] [update_service_instance] Exception; Returning None ...") return None
def get_from_DB_DOCKER_PORTS(port): try: # debug DB # print_records(DB_DOCKER_PORTS) records = [r for r in DB_DOCKER_PORTS if r['port'] == port] LOG.debug( "[lifecycle.data.app.db] [get_from_DB_DOCKER_PORTS] records: " + str(records)) #records = DB_DOCKER_PORTS(port=port) if len(records) >= 1: return records[0] else: LOG.warning( '[lifecycle.data.app.db] [get_from_DB_DOCKER_PORTS] No records found' ) except: LOG.exception( '[lifecycle.data.app.db] [get_from_DB_DOCKER_PORTS] Exception') return None
def db_si_update(id, service_instance): LOG.debug("[lifecycle.data.app.lm_db] [db_si_update] Updating record [" + id + "] ...") try: record = db_si_get(id) if record is not None: DB_LM_SERVICE_INSTANCES.insert(id=id, service_instance=service_instance) # save changes on disk DB_LM_SERVICE_INSTANCES.commit() # debug DB __print_records(DB_LM_SERVICE_INSTANCES) return True else: LOG.warning( '[lifecycle.data.app.lm_db] [db_si_update] Service Instance not found' ) return False except: LOG.exception('[lifecycle.data.app.lm_db] [db_si_update] Exception') return False
def operation_service(agent, operation): LOG.debug( "[lifecycle.modules.apps.kubernetes.adapter] [operation_service] [" + operation + "]: " + str(agent)) try: if operation == OPERATION_TERMINATE or operation == OPERATION_STOP: # app name is stored in 'container_id' field # deployment r = requests.delete(config.dic['K8S_PROXY'] + "/apis/apps/v1/namespaces/" + config.dic['K8S_NAMESPACE'] + "/deployments/" + agent['container_id'], json={}, verify=config.dic['VERIFY_SSL']) # service r = requests.delete( config.dic['K8S_PROXY'] + "/api/v1/namespaces/" + config.dic['K8S_NAMESPACE'] + "/services/serv_" + agent['container_id'], json={}, verify=config.dic['VERIFY_SSL']) agent['status'] = STATUS_TERMINATED else: LOG.warning( "[lifecycle.modules.apps.kubernetes.adapter] [operation_service] [" + operation + "]: " + str(agent) + ": operation not supported") # return status return agent['status'] except: agent['status'] = STATUS_ERROR LOG.exception( '[lifecycle.modules.apps.kubernetes.adapter] [operation_service] Exception. Returning STATUS_ERROR ...' ) return STATUS_ERROR
def save_to_DB_DOCKER_PORTS(port, mapped_to): LOG.log( TRACE, '[lifecycle.data.app.db] [save_to_DB_DOCKER_PORTS] Saving record ...') try: record = get_from_DB_DOCKER_PORTS(port) if record is None: DB_DOCKER_PORTS.insert(port=port, mapped_to=mapped_to) # save changes on disk DB_DOCKER_PORTS.commit() # debug DB __print_records(DB_DOCKER_PORTS) return True else: LOG.warning( '[lifecycle.data.app.db] [save_to_DB_DOCKER_PORTS] Port already added to DB' ) return False except: LOG.exception( '[lifecycle.data.app.db] [save_to_DB_DOCKER_PORTS] Exception') return False
def __select_agents_service_docker(service_instance, num_agents): # docker ==> deploy in only 1 agent if len(service_instance['agents']) > 0 and num_agents == -1: list_of_agents = [service_instance['agents'][0]] LOG.debug( "[lifecycle.modules.agent_decision] [__select_agents_service_docker] first agent: " + str(list_of_agents)) service_instance['agents'] = list_of_agents # docker ==> deploy in 'num_agents' elif len(service_instance['agents']) > 0: if len(service_instance['agents']) >= num_agents: LOG.debug( "[lifecycle.modules.agent_decision] [__select_agents_service_docker] [DOCKER] service will be deployed in " + str(num_agents) + " agents") list_of_agents = [] i = 0 while i < num_agents: # len(service_instance['agents']): list_of_agents.append(service_instance['agents'][i]) i += 1 service_instance['agents'] = list_of_agents else: LOG.warning( "[lifecycle.modules.agent_decision] [__select_agents_service_compss] [DOCKER] service should be deployed in " + str(num_agents) + " agents, but only " + str(len(service_instance['agents'])) + " are available") return service_instance, "not-enough-resources-found" # compss: not-enough-resources-found else: return service_instance, "not-enough-resources-found" LOG.debug( "[lifecycle.modules.agent_decision] [__select_agents_service_docker] agents FINAL list: " + str(service_instance['agents'])) return service_instance, "ok"
def thr(warning): try: LOG.debug("[lifecycle.events.handler_um] [thr] Handling UM notifications [" + str(warning) + "] ...") # battery_limit_violation if warning['result']['battery_limit_violation']: LOG.warning("[lifecycle.events.handler_um] [thr] battery_limit_violation") # TODO # resource_contributor_violation elif warning['result']['resource_contributor_violation']: LOG.warning("[lifecycle.events.handler_um] [thr] resource_contributor_violation") # TODO # max_apps_violation elif warning['result']['max_apps_violation']: LOG.warning("[lifecycle.events.handler_um] [thr] max_apps_violation") # TODO LOG.debug("[lifecycle.events.handler_um] [thr] UM Notification handled") except: LOG.exception('[lifecycle.events.handler_um] [thr] Exception')