def __init__(self): """ instantiate the SubmitterEngine class. Retrieving the JSON_DATA file to see if there's any other application that were already launched previously, if not creation of it. """ super(SubmitterEngine, self).__init__() logger.debug("init of submitter engine class") try: with open(JSON_FILE, 'r') as json_data: logger.debug( "instantiation of dictionary app_list with {}".format( JSON_FILE)) self.app_list = json.load(json_data) except FileNotFoundError: logger.debug( "file {} doesn't exist so isntantiation of empty directory of app_list" .format(JSON_FILE)) self.app_list = dict() logger.debug("load configurations") self.object_config = SubmitterConfig() self.adaptors_class_name = [] self._get_adaptors_class() self.translated_adaptors = {} self.executed_adaptors = {}
def setUp(self): """ Prep a good TOSCA template """ self.tpl = ToscaTemplate("tests/templates/good_tosca.yaml") instantiation_config = SubmitterConfig() instantiation_config.mapping(self.tpl) self.config = instantiation_config.adaptor_config['DockerAdaptor'] self.adapt = DockerAdaptor("main_adapt", self.config, self.tpl) self.compose_data = {}
from micado_parser import MiCADOParser from plugins_gestion import PluginsGestion import sys from micado_validator import MultiError from abstracts.exceptions import AdaptorCritical, AdaptorError import utils import json import ruamel.yaml as yaml import os import time from random import randint from submitter_config import SubmitterConfig import logging import logging.config """ set up of Logging """ config = SubmitterConfig() logging.config.dictConfig(config.logging_config) logger = logging.getLogger("submitter." + __name__) JSON_FILE = "system/ids.json" class SubmitterEngine(object): """ SubmitterEngine class that is the main one that is used to treat with the application. """ def __init__(self): """ instantiate the SubmitterEngine class. Retrieving the JSON_DATA file to see if there's any other application that were already launched previously, if not creation of it. """ super(SubmitterEngine, self).__init__() logger.debug("init of submitter engine class")
class SubmitterEngine(object): """ SubmitterEngine class that is the main one that is used to treat with the application. """ def __init__(self): """ instantiate the SubmitterEngine class. Retrieving the JSON_DATA file to see if there's any other application that were already launched previously, if not creation of it. """ super(SubmitterEngine, self).__init__() logger.debug("init of submitter engine class") try: with open(JSON_FILE, 'r') as json_data: logger.debug( "instantiation of dictionary app_list with {}".format( JSON_FILE)) self.app_list = json.load(json_data) except FileNotFoundError: logger.debug( "file {} doesn't exist so isntantiation of empty directory of app_list" .format(JSON_FILE)) self.app_list = dict() logger.debug("load configurations") self.object_config = SubmitterConfig() self.adaptors_class_name = [] self._get_adaptors_class() self.translated_adaptors = {} self.executed_adaptors = {} #def launch(self, path_to_file, id_app, dry_run=False, parsed_params=None): def launch(self, template, dict_object_adaptors, id_app, dry_run): """ Launch method, that will call the in-method egine to execute the application Creating empty list for the whole class adaptor and executed adaptor :params: path_to_file, parsed_params :types: string, dictionary .. note:: For the time being we only have one "workflow engine" but we could extend this launch method to accept another parameter to be able to choose which engine to launch """ logger.info("****** Launching the application ******") if self.app_list: raise Exception( "An application is already running, MiCADO doesn't currently support multi applications" ) #template = self._micado_parser_upload(path_to_file, parsed_params) #self.object_config.mapping(template) #template, dict_object_adaptors = self._validate(path_to_file, dry_run, False, id_app, parsed_params) #dict_object_adaptors = self._instantiate_adaptors(id_app, dryrun, template) #logger.debug("list of objects adaptor: {}".format(dict_object_adaptors)) #self._save_file(id_app, path_to_file) self.app_list.update({ id_app: { "components": list(dict_object_adaptors.keys()), "adaptors_object": dict_object_adaptors, "dry_run": dry_run } }) self._update_json() logger.debug("dictionnaty of id is: {}".format(self.app_list)) self._engine(dict_object_adaptors, template, id_app) logger.info("launched process done") logger.info("*********************") return id_app def undeploy(self, id_app, force=False): """ Undeploy method will remove the application from the infrastructure. :params: id :type: string """ logger.info( "****** proceding to the undeployment of the application *****") try: if id_app not in self.app_list.keys() and not force: raise Exception("application doesn't exist") except AttributeError: logger.error( "no application has been detected on the infrastructure trying to see if force flag present" ) if not force: raise Exception("no application detected") else: logger.info("force flag detected, preceeding to undeploy") dict_object_adaptors = self._instantiate_adaptors( id_app, self.app_list[id_app]['dry_run']) logger.debug("{}".format(dict_object_adaptors)) self._undeploy(dict_object_adaptors) self._cleanup(id_app, dict_object_adaptors) if self.app_list: self.app_list.pop(id_app) self._update_json() logger.info("undeploy process done") logger.info("*********************") def update(self, id_app, template, dict_object_adaptors): """ Update method that will be updating the application we want to update. :params id: id of the application we want to update :params type: string :params path_to_file: path to the template file :params type: string :params parse_params: dictionary containing the value we want to use as the value of the input section :params type: dictionary """ logger.info( "****** proceding to the update of the application {}******". format(id_app)) #template = self._micado_parser_upload(path_to_file, parsed_params) self.object_config.mapping(template) dry_run = self.app_list[id_app]['dry_run'] #dict_object_adaptors = self._instantiate_adaptors(id_app, dry_run, False, template) logger.debug( "list of adaptor created: {}".format(dict_object_adaptors)) self.app_list.update({ id_app: { "components": list(dict_object_adaptors.keys()), "adaptors_object": dict_object_adaptors, "dry_run": dry_run } }) self._update(dict_object_adaptors, id_app) logger.info("update process done") self._update_json() logger.info("*******************") def _validate(self, path_to_file, dry_run=False, validate=False, app_id=None, parsed_params=None): """Validates app template, instantiate adaptors and validate adaptor translation Arguments: path_to_file {str} -- path to the app template Keyword Arguments: app_id {str} -- application id (default: {None}) parsed_params -- (default: {None}) Returns: tuple -- template and dictionary of adaptors """ # MiCADO Validation logger.info( "****** Starting the validation process of {} *****".format( path_to_file)) template = self._micado_parser_upload(path_to_file, parsed_params) self.object_config.mapping(template) #if validate is True: # dry_run = True # Adaptors instantiation logger.debug("Instantiating the required adaptors") dict_object_adaptors = self._instantiate_adaptors( app_id, dry_run, validate, template) logger.info("Adaptors are successfully instantiated") logger.debug( "list of objects adaptor: {}".format(dict_object_adaptors)) # Adaptors translation try: self._translate(dict_object_adaptors) except MultiError: raise except AdaptorCritical as error: logger.error( "******* Critical error during deployment, starting to roll back *********", error) if self.translated_adaptors: logger.info("Starting clean-up on translated files") self._cleanup(app_id, self.translated_adaptors) logger.info("Adaptor translation wasn't successful...") logger.info("*******************") raise logger.info("Adaptors are successfully translated") return template, dict_object_adaptors def _engine(self, adaptors, template, app_id): """ Engine itself. Creates first an id, then parse the input file. Retreive the list of id created by the translate methods of the adaptors. Excute those id in their respective adaptor. Update the app_list and the json file. """ try: #self._translate(adaptors) self._execute(app_id, adaptors) logger.debug(self.executed_adaptors) except MultiError: raise except AdaptorCritical: logger.info( "******* Critical error during deployment, starting to roll back *********" ) if self.executed_adaptors: logger.info("Starting undeploy on executed components") self._undeploy(self.executed_adaptors) if self.translated_adaptors: logger.info("Starting clean-up on translated files") self._cleanup(app_id, self.translated_adaptors) if self.app_list: logger.info("Removing application ID from deployment") self.app_list.pop(app_id) self._update_json() logger.info("The deployment wasn't successful...") logger.info("*******************") raise def _micado_parser_upload(self, path, parsed_params): """ Parse the file and retrieve the object """ logger.debug( "Instantiation of the submitter and retrieving the template") parser = MiCADOParser() template = parser.set_template(path=path, parsed_params=parsed_params) logger.info("Valid & Compatible TOSCA template") return template def _get_adaptors_class(self): """ Retrieve the list of the differrent class adaptors """ logger.debug("retreive the adaptors class") adaptor_list = self.object_config.get_list_adaptors() PG = PluginsGestion() for k in adaptor_list: adaptor = PG.get_plugin(k) logger.debug("adaptor found {}".format(adaptor)) self.adaptors_class_name.append(adaptor) logger.debug("list of adaptors instantiated: {}".format( self.adaptors_class_name)) def _instantiate_adaptors(self, app_id, dry_run=False, validate=False, template=None): """ Instantiate the list of adaptors from the adaptors class list :params app_id: id of the application :params app_ids: list of ids to specify the adaptors (can be None) :params template: template of the application if provide list of adaptors object and app_id :returns: list of adaptors """ adaptors = dict() if template is not None: for adaptor in self.adaptors_class_name: logger.debug("instantiate {}, template".format(adaptor)) if app_id: adaptor_id = "{}_{}".format(app_id, adaptor.__name__) else: adaptor_id = adaptor.__name__ obj = adaptor( adaptor_id, self.object_config.adaptor_config[adaptor.__name__], dry_run, validate, template=template) adaptors[adaptor.__name__] = obj #adaptors.append(obj) return adaptors elif template is None: for adaptor in self.adaptors_class_name: logger.debug("instantiate {}, no template".format(adaptor)) if app_id: adaptor_id = "{}_{}".format(app_id, adaptor.__name__) else: adaptor_id = adaptor.__name__ obj = adaptor( adaptor_id, self.object_config.adaptor_config[adaptor.__name__], dry_run, validate) #adaptors.append(obj) adaptors[adaptor.__name__] = obj logger.debug("done instntiation of {}".format(adaptor)) return adaptors def _translate(self, adaptors): """ Launch the translate engine """ logger.debug("launch of translate method") logger.info("translate method called in all the adaptors") self.translated_adaptors = {} for step in self.object_config.step_config['translate']: logger.info("translating method call from {}".format(step)) while True: try: self.translated_adaptors[step] = adaptors[step] adaptors[step].translate() except AdaptorError: continue break def _execute(self, app_id, adaptors): """ method called by the engine to launch the adaptors execute methods """ logger.info( "launch of the execute methods in each adaptors in a serial way") self.executed_adaptors = {} self.app_list.setdefault(app_id, {}).setdefault("output", {}) for step in self.object_config.step_config['execute']: self.executed_adaptors[step] = adaptors[step] adaptors[step].execute() output = getattr(adaptors[step], "output", None) if output: self.app_list[app_id]["output"].update({step: output}) self._update_json() def _undeploy(self, adaptors): """ method called by the engine to launch the adaptor undeploy method of a specific component identified by its ID""" logger.info("undeploying component") for step in self.object_config.step_config['undeploy']: try: adaptors[step].undeploy() except KeyError as e: logger.debug( "{} not in initialised/executed adaptors, skipping...". format(e)) except Exception as e: logger.error( "error: {}; proceeding to undepployment of the other adaptors" .format(e)) def _update(self, adaptors, app_id): """ method that will translate first the new component and then see if there's a difference, and then execute""" logger.info( "update of each component related to the application wanted") self.app_list.setdefault(app_id, {}).setdefault("output", {}) for step in self.object_config.step_config['update']: adaptors[step].update() output = getattr(adaptors[step], "output", None) if output: self.app_list[app_id]["output"].update({step: output}) def query(self, query, app_id, dry_run=False): """ query """ for adaptor in self._instantiate_adaptors(app_id, dry_run).values(): try: result = adaptor.query(query) except AttributeError: continue else: return result else: raise AdaptorCritical("No query method available") def get_status(self, app_id): """ method to retrieve the status of the differents adaptor""" try: result = dict() for key, value in self.app_list[app_id].get("adaptors_object", {}).items(): result[key] = value.status except KeyError: logger.error("application id {} doesn't exist".format(app_id)) raise KeyError return result def _cleanup(self, id, adaptors): """ method called by the engine to launch the celanup method of all the components for a specific application identified by it's ID, and removing the template from files/templates""" logger.info("cleaning up the file after undeployment") for step in self.object_config.step_config['cleanup']: try: adaptors[step].cleanup() except KeyError as e: logger.debug( "{} not in initialised/translated adaptors, skipping...". format(e)) except Exception as e: logger.error( "error: {}; proceeding to cleanup of the other adaptors". format(e)) def _update_json(self): """ method called by the engine to update the json file that will contain the dictionary of the IDs of the applications and the list of the IDs of its components link to the ID of the app. """ data_to_save = dict() if isinstance(self.app_list, dict): for key, value in self.app_list.items(): data_to_save.setdefault(key, {}) if isinstance(self.app_list[key], dict): for k, v in self.app_list[key].items(): if "components" in k or "outputs" in k or "dry_run" in k: data_to_save[key].setdefault(k, v) if not data_to_save: logger.debug("data to save is empty") data_to_save = {} try: with open(JSON_FILE, 'w') as outfile: json.dump(data_to_save, outfile) except Exception as e: logger.warning("{}".format(e)) def _save_file(self, id_app, path): """ method called by the engine to dump the current template being treated to the files/templates directory, with as name the ID of the app. """ data = utils.get_yaml_data(path) utils.dump_order_yaml(data, "files/templates/{}.yaml".format(id_app))
class SubmitterEngine(object): """ SubmitterEngine class that is the main one that is used to treat with the application. """ def __init__(self): """ instantiate the SubmitterEngine class. Retrieving the JSON_DATA file to see if there's any other application that were already launched previously, if not creation of it. """ super(SubmitterEngine, self).__init__() logger.debug("init of submitter engine class") try: with open(JSON_FILE, 'r') as json_data: logger.debug( "instantiation of dictionary app_list with {}".format( JSON_FILE)) self.app_list = json.load(json_data) except FileNotFoundError: logger.debug( "file {} doesn't exist so isntantiation of empty directory of app_list" .format(JSON_FILE)) self.app_list = dict() logger.debug("load configurations") self.object_config = SubmitterConfig() self.adaptors_class_name = [] self._get_adaptors_class() def launch(self, path_to_file, parsed_params=None, id_app=None): """ Launch method, that will call the in-method egine to execute the application Creating empty list for the whole class adaptor and executed adaptor :params: path_to_file, parsed_params :types: string, dictionary .. note:: For the time being we only have one "workflow engine" but we could extend this launch method to accept another parameter to be able to choose which engine to launch """ logger.info( "Launching the application located there {} and with params {}". format(path_to_file, parsed_params)) if self.app_list and not self.object_config.main_config['dry_run']: raise Exception( "An application is already running, MiCADO doesn't currently support multi applications" ) return template = self._micado_parser_upload(path_to_file, parsed_params) self.object_config.mapping(template) if id_app is None: id_app = utils.id_generator() dict_object_adaptors = self._instantiate_adaptors(id_app, template) logger.debug( "list of objects adaptor: {}".format(dict_object_adaptors)) #self._save_file(id_app, path_to_file) self.app_list.update( {id_app: { "components": list(dict_object_adaptors.keys()) }}) self._update_json() logger.info("dictionnaty of id is: {}".format(self.app_list)) self._engine(dict_object_adaptors, template, id_app) return id_app def undeploy(self, id_app): """ Undeploy method will remove the application from the infrastructure. :params: id :type: string """ logger.info("proceding to the undeployment of the application") dict_object_adaptors = self._instantiate_adaptors(id_app) logger.debug("{}".format(dict_object_adaptors)) self._undeploy(dict_object_adaptors) self._cleanup(id_app, dict_object_adaptors) self.app_list.pop(id_app) self._update_json() def update(self, id_app, path_to_file, parsed_params=None): """ Update method that will be updating the application we want to update. :params id: id of the application we want to update :params type: string :params path_to_file: path to the template file :params type: string :params parse_params: dictionary containing the value we want to use as the value of the input section :params type: dictionary """ logger.info( "proceding to the update of the application {}".format(id_app)) template = self._micado_parser_upload(path_to_file, parsed_params) self.object_config.mapping(template) dict_object_adaptors = self._instantiate_adaptors(id_app, template) logger.debug( "list of adaptor created: {}".format(dict_object_adaptors)) self._update(dict_object_adaptors, id_app) def _engine(self, adaptors, template, app_id): """ Engine itself. Creates first a id, then parse the input file. Retreive the list of id created by the translate methods of the adaptors. Excute those id in their respective adaptor. Update the app_list and the json file. """ try: self._translate(adaptors) executed_adaptors = self._execute(app_id, adaptors) logger.debug(executed_adaptors) except MultiError as e: raise except AdaptorCritical as e: raise except AdaptorCritical as e: for adaptor in reversed(executed_adaptors): self._undeploy(adaptor, id_app) raise def _micado_parser_upload(self, path, parsed_params): """ Parse the file and retrieve the object """ logger.debug("instantiation of submitter and retrieve template") parser = MiCADOParser() template = parser.set_template(path=path, parsed_params=parsed_params) logger.info("Valid & Compatible TOSCA template") return template def _get_adaptors_class(self): """ Retrieve the list of the differrent class adaptors """ logger.debug("retreive the adaptors class") adaptor_list = self.object_config.get_list_adaptors() PG = PluginsGestion() for k in adaptor_list: adaptor = PG.get_plugin(k) logger.debug("adaptor found {}".format(adaptor)) self.adaptors_class_name.append(adaptor) logger.debug("list of adaptors instantiated: {}".format( self.adaptors_class_name)) def _instantiate_adaptors(self, app_id, template=None): """ Instantiate the list of adaptors from the adaptors class list :params app_id: id of the application :params app_ids: list of ids to specify the adaptors (can be None) :params template: template of the application if provide list of adaptors object and app_id :returns: list of adaptors """ adaptors = dict() if template is not None: for adaptor in self.adaptors_class_name: logger.debug("instantiate {}, template".format(adaptor)) adaptor_id = "{}_{}".format(app_id, adaptor.__name__) obj = adaptor( adaptor_id, self.object_config.adaptor_config[adaptor.__name__], template=template) adaptors[adaptor.__name__] = obj #adaptors.append(obj) return adaptors elif template is None: for adaptor in self.adaptors_class_name: logger.debug("instantiate {}, no template".format(adaptor)) adaptor_id = "{}_{}".format(app_id, adaptor.__name__) obj = adaptor( adaptor_id, self.object_config.adaptor_config[adaptor.__name__]) #adaptors.append(obj) adaptors[adaptor.__name__] = obj logger.debug("done instntiation of {}".format(adaptor)) return adaptors def _translate(self, adaptors): """ Launch the translate engine """ logger.debug("launch of translate method") logger.info("translate method called in all the adaptors") for step in self.object_config.step_config['translate']: logger.info("translating method call from {}".format(step)) while True: try: adaptors[step].translate() except AdaptorError as e: continue break def _execute(self, app_id, adaptors): """ method called by the engine to launch the adaptors execute methods """ logger.info( "launch of the execute methods in each adaptors in a serial way") executed_adaptors = [] for step in self.object_config.step_config['execute']: adaptors[step].execute() executed_adaptors.append(adaptors[step]) try: self.app_list[app_id]["output"] = adaptors[step].output except AttributeError as e: self.app_list[app_id]["output"] = "no output available" logger.warning( "the adaptor doesn't provice a output attribute") # for adaptor in adaptors: # logger.debug("\t execute adaptor: {}".format(adaptor)) # #Step(adaptor).execute() # adaptor.execute() # try: # #self.app_list.update(app_id, Step(adaptor).output) # #self.app_list.update(app_id, adaptor.output) # self.app_list[app_id] = adaptor.output # # except AttributeError as e: # logger.warning("the Adaptor doesn't provide a output attribute") self._update_json() return executed_adaptors def _undeploy(self, adaptors): """ method called by the engine to launch the adaptor undeploy method of a specific component identified by its ID""" logger.info("undeploying component") for step in self.object_config.step_config['undeploy']: adaptors[step].undeploy() def _update(self, adaptors, app_id): """ method that will translate first the new component and then see if there's a difference, and then execute""" logger.info( "update of each components related to the application wanted") for step in self.object_config.step_config['update']: #Step(adaptor).update() adaptors[step].update() try: #self.app_list.update(app_id, Step(adaptor).output) logger.info(adaptors[step].output) self.app_list[app_id] = adaptors[step].output except AttributeError as e: logger.warning( "the Adaptor doesn't provide a output attribute") def _cleanup(self, id, adaptors): """ method called by the engine to launch the celanup method of all the components for a specific application identified by it's ID, and removing the template from files/templates""" logger.info("cleaning up the file after undeployment") for step in self.object_config.step_config['cleanup']: adaptors[step].cleanup() def _update_json(self): """ method called by the engine to update the json file that will contain the dictionary of the IDs of the applications and the list of the IDs of its components link to the ID of the app. """ try: with open(JSON_FILE, 'w') as outfile: json.dump(self.app_list, outfile) except Exception as e: logger.warning("{}".format(e)) def _save_file(self, id_app, path): """ method called by the engine to dump the current template being treated to the files/templates directory, with as name the ID of the app. """ data = utils.get_yaml_data(path) utils.dump_order_yaml(data, "files/templates/{}.yaml".format(id_app))