def get_config_template_data(self, serialnumber=None, templateName=None, groupName=None, isRaw=None): status, data = self.get_group_data(serialnumber=serialnumber, groupName=groupName) if status: grp_cfg = Tools.create_config_view(config_type=c.CONFIG_TYPE_GROUP, stream=data) t_dir = grp_cfg.TASKS.Provision.Configuration.DeviceConfTemplateDir t_file = grp_cfg.TASKS.Provision.Configuration.DeviceConfTemplateFile if isRaw: try: with open(t_dir + t_file) as tfile: t_data = tfile.read() return True, t_data except IOError as ioe: return False, ioe.message else: try: env = Environment(autoescape=False, loader=FileSystemLoader(t_dir), trim_blocks=True, lstrip_blocks=True) self.logger.info( Tools.create_log_msg(self.name, serialnumber, 'Found template <{0}>)'.format(t_file))) return True, env.get_template(t_file) except (TemplateNotFound, IOError) as err: self.logger.info(Tools.create_log_msg(self.name, serialnumber, 'Error({0}): {1} --> {2})'.format(err.errno, err.strerror, err.filename))) return False, err else: return status, data
def __init__(self, sample_device=None, shared=None): self.logger = c.logger self.task_state = c.TASK_STATE_INIT self.task_progress = 0.0 self.messages = { 'error': 'error', 'valid': 'valid', 'invalid': 'invalid', 'deactivated': 'deactivated' } self.task_name = self.__class__.__name__.split('Task')[0] self.sample_device = sample_device self.shared = shared self.grp_cfg = Tools.create_config_view( c.CONFIG_TYPE_GROUP, stream=sample_device.deviceGroupData) self.task_type = self.__class__.TASK_TYPE self.check_schema = self.__class__.CHECK_SCHEMA self.task_version = self.__class__.TASK_VERSION self._backendp = BackendClientProcessor( exchange='', routing_key=c.AMQP_RPC_BACKEND_QUEUE) self.logger.info( Tools.create_log_msg(self.task_name, self.sample_device.deviceSerial, 'Validating task configuration')) isValid, err = self.__validate_task_config( grp_cfg=sample_device.deviceGroupData) if isValid == self.messages['valid']: self.logger.info( Tools.create_log_msg(self.task_name, self.sample_device.deviceSerial, 'Task configuration is valid')) # self.pre_run_task() # self.run_task() # self.post_run_task() elif isValid == self.messages['invalid']: Tools.emit_log( task_name=self.task_name, sample_device=self.sample_device, message='Task configuration is invalid. <{0}>'.format(err)) self.update_task_state(new_task_state=c.TASK_STATE_FAILED, task_state_message=c.TASK_STATE_MSG_FAILED) return elif isValid == self.messages['error']: Tools.emit_log(task_name=self.task_name, sample_device=self.sample_device, message='Task configuration file not found') self.update_task_state(new_task_state=c.TASK_STATE_FAILED, task_state_message=c.TASK_STATE_MSG_FAILED) return else: Tools.emit_log(task_name=self.task_name, sample_device=self.sample_device, message='Task configuration validation deactivated')
def prepare_device_config(self, sample_device=None): version = sample_device.softwareVersion now = datetime.datetime.now().strftime('%Y-%m-%d-%H%M') grp_cfg = Tools.create_config_view( config_type=c.CONFIG_TYPE_GROUP, stream=sample_device.deviceGroupData) if c.SERVICEPLUGIN_OSSH in sample_device.deviceServicePlugin: sample_device.deviceConfigData['device'][ 'ossh_secret'] = sample_device.deviceConfigData['device'][ 'ossh_secret'] sample_device.deviceConfigData['device'][ 'ossh_ip'] = c.conf.SERVICES.Ossh.ServiceBindAddress sample_device.deviceConfigData['device'][ 'ossh_port'] = c.conf.SERVICES.Ossh.ServiceListenPort if sample_device.deviceConfigData: heading = "## Last changed: " + now + "\n" heading += "version " + version + ";" sample_device.deviceConfigData['heading'] = heading status, data = Tools.get_config( lookup_type=c.CONFIG_LOOKUP_TYPE_GET_TEMPLATE, sample_device=sample_device) if status: config = data.render(sample_device.deviceConfigData) sample_device.deviceConfiguration = config _device_config_file = '{0}-{1}.conf'.format( sample_device.deviceSerial, now) target = open( grp_cfg.TASKS.Provision.Configuration.ConfigFileHistory + _device_config_file, 'w') target.write(sample_device.deviceConfiguration) target.close() return { 'status': True, 'sample_device': sample_device, 'configfilename': _device_config_file } else: return { 'status': False, 'sample_device': sample_device, 'configfilename': data } else: self.logger.info( Tools.create_log_msg(logmsg.CONF_DEV_CFG, sample_device.deviceSerial, logmsg.CONF_DEV_CFG_DEV_DATA_ERROR)) return None
class Yapt(object): if __name__ == '__main__': Tools.create_config_view(c.CONFIG_TYPE_MAIN) EmitterPlgFact() uiprocessor = UiProcessor(target=UiProcessor, name=c.AMQP_PROCESSOR_UI, args=( c.conf.AMQP.Exchange, c.conf.AMQP.Type, c.AMQP_PROCESSOR_UI, c.AMQP_PROCESSOR_UI, )) uiprocessor.start() #multiprocessing.set_start_method('spawn') p = multiprocessing.Process(target=UiInit, args=()) p.start() c.fc = FactoryContainer().get_factory_container() c.taskq = c.fc.taskq() BackendPluginFactory(plugin_name=c.conf.BACKEND.Module, target=Backend, name=c.AMQP_PROCESSOR_BACKEND) for item in range(c.conf.YAPT.WorkerThreads): taskprocessor = TaskProcessor(target=TaskProcessor, name=c.AMQP_PROCESSOR_TASK + str(item), args=( c.conf.AMQP.Exchange, c.conf.AMQP.Type, c.AMQP_PROCESSOR_TASK, c.AMQP_PROCESSOR_TASK, )) taskprocessor.start() if c.conf.JUNOSSPACE.Enabled: spf = SpacePluginFactory(c.conf.JUNOSSPACE.Version) c.SRC = spf.init_plugin() serviceprocessor = ServiceProcessor(target=ServiceProcessor, name=c.AMQP_PROCESSOR_SVC, args=( c.conf.AMQP.Exchange, c.conf.AMQP.Type, c.AMQP_RPC_SERVICE_QUEUE, )) serviceprocessor.start()
class YaptBackend(object): if __name__ == '__main__': Tools.create_config_view(c.CONFIG_TYPE_MAIN) EmitterPlgFact() parser = argparse.ArgumentParser() parser.add_argument("amqpIp", help="provide amqp bus ip") args = parser.parse_args() c.conf.AMQP.Host = args.amqpIp BackendPluginFactory(plugin_name=c.conf.BACKEND.Module, target=Backend, name=c.AMQP_PROCESSOR_BACKEND)
def get_config_template_file(self, serialnumber=None, templateName=None, groupName=None): status, data = self.get_group_data(serialnumber=serialnumber, groupName=groupName) if status: grp_cfg = Tools.create_config_view(config_type=c.CONFIG_TYPE_GROUP, stream=data) t_dir = grp_cfg.TASKS.Provision.Configuration.DeviceConfTemplateDir t_file = grp_cfg.TASKS.Provision.Configuration.DeviceConfTemplateFile if os.path.exists(t_dir + t_file) and os.path.isfile(t_dir + t_file): self.logger.info(Tools.create_log_msg(self.name, serialnumber, logmsg.LOCAL_TEMPLATE_FILE_OK.format( t_dir + t_file))) return True, t_dir + t_file else: self.logger.info(Tools.create_log_msg(self.name, serialnumber, logmsg.LOCAL_TEMPLATE_FILE_NOK.format( t_dir + t_file))) return False, None else: return status, data
def del_config_template_data(self, templateName=None, groupName=None): if groupName and templateName: isFile, grp_cfg = self.get_group_data(serialnumber=groupName, groupName=groupName) if isFile: grp_cfg = Tools.create_config_view(config_type=c.CONFIG_TYPE_GROUP, stream=grp_cfg) if os.path.exists( grp_cfg.TASKS.Provision.Configuration.DeviceConfTemplateDir + templateName + c.CONFIG_FILE_SUFFIX_TEMPLATE): os.remove( grp_cfg.TASKS.Provision.Configuration.DeviceConfTemplateDir + templateName + c.CONFIG_FILE_SUFFIX_TEMPLATE) return True, 'Successfully deleted template <{0}>'.format(templateName) else: return False, 'File \<{0}\> not found'.format(templateName)
def add_config_template_data(self, templateName=None, templateData=None, groupName=None): if groupName and templateName and templateData: isFile, grp_cfg = self.get_group_data(serialnumber=groupName, groupName=groupName) if isFile: grp_cfg = Tools.create_config_view(config_type=c.CONFIG_TYPE_GROUP, stream=grp_cfg) try: with open( grp_cfg.TASKS.Provision.Configuration.DeviceConfTemplateDir + templateName + c.CONFIG_FILE_SUFFIX_TEMPLATE, 'w') as fp: fp.write(templateData) return True, 'Successfully added template <{0}>'.format(templateName) except IOError as ioe: return False, ioe.message
class YaptSvc(object): if __name__ == '__main__': Tools.create_config_view(c.CONFIG_TYPE_MAIN) EmitterPlgFact() parser = argparse.ArgumentParser() parser.add_argument("amqpIp", help="provide amqp bus ip") args = parser.parse_args() c.conf.AMQP.Host = args.amqpIp if c.conf.JUNOSSPACE.Enabled: spf = SpacePluginFactory(c.conf.JUNOSSPACE.Version) c.SRC = spf.init_plugin() for item in range(c.conf.YAPT.WorkerThreads): taskprocessor = TaskProcessor(target=TaskProcessor, name=c.AMQP_PROCESSOR_TASK + str(item), args=( c.conf.AMQP.Exchange, c.conf.AMQP.Type, c.AMQP_PROCESSOR_TASK, c.AMQP_PROCESSOR_TASK, )) taskprocessor.start() serviceprocessor = ServiceProcessor(target=ServiceProcessor, name=c.AMQP_PROCESSOR_SVC, args=( c.conf.AMQP.Exchange, c.conf.AMQP.Type, c.AMQP_PROCESSOR_SVC, c.AMQP_PROCESSOR_SVC, )) serviceprocessor.start()
class YaptUi(object): if __name__ == '__main__': Tools.create_config_view(c.CONFIG_TYPE_MAIN) EmitterPlgFact() parser = argparse.ArgumentParser() parser.add_argument("amqpIp", help="provide amqp bus ip") args = parser.parse_args() c.conf.AMQP.Host = args.amqpIp uiprocessor = UiProcessor(target=UiProcessor, name=c.AMQP_PROCESSOR_UI, args=( c.conf.AMQP.Exchange, c.conf.AMQP.Type, c.AMQP_PROCESSOR_UI, c.AMQP_PROCESSOR_UI, )) uiprocessor.start() p = multiprocessing.Process(target=UiInit, args=()) p.start()
def receive_message(self, ch, method, properties, body): if body is not None: ch.basic_ack(delivery_tag=method.delivery_tag) body_decoded = jsonpickle.decode(body) Tools.amqp_receive_to_logger(routing_key=method.routing_key, body_decoded=body_decoded) if isinstance(body_decoded, AMQPMessage) and c.AMQP_MSG_TYPE_DEVICE_ADD == body_decoded.message_type: sample_device = body_decoded.payload status, sample_device = Tools.create_dev_conn(sample_device=sample_device) if status: sample_device = Tools.get_device_facts(sample_device=sample_device) status, data = Tools.get_config(lookup_type=c.CONFIG_LOOKUP_TYPE_GET_DEVICE_CFG, sample_device=sample_device) if status: sample_device.deviceConfigData = data try: sample_device.deviceGroup = data['yapt']['device_group'] except KeyError as ke: self._logger.info(Tools.create_log_msg(logmsg.TASKP, sample_device.deviceSerial, logmsg.TASKP_STOP_NO_DEVGRP.format(ke.message))) return status, grp_cfg = Tools.get_config(lookup_type=c.CONFIG_LOOKUP_TYPE_GET_GROUP, sample_device=sample_device) if status: sample_device.deviceGroupData = grp_cfg grp_cfg = Tools.create_config_view(config_type=c.CONFIG_TYPE_GROUP, stream=grp_cfg) sample_device.deviceTemplate = grp_cfg.TASKS.Provision.Configuration.DeviceConfTemplateFile sample_device.deviceTaskSeq = list(grp_cfg.TASKS.Sequence) dev_conn = sample_device.deviceConnection sample_device.deviceConnection = hex(id(sample_device.deviceConnection)) message = AMQPMessage(message_type=c.AMQP_MSG_TYPE_DEVICE_ADD, payload=sample_device, source=c.AMQP_PROCESSOR_TASK) resp = self._backendp.call(message=message) resp = jsonpickle.decode(resp) sample_device = resp.payload[1] sample_device.deviceConnection = dev_conn self._logger.info(Tools.create_log_msg(logmsg.TASKP, sample_device.deviceSerial, logmsg.TASKP_LOAD_TASK_SEQ.format( sample_device.deviceTaskSeq))) c.taskq.add_device_task_q(sample_device=sample_device, grp_cfg=grp_cfg) taskq = c.taskq.get_device_task_q(sn=sample_device.deviceSerial) self._logger.debug(Tools.create_log_msg(self.name, sample_device.deviceSerial, 'Device status: <{0}>'.format( sample_device.deviceStatus))) if taskq: for task in taskq: self._logger.debug(Tools.create_log_msg(self.name, sample_device.deviceSerial, 'Task: <{0}> status: <{1}>'.format( task.task_name, task.task_state))) if task.task_state == c.TASK_STATE_INIT: task.pre_run_task() task.run_task() task.post_run_task() if task.task_state == c.TASK_STATE_REBOOTING: self._logger.debug( Tools.create_log_msg(self.name, sample_device.deviceSerial, 'Task: <{0}> status: <{1}>'.format( task.task_name, task.task_state))) self._logger.info( Tools.create_log_msg(logmsg.TASKP, sample_device.deviceSerial, logmsg.TASKP_STOP_DEV_REBOOT.format( sample_device.deviceSerial))) break elif task.task_state == c.TASK_STATE_FAILED: self._logger.debug( Tools.create_log_msg(self.name, sample_device.deviceSerial, 'Task: <{0}> status: <{1}>'.format( task.task_name, task.task_state))) self._logger.info( Tools.create_log_msg(logmsg.TASKP, sample_device.deviceSerial, logmsg.TASKP_TASK_ERROR.format( task.task_name, ))) self.close_dev_conn(sample_device=sample_device) break elif task.task_state == c.TASK_STATE_REBOOTING: task.pre_run_task() task.run_task() task.post_run_task() if task.task_state == c.TASK_STATE_FAILED: self._logger.debug( Tools.create_log_msg(self.name, sample_device.deviceSerial, 'Task: <{0}> status: <{1}>'.format( task.task_name, task.task_state))) self._logger.info( Tools.create_log_msg(logmsg.TASKP, sample_device.deviceSerial, logmsg.TASKP_TASK_ERROR.format( task.task_name, ))) self.close_dev_conn(sample_device=sample_device) break elif task.task_state == c.TASK_STATE_DONE: pass elif task.task_state == c.TASK_STATE_FAILED: self._logger.debug( Tools.create_log_msg(self.name, sample_device.deviceSerial, 'Task: <{0}> status: <{1}>'.format( task.task_name, task.task_state))) self._logger.info( Tools.create_log_msg(logmsg.TASKP, sample_device.deviceSerial, logmsg.TASKP_TASK_ERROR.format( task.task_name, ))) self.close_dev_conn(sample_device=sample_device) break else: self._logger.info(Tools.create_log_msg(logmsg.TASKP, sample_device.deviceSerial, 'Task <{0}> state: <{1}> unknown'.format( task.task_name, task.task_state))) break # print 'After running all tasks device status is: <{0}>'.format( # sample_device.deviceStatus) if sample_device.deviceStatus == c.DEVICE_STATUS_DONE: c.taskq.del_device_task_q(sample_device.deviceSerial) else: self._logger.info(Tools.create_log_msg(logmsg.TASKP, sample_device.deviceSerial, logmsg.TASKP_STOP_NO_DEVGRP_CFG.format(grp_cfg))) self.close_dev_conn(sample_device=sample_device) return else: self._logger.info(Tools.create_log_msg(logmsg.TASKP, sample_device.deviceSerial, logmsg.TASKP_STOP_NO_DEV_CFG.format(data))) self.close_dev_conn(sample_device=sample_device) return else: self._logger.info(Tools.create_log_msg(logmsg.TASKP, sample_device.deviceSerial, logmsg.TASKP_CONN_ERR.format(sample_device.deviceSerial))) return else: Tools.amqp_receive_error_to_logger(routing_key=method.routing_key, body_decoded=body_decoded) else: self._logger.info(Tools.create_log_msg(logmsg.TASKP, None, logmsg.TASKP_DEFECT_MSG))
def get_config_template_data(self, serialnumber=None, templateName=None, groupName=None, isRaw=None): auth_status, data = self.authenticate_oauth() if auth_status: if groupName: status, groupData = self.get_group_data(serialnumber=serialnumber, groupName=groupName, isRaw=False) if status: grp_cfg = Tools.create_config_view(config_type=c.CONFIG_TYPE_GROUP, stream=groupData) template_file = grp_cfg.TASKS.Provision.Configuration.DeviceConfTemplateFile try: project = self.gl.projects.get(c.conf.STORAGE.Cgitlab.DevCfgTemplate) path = '{0}'.format(template_file) except (GitlabConnectionError, GitlabError) as gle: return False, 'Failed to get project with error: <0>'.format(gle.message) try: f = project.files.get(file_path=path, ref='master') except GitlabError as ge: return False, 'Failed to get template config with error: <{0}>'.format(ge.message) if isRaw: return True, f.decode() else: try: template = Environment(loader=BaseLoader).from_string(f.decode()) return True, template except TypeError as te: return False, te.message else: return False, groupData else: try: project = self.gl.projects.get(c.conf.STORAGE.Cgitlab.DevCfgTemplate) path = '{0}'.format(templateName) except (GitlabConnectionError, GitlabError) as gle: return False, 'Failed to get project with error: <0>'.format(gle.message) try: f = project.files.get(file_path=path, ref='master') except GitlabError as ge: return False, 'Failed to get template config with error: <{0}>'.format(ge.message) if isRaw: return True, f.decode() else: try: template = Environment(loader=BaseLoader).from_string(f.decode()) return True, template except TypeError as te: return False, te.message else: return auth_status, data
def discover_by_configlet(self, sample_device=None, shared=None): if sample_device.deviceConnection.connected: self.logger.info( 'PROVCONFIGLET-[%s]: Start connection provisioning process for device: %s', sample_device.deviceIP, sample_device.deviceIP) grp_cfg = Tools.create_config_view( config_type=c.CONFIG_TYPE_GROUP, stream=sample_device.deviceGroupData) sample_device.deviceTasks.taskState['Discovery'] = 'Copy configlet' configlet_dir = grp_cfg.TASKS.Provision.Discovery.ConnectionConfigletDir configlet_template_file = configlet_dir + sample_device.deviceModel + \ "/connectionConfiglet.j2" configlet_vars_file = configlet_dir + sample_device.deviceModel + \ "/connectionConfigletVars.yml" self.logger.info( 'PROVCONFIGLET-[%s]: Configlet file to be used: %s', sample_device.deviceIP, configlet_template_file) datavars = yaml.load(open(configlet_vars_file).read()) template = Template(open(configlet_template_file).read()) try: sample_device.deviceConnection.cu.lock() except LockError as err: self.logger.info( "PROVCONFIGLET-[%s]: Unable to lock configuration: %s", sample_device.deviceIP, err.message) sample_device.deviceTasks.taskState[ 'Discovery'] = 'Error: Unable to lock configuration' shared[ Tools.TASK_SHARED_STATE] = Tools.TASK_STATE_RESULT_FAILURE return try: sample_device.deviceConnection.cu.load(template_path=template, template_vars=datavars, merge=True, overwrite=False, format='text') except ValueError as err: self.logger.info(err.message) shared[ Tools.TASK_SHARED_STATE] = Tools.TASK_STATE_RESULT_FAILURE except (ConfigLoadError, Exception) as err: self.logger.info( "PROVCONFIGLET-[%s]: Unable to load configuration changes: %s", sample_device.deviceIP, err.message) self.logger.info( 'PROVCONFIGLET-[%s]: Unlocking the configuration', sample_device.deviceIP) sample_device.deviceTasks.taskState[ 'Discovery'] = 'Unable to load configuration changes' shared[ Tools.TASK_SHARED_STATE] = Tools.TASK_STATE_RESULT_FAILURE try: sample_device.deviceConnection.cu.unlock() except UnlockError: self.logger.info( "PROVCONFIGLET-[%s]: Error --> Unable to unlock configuration", sample_device.deviceIP) sample_device.deviceTasks.taskState[ 'Discovery'] = 'Error: Unable to unlock configuration' shared[Tools. TASK_SHARED_STATE] = Tools.TASK_STATE_RESULT_FAILURE return self.logger.info('PROVCONFIGLET-[%s]: Committing configuration', sample_device.deviceIP) sample_device.deviceTasks.taskState[ 'Discovery'] = 'Committing configuration' try: sample_device.deviceConnection.cu.commit() except (CommitError, ConnectClosedError, RpcTimeoutError): self.logger.info( "PROVCONFIGLET-[%s]: Error --> Unable to commit configuration", sample_device.deviceIP) self.logger.info( "PROVCONFIGLET-[%s]: Unlocking the configuration", sample_device.deviceIP) sample_device.deviceTasks.taskState[ 'Discovery'] = 'Error: Unable to commit configuration' shared[ Tools.TASK_SHARED_STATE] = Tools.TASK_STATE_RESULT_FAILURE try: sample_device.deviceConnection.cu.unlock() except UnlockError: self.logger.info( "PROVCONFIGLET-[%s]: Error --> Unable to unlock configuration", sample_device.deviceIP) sample_device.deviceTasks.taskState[ 'Discovery'] = 'Error: Unable to unlock configuration' shared[Tools. TASK_SHARED_STATE] = Tools.TASK_STATE_RESULT_FAILURE return self.logger.info('PROVCONFIGLET-[%s]: Unlocking the configuration', sample_device.deviceIP) sample_device.deviceTasks.taskState[ 'Discovery'] = 'Unlocking the configuration' try: sample_device.deviceConnection.cu.unlock() except UnlockError as err: self.logger.info( 'PROVCONFIGLET-[%s]: Unable to unlock configuration: %s', sample_device.deviceIP, err.message) sample_device.deviceTasks.taskState[ 'Discovery'] = 'Error: Unable to unlock configuration' shared[ Tools.TASK_SHARED_STATE] = Tools.TASK_STATE_RESULT_FAILURE self.logger.info( 'PROVCONFIGLET-[%s]: Configuration template: %s committed on device: %s', sample_device.deviceIP, template, sample_device.deviceIP) sample_device.deviceTasks.taskState[ 'Discovery'] = Tools.TASK_STATE_DONE shared[Tools.TASK_SHARED_STATE] = Tools.TASK_STATE_RESULT_DONE else: self.logger.info('PROVCONFIGLET-[%s]: No connection to device %s', sample_device.deviceIP, sample_device.deviceIP) shared[Tools.TASK_SHARED_STATE] = Tools.TASK_STATE_RESULT_FAILURE sample_device.deviceTasks.taskState[ 'Discovery'] = 'Add new device instance' instance_id = self.get_modeled_device_instance_id( sample_device.deviceModel) if instance_id != 'False': response = self.add_modeled_device_instance(instance_id['id'], 1) if response == c.SRC_RESPONSE_FAILURE: sample_device.deviceTasks.taskState[ 'Discovery'] = 'Error adding device' shared[ Tools.TASK_SHARED_STATE] = Tools.TASK_STATE_RESULT_FAILURE else: sample_device.deviceTasks.taskState[ 'Discovery'] = Tools.TASK_STATE_DONE shared[Tools.TASK_SHARED_STATE] = Tools.TASK_STATE_RESULT_DONE else: self.logger.info('PROVCONFIGLET-[%s]: Instance <%s> not found', sample_device.deviceIP, sample_device.deviceModel) sample_device.deviceTasks.taskState[ 'Discovery'] = 'Instance <{0}> not found'.format( sample_device.deviceModel) shared[Tools.TASK_SHARED_STATE] = Tools.TASK_STATE_RESULT_FAILURE