def receive_message(self, ch, method, properties, body): if body is not None: ch.basic_ack(delivery_tag=method.delivery_tag) body_decoded = jsonpickle.decode(body) Tools.amqp_receive_to_logger(routing_key=method.routing_key, body_decoded=body_decoded) if isinstance(body_decoded, AMQPMessage) and c.AMQP_MSG_TYPE_DEVICE_ADD == body_decoded.message_type: sample_device = body_decoded.payload status, sample_device = Tools.create_dev_conn(sample_device=sample_device) if status: sample_device = Tools.get_device_facts(sample_device=sample_device) status, data = Tools.get_config(lookup_type=c.CONFIG_LOOKUP_TYPE_GET_DEVICE_CFG, sample_device=sample_device) if status: sample_device.deviceConfigData = data try: sample_device.deviceGroup = data['yapt']['device_group'] except KeyError as ke: self._logger.info(Tools.create_log_msg(logmsg.TASKP, sample_device.deviceSerial, logmsg.TASKP_STOP_NO_DEVGRP.format(ke.message))) return status, grp_cfg = Tools.get_config(lookup_type=c.CONFIG_LOOKUP_TYPE_GET_GROUP, sample_device=sample_device) if status: sample_device.deviceGroupData = grp_cfg grp_cfg = Tools.create_config_view(config_type=c.CONFIG_TYPE_GROUP, stream=grp_cfg) sample_device.deviceTemplate = grp_cfg.TASKS.Provision.Configuration.DeviceConfTemplateFile sample_device.deviceTaskSeq = list(grp_cfg.TASKS.Sequence) dev_conn = sample_device.deviceConnection sample_device.deviceConnection = hex(id(sample_device.deviceConnection)) message = AMQPMessage(message_type=c.AMQP_MSG_TYPE_DEVICE_ADD, payload=sample_device, source=c.AMQP_PROCESSOR_TASK) resp = self._backendp.call(message=message) resp = jsonpickle.decode(resp) sample_device = resp.payload[1] sample_device.deviceConnection = dev_conn self._logger.info(Tools.create_log_msg(logmsg.TASKP, sample_device.deviceSerial, logmsg.TASKP_LOAD_TASK_SEQ.format( sample_device.deviceTaskSeq))) c.taskq.add_device_task_q(sample_device=sample_device, grp_cfg=grp_cfg) taskq = c.taskq.get_device_task_q(sn=sample_device.deviceSerial) self._logger.debug(Tools.create_log_msg(self.name, sample_device.deviceSerial, 'Device status: <{0}>'.format( sample_device.deviceStatus))) if taskq: for task in taskq: self._logger.debug(Tools.create_log_msg(self.name, sample_device.deviceSerial, 'Task: <{0}> status: <{1}>'.format( task.task_name, task.task_state))) if task.task_state == c.TASK_STATE_INIT: task.pre_run_task() task.run_task() task.post_run_task() if task.task_state == c.TASK_STATE_REBOOTING: self._logger.debug( Tools.create_log_msg(self.name, sample_device.deviceSerial, 'Task: <{0}> status: <{1}>'.format( task.task_name, task.task_state))) self._logger.info( Tools.create_log_msg(logmsg.TASKP, sample_device.deviceSerial, logmsg.TASKP_STOP_DEV_REBOOT.format( sample_device.deviceSerial))) break elif task.task_state == c.TASK_STATE_FAILED: self._logger.debug( Tools.create_log_msg(self.name, sample_device.deviceSerial, 'Task: <{0}> status: <{1}>'.format( task.task_name, task.task_state))) self._logger.info( Tools.create_log_msg(logmsg.TASKP, sample_device.deviceSerial, logmsg.TASKP_TASK_ERROR.format( task.task_name, ))) self.close_dev_conn(sample_device=sample_device) break elif task.task_state == c.TASK_STATE_REBOOTING: task.pre_run_task() task.run_task() task.post_run_task() if task.task_state == c.TASK_STATE_FAILED: self._logger.debug( Tools.create_log_msg(self.name, sample_device.deviceSerial, 'Task: <{0}> status: <{1}>'.format( task.task_name, task.task_state))) self._logger.info( Tools.create_log_msg(logmsg.TASKP, sample_device.deviceSerial, logmsg.TASKP_TASK_ERROR.format( task.task_name, ))) self.close_dev_conn(sample_device=sample_device) break elif task.task_state == c.TASK_STATE_DONE: pass elif task.task_state == c.TASK_STATE_FAILED: self._logger.debug( Tools.create_log_msg(self.name, sample_device.deviceSerial, 'Task: <{0}> status: <{1}>'.format( task.task_name, task.task_state))) self._logger.info( Tools.create_log_msg(logmsg.TASKP, sample_device.deviceSerial, logmsg.TASKP_TASK_ERROR.format( task.task_name, ))) self.close_dev_conn(sample_device=sample_device) break else: self._logger.info(Tools.create_log_msg(logmsg.TASKP, sample_device.deviceSerial, 'Task <{0}> state: <{1}> unknown'.format( task.task_name, task.task_state))) break # print 'After running all tasks device status is: <{0}>'.format( # sample_device.deviceStatus) if sample_device.deviceStatus == c.DEVICE_STATUS_DONE: c.taskq.del_device_task_q(sample_device.deviceSerial) else: self._logger.info(Tools.create_log_msg(logmsg.TASKP, sample_device.deviceSerial, logmsg.TASKP_STOP_NO_DEVGRP_CFG.format(grp_cfg))) self.close_dev_conn(sample_device=sample_device) return else: self._logger.info(Tools.create_log_msg(logmsg.TASKP, sample_device.deviceSerial, logmsg.TASKP_STOP_NO_DEV_CFG.format(data))) self.close_dev_conn(sample_device=sample_device) return else: self._logger.info(Tools.create_log_msg(logmsg.TASKP, sample_device.deviceSerial, logmsg.TASKP_CONN_ERR.format(sample_device.deviceSerial))) return else: Tools.amqp_receive_error_to_logger(routing_key=method.routing_key, body_decoded=body_decoded) else: self._logger.info(Tools.create_log_msg(logmsg.TASKP, None, logmsg.TASKP_DEFECT_MSG))
def install_device_software(self, path, image, target_version): """ Call PyEz to install new JUNOS image to device :param sample_device: :param path: :param image: :return: """ package = os.path.join(os.getcwd(), path) if c.SERVICEPLUGIN_OSSH in self.sample_device.deviceServicePlugin: try: Tools.emit_log(task_name=self.task_name, sample_device=self.sample_device, message=logmsg.SW_CLEANUP_STORAGE) self.update_task_state( new_task_state=c.TASK_STATE_PROGRESS, task_state_message=logmsg.SW_CLEANUP_STORAGE) self.sample_device.deviceConnection.rpc.request_system_storage_cleanup( ) Tools.emit_log(task_name=self.task_name, sample_device=self.sample_device, message=logmsg.SW_COPY_IMG.format(image)) self.update_task_state( new_task_state=c.TASK_STATE_PROGRESS, task_state_message=logmsg.SW_COPY_IMG.format(image)) # progress = SoftwareTask.copy_progress with SCPClient(transport=self.sample_device.deviceConnection. _conn._session.transport) as scp: scp.put(package, remote_path=self.grp_cfg.TASKS.Provision.Software. RemoteDir) except (BadHostKeyException, AuthenticationException) as e: Tools.emit_log(task_name=self.task_name, sample_device=self.sample_device, message=logmsg.SW_COPY_IMG_NOK.format( e.message)) self.update_task_state( new_task_state=c.TASK_STATE_FAILED, task_state_message=logmsg.SW_COPY_IMG_NOK.format( e.message)) return self.sample_device try: Tools.emit_log( task_name=self.task_name, sample_device=self.sample_device, message=logmsg.SW_INSTALL_VERS.format(target_version)) self.update_task_state( new_task_state=c.TASK_STATE_PROGRESS, task_state_message=logmsg.SW_INSTALL_VERS.format( target_version)) result = self.sample_device.deviceConnection.sw.pkgadd( self.grp_cfg.TASKS.Provision.Software.RemoteDir + image, dev_timeout=self.grp_cfg.TASKS.Provision.Software. PkgAddDevTimeout) except Exception as err: Tools.emit_log(task_name=self.task_name, sample_device=self.sample_device, message=logmsg.SW_INSTALL_NOK.format(str(err))) self.update_task_state( new_task_state=c.TASK_STATE_FAILED, task_state_message=logmsg.SW_INSTALL_NOK.format(str(err))) return self.sample_device if result is True: Tools.emit_log(task_name=self.task_name, sample_device=self.sample_device, message=logmsg.SW_INSTALL_OK.format( self.sample_device.deviceIP)) self.update_task_state( new_task_state=c.TASK_STATE_PROGRESS, task_state_message=logmsg.SW_INSTALL_OK.format( self.sample_device.deviceIP)) else: Tools.emit_log(task_name=self.task_name, sample_device=self.sample_device, message=logmsg.SW_INSTALL_NOK.format( str(result))) self.update_task_state( new_task_state=c.TASK_STATE_FAILED, task_state_message=logmsg.SW_INSTALL_NOK.format( str(result))) time.sleep(3) return self.sample_device Tools.emit_log(task_name=self.task_name, sample_device=self.sample_device, message=logmsg.SW_REBOOT.format( self.sample_device.deviceIP)) self.update_task_state(new_task_state=c.TASK_STATE_PROGRESS, task_state_message=logmsg.SW_REBOOT.format( self.sample_device.deviceIP)) try: rsp = self.sample_device.deviceConnection.sw.reboot() Tools.emit_log(task_name=self.task_name, sample_device=self.sample_device, message=logmsg.SW_REBOOT_DEV_RESP.format( rsp.replace('\n', " "))) self.sample_device.deviceConnection.close() self.sample_device.deviceIsRebooted = True self.update_task_state(new_task_state=c.TASK_STATE_REBOOTING, task_state_message='Rebooting...') c.oss_seen_devices_lck.acquire() try: if self.sample_device.deviceIP in c.oss_seen_devices: c.oss_seen_devices.pop(self.sample_device.deviceIP, None) finally: c.oss_seen_devices_lck.release() return self.sample_device except exception.ConnectClosedError: Tools.emit_log(task_name=self.task_name, sample_device=self.sample_device, message=logmsg.SW_CONN_LOOSE_REBOOT) self.update_task_state( new_task_state=c.TASK_STATE_REBOOTING, task_state_message=logmsg.SW_CONN_LOOSE_REBOOT) return self.sample_device else: try: result = self.sample_device.deviceConnection.sw.install( package=package, remote_path=self.grp_cfg.TASKS.Provision.Software. RemoteDir, cleanfs=True, no_copy=False, progress=SoftwareTask.install_progress) except Exception as err: Tools.emit_log(task_name=self.task_name, sample_device=self.sample_device, message=logmsg.SW_INSTALL_NOK.format(str(err))) self.update_task_state(new_task_state=c.TASK_STATE_FAILED, task_state_message=str(err)) return self.sample_device if result is True: Tools.emit_log(task_name=self.task_name, sample_device=self.sample_device, message=logmsg.SW_INSTALL_OK.format( self.sample_device.deviceIP)) self.update_task_state( new_task_state=c.TASK_STATE_PROGRESS, task_state_message=logmsg.SW_INSTALL_OK.format( self.sample_device.deviceIP)) else: Tools.emit_log(task_name=self.task_name, sample_device=self.sample_device, message=logmsg.SW_INSTALL_NOK.format( str(result))) self.update_task_state( new_task_state=c.TASK_STATE_FAILED, task_state_message=logmsg.SW_INSTALL_NOK.format( str(result))) time.sleep(3) return self.sample_device Tools.emit_log(task_name=self.task_name, sample_device=self.sample_device, message=logmsg.SW_REBOOT.format( self.sample_device.deviceIP)) self.update_task_state(new_task_state=c.TASK_STATE_PROGRESS, task_state_message=logmsg.SW_REBOOT.format( self.sample_device.deviceIP)) try: rsp = self.sample_device.deviceConnection.sw.reboot() Tools.emit_log(task_name=self.task_name, sample_device=self.sample_device, message=logmsg.SW_REBOOT_DEV_RESP.format( rsp.replace('\n', " "))) # self.sample_device.deviceConnection.close() except exception.ConnectClosedError: Tools.emit_log(task_name=self.task_name, sample_device=self.sample_device, message=logmsg.SW_CONN_LOOSE_REBOOT) self.update_task_state( new_task_state=c.TASK_STATE_REBOOTING, task_state_message=logmsg.SW_CONN_LOOSE_REBOOT) finally: alive = self.probe_device_not_alive( self.sample_device, self.grp_cfg.TASKS.Provision.Software.RetryProbeCounter) if not alive: self.sample_device.deviceIsRebooted = True Tools.emit_log(task_name=self.task_name, sample_device=self.sample_device, message=logmsg.SW_PROBE_WAKEUP.format( self.sample_device.deviceIP)) status, self.sample_device = Tools.create_dev_conn( self.sample_device, connect=False) if status: alive = self.probe_device_alive( self.sample_device, self.grp_cfg.TASKS.Provision. Software.RebootProbeTimeout) if alive: Tools.emit_log( task_name=self.task_name, sample_device=self.sample_device, message=logmsg.SW_PROBE_WAKUP_OK.format( self.sample_device.deviceIP)) self.sample_device.deviceIsRebooted = False self.update_task_state( new_task_state=c.TASK_STATE_PROGRESS, task_state_message=logmsg.SW_PROBE_WAKUP_OK. format(self.sample_device.deviceIP)) status, self.sample_device = Tools.create_dev_conn( self.sample_device) if status: self.sample_device.deviceConnection.bind( cu=Config, sw=SW) # Tools.emit_log(task_name=self.task_name, sample_device=self.sample_device, # message=logmsg.SW_CONN_OK.format(self.sample_device.deviceIP)) self.update_task_state( new_task_state=c.TASK_STATE_PROGRESS, task_state_message=logmsg.SW_CONN_OK. format(self.sample_device.deviceIP)) return self.sample_device else: return self.sample_device else: self.update_task_state( new_task_state=c.TASK_STATE_FAILED, task_state_message=c.TASK_STATE_MSG_FAILED) self.sample_device.deviceConnection = None return self.sample_device else: Tools.emit_log(task_name=self.task_name, sample_device=self.sample_device, message=logmsg.SW_PROBE_DEV_NOK.format( self.sample_device.deviceIP, self.grp_cfg.TASKS.Provision.Software. RebootProbeCounter)) self.update_task_state( new_task_state=c.TASK_STATE_FAILED, task_state_message=logmsg.SW_PROBE_DEV_NOK.format( self.sample_device.deviceIP, self.grp_cfg.TASKS. Provision.Software.RebootProbeCounter))
def run_task(self): cancel_chan = ChannelCancellation() e = threading.Event() status, data = Tools.get_config( lookup_type=c.CONFIG_LOOKUP_TYPE_GET_DEVICE_CFG, sample_device=self.sample_device) if status: if self.grp_cfg.TASKS.Provision.Cert.PortForwarding: if self.sample_device.deviceServicePlugin != c.SERVICEPLUGIN_OSSH: with SCP(self.sample_device.deviceConnection, progress=False) as scp: scp.put(c.conf.SERVICES.Ossh.LocalConfigFile, c.SSHD_PORT_FWD_PATH) self.logger.info( Tools.create_log_msg( self.task_name, self.sample_device.deviceSerial, logmsg.CERT_FILE_OK.format( self.sample_device.deviceSerial))) self.sample_device.deviceConnection.close() status, self.sample_device = Tools.create_dev_conn( self.sample_device) if status: thr = threading.Thread(target=self.do_cert_requests, args=( data, e, cancel_chan, )) thr.start() ssh_pfwd = SSHPortForward( sample_device=self.sample_device, grp_cfg=self.grp_cfg, event=e, cancel_chan=cancel_chan) ssh_pfwd.init_port_fwd() else: return False, 'Error in device connection' else: thr = threading.Thread(target=self.do_cert_requests, args=( data, e, cancel_chan, )) thr.start() ssh_pfwd = SSHPortForward(sample_device=self.sample_device, grp_cfg=self.grp_cfg, event=e, cancel_chan=cancel_chan) ssh_pfwd.init_port_fwd() else: self.do_cert_requests(datavars=data, event=None) else: self.logger.info( Tools.create_log_msg(self.task_name, self.sample_device.deviceSerial, logmsg.CERT_DEV_DATA_NOK))