def __trigger_switch_discovery(self): payload = { 'broadcast_frequency': 100, 'interface_name': CPARAMS.WIFI_DEV_FLAG, 'config_file': CPARAMS.WIFI_CONFIG_FILE, 'leader_id': self.deviceID } r = requests.post(self.URL_DISCOVERY_SWITCH_LEADER, json=payload) rjson = r.json() LOG.debug( self.TAG + 'Discovery broadcast start trigger received code: {} with msg: {}'. format(r.status_code, rjson['message'])) self.discovery_switched = rjson['message'] if r.status_code != 200: LOG.warning( self.TAG + 'Discovery broadcast failed with code {}. DHCP trigger is not performed.' .format(r.status_code)) else: payload2 = {'interface_name': CPARAMS.WIFI_DEV_FLAG} r2 = requests.post(self.URL_DISCOVERY_DHCP, json=payload2) rjson2 = r2.json() self.discovery_switched += '|| DHCP: ' + rjson2['message'] if r2.status_code == 200: LOG.debug( self.TAG + 'Discovery DHCP trigger successfully done with message: {}.' .format(rjson2['message'])) return True else: LOG.warning(self.TAG + 'Discovery DHCP trigger failed with code {}'. format(r2.status_code)) return False
def getIPfromFile(file_path=CPARAMS.VPN_FILE_PATH): """ Get the IP of the device in the VPN network from the JSON file generated by the VPN client component. :param file_path: Path to the VPN JSON file :return: string with IP of the device, empty if failure """ ret_ip = '' try: with open(file_path, mode='r') as json_file: json_txt = json_file.readlines()[0] ljson = json.loads(json_txt) if ljson['status'] == 'connected': ret_ip = str(ljson['ip']) LOG.debug( 'VPN IP successfully parsed from JSON file at \'{}\'. Content: {} IP: {}' .format(file_path, str(ljson), ret_ip)) else: LOG.warning( 'VPN JSON status != \'connected\': Content: {}'.format( str(ljson))) except OSError: LOG.exception('VPN file cannot be open or found at \'{}\'.'.format( file_path)) except (IndexError, KeyError): LOG.exception('VPN error on parsing the IP.') except: LOG.exception('VPN generic error.') finally: return ret_ip
def __trigger_triggerCAUclient(self): # payload = { # 'MACaddr': self.MACaddr, # 'detectedLeaderID': self.detectedLeaderID, # 'deviceID': self.deviceID, # 'IDkey': self.IDkey # } s_caucl = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s_caucl.connect(CPARAMS.CAU_CLIENT_ADDR) s_caucl.settimeout(15.) s_caucl.send('detectedLeaderID={},deviceID={}\n'.format( self.detectedLeaderID, self.deviceID).encode()) reply = s_caucl.recv(4092).decode() LOG.debug(self.TAG + 'CAU_client reply: {}'.format(reply)) s_caucl.close() if 'OK' in reply: self.isAuthenticated = True self.secureConnection = True return True else: LOG.warning( self.TAG + 'CAU_client reply \'{}\' evaluated as error'.format(reply)) self.isAuthenticated = False self.secureConnection = False return False
def switch(self, imLeader): if not self.isStarted: LOG.error('Agent is not started!') return False if self.th_proc.is_alive(): LOG.debug( self.TAG + 'Stopping thread {} to switch...'.format(self.th_proc.name)) self._connected = False self.th_proc.join() LOG.debug('Thread successfully stopped.') self._connected = True if self.imLeader != imLeader: LOG.warning( 'imLeader state is not consequent!') # TODO: Action required if self.imLeader: # Switch to Agent LOG.info(self.TAG + 'Switch to Agent') self.imLeader = False self.th_proc = threading.Thread(name='fcjp_agent', target=self.__agent_switch_flow, daemon=True) else: # Switch to Leader LOG.info(self.TAG + 'Switch to Leader') # TODO: Create a thread if we don't want blocking feature (AR wait until leader is set - slow) self.imLeader = True self.th_proc = threading.Thread(name='fcjp_leader', target=self.__leader_switch_flow, daemon=True) self.th_proc.start() return True
def __keeper(self): """ Thread that reduces the TTL of a backup and demotes if TTL <= 0 :return: """ LOG.debug('Keeper is running') with self.backupDatabaseLock: self.backupDatabase = [] # Restart database while self._connected: with self.backupDatabaseLock: for backup in self.backupDatabase: # Reduce TTL backup.TTL -= 1 if backup.TTL < 0: # Backup is down LOG.warning( 'Backup {}[{}] is DOWN with TTL: {}'.format( backup.deviceID, backup.deviceIP, backup.TTL)) self.__send_demotion_message(backup.deviceIP) # Remove from list self.backupDatabase.remove( backup) # TODO: Inform CIMI? LOG.debug('Backup removed from database.') else: # Backup is ok LOG.debug('Backup {}[{}] is OK with TTL: {}'.format( backup.deviceID, backup.deviceIP, backup.TTL)) if self._connected: sleep(self._lpp.get(self._lpp.TIME_KEEPER)) LOG.warning('Keeper thread stopped')
def __send_election_message(self, address): """ :param address: IP address for election :return: True if correct election, False otherwise """ try: r = requests.get('{}backup'.format( URLS.build_url_address(URLS.URL_POLICIES_ROLECHANGE, addr=address, port=self.POLICIES_EXTERNAL_PORT, secure=CPARAMS.MF2C_FLAG)), timeout=1.5, verify=False) if r.status_code == 200: # Correct return True else: LOG.warning( 'Selected device [{}] received {} status code received on electing a new backup' .format(address, r.status_code)) return False except timeout: LOG.warning( 'Selected device [{}] cannot become Backup due timeout'.format( address)) return False except: LOG.exception( 'Selected device [{}] cannot become Backup due error in election message' .format(address)) return False
def __send_demotion_message(self, address): """ Demote a backup to normal agent :param address: IP address for demotion :return: True if correct demotion, False otherwise """ try: r = requests.get('{}agent'.format( URLS.build_url_address(URLS.URL_POLICIES_ROLECHANGE, addr=address, port=CPARAMS.POLICIES_PORT)), timeout=1.5) if r.status_code == 200: # Correct return True else: LOG.warning( 'Selected device [{}] received {} status code received on removing a backup' .format(address, r.status_code)) return False except timeout: LOG.warning( 'Selected device [{}] cannot be demoted to Agent due timeout'. format(address)) return False except: LOG.exception( 'Selected device cannot become Agent due error in demotion message' ) return False
def reelection(arearesilience, deviceID, deviceIP): # 0. Check if is not a backup backups = arearesilience.getBackupDatabase() LOG.debug('Backup database query: {}'.format(backups)) found = False for backup in backups: if backup.deviceID == deviceID: found = True break # If backup: Go to step # 2 if not found: LOG.debug('Device {} is not an active backup.'.format(deviceID)) # 1. Promote device to Backup # Ok? Go to 3, return False otherwise ok = arearesilience.addBackup( deviceID, deviceIP, arearesilience.PRIORITY_ON_REELECTION) if not ok: LOG.error( 'Proposed device cannot be promoted to backup for reelection' ) return False else: LOG.info('Device {} is an active backup.'.format(deviceID)) # 2. Change preference to 0 backup.priority = arearesilience.PRIORITY_ON_REELECTION # 3. Demote other backups (if any) backups = arearesilience.getBackupDatabase() for backup in backups: if backup.deviceID != deviceID: # Delete ok = arearesilience.deleteBackup(backup.deviceID) if ok: LOG.info( 'Backup {}[{}] demoted successfully due Leader Reelection.' .format(backup.deviceID, backup.deviceIP)) else: LOG.error( 'Error on Backup deletion {}[{}] in Leader Reelection.' .format(backup.deviceID, backup.deviceIP)) # 4. Demote leader (rest call self) r = requests.get('{}agent'.format( URLS.build_url_address(URLS.URL_POLICIES_ROLECHANGE, portaddr=('127.0.0.1', CPARAMS.POLICIES_PORT)))) if r.status_code == 200: # Correct LOG.info('Leader (self) demoted successfully') return True LOG.warning('Leader not demoted or confirmation not received') return False
def __backupLeader_flow(self): if not self._connected: LOG.error('Module stoped due _connected = False') return if not self._imLeader: # I've been promoted as backup LOG.info(self.TAG + 'I\'m selected to be a backup. Seting up') self.__preBackupSetup() self.__becomeBackup() if not self._connected: return # Multiple backups support if self._backupPriority > 0: sleep_time = 1. + 10 * (self._backupPriority - 1) LOG.info( 'Waiting {}s before leader takeover...'.format(sleep_time)) sleep(sleep_time) if not self._connected: return LOG.debug('Checking if new Leader is up...') new_leader = self.__getCIMIData('disc_leaderIP', default='') LOG.debug('Stored Leader = [{}], Detected Leader = [{}]'.format( self._leaderIP, new_leader)) if new_leader == '' or new_leader == self._leaderIP: LOG.warning('Leader not detected by Discovery') elif self._leaderIP != new_leader: LOG.info( 'Correct Leader takeover by a backup with more preference.' ) try: # TODO: Clean solution r = requests.get('{}agent'.format( URLS.build_url_address(URLS.URL_POLICIES_ROLECHANGE, addr='127.0.0.1', port=CPARAMS.POLICIES_PORT)), timeout=.5) except: pass finally: return if not self._connected: return if self._imLeader or self._leaderFailed: # I'm a leader LOG.info(self.TAG + 'Leader seting up') self.__becomeLeader() self.__backupSelection() return
def __scanning_flow(self): # 1. Get Beacon # 2. Categorize # 3. Send Categorization info self._socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) try: self._socket.bind(('0.0.0.0', CPARAMS.LDISCOVERY_PORT)) LOG.info('Scan server created correctly') except: LOG.exception('Error on creating the scan receiver') return while self._connected: try: data, addr = self._socket.recvfrom(4096) if not self._connected: break LOG.debug('Received beacon from [{}]: \"{}\"'.format( addr[0], data.decode())) self.leaderIP = addr[0] try: ddata = loads(data.decode()) self.leaderID = ddata.get('leaderID') except JSONDecodeError: LOG.warning('Beacon payload malformed') cpu, mem, stg = self.__categorize_device() LOG.debug('CPU: {}, MEM: {}, STG: {}'.format(cpu, mem, stg)) payload = DeviceInformation(deviceID=self._deviceID, cpuCores=cpu, memAvail=mem, stgAvail=stg).getDict() LOG.info('Sending beacon reply to Leader...') r = requests.post(URLS.build_url_address( URLS.URL_BEACONREPLY, portaddr=(addr[0], CPARAMS.POLICIES_PORT)), json=payload, timeout=2) if r.status_code == 200: LOG.info('Discovery Message successfully sent to Leader') else: LOG.warning( 'Discovery Message received error status code {}'. format(r.status_code)) except: LOG.exception('Error on beacon received') try: self._socket.close() LOG.info('Scan Server Stopped') except: LOG.exception('Server Stop not successful')
def deleteBackupOnAgentResource(backupIP): try: agent_resource, agent_resource_id = CIMI.getAgentResource() ar = AgentResource.load(agent_resource) if ar.backupIP != backupIP: LOG.warning( 'Backup [{}] does not match [{}] stored in Agent Resource'. format(backupIP, ar.backupIP)) ar2 = AgentResource(None, None, None, None, None, backupIP='') LOG.debug( 'Removing backup [{}] in Agent Resource. Updated agent resource: {}' .format(backupIP, ar2.getCIMIdicc())) CIMI.modify_resource(agent_resource_id, ar2.getCIMIdicc()) except: LOG.exception('Add backup in Agent resource failed')
def startScanning(self): if self._isStarted or self._isScanning: LOG.warning( 'LDiscovery is already started: isStarted={} isScanning={}'. format(self._isStarted, self._isScanning)) return False self._th_proc = threading.Thread(name='LDiscS', target=self.__scanning_flow, daemon=True) self._connected = True self._isStarted = True self._isScanning = True self.leaderIP = None self.leaderID = None self._th_proc.start() LOG.info('LDiscovery successfully started in Scan Mode.') return True
def start(self, deviceID): # TODO: Give deviceID at startup? """ :return: """ self._deviceID = deviceID if self.isStarted: LOG.warning(self.TAG + 'Procedure is already started...') return False else: self.th_proc = threading.Thread(name='area_res', target=self.__common_flow, daemon=True) self.th_proc.start() self.isStarted = True LOG.info(self.TAG + 'Module Started') return True
def stopScanning(self): if self._isStarted and self._isScanning: self._connected = False try: self._socket.shutdown(socket.SHUT_RDWR) self._socket.close() LOG.debug('Socket closed on scanning') except: pass self._th_proc.join() LOG.info('LDisc Scanning Stopped') self._isScanning = False self._isStarted = False self.leaderIP = None self.leaderID = None else: LOG.warning('LDisc is not Scanning.') return True
def startBeaconning(self): if self._isStarted or self._isBroadcasting: LOG.warning( 'LDiscovery is already started: isStarted={} isBroadcasting={}' .format(self._isStarted, self._isBroadcasting)) return False self._th_proc = threading.Thread(name='LDiscB', target=self.__beaconning_flow, daemon=True) self._connected = True self._isStarted = True self._isBroadcasting = True self.leaderIP = None self.leaderID = self._deviceID self._db = {} self._th_proc.start() LOG.info('LDiscovery successfully started in Beacon Mode.') return True
def promotedToBackup(self, leaderIP): """ The agent is promoted to be a backup :return: """ # First check if Agent was electable self._leaderIP = leaderIP if self._imCapable: LOG.info(self.TAG + 'Becoming backup due leader selection.') # Then, check if AreaResilience thread is running if self.th_proc is None: pass elif not self.th_proc.is_alive(): pass elif self._imLeader or self._imBackup: LOG.error( 'Agent is already a Backup/Leader. Cannot become a Backup.' ) return False else: LOG.warning( 'Area Resilience still starting. Cannot promote on this state. Waiting...' ) while self.th_proc.is_alive(): sleep(0.1) LOG.debug('Successful waiting.') LOG.debug('Module is ready for promotion.') self.th_proc = threading.Thread(name='area_res', target=self.__backupLeader_flow, daemon=True) self.th_proc.start() self.isStarted = True return True else: if not self._startupCorrect: LOG.warning( 'Area Resilience still starting. Cannot promote on this state.' ) else: LOG.error('Agent not capable to be Backup/Leader') return False
def addBackupOnAgentResource(backupIP): try: agent_resource, agent_resource_id = CIMI.getAgentResource() ar = AgentResource.load(agent_resource) if ar.backupIP is not None or (ar.backupIP != '' and ar.backupIP is not None): LOG.warning( 'Non-empty backupIP value when adding a new backup! Agent resource: {}' .format(ar.getCIMIdicc())) ar2 = AgentResource(None, None, None, None, None, backupIP='{}'.format(backupIP)) LOG.debug( 'Adding backup [{}] in Agent Resource. Updated agent resource: {}'. format(backupIP, ar2.getCIMIdicc())) CIMI.modify_resource(agent_resource_id, ar2.getCIMIdicc()) except: LOG.exception('Add backup in Agent resource failed')
def __trigger_joinDiscovery(self): payload = {'interface': CPARAMS.WIFI_DEV_FLAG, 'bssid': self.bssid} try: r = requests.post(self.URL_DISCOVERY_JOIN, json=payload, timeout=20.) except (timeout, timeout2): LOG.error(self.TAG + 'JOIN trigger timeout.') return False rjson = r.json() if r.status_code != 200: LOG.warning(self.TAG + 'JOIN operation failed. Reply from discovery: {}'. format(rjson['message'])) return False else: LOG.debug(self.TAG + 'Sending MYIP trigger to Discovery...') try: r2 = requests.get(self.URL_DISCOVERY_MYIP, timeout=20.) except (timeout, timeout2): LOG.error(self.TAG + 'MYIP trigger timeout.') return False rjson2 = r2.json() if r2.status_code != 200: LOG.warning(self.TAG + 'MYIP operation failed. Reply from discovery: {}'. format(rjson2)) return False elif 'IP_address' not in rjson2 or rjson2[ 'IP_address'] is None or len(rjson2['IP_address']) == 0: LOG.error( self.TAG + 'MYIP operation returned an OK code BUT IP_address is empty! json: {}' .format(rjson2)) return False LOG.debug(self.TAG + 'MYIP trigger success. Reply form Discovery: {}'.format( rjson2['IP_address'])) self.deviceIP = rjson2['IP_address'] return True
def debug(): sleep(10) LOG.info('Device registration procedure...') attempt = 0 successful = False while attempt < CPARAMS.REGISTRATION_MAX_RETRY and not successful: try: r = requests.post( URLS.build_url_address(URLS.URL_IDENTIFICATION_START, portaddr=('identification', '46060'))) rjson = r.json() LOG.debug('Identification request result: {}'.format(rjson)) if rjson['status'] == '412' and CPARAMS.CLOUD_FLAG: # We need to wait until user and password is registered LOG.warning('Cloud user not registered yet... Retry in 10s.') elif rjson['status'] in ('200', '201'): LOG.info('Successful registration of the device.') successful = True elif CPARAMS.DEBUG_FLAG: LOG.warning( 'Status code received different from 200 or 201. Debug mode skips this failure.' ) successful = True else: LOG.warning('Error on registration trigger. Retry in 10s...') successful = False except ValueError: LOG.debug('ValueError raised on Identification: {}'.format(r.text)) except: LOG.warning('Early start of Identification not successful.') finally: if not successful: sleep(10) attempt += 1 if not CPARAMS.DEBUG_FLAG and not successful: LOG.critical( 'Critical Error: Registration of the device not successful. Stopping module' ) exit(4) sleep(5) LOG.info('Starting Agent Flow...') r = requests.get( URLS.build_url_address(URLS.URL_START_FLOW, portaddr=('127.0.0.1', CPARAMS.POLICIES_PORT))) # r = requests.get(URLS.build_url_address(URLS.URL_POLICIES, portaddr=('127.0.0.1', CPARAMS.POLICIES_PORT))) LOG.debug('Agent Flow request result: {}'.format(r.json())) LOG.debug('Stopping thread activity.') return
def get(self, role): global arearesilience """Promotion/Demotion of the agent role.""" imLeader = arearesilience.imLeader() imBackup = arearesilience.imBackup() if role.lower() == 'leader': # Do you want to be a leader? if imLeader: # If a leader is promoted to leader, it becomes a super-leader? LOG.debug('Role change: Leader -> Leader') return {'imLeader': imLeader, 'imBackup': imBackup}, 403 elif imBackup: # Hi, I'm backup-kun - It's my time to shine!! LOG.debug('Role change: Backup -> Leader') ret = agentstart.switch(imLeader=True) if ret: LOG.info('Successful promotion to Leader') else: LOG.warning('Unsuccessful promotion from Backup to Leader') return {'imLeader': True, 'imBackup': False}, 200 else: # Nor leader, nor Backup, just a normal agent # For reelection, first you must be a backup! LOG.debug('Role change: Agent -> Leader') return {'imLeader': imLeader, 'imBackup': imBackup}, 403 elif role.lower() == 'backup': # Always have a B plan if imLeader: # Why in the hell a Leader'll become a backup? LOG.debug('Role change: Leader -> Backup') return {'imLeader': imLeader, 'imBackup': imBackup}, 403 elif imBackup: # Emm... no pls. LOG.debug('Role change: Backup -> Backup') return {'imLeader': imLeader, 'imBackup': imBackup}, 403 else: # Can you watch my shoulder? LOG.debug('Role change: Agent -> Backup') leaderIP = request.environ.get('HTTP_X_REAL_IP', request.remote_addr) LOG.debug( 'Leader at {} is selecting me as Backup'.format(leaderIP)) ret = arearesilience.promotedToBackup( leaderIP=agentstart.leaderIP ) # TODO: get leaderIP from CIMI if ret: LOG.info('Successful promotion to Backup') return {'imLeader': imLeader, 'imBackup': True}, 200 else: LOG.warning('Unsuccessful promotion from Agent to Backup') return { 'imLeader': arearesilience.imLeader(), 'imBackup': arearesilience.imBackup() }, 403 elif role.lower() == 'agent': # Bigger will be the fall.... if imLeader: # You are shuch an incompetent, you're FIRED! # Leader demotion LOG.debug('Role change: Leader -> Agent') arearesilience.stop() agentstart.switch(imLeader=False) CPARAMS.LEADER_FLAG = False arearesilience = AreaResilience(cimi) arearesilience.start(agentstart.deviceID) return {'imLeader': False, 'imBackup': False}, 200 elif imBackup: # Maybe we are gona call you latter.... or not # Backup demotion LOG.debug('Role change: Backup -> Agent') arearesilience.stop() arearesilience = AreaResilience(cimi) arearesilience.start(agentstart.deviceID) return {'imLeader': False, 'imBackup': False}, 200 else: # You're so tiny that I don't even care. LOG.debug('Role change: Agent -> Agent') return {'imLeader': False, 'imBackup': False}, 403 else: # keikaku doori... Weird syntax maybe? return {'imLeader': imLeader, 'imBackup': imBackup}, 404
def __agent_startup_flow(self): while self._connected: # 0. Init self.detectedLeaderID, self.MACaddr = None, None # 0.1 Check CIMI is UP CIMIon = False while self._connected and not CIMIon: CIMIon = CIMI.checkCIMIstarted() if not CIMIon: LOG.debug(self.TAG + 'CIMI is not ready... Retry in {}s'.format( self.WAIT_TIME_CIMI)) sleep(self.WAIT_TIME_CIMI) LOG.info(self.TAG + 'CIMI is ready!') # 1. Identification if self._connected: self.identification_failed = True # Reset variable to avoid false positives LOG.debug(self.TAG + 'Sending trigger to Identification...') try: self.__trigger_requestID() self.identification_failed = False except Exception: LOG.exception(self.TAG + 'Identification trigger failed!') self.identification_failed = True LOG.info(self.TAG + 'Identification Trigger Done.') else: return if not CPARAMS.DEBUG_FLAG and self.identification_failed: LOG.critical( self.TAG + 'Identification failed, interrupting agent start.') return # 2. Check if im a Leader - PLE if self.imLeader: # switch to leader self.__leader_switch_flow() # TODO: imCapable? return # remain as agent # 3. Scan for Leaders count = 0 self.discovery_failed = True while self._connected and count < self.MAX_MISSING_SCANS and self.detectedLeaderID is None and self.MACaddr is None: # TODO: new protocol required LOG.debug(self.TAG + 'Sending scan trigger to Discovery...') try: self.__trigger_startScan() self.discovery_failed = False except Exception: LOG.debug(self.TAG + 'Discovery failed on attepmt {}.'.format(count)) self.discovery_failed = True if self.detectedLeaderID is not None and self.MACaddr is not None: LOG.info(self.TAG + 'Discovery Scan Trigger Done.') count += 1 LOG.info( self.TAG + 'Discovery trigger finished in #{} attempts and ok={}'.format( count, self.detectedLeaderID is not None and self.MACaddr is not None)) if not self._connected: return if not CPARAMS.DEBUG_FLAG and self.discovery_failed: LOG.critical(self.TAG + 'Discovery failed, interrupting agent start.') return # 4. If no leader detected, switch to leader IF policy and capable - ALE if not self.discovery_failed and self.detectedLeaderID is None and self.MACaddr is None and self.ALE_ENABLED: self.__leader_switch_flow() # TODO: imCapable? return # 5. CAU client if self._connected: self.cauclient_failed = True LOG.debug(self.TAG + 'Sending trigger to CAU client...') try: self.__trigger_triggerCAUclient() self.cauclient_failed = False except Exception: LOG.exception(self.TAG + 'CAUclient failed.') self.cauclient_failed = True LOG.info(self.TAG + 'CAU client Trigger Done.') else: return if not CPARAMS.DEBUG_FLAG and self.cauclient_failed: LOG.critical(self.TAG + 'CAU-Client failed, interrupting agent start.') return # 5. Categorization if self._connected and not self.categorization_started: self.categorization_failed = True LOG.debug(self.TAG + 'Sending start trigger to Categorization...') try: self.__trigger_startCategorization() self.categorization_failed = False self.categorization_started = True except Exception: LOG.exception(self.TAG + 'Categorization failed') self.categorization_failed = True LOG.info(self.TAG + 'Categorization Start Trigger Done.') elif not self._connected: return if not CPARAMS.DEBUG_FLAG and self.categorization_failed: LOG.critical( self.TAG + 'Categorization failed, interrupting agent start.') return # 6. Area Resilience if self._connected and not self.arearesilience_started: self.policies_failed = True LOG.debug(self.TAG + 'Sending start trigger to Policies...') try: success = self.__trigger_startLeaderProtectionPolicies() self.policies_failed = not success self.arearesilience_started = success except Exception: LOG.exception(self.TAG + 'Policies Area Resilience failed!') LOG.info(self.TAG + 'Policies Area Resilience Start Trigger Done.') elif not self._connected: return if not CPARAMS.DEBUG_FLAG and self.policies_failed: LOG.critical( self.TAG + 'Policies Area Resilience failed, interrupting agent start.' ) return # Print summary self.__print_summary() # Create/Modify Agent Resource # TODO. self.deviceIP = '' # TODO: Real value here (from categorization) self._cimi_agent_resource = AgentResource(self.deviceID, self.deviceIP, self.isAuthenticated, self.secureConnection, self.imLeader, leaderIP=self.leaderIP) if self._cimi_agent_resource_id is None: # Create agent resource self._cimi_agent_resource_id = CIMI.createAgentResource( self._cimi_agent_resource.getCIMIdicc()) else: # Agent resource already exists status = CIMI.modify_resource( self._cimi_agent_resource_id, self._cimi_agent_resource.getCIMIdicc()) # 7. Watch Leader if self._connected and not self.discovery_failed: LOG.debug(self.TAG + 'Start Discovery Leader Watch...') try: self.__trigger_startDiscoveryWatch() except Exception: LOG.exception(self.TAG + 'Watch Discovery Start Fail.') LOG.info(self.TAG + 'Watch Discovery Start Trigger Done.') elif self.discovery_failed: LOG.warning( self.TAG + 'Discovery Watch cancelled due Discovery Trigger failed') else: return alive = True while self._connected and not self.discovery_failed and alive: # 6 Check if discovery connection is alive LOG.debug(self.TAG + 'Discovery Alive Start Trigger.') try: alive = not self.__trigger_aliveDiscovery( ) # not disconnected except Exception: LOG.exception(self.TAG + 'Discovery Alive failed') alive = False if self._connected: sleep(CPARAMS.TIME_WAIT_ALIVE) LOG.info(self.TAG + 'Discovery Alive Start Trigger Done.') if not self._connected: return if CPARAMS.DEBUG_FLAG and self.discovery_failed: # TODO: Delete this in future versions LOG.debug(self.TAG + 'No rescan available. Stoping activity') return
def __agent_startup_flow(self): while self._connected: # 0. Init self.detectedLeaderID, self.MACaddr, self.bssid = None, None, None # 0.1 Check CIMI is UP CIMIon = False while self._connected and not CIMIon: CIMIon = CIMI.checkCIMIstarted() if not CIMIon: LOG.debug(self.TAG + 'CIMI is not ready... Retry in {}s'.format( self.WAIT_TIME_CIMI)) sleep(self.WAIT_TIME_CIMI) LOG.info(self.TAG + 'CIMI is ready!') # 1. Identification if self._connected: self.identification_failed = True # Reset variable to avoid false positives LOG.debug(self.TAG + 'Sending trigger to Identification...') try: self.__trigger_requestID() self.identification_failed = False except Exception: LOG.exception(self.TAG + 'Identification trigger failed!') self.identification_failed = True LOG.info(self.TAG + 'Identification Trigger Done.') else: return if not CPARAMS.DEBUG_FLAG and self.identification_failed: LOG.critical( self.TAG + 'Identification failed, interrupting agent start.') return # 2.1. Check if im Cloud Agent if self.imCloud: # start cloud flow self.__cloud_flow() return # 2.2. Check if im a Leader - PLE if self.imLeader: # switch to leader self.__leader_switch_flow() return # remain as agent # 3. Scan for Leaders count = 0 self.discovery_failed = True while self._connected and count < self.MAX_MISSING_SCANS and self.detectedLeaderID is None and self.MACaddr is None: # TODO: new protocol required LOG.debug(self.TAG + 'Sending SCAN trigger to Discovery...') try: self.__trigger_startScan() self.discovery_failed = False except Exception: LOG.debug(self.TAG + 'Discovery failed on attepmt {}.'.format(count)) self.discovery_failed = True if self.detectedLeaderID is not None and self.MACaddr is not None and self.bssid is not None: LOG.info(self.TAG + 'Discovery Scan Trigger Done.') count += 1 LOG.info( self.TAG + 'Discovery trigger finished in #{} attempts and ok={}'.format( count, self.detectedLeaderID is not None and self.MACaddr is not None and self.bssid is not None)) if not self._connected: return if not CPARAMS.DEBUG_FLAG and self.discovery_failed: LOG.critical(self.TAG + 'Discovery failed, interrupting agent start.') return # 4.1. If no leader detected, switch to leader IF policy and capable - ALE if not self.discovery_failed and self.detectedLeaderID is None and self.MACaddr is None and self.bssid is None and self.ALE_ENABLED: self.__leader_switch_flow() # TODO: imCapable? return # 4.2 If detected, join to the Leader if not self.discovery_failed and self.bssid is not None and self._connected: LOG.debug(self.TAG + 'Sending JOIN trigger to discovery...') try: r = self.__trigger_joinDiscovery() self.discovery_failed = not r self.discovery_joined = r if not self.discovery_failed: self.leaderIP = CPARAMS.LEADER_DISCOVERY_IP except Exception: LOG.exception(self.TAG + 'Discovery JOIN trigger failed.') self.discovery_failed = True self.discovery_joined = False LOG.debug(self.TAG + 'Discovery JOIN trigger Done.') # 4.3 If not detected or failed, static configuration if setup if self.discovery_failed or (self.detectedLeaderID is None and self.MACaddr is None and self.bssid is None): LOG.debug( self.TAG + 'Discovery failed or leader was not detected. Fetching deviceIP and leaderIP from env variables.' ) self.deviceIP = CPARAMS.DEVICE_IP_FLAG self.leaderIP = CPARAMS.LEADER_IP_FLAG # 5. CAU client if self._connected: self.cauclient_failed = True LOG.debug(self.TAG + 'Sending trigger to CAU client...') attempt = 0 r = False while self._connected and not r and attempt < self.MAX_CAUCLIENT_FAILURES: try: r = self.__trigger_triggerCAUclient() self.cauclient_failed = not r except Exception: LOG.exception(self.TAG + 'CAUclient failed.') self.cauclient_failed = True finally: attempt += 1 if not r: sleep(self.WAIT_TIME_CAUCLIENT) LOG.info(self.TAG + 'CAU client Trigger Done in {} attempts and ok={}.'. format(attempt, r)) else: return if not CPARAMS.DEBUG_FLAG and self.cauclient_failed: LOG.critical(self.TAG + 'CAU-Client failed, interrupting agent start.') return # 5.1. VPN get IP attempt = 0 while self._connected and self.vpnIP is None and attempt < self.MAX_VPN_FAILURES: vpn_ip = VPN.getIPfromFile() self.vpnIP = vpn_ip if vpn_ip != '' else None if self.vpnIP is None: LOG.debug(self.TAG + 'VPN IP cannot be obtained... Retry in {}s'. format(self.WAIT_TIME_VPN)) sleep(self.WAIT_TIME_VPN) attempt += 1 if self.vpnIP is None: LOG.warning(self.TAG + 'VPN IP cannot be obtained.') if not CPARAMS.DEBUG_FLAG: LOG.critical( self.TAG + 'Policies module cannot continue its activity without VPN IP' ) exit(4) else: LOG.info(self.TAG + 'VPN IP: [{}]'.format(self.vpnIP)) # 5.2 If not static configuration and no leader detected, VPN configuration if self.deviceIP is None and self.leaderIP is None: LOG.debug( self.TAG + 'Static configuration for deviceIP and leaderIP not found. Using VPN values' ) self.deviceIP = self.vpnIP self.leaderIP = self.cloudIP LOG.info(self.TAG + 'deviceIP={}, leaderIP={}'.format( self.deviceIP, self.leaderIP)) # 6. Categorization if self._connected and not self.categorization_started: self.categorization_failed = True LOG.debug(self.TAG + 'Sending start trigger to Categorization...') try: self.__trigger_startCategorization() self.categorization_failed = False self.categorization_started = True except Exception: LOG.exception(self.TAG + 'Categorization failed') self.categorization_failed = True LOG.info(self.TAG + 'Categorization Start Trigger Done.') elif not self._connected: return if not CPARAMS.DEBUG_FLAG and self.categorization_failed: LOG.critical( self.TAG + 'Categorization failed, interrupting agent start.') return # 7. Area Resilience if self._connected and not self.arearesilience_started: self.policies_failed = True LOG.debug(self.TAG + 'Sending start trigger to Policies...') try: success = self.__trigger_startLeaderProtectionPolicies() self.policies_failed = not success self.arearesilience_started = success except Exception: LOG.exception(self.TAG + 'Policies Area Resilience failed!') LOG.info(self.TAG + 'Policies Area Resilience Start Trigger Done.') elif not self._connected: return if not CPARAMS.DEBUG_FLAG and self.policies_failed: LOG.critical( self.TAG + 'Policies Area Resilience failed, interrupting agent start.' ) return # Print summary self.__print_summary() # Create/Modify Agent Resource self.deviceIP = '' if self.deviceIP is None else self.deviceIP self._cimi_agent_resource = AgentResource(self.deviceID, self.deviceIP, self.isAuthenticated, self.secureConnection, self.imLeader) LOG.debug(self.TAG + 'CIMI Agent Resource payload: {}'.format( self._cimi_agent_resource.getCIMIdicc())) if self._cimi_agent_resource_id is None: # Create agent resource self._cimi_agent_resource_id = CIMI.createAgentResource( self._cimi_agent_resource.getCIMIdicc()) sleep(.1) self._cimi_agent_resource = AgentResource( self.deviceID, self.deviceIP, self.isAuthenticated, self.secureConnection, self.imLeader, leaderIP=self.leaderIP) LOG.debug(self.TAG + 'CIMI Agent Resource payload: {}'.format( self._cimi_agent_resource.getCIMIdicc())) status = CIMI.modify_resource( self._cimi_agent_resource_id, self._cimi_agent_resource.getCIMIdicc()) if self._cimi_agent_resource_id == '': LOG.warning(self.TAG + 'Agent resource creation failed.') if not CPARAMS.DEBUG_FLAG: LOG.error( 'Stopping Policies module due to resource creation failure.' ) exit(4) else: # Agent resource already exists status = CIMI.modify_resource( self._cimi_agent_resource_id, self._cimi_agent_resource.getCIMIdicc()) sleep(.1) self._cimi_agent_resource = AgentResource( self.deviceID, self.deviceIP, self.isAuthenticated, self.secureConnection, self.imLeader, leaderIP=self.leaderIP) LOG.debug(self.TAG + 'CIMI Agent Resource payload: {}'.format( self._cimi_agent_resource.getCIMIdicc())) status = CIMI.modify_resource( self._cimi_agent_resource_id, self._cimi_agent_resource.getCIMIdicc()) # 8. Watch Leader if self._connected and not self.discovery_failed: LOG.debug(self.TAG + 'Start Discovery Leader Watch...') try: self.__trigger_startDiscoveryWatch() except Exception: LOG.exception(self.TAG + 'Watch Discovery Start Fail.') LOG.info(self.TAG + 'Watch Discovery Start Trigger Done.') elif self.discovery_failed: LOG.warning( self.TAG + 'Discovery Watch cancelled due Discovery Trigger failed') else: return self.isCompleted = True alive = True while self._connected and not self.discovery_failed and alive: # 6 Check if discovery connection is alive LOG.debug(self.TAG + 'Discovery Alive Start Trigger.') try: alive = not self.__trigger_aliveDiscovery( ) # not disconnected except Exception: LOG.exception(self.TAG + 'Discovery Alive failed') alive = False if self._connected: sleep(CPARAMS.TIME_WAIT_ALIVE) LOG.info(self.TAG + 'Discovery Alive Start Trigger Done.') if not self._connected: return if CPARAMS.DEBUG_FLAG and self.discovery_failed: LOG.debug(self.TAG + 'No rescan available. Stoping activity') return
def __leader_switch_flow(self): """ Agent become leader :return: """ # 1. Start sending beacons if self._connected: self.discovery_leader_failed = True self.discovery_failed = True LOG.debug(self.TAG + 'Sending Broadcast trigger to discovery...') try: r = self.__trigger_switch_discovery() self.detectedLeaderID = self.deviceID self.discovery_leader_failed = not r except Exception as ex: LOG.exception(self.TAG + 'Discovery broadcast trigger failed!') self.detectedLeaderID = self.deviceID LOG.info(self.TAG + 'Discovery Broadcast Trigger Done.') else: return if not CPARAMS.DEBUG_FLAG and self.discovery_leader_failed: LOG.critical( self.TAG + 'Discovery broadcast failed, interrupting leader switch.') return self.discovery_failed = self.discovery_leader_failed # 2. Start CAU-client if self._connected: self.cauclient_failed = True LOG.debug(self.TAG + 'Sending trigger to CAU client...') attempt = 0 r = False while self._connected and not r and attempt < self.MAX_CAUCLIENT_FAILURES: try: r = self.__trigger_triggerCAUclient() self.cauclient_failed = not r except Exception: LOG.exception(self.TAG + 'CAUclient failed.') self.cauclient_failed = True finally: attempt += 1 if not r: sleep(self.WAIT_TIME_CAUCLIENT) LOG.info(self.TAG + 'CAU client Trigger Done in {} attempts and ok={}.'. format(attempt, r)) else: return if not CPARAMS.DEBUG_FLAG and self.cauclient_failed: LOG.critical(self.TAG + 'CAU-Client failed, interrupting agent start.') return # 3. VPN get IP attempt = 0 while self._connected and self.vpnIP is None and attempt < self.MAX_VPN_FAILURES: vpn_ip = VPN.getIPfromFile() self.vpnIP = vpn_ip if vpn_ip != '' else None if self.vpnIP is None: LOG.debug(self.TAG + 'VPN IP cannot be obtained... Retry in {}s'.format( self.WAIT_TIME_VPN)) sleep(self.WAIT_TIME_VPN) attempt += 1 if self.vpnIP is None: LOG.warning(self.TAG + 'VPN IP cannot be obtained.') if not CPARAMS.DEBUG_FLAG: LOG.critical( self.TAG + 'Policies module cannot continue its activity without VPN IP' ) exit(4) else: LOG.info(self.TAG + 'VPN IP: [{}]'.format(self.vpnIP)) # 4. Switch leader categorization (or start if not started) if self.categorization_started: self.categorization_leader_failed = True # Switch! LOG.debug(self.TAG + 'Sending switch trigger to Categorization...') try: self.__trigger_switch_categorization() self.categorization_leader_failed = False except Exception: LOG.exception(self.TAG + 'Categorization switch to leader failed') LOG.info(self.TAG + 'Categorization Switch Trigger Done.') else: # Start as leader! LOG.debug(self.TAG + 'Sending start trigger to Categorization...') try: self.__trigger_startCategorization() self.categorization_leader_failed = False self.categorization_started = True except Exception: LOG.exception(self.TAG + 'Categorization failed') LOG.info(self.TAG + 'Categorization Start Trigger Done.') if not CPARAMS.DEBUG_FLAG and self.categorization_leader_failed: LOG.critical(self.TAG + 'Categorization failed, interrupting leader switch.') return self.categorization_failed = self.categorization_leader_failed # 5. Start Area Resilience (if not started) if not self.arearesilience_started: self.policies_failed = True LOG.debug(self.TAG + 'Sending start trigger to Policies...') try: self.__trigger_startLeaderProtectionPolicies() self.policies_failed = False self.arearesilience_started = True except Exception: LOG.exception(self.TAG + 'Policies Area Resilience failed!') LOG.info(self.TAG + 'Policies Area Resilience Start Trigger Done.') if not CPARAMS.DEBUG_FLAG and self.policies_failed: LOG.critical( self.TAG + 'Policies Area Resilience failed, interrupting agent start.') return # 8. Watch Leader if self._connected and not self.discovery_failed: LOG.debug(self.TAG + 'Start Discovery Leader Watch...') try: self.__trigger_startDiscoveryWatchLeader() except Exception: LOG.exception(self.TAG + 'Watch Discovery Start Fail.') LOG.info(self.TAG + 'Watch Discovery Start Trigger Done.') elif self.discovery_failed: LOG.warning( self.TAG + 'Discovery Watch cancelled due Discovery Trigger failed') # Create/Modify Agent Resource # IF static IP configuration setup self.deviceIP = CPARAMS.DEVICE_IP_FLAG self.leaderIP = CPARAMS.LEADER_IP_FLAG if self.deviceIP is None or self.leaderIP is None: LOG.debug( self.TAG + 'No static configuration was detected. Applying VPN values') self.deviceIP = self.vpnIP self.leaderIP = self.cloudIP LOG.info( self.TAG + 'deviceIP={}, leaderIP={}'.format(self.deviceIP, self.leaderIP)) # Print summary self.__print_summary() self._cimi_agent_resource = AgentResource(self.deviceID, self.deviceIP, True, True, self.imLeader, leaderIP=self.leaderIP) # deprecated: real values of Auth and Conn (as now are None in the Leader) if self._cimi_agent_resource_id is None: # Create agent resource self._cimi_agent_resource_id = CIMI.createAgentResource( self._cimi_agent_resource.getCIMIdicc()) else: # Agent resource already exists status = CIMI.modify_resource( self._cimi_agent_resource_id, self._cimi_agent_resource.getCIMIdicc()) # 6. Finish self.isCompleted = True return
def __cloud_flow(self): LOG.info(self.TAG + 'Cloud flow started.') # 0. Cloud Agent is Leader by definition self.imLeader = self.imCloud # 1. Discovery LOG.debug(self.TAG + 'Discovery trigger ignored in Cloud flow.') self.discovery_failed = False self.discovery_leader_failed = False self.detectedLeaderID = self.deviceID # 2. Start CAU-client if self._connected: self.cauclient_failed = True LOG.debug(self.TAG + 'Sending trigger to CAU client...') attempt = 0 r = False while self._connected and not r and attempt < self.MAX_CAUCLIENT_FAILURES: try: r = self.__trigger_triggerCAUclient() self.cauclient_failed = not r except Exception: LOG.exception(self.TAG + 'CAUclient failed.') self.cauclient_failed = True finally: attempt += 1 if not r: sleep(self.WAIT_TIME_CAUCLIENT) LOG.info(self.TAG + 'CAU client Trigger Done in {} attempts and ok={}.'. format(attempt, r)) else: return if not CPARAMS.DEBUG_FLAG and self.cauclient_failed: LOG.critical(self.TAG + 'CAU-Client failed, interrupting agent start.') return # 3. VPN get IP attempt = 0 while self._connected and self.vpnIP is None and attempt < self.MAX_VPN_FAILURES: vpn_ip = VPN.getIPfromFile() self.vpnIP = vpn_ip if vpn_ip != '' else None if self.vpnIP is None: LOG.debug(self.TAG + 'VPN IP cannot be obtained... Retry in {}s'.format( self.WAIT_TIME_VPN)) sleep(self.WAIT_TIME_VPN) attempt += 1 if self.vpnIP is None: LOG.warning(self.TAG + 'VPN IP cannot be obtained.') if not CPARAMS.DEBUG_FLAG: LOG.critical( self.TAG + 'Policies module cannot continue its activity without VPN IP' ) exit(4) else: LOG.info(self.TAG + 'VPN IP: [{}]'.format(self.vpnIP)) # 4. Switch leader categorization (or start if not started) if self._connected and not self.categorization_started: self.categorization_failed = True LOG.debug(self.TAG + 'Sending start trigger to Categorization...') try: self.__trigger_startCategorization() self.categorization_failed = False self.categorization_started = True except Exception: LOG.exception(self.TAG + 'Categorization failed') self.categorization_failed = True LOG.info(self.TAG + 'Categorization Start Trigger Done.') elif not self._connected: return if not CPARAMS.DEBUG_FLAG and self.categorization_failed: LOG.critical(self.TAG + 'Categorization failed, interrupting agent start.') return # 5. Area Resilience LOG.debug(self.TAG + 'Area Resilience trigger ignored in Cloud flow.') self.policies_failed = False # Print summary self.__print_summary() # Create Agent Resource self.deviceIP = self.vpnIP self.leaderIP = None self._cimi_agent_resource = AgentResource(self.deviceID, self.deviceIP, True, True, self.imLeader) LOG.debug(self.TAG + 'CIMI Agent Resource payload: {}'.format( self._cimi_agent_resource.getCIMIdicc())) if self._cimi_agent_resource_id is None: # Create agent resource self._cimi_agent_resource_id = CIMI.createAgentResource( self._cimi_agent_resource.getCIMIdicc()) if self._cimi_agent_resource_id == '': LOG.warning(self.TAG + 'Agent resource creation failed.') if not CPARAMS.DEBUG_FLAG: LOG.error( 'Stopping Policies module due to resource creation failure.' ) exit(4) else: # Agent resource already exists status = CIMI.modify_resource( self._cimi_agent_resource_id, self._cimi_agent_resource.getCIMIdicc()) self.isCompleted = True return
def __backupSelection(self): """ :return: """ # TODO: # 1- Check Backups # 2- Enough backups? # YES: End sleep(X) # NO: # 3- Get topology and select one agent # If not capable: Go to 3 # 4- promote to Backup # Success: Correct_backups++ # Go to 2 while self._connected: correct_backups = 0 with self.backupDatabaseLock: # Check backups for backup in self.backupDatabase: if backup.TTL >= 0: correct_backups += 1 # Enough? if correct_backups >= self._lpp.get(self._lpp.BACKUP_MINIMUM, default=1): # Enough backups LOG.debug( '{} correct backup detected in Leader. Everything is OK.'. format(correct_backups)) else: # Not enough if not self._connected: break LOG.warning( '{} backup dettected are not enough. Electing new ones...'. format(correct_backups)) topology = self.__getTopology() new_backups = [] while self._connected and correct_backups < self._lpp.get( self._lpp.BACKUP_MINIMUM, default=1) and len(topology) > 0: device = topology[0] topology.remove(device) found = False with self.backupDatabaseLock: for backup in self.backupDatabase: if backup.deviceID == device.get('deviceID'): found = True break if not found: # Todo: Evaluate if selected device is capable correct = self.__send_election_message( device.get('deviceIP')) if correct: new_backup = BackupEntry(device.get('deviceID'), device.get('deviceIP'), self._nextPriority) with self.backupDatabaseLock: self.backupDatabase.append(new_backup) LOG.info( 'Backup {}[{}] added with priority {}'.format( device.get('deviceID'), device.get('deviceIP'), self._nextPriority)) correct_backups += 1 self._nextPriority += 1 new_backups.append(new_backups) if correct_backups >= self._lpp.get(self._lpp.BACKUP_MINIMUM, default=1): # Now we have enough LOG.info( '{} correct backups dettected in Leader. {} new backups added.' .format(correct_backups, len(new_backups))) else: LOG.warning( '{} backups dettected are not enough. Waiting for new election.' .format(correct_backups)) # Sleep if self._connected: sleep(self._lpp.get(self._lpp.TIME_TO_WAIT_BACKUP_SELECTION)) LOG.info('Leader stopped...')
def __becomeBackup(self): """ :return: """ # 1- Send the KeepAlive message to the leader. # 2- Receive the reply (with preference number). # If leader down, Backup becomes leader. # Else repeat. attempt = 0 counter = 0 payload = {'deviceID': self._deviceID} self._imBackup = True while self._connected and attempt < self._lpp.get( self._lpp.MAX_RETRY_ATTEMPTS): stopLoop = False while self._connected and not stopLoop: try: # 1. Requests to Leader Keepalive endpoint r = requests.post(URLS.build_url_address( URLS.URL_POLICIES_KEEPALIVE, portaddr=(self._leaderIP, CPARAMS.POLICIES_PORT)), json=payload, timeout=0.5) LOG.debug(self.TAG + 'Keepalive sent [#{}]'.format(counter)) # 2. Process Reply jreply = r.json() if r.status_code == 200: leaderID = jreply['deviceID'] # Todo: Use this priority = jreply['backupPriority'] # 3. Update Preference self._backupPriority = priority LOG.debug( self.TAG + 'Reply received, Leader still alive: LeaderID: {}'. format(leaderID)) attempt = 0 else: # Error? LOG.error('KeepAlive status_code = {}'.format( r.status_code)) if r.status_code == 403 and self.PRIORITY_ON_DEMOTION == jreply[ 'backupPriority']: LOG.warning( 'Backup has been removed from database or not authorized to send keepalive messages' ) elif r.status_code == 405 and self.PRIORITY_ON_FAILURE == jreply[ 'backupPriority']: LOG.warning( 'Sending message to a Device that is not a Leader!' ) stopLoop = True else: stopLoop = True if not stopLoop: # 4. Sleep sleep(self._lpp.get(self._lpp.TIME_KEEPALIVE)) counter += 1 except: # Connection broke, backup assumes that Leader is down. LOG.debug('Keepalive connection refused') stopLoop = True LOG.warning( 'Keepalive connection is broken... Retry Attempts: {}'.format( self._lpp.get(self._lpp.MAX_RETRY_ATTEMPTS) - (attempt + 1))) attempt += 1 if not self._connected: LOG.info('Backup stopped.') else: LOG.warning(self.TAG + '## LEADER IS DOWN! ##') self._leaderFailed = True return