def __trigger_switch_discovery(self):
payload = {
'broadcast_frequency': 100,
'interface_name': CPARAMS.WIFI_DEV_FLAG,
'config_file': CPARAMS.WIFI_CONFIG_FILE,
'leader_id': self.deviceID
}
r = requests.post(self.URL_DISCOVERY_SWITCH_LEADER, json=payload)
rjson = r.json()
LOG.debug(
self.TAG +
'Discovery broadcast start trigger received code: {} with msg: {}'.
format(r.status_code, rjson['message']))
self.discovery_switched = rjson['message']
if r.status_code != 200:
LOG.warning(
self.TAG +
'Discovery broadcast failed with code {}. DHCP trigger is not performed.'
.format(r.status_code))
else:
payload2 = {'interface_name': CPARAMS.WIFI_DEV_FLAG}
r2 = requests.post(self.URL_DISCOVERY_DHCP, json=payload2)
rjson2 = r2.json()
self.discovery_switched += '|| DHCP: ' + rjson2['message']
if r2.status_code == 200:
LOG.debug(
self.TAG +
'Discovery DHCP trigger successfully done with message: {}.'
.format(rjson2['message']))
return True
else:
LOG.warning(self.TAG +
'Discovery DHCP trigger failed with code {}'.
format(r2.status_code))
return False
def getIPfromFile(file_path=CPARAMS.VPN_FILE_PATH):
"""
Get the IP of the device in the VPN network from the JSON file generated by the VPN client component.
:param file_path: Path to the VPN JSON file
:return: string with IP of the device, empty if failure
"""
ret_ip = ''
try:
with open(file_path, mode='r') as json_file:
json_txt = json_file.readlines()[0]
ljson = json.loads(json_txt)
if ljson['status'] == 'connected':
ret_ip = str(ljson['ip'])
LOG.debug(
'VPN IP successfully parsed from JSON file at \'{}\'. Content: {} IP: {}'
.format(file_path, str(ljson), ret_ip))
else:
LOG.warning(
'VPN JSON status != \'connected\': Content: {}'.format(
str(ljson)))
except OSError:
LOG.exception('VPN file cannot be open or found at \'{}\'.'.format(
file_path))
except (IndexError, KeyError):
LOG.exception('VPN error on parsing the IP.')
except:
LOG.exception('VPN generic error.')
finally:
return ret_ip
def __trigger_triggerCAUclient(self):
# payload = {
# 'MACaddr': self.MACaddr,
# 'detectedLeaderID': self.detectedLeaderID,
# 'deviceID': self.deviceID,
# 'IDkey': self.IDkey
# }
s_caucl = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s_caucl.connect(CPARAMS.CAU_CLIENT_ADDR)
s_caucl.settimeout(15.)
s_caucl.send('detectedLeaderID={},deviceID={}\n'.format(
self.detectedLeaderID, self.deviceID).encode())
reply = s_caucl.recv(4092).decode()
LOG.debug(self.TAG + 'CAU_client reply: {}'.format(reply))
s_caucl.close()
if 'OK' in reply:
self.isAuthenticated = True
self.secureConnection = True
return True
else:
LOG.warning(
self.TAG +
'CAU_client reply \'{}\' evaluated as error'.format(reply))
self.isAuthenticated = False
self.secureConnection = False
return False
def switch(self, imLeader):
if not self.isStarted:
LOG.error('Agent is not started!')
return False
if self.th_proc.is_alive():
LOG.debug(
self.TAG +
'Stopping thread {} to switch...'.format(self.th_proc.name))
self._connected = False
self.th_proc.join()
LOG.debug('Thread successfully stopped.')
self._connected = True
if self.imLeader != imLeader:
LOG.warning(
'imLeader state is not consequent!') # TODO: Action required
if self.imLeader:
# Switch to Agent
LOG.info(self.TAG + 'Switch to Agent')
self.imLeader = False
self.th_proc = threading.Thread(name='fcjp_agent',
target=self.__agent_switch_flow,
daemon=True)
else:
# Switch to Leader
LOG.info(self.TAG + 'Switch to Leader')
# TODO: Create a thread if we don't want blocking feature (AR wait until leader is set - slow)
self.imLeader = True
self.th_proc = threading.Thread(name='fcjp_leader',
target=self.__leader_switch_flow,
daemon=True)
self.th_proc.start()
return True
def __keeper(self):
"""
Thread that reduces the TTL of a backup and demotes if TTL <= 0
:return:
"""
LOG.debug('Keeper is running')
with self.backupDatabaseLock:
self.backupDatabase = [] # Restart database
while self._connected:
with self.backupDatabaseLock:
for backup in self.backupDatabase:
# Reduce TTL
backup.TTL -= 1
if backup.TTL < 0:
# Backup is down
LOG.warning(
'Backup {}[{}] is DOWN with TTL: {}'.format(
backup.deviceID, backup.deviceIP, backup.TTL))
self.__send_demotion_message(backup.deviceIP)
# Remove from list
self.backupDatabase.remove(
backup) # TODO: Inform CIMI?
LOG.debug('Backup removed from database.')
else:
# Backup is ok
LOG.debug('Backup {}[{}] is OK with TTL: {}'.format(
backup.deviceID, backup.deviceIP, backup.TTL))
if self._connected:
sleep(self._lpp.get(self._lpp.TIME_KEEPER))
LOG.warning('Keeper thread stopped')
def __send_election_message(self, address):
"""
:param address: IP address for election
:return: True if correct election, False otherwise
"""
try:
r = requests.get('{}backup'.format(
URLS.build_url_address(URLS.URL_POLICIES_ROLECHANGE,
addr=address,
port=self.POLICIES_EXTERNAL_PORT,
secure=CPARAMS.MF2C_FLAG)),
timeout=1.5,
verify=False)
if r.status_code == 200:
# Correct
return True
else:
LOG.warning(
'Selected device [{}] received {} status code received on electing a new backup'
.format(address, r.status_code))
return False
except timeout:
LOG.warning(
'Selected device [{}] cannot become Backup due timeout'.format(
address))
return False
except:
LOG.exception(
'Selected device [{}] cannot become Backup due error in election message'
.format(address))
return False
def __send_demotion_message(self, address):
"""
Demote a backup to normal agent
:param address: IP address for demotion
:return: True if correct demotion, False otherwise
"""
try:
r = requests.get('{}agent'.format(
URLS.build_url_address(URLS.URL_POLICIES_ROLECHANGE,
addr=address,
port=CPARAMS.POLICIES_PORT)),
timeout=1.5)
if r.status_code == 200:
# Correct
return True
else:
LOG.warning(
'Selected device [{}] received {} status code received on removing a backup'
.format(address, r.status_code))
return False
except timeout:
LOG.warning(
'Selected device [{}] cannot be demoted to Agent due timeout'.
format(address))
return False
except:
LOG.exception(
'Selected device cannot become Agent due error in demotion message'
)
return False
def reelection(arearesilience, deviceID, deviceIP):
# 0. Check if is not a backup
backups = arearesilience.getBackupDatabase()
LOG.debug('Backup database query: {}'.format(backups))
found = False
for backup in backups:
if backup.deviceID == deviceID:
found = True
break
# If backup: Go to step # 2
if not found:
LOG.debug('Device {} is not an active backup.'.format(deviceID))
# 1. Promote device to Backup
# Ok? Go to 3, return False otherwise
ok = arearesilience.addBackup(
deviceID, deviceIP, arearesilience.PRIORITY_ON_REELECTION)
if not ok:
LOG.error(
'Proposed device cannot be promoted to backup for reelection'
)
return False
else:
LOG.info('Device {} is an active backup.'.format(deviceID))
# 2. Change preference to 0
backup.priority = arearesilience.PRIORITY_ON_REELECTION
# 3. Demote other backups (if any)
backups = arearesilience.getBackupDatabase()
for backup in backups:
if backup.deviceID != deviceID:
# Delete
ok = arearesilience.deleteBackup(backup.deviceID)
if ok:
LOG.info(
'Backup {}[{}] demoted successfully due Leader Reelection.'
.format(backup.deviceID, backup.deviceIP))
else:
LOG.error(
'Error on Backup deletion {}[{}] in Leader Reelection.'
.format(backup.deviceID, backup.deviceIP))
# 4. Demote leader (rest call self)
r = requests.get('{}agent'.format(
URLS.build_url_address(URLS.URL_POLICIES_ROLECHANGE,
portaddr=('127.0.0.1',
CPARAMS.POLICIES_PORT))))
if r.status_code == 200:
# Correct
LOG.info('Leader (self) demoted successfully')
return True
LOG.warning('Leader not demoted or confirmation not received')
return False
def __backupLeader_flow(self):
if not self._connected:
LOG.error('Module stoped due _connected = False')
return
if not self._imLeader:
# I've been promoted as backup
LOG.info(self.TAG + 'I\'m selected to be a backup. Seting up')
self.__preBackupSetup()
self.__becomeBackup()
if not self._connected:
return
# Multiple backups support
if self._backupPriority > 0:
sleep_time = 1. + 10 * (self._backupPriority - 1)
LOG.info(
'Waiting {}s before leader takeover...'.format(sleep_time))
sleep(sleep_time)
if not self._connected:
return
LOG.debug('Checking if new Leader is up...')
new_leader = self.__getCIMIData('disc_leaderIP', default='')
LOG.debug('Stored Leader = [{}], Detected Leader = [{}]'.format(
self._leaderIP, new_leader))
if new_leader == '' or new_leader == self._leaderIP:
LOG.warning('Leader not detected by Discovery')
elif self._leaderIP != new_leader:
LOG.info(
'Correct Leader takeover by a backup with more preference.'
)
try: # TODO: Clean solution
r = requests.get('{}agent'.format(
URLS.build_url_address(URLS.URL_POLICIES_ROLECHANGE,
addr='127.0.0.1',
port=CPARAMS.POLICIES_PORT)),
timeout=.5)
except:
pass
finally:
return
if not self._connected:
return
if self._imLeader or self._leaderFailed:
# I'm a leader
LOG.info(self.TAG + 'Leader seting up')
self.__becomeLeader()
self.__backupSelection()
return
def __scanning_flow(self):
# 1. Get Beacon
# 2. Categorize
# 3. Send Categorization info
self._socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
self._socket.bind(('0.0.0.0', CPARAMS.LDISCOVERY_PORT))
LOG.info('Scan server created correctly')
except:
LOG.exception('Error on creating the scan receiver')
return
while self._connected:
try:
data, addr = self._socket.recvfrom(4096)
if not self._connected:
break
LOG.debug('Received beacon from [{}]: \"{}\"'.format(
addr[0], data.decode()))
self.leaderIP = addr[0]
try:
ddata = loads(data.decode())
self.leaderID = ddata.get('leaderID')
except JSONDecodeError:
LOG.warning('Beacon payload malformed')
cpu, mem, stg = self.__categorize_device()
LOG.debug('CPU: {}, MEM: {}, STG: {}'.format(cpu, mem, stg))
payload = DeviceInformation(deviceID=self._deviceID,
cpuCores=cpu,
memAvail=mem,
stgAvail=stg).getDict()
LOG.info('Sending beacon reply to Leader...')
r = requests.post(URLS.build_url_address(
URLS.URL_BEACONREPLY,
portaddr=(addr[0], CPARAMS.POLICIES_PORT)),
json=payload,
timeout=2)
if r.status_code == 200:
LOG.info('Discovery Message successfully sent to Leader')
else:
LOG.warning(
'Discovery Message received error status code {}'.
format(r.status_code))
except:
LOG.exception('Error on beacon received')
try:
self._socket.close()
LOG.info('Scan Server Stopped')
except:
LOG.exception('Server Stop not successful')
def deleteBackupOnAgentResource(backupIP):
try:
agent_resource, agent_resource_id = CIMI.getAgentResource()
ar = AgentResource.load(agent_resource)
if ar.backupIP != backupIP:
LOG.warning(
'Backup [{}] does not match [{}] stored in Agent Resource'.
format(backupIP, ar.backupIP))
ar2 = AgentResource(None, None, None, None, None, backupIP='')
LOG.debug(
'Removing backup [{}] in Agent Resource. Updated agent resource: {}'
.format(backupIP, ar2.getCIMIdicc()))
CIMI.modify_resource(agent_resource_id, ar2.getCIMIdicc())
except:
LOG.exception('Add backup in Agent resource failed')
def startScanning(self):
if self._isStarted or self._isScanning:
LOG.warning(
'LDiscovery is already started: isStarted={} isScanning={}'.
format(self._isStarted, self._isScanning))
return False
self._th_proc = threading.Thread(name='LDiscS',
target=self.__scanning_flow,
daemon=True)
self._connected = True
self._isStarted = True
self._isScanning = True
self.leaderIP = None
self.leaderID = None
self._th_proc.start()
LOG.info('LDiscovery successfully started in Scan Mode.')
return True
def start(self, deviceID): # TODO: Give deviceID at startup?
"""
:return:
"""
self._deviceID = deviceID
if self.isStarted:
LOG.warning(self.TAG + 'Procedure is already started...')
return False
else:
self.th_proc = threading.Thread(name='area_res',
target=self.__common_flow,
daemon=True)
self.th_proc.start()
self.isStarted = True
LOG.info(self.TAG + 'Module Started')
return True
def stopScanning(self):
if self._isStarted and self._isScanning:
self._connected = False
try:
self._socket.shutdown(socket.SHUT_RDWR)
self._socket.close()
LOG.debug('Socket closed on scanning')
except:
pass
self._th_proc.join()
LOG.info('LDisc Scanning Stopped')
self._isScanning = False
self._isStarted = False
self.leaderIP = None
self.leaderID = None
else:
LOG.warning('LDisc is not Scanning.')
return True
def startBeaconning(self):
if self._isStarted or self._isBroadcasting:
LOG.warning(
'LDiscovery is already started: isStarted={} isBroadcasting={}'
.format(self._isStarted, self._isBroadcasting))
return False
self._th_proc = threading.Thread(name='LDiscB',
target=self.__beaconning_flow,
daemon=True)
self._connected = True
self._isStarted = True
self._isBroadcasting = True
self.leaderIP = None
self.leaderID = self._deviceID
self._db = {}
self._th_proc.start()
LOG.info('LDiscovery successfully started in Beacon Mode.')
return True
def promotedToBackup(self, leaderIP):
"""
The agent is promoted to be a backup
:return:
"""
# First check if Agent was electable
self._leaderIP = leaderIP
if self._imCapable:
LOG.info(self.TAG + 'Becoming backup due leader selection.')
# Then, check if AreaResilience thread is running
if self.th_proc is None:
pass
elif not self.th_proc.is_alive():
pass
elif self._imLeader or self._imBackup:
LOG.error(
'Agent is already a Backup/Leader. Cannot become a Backup.'
)
return False
else:
LOG.warning(
'Area Resilience still starting. Cannot promote on this state. Waiting...'
)
while self.th_proc.is_alive():
sleep(0.1)
LOG.debug('Successful waiting.')
LOG.debug('Module is ready for promotion.')
self.th_proc = threading.Thread(name='area_res',
target=self.__backupLeader_flow,
daemon=True)
self.th_proc.start()
self.isStarted = True
return True
else:
if not self._startupCorrect:
LOG.warning(
'Area Resilience still starting. Cannot promote on this state.'
)
else:
LOG.error('Agent not capable to be Backup/Leader')
return False
def addBackupOnAgentResource(backupIP):
try:
agent_resource, agent_resource_id = CIMI.getAgentResource()
ar = AgentResource.load(agent_resource)
if ar.backupIP is not None or (ar.backupIP != ''
and ar.backupIP is not None):
LOG.warning(
'Non-empty backupIP value when adding a new backup! Agent resource: {}'
.format(ar.getCIMIdicc()))
ar2 = AgentResource(None,
None,
None,
None,
None,
backupIP='{}'.format(backupIP))
LOG.debug(
'Adding backup [{}] in Agent Resource. Updated agent resource: {}'.
format(backupIP, ar2.getCIMIdicc()))
CIMI.modify_resource(agent_resource_id, ar2.getCIMIdicc())
except:
LOG.exception('Add backup in Agent resource failed')
def __trigger_joinDiscovery(self):
payload = {'interface': CPARAMS.WIFI_DEV_FLAG, 'bssid': self.bssid}
try:
r = requests.post(self.URL_DISCOVERY_JOIN,
json=payload,
timeout=20.)
except (timeout, timeout2):
LOG.error(self.TAG + 'JOIN trigger timeout.')
return False
rjson = r.json()
if r.status_code != 200:
LOG.warning(self.TAG +
'JOIN operation failed. Reply from discovery: {}'.
format(rjson['message']))
return False
else:
LOG.debug(self.TAG + 'Sending MYIP trigger to Discovery...')
try:
r2 = requests.get(self.URL_DISCOVERY_MYIP, timeout=20.)
except (timeout, timeout2):
LOG.error(self.TAG + 'MYIP trigger timeout.')
return False
rjson2 = r2.json()
if r2.status_code != 200:
LOG.warning(self.TAG +
'MYIP operation failed. Reply from discovery: {}'.
format(rjson2))
return False
elif 'IP_address' not in rjson2 or rjson2[
'IP_address'] is None or len(rjson2['IP_address']) == 0:
LOG.error(
self.TAG +
'MYIP operation returned an OK code BUT IP_address is empty! json: {}'
.format(rjson2))
return False
LOG.debug(self.TAG +
'MYIP trigger success. Reply form Discovery: {}'.format(
rjson2['IP_address']))
self.deviceIP = rjson2['IP_address']
return True
def debug():
sleep(10)
LOG.info('Device registration procedure...')
attempt = 0
successful = False
while attempt < CPARAMS.REGISTRATION_MAX_RETRY and not successful:
try:
r = requests.post(
URLS.build_url_address(URLS.URL_IDENTIFICATION_START,
portaddr=('identification', '46060')))
rjson = r.json()
LOG.debug('Identification request result: {}'.format(rjson))
if rjson['status'] == '412' and CPARAMS.CLOUD_FLAG:
# We need to wait until user and password is registered
LOG.warning('Cloud user not registered yet... Retry in 10s.')
elif rjson['status'] in ('200', '201'):
LOG.info('Successful registration of the device.')
successful = True
elif CPARAMS.DEBUG_FLAG:
LOG.warning(
'Status code received different from 200 or 201. Debug mode skips this failure.'
)
successful = True
else:
LOG.warning('Error on registration trigger. Retry in 10s...')
successful = False
except ValueError:
LOG.debug('ValueError raised on Identification: {}'.format(r.text))
except:
LOG.warning('Early start of Identification not successful.')
finally:
if not successful:
sleep(10)
attempt += 1
if not CPARAMS.DEBUG_FLAG and not successful:
LOG.critical(
'Critical Error: Registration of the device not successful. Stopping module'
)
exit(4)
sleep(5)
LOG.info('Starting Agent Flow...')
r = requests.get(
URLS.build_url_address(URLS.URL_START_FLOW,
portaddr=('127.0.0.1', CPARAMS.POLICIES_PORT)))
# r = requests.get(URLS.build_url_address(URLS.URL_POLICIES, portaddr=('127.0.0.1', CPARAMS.POLICIES_PORT)))
LOG.debug('Agent Flow request result: {}'.format(r.json()))
LOG.debug('Stopping thread activity.')
return
def get(self, role):
global arearesilience
"""Promotion/Demotion of the agent role."""
imLeader = arearesilience.imLeader()
imBackup = arearesilience.imBackup()
if role.lower() == 'leader':
# Do you want to be a leader?
if imLeader:
# If a leader is promoted to leader, it becomes a super-leader?
LOG.debug('Role change: Leader -> Leader')
return {'imLeader': imLeader, 'imBackup': imBackup}, 403
elif imBackup:
# Hi, I'm backup-kun - It's my time to shine!!
LOG.debug('Role change: Backup -> Leader')
ret = agentstart.switch(imLeader=True)
if ret:
LOG.info('Successful promotion to Leader')
else:
LOG.warning('Unsuccessful promotion from Backup to Leader')
return {'imLeader': True, 'imBackup': False}, 200
else:
# Nor leader, nor Backup, just a normal agent
# For reelection, first you must be a backup!
LOG.debug('Role change: Agent -> Leader')
return {'imLeader': imLeader, 'imBackup': imBackup}, 403
elif role.lower() == 'backup':
# Always have a B plan
if imLeader:
# Why in the hell a Leader'll become a backup?
LOG.debug('Role change: Leader -> Backup')
return {'imLeader': imLeader, 'imBackup': imBackup}, 403
elif imBackup:
# Emm... no pls.
LOG.debug('Role change: Backup -> Backup')
return {'imLeader': imLeader, 'imBackup': imBackup}, 403
else:
# Can you watch my shoulder?
LOG.debug('Role change: Agent -> Backup')
leaderIP = request.environ.get('HTTP_X_REAL_IP',
request.remote_addr)
LOG.debug(
'Leader at {} is selecting me as Backup'.format(leaderIP))
ret = arearesilience.promotedToBackup(
leaderIP=agentstart.leaderIP
) # TODO: get leaderIP from CIMI
if ret:
LOG.info('Successful promotion to Backup')
return {'imLeader': imLeader, 'imBackup': True}, 200
else:
LOG.warning('Unsuccessful promotion from Agent to Backup')
return {
'imLeader': arearesilience.imLeader(),
'imBackup': arearesilience.imBackup()
}, 403
elif role.lower() == 'agent':
# Bigger will be the fall....
if imLeader:
# You are shuch an incompetent, you're FIRED!
# Leader demotion
LOG.debug('Role change: Leader -> Agent')
arearesilience.stop()
agentstart.switch(imLeader=False)
CPARAMS.LEADER_FLAG = False
arearesilience = AreaResilience(cimi)
arearesilience.start(agentstart.deviceID)
return {'imLeader': False, 'imBackup': False}, 200
elif imBackup:
# Maybe we are gona call you latter.... or not
# Backup demotion
LOG.debug('Role change: Backup -> Agent')
arearesilience.stop()
arearesilience = AreaResilience(cimi)
arearesilience.start(agentstart.deviceID)
return {'imLeader': False, 'imBackup': False}, 200
else:
# You're so tiny that I don't even care.
LOG.debug('Role change: Agent -> Agent')
return {'imLeader': False, 'imBackup': False}, 403
else:
# keikaku doori... Weird syntax maybe?
return {'imLeader': imLeader, 'imBackup': imBackup}, 404
def __agent_startup_flow(self):
while self._connected:
# 0. Init
self.detectedLeaderID, self.MACaddr = None, None
# 0.1 Check CIMI is UP
CIMIon = False
while self._connected and not CIMIon:
CIMIon = CIMI.checkCIMIstarted()
if not CIMIon:
LOG.debug(self.TAG +
'CIMI is not ready... Retry in {}s'.format(
self.WAIT_TIME_CIMI))
sleep(self.WAIT_TIME_CIMI)
LOG.info(self.TAG + 'CIMI is ready!')
# 1. Identification
if self._connected:
self.identification_failed = True # Reset variable to avoid false positives
LOG.debug(self.TAG + 'Sending trigger to Identification...')
try:
self.__trigger_requestID()
self.identification_failed = False
except Exception:
LOG.exception(self.TAG + 'Identification trigger failed!')
self.identification_failed = True
LOG.info(self.TAG + 'Identification Trigger Done.')
else:
return
if not CPARAMS.DEBUG_FLAG and self.identification_failed:
LOG.critical(
self.TAG +
'Identification failed, interrupting agent start.')
return
# 2. Check if im a Leader - PLE
if self.imLeader:
# switch to leader
self.__leader_switch_flow() # TODO: imCapable?
return
# remain as agent
# 3. Scan for Leaders
count = 0
self.discovery_failed = True
while self._connected and count < self.MAX_MISSING_SCANS and self.detectedLeaderID is None and self.MACaddr is None: # TODO: new protocol required
LOG.debug(self.TAG + 'Sending scan trigger to Discovery...')
try:
self.__trigger_startScan()
self.discovery_failed = False
except Exception:
LOG.debug(self.TAG +
'Discovery failed on attepmt {}.'.format(count))
self.discovery_failed = True
if self.detectedLeaderID is not None and self.MACaddr is not None:
LOG.info(self.TAG + 'Discovery Scan Trigger Done.')
count += 1
LOG.info(
self.TAG +
'Discovery trigger finished in #{} attempts and ok={}'.format(
count, self.detectedLeaderID is not None
and self.MACaddr is not None))
if not self._connected:
return
if not CPARAMS.DEBUG_FLAG and self.discovery_failed:
LOG.critical(self.TAG +
'Discovery failed, interrupting agent start.')
return
# 4. If no leader detected, switch to leader IF policy and capable - ALE
if not self.discovery_failed and self.detectedLeaderID is None and self.MACaddr is None and self.ALE_ENABLED:
self.__leader_switch_flow() # TODO: imCapable?
return
# 5. CAU client
if self._connected:
self.cauclient_failed = True
LOG.debug(self.TAG + 'Sending trigger to CAU client...')
try:
self.__trigger_triggerCAUclient()
self.cauclient_failed = False
except Exception:
LOG.exception(self.TAG + 'CAUclient failed.')
self.cauclient_failed = True
LOG.info(self.TAG + 'CAU client Trigger Done.')
else:
return
if not CPARAMS.DEBUG_FLAG and self.cauclient_failed:
LOG.critical(self.TAG +
'CAU-Client failed, interrupting agent start.')
return
# 5. Categorization
if self._connected and not self.categorization_started:
self.categorization_failed = True
LOG.debug(self.TAG +
'Sending start trigger to Categorization...')
try:
self.__trigger_startCategorization()
self.categorization_failed = False
self.categorization_started = True
except Exception:
LOG.exception(self.TAG + 'Categorization failed')
self.categorization_failed = True
LOG.info(self.TAG + 'Categorization Start Trigger Done.')
elif not self._connected:
return
if not CPARAMS.DEBUG_FLAG and self.categorization_failed:
LOG.critical(
self.TAG +
'Categorization failed, interrupting agent start.')
return
# 6. Area Resilience
if self._connected and not self.arearesilience_started:
self.policies_failed = True
LOG.debug(self.TAG + 'Sending start trigger to Policies...')
try:
success = self.__trigger_startLeaderProtectionPolicies()
self.policies_failed = not success
self.arearesilience_started = success
except Exception:
LOG.exception(self.TAG +
'Policies Area Resilience failed!')
LOG.info(self.TAG +
'Policies Area Resilience Start Trigger Done.')
elif not self._connected:
return
if not CPARAMS.DEBUG_FLAG and self.policies_failed:
LOG.critical(
self.TAG +
'Policies Area Resilience failed, interrupting agent start.'
)
return
# Print summary
self.__print_summary()
# Create/Modify Agent Resource # TODO.
self.deviceIP = '' # TODO: Real value here (from categorization)
self._cimi_agent_resource = AgentResource(self.deviceID,
self.deviceIP,
self.isAuthenticated,
self.secureConnection,
self.imLeader,
leaderIP=self.leaderIP)
if self._cimi_agent_resource_id is None:
# Create agent resource
self._cimi_agent_resource_id = CIMI.createAgentResource(
self._cimi_agent_resource.getCIMIdicc())
else:
# Agent resource already exists
status = CIMI.modify_resource(
self._cimi_agent_resource_id,
self._cimi_agent_resource.getCIMIdicc())
# 7. Watch Leader
if self._connected and not self.discovery_failed:
LOG.debug(self.TAG + 'Start Discovery Leader Watch...')
try:
self.__trigger_startDiscoveryWatch()
except Exception:
LOG.exception(self.TAG + 'Watch Discovery Start Fail.')
LOG.info(self.TAG + 'Watch Discovery Start Trigger Done.')
elif self.discovery_failed:
LOG.warning(
self.TAG +
'Discovery Watch cancelled due Discovery Trigger failed')
else:
return
alive = True
while self._connected and not self.discovery_failed and alive:
# 6 Check if discovery connection is alive
LOG.debug(self.TAG + 'Discovery Alive Start Trigger.')
try:
alive = not self.__trigger_aliveDiscovery(
) # not disconnected
except Exception:
LOG.exception(self.TAG + 'Discovery Alive failed')
alive = False
if self._connected:
sleep(CPARAMS.TIME_WAIT_ALIVE)
LOG.info(self.TAG + 'Discovery Alive Start Trigger Done.')
if not self._connected:
return
if CPARAMS.DEBUG_FLAG and self.discovery_failed:
# TODO: Delete this in future versions
LOG.debug(self.TAG + 'No rescan available. Stoping activity')
return
def __agent_startup_flow(self):
while self._connected:
# 0. Init
self.detectedLeaderID, self.MACaddr, self.bssid = None, None, None
# 0.1 Check CIMI is UP
CIMIon = False
while self._connected and not CIMIon:
CIMIon = CIMI.checkCIMIstarted()
if not CIMIon:
LOG.debug(self.TAG +
'CIMI is not ready... Retry in {}s'.format(
self.WAIT_TIME_CIMI))
sleep(self.WAIT_TIME_CIMI)
LOG.info(self.TAG + 'CIMI is ready!')
# 1. Identification
if self._connected:
self.identification_failed = True # Reset variable to avoid false positives
LOG.debug(self.TAG + 'Sending trigger to Identification...')
try:
self.__trigger_requestID()
self.identification_failed = False
except Exception:
LOG.exception(self.TAG + 'Identification trigger failed!')
self.identification_failed = True
LOG.info(self.TAG + 'Identification Trigger Done.')
else:
return
if not CPARAMS.DEBUG_FLAG and self.identification_failed:
LOG.critical(
self.TAG +
'Identification failed, interrupting agent start.')
return
# 2.1. Check if im Cloud Agent
if self.imCloud:
# start cloud flow
self.__cloud_flow()
return
# 2.2. Check if im a Leader - PLE
if self.imLeader:
# switch to leader
self.__leader_switch_flow()
return
# remain as agent
# 3. Scan for Leaders
count = 0
self.discovery_failed = True
while self._connected and count < self.MAX_MISSING_SCANS and self.detectedLeaderID is None and self.MACaddr is None: # TODO: new protocol required
LOG.debug(self.TAG + 'Sending SCAN trigger to Discovery...')
try:
self.__trigger_startScan()
self.discovery_failed = False
except Exception:
LOG.debug(self.TAG +
'Discovery failed on attepmt {}.'.format(count))
self.discovery_failed = True
if self.detectedLeaderID is not None and self.MACaddr is not None and self.bssid is not None:
LOG.info(self.TAG + 'Discovery Scan Trigger Done.')
count += 1
LOG.info(
self.TAG +
'Discovery trigger finished in #{} attempts and ok={}'.format(
count, self.detectedLeaderID is not None
and self.MACaddr is not None and self.bssid is not None))
if not self._connected:
return
if not CPARAMS.DEBUG_FLAG and self.discovery_failed:
LOG.critical(self.TAG +
'Discovery failed, interrupting agent start.')
return
# 4.1. If no leader detected, switch to leader IF policy and capable - ALE
if not self.discovery_failed and self.detectedLeaderID is None and self.MACaddr is None and self.bssid is None and self.ALE_ENABLED:
self.__leader_switch_flow() # TODO: imCapable?
return
# 4.2 If detected, join to the Leader
if not self.discovery_failed and self.bssid is not None and self._connected:
LOG.debug(self.TAG + 'Sending JOIN trigger to discovery...')
try:
r = self.__trigger_joinDiscovery()
self.discovery_failed = not r
self.discovery_joined = r
if not self.discovery_failed:
self.leaderIP = CPARAMS.LEADER_DISCOVERY_IP
except Exception:
LOG.exception(self.TAG + 'Discovery JOIN trigger failed.')
self.discovery_failed = True
self.discovery_joined = False
LOG.debug(self.TAG + 'Discovery JOIN trigger Done.')
# 4.3 If not detected or failed, static configuration if setup
if self.discovery_failed or (self.detectedLeaderID is None
and self.MACaddr is None
and self.bssid is None):
LOG.debug(
self.TAG +
'Discovery failed or leader was not detected. Fetching deviceIP and leaderIP from env variables.'
)
self.deviceIP = CPARAMS.DEVICE_IP_FLAG
self.leaderIP = CPARAMS.LEADER_IP_FLAG
# 5. CAU client
if self._connected:
self.cauclient_failed = True
LOG.debug(self.TAG + 'Sending trigger to CAU client...')
attempt = 0
r = False
while self._connected and not r and attempt < self.MAX_CAUCLIENT_FAILURES:
try:
r = self.__trigger_triggerCAUclient()
self.cauclient_failed = not r
except Exception:
LOG.exception(self.TAG + 'CAUclient failed.')
self.cauclient_failed = True
finally:
attempt += 1
if not r:
sleep(self.WAIT_TIME_CAUCLIENT)
LOG.info(self.TAG +
'CAU client Trigger Done in {} attempts and ok={}.'.
format(attempt, r))
else:
return
if not CPARAMS.DEBUG_FLAG and self.cauclient_failed:
LOG.critical(self.TAG +
'CAU-Client failed, interrupting agent start.')
return
# 5.1. VPN get IP
attempt = 0
while self._connected and self.vpnIP is None and attempt < self.MAX_VPN_FAILURES:
vpn_ip = VPN.getIPfromFile()
self.vpnIP = vpn_ip if vpn_ip != '' else None
if self.vpnIP is None:
LOG.debug(self.TAG +
'VPN IP cannot be obtained... Retry in {}s'.
format(self.WAIT_TIME_VPN))
sleep(self.WAIT_TIME_VPN)
attempt += 1
if self.vpnIP is None:
LOG.warning(self.TAG + 'VPN IP cannot be obtained.')
if not CPARAMS.DEBUG_FLAG:
LOG.critical(
self.TAG +
'Policies module cannot continue its activity without VPN IP'
)
exit(4)
else:
LOG.info(self.TAG + 'VPN IP: [{}]'.format(self.vpnIP))
# 5.2 If not static configuration and no leader detected, VPN configuration
if self.deviceIP is None and self.leaderIP is None:
LOG.debug(
self.TAG +
'Static configuration for deviceIP and leaderIP not found. Using VPN values'
)
self.deviceIP = self.vpnIP
self.leaderIP = self.cloudIP
LOG.info(self.TAG + 'deviceIP={}, leaderIP={}'.format(
self.deviceIP, self.leaderIP))
# 6. Categorization
if self._connected and not self.categorization_started:
self.categorization_failed = True
LOG.debug(self.TAG +
'Sending start trigger to Categorization...')
try:
self.__trigger_startCategorization()
self.categorization_failed = False
self.categorization_started = True
except Exception:
LOG.exception(self.TAG + 'Categorization failed')
self.categorization_failed = True
LOG.info(self.TAG + 'Categorization Start Trigger Done.')
elif not self._connected:
return
if not CPARAMS.DEBUG_FLAG and self.categorization_failed:
LOG.critical(
self.TAG +
'Categorization failed, interrupting agent start.')
return
# 7. Area Resilience
if self._connected and not self.arearesilience_started:
self.policies_failed = True
LOG.debug(self.TAG + 'Sending start trigger to Policies...')
try:
success = self.__trigger_startLeaderProtectionPolicies()
self.policies_failed = not success
self.arearesilience_started = success
except Exception:
LOG.exception(self.TAG +
'Policies Area Resilience failed!')
LOG.info(self.TAG +
'Policies Area Resilience Start Trigger Done.')
elif not self._connected:
return
if not CPARAMS.DEBUG_FLAG and self.policies_failed:
LOG.critical(
self.TAG +
'Policies Area Resilience failed, interrupting agent start.'
)
return
# Print summary
self.__print_summary()
# Create/Modify Agent Resource
self.deviceIP = '' if self.deviceIP is None else self.deviceIP
self._cimi_agent_resource = AgentResource(self.deviceID,
self.deviceIP,
self.isAuthenticated,
self.secureConnection,
self.imLeader)
LOG.debug(self.TAG + 'CIMI Agent Resource payload: {}'.format(
self._cimi_agent_resource.getCIMIdicc()))
if self._cimi_agent_resource_id is None:
# Create agent resource
self._cimi_agent_resource_id = CIMI.createAgentResource(
self._cimi_agent_resource.getCIMIdicc())
sleep(.1)
self._cimi_agent_resource = AgentResource(
self.deviceID,
self.deviceIP,
self.isAuthenticated,
self.secureConnection,
self.imLeader,
leaderIP=self.leaderIP)
LOG.debug(self.TAG + 'CIMI Agent Resource payload: {}'.format(
self._cimi_agent_resource.getCIMIdicc()))
status = CIMI.modify_resource(
self._cimi_agent_resource_id,
self._cimi_agent_resource.getCIMIdicc())
if self._cimi_agent_resource_id == '':
LOG.warning(self.TAG + 'Agent resource creation failed.')
if not CPARAMS.DEBUG_FLAG:
LOG.error(
'Stopping Policies module due to resource creation failure.'
)
exit(4)
else:
# Agent resource already exists
status = CIMI.modify_resource(
self._cimi_agent_resource_id,
self._cimi_agent_resource.getCIMIdicc())
sleep(.1)
self._cimi_agent_resource = AgentResource(
self.deviceID,
self.deviceIP,
self.isAuthenticated,
self.secureConnection,
self.imLeader,
leaderIP=self.leaderIP)
LOG.debug(self.TAG + 'CIMI Agent Resource payload: {}'.format(
self._cimi_agent_resource.getCIMIdicc()))
status = CIMI.modify_resource(
self._cimi_agent_resource_id,
self._cimi_agent_resource.getCIMIdicc())
# 8. Watch Leader
if self._connected and not self.discovery_failed:
LOG.debug(self.TAG + 'Start Discovery Leader Watch...')
try:
self.__trigger_startDiscoveryWatch()
except Exception:
LOG.exception(self.TAG + 'Watch Discovery Start Fail.')
LOG.info(self.TAG + 'Watch Discovery Start Trigger Done.')
elif self.discovery_failed:
LOG.warning(
self.TAG +
'Discovery Watch cancelled due Discovery Trigger failed')
else:
return
self.isCompleted = True
alive = True
while self._connected and not self.discovery_failed and alive:
# 6 Check if discovery connection is alive
LOG.debug(self.TAG + 'Discovery Alive Start Trigger.')
try:
alive = not self.__trigger_aliveDiscovery(
) # not disconnected
except Exception:
LOG.exception(self.TAG + 'Discovery Alive failed')
alive = False
if self._connected:
sleep(CPARAMS.TIME_WAIT_ALIVE)
LOG.info(self.TAG + 'Discovery Alive Start Trigger Done.')
if not self._connected:
return
if CPARAMS.DEBUG_FLAG and self.discovery_failed:
LOG.debug(self.TAG + 'No rescan available. Stoping activity')
return
def __leader_switch_flow(self):
"""
Agent become leader
:return:
"""
# 1. Start sending beacons
if self._connected:
self.discovery_leader_failed = True
self.discovery_failed = True
LOG.debug(self.TAG + 'Sending Broadcast trigger to discovery...')
try:
r = self.__trigger_switch_discovery()
self.detectedLeaderID = self.deviceID
self.discovery_leader_failed = not r
except Exception as ex:
LOG.exception(self.TAG + 'Discovery broadcast trigger failed!')
self.detectedLeaderID = self.deviceID
LOG.info(self.TAG + 'Discovery Broadcast Trigger Done.')
else:
return
if not CPARAMS.DEBUG_FLAG and self.discovery_leader_failed:
LOG.critical(
self.TAG +
'Discovery broadcast failed, interrupting leader switch.')
return
self.discovery_failed = self.discovery_leader_failed
# 2. Start CAU-client
if self._connected:
self.cauclient_failed = True
LOG.debug(self.TAG + 'Sending trigger to CAU client...')
attempt = 0
r = False
while self._connected and not r and attempt < self.MAX_CAUCLIENT_FAILURES:
try:
r = self.__trigger_triggerCAUclient()
self.cauclient_failed = not r
except Exception:
LOG.exception(self.TAG + 'CAUclient failed.')
self.cauclient_failed = True
finally:
attempt += 1
if not r:
sleep(self.WAIT_TIME_CAUCLIENT)
LOG.info(self.TAG +
'CAU client Trigger Done in {} attempts and ok={}.'.
format(attempt, r))
else:
return
if not CPARAMS.DEBUG_FLAG and self.cauclient_failed:
LOG.critical(self.TAG +
'CAU-Client failed, interrupting agent start.')
return
# 3. VPN get IP
attempt = 0
while self._connected and self.vpnIP is None and attempt < self.MAX_VPN_FAILURES:
vpn_ip = VPN.getIPfromFile()
self.vpnIP = vpn_ip if vpn_ip != '' else None
if self.vpnIP is None:
LOG.debug(self.TAG +
'VPN IP cannot be obtained... Retry in {}s'.format(
self.WAIT_TIME_VPN))
sleep(self.WAIT_TIME_VPN)
attempt += 1
if self.vpnIP is None:
LOG.warning(self.TAG + 'VPN IP cannot be obtained.')
if not CPARAMS.DEBUG_FLAG:
LOG.critical(
self.TAG +
'Policies module cannot continue its activity without VPN IP'
)
exit(4)
else:
LOG.info(self.TAG + 'VPN IP: [{}]'.format(self.vpnIP))
# 4. Switch leader categorization (or start if not started)
if self.categorization_started:
self.categorization_leader_failed = True
# Switch!
LOG.debug(self.TAG + 'Sending switch trigger to Categorization...')
try:
self.__trigger_switch_categorization()
self.categorization_leader_failed = False
except Exception:
LOG.exception(self.TAG +
'Categorization switch to leader failed')
LOG.info(self.TAG + 'Categorization Switch Trigger Done.')
else:
# Start as leader!
LOG.debug(self.TAG + 'Sending start trigger to Categorization...')
try:
self.__trigger_startCategorization()
self.categorization_leader_failed = False
self.categorization_started = True
except Exception:
LOG.exception(self.TAG + 'Categorization failed')
LOG.info(self.TAG + 'Categorization Start Trigger Done.')
if not CPARAMS.DEBUG_FLAG and self.categorization_leader_failed:
LOG.critical(self.TAG +
'Categorization failed, interrupting leader switch.')
return
self.categorization_failed = self.categorization_leader_failed
# 5. Start Area Resilience (if not started)
if not self.arearesilience_started:
self.policies_failed = True
LOG.debug(self.TAG + 'Sending start trigger to Policies...')
try:
self.__trigger_startLeaderProtectionPolicies()
self.policies_failed = False
self.arearesilience_started = True
except Exception:
LOG.exception(self.TAG + 'Policies Area Resilience failed!')
LOG.info(self.TAG + 'Policies Area Resilience Start Trigger Done.')
if not CPARAMS.DEBUG_FLAG and self.policies_failed:
LOG.critical(
self.TAG +
'Policies Area Resilience failed, interrupting agent start.')
return
# 8. Watch Leader
if self._connected and not self.discovery_failed:
LOG.debug(self.TAG + 'Start Discovery Leader Watch...')
try:
self.__trigger_startDiscoveryWatchLeader()
except Exception:
LOG.exception(self.TAG + 'Watch Discovery Start Fail.')
LOG.info(self.TAG + 'Watch Discovery Start Trigger Done.')
elif self.discovery_failed:
LOG.warning(
self.TAG +
'Discovery Watch cancelled due Discovery Trigger failed')
# Create/Modify Agent Resource
# IF static IP configuration setup
self.deviceIP = CPARAMS.DEVICE_IP_FLAG
self.leaderIP = CPARAMS.LEADER_IP_FLAG
if self.deviceIP is None or self.leaderIP is None:
LOG.debug(
self.TAG +
'No static configuration was detected. Applying VPN values')
self.deviceIP = self.vpnIP
self.leaderIP = self.cloudIP
LOG.info(
self.TAG +
'deviceIP={}, leaderIP={}'.format(self.deviceIP, self.leaderIP))
# Print summary
self.__print_summary()
self._cimi_agent_resource = AgentResource(self.deviceID,
self.deviceIP,
True,
True,
self.imLeader,
leaderIP=self.leaderIP)
# deprecated: real values of Auth and Conn (as now are None in the Leader)
if self._cimi_agent_resource_id is None:
# Create agent resource
self._cimi_agent_resource_id = CIMI.createAgentResource(
self._cimi_agent_resource.getCIMIdicc())
else:
# Agent resource already exists
status = CIMI.modify_resource(
self._cimi_agent_resource_id,
self._cimi_agent_resource.getCIMIdicc())
# 6. Finish
self.isCompleted = True
return
def __cloud_flow(self):
LOG.info(self.TAG + 'Cloud flow started.')
# 0. Cloud Agent is Leader by definition
self.imLeader = self.imCloud
# 1. Discovery
LOG.debug(self.TAG + 'Discovery trigger ignored in Cloud flow.')
self.discovery_failed = False
self.discovery_leader_failed = False
self.detectedLeaderID = self.deviceID
# 2. Start CAU-client
if self._connected:
self.cauclient_failed = True
LOG.debug(self.TAG + 'Sending trigger to CAU client...')
attempt = 0
r = False
while self._connected and not r and attempt < self.MAX_CAUCLIENT_FAILURES:
try:
r = self.__trigger_triggerCAUclient()
self.cauclient_failed = not r
except Exception:
LOG.exception(self.TAG + 'CAUclient failed.')
self.cauclient_failed = True
finally:
attempt += 1
if not r:
sleep(self.WAIT_TIME_CAUCLIENT)
LOG.info(self.TAG +
'CAU client Trigger Done in {} attempts and ok={}.'.
format(attempt, r))
else:
return
if not CPARAMS.DEBUG_FLAG and self.cauclient_failed:
LOG.critical(self.TAG +
'CAU-Client failed, interrupting agent start.')
return
# 3. VPN get IP
attempt = 0
while self._connected and self.vpnIP is None and attempt < self.MAX_VPN_FAILURES:
vpn_ip = VPN.getIPfromFile()
self.vpnIP = vpn_ip if vpn_ip != '' else None
if self.vpnIP is None:
LOG.debug(self.TAG +
'VPN IP cannot be obtained... Retry in {}s'.format(
self.WAIT_TIME_VPN))
sleep(self.WAIT_TIME_VPN)
attempt += 1
if self.vpnIP is None:
LOG.warning(self.TAG + 'VPN IP cannot be obtained.')
if not CPARAMS.DEBUG_FLAG:
LOG.critical(
self.TAG +
'Policies module cannot continue its activity without VPN IP'
)
exit(4)
else:
LOG.info(self.TAG + 'VPN IP: [{}]'.format(self.vpnIP))
# 4. Switch leader categorization (or start if not started)
if self._connected and not self.categorization_started:
self.categorization_failed = True
LOG.debug(self.TAG + 'Sending start trigger to Categorization...')
try:
self.__trigger_startCategorization()
self.categorization_failed = False
self.categorization_started = True
except Exception:
LOG.exception(self.TAG + 'Categorization failed')
self.categorization_failed = True
LOG.info(self.TAG + 'Categorization Start Trigger Done.')
elif not self._connected:
return
if not CPARAMS.DEBUG_FLAG and self.categorization_failed:
LOG.critical(self.TAG +
'Categorization failed, interrupting agent start.')
return
# 5. Area Resilience
LOG.debug(self.TAG + 'Area Resilience trigger ignored in Cloud flow.')
self.policies_failed = False
# Print summary
self.__print_summary()
# Create Agent Resource
self.deviceIP = self.vpnIP
self.leaderIP = None
self._cimi_agent_resource = AgentResource(self.deviceID, self.deviceIP,
True, True, self.imLeader)
LOG.debug(self.TAG + 'CIMI Agent Resource payload: {}'.format(
self._cimi_agent_resource.getCIMIdicc()))
if self._cimi_agent_resource_id is None:
# Create agent resource
self._cimi_agent_resource_id = CIMI.createAgentResource(
self._cimi_agent_resource.getCIMIdicc())
if self._cimi_agent_resource_id == '':
LOG.warning(self.TAG + 'Agent resource creation failed.')
if not CPARAMS.DEBUG_FLAG:
LOG.error(
'Stopping Policies module due to resource creation failure.'
)
exit(4)
else:
# Agent resource already exists
status = CIMI.modify_resource(
self._cimi_agent_resource_id,
self._cimi_agent_resource.getCIMIdicc())
self.isCompleted = True
return
def __backupSelection(self):
"""
:return:
"""
# TODO:
# 1- Check Backups
# 2- Enough backups?
# YES: End sleep(X)
# NO:
# 3- Get topology and select one agent
# If not capable: Go to 3
# 4- promote to Backup
# Success: Correct_backups++
# Go to 2
while self._connected:
correct_backups = 0
with self.backupDatabaseLock:
# Check backups
for backup in self.backupDatabase:
if backup.TTL >= 0:
correct_backups += 1
# Enough?
if correct_backups >= self._lpp.get(self._lpp.BACKUP_MINIMUM,
default=1):
# Enough backups
LOG.debug(
'{} correct backup detected in Leader. Everything is OK.'.
format(correct_backups))
else:
# Not enough
if not self._connected:
break
LOG.warning(
'{} backup dettected are not enough. Electing new ones...'.
format(correct_backups))
topology = self.__getTopology()
new_backups = []
while self._connected and correct_backups < self._lpp.get(
self._lpp.BACKUP_MINIMUM,
default=1) and len(topology) > 0:
device = topology[0]
topology.remove(device)
found = False
with self.backupDatabaseLock:
for backup in self.backupDatabase:
if backup.deviceID == device.get('deviceID'):
found = True
break
if not found:
# Todo: Evaluate if selected device is capable
correct = self.__send_election_message(
device.get('deviceIP'))
if correct:
new_backup = BackupEntry(device.get('deviceID'),
device.get('deviceIP'),
self._nextPriority)
with self.backupDatabaseLock:
self.backupDatabase.append(new_backup)
LOG.info(
'Backup {}[{}] added with priority {}'.format(
device.get('deviceID'),
device.get('deviceIP'),
self._nextPriority))
correct_backups += 1
self._nextPriority += 1
new_backups.append(new_backups)
if correct_backups >= self._lpp.get(self._lpp.BACKUP_MINIMUM,
default=1):
# Now we have enough
LOG.info(
'{} correct backups dettected in Leader. {} new backups added.'
.format(correct_backups, len(new_backups)))
else:
LOG.warning(
'{} backups dettected are not enough. Waiting for new election.'
.format(correct_backups))
# Sleep
if self._connected:
sleep(self._lpp.get(self._lpp.TIME_TO_WAIT_BACKUP_SELECTION))
LOG.info('Leader stopped...')
def __becomeBackup(self):
"""
:return:
"""
# 1- Send the KeepAlive message to the leader.
# 2- Receive the reply (with preference number).
# If leader down, Backup becomes leader.
# Else repeat.
attempt = 0
counter = 0
payload = {'deviceID': self._deviceID}
self._imBackup = True
while self._connected and attempt < self._lpp.get(
self._lpp.MAX_RETRY_ATTEMPTS):
stopLoop = False
while self._connected and not stopLoop:
try:
# 1. Requests to Leader Keepalive endpoint
r = requests.post(URLS.build_url_address(
URLS.URL_POLICIES_KEEPALIVE,
portaddr=(self._leaderIP, CPARAMS.POLICIES_PORT)),
json=payload,
timeout=0.5)
LOG.debug(self.TAG +
'Keepalive sent [#{}]'.format(counter))
# 2. Process Reply
jreply = r.json()
if r.status_code == 200:
leaderID = jreply['deviceID'] # Todo: Use this
priority = jreply['backupPriority']
# 3. Update Preference
self._backupPriority = priority
LOG.debug(
self.TAG +
'Reply received, Leader still alive: LeaderID: {}'.
format(leaderID))
attempt = 0
else:
# Error?
LOG.error('KeepAlive status_code = {}'.format(
r.status_code))
if r.status_code == 403 and self.PRIORITY_ON_DEMOTION == jreply[
'backupPriority']:
LOG.warning(
'Backup has been removed from database or not authorized to send keepalive messages'
)
elif r.status_code == 405 and self.PRIORITY_ON_FAILURE == jreply[
'backupPriority']:
LOG.warning(
'Sending message to a Device that is not a Leader!'
)
stopLoop = True
else:
stopLoop = True
if not stopLoop:
# 4. Sleep
sleep(self._lpp.get(self._lpp.TIME_KEEPALIVE))
counter += 1
except:
# Connection broke, backup assumes that Leader is down.
LOG.debug('Keepalive connection refused')
stopLoop = True
LOG.warning(
'Keepalive connection is broken... Retry Attempts: {}'.format(
self._lpp.get(self._lpp.MAX_RETRY_ATTEMPTS) -
(attempt + 1)))
attempt += 1
if not self._connected:
LOG.info('Backup stopped.')
else:
LOG.warning(self.TAG + '## LEADER IS DOWN! ##')
self._leaderFailed = True
return