def get_active_nodes(self, threshold=30):
""" Returns a list of nodes that have been active within the last threshold (120) seconds
-> [(1629312332, 'node-id')] """
now = epoch()
other_nodes = []
for node_id in self.redis.keys('miso:nodes:*:last_seen', idx=-2):
last_seen = self.redis.getj(f'miso:nodes:{node_id}:last_seen')
if last_seen and now - last_seen < threshold:
other_nodes.append((last_seen, node_id))
return sorted(other_nodes)
def invoke_rpc(self, result_key, service, method, *args, **kwargs):
self.logger.info('invoking [%s.%s] and storing result at [%s]', service, method, result_key)
with ClusterRpcProxy(fake_proxy_config()) as cluster_rpc:
func = getattr(getattr(cluster_rpc, service), method)
result = func(*args, **kwargs)
if hasattr(result, 'to_dict'):
result = result.to_dict()
self.redis.setj(result_key+':result', result)
self.redis.setj(result_key+':when', epoch())
self.redis.expire(result_key, 3600*24*2)
self.redis.expire(result_key+':when', 3600*24*2)
def confirm_master(self):
now = epoch()
if not self.master_node:
self.master_node = self.redis.get('miso:cluster:master_node')
if self.master_node:
master_last_seen = now - self.redis.getj(f'miso:nodes:{self.master_node}:last_seen')
if master_last_seen is None or master_last_seen > 20:
self.logger.warning('We %s have not seen our master (%s) for 20 seconds now', id(self), self.master_node)
self.master_node = None
if not self.master_node:
for last_seen, other_node in self.get_active_nodes():
if self.redis.getj(f'miso:nodes:{other_node}:never_promote'):
continue # This node does not want to be master, move on!
self.redis.set('miso:cluster:master_node', other_node)
self.master_node = other_node
self.logger.info('%s has been nominated as master by %s', other_node, self.node_id)
break
return self.master_node
def get_available_services(self):
for service_name, service_class, service_file, module in discover_services_in_path(self.service_path):
file_hash = md5file(service_file)
file_mtime = int(os.stat(service_file).st_mtime)
file_key = f'{service_file}'[(len(self.service_path) + 1):]
old_mtime = self.redis.getj(f'miso:services:{service_name}:mtime')
self.redis.setj(f'miso:services:{service_name}:file_key', file_key)
self.redis.setj(f'miso:services:{service_name}:last_seen', epoch())
if not old_mtime or old_mtime < file_mtime:
self.redis.setj(f'miso:services:{service_name}:mtime', file_mtime)
self.redis.setj(f'miso:services:{service_name}:hash', file_hash)
if old_mtime:
self.logger.debug('Reloading %s due to mtime', module)
importlib.reload(module)
self._services[service_name] = {
'class': service_class,
'file': service_file,
'hash': file_hash,
'load_hash': self.redis.getj(f'miso:services:{service_name}:hash')
}
return [self._services[svc]['class'] for svc in self._services]
def epoch(self):
return epoch()