def get_devices(self, refresh=False):
if self.cache:
if refresh is True or not Cache.get(self.cache):
result = self._get_devices()
Cache.set(self.cache, result)
else:
result = Cache.get(self.cache)
else:
result = self._get_devices()
return result
def average_runs(name, method):
num_runs = Config.get()["num_runs"]
new_percent_list = np.arange(
0.1, # self.label_percent,
0.4, # self.max_percent,
0.015 # self.batch_percent / 2.0
)
rmse_avg_list = np.zeros((len(new_percent_list)))
mae_avg_list = np.zeros((len(new_percent_list)))
for run_id in range(num_runs):
key = "{}_{}_{}".format(name, method, run_id)
percent_list, rmse_list, mae_list = Cache.get(key)
rmse_avg_list += np.interp(new_percent_list, percent_list, rmse_list)
mae_avg_list += np.interp(new_percent_list, percent_list, mae_list)
rmse_avg_list /= num_runs
mae_avg_list /= num_runs
return new_percent_list, rmse_avg_list, mae_avg_list
class Zabbix(object):
def __init__(self,
host,
uri_path,
user,
noverify=False,
cacert=None,
http=False,
timeout=30):
"""
Initializes a Zabbix instance
:param host: hostname to connect to (ex. zabbix.yourdomain.net)
:param user: username to connect with (ex. Admin)
:param uri_path: uri path to zabbix api (ex. zabbix)
:param noverify: turns off verification
:param cacert: the certificate authority to use
:param http: flag to use http over https
:param timeout: API timeout parameter
:return: Zabbix instance
"""
self.cache = Cache('/tmp/zabbix.cache')
self.host = host
self.cache_slug = '{0}-{1}'.format(host, user)
zabbix_url = urlunparse([
'http' if http else 'https',
host.strip('/'), uri_path, '', '', ''
])
log.debug("Creating instance of Zabbic with url: %s", zabbix_url)
self.zapi = ZabbixAPI(zabbix_url)
if cacert is not None:
log.debug('Setting zapi.session.verify to {0}' ''.format(cacert))
self.zapi.session.verify = cacert
if noverify:
log.debug('Setting zapi.session.verify to False')
self.zapi.session.verify = False
self.zapi.timeout = timeout
self.fetch_zabbix_api_version() # Check the api
token = self.cache.get(self.cache_slug)
if token:
log.debug('Found token for {0}'.format(host))
self.zapi.auth = token
# Let's test the token
try:
self.verify_token()
except ZabbixNotAuthorized:
self.zapi.auth = ''
self.cache.delete(self.cache_slug)
def fetch_zabbix_api_version(self):
"""
reaches out to the zapi api info to parse the string
:return: Version string or False
"""
try:
return self.zapi.apiinfo.version()
except (HTTPError, ConnectionError, ZabbixAPIException) as e:
raise ZabbixError(e)
def verify_token(self):
"""
Runs the zapi.host.get(limit=1) call to test the current token
:return: Nothing
"""
try:
self.zapi.host.get(limit=1)
except (HTTPError, ConnectionError, ZabbixAPIException) as e:
# todo: cant we check by the error, not its string?
if any([
'Not authorised' in str(e), 'Not authorized' in str(e),
'Session terminated,' in str(e)
]):
log.debug('Token not authorized for {0}'.format(self.host))
raise ZabbixNotAuthorized
raise ZabbixError(e)
def auth(self, username, password):
"""
Performs the loggin function of the api with the supplied credentials
:param username: username
:param password: password
:return: True is valid, False otherwise
"""
try:
self.zapi.login(username, password)
self.cache.write(self.cache_slug, self.zapi.auth)
except ZabbixAPIException as e:
raise ZabbixNotAuthorized('Username or password invalid')
return True
class Zabbix(object):
def __init__(self, host, uri_path, user, noverify=False, cacert=None, http=False, timeout=30):
"""
Initializes a Zabbix instance
:param host: hostname to connect to (ex. zabbix.yourdomain.net)
:param user: username to connect with (ex. Admin)
:param uri_path: uri path to zabbix api (ex. zabbix)
:param noverify: turns off verification
:param cacert: the certificate authority to use
:param http: flag to use http over https
:param timeout: API timeout parameter
:return: Zabbix instance
"""
self.cache = Cache('/tmp/zabbix.cache')
self.host = host
self.cache_slug = '{0}-{1}'.format(host, user)
zabbix_url = urlunparse([
'http' if http else 'https',
host.strip('/'),
uri_path,
'', '', ''
]
)
log.debug("Creating instance of Zabbic with url: %s", zabbix_url)
self.zapi = ZabbixAPI(zabbix_url)
if cacert is not None:
log.debug('Setting zapi.session.verify to {0}'
''.format(cacert))
self.zapi.session.verify = cacert
if noverify:
log.debug('Setting zapi.session.verify to False')
self.zapi.session.verify = False
self.zapi.timeout = timeout
self.fetch_zabbix_api_version() # Check the api
token = self.cache.get(self.cache_slug)
if token:
log.debug('Found token for {0}'.format(host))
self.zapi.auth = token
# Let's test the token by grabbing the api version
try:
self.fetch_zabbix_api_version()
except ZabbixNotAuthorized:
self.zapi.auth = ''
def fetch_zabbix_api_version(self):
"""
reaches out to the zapi api info to parse the string
:return: Version string or False
"""
try:
return self.zapi.apiinfo.version()
except (HTTPError, ConnectionError, ZabbixAPIException) as e:
# todo: cant we check by the error, not its string?
if 'Not authorized' in str(e):
log.debug('Token not authorized for {0}'.format(self.host))
raise ZabbixNotAuthorized
raise ZabbixError(e)
def auth(self, username, password):
"""
Performs the loggin function of the api with the supplied credentials
:param username: username
:param password: password
:return: True is valid, False otherwise
"""
try:
self.zapi.login(username, password)
self.cache.write(self.cache_slug, self.zapi.auth)
except ZabbixAPIException as e:
raise ZabbixNotAuthorized('Username or password invalid')
return True
class HealthDispatcher(object):
def __init__(self, **kwargs):
super().__init__()
self._request_inter = Interaction("receive")
self._sender_inter = Interaction("sender")
self._handlers = {
"alive":
TaskThread(target=kwargs.get("alive", self._alive), name="alive"),
"status":
TaskThread(target=kwargs.get("status", self._status),
name="status"),
"init":
TaskThread(target=kwargs.get("init", self._init), name="init"),
}
self._cache = Cache("cluster_manager")
self._cache['node_load'] = {}
self._mapper = HealthMapper()
# self._cache['fd'] = {}
def dispatch(self, data, address):
command, *payload = data.decode("utf-8").split('&')
print(command, payload)
handler = self._handlers.get(command, None)
if handler:
handler.add_task((payload, address))
else:
print("Bad command: ", command)
#handler.delay((payload, address))
def add_handler(self, key, handler_func):
self._handlers[key] = TaskThread(target=handler_func, name=key)
self._handlers[key].start()
def start(self):
for handler in self._handlers.values():
handler.start()
def stop(self):
for handler in self._handlers.values():
handler.stop()
handler.join()
def _alive(self, data, *args, **kwargs):
payload, _ = data
node_id, status = payload
node_id = int(node_id)
if self._cache.get(node_id, None):
if self._cache[node_id]['timer']:
self._cache[node_id]['timer'].cancel()
if status == "True":
self._cache[node_id]['timer'] = Timer(
int(CF.get("Node", "node_live_timeout")),
self._alive,
args=[((node_id, "False"), ())])
self._cache[node_id]['timer'].start()
elif self._cache.get(node_id, None):
del self._cache[node_id]
del self._cache['node_load'][node_id]
def _status(self, data, *args, **kwargs):
payload, _ = data
new_size, node_id, pack_id, status = payload
node_id = int(node_id)
self._mapper.query("update_package_status", (status, pack_id))
res, file_id = self._mapper.query("get_unready_package",
(pack_id, pack_id))
res = res[0]
file_id = file_id[0]
self._mapper.query("update_file_size", (new_size, file_id))
self._cache[node_id]['pack_idle_count'] -= 1
self._cache['node_load'][node_id] = self._cache[node_id][
'pack_idle_count'] * self._cache[node_id]['free_size']
if not res:
message = "&".join(("status", str(file_id), "True"))
self._sender_inter.insert(
(message, (CF.get("Receiver",
"ip"), int(CF.get("Receiver", "tcp_port")))))
def _init(self, data, *args, **kwargs):
payload, address = data
tcp_port, udp_port, free_size = payload
res = self._mapper.query("get_node_by_address",
(address[0], int(tcp_port)))
if not res:
node_id = self._mapper.query(
"insert_new_node", (address[0], tcp_port, udp_port))[0][0]
else:
node_id = int(res[0][0])
res = "&".join(("init", str(node_id)))
self._sender_inter.insert((res, (address[0], int(tcp_port))))
self._cache[node_id] = {
'timer': None,
'ip': address[0],
'tcp_port': int(tcp_port),
'udp_port': int(udp_port),
'free_size': int(free_size),
'pack_idle_count': 0
}
self._cache['node_load'][node_id] = 0
