class RTCDevice(I2CDevice):
def __init__(self):
super(RTCDevice, self).__init__(0b1010001, "RTC")
self._current_time = datetime.datetime.now()
self._offset = datetime.timedelta()
self._raw_response = None
self._timer = RepeatedTimer(1, self._tick_time)
self._lock = Lock()
self._advance_time_interval = datetime.timedelta(seconds=1)
def set_response_array(self, response_array):
self._raw_response = response_array
def set_response_time(self, response_time):
with self._lock:
self._current_time = response_time
self._raw_response = None
def response_time(self):
with self._lock:
return self._current_time + self._offset
def start_running(self):
self._timer.start()
def stop_running(self):
self._timer.stop()
def advance_by(self, interval):
with self._lock:
self._offset += interval
def _time_to_bcd(self, dt):
time = dt.time()
date = dt.date()
return [
self.to_bcd(time.second),
self.to_bcd(time.minute),
self.to_bcd(time.hour),
self.to_bcd(date.day), 0,
self.to_bcd(date.month),
self.to_bcd(date.year - 2000)
]
def _tick_time(self):
with self._lock:
self._current_time += self._advance_time_interval
def to_bcd(self, byte):
return ((byte / 10) << 4) | (byte % 10)
@i2cMock.command([0x02])
def read_time(self):
self.log.debug("Read Time")
with self._lock:
return self._raw_response or self._time_to_bcd(self._current_time +
self._offset)
def __init__(self):
super(RTCDevice, self).__init__(0b1010001, "RTC")
self._current_time = datetime.datetime.now()
self._offset = datetime.timedelta()
self._raw_response = None
self._timer = RepeatedTimer(1, self._tick_time)
self._lock = Lock()
self._advance_time_interval = datetime.timedelta(seconds=1)
def __init__(self, service_manager, configuration):
"""The initialization function of the class ServiceHandler.
The function initializes the ServiceHandler instance and extracts all needed information to run and stop the service.
Args:
service_manager (:obj:`ServiceManager`): The instance of the
service manager core to extract the informations.
configuration (:obj:`optparse.Option`): The configuration of the
command line interface, e.g. to extract the service file path.
"""
self.configuration = configuration
self.testing_flag = configuration.testing
self.service_file_name_path = configuration.service_file
self.own_node_id = service_manager.get_own_hostname_callback()
self.cb_new_service_ports_found = service_manager.found_service_ports_callback
self.service = None
self.service_ports = []
self.service_status = (ServiceStatusCodes.NOT_STARTED_YET, None)
self.service_status_lock = threading.RLock()
self.service_id = 1 # the id of the current service instance in the network
# (will be updated after receiving a service)
self.open_ports_check = RepeatedTimer(5,
self.get_open_ports_of_service)
self.server_broadcast_socket = socket.socket(socket.AF_INET,
socket.SOCK_DGRAM)
self.server_broadcast_socket.setsockopt(socket.SOL_SOCKET,
socket.SO_REUSEADDR, 1)
self.server_broadcast_socket.setsockopt(socket.SOL_SOCKET,
socket.SO_REUSEPORT, 1)
self.server_broadcast_socket.bind(('', self.BROADCAST_PORT))
self.server_broadcast_event = threading.Event()
self.server_broadcast_thread = threading.Thread(
target=self.listen_for_whois_requests, args=())
self.server_broadcast_thread.daemon = True
self.server_broadcast_thread.start()
LOGGER.debug("service handler init")
def start_recent_cpu_and_ram_usage_timer(self, pid=None):
"""This method starts the check the current CPU and RAM of the service.
The method is starts the repeated check for the current CPU and RAM
usage of the service.
Returns:
True if successful start of the timer, False if the timer has not
been started.
"""
if self.migration_flag is True and self.recent_cpu_ram_check_flag is True:
self.service_pid = pid
self.check_cpu_ram_timer = RepeatedTimer(
self.cpu_ram_check_time, self.check_recent_cpu_and_ram_usage,
self.service_pid)
self.check_cpu_ram_timer.start()
return True
else:
return False
class SunSTest(RestartPerSuite):
def __init__(self, *args, **kwargs):
super(SunSTest, self).__init__(*args, **kwargs)
self._high = True
self._timer = RepeatedTimer(1, self._tick_time)
def test_measure(self):
self._timer.start()
res = self.system.obc.measure_suns(1, 2)
self.assertEqual(res, [
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33
])
self._timer.stop()
def _tick_time(self):
if self._high:
self.system.suns.gpio_interrupt_low()
else:
self.system.suns.gpio_interrupt_high()
self._high = not self._high
def start_forever(self):
"""The external interface method to start the migration check.
The method starts the main functionality of the NetworkUtilizationInspector by starting the repeated timers.
Returns:
True on successful start of the migration check cycle,
False on any error.
"""
if self.migration_flag is True:
LOGGER.info("starting migration check forever")
try:
if self.check_connections_timer and self.check_connections_timer.is_alive(
):
self.check_connections_timer.cancel()
self.check_connections_timer.join()
if self.check_cpu_ram_timer and self.check_cpu_ram_timer.is_alive(
):
self.check_cpu_ram_timer.cancel()
self.check_cpu_ram_timer.join()
self.check_connections_timer = RepeatedTimer(
self.connection_check_time,
self.check_recent_connections_for_best_server)
self.check_connections_timer.start()
# self.check_cpu_ram_timer = RepeatedTimer(
# self.cpu_ram_check_time,
# self.check_recent_cpu_and_ram_usage,
# self.service_pid)
# self.check_cpu_ram_timer.start()
return True
except Exception as exc:
LOGGER.error("cannot start migration check, Error=" + str(exc))
return False
else:
return False
def prepare_validation_timer(tparams, process_queue, model_filename,
model_option_filename, eval_intv, valid_ret_queue,
translator, evaluator, nproc, beam_size,
src_vocab, trg_vocab, valid_src, valid_trg,
trans_valid_src):
translator_cmd = [
"python",
translator,
"-p",
str(nproc),
"-k",
str(beam_size),
"-n",
"-u",
model_filename,
model_option_filename,
src_vocab,
trg_vocab,
valid_src,
trans_valid_src,
]
evaluator_cmd = [
"perl",
evaluator,
valid_trg,
]
args = (tparams, process_queue)
kwargs = {
'translator_cmd': translator_cmd,
'evaluator_cmd': evaluator_cmd,
'model_filename': model_filename,
'trans_valid_src': trans_valid_src
}
return RepeatedTimer(eval_intv * 60, validation, valid_ret_queue, *args,
**kwargs)
def __init__(self, *args, **kwargs):
super(SunSTest, self).__init__(*args, **kwargs)
self._high = True
self._timer = RepeatedTimer(1, self._tick_time)
monitor_port = 5000
app_port = 5001
if len(sys.argv) == 2:
app_port = sys.argv[1]
interval = 15
log.info('PORT {} ARGS {}'.format(app_port, len(sys.argv)))
log.info(sys.argv)
pinger = Pinger(
app_name='reporting',
app_host='127.0.0.1',
app_port=app_port,
monitor_host='127.0.0.1',
monitor_port=monitor_port,
interval=interval
)
rt = RepeatedTimer(interval=interval, function=pinger.ping)
try:
# app = Flask(__name__)
# app.run(host='127.0.0.1', port=5001)
app = connexion.App(__name__, specification_dir='./openapi')
app.add_api('swagger.yml')
# NOTE: debug=True causes the restart
app.run(host='127.0.0.1', port=app_port, debug=False)
finally:
pinger.shutdown()
rt.stop()
class ServiceHandler(object):
"""This class handles the service itself.
This ServiceHandler class starts and stops the service in a separate
process and informs all hosts using this service by sending a broadcast
message through the network.
Attributes:
BROADCAST_PORT (:obj:`int`): The port to broadcast changes to the
service status.
service (:obj:`subprocess`): The process of the service.
service_ports (:obj:`list` of :obj:`int`): The opend and used ports of
the service.
service_status (:obj:`(ServiceStatusCodes,str)`): The current status
of the service.
service_id (:obj:`int`): The unique ID of the service instance.
open_ports_check (:obj:`RepeatedTimer`): A timer to check the ports of
the service.
server_broadcast_socket (:obj:`socket`): Socket to broadcast changes
to the service status.
"""
BROADCAST_PORT = 6500
def __init__(self, service_manager, configuration):
"""The initialization function of the class ServiceHandler.
The function initializes the ServiceHandler instance and extracts all needed information to run and stop the service.
Args:
service_manager (:obj:`ServiceManager`): The instance of the
service manager core to extract the informations.
configuration (:obj:`optparse.Option`): The configuration of the
command line interface, e.g. to extract the service file path.
"""
self.configuration = configuration
self.testing_flag = configuration.testing
self.service_file_name_path = configuration.service_file
self.own_node_id = service_manager.get_own_hostname_callback()
self.cb_new_service_ports_found = service_manager.found_service_ports_callback
self.service = None
self.service_ports = []
self.service_status = (ServiceStatusCodes.NOT_STARTED_YET, None)
self.service_status_lock = threading.RLock()
self.service_id = 1 # the id of the current service instance in the network
# (will be updated after receiving a service)
self.open_ports_check = RepeatedTimer(5,
self.get_open_ports_of_service)
self.server_broadcast_socket = socket.socket(socket.AF_INET,
socket.SOCK_DGRAM)
self.server_broadcast_socket.setsockopt(socket.SOL_SOCKET,
socket.SO_REUSEADDR, 1)
self.server_broadcast_socket.setsockopt(socket.SOL_SOCKET,
socket.SO_REUSEPORT, 1)
self.server_broadcast_socket.bind(('', self.BROADCAST_PORT))
self.server_broadcast_event = threading.Event()
self.server_broadcast_thread = threading.Thread(
target=self.listen_for_whois_requests, args=())
self.server_broadcast_thread.daemon = True
self.server_broadcast_thread.start()
LOGGER.debug("service handler init")
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
LOGGER.debug("service handler exit")
self.open_ports_check.cancel()
self.server_broadcast_event.set()
self.stop_service()
return self
def listen_for_whois_requests(self):
"""Listens for requests from hosts who don't know the current server.
The function listens for hosts and answers them directly, if they don't know the current host, who runs the service.
"""
while self.server_broadcast_event.is_set() is False:
try:
new_message, address = self.server_broadcast_socket.recvfrom(
16384)
new_message = json.loads(new_message)
if new_message['event'] == "who_is":
status, _ = self.get_service_status()
if status == ServiceStatusCodes.STARTED_NORMALLY:
own_server_ip = Networking.translate_node_id_to_ip_addr(
self.own_node_id)
LOGGER.info("who_is message from %s = %s", address,
new_message)
publish_options = {}
publish_options['service_name'] = "service"
publish_options['event'] = "who_is_answer"
publish_options['server_ip'] = own_server_ip
publish_options['counter'] = self.service_id
try:
who_is_answer_socket = socket.socket(
socket.AF_INET, socket.SOCK_DGRAM)
who_is_answer_socket.sendto(
json.dumps(publish_options),
(address[0], self.BROADCAST_PORT))
who_is_answer_socket.close()
except socket.error as exc:
LOGGER.error(
'Failed to answer who_is (socket.error): ' +
str(exc),
exc_info=True)
except Exception as exc:
LOGGER.error(
"Error while receiving broadcast message,\naddress=%s,\nmessage=%s\nError=%s",
address,
new_message,
exc,
exc_info=True)
#except Exception as exc:
# LOGGER.error("Error while releasing the current lock, Error={}".format(exc))
self.server_broadcast_socket.close()
@staticmethod
def __get_class_from_frame(depth):
frame = inspect.stack()[depth][0]
args, _, _, value_dict = inspect.getargvalues(frame)
# we check the first parameter for the frame function is
# named 'self'
if len(args) and args[0] == 'self':
# in that case, 'self' will be referenced in value_dict
instance = value_dict.get('self', None)
if instance:
# return its class
return getattr(instance, '__class__', None)
# return None otherwise
return None
@staticmethod
def __get_calling_class():
return ServiceHandler.__get_class_from_frame(4)
@staticmethod
def __get_full_calling_class_name():
calling_class = ServiceHandler.__get_class_from_frame(4)
return calling_class.__module__ + "." + calling_class.__name__
def get_service_configuration(self):
"""Gets the current service configuration as a tuple.
If the service is running, the service ID and the open ports are the informations of the service needed in other classes.
Returns:
the service ID and the opened ports of the service.
"""
return self.service_id, self.service_ports
def set_service_configuration(self, service_id, service_ports):
"""Sets the current service configuration.
If the service is freshly received on the service manager, this
function will be called for the first configuration. Since the
previous service manager already collected ports, they will be used
and set here for sniffing purposes.
Args:
service_id (:obj:`int`): The new ID of the service.
service_ports (:obj:`list` of :obj:`int`): The current ports of
the service.
"""
self.service_id = service_id
if self.BROADCAST_PORT in service_ports:
service_ports.remove(self.BROADCAST_PORT)
if self.configuration.service_transporter_port in service_ports:
service_ports.remove(self.configuration.service_transporter_port)
self.service_ports = service_ports
def set_service_status(self, status, error=None):
"""Sets the current service status.
If the service is in transportation, an error occurred or started successfully, the service status is used to inform this.
Args:
status (:obj:`str`): status of the service from ServiceStatusCodes.
error (:obj:`str`, optional): The occurred error of the status, if
available.
"""
try:
self.service_status_lock.acquire()
self.service_status = status, error
finally:
self.service_status_lock.release()
def get_service_status(self):
"""Gets the current service status.
If the service is in transportation, an error occurred or started successfully, the service status is used to inform this.
Returns:
the current status of the service.
"""
try:
calling_class = self.__get_full_calling_class_name()
LOGGER.info("Returning service status %s to %s",
self.service_status, calling_class)
#self.service_status_lock.acquire()
return self.service_status
finally:
pass
#self.service_status_lock.release()
def delete_service_and_reset_status(self):
"""Deletes all files of the service and resets the status.
In any error case the service can be reset back with this method.
Returns:
True if reset was successful, False in any other case.
"""
try:
LOGGER.info("Resetting service now.")
if os.path.exists(self.service_file_name_path):
os.remove(self.service_file_name_path)
self.open_ports_check.cancel()
self.service_ports = []
self.set_service_status(ServiceStatusCodes.NOT_STARTED_YET, None)
return True
except Exception as exc:
LOGGER.error("Error while resetting service: %s", exc)
return False
return False
def seperate_ipv4_and_port(self, combined_ip_port):
"""Helper function to seperate the IPv4 address and the port.
Returns:
A tuple of the IP address and the port number.
"""
ip_and_port = combined_ip_port.split(':')
port = ip_and_port.pop()
#support for IPv6 addresses
ip_address = ':'.join(ip_and_port)
return ip_address, port
def get_open_ports_of_service(self):
"""Callback method of the repeated timer to find open ports.
This method is called after several seconds to check if there are any open ports of the service in the system.
"""
try:
pid = self.service.pid
command = "netstat -tlnup | awk 'NR>2 { print $4, $7; }'"
command = "netstat -tlnup | awk 'NR>2 { print $1, $4, $6, $7;}'"
netstat_output = subprocess.check_output(command, shell=True)
except subprocess.CalledProcessError as e:
LOGGER.error("Error while getting open ports of service, Error=" +
str(e))
else:
found_ports = []
for line in netstat_output.split('\n'):
values = line.split(' ')
if len(values) > 1:
if values[0] == "tcp" and values[3] == (str(pid) + "/python") or \
values[0] == "udp" and values[2] == (str(pid) + "/python"):
ip_address, port = self.seperate_ipv4_and_port(
values[1])
if port not in found_ports:
found_ports.append(int(port))
if found_ports:
self.open_ports_check.cancel()
for port in found_ports:
if port not in self.service_ports:
if port != self.BROADCAST_PORT and \
port != self.configuration.service_transporter_port:
self.service_ports.append(port)
self.cb_new_service_ports_found(self.service.pid,
self.service_ports)
def send_broadcast_event(self, service_name, event):
"""Method to send an event to all hosts in the network.
This method sends a broadcast message with the current service status
to all hosts in the network.
"""
broadcast_addresses = ["10.0.0.255"]
if self.testing_flag is False:
broadcast_addresses.append("10.0.1.255")
broadcast_addresses.append("10.0.2.255")
Networking.broadcast_service_status(broadcast_addresses,
self.own_node_id, service_name,
event, self.service_id)
def start_service(self):
"""Method to start the service in an own subprocess.
This method starts the service in an own subprocess, informs all hosts
in the network via broadcast and returns the new service status to the
calling functions.
Returns:
The new service status.
"""
try:
# The os.setsid() is passed in the argument preexec_fn so
# it's run after the fork() and before exec() to run the shell.
self.service = subprocess.Popen(
[sys.executable, self.service_file_name_path])
#close_fds=True,# shell=True,
#preexec_fn=os.setsid)
LOGGER.info("Started service with pid=" + str(self.service.pid))
self.open_ports_check.start()
# if there are already service ports inside the list (from the last migration process)
# inform the service manager core about these
if self.service_ports:
self.cb_new_service_ports_found(self.service.pid,
self.service_ports)
self.set_service_status(ServiceStatusCodes.STARTED_NORMALLY, None)
self.send_broadcast_event("service", "started")
except OSError as exc:
LOGGER.error("Failed to start service (OSError): %s",
exc,
exc_info=True)
self.set_service_status(ServiceStatusCodes.ERROR_STARTING_SERVICE,
exc)
except Exception as exc:
LOGGER.error("Failed to start service: %s", exc, exc_info=True)
self.set_service_status(ServiceStatusCodes.ERROR_STARTING_SERVICE,
exc)
return self.get_service_status()
def stop_service(self):
"""Method to stop the service.
This method stops the service and informs all hosts in the network via broadcast, that the service has been stopped.
Returns:
True if the service has been stopped, False in any occurred error.
"""
try:
LOGGER.info("trying to stop service")
self.send_broadcast_event("service", "stopped")
if self.service:
#os.killpg(os.getpgid(self.service.pid), signal.SIGTERM)
os.kill(self.service.pid, signal.SIGINT)
# self.service.send_signal(signal.SIGINT)
# if self.service.wait() == 0:
# logger.info("Successful terminated the running service")
# else:
# logger.info("Error while terminating running service. Kill it instead")
# os.kill(self.service.pid, signal.SIGINT)
self.service = None
self.set_service_status(ServiceStatusCodes.NOT_STARTED_YET,
None)
except Exception as exc:
LOGGER.error("Could not kill process group: %s",
exc,
exc_info=True)
return False
return True
"""Old functions from the previous middleware concept.
from utils import RepeatedTimer
def _update_times():
state = [0]
def to_raw(sec):
return sec * (1000 / 50)
def callback():
i = state[0]
system.primary_antenna.antenna_state[0].activation_time = to_raw(i + 0)
system.primary_antenna.antenna_state[1].activation_time = to_raw(i + 2)
system.primary_antenna.antenna_state[2].activation_time = to_raw(i + 4)
system.primary_antenna.antenna_state[3].activation_time = to_raw(i + 8)
system.backup_antenna.antenna_state[0].activation_time = to_raw(i + 8)
system.backup_antenna.antenna_state[1].activation_time = to_raw(i + 4)
system.backup_antenna.antenna_state[2].activation_time = to_raw(i + 2)
system.backup_antenna.antenna_state[3].activation_time = to_raw(i + 0)
state[0] += 1
return callback
antenna_timer = RepeatedTimer(1, _update_times())
class NetworkUtilizationInspector(object):
"""This class checks a possible migration for the platform.
This NetworkUtilizationInspector class checks the collected data from the
NetworkSniffer and calculates with functions from the NetworkRouter the
next possible service host. If the hosts is on a better position than this
host, the migration will be informed to the service manager core.
Attributes:
check_connections_timer (:obj:`RepeatedTimer`): Timer to repeat the
process of the migration cycle
connection_check_lock (:obj:`threading.Lock`): A lock that no two or
more checks could occur in the same time.
recent_in_out_packets (:obj:`dict` of :obj:`dict` of :obj:`int`):
A dictionary, containing all recently with the service
connected client hosts and the amount of the incoming and
outgoing traffic in this time slot.
recent_in_out_packets_total(:obj:`int`): The total number of recent
incoming and outgoing data traffic, measured in bytes.
best_new_choosen_nodes (:obj:`list` of :obj:`(int, int)`): A sorted
list of all possible service hosts.
check_cpu_ram_timer (:obj:`RepeatedTimer`): Timer to repeatedly check
the current CPU and RAM status of the service on the host.
cpu_ram_check_lock (:obj:`threading.Lock`): A lock that no two or
more checks could occur in the same time.
"""
def __init__(self, service_manager, configuration):
"""The initialization function of the NetworkUtilizationInspector.
The function initializes the NetworkUtilizationInspector instance and
registers all callback functions of the service manager core for its
own functions.
Args:
service_manager (:obj:`ServiceManager`): The instance of the
service manager core to extract the callback functions.
configuration (:obj:`optparse.Option`): The configuration of the
command line interface, e.g. to extract the possible server
hosts.
"""
LOGGER.debug("migration_checker init")
self.recent_cpu_ram_check_flag = False
self.migration_flag = configuration.migration
self.testing_flag = configuration.testing
self.server_hosts = configuration.server_hosts
self.own_node_id = service_manager.get_own_hostname_callback()
self.connection_check_time = configuration.connection_check_time
self.cpu_ram_check_time = configuration.cpu_ram_check_time
self.cpu_threshold = configuration.cpu_threshold
self.ram_threshold = configuration.ram_threshold
self.migration_threshold = configuration.migration_threshold
self.cb_send_service = service_manager.do_service_send_callback
self.cb_no_recent_connections = service_manager.no_recent_connections_callback
self.cb_calculate_central_node = service_manager.calculate_central_node_callback
#self.own_pid = os.getpid()
self.check_connections_timer = None
self.connection_check_lock = threading.Lock()
self.recent_in_out_packets = defaultdict(lambda: defaultdict(int))
self.recent_in_out_packets_total = 0
self.best_new_choosen_nodes = None
self.check_cpu_ram_timer = None
self.service_pid = None
self.cpu_ram_check_lock = threading.Lock()
self.recent_cpu_ram_usage = {'cpu': 0.0, 'ram': 0.0, 'counter': 0}
def __enter__(self):
LOGGER.debug("migration_checker enter")
return self
def __exit__(self, exc_type, exc_value, traceback):
LOGGER.debug("migration_checker exit")
self.cancel_migration_check()
return self
def start_forever(self):
"""The external interface method to start the migration check.
The method starts the main functionality of the NetworkUtilizationInspector by starting the repeated timers.
Returns:
True on successful start of the migration check cycle,
False on any error.
"""
if self.migration_flag is True:
LOGGER.info("starting migration check forever")
try:
if self.check_connections_timer and self.check_connections_timer.is_alive(
):
self.check_connections_timer.cancel()
self.check_connections_timer.join()
if self.check_cpu_ram_timer and self.check_cpu_ram_timer.is_alive(
):
self.check_cpu_ram_timer.cancel()
self.check_cpu_ram_timer.join()
self.check_connections_timer = RepeatedTimer(
self.connection_check_time,
self.check_recent_connections_for_best_server)
self.check_connections_timer.start()
# self.check_cpu_ram_timer = RepeatedTimer(
# self.cpu_ram_check_time,
# self.check_recent_cpu_and_ram_usage,
# self.service_pid)
# self.check_cpu_ram_timer.start()
return True
except Exception as exc:
LOGGER.error("cannot start migration check, Error=" + str(exc))
return False
else:
return False
def cancel_migration_check(self):
"""The external interface method to cancel the migration check.
The method cancels the main functionality of the
NetworkUtilizationInspector by canceling the repeated timers.
"""
if self.check_connections_timer is not None:
self.check_connections_timer.cancel()
if self.check_cpu_ram_timer:
self.check_cpu_ram_timer.cancel()
def check_recent_cpu_and_ram_usage(self, args=None):
"""The callback method to check the current CPU and RAM of the service.
The method is called repeatedly and checks the current CPU and RAM
usage of the service. The values are summed up, so that an average
value can be created.
"""
try:
self.cpu_ram_check_lock.acquire()
if args is None:
pid = None
else:
pid = args[0]
#LOGGER.info("check_recent_cpu_and_ram_usage, pid="+str(pid))
#remove for further investigations in total CPU and RAM usage
if pid is None:
return
command = "/usr/bin/top -b -n 1 -p " + str(
pid
) + " | awk 'NR>7 { cpu = $9; ram = $10 } END { print cpu, ram; }'"
proc = subprocess.Popen(command,
stdout=subprocess.PIPE,
shell=True)
stdout = proc.stdout.read().replace(',', '.').split()
#LOGGER.info(stdout)
self.recent_cpu_ram_usage['cpu'] += float(stdout[0])
self.recent_cpu_ram_usage['ram'] += float(stdout[1])
self.recent_cpu_ram_usage['counter'] += 1
#LOGGER.info(self.recent_cpu_ram_usage)
except Exception as e:
LOGGER.error("Failed while searching cpu and ram, pid=" +
str(pid) + ", Error=" + str(e),
exc_info=True)
finally:
self.cpu_ram_check_lock.release()
def start_recent_cpu_and_ram_usage_timer(self, pid=None):
"""This method starts the check the current CPU and RAM of the service.
The method is starts the repeated check for the current CPU and RAM
usage of the service.
Returns:
True if successful start of the timer, False if the timer has not
been started.
"""
if self.migration_flag is True and self.recent_cpu_ram_check_flag is True:
self.service_pid = pid
self.check_cpu_ram_timer = RepeatedTimer(
self.cpu_ram_check_time, self.check_recent_cpu_and_ram_usage,
self.service_pid)
self.check_cpu_ram_timer.start()
return True
else:
return False
def get_best_new_choosen_nodes(self):
"""This method returns the list of best hosts for running the service.
Returns:
A list of all possible hosts to run the service in descending
order.
"""
try:
self.connection_check_lock.acquire()
return self.best_new_choosen_nodes
finally:
self.connection_check_lock.release()
def add_new_connection(self, node_id, packet_size, is_incoming_packet):
"""Adds a new incoming connection to the service to the list.
If a new host has send a packet to the service, this will be saved
into a dictionary. This information can be used to calculate the next
best host to run the service.
Args:
node_id: (:obj:`int`): The ID of the connected host.
packet_size (:obj:`int`): The size of the sent or received packet.
is_incoming_packet (bool): Indicator, if the packet was incoming
from or outgoing to the other host.
"""
try:
self.connection_check_lock.acquire()
#if node_id not in self.recent_in_out_packets:
# self.recent_in_out_packets[node_id]['in'] = 0
# self.recent_in_out_packets[node_id]['out'] = 0
if is_incoming_packet:
self.recent_in_out_packets[node_id]['in'] += packet_size
else:
self.recent_in_out_packets[node_id]['out'] += packet_size
self.recent_in_out_packets_total += packet_size
#LOGGER.info("Added new connection from node {}".format(node_id))
except Exception as exc:
LOGGER.error("Failed while adding new connection: " + str(exc),
exc_info=True)
finally:
#LOGGER.info("total counter: " + str(self.recent_in_out_packets_total))
self.connection_check_lock.release()
def calculate_avg_cpu_and_ram_out_of_recent_usage(self):
"""Checks the average cpu and ram usage of the service.
This method is called once every migration cycle to check the average
CPU and RAM usage value of the service. If the values is higher than
threshold value, a duplication of the service could be made instead of
a migration.
Returns:
the recent average value of the CPU and RAM usage of the service.
"""
try:
avg_cpu, avg_ram = 0.0, 0.0
self.cpu_ram_check_lock.acquire()
LOGGER.info(self.recent_cpu_ram_usage)
if self.recent_cpu_ram_usage['counter'] > 0:
avg_cpu = self.recent_cpu_ram_usage[
'cpu'] / self.recent_cpu_ram_usage['counter']
avg_ram = self.recent_cpu_ram_usage[
'ram'] / self.recent_cpu_ram_usage['counter']
self.recent_cpu_ram_usage = {'cpu': 0.0, 'ram': 0.0, 'counter': 0}
return avg_cpu, avg_ram
except Exception as exc:
LOGGER.error("Failed while checking recent connections: " +
str(exc),
exc_info=True)
finally:
self.cpu_ram_check_lock.release()
def check_recent_connections_for_best_server(self):
"""Checks the recent connections to calculate the best host.
This method is called once every migration cycle to check, which host
is the best to run the service instance. It uses all gathered
information from packets and hosts to allocate the best service. The
results will be informed to the service manager core.
"""
LOGGER.debug("enter")
try:
self.connection_check_lock.acquire()
if self.recent_in_out_packets_total == 0:
self.cb_no_recent_connections()
return
#LOGGER.debug("start calculating central node from recent connections\ntotal={},\nseperated={}".format(
# self.recent_in_out_packets_total,
# self.recent_in_out_packets))
best_nodes = self.cb_calculate_central_node(
self.recent_in_out_packets, self.recent_in_out_packets_total)
if best_nodes is not [] and best_nodes[0][0] != self.own_node_id:
# the instance variable can be set,
# since the general connection_check_lock has been aquired
self.best_new_choosen_nodes = []
# remove the nodes from the list, which are not in the list of possible servers
if self.server_hosts:
for node in best_nodes:
if node[0] in self.server_hosts:
self.best_new_choosen_nodes.append(node)
else:
self.best_new_choosen_nodes = best_nodes
LOGGER.info("Best Nodes: " + str(self.best_new_choosen_nodes))
# if the value between the best new server and the already running server is smaller than the migration threshold, do not migrate
for node in self.best_new_choosen_nodes:
if node[0] == self.own_node_id:
best_node_value = self.best_new_choosen_nodes[0][1]
own_value = node[1]
LOGGER.info("Me=%s, Other=%s", node,
self.best_new_choosen_nodes[0])
# if there is only one host connecting, this host should be the new server, since it has a value of 0.0
if best_node_value > 0.0:
percentage_difference = own_value / best_node_value
LOGGER.info(
"own_val=%s, best_val=%s, per_diff=%s, mig_trsh=%s",
own_value, best_node_value,
percentage_difference,
self.migration_threshold)
if percentage_difference < (
1 + (self.migration_threshold / 100)):
LOGGER.info(
"The new server wouldn't be really better as a server. Migration rejected."
)
return
avg_cpu, avg_ram = self.calculate_avg_cpu_and_ram_out_of_recent_usage(
)
LOGGER.info("avg cpu=" + str(avg_cpu) + ", avg ram=" +
str(avg_ram))
if avg_cpu > self.cpu_threshold or avg_ram > self.ram_threshold:
self.cb_send_service(True)
else:
self.cb_send_service(False)
else:
LOGGER.info("No node as best server found! best_node=" +
str(best_nodes))
except Exception as exc:
LOGGER.error("Failed while checking recent connections: " +
str(exc),
exc_info=True)
finally:
#self.recently_connected_nodes_counter = {}
self.recent_in_out_packets = defaultdict(lambda: defaultdict(int))
self.recent_in_out_packets_total = 0
self.connection_check_lock.release()
LOGGER.debug("exit")
def __init__(self, master):
self._NAME = "Coinz.py"
self._SOUNDS = "sounds"
self.master = master
self.master.title(self._NAME)
self.sound_name = "320181__dland__hint.wav"
self.sound_array = self.get_sounds()
root.protocol("WM_DELETE_WINDOW", self.on_closing)
self.market_prices = {
'btc': get_last_price['Bittrex']("usdt", "btc"),
'eth': get_last_price['Bittrex']("usdt", "eth"),
'bnb': get_last_price['Binance']("usdt", "bnb")
}
self.rt = RepeatedTimer(15, self.update_coins)
self.coin_list = []
#create menubar
self.menubar = Menu(master)
self.settingsmenu = Menu(self.menubar, tearoff=0)
self.settingsmenu.add_command(label="Settings",
command=self.edit_settings)
self.settingsmenu.add_command(label="Sounds", command=self.edit_sounds)
self.menubar.add_cascade(label="File", menu=self.settingsmenu)
root.config(menu=self.menubar)
#create treeview
ct_header = [
"exchange", "market", "shitcoin", "market price", "usd price",
"alerts"
]
ct_width = [80, 80, 128, 100, 100, 40]
self.cointree = Treeview(master,
selectmode="browse",
columns=ct_header,
show="headings")
for i in range(len(ct_header)):
self.cointree.heading(ct_header[i], text=ct_header[i])
self.cointree.column(ct_header[i],
width=ct_width[i],
stretch=False)
self.cointree.grid(column=0, row=0, columnspan=6, rowspan=6)
#create buttons
button_frame = Frame(master)
self.addcoin_button = Button(button_frame,
text="Add Coin",
command=self.add_coin)
self.addcoin_button.grid(column=0, row=0, sticky="NEWS")
self.removecoin_button = Button(button_frame,
text="Remove Coin",
command=self.remove_coin)
self.removecoin_button.grid(column=0, row=1, sticky="NEWS")
self.button_separator = Separator(button_frame, orient="horizontal")
self.button_separator.grid(column=0, row=2, sticky="NEWS", pady=5)
self.addalert_button = Button(button_frame,
text="Add Alert",
command=self.add_alert)
self.addalert_button.grid(column=0, row=3, sticky="NEWS")
self.viewalerts_button = Button(button_frame,
text="View Alerts",
command=self.view_alerts)
self.viewalerts_button.grid(column=0, row=4, sticky="NEWS")
button_frame.grid(column=6, row=0)
#status label
status_frame = Frame(master)
self.status_info_text = StringVar()
self.status_info_text.set("example info")
self.status_price_text = StringVar()
self.status_price_text.set("example price")
self.status_info = Label(status_frame,
relief="sunken",
textvariable=self.status_info_text)
self.status_price = Label(status_frame,
relief="sunken",
textvariable=self.status_price_text)
self.status_info.grid(column=0, row=0)
self.status_price.grid(column=1, row=0)
status_frame.grid(column=0, row=6, columnspan=6)
##load shit
if os.path.exists(os.path.join(os.getcwd(), "coinlist.pickle")):
with open('coinlist.pickle', 'rb') as f:
self.coin_list = pickle.load(f)
temp_sound_name = ""
if os.path.exists(os.path.join(os.getcwd(), "soundname.pickle")):
with open('soundname.pickle', 'rb') as f:
temp_sound_name = pickle.load(f)
ind = next(
iter([
i for i in range(len(self.sound_array))
if self.sound_array[i].find(temp_sound_name) != -1
]), None)
if ind != None:
self.sound_name = temp_sound_name
else:
ErrorPopup(
self.master, "saved sound " + temp_sound_name +
" not found! \\n it will be replaced with the default on close"
)
del (temp_sound_name)
#update treeview
self.display_coinlist(True)
class Overview:
def __init__(self, master):
self._NAME = "Coinz.py"
self._SOUNDS = "sounds"
self.master = master
self.master.title(self._NAME)
self.sound_name = "320181__dland__hint.wav"
self.sound_array = self.get_sounds()
root.protocol("WM_DELETE_WINDOW", self.on_closing)
self.market_prices = {
'btc': get_last_price['Bittrex']("usdt", "btc"),
'eth': get_last_price['Bittrex']("usdt", "eth"),
'bnb': get_last_price['Binance']("usdt", "bnb")
}
self.rt = RepeatedTimer(15, self.update_coins)
self.coin_list = []
#create menubar
self.menubar = Menu(master)
self.settingsmenu = Menu(self.menubar, tearoff=0)
self.settingsmenu.add_command(label="Settings",
command=self.edit_settings)
self.settingsmenu.add_command(label="Sounds", command=self.edit_sounds)
self.menubar.add_cascade(label="File", menu=self.settingsmenu)
root.config(menu=self.menubar)
#create treeview
ct_header = [
"exchange", "market", "shitcoin", "market price", "usd price",
"alerts"
]
ct_width = [80, 80, 128, 100, 100, 40]
self.cointree = Treeview(master,
selectmode="browse",
columns=ct_header,
show="headings")
for i in range(len(ct_header)):
self.cointree.heading(ct_header[i], text=ct_header[i])
self.cointree.column(ct_header[i],
width=ct_width[i],
stretch=False)
self.cointree.grid(column=0, row=0, columnspan=6, rowspan=6)
#create buttons
button_frame = Frame(master)
self.addcoin_button = Button(button_frame,
text="Add Coin",
command=self.add_coin)
self.addcoin_button.grid(column=0, row=0, sticky="NEWS")
self.removecoin_button = Button(button_frame,
text="Remove Coin",
command=self.remove_coin)
self.removecoin_button.grid(column=0, row=1, sticky="NEWS")
self.button_separator = Separator(button_frame, orient="horizontal")
self.button_separator.grid(column=0, row=2, sticky="NEWS", pady=5)
self.addalert_button = Button(button_frame,
text="Add Alert",
command=self.add_alert)
self.addalert_button.grid(column=0, row=3, sticky="NEWS")
self.viewalerts_button = Button(button_frame,
text="View Alerts",
command=self.view_alerts)
self.viewalerts_button.grid(column=0, row=4, sticky="NEWS")
button_frame.grid(column=6, row=0)
#status label
status_frame = Frame(master)
self.status_info_text = StringVar()
self.status_info_text.set("example info")
self.status_price_text = StringVar()
self.status_price_text.set("example price")
self.status_info = Label(status_frame,
relief="sunken",
textvariable=self.status_info_text)
self.status_price = Label(status_frame,
relief="sunken",
textvariable=self.status_price_text)
self.status_info.grid(column=0, row=0)
self.status_price.grid(column=1, row=0)
status_frame.grid(column=0, row=6, columnspan=6)
##load shit
if os.path.exists(os.path.join(os.getcwd(), "coinlist.pickle")):
with open('coinlist.pickle', 'rb') as f:
self.coin_list = pickle.load(f)
temp_sound_name = ""
if os.path.exists(os.path.join(os.getcwd(), "soundname.pickle")):
with open('soundname.pickle', 'rb') as f:
temp_sound_name = pickle.load(f)
ind = next(
iter([
i for i in range(len(self.sound_array))
if self.sound_array[i].find(temp_sound_name) != -1
]), None)
if ind != None:
self.sound_name = temp_sound_name
else:
ErrorPopup(
self.master, "saved sound " + temp_sound_name +
" not found! \\n it will be replaced with the default on close"
)
del (temp_sound_name)
#update treeview
self.display_coinlist(True)
def edit_sounds(self):
self.edit_sounds_popup = EditSoundsPopup(self, self.master)
self.master.wait_window(self.edit_sounds_popup.top)
def edit_settings(self):
self.edit_settings_popup = EditSettingsPopup(self, self.master)
self.master.wait_window(self.edit_settings_popup.top)
def get_sounds(self):
sound_array = []
for sound in [
f for f in listdir(self._SOUNDS)
if isfile(join(self._SOUNDS, f))
]:
sound_array.append(join(self._SOUNDS, sound))
return sound_array
def get_selected_sound(self):
return self.sound_array[self.sound_index]
def update_coins(self):
#update btc price and btc display
self.market_prices = {
'btc': get_last_price['Bittrex']("usdt", "btc"),
'eth': get_last_price['Bittrex']("usdt", "eth"),
'bnb': get_last_price['Binance']("usdt", "bnb")
}
self.status_price_text.set(
("BTC: " + self.format_dollar(self.market_prices['btc'])))
#update coin objects
for c in self.coin_list:
if c.market is not Market.USDT.name:
c.market_price = self.format_satoshi(
get_last_price[c.exchange](c.market, c.name))
c.usd_price = self.format_dollar(
get_last_price[c.exchange](c.market, c.name) *
self.market_prices[c.market.lower()])
else:
c.market_price = self.format_dollar(get_last_price[c.exchange](
c.market, c.name))
c.usd_price = "-"
#this will return alert events if any are tripped
alert_events = c.check_alerts()
if alert_events is not None and len(alert_events) > 0:
#dont spam alert sounds, one will suffice
self.master.deiconify()
self.master.focus_force()
self.play_notif_sound()
#display each one
for a in alert_events:
AlertPopup(self.master, c, a.info)
#coin object is updated, time to update display
item = self.find_item_from_coin(c)
if item is None:
ErrorPopup(self.master, "item not found!")
return
self.cointree.set(item,
column="market price",
value=c.market_price)
self.cointree.set(item, column="usd price", value=c.usd_price)
self.cointree.set(item, column="alerts", value=len(c.alerts))
def add_coin(self):
self.add_coin_popup = AddCoinPopup(self.master, self)
self.addcoin_button["state"] = "disabled"
self.master.wait_window(self.add_coin_popup.top)
self.addcoin_button["state"] = "normal"
##check if user did not fill out form
if not hasattr(self.add_coin_popup, "value"):
print("no value 'value' in add_coin_popup")
return
#get ticker from input
c_form = self.add_coin_popup.value
print(c_form)
for c in self.coin_list:
if c_form[2].lower() == c.name.lower() and c_form[0].lower(
) == c.exchange.lower() and c_form[1].lower() == c.market.lower():
InfoPopup(self.master, "You already added this coin!")
return
_market_price = 0
_usd_price = 0
if c_form[1] is not Market.USDT.name:
_market_price = self.format_satoshi(get_last_price[c_form[0]](
c_form[1], c_form[2]))
_usd_price = self.format_dollar(
get_last_price[c_form[0]](c_form[1], c_form[2]) *
self.market_prices[c_form[1].lower()])
else:
_market_price = self.format_dollar(get_last_price[c_form[0]](
c_form[1], c_form[2]))
_usd_price = "-"
_coin = Coin(c_form[2].upper(), c_form[0], c_form[1], _market_price,
_usd_price)
self.coin_list.append(_coin)
self.display_coinlist(False)
def remove_coin(self):
selected_coin = self.cointree.selection()[0] if len(
self.cointree.selection()) > 0 else None
#if shit in None show a popup
if selected_coin is None:
InfoPopup(self.master, "You must select a coin to remove it!")
return
#do you wanna do dis?
AreYouSure = AreYouSurePopup(
self.master,
"remove " + self.cointree.item(selected_coin)["values"][2])
self.removecoin_button["state"] = "disabled"
self.master.wait_window(AreYouSure.top)
self.removecoin_button["state"] = "normal"
remove_successful = False
if AreYouSure.value:
coin = self.find_coin_from_item(selected_coin)
if coin is None:
ErrorPopup(self.master, "could not find coin")
#found it
#remove from treeview
self.cointree.delete(selected_coin)
#remove from coinlist
self.coin_list = [x for x in self.coin_list if x != coin]
#ool flip
remove_successful = True
if not remove_successful:
ErrorPopup(
self.master,
"Something went wrong during removal! The coin was not deleted."
)
def display_coinlist(self, first_time):
#this adds a coin to the treeview only if the "displayed" internal value of it is false
if first_time:
for c in self.coin_list:
if c.market == Market.USDT.name:
c.market_price = self.format_dollar(float(c.market_price))
self.cointree.insert('',
'end',
values=(c.exchange, c.market, c.name,
c.market_price, c.usd_price,
len(c.alerts)))
else:
for c in [x for x in self.coin_list if x.displayed != True]:
c.displayed = True
self.cointree.insert('',
'end',
values=(c.exchange, c.market, c.name,
c.market_price, c.usd_price,
len(c.alerts)))
def view_alerts(self):
selected_coin = self.cointree.selection()[0] if len(
self.cointree.selection()) > 0 else None
#if shit in None show a popup
if selected_coin is None:
InfoPopup(self.master, "You must select a coin to view alerts!")
return
#find coin obj from selected treeview item
coin_obj = self.find_coin_from_item(selected_coin)
#make sure shit is there
if coin_obj is None:
ErrorPopup(self.master,
"Selected coin not found in the internal coin list!")
return
self.view_alerts_popup = ViewAlertsPopup(self.master, coin_obj)
self.viewalerts_button["state"] = "disabled"
self.master.wait_window(self.view_alerts_popup.top)
self.viewalerts_button["state"] = "normal"
##user might have deleted alerts, so update the alert column
self.cointree.set(selected_coin,
column="alerts",
value=len(coin_obj.alerts))
def find_coin_from_item(self, selection):
c = next(
iter([
x for x in self.coin_list
if x.name.lower() == self.cointree.item(selection)["values"]
[2].lower() and x.exchange.lower() == self.cointree.item(
selection)["values"][0].lower() and x.market.lower() ==
self.cointree.item(selection)["values"][1].lower()
]), None)
if c is None:
print("not found!")
return
return c
def find_item_from_coin(self, coin):
i = next(
iter([
x for x in self.cointree.get_children()
if self.cointree.item(x)["values"][2].lower() ==
coin.name.lower() and self.cointree.item(x)["values"]
[0].lower() == coin.exchange.lower() and self.cointree.item(
x)["values"][1].lower() == coin.market.lower()
]), None)
if i is None:
print("not found!")
return
return i
def add_alert(self):
selected_coin = self.cointree.selection()[0] if len(
self.cointree.selection()) > 0 else None
#if shit in None show a popup
if selected_coin is None:
InfoPopup(self.master, "You must select a coin to add an alert!")
return
#find coin obj from selected treeview item
coin_obj = self.find_coin_from_item(selected_coin)
self.add_alert_popup = AddAlertPopup(self.master, coin_obj)
self.addalert_button["state"] = "disabled"
self.master.wait_window(self.add_alert_popup.top)
self.addalert_button["state"] = "normal"
##check if user did not fill out form
if not hasattr(self.add_alert_popup, "value"):
print("no value 'value' in add_alert_popup")
return
alert_data = self.add_alert_popup.value
#todo:check data for validity
print(alert_data)
##add alert to the coin object if shit is there
coin_obj.add_alert(
Alert(alert_data[2], alert_data[1],
True if alert_data[0] == 'above' else False))
##update alert count in the treeview
self.cointree.set(selected_coin,
column="alerts",
value=len(coin_obj.alerts))
def format_satoshi(self, val):
return "{:10.8f}".format(val)
def format_dollar(self, val):
return "{:10.2f}".format(val)
def play_notif_sound(self):
ind = next(
iter([
i for i in range(len(self.sound_array))
if self.sound_array[i].find(self.sound_name) != -1
]), None)
if ind is None:
ErrorPopup(self.master, "sound " + self.sound_name + " not found!")
return
ns = AudioSegment.from_file(self.sound_array[ind])
ns = ns[:3 * 1000]
play(ns)
def on_closing(self):
with open("soundname.pickle", 'wb') as f:
pickle.dump(self.sound_name, f)
with open("coinlist.pickle", 'wb') as f:
pickle.dump(self.coin_list, f)
self.rt.stop()
root.destroy()