def mqtt_incoming_gw(self, topics, message):
# ybo_gw/src_gwid/dest_gwid
body = message["body"]
source_id = body["source_id"]
component_type = body["component_type"]
component_name = body["component_name"]
payload = body["payload"]
if component_type == "module":
try:
module = self._Modules[component_name]
except Exception as e:
logger.info("Received inter-gateway MQTT coms for module {module}, but module not found. Dropping.", module=component_name)
return False
try:
yield maybeDeferred(module._inter_gateway_mqtt_, topics, message)
except Exception as e:
logger.info("Received inter-gateway MQTT coms for module {module}, but module doesn't have function '_inter_gateway_mqtt_' Dropping.", module=component_name)
return False
elif component_type == "lib":
try:
if component_name == "system_ping_response":
reply_to = body["reply_to"]
if reply_to is None:
raise YomboWarning("lib.system_ping requires a reply_id, but not found. Dropping.")
if source_id in self._Gateways.gateways and \
self._Gateways.gateways[source_id].ping_request_id == reply_to:
gateway = self._Gateways.gateways[source_id]
gateway.ping_roundtrip = round(
(body["payload"] - gateway.ping_request_at) * 100, 2)
gateway.ping_response_at = body["time_received"]
gateway.ping_time_offset = round(body["payload"] - body["time_received"], 2)
elif component_name == "atoms_set":
for name, value in payload.items():
self._Atoms.set_from_gateway_communications(name, value, self)
elif component_name == "device_command":
self.incoming_data_device_command(body)
elif component_name == "device_command_status":
self.incoming_data_device_command_status(body)
elif component_name == "device_states":
self.incoming_data_device_states(body)
elif component_name == "notification":
self.incoming_data_notification(body)
elif component_name == "states_set":
for name, value in payload.items():
self._States.set_from_gateway_communications(name, value, self)
# elif component_name == "scenes":
# for name, value in message["payload"].items():
# self._Scenes.set_from_gateway_communications(name, value, self)
elif component_name == "system_state":
if payload == "online":
self._Gateways.gateways[source_id].com_status = "online"
reactor.callLater(random_int(2, .8), self.send_all_info, destination_id=source_id)
if payload == "offline":
self._Gateways.gateways[source_id].com_status = "offline"
except Exception as e: # catch anything here...so can display details.
logger.error("---------------==(Traceback)==--------------------------")
logger.error("{trace}", trace=traceback.format_exc())
logger.error("--------------------------------------------------------")
return True
def _init_(self, **kwargs):
self.gateway_id = self._Configs.get('core', 'gwid', 'local', False)
cookie_id = hashlib.sha224(
str(self._Gateways.get_master_gateway_id()).encode(
'utf-8')).hexdigest()
self.config = DictObject({
'cookie_session_name': 'yombo_' + cookie_id,
'cookie_path': '/',
'max_session':
15552000, # How long session can be good for: 180 days
'max_idle': 5184000, # Max idle timeout: 60 days
'max_session_no_auth':
600, # If not auth in 10 mins, delete session
'ignore_expiry': True,
'ignore_change_ip': True,
'expired_message': 'Session expired',
'httponly': True,
'secure':
False, # will change to true after SSL system/dns complete. - Mitch
})
self.active_sessions = {}
self.clean_sessions_loop = LoopingCall(self.clean_sessions)
self.clean_sessions_loop.start(random_int(
60 * 60,
.2)) # Every hour-ish. Save to disk, or remove from memory.
def _load_(self, **kwargs):
"""
Starts the loop to check if any certs need to be updated.
:return:
"""
self.check_if_certs_need_update_loop = LoopingCall(
self.check_if_certs_need_update)
self.check_if_certs_need_update_loop.start(
self._Configs.get("sqldict", "save_interval",
random_int(60 * 60 * 24, .1), False), False)
# Check if any libraries or modules need certs.
if self.local_gateway.dns_name is None:
logger.warn(
"Unable to generate sign ssl/tls certs, gateway has no domain name."
)
return
if self._Loader.operating_mode != "run":
return
sslcerts = yield global_invoke_all(
"_sslcerts_",
called_by=self,
)
for component_name, ssl_certs in sslcerts.items():
logger.debug(
f"Adding new managed certs from hook: {component_name}")
if isinstance(ssl_certs, tuple) is False and isinstance(
ssl_certs, list) is False:
ssl_certs = [ssl_certs]
for ssl_item in ssl_certs:
yield self.add_sslcert(ssl_item)
def _init_(self, **kwargs):
self.session_type = "apiauth"
self.active_api_auth = {}
self.active_api_auth_by_label = {}
# self.active_api_auth_cache = ExpiringDict(200, 5) # keep 200 entries, for at most 1 second...???
self._periodic_clean_sessions = LoopingCall(self.clean_sessions)
self._periodic_clean_sessions.start(random_int(
60 * 60, .2)) # Every hours-ish. Save or update
def _start_(self, **kwargs):
"""
We don't do anything, but "pass" so we don't generate an exception.
"""
if self._Loader.operating_mode != "run":
return
self.remote_get_root_key()
self.send_my_gpg_key_loop = LoopingCall(self.send_my_gpg_key)
self.send_my_gpg_key_loop.start(random_int(60 * 60 * 2, .2))
def _init_(self, **kwargs):
self.library_phase = 1
self.gateway_id = self._Configs.get('core', 'gwid', 'local', False)
self.__States = {self.gateway_id: {}}
self.automation_startup_check = {}
self.db_save_states_data = deque()
self.db_save_states_loop = LoopingCall(self.db_save_states)
self.db_save_states_loop.start(random_int(60, .10), False) # clean the database every 6 hours.
self.init_deferred = Deferred()
self.load_states()
return self.init_deferred
def _start_(self, **kwargs):
self.module_phase = 3
self.clean_states_loop = LoopingCall(self.clean_states_table)
self.clean_states_loop.start(random_int(60*60*6, .10)) # clean the database every 6 hours.
if self._States['loader.operating_mode'] == 'run':
self.mqtt = self._MQTT.new(mqtt_incoming_callback=self.mqtt_incoming, client_id='Yombo-states-%s' %
self.gateway_id)
self.mqtt.subscribe("yombo/states/+/get")
self.mqtt.subscribe("yombo/states/+/get/+")
self.mqtt.subscribe("yombo/states/+/set")
self.mqtt.subscribe("yombo/states/+/set/+")
def _init_(self, **kwargs):
self.library_phase = 1
self.gateway_id = self._Configs.get("core", "gwid", "local", False)
self.states = {self.gateway_id: {}}
self.db_save_states_data = deque()
self.db_save_states_loop = LoopingCall(self.db_save_states)
self.db_save_states_loop.start(random_int(30, .10), False)
yield self.load_states()
# setup cluster state defaults if not set
try:
self.get("is.away", gateway_id="cluster")
except KeyError:
self.set("is.away", False, gateway_id="cluster")
def amqp_connected(self):
"""
Called by AQMP when connected. This function was define above when setting up self.ampq.
:return:
"""
self._States.set('amqp.amqpyombo.state', True)
if self.send_local_information_loop is None:
self.send_local_information_loop = LoopingCall(
self.send_local_information)
# Sends various information, helps Yombo cloud know we are alive and where to find us.
if self.send_local_information_loop.running is False:
self.send_local_information_loop.start(random_int(60 * 60 * 4, .2))
def _load_(self, **kwargs):
"""
Starts the loop to check if any certs need to be updated.
:return:
"""
self.check_if_certs_need_update_loop = LoopingCall(
self.check_if_certs_need_update)
self.check_if_certs_need_update_loop.start(
self._Configs.get('sqldict', 'save_interval',
random_int(60 * 60 * 24, .1), False))
# Check if any libraries or modules need certs.
sslcerts = yield global_invoke_all('_sslcerts_', called_by=self)
# print("about to add sslcerts")
yield self._add_sslcerts(sslcerts)
def amqp_connected(self):
"""
Called by AQMP when connected. This function was define above when setting up self.ampq.
:return:
"""
self.connected = True
for the_callback in self.amqpyombo_options["connected"]:
the_callback()
if self.send_local_information_loop is None:
self.send_local_information_loop = LoopingCall(
self.send_local_information)
# Sends various information, helps Yombo cloud know we are alive and where to find us.
self.send_local_information(full=True)
yield sleep(1)
self.init_deferred.callback(1)
if self.send_local_information_loop.running is False:
self.send_local_information_loop.start(random_int(60 * 60 * 4, .2),
False)
def amqp_connected(self):
"""
Setup from the 'connect' method, set in the 'connected_callback' argument.
This method is called once the AMQPYombo connection is complete. This setups the loop that updates
Yombo about our current status. Primarily used for Dynamic DNS updates.
:return:
"""
self.connected = True
for callback in self.amqpyombo_options["connected"]:
callback()
if self.send_local_information_loop is None:
self.send_local_information_loop = LoopingCall(self.send_local_information)
# Sends various information, helps Yombo cloud know we are alive and where to find us.
self.send_local_information()
if self.send_local_information_loop.running is False:
self.send_local_information_loop.start(random_int(60 * 60 * 4, .2), False)
def _load_(self, **kwargs):
"""
Starts the loop to check if any certs need to be updated.
:return:
"""
self.check_if_certs_need_update_loop = LoopingCall(
self.check_if_certs_need_update)
self.check_if_certs_need_update_loop.start(
self._Configs.get('sqldict', 'save_interval',
random_int(60 * 60 * 2, .2), False), False)
# Check if any libraries or modules need certs.
fqdn = self._Configs.get('dns', 'fqdn', None, False)
if fqdn is None:
logger.warn(
"Unable to create webinterface SSL cert: DNS not set properly."
)
return
sslcerts = yield global_invoke_all(
'_sslcerts_',
called_by=self,
)
yield self._add_sslcerts(sslcerts)
def init(self):
self._periodic_clean_sessions = LoopingCall(self.clean_sessions)
self._periodic_clean_sessions.start(random_int(
60 * 60 * 2,
.7)) # Every 60-ish seconds. Save to disk, or remove from memory.
def generate_key(self, sync_when_done=None):
"""
Generates a new GPG key pair. Updates yombo.toml and marks it to be sent when gateway
connects to server again.
"""
operating_mode = self._Loader.operating_mode
if operating_mode != "run":
logger.info("Not creating GPG key, in wrong run mode: {mode}",
mode=operating_mode)
if self._generating_key is True:
return
self._generating_key = True
gwid = self._gateway_id
if gwid is "local":
self.key_generation_status = "failed: gateway not setup, gateway id is missing"
self._generating_key = False
return
passphrase = random_string(length=random_int(200, .1),
char_set="extended")
expire_date = "10y"
# if self.debug_mode is True:
# logger.warn("Setting GPG key to expire in one day due to debug mode.")
# expire_date = "1d"
user_prefix = self._Configs.get("core.system_user_prefix")
input_data = self.gpg_module.gen_key_input(
name_email=f"{gwid}@{user_prefix}.gpg.yombo.net",
name_real="Yombo Gateway",
name_comment="Created by https://Yombo.net",
key_type="RSA",
key_length=4096,
expire_date=expire_date,
preferences=
"SHA512 SHA384 SHA256 SHA224 AES256 AES192 AES CAST5 ZLIB BZIP2 ZIP Uncompressed",
keyserver="hkp://pgp.mit.edu",
revoker="1:9C69E1F8A7C39961C223C485BCEAA0E429FA3EF8",
passphrase=passphrase)
self.key_generation_status = "working"
def do_generate_key():
logger.warn(
"Generating new system GPG key. This can take a little while on slower systems."
)
return self.gpg_module.gen_key(input_data)
newkey = yield threads.deferToThread(do_generate_key)
# print("bb 3: newkey: %s" % newkey)
# print("bb 3: newkey: %s" % newkey.__dict__)
# print("bb 3: newkey: %s" % type(newkey))
self.key_generation_status = "done"
if str(newkey) == "":
logger.error(
"ERROR: Unable to generate GPG keys.... Is GPG installed and configured? Is it in your path?"
)
self._generating_key = False
raise YomboCritical(
"Error with python GPG interface. Is it installed?")
newfingerprint = newkey.fingerprint
self._Configs.set("gpg.fingerprint", newfingerprint)
secret_file = f"{self._working_dir}/etc/gpg/{newfingerprint}.pass"
yield self._Files.save(secret_file, passphrase)
secret_file = f"{self._working_dir}/etc/gpg/last.pass"
yield self._Files.save(secret_file, passphrase)
yield self.load_keys()
if newfingerprint not in self.gpg_keys:
raise YomboWarning(
f"Unable to find newly generated key: {newfingerprint}")
self._Configs.set("gpg.last_sent_yombo", None)
self._Configs.set("gpg.last_sent_keyserver", None)
self._Configs.set("gpg.last_received_keyserver", None)
# self.send_my_gpg_key()
self._generating_key = False
if self._generating_key_deferred is not None and self._generating_key_deferred.called is False:
self._generating_key_deferred.callback(1)
def _start_(self, **kwargs):
"""
We don't do anything, but "pass" so we don't generate an exception.
"""
self.send_my_gpg_key_loop = LoopingCall(self.send_my_gpg_key)
self.send_my_gpg_key_loop.start(random_int(60 * 60 * 6, .2))
def _start_(self, **kwargs):
if self.enabled is True:
self._upload_statistics_loop.start(random_int(60 * 10, .2),
False) # about every 10 minutes
def _started_(self, **kwargs):
self.library_phase = 4
self.memory_usage_checker_loop = LoopingCall(self.memory_usage_checker)
self.memory_usage_checker_loop.start(random_int(600, .10))
def generate_key(self, sync_when_done = None):
"""
Generates a new GPG key pair. Updates yombo.ini and marks it to be sent when gateway
connects to server again.
"""
operating_mode = self._Loader.operating_mode
if operating_mode != "run":
logger.info("Not creating GPG key, in wrong run mode: {mode}", mode=operating_mode)
if self._generating_key is True:
return
self._generating_key = True
gwid = self.gateway_id
gwuuid = self.gwuuid()
if gwid is "local" or gwuuid is None:
self.key_generation_status = "failed: gateway not setup, gateway id or uuid is missing"
self._generating_key = False
return
passphrase = random_string(length=random_int(200, .1), char_set="extended")
expire_date = "5y"
# if self.debug_mode is True:
# logger.warn("Setting GPG key to expire in one day due to debug mode.")
# expire_date = "1d"
input_data = self.gpg.gen_key_input(
name_email=f"{gwid}@gw.gpg.yombo.net",
name_real="Yombo Gateway",
name_comment="Created by https://Yombo.net",
key_type="RSA",
key_length=4096,
expire_date=expire_date,
preferences="SHA512 SHA384 SHA256 SHA224 AES256 AES192 AES CAST5 ZLIB BZIP2 ZIP Uncompressed",
keyserver="hkp://gpg.nebrwesleyan.edu",
revoker="1:9C69E1F8A7C39961C223C485BCEAA0E429FA3EF8",
passphrase=passphrase)
self.key_generation_status = "working"
newkey = yield threads.deferToThread(self.do_generate_key, input_data)
# print("bb 3: newkey: %s" % newkey)
# print("bb 3: newkey: %s" % newkey.__dict__)
# print("bb 3: newkey: %s" % type(newkey))
self.key_generation_status = "done"
if str(newkey) == "":
logger.error("ERROR: Unable to generate GPG keys.... Is GPG installed and configured? Is it in your path?")
self._generating_key = False
raise YomboCritical("Error with python GPG interface. Is it installed?")
private_keys = yield self.get_keyring_keys(True)
newfingerprint = ""
for existing_key_id, key_data in private_keys.items():
# print("inspecting key: %s" % existing_key_id)
if key_data["fingerprint"] == str(newkey):
newfingerprint = key_data["fingerprint"]
break
asciiArmoredPublicKey = self.gpg.export_keys(newfingerprint)
print(f"saving new gpg fingerprint: {newfingerprint}")
self._Configs.set("gpg", "fingerprint", newfingerprint)
secret_file = f"{self._Atoms.get('working_dir')}/etc/gpg/{newfingerprint}.pass"
# print("saveing pass to : %s" % secret_file)
yield save_file(secret_file, passphrase)
secret_file = f"{self._Atoms.get('working_dir')}/etc/gpg/last.pass"
yield save_file(secret_file, passphrase)
self.__mypassphrase = passphrase
self._Configs.set("gpg", "last_sent_yombo", None)
self._Configs.set("gpg", "last_sent_keyserver", None)
self._Configs.set("gpg", "last_received_keyserver", None)
yield self.sync_keyring_to_db()
# self.send_my_gpg_key()
#
# gpg_keys = yield self.gpg.get_keyring_keys(keys=fingerprint)
#
# # print("keys: %s" % type(keys))
# # print("newkey1: %s" % newkey)
# print("newkey2: %s" % str(newkey))
# print("keys: %s" % gpg_keys)
#
# mykey = gpg_keys[fingerprint]
# mykey["publickey"] = asciiArmoredPublicKey
# mykey["notes"] = "Autogenerated."
# mykey["have_private"] = 1
self._generating_key = False
if self._generating_key_deferred is not None and self._generating_key_deferred.called is False:
self._generating_key_deferred.callback(1)
def _start_(self, **kwargs):
if self.is_master is False:
logger.warn(
"Amazon Alexa disabled, only works on the master gateway of a cluster."
)
self._Notifications.add({
'title': 'Alexa disabled',
'message':
'The Amazon Alexa module can only be used on a master gateway node.',
'source': 'Amazon Alexa Module',
'persist': False,
'priority': 'high',
'always_show': True,
'always_show_allow_clear': True,
})
return
if self.module_enabled is False:
return
self.gwid = self._Gateways.local_id
nodes = self._Nodes.search({'node_type': 'module_amazonalexa'})
if len(nodes) == 0:
logger.info("alexa creating new node...")
self.node = yield self._Nodes.create(
label='Module Amazon Alexa',
machine_label='module_amazonalexa',
node_type='module_amazonalexa',
data={
'scenes': {},
'devices': {},
'config': {},
'alexa': {}
},
data_content_type='json',
gateway_id=self.gwid,
destination='gw')
if isinstance(self.node,
dict) and self.node['status'] != 'success':
self.working = False
self._Notifications.add({
'title':
'Alexa unable to create node',
'message':
'The Amazon Alexa module was unable to create a new node to save'
' configuration in. Reason: \n %s' % self.node['msg'],
'source':
'Amazon Alexa Module',
'persist':
False,
'priority':
'high',
'always_show':
True,
'always_show_allow_clear':
True,
})
elif nodes is not None and len(nodes) > 1:
logger.warn(
"Too many node instances. Taking the first one and dropping old ones: {count}",
count=len(nodes))
for node_id, node in nodes.items():
# print("amazon alex has node: %s - %s" % (node_id, node.data))
# print("amazon alex has node: %s - %s" % (node_id, type(node.data)))
self.node = node
if 'alexa' not in self.node.data:
self.node.data['alexa'] = {}
if 'devices' not in self.node.data:
self.node.data['devices'] = {}
if 'allowed' not in self.node.data['devices']:
self.node.data['devices']['allowed'] = []
if 'configs' not in self.node.data:
self.node.data['configs'] = {}
if 'scenes' not in self.node.data:
self.node.data['scenes'] = {}
if 'allowed' not in self.node.data['scenes']:
self.node.data['scenes']['allowed'] = []
break
self.discovery_loop = LoopingCall(self.discovery)
self.discovery_loop.start(random_int(60 * 60 * 12, .25))
def _started_(self, **kwargs):
self.memory_usage_checker_loop = LoopingCall(self.memory_usage_checker)
self.memory_usage_checker_loop.start(random_int(1800, .10))
def _start_(self, **kwargs):
return # Yombo doesn't currently store statistics in the cloud.
self._upload_statistics_loop.start(random_int(60 * 10, .2),
False) # about every 10 minutes
