def mqtt_incoming_gw(self, topics, message): # ybo_gw/src_gwid/dest_gwid body = message["body"] source_id = body["source_id"] component_type = body["component_type"] component_name = body["component_name"] payload = body["payload"] if component_type == "module": try: module = self._Modules[component_name] except Exception as e: logger.info("Received inter-gateway MQTT coms for module {module}, but module not found. Dropping.", module=component_name) return False try: yield maybeDeferred(module._inter_gateway_mqtt_, topics, message) except Exception as e: logger.info("Received inter-gateway MQTT coms for module {module}, but module doesn't have function '_inter_gateway_mqtt_' Dropping.", module=component_name) return False elif component_type == "lib": try: if component_name == "system_ping_response": reply_to = body["reply_to"] if reply_to is None: raise YomboWarning("lib.system_ping requires a reply_id, but not found. Dropping.") if source_id in self._Gateways.gateways and \ self._Gateways.gateways[source_id].ping_request_id == reply_to: gateway = self._Gateways.gateways[source_id] gateway.ping_roundtrip = round( (body["payload"] - gateway.ping_request_at) * 100, 2) gateway.ping_response_at = body["time_received"] gateway.ping_time_offset = round(body["payload"] - body["time_received"], 2) elif component_name == "atoms_set": for name, value in payload.items(): self._Atoms.set_from_gateway_communications(name, value, self) elif component_name == "device_command": self.incoming_data_device_command(body) elif component_name == "device_command_status": self.incoming_data_device_command_status(body) elif component_name == "device_states": self.incoming_data_device_states(body) elif component_name == "notification": self.incoming_data_notification(body) elif component_name == "states_set": for name, value in payload.items(): self._States.set_from_gateway_communications(name, value, self) # elif component_name == "scenes": # for name, value in message["payload"].items(): # self._Scenes.set_from_gateway_communications(name, value, self) elif component_name == "system_state": if payload == "online": self._Gateways.gateways[source_id].com_status = "online" reactor.callLater(random_int(2, .8), self.send_all_info, destination_id=source_id) if payload == "offline": self._Gateways.gateways[source_id].com_status = "offline" except Exception as e: # catch anything here...so can display details. logger.error("---------------==(Traceback)==--------------------------") logger.error("{trace}", trace=traceback.format_exc()) logger.error("--------------------------------------------------------") return True
def _init_(self, **kwargs): self.gateway_id = self._Configs.get('core', 'gwid', 'local', False) cookie_id = hashlib.sha224( str(self._Gateways.get_master_gateway_id()).encode( 'utf-8')).hexdigest() self.config = DictObject({ 'cookie_session_name': 'yombo_' + cookie_id, 'cookie_path': '/', 'max_session': 15552000, # How long session can be good for: 180 days 'max_idle': 5184000, # Max idle timeout: 60 days 'max_session_no_auth': 600, # If not auth in 10 mins, delete session 'ignore_expiry': True, 'ignore_change_ip': True, 'expired_message': 'Session expired', 'httponly': True, 'secure': False, # will change to true after SSL system/dns complete. - Mitch }) self.active_sessions = {} self.clean_sessions_loop = LoopingCall(self.clean_sessions) self.clean_sessions_loop.start(random_int( 60 * 60, .2)) # Every hour-ish. Save to disk, or remove from memory.
def _load_(self, **kwargs): """ Starts the loop to check if any certs need to be updated. :return: """ self.check_if_certs_need_update_loop = LoopingCall( self.check_if_certs_need_update) self.check_if_certs_need_update_loop.start( self._Configs.get("sqldict", "save_interval", random_int(60 * 60 * 24, .1), False), False) # Check if any libraries or modules need certs. if self.local_gateway.dns_name is None: logger.warn( "Unable to generate sign ssl/tls certs, gateway has no domain name." ) return if self._Loader.operating_mode != "run": return sslcerts = yield global_invoke_all( "_sslcerts_", called_by=self, ) for component_name, ssl_certs in sslcerts.items(): logger.debug( f"Adding new managed certs from hook: {component_name}") if isinstance(ssl_certs, tuple) is False and isinstance( ssl_certs, list) is False: ssl_certs = [ssl_certs] for ssl_item in ssl_certs: yield self.add_sslcert(ssl_item)
def _init_(self, **kwargs): self.session_type = "apiauth" self.active_api_auth = {} self.active_api_auth_by_label = {} # self.active_api_auth_cache = ExpiringDict(200, 5) # keep 200 entries, for at most 1 second...??? self._periodic_clean_sessions = LoopingCall(self.clean_sessions) self._periodic_clean_sessions.start(random_int( 60 * 60, .2)) # Every hours-ish. Save or update
def _start_(self, **kwargs): """ We don't do anything, but "pass" so we don't generate an exception. """ if self._Loader.operating_mode != "run": return self.remote_get_root_key() self.send_my_gpg_key_loop = LoopingCall(self.send_my_gpg_key) self.send_my_gpg_key_loop.start(random_int(60 * 60 * 2, .2))
def _init_(self, **kwargs): self.library_phase = 1 self.gateway_id = self._Configs.get('core', 'gwid', 'local', False) self.__States = {self.gateway_id: {}} self.automation_startup_check = {} self.db_save_states_data = deque() self.db_save_states_loop = LoopingCall(self.db_save_states) self.db_save_states_loop.start(random_int(60, .10), False) # clean the database every 6 hours. self.init_deferred = Deferred() self.load_states() return self.init_deferred
def _start_(self, **kwargs): self.module_phase = 3 self.clean_states_loop = LoopingCall(self.clean_states_table) self.clean_states_loop.start(random_int(60*60*6, .10)) # clean the database every 6 hours. if self._States['loader.operating_mode'] == 'run': self.mqtt = self._MQTT.new(mqtt_incoming_callback=self.mqtt_incoming, client_id='Yombo-states-%s' % self.gateway_id) self.mqtt.subscribe("yombo/states/+/get") self.mqtt.subscribe("yombo/states/+/get/+") self.mqtt.subscribe("yombo/states/+/set") self.mqtt.subscribe("yombo/states/+/set/+")
def _init_(self, **kwargs): self.library_phase = 1 self.gateway_id = self._Configs.get("core", "gwid", "local", False) self.states = {self.gateway_id: {}} self.db_save_states_data = deque() self.db_save_states_loop = LoopingCall(self.db_save_states) self.db_save_states_loop.start(random_int(30, .10), False) yield self.load_states() # setup cluster state defaults if not set try: self.get("is.away", gateway_id="cluster") except KeyError: self.set("is.away", False, gateway_id="cluster")
def amqp_connected(self): """ Called by AQMP when connected. This function was define above when setting up self.ampq. :return: """ self._States.set('amqp.amqpyombo.state', True) if self.send_local_information_loop is None: self.send_local_information_loop = LoopingCall( self.send_local_information) # Sends various information, helps Yombo cloud know we are alive and where to find us. if self.send_local_information_loop.running is False: self.send_local_information_loop.start(random_int(60 * 60 * 4, .2))
def _load_(self, **kwargs): """ Starts the loop to check if any certs need to be updated. :return: """ self.check_if_certs_need_update_loop = LoopingCall( self.check_if_certs_need_update) self.check_if_certs_need_update_loop.start( self._Configs.get('sqldict', 'save_interval', random_int(60 * 60 * 24, .1), False)) # Check if any libraries or modules need certs. sslcerts = yield global_invoke_all('_sslcerts_', called_by=self) # print("about to add sslcerts") yield self._add_sslcerts(sslcerts)
def amqp_connected(self): """ Called by AQMP when connected. This function was define above when setting up self.ampq. :return: """ self.connected = True for the_callback in self.amqpyombo_options["connected"]: the_callback() if self.send_local_information_loop is None: self.send_local_information_loop = LoopingCall( self.send_local_information) # Sends various information, helps Yombo cloud know we are alive and where to find us. self.send_local_information(full=True) yield sleep(1) self.init_deferred.callback(1) if self.send_local_information_loop.running is False: self.send_local_information_loop.start(random_int(60 * 60 * 4, .2), False)
def amqp_connected(self): """ Setup from the 'connect' method, set in the 'connected_callback' argument. This method is called once the AMQPYombo connection is complete. This setups the loop that updates Yombo about our current status. Primarily used for Dynamic DNS updates. :return: """ self.connected = True for callback in self.amqpyombo_options["connected"]: callback() if self.send_local_information_loop is None: self.send_local_information_loop = LoopingCall(self.send_local_information) # Sends various information, helps Yombo cloud know we are alive and where to find us. self.send_local_information() if self.send_local_information_loop.running is False: self.send_local_information_loop.start(random_int(60 * 60 * 4, .2), False)
def _load_(self, **kwargs): """ Starts the loop to check if any certs need to be updated. :return: """ self.check_if_certs_need_update_loop = LoopingCall( self.check_if_certs_need_update) self.check_if_certs_need_update_loop.start( self._Configs.get('sqldict', 'save_interval', random_int(60 * 60 * 2, .2), False), False) # Check if any libraries or modules need certs. fqdn = self._Configs.get('dns', 'fqdn', None, False) if fqdn is None: logger.warn( "Unable to create webinterface SSL cert: DNS not set properly." ) return sslcerts = yield global_invoke_all( '_sslcerts_', called_by=self, ) yield self._add_sslcerts(sslcerts)
def init(self): self._periodic_clean_sessions = LoopingCall(self.clean_sessions) self._periodic_clean_sessions.start(random_int( 60 * 60 * 2, .7)) # Every 60-ish seconds. Save to disk, or remove from memory.
def generate_key(self, sync_when_done=None): """ Generates a new GPG key pair. Updates yombo.toml and marks it to be sent when gateway connects to server again. """ operating_mode = self._Loader.operating_mode if operating_mode != "run": logger.info("Not creating GPG key, in wrong run mode: {mode}", mode=operating_mode) if self._generating_key is True: return self._generating_key = True gwid = self._gateway_id if gwid is "local": self.key_generation_status = "failed: gateway not setup, gateway id is missing" self._generating_key = False return passphrase = random_string(length=random_int(200, .1), char_set="extended") expire_date = "10y" # if self.debug_mode is True: # logger.warn("Setting GPG key to expire in one day due to debug mode.") # expire_date = "1d" user_prefix = self._Configs.get("core.system_user_prefix") input_data = self.gpg_module.gen_key_input( name_email=f"{gwid}@{user_prefix}.gpg.yombo.net", name_real="Yombo Gateway", name_comment="Created by https://Yombo.net", key_type="RSA", key_length=4096, expire_date=expire_date, preferences= "SHA512 SHA384 SHA256 SHA224 AES256 AES192 AES CAST5 ZLIB BZIP2 ZIP Uncompressed", keyserver="hkp://pgp.mit.edu", revoker="1:9C69E1F8A7C39961C223C485BCEAA0E429FA3EF8", passphrase=passphrase) self.key_generation_status = "working" def do_generate_key(): logger.warn( "Generating new system GPG key. This can take a little while on slower systems." ) return self.gpg_module.gen_key(input_data) newkey = yield threads.deferToThread(do_generate_key) # print("bb 3: newkey: %s" % newkey) # print("bb 3: newkey: %s" % newkey.__dict__) # print("bb 3: newkey: %s" % type(newkey)) self.key_generation_status = "done" if str(newkey) == "": logger.error( "ERROR: Unable to generate GPG keys.... Is GPG installed and configured? Is it in your path?" ) self._generating_key = False raise YomboCritical( "Error with python GPG interface. Is it installed?") newfingerprint = newkey.fingerprint self._Configs.set("gpg.fingerprint", newfingerprint) secret_file = f"{self._working_dir}/etc/gpg/{newfingerprint}.pass" yield self._Files.save(secret_file, passphrase) secret_file = f"{self._working_dir}/etc/gpg/last.pass" yield self._Files.save(secret_file, passphrase) yield self.load_keys() if newfingerprint not in self.gpg_keys: raise YomboWarning( f"Unable to find newly generated key: {newfingerprint}") self._Configs.set("gpg.last_sent_yombo", None) self._Configs.set("gpg.last_sent_keyserver", None) self._Configs.set("gpg.last_received_keyserver", None) # self.send_my_gpg_key() self._generating_key = False if self._generating_key_deferred is not None and self._generating_key_deferred.called is False: self._generating_key_deferred.callback(1)
def _start_(self, **kwargs): """ We don't do anything, but "pass" so we don't generate an exception. """ self.send_my_gpg_key_loop = LoopingCall(self.send_my_gpg_key) self.send_my_gpg_key_loop.start(random_int(60 * 60 * 6, .2))
def _start_(self, **kwargs): if self.enabled is True: self._upload_statistics_loop.start(random_int(60 * 10, .2), False) # about every 10 minutes
def _started_(self, **kwargs): self.library_phase = 4 self.memory_usage_checker_loop = LoopingCall(self.memory_usage_checker) self.memory_usage_checker_loop.start(random_int(600, .10))
def generate_key(self, sync_when_done = None): """ Generates a new GPG key pair. Updates yombo.ini and marks it to be sent when gateway connects to server again. """ operating_mode = self._Loader.operating_mode if operating_mode != "run": logger.info("Not creating GPG key, in wrong run mode: {mode}", mode=operating_mode) if self._generating_key is True: return self._generating_key = True gwid = self.gateway_id gwuuid = self.gwuuid() if gwid is "local" or gwuuid is None: self.key_generation_status = "failed: gateway not setup, gateway id or uuid is missing" self._generating_key = False return passphrase = random_string(length=random_int(200, .1), char_set="extended") expire_date = "5y" # if self.debug_mode is True: # logger.warn("Setting GPG key to expire in one day due to debug mode.") # expire_date = "1d" input_data = self.gpg.gen_key_input( name_email=f"{gwid}@gw.gpg.yombo.net", name_real="Yombo Gateway", name_comment="Created by https://Yombo.net", key_type="RSA", key_length=4096, expire_date=expire_date, preferences="SHA512 SHA384 SHA256 SHA224 AES256 AES192 AES CAST5 ZLIB BZIP2 ZIP Uncompressed", keyserver="hkp://gpg.nebrwesleyan.edu", revoker="1:9C69E1F8A7C39961C223C485BCEAA0E429FA3EF8", passphrase=passphrase) self.key_generation_status = "working" newkey = yield threads.deferToThread(self.do_generate_key, input_data) # print("bb 3: newkey: %s" % newkey) # print("bb 3: newkey: %s" % newkey.__dict__) # print("bb 3: newkey: %s" % type(newkey)) self.key_generation_status = "done" if str(newkey) == "": logger.error("ERROR: Unable to generate GPG keys.... Is GPG installed and configured? Is it in your path?") self._generating_key = False raise YomboCritical("Error with python GPG interface. Is it installed?") private_keys = yield self.get_keyring_keys(True) newfingerprint = "" for existing_key_id, key_data in private_keys.items(): # print("inspecting key: %s" % existing_key_id) if key_data["fingerprint"] == str(newkey): newfingerprint = key_data["fingerprint"] break asciiArmoredPublicKey = self.gpg.export_keys(newfingerprint) print(f"saving new gpg fingerprint: {newfingerprint}") self._Configs.set("gpg", "fingerprint", newfingerprint) secret_file = f"{self._Atoms.get('working_dir')}/etc/gpg/{newfingerprint}.pass" # print("saveing pass to : %s" % secret_file) yield save_file(secret_file, passphrase) secret_file = f"{self._Atoms.get('working_dir')}/etc/gpg/last.pass" yield save_file(secret_file, passphrase) self.__mypassphrase = passphrase self._Configs.set("gpg", "last_sent_yombo", None) self._Configs.set("gpg", "last_sent_keyserver", None) self._Configs.set("gpg", "last_received_keyserver", None) yield self.sync_keyring_to_db() # self.send_my_gpg_key() # # gpg_keys = yield self.gpg.get_keyring_keys(keys=fingerprint) # # # print("keys: %s" % type(keys)) # # print("newkey1: %s" % newkey) # print("newkey2: %s" % str(newkey)) # print("keys: %s" % gpg_keys) # # mykey = gpg_keys[fingerprint] # mykey["publickey"] = asciiArmoredPublicKey # mykey["notes"] = "Autogenerated." # mykey["have_private"] = 1 self._generating_key = False if self._generating_key_deferred is not None and self._generating_key_deferred.called is False: self._generating_key_deferred.callback(1)
def _start_(self, **kwargs): if self.is_master is False: logger.warn( "Amazon Alexa disabled, only works on the master gateway of a cluster." ) self._Notifications.add({ 'title': 'Alexa disabled', 'message': 'The Amazon Alexa module can only be used on a master gateway node.', 'source': 'Amazon Alexa Module', 'persist': False, 'priority': 'high', 'always_show': True, 'always_show_allow_clear': True, }) return if self.module_enabled is False: return self.gwid = self._Gateways.local_id nodes = self._Nodes.search({'node_type': 'module_amazonalexa'}) if len(nodes) == 0: logger.info("alexa creating new node...") self.node = yield self._Nodes.create( label='Module Amazon Alexa', machine_label='module_amazonalexa', node_type='module_amazonalexa', data={ 'scenes': {}, 'devices': {}, 'config': {}, 'alexa': {} }, data_content_type='json', gateway_id=self.gwid, destination='gw') if isinstance(self.node, dict) and self.node['status'] != 'success': self.working = False self._Notifications.add({ 'title': 'Alexa unable to create node', 'message': 'The Amazon Alexa module was unable to create a new node to save' ' configuration in. Reason: \n %s' % self.node['msg'], 'source': 'Amazon Alexa Module', 'persist': False, 'priority': 'high', 'always_show': True, 'always_show_allow_clear': True, }) elif nodes is not None and len(nodes) > 1: logger.warn( "Too many node instances. Taking the first one and dropping old ones: {count}", count=len(nodes)) for node_id, node in nodes.items(): # print("amazon alex has node: %s - %s" % (node_id, node.data)) # print("amazon alex has node: %s - %s" % (node_id, type(node.data))) self.node = node if 'alexa' not in self.node.data: self.node.data['alexa'] = {} if 'devices' not in self.node.data: self.node.data['devices'] = {} if 'allowed' not in self.node.data['devices']: self.node.data['devices']['allowed'] = [] if 'configs' not in self.node.data: self.node.data['configs'] = {} if 'scenes' not in self.node.data: self.node.data['scenes'] = {} if 'allowed' not in self.node.data['scenes']: self.node.data['scenes']['allowed'] = [] break self.discovery_loop = LoopingCall(self.discovery) self.discovery_loop.start(random_int(60 * 60 * 12, .25))
def _started_(self, **kwargs): self.memory_usage_checker_loop = LoopingCall(self.memory_usage_checker) self.memory_usage_checker_loop.start(random_int(1800, .10))
def _start_(self, **kwargs): return # Yombo doesn't currently store statistics in the cloud. self._upload_statistics_loop.start(random_int(60 * 10, .2), False) # about every 10 minutes