def install_python_requirement(self, requirement, source=None):
if source is None:
source = "System"
line = yombo.utils.bytes_to_unicode(requirement)
line = line.strip()
if len(line) == 0 or line.startswith("#") or line.startswith("git+"):
return
logger.debug("Processing requirement: {requirement}", requirement=line)
requirement = pkg_requirement(line)
package_name = requirement.name
package_specifier = requirement.specifier
if package_name in self.requirements:
if self.requirements[package_name][
'specifier'] == package_specifier:
self.requirements[package_name]['used_by'].append(source)
return
else:
logger.warn(
"Unable to install conflicting python module '{name}'. Version '{current}' already set,"
" requested '{new}' by: {source}",
name=package_name,
current=self.requirements[package_name]['specifier'],
new=package_specifier,
source=source)
return
try:
pkg_info = yield yombo.utils.get_python_package_info(
line, events_queue=self.startup_events_queue)
# logger.debug("Processing requirement: results: {results}", results=pkg_info)
if pkg_info is None:
return
except YomboWarning as e:
raise YomboCritical(e.message)
# logger.debug("Have requirement details...")
if pkg_info is not None:
save_info = {
"name": pkg_info.project_name,
"version": pkg_info._version,
"path": pkg_info.location,
"used_by": [
source,
],
"specifier": package_specifier,
}
else:
save_info = {
"name": package_name,
"version": "Invalid python module",
"path": "Invalid module",
"used_by": [
source,
],
"specifier": package_specifier,
}
self.requirements[package_name] = save_info
def generate_key(self, request_id=None):
"""
Generates a new GPG key pair. Updates yombo.ini and marks it to be sent when gateway conencts to server
again.
"""
if self.gateway_id is 'local' or self.gwuuid is None:
self._key_generation_status[
request_id] = 'failed-gateway not setup'
return
input_data = self.gpg.gen_key_input(name_email=self.gwuuid() +
"@yombo.net",
name_real="Yombo Gateway",
name_comment="gw_" + self.gwuuid(),
key_type='RSA',
key_length=2048,
expire_date='5y')
if request_id is None:
request_id = random_string(length=16)
self._key_generation_status[request_id] = 'working'
newkey = yield self.gpg.gen_key(input_data)
self._key_generation_status[request_id] = 'done'
# print("newkey!!!!!! ====")
# print(format(newkey))
# print("request id =")
# print(request_id)
if newkey == '':
logger.error(
"ERROR: Unable to generate GPG keys.... Is GPG installed and configured? Is it in your path?"
)
raise YomboCritical(
"Error with python GPG interface. Is it installed?")
private_keys = self.gpg.list_keys(True)
keyid = ''
for key in private_keys:
if str(key['fingerprint']) == str(newkey):
keyid = key['keyid']
asciiArmoredPublicKey = self.gpg.export_keys(keyid)
gpg_keys = yield self.gpg.list_keys(keyid)
keys = self._format_list_keys(gpg_keys)
# print("keys: %s" % type(keys))
# print("keys: %s" % keys)
data = keys[format(newkey)]
data['publickey'] = asciiArmoredPublicKey
data['notes'] = 'Key generated during wizard setup.'
data['have_private'] = 1
yield self.local_db.insert_gpg_key(data)
returnValue({'keyid': keyid, 'keypublicascii': asciiArmoredPublicKey})
def _init_(self, **kwargs):
"""
Reads the yombo.toml file and stores the settings in memory for fast access.
"""
self.pid = os.getpid()
self.exit_config_file = None # Holds a complete configuration file to save when exiting.
self.yombo_toml_path = f"{self._working_dir}/yombo.toml"
self.yombo_toml_meta_path = f"{self._working_dir}/etc/yombo_meta.toml"
self.save_loop = LoopingCall(self.save,
force_save=False,
display_extra_warning=True)
self.save_loop.start(randint(12600, 14400), False) # every 3.5-4 hours
self.setup_defaults()
has_cfg_file = self.check_yombo_toml_file()
if has_cfg_file is False:
logger.warn("yombo.toml file not found, creating a new one.")
yield self.save(force_save=True)
try:
yield self.import_toml_file()
except TypeError as e:
logger.error("{e}", e=e)
logger.error(
"Check validate the yombo.toml file at: {yombo_toml_path}",
yombo_toml_path=self.yombo_toml_path)
logger.error(
"Validate the syntax at the line/position at the line noted above."
)
raise YomboCritical(str(e))
else:
self.yombo_toml_loaded = True
if self.get("core.gwid") == "local":
self._Loader.operating_mode = "config"
else:
self._Loader.operating_mode = "run"
yield self.update_network_details()
self._loaded_toml = True
def import_libraries_failure(self, failure):
logger.error(
"Got failure during import of library: {failure}. Going to stop now.",
failure=failure)
raise YomboCritical("Load failure for gateway library.")
def do_shutdown(self):
raise YomboCritical("Web Interface setup wizard complete.")
def _load_(self, **kwargs):
if self.server_enabled is False:
logger.info("Embedded MQTT Disabled.")
return
if self.is_master is not True:
logger.info("Not managing MQTT broker, we are not the master!")
if self.mosquitto_enabled is True:
logger.info("Disabling mosquitto MQTT broker.")
try:
yield getProcessOutput(
"sudo", ["systemctl", "disable", "mosquitto.service"])
except Exception as e:
logger.warn(
"Error while trying to disable mosquitto (mqtt) service: {e}",
e=e)
yield self.start_mqtt_broker()
logger.info("Sleeping for 2 seconds while MQTT broker stops.")
self._Configs.set("mqtt", "mosquitto_enabled", False)
self.mosquitto_enabled = False
yield sleep(2)
return
ssl_self_signed = self._SSLCerts.get("selfsigned")
ssl_lib_webinterface = self._SSLCerts.get("lib_webinterface")
mosquitto_config = [
"allow_anonymous false",
"user mosquitto",
"persistent_client_expiration 4h",
"max_connections 512",
"",
]
if self.server_listen_port > 0:
mosquitto_config.extend([
"#", "# Insecure listen MQTT port", "#",
f"port {self.server_listen_port}", ""
])
if self.server_listen_port_ss_ssl > 0:
mosquitto_config.extend([
"#",
"# Self-signed cert for mqtt",
"#",
f"listener {self.server_listen_port_ss_ssl}",
f"certfile {ssl_self_signed['cert_file']}",
f"keyfile {ssl_self_signed['key_file']}",
"ciphers ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS",
"tls_version tlsv1.2",
"protocol mqtt",
"",
])
if self.server_listen_port_le_ssl > 0 and ssl_lib_webinterface[
"self_signed"] is False:
mosquitto_config.extend([
"#",
"# Lets encrypt signed cert for mqtt",
"#",
f"listener {self.server_listen_port_le_ssl}",
f"cafile {ssl_lib_webinterface['chain_file']}",
f"certfile {ssl_lib_webinterface['cert_file']}",
f"keyfile {ssl_lib_webinterface['key_file']}",
"ciphers ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS",
"tls_version tlsv1.2",
"protocol mqtt",
"",
])
if self.server_listen_port_websockets > 0:
mosquitto_config.extend([
"#",
"# Unecrypted websockets",
"#",
f"listener {self.server_listen_port_websockets}",
"protocol websockets",
"max_connections 512",
"",
])
if self.server_listen_port_websockets_ss_ssl > 0:
mosquitto_config.extend([
"#",
"# Self-signed cert for websockets",
"#",
f"listener {self.server_listen_port_websockets_ss_ssl}",
f"certfile {ssl_self_signed['cert_file']}",
f"keyfile {ssl_self_signed['key_file']}",
"ciphers ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS",
"tls_version tlsv1.2",
"protocol websockets",
"",
])
if self.server_listen_port_websockets_le_ssl > 0 and ssl_lib_webinterface[
"self_signed"] is False:
mosquitto_config.extend([
"#",
"# Lets encrypt signed cert for websockets",
"#",
f"listener {self.server_listen_port_websockets_le_ssl}",
f"cafile {ssl_lib_webinterface['chain_file']}",
f"certfile {ssl_lib_webinterface['cert_file']}",
f"keyfile {ssl_lib_webinterface['key_file']}",
"ciphers ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS",
"tls_version tlsv1.2",
"protocol websockets",
"",
])
if ssl_lib_webinterface["self_signed"] is False:
self.server_listen_port_websockets_le_ssl = self.server_listen_port_websockets_ss_ssl
logger.debug(
"Writting mosquitto_config_file to: {mosquitto_config_file}",
mosquitto_config_file=self.mosquitto_config_file)
config_file = FileWriter(filename=self.mosquitto_config_file, mode="w")
config_file.write(
"# File automatically generated by Yombo Gateway. Edits will be lost.\n"
)
config_file.write(
f"# Created {f'{datetime.now():%Y-%m-%d %H%M%S}'}\n\n")
config_file.write("# HTTP Auth plugin...\n")
config_file.write(
"auth_plugin /usr/local/src/yombo/mosquitto-auth-plug/auth-plug.so\n"
)
config_file.write("auth_opt_backends http\n")
# config_file.write("auth_opt_acl_log_quiet true\n")
config_file.write("auth_opt_acl_cacheseconds 600\n")
config_file.write("auth_opt_auth_cacheseconds 30\n")
config_file.write("auth_opt_http_ip 127.0.0.1\n")
webinterface_port = self._Configs.get("webinterface",
"nonsecure_port",
fallback=8080)
config_file.write(f"auth_opt_http_port {webinterface_port}\n")
config_file.write("auth_opt_http_getuser_uri /api/v1/mqtt/auth/user\n")
config_file.write(
"auth_opt_http_superuser_uri /api/v1/mqtt/auth/superuser\n")
config_file.write("auth_opt_http_aclcheck_uri /api/v1/mqtt/auth/acl\n")
config_file.write("# Base configs\n\n")
for line_out in mosquitto_config:
config_file.write(f"{line_out}\n")
yield config_file.close_while_waiting()
if self.mosquitto_enabled is False:
logger.info("Enabling mosquitto MQTT broker.")
try:
yield getProcessOutput(
"sudo", ["systemctl", "enable", "mosquitto.service"])
except Exception as e:
logger.warn(
"Error while trying to enable mosquitto (mqtt) service: {e}",
e=e)
self._Configs.set("mqtt", "mosquitto_enabled", True)
self.mosquitto_enabled = True
yield self.check_mqtt_broker_running()
if self.mosquitto_running is False:
yield self.start_mqtt_broker()
logger.info("Sleeping for 3 seconds while MQTT broker starts up.")
yield sleep(3)
if self.mosquitto_running is False:
logger.error("Cannot connect to MQTT broker.")
raise YomboCritical(
"MQTT failed to connect and/or start, shutting down.")
def import_component(self,
pathName,
componentName,
componentType,
componentUUID=None):
"""
Load component of given name. Can be a core library, or a module.
"""
pymodulename = pathName.lower()
self._log_loader('debug', componentName, componentType, 'import',
'About to import.')
try:
pyclassname = ReSearch("(?<=\.)([^.]+)$", pathName).group(1)
except AttributeError:
self._log_loader('error', componentName, componentType, 'import',
'Not found. Path: %s' % pathName)
logger.error("Library or Module not found: {pathName}",
pathName=pathName)
raise YomboCritical("Library or Module not found: %s", pathName)
try:
module_root = __import__(pymodulename, globals(), locals(), [], 0)
except ImportError as detail:
self._log_loader('error', componentName, componentType, 'import',
'Not found. Path: %s' % pathName)
logger.error(
"--------==(Error: Library or Module not found)==--------")
logger.error("----Name: {pathName}, Details: {detail}",
pathName=pathName,
detail=detail)
logger.error(
"---------------==(Traceback)==--------------------------")
logger.error("{trace}", trace=traceback.format_exc())
logger.error(
"--------------------------------------------------------")
raise ImportError("Cannot import module, not found.")
except Exception as e:
logger.error(
"---------------==(Traceback)==--------------------------")
logger.error("{trace}", trace=traceback.format_exc())
logger.error(
"--------------------------------------------------------")
logger.warn(
"An exception of type {etype} occurred in yombo.lib.nodes:import_component. Message: {msg}",
etype=type(e),
msg=e)
raise ImportError(e)
module_tail = reduce(lambda p1, p2: getattr(p1, p2), [
module_root,
] + pymodulename.split('.')[1:])
# print "module_tail: %s pyclassname: %s" % (module_tail, pyclassname)
klass = getattr(module_tail, pyclassname)
# print "klass: %s " % klass
# Put the component into various lists for mgmt
if not isinstance(klass, Callable):
logger.error(
"Unable to start class '{classname}', it's not callable.",
classname=pyclassname)
raise ImportError(
"Unable to start class '%s', it's not callable." % pyclassname)
try:
# Instantiate the class
# logger.debug("Instantiate class: {pyclassname}", pyclassname=pyclassname)
moduleinst = klass(
) # start the class, only libraries get the loader
if componentType == 'library':
if componentName.lower() == 'modules':
self._moduleLibrary = moduleinst
self.loadedComponents["yombo.gateway.lib." +
str(componentName.lower())] = moduleinst
self.loadedLibraries[str(componentName.lower())] = moduleinst
# this is mostly for manhole module, but maybe useful elsewhere?
temp = componentName.split(".")
self.libraryNames[temp[-1]] = moduleinst
else:
self.loadedComponents["yombo.gateway.modules." +
str(componentName.lower())] = moduleinst
return moduleinst, componentName.lower()
except YomboCritical as e:
logger.debug(
"@!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
)
logger.debug("{e}", e=e)
logger.debug(
"@!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
)
e.exit()
raise
def generate_key(self, sync_when_done=None):
"""
Generates a new GPG key pair. Updates yombo.toml and marks it to be sent when gateway
connects to server again.
"""
operating_mode = self._Loader.operating_mode
if operating_mode != "run":
logger.info("Not creating GPG key, in wrong run mode: {mode}",
mode=operating_mode)
if self._generating_key is True:
return
self._generating_key = True
gwid = self._gateway_id
if gwid is "local":
self.key_generation_status = "failed: gateway not setup, gateway id is missing"
self._generating_key = False
return
passphrase = random_string(length=random_int(200, .1),
char_set="extended")
expire_date = "10y"
# if self.debug_mode is True:
# logger.warn("Setting GPG key to expire in one day due to debug mode.")
# expire_date = "1d"
user_prefix = self._Configs.get("core.system_user_prefix")
input_data = self.gpg_module.gen_key_input(
name_email=f"{gwid}@{user_prefix}.gpg.yombo.net",
name_real="Yombo Gateway",
name_comment="Created by https://Yombo.net",
key_type="RSA",
key_length=4096,
expire_date=expire_date,
preferences=
"SHA512 SHA384 SHA256 SHA224 AES256 AES192 AES CAST5 ZLIB BZIP2 ZIP Uncompressed",
keyserver="hkp://pgp.mit.edu",
revoker="1:9C69E1F8A7C39961C223C485BCEAA0E429FA3EF8",
passphrase=passphrase)
self.key_generation_status = "working"
def do_generate_key():
logger.warn(
"Generating new system GPG key. This can take a little while on slower systems."
)
return self.gpg_module.gen_key(input_data)
newkey = yield threads.deferToThread(do_generate_key)
# print("bb 3: newkey: %s" % newkey)
# print("bb 3: newkey: %s" % newkey.__dict__)
# print("bb 3: newkey: %s" % type(newkey))
self.key_generation_status = "done"
if str(newkey) == "":
logger.error(
"ERROR: Unable to generate GPG keys.... Is GPG installed and configured? Is it in your path?"
)
self._generating_key = False
raise YomboCritical(
"Error with python GPG interface. Is it installed?")
newfingerprint = newkey.fingerprint
self._Configs.set("gpg.fingerprint", newfingerprint)
secret_file = f"{self._working_dir}/etc/gpg/{newfingerprint}.pass"
yield self._Files.save(secret_file, passphrase)
secret_file = f"{self._working_dir}/etc/gpg/last.pass"
yield self._Files.save(secret_file, passphrase)
yield self.load_keys()
if newfingerprint not in self.gpg_keys:
raise YomboWarning(
f"Unable to find newly generated key: {newfingerprint}")
self._Configs.set("gpg.last_sent_yombo", None)
self._Configs.set("gpg.last_sent_keyserver", None)
self._Configs.set("gpg.last_received_keyserver", None)
# self.send_my_gpg_key()
self._generating_key = False
if self._generating_key_deferred is not None and self._generating_key_deferred.called is False:
self._generating_key_deferred.callback(1)
def _load_(self, **kwargs):
if self.server_enabled is False:
logger.info("Embedded MQTT Disabled.")
return
if self.is_master is not True:
logger.info("Not managing MQTT broker, we are not the master!")
if self.mosquitto_enabled is True:
logger.info("Disabling mosquitto MQTT broker.")
try:
yield getProcessOutput(
"sudo", ['systemctl', 'disable', 'mosquitto.service'])
except Exception as e:
logger.warn(
"Error while trying to disable mosquitto (mqtt) service: {e}",
e=e)
yield self.start_mqtt_broker()
logger.info("Sleeping for 2 seconds while MQTT broker stops.")
self._Configs.set('mqtt', 'mosquitto_enabled', False)
self.mosquitto_enabled = False
yield sleep(2)
return
ssl_self_signed = self._SSLCerts.get('selfsigned')
ssl_lib_webinterface = self._SSLCerts.get('lib_webinterface')
mosquitto_config = [
'allow_anonymous false',
'password_file /etc/mosquitto/yombo/passwd',
'user mosquitto',
'persistent_client_expiration 4h',
'max_connections 512',
'',
]
if self.server_listen_port > 0:
mosquitto_config.append("port %s" % self.server_listen_port)
if self.server_listen_port_ss_ssl > 0:
mosquitto_config.extend([
'#',
'# Self-signed cert for mqtt',
'#',
'listener %s' % self.server_listen_port_ss_ssl,
'certfile %s' % ssl_self_signed['cert_file'],
'keyfile %s' % ssl_self_signed['key_file'],
'ciphers ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS',
'tls_version tlsv1.2',
'protocol mqtt',
'',
])
if self.server_listen_port_le_ssl > 0 and ssl_lib_webinterface[
'self_signed'] is False:
mosquitto_config.extend([
'#',
'# Lets encrypt signed cert for mqtt',
'#',
'listener %s' % self.server_listen_port_le_ssl,
'cafile %s' % ssl_lib_webinterface['chain_file'],
'certfile %s' % ssl_lib_webinterface['cert_file'],
'keyfile %s' % ssl_lib_webinterface['key_file'],
'ciphers ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS',
'tls_version tlsv1.2',
'protocol mqtt',
'',
])
if self.server_listen_port_websockets > 0:
mosquitto_config.extend([
'#',
'# Unecrypted websockets',
'#',
'listener %s' % self.server_listen_port_websockets,
'protocol websockets',
'max_connections 512',
'',
])
if self.server_listen_port_websockets_ss_ssl > 0:
mosquitto_config.extend([
'#',
'# Self-signed cert for websockets',
'#',
'listener %s' % self.server_listen_port_websockets_ss_ssl,
'certfile %s' % ssl_self_signed['cert_file'],
'keyfile %s' % ssl_self_signed['key_file'],
'ciphers ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS',
'tls_version tlsv1.2',
'protocol websockets',
'',
])
if self.server_listen_port_websockets_le_ssl > 0 and ssl_lib_webinterface[
'self_signed'] is False:
mosquitto_config.extend([
'#',
'# Lets encrypt signed cert for websockets',
'#',
'listener %s' % self.server_listen_port_websockets_le_ssl,
'cafile %s' % ssl_lib_webinterface['chain_file'],
'certfile %s' % ssl_lib_webinterface['cert_file'],
'keyfile %s' % ssl_lib_webinterface['key_file'],
'ciphers ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS',
'tls_version tlsv1.2',
'protocol websockets',
'',
])
if ssl_lib_webinterface['self_signed'] is False:
self.server_listen_port_websockets_le_ssl = self.server_listen_port_websockets_ss_ssl
self.mqtt_available_ports = {
'ws': self.server_listen_port_websockets,
'wss': self.server_listen_port_websockets_le_ssl,
'wss-ss': self.server_listen_port_websockets_ss_ssl,
}
mosquitto_config_filepointer = open(self.mosquitto_config_file, 'w')
print(
"# File automatically generated by Yombo Gateway. Edits will be lost.",
file=mosquitto_config_filepointer)
print("# Created %s" % f"{datetime.now():%Y-%m-%d %H%M%S}",
file=mosquitto_config_filepointer)
print("#", file=mosquitto_config_filepointer)
print("# Base configs", file=mosquitto_config_filepointer)
print("#", file=mosquitto_config_filepointer)
for line_out in mosquitto_config:
print(line_out, file=mosquitto_config_filepointer)
mosquitto_config_filepointer.close()
password_file = open(self.mosquitto_pass_file, 'w')
print(
"# File automatically generated by Yombo Gateway. Edits will be lost.",
file=password_file)
print("# Created %s" % f"{datetime.now():%Y-%m-%d %H%M%S}",
file=password_file)
cfg_users = self._Configs.get('mqtt_users', '*')
if cfg_users is not None:
print("# ", file=password_file)
print("# Users from yombo.ini", file=password_file)
print("# ", file=password_file)
for username, password in cfg_users.items():
print("%s:%s" % (username, sha512_crypt_mosquitto(password)),
file=password_file)
gateway_passwords = self._Gateways.get_mqtt_passwords()
for gateway, passwords in gateway_passwords.items():
print("# ", file=password_file)
print("# Gateways", file=password_file)
print("# ", file=password_file)
print("yombogw_%s:%s" %
(gateway, sha512_crypt_mosquitto(passwords['current'])),
file=password_file)
password_file.close()
if self.mosquitto_enabled is False:
logger.info("Enabling mosquitto MQTT broker.")
try:
yield getProcessOutput(
"sudo", ['systemctl', 'enable', 'mosquitto.service'])
except Exception as e:
logger.warn(
"Error while trying to enable mosquitto (mqtt) service: {e}",
e=e)
self._Configs.set('mqtt', 'mosquitto_enabled', True)
self.mosquitto_enabled = True
yield self.check_mqtt_broker_running()
if self.mosquitto_running is False:
yield self.start_mqtt_broker()
logger.info("Sleeping for 2 seconds while MQTT broker starts up.")
yield sleep(2)
if self.mosquitto_running is False:
logger.error("MQTT failed to start!")
raise YomboCritical("MQTT failed to start, shutting down.")
else:
yield self.reload_mqtt_broker() #reload the configs
def generate_key(self, sync_when_done = None):
"""
Generates a new GPG key pair. Updates yombo.ini and marks it to be sent when gateway
connects to server again.
"""
operating_mode = self._Loader.operating_mode
if operating_mode != "run":
logger.info("Not creating GPG key, in wrong run mode: {mode}", mode=operating_mode)
if self._generating_key is True:
return
self._generating_key = True
gwid = self.gateway_id
gwuuid = self.gwuuid()
if gwid is "local" or gwuuid is None:
self.key_generation_status = "failed: gateway not setup, gateway id or uuid is missing"
self._generating_key = False
return
passphrase = random_string(length=random_int(200, .1), char_set="extended")
expire_date = "5y"
# if self.debug_mode is True:
# logger.warn("Setting GPG key to expire in one day due to debug mode.")
# expire_date = "1d"
input_data = self.gpg.gen_key_input(
name_email=f"{gwid}@gw.gpg.yombo.net",
name_real="Yombo Gateway",
name_comment="Created by https://Yombo.net",
key_type="RSA",
key_length=4096,
expire_date=expire_date,
preferences="SHA512 SHA384 SHA256 SHA224 AES256 AES192 AES CAST5 ZLIB BZIP2 ZIP Uncompressed",
keyserver="hkp://gpg.nebrwesleyan.edu",
revoker="1:9C69E1F8A7C39961C223C485BCEAA0E429FA3EF8",
passphrase=passphrase)
self.key_generation_status = "working"
newkey = yield threads.deferToThread(self.do_generate_key, input_data)
# print("bb 3: newkey: %s" % newkey)
# print("bb 3: newkey: %s" % newkey.__dict__)
# print("bb 3: newkey: %s" % type(newkey))
self.key_generation_status = "done"
if str(newkey) == "":
logger.error("ERROR: Unable to generate GPG keys.... Is GPG installed and configured? Is it in your path?")
self._generating_key = False
raise YomboCritical("Error with python GPG interface. Is it installed?")
private_keys = yield self.get_keyring_keys(True)
newfingerprint = ""
for existing_key_id, key_data in private_keys.items():
# print("inspecting key: %s" % existing_key_id)
if key_data["fingerprint"] == str(newkey):
newfingerprint = key_data["fingerprint"]
break
asciiArmoredPublicKey = self.gpg.export_keys(newfingerprint)
print(f"saving new gpg fingerprint: {newfingerprint}")
self._Configs.set("gpg", "fingerprint", newfingerprint)
secret_file = f"{self._Atoms.get('working_dir')}/etc/gpg/{newfingerprint}.pass"
# print("saveing pass to : %s" % secret_file)
yield save_file(secret_file, passphrase)
secret_file = f"{self._Atoms.get('working_dir')}/etc/gpg/last.pass"
yield save_file(secret_file, passphrase)
self.__mypassphrase = passphrase
self._Configs.set("gpg", "last_sent_yombo", None)
self._Configs.set("gpg", "last_sent_keyserver", None)
self._Configs.set("gpg", "last_received_keyserver", None)
yield self.sync_keyring_to_db()
# self.send_my_gpg_key()
#
# gpg_keys = yield self.gpg.get_keyring_keys(keys=fingerprint)
#
# # print("keys: %s" % type(keys))
# # print("newkey1: %s" % newkey)
# print("newkey2: %s" % str(newkey))
# print("keys: %s" % gpg_keys)
#
# mykey = gpg_keys[fingerprint]
# mykey["publickey"] = asciiArmoredPublicKey
# mykey["notes"] = "Autogenerated."
# mykey["have_private"] = 1
self._generating_key = False
if self._generating_key_deferred is not None and self._generating_key_deferred.called is False:
self._generating_key_deferred.callback(1)
def generate_key(self, sync_when_done=None):
"""
Generates a new GPG key pair. Updates yombo.ini and marks it to be sent when gateway
connects to server again.
"""
operating_mode = self._Loader.operating_mode
if operating_mode != "run":
logger.info("Not creating GPG key, in wrong run mode: {mode}",
mode=operating_mode)
if self._generating_key is True:
return
self._generating_key = True
gwid = self.gateway_id()
gwuuid = self.gwuuid()
if gwid is 'local' or gwuuid is None:
self.key_generation_status = 'failed-gateway not setup'
self._generating_key = False
return
passphrase = random_string(length=120)
expire_date = '1y'
if self.debug_mode is True:
logger.warn(
'Setting GPG key to expire in one day due to debug mode.')
expire_date = '1d'
input_data = self.gpg.gen_key_input(
name_email="*****@*****.**" % gwuuid,
name_real="Yombo Gateway",
name_comment="Created by https://Yombo.net Automation",
key_type='RSA',
key_length=4096,
expire_date=expire_date,
preferences=
'SHA512 SHA384 SHA256 SHA224 AES256 AES192 AES CAST5 ZLIB BZIP2 ZIP Uncompressed',
keyserver='hkp://pool.sks-keyservers.net',
revoker="1:9C69E1F8A7C39961C223C485BCEAA0E429FA3EF8",
passphrase=passphrase)
self.key_generation_status = 'working'
newkey = yield threads.deferToThread(self._gen_key, input_data)
# print("bb 3: newkey: %s" % newkey)
# print("bb 3: newkey: %s" % newkey.__dict__)
# print("bb 3: newkey: %s" % type(newkey))
self.key_generation_status = 'done'
if str(newkey) == '':
logger.error(
"ERROR: Unable to generate GPG keys.... Is GPG installed and configured? Is it in your path?"
)
self._generating_key = False
raise YomboCritical(
"Error with python GPG interface. Is it installed?")
private_keys = yield self.get_keyring_keys(True)
newkeyid = ''
for existing_key_id, key_data in private_keys.items():
# print("inspecting key: %s" % existing_key_id)
if key_data['fingerprint'] == str(newkey):
newkeyid = key_data['keyid']
break
asciiArmoredPublicKey = self.gpg.export_keys(newkeyid)
self._Configs.set('gpg', 'keyid', newkeyid)
secret_file = "%s/usr/etc/gpg/%s.pass" % (
self._Atoms.get('yombo.path'), newkeyid)
# print("saveing pass to : %s" % secret_file)
yield save_file(secret_file, passphrase)
self.__mypassphrase = passphrase
self._Configs.set('gpg', 'last_sent_yombo', None)
self._Configs.set('gpg', 'last_sent_keyserver', None)
self._Configs.set('gpg', 'last_received_keyserver', None)
yield self.sync_keyring_to_db()
# self.send_my_gpg_key()
#
# gpg_keys = yield self.gpg.get_keyring_keys(keys=keyid)
#
# # print("keys: %s" % type(keys))
# # print("newkey1: %s" % newkey)
# print("newkey2: %s" % str(newkey))
# print("keys: %s" % gpg_keys)
#
# mykey = gpg_keys[keyid]
# mykey['publickey'] = asciiArmoredPublicKey
# mykey['notes'] = 'Autogenerated.'
# mykey['have_private'] = 1
self._generating_key = False
if self._generating_key_deferred is not None and self._generating_key_deferred.called is False:
self._generating_key_deferred.callback(1)
def setup_database_connection(self) -> None:
"""
Setup various database attributes. Will call the migration tool YoYo and perform any migrations
Check to make sure the database exists. Will create if missing, will also update schema if any
changes are required.
Ensure the database is available:
sqlite - That the file is available for reading
mysql/mariadb - That we can connect to the database with the provided credentials.
Also ensures that any pending migrations are complete.
:return:
"""
if self.database_type not in ("sqlite", "mysql", "mariadb"):
raise YomboCritical(
"Only sqlite, mariabdb, or mysql databases are currently supported."
)
self.database = None # Reference to the database connection.
self.db_bulk_queue: Dict[str, Any] = {}
self.db_bulk_queue_id_cols: Dict[str, Any] = {}
self.db_pool = None
self.db_model: Dict[str,
dict] = {} # store generated database model here.
self.db_save_bulk_queue_running = False
self.db_save_bulk_queue_run_again = False
self.db_save_bulk_queue_loop = None
self.db_cleanup_loop = None
self.db_cleanup_running = False
# self.db_model: dict = {} # store generated database model here.
logger.debug("Connecting to database, database type: '{db_type}'",
db_type=self.database_type)
if self.database_type == "sqlite":
try:
from yombo.mixins.database_mixin.connections.sqlite import SQLiteDB
except Exception as e:
logger.error(
"---------------==(Traceback)==--------------------------")
logger.error("{trace}",
trace=traceback.print_exc(file=sys.stdout))
logger.error(
"--------------------------------------------------------")
logger.warn("Error loading SQLite connection handler: {e}",
e=e)
self.database = SQLiteDB(self)
elif self.database_type == "mysql" or self.database_type == "mariadb":
try:
from yombo.mixins.database_mixin.connections.mysql import MySQL
except Exception as e:
logger.error(
"---------------==(Traceback)==--------------------------")
logger.error("{trace}",
trace=traceback.print_exc(file=sys.stdout))
logger.error(
"--------------------------------------------------------")
logger.warn("Error loading SQLite connection handler: {e}",
e=e)
self.database = MySQL(self)
else:
logger.info("Connecting to database, database type: {db_type}",
db_type=self.database_type)
logger.info("Connecting to database, database type: {db_type}",
db_type=type(self.database_type))
logger.info("Connecting to database, database type: {db_type}",
db_type=type(str(self.database_type)))
raise YomboCritical(f"Unknown database type: {self.database_type}")
yield maybeDeferred(self.database.init)
self.db_pool = self.database.db_pool # to be removed in future
self.db_model = yield maybeDeferred(self.database.db_generate_model)
# print(f"self.db_model: {self.db_model}")
self.db_save_bulk_queue_loop = LoopingCall(self.db_save_bulk_queue,
slow=True)
self.db_save_bulk_queue_loop.start(5, False)
self._CronTab.new(
self.database.db_cleanup,
mins=0,
hours=3, # Clean database at 3am every day.
label="Periodically clean the database.",
load_source="system")