def get_session_from_request(self, request: Type["twisted.web.http.Request"]) -> AuthKey:
"""
Called by the web interface auth system to check if the provided request
has an auth key. Can be in the query string as "?_auth_key=key" or "?_api_auth=key"
or in the header as "x-auth-key: key" or "x-api-auth: key"
Returns the auth object if found otherwise raises YomboWarning.
:param request: The web request instance.
:return: bool
"""
auth_key_id_full = None
if request is not None:
auth_key_id_full = bytes_to_unicode(request.getHeader(b"x-auth-key"))
if auth_key_id_full is None:
auth_key_id_full = bytes_to_unicode(request.getHeader(b"x-api-auth"))
if auth_key_id_full is None:
try:
auth_key_id_full = request.args.get("_auth_key")[0]
except:
try:
auth_key_id_full = request.args.get("_api_auth")[0]
except:
pass
if auth_key_id_full is None:
raise YomboWarning("x-auth-key or x-api-auth header missing, nor _api_auth or _api_query query string is not found.")
request.session_long_id = auth_key_id_full
return self.get_session_by_id(auth_key_id_full)
def update_request(webinterface, request):
"""
Modifies the request to add "received_cookies in unicode. Also, adds a "args"
attribute that contains the incoming arguments, but in unicode. Also adds "_" to the
templates, but it for the current user's language.
Finally, it adds cache-control and expires items to ensure the content isnt" cached.
:param request:
:return:
"""
request.auth = None
request.received_cookies = bytes_to_unicode(request.received_cookies)
request.args = bytes_to_unicode(request.args)
request.setHeader("server", "Apache/2.4.38 (Unix)")
request.setHeader("X-Powered-By", "YGW")
request.setHeader("Cache-Control", "no-cache, no-store, must-revalidate") # don't cache!
origin_final = "*"
if request.requestHeaders.hasHeader("origin"):
origin = request.requestHeaders.getRawHeaders("origin")[0]
if origin is not None:
origin = urlparse(origin)
if origin.scheme in ("http", "https") and len(origin.hostname) < 150 \
and origin.port > 60 and origin.port < 65535:
origin_final = f"{origin.scheme}://{origin.hostname}:{origin.port}" # For the API
request.setHeader("Access-Control-Allow-Origin", origin_final) # For the API, TODO: Make this more restrictive.
request.setHeader("Access-Control-Allow-Credentials", "true") # For the API, TODO: Make this more restrictive.
request.setHeader("Expires", "-1") # don't cache!
request.setHeader("X-Frame-Options", "SAMEORIGIN") # Prevent nesting frames
request.setHeader("X-Content-Type-Options", "nosniff"); # We"ll do our best to be accurate!
request.webinterface = webinterface
def process_response(self, response):
"""
Receives a treq response, collects the content, and attempts to smartly decode
the content based on the response headers.
If headers don't match, a YomboWarning will be raised.
:param response:
:param headers:
:return:
"""
raw_content = yield treq.content(response)
content = raw_content
headers = self.clean_headers(response, True)
if HEADER_CONTENT_TYPE in headers:
content_type = headers[HEADER_CONTENT_TYPE][0]
else:
content_type = None
if content_type == HEADER_CONTENT_TYPE:
try:
content = yield treq.json_content(response)
content_type = "dict"
except Exception as e:
raise YomboWarning(f"Receive response reported json, but found an error: {e}")
elif content_type == CONTENT_TYPE_MSGPACK:
try:
content = msgpack.loads(raw_content)
except Exception:
if len(content) == 0:
return "dict", {}
raise YomboWarning(f"Receive response reported msgpack, but isn't: {content}")
else:
content_type = "string"
try:
content = json.loads(raw_content)
content_type = "dict"
except Exception:
try:
content = msgpack.loads(raw_content)
content_type = "dict"
except Exception:
content = raw_content
content = bytes_to_unicode(content)
# return {
# "content_type": content_type,
# "headers": response.all_headers,
# }
self.content = bytes_to_unicode(content)
self.content_raw = raw_content
self.request = response.request.original
self.response = response
self.content_type = content_type
self.response_phrase = bytes_to_unicode(response.phrase)
self.response_code = response.code
self.headers = headers
def update_request(webinterface, request):
"""
Does some basic conversions for us.
:param request:
:return:
"""
request.received_cookies = bytes_to_unicode(request.received_cookies)
request.args = bytes_to_unicode(request.args)
webinterface.webapp.templates.globals['_'] = webinterface.i18n(request)
request.setHeader('Cache-Control', 'no-cache, no-store, must-revalidate')
request.setHeader('Expires', '0')
def read(self,
filename: str,
convert_to_unicode: Optional[bool] = None,
unpickle: Optional[str] = None):
"""
Reads a file in a non-blocking method and returns a deferred.
:param filename:
:param convert_to_unicode: If True, converts the read file from bytes to unicode (string).
:param unpickle: If string is set, run through self._Tools.data_unpickle(unpickle)
:return:
"""
def _read(read_filename):
f = open(read_filename, "r")
data = f.read()
f.close()
return data
contents = yield threads.deferToThread(_read, filename)
if convert_to_unicode is True:
contents = bytes_to_unicode(contents)
if unpickle is not None:
contents = self._Tools.data_unpickle(contents, "json")
return contents
def ajax_devices_command_get_post(webinterface, request, session,
device_id, command_id):
json_output = bytes_to_unicode(
request.args.get('json_output', ["{}"])[0])
# print("json_output %s" % json_output)
# print("json_output type: %s" % type(json_output))
json_output = json.loads(json_output)
inputs = json_output.get('inputs', {})
if device_id in webinterface._Devices:
device = webinterface._Devices[device_id]
else:
return return_not_found(request, 'device not found')
try:
device.command(
cmd=command_id,
requested_by={
'user_id': session['auth_id'],
'component':
'yombo.gateway.lib.WebInterface.api_v1.devices_get',
'gateway': webinterface.gateway_id()
},
inputs=inputs,
)
a = return_good(request, 'Command executed.')
request.setHeader('Content-Type', 'application/json')
return json.dumps(a)
except KeyError as e:
return return_not_found(request, 'Error with command: %s' % e)
def _init_(self, **kwargs):
"""
Get the GnuPG subsystem up and loaded.
"""
self.key_generation_status = None
self._generating_key = False
self._gpg_keys = {}
self._generating_key_deferred = None
self.sks_pools = [ # Send to a few to ensure we get our key seeded
"ipv4.pool.sks-keyservers.net",
"na.pool.sks-keyservers.net",
"eu.pool.sks-keyservers.net",
"oc.pool.sks-keyservers.net",
"pool.sks-keyservers.net",
"ha.pool.sks-keyservers.net"
]
self.working_dir = settings.arguments["working_dir"]
self.gpg = gnupg.GPG(gnupghome=f"{self.working_dir}/etc/gpg")
self.__mypassphrase = None # will be loaded by sync_keyring_to_db() calls
secret_file = f"{self.working_dir}/etc/gpg/last.pass"
if os.path.exists(secret_file):
phrase = yield read_file(secret_file)
self.__mypassphrase = bytes_to_unicode(phrase)
def _format_list_keys(self, keys):
"""
Formats the results of gnupg.list_keys() into a more usable form.
:param keys:
:return:
"""
variables = {}
# if isinstance(keys, dict):
# keys = [keys]
for record in keys:
# print "list keys: %s" % record
uid = record['uids'][0]
key_comment = uid[uid.find("(") + 1:uid.find(")")]
key = {
'endpoint': key_comment,
'keyid': record['keyid'],
'fingerprint': record['fingerprint'],
'expires': record['expires'],
'sigs': record['sigs'],
'subkeys': record['subkeys'],
'length': record['length'],
'ownertrust': record['ownertrust'],
'algo': record['algo'],
'created_at': record['date'],
'trust': record['trust'],
'type': record['type'],
'uids': record['uids'],
}
key = bytes_to_unicode(key)
variables[record['fingerprint']] = key
return variables
def duplicate_scene(self, scene_id):
"""
Deletes the scene. Will disappear on next restart. This allows the user to recover it.
This marks the node to be deleted!
:param scene_id:
:return:
"""
scene = self.get(scene_id)
label = f"{scene.label} ({_('common::copy')})"
machine_label = f"{scene.machine_label}_{_('common::copy')}"
if label is not None and machine_label is not None:
self.check_duplicate_scene(label, machine_label, scene_id)
new_data = bytes_to_unicode(msgpack.unpackb(msgpack.packb(scene.data))) # had issues with deepcopy
new_scene = yield self._Nodes.new(label=label,
machine_label=machine_label,
node_type="scene",
data=new_data,
data_content_type="json",
gateway_id=self._gateway_id,
destination="gw",
status=1)
self.scenes[new_scene.node_id] = new_scene
yield global_invoke_all("_scene_added_",
called_by=self,
arguments={
"scene_id": scene_id,
"scene": scene,
}
)
return new_scene
def amqp_incoming_response_to_csr_request(self,
body=None,
properties=None,
correlation_info=None,
**kwargs):
"""
Called when we get a signed cert back from a CSR.
:param body:
:param properties:
:param correlation_info:
:param kwargs:
:return:
"""
logger.debug("Received CSR response message: {body}", body=body)
if "sslname" not in body:
logger.warn(
"Discarding response, doesn't have an sslname attached."
) # can't raise exception due to AMPQ processing.
return
logger.info("Received a new signed SSL/TLS certificate for: {sslname}",
sslname=body["sslname"])
sslname = bytes_to_unicode(body["sslname"])
if sslname not in self.managed_certs:
logger.warn(
"It doesn't appear we have a managed cert for the given SSL name. Lets store it for a few minutes: {sslname}",
sslname=sslname)
if sslname in self.received_message_for_unknown:
self.received_message_for_unknown[sslname].append(body)
else:
self.received_message_for_unknown[sslname] = [body]
else:
self.managed_certs[sslname].amqp_incoming_response_to_csr_request(
properties, body, correlation_info)
def encrypt(self, in_text, destination=None, unicode=None):
"""
Encrypt text and output as ascii armor text.
:param in_text: Plain text to encrypt.
:type in_text: string
:param destination: Key id of the destination.
:type destination: string
:return: Ascii armored text.
:rtype: string
:raises: YomboException - If encryption failed.
"""
if in_text.startswith("-----BEGIN PGP MESSAGE-----"):
return in_text
if hasattr(self, "myfingerprint") is False:
return in_text
if destination is None:
destination = self.myfingerprint()
# print("gpg encrypt destination: %s" % destination)
try:
# output = self.gpg.encrypt(in_text, destination, sign=self.myfingerprint())
output = yield threads.deferToThread(self._gpg_encrypt, in_text, destination)
# output = self.gpg.encrypt(in_text, destination)
# print("gpg output: %s" % output)
# print("gpg %s: %s" % (in_text, output.status))
if output.status != "encryption ok":
raise YomboWarning("Unable to encrypt string. Error 1.")
if unicode is False:
return output.data
return bytes_to_unicode(output.data)
except Exception as e:
raise YomboWarning(f"Unable to encrypt string. Error 2.: {e}")
def generate_new_csr_done(self, results, args):
"""
Our CSR has been generated. Lets save it, and maybe submit it.
:param results: The CSR and KEY.
:param args: Any args from the queue.
:param submit: True if we should submit it to yombo for signing.
:return:
"""
# logger.warn("generate_new_csr_done: {sslname}", sslname=self.sslname)
results = bytes_to_unicode(results)
# logger.info("generate_new_csr_done:results: results {results}", results=results)
# logger.info("generate_new_csr_done:results: args {results}", results=args)
self.next_key = results['key']
self.next_csr = results['csr']
# print("generate_new_csr_done csr: %s " % self.next_csr)
yield save_file('usr/etc/certs/%s.next.csr.pem' % self.sslname,
self.next_csr)
yield save_file('usr/etc/certs/%s.next.key.pem' % self.sslname,
self.next_key)
self.next_created = int(time())
self.dirty = True
self.next_csr_generation_in_progress = False
logger.debug("generate_new_csr_done:args: {args}", args=args)
if args['submit'] is True:
# print("calling submit_csr from generate_new_csr_done")
self.submit_csr()
def load_pass_phrase(self, fingerprint):
secret_file = f"{self._working_dir}/etc/gpg/{fingerprint}.pass"
if os.path.exists(secret_file):
phrase = yield self._Files.read(secret_file)
passphrase = bytes_to_unicode(phrase)
return passphrase
return IOError(
f"Unable to read gpg file for fingerprint: {fingerprint}")
def decrypt(self, in_text, unicode=None):
"""
Decrypt a PGP / GPG ascii armor text. If passed in string/text is not detected as encrypted,
will simply return the input.
:param in_text: Ascii armored encoded text.
:type in_text: string
:return: Decoded string.
:rtype: string
:raises: YomboException - If decoding failed.
"""
if in_text is None:
raise YomboWarning("Cannot decrypt NoneType.")
if in_text.startswith("-----BEGIN PGP SIGNED MESSAGE-----"):
verify = yield self.verify_asymmetric(in_text)
return verify
elif in_text.startswith("-----BEGIN PGP MESSAGE-----"):
try:
output = yield threads.deferToThread(self._gpg_decrypt,
in_text,
self.gpg_key.passphrase)
if output.status == "decryption ok":
if unicode is False:
return output.data
return bytes_to_unicode(output.data)
# print("Trying more GPG keys.")
for fingerprint, data in self.gpg_keys.items():
if not data.have_private:
continue
if fingerprint == self.myfingerprint.value:
continue
output = yield threads.deferToThread(
self._gpg_decrypt, in_text, data.passphrase)
if output.status == "decryption ok":
# print("GPG decryption ok with key: %s" % data["email"])
if unicode is False:
return output.data
return bytes_to_unicode(output.data)
raise YomboWarning("No more GPG keys to try.")
except Exception as e:
raise YomboWarning(f"Unable to decrypt string. Reason: {e}")
return in_text
def edit_node(self, node_id, node_data, source=None, authorization=None, **kwargs):
"""
This shouldn't be used by outside calls, instead, tp update the node, simply edit
the node attributes directly. That will cause the node to update the database and Yombo API.
This is used by other internal libraries to update a node's data in bulk and
optionally
:param node_id: Node ID to bulk update.
:param node_data: Dictionary of items to update
:param source: Should be: local or remote. Default is local.
:param kwargs:
:return:
"""
gateway_id = self.gateway_id
if source is None:
source = "local"
# print("editing node: %s" % node_id)
if isinstance(node_data, dict) is False:
raise YomboWarning("edit_node() only accepts dictionaries for 'node_data' argument.")
if "data" not in node_data:
raise YomboWarning("Cannot edit node, 'data' not found")
global_invoke_all("_node_before_update_",
called_by=self,
node_id=node_id,
node=node_data,
in_memory=node_id in self.nodes,
)
if source == "local":
api_data = deepcopy(node_data)
node_data["data"] = data_pickle(api_data["data"], api_data["data_content_type"])
# print("new node data: %s" % api_data)
api_to_send = {k: v for k, v in bytes_to_unicode(api_data).items() if v}
response = yield self.patch_node(node_id=node_id, api_data=api_to_send, authorization=authorization)
node_data = response.content["data"]['attributes']
# Now we have the final data, lets update the local info.
node_id = node_data["id"]
if node_id in self.nodes:
self.nodes[node_id].update_attributes(node_data) # Update existing node data.
self.nodes[node_id].save_to_db()
global_invoke_all("_node_updated_",
called_by=self,
node_id=node_id,
node=self.nodes[node_id],
)
return self.nodes[node_id]
def patch_put_hotfix(request):
"""
Twisted doesn't get arguments for PATCH/PUT. Ticket submitted:
https://twistedmatrix.com/trac/ticket/9759#ticket
This method attempts to get the arguments for PATCH and PUT.
:param request:
:return:
"""
ctype = request.requestHeaders.getRawHeaders(b'content-type')[0]
content = request.content.read()
clength = len(content)
request.args = {}
# print(f"request.method: {request.method}")
# print(f"ctype: {ctype}")
# print(f"clength: {clength}")
# print(f"content: {content}")
if request.method in (b"POST", b"PATCH", b"PUT") and ctype and clength:
mfd = b'multipart/form-data'
key, pdict = _parseHeader(ctype)
# This weird CONTENT-LENGTH param is required by
# cgi.parse_multipart() in some versions of Python 3.7+, see
# bpo-29979. It looks like this will be relaxed and backported, see
# https://github.com/python/cpython/pull/8530.
pdict["CONTENT-LENGTH"] = clength
if key == b'application/x-www-form-urlencoded':
# print("parsing url encoded")
# print(parse_qs(content, 1))
request.args.update(parse_qs(content, 1))
elif key == mfd: # This isnt' working. :(
try:
# print(f"FD 1: pdict: {pdict}")
boundary = pdict["boundary"].decode("charmap").replace("-", "")
pdict["boundary"] = boundary.encode("charmap")
# print(f"FD 2: pdict: {pdict}")
cgiArgs = cgi.parse_multipart(
request.content, pdict, encoding='utf8',
errors="surrogateescape")
# print(f"FD: cgiArgs: {cgiArgs}")
# The parse_multipart function on Python 3.7+
# decodes the header bytes as iso-8859-1 and
# decodes the body bytes as utf8 with
# surrogateescape -- we want bytes
request.args.update({
x.encode('iso-8859-1'): \
[z.encode('utf8', "surrogateescape")
if isinstance(z, str) else z for z in y]
for x, y in cgiArgs.items()})
except Exception as e:
print(f"error parsing form data: {e}")
pass
request.args = bytes_to_unicode(request.args)
def render_find_accepts(accepts):
accepts = bytes_to_unicode(accepts)
for accept in [
CONTENT_TYPE_MSGPACK, CONTENT_TYPE_JSON,
CONTENT_TYPE_TEXT_HTML, CONTENT_TYPE_TEXT_PLAIN
]:
if accept in accepts:
return accept
if "*/*" in accepts:
return "text/plain"
return None
def load_passphrase(self, keyid=None):
if keyid is None:
keyid = self.mykeyid()
if keyid is not None:
secret_file = "%s/usr/etc/gpg/%s.pass" % (
self._Atoms.get('yombo.path'), keyid)
if os.path.exists(secret_file):
phrase = yield read_file(secret_file)
if keyid == self.mykeyid():
self.__mypassphrase = phrase
return bytes_to_unicode(phrase)
return None
def load_passphrase(self, fingerprint=None):
if fingerprint is None:
fingerprint = self.myfingerprint()
if fingerprint is not None:
secret_file = f"{self.working_dir}/etc/gpg/{fingerprint}.pass"
if os.path.exists(secret_file):
phrase = yield read_file(secret_file)
phrase = bytes_to_unicode(phrase)
if fingerprint == self.myfingerprint():
self.__mypassphrase = phrase
return phrase
return None
def clean_headers(self, response, update_response=None):
"""
Take a treq response and get friendly headers.
:param response:
:return:
"""
all_headers = CaseInsensitiveDict()
raw_headers = bytes_to_unicode(response.headers._rawHeaders)
for key, value in raw_headers.items():
all_headers[key.lower()] = value[0]
if update_response is not False:
response.all_headers = all_headers
return all_headers
def check_web_request(self, request=None):
"""
Called by the web interface auth system to check if the provided request
has an API key. Can be in the query string as '?_api_auth=key' or in
the header as "x-api-auth: key"
Returns the auth object if found otherwise raises YomboWarning.
:param request: The web request instance.
:return: bool
"""
auth_id = None
if request is not None:
# print("request: %s" % request)
auth_id = bytes_to_unicode(request.getHeader(b'x-api-auth'))
if auth_id is None:
try:
auth_id = request.args.get('_api_auth')[0]
except:
auth_id = None
if auth_id is None:
raise YomboWarning("x-api-auth header nor query string is found.")
if self.validate_auth_id(auth_id) is False:
raise YomboWarning("api auth key has invalid characters.")
if auth_id in self.active_api_auth:
if self.active_api_auth[auth_id].is_valid is True:
return self.active_api_auth[auth_id]
else:
raise YomboWarning("api auth key is no longer valid.")
else:
logger.debug("has_session is looking in database for session...")
try:
db_api_auth = yield self._LocalDB.get_api_auth(auth_id)
except Exception as e:
raise YomboWarning("api auth isn't found")
self.active_api_auth[auth_id] = Auth(self,
db_api_auth,
source='database')
self.active_api_auth_by_label[
self.active_api_auth[auth_id].label] = auth_id
return self.active_api_auth[auth_id]
raise YomboWarning("x-api-auth header is invalid, other reasons.")
def update_request(request, api):
"""
Modifies the request to add "received_cookies in unicode. Also, adds a "args"
attribute that contains the incoming arguments, but in unicode. Also adds "_" to the
templates, but it for the current user's language.
Finally, it adds cache-control and expires items to ensure the content isn't cached.
:param request:
:return:
"""
request.auth = None
if api in ("true", True, 1, "yes"):
request.api = True
else:
request.api = False
request.received_cookies = bytes_to_unicode(request.received_cookies)
request.args = bytes_to_unicode(request.args)
request.request_id = random_string(length=25)
request.setHeader("Cache-Control", "no-cache, no-store, must-revalidate") # don't cache!
request.setHeader("Expires", "-1") # don't cache!
# Make uniform arguments available as 'processed_arguments'. First, if the request is type
# POST, PATCH, or PUT, then first try to decode the body request. If not, then use the query string args.
request.processed_body = None
request.processed_body_encoding = None
if bytes_to_unicode(request.method).lower() in ("post", "patch", "put"):
content_type = bytes_to_unicode(request.getHeader("content-type"))
if isinstance(content_type, str):
content_type = content_type.lower()
else:
content_type = ""
if content_type == CONTENT_TYPE_JSON:
try:
request.processed_body = bytes_to_unicode(json.loads(request.content.read()))
request.processed_body_encoding = "json"
except Exception as e:
logger.info("Error decoding web request 'json' data: {e}", e=e)
elif content_type == CONTENT_TYPE_MSGPACK:
try:
request.processed_body = bytes_to_unicode(msgpack.unpackb(request.content.read()))
request.processed_body_encoding = "msgpack"
except Exception as e:
logger.info("Error decoding web request 'msgpack' data: {e}", e=e)
common_headers(request)
def get(self, request=None, api_auth_id=None):
"""
Checks the request for an x-api-auth header and then tries to validate it.
Returns True if everything is good, otherwise raises YomboWarning with
status reason.
:param request: The request instance.
:return: bool
"""
if api_auth_id is None and request is not None:
# print("request: %s" % request)
api_auth_id = bytes_to_unicode(request.getHeader(b'x-api-auth'))
if api_auth_id is None:
try:
api_auth_id = request.args.get('_api_auth')[0]
except:
api_auth_id = None
if api_auth_id is None:
raise YomboWarning("x-api-auth header is missing or blank.")
if self.validate_api_auth_id(api_auth_id) is False:
raise YomboWarning("x-api-auth header has invalid characters.")
if api_auth_id in self.active_api_auth:
if self.active_api_auth[api_auth_id].is_valid is True:
return self.active_api_auth[api_auth_id]
else:
raise YomboWarning("x-api-auth header is no longer valid.")
else:
logger.debug("has_session is looking in database for session...")
try:
db_api_auth = yield self._LocalDB.get_api_auth(api_auth_id)
except Exception as e:
raise YomboWarning("x-api-auth is not valid")
# logger.debug("has_session - found in DB! {db_session}", db_session=db_session)
self.active_api_auth[api_auth_id] = ApiAuth(self,
db_api_auth,
source='database')
return self.active_api_auth[api_auth_id]
raise YomboWarning("x-api-auth header is invalid, other reasons.")
def read_configs(self):
try:
content = yield read_file('%s/config.json' % self.module_path)
except Exception as e:
logger.info(
"Couldn't read existing config.json file for homebridge.")
return False
config_file = bytes_to_unicode(json.loads(content))
if 'bridge' not in config_file:
logger.warn(
"'bridge' section missing from homebridge config.json file.")
return False
if 'username' in config_file['bridge']:
self.username = config_file['bridge']['username']
else:
logger.warn(
"'username' missing within 'bridge' section from homebridge config.json file."
)
return False
if 'pin' in config_file['bridge']:
self.pin_code = config_file['bridge']['pin']
else:
logger.warn(
"'pin' missing within 'bridge' section from homebridge config.json file."
)
return False
for idx, platform in enumerate(config_file['platforms']):
if platform['platform'] == 'Yombo':
config_file['platforms'][idx]['apiauth'] = self.apiauth.auth_id
break
self.config_file = config_file
def send_csr_request_response(self,
msg=None,
properties=None,
correlation_info=None,
**kwargs):
"""
Called when we get a signed cert back from a CSR.
:param msg:
:param properties:
:param correlation_info:
:param kwargs:
:return:
"""
logger.debug("Received CSR response mesage: {msg}", msg=msg)
if 'sslname' not in msg:
logger.warn(
"Discarding response, doesn't have an sslname attached."
) # can't raise exception due to AMPQ processing.
return
sslname = bytes_to_unicode(msg['sslname'])
# print("sslname: %s" % sslname)
# print("sslname: %s" % type(sslname))
# print("managed_certs: %s" % self.managed_certs)
# print("managed_certs: %s" % type(self.managed_certs))
if sslname not in self.managed_certs:
logger.warn(
"It doesn't appear we have a managed cert for the given SSL name. Lets store it for a few minutes: %s"
% sslname)
if sslname in self.received_message_for_unknown:
self.received_message_for_unknown[sslname].append(msg)
else:
self.received_message_for_unknown[sslname] = [msg]
else:
self.managed_certs[sslname].yombo_csr_response(
properties, msg, correlation_info)
def sha512_crypt_mosquitto(password, salt_base64=None, rounds=None):
"""
Used for generating a crypted version for mosquitto pasword file.
:param password:
:param salt_base64:
:param rounds:
:return:
"""
if salt_base64 is None:
rand = random.SystemRandom()
salt_base64 = ''.join([
rand.choice(string.ascii_letters + string.digits)
for _ in range(16)
])
salt = base64.b64decode(salt_base64)
m = hashlib.sha512()
m.update(unicode_to_bytes(password))
m.update(salt)
hashed_password = m.digest()
encoded_password = bytes_to_unicode(base64.b64encode(hashed_password))
return '$6$' + salt_base64 + "$" + encoded_password
def set(self,
config: str,
value: Any,
value_type: Optional[str] = None,
ignore_case: Optional[bool] = None,
source: Optional = None,
**kwargs) -> ConfigItem:
"""
Set value of configuration option for a given config. The option length
**cannot exceed 1000 characters**. The value cannot exceed 5000 bytes.
Section and option will be converted to lowercase, rending the set/get function case insenstive.
**Usage**:
.. code-block:: python
self._Config.set("core.something", "New Value")
:raises YomboInvalidArgument: When an argument is invalid or illegal.
:raises KeyError: When the requested config is not found.
:param config: The configuration to use.
:param value: What to return if no result is found, default = None.
:param value_type:
:param ignore_case:
:param source:
"""
if config.endswith("*"):
raise YomboInvalidArgument(
"Config item label cannot end with '*'.")
if len(config) > MAX_CONFIG_LENGTH:
raise YomboInvalidArgument(
f"section cannot be more than {MAX_CONFIG_LENGTH:d} chars")
# Can't set value!
if config.startswith(
"security") and self._Loader.operating_mode == "run":
raise YomboInvalidArgument(
"Not allowed to set value: cannot edit security items.")
value = bytes_to_unicode(value)
if isinstance(value, str):
if len(value) > MAX_VALUE_LENGTH:
raise YomboInvalidArgument(
f"value cannot be more than {MAX_VALUE_LENGTH} chars")
if ignore_case is not True:
config = config.lower()
# If the gateway ID changes.
if config == "core.gwid" and config in self.configs:
self._Atoms.change_gateway_id(self.configs[config].value, value)
self._States.change_gateway_id(self.configs[config].value, value)
if config in self.configs:
config_item = self.configs[config]
if self.configs[config].value == value:
return value
config_item.set(value, value_type=value_type)
else:
self.configs[config] = ConfigItem(self,
config=config,
value=value,
value_type=value_type,
source=source)
self.configs_dirty = True
return self.configs[config]
def page_restore_details(webinterface, request, session):
"""
Prompts user to upload the configuration file to restore the gateway.
:param webinterface:
:param request:
:param session:
:return:
"""
try:
restorefile = request.args.get("restorefile")[0]
try:
restorefile = json.loads(restorefile)
logger.info("Received configuration backup file.")
try:
if restorefile["hash"] != hashlib.sha256(
unicode_to_bytes(
restorefile["data"])).hexdigest():
webinterface.add_alert(
"Backup file appears to be corrupt: Invalid checksum."
)
return webinterface.redirect("/setup_wizard/start")
restorefile["data"] = base64.b64decode(
unicode_to_bytes(restorefile["data"]))
except Exception as e:
logger.warn("Unable to b64decode data: {e}", e=e)
webinterface.add_alert(
"Unable to properly decode pass 1 of data segment of restore file."
)
return webinterface.redirect("/setup_wizard/start")
session.set("restore_backup_file", restorefile)
except Exception as e:
logger.warn("Unable to parse JSON phase 2: {e}", e=e)
webinterface.add_alert("Invalid restore file contents.")
return webinterface.redirect("/setup_wizard/start")
except Exception:
restorefile = session.get("restore_backup_file", None)
if restorefile is None:
webinterface.add_alert("No restore file found.")
return webinterface.redirect("/setup_wizard/start")
required_keys = ("encrypted", "time", "file_type", "created",
"backup_version")
if all(required in restorefile
for required in required_keys) is False:
webinterface.add_alert(
"Backup file appears to be missing important parts.")
return webinterface.redirect("/setup_wizard/start")
if restorefile["encrypted"] is True:
try:
password = request.args.get("password", )[0]
except Exception:
password = session.get("restorepassword", None)
if password is None:
page = webinterface.get_template(
request, webinterface.wi_dir +
"/pages/setup_wizard/restore_password.html")
return page.render(alerts=session.get_alerts(),
restore=restorefile)
try:
decrypted = yield webinterface._Tools.data_unpickle(
restorefile["data"],
"msgpack_aes256_zip_base85",
passphrase=password,
)
restorefile["data_processed"] = json.loads(
bytes_to_unicode(decrypted))
except Exception as e:
logger.warn("Unable to decrypt restoration file: {e}", e=e)
webinterface.add_alert(
"It appears the password is incorrect.", "danger")
page = webinterface.get_template(
request, webinterface.wi_dir +
"/pages/setup_wizard/restore_password.html")
return page.render(alerts=session.get_alerts(),
restore=restorefile)
else: #no password
restorefile["data_processed"] = json.loads(
bytes_to_unicode(restorefile["data"]))
session.set("restore_backup_file", restorefile)
page = webinterface.get_template(
request,
webinterface.wi_dir + "/pages/restore/restore_ready.html")
return page.render(alerts=session.get_alerts(),
restore=session.get("restore_backup_file"))
def add_node(self, node_data, source=None, authorization=None, **kwargs):
"""
Used to create new nodes. Node data should be a dictionary. This will:
1) Send the node information to Yombo cloud for persistence.
2) Save the node information to local database.
3) Load the node into memory for usage.
This adds the node at Yombo, adds to the local DB store if the gateway_id matches outs,
and loads it into memory if the gateways is ours and destination is 'gw' or 'always_load' is 1.
Required:
node_type
weight (defaults to 0 if not set)
always_load (defaults to 1 - true if not set)
data
data_content_type - Usually msgpack_base85 or json.
status (defaults to 1 - enabled)
Optional:
gateway_id - Will not save to localdb or load into memory if not set to this gateway.
machine_label
label
destination
:param node_data:
:param kwargs:
:return:
"""
print("nodes:: new_new 1")
if source is None:
source = "local"
gateway_id = self.gateway_id
if "data" not in node_data or node_data["data"] is None:
raise YomboWarning("Node must have data!")
if "data_content_type" not in node_data or node_data["data_content_type"] is None:
if isinstance(node_data["data"], dict) or isinstance(node_data["data"], list):
node_data["data_content_type"] = "json"
elif isinstance(node_data["data"], bool):
node_data["data_content_type"] = "bool"
else:
node_data["data_content_type"] = "string"
if "parent_id" not in node_data:
node_data["parent_id"] = None
if "gateway_id" not in node_data:
node_data["gateway_id"] = gateway_id
if "destination" in node_data and node_data["destination"] == "gw" and \
("gateway_id" not in node_data or node_data["gateway_id"] is None):
node_data["gateway_id"] = gateway_id
if "always_load" not in node_data or node_data["always_load"] is None:
node_data["always_load"] = 1
if "weight" not in node_data or node_data["weight"] is None:
node_data["weight"] = 0
if "status" not in node_data or node_data["status"] is None:
node_data["status"] = 1
if source == "local":
# api_data = deepcopy(node_data)
node_data["data"] = data_pickle(node_data["data"], node_data["data_content_type"])
api_data = {k: v for k, v in bytes_to_unicode(node_data).items() if v}
print("nodes:: new_new 10")
response = yield self._YomboAPI.request("POST", "/v1/node",
api_data,
authorization_header=authorization)
# print("added node results: %s" % node_results)
node_data = response.content["data"]['attributes']
print(f"new node data: {node_data}")
node_id = node_data["id"]
if "gateway_id" in node_data and node_data["gateway_id"] == gateway_id:
self.nodes[node_id].add_to_db()
if "destination" in node_data and node_data["destination"] == "gw" and \
"gateway_id" in node_data and node_data["gateway_id"] == gateway_id:
print("Loading new node data into memory...")
self._load_node_into_memory(node_data)
global_invoke_all("_node_added_",
called_by=self,
node_id=node_id,
node=self.nodes[node_id],
)
return node_id
def amqp_incoming_parse(self, channel, deliver, properties, msg):
"""
:param deliver:
:param properties:
:param msg:
:param queue:
:return:
"""
# print("amqp_incoming_parse............")
if not hasattr(properties, 'user_id') or properties.user_id is None:
self._Statistics.increment(
"lib.amqpyombo.received.discarded.nouserid",
bucket_size=15,
anon=True)
raise YomboWarning("user_id missing.")
if not hasattr(properties,
'content_type') or properties.content_type is None:
self._Statistics.increment(
"lib.amqpyombo.received.discarded.content_type_missing",
bucket_size=15,
anon=True)
raise YomboWarning("content_type missing.")
if not hasattr(
properties,
'content_encoding') or properties.content_encoding is None:
self._Statistics.increment(
"lib.amqpyombo.received.discarded.content_encoding_missing",
bucket_size=15,
anon=True)
raise YomboWarning("content_encoding missing.")
if properties.content_encoding != 'text' and properties.content_encoding != 'zlib':
self._Statistics.increment(
"lib.amqpyombo.received.discarded.content_encoding_invalid",
bucket_size=15,
anon=True)
raise YomboWarning(
"Content Encoding must be either 'text' or 'zlib'. Got: " +
properties.content_encoding)
if properties.content_type != 'text/plain' and properties.content_type != 'application/msgpack' and properties.content_type != 'application/json':
self._Statistics.increment(
"lib.amqpyombo.received.discarded.content_type_invalid",
bucket_size=15,
anon=True)
logger.warn('Error with contentType!')
raise YomboWarning(
"Content type must be 'application/msgpack', 'application/json' or 'text/plain'. Got: "
+ properties.content_type)
received_message_meta = {}
received_message_meta['content_encoding'] = properties.content_encoding
received_message_meta['content_type'] = properties.content_type
if properties.content_encoding == 'zlib':
compressed_size = len(msg)
msg = zlib.decompress(msg)
uncompressed_size = len(msg)
# logger.info(
# "Message sizes: msg_size_compressed = {compressed}, non-compressed = {uncompressed}, percent: {percent}",
# compressed=beforeZlib, uncompressed=afterZlib, percent=abs(percentage(beforeZlib, afterZlib)-1))
received_message_meta['payload_size'] = uncompressed_size
received_message_meta['compressed_size'] = compressed_size
received_message_meta['compression_percent'] = abs(
(compressed_size / uncompressed_size) - 1) * 100
else:
received_message_meta['payload_size'] = len(msg)
received_message_meta['compressed_size'] = len(msg)
received_message_meta['compression_percent'] = None
if properties.content_type == 'application/json':
if self._Validate.is_json(msg):
msg = bytes_to_unicode(json.loads(msg))
else:
raise YomboWarning("Receive msg reported json, but isn't: %s" %
msg)
elif properties.content_type == 'application/msgpack':
if self._Validate.is_msgpack(msg):
msg = bytes_to_unicode(msgpack.loads(msg))
# print("msg: %s" % type(msg))
# print("msg: %s" % msg['headers'])
else:
raise YomboWarning(
"Received msg reported msgpack, but isn't: %s" % msg)
# todo: Validate signatures/encryption here!
return {
'headers': msg['headers'],
'body': msg['body'],
'received_message_meta': received_message_meta,
}