def testSafeMatch(self): assert SafeRe.match( "((js|css)/(?!all.(js|css))|data/users/.*db|data/users/.*/.*|data/archived|.*.py)", "js/ZeroTalk.coffee") assert SafeRe.match( ".+/data.json", "data/users/1J3rJ8ecnwH2EPYa6MrgZttBNc61ACFiCj/data.json")
def verifyContentInclude(self, inner_path, content, content_size, content_size_optional): # Load include details rules = self.getRules(inner_path, content) if not rules: raise VerifyError("No rules") # Check include size limit if rules.get("max_size") is not None: # Include size limit if content_size > rules["max_size"]: raise VerifyError("Include too large %sB > %sB" % (content_size, rules["max_size"])) if rules.get("max_size_optional") is not None: # Include optional files limit if content_size_optional > rules["max_size_optional"]: raise VerifyError("Include optional files too large %sB > %sB" % ( content_size_optional, rules["max_size_optional"]) ) # Filename limit if rules.get("files_allowed"): for file_inner_path in content["files"].keys(): if not SafeRe.match("^%s$" % rules["files_allowed"], file_inner_path): raise VerifyError("File not allowed: %s" % file_inner_path) if rules.get("files_allowed_optional"): for file_inner_path in content.get("files_optional", {}).keys(): if not SafeRe.match("^%s$" % rules["files_allowed_optional"], file_inner_path): raise VerifyError("Optional file not allowed: %s" % file_inner_path) # Check if content includes allowed if rules.get("includes_allowed") is False and content.get("includes"): raise VerifyError("Includes not allowed") return True # All good
def hashFiles(self, dir_inner_path, ignore_pattern=None, optional_pattern=None): files_node = {} files_optional_node = {} if dir_inner_path and not self.isValidRelativePath(dir_inner_path): ignored = True self.log.error( "- [ERROR] Only ascii encoded directories allowed: %s" % dir_inner_path) for file_relative_path in self.site.storage.walk(dir_inner_path): file_name = helper.getFilename(file_relative_path) ignored = optional = False if file_name == "content.json": ignored = True elif ignore_pattern and SafeRe.match(ignore_pattern, file_relative_path): ignored = True elif file_name.startswith(".") or file_name.endswith( "-old") or file_name.endswith("-new"): ignored = True elif not self.isValidRelativePath(file_relative_path): ignored = True self.log.error("- [ERROR] Invalid filename: %s" % file_relative_path) elif dir_inner_path == "" and file_relative_path == self.site.storage.getDbFile( ): ignored = True elif optional_pattern and SafeRe.match(optional_pattern, file_relative_path): optional = True if ignored: # Ignore content.json, defined regexp and files starting with . self.log.info("- [SKIPPED] %s" % file_relative_path) else: if optional: self.log.info("- [OPTIONAL] %s" % file_relative_path) files_optional_node.update( self.hashFile(dir_inner_path, file_relative_path, optional=True)) else: self.log.info("- %s" % file_relative_path) files_node.update( self.hashFile(dir_inner_path, file_relative_path)) return files_node, files_optional_node
def verifyCert(self, inner_path, content): from Crypt import CryptBitcoin rules = self.getRules(inner_path, content) if not rules: raise VerifyError("No rules for this file") if not rules.get("cert_signers") and not rules.get("cert_signers_pattern"): return True # Does not need cert if "cert_user_id" not in content: raise VerifyError("Missing cert_user_id") if content["cert_user_id"].count("@") != 1: raise VerifyError("Invalid domain in cert_user_id") name, domain = content["cert_user_id"].rsplit("@", 1) cert_address = rules["cert_signers"].get(domain) if not cert_address: # Unknown Cert signer if rules.get("cert_signers_pattern") and SafeRe.match(rules["cert_signers_pattern"], domain): cert_address = domain else: raise VerifyError("Invalid cert signer: %s" % domain) try: cert_subject = "%s#%s/%s" % (rules["user_address"], content["cert_auth_type"], name) result = CryptBitcoin.verify(cert_subject, cert_address, content["cert_sign"]) except Exception, err: raise VerifyError("Certificate verify error: %s" % err)
def getUserContentRules(self, parent_content, inner_path, content): user_contents = parent_content["user_contents"] # Delivered for directory if "inner_path" in parent_content: parent_content_dir = helper.getDirname(parent_content["inner_path"]) user_address = re.match("([A-Za-z0-9]*?)/", inner_path[len(parent_content_dir):]).group(1) else: user_address = re.match(".*/([A-Za-z0-9]*?)/.*?$", inner_path).group(1) try: if not content: content = self.site.storage.loadJson(inner_path) # Read the file if no content specified user_urn = "%s/%s" % (content["cert_auth_type"], content["cert_user_id"]) # web/[email protected] cert_user_id = content["cert_user_id"] except Exception: # Content.json not exist user_urn = "n-a/n-a" cert_user_id = "n-a" if user_address in user_contents["permissions"]: rules = copy.copy(user_contents["permissions"].get(user_address, {})) # Default rules based on address else: rules = copy.copy(user_contents["permissions"].get(cert_user_id, {})) # Default rules based on username if rules is False: banned = True rules = {} else: banned = False if "signers" in rules: rules["signers"] = rules["signers"][:] # Make copy of the signers for permission_pattern, permission_rules in user_contents["permission_rules"].items(): # Regexp rules if not SafeRe.match(permission_pattern, user_urn): continue # Rule is not valid for user # Update rules if its better than current recorded ones for key, val in permission_rules.iteritems(): if key not in rules: if type(val) is list: rules[key] = val[:] # Make copy else: rules[key] = val elif type(val) is int: # Int, update if larger if val > rules[key]: rules[key] = val elif hasattr(val, "startswith"): # String, update if longer if len(val) > len(rules[key]): rules[key] = val elif type(val) is list: # List, append rules[key] += val rules["cert_signers"] = user_contents["cert_signers"] # Add valid cert signers if "signers" not in rules: rules["signers"] = [] if not banned: rules["signers"].append(user_address) # Add user as valid signer rules["user_address"] = user_address rules["includes_allowed"] = False return rules
def peerCheckMessage(self, to, message): # Check whether there is p2p.json if not self.site.storage.isFile("p2p.json"): self.response(to, {"error": "Site %s doesn't support P2P messages" % self.site.address}) return False # Check whether P2P messages are supported p2p_json = self.site.storage.loadJson("p2p.json") if "filter" not in p2p_json: self.response(to, {"error": "Site %s doesn't support P2P messages" % self.site.address}) return False # Check whether the message matches passive filter if not SafeRe.match(p2p_json["filter"], json.dumps(message)): self.response(to, {"error": "Invalid message for site %s: %s" % (self.site.address, message)}) return False # Not so fast if "freq_limit" in p2p_json and time.time() - self.site.p2p_last_recv.get("self", 0) < p2p_json["freq_limit"]: self.response(to, {"error": "Too fast messages"}) return False self.site.p2p_last_recv["self"] = time.time() # Not so much if "size_limit" in p2p_json and len(json.dumps(message)) > p2p_json["size_limit"]: self.response(to, {"error": "Too big message"}) return False return True
def walk(self, dir_inner_path, ignore=None): directory = self.getPath(dir_inner_path) for root, dirs, files in os.walk(directory): root = root.replace("\\", "/") root_relative_path = re.sub("^%s" % re.escape(directory), "", root).lstrip("/") for file_name in files: if root_relative_path: # Not root dir file_relative_path = root_relative_path + "/" + file_name else: file_relative_path = file_name if ignore and SafeRe.match(ignore, file_relative_path): continue yield file_relative_path # Don't scan directory that is in the ignore pattern if ignore: dirs_filtered = [] for dir_name in dirs: if root_relative_path: dir_relative_path = root_relative_path + "/" + dir_name else: dir_relative_path = dir_name if ignore == ".*" or re.match(".*([|(]|^)%s([|)]|$)" % re.escape(dir_relative_path + "/.*"), ignore): continue dirs_filtered.append(dir_name) dirs[:] = dirs_filtered
def updateJson(self, file_path, file=None, cur=None): if not file_path.startswith(self.db_dir): return False # Not from the db dir: Skipping relative_path = file_path[len(self.db_dir):] # File path realative to db file # Check if filename matches any of mappings in schema matched_maps = [] for match, map_settings in self.schema["maps"].items(): try: if SafeRe.match(match, relative_path): matched_maps.append(map_settings) except SafeRe.UnsafePatternError as err: self.log.error(err) # No match found for the file if not matched_maps: return False # Load the json file try: if file is None: # Open file is not file object passed file = open(file_path, "rb") if file is False: # File deleted data = {} else: if file_path.endswith("json.gz"): data = json.load(helper.limitedGzipFile(fileobj=file)) else: data = json.load(file) except Exception, err: self.log.debug("Json file %s load error: %s" % (file_path, err)) data = {}
def walk(self, dir_inner_path, ignore=None): directory = self.getPath(dir_inner_path) for root, dirs, files in os.walk(directory): root = root.replace("\\", "/") root_relative_path = re.sub("^%s" % re.escape(directory), "", root).lstrip("/") for file_name in files: if root_relative_path: # Not root dir file_relative_path = root_relative_path + "/" + file_name else: file_relative_path = file_name if ignore and SafeRe.match(ignore, file_relative_path): continue yield file_relative_path # Don't scan directory that is in the ignore pattern if ignore: dirs_filtered = [] for dir_name in dirs: if root_relative_path: dir_relative_path = root_relative_path + "/" + dir_name else: dir_relative_path = dir_name if ignore == ".*" or re.match( ".*([|(]|^)%s([|)]|$)" % re.escape(dir_relative_path + "/.*"), ignore): continue dirs_filtered.append(dir_name) dirs[:] = dirs_filtered
def actionCertSelect(self, to, accepted_domains=[], accept_any=False, accepted_pattern=None): accounts = [] accounts.append(["", _["No certificate"], ""]) # Default option active = "" # Make it active if no other option found # Add my certs auth_address = self.user.getAuthAddress(self.site.address) # Current auth address site_data = self.user.getSiteData(self.site.address) # Current auth address if not accepted_domains and not accepted_pattern: # Accept any if no filter defined accept_any = True for domain, cert in self.user.certs.items(): if auth_address == cert["auth_address"] and domain == site_data.get("cert"): active = domain title = cert["auth_user_name"] + "@" + domain accepted_pattern_match = accepted_pattern and SafeRe.match(accepted_pattern, domain) if domain in accepted_domains or accept_any or accepted_pattern_match: accounts.append([domain, title, ""]) else: accounts.append([domain, title, "disabled"]) # Render the html body = "<span style='padding-bottom: 5px; display: inline-block'>" + _["Select account you want to use in this site:"] + "</span>" # Accounts for domain, account, css_class in accounts: if domain == active: css_class += " active" # Currently selected option title = _(u"<b>%s</b> <small>({_[currently selected]})</small>") % account else: title = "<b>%s</b>" % account body += "<a href='#Select+account' class='select select-close cert %s' title='%s'>%s</a>" % (css_class, domain, title) # More available providers more_domains = [domain for domain in accepted_domains if domain not in self.user.certs] # Domains we not displayed yet if more_domains: # body+= "<small style='margin-top: 10px; display: block'>Accepted authorization providers by the site:</small>" body += "<div style='background-color: #F7F7F7; margin-right: -30px'>" for domain in more_domains: body += _(u""" <a href='/{domain}' onclick='zeroframe.certSelectGotoSite(this)' class='select'> <small style='float: right; margin-right: 40px; margin-top: -1px'>{_[Register]} »</small>{domain} </a> """) body += "</div>" body += """ <script> $(".notification .select.cert").on("click", function() { $(".notification .select").removeClass('active') zeroframe.response(%s, this.title) return false }) </script> """ % self.next_message_id # Send the notification self.cmd("notification", ["ask", body], lambda domain: self.actionCertSet(to, domain))
def actionCertSelect(self, to, accepted_domains=[], accept_any=False, accepted_pattern=None): accounts = [] accounts.append(["", _["No certificate"], ""]) # Default option active = "" # Make it active if no other option found # Add my certs auth_address = self.user.getAuthAddress(self.site.address) # Current auth address site_data = self.user.getSiteData(self.site.address) # Current auth address if not accepted_domains and not accepted_pattern: # Accept any if no filter defined accept_any = True for domain, cert in self.user.certs.items(): if auth_address == cert["auth_address"] and domain == site_data.get("cert"): active = domain title = cert["auth_user_name"] + "@" + domain accepted_pattern_match = accepted_pattern and SafeRe.match(accepted_pattern, domain) if domain in accepted_domains or accept_any or accepted_pattern_match: accounts.append([domain, title, ""]) else: accounts.append([domain, title, "disabled"]) # Render the html body = "<span style='padding-bottom: 5px; display: inline-block'>" + _["Select account you want to use in this site:"] + "</span>" # Accounts for domain, account, css_class in accounts: if domain == active: css_class += " active" # Currently selected option title = _(u"<b>%s</b> <small>({_[currently selected]})</small>") % account else: title = "<b>%s</b>" % account body += "<a href='#Select+account' class='select select-close cert %s' title='%s'>%s</a>" % (css_class, domain, title) # More available providers more_domains = [domain for domain in accepted_domains if domain not in self.user.certs] # Domains we not displayed yet if more_domains: # body+= "<small style='margin-top: 10px; display: block'>Accepted authorization providers by the site:</small>" body += "<div style='background-color: #F7F7F7; margin-right: -30px'>" for domain in more_domains: body += _(u""" <a href='/{domain}' target='_top' class='select'> <small style='float: right; margin-right: 40px; margin-top: -1px'>{_[Register]} »</small>{domain} </a> """) body += "</div>" script = """ $(".notification .select.cert").on("click", function() { $(".notification .select").removeClass('active') zeroframe.response(%s, this.title) return false }) """ % self.next_message_id self.cmd("notification", ["ask", body], lambda domain: self.actionCertSet(to, domain)) self.cmd("injectScript", script)
def hashFiles(self, dir_inner_path, ignore_pattern=None, optional_pattern=None): files_node = {} files_optional_node = {} if dir_inner_path and not self.isValidRelativePath(dir_inner_path): ignored = True self.log.error("- [ERROR] Only ascii encoded directories allowed: %s" % dir_inner_path) for file_relative_path in self.site.storage.walk(dir_inner_path): file_name = helper.getFilename(file_relative_path) ignored = optional = False if file_name == "content.json": ignored = True elif ignore_pattern and SafeRe.match(ignore_pattern, file_relative_path): ignored = True elif file_name.startswith(".") or file_name.endswith("-old") or file_name.endswith("-new"): ignored = True elif not self.isValidRelativePath(file_relative_path): ignored = True self.log.error("- [ERROR] Invalid filename: %s" % file_relative_path) elif optional_pattern and SafeRe.match(optional_pattern, file_relative_path): optional = True if ignored: # Ignore content.json, defined regexp and files starting with . self.log.info("- [SKIPPED] %s" % file_relative_path) else: if optional: self.log.info("- [OPTIONAL] %s" % file_relative_path) files_optional_node.update( self.hashFile(dir_inner_path, file_relative_path, optional=True) ) else: self.log.info("- %s" % file_relative_path) files_node.update( self.hashFile(dir_inner_path, file_relative_path) ) return files_node, files_optional_node
def updateJson(self, file_path, file=None, cur=None): if not file_path.startswith(self.db_dir): return False # Not from the db dir: Skipping relative_path = file_path[len(self.db_dir):] # File path realative to db file # Check if filename matches any of mappings in schema matched_maps = [] for match, map_settings in self.schema["maps"].items(): try: if SafeRe.match(match, relative_path): matched_maps.append(map_settings) except SafeRe.UnsafePatternError as err: self.log.error(err) # No match found for the file if not matched_maps: return False # Load the json file try: if file is None: # Open file is not file object passed file = open(file_path, "rb") if file is False: # File deleted data = {} else: if file_path.endswith("json.gz"): file = helper.limitedGzipFile(fileobj=file) if sys.version_info.major == 3 and sys.version_info.minor < 6: data = json.loads(file.read().decode("utf8")) else: data = json.load(file) except Exception as err: self.log.debug("Json file %s load error: %s" % (file_path, err)) data = {} # No cursor specificed if not cur: cur = self.getSharedCursor() cur.logging = False # Row for current json file if required if not data or [dbmap for dbmap in matched_maps if "to_keyvalue" in dbmap or "to_table" in dbmap]: json_row = cur.getJsonRow(relative_path) # Check matched mappings in schema for dbmap in matched_maps: # Insert non-relational key values if dbmap.get("to_keyvalue"): # Get current values res = cur.execute("SELECT * FROM keyvalue WHERE json_id = ?", (json_row["json_id"],)) current_keyvalue = {} current_keyvalue_id = {} for row in res: current_keyvalue[row["key"]] = row["value"] current_keyvalue_id[row["key"]] = row["keyvalue_id"] for key in dbmap["to_keyvalue"]: if key not in current_keyvalue: # Keyvalue not exist yet in the db cur.execute( "INSERT INTO keyvalue ?", {"key": key, "value": data.get(key), "json_id": json_row["json_id"]} ) elif data.get(key) != current_keyvalue[key]: # Keyvalue different value cur.execute( "UPDATE keyvalue SET value = ? WHERE keyvalue_id = ?", (data.get(key), current_keyvalue_id[key]) ) # Insert data to json table for easier joins if dbmap.get("to_json_table"): directory, file_name = re.match("^(.*?)/*([^/]*)$", relative_path).groups() data_json_row = dict(cur.getJsonRow(directory + "/" + dbmap.get("file_name", file_name))) changed = False for key in dbmap["to_json_table"]: if data.get(key) != data_json_row.get(key): changed = True if changed: # Add the custom col values data_json_row.update({key: val for key, val in data.items() if key in dbmap["to_json_table"]}) cur.execute("INSERT OR REPLACE INTO json ?", data_json_row) # Insert data to tables for table_settings in dbmap.get("to_table", []): if isinstance(table_settings, dict): # Custom settings table_name = table_settings["table"] # Table name to insert datas node = table_settings.get("node", table_name) # Node keyname in data json file key_col = table_settings.get("key_col") # Map dict key as this col val_col = table_settings.get("val_col") # Map dict value as this col import_cols = table_settings.get("import_cols") replaces = table_settings.get("replaces") else: # Simple settings table_name = table_settings node = table_settings key_col = None val_col = None import_cols = None replaces = None # Fill import cols from table cols if not import_cols: import_cols = set([item[0] for item in self.schema["tables"][table_name]["cols"]]) cur.execute("DELETE FROM %s WHERE json_id = ?" % table_name, (json_row["json_id"],)) if node not in data: continue if key_col: # Map as dict for key, val in data[node].items(): if val_col: # Single value cur.execute( "INSERT OR REPLACE INTO %s ?" % table_name, {key_col: key, val_col: val, "json_id": json_row["json_id"]} ) else: # Multi value if type(val) is dict: # Single row row = val if import_cols: row = {key: row[key] for key in row if key in import_cols} # Filter row by import_cols row[key_col] = key # Replace in value if necessary if replaces: for replace_key, replace in replaces.items(): if replace_key in row: for replace_from, replace_to in replace.items(): row[replace_key] = row[replace_key].replace(replace_from, replace_to) row["json_id"] = json_row["json_id"] cur.execute("INSERT OR REPLACE INTO %s ?" % table_name, row) elif type(val) is list: # Multi row for row in val: row[key_col] = key row["json_id"] = json_row["json_id"] cur.execute("INSERT OR REPLACE INTO %s ?" % table_name, row) else: # Map as list for row in data[node]: row["json_id"] = json_row["json_id"] if import_cols: row = {key: row[key] for key in row if key in import_cols} # Filter row by import_cols cur.execute("INSERT OR REPLACE INTO %s ?" % table_name, row) # Cleanup json row if not data: self.log.debug("Cleanup json row for %s" % file_path) cur.execute("DELETE FROM json WHERE json_id = %s" % json_row["json_id"]) return True
def peerCheckMessage(self, raw, params, ip): # Calculate hash from nonce msg_hash = hashlib.sha256( "%s,%s" % (params["nonce"], params["raw"])).hexdigest() # Check whether P2P messages are supported site = self.sites.get(raw["site"]) content_json = site.storage.loadJson("content.json") if "p2p_filter" not in content_json: self.connection.log("Site %s doesn't support P2P messages" % raw["site"]) self.connection.badAction(5) self.response({ "error": "Site %s doesn't support P2P messages" % raw["site"] }) return False, "" # Was the message received yet? if msg_hash in site.p2p_received: self.response({"warning": "Already received, thanks"}) return False, "" site.p2p_received.append(msg_hash) # Check whether the message matches passive filter if not SafeRe.match(content_json["p2p_filter"], json.dumps(raw["message"])): self.connection.log("Invalid message for site %s: %s" % (raw["site"], raw["message"])) self.connection.badAction(5) self.response({ "error": "Invalid message for site %s: %s" % (raw["site"], raw["message"]) }) return False, "" # Not so fast if "p2p_freq_limit" in content_json and time.time( ) - site.p2p_last_recv.get(ip, 0) < content_json["p2p_freq_limit"]: self.connection.log("Too fast messages from %s" % raw["site"]) self.connection.badAction(2) self.response({"error": "Too fast messages from %s" % raw["site"]}) return False, "" site.p2p_last_recv[ip] = time.time() # Not so much if "p2p_size_limit" in content_json and len(json.dumps( raw["message"])) > content_json["p2p_size_limit"]: self.connection.log("Too big message from %s" % raw["site"]) self.connection.badAction(7) self.response({"error": "Too big message from %s" % raw["site"]}) return False, "" # Verify signature if params["signature"]: signature_address, signature = params["signature"].split("|") what = "%s|%s|%s" % (signature_address, msg_hash, params["raw"]) from Crypt import CryptBitcoin if not CryptBitcoin.verify(what, signature_address, signature): self.connection.log("Invalid signature") self.connection.badAction(7) self.response({"error": "Invalid signature"}) return False, "" else: signature_address = "" # Check that the signature address is correct if "p2p_signed_only" in content_json: valid = content_json["p2p_signed_only"] if valid is True and not signature_address: self.connection.log("Not signed message") self.connection.badAction(5) self.response({"error": "Not signed message"}) return False, "" elif isinstance(valid, str) and signature_address != valid: self.connection.log( "Message signature is invalid: %s not in [%r]" % (signature_address, valid)) self.connection.badAction(5) self.response({ "error": "Message signature is invalid: %s not in [%r]" % (signature_address, valid) }) return False, "" elif isinstance(valid, list) and signature_address not in valid: self.connection.log( "Message signature is invalid: %s not in %r" % (signature_address, valid)) self.connection.badAction(5) self.response({ "error": "Message signature is invalid: %s not in %r" % (signature_address, valid) }) return False, "" return True, signature_address
def testUnsafeRepetition(self, pattern): with pytest.raises(SafeRe.UnsafePatternError) as err: SafeRe.match(pattern, "aaaaaaaaaaaaaaaaaaaaaaaa!") assert "More than" in str(err.value)
def testUnsafeRepetition(self, pattern): with pytest.raises(SafeRe.UnsafePatternError) as err: SafeRe.match(pattern, "aaaaaaaaaaaaaaaaaaaaaaaa!") assert "More than" in str(err)
def peerCheckMessage(self, raw, params, ip): # Calculate hash from nonce msg_hash = hashlib.sha256("%s,%s" % (params["nonce"], params["raw"])).hexdigest() # Check that p2p.json exists site = self.sites.get(raw["site"]) if not site.storage.isFile("p2p.json"): self.connection.log("Site %s doesn't support P2P messages" % raw["site"]) self.connection.badAction(5) self.response({ "error": "Site %s doesn't support P2P messages" % raw["site"] }) return False, "", None, msg_hash # Check whether P2P messages are supported p2p_json = site.storage.loadJson("p2p.json") if "filter" not in p2p_json: self.connection.log("Site %s doesn't support P2P messages" % raw["site"]) self.connection.badAction(5) self.response({ "error": "Site %s doesn't support P2P messages" % raw["site"] }) return False, "", None, msg_hash # Was the message received yet? if msg_hash in site.p2p_received: self.response({ "warning": "Already received, thanks" }) return False, "", None, msg_hash site.p2p_received.append(msg_hash) # Check whether the message matches passive filter if not SafeRe.match(p2p_json["filter"], json.dumps(raw["message"])): self.connection.log("Invalid message for site %s: %s" % (raw["site"], raw["message"])) self.connection.badAction(5) self.response({ "error": "Invalid message for site %s: %s" % (raw["site"], raw["message"]) }) return False, "", None, msg_hash # Not so fast if "freq_limit" in p2p_json and time.time() - site.p2p_last_recv.get(ip, 0) < p2p_json["freq_limit"]: self.connection.log("Too fast messages from %s" % raw["site"]) self.connection.badAction(2) self.response({ "error": "Too fast messages from %s" % raw["site"] }) return False, "", None, msg_hash site.p2p_last_recv[ip] = time.time() # Not so much if "size_limit" in p2p_json and len(json.dumps(raw["message"])) > p2p_json["size_limit"]: self.connection.log("Too big message from %s" % raw["site"]) self.connection.badAction(7) self.response({ "error": "Too big message from %s" % raw["site"] }) return False, "", None, msg_hash # Verify signature if params["signature"]: signature_address, signature = params["signature"].split("|") what = "%s|%s|%s" % (signature_address, msg_hash, params["raw"]) from Crypt import CryptBitcoin if not CryptBitcoin.verify(what, signature_address, signature): self.connection.log("Invalid signature") self.connection.badAction(7) self.response({ "error": "Invalid signature" }) return False, "", None, msg_hash # Now check auth providers if params.get("cert"): # Read all info cert_auth_type, cert_auth_user_name, cert_issuer, cert_sign = params["cert"] # This is what certificate issuer signs cert_subject = "%s#%s/%s" % (signature_address, cert_auth_type, cert_auth_user_name) # Now get cert issuer address cert_signers = p2p_json.get("cert_signers", {}) cert_addresses = cert_signers.get(cert_issuer, []) # And verify it if not CryptBitcoin.verify(cert_subject, cert_addresses, cert_sign): self.connection.log("Invalid signature certificate") self.connection.badAction(7) self.response({ "error": "Invalid signature certificate" }) return False, "", None, msg_hash # And save the ID cert = "%s/%s@%s" % (cert_auth_type, cert_auth_user_name, cert_issuer) else: # Old-style sign cert = "" else: signature_address = "" cert = "" # Check that the signature address is correct if "signed_only" in p2p_json: valid = p2p_json["signed_only"] if valid is True and not signature_address: self.connection.log("Not signed message") self.connection.badAction(5) self.response({ "error": "Not signed message" }) return False, "", None, msg_hash elif isinstance(valid, str) and signature_address != valid: self.connection.log("Message signature is invalid: %s not in [%r]" % (signature_address, valid)) self.connection.badAction(5) self.response({ "error": "Message signature is invalid: %s not in [%r]" % (signature_address, valid) }) return False, "", None, msg_hash elif isinstance(valid, list) and signature_address not in valid: self.connection.log("Message signature is invalid: %s not in %r" % (signature_address, valid)) self.connection.badAction(5) self.response({ "error": "Message signature is invalid: %s not in %r" % (signature_address, valid) }) return False, "", None, msg_hash return True, signature_address, cert, msg_hash
def actionPeerBroadcast(self, to, message, privatekey=None, peer_count=5, broadcast=True, immediate=False, timeout=60): # Check whether P2P messages are supported content_json = self.site.storage.loadJson("content.json") if "p2p_filter" not in content_json: self.response( to, { "error": "Site %s doesn't support P2P messages" % self.site.address }) return # Check whether the message matches passive filter if not SafeRe.match(content_json["p2p_filter"], json.dumps(message)): self.response( to, { "error": "Invalid message for site %s: %s" % (self.site.address, message) }) return # Not so fast if "p2p_freq_limit" in content_json and time.time( ) - self.site.p2p_last_recv.get("self", 0) < content_json["p2p_freq_limit"]: self.response(to, {"error": "Too fast messages"}) return self.site.p2p_last_recv["self"] = time.time() # Not so much if "p2p_size_limit" in content_json and len( json.dumps(message)) > content_json["p2p_size_limit"]: self.response(to, {"error": "Too big message"}) return # Generate message and sign it all_message = { "message": message, "peer_count": peer_count, "broadcast": broadcast, "immediate": immediate, "site": self.site.address } all_message = json.dumps(all_message) nonce = str(random.randint(0, 1000000000)) msg_hash = hashlib.md5("%s,%s" % (nonce, all_message)).hexdigest() signature = self.p2pGetSignature(msg_hash, all_message, privatekey) all_message = { "raw": all_message, "signature": signature, "hash": msg_hash } peers = self.site.getConnectedPeers() if len(peers ) < peer_count: # Add more, non-connected peers if necessary peers += self.site.getRecentPeers(peer_count - len(peers)) # Send message to peers jobs = [] for peer in peers: jobs.append(gevent.spawn(self.p2pBroadcast, peer, all_message)) if not broadcast: # Makes sense to return result res = gevent.joinall(jobs, timeout) self.response(to, res) return # Reply self.response(to, {"sent": True}) # Send message to myself self.site.p2p_received.append(msg_hash) websockets = [ ws for ws in self.site.websockets if "peerReceive" in ws.channels ] for ws in websockets: ws.cmd( "peerReceive", { "ip": "self", "hash": msg_hash, "message": message, "signed_by": all_message["signature"].split("|")[0] if all_message["signature"] else "" }) if not websockets and immediate: self.site.p2p_unread.append({ "ip": "self", "hash": msg_hash, "message": message, "signed_by": all_message["signature"].split("|")[0] if all_message["signature"] else "" })
def actionPeerBroadcast(self, params): ip = "%s:%s" % (self.connection.ip, self.connection.port) raw = json.loads(params["raw"]) # Check whether P2P messages are supported site = self.sites.get(raw["site"]) content_json = site.storage.loadJson("content.json") if "p2p_filter" not in content_json: self.connection.log("Site %s doesn't support P2P messages" % raw["site"]) self.connection.badAction(5) return # Was the message received yet? if params["hash"] in site.p2p_received: return site.p2p_received.append(params["hash"]) # Check whether the message matches passive filter if not SafeRe.match(content_json["p2p_filter"], json.dumps(raw["message"])): self.connection.log("Invalid message for site %s: %s" % (raw["site"], raw["message"])) self.connection.badAction(5) return # Not so fast if "p2p_freq_limit" in content_json and time.time( ) - site.p2p_last_recv.get(ip, 0) < content_json["p2p_freq_limit"]: self.connection.log("Too fast messages from %s" % raw["site"]) self.connection.badAction(2) return site.p2p_last_recv[ip] = time.time() # Not so much if "p2p_size_limit" in content_json and len(json.dumps( raw["message"])) > content_json["p2p_size_limit"]: self.connection.log("Too big message from %s" % raw["site"]) self.connection.badAction(7) return # Verify signature if params["signature"]: signature_address, signature = params["signature"].split("|") what = "%s|%s|%s" % (signature_address, params["hash"], params["raw"]) from Crypt import CryptBitcoin if not CryptBitcoin.verify(what, signature_address, signature): self.connection.log("Invalid signature") self.connection.badAction(7) return else: signature_address = "" # Check that the signature address is correct if "p2p_signed_only" in content_json: valid = content_json["p2p_signed_only"] if valid is True and not signature_address: self.connection.log("Not signed message") self.connection.badAction(5) return elif isinstance(valid, str) and signature_address != valid: self.connection.log( "Message signature is invalid: %s not in [%r]" % (signature_address, valid)) self.connection.badAction(5) return elif isinstance(valid, list) and signature_address not in valid: self.connection.log( "Message signature is invalid: %s not in %r" % (signature_address, valid)) self.connection.badAction(5) return # Send to WebSocket websockets = [ ws for ws in site.websockets if "peerReceive" in ws.channels ] for ws in websockets: ws.cmd( "peerReceive", { "ip": ip, "hash": params["hash"], "message": raw["message"], "signed_by": signature_address }) # Maybe active filter will reply? if websockets: # Wait for p2p_result result = gevent.spawn(self.p2pWaitMessage, site, params["hash"]).join() del site.p2p_result[params["hash"]] if not result: self.connection.badAction(10) return # Save to cache if not websockets and raw["immediate"]: site.p2p_unread.append({ "ip": "%s:%s" % (self.connection.ip, self.connection.port), "hash": params["hash"], "message": raw["message"], "signed_by": signature_address }) # Now send to neighbour peers if raw["broadcast"]: # Get peer list peers = site.getConnectedPeers() if len(peers) < raw[ "peer_count"]: # Add more, non-connected peers if necessary peers += site.getRecentPeers(raw["peer_count"] - len(peers)) # Send message to peers for peer in peers: gevent.spawn(peer.connection.request, "peerBroadcast", params)
def testSafeMatch(self): assert SafeRe.match( "((js|css)/(?!all.(js|css))|data/users/.*db|data/users/.*/.*|data/archived|.*.py)", "js/ZeroTalk.coffee" ) assert SafeRe.match(".+/data.json", "data/users/1J3rJ8ecnwH2EPYa6MrgZttBNc61ACFiCj/data.json")
def testUnsafeMatch(self, pattern): with pytest.raises(SafeRe.UnsafePatternError) as err: SafeRe.match(pattern, "aaaaaaaaaaaaaaaaaaaaaaaa!") assert "Potentially unsafe" in str(err.value)
def testUnsafeMatch(self, pattern): with pytest.raises(SafeRe.UnsafePatternError) as err: SafeRe.match(pattern, "aaaaaaaaaaaaaaaaaaaaaaaa!") assert "Potentially unsafe" in str(err)