def testSafeMatch(self):
assert SafeRe.match(
"((js|css)/(?!all.(js|css))|data/users/.*db|data/users/.*/.*|data/archived|.*.py)",
"js/ZeroTalk.coffee")
assert SafeRe.match(
".+/data.json",
"data/users/1J3rJ8ecnwH2EPYa6MrgZttBNc61ACFiCj/data.json")
def verifyContentInclude(self, inner_path, content, content_size, content_size_optional):
# Load include details
rules = self.getRules(inner_path, content)
if not rules:
raise VerifyError("No rules")
# Check include size limit
if rules.get("max_size") is not None: # Include size limit
if content_size > rules["max_size"]:
raise VerifyError("Include too large %sB > %sB" % (content_size, rules["max_size"]))
if rules.get("max_size_optional") is not None: # Include optional files limit
if content_size_optional > rules["max_size_optional"]:
raise VerifyError("Include optional files too large %sB > %sB" % (
content_size_optional, rules["max_size_optional"])
)
# Filename limit
if rules.get("files_allowed"):
for file_inner_path in content["files"].keys():
if not SafeRe.match("^%s$" % rules["files_allowed"], file_inner_path):
raise VerifyError("File not allowed: %s" % file_inner_path)
if rules.get("files_allowed_optional"):
for file_inner_path in content.get("files_optional", {}).keys():
if not SafeRe.match("^%s$" % rules["files_allowed_optional"], file_inner_path):
raise VerifyError("Optional file not allowed: %s" % file_inner_path)
# Check if content includes allowed
if rules.get("includes_allowed") is False and content.get("includes"):
raise VerifyError("Includes not allowed")
return True # All good
def verifyContentInclude(self, inner_path, content, content_size, content_size_optional):
# Load include details
rules = self.getRules(inner_path, content)
if not rules:
raise VerifyError("No rules")
# Check include size limit
if rules.get("max_size") is not None: # Include size limit
if content_size > rules["max_size"]:
raise VerifyError("Include too large %sB > %sB" % (content_size, rules["max_size"]))
if rules.get("max_size_optional") is not None: # Include optional files limit
if content_size_optional > rules["max_size_optional"]:
raise VerifyError("Include optional files too large %sB > %sB" % (
content_size_optional, rules["max_size_optional"])
)
# Filename limit
if rules.get("files_allowed"):
for file_inner_path in content["files"].keys():
if not SafeRe.match("^%s$" % rules["files_allowed"], file_inner_path):
raise VerifyError("File not allowed: %s" % file_inner_path)
if rules.get("files_allowed_optional"):
for file_inner_path in content.get("files_optional", {}).keys():
if not SafeRe.match("^%s$" % rules["files_allowed_optional"], file_inner_path):
raise VerifyError("Optional file not allowed: %s" % file_inner_path)
# Check if content includes allowed
if rules.get("includes_allowed") is False and content.get("includes"):
raise VerifyError("Includes not allowed")
return True # All good
def hashFiles(self,
dir_inner_path,
ignore_pattern=None,
optional_pattern=None):
files_node = {}
files_optional_node = {}
if dir_inner_path and not self.isValidRelativePath(dir_inner_path):
ignored = True
self.log.error(
"- [ERROR] Only ascii encoded directories allowed: %s" %
dir_inner_path)
for file_relative_path in self.site.storage.walk(dir_inner_path):
file_name = helper.getFilename(file_relative_path)
ignored = optional = False
if file_name == "content.json":
ignored = True
elif ignore_pattern and SafeRe.match(ignore_pattern,
file_relative_path):
ignored = True
elif file_name.startswith(".") or file_name.endswith(
"-old") or file_name.endswith("-new"):
ignored = True
elif not self.isValidRelativePath(file_relative_path):
ignored = True
self.log.error("- [ERROR] Invalid filename: %s" %
file_relative_path)
elif dir_inner_path == "" and file_relative_path == self.site.storage.getDbFile(
):
ignored = True
elif optional_pattern and SafeRe.match(optional_pattern,
file_relative_path):
optional = True
if ignored: # Ignore content.json, defined regexp and files starting with .
self.log.info("- [SKIPPED] %s" % file_relative_path)
else:
if optional:
self.log.info("- [OPTIONAL] %s" % file_relative_path)
files_optional_node.update(
self.hashFile(dir_inner_path,
file_relative_path,
optional=True))
else:
self.log.info("- %s" % file_relative_path)
files_node.update(
self.hashFile(dir_inner_path, file_relative_path))
return files_node, files_optional_node
def walk(self, dir_inner_path, ignore=None):
directory = self.getPath(dir_inner_path)
for root, dirs, files in os.walk(directory):
root = root.replace("\\", "/")
root_relative_path = re.sub("^%s" % re.escape(directory), "", root).lstrip("/")
for file_name in files:
if root_relative_path: # Not root dir
file_relative_path = root_relative_path + "/" + file_name
else:
file_relative_path = file_name
if ignore and SafeRe.match(ignore, file_relative_path):
continue
yield file_relative_path
# Don't scan directory that is in the ignore pattern
if ignore:
dirs_filtered = []
for dir_name in dirs:
if root_relative_path:
dir_relative_path = root_relative_path + "/" + dir_name
else:
dir_relative_path = dir_name
if ignore == ".*" or re.match(".*([|(]|^)%s([|)]|$)" % re.escape(dir_relative_path + "/.*"), ignore):
continue
dirs_filtered.append(dir_name)
dirs[:] = dirs_filtered
def updateJson(self, file_path, file=None, cur=None):
if not file_path.startswith(self.db_dir):
return False # Not from the db dir: Skipping
relative_path = file_path[len(self.db_dir):] # File path realative to db file
# Check if filename matches any of mappings in schema
matched_maps = []
for match, map_settings in self.schema["maps"].items():
try:
if SafeRe.match(match, relative_path):
matched_maps.append(map_settings)
except SafeRe.UnsafePatternError as err:
self.log.error(err)
# No match found for the file
if not matched_maps:
return False
# Load the json file
try:
if file is None: # Open file is not file object passed
file = open(file_path, "rb")
if file is False: # File deleted
data = {}
else:
if file_path.endswith("json.gz"):
data = json.load(helper.limitedGzipFile(fileobj=file))
else:
data = json.load(file)
except Exception, err:
self.log.debug("Json file %s load error: %s" % (file_path, err))
data = {}
def updateJson(self, file_path, file=None, cur=None):
if not file_path.startswith(self.db_dir):
return False # Not from the db dir: Skipping
relative_path = file_path[len(self.db_dir):] # File path realative to db file
# Check if filename matches any of mappings in schema
matched_maps = []
for match, map_settings in self.schema["maps"].items():
try:
if SafeRe.match(match, relative_path):
matched_maps.append(map_settings)
except SafeRe.UnsafePatternError as err:
self.log.error(err)
# No match found for the file
if not matched_maps:
return False
# Load the json file
try:
if file is None: # Open file is not file object passed
file = open(file_path, "rb")
if file is False: # File deleted
data = {}
else:
if file_path.endswith("json.gz"):
data = json.load(helper.limitedGzipFile(fileobj=file))
else:
data = json.load(file)
except Exception, err:
self.log.debug("Json file %s load error: %s" % (file_path, err))
data = {}
def walk(self, dir_inner_path, ignore=None):
directory = self.getPath(dir_inner_path)
for root, dirs, files in os.walk(directory):
root = root.replace("\\", "/")
root_relative_path = re.sub("^%s" % re.escape(directory), "",
root).lstrip("/")
for file_name in files:
if root_relative_path: # Not root dir
file_relative_path = root_relative_path + "/" + file_name
else:
file_relative_path = file_name
if ignore and SafeRe.match(ignore, file_relative_path):
continue
yield file_relative_path
# Don't scan directory that is in the ignore pattern
if ignore:
dirs_filtered = []
for dir_name in dirs:
if root_relative_path:
dir_relative_path = root_relative_path + "/" + dir_name
else:
dir_relative_path = dir_name
if ignore == ".*" or re.match(
".*([|(]|^)%s([|)]|$)" %
re.escape(dir_relative_path + "/.*"), ignore):
continue
dirs_filtered.append(dir_name)
dirs[:] = dirs_filtered
def verifyCert(self, inner_path, content):
from Crypt import CryptBitcoin
rules = self.getRules(inner_path, content)
if not rules:
raise VerifyError("No rules for this file")
if not rules.get("cert_signers") and not rules.get("cert_signers_pattern"):
return True # Does not need cert
if "cert_user_id" not in content:
raise VerifyError("Missing cert_user_id")
if content["cert_user_id"].count("@") != 1:
raise VerifyError("Invalid domain in cert_user_id")
name, domain = content["cert_user_id"].rsplit("@", 1)
cert_address = rules["cert_signers"].get(domain)
if not cert_address: # Unknown Cert signer
if rules.get("cert_signers_pattern") and SafeRe.match(rules["cert_signers_pattern"], domain):
cert_address = domain
else:
raise VerifyError("Invalid cert signer: %s" % domain)
try:
cert_subject = "%s#%s/%s" % (rules["user_address"], content["cert_auth_type"], name)
result = CryptBitcoin.verify(cert_subject, cert_address, content["cert_sign"])
except Exception, err:
raise VerifyError("Certificate verify error: %s" % err)
def getUserContentRules(self, parent_content, inner_path, content):
user_contents = parent_content["user_contents"]
# Delivered for directory
if "inner_path" in parent_content:
parent_content_dir = helper.getDirname(parent_content["inner_path"])
user_address = re.match("([A-Za-z0-9]*?)/", inner_path[len(parent_content_dir):]).group(1)
else:
user_address = re.match(".*/([A-Za-z0-9]*?)/.*?$", inner_path).group(1)
try:
if not content:
content = self.site.storage.loadJson(inner_path) # Read the file if no content specified
user_urn = "%s/%s" % (content["cert_auth_type"], content["cert_user_id"]) # web/[email protected]
cert_user_id = content["cert_user_id"]
except Exception: # Content.json not exist
user_urn = "n-a/n-a"
cert_user_id = "n-a"
if user_address in user_contents["permissions"]:
rules = copy.copy(user_contents["permissions"].get(user_address, {})) # Default rules based on address
else:
rules = copy.copy(user_contents["permissions"].get(cert_user_id, {})) # Default rules based on username
if rules is False:
banned = True
rules = {}
else:
banned = False
if "signers" in rules:
rules["signers"] = rules["signers"][:] # Make copy of the signers
for permission_pattern, permission_rules in user_contents["permission_rules"].items(): # Regexp rules
if not SafeRe.match(permission_pattern, user_urn):
continue # Rule is not valid for user
# Update rules if its better than current recorded ones
for key, val in permission_rules.iteritems():
if key not in rules:
if type(val) is list:
rules[key] = val[:] # Make copy
else:
rules[key] = val
elif type(val) is int: # Int, update if larger
if val > rules[key]:
rules[key] = val
elif hasattr(val, "startswith"): # String, update if longer
if len(val) > len(rules[key]):
rules[key] = val
elif type(val) is list: # List, append
rules[key] += val
rules["cert_signers"] = user_contents["cert_signers"] # Add valid cert signers
if "signers" not in rules:
rules["signers"] = []
if not banned:
rules["signers"].append(user_address) # Add user as valid signer
rules["user_address"] = user_address
rules["includes_allowed"] = False
return rules
def verifyCert(self, inner_path, content):
from Crypt import CryptBitcoin
rules = self.getRules(inner_path, content)
if not rules:
raise VerifyError("No rules for this file")
if not rules.get("cert_signers") and not rules.get("cert_signers_pattern"):
return True # Does not need cert
if "cert_user_id" not in content:
raise VerifyError("Missing cert_user_id")
if content["cert_user_id"].count("@") != 1:
raise VerifyError("Invalid domain in cert_user_id")
name, domain = content["cert_user_id"].rsplit("@", 1)
cert_address = rules["cert_signers"].get(domain)
if not cert_address: # Unknown Cert signer
if rules.get("cert_signers_pattern") and SafeRe.match(rules["cert_signers_pattern"], domain):
cert_address = domain
else:
raise VerifyError("Invalid cert signer: %s" % domain)
try:
cert_subject = "%s#%s/%s" % (rules["user_address"], content["cert_auth_type"], name)
result = CryptBitcoin.verify(cert_subject, cert_address, content["cert_sign"])
except Exception, err:
raise VerifyError("Certificate verify error: %s" % err)
def peerCheckMessage(self, to, message):
# Check whether there is p2p.json
if not self.site.storage.isFile("p2p.json"):
self.response(to, {"error": "Site %s doesn't support P2P messages" % self.site.address})
return False
# Check whether P2P messages are supported
p2p_json = self.site.storage.loadJson("p2p.json")
if "filter" not in p2p_json:
self.response(to, {"error": "Site %s doesn't support P2P messages" % self.site.address})
return False
# Check whether the message matches passive filter
if not SafeRe.match(p2p_json["filter"], json.dumps(message)):
self.response(to, {"error": "Invalid message for site %s: %s" % (self.site.address, message)})
return False
# Not so fast
if "freq_limit" in p2p_json and time.time() - self.site.p2p_last_recv.get("self", 0) < p2p_json["freq_limit"]:
self.response(to, {"error": "Too fast messages"})
return False
self.site.p2p_last_recv["self"] = time.time()
# Not so much
if "size_limit" in p2p_json and len(json.dumps(message)) > p2p_json["size_limit"]:
self.response(to, {"error": "Too big message"})
return False
return True
def getUserContentRules(self, parent_content, inner_path, content):
user_contents = parent_content["user_contents"]
# Delivered for directory
if "inner_path" in parent_content:
parent_content_dir = helper.getDirname(parent_content["inner_path"])
user_address = re.match("([A-Za-z0-9]*?)/", inner_path[len(parent_content_dir):]).group(1)
else:
user_address = re.match(".*/([A-Za-z0-9]*?)/.*?$", inner_path).group(1)
try:
if not content:
content = self.site.storage.loadJson(inner_path) # Read the file if no content specified
user_urn = "%s/%s" % (content["cert_auth_type"], content["cert_user_id"]) # web/[email protected]
cert_user_id = content["cert_user_id"]
except Exception: # Content.json not exist
user_urn = "n-a/n-a"
cert_user_id = "n-a"
if user_address in user_contents["permissions"]:
rules = copy.copy(user_contents["permissions"].get(user_address, {})) # Default rules based on address
else:
rules = copy.copy(user_contents["permissions"].get(cert_user_id, {})) # Default rules based on username
if rules is False:
banned = True
rules = {}
else:
banned = False
if "signers" in rules:
rules["signers"] = rules["signers"][:] # Make copy of the signers
for permission_pattern, permission_rules in user_contents["permission_rules"].items(): # Regexp rules
if not SafeRe.match(permission_pattern, user_urn):
continue # Rule is not valid for user
# Update rules if its better than current recorded ones
for key, val in permission_rules.iteritems():
if key not in rules:
if type(val) is list:
rules[key] = val[:] # Make copy
else:
rules[key] = val
elif type(val) is int: # Int, update if larger
if val > rules[key]:
rules[key] = val
elif hasattr(val, "startswith"): # String, update if longer
if len(val) > len(rules[key]):
rules[key] = val
elif type(val) is list: # List, append
rules[key] += val
rules["cert_signers"] = user_contents["cert_signers"] # Add valid cert signers
if "signers" not in rules:
rules["signers"] = []
if not banned:
rules["signers"].append(user_address) # Add user as valid signer
rules["user_address"] = user_address
rules["includes_allowed"] = False
return rules
def actionConsoleLogRead(self,
to,
filter=None,
read_size=32 * 1024,
limit=500):
log_file_path = "%s/debug.log" % config.log_dir
log_file = open(log_file_path, encoding="utf-8")
log_file.seek(0, 2)
end_pos = log_file.tell()
log_file.seek(max(0, end_pos - read_size))
if log_file.tell() != 0:
log_file.readline() # Partial line junk
pos_start = log_file.tell()
lines = []
if filter:
assert SafeRe.isSafePattern(filter)
filter_re = re.compile(".*" + filter)
for line in log_file:
if filter and not filter_re.match(line):
continue
lines.append(line)
num_found = len(lines)
lines = lines[-limit:]
return {
"lines": lines,
"pos_end": log_file.tell(),
"pos_start": pos_start,
"num_found": num_found
}
def actionCertSelect(self, to, accepted_domains=[], accept_any=False, accepted_pattern=None):
accounts = []
accounts.append(["", _["No certificate"], ""]) # Default option
active = "" # Make it active if no other option found
# Add my certs
auth_address = self.user.getAuthAddress(self.site.address) # Current auth address
site_data = self.user.getSiteData(self.site.address) # Current auth address
if not accepted_domains and not accepted_pattern: # Accept any if no filter defined
accept_any = True
for domain, cert in self.user.certs.items():
if auth_address == cert["auth_address"] and domain == site_data.get("cert"):
active = domain
title = cert["auth_user_name"] + "@" + domain
accepted_pattern_match = accepted_pattern and SafeRe.match(accepted_pattern, domain)
if domain in accepted_domains or accept_any or accepted_pattern_match:
accounts.append([domain, title, ""])
else:
accounts.append([domain, title, "disabled"])
# Render the html
body = "<span style='padding-bottom: 5px; display: inline-block'>" + _["Select account you want to use in this site:"] + "</span>"
# Accounts
for domain, account, css_class in accounts:
if domain == active:
css_class += " active" # Currently selected option
title = _(u"<b>%s</b> <small>({_[currently selected]})</small>") % account
else:
title = "<b>%s</b>" % account
body += "<a href='#Select+account' class='select select-close cert %s' title='%s'>%s</a>" % (css_class, domain, title)
# More available providers
more_domains = [domain for domain in accepted_domains if domain not in self.user.certs] # Domains we not displayed yet
if more_domains:
# body+= "<small style='margin-top: 10px; display: block'>Accepted authorization providers by the site:</small>"
body += "<div style='background-color: #F7F7F7; margin-right: -30px'>"
for domain in more_domains:
body += _(u"""
<a href='/{domain}' onclick='zeroframe.certSelectGotoSite(this)' class='select'>
<small style='float: right; margin-right: 40px; margin-top: -1px'>{_[Register]} »</small>{domain}
</a>
""")
body += "</div>"
body += """
<script>
$(".notification .select.cert").on("click", function() {
$(".notification .select").removeClass('active')
zeroframe.response(%s, this.title)
return false
})
</script>
""" % self.next_message_id
# Send the notification
self.cmd("notification", ["ask", body], lambda domain: self.actionCertSet(to, domain))
def actionCertSelect(self, to, accepted_domains=[], accept_any=False, accepted_pattern=None):
accounts = []
accounts.append(["", _["No certificate"], ""]) # Default option
active = "" # Make it active if no other option found
# Add my certs
auth_address = self.user.getAuthAddress(self.site.address) # Current auth address
site_data = self.user.getSiteData(self.site.address) # Current auth address
if not accepted_domains and not accepted_pattern: # Accept any if no filter defined
accept_any = True
for domain, cert in self.user.certs.items():
if auth_address == cert["auth_address"] and domain == site_data.get("cert"):
active = domain
title = cert["auth_user_name"] + "@" + domain
accepted_pattern_match = accepted_pattern and SafeRe.match(accepted_pattern, domain)
if domain in accepted_domains or accept_any or accepted_pattern_match:
accounts.append([domain, title, ""])
else:
accounts.append([domain, title, "disabled"])
# Render the html
body = "<span style='padding-bottom: 5px; display: inline-block'>" + _["Select account you want to use in this site:"] + "</span>"
# Accounts
for domain, account, css_class in accounts:
if domain == active:
css_class += " active" # Currently selected option
title = _(u"<b>%s</b> <small>({_[currently selected]})</small>") % account
else:
title = "<b>%s</b>" % account
body += "<a href='#Select+account' class='select select-close cert %s' title='%s'>%s</a>" % (css_class, domain, title)
# More available providers
more_domains = [domain for domain in accepted_domains if domain not in self.user.certs] # Domains we not displayed yet
if more_domains:
# body+= "<small style='margin-top: 10px; display: block'>Accepted authorization providers by the site:</small>"
body += "<div style='background-color: #F7F7F7; margin-right: -30px'>"
for domain in more_domains:
body += _(u"""
<a href='/{domain}' target='_top' class='select'>
<small style='float: right; margin-right: 40px; margin-top: -1px'>{_[Register]} »</small>{domain}
</a>
""")
body += "</div>"
script = """
$(".notification .select.cert").on("click", function() {
$(".notification .select").removeClass('active')
zeroframe.response(%s, this.title)
return false
})
""" % self.next_message_id
self.cmd("notification", ["ask", body], lambda domain: self.actionCertSet(to, domain))
self.cmd("injectScript", script)
def __init__(self, stream_id, ui_websocket, filter):
self.stream_id = stream_id
self.ui_websocket = ui_websocket
if filter:
if not SafeRe.isSafePattern(filter):
raise Exception("Not a safe prex pattern")
self.filter_re = re.compile(".*" + filter)
else:
self.filter_re = None
return super(WsLogStreamer, self).__init__()
def hashFiles(self, dir_inner_path, ignore_pattern=None, optional_pattern=None):
files_node = {}
files_optional_node = {}
if dir_inner_path and not self.isValidRelativePath(dir_inner_path):
ignored = True
self.log.error("- [ERROR] Only ascii encoded directories allowed: %s" % dir_inner_path)
for file_relative_path in self.site.storage.walk(dir_inner_path):
file_name = helper.getFilename(file_relative_path)
ignored = optional = False
if file_name == "content.json":
ignored = True
elif ignore_pattern and SafeRe.match(ignore_pattern, file_relative_path):
ignored = True
elif file_name.startswith(".") or file_name.endswith("-old") or file_name.endswith("-new"):
ignored = True
elif not self.isValidRelativePath(file_relative_path):
ignored = True
self.log.error("- [ERROR] Invalid filename: %s" % file_relative_path)
elif optional_pattern and SafeRe.match(optional_pattern, file_relative_path):
optional = True
if ignored: # Ignore content.json, defined regexp and files starting with .
self.log.info("- [SKIPPED] %s" % file_relative_path)
else:
if optional:
self.log.info("- [OPTIONAL] %s" % file_relative_path)
files_optional_node.update(
self.hashFile(dir_inner_path, file_relative_path, optional=True)
)
else:
self.log.info("- %s" % file_relative_path)
files_node.update(
self.hashFile(dir_inner_path, file_relative_path)
)
return files_node, files_optional_node
def peerCheckMessage(self, raw, params, ip):
# Calculate hash from nonce
msg_hash = hashlib.sha256(
"%s,%s" % (params["nonce"], params["raw"])).hexdigest()
# Check whether P2P messages are supported
site = self.sites.get(raw["site"])
content_json = site.storage.loadJson("content.json")
if "p2p_filter" not in content_json:
self.connection.log("Site %s doesn't support P2P messages" %
raw["site"])
self.connection.badAction(5)
self.response({
"error":
"Site %s doesn't support P2P messages" % raw["site"]
})
return False, ""
# Was the message received yet?
if msg_hash in site.p2p_received:
self.response({"warning": "Already received, thanks"})
return False, ""
site.p2p_received.append(msg_hash)
# Check whether the message matches passive filter
if not SafeRe.match(content_json["p2p_filter"],
json.dumps(raw["message"])):
self.connection.log("Invalid message for site %s: %s" %
(raw["site"], raw["message"]))
self.connection.badAction(5)
self.response({
"error":
"Invalid message for site %s: %s" %
(raw["site"], raw["message"])
})
return False, ""
# Not so fast
if "p2p_freq_limit" in content_json and time.time(
) - site.p2p_last_recv.get(ip, 0) < content_json["p2p_freq_limit"]:
self.connection.log("Too fast messages from %s" % raw["site"])
self.connection.badAction(2)
self.response({"error": "Too fast messages from %s" % raw["site"]})
return False, ""
site.p2p_last_recv[ip] = time.time()
# Not so much
if "p2p_size_limit" in content_json and len(json.dumps(
raw["message"])) > content_json["p2p_size_limit"]:
self.connection.log("Too big message from %s" % raw["site"])
self.connection.badAction(7)
self.response({"error": "Too big message from %s" % raw["site"]})
return False, ""
# Verify signature
if params["signature"]:
signature_address, signature = params["signature"].split("|")
what = "%s|%s|%s" % (signature_address, msg_hash, params["raw"])
from Crypt import CryptBitcoin
if not CryptBitcoin.verify(what, signature_address, signature):
self.connection.log("Invalid signature")
self.connection.badAction(7)
self.response({"error": "Invalid signature"})
return False, ""
else:
signature_address = ""
# Check that the signature address is correct
if "p2p_signed_only" in content_json:
valid = content_json["p2p_signed_only"]
if valid is True and not signature_address:
self.connection.log("Not signed message")
self.connection.badAction(5)
self.response({"error": "Not signed message"})
return False, ""
elif isinstance(valid, str) and signature_address != valid:
self.connection.log(
"Message signature is invalid: %s not in [%r]" %
(signature_address, valid))
self.connection.badAction(5)
self.response({
"error":
"Message signature is invalid: %s not in [%r]" %
(signature_address, valid)
})
return False, ""
elif isinstance(valid, list) and signature_address not in valid:
self.connection.log(
"Message signature is invalid: %s not in %r" %
(signature_address, valid))
self.connection.badAction(5)
self.response({
"error":
"Message signature is invalid: %s not in %r" %
(signature_address, valid)
})
return False, ""
return True, signature_address
def updateJson(self, file_path, file=None, cur=None):
if not file_path.startswith(self.db_dir):
return False # Not from the db dir: Skipping
relative_path = file_path[len(self.db_dir):] # File path realative to db file
# Check if filename matches any of mappings in schema
matched_maps = []
for match, map_settings in self.schema["maps"].items():
try:
if SafeRe.match(match, relative_path):
matched_maps.append(map_settings)
except SafeRe.UnsafePatternError as err:
self.log.error(err)
# No match found for the file
if not matched_maps:
return False
# Load the json file
try:
if file is None: # Open file is not file object passed
file = open(file_path, "rb")
if file is False: # File deleted
data = {}
else:
if file_path.endswith("json.gz"):
file = helper.limitedGzipFile(fileobj=file)
if sys.version_info.major == 3 and sys.version_info.minor < 6:
data = json.loads(file.read().decode("utf8"))
else:
data = json.load(file)
except Exception as err:
self.log.debug("Json file %s load error: %s" % (file_path, err))
data = {}
# No cursor specificed
if not cur:
cur = self.getSharedCursor()
cur.logging = False
# Row for current json file if required
if not data or [dbmap for dbmap in matched_maps if "to_keyvalue" in dbmap or "to_table" in dbmap]:
json_row = cur.getJsonRow(relative_path)
# Check matched mappings in schema
for dbmap in matched_maps:
# Insert non-relational key values
if dbmap.get("to_keyvalue"):
# Get current values
res = cur.execute("SELECT * FROM keyvalue WHERE json_id = ?", (json_row["json_id"],))
current_keyvalue = {}
current_keyvalue_id = {}
for row in res:
current_keyvalue[row["key"]] = row["value"]
current_keyvalue_id[row["key"]] = row["keyvalue_id"]
for key in dbmap["to_keyvalue"]:
if key not in current_keyvalue: # Keyvalue not exist yet in the db
cur.execute(
"INSERT INTO keyvalue ?",
{"key": key, "value": data.get(key), "json_id": json_row["json_id"]}
)
elif data.get(key) != current_keyvalue[key]: # Keyvalue different value
cur.execute(
"UPDATE keyvalue SET value = ? WHERE keyvalue_id = ?",
(data.get(key), current_keyvalue_id[key])
)
# Insert data to json table for easier joins
if dbmap.get("to_json_table"):
directory, file_name = re.match("^(.*?)/*([^/]*)$", relative_path).groups()
data_json_row = dict(cur.getJsonRow(directory + "/" + dbmap.get("file_name", file_name)))
changed = False
for key in dbmap["to_json_table"]:
if data.get(key) != data_json_row.get(key):
changed = True
if changed:
# Add the custom col values
data_json_row.update({key: val for key, val in data.items() if key in dbmap["to_json_table"]})
cur.execute("INSERT OR REPLACE INTO json ?", data_json_row)
# Insert data to tables
for table_settings in dbmap.get("to_table", []):
if isinstance(table_settings, dict): # Custom settings
table_name = table_settings["table"] # Table name to insert datas
node = table_settings.get("node", table_name) # Node keyname in data json file
key_col = table_settings.get("key_col") # Map dict key as this col
val_col = table_settings.get("val_col") # Map dict value as this col
import_cols = table_settings.get("import_cols")
replaces = table_settings.get("replaces")
else: # Simple settings
table_name = table_settings
node = table_settings
key_col = None
val_col = None
import_cols = None
replaces = None
# Fill import cols from table cols
if not import_cols:
import_cols = set([item[0] for item in self.schema["tables"][table_name]["cols"]])
cur.execute("DELETE FROM %s WHERE json_id = ?" % table_name, (json_row["json_id"],))
if node not in data:
continue
if key_col: # Map as dict
for key, val in data[node].items():
if val_col: # Single value
cur.execute(
"INSERT OR REPLACE INTO %s ?" % table_name,
{key_col: key, val_col: val, "json_id": json_row["json_id"]}
)
else: # Multi value
if type(val) is dict: # Single row
row = val
if import_cols:
row = {key: row[key] for key in row if key in import_cols} # Filter row by import_cols
row[key_col] = key
# Replace in value if necessary
if replaces:
for replace_key, replace in replaces.items():
if replace_key in row:
for replace_from, replace_to in replace.items():
row[replace_key] = row[replace_key].replace(replace_from, replace_to)
row["json_id"] = json_row["json_id"]
cur.execute("INSERT OR REPLACE INTO %s ?" % table_name, row)
elif type(val) is list: # Multi row
for row in val:
row[key_col] = key
row["json_id"] = json_row["json_id"]
cur.execute("INSERT OR REPLACE INTO %s ?" % table_name, row)
else: # Map as list
for row in data[node]:
row["json_id"] = json_row["json_id"]
if import_cols:
row = {key: row[key] for key in row if key in import_cols} # Filter row by import_cols
cur.execute("INSERT OR REPLACE INTO %s ?" % table_name, row)
# Cleanup json row
if not data:
self.log.debug("Cleanup json row for %s" % file_path)
cur.execute("DELETE FROM json WHERE json_id = %s" % json_row["json_id"])
return True
def testUnsafeMatch(self, pattern):
with pytest.raises(SafeRe.UnsafePatternError) as err:
SafeRe.match(pattern, "aaaaaaaaaaaaaaaaaaaaaaaa!")
assert "Potentially unsafe" in str(err)
def testUnsafeRepetition(self, pattern):
with pytest.raises(SafeRe.UnsafePatternError) as err:
SafeRe.match(pattern, "aaaaaaaaaaaaaaaaaaaaaaaa!")
assert "More than" in str(err)
def testSafeMatch(self):
assert SafeRe.match(
"((js|css)/(?!all.(js|css))|data/users/.*db|data/users/.*/.*|data/archived|.*.py)",
"js/ZeroTalk.coffee"
)
assert SafeRe.match(".+/data.json", "data/users/1J3rJ8ecnwH2EPYa6MrgZttBNc61ACFiCj/data.json")
def testUnsafeRepetition(self, pattern):
with pytest.raises(SafeRe.UnsafePatternError) as err:
SafeRe.match(pattern, "aaaaaaaaaaaaaaaaaaaaaaaa!")
assert "More than" in str(err.value)
def testUnsafeMatch(self, pattern):
with pytest.raises(SafeRe.UnsafePatternError) as err:
SafeRe.match(pattern, "aaaaaaaaaaaaaaaaaaaaaaaa!")
assert "Potentially unsafe" in str(err.value)
def actionPeerBroadcast(self, params):
ip = "%s:%s" % (self.connection.ip, self.connection.port)
raw = json.loads(params["raw"])
# Check whether P2P messages are supported
site = self.sites.get(raw["site"])
content_json = site.storage.loadJson("content.json")
if "p2p_filter" not in content_json:
self.connection.log("Site %s doesn't support P2P messages" %
raw["site"])
self.connection.badAction(5)
return
# Was the message received yet?
if params["hash"] in site.p2p_received:
return
site.p2p_received.append(params["hash"])
# Check whether the message matches passive filter
if not SafeRe.match(content_json["p2p_filter"],
json.dumps(raw["message"])):
self.connection.log("Invalid message for site %s: %s" %
(raw["site"], raw["message"]))
self.connection.badAction(5)
return
# Not so fast
if "p2p_freq_limit" in content_json and time.time(
) - site.p2p_last_recv.get(ip, 0) < content_json["p2p_freq_limit"]:
self.connection.log("Too fast messages from %s" % raw["site"])
self.connection.badAction(2)
return
site.p2p_last_recv[ip] = time.time()
# Not so much
if "p2p_size_limit" in content_json and len(json.dumps(
raw["message"])) > content_json["p2p_size_limit"]:
self.connection.log("Too big message from %s" % raw["site"])
self.connection.badAction(7)
return
# Verify signature
if params["signature"]:
signature_address, signature = params["signature"].split("|")
what = "%s|%s|%s" % (signature_address, params["hash"],
params["raw"])
from Crypt import CryptBitcoin
if not CryptBitcoin.verify(what, signature_address, signature):
self.connection.log("Invalid signature")
self.connection.badAction(7)
return
else:
signature_address = ""
# Check that the signature address is correct
if "p2p_signed_only" in content_json:
valid = content_json["p2p_signed_only"]
if valid is True and not signature_address:
self.connection.log("Not signed message")
self.connection.badAction(5)
return
elif isinstance(valid, str) and signature_address != valid:
self.connection.log(
"Message signature is invalid: %s not in [%r]" %
(signature_address, valid))
self.connection.badAction(5)
return
elif isinstance(valid, list) and signature_address not in valid:
self.connection.log(
"Message signature is invalid: %s not in %r" %
(signature_address, valid))
self.connection.badAction(5)
return
# Send to WebSocket
websockets = [
ws for ws in site.websockets if "peerReceive" in ws.channels
]
for ws in websockets:
ws.cmd(
"peerReceive", {
"ip": ip,
"hash": params["hash"],
"message": raw["message"],
"signed_by": signature_address
})
# Maybe active filter will reply?
if websockets:
# Wait for p2p_result
result = gevent.spawn(self.p2pWaitMessage, site,
params["hash"]).join()
del site.p2p_result[params["hash"]]
if not result:
self.connection.badAction(10)
return
# Save to cache
if not websockets and raw["immediate"]:
site.p2p_unread.append({
"ip":
"%s:%s" % (self.connection.ip, self.connection.port),
"hash":
params["hash"],
"message":
raw["message"],
"signed_by":
signature_address
})
# Now send to neighbour peers
if raw["broadcast"]:
# Get peer list
peers = site.getConnectedPeers()
if len(peers) < raw[
"peer_count"]: # Add more, non-connected peers if necessary
peers += site.getRecentPeers(raw["peer_count"] - len(peers))
# Send message to peers
for peer in peers:
gevent.spawn(peer.connection.request, "peerBroadcast", params)
def peerCheckMessage(self, raw, params, ip):
# Calculate hash from nonce
msg_hash = hashlib.sha256("%s,%s" % (params["nonce"], params["raw"])).hexdigest()
# Check that p2p.json exists
site = self.sites.get(raw["site"])
if not site.storage.isFile("p2p.json"):
self.connection.log("Site %s doesn't support P2P messages" % raw["site"])
self.connection.badAction(5)
self.response({
"error": "Site %s doesn't support P2P messages" % raw["site"]
})
return False, "", None, msg_hash
# Check whether P2P messages are supported
p2p_json = site.storage.loadJson("p2p.json")
if "filter" not in p2p_json:
self.connection.log("Site %s doesn't support P2P messages" % raw["site"])
self.connection.badAction(5)
self.response({
"error": "Site %s doesn't support P2P messages" % raw["site"]
})
return False, "", None, msg_hash
# Was the message received yet?
if msg_hash in site.p2p_received:
self.response({
"warning": "Already received, thanks"
})
return False, "", None, msg_hash
site.p2p_received.append(msg_hash)
# Check whether the message matches passive filter
if not SafeRe.match(p2p_json["filter"], json.dumps(raw["message"])):
self.connection.log("Invalid message for site %s: %s" % (raw["site"], raw["message"]))
self.connection.badAction(5)
self.response({
"error": "Invalid message for site %s: %s" % (raw["site"], raw["message"])
})
return False, "", None, msg_hash
# Not so fast
if "freq_limit" in p2p_json and time.time() - site.p2p_last_recv.get(ip, 0) < p2p_json["freq_limit"]:
self.connection.log("Too fast messages from %s" % raw["site"])
self.connection.badAction(2)
self.response({
"error": "Too fast messages from %s" % raw["site"]
})
return False, "", None, msg_hash
site.p2p_last_recv[ip] = time.time()
# Not so much
if "size_limit" in p2p_json and len(json.dumps(raw["message"])) > p2p_json["size_limit"]:
self.connection.log("Too big message from %s" % raw["site"])
self.connection.badAction(7)
self.response({
"error": "Too big message from %s" % raw["site"]
})
return False, "", None, msg_hash
# Verify signature
if params["signature"]:
signature_address, signature = params["signature"].split("|")
what = "%s|%s|%s" % (signature_address, msg_hash, params["raw"])
from Crypt import CryptBitcoin
if not CryptBitcoin.verify(what, signature_address, signature):
self.connection.log("Invalid signature")
self.connection.badAction(7)
self.response({
"error": "Invalid signature"
})
return False, "", None, msg_hash
# Now check auth providers
if params.get("cert"):
# Read all info
cert_auth_type, cert_auth_user_name, cert_issuer, cert_sign = params["cert"]
# This is what certificate issuer signs
cert_subject = "%s#%s/%s" % (signature_address, cert_auth_type, cert_auth_user_name)
# Now get cert issuer address
cert_signers = p2p_json.get("cert_signers", {})
cert_addresses = cert_signers.get(cert_issuer, [])
# And verify it
if not CryptBitcoin.verify(cert_subject, cert_addresses, cert_sign):
self.connection.log("Invalid signature certificate")
self.connection.badAction(7)
self.response({
"error": "Invalid signature certificate"
})
return False, "", None, msg_hash
# And save the ID
cert = "%s/%s@%s" % (cert_auth_type, cert_auth_user_name, cert_issuer)
else:
# Old-style sign
cert = ""
else:
signature_address = ""
cert = ""
# Check that the signature address is correct
if "signed_only" in p2p_json:
valid = p2p_json["signed_only"]
if valid is True and not signature_address:
self.connection.log("Not signed message")
self.connection.badAction(5)
self.response({
"error": "Not signed message"
})
return False, "", None, msg_hash
elif isinstance(valid, str) and signature_address != valid:
self.connection.log("Message signature is invalid: %s not in [%r]" % (signature_address, valid))
self.connection.badAction(5)
self.response({
"error": "Message signature is invalid: %s not in [%r]" % (signature_address, valid)
})
return False, "", None, msg_hash
elif isinstance(valid, list) and signature_address not in valid:
self.connection.log("Message signature is invalid: %s not in %r" % (signature_address, valid))
self.connection.badAction(5)
self.response({
"error": "Message signature is invalid: %s not in %r" % (signature_address, valid)
})
return False, "", None, msg_hash
return True, signature_address, cert, msg_hash
def actionPeerBroadcast(self,
to,
message,
privatekey=None,
peer_count=5,
broadcast=True,
immediate=False,
timeout=60):
# Check whether P2P messages are supported
content_json = self.site.storage.loadJson("content.json")
if "p2p_filter" not in content_json:
self.response(
to, {
"error":
"Site %s doesn't support P2P messages" % self.site.address
})
return
# Check whether the message matches passive filter
if not SafeRe.match(content_json["p2p_filter"], json.dumps(message)):
self.response(
to, {
"error":
"Invalid message for site %s: %s" %
(self.site.address, message)
})
return
# Not so fast
if "p2p_freq_limit" in content_json and time.time(
) - self.site.p2p_last_recv.get("self",
0) < content_json["p2p_freq_limit"]:
self.response(to, {"error": "Too fast messages"})
return
self.site.p2p_last_recv["self"] = time.time()
# Not so much
if "p2p_size_limit" in content_json and len(
json.dumps(message)) > content_json["p2p_size_limit"]:
self.response(to, {"error": "Too big message"})
return
# Generate message and sign it
all_message = {
"message": message,
"peer_count": peer_count,
"broadcast": broadcast,
"immediate": immediate,
"site": self.site.address
}
all_message = json.dumps(all_message)
nonce = str(random.randint(0, 1000000000))
msg_hash = hashlib.md5("%s,%s" % (nonce, all_message)).hexdigest()
signature = self.p2pGetSignature(msg_hash, all_message, privatekey)
all_message = {
"raw": all_message,
"signature": signature,
"hash": msg_hash
}
peers = self.site.getConnectedPeers()
if len(peers
) < peer_count: # Add more, non-connected peers if necessary
peers += self.site.getRecentPeers(peer_count - len(peers))
# Send message to peers
jobs = []
for peer in peers:
jobs.append(gevent.spawn(self.p2pBroadcast, peer, all_message))
if not broadcast:
# Makes sense to return result
res = gevent.joinall(jobs, timeout)
self.response(to, res)
return
# Reply
self.response(to, {"sent": True})
# Send message to myself
self.site.p2p_received.append(msg_hash)
websockets = [
ws for ws in self.site.websockets if "peerReceive" in ws.channels
]
for ws in websockets:
ws.cmd(
"peerReceive", {
"ip":
"self",
"hash":
msg_hash,
"message":
message,
"signed_by":
all_message["signature"].split("|")[0]
if all_message["signature"] else ""
})
if not websockets and immediate:
self.site.p2p_unread.append({
"ip":
"self",
"hash":
msg_hash,
"message":
message,
"signed_by":
all_message["signature"].split("|")[0]
if all_message["signature"] else ""
})
