def testVerifyAddress(self, site): privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv" # For 1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT user_inner_path = "data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/content.json" data_dict = site.storage.loadJson(user_inner_path) users_content = site.content_manager.contents["data/users/content.json"] data = StringIO(json.dumps(data_dict)) assert site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) # Test error on 15k data.json data_dict["files"]["data.json"]["size"] = 1024 * 15 del data_dict["signs"] # Remove signs before signing data_dict["signs"] = { "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) } data = StringIO(json.dumps(data_dict)) with pytest.raises(VerifyError) as err: site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) assert "Content too large" in str(err) # Give more space based on address users_content["user_contents"]["permissions"]["1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9"] = {"max_size": 20000} del data_dict["signs"] # Remove signs before signing data_dict["signs"] = { "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) } data = StringIO(json.dumps(data_dict)) assert site.content_manager.verifyFile(user_inner_path, data, ignore_same=False)
def siteCreate(self): logging.info("Generating new privatekey...") from Crypt import CryptBitcoin privatekey = CryptBitcoin.newPrivatekey() logging.info("----------------------------------------------------------------------") logging.info("Site private key: %s" % privatekey) logging.info(" !!! ^ Save it now, required to modify the site ^ !!!") address = CryptBitcoin.privatekeyToAddress(privatekey) logging.info("Site address: %s" % address) logging.info("----------------------------------------------------------------------") while True and not config.batch: if raw_input("? Have you secured your private key? (yes, no) > ").lower() == "yes": break else: logging.info("Please, secure it now, you going to need it to modify your site!") logging.info("Creating directory structure...") from Site import Site from Site import SiteManager SiteManager.site_manager.load() os.mkdir("%s/%s" % (config.data_dir, address)) open("%s/%s/index.html" % (config.data_dir, address), "w").write("Hello %s!" % address) logging.info("Creating content.json...") site = Site(address) site.content_manager.sign(privatekey=privatekey, extend={"postmessage_nonce_security": True}) site.settings["own"] = True site.saveSettings() logging.info("Site created!")
def testVerifyInnerPath(self, site): inner_path = "content.json" data_dict = site.storage.loadJson(inner_path) for good_relative_path in ["data.json", "out/data.json", "Any File [by none] (1).jpg"]: data_dict["files"] = {good_relative_path: {"sha512": "369d4e780cc80504285f13774ca327fe725eed2d813aad229e62356b07365906", "size": 505}} if "sign" in data_dict: del data_dict["sign"] del data_dict["signs"] data_dict["signs"] = { "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), self.privatekey) } data = StringIO(json.dumps(data_dict)) assert site.content_manager.verifyFile(inner_path, data, ignore_same=False) for bad_relative_path in ["../data.json", "data/" * 100, "invalid|file.jpg"]: data_dict["files"] = {bad_relative_path: {"sha512": "369d4e780cc80504285f13774ca327fe725eed2d813aad229e62356b07365906", "size": 505}} if "sign" in data_dict: del data_dict["sign"] del data_dict["signs"] data_dict["signs"] = { "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), self.privatekey) } data = StringIO(json.dumps(data_dict)) with pytest.raises(VerifyError) as err: site.content_manager.verifyFile(inner_path, data, ignore_same=False) assert "Invalid relative path" in str(err)
def siteCreate(): logging.info("Generating new privatekey...") from Crypt import CryptBitcoin privatekey = CryptBitcoin.newPrivatekey() logging.info("----------------------------------------------------------------------") logging.info("Site private key: %s" % privatekey) logging.info(" !!! ^ Save it now, required to modify the site ^ !!!") address = CryptBitcoin.privatekeyToAddress(privatekey) logging.info("Site address: %s" % address) logging.info("----------------------------------------------------------------------") while True: if raw_input("? Have you secured your private key? (yes, no) > ").lower() == "yes": break else: logging.info("Please, secure it now, you going to need it to modify your site!") logging.info("Creating directory structure...") from Site import Site os.mkdir("data/%s" % address) open("data/%s/index.html" % address, "w").write("Hello %s!" % address) logging.info("Creating content.json...") site = Site(address) site.content_manager.sign(privatekey=privatekey) site.settings["own"] = True site.saveSettings() logging.info("Site created!")
def create_phantom_site(self): from Config import config from Crypt import CryptBitcoin from Site import Site import os config.parse(silent=True) if not config.arguments: config.parse() self.private_key = CryptBitcoin.newPrivatekey() self.address = CryptBitcoin.privatekeyToAddress(self.private_key) try: os.mkdir("%s/%s" % (config.data_dir, self.address)) open("%s/%s/index.html" % (config.data_dir, self.address), "w").write("Hello %s!" % self.address) except Exception as e: return self.error_msg.error_response("err_create_sitedir") try: self.site = Site(self.address) self.site.content_manager.sign(privatekey=self.private_key, extend={"postmessage_nonce_security": True}) self.site.settings["own"] = True self.site.saveSettings() except Exception as e: print e return self.error_msg.error_response("err_create_site") return {"jsonrpc": "2.0", "id": "1", "result": ["true", str(self.address), str(self.private_key)]}
def cryptPrivatekeyToAddress(self, privatekey=None): from Crypt import CryptBitcoin if not privatekey: # If no privatekey in args then ask it now import getpass privatekey = getpass.getpass("Private key (input hidden):") print CryptBitcoin.privatekeyToAddress(privatekey)
def testVerify(self, site): privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv" # For 1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT user_inner_path = "data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/content.json" data_dict = site.content_manager.contents[user_inner_path] users_content = site.content_manager.contents["data/users/content.json"] data = StringIO(json.dumps(data_dict)) assert site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) # Test max size exception by setting allowed to 0 rules = site.content_manager.getRules(user_inner_path, data_dict) assert rules["max_size"] == 10000 assert users_content["user_contents"]["permission_rules"][".*"]["max_size"] == 10000 users_content["user_contents"]["permission_rules"][".*"]["max_size"] = 0 rules = site.content_manager.getRules(user_inner_path, data_dict) assert rules["max_size"] == 0 data = StringIO(json.dumps(data_dict)) assert not site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) users_content["user_contents"]["permission_rules"][".*"]["max_size"] = 10000 # Reset # Test max optional size exception # 1 MB gif = Allowed data_dict["files_optional"]["peanut-butter-jelly-time.gif"]["size"] = 1024 * 1024 del data_dict["signs"] # Remove signs before signing data_dict["signs"] = { "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) } data = StringIO(json.dumps(data_dict)) assert site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) # 100 MB gif = Not allowed data_dict["files_optional"]["peanut-butter-jelly-time.gif"]["size"] = 100 * 1024 * 1024 del data_dict["signs"] # Remove signs before signing data_dict["signs"] = { "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) } data = StringIO(json.dumps(data_dict)) assert not site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) data_dict["files_optional"]["peanut-butter-jelly-time.gif"]["size"] = 1024 * 1024 # Reset # hello.exe = Not allowed data_dict["files_optional"]["hello.exe"] = data_dict["files_optional"]["peanut-butter-jelly-time.gif"] del data_dict["signs"] # Remove signs before signing data_dict["signs"] = { "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) } data = StringIO(json.dumps(data_dict)) assert not site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) del data_dict["files_optional"]["hello.exe"] # Reset # Includes not allowed in user content data_dict["includes"] = { "other.json": { } } del data_dict["signs"] # Remove signs before signing data_dict["signs"] = { "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) } data = StringIO(json.dumps(data_dict)) assert not site.content_manager.verifyFile(user_inner_path, data, ignore_same=False)
def testUserContentCert(self): from Site import Site from cStringIO import StringIO import json # user_addr = "1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C" user_priv = "5Kk7FSA63FC2ViKmKLuBxk9gQkaQ5713hKq8LmFAf4cVeXh6K6A" # cert_addr = "14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet" cert_priv = "5JusJDSjHaMHwUjDT3o6eQ54pA6poo8La5fAgn1wNc3iK59jxjA" site = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT") # user_content = site.storage.loadJson("data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json") # site.content_manager.contents["data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json"] = user_content # Add to content manager # Check if the user file is loaded self.assertTrue("data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json" in site.content_manager.contents) user_content = site.content_manager.contents["data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json"] cert_content = site.content_manager.contents["data/users/content.json"] # Override cert signer cert_content["user_contents"]["cert_signers"]["zeroid.bit"] = ["14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet", "1iD5ZQJMNXu43w1qLB8sfdHVKppVMduGz"] # Valid cert providers rules = site.content_manager.getRules("data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_content) self.assertEqual(rules["cert_signers"], {"zeroid.bit": ["14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet", "1iD5ZQJMNXu43w1qLB8sfdHVKppVMduGz"]}) # Add cert user_content["cert_sign"] = CryptBitcoin.sign( "1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C#%s/%s" % (user_content["cert_auth_type"], user_content["cert_user_id"].split("@")[0]), cert_priv ) # Verify cert self.assertTrue(site.content_manager.verifyCert("data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_content)) self.assertFalse(site.content_manager.verifyCert("data/users/badaddress/content.json", user_content)) # Sign user content signed_content = site.content_manager.sign("data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_priv, filewrite=False) # Test user cert self.assertTrue(site.content_manager.verifyFile( "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", StringIO(json.dumps(signed_content)), ignore_same=False )) # Test banned user site.content_manager.contents["data/users/content.json"]["user_contents"]["permissions"][user_content["cert_user_id"]] = False self.assertFalse(site.content_manager.verifyFile( "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", StringIO(json.dumps(signed_content)), ignore_same=False )) # Test invalid cert user_content["cert_sign"] = CryptBitcoin.sign( "badaddress#%s/%s" % (user_content["cert_auth_type"], user_content["cert_user_id"]), cert_priv ) signed_content = site.content_manager.sign("data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_priv, filewrite=False) self.assertFalse(site.content_manager.verifyFile( "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", StringIO(json.dumps(signed_content)), ignore_same=False ))
def verifyFile(self, inner_path, file, ignore_same=True): if inner_path.endswith("content.json"): # content.json: Check using sign from Crypt import CryptBitcoin try: new_content = json.load(file) if inner_path in self.contents: old_content = self.contents.get(inner_path) # Checks if its newer the ours if old_content["modified"] == new_content["modified"] and ignore_same: # Ignore, have the same content.json return None elif old_content["modified"] > new_content["modified"]: # We have newer self.log.debug("We have newer %s (Our: %s, Sent: %s)" % (inner_path, old_content["modified"], new_content["modified"])) gevent.spawn(self.site.publish, inner_path=inner_path) # Try to fix the broken peers return False if new_content["modified"] > time.time() + 60 * 60 * 24: # Content modified in the far future (allow 1 day window) self.log.error("%s modify is in the future!" % inner_path) return False # Check sign sign = new_content.get("sign") signs = new_content.get("signs", {}) if "sign" in new_content: del(new_content["sign"]) # The file signed without the sign if "signs" in new_content: del(new_content["signs"]) # The file signed without the signs sign_content = json.dumps(new_content, sort_keys=True) # Dump the json to string to remove whitepsace if not self.validContent(inner_path, new_content): return False # Content not valid (files too large, invalid files) if signs: # New style signing valid_signers = self.getValidSigners(inner_path, new_content) signs_required = self.getSignsRequired(inner_path, new_content) if inner_path == "content.json" and len(valid_signers) > 1: # Check signers_sign on root content.json if not CryptBitcoin.verify("%s:%s" % (signs_required, ",".join(valid_signers)), self.site.address, new_content["signers_sign"]): self.log.error("%s invalid signers_sign!" % inner_path) return False if inner_path != "content.json" and not self.verifyCert(inner_path, new_content): # Check if cert valid self.log.error("%s invalid cert!" % inner_path) return False valid_signs = 0 for address in valid_signers: if address in signs: valid_signs += CryptBitcoin.verify(sign_content, address, signs[address]) if valid_signs >= signs_required: break # Break if we has enough signs return valid_signs >= signs_required else: # Old style signing return CryptBitcoin.verify(sign_content, self.site.address, sign) except Exception, err: self.log.error("Verify sign error: %s" % Debug.formatException(err)) return False
def generateAuthAddress(self, address): s = time.time() address_id = self.getAddressAuthIndex(address) # Convert site address to int auth_privatekey = CryptBitcoin.hdPrivatekey(self.master_seed, address_id) self.sites[address] = { "auth_address": CryptBitcoin.privatekeyToAddress(auth_privatekey), "auth_privatekey": auth_privatekey } self.saveDelayed() self.log.debug("Added new site: %s in %.3fs" % (address, time.time() - s)) return self.sites[address]
def getNewSiteData(self): import random bip32_index = random.randrange(2**256) % 100000000 site_privatekey = CryptBitcoin.hdPrivatekey(self.master_seed, bip32_index) site_address = CryptBitcoin.privatekeyToAddress(site_privatekey) if site_address in self.sites: raise Exception("Random error: site exist!") # Save to sites self.getSiteData(site_address) self.sites[site_address]["privatekey"] = site_privatekey self.save() return site_address, bip32_index, self.sites[site_address]
def __init__(self, master_address=None, master_seed=None): if master_seed: self.master_seed = master_seed self.master_address = CryptBitcoin.privatekeyToAddress(self.master_seed) elif master_address: self.master_address = master_address self.master_seed = None else: self.master_seed = CryptBitcoin.newSeed() self.master_address = CryptBitcoin.privatekeyToAddress(self.master_seed) self.sites = {} self.log = logging.getLogger("User:%s" % self.master_address)
def getSiteData(self, address, create=True): if not address in self.sites: # Genreate new BIP32 child key based on site address if not create: return {"auth_address": None, "auth_privatekey": None} # Dont create user yet s = time.time() address_id = int(address.encode("hex"), 16) # Convert site address to int auth_privatekey = CryptBitcoin.hdPrivatekey(self.master_seed, address_id) self.sites[address] = { "auth_address": CryptBitcoin.privatekeyToAddress(auth_privatekey), "auth_privatekey": auth_privatekey } self.save() self.log.debug("Added new site: %s in %.3fs" % (address, time.time()-s)) return self.sites[address]
def __init__(self, master_address=None, master_seed=None, data={}): if master_seed: self.master_seed = master_seed self.master_address = CryptBitcoin.privatekeyToAddress(self.master_seed) elif master_address: self.master_address = master_address self.master_seed = data.get("master_seed") else: self.master_seed = CryptBitcoin.newSeed() self.master_address = CryptBitcoin.privatekeyToAddress(self.master_seed) self.sites = data.get("sites", {}) self.certs = data.get("certs", {}) self.log = logging.getLogger("User:%s" % self.master_address)
def testNewSeed(self): assert CryptBitcoin.newSeed() != CryptBitcoin.newSeed() assert CryptBitcoin.privatekeyToAddress( CryptBitcoin.hdPrivatekey(CryptBitcoin.newSeed(), 0) ) assert CryptBitcoin.privatekeyToAddress( CryptBitcoin.hdPrivatekey(CryptBitcoin.newSeed(), 2**256) )
def testInlcudeLimits(self, site): privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv" # Data validation data_dict = { "files": { "data.json": { "sha512": "369d4e780cc80504285f13774ca327fe725eed2d813aad229e62356b07365906", "size": 505 } }, "modified": time.time() } # Normal data data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey)} data = StringIO(json.dumps(data_dict)) assert site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False) # Reset del data_dict["signs"] # Too large data_dict["files"]["data.json"]["size"] = 200000 # Emulate 2MB sized data.json data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey)} data = StringIO(json.dumps(data_dict)) with pytest.raises(VerifyError) as err: site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False) assert "Include too large" in str(err) # Reset data_dict["files"]["data.json"]["size"] = 505 del data_dict["signs"] # Not allowed file data_dict["files"]["notallowed.exe"] = data_dict["files"]["data.json"] data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey)} data = StringIO(json.dumps(data_dict)) with pytest.raises(VerifyError) as err: site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False) assert "File not allowed" in str(err) # Reset del data_dict["files"]["notallowed.exe"] del data_dict["signs"] # Should work again data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey)} data = StringIO(json.dumps(data_dict)) assert site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False)
def verifyCert(self, inner_path, content): from Crypt import CryptBitcoin rules = self.getRules(inner_path, content) if not rules: raise VerifyError("No rules for this file") if not rules.get("cert_signers") and not rules.get("cert_signers_pattern"): return True # Does not need cert if "cert_user_id" not in content: raise VerifyError("Missing cert_user_id") if content["cert_user_id"].count("@") != 1: raise VerifyError("Invalid domain in cert_user_id") name, domain = content["cert_user_id"].rsplit("@", 1) cert_address = rules["cert_signers"].get(domain) if not cert_address: # Unknown Cert signer if rules.get("cert_signers_pattern") and SafeRe.match(rules["cert_signers_pattern"], domain): cert_address = domain else: raise VerifyError("Invalid cert signer: %s" % domain) try: cert_subject = "%s#%s/%s" % (rules["user_address"], content["cert_auth_type"], name) result = CryptBitcoin.verify(cert_subject, cert_address, content["cert_sign"]) except Exception, err: raise VerifyError("Certificate verify error: %s" % err)
def testVerify(self, site): privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv" inner_path = "data/test_include/content.json" data_dict = site.storage.loadJson(inner_path) data = StringIO(json.dumps(data_dict)) # Re-sign data_dict["signs"] = { "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) } assert site.content_manager.verifyFile(inner_path, data, ignore_same=False) # Wrong address data_dict["address"] = "Othersite" del data_dict["signs"] data_dict["signs"] = { "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) } data = StringIO(json.dumps(data_dict)) with pytest.raises(VerifyError) as err: site.content_manager.verifyFile(inner_path, data, ignore_same=False) assert "Wrong site address" in str(err) # Wrong inner_path data_dict["address"] = "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT" data_dict["inner_path"] = "content.json" del data_dict["signs"] data_dict["signs"] = { "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) } data = StringIO(json.dumps(data_dict)) with pytest.raises(VerifyError) as err: site.content_manager.verifyFile(inner_path, data, ignore_same=False) assert "Wrong inner_path" in str(err) # Everything right again data_dict["address"] = "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT" data_dict["inner_path"] = inner_path del data_dict["signs"] data_dict["signs"] = { "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) } data = StringIO(json.dumps(data_dict)) assert site.content_manager.verifyFile(inner_path, data, ignore_same=False)
def testNewSite(self, user): address, address_index, site_data = user.getNewSiteData() # Create a new random site assert CryptBitcoin.hdPrivatekey(user.master_seed, address_index) == site_data["privatekey"] user.sites = {} # Reset user data # Site address and auth address is different assert user.getSiteData(address)["auth_address"] != address # Re-generate auth_privatekey for site assert user.getSiteData(address)["auth_privatekey"] == site_data["auth_privatekey"]
def verifyCert(self, inner_path, content): from Crypt import CryptBitcoin rules = self.getRules(inner_path, content) if not rules.get("cert_signers"): return True # Does not need cert name, domain = content["cert_user_id"].split("@") cert_address = rules["cert_signers"].get(domain) if not cert_address: # Cert signer not allowed self.log.error("Invalid cert signer: %s" % domain) return False return CryptBitcoin.verify("%s#%s/%s" % (rules["user_address"], content["cert_auth_type"], name), cert_address, content["cert_sign"])
def testMissingCert(self, site): user_priv = "5Kk7FSA63FC2ViKmKLuBxk9gQkaQ5713hKq8LmFAf4cVeXh6K6A" cert_priv = "5JusJDSjHaMHwUjDT3o6eQ54pA6poo8La5fAgn1wNc3iK59jxjA" user_content = site.content_manager.contents["data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json"] rules_content = site.content_manager.contents["data/users/content.json"] # Override valid cert signers for the test rules_content["user_contents"]["cert_signers"]["zeroid.bit"] = [ "14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet", "1iD5ZQJMNXu43w1qLB8sfdHVKppVMduGz" ] # Sign a valid cert user_content["cert_sign"] = CryptBitcoin.sign("1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C#%s/%s" % ( user_content["cert_auth_type"], user_content["cert_user_id"].split("@")[0] ), cert_priv) signed_content = site.content_manager.sign( "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_priv, filewrite=False ) assert site.content_manager.verifyFile( "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", StringIO(json.dumps(signed_content)), ignore_same=False ) # Test removed cert del user_content["cert_user_id"] del user_content["cert_auth_type"] del user_content["signs"] # Remove signs before signing user_content["signs"] = { "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(user_content, sort_keys=True), user_priv) } with pytest.raises(VerifyError) as err: site.content_manager.verifyFile( "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", StringIO(json.dumps(user_content)), ignore_same=False ) assert "Missing cert_user_id" in str(err)
def responseUserLogin(self, master_seed): user_manager = sys.modules["User.UserManager"].user_manager user = user_manager.get(CryptBitcoin.privatekeyToAddress(master_seed)) if not user: user = user_manager.create(master_seed=master_seed) if user.master_address: message = "Successfull login, reloading page..." message += "<script>document.cookie = 'master_address=%s;path=/;max-age=2592000;'</script>" % user.master_address message += "<script>wrapper.reload('login=done')</script>" self.cmd("notification", ["done", message]) else: self.cmd("notification", ["error", "Error: Invalid master seed"]) self.actionUserLoginForm(0)
def testUserNewsite(self): from User import UserManager user = UserManager.user_manager.get() user.sites = {} # Reset user data self.assertEqual(user.master_address, "15E5rhcAUD69WbiYsYARh4YHJ4sLm2JEyc") self.assertEqual(user.getAddressAuthIndex("15E5rhcAUD69WbiYsYARh4YHJ4sLm2JEyc"), 1458664252141532163166741013621928587528255888800826689784628722366466547364755811L) address, address_index, site_data = user.getNewSiteData() self.assertEqual(CryptBitcoin.hdPrivatekey(user.master_seed, address_index), site_data["privatekey"]) # Re-generate privatekey based on address_index user.sites = {} # Reset user data self.assertNotEqual(user.getSiteData(address)["auth_address"], address) # Site address and auth address is different self.assertEqual(user.getSiteData(address)["auth_privatekey"], site_data["auth_privatekey"]) # Re-generate auth_privatekey for site
def getEncryptPrivatekey(self, address, param_index=0): assert param_index >= 0 and param_index <= 1000 site_data = self.getSiteData(address) if site_data.get("cert"): # Different privatekey for different cert provider index = param_index + self.getAddressAuthIndex(site_data["cert"]) else: index = param_index if "encrypt_privatekey_%s" % index not in site_data: address_index = self.getAddressAuthIndex(address) crypt_index = address_index + 1000 + index site_data["encrypt_privatekey_%s" % index] = CryptBitcoin.hdPrivatekey(self.master_seed, crypt_index) self.log.debug("New encrypt privatekey generated for %s:%s" % (address, index)) return site_data["encrypt_privatekey_%s" % index]
def testCert(self, user): cert_auth_address = user.getAuthAddress("1iD5ZQJMNXu43w1qLB8sfdHVKppVMduGz") # Add site to user's registry # Add cert user.addCert(cert_auth_address, "zeroid.bit", "faketype", "fakeuser", "fakesign") user.setCert("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr", "zeroid.bit") # By using certificate the auth address should be same as the certificate provider assert user.getAuthAddress("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr") == cert_auth_address auth_privatekey = user.getAuthPrivatekey("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr") assert CryptBitcoin.privatekeyToAddress(auth_privatekey) == cert_auth_address # Test delete site data assert "1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr" in user.sites user.deleteSiteData("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr") assert "1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr" not in user.sites # Re-create add site should generate normal, unique auth_address assert not user.getAuthAddress("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr") == cert_auth_address assert user.getAuthAddress("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr") == "1MyJgYQjeEkR9QD66nkfJc9zqi9uUy5Lr2"
def verifyCert(self, inner_path, content): from Crypt import CryptBitcoin rules = self.getRules(inner_path, content) if not rules.get("cert_signers"): return True # Does not need cert name, domain = content["cert_user_id"].split("@") cert_address = rules["cert_signers"].get(domain) if not cert_address: # Cert signer not allowed self.log.warning("Invalid cert signer: %s" % domain) return False try: cert_subject = "%s#%s/%s" % (rules["user_address"], content["cert_auth_type"], name) result = CryptBitcoin.verify(cert_subject, cert_address, content["cert_sign"]) except Exception, err: self.log.warning("Certificate verify error: %s" % err) result = False
def testSignOld(self): privatekey = "23DKQpDz7bXM7w5KN5Wnmz7bwRNqNHcdQjb2WwrdB1QtTf5gM3pFdf" privatekey_bad = "23DKQpDz7bXM7w5KN5Wnmz6bwRNqNHcdQjb2WwrdB1QtTf5gM3pFdf" # Get address by privatekey address = CryptBitcoin.privatekeyToAddress(privatekey) assert address == "12vTsjscg4hYPewUL2onma5pgQmWPMs3ez" address_bad = CryptBitcoin.privatekeyToAddress(privatekey_bad) assert not address_bad == "12vTsjscg4hYPewUL2onma5pgQmWPMs3ez" # Text signing sign = CryptBitcoin.signOld("hello", privatekey) assert CryptBitcoin.verify("hello", address, sign) # Original text assert not CryptBitcoin.verify("not hello", address, sign) # Different text # Signed by bad privatekey sign_bad = CryptBitcoin.signOld("hello", privatekey_bad) assert not CryptBitcoin.verify("hello", address, sign_bad)
def testBitcoinSign(self): s = time.time() privatekey = "5K9S6dVpufGnroRgFrT6wsKiz2mJRYsC73eWDmajaHserAp3F1C" privatekey_bad = "5Jbm9rrusXyApAoM8YoM4Rja337zMMoBUMRJ1uijiguU2aZRnwC" address = CryptBitcoin.privatekeyToAddress(privatekey) self.assertEqual(address, "1MpDMxFeDUkiHohxx9tbGLeEGEuR4ZNsJz") address_bad = CryptBitcoin.privatekeyToAddress(privatekey_bad) self.assertNotEqual(address_bad, "1MpDMxFeDUkiHohxx9tbGLeEGEuR4ZNsJz") sign = CryptBitcoin.sign("hello", privatekey) self.assertTrue(CryptBitcoin.verify("hello", address, sign)) self.assertFalse(CryptBitcoin.verify("not hello", address, sign)) sign_bad = CryptBitcoin.sign("hello", privatekey_bad) self.assertFalse(CryptBitcoin.verify("hello", address, sign_bad)) print "Taken: %.3fs, " % (time.time()-s),
def testBitcoinSignOld(self): s = time.time() privatekey = "23DKQpDz7bXM7w5KN5Wnmz7bwRNqNHcdQjb2WwrdB1QtTf5gM3pFdf" privatekey_bad = "23DKQpDz7bXM7w5KN5Wnmz6bwRNqNHcdQjb2WwrdB1QtTf5gM3pFdf" address = CryptBitcoin.privatekeyToAddress(privatekey) self.assertEqual(address, "12vTsjscg4hYPewUL2onma5pgQmWPMs3ez") address_bad = CryptBitcoin.privatekeyToAddress(privatekey_bad) self.assertNotEqual(address_bad, "12vTsjscg4hYPewUL2onma5pgQmWPMs3ez") sign = CryptBitcoin.signOld("hello", privatekey) self.assertTrue(CryptBitcoin.verify("hello", address, sign)) self.assertFalse(CryptBitcoin.verify("not hello", address, sign)) sign_bad = CryptBitcoin.signOld("hello", privatekey_bad) self.assertFalse(CryptBitcoin.verify("hello", address, sign_bad)) print "Taken: %.3fs, " % (time.time()-s),
def testSign(self): privatekey = "5K9S6dVpufGnroRgFrT6wsKiz2mJRYsC73eWDmajaHserAp3F1C" privatekey_bad = "5Jbm9rrusXyApAoM8YoM4Rja337zMMoBUMRJ1uijiguU2aZRnwC" # Get address by privatekey address = CryptBitcoin.privatekeyToAddress(privatekey) assert address == "1MpDMxFeDUkiHohxx9tbGLeEGEuR4ZNsJz" address_bad = CryptBitcoin.privatekeyToAddress(privatekey_bad) assert address_bad != "1MpDMxFeDUkiHohxx9tbGLeEGEuR4ZNsJz" # Text signing sign = CryptBitcoin.sign("hello", privatekey) assert CryptBitcoin.verify("hello", address, sign) assert not CryptBitcoin.verify("not hello", address, sign) # Signed by bad privatekey sign_bad = CryptBitcoin.sign("hello", privatekey_bad) assert not CryptBitcoin.verify("hello", address, sign_bad)
class UiRequestPlugin(object): def formatTableRow(self, row): back = [] for format, val in row: if val is None: formatted = "n/a" elif format == "since": if val: formatted = "%.0f" % (time.time() - val) else: formatted = "n/a" else: formatted = format % val back.append("<td>%s</td>" % formatted) return "<tr>%s</tr>" % "".join(back) def getObjSize(self, obj, hpy=None): if hpy: return float(hpy.iso(obj).domisize) / 1024 else: return 0 # /Stats entry point def actionStats(self): import gc import sys from Ui import UiRequest from Crypt import CryptConnection hpy = None if self.get.get("size") == "1": # Calc obj size try: import guppy hpy = guppy.hpy() except: pass self.sendHeader() s = time.time() main = sys.modules["main"] # Style yield """ <style> * { font-family: monospace } table td, table th { text-align: right; padding: 0px 10px } </style> """ # Memory try: yield "rev%s | " % config.rev yield "%s | " % config.ip_external yield "Opened: %s | " % main.file_server.port_opened yield "Crypt: %s | " % CryptConnection.manager.crypt_supported yield "In: %.2fMB, Out: %.2fMB | " % ( float(main.file_server.bytes_recv) / 1024 / 1024, float(main.file_server.bytes_sent) / 1024 / 1024 ) yield "Peerid: %s | " % main.file_server.peer_id import psutil process = psutil.Process(os.getpid()) mem = process.get_memory_info()[0] / float(2 ** 20) yield "Mem: %.2fMB | " % mem yield "Threads: %s | " % len(process.threads()) yield "CPU: usr %.2fs sys %.2fs | " % process.cpu_times() yield "Files: %s | " % len(process.open_files()) yield "Sockets: %s | " % len(process.connections()) yield "Calc size <a href='?size=1'>on</a> <a href='?size=0'>off</a>" except Exception: pass yield "<br>" # Connections yield "<b>Connections</b> (%s, total made: %s):<br>" % ( len(main.file_server.connections), main.file_server.last_connection_id ) yield "<table><tr> <th>id</th> <th>proto</th> <th>type</th> <th>ip</th> <th>open</th> <th>crypt</th> <th>ping</th>" yield "<th>buff</th> <th>idle</th> <th>open</th> <th>delay</th> <th>out</th> <th>in</th> <th>last sent</th>" yield "<th>waiting</th> <th>version</th> <th>peerid</th> </tr>" for connection in main.file_server.connections: if "cipher" in dir(connection.sock): cipher = connection.sock.cipher()[0] else: cipher = connection.crypt yield self.formatTableRow([ ("%3d", connection.id), ("%s", connection.protocol), ("%s", connection.type), ("%s:%s", (connection.ip, connection.port)), ("%s", connection.handshake.get("port_opened")), ("<span title='%s'>%s</span>", (connection.crypt, cipher)), ("%6.3f", connection.last_ping_delay), ("%s", connection.incomplete_buff_recv), ("since", max(connection.last_send_time, connection.last_recv_time)), ("since", connection.start_time), ("%.3f", connection.last_sent_time - connection.last_send_time), ("%.0fkB", connection.bytes_sent / 1024), ("%.0fkB", connection.bytes_recv / 1024), ("%s", connection.last_cmd), ("%s", connection.waiting_requests.keys()), ("%s r%s", (connection.handshake.get("version"), connection.handshake.get("rev", "?"))), ("%s", connection.handshake.get("peer_id")), ]) yield "</table>" # Sites yield "<br><br><b>Sites</b>:" yield "<table>" yield "<tr><th>address</th> <th>connected</th> <th title='connected/good/total'>peers</th> <th>content.json</th> <th>out</th> <th>in</th> </tr>" for site in self.server.sites.values(): yield self.formatTableRow([ ( """<a href='#' onclick='document.getElementById("peers_%s").style.display="initial"; return false'>%s</a>""", (site.address, site.address) ), ("%s", [peer.connection.id for peer in site.peers.values() if peer.connection and peer.connection.connected]), ("%s/%s/%s", ( len([peer for peer in site.peers.values() if peer.connection and peer.connection.connected]), len(site.getConnectablePeers(100)), len(site.peers) )), ("%s", len(site.content_manager.contents)), ("%.0fkB", site.settings.get("bytes_sent", 0) / 1024), ("%.0fkB", site.settings.get("bytes_recv", 0) / 1024), ]) yield "<tr><td id='peers_%s' style='display: none; white-space: pre' colspan=2>" % site.address for key, peer in site.peers.items(): if peer.time_found: time_found = int(time.time()-peer.time_found)/60 else: time_found = "--" if peer.connection: connection_id = peer.connection.id else: connection_id = None if site.content_manager.hashfield: yield "Optional files: %4s " % len(peer.hashfield) yield "(#%4s, err: %s, found: %5s min ago) %22s -<br>" % (connection_id, peer.connection_error, time_found, key) yield "<br></td></tr>" yield "</table>" # No more if not in debug mode if not config.debug: raise StopIteration # Object types obj_count = {} for obj in gc.get_objects(): obj_type = str(type(obj)) if obj_type not in obj_count: obj_count[obj_type] = [0, 0] obj_count[obj_type][0] += 1 # Count obj_count[obj_type][1] += float(sys.getsizeof(obj)) / 1024 # Size yield "<br><br><b>Objects in memory (types: %s, total: %s, %.2fkb):</b><br>" % ( len(obj_count), sum([stat[0] for stat in obj_count.values()]), sum([stat[1] for stat in obj_count.values()]) ) for obj, stat in sorted(obj_count.items(), key=lambda x: x[1][0], reverse=True): # Sorted by count yield " - %.1fkb = %s x <a href=\"/Listobj?type=%s\">%s</a><br>" % (stat[1], stat[0], obj, cgi.escape(obj)) # Classes class_count = {} for obj in gc.get_objects(): obj_type = str(type(obj)) if obj_type != "<type 'instance'>": continue class_name = obj.__class__.__name__ if class_name not in class_count: class_count[class_name] = [0, 0] class_count[class_name][0] += 1 # Count class_count[class_name][1] += float(sys.getsizeof(obj)) / 1024 # Size yield "<br><br><b>Classes in memory (types: %s, total: %s, %.2fkb):</b><br>" % ( len(class_count), sum([stat[0] for stat in class_count.values()]), sum([stat[1] for stat in class_count.values()]) ) for obj, stat in sorted(class_count.items(), key=lambda x: x[1][0], reverse=True): # Sorted by count yield " - %.1fkb = %s x <a href=\"/Dumpobj?class=%s\">%s</a><br>" % (stat[1], stat[0], obj, cgi.escape(obj)) from greenlet import greenlet objs = [obj for obj in gc.get_objects() if isinstance(obj, greenlet)] yield "<br>Greenlets (%s):<br>" % len(objs) for obj in objs: yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj))) from Worker import Worker objs = [obj for obj in gc.get_objects() if isinstance(obj, Worker)] yield "<br>Workers (%s):<br>" % len(objs) for obj in objs: yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj))) from Connection import Connection objs = [obj for obj in gc.get_objects() if isinstance(obj, Connection)] yield "<br>Connections (%s):<br>" % len(objs) for obj in objs: yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj))) from socket import socket objs = [obj for obj in gc.get_objects() if isinstance(obj, socket)] yield "<br>Sockets (%s):<br>" % len(objs) for obj in objs: yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj))) from msgpack import Unpacker objs = [obj for obj in gc.get_objects() if isinstance(obj, Unpacker)] yield "<br>Msgpack unpacker (%s):<br>" % len(objs) for obj in objs: yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj))) from Site import Site objs = [obj for obj in gc.get_objects() if isinstance(obj, Site)] yield "<br>Sites (%s):<br>" % len(objs) for obj in objs: yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj))) objs = [obj for obj in gc.get_objects() if isinstance(obj, self.server.log.__class__)] yield "<br>Loggers (%s):<br>" % len(objs) for obj in objs: yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj.name))) objs = [obj for obj in gc.get_objects() if isinstance(obj, UiRequest)] yield "<br>UiRequests (%s):<br>" % len(objs) for obj in objs: yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj))) from Peer import Peer objs = [obj for obj in gc.get_objects() if isinstance(obj, Peer)] yield "<br>Peers (%s):<br>" % len(objs) for obj in objs: yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj))) objs = [(key, val) for key, val in sys.modules.iteritems() if val is not None] objs.sort() yield "<br>Modules (%s):<br>" % len(objs) for module_name, module in objs: yield " - %.3fkb: %s %s<br>" % (self.getObjSize(module, hpy), module_name, cgi.escape(repr(module))) gc.collect() # Implicit grabage collection yield "Done in %.1f" % (time.time() - s) def actionDumpobj(self): import gc import sys self.sendHeader() # No more if not in debug mode if not config.debug: yield "Not in debug mode" raise StopIteration class_filter = self.get.get("class") yield """ <style> * { font-family: monospace; white-space: pre } table * { text-align: right; padding: 0px 10px } </style> """ objs = gc.get_objects() for obj in objs: obj_type = str(type(obj)) if obj_type != "<type 'instance'>" or obj.__class__.__name__ != class_filter: continue yield "%.1fkb %s... " % (float(sys.getsizeof(obj)) / 1024, cgi.escape(str(obj))) for attr in dir(obj): yield "- %s: %s<br>" % (attr, cgi.escape(str(getattr(obj, attr)))) yield "<br>" gc.collect() # Implicit grabage collection def actionListobj(self): import gc import sys self.sendHeader() # No more if not in debug mode if not config.debug: yield "Not in debug mode" raise StopIteration type_filter = self.get.get("type") yield """ <style> * { font-family: monospace; white-space: pre } table * { text-align: right; padding: 0px 10px } </style> """ yield "Listing all %s objects in memory...<br>" % cgi.escape(type_filter) ref_count = {} objs = gc.get_objects() for obj in objs: obj_type = str(type(obj)) if obj_type != type_filter: continue refs = [ ref for ref in gc.get_referrers(obj) if hasattr(ref, "__class__") and ref.__class__.__name__ not in ["list", "dict", "function", "type", "frame", "WeakSet", "tuple"] ] if not refs: continue yield "%.1fkb <span title=\"%s\">%s</span>... " % ( float(sys.getsizeof(obj)) / 1024, cgi.escape(str(obj)), cgi.escape(str(obj)[0:100].ljust(100)) ) for ref in refs: yield " [" if "object at" in str(ref) or len(str(ref)) > 100: yield str(ref.__class__.__name__) else: yield str(ref.__class__.__name__) + ":" + cgi.escape(str(ref)) yield "] " ref_type = ref.__class__.__name__ if ref_type not in ref_count: ref_count[ref_type] = [0, 0] ref_count[ref_type][0] += 1 # Count ref_count[ref_type][1] += float(sys.getsizeof(obj)) / 1024 # Size yield "<br>" yield "<br>Object referrer (total: %s, %.2fkb):<br>" % (len(ref_count), sum([stat[1] for stat in ref_count.values()])) for obj, stat in sorted(ref_count.items(), key=lambda x: x[1][0], reverse=True)[0:30]: # Sorted by count yield " - %.1fkb = %s x %s<br>" % (stat[1], stat[0], cgi.escape(str(obj))) gc.collect() # Implicit grabage collection def actionBenchmark(self): import sys import gc from contextlib import contextmanager output = self.sendHeader() @contextmanager def benchmark(name, standard): s = time.time() output("- %s" % name) try: yield 1 except Exception, err: output("<br><b>! Error: %s</b><br>" % err) taken = time.time() - s multipler = standard / taken if multipler < 0.3: speed = "Sloooow" elif multipler < 0.5: speed = "Ehh" elif multipler < 0.8: speed = "Goodish" elif multipler < 1.2: speed = "OK" elif multipler < 1.7: speed = "Fine" elif multipler < 2.5: speed = "Fast" elif multipler < 3.5: speed = "WOW" else: speed = "Insane!!" output("%.3fs [x%.2f: %s]<br>" % (taken, multipler, speed)) time.sleep(0.01) yield """ <style> * { font-family: monospace } table * { text-align: right; padding: 0px 10px } </style> """ yield "Benchmarking ZeroNet %s (rev%s) Python %s on: %s...<br>" % (config.version, config.rev, sys.version, sys.platform) t = time.time() # CryptBitcoin yield "<br>CryptBitcoin:<br>" from Crypt import CryptBitcoin # seed = CryptBitcoin.newSeed() # yield "- Seed: %s<br>" % seed seed = "e180efa477c63b0f2757eac7b1cce781877177fe0966be62754ffd4c8592ce38" with benchmark("hdPrivatekey x 10", 0.7): for i in range(10): privatekey = CryptBitcoin.hdPrivatekey(seed, i * 10) yield "." valid = "5JsunC55XGVqFQj5kPGK4MWgTL26jKbnPhjnmchSNPo75XXCwtk" assert privatekey == valid, "%s != %s" % (privatekey, valid) data = "Hello" * 1024 # 5k with benchmark("sign x 10", 0.35): for i in range(10): yield "." sign = CryptBitcoin.sign(data, privatekey) valid = "HFGXaDauZ8vX/N9Jn+MRiGm9h+I94zUhDnNYFaqMGuOi+4+BbWHjuwmx0EaKNV1G+kP0tQDxWu0YApxwxZbSmZU=" assert sign == valid, "%s != %s" % (sign, valid) address = CryptBitcoin.privatekeyToAddress(privatekey) if CryptBitcoin.opensslVerify: # Openssl avalible with benchmark("openssl verify x 100", 0.37): for i in range(100): if i % 10 == 0: yield "." ok = CryptBitcoin.verify(data, address, sign) assert ok, "does not verify from %s" % address else: yield " - openssl verify x 100...not avalible :(<br>" openssl_verify_bk = CryptBitcoin.opensslVerify # Emulate openssl not found in any way CryptBitcoin.opensslVerify = None with benchmark("pure-python verify x 10", 1.6): for i in range(10): yield "." ok = CryptBitcoin.verify(data, address, sign) assert ok, "does not verify from %s" % address CryptBitcoin.opensslVerify = openssl_verify_bk # CryptHash yield "<br>CryptHash:<br>" from Crypt import CryptHash from cStringIO import StringIO data = StringIO("Hello" * 1024 * 1024) # 5m with benchmark("sha512 x 100 000", 1): for i in range(10): for y in range(10000): hash = CryptHash.sha512sum(data) yield "." valid = "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce" assert hash == valid, "%s != %s" % (hash, valid) with benchmark("os.urandom(256) x 100 000", 0.65): for i in range(10): for y in range(10000): data = os.urandom(256) yield "." # Msgpack yield "<br>Msgpack:<br>" import msgpack binary = 'fqv\xf0\x1a"e\x10,\xbe\x9cT\x9e(\xa5]u\x072C\x8c\x15\xa2\xa8\x93Sw)\x19\x02\xdd\t\xfb\xf67\x88\xd9\xee\x86\xa1\xe4\xb6,\xc6\x14\xbb\xd7$z\x1d\xb2\xda\x85\xf5\xa0\x97^\x01*\xaf\xd3\xb0!\xb7\x9d\xea\x89\xbbh8\xa1"\xa7]e(@\xa2\xa5g\xb7[\xae\x8eE\xc2\x9fL\xb6s\x19\x19\r\xc8\x04S\xd0N\xe4]?/\x01\xea\xf6\xec\xd1\xb3\xc2\x91\x86\xd7\xf4K\xdf\xc2lV\xf4\xe8\x80\xfc\x8ep\xbb\x82\xb3\x86\x98F\x1c\xecS\xc8\x15\xcf\xdc\xf1\xed\xfc\xd8\x18r\xf9\x80\x0f\xfa\x8cO\x97(\x0b]\xf1\xdd\r\xe7\xbf\xed\x06\xbd\x1b?\xc5\xa0\xd7a\x82\xf3\xa8\xe6@\xf3\ri\xa1\xb10\xf6\xd4W\xbc\x86\x1a\xbb\xfd\x94!bS\xdb\xaeM\x92\x00#\x0b\xf7\xad\xe9\xc2\x8e\x86\xbfi![%\xd31]\xc6\xfc2\xc9\xda\xc6v\x82P\xcc\xa9\xea\xb9\xff\xf6\xc8\x17iD\xcf\xf3\xeeI\x04\xe9\xa1\x19\xbb\x01\x92\xf5nn4K\xf8\xbb\xc6\x17e>\xa7 \xbbv' data = {"int": 1024*1024*1024, "float": 12345.67890, "text": "hello"*1024, "binary": binary} with benchmark("pack 5K x 10 000", 0.78): for i in range(10): for y in range(1000): data_packed = msgpack.packb(data) yield "." valid = """\x84\xa3int\xce@\x00\x00\x00\xa4text\xda\x14\x00hellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohello\xa5float\xcb@\xc8\x1c\xd6\xe61\xf8\xa1\xa6binary\xda\x01\x00fqv\xf0\x1a"e\x10,\xbe\x9cT\x9e(\xa5]u\x072C\x8c\x15\xa2\xa8\x93Sw)\x19\x02\xdd\t\xfb\xf67\x88\xd9\xee\x86\xa1\xe4\xb6,\xc6\x14\xbb\xd7$z\x1d\xb2\xda\x85\xf5\xa0\x97^\x01*\xaf\xd3\xb0!\xb7\x9d\xea\x89\xbbh8\xa1"\xa7]e(@\xa2\xa5g\xb7[\xae\x8eE\xc2\x9fL\xb6s\x19\x19\r\xc8\x04S\xd0N\xe4]?/\x01\xea\xf6\xec\xd1\xb3\xc2\x91\x86\xd7\xf4K\xdf\xc2lV\xf4\xe8\x80\xfc\x8ep\xbb\x82\xb3\x86\x98F\x1c\xecS\xc8\x15\xcf\xdc\xf1\xed\xfc\xd8\x18r\xf9\x80\x0f\xfa\x8cO\x97(\x0b]\xf1\xdd\r\xe7\xbf\xed\x06\xbd\x1b?\xc5\xa0\xd7a\x82\xf3\xa8\xe6@\xf3\ri\xa1\xb10\xf6\xd4W\xbc\x86\x1a\xbb\xfd\x94!bS\xdb\xaeM\x92\x00#\x0b\xf7\xad\xe9\xc2\x8e\x86\xbfi![%\xd31]\xc6\xfc2\xc9\xda\xc6v\x82P\xcc\xa9\xea\xb9\xff\xf6\xc8\x17iD\xcf\xf3\xeeI\x04\xe9\xa1\x19\xbb\x01\x92\xf5nn4K\xf8\xbb\xc6\x17e>\xa7 \xbbv""" assert data_packed == valid, "%s<br>!=<br>%s" % (repr(data_packed), repr(valid)) with benchmark("unpack 5K x 10 000", 1.2): for i in range(10): for y in range(1000): data_unpacked = msgpack.unpackb(data_packed) yield "." assert data == data_unpacked, "%s != %s" % (data_unpack, data) with benchmark("streaming unpack 5K x 10 000", 1.4): for i in range(10): unpacker = msgpack.Unpacker() for y in range(1000): unpacker.feed(data_packed) for data_unpacked in unpacker: pass yield "." assert data == data_unpacked, "%s != %s" % (data_unpack, data) # Db yield "<br>Db:<br>" from Db import Db schema = { "db_name": "TestDb", "db_file": "%s/benchmark.db" % config.data_dir, "maps": { ".*": { "to_table": { "test": "test" } } }, "tables": { "test": { "cols": [ ["test_id", "INTEGER"], ["title", "TEXT"], ["json_id", "INTEGER REFERENCES json (json_id)"] ], "indexes": ["CREATE UNIQUE INDEX test_key ON test(test_id, json_id)"], "schema_changed": 1426195822 } } } if os.path.isfile("%s/benchmark.db" % config.data_dir): os.unlink("%s/benchmark.db" % config.data_dir) with benchmark("Open x 10", 0.13): for i in range(10): db = Db(schema, "%s/benchmark.db" % config.data_dir) db.checkTables() db.close() yield "." db = Db(schema, "%s/benchmark.db" % config.data_dir) db.checkTables() import json with benchmark("Insert x 10 x 1000", 1.0): for u in range(10): # 10 user data = {"test": []} for i in range(1000): # 1000 line of data data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)}) json.dump(data, open("%s/test_%s.json" % (config.data_dir, u), "w")) db.loadJson("%s/test_%s.json" % (config.data_dir, u)) os.unlink("%s/test_%s.json" % (config.data_dir, u)) yield "." with benchmark("Buffered insert x 100 x 100", 1.3): cur = db.getCursor() cur.execute("BEGIN") cur.logging = False for u in range(100, 200): # 100 user data = {"test": []} for i in range(100): # 1000 line of data data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)}) json.dump(data, open("%s/test_%s.json" % (config.data_dir, u), "w")) db.loadJson("%s/test_%s.json" % (config.data_dir, u), cur=cur) os.unlink("%s/test_%s.json" % (config.data_dir, u)) if u % 10 == 0: yield "." cur.execute("COMMIT") yield " - Total rows in db: %s<br>" % db.execute("SELECT COUNT(*) AS num FROM test").fetchone()[0] with benchmark("Indexed query x 1000", 0.25): found = 0 cur = db.getCursor() cur.logging = False for i in range(1000): # 1000x by test_id res = cur.execute("SELECT * FROM test WHERE test_id = %s" % i) for row in res: found += 1 if i % 100 == 0: yield "." assert found == 20000, "Found: %s != 20000" % found with benchmark("Not indexed query x 100", 0.6): found = 0 cur = db.getCursor() cur.logging = False for i in range(100): # 1000x by test_id res = cur.execute("SELECT * FROM test WHERE json_id = %s" % i) for row in res: found += 1 if i % 10 == 0: yield "." assert found == 18900, "Found: %s != 18900" % found with benchmark("Like query x 100", 1.8): found = 0 cur = db.getCursor() cur.logging = False for i in range(100): # 1000x by test_id res = cur.execute("SELECT * FROM test WHERE title LIKE '%%message %s%%'" % i) for row in res: found += 1 if i % 10 == 0: yield "." assert found == 38900, "Found: %s != 11000" % found db.close() if os.path.isfile("%s/benchmark.db" % config.data_dir): os.unlink("%s/benchmark.db" % config.data_dir) gc.collect() # Implicit grabage collection yield "<br>Done. Total: %.2fs" % (time.time() - t)
def peerCheckMessage(self, raw, params, ip): # Calculate hash from nonce msg_hash = hashlib.sha256( "%s,%s" % (params["nonce"], params["raw"])).hexdigest() # Check that p2p.json exists site = self.sites.get(raw["site"]) if not site.storage.isFile("p2p.json"): self.connection.log("Site %s doesn't support P2P messages" % raw["site"]) self.connection.badAction(5) self.response({ "error": "Site %s doesn't support P2P messages" % raw["site"] }) return False, "", None, msg_hash # Check whether P2P messages are supported p2p_json = site.storage.loadJson("p2p.json") if "filter" not in p2p_json: self.connection.log("Site %s doesn't support P2P messages" % raw["site"]) self.connection.badAction(5) self.response({ "error": "Site %s doesn't support P2P messages" % raw["site"] }) return False, "", None, msg_hash # Was the message received yet? if msg_hash in site.p2p_received: self.response({"warning": "Already received, thanks"}) return False, "", None, msg_hash site.p2p_received.append(msg_hash) # Check whether the message matches passive filter if not SafeRe.match(p2p_json["filter"], json.dumps(raw["message"])): self.connection.log("Invalid message for site %s: %s" % (raw["site"], raw["message"])) self.connection.badAction(5) self.response({ "error": "Invalid message for site %s: %s" % (raw["site"], raw["message"]) }) return False, "", None, msg_hash # Not so fast if "freq_limit" in p2p_json and time.time() - site.p2p_last_recv.get( ip, 0) < p2p_json["freq_limit"]: self.connection.log("Too fast messages from %s" % raw["site"]) self.connection.badAction(2) self.response({"error": "Too fast messages from %s" % raw["site"]}) return False, "", None, msg_hash site.p2p_last_recv[ip] = time.time() # Not so much if "size_limit" in p2p_json and len(json.dumps( raw["message"])) > p2p_json["size_limit"]: self.connection.log("Too big message from %s" % raw["site"]) self.connection.badAction(7) self.response({"error": "Too big message from %s" % raw["site"]}) return False, "", None, msg_hash # Verify signature if params["signature"]: signature_address, signature = params["signature"].split("|") what = "%s|%s|%s" % (signature_address, msg_hash, params["raw"]) from Crypt import CryptBitcoin if not CryptBitcoin.verify(what, signature_address, signature): self.connection.log("Invalid signature") self.connection.badAction(7) self.response({"error": "Invalid signature"}) return False, "", None, msg_hash # Now check auth providers if params.get("cert"): # Read all info cert_auth_type, cert_auth_user_name, cert_issuer, cert_sign = params[ "cert"] # This is what certificate issuer signs cert_subject = "%s#%s/%s" % (signature_address, cert_auth_type, cert_auth_user_name) # Now get cert issuer address cert_signers = p2p_json.get("cert_signers", {}) cert_addresses = cert_signers.get(cert_issuer, []) # And verify it if not CryptBitcoin.verify(cert_subject, cert_addresses, cert_sign): self.connection.log("Invalid signature certificate") self.connection.badAction(7) self.response({"error": "Invalid signature certificate"}) return False, "", None, msg_hash # And save the ID cert = "%s/%s@%s" % (cert_auth_type, cert_auth_user_name, cert_issuer) else: # Old-style sign cert = "" else: signature_address = "" cert = "" # Check that the signature address is correct if "signed_only" in p2p_json: valid = p2p_json["signed_only"] if valid is True and not signature_address: self.connection.log("Not signed message") self.connection.badAction(5) self.response({"error": "Not signed message"}) return False, "", None, msg_hash elif isinstance(valid, str) and signature_address != valid: self.connection.log( "Message signature is invalid: %s not in [%r]" % (signature_address, valid)) self.connection.badAction(5) self.response({ "error": "Message signature is invalid: %s not in [%r]" % (signature_address, valid) }) return False, "", None, msg_hash elif isinstance(valid, list) and signature_address not in valid: self.connection.log( "Message signature is invalid: %s not in %r" % (signature_address, valid)) self.connection.badAction(5) self.response({ "error": "Message signature is invalid: %s not in %r" % (signature_address, valid) }) return False, "", None, msg_hash return True, signature_address, cert, msg_hash
def testInlcudeLimits(self, site): # Data validation data_dict = { "files": { "data.json": { "sha512": "369d4e780cc80504285f13774ca327fe725eed2d813aad229e62356b07365906", "size": 505 } }, "modified": time.time() } # Normal data data_dict["signs"] = { "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), self.privatekey) } data = StringIO(json.dumps(data_dict)) assert site.content_manager.verifyFile( "data/test_include/content.json", data, ignore_same=False) # Reset del data_dict["signs"] # Too large data_dict["files"]["data.json"][ "size"] = 200000 # Emulate 2MB sized data.json data_dict["signs"] = { "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), self.privatekey) } data = StringIO(json.dumps(data_dict)) with pytest.raises(VerifyError) as err: site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False) assert "Include too large" in str(err) # Reset data_dict["files"]["data.json"]["size"] = 505 del data_dict["signs"] # Not allowed file data_dict["files"]["notallowed.exe"] = data_dict["files"]["data.json"] data_dict["signs"] = { "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), self.privatekey) } data = StringIO(json.dumps(data_dict)) with pytest.raises(VerifyError) as err: site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False) assert "File not allowed" in str(err) # Reset del data_dict["files"]["notallowed.exe"] del data_dict["signs"] # Should work again data_dict["signs"] = { "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), self.privatekey) } data = StringIO(json.dumps(data_dict)) assert site.content_manager.verifyFile( "data/test_include/content.json", data, ignore_same=False)
def verifyFile(self, inner_path, file, ignore_same=True): if inner_path.endswith("content.json"): # content.json: Check using sign from Crypt import CryptBitcoin try: if type(file) is dict: new_content = file else: new_content = json.load(file) if inner_path in self.contents: old_content = self.contents.get(inner_path, {"modified": 0}) # Checks if its newer the ours if old_content["modified"] == new_content["modified"] and ignore_same: # Ignore, have the same content.json return None elif old_content["modified"] > new_content["modified"]: # We have newer raise VerifyError( "We have newer (Our: %s, Sent: %s)" % (old_content["modified"], new_content["modified"]) ) if new_content["modified"] > time.time() + 60 * 60 * 24: # Content modified in the far future (allow 1 day+) raise VerifyError("Modify timestamp is in the far future!") if self.isArchived(inner_path, new_content["modified"]): if inner_path in self.site.bad_files: del self.site.bad_files[inner_path] raise VerifyError("This file is archived!") # Check sign sign = new_content.get("sign") signs = new_content.get("signs", {}) if "sign" in new_content: del(new_content["sign"]) # The file signed without the sign if "signs" in new_content: del(new_content["signs"]) # The file signed without the signs sign_content = json.dumps(new_content, sort_keys=True) # Dump the json to string to remove whitepsace # Fix float representation error on Android modified = new_content["modified"] if config.fix_float_decimals and type(modified) is float and not str(modified).endswith(".0"): modified_fixed = "{:.6f}".format(modified).strip("0.") sign_content = sign_content.replace( '"modified": %s' % repr(modified), '"modified": %s' % modified_fixed ) self.verifyContent(inner_path, new_content) if signs: # New style signing valid_signers = self.getValidSigners(inner_path, new_content) signs_required = self.getSignsRequired(inner_path, new_content) if inner_path == "content.json" and len(valid_signers) > 1: # Check signers_sign on root content.json signers_data = "%s:%s" % (signs_required, ",".join(valid_signers)) if not CryptBitcoin.verify(signers_data, self.site.address, new_content["signers_sign"]): raise VerifyError("Invalid signers_sign!") if inner_path != "content.json" and not self.verifyCert(inner_path, new_content): # Check if cert valid raise VerifyError("Invalid cert!") valid_signs = 0 for address in valid_signers: if address in signs: valid_signs += CryptBitcoin.verify(sign_content, address, signs[address]) if valid_signs >= signs_required: break # Break if we has enough signs if valid_signs < signs_required: raise VerifyError("Valid signs: %s/%s" % (valid_signs, signs_required)) else: return True else: # Old style signing if CryptBitcoin.verify(sign_content, self.site.address, sign): return True else: raise VerifyError("Invalid old-style sign") except Exception, err: self.log.warning("%s: verify sign error: %s" % (inner_path, Debug.formatException(err))) raise err
yield "Benchmarking ZeroNet %s (rev%s) Python %s, platform: %s...<br>" % ( config.version, config.rev, sys.version, sys.platform) t = time.time() yield "<br>CryptBitcoin:<br>" from Crypt import CryptBitcoin # seed = CryptBitcoin.newSeed() # yield "- Seed: %s<br>" % seed seed = "e180efa477c63b0f2757eac7b1cce781877177fe0966be62754ffd4c8592ce38" with benchmark("hdPrivatekey x 10", 0.7): for i in range(10): privatekey = CryptBitcoin.hdPrivatekey(seed, i * 10) yield "." valid = "5JsunC55XGVqFQj5kPGK4MWgTL26jKbnPhjnmchSNPo75XXCwtk" assert privatekey == valid, "%s != %s" % (privatekey, valid) data = "Hello" * 1024 #5k with benchmark("sign x 10", 0.35): for i in range(10): yield "." sign = CryptBitcoin.sign(data, privatekey) valid = "HFGXaDauZ8vX/N9Jn+MRiGm9h+I94zUhDnNYFaqMGuOi+4+BbWHjuwmx0EaKNV1G+kP0tQDxWu0YApxwxZbSmZU=" assert sign == valid, "%s != %s" % (sign, valid) address = CryptBitcoin.privatekeyToAddress(privatekey) with benchmark("verify x 10", 1.6): for i in range(10):
def testNewPrivatekey(self): assert CryptBitcoin.newPrivatekey() != CryptBitcoin.newPrivatekey() assert CryptBitcoin.privatekeyToAddress(CryptBitcoin.newPrivatekey())
def testCert(self, site): # user_addr = "1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C" user_priv = "5Kk7FSA63FC2ViKmKLuBxk9gQkaQ5713hKq8LmFAf4cVeXh6K6A" # cert_addr = "14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet" cert_priv = "5JusJDSjHaMHwUjDT3o6eQ54pA6poo8La5fAgn1wNc3iK59jxjA" # Check if the user file is loaded assert "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json" in site.content_manager.contents user_content = site.content_manager.contents[ "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json"] rules_content = site.content_manager.contents[ "data/users/content.json"] # Override valid cert signers for the test rules_content["user_contents"]["cert_signers"]["zeroid.bit"] = [ "14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet", "1iD5ZQJMNXu43w1qLB8sfdHVKppVMduGz" ] # Check valid cert signers rules = site.content_manager.getRules( "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_content) assert rules["cert_signers"] == { "zeroid.bit": [ "14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet", "1iD5ZQJMNXu43w1qLB8sfdHVKppVMduGz" ] } # Sign a valid cert user_content["cert_sign"] = CryptBitcoin.sign( "1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C#%s/%s" % (user_content["cert_auth_type"], user_content["cert_user_id"].split("@")[0]), cert_priv) # Verify cert assert site.content_manager.verifyCert( "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_content) # Verify if the cert is valid for other address assert not site.content_manager.verifyCert( "data/users/badaddress/content.json", user_content) # Sign user content signed_content = site.content_manager.sign( "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_priv, filewrite=False) # Test user cert assert site.content_manager.verifyFile( "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", io.BytesIO(json.dumps(signed_content).encode()), ignore_same=False) # Test banned user cert_user_id = user_content["cert_user_id"] # My username site.content_manager.contents["data/users/content.json"][ "user_contents"]["permissions"][cert_user_id] = False with pytest.raises(VerifyError) as err: site.content_manager.verifyFile( "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", io.BytesIO(json.dumps(signed_content).encode()), ignore_same=False) assert "Valid signs: 0/1" in str(err.value) del site.content_manager.contents["data/users/content.json"][ "user_contents"]["permissions"][cert_user_id] # Reset # Test invalid cert user_content["cert_sign"] = CryptBitcoin.sign( "badaddress#%s/%s" % (user_content["cert_auth_type"], user_content["cert_user_id"]), cert_priv) signed_content = site.content_manager.sign( "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_priv, filewrite=False) with pytest.raises(VerifyError) as err: site.content_manager.verifyFile( "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", io.BytesIO(json.dumps(signed_content).encode()), ignore_same=False) assert "Invalid cert" in str(err.value) # Test banned user, signed by the site owner user_content["cert_sign"] = CryptBitcoin.sign( "1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C#%s/%s" % (user_content["cert_auth_type"], user_content["cert_user_id"].split("@")[0]), cert_priv) cert_user_id = user_content["cert_user_id"] # My username site.content_manager.contents["data/users/content.json"][ "user_contents"]["permissions"][cert_user_id] = False site_privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv" # For 1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT del user_content["signs"] # Remove signs before signing user_content["signs"] = { "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(user_content, sort_keys=True), site_privatekey) } assert site.content_manager.verifyFile( "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", io.BytesIO(json.dumps(user_content).encode()), ignore_same=False)
def testAuthAddress(self, user): # Auth address without Cert auth_address = user.getAuthAddress("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr") assert auth_address == "1MyJgYQjeEkR9QD66nkfJc9zqi9uUy5Lr2" auth_privatekey = user.getAuthPrivatekey("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr") assert CryptBitcoin.privatekeyToAddress(auth_privatekey) == auth_address
def testNewSeed(self): assert CryptBitcoin.newSeed() != CryptBitcoin.newSeed() assert CryptBitcoin.privatekeyToAddress( CryptBitcoin.hdPrivatekey(CryptBitcoin.newSeed(), 0)) assert CryptBitcoin.privatekeyToAddress( CryptBitcoin.hdPrivatekey(CryptBitcoin.newSeed(), 2**256))
def cryptVerify(self, message, sign, address): from Crypt import CryptBitcoin print CryptBitcoin.verify(message, address, sign)
def actionEcdsaSign(self, to, data, privatekey=None): if privatekey is None: # Sign using user's privatekey privatekey = self.user.getAuthPrivatekey(self.site.address) self.response(to, CryptBitcoin.sign(data, privatekey))
def crypt_bitcoin_lib(request, monkeypatch): monkeypatch.setattr(CryptBitcoin, "lib_verify_best", request.param) CryptBitcoin.loadLib(request.param) return CryptBitcoin
class UiRequestPlugin(object): def formatTableRow(self, row, class_name=""): back = [] for format, val in row: if val is None: formatted = "n/a" elif format == "since": if val: formatted = "%.0f" % (time.time() - val) else: formatted = "n/a" else: formatted = format % val back.append("<td>%s</td>" % formatted) return "<tr class='%s'>%s</tr>" % (class_name, "".join(back)) def getObjSize(self, obj, hpy=None): if hpy: return float(hpy.iso(obj).domisize) / 1024 else: return 0 # /Stats entry point def actionStats(self): import gc import sys from Ui import UiRequest from Db import Db from Crypt import CryptConnection hpy = None if self.get.get("size") == "1": # Calc obj size try: import guppy hpy = guppy.hpy() except: pass self.sendHeader() if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local: yield "This function is disabled on this proxy" raise StopIteration s = time.time() main = sys.modules["main"] # Style yield """ <style> * { font-family: monospace } table td, table th { text-align: right; padding: 0px 10px } .connections td { white-space: nowrap } .serving-False { opacity: 0.3 } </style> """ # Memory try: yield "rev%s | " % config.rev yield "%s | " % config.ip_external yield "Opened: %s | " % main.file_server.port_opened yield "Crypt: %s | " % CryptConnection.manager.crypt_supported yield "In: %.2fMB, Out: %.2fMB | " % ( float(main.file_server.bytes_recv) / 1024 / 1024, float(main.file_server.bytes_sent) / 1024 / 1024) yield "Peerid: %s | " % main.file_server.peer_id import psutil process = psutil.Process(os.getpid()) mem = process.get_memory_info()[0] / float(2**20) yield "Mem: %.2fMB | " % mem yield "Threads: %s | " % len(process.threads()) yield "CPU: usr %.2fs sys %.2fs | " % process.cpu_times() yield "Files: %s | " % len(process.open_files()) yield "Sockets: %s | " % len(process.connections()) yield "Calc size <a href='?size=1'>on</a> <a href='?size=0'>off</a>" except Exception: pass yield "<br>" # Connections yield "<b>Connections</b> (%s, total made: %s):<br>" % (len( main.file_server.connections), main.file_server.last_connection_id) yield "<table class='connections'><tr> <th>id</th> <th>type</th> <th>ip</th> <th>open</th> <th>crypt</th> <th>ping</th>" yield "<th>buff</th> <th>bad</th> <th>idle</th> <th>open</th> <th>delay</th> <th>cpu</th> <th>out</th> <th>in</th> <th>last sent</th>" yield "<th>wait</th> <th>version</th> <th>sites</th> </tr>" for connection in main.file_server.connections: if "cipher" in dir(connection.sock): cipher = connection.sock.cipher()[0] else: cipher = connection.crypt yield self.formatTableRow([ ("%3d", connection.id), ("%s", connection.type), ("%s:%s", (connection.ip, connection.port)), ("%s", connection.handshake.get("port_opened")), ("<span title='%s'>%s</span>", (connection.crypt, cipher)), ("%6.3f", connection.last_ping_delay), ("%s", connection.incomplete_buff_recv), ("%s", connection.bad_actions), ("since", max(connection.last_send_time, connection.last_recv_time)), ("since", connection.start_time), ("%.3f", connection.last_sent_time - connection.last_send_time), ("%.3f", connection.cpu_time), ("%.0fkB", connection.bytes_sent / 1024), ("%.0fkB", connection.bytes_recv / 1024), ("%s", connection.last_cmd), ("%s", connection.waiting_requests.keys()), ("%s r%s", (connection.handshake.get("version"), connection.handshake.get("rev", "?"))), ("%s", connection.sites) ]) yield "</table>" # Tor hidden services yield "<br><br><b>Tor hidden services (status: %s):</b><br>" % main.file_server.tor_manager.status for site_address, onion in main.file_server.tor_manager.site_onions.items( ): yield "- %-34s: %s<br>" % (site_address, onion) # Db yield "<br><br><b>Db</b>:<br>" for db in sys.modules["Db.Db"].opened_dbs: yield "- %.3fs: %s<br>" % (time.time() - db.last_query_time, db.db_path.encode("utf8")) # Sites yield "<br><br><b>Sites</b>:" yield "<table>" yield "<tr><th>address</th> <th>connected</th> <th title='connected/good/total'>peers</th> <th>content.json</th> <th>out</th> <th>in</th> </tr>" for site in sorted(self.server.sites.values(), lambda a, b: cmp(a.address, b.address)): yield self.formatTableRow([ ("""<a href='#' onclick='document.getElementById("peers_%s").style.display="initial"; return false'>%s</a>""", (site.address, site.address)), ("%s", [ peer.connection.id for peer in site.peers.values() if peer.connection and peer.connection.connected ]), ("%s/%s/%s", (len([ peer for peer in site.peers.values() if peer.connection and peer.connection.connected ]), len(site.getConnectablePeers(100)), len(site.peers))), ("%s (loaded: %s)", (len(site.content_manager.contents), len([ key for key, val in dict( site.content_manager.contents).iteritems() if val ]))), ("%.0fkB", site.settings.get("bytes_sent", 0) / 1024), ("%.0fkB", site.settings.get("bytes_recv", 0) / 1024), ], "serving-%s" % site.settings["serving"]) yield "<tr><td id='peers_%s' style='display: none; white-space: pre' colspan=6>" % site.address for key, peer in site.peers.items(): if peer.time_found: time_found = int(time.time() - peer.time_found) / 60 else: time_found = "--" if peer.connection: connection_id = peer.connection.id else: connection_id = None if site.content_manager.has_optional_files: yield "Optional files: %4s " % len(peer.hashfield) time_added = (time.time() - peer.time_added) / (60 * 60 * 24) yield "(#%4s, err: %s, found: %3s min, add: %.1f day) %30s -<br>" % ( connection_id, peer.connection_error, time_found, time_added, key) yield "<br></td></tr>" yield "</table>" # No more if not in debug mode if not config.debug: raise StopIteration # Object types obj_count = {} for obj in gc.get_objects(): obj_type = str(type(obj)) if obj_type not in obj_count: obj_count[obj_type] = [0, 0] obj_count[obj_type][0] += 1 # Count obj_count[obj_type][1] += float(sys.getsizeof(obj)) / 1024 # Size yield "<br><br><b>Objects in memory (types: %s, total: %s, %.2fkb):</b><br>" % ( len(obj_count), sum([stat[0] for stat in obj_count.values()]), sum([stat[1] for stat in obj_count.values()])) for obj, stat in sorted(obj_count.items(), key=lambda x: x[1][0], reverse=True): # Sorted by count yield " - %.1fkb = %s x <a href=\"/Listobj?type=%s\">%s</a><br>" % ( stat[1], stat[0], obj, cgi.escape(obj)) # Classes class_count = {} for obj in gc.get_objects(): obj_type = str(type(obj)) if obj_type != "<type 'instance'>": continue class_name = obj.__class__.__name__ if class_name not in class_count: class_count[class_name] = [0, 0] class_count[class_name][0] += 1 # Count class_count[class_name][1] += float( sys.getsizeof(obj)) / 1024 # Size yield "<br><br><b>Classes in memory (types: %s, total: %s, %.2fkb):</b><br>" % ( len(class_count), sum([stat[0] for stat in class_count.values()]), sum([stat[1] for stat in class_count.values()])) for obj, stat in sorted(class_count.items(), key=lambda x: x[1][0], reverse=True): # Sorted by count yield " - %.1fkb = %s x <a href=\"/Dumpobj?class=%s\">%s</a><br>" % ( stat[1], stat[0], obj, cgi.escape(obj)) from greenlet import greenlet objs = [obj for obj in gc.get_objects() if isinstance(obj, greenlet)] yield "<br>Greenlets (%s):<br>" % len(objs) for obj in objs: yield " - %.1fkb: %s<br>" % (self.getObjSize( obj, hpy), cgi.escape(repr(obj).encode("utf8"))) from Worker import Worker objs = [obj for obj in gc.get_objects() if isinstance(obj, Worker)] yield "<br>Workers (%s):<br>" % len(objs) for obj in objs: yield " - %.1fkb: %s<br>" % (self.getObjSize( obj, hpy), cgi.escape(repr(obj))) from Connection import Connection objs = [obj for obj in gc.get_objects() if isinstance(obj, Connection)] yield "<br>Connections (%s):<br>" % len(objs) for obj in objs: yield " - %.1fkb: %s<br>" % (self.getObjSize( obj, hpy), cgi.escape(repr(obj))) from socket import socket objs = [obj for obj in gc.get_objects() if isinstance(obj, socket)] yield "<br>Sockets (%s):<br>" % len(objs) for obj in objs: yield " - %.1fkb: %s<br>" % (self.getObjSize( obj, hpy), cgi.escape(repr(obj))) from msgpack import Unpacker objs = [obj for obj in gc.get_objects() if isinstance(obj, Unpacker)] yield "<br>Msgpack unpacker (%s):<br>" % len(objs) for obj in objs: yield " - %.1fkb: %s<br>" % (self.getObjSize( obj, hpy), cgi.escape(repr(obj))) from Site import Site objs = [obj for obj in gc.get_objects() if isinstance(obj, Site)] yield "<br>Sites (%s):<br>" % len(objs) for obj in objs: yield " - %.1fkb: %s<br>" % (self.getObjSize( obj, hpy), cgi.escape(repr(obj))) objs = [ obj for obj in gc.get_objects() if isinstance(obj, self.server.log.__class__) ] yield "<br>Loggers (%s):<br>" % len(objs) for obj in objs: yield " - %.1fkb: %s<br>" % (self.getObjSize( obj, hpy), cgi.escape(repr(obj.name))) objs = [obj for obj in gc.get_objects() if isinstance(obj, UiRequest)] yield "<br>UiRequests (%s):<br>" % len(objs) for obj in objs: yield " - %.1fkb: %s<br>" % (self.getObjSize( obj, hpy), cgi.escape(repr(obj))) from Peer import Peer objs = [obj for obj in gc.get_objects() if isinstance(obj, Peer)] yield "<br>Peers (%s):<br>" % len(objs) for obj in objs: yield " - %.1fkb: %s<br>" % (self.getObjSize( obj, hpy), cgi.escape(repr(obj))) objs = [(key, val) for key, val in sys.modules.iteritems() if val is not None] objs.sort() yield "<br>Modules (%s):<br>" % len(objs) for module_name, module in objs: yield " - %.3fkb: %s %s<br>" % (self.getObjSize( module, hpy), module_name, cgi.escape(repr(module))) gc.collect() # Implicit grabage collection yield "Done in %.1f" % (time.time() - s) def actionDumpobj(self): import gc import sys self.sendHeader() if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local: yield "This function is disabled on this proxy" raise StopIteration # No more if not in debug mode if not config.debug: yield "Not in debug mode" raise StopIteration class_filter = self.get.get("class") yield """ <style> * { font-family: monospace; white-space: pre } table * { text-align: right; padding: 0px 10px } </style> """ objs = gc.get_objects() for obj in objs: obj_type = str(type(obj)) if obj_type != "<type 'instance'>" or obj.__class__.__name__ != class_filter: continue yield "%.1fkb %s... " % (float(sys.getsizeof(obj)) / 1024, cgi.escape(str(obj))) for attr in dir(obj): yield "- %s: %s<br>" % (attr, cgi.escape(str(getattr(obj, attr)))) yield "<br>" gc.collect() # Implicit grabage collection def actionListobj(self): import gc import sys self.sendHeader() if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local: yield "This function is disabled on this proxy" raise StopIteration # No more if not in debug mode if not config.debug: yield "Not in debug mode" raise StopIteration type_filter = self.get.get("type") yield """ <style> * { font-family: monospace; white-space: pre } table * { text-align: right; padding: 0px 10px } </style> """ yield "Listing all %s objects in memory...<br>" % cgi.escape( type_filter) ref_count = {} objs = gc.get_objects() for obj in objs: obj_type = str(type(obj)) if obj_type != type_filter: continue refs = [ ref for ref in gc.get_referrers(obj) if hasattr(ref, "__class__") and ref.__class__.__name__ not in [ "list", "dict", "function", "type", "frame", "WeakSet", "tuple" ] ] if not refs: continue try: yield "%.1fkb <span title=\"%s\">%s</span>... " % ( float(sys.getsizeof(obj)) / 1024, cgi.escape( str(obj)), cgi.escape(str(obj)[0:100].ljust(100))) except: continue for ref in refs: yield " [" if "object at" in str(ref) or len(str(ref)) > 100: yield str(ref.__class__.__name__) else: yield str(ref.__class__.__name__) + ":" + cgi.escape( str(ref)) yield "] " ref_type = ref.__class__.__name__ if ref_type not in ref_count: ref_count[ref_type] = [0, 0] ref_count[ref_type][0] += 1 # Count ref_count[ref_type][1] += float( sys.getsizeof(obj)) / 1024 # Size yield "<br>" yield "<br>Object referrer (total: %s, %.2fkb):<br>" % ( len(ref_count), sum([stat[1] for stat in ref_count.values()])) for obj, stat in sorted(ref_count.items(), key=lambda x: x[1][0], reverse=True)[0:30]: # Sorted by count yield " - %.1fkb = %s x %s<br>" % (stat[1], stat[0], cgi.escape(str(obj))) gc.collect() # Implicit grabage collection def actionBenchmark(self): import sys import gc from contextlib import contextmanager output = self.sendHeader() if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local: yield "This function is disabled on this proxy" raise StopIteration @contextmanager def benchmark(name, standard): s = time.time() output("- %s" % name) try: yield 1 except Exception, err: output("<br><b>! Error: %s</b><br>" % err) taken = time.time() - s if taken > 0: multipler = standard / taken else: multipler = 99 if multipler < 0.3: speed = "Sloooow" elif multipler < 0.5: speed = "Ehh" elif multipler < 0.8: speed = "Goodish" elif multipler < 1.2: speed = "OK" elif multipler < 1.7: speed = "Fine" elif multipler < 2.5: speed = "Fast" elif multipler < 3.5: speed = "WOW" else: speed = "Insane!!" output("%.3fs [x%.2f: %s]<br>" % (taken, multipler, speed)) time.sleep(0.01) yield """ <style> * { font-family: monospace } table * { text-align: right; padding: 0px 10px } </style> """ yield "Benchmarking ZeroNet %s (rev%s) Python %s on: %s...<br>" % ( config.version, config.rev, sys.version, sys.platform) t = time.time() # CryptBitcoin yield "<br>CryptBitcoin:<br>" from Crypt import CryptBitcoin # seed = CryptBitcoin.newSeed() # yield "- Seed: %s<br>" % seed seed = "e180efa477c63b0f2757eac7b1cce781877177fe0966be62754ffd4c8592ce38" with benchmark("hdPrivatekey x 10", 0.7): for i in range(10): privatekey = CryptBitcoin.hdPrivatekey(seed, i * 10) yield "." valid = "5JsunC55XGVqFQj5kPGK4MWgTL26jKbnPhjnmchSNPo75XXCwtk" assert privatekey == valid, "%s != %s" % (privatekey, valid) data = "Hello" * 1024 # 5k with benchmark("sign x 10", 0.35): for i in range(10): yield "." sign = CryptBitcoin.sign(data, privatekey) valid = "G1GXaDauZ8vX/N9Jn+MRiGm9h+I94zUhDnNYFaqMGuOiBHB+kp4cRPZOL7l1yqK5BHa6J+W97bMjvTXtxzljp6w=" assert sign == valid, "%s != %s" % (sign, valid) address = CryptBitcoin.privatekeyToAddress(privatekey) if CryptBitcoin.opensslVerify: # Openssl avalible with benchmark("openssl verify x 100", 0.37): for i in range(100): if i % 10 == 0: yield "." ok = CryptBitcoin.verify(data, address, sign) assert ok, "does not verify from %s" % address else: yield " - openssl verify x 100...not avalible :(<br>" openssl_verify_bk = CryptBitcoin.opensslVerify # Emulate openssl not found in any way CryptBitcoin.opensslVerify = None with benchmark("pure-python verify x 10", 1.6): for i in range(10): yield "." ok = CryptBitcoin.verify(data, address, sign) assert ok, "does not verify from %s" % address CryptBitcoin.opensslVerify = openssl_verify_bk # CryptHash yield "<br>CryptHash:<br>" from Crypt import CryptHash from cStringIO import StringIO data = StringIO("Hello" * 1024 * 1024) # 5m with benchmark("sha256 5M x 10", 0.6): for i in range(10): data.seek(0) hash = CryptHash.sha256sum(data) yield "." valid = "8cd629d9d6aff6590da8b80782a5046d2673d5917b99d5603c3dcb4005c45ffa" assert hash == valid, "%s != %s" % (hash, valid) data = StringIO("Hello" * 1024 * 1024) # 5m with benchmark("sha512 5M x 10", 0.6): for i in range(10): data.seek(0) hash = CryptHash.sha512sum(data) yield "." valid = "9ca7e855d430964d5b55b114e95c6bbb114a6d478f6485df93044d87b108904d" assert hash == valid, "%s != %s" % (hash, valid) with benchmark("os.urandom(256) x 1000", 0.0065): for i in range(10): for y in range(100): data = os.urandom(256) yield "." # Msgpack import msgpack yield "<br>Msgpack: (version: %s)<br>" % ".".join( map(str, msgpack.version)) binary = 'fqv\xf0\x1a"e\x10,\xbe\x9cT\x9e(\xa5]u\x072C\x8c\x15\xa2\xa8\x93Sw)\x19\x02\xdd\t\xfb\xf67\x88\xd9\xee\x86\xa1\xe4\xb6,\xc6\x14\xbb\xd7$z\x1d\xb2\xda\x85\xf5\xa0\x97^\x01*\xaf\xd3\xb0!\xb7\x9d\xea\x89\xbbh8\xa1"\xa7]e(@\xa2\xa5g\xb7[\xae\x8eE\xc2\x9fL\xb6s\x19\x19\r\xc8\x04S\xd0N\xe4]?/\x01\xea\xf6\xec\xd1\xb3\xc2\x91\x86\xd7\xf4K\xdf\xc2lV\xf4\xe8\x80\xfc\x8ep\xbb\x82\xb3\x86\x98F\x1c\xecS\xc8\x15\xcf\xdc\xf1\xed\xfc\xd8\x18r\xf9\x80\x0f\xfa\x8cO\x97(\x0b]\xf1\xdd\r\xe7\xbf\xed\x06\xbd\x1b?\xc5\xa0\xd7a\x82\xf3\xa8\xe6@\xf3\ri\xa1\xb10\xf6\xd4W\xbc\x86\x1a\xbb\xfd\x94!bS\xdb\xaeM\x92\x00#\x0b\xf7\xad\xe9\xc2\x8e\x86\xbfi![%\xd31]\xc6\xfc2\xc9\xda\xc6v\x82P\xcc\xa9\xea\xb9\xff\xf6\xc8\x17iD\xcf\xf3\xeeI\x04\xe9\xa1\x19\xbb\x01\x92\xf5nn4K\xf8\xbb\xc6\x17e>\xa7 \xbbv' data = { "int": 1024 * 1024 * 1024, "float": 12345.67890, "text": "hello" * 1024, "binary": binary } with benchmark("pack 5K x 10 000", 0.78): for i in range(10): for y in range(1000): data_packed = msgpack.packb(data) yield "." valid = """\x84\xa3int\xce@\x00\x00\x00\xa4text\xda\x14\x00hellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohello\xa5float\xcb@\xc8\x1c\xd6\xe61\xf8\xa1\xa6binary\xda\x01\x00fqv\xf0\x1a"e\x10,\xbe\x9cT\x9e(\xa5]u\x072C\x8c\x15\xa2\xa8\x93Sw)\x19\x02\xdd\t\xfb\xf67\x88\xd9\xee\x86\xa1\xe4\xb6,\xc6\x14\xbb\xd7$z\x1d\xb2\xda\x85\xf5\xa0\x97^\x01*\xaf\xd3\xb0!\xb7\x9d\xea\x89\xbbh8\xa1"\xa7]e(@\xa2\xa5g\xb7[\xae\x8eE\xc2\x9fL\xb6s\x19\x19\r\xc8\x04S\xd0N\xe4]?/\x01\xea\xf6\xec\xd1\xb3\xc2\x91\x86\xd7\xf4K\xdf\xc2lV\xf4\xe8\x80\xfc\x8ep\xbb\x82\xb3\x86\x98F\x1c\xecS\xc8\x15\xcf\xdc\xf1\xed\xfc\xd8\x18r\xf9\x80\x0f\xfa\x8cO\x97(\x0b]\xf1\xdd\r\xe7\xbf\xed\x06\xbd\x1b?\xc5\xa0\xd7a\x82\xf3\xa8\xe6@\xf3\ri\xa1\xb10\xf6\xd4W\xbc\x86\x1a\xbb\xfd\x94!bS\xdb\xaeM\x92\x00#\x0b\xf7\xad\xe9\xc2\x8e\x86\xbfi![%\xd31]\xc6\xfc2\xc9\xda\xc6v\x82P\xcc\xa9\xea\xb9\xff\xf6\xc8\x17iD\xcf\xf3\xeeI\x04\xe9\xa1\x19\xbb\x01\x92\xf5nn4K\xf8\xbb\xc6\x17e>\xa7 \xbbv""" assert data_packed == valid, "%s<br>!=<br>%s" % (repr(data_packed), repr(valid)) with benchmark("unpack 5K x 10 000", 1.2): for i in range(10): for y in range(1000): data_unpacked = msgpack.unpackb(data_packed) yield "." assert data == data_unpacked, "%s != %s" % (data_unpacked, data) with benchmark("streaming unpack 5K x 10 000", 1.4): for i in range(10): unpacker = msgpack.Unpacker() for y in range(1000): unpacker.feed(data_packed) for data_unpacked in unpacker: pass yield "." assert data == data_unpacked, "%s != %s" % (data_unpacked, data) # Db from Db import Db import sqlite3 yield "<br>Db: (version: %s, API: %s)<br>" % (sqlite3.sqlite_version, sqlite3.version) schema = { "db_name": "TestDb", "db_file": "%s/benchmark.db" % config.data_dir, "maps": { ".*": { "to_table": { "test": "test" } } }, "tables": { "test": { "cols": [["test_id", "INTEGER"], ["title", "TEXT"], ["json_id", "INTEGER REFERENCES json (json_id)"]], "indexes": ["CREATE UNIQUE INDEX test_key ON test(test_id, json_id)"], "schema_changed": 1426195822 } } } if os.path.isfile("%s/benchmark.db" % config.data_dir): os.unlink("%s/benchmark.db" % config.data_dir) with benchmark("Open x 10", 0.13): for i in range(10): db = Db(schema, "%s/benchmark.db" % config.data_dir) db.checkTables() db.close() yield "." db = Db(schema, "%s/benchmark.db" % config.data_dir) db.checkTables() import json with benchmark("Insert x 10 x 1000", 1.0): for u in range(10): # 10 user data = {"test": []} for i in range(1000): # 1000 line of data data["test"].append({ "test_id": i, "title": "Testdata for %s message %s" % (u, i) }) json.dump(data, open("%s/test_%s.json" % (config.data_dir, u), "w")) db.updateJson("%s/test_%s.json" % (config.data_dir, u)) os.unlink("%s/test_%s.json" % (config.data_dir, u)) yield "." with benchmark("Buffered insert x 100 x 100", 1.3): cur = db.getCursor() cur.execute("BEGIN") cur.logging = False for u in range(100, 200): # 100 user data = {"test": []} for i in range(100): # 1000 line of data data["test"].append({ "test_id": i, "title": "Testdata for %s message %s" % (u, i) }) json.dump(data, open("%s/test_%s.json" % (config.data_dir, u), "w")) db.updateJson("%s/test_%s.json" % (config.data_dir, u), cur=cur) os.unlink("%s/test_%s.json" % (config.data_dir, u)) if u % 10 == 0: yield "." cur.execute("COMMIT") yield " - Total rows in db: %s<br>" % db.execute( "SELECT COUNT(*) AS num FROM test").fetchone()[0] with benchmark("Indexed query x 1000", 0.25): found = 0 cur = db.getCursor() cur.logging = False for i in range(1000): # 1000x by test_id res = cur.execute("SELECT * FROM test WHERE test_id = %s" % i) for row in res: found += 1 if i % 100 == 0: yield "." assert found == 20000, "Found: %s != 20000" % found with benchmark("Not indexed query x 100", 0.6): found = 0 cur = db.getCursor() cur.logging = False for i in range(100): # 1000x by test_id res = cur.execute("SELECT * FROM test WHERE json_id = %s" % i) for row in res: found += 1 if i % 10 == 0: yield "." assert found == 18900, "Found: %s != 18900" % found with benchmark("Like query x 100", 1.8): found = 0 cur = db.getCursor() cur.logging = False for i in range(100): # 1000x by test_id res = cur.execute( "SELECT * FROM test WHERE title LIKE '%%message %s%%'" % i) for row in res: found += 1 if i % 10 == 0: yield "." assert found == 38900, "Found: %s != 11000" % found db.close() if os.path.isfile("%s/benchmark.db" % config.data_dir): os.unlink("%s/benchmark.db" % config.data_dir) gc.collect() # Implicit grabage collection # Zip yield "<br>Compression:<br>" import zipfile test_data = "Test" * 1024 file_name = "\xc3\x81rv\xc3\xadzt\xc5\xb0r\xc5\x91t\xc3\xbck\xc3\xb6r\xc3\xb3g\xc3\xa9p\xe4\xb8\xad\xe5\x8d\x8e%s.txt" with benchmark("Zip pack x 10", 0.12): for i in range(10): with zipfile.ZipFile('%s/test.zip' % config.data_dir, 'w') as archive: for y in range(100): zip_info = zipfile.ZipInfo(file_name % y, (1980, 1, 1, 0, 0, 0)) zip_info.compress_type = zipfile.ZIP_DEFLATED zip_info.create_system = 3 archive.writestr(zip_info, test_data) yield "." hash = CryptHash.sha512sum( open("%s/test.zip" % config.data_dir, "rb")) valid = "f6ef623e6653883a1758db14aa593350e26c9dc53a8406d6e6defd6029dbd483" assert hash == valid, "Invalid hash: %s != %s<br>" % (hash, valid) with benchmark("Zip unpack x 10", 0.2): for i in range(10): with zipfile.ZipFile('%s/test.zip' % config.data_dir) as archive: for y in range(100): assert archive.read(file_name % y) == test_data yield "." if os.path.isfile("%s/test.zip" % config.data_dir): os.unlink("%s/test.zip" % config.data_dir) # Tar.gz import tarfile import struct # Monkey patch _init_write_gz to use fixed date in order to keep the hash independent from datetime def nodate_write_gzip_header(self): self.mtime = 0 original_write_gzip_header(self) import gzip original_write_gzip_header = gzip.GzipFile._write_gzip_header gzip.GzipFile._write_gzip_header = nodate_write_gzip_header test_data_io = StringIO("Test" * 1024) with benchmark("Tar.gz pack x 10", 0.3): for i in range(10): with tarfile.open('%s/test.tar.gz' % config.data_dir, 'w:gz') as archive: for y in range(100): test_data_io.seek(0) tar_info = tarfile.TarInfo(file_name % y) tar_info.size = 4 * 1024 archive.addfile(tar_info, test_data_io) yield "." hash = CryptHash.sha512sum( open("%s/test.tar.gz" % config.data_dir, "rb")) valid = "4704ebd8c987ed6f833059f1de9c475d443b0539b8d4c4cb8b49b26f7bbf2d19" assert hash == valid, "Invalid hash: %s != %s<br>" % (hash, valid) with benchmark("Tar.gz unpack x 10", 0.2): for i in range(10): with tarfile.open('%s/test.tar.gz' % config.data_dir, 'r:gz') as archive: for y in range(100): assert archive.extractfile(file_name % y).read() == test_data yield "." if os.path.isfile("%s/test.tar.gz" % config.data_dir): os.unlink("%s/test.tar.gz" % config.data_dir) # Tar.bz2 import tarfile test_data_io = StringIO("Test" * 1024) with benchmark("Tar.bz2 pack x 10", 2.0): for i in range(10): with tarfile.open('%s/test.tar.bz2' % config.data_dir, 'w:bz2') as archive: for y in range(100): test_data_io.seek(0) tar_info = tarfile.TarInfo(file_name % y) tar_info.size = 4 * 1024 archive.addfile(tar_info, test_data_io) yield "." hash = CryptHash.sha512sum( open("%s/test.tar.bz2" % config.data_dir, "rb")) valid = "90cba0b4d9abaa37b830bf37e4adba93bfd183e095b489ebee62aaa94339f3b5" assert hash == valid, "Invalid hash: %s != %s<br>" % (hash, valid) with benchmark("Tar.bz2 unpack x 10", 0.5): for i in range(10): with tarfile.open('%s/test.tar.bz2' % config.data_dir, 'r:bz2') as archive: for y in range(100): assert archive.extractfile(file_name % y).read() == test_data yield "." if os.path.isfile("%s/test.tar.bz2" % config.data_dir): os.unlink("%s/test.tar.bz2" % config.data_dir) yield "<br>Done. Total: %.2fs" % (time.time() - t)
def actionPeerBroadcast(self, params): ip = "%s:%s" % (self.connection.ip, self.connection.port) raw = json.loads(params["raw"]) # Check whether P2P messages are supported site = self.sites.get(raw["site"]) content_json = site.storage.loadJson("content.json") if "p2p_filter" not in content_json: self.connection.log("Site %s doesn't support P2P messages" % raw["site"]) self.connection.badAction(5) return # Was the message received yet? if params["hash"] in site.p2p_received: return site.p2p_received.append(params["hash"]) # Check whether the message matches passive filter if not SafeRe.match(content_json["p2p_filter"], json.dumps(raw["message"])): self.connection.log("Invalid message for site %s: %s" % (raw["site"], raw["message"])) self.connection.badAction(5) return # Not so fast if "p2p_freq_limit" in content_json and time.time( ) - site.p2p_last_recv.get(ip, 0) < content_json["p2p_freq_limit"]: self.connection.log("Too fast messages from %s" % raw["site"]) self.connection.badAction(2) return site.p2p_last_recv[ip] = time.time() # Not so much if "p2p_size_limit" in content_json and len(json.dumps( raw["message"])) > content_json["p2p_size_limit"]: self.connection.log("Too big message from %s" % raw["site"]) self.connection.badAction(7) return # Verify signature if params["signature"]: signature_address, signature = params["signature"].split("|") what = "%s|%s|%s" % (signature_address, params["hash"], params["raw"]) from Crypt import CryptBitcoin if not CryptBitcoin.verify(what, signature_address, signature): self.connection.log("Invalid signature") self.connection.badAction(7) return else: signature_address = "" # Check that the signature address is correct if "p2p_signed_only" in content_json: valid = content_json["p2p_signed_only"] if valid is True and not signature_address: self.connection.log("Not signed message") self.connection.badAction(5) return elif isinstance(valid, str) and signature_address != valid: self.connection.log( "Message signature is invalid: %s not in [%r]" % (signature_address, valid)) self.connection.badAction(5) return elif isinstance(valid, list) and signature_address not in valid: self.connection.log( "Message signature is invalid: %s not in %r" % (signature_address, valid)) self.connection.badAction(5) return # Send to WebSocket websockets = [ ws for ws in site.websockets if "peerReceive" in ws.channels ] for ws in websockets: ws.cmd( "peerReceive", { "ip": ip, "hash": params["hash"], "message": raw["message"], "signed_by": signature_address }) # Maybe active filter will reply? if websockets: # Wait for p2p_result result = gevent.spawn(self.p2pWaitMessage, site, params["hash"]).join() del site.p2p_result[params["hash"]] if not result: self.connection.badAction(10) return # Save to cache if not websockets and raw["immediate"]: site.p2p_unread.append({ "ip": "%s:%s" % (self.connection.ip, self.connection.port), "hash": params["hash"], "message": raw["message"], "signed_by": signature_address }) # Now send to neighbour peers if raw["broadcast"]: # Get peer list peers = site.getConnectedPeers() if len(peers) < raw[ "peer_count"]: # Add more, non-connected peers if necessary peers += site.getRecentPeers(raw["peer_count"] - len(peers)) # Send message to peers for peer in peers: gevent.spawn(peer.connection.request, "peerBroadcast", params)
def actionEcdsaVerify(self, to, data, address, signature): self.response(to, CryptBitcoin.verify(data, address, signature))
def testMissingCert(self, site): user_priv = "5Kk7FSA63FC2ViKmKLuBxk9gQkaQ5713hKq8LmFAf4cVeXh6K6A" cert_priv = "5JusJDSjHaMHwUjDT3o6eQ54pA6poo8La5fAgn1wNc3iK59jxjA" user_content = site.content_manager.contents[ "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json"] rules_content = site.content_manager.contents[ "data/users/content.json"] # Override valid cert signers for the test rules_content["user_contents"]["cert_signers"]["zeroid.bit"] = [ "14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet", "1iD5ZQJMNXu43w1qLB8sfdHVKppVMduGz" ] # Sign a valid cert user_content["cert_sign"] = CryptBitcoin.sign( "1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C#%s/%s" % (user_content["cert_auth_type"], user_content["cert_user_id"].split("@")[0]), cert_priv) signed_content = site.content_manager.sign( "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_priv, filewrite=False) assert site.content_manager.verifyFile( "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", io.BytesIO(json.dumps(signed_content).encode()), ignore_same=False) # Test invalid cert_user_id user_content["cert_user_id"] = "nodomain" user_content["signs"] = { "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(user_content, sort_keys=True), user_priv) } signed_content = site.content_manager.sign( "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_priv, filewrite=False) with pytest.raises(VerifyError) as err: site.content_manager.verifyFile( "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", io.BytesIO(json.dumps(signed_content).encode()), ignore_same=False) assert "Invalid domain in cert_user_id" in str(err.value) # Test removed cert del user_content["cert_user_id"] del user_content["cert_auth_type"] del user_content["signs"] # Remove signs before signing user_content["signs"] = { "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(user_content, sort_keys=True), user_priv) } signed_content = site.content_manager.sign( "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_priv, filewrite=False) with pytest.raises(VerifyError) as err: site.content_manager.verifyFile( "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", io.BytesIO(json.dumps(signed_content).encode()), ignore_same=False) assert "Missing cert_user_id" in str(err.value)
def sign(self, inner_path="content.json", privatekey=None, filewrite=True, update_changed_files=False): content = self.contents.get(inner_path) if not content: # Content not exits yet, load default one self.log.info("File %s not exits yet, loading default values..." % inner_path) content = {"files": {}, "signs": {}} # Default content.json if inner_path == "content.json": # Its the root content.json, add some more fields content["title"] = "%s - ZeroNet_" % self.site.address content["description"] = "" content["signs_required"] = 1 content["ignore"] = "" directory = self.toDir(self.site.storage.getPath(inner_path)) self.log.info("Opening site data directory: %s..." % directory) hashed_files = {} changed_files = [inner_path] for root, dirs, files in os.walk(directory): for file_name in files: file_path = self.site.storage.getPath( "%s/%s" % (root.strip("/"), file_name)) file_inner_path = re.sub(re.escape(directory), "", file_path) if file_name == "content.json" or ( content.get("ignore") and re.match(content["ignore"], file_inner_path) ) or file_name.startswith( "." ): # Ignore content.json, definied regexp and files starting with . self.log.info("- [SKIPPED] %s" % file_inner_path) else: sha512sum = CryptHash.sha512sum( file_path) # Calculate sha512 sum of file self.log.info("- %s (SHA512: %s)" % (file_inner_path, sha512sum)) hashed_files[file_inner_path] = { "sha512": sha512sum, "size": os.path.getsize(file_path) } if file_inner_path in content["files"].keys( ) and hashed_files[file_inner_path]["sha512"] != content[ "files"][file_inner_path].get("sha512"): changed_files.append(file_path) self.log.debug("Changed files: %s" % changed_files) if update_changed_files: for file_path in changed_files: self.site.storage.onUpdated(file_path) # Generate new content.json self.log.info("Adding timestamp and sha512sums to new content.json...") new_content = content.copy() # Create a copy of current content.json new_content["files"] = hashed_files # Add files sha512 hash new_content["modified"] = time.time() # Add timestamp if inner_path == "content.json": new_content["address"] = self.site.address new_content["zeronet_version"] = config.version new_content["signs_required"] = content.get("signs_required", 1) from Crypt import CryptBitcoin self.log.info("Verifying private key...") privatekey_address = CryptBitcoin.privatekeyToAddress(privatekey) valid_signers = self.getValidSigners(inner_path) if privatekey_address not in valid_signers: return self.log.error( "Private key invalid! Valid signers: %s, Private key address: %s" % (valid_signers, privatekey_address)) self.log.info("Correct %s in valid signers: %s" % (privatekey_address, valid_signers)) if inner_path == "content.json" and privatekey_address == self.site.address: # If signing using the root key sign the valid signers new_content["signers_sign"] = CryptBitcoin.sign( "%s:%s" % (new_content["signs_required"], ",".join(valid_signers)), privatekey) if not new_content["signers_sign"]: self.log.info("Old style address, signers_sign is none") self.log.info("Signing %s..." % inner_path) if "signs" in new_content: del (new_content["signs"]) # Delete old signs if "sign" in new_content: del (new_content["sign"] ) # Delete old sign (backward compatibility) sign_content = json.dumps(new_content, sort_keys=True) sign = CryptBitcoin.sign(sign_content, privatekey) #new_content["signs"] = content.get("signs", {}) # TODO: Multisig if sign: # If signing is successful (not an old address) new_content["signs"] = {} new_content["signs"][privatekey_address] = sign if inner_path == "content.json": # To root content.json add old format sign for backward compatibility oldsign_content = json.dumps(new_content, sort_keys=True) new_content["sign"] = CryptBitcoin.signOld(oldsign_content, privatekey) if not self.validContent(inner_path, new_content): self.log.error("Sign failed: Invalid content") return False if filewrite: self.log.info("Saving to %s..." % inner_path) json.dump(new_content, open(self.site.storage.getPath(inner_path), "w"), indent=2, sort_keys=True) self.log.info("File %s signed!" % inner_path) if filewrite: # Written to file return True else: # Return the new content return new_content
def cryptSign(self, message, privatekey): from Crypt import CryptBitcoin print CryptBitcoin.sign(message, privatekey)
def testVerify(self, site): privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv" # For 1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT user_inner_path = "data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/content.json" data_dict = site.storage.loadJson(user_inner_path) users_content = site.content_manager.contents[ "data/users/content.json"] data = io.BytesIO(json.dumps(data_dict).encode()) assert site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) # Test max size exception by setting allowed to 0 rules = site.content_manager.getRules(user_inner_path, data_dict) assert rules["max_size"] == 10000 assert users_content["user_contents"]["permission_rules"][".*"][ "max_size"] == 10000 users_content["user_contents"]["permission_rules"][".*"][ "max_size"] = 0 rules = site.content_manager.getRules(user_inner_path, data_dict) assert rules["max_size"] == 0 data = io.BytesIO(json.dumps(data_dict).encode()) with pytest.raises(VerifyError) as err: site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) assert "Include too large" in str(err.value) users_content["user_contents"]["permission_rules"][".*"][ "max_size"] = 10000 # Reset # Test max optional size exception # 1 MB gif = Allowed data_dict["files_optional"]["peanut-butter-jelly-time.gif"][ "size"] = 1024 * 1024 del data_dict["signs"] # Remove signs before signing data_dict["signs"] = { "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) } data = io.BytesIO(json.dumps(data_dict).encode()) assert site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) # 100 MB gif = Not allowed data_dict["files_optional"]["peanut-butter-jelly-time.gif"][ "size"] = 100 * 1024 * 1024 del data_dict["signs"] # Remove signs before signing data_dict["signs"] = { "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) } data = io.BytesIO(json.dumps(data_dict).encode()) with pytest.raises(VerifyError) as err: site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) assert "Include optional files too large" in str(err.value) data_dict["files_optional"]["peanut-butter-jelly-time.gif"][ "size"] = 1024 * 1024 # Reset # hello.exe = Not allowed data_dict["files_optional"]["hello.exe"] = data_dict["files_optional"][ "peanut-butter-jelly-time.gif"] del data_dict["signs"] # Remove signs before signing data_dict["signs"] = { "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) } data = io.BytesIO(json.dumps(data_dict).encode()) with pytest.raises(VerifyError) as err: site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) assert "Optional file not allowed" in str(err.value) del data_dict["files_optional"]["hello.exe"] # Reset # Includes not allowed in user content data_dict["includes"] = {"other.json": {}} del data_dict["signs"] # Remove signs before signing data_dict["signs"] = { "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey) } data = io.BytesIO(json.dumps(data_dict).encode()) with pytest.raises(VerifyError) as err: site.content_manager.verifyFile(user_inner_path, data, ignore_same=False) assert "Includes not allowed" in str(err.value)
def sign(self, inner_path="content.json", privatekey=None, filewrite=True, update_changed_files=False, extend=None): content = self.contents.get(inner_path) if not content: # Content not exist yet, load default one self.log.info("File %s not exist yet, loading default values..." % inner_path) content = {"files": {}, "signs": {}} # Default content.json if inner_path == "content.json": # It's the root content.json, add some more fields content["title"] = "%s - ZeroNet_" % self.site.address content["description"] = "" content["signs_required"] = 1 content["ignore"] = "" if extend: content.update(extend) # Add custom fields directory = helper.getDirname(self.site.storage.getPath(inner_path)) inner_directory = helper.getDirname(inner_path) self.log.info("Opening site data directory: %s..." % directory) changed_files = [inner_path] files_node, files_optional_node = self.hashFiles( helper.getDirname(inner_path), content.get("ignore"), content.get("optional") ) # Find changed files files_merged = files_node.copy() files_merged.update(files_optional_node) for file_relative_path, file_details in files_merged.iteritems(): old_hash = content["files"].get(file_relative_path, {}).get("sha512") new_hash = files_merged[file_relative_path]["sha512"] if old_hash != new_hash: changed_files.append(inner_directory + file_relative_path) self.log.debug("Changed files: %s" % changed_files) if update_changed_files: for file_path in changed_files: self.site.storage.onUpdated(file_path) # Generate new content.json self.log.info("Adding timestamp and sha512sums to new content.json...") new_content = content.copy() # Create a copy of current content.json new_content["files"] = files_node # Add files sha512 hash if files_optional_node: new_content["files_optional"] = files_optional_node elif "files_optional" in new_content: del new_content["files_optional"] new_content["modified"] = time.time() # Add timestamp if inner_path == "content.json": new_content["zeronet_version"] = config.version new_content["signs_required"] = content.get("signs_required", 1) new_content["address"] = self.site.address new_content["inner_path"] = inner_path # Verify private key from Crypt import CryptBitcoin self.log.info("Verifying private key...") privatekey_address = CryptBitcoin.privatekeyToAddress(privatekey) valid_signers = self.getValidSigners(inner_path, new_content) if privatekey_address not in valid_signers: return self.log.error( "Private key invalid! Valid signers: %s, Private key address: %s" % (valid_signers, privatekey_address) ) self.log.info("Correct %s in valid signers: %s" % (privatekey_address, valid_signers)) if inner_path == "content.json" and privatekey_address == self.site.address: # If signing using the root key, then sign the valid signers new_content["signers_sign"] = CryptBitcoin.sign( "%s:%s" % (new_content["signs_required"], ",".join(valid_signers)), privatekey ) if not new_content["signers_sign"]: self.log.info("Old style address, signers_sign is none") self.log.info("Signing %s..." % inner_path) if "signs" in new_content: del(new_content["signs"]) # Delete old signs if "sign" in new_content: del(new_content["sign"]) # Delete old sign (backward compatibility) sign_content = json.dumps(new_content, sort_keys=True) sign = CryptBitcoin.sign(sign_content, privatekey) # new_content["signs"] = content.get("signs", {}) # TODO: Multisig if sign: # If signing is successful (not an old address) new_content["signs"] = {} new_content["signs"][privatekey_address] = sign if inner_path == "content.json": # To root content.json add old format sign for backward compatibility oldsign_content = json.dumps(new_content, sort_keys=True) new_content["sign"] = CryptBitcoin.signOld(oldsign_content, privatekey) if not self.verifyContent(inner_path, new_content): self.log.error("Sign failed: Invalid content") return False if filewrite: self.log.info("Saving to %s..." % inner_path) self.site.storage.writeJson(inner_path, new_content) self.contents[inner_path] = new_content self.log.info("File %s signed!" % inner_path) if filewrite: # Written to file return True else: # Return the new content return new_content
def cryptGetPrivatekey(self, master_seed, site_address_index=None): from Crypt import CryptBitcoin print CryptBitcoin.hdPrivatekey(master_seed, site_address_index)
def verifyFile(self, inner_path, file, ignore_same=True): if inner_path.endswith("content.json"): # content.json: Check using sign from Crypt import CryptBitcoin try: new_content = json.load(file) if inner_path in self.contents: old_content = self.contents.get(inner_path) # Checks if its newer the ours if old_content["modified"] == new_content["modified"] and ignore_same: # Ignore, have the same content.json return None elif old_content["modified"] > new_content["modified"]: # We have newer self.log.debug( "We have newer %s (Our: %s, Sent: %s)" % (inner_path, old_content["modified"], new_content["modified"]) ) # gevent.spawn(self.site.publish, inner_path=inner_path) # Try to fix the broken peers return False if new_content["modified"] > time.time() + 60 * 60 * 24: # Content modified in the far future (allow 1 day+) self.log.error("%s modify is in the future!" % inner_path) return False # Check sign sign = new_content.get("sign") signs = new_content.get("signs", {}) if "sign" in new_content: del(new_content["sign"]) # The file signed without the sign if "signs" in new_content: del(new_content["signs"]) # The file signed without the signs sign_content = json.dumps(new_content, sort_keys=True) # Dump the json to string to remove whitepsace if not self.verifyContent(inner_path, new_content): return False # Content not valid (files too large, invalid files) if signs: # New style signing valid_signers = self.getValidSigners(inner_path, new_content) signs_required = self.getSignsRequired(inner_path, new_content) if inner_path == "content.json" and len(valid_signers) > 1: # Check signers_sign on root content.json if not CryptBitcoin.verify( "%s:%s" % (signs_required, ",".join(valid_signers)), self.site.address, new_content["signers_sign"] ): self.log.error("%s invalid signers_sign!" % inner_path) return False if inner_path != "content.json" and not self.verifyCert(inner_path, new_content): # Check if cert valid self.log.error("%s invalid cert!" % inner_path) return False valid_signs = 0 for address in valid_signers: if address in signs: valid_signs += CryptBitcoin.verify(sign_content, address, signs[address]) if valid_signs >= signs_required: break # Break if we has enough signs self.log.debug("%s: Valid signs: %s/%s" % (inner_path, valid_signs, signs_required)) return valid_signs >= signs_required else: # Old style signing return CryptBitcoin.verify(sign_content, self.site.address, sign) except Exception, err: self.log.error("Verify sign error: %s" % Debug.formatException(err)) return False
def signContent(self, privatekey=None): if not self.content: # New site self.log.info( "Site not exits yet, loading default content.json values...") self.content = { "files": {}, "title": "%s - ZeroNet_" % self.address, "sign": "", "modified": 0.0, "description": "", "address": self.address, "ignore": "", "zeronet_version": config.version } # Default content.json self.log.info("Opening site data directory: %s..." % self.directory) hashed_files = {} for root, dirs, files in os.walk(self.directory): for file_name in files: file_path = self.getPath("%s/%s" % (root, file_name)) if file_name == "content.json" or ( self.content["ignore"] and re.match( self.content["ignore"], file_path.replace(self.directory + "/", "")) ): # Dont add content.json and ignore regexp pattern definied in content.json self.log.info("- [SKIPPED] %s" % file_path) else: sha1sum = CryptHash.sha1sum( file_path) # Calculate sha1 sum of file sha512sum = CryptHash.sha512sum( file_path) # Calculate sha512 sum of file inner_path = re.sub("^%s/" % re.escape(self.directory), "", file_path) self.log.info("- %s (SHA512: %s)" % (file_path, sha512sum)) hashed_files[inner_path] = { "sha1": sha1sum, "sha512": sha512sum, "size": os.path.getsize(file_path) } # Generate new content.json self.log.info("Adding timestamp and sha512sums to new content.json...") content = self.content.copy() # Create a copy of current content.json content["address"] = self.address content["files"] = hashed_files # Add files sha512 hash content["modified"] = time.time() # Add timestamp content["zeronet_version"] = config.version # Signer's zeronet version del (content["sign"]) # Delete old sign # Signing content from Crypt import CryptBitcoin self.log.info("Verifying private key...") privatekey_address = CryptBitcoin.privatekeyToAddress(privatekey) if self.address != privatekey_address: return self.log.error( "Private key invalid! Site address: %s, Private key address: %s" % (self.address, privatekey_address)) self.log.info("Signing modified content.json...") sign_content = json.dumps(content, sort_keys=True) sign = CryptBitcoin.sign(sign_content, privatekey) content["sign"] = sign # Saving modified content.json self.log.info("Saving to %s/content.json..." % self.directory) open("%s/content.json" % self.directory, "w").write(json.dumps(content, indent=4, sort_keys=True)) self.log.info("Site signed!") return True