Exemple #1
0
    def testVerifyAddress(self, site):
        privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv"  # For 1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT
        user_inner_path = "data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/content.json"
        data_dict = site.storage.loadJson(user_inner_path)
        users_content = site.content_manager.contents["data/users/content.json"]

        data = StringIO(json.dumps(data_dict))
        assert site.content_manager.verifyFile(user_inner_path, data, ignore_same=False)

        # Test error on 15k data.json
        data_dict["files"]["data.json"]["size"] = 1024 * 15
        del data_dict["signs"]  # Remove signs before signing
        data_dict["signs"] = {
            "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey)
        }
        data = StringIO(json.dumps(data_dict))
        with pytest.raises(VerifyError) as err:
            site.content_manager.verifyFile(user_inner_path, data, ignore_same=False)
            assert "Content too large" in str(err)

        # Give more space based on address
        users_content["user_contents"]["permissions"]["1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9"] = {"max_size": 20000}
        del data_dict["signs"]  # Remove signs before signing
        data_dict["signs"] = {
            "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey)
        }
        data = StringIO(json.dumps(data_dict))
        assert site.content_manager.verifyFile(user_inner_path, data, ignore_same=False)
Exemple #2
0
    def siteCreate(self):
        logging.info("Generating new privatekey...")
        from Crypt import CryptBitcoin
        privatekey = CryptBitcoin.newPrivatekey()
        logging.info("----------------------------------------------------------------------")
        logging.info("Site private key: %s" % privatekey)
        logging.info("                  !!! ^ Save it now, required to modify the site ^ !!!")
        address = CryptBitcoin.privatekeyToAddress(privatekey)
        logging.info("Site address:     %s" % address)
        logging.info("----------------------------------------------------------------------")

        while True and not config.batch:
            if raw_input("? Have you secured your private key? (yes, no) > ").lower() == "yes":
                break
            else:
                logging.info("Please, secure it now, you going to need it to modify your site!")

        logging.info("Creating directory structure...")
        from Site import Site
        from Site import SiteManager
        SiteManager.site_manager.load()

        os.mkdir("%s/%s" % (config.data_dir, address))
        open("%s/%s/index.html" % (config.data_dir, address), "w").write("Hello %s!" % address)

        logging.info("Creating content.json...")
        site = Site(address)
        site.content_manager.sign(privatekey=privatekey, extend={"postmessage_nonce_security": True})
        site.settings["own"] = True
        site.saveSettings()

        logging.info("Site created!")
Exemple #3
0
    def testVerifyInnerPath(self, site):
        inner_path = "content.json"
        data_dict = site.storage.loadJson(inner_path)

        for good_relative_path in ["data.json", "out/data.json", "Any File [by none] (1).jpg"]:
            data_dict["files"] = {good_relative_path: {"sha512": "369d4e780cc80504285f13774ca327fe725eed2d813aad229e62356b07365906", "size": 505}}

            if "sign" in data_dict:
                del data_dict["sign"]
            del data_dict["signs"]
            data_dict["signs"] = {
                "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), self.privatekey)
            }
            data = StringIO(json.dumps(data_dict))
            assert site.content_manager.verifyFile(inner_path, data, ignore_same=False)

        for bad_relative_path in ["../data.json", "data/" * 100, "invalid|file.jpg"]:
            data_dict["files"] = {bad_relative_path: {"sha512": "369d4e780cc80504285f13774ca327fe725eed2d813aad229e62356b07365906", "size": 505}}

            if "sign" in data_dict:
                del data_dict["sign"]
            del data_dict["signs"]
            data_dict["signs"] = {
                "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), self.privatekey)
            }
            data = StringIO(json.dumps(data_dict))
            with pytest.raises(VerifyError) as err:
                site.content_manager.verifyFile(inner_path, data, ignore_same=False)
            assert "Invalid relative path" in str(err)
Exemple #4
0
def siteCreate():
	logging.info("Generating new privatekey...")
	from Crypt import CryptBitcoin
	privatekey = CryptBitcoin.newPrivatekey()
	logging.info("----------------------------------------------------------------------")
	logging.info("Site private key: %s" % privatekey)
	logging.info("                  !!! ^ Save it now, required to modify the site ^ !!!")
	address = CryptBitcoin.privatekeyToAddress(privatekey)
	logging.info("Site address:     %s" % address)
	logging.info("----------------------------------------------------------------------")

	while True:
		if raw_input("? Have you secured your private key? (yes, no) > ").lower() == "yes": break
		else: logging.info("Please, secure it now, you going to need it to modify your site!")

	logging.info("Creating directory structure...")
	from Site import Site
	os.mkdir("data/%s" % address)
	open("data/%s/index.html" % address, "w").write("Hello %s!" % address)

	logging.info("Creating content.json...")
	site = Site(address)
	site.content_manager.sign(privatekey=privatekey)
	site.settings["own"] = True
	site.saveSettings()

	logging.info("Site created!")
Exemple #5
0
    def create_phantom_site(self):

        from Config import config
        from Crypt import CryptBitcoin
        from Site import Site
        import os

        config.parse(silent=True)
        if not config.arguments:
            config.parse()

        self.private_key = CryptBitcoin.newPrivatekey()
        self.address = CryptBitcoin.privatekeyToAddress(self.private_key)

        try:
            os.mkdir("%s/%s" % (config.data_dir, self.address))
            open("%s/%s/index.html" % (config.data_dir, self.address), "w").write("Hello %s!" % self.address)
        except Exception as e:
            return self.error_msg.error_response("err_create_sitedir")
    
        try:
            self.site = Site(self.address)
            self.site.content_manager.sign(privatekey=self.private_key, extend={"postmessage_nonce_security": True})
            self.site.settings["own"] = True
            self.site.saveSettings()
        except Exception as e:
            print e
            return self.error_msg.error_response("err_create_site")
        return {"jsonrpc": "2.0", "id": "1", "result": ["true", str(self.address), str(self.private_key)]}
Exemple #6
0
    def cryptPrivatekeyToAddress(self, privatekey=None):
        from Crypt import CryptBitcoin
        if not privatekey:  # If no privatekey in args then ask it now
            import getpass
            privatekey = getpass.getpass("Private key (input hidden):")

        print CryptBitcoin.privatekeyToAddress(privatekey)
Exemple #7
0
    def testVerify(self, site):
        privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv"  # For 1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT
        user_inner_path = "data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/content.json"
        data_dict = site.content_manager.contents[user_inner_path]
        users_content = site.content_manager.contents["data/users/content.json"]

        data = StringIO(json.dumps(data_dict))
        assert site.content_manager.verifyFile(user_inner_path, data, ignore_same=False)

        # Test max size exception by setting allowed to 0
        rules = site.content_manager.getRules(user_inner_path, data_dict)
        assert rules["max_size"] == 10000
        assert users_content["user_contents"]["permission_rules"][".*"]["max_size"] == 10000

        users_content["user_contents"]["permission_rules"][".*"]["max_size"] = 0
        rules = site.content_manager.getRules(user_inner_path, data_dict)
        assert rules["max_size"] == 0
        data = StringIO(json.dumps(data_dict))
        assert not site.content_manager.verifyFile(user_inner_path, data, ignore_same=False)
        users_content["user_contents"]["permission_rules"][".*"]["max_size"] = 10000  # Reset

        # Test max optional size exception
        # 1 MB gif = Allowed
        data_dict["files_optional"]["peanut-butter-jelly-time.gif"]["size"] = 1024 * 1024
        del data_dict["signs"]  # Remove signs before signing
        data_dict["signs"] = {
            "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey)
        }
        data = StringIO(json.dumps(data_dict))
        assert site.content_manager.verifyFile(user_inner_path, data, ignore_same=False)

        # 100 MB gif = Not allowed
        data_dict["files_optional"]["peanut-butter-jelly-time.gif"]["size"] = 100 * 1024 * 1024
        del data_dict["signs"]  # Remove signs before signing
        data_dict["signs"] = {
            "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey)
        }
        data = StringIO(json.dumps(data_dict))
        assert not site.content_manager.verifyFile(user_inner_path, data, ignore_same=False)
        data_dict["files_optional"]["peanut-butter-jelly-time.gif"]["size"] = 1024 * 1024  # Reset

        # hello.exe = Not allowed
        data_dict["files_optional"]["hello.exe"] = data_dict["files_optional"]["peanut-butter-jelly-time.gif"]
        del data_dict["signs"]  # Remove signs before signing
        data_dict["signs"] = {
            "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey)
        }
        data = StringIO(json.dumps(data_dict))
        assert not site.content_manager.verifyFile(user_inner_path, data, ignore_same=False)
        del data_dict["files_optional"]["hello.exe"]  # Reset

        # Includes not allowed in user content
        data_dict["includes"] = { "other.json": { } }
        del data_dict["signs"]  # Remove signs before signing
        data_dict["signs"] = {
            "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey)
        }
        data = StringIO(json.dumps(data_dict))
        assert not site.content_manager.verifyFile(user_inner_path, data, ignore_same=False)
Exemple #8
0
    def testUserContentCert(self):
        from Site import Site
        from cStringIO import StringIO
        import json
        # user_addr = "1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C"
        user_priv = "5Kk7FSA63FC2ViKmKLuBxk9gQkaQ5713hKq8LmFAf4cVeXh6K6A"
        # cert_addr = "14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet"
        cert_priv = "5JusJDSjHaMHwUjDT3o6eQ54pA6poo8La5fAgn1wNc3iK59jxjA"

        site = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT")
        # user_content = site.storage.loadJson("data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json")
        # site.content_manager.contents["data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json"] = user_content # Add to content manager
        # Check if the user file is loaded
        self.assertTrue("data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json" in site.content_manager.contents)
        user_content = site.content_manager.contents["data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json"]
        cert_content = site.content_manager.contents["data/users/content.json"]
        # Override cert signer
        cert_content["user_contents"]["cert_signers"]["zeroid.bit"] = ["14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet", "1iD5ZQJMNXu43w1qLB8sfdHVKppVMduGz"]

        # Valid cert providers
        rules = site.content_manager.getRules("data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_content)
        self.assertEqual(rules["cert_signers"], {"zeroid.bit": ["14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet", "1iD5ZQJMNXu43w1qLB8sfdHVKppVMduGz"]})

        # Add cert
        user_content["cert_sign"] = CryptBitcoin.sign(
            "1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C#%s/%s" % (user_content["cert_auth_type"], user_content["cert_user_id"].split("@")[0]), cert_priv
        )

        # Verify cert
        self.assertTrue(site.content_manager.verifyCert("data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_content))
        self.assertFalse(site.content_manager.verifyCert("data/users/badaddress/content.json", user_content))

        # Sign user content
        signed_content = site.content_manager.sign("data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_priv, filewrite=False)

        # Test user cert
        self.assertTrue(site.content_manager.verifyFile(
            "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json",
            StringIO(json.dumps(signed_content)), ignore_same=False
        ))

        # Test banned user
        site.content_manager.contents["data/users/content.json"]["user_contents"]["permissions"][user_content["cert_user_id"]] = False
        self.assertFalse(site.content_manager.verifyFile(
            "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json",
            StringIO(json.dumps(signed_content)), ignore_same=False
        ))

        # Test invalid cert
        user_content["cert_sign"] = CryptBitcoin.sign(
            "badaddress#%s/%s" % (user_content["cert_auth_type"], user_content["cert_user_id"]), cert_priv
        )
        signed_content = site.content_manager.sign("data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_priv, filewrite=False)
        self.assertFalse(site.content_manager.verifyFile(
            "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json",
            StringIO(json.dumps(signed_content)), ignore_same=False
        ))
Exemple #9
0
    def verifyFile(self, inner_path, file, ignore_same=True):
        if inner_path.endswith("content.json"):  # content.json: Check using sign
            from Crypt import CryptBitcoin
            try:
                new_content = json.load(file)
                if inner_path in self.contents:
                    old_content = self.contents.get(inner_path)
                    # Checks if its newer the ours
                    if old_content["modified"] == new_content["modified"] and ignore_same:  # Ignore, have the same content.json
                        return None
                    elif old_content["modified"] > new_content["modified"]:  # We have newer
                        self.log.debug("We have newer %s (Our: %s, Sent: %s)" % (inner_path, old_content["modified"], new_content["modified"]))
                        gevent.spawn(self.site.publish, inner_path=inner_path)  # Try to fix the broken peers
                        return False
                if new_content["modified"] > time.time() + 60 * 60 * 24:  # Content modified in the far future (allow 1 day window)
                    self.log.error("%s modify is in the future!" % inner_path)
                    return False
                # Check sign
                sign = new_content.get("sign")
                signs = new_content.get("signs", {})
                if "sign" in new_content:
                    del(new_content["sign"])  # The file signed without the sign
                if "signs" in new_content:
                    del(new_content["signs"])  # The file signed without the signs
                sign_content = json.dumps(new_content, sort_keys=True)  # Dump the json to string to remove whitepsace

                if not self.validContent(inner_path, new_content):
                    return False  # Content not valid (files too large, invalid files)

                if signs:  # New style signing
                    valid_signers = self.getValidSigners(inner_path, new_content)
                    signs_required = self.getSignsRequired(inner_path, new_content)

                    if inner_path == "content.json" and len(valid_signers) > 1:  # Check signers_sign on root content.json
                        if not CryptBitcoin.verify("%s:%s" % (signs_required, ",".join(valid_signers)), self.site.address, new_content["signers_sign"]):
                            self.log.error("%s invalid signers_sign!" % inner_path)
                            return False

                    if inner_path != "content.json" and not self.verifyCert(inner_path, new_content):  # Check if cert valid
                        self.log.error("%s invalid cert!" % inner_path)
                        return False

                    valid_signs = 0
                    for address in valid_signers:
                        if address in signs:
                            valid_signs += CryptBitcoin.verify(sign_content, address, signs[address])
                        if valid_signs >= signs_required:
                            break  # Break if we has enough signs

                    return valid_signs >= signs_required
                else:  # Old style signing
                    return CryptBitcoin.verify(sign_content, self.site.address, sign)

            except Exception, err:
                self.log.error("Verify sign error: %s" % Debug.formatException(err))
                return False
Exemple #10
0
 def generateAuthAddress(self, address):
     s = time.time()
     address_id = self.getAddressAuthIndex(address)  # Convert site address to int
     auth_privatekey = CryptBitcoin.hdPrivatekey(self.master_seed, address_id)
     self.sites[address] = {
         "auth_address": CryptBitcoin.privatekeyToAddress(auth_privatekey),
         "auth_privatekey": auth_privatekey
     }
     self.saveDelayed()
     self.log.debug("Added new site: %s in %.3fs" % (address, time.time() - s))
     return self.sites[address]
Exemple #11
0
 def getNewSiteData(self):
     import random
     bip32_index = random.randrange(2**256) % 100000000
     site_privatekey = CryptBitcoin.hdPrivatekey(self.master_seed, bip32_index)
     site_address = CryptBitcoin.privatekeyToAddress(site_privatekey)
     if site_address in self.sites: raise Exception("Random error: site exist!")
     # Save to sites
     self.getSiteData(site_address)
     self.sites[site_address]["privatekey"] = site_privatekey
     self.save()
     return site_address, bip32_index, self.sites[site_address]
Exemple #12
0
	def __init__(self, master_address=None, master_seed=None):
		if master_seed:
			self.master_seed = master_seed
			self.master_address = CryptBitcoin.privatekeyToAddress(self.master_seed)
		elif master_address:
			self.master_address = master_address
			self.master_seed = None
		else:
			self.master_seed = CryptBitcoin.newSeed()
			self.master_address = CryptBitcoin.privatekeyToAddress(self.master_seed)
		self.sites = {}
		self.log = logging.getLogger("User:%s" % self.master_address)
Exemple #13
0
	def getSiteData(self, address, create=True):
		if not address in self.sites: # Genreate new BIP32 child key based on site address
			if not create: return {"auth_address": None, "auth_privatekey": None} # Dont create user yet
			s = time.time()
			address_id = int(address.encode("hex"), 16) # Convert site address to int
			auth_privatekey = CryptBitcoin.hdPrivatekey(self.master_seed, address_id)
			self.sites[address] = {
				"auth_address": CryptBitcoin.privatekeyToAddress(auth_privatekey),
				"auth_privatekey": auth_privatekey
			}
			self.save()
			self.log.debug("Added new site: %s in %.3fs" % (address, time.time()-s))
		return self.sites[address]
Exemple #14
0
    def __init__(self, master_address=None, master_seed=None, data={}):
        if master_seed:
            self.master_seed = master_seed
            self.master_address = CryptBitcoin.privatekeyToAddress(self.master_seed)
        elif master_address:
            self.master_address = master_address
            self.master_seed = data.get("master_seed")
        else:
            self.master_seed = CryptBitcoin.newSeed()
            self.master_address = CryptBitcoin.privatekeyToAddress(self.master_seed)
        self.sites = data.get("sites", {})
        self.certs = data.get("certs", {})

        self.log = logging.getLogger("User:%s" % self.master_address)
Exemple #15
0
 def testNewSeed(self):
     assert CryptBitcoin.newSeed() != CryptBitcoin.newSeed()
     assert CryptBitcoin.privatekeyToAddress(
         CryptBitcoin.hdPrivatekey(CryptBitcoin.newSeed(), 0)
     )
     assert CryptBitcoin.privatekeyToAddress(
         CryptBitcoin.hdPrivatekey(CryptBitcoin.newSeed(), 2**256)
     )
Exemple #16
0
    def testInlcudeLimits(self, site):
        privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv"
        # Data validation
        data_dict = {
            "files": {
                "data.json": {
                    "sha512": "369d4e780cc80504285f13774ca327fe725eed2d813aad229e62356b07365906",
                    "size": 505
                }
            },
            "modified": time.time()
        }

        # Normal data
        data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey)}
        data = StringIO(json.dumps(data_dict))
        assert site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False)
        # Reset
        del data_dict["signs"]

        # Too large
        data_dict["files"]["data.json"]["size"] = 200000  # Emulate 2MB sized data.json
        data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey)}
        data = StringIO(json.dumps(data_dict))
        with pytest.raises(VerifyError) as err:
            site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False)
            assert "Include too large" in str(err)

        # Reset
        data_dict["files"]["data.json"]["size"] = 505
        del data_dict["signs"]

        # Not allowed file
        data_dict["files"]["notallowed.exe"] = data_dict["files"]["data.json"]
        data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey)}
        data = StringIO(json.dumps(data_dict))
        with pytest.raises(VerifyError) as err:
            site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False)
            assert "File not allowed" in str(err)

        # Reset
        del data_dict["files"]["notallowed.exe"]
        del data_dict["signs"]

        # Should work again
        data_dict["signs"] = {"1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict), privatekey)}
        data = StringIO(json.dumps(data_dict))
        assert site.content_manager.verifyFile("data/test_include/content.json", data, ignore_same=False)
Exemple #17
0
    def verifyCert(self, inner_path, content):
        from Crypt import CryptBitcoin

        rules = self.getRules(inner_path, content)

        if not rules:
            raise VerifyError("No rules for this file")

        if not rules.get("cert_signers") and not rules.get("cert_signers_pattern"):
            return True  # Does not need cert

        if "cert_user_id" not in content:
            raise VerifyError("Missing cert_user_id")

        if content["cert_user_id"].count("@") != 1:
            raise VerifyError("Invalid domain in cert_user_id")

        name, domain = content["cert_user_id"].rsplit("@", 1)
        cert_address = rules["cert_signers"].get(domain)
        if not cert_address:  # Unknown Cert signer
            if rules.get("cert_signers_pattern") and SafeRe.match(rules["cert_signers_pattern"], domain):
                cert_address = domain
            else:
                raise VerifyError("Invalid cert signer: %s" % domain)

        try:
            cert_subject = "%s#%s/%s" % (rules["user_address"], content["cert_auth_type"], name)
            result = CryptBitcoin.verify(cert_subject, cert_address, content["cert_sign"])
        except Exception, err:
            raise VerifyError("Certificate verify error: %s" % err)
Exemple #18
0
    def testVerify(self, site):
        privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv"
        inner_path = "data/test_include/content.json"
        data_dict = site.storage.loadJson(inner_path)
        data = StringIO(json.dumps(data_dict))

        # Re-sign
        data_dict["signs"] = {
            "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey)
        }
        assert site.content_manager.verifyFile(inner_path, data, ignore_same=False)

        # Wrong address
        data_dict["address"] = "Othersite"
        del data_dict["signs"]
        data_dict["signs"] = {
            "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey)
        }
        data = StringIO(json.dumps(data_dict))
        with pytest.raises(VerifyError) as err:
            site.content_manager.verifyFile(inner_path, data, ignore_same=False)
            assert "Wrong site address" in str(err)

        # Wrong inner_path
        data_dict["address"] = "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT"
        data_dict["inner_path"] = "content.json"
        del data_dict["signs"]
        data_dict["signs"] = {
            "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey)
        }
        data = StringIO(json.dumps(data_dict))
        with pytest.raises(VerifyError) as err:
            site.content_manager.verifyFile(inner_path, data, ignore_same=False)
            assert "Wrong inner_path" in str(err)

        # Everything right again
        data_dict["address"] = "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT"
        data_dict["inner_path"] = inner_path
        del data_dict["signs"]
        data_dict["signs"] = {
            "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True), privatekey)
        }
        data = StringIO(json.dumps(data_dict))
        assert site.content_manager.verifyFile(inner_path, data, ignore_same=False)
Exemple #19
0
    def testNewSite(self, user):
        address, address_index, site_data = user.getNewSiteData()  # Create a new random site
        assert CryptBitcoin.hdPrivatekey(user.master_seed, address_index) == site_data["privatekey"]

        user.sites = {}  # Reset user data

        # Site address and auth address is different
        assert user.getSiteData(address)["auth_address"] != address
        # Re-generate auth_privatekey for site
        assert user.getSiteData(address)["auth_privatekey"] == site_data["auth_privatekey"]
Exemple #20
0
	def verifyCert(self, inner_path, content):
		from Crypt import CryptBitcoin

		rules = self.getRules(inner_path, content)
		if not rules.get("cert_signers"): return True # Does not need cert

		name, domain = content["cert_user_id"].split("@")
		cert_address = rules["cert_signers"].get(domain)
		if not cert_address: # Cert signer not allowed
			self.log.error("Invalid cert signer: %s" % domain)
			return False
		return CryptBitcoin.verify("%s#%s/%s" % (rules["user_address"], content["cert_auth_type"], name), cert_address, content["cert_sign"])
Exemple #21
0
    def testMissingCert(self, site):
        user_priv = "5Kk7FSA63FC2ViKmKLuBxk9gQkaQ5713hKq8LmFAf4cVeXh6K6A"
        cert_priv = "5JusJDSjHaMHwUjDT3o6eQ54pA6poo8La5fAgn1wNc3iK59jxjA"

        user_content = site.content_manager.contents["data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json"]
        rules_content = site.content_manager.contents["data/users/content.json"]

        # Override valid cert signers for the test
        rules_content["user_contents"]["cert_signers"]["zeroid.bit"] = [
            "14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet",
            "1iD5ZQJMNXu43w1qLB8sfdHVKppVMduGz"
        ]

        # Sign a valid cert
        user_content["cert_sign"] = CryptBitcoin.sign("1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C#%s/%s" % (
            user_content["cert_auth_type"],
            user_content["cert_user_id"].split("@")[0]
        ), cert_priv)
        signed_content = site.content_manager.sign(
            "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json", user_priv, filewrite=False
        )

        assert site.content_manager.verifyFile(
            "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json",
            StringIO(json.dumps(signed_content)), ignore_same=False
        )

        # Test removed cert
        del user_content["cert_user_id"]
        del user_content["cert_auth_type"]
        del user_content["signs"]  # Remove signs before signing
        user_content["signs"] = {
            "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT": CryptBitcoin.sign(json.dumps(user_content, sort_keys=True), user_priv)
        }
        with pytest.raises(VerifyError) as err:
            site.content_manager.verifyFile(
                "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json",
                StringIO(json.dumps(user_content)), ignore_same=False
            )
            assert "Missing cert_user_id" in str(err)
Exemple #22
0
 def responseUserLogin(self, master_seed):
     user_manager = sys.modules["User.UserManager"].user_manager
     user = user_manager.get(CryptBitcoin.privatekeyToAddress(master_seed))
     if not user:
         user = user_manager.create(master_seed=master_seed)
     if user.master_address:
         message = "Successfull login, reloading page..."
         message += "<script>document.cookie = 'master_address=%s;path=/;max-age=2592000;'</script>" % user.master_address
         message += "<script>wrapper.reload('login=done')</script>"
         self.cmd("notification", ["done", message])
     else:
         self.cmd("notification", ["error", "Error: Invalid master seed"])
         self.actionUserLoginForm(0)
Exemple #23
0
	def testUserNewsite(self):
		from User import UserManager
		user = UserManager.user_manager.get()
		user.sites = {} # Reset user data
		self.assertEqual(user.master_address, "15E5rhcAUD69WbiYsYARh4YHJ4sLm2JEyc")
		self.assertEqual(user.getAddressAuthIndex("15E5rhcAUD69WbiYsYARh4YHJ4sLm2JEyc"), 1458664252141532163166741013621928587528255888800826689784628722366466547364755811L)

		address, address_index, site_data = user.getNewSiteData()
		self.assertEqual(CryptBitcoin.hdPrivatekey(user.master_seed, address_index), site_data["privatekey"]) # Re-generate privatekey based on address_index
		
		user.sites = {} # Reset user data

		self.assertNotEqual(user.getSiteData(address)["auth_address"], address) # Site address and auth address is different
		self.assertEqual(user.getSiteData(address)["auth_privatekey"], site_data["auth_privatekey"]) # Re-generate auth_privatekey for site
    def getEncryptPrivatekey(self, address, param_index=0):
        assert param_index >= 0 and param_index <= 1000
        site_data = self.getSiteData(address)

        if site_data.get("cert"):  # Different privatekey for different cert provider
            index = param_index + self.getAddressAuthIndex(site_data["cert"])
        else:
            index = param_index

        if "encrypt_privatekey_%s" % index not in site_data:
            address_index = self.getAddressAuthIndex(address)
            crypt_index = address_index + 1000 + index
            site_data["encrypt_privatekey_%s" % index] = CryptBitcoin.hdPrivatekey(self.master_seed, crypt_index)
            self.log.debug("New encrypt privatekey generated for %s:%s" % (address, index))
        return site_data["encrypt_privatekey_%s" % index]
Exemple #25
0
    def testCert(self, user):
        cert_auth_address = user.getAuthAddress("1iD5ZQJMNXu43w1qLB8sfdHVKppVMduGz")  # Add site to user's registry
        # Add cert
        user.addCert(cert_auth_address, "zeroid.bit", "faketype", "fakeuser", "fakesign")
        user.setCert("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr", "zeroid.bit")

        # By using certificate the auth address should be same as the certificate provider
        assert user.getAuthAddress("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr") == cert_auth_address
        auth_privatekey = user.getAuthPrivatekey("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr")
        assert CryptBitcoin.privatekeyToAddress(auth_privatekey) == cert_auth_address

        # Test delete site data
        assert "1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr" in user.sites
        user.deleteSiteData("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr")
        assert "1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr" not in user.sites

        # Re-create add site should generate normal, unique auth_address
        assert not user.getAuthAddress("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr") == cert_auth_address
        assert user.getAuthAddress("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr") == "1MyJgYQjeEkR9QD66nkfJc9zqi9uUy5Lr2"
Exemple #26
0
    def verifyCert(self, inner_path, content):
        from Crypt import CryptBitcoin

        rules = self.getRules(inner_path, content)

        if not rules.get("cert_signers"):
            return True  # Does not need cert

        name, domain = content["cert_user_id"].split("@")
        cert_address = rules["cert_signers"].get(domain)
        if not cert_address:  # Cert signer not allowed
            self.log.warning("Invalid cert signer: %s" % domain)
            return False

        try:
            cert_subject = "%s#%s/%s" % (rules["user_address"], content["cert_auth_type"], name)
            result = CryptBitcoin.verify(cert_subject, cert_address, content["cert_sign"])
        except Exception, err:
            self.log.warning("Certificate verify error: %s" % err)
            result = False
Exemple #27
0
    def testSignOld(self):
        privatekey = "23DKQpDz7bXM7w5KN5Wnmz7bwRNqNHcdQjb2WwrdB1QtTf5gM3pFdf"
        privatekey_bad = "23DKQpDz7bXM7w5KN5Wnmz6bwRNqNHcdQjb2WwrdB1QtTf5gM3pFdf"

        # Get address by privatekey
        address = CryptBitcoin.privatekeyToAddress(privatekey)
        assert address == "12vTsjscg4hYPewUL2onma5pgQmWPMs3ez"

        address_bad = CryptBitcoin.privatekeyToAddress(privatekey_bad)
        assert not address_bad == "12vTsjscg4hYPewUL2onma5pgQmWPMs3ez"

        # Text signing
        sign = CryptBitcoin.signOld("hello", privatekey)
        assert CryptBitcoin.verify("hello", address, sign)  # Original text
        assert not CryptBitcoin.verify("not hello", address, sign)  # Different text

        # Signed by bad privatekey
        sign_bad = CryptBitcoin.signOld("hello", privatekey_bad)
        assert not CryptBitcoin.verify("hello", address, sign_bad)
Exemple #28
0
	def testBitcoinSign(self):
		s = time.time()
		privatekey = "5K9S6dVpufGnroRgFrT6wsKiz2mJRYsC73eWDmajaHserAp3F1C"
		privatekey_bad = "5Jbm9rrusXyApAoM8YoM4Rja337zMMoBUMRJ1uijiguU2aZRnwC"

		address = CryptBitcoin.privatekeyToAddress(privatekey)
		self.assertEqual(address, "1MpDMxFeDUkiHohxx9tbGLeEGEuR4ZNsJz")

		address_bad = CryptBitcoin.privatekeyToAddress(privatekey_bad)
		self.assertNotEqual(address_bad, "1MpDMxFeDUkiHohxx9tbGLeEGEuR4ZNsJz")

		sign = CryptBitcoin.sign("hello", privatekey)

		self.assertTrue(CryptBitcoin.verify("hello", address, sign))
		self.assertFalse(CryptBitcoin.verify("not hello", address, sign))

		sign_bad = CryptBitcoin.sign("hello", privatekey_bad)
		self.assertFalse(CryptBitcoin.verify("hello", address, sign_bad))

		print "Taken: %.3fs, " % (time.time()-s),
Exemple #29
0
	def testBitcoinSignOld(self):
		s = time.time()
		privatekey = "23DKQpDz7bXM7w5KN5Wnmz7bwRNqNHcdQjb2WwrdB1QtTf5gM3pFdf"
		privatekey_bad = "23DKQpDz7bXM7w5KN5Wnmz6bwRNqNHcdQjb2WwrdB1QtTf5gM3pFdf"

		address = CryptBitcoin.privatekeyToAddress(privatekey)
		self.assertEqual(address, "12vTsjscg4hYPewUL2onma5pgQmWPMs3ez")

		address_bad = CryptBitcoin.privatekeyToAddress(privatekey_bad)
		self.assertNotEqual(address_bad, "12vTsjscg4hYPewUL2onma5pgQmWPMs3ez")

		sign = CryptBitcoin.signOld("hello", privatekey)

		self.assertTrue(CryptBitcoin.verify("hello", address, sign))
		self.assertFalse(CryptBitcoin.verify("not hello", address, sign))

		sign_bad = CryptBitcoin.signOld("hello", privatekey_bad)
		self.assertFalse(CryptBitcoin.verify("hello", address, sign_bad))

		print "Taken: %.3fs, " % (time.time()-s),
Exemple #30
0
    def testSign(self):
        privatekey = "5K9S6dVpufGnroRgFrT6wsKiz2mJRYsC73eWDmajaHserAp3F1C"
        privatekey_bad = "5Jbm9rrusXyApAoM8YoM4Rja337zMMoBUMRJ1uijiguU2aZRnwC"

        # Get address by privatekey
        address = CryptBitcoin.privatekeyToAddress(privatekey)
        assert address == "1MpDMxFeDUkiHohxx9tbGLeEGEuR4ZNsJz"

        address_bad = CryptBitcoin.privatekeyToAddress(privatekey_bad)
        assert address_bad != "1MpDMxFeDUkiHohxx9tbGLeEGEuR4ZNsJz"

        # Text signing
        sign = CryptBitcoin.sign("hello", privatekey)

        assert CryptBitcoin.verify("hello", address, sign)
        assert not CryptBitcoin.verify("not hello", address, sign)

        # Signed by bad privatekey
        sign_bad = CryptBitcoin.sign("hello", privatekey_bad)
        assert not CryptBitcoin.verify("hello", address, sign_bad)
Exemple #31
0
class UiRequestPlugin(object):

    def formatTableRow(self, row):
        back = []
        for format, val in row:
            if val is None:
                formatted = "n/a"
            elif format == "since":
                if val:
                    formatted = "%.0f" % (time.time() - val)
                else:
                    formatted = "n/a"
            else:
                formatted = format % val
            back.append("<td>%s</td>" % formatted)
        return "<tr>%s</tr>" % "".join(back)

    def getObjSize(self, obj, hpy=None):
        if hpy:
            return float(hpy.iso(obj).domisize) / 1024
        else:
            return 0

    # /Stats entry point
    def actionStats(self):
        import gc
        import sys
        from Ui import UiRequest
        from Crypt import CryptConnection

        hpy = None
        if self.get.get("size") == "1":  # Calc obj size
            try:
                import guppy
                hpy = guppy.hpy()
            except:
                pass
        self.sendHeader()
        s = time.time()
        main = sys.modules["main"]

        # Style
        yield """
        <style>
         * { font-family: monospace }
         table td, table th { text-align: right; padding: 0px 10px }
        </style>
        """

        # Memory
        try:
            yield "rev%s | " % config.rev
            yield "%s | " % config.ip_external
            yield "Opened: %s | " % main.file_server.port_opened
            yield "Crypt: %s | " % CryptConnection.manager.crypt_supported
            yield "In: %.2fMB, Out: %.2fMB  | " % (
                float(main.file_server.bytes_recv) / 1024 / 1024,
                float(main.file_server.bytes_sent) / 1024 / 1024
            )
            yield "Peerid: %s  | " % main.file_server.peer_id
            import psutil
            process = psutil.Process(os.getpid())
            mem = process.get_memory_info()[0] / float(2 ** 20)
            yield "Mem: %.2fMB | " % mem
            yield "Threads: %s | " % len(process.threads())
            yield "CPU: usr %.2fs sys %.2fs | " % process.cpu_times()
            yield "Files: %s | " % len(process.open_files())
            yield "Sockets: %s | " % len(process.connections())
            yield "Calc size <a href='?size=1'>on</a> <a href='?size=0'>off</a>"
        except Exception:
            pass
        yield "<br>"

        # Connections
        yield "<b>Connections</b> (%s, total made: %s):<br>" % (
            len(main.file_server.connections), main.file_server.last_connection_id
        )
        yield "<table><tr> <th>id</th> <th>proto</th>  <th>type</th> <th>ip</th> <th>open</th> <th>crypt</th> <th>ping</th>"
        yield "<th>buff</th> <th>idle</th> <th>open</th> <th>delay</th> <th>out</th> <th>in</th> <th>last sent</th>"
        yield "<th>waiting</th> <th>version</th> <th>peerid</th> </tr>"
        for connection in main.file_server.connections:
            if "cipher" in dir(connection.sock):
                cipher = connection.sock.cipher()[0]
            else:
                cipher = connection.crypt
            yield self.formatTableRow([
                ("%3d", connection.id),
                ("%s", connection.protocol),
                ("%s", connection.type),
                ("%s:%s", (connection.ip, connection.port)),
                ("%s", connection.handshake.get("port_opened")),
                ("<span title='%s'>%s</span>", (connection.crypt, cipher)),
                ("%6.3f", connection.last_ping_delay),
                ("%s", connection.incomplete_buff_recv),
                ("since", max(connection.last_send_time, connection.last_recv_time)),
                ("since", connection.start_time),
                ("%.3f", connection.last_sent_time - connection.last_send_time),
                ("%.0fkB", connection.bytes_sent / 1024),
                ("%.0fkB", connection.bytes_recv / 1024),
                ("%s", connection.last_cmd),
                ("%s", connection.waiting_requests.keys()),
                ("%s r%s", (connection.handshake.get("version"), connection.handshake.get("rev", "?"))),
                ("%s", connection.handshake.get("peer_id")),
            ])
        yield "</table>"

        # Sites
        yield "<br><br><b>Sites</b>:"
        yield "<table>"
        yield "<tr><th>address</th> <th>connected</th> <th title='connected/good/total'>peers</th> <th>content.json</th> <th>out</th> <th>in</th>  </tr>"
        for site in self.server.sites.values():
            yield self.formatTableRow([
                (
                    """<a href='#' onclick='document.getElementById("peers_%s").style.display="initial"; return false'>%s</a>""",
                    (site.address, site.address)
                ),
                ("%s", [peer.connection.id for peer in site.peers.values() if peer.connection and peer.connection.connected]),
                ("%s/%s/%s", (
                    len([peer for peer in site.peers.values() if peer.connection and peer.connection.connected]),
                    len(site.getConnectablePeers(100)),
                    len(site.peers)
                )),
                ("%s", len(site.content_manager.contents)),
                ("%.0fkB", site.settings.get("bytes_sent", 0) / 1024),
                ("%.0fkB", site.settings.get("bytes_recv", 0) / 1024),
            ])
            yield "<tr><td id='peers_%s' style='display: none; white-space: pre' colspan=2>" % site.address
            for key, peer in site.peers.items():
                if peer.time_found:
                    time_found = int(time.time()-peer.time_found)/60
                else:
                    time_found = "--"
                if peer.connection:
                    connection_id = peer.connection.id
                else:
                    connection_id = None
                if site.content_manager.hashfield:
                    yield "Optional files: %4s " % len(peer.hashfield)
                yield "(#%4s, err: %s, found: %5s min ago) %22s -<br>" % (connection_id, peer.connection_error, time_found, key)
            yield "<br></td></tr>"
        yield "</table>"

        # No more if not in debug mode
        if not config.debug:
            raise StopIteration

        # Object types


        obj_count = {}
        for obj in gc.get_objects():
            obj_type = str(type(obj))
            if obj_type not in obj_count:
                obj_count[obj_type] = [0, 0]
            obj_count[obj_type][0] += 1  # Count
            obj_count[obj_type][1] += float(sys.getsizeof(obj)) / 1024  # Size

        yield "<br><br><b>Objects in memory (types: %s, total: %s, %.2fkb):</b><br>" % (
            len(obj_count),
            sum([stat[0] for stat in obj_count.values()]),
            sum([stat[1] for stat in obj_count.values()])
        )

        for obj, stat in sorted(obj_count.items(), key=lambda x: x[1][0], reverse=True):  # Sorted by count
            yield " - %.1fkb = %s x <a href=\"/Listobj?type=%s\">%s</a><br>" % (stat[1], stat[0], obj, cgi.escape(obj))

        # Classes

        class_count = {}
        for obj in gc.get_objects():
            obj_type = str(type(obj))
            if obj_type != "<type 'instance'>":
                continue
            class_name = obj.__class__.__name__
            if class_name not in class_count:
                class_count[class_name] = [0, 0]
            class_count[class_name][0] += 1  # Count
            class_count[class_name][1] += float(sys.getsizeof(obj)) / 1024  # Size

        yield "<br><br><b>Classes in memory (types: %s, total: %s, %.2fkb):</b><br>" % (
            len(class_count),
            sum([stat[0] for stat in class_count.values()]),
            sum([stat[1] for stat in class_count.values()])
        )

        for obj, stat in sorted(class_count.items(), key=lambda x: x[1][0], reverse=True):  # Sorted by count
            yield " - %.1fkb = %s x <a href=\"/Dumpobj?class=%s\">%s</a><br>" % (stat[1], stat[0], obj, cgi.escape(obj))

        from greenlet import greenlet
        objs = [obj for obj in gc.get_objects() if isinstance(obj, greenlet)]
        yield "<br>Greenlets (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))

        from Worker import Worker
        objs = [obj for obj in gc.get_objects() if isinstance(obj, Worker)]
        yield "<br>Workers (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))

        from Connection import Connection
        objs = [obj for obj in gc.get_objects() if isinstance(obj, Connection)]
        yield "<br>Connections (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))

        from socket import socket
        objs = [obj for obj in gc.get_objects() if isinstance(obj, socket)]
        yield "<br>Sockets (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))

        from msgpack import Unpacker
        objs = [obj for obj in gc.get_objects() if isinstance(obj, Unpacker)]
        yield "<br>Msgpack unpacker (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))

        from Site import Site
        objs = [obj for obj in gc.get_objects() if isinstance(obj, Site)]
        yield "<br>Sites (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))

        objs = [obj for obj in gc.get_objects() if isinstance(obj, self.server.log.__class__)]
        yield "<br>Loggers (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj.name)))

        objs = [obj for obj in gc.get_objects() if isinstance(obj, UiRequest)]
        yield "<br>UiRequests (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))

        from Peer import Peer
        objs = [obj for obj in gc.get_objects() if isinstance(obj, Peer)]
        yield "<br>Peers (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))

        objs = [(key, val) for key, val in sys.modules.iteritems() if val is not None]
        objs.sort()
        yield "<br>Modules (%s):<br>" % len(objs)
        for module_name, module in objs:
            yield " - %.3fkb: %s %s<br>" % (self.getObjSize(module, hpy), module_name, cgi.escape(repr(module)))

        gc.collect()  # Implicit grabage collection
        yield "Done in %.1f" % (time.time() - s)

    def actionDumpobj(self):

        import gc
        import sys

        self.sendHeader()

        # No more if not in debug mode
        if not config.debug:
            yield "Not in debug mode"
            raise StopIteration

        class_filter = self.get.get("class")

        yield """
        <style>
         * { font-family: monospace; white-space: pre }
         table * { text-align: right; padding: 0px 10px }
        </style>
        """

        objs = gc.get_objects()
        for obj in objs:
            obj_type = str(type(obj))
            if obj_type != "<type 'instance'>" or obj.__class__.__name__ != class_filter:
                continue
            yield "%.1fkb %s... " % (float(sys.getsizeof(obj)) / 1024, cgi.escape(str(obj)))
            for attr in dir(obj):
                yield "- %s: %s<br>" % (attr, cgi.escape(str(getattr(obj, attr))))
            yield "<br>"

        gc.collect()  # Implicit grabage collection

    def actionListobj(self):

        import gc
        import sys

        self.sendHeader()

        # No more if not in debug mode
        if not config.debug:
            yield "Not in debug mode"
            raise StopIteration

        type_filter = self.get.get("type")

        yield """
        <style>
         * { font-family: monospace; white-space: pre }
         table * { text-align: right; padding: 0px 10px }
        </style>
        """

        yield "Listing all %s objects in memory...<br>" % cgi.escape(type_filter)

        ref_count = {}
        objs = gc.get_objects()
        for obj in objs:
            obj_type = str(type(obj))
            if obj_type != type_filter:
                continue
            refs = [
                ref for ref in gc.get_referrers(obj)
                if hasattr(ref, "__class__") and
                ref.__class__.__name__ not in ["list", "dict", "function", "type", "frame", "WeakSet", "tuple"]
            ]
            if not refs:
                continue
            yield "%.1fkb <span title=\"%s\">%s</span>... " % (
                float(sys.getsizeof(obj)) / 1024, cgi.escape(str(obj)), cgi.escape(str(obj)[0:100].ljust(100))
            )
            for ref in refs:
                yield " ["
                if "object at" in str(ref) or len(str(ref)) > 100:
                    yield str(ref.__class__.__name__)
                else:
                    yield str(ref.__class__.__name__) + ":" + cgi.escape(str(ref))
                yield "] "
                ref_type = ref.__class__.__name__
                if ref_type not in ref_count:
                    ref_count[ref_type] = [0, 0]
                ref_count[ref_type][0] += 1  # Count
                ref_count[ref_type][1] += float(sys.getsizeof(obj)) / 1024  # Size
            yield "<br>"

        yield "<br>Object referrer (total: %s, %.2fkb):<br>" % (len(ref_count), sum([stat[1] for stat in ref_count.values()]))

        for obj, stat in sorted(ref_count.items(), key=lambda x: x[1][0], reverse=True)[0:30]:  # Sorted by count
            yield " - %.1fkb = %s x %s<br>" % (stat[1], stat[0], cgi.escape(str(obj)))

        gc.collect()  # Implicit grabage collection

    def actionBenchmark(self):
        import sys
        import gc
        from contextlib import contextmanager

        output = self.sendHeader()

        @contextmanager
        def benchmark(name, standard):
            s = time.time()
            output("- %s" % name)
            try:
                yield 1
            except Exception, err:
                output("<br><b>! Error: %s</b><br>" % err)
            taken = time.time() - s
            multipler = standard / taken
            if multipler < 0.3:
                speed = "Sloooow"
            elif multipler < 0.5:
                speed = "Ehh"
            elif multipler < 0.8:
                speed = "Goodish"
            elif multipler < 1.2:
                speed = "OK"
            elif multipler < 1.7:
                speed = "Fine"
            elif multipler < 2.5:
                speed = "Fast"
            elif multipler < 3.5:
                speed = "WOW"
            else:
                speed = "Insane!!"
            output("%.3fs [x%.2f: %s]<br>" % (taken, multipler, speed))
            time.sleep(0.01)

        yield """
        <style>
         * { font-family: monospace }
         table * { text-align: right; padding: 0px 10px }
        </style>
        """

        yield "Benchmarking ZeroNet %s (rev%s) Python %s on: %s...<br>" % (config.version, config.rev, sys.version, sys.platform)

        t = time.time()

        # CryptBitcoin
        yield "<br>CryptBitcoin:<br>"
        from Crypt import CryptBitcoin

        # seed = CryptBitcoin.newSeed()
        # yield "- Seed: %s<br>" % seed
        seed = "e180efa477c63b0f2757eac7b1cce781877177fe0966be62754ffd4c8592ce38"

        with benchmark("hdPrivatekey x 10", 0.7):
            for i in range(10):
                privatekey = CryptBitcoin.hdPrivatekey(seed, i * 10)
                yield "."
            valid = "5JsunC55XGVqFQj5kPGK4MWgTL26jKbnPhjnmchSNPo75XXCwtk"
            assert privatekey == valid, "%s != %s" % (privatekey, valid)

        data = "Hello" * 1024  # 5k
        with benchmark("sign x 10", 0.35):
            for i in range(10):
                yield "."
                sign = CryptBitcoin.sign(data, privatekey)
            valid = "HFGXaDauZ8vX/N9Jn+MRiGm9h+I94zUhDnNYFaqMGuOi+4+BbWHjuwmx0EaKNV1G+kP0tQDxWu0YApxwxZbSmZU="
            assert sign == valid, "%s != %s" % (sign, valid)

        address = CryptBitcoin.privatekeyToAddress(privatekey)
        if CryptBitcoin.opensslVerify:  # Openssl avalible
            with benchmark("openssl verify x 100", 0.37):
                for i in range(100):
                    if i % 10 == 0:
                        yield "."
                    ok = CryptBitcoin.verify(data, address, sign)
                assert ok, "does not verify from %s" % address
        else:
            yield " - openssl verify x 100...not avalible :(<br>"

        openssl_verify_bk = CryptBitcoin.opensslVerify  # Emulate openssl not found in any way
        CryptBitcoin.opensslVerify = None
        with benchmark("pure-python verify x 10", 1.6):
            for i in range(10):
                yield "."
                ok = CryptBitcoin.verify(data, address, sign)
            assert ok, "does not verify from %s" % address
        CryptBitcoin.opensslVerify = openssl_verify_bk

        # CryptHash
        yield "<br>CryptHash:<br>"
        from Crypt import CryptHash
        from cStringIO import StringIO

        data = StringIO("Hello" * 1024 * 1024)  # 5m
        with benchmark("sha512 x 100 000", 1):
            for i in range(10):
                for y in range(10000):
                    hash = CryptHash.sha512sum(data)
                yield "."
            valid = "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce"
            assert hash == valid, "%s != %s" % (hash, valid)

        with benchmark("os.urandom(256) x 100 000", 0.65):
            for i in range(10):
                for y in range(10000):
                    data = os.urandom(256)
                yield "."

        # Msgpack
        yield "<br>Msgpack:<br>"
        import msgpack
        binary = 'fqv\xf0\x1a"e\x10,\xbe\x9cT\x9e(\xa5]u\x072C\x8c\x15\xa2\xa8\x93Sw)\x19\x02\xdd\t\xfb\xf67\x88\xd9\xee\x86\xa1\xe4\xb6,\xc6\x14\xbb\xd7$z\x1d\xb2\xda\x85\xf5\xa0\x97^\x01*\xaf\xd3\xb0!\xb7\x9d\xea\x89\xbbh8\xa1"\xa7]e(@\xa2\xa5g\xb7[\xae\x8eE\xc2\x9fL\xb6s\x19\x19\r\xc8\x04S\xd0N\xe4]?/\x01\xea\xf6\xec\xd1\xb3\xc2\x91\x86\xd7\xf4K\xdf\xc2lV\xf4\xe8\x80\xfc\x8ep\xbb\x82\xb3\x86\x98F\x1c\xecS\xc8\x15\xcf\xdc\xf1\xed\xfc\xd8\x18r\xf9\x80\x0f\xfa\x8cO\x97(\x0b]\xf1\xdd\r\xe7\xbf\xed\x06\xbd\x1b?\xc5\xa0\xd7a\x82\xf3\xa8\xe6@\xf3\ri\xa1\xb10\xf6\xd4W\xbc\x86\x1a\xbb\xfd\x94!bS\xdb\xaeM\x92\x00#\x0b\xf7\xad\xe9\xc2\x8e\x86\xbfi![%\xd31]\xc6\xfc2\xc9\xda\xc6v\x82P\xcc\xa9\xea\xb9\xff\xf6\xc8\x17iD\xcf\xf3\xeeI\x04\xe9\xa1\x19\xbb\x01\x92\xf5nn4K\xf8\xbb\xc6\x17e>\xa7 \xbbv'
        data = {"int": 1024*1024*1024, "float": 12345.67890, "text": "hello"*1024, "binary": binary}
        with benchmark("pack 5K x 10 000", 0.78):
            for i in range(10):
                for y in range(1000):
                    data_packed = msgpack.packb(data)
                yield "."
            valid = """\x84\xa3int\xce@\x00\x00\x00\xa4text\xda\x14\x00hellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohello\xa5float\xcb@\xc8\x1c\xd6\xe61\xf8\xa1\xa6binary\xda\x01\x00fqv\xf0\x1a"e\x10,\xbe\x9cT\x9e(\xa5]u\x072C\x8c\x15\xa2\xa8\x93Sw)\x19\x02\xdd\t\xfb\xf67\x88\xd9\xee\x86\xa1\xe4\xb6,\xc6\x14\xbb\xd7$z\x1d\xb2\xda\x85\xf5\xa0\x97^\x01*\xaf\xd3\xb0!\xb7\x9d\xea\x89\xbbh8\xa1"\xa7]e(@\xa2\xa5g\xb7[\xae\x8eE\xc2\x9fL\xb6s\x19\x19\r\xc8\x04S\xd0N\xe4]?/\x01\xea\xf6\xec\xd1\xb3\xc2\x91\x86\xd7\xf4K\xdf\xc2lV\xf4\xe8\x80\xfc\x8ep\xbb\x82\xb3\x86\x98F\x1c\xecS\xc8\x15\xcf\xdc\xf1\xed\xfc\xd8\x18r\xf9\x80\x0f\xfa\x8cO\x97(\x0b]\xf1\xdd\r\xe7\xbf\xed\x06\xbd\x1b?\xc5\xa0\xd7a\x82\xf3\xa8\xe6@\xf3\ri\xa1\xb10\xf6\xd4W\xbc\x86\x1a\xbb\xfd\x94!bS\xdb\xaeM\x92\x00#\x0b\xf7\xad\xe9\xc2\x8e\x86\xbfi![%\xd31]\xc6\xfc2\xc9\xda\xc6v\x82P\xcc\xa9\xea\xb9\xff\xf6\xc8\x17iD\xcf\xf3\xeeI\x04\xe9\xa1\x19\xbb\x01\x92\xf5nn4K\xf8\xbb\xc6\x17e>\xa7 \xbbv"""
            assert data_packed == valid, "%s<br>!=<br>%s" % (repr(data_packed), repr(valid))

        with benchmark("unpack 5K x 10 000", 1.2):
            for i in range(10):
                for y in range(1000):
                    data_unpacked = msgpack.unpackb(data_packed)
                yield "."
            assert data == data_unpacked, "%s != %s" % (data_unpack, data)

        with benchmark("streaming unpack 5K x 10 000", 1.4):
            for i in range(10):
                unpacker = msgpack.Unpacker()
                for y in range(1000):
                    unpacker.feed(data_packed)
                    for data_unpacked in unpacker:
                        pass
                yield "."
            assert data == data_unpacked, "%s != %s" % (data_unpack, data)

        # Db
        yield "<br>Db:<br>"
        from Db import Db

        schema = {
            "db_name": "TestDb",
            "db_file": "%s/benchmark.db" % config.data_dir,
            "maps": {
                ".*": {
                    "to_table": {
                        "test": "test"
                    }
                }
            },
            "tables": {
                "test": {
                    "cols": [
                        ["test_id", "INTEGER"],
                        ["title", "TEXT"],
                        ["json_id", "INTEGER REFERENCES json (json_id)"]
                    ],
                    "indexes": ["CREATE UNIQUE INDEX test_key ON test(test_id, json_id)"],
                    "schema_changed": 1426195822
                }
            }
        }

        if os.path.isfile("%s/benchmark.db" % config.data_dir):
            os.unlink("%s/benchmark.db" % config.data_dir)

        with benchmark("Open x 10", 0.13):
            for i in range(10):
                db = Db(schema, "%s/benchmark.db" % config.data_dir)
                db.checkTables()
                db.close()
                yield "."

        db = Db(schema, "%s/benchmark.db" % config.data_dir)
        db.checkTables()
        import json

        with benchmark("Insert x 10 x 1000", 1.0):
            for u in range(10):  # 10 user
                data = {"test": []}
                for i in range(1000):  # 1000 line of data
                    data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)})
                json.dump(data, open("%s/test_%s.json" % (config.data_dir, u), "w"))
                db.loadJson("%s/test_%s.json" % (config.data_dir, u))
                os.unlink("%s/test_%s.json" % (config.data_dir, u))
                yield "."

        with benchmark("Buffered insert x 100 x 100", 1.3):
            cur = db.getCursor()
            cur.execute("BEGIN")
            cur.logging = False
            for u in range(100, 200):  # 100 user
                data = {"test": []}
                for i in range(100):  # 1000 line of data
                    data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)})
                json.dump(data, open("%s/test_%s.json" % (config.data_dir, u), "w"))
                db.loadJson("%s/test_%s.json" % (config.data_dir, u), cur=cur)
                os.unlink("%s/test_%s.json" % (config.data_dir, u))
                if u % 10 == 0:
                    yield "."
            cur.execute("COMMIT")

        yield " - Total rows in db: %s<br>" % db.execute("SELECT COUNT(*) AS num FROM test").fetchone()[0]

        with benchmark("Indexed query x 1000", 0.25):
            found = 0
            cur = db.getCursor()
            cur.logging = False
            for i in range(1000):  # 1000x by test_id
                res = cur.execute("SELECT * FROM test WHERE test_id = %s" % i)
                for row in res:
                    found += 1
                if i % 100 == 0:
                    yield "."

            assert found == 20000, "Found: %s != 20000" % found

        with benchmark("Not indexed query x 100", 0.6):
            found = 0
            cur = db.getCursor()
            cur.logging = False
            for i in range(100):  # 1000x by test_id
                res = cur.execute("SELECT * FROM test WHERE json_id = %s" % i)
                for row in res:
                    found += 1
                if i % 10 == 0:
                    yield "."

            assert found == 18900, "Found: %s != 18900" % found

        with benchmark("Like query x 100", 1.8):
            found = 0
            cur = db.getCursor()
            cur.logging = False
            for i in range(100):  # 1000x by test_id
                res = cur.execute("SELECT * FROM test WHERE title LIKE '%%message %s%%'" % i)
                for row in res:
                    found += 1
                if i % 10 == 0:
                    yield "."

            assert found == 38900, "Found: %s != 11000" % found

        db.close()
        if os.path.isfile("%s/benchmark.db" % config.data_dir):
            os.unlink("%s/benchmark.db" % config.data_dir)

        gc.collect()  # Implicit grabage collection

        yield "<br>Done. Total: %.2fs" % (time.time() - t)
Exemple #32
0
    def peerCheckMessage(self, raw, params, ip):
        # Calculate hash from nonce
        msg_hash = hashlib.sha256(
            "%s,%s" % (params["nonce"], params["raw"])).hexdigest()

        # Check that p2p.json exists
        site = self.sites.get(raw["site"])
        if not site.storage.isFile("p2p.json"):
            self.connection.log("Site %s doesn't support P2P messages" %
                                raw["site"])
            self.connection.badAction(5)
            self.response({
                "error":
                "Site %s doesn't support P2P messages" % raw["site"]
            })
            return False, "", None, msg_hash

        # Check whether P2P messages are supported
        p2p_json = site.storage.loadJson("p2p.json")
        if "filter" not in p2p_json:
            self.connection.log("Site %s doesn't support P2P messages" %
                                raw["site"])
            self.connection.badAction(5)
            self.response({
                "error":
                "Site %s doesn't support P2P messages" % raw["site"]
            })
            return False, "", None, msg_hash

        # Was the message received yet?
        if msg_hash in site.p2p_received:
            self.response({"warning": "Already received, thanks"})
            return False, "", None, msg_hash
        site.p2p_received.append(msg_hash)

        # Check whether the message matches passive filter
        if not SafeRe.match(p2p_json["filter"], json.dumps(raw["message"])):
            self.connection.log("Invalid message for site %s: %s" %
                                (raw["site"], raw["message"]))
            self.connection.badAction(5)
            self.response({
                "error":
                "Invalid message for site %s: %s" %
                (raw["site"], raw["message"])
            })
            return False, "", None, msg_hash

        # Not so fast
        if "freq_limit" in p2p_json and time.time() - site.p2p_last_recv.get(
                ip, 0) < p2p_json["freq_limit"]:
            self.connection.log("Too fast messages from %s" % raw["site"])
            self.connection.badAction(2)
            self.response({"error": "Too fast messages from %s" % raw["site"]})
            return False, "", None, msg_hash
        site.p2p_last_recv[ip] = time.time()

        # Not so much
        if "size_limit" in p2p_json and len(json.dumps(
                raw["message"])) > p2p_json["size_limit"]:
            self.connection.log("Too big message from %s" % raw["site"])
            self.connection.badAction(7)
            self.response({"error": "Too big message from %s" % raw["site"]})
            return False, "", None, msg_hash

        # Verify signature
        if params["signature"]:
            signature_address, signature = params["signature"].split("|")
            what = "%s|%s|%s" % (signature_address, msg_hash, params["raw"])
            from Crypt import CryptBitcoin
            if not CryptBitcoin.verify(what, signature_address, signature):
                self.connection.log("Invalid signature")
                self.connection.badAction(7)
                self.response({"error": "Invalid signature"})
                return False, "", None, msg_hash

            # Now check auth providers
            if params.get("cert"):
                # Read all info
                cert_auth_type, cert_auth_user_name, cert_issuer, cert_sign = params[
                    "cert"]
                # This is what certificate issuer signs
                cert_subject = "%s#%s/%s" % (signature_address, cert_auth_type,
                                             cert_auth_user_name)
                # Now get cert issuer address
                cert_signers = p2p_json.get("cert_signers", {})
                cert_addresses = cert_signers.get(cert_issuer, [])
                # And verify it
                if not CryptBitcoin.verify(cert_subject, cert_addresses,
                                           cert_sign):
                    self.connection.log("Invalid signature certificate")
                    self.connection.badAction(7)
                    self.response({"error": "Invalid signature certificate"})
                    return False, "", None, msg_hash
                # And save the ID
                cert = "%s/%s@%s" % (cert_auth_type, cert_auth_user_name,
                                     cert_issuer)
            else:
                # Old-style sign
                cert = ""
        else:
            signature_address = ""
            cert = ""

        # Check that the signature address is correct
        if "signed_only" in p2p_json:
            valid = p2p_json["signed_only"]
            if valid is True and not signature_address:
                self.connection.log("Not signed message")
                self.connection.badAction(5)
                self.response({"error": "Not signed message"})
                return False, "", None, msg_hash
            elif isinstance(valid, str) and signature_address != valid:
                self.connection.log(
                    "Message signature is invalid: %s not in [%r]" %
                    (signature_address, valid))
                self.connection.badAction(5)
                self.response({
                    "error":
                    "Message signature is invalid: %s not in [%r]" %
                    (signature_address, valid)
                })
                return False, "", None, msg_hash
            elif isinstance(valid, list) and signature_address not in valid:
                self.connection.log(
                    "Message signature is invalid: %s not in %r" %
                    (signature_address, valid))
                self.connection.badAction(5)
                self.response({
                    "error":
                    "Message signature is invalid: %s not in %r" %
                    (signature_address, valid)
                })
                return False, "", None, msg_hash

        return True, signature_address, cert, msg_hash
Exemple #33
0
    def testInlcudeLimits(self, site):
        # Data validation
        data_dict = {
            "files": {
                "data.json": {
                    "sha512":
                    "369d4e780cc80504285f13774ca327fe725eed2d813aad229e62356b07365906",
                    "size": 505
                }
            },
            "modified": time.time()
        }

        # Normal data
        data_dict["signs"] = {
            "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT":
            CryptBitcoin.sign(json.dumps(data_dict), self.privatekey)
        }
        data = StringIO(json.dumps(data_dict))
        assert site.content_manager.verifyFile(
            "data/test_include/content.json", data, ignore_same=False)
        # Reset
        del data_dict["signs"]

        # Too large
        data_dict["files"]["data.json"][
            "size"] = 200000  # Emulate 2MB sized data.json
        data_dict["signs"] = {
            "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT":
            CryptBitcoin.sign(json.dumps(data_dict), self.privatekey)
        }
        data = StringIO(json.dumps(data_dict))
        with pytest.raises(VerifyError) as err:
            site.content_manager.verifyFile("data/test_include/content.json",
                                            data,
                                            ignore_same=False)
            assert "Include too large" in str(err)

        # Reset
        data_dict["files"]["data.json"]["size"] = 505
        del data_dict["signs"]

        # Not allowed file
        data_dict["files"]["notallowed.exe"] = data_dict["files"]["data.json"]
        data_dict["signs"] = {
            "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT":
            CryptBitcoin.sign(json.dumps(data_dict), self.privatekey)
        }
        data = StringIO(json.dumps(data_dict))
        with pytest.raises(VerifyError) as err:
            site.content_manager.verifyFile("data/test_include/content.json",
                                            data,
                                            ignore_same=False)
            assert "File not allowed" in str(err)

        # Reset
        del data_dict["files"]["notallowed.exe"]
        del data_dict["signs"]

        # Should work again
        data_dict["signs"] = {
            "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT":
            CryptBitcoin.sign(json.dumps(data_dict), self.privatekey)
        }
        data = StringIO(json.dumps(data_dict))
        assert site.content_manager.verifyFile(
            "data/test_include/content.json", data, ignore_same=False)
Exemple #34
0
    def verifyFile(self, inner_path, file, ignore_same=True):
        if inner_path.endswith("content.json"):  # content.json: Check using sign
            from Crypt import CryptBitcoin
            try:
                if type(file) is dict:
                    new_content = file
                else:
                    new_content = json.load(file)
                if inner_path in self.contents:
                    old_content = self.contents.get(inner_path, {"modified": 0})
                    # Checks if its newer the ours
                    if old_content["modified"] == new_content["modified"] and ignore_same:  # Ignore, have the same content.json
                        return None
                    elif old_content["modified"] > new_content["modified"]:  # We have newer
                        raise VerifyError(
                            "We have newer (Our: %s, Sent: %s)" %
                            (old_content["modified"], new_content["modified"])
                        )
                if new_content["modified"] > time.time() + 60 * 60 * 24:  # Content modified in the far future (allow 1 day+)
                    raise VerifyError("Modify timestamp is in the far future!")
                if self.isArchived(inner_path, new_content["modified"]):
                    if inner_path in self.site.bad_files:
                        del self.site.bad_files[inner_path]
                    raise VerifyError("This file is archived!")
                # Check sign
                sign = new_content.get("sign")
                signs = new_content.get("signs", {})
                if "sign" in new_content:
                    del(new_content["sign"])  # The file signed without the sign
                if "signs" in new_content:
                    del(new_content["signs"])  # The file signed without the signs

                sign_content = json.dumps(new_content, sort_keys=True)  # Dump the json to string to remove whitepsace

                # Fix float representation error on Android
                modified = new_content["modified"]
                if config.fix_float_decimals and type(modified) is float and not str(modified).endswith(".0"):
                    modified_fixed = "{:.6f}".format(modified).strip("0.")
                    sign_content = sign_content.replace(
                        '"modified": %s' % repr(modified),
                        '"modified": %s' % modified_fixed
                    )

                self.verifyContent(inner_path, new_content)

                if signs:  # New style signing
                    valid_signers = self.getValidSigners(inner_path, new_content)
                    signs_required = self.getSignsRequired(inner_path, new_content)

                    if inner_path == "content.json" and len(valid_signers) > 1:  # Check signers_sign on root content.json
                        signers_data = "%s:%s" % (signs_required, ",".join(valid_signers))
                        if not CryptBitcoin.verify(signers_data, self.site.address, new_content["signers_sign"]):
                            raise VerifyError("Invalid signers_sign!")

                    if inner_path != "content.json" and not self.verifyCert(inner_path, new_content):  # Check if cert valid
                        raise VerifyError("Invalid cert!")

                    valid_signs = 0
                    for address in valid_signers:
                        if address in signs:
                            valid_signs += CryptBitcoin.verify(sign_content, address, signs[address])
                        if valid_signs >= signs_required:
                            break  # Break if we has enough signs
                    if valid_signs < signs_required:
                        raise VerifyError("Valid signs: %s/%s" % (valid_signs, signs_required))
                    else:
                        return True
                else:  # Old style signing
                    if CryptBitcoin.verify(sign_content, self.site.address, sign):
                        return True
                    else:
                        raise VerifyError("Invalid old-style sign")

            except Exception, err:
                self.log.warning("%s: verify sign error: %s" % (inner_path, Debug.formatException(err)))
                raise err
Exemple #35
0
        yield "Benchmarking ZeroNet %s (rev%s) Python %s, platform: %s...<br>" % (
            config.version, config.rev, sys.version, sys.platform)

        t = time.time()

        yield "<br>CryptBitcoin:<br>"
        from Crypt import CryptBitcoin

        # seed = CryptBitcoin.newSeed()
        # yield "- Seed: %s<br>" % seed
        seed = "e180efa477c63b0f2757eac7b1cce781877177fe0966be62754ffd4c8592ce38"

        with benchmark("hdPrivatekey x 10", 0.7):
            for i in range(10):
                privatekey = CryptBitcoin.hdPrivatekey(seed, i * 10)
                yield "."
            valid = "5JsunC55XGVqFQj5kPGK4MWgTL26jKbnPhjnmchSNPo75XXCwtk"
            assert privatekey == valid, "%s != %s" % (privatekey, valid)

        data = "Hello" * 1024  #5k
        with benchmark("sign x 10", 0.35):
            for i in range(10):
                yield "."
                sign = CryptBitcoin.sign(data, privatekey)
            valid = "HFGXaDauZ8vX/N9Jn+MRiGm9h+I94zUhDnNYFaqMGuOi+4+BbWHjuwmx0EaKNV1G+kP0tQDxWu0YApxwxZbSmZU="
            assert sign == valid, "%s != %s" % (sign, valid)

        address = CryptBitcoin.privatekeyToAddress(privatekey)
        with benchmark("verify x 10", 1.6):
            for i in range(10):
Exemple #36
0
 def testNewPrivatekey(self):
     assert CryptBitcoin.newPrivatekey() != CryptBitcoin.newPrivatekey()
     assert CryptBitcoin.privatekeyToAddress(CryptBitcoin.newPrivatekey())
Exemple #37
0
    def testCert(self, site):
        # user_addr = "1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C"
        user_priv = "5Kk7FSA63FC2ViKmKLuBxk9gQkaQ5713hKq8LmFAf4cVeXh6K6A"
        # cert_addr = "14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet"
        cert_priv = "5JusJDSjHaMHwUjDT3o6eQ54pA6poo8La5fAgn1wNc3iK59jxjA"

        # Check if the user file is loaded
        assert "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json" in site.content_manager.contents
        user_content = site.content_manager.contents[
            "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json"]
        rules_content = site.content_manager.contents[
            "data/users/content.json"]

        # Override valid cert signers for the test
        rules_content["user_contents"]["cert_signers"]["zeroid.bit"] = [
            "14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet",
            "1iD5ZQJMNXu43w1qLB8sfdHVKppVMduGz"
        ]

        # Check valid cert signers
        rules = site.content_manager.getRules(
            "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json",
            user_content)
        assert rules["cert_signers"] == {
            "zeroid.bit": [
                "14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet",
                "1iD5ZQJMNXu43w1qLB8sfdHVKppVMduGz"
            ]
        }

        # Sign a valid cert
        user_content["cert_sign"] = CryptBitcoin.sign(
            "1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C#%s/%s" %
            (user_content["cert_auth_type"],
             user_content["cert_user_id"].split("@")[0]), cert_priv)

        # Verify cert
        assert site.content_manager.verifyCert(
            "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json",
            user_content)

        # Verify if the cert is valid for other address
        assert not site.content_manager.verifyCert(
            "data/users/badaddress/content.json", user_content)

        # Sign user content
        signed_content = site.content_manager.sign(
            "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json",
            user_priv,
            filewrite=False)

        # Test user cert
        assert site.content_manager.verifyFile(
            "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json",
            io.BytesIO(json.dumps(signed_content).encode()),
            ignore_same=False)

        # Test banned user
        cert_user_id = user_content["cert_user_id"]  # My username
        site.content_manager.contents["data/users/content.json"][
            "user_contents"]["permissions"][cert_user_id] = False
        with pytest.raises(VerifyError) as err:
            site.content_manager.verifyFile(
                "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json",
                io.BytesIO(json.dumps(signed_content).encode()),
                ignore_same=False)
        assert "Valid signs: 0/1" in str(err.value)
        del site.content_manager.contents["data/users/content.json"][
            "user_contents"]["permissions"][cert_user_id]  # Reset

        # Test invalid cert
        user_content["cert_sign"] = CryptBitcoin.sign(
            "badaddress#%s/%s" %
            (user_content["cert_auth_type"], user_content["cert_user_id"]),
            cert_priv)
        signed_content = site.content_manager.sign(
            "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json",
            user_priv,
            filewrite=False)
        with pytest.raises(VerifyError) as err:
            site.content_manager.verifyFile(
                "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json",
                io.BytesIO(json.dumps(signed_content).encode()),
                ignore_same=False)
        assert "Invalid cert" in str(err.value)

        # Test banned user, signed by the site owner
        user_content["cert_sign"] = CryptBitcoin.sign(
            "1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C#%s/%s" %
            (user_content["cert_auth_type"],
             user_content["cert_user_id"].split("@")[0]), cert_priv)
        cert_user_id = user_content["cert_user_id"]  # My username
        site.content_manager.contents["data/users/content.json"][
            "user_contents"]["permissions"][cert_user_id] = False

        site_privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv"  # For 1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT
        del user_content["signs"]  # Remove signs before signing
        user_content["signs"] = {
            "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT":
            CryptBitcoin.sign(json.dumps(user_content, sort_keys=True),
                              site_privatekey)
        }
        assert site.content_manager.verifyFile(
            "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json",
            io.BytesIO(json.dumps(user_content).encode()),
            ignore_same=False)
Exemple #38
0
 def testAuthAddress(self, user):
     # Auth address without Cert
     auth_address = user.getAuthAddress("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr")
     assert auth_address == "1MyJgYQjeEkR9QD66nkfJc9zqi9uUy5Lr2"
     auth_privatekey = user.getAuthPrivatekey("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr")
     assert CryptBitcoin.privatekeyToAddress(auth_privatekey) == auth_address
Exemple #39
0
 def testNewSeed(self):
     assert CryptBitcoin.newSeed() != CryptBitcoin.newSeed()
     assert CryptBitcoin.privatekeyToAddress(
         CryptBitcoin.hdPrivatekey(CryptBitcoin.newSeed(), 0))
     assert CryptBitcoin.privatekeyToAddress(
         CryptBitcoin.hdPrivatekey(CryptBitcoin.newSeed(), 2**256))
Exemple #40
0
 def cryptVerify(self, message, sign, address):
     from Crypt import CryptBitcoin
     print CryptBitcoin.verify(message, address, sign)
Exemple #41
0
    def actionEcdsaSign(self, to, data, privatekey=None):
        if privatekey is None:  # Sign using user's privatekey
            privatekey = self.user.getAuthPrivatekey(self.site.address)

        self.response(to, CryptBitcoin.sign(data, privatekey))
Exemple #42
0
def crypt_bitcoin_lib(request, monkeypatch):
    monkeypatch.setattr(CryptBitcoin, "lib_verify_best", request.param)
    CryptBitcoin.loadLib(request.param)
    return CryptBitcoin
Exemple #43
0
class UiRequestPlugin(object):
    def formatTableRow(self, row, class_name=""):
        back = []
        for format, val in row:
            if val is None:
                formatted = "n/a"
            elif format == "since":
                if val:
                    formatted = "%.0f" % (time.time() - val)
                else:
                    formatted = "n/a"
            else:
                formatted = format % val
            back.append("<td>%s</td>" % formatted)
        return "<tr class='%s'>%s</tr>" % (class_name, "".join(back))

    def getObjSize(self, obj, hpy=None):
        if hpy:
            return float(hpy.iso(obj).domisize) / 1024
        else:
            return 0

    # /Stats entry point
    def actionStats(self):
        import gc
        import sys
        from Ui import UiRequest
        from Db import Db
        from Crypt import CryptConnection

        hpy = None
        if self.get.get("size") == "1":  # Calc obj size
            try:
                import guppy
                hpy = guppy.hpy()
            except:
                pass
        self.sendHeader()

        if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local:
            yield "This function is disabled on this proxy"
            raise StopIteration

        s = time.time()
        main = sys.modules["main"]

        # Style
        yield """
        <style>
         * { font-family: monospace }
         table td, table th { text-align: right; padding: 0px 10px }
         .connections td { white-space: nowrap }
         .serving-False { opacity: 0.3 }
        </style>
        """

        # Memory
        try:
            yield "rev%s | " % config.rev
            yield "%s | " % config.ip_external
            yield "Opened: %s | " % main.file_server.port_opened
            yield "Crypt: %s | " % CryptConnection.manager.crypt_supported
            yield "In: %.2fMB, Out: %.2fMB  | " % (
                float(main.file_server.bytes_recv) / 1024 / 1024,
                float(main.file_server.bytes_sent) / 1024 / 1024)
            yield "Peerid: %s  | " % main.file_server.peer_id
            import psutil
            process = psutil.Process(os.getpid())
            mem = process.get_memory_info()[0] / float(2**20)
            yield "Mem: %.2fMB | " % mem
            yield "Threads: %s | " % len(process.threads())
            yield "CPU: usr %.2fs sys %.2fs | " % process.cpu_times()
            yield "Files: %s | " % len(process.open_files())
            yield "Sockets: %s | " % len(process.connections())
            yield "Calc size <a href='?size=1'>on</a> <a href='?size=0'>off</a>"
        except Exception:
            pass
        yield "<br>"

        # Connections
        yield "<b>Connections</b> (%s, total made: %s):<br>" % (len(
            main.file_server.connections), main.file_server.last_connection_id)
        yield "<table class='connections'><tr> <th>id</th> <th>type</th> <th>ip</th> <th>open</th> <th>crypt</th> <th>ping</th>"
        yield "<th>buff</th> <th>bad</th> <th>idle</th> <th>open</th> <th>delay</th> <th>cpu</th> <th>out</th> <th>in</th> <th>last sent</th>"
        yield "<th>wait</th> <th>version</th> <th>sites</th> </tr>"
        for connection in main.file_server.connections:
            if "cipher" in dir(connection.sock):
                cipher = connection.sock.cipher()[0]
            else:
                cipher = connection.crypt
            yield self.formatTableRow([
                ("%3d", connection.id), ("%s", connection.type),
                ("%s:%s", (connection.ip, connection.port)),
                ("%s", connection.handshake.get("port_opened")),
                ("<span title='%s'>%s</span>", (connection.crypt, cipher)),
                ("%6.3f", connection.last_ping_delay),
                ("%s", connection.incomplete_buff_recv),
                ("%s", connection.bad_actions),
                ("since",
                 max(connection.last_send_time, connection.last_recv_time)),
                ("since", connection.start_time),
                ("%.3f",
                 connection.last_sent_time - connection.last_send_time),
                ("%.3f", connection.cpu_time),
                ("%.0fkB", connection.bytes_sent / 1024),
                ("%.0fkB", connection.bytes_recv / 1024),
                ("%s", connection.last_cmd),
                ("%s", connection.waiting_requests.keys()),
                ("%s r%s", (connection.handshake.get("version"),
                            connection.handshake.get("rev", "?"))),
                ("%s", connection.sites)
            ])
        yield "</table>"

        # Tor hidden services
        yield "<br><br><b>Tor hidden services (status: %s):</b><br>" % main.file_server.tor_manager.status
        for site_address, onion in main.file_server.tor_manager.site_onions.items(
        ):
            yield "- %-34s: %s<br>" % (site_address, onion)

        # Db
        yield "<br><br><b>Db</b>:<br>"
        for db in sys.modules["Db.Db"].opened_dbs:
            yield "- %.3fs: %s<br>" % (time.time() - db.last_query_time,
                                       db.db_path.encode("utf8"))

        # Sites
        yield "<br><br><b>Sites</b>:"
        yield "<table>"
        yield "<tr><th>address</th> <th>connected</th> <th title='connected/good/total'>peers</th> <th>content.json</th> <th>out</th> <th>in</th>  </tr>"
        for site in sorted(self.server.sites.values(),
                           lambda a, b: cmp(a.address, b.address)):
            yield self.formatTableRow([
                ("""<a href='#' onclick='document.getElementById("peers_%s").style.display="initial"; return false'>%s</a>""",
                 (site.address, site.address)),
                ("%s", [
                    peer.connection.id for peer in site.peers.values()
                    if peer.connection and peer.connection.connected
                ]),
                ("%s/%s/%s", (len([
                    peer for peer in site.peers.values()
                    if peer.connection and peer.connection.connected
                ]), len(site.getConnectablePeers(100)), len(site.peers))),
                ("%s (loaded: %s)",
                 (len(site.content_manager.contents),
                  len([
                      key for key, val in dict(
                          site.content_manager.contents).iteritems() if val
                  ]))),
                ("%.0fkB", site.settings.get("bytes_sent", 0) / 1024),
                ("%.0fkB", site.settings.get("bytes_recv", 0) / 1024),
            ], "serving-%s" % site.settings["serving"])
            yield "<tr><td id='peers_%s' style='display: none; white-space: pre' colspan=6>" % site.address
            for key, peer in site.peers.items():
                if peer.time_found:
                    time_found = int(time.time() - peer.time_found) / 60
                else:
                    time_found = "--"
                if peer.connection:
                    connection_id = peer.connection.id
                else:
                    connection_id = None
                if site.content_manager.has_optional_files:
                    yield "Optional files: %4s " % len(peer.hashfield)
                time_added = (time.time() - peer.time_added) / (60 * 60 * 24)
                yield "(#%4s, err: %s, found: %3s min, add: %.1f day) %30s -<br>" % (
                    connection_id, peer.connection_error, time_found,
                    time_added, key)
            yield "<br></td></tr>"
        yield "</table>"

        # No more if not in debug mode
        if not config.debug:
            raise StopIteration

        # Object types

        obj_count = {}
        for obj in gc.get_objects():
            obj_type = str(type(obj))
            if obj_type not in obj_count:
                obj_count[obj_type] = [0, 0]
            obj_count[obj_type][0] += 1  # Count
            obj_count[obj_type][1] += float(sys.getsizeof(obj)) / 1024  # Size

        yield "<br><br><b>Objects in memory (types: %s, total: %s, %.2fkb):</b><br>" % (
            len(obj_count), sum([stat[0] for stat in obj_count.values()]),
            sum([stat[1] for stat in obj_count.values()]))

        for obj, stat in sorted(obj_count.items(),
                                key=lambda x: x[1][0],
                                reverse=True):  # Sorted by count
            yield " - %.1fkb = %s x <a href=\"/Listobj?type=%s\">%s</a><br>" % (
                stat[1], stat[0], obj, cgi.escape(obj))

        # Classes

        class_count = {}
        for obj in gc.get_objects():
            obj_type = str(type(obj))
            if obj_type != "<type 'instance'>":
                continue
            class_name = obj.__class__.__name__
            if class_name not in class_count:
                class_count[class_name] = [0, 0]
            class_count[class_name][0] += 1  # Count
            class_count[class_name][1] += float(
                sys.getsizeof(obj)) / 1024  # Size

        yield "<br><br><b>Classes in memory (types: %s, total: %s, %.2fkb):</b><br>" % (
            len(class_count), sum([stat[0] for stat in class_count.values()]),
            sum([stat[1] for stat in class_count.values()]))

        for obj, stat in sorted(class_count.items(),
                                key=lambda x: x[1][0],
                                reverse=True):  # Sorted by count
            yield " - %.1fkb = %s x <a href=\"/Dumpobj?class=%s\">%s</a><br>" % (
                stat[1], stat[0], obj, cgi.escape(obj))

        from greenlet import greenlet
        objs = [obj for obj in gc.get_objects() if isinstance(obj, greenlet)]
        yield "<br>Greenlets (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(
                obj, hpy), cgi.escape(repr(obj).encode("utf8")))

        from Worker import Worker
        objs = [obj for obj in gc.get_objects() if isinstance(obj, Worker)]
        yield "<br>Workers (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(
                obj, hpy), cgi.escape(repr(obj)))

        from Connection import Connection
        objs = [obj for obj in gc.get_objects() if isinstance(obj, Connection)]
        yield "<br>Connections (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(
                obj, hpy), cgi.escape(repr(obj)))

        from socket import socket
        objs = [obj for obj in gc.get_objects() if isinstance(obj, socket)]
        yield "<br>Sockets (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(
                obj, hpy), cgi.escape(repr(obj)))

        from msgpack import Unpacker
        objs = [obj for obj in gc.get_objects() if isinstance(obj, Unpacker)]
        yield "<br>Msgpack unpacker (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(
                obj, hpy), cgi.escape(repr(obj)))

        from Site import Site
        objs = [obj for obj in gc.get_objects() if isinstance(obj, Site)]
        yield "<br>Sites (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(
                obj, hpy), cgi.escape(repr(obj)))

        objs = [
            obj for obj in gc.get_objects()
            if isinstance(obj, self.server.log.__class__)
        ]
        yield "<br>Loggers (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(
                obj, hpy), cgi.escape(repr(obj.name)))

        objs = [obj for obj in gc.get_objects() if isinstance(obj, UiRequest)]
        yield "<br>UiRequests (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(
                obj, hpy), cgi.escape(repr(obj)))

        from Peer import Peer
        objs = [obj for obj in gc.get_objects() if isinstance(obj, Peer)]
        yield "<br>Peers (%s):<br>" % len(objs)
        for obj in objs:
            yield " - %.1fkb: %s<br>" % (self.getObjSize(
                obj, hpy), cgi.escape(repr(obj)))

        objs = [(key, val) for key, val in sys.modules.iteritems()
                if val is not None]
        objs.sort()
        yield "<br>Modules (%s):<br>" % len(objs)
        for module_name, module in objs:
            yield " - %.3fkb: %s %s<br>" % (self.getObjSize(
                module, hpy), module_name, cgi.escape(repr(module)))

        gc.collect()  # Implicit grabage collection
        yield "Done in %.1f" % (time.time() - s)

    def actionDumpobj(self):

        import gc
        import sys

        self.sendHeader()

        if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local:
            yield "This function is disabled on this proxy"
            raise StopIteration

        # No more if not in debug mode
        if not config.debug:
            yield "Not in debug mode"
            raise StopIteration

        class_filter = self.get.get("class")

        yield """
        <style>
         * { font-family: monospace; white-space: pre }
         table * { text-align: right; padding: 0px 10px }
        </style>
        """

        objs = gc.get_objects()
        for obj in objs:
            obj_type = str(type(obj))
            if obj_type != "<type 'instance'>" or obj.__class__.__name__ != class_filter:
                continue
            yield "%.1fkb %s... " % (float(sys.getsizeof(obj)) / 1024,
                                     cgi.escape(str(obj)))
            for attr in dir(obj):
                yield "- %s: %s<br>" % (attr,
                                        cgi.escape(str(getattr(obj, attr))))
            yield "<br>"

        gc.collect()  # Implicit grabage collection

    def actionListobj(self):

        import gc
        import sys

        self.sendHeader()

        if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local:
            yield "This function is disabled on this proxy"
            raise StopIteration

        # No more if not in debug mode
        if not config.debug:
            yield "Not in debug mode"
            raise StopIteration

        type_filter = self.get.get("type")

        yield """
        <style>
         * { font-family: monospace; white-space: pre }
         table * { text-align: right; padding: 0px 10px }
        </style>
        """

        yield "Listing all %s objects in memory...<br>" % cgi.escape(
            type_filter)

        ref_count = {}
        objs = gc.get_objects()
        for obj in objs:
            obj_type = str(type(obj))
            if obj_type != type_filter:
                continue
            refs = [
                ref for ref in gc.get_referrers(obj) if
                hasattr(ref, "__class__") and ref.__class__.__name__ not in [
                    "list", "dict", "function", "type", "frame", "WeakSet",
                    "tuple"
                ]
            ]
            if not refs:
                continue
            try:
                yield "%.1fkb <span title=\"%s\">%s</span>... " % (
                    float(sys.getsizeof(obj)) / 1024, cgi.escape(
                        str(obj)), cgi.escape(str(obj)[0:100].ljust(100)))
            except:
                continue
            for ref in refs:
                yield " ["
                if "object at" in str(ref) or len(str(ref)) > 100:
                    yield str(ref.__class__.__name__)
                else:
                    yield str(ref.__class__.__name__) + ":" + cgi.escape(
                        str(ref))
                yield "] "
                ref_type = ref.__class__.__name__
                if ref_type not in ref_count:
                    ref_count[ref_type] = [0, 0]
                ref_count[ref_type][0] += 1  # Count
                ref_count[ref_type][1] += float(
                    sys.getsizeof(obj)) / 1024  # Size
            yield "<br>"

        yield "<br>Object referrer (total: %s, %.2fkb):<br>" % (
            len(ref_count), sum([stat[1] for stat in ref_count.values()]))

        for obj, stat in sorted(ref_count.items(),
                                key=lambda x: x[1][0],
                                reverse=True)[0:30]:  # Sorted by count
            yield " - %.1fkb = %s x %s<br>" % (stat[1], stat[0],
                                               cgi.escape(str(obj)))

        gc.collect()  # Implicit grabage collection

    def actionBenchmark(self):
        import sys
        import gc
        from contextlib import contextmanager

        output = self.sendHeader()

        if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local:
            yield "This function is disabled on this proxy"
            raise StopIteration

        @contextmanager
        def benchmark(name, standard):
            s = time.time()
            output("- %s" % name)
            try:
                yield 1
            except Exception, err:
                output("<br><b>! Error: %s</b><br>" % err)
            taken = time.time() - s
            if taken > 0:
                multipler = standard / taken
            else:
                multipler = 99
            if multipler < 0.3:
                speed = "Sloooow"
            elif multipler < 0.5:
                speed = "Ehh"
            elif multipler < 0.8:
                speed = "Goodish"
            elif multipler < 1.2:
                speed = "OK"
            elif multipler < 1.7:
                speed = "Fine"
            elif multipler < 2.5:
                speed = "Fast"
            elif multipler < 3.5:
                speed = "WOW"
            else:
                speed = "Insane!!"
            output("%.3fs [x%.2f: %s]<br>" % (taken, multipler, speed))
            time.sleep(0.01)

        yield """
        <style>
         * { font-family: monospace }
         table * { text-align: right; padding: 0px 10px }
        </style>
        """

        yield "Benchmarking ZeroNet %s (rev%s) Python %s on: %s...<br>" % (
            config.version, config.rev, sys.version, sys.platform)

        t = time.time()

        # CryptBitcoin
        yield "<br>CryptBitcoin:<br>"
        from Crypt import CryptBitcoin

        # seed = CryptBitcoin.newSeed()
        # yield "- Seed: %s<br>" % seed
        seed = "e180efa477c63b0f2757eac7b1cce781877177fe0966be62754ffd4c8592ce38"

        with benchmark("hdPrivatekey x 10", 0.7):
            for i in range(10):
                privatekey = CryptBitcoin.hdPrivatekey(seed, i * 10)
                yield "."
            valid = "5JsunC55XGVqFQj5kPGK4MWgTL26jKbnPhjnmchSNPo75XXCwtk"
            assert privatekey == valid, "%s != %s" % (privatekey, valid)

        data = "Hello" * 1024  # 5k
        with benchmark("sign x 10", 0.35):
            for i in range(10):
                yield "."
                sign = CryptBitcoin.sign(data, privatekey)
            valid = "G1GXaDauZ8vX/N9Jn+MRiGm9h+I94zUhDnNYFaqMGuOiBHB+kp4cRPZOL7l1yqK5BHa6J+W97bMjvTXtxzljp6w="
            assert sign == valid, "%s != %s" % (sign, valid)

        address = CryptBitcoin.privatekeyToAddress(privatekey)
        if CryptBitcoin.opensslVerify:  # Openssl avalible
            with benchmark("openssl verify x 100", 0.37):
                for i in range(100):
                    if i % 10 == 0:
                        yield "."
                    ok = CryptBitcoin.verify(data, address, sign)
                assert ok, "does not verify from %s" % address
        else:
            yield " - openssl verify x 100...not avalible :(<br>"

        openssl_verify_bk = CryptBitcoin.opensslVerify  # Emulate openssl not found in any way
        CryptBitcoin.opensslVerify = None
        with benchmark("pure-python verify x 10", 1.6):
            for i in range(10):
                yield "."
                ok = CryptBitcoin.verify(data, address, sign)
            assert ok, "does not verify from %s" % address
        CryptBitcoin.opensslVerify = openssl_verify_bk

        # CryptHash
        yield "<br>CryptHash:<br>"
        from Crypt import CryptHash
        from cStringIO import StringIO

        data = StringIO("Hello" * 1024 * 1024)  # 5m
        with benchmark("sha256 5M x 10", 0.6):
            for i in range(10):
                data.seek(0)
                hash = CryptHash.sha256sum(data)
                yield "."
            valid = "8cd629d9d6aff6590da8b80782a5046d2673d5917b99d5603c3dcb4005c45ffa"
            assert hash == valid, "%s != %s" % (hash, valid)

        data = StringIO("Hello" * 1024 * 1024)  # 5m
        with benchmark("sha512 5M x 10", 0.6):
            for i in range(10):
                data.seek(0)
                hash = CryptHash.sha512sum(data)
                yield "."
            valid = "9ca7e855d430964d5b55b114e95c6bbb114a6d478f6485df93044d87b108904d"
            assert hash == valid, "%s != %s" % (hash, valid)

        with benchmark("os.urandom(256) x 1000", 0.0065):
            for i in range(10):
                for y in range(100):
                    data = os.urandom(256)
                yield "."

        # Msgpack
        import msgpack
        yield "<br>Msgpack: (version: %s)<br>" % ".".join(
            map(str, msgpack.version))
        binary = 'fqv\xf0\x1a"e\x10,\xbe\x9cT\x9e(\xa5]u\x072C\x8c\x15\xa2\xa8\x93Sw)\x19\x02\xdd\t\xfb\xf67\x88\xd9\xee\x86\xa1\xe4\xb6,\xc6\x14\xbb\xd7$z\x1d\xb2\xda\x85\xf5\xa0\x97^\x01*\xaf\xd3\xb0!\xb7\x9d\xea\x89\xbbh8\xa1"\xa7]e(@\xa2\xa5g\xb7[\xae\x8eE\xc2\x9fL\xb6s\x19\x19\r\xc8\x04S\xd0N\xe4]?/\x01\xea\xf6\xec\xd1\xb3\xc2\x91\x86\xd7\xf4K\xdf\xc2lV\xf4\xe8\x80\xfc\x8ep\xbb\x82\xb3\x86\x98F\x1c\xecS\xc8\x15\xcf\xdc\xf1\xed\xfc\xd8\x18r\xf9\x80\x0f\xfa\x8cO\x97(\x0b]\xf1\xdd\r\xe7\xbf\xed\x06\xbd\x1b?\xc5\xa0\xd7a\x82\xf3\xa8\xe6@\xf3\ri\xa1\xb10\xf6\xd4W\xbc\x86\x1a\xbb\xfd\x94!bS\xdb\xaeM\x92\x00#\x0b\xf7\xad\xe9\xc2\x8e\x86\xbfi![%\xd31]\xc6\xfc2\xc9\xda\xc6v\x82P\xcc\xa9\xea\xb9\xff\xf6\xc8\x17iD\xcf\xf3\xeeI\x04\xe9\xa1\x19\xbb\x01\x92\xf5nn4K\xf8\xbb\xc6\x17e>\xa7 \xbbv'
        data = {
            "int": 1024 * 1024 * 1024,
            "float": 12345.67890,
            "text": "hello" * 1024,
            "binary": binary
        }
        with benchmark("pack 5K x 10 000", 0.78):
            for i in range(10):
                for y in range(1000):
                    data_packed = msgpack.packb(data)
                yield "."
            valid = """\x84\xa3int\xce@\x00\x00\x00\xa4text\xda\x14\x00hellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohello\xa5float\xcb@\xc8\x1c\xd6\xe61\xf8\xa1\xa6binary\xda\x01\x00fqv\xf0\x1a"e\x10,\xbe\x9cT\x9e(\xa5]u\x072C\x8c\x15\xa2\xa8\x93Sw)\x19\x02\xdd\t\xfb\xf67\x88\xd9\xee\x86\xa1\xe4\xb6,\xc6\x14\xbb\xd7$z\x1d\xb2\xda\x85\xf5\xa0\x97^\x01*\xaf\xd3\xb0!\xb7\x9d\xea\x89\xbbh8\xa1"\xa7]e(@\xa2\xa5g\xb7[\xae\x8eE\xc2\x9fL\xb6s\x19\x19\r\xc8\x04S\xd0N\xe4]?/\x01\xea\xf6\xec\xd1\xb3\xc2\x91\x86\xd7\xf4K\xdf\xc2lV\xf4\xe8\x80\xfc\x8ep\xbb\x82\xb3\x86\x98F\x1c\xecS\xc8\x15\xcf\xdc\xf1\xed\xfc\xd8\x18r\xf9\x80\x0f\xfa\x8cO\x97(\x0b]\xf1\xdd\r\xe7\xbf\xed\x06\xbd\x1b?\xc5\xa0\xd7a\x82\xf3\xa8\xe6@\xf3\ri\xa1\xb10\xf6\xd4W\xbc\x86\x1a\xbb\xfd\x94!bS\xdb\xaeM\x92\x00#\x0b\xf7\xad\xe9\xc2\x8e\x86\xbfi![%\xd31]\xc6\xfc2\xc9\xda\xc6v\x82P\xcc\xa9\xea\xb9\xff\xf6\xc8\x17iD\xcf\xf3\xeeI\x04\xe9\xa1\x19\xbb\x01\x92\xf5nn4K\xf8\xbb\xc6\x17e>\xa7 \xbbv"""
            assert data_packed == valid, "%s<br>!=<br>%s" % (repr(data_packed),
                                                             repr(valid))

        with benchmark("unpack 5K x 10 000", 1.2):
            for i in range(10):
                for y in range(1000):
                    data_unpacked = msgpack.unpackb(data_packed)
                yield "."
            assert data == data_unpacked, "%s != %s" % (data_unpacked, data)

        with benchmark("streaming unpack 5K x 10 000", 1.4):
            for i in range(10):
                unpacker = msgpack.Unpacker()
                for y in range(1000):
                    unpacker.feed(data_packed)
                    for data_unpacked in unpacker:
                        pass
                yield "."
            assert data == data_unpacked, "%s != %s" % (data_unpacked, data)

        # Db
        from Db import Db
        import sqlite3
        yield "<br>Db: (version: %s, API: %s)<br>" % (sqlite3.sqlite_version,
                                                      sqlite3.version)

        schema = {
            "db_name": "TestDb",
            "db_file": "%s/benchmark.db" % config.data_dir,
            "maps": {
                ".*": {
                    "to_table": {
                        "test": "test"
                    }
                }
            },
            "tables": {
                "test": {
                    "cols": [["test_id", "INTEGER"], ["title", "TEXT"],
                             ["json_id", "INTEGER REFERENCES json (json_id)"]],
                    "indexes":
                    ["CREATE UNIQUE INDEX test_key ON test(test_id, json_id)"],
                    "schema_changed":
                    1426195822
                }
            }
        }

        if os.path.isfile("%s/benchmark.db" % config.data_dir):
            os.unlink("%s/benchmark.db" % config.data_dir)

        with benchmark("Open x 10", 0.13):
            for i in range(10):
                db = Db(schema, "%s/benchmark.db" % config.data_dir)
                db.checkTables()
                db.close()
                yield "."

        db = Db(schema, "%s/benchmark.db" % config.data_dir)
        db.checkTables()
        import json

        with benchmark("Insert x 10 x 1000", 1.0):
            for u in range(10):  # 10 user
                data = {"test": []}
                for i in range(1000):  # 1000 line of data
                    data["test"].append({
                        "test_id":
                        i,
                        "title":
                        "Testdata for %s message %s" % (u, i)
                    })
                json.dump(data,
                          open("%s/test_%s.json" % (config.data_dir, u), "w"))
                db.updateJson("%s/test_%s.json" % (config.data_dir, u))
                os.unlink("%s/test_%s.json" % (config.data_dir, u))
                yield "."

        with benchmark("Buffered insert x 100 x 100", 1.3):
            cur = db.getCursor()
            cur.execute("BEGIN")
            cur.logging = False
            for u in range(100, 200):  # 100 user
                data = {"test": []}
                for i in range(100):  # 1000 line of data
                    data["test"].append({
                        "test_id":
                        i,
                        "title":
                        "Testdata for %s message %s" % (u, i)
                    })
                json.dump(data,
                          open("%s/test_%s.json" % (config.data_dir, u), "w"))
                db.updateJson("%s/test_%s.json" % (config.data_dir, u),
                              cur=cur)
                os.unlink("%s/test_%s.json" % (config.data_dir, u))
                if u % 10 == 0:
                    yield "."
            cur.execute("COMMIT")

        yield " - Total rows in db: %s<br>" % db.execute(
            "SELECT COUNT(*) AS num FROM test").fetchone()[0]

        with benchmark("Indexed query x 1000", 0.25):
            found = 0
            cur = db.getCursor()
            cur.logging = False
            for i in range(1000):  # 1000x by test_id
                res = cur.execute("SELECT * FROM test WHERE test_id = %s" % i)
                for row in res:
                    found += 1
                if i % 100 == 0:
                    yield "."

            assert found == 20000, "Found: %s != 20000" % found

        with benchmark("Not indexed query x 100", 0.6):
            found = 0
            cur = db.getCursor()
            cur.logging = False
            for i in range(100):  # 1000x by test_id
                res = cur.execute("SELECT * FROM test WHERE json_id = %s" % i)
                for row in res:
                    found += 1
                if i % 10 == 0:
                    yield "."

            assert found == 18900, "Found: %s != 18900" % found

        with benchmark("Like query x 100", 1.8):
            found = 0
            cur = db.getCursor()
            cur.logging = False
            for i in range(100):  # 1000x by test_id
                res = cur.execute(
                    "SELECT * FROM test WHERE title LIKE '%%message %s%%'" % i)
                for row in res:
                    found += 1
                if i % 10 == 0:
                    yield "."

            assert found == 38900, "Found: %s != 11000" % found

        db.close()
        if os.path.isfile("%s/benchmark.db" % config.data_dir):
            os.unlink("%s/benchmark.db" % config.data_dir)

        gc.collect()  # Implicit grabage collection

        # Zip
        yield "<br>Compression:<br>"
        import zipfile
        test_data = "Test" * 1024
        file_name = "\xc3\x81rv\xc3\xadzt\xc5\xb0r\xc5\x91t\xc3\xbck\xc3\xb6r\xc3\xb3g\xc3\xa9p\xe4\xb8\xad\xe5\x8d\x8e%s.txt"

        with benchmark("Zip pack x 10", 0.12):
            for i in range(10):
                with zipfile.ZipFile('%s/test.zip' % config.data_dir,
                                     'w') as archive:
                    for y in range(100):
                        zip_info = zipfile.ZipInfo(file_name % y,
                                                   (1980, 1, 1, 0, 0, 0))
                        zip_info.compress_type = zipfile.ZIP_DEFLATED
                        zip_info.create_system = 3
                        archive.writestr(zip_info, test_data)
                yield "."

            hash = CryptHash.sha512sum(
                open("%s/test.zip" % config.data_dir, "rb"))
            valid = "f6ef623e6653883a1758db14aa593350e26c9dc53a8406d6e6defd6029dbd483"
            assert hash == valid, "Invalid hash: %s != %s<br>" % (hash, valid)

        with benchmark("Zip unpack x 10", 0.2):
            for i in range(10):
                with zipfile.ZipFile('%s/test.zip' %
                                     config.data_dir) as archive:
                    for y in range(100):
                        assert archive.read(file_name % y) == test_data
                yield "."

        if os.path.isfile("%s/test.zip" % config.data_dir):
            os.unlink("%s/test.zip" % config.data_dir)

        # Tar.gz
        import tarfile
        import struct

        # Monkey patch _init_write_gz to use fixed date in order to keep the hash independent from datetime
        def nodate_write_gzip_header(self):
            self.mtime = 0
            original_write_gzip_header(self)

        import gzip
        original_write_gzip_header = gzip.GzipFile._write_gzip_header
        gzip.GzipFile._write_gzip_header = nodate_write_gzip_header

        test_data_io = StringIO("Test" * 1024)
        with benchmark("Tar.gz pack x 10", 0.3):
            for i in range(10):
                with tarfile.open('%s/test.tar.gz' % config.data_dir,
                                  'w:gz') as archive:
                    for y in range(100):
                        test_data_io.seek(0)
                        tar_info = tarfile.TarInfo(file_name % y)
                        tar_info.size = 4 * 1024
                        archive.addfile(tar_info, test_data_io)
                yield "."

            hash = CryptHash.sha512sum(
                open("%s/test.tar.gz" % config.data_dir, "rb"))
            valid = "4704ebd8c987ed6f833059f1de9c475d443b0539b8d4c4cb8b49b26f7bbf2d19"
            assert hash == valid, "Invalid hash: %s != %s<br>" % (hash, valid)

        with benchmark("Tar.gz unpack x 10", 0.2):
            for i in range(10):
                with tarfile.open('%s/test.tar.gz' % config.data_dir,
                                  'r:gz') as archive:
                    for y in range(100):
                        assert archive.extractfile(file_name %
                                                   y).read() == test_data
                yield "."

        if os.path.isfile("%s/test.tar.gz" % config.data_dir):
            os.unlink("%s/test.tar.gz" % config.data_dir)

        # Tar.bz2
        import tarfile
        test_data_io = StringIO("Test" * 1024)
        with benchmark("Tar.bz2 pack x 10", 2.0):
            for i in range(10):
                with tarfile.open('%s/test.tar.bz2' % config.data_dir,
                                  'w:bz2') as archive:
                    for y in range(100):
                        test_data_io.seek(0)
                        tar_info = tarfile.TarInfo(file_name % y)
                        tar_info.size = 4 * 1024
                        archive.addfile(tar_info, test_data_io)
                yield "."

            hash = CryptHash.sha512sum(
                open("%s/test.tar.bz2" % config.data_dir, "rb"))
            valid = "90cba0b4d9abaa37b830bf37e4adba93bfd183e095b489ebee62aaa94339f3b5"
            assert hash == valid, "Invalid hash: %s != %s<br>" % (hash, valid)

        with benchmark("Tar.bz2 unpack x 10", 0.5):
            for i in range(10):
                with tarfile.open('%s/test.tar.bz2' % config.data_dir,
                                  'r:bz2') as archive:
                    for y in range(100):
                        assert archive.extractfile(file_name %
                                                   y).read() == test_data
                yield "."

        if os.path.isfile("%s/test.tar.bz2" % config.data_dir):
            os.unlink("%s/test.tar.bz2" % config.data_dir)

        yield "<br>Done. Total: %.2fs" % (time.time() - t)
Exemple #44
0
    def actionPeerBroadcast(self, params):
        ip = "%s:%s" % (self.connection.ip, self.connection.port)

        raw = json.loads(params["raw"])

        # Check whether P2P messages are supported
        site = self.sites.get(raw["site"])
        content_json = site.storage.loadJson("content.json")
        if "p2p_filter" not in content_json:
            self.connection.log("Site %s doesn't support P2P messages" %
                                raw["site"])
            self.connection.badAction(5)
            return

        # Was the message received yet?
        if params["hash"] in site.p2p_received:
            return
        site.p2p_received.append(params["hash"])

        # Check whether the message matches passive filter
        if not SafeRe.match(content_json["p2p_filter"],
                            json.dumps(raw["message"])):
            self.connection.log("Invalid message for site %s: %s" %
                                (raw["site"], raw["message"]))
            self.connection.badAction(5)
            return

        # Not so fast
        if "p2p_freq_limit" in content_json and time.time(
        ) - site.p2p_last_recv.get(ip, 0) < content_json["p2p_freq_limit"]:
            self.connection.log("Too fast messages from %s" % raw["site"])
            self.connection.badAction(2)
            return
        site.p2p_last_recv[ip] = time.time()

        # Not so much
        if "p2p_size_limit" in content_json and len(json.dumps(
                raw["message"])) > content_json["p2p_size_limit"]:
            self.connection.log("Too big message from %s" % raw["site"])
            self.connection.badAction(7)
            return

        # Verify signature
        if params["signature"]:
            signature_address, signature = params["signature"].split("|")
            what = "%s|%s|%s" % (signature_address, params["hash"],
                                 params["raw"])
            from Crypt import CryptBitcoin
            if not CryptBitcoin.verify(what, signature_address, signature):
                self.connection.log("Invalid signature")
                self.connection.badAction(7)
                return
        else:
            signature_address = ""

        # Check that the signature address is correct
        if "p2p_signed_only" in content_json:
            valid = content_json["p2p_signed_only"]
            if valid is True and not signature_address:
                self.connection.log("Not signed message")
                self.connection.badAction(5)
                return
            elif isinstance(valid, str) and signature_address != valid:
                self.connection.log(
                    "Message signature is invalid: %s not in [%r]" %
                    (signature_address, valid))
                self.connection.badAction(5)
                return
            elif isinstance(valid, list) and signature_address not in valid:
                self.connection.log(
                    "Message signature is invalid: %s not in %r" %
                    (signature_address, valid))
                self.connection.badAction(5)
                return

        # Send to WebSocket
        websockets = [
            ws for ws in site.websockets if "peerReceive" in ws.channels
        ]
        for ws in websockets:
            ws.cmd(
                "peerReceive", {
                    "ip": ip,
                    "hash": params["hash"],
                    "message": raw["message"],
                    "signed_by": signature_address
                })

        # Maybe active filter will reply?
        if websockets:
            # Wait for p2p_result
            result = gevent.spawn(self.p2pWaitMessage, site,
                                  params["hash"]).join()
            del site.p2p_result[params["hash"]]
            if not result:
                self.connection.badAction(10)
                return

        # Save to cache
        if not websockets and raw["immediate"]:
            site.p2p_unread.append({
                "ip":
                "%s:%s" % (self.connection.ip, self.connection.port),
                "hash":
                params["hash"],
                "message":
                raw["message"],
                "signed_by":
                signature_address
            })

        # Now send to neighbour peers
        if raw["broadcast"]:
            # Get peer list
            peers = site.getConnectedPeers()
            if len(peers) < raw[
                    "peer_count"]:  # Add more, non-connected peers if necessary
                peers += site.getRecentPeers(raw["peer_count"] - len(peers))

            # Send message to peers
            for peer in peers:
                gevent.spawn(peer.connection.request, "peerBroadcast", params)
Exemple #45
0
 def actionEcdsaVerify(self, to, data, address, signature):
     self.response(to, CryptBitcoin.verify(data, address, signature))
Exemple #46
0
    def testMissingCert(self, site):
        user_priv = "5Kk7FSA63FC2ViKmKLuBxk9gQkaQ5713hKq8LmFAf4cVeXh6K6A"
        cert_priv = "5JusJDSjHaMHwUjDT3o6eQ54pA6poo8La5fAgn1wNc3iK59jxjA"

        user_content = site.content_manager.contents[
            "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json"]
        rules_content = site.content_manager.contents[
            "data/users/content.json"]

        # Override valid cert signers for the test
        rules_content["user_contents"]["cert_signers"]["zeroid.bit"] = [
            "14wgQ4VDDZNoRMFF4yCDuTrBSHmYhL3bet",
            "1iD5ZQJMNXu43w1qLB8sfdHVKppVMduGz"
        ]

        # Sign a valid cert
        user_content["cert_sign"] = CryptBitcoin.sign(
            "1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C#%s/%s" %
            (user_content["cert_auth_type"],
             user_content["cert_user_id"].split("@")[0]), cert_priv)
        signed_content = site.content_manager.sign(
            "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json",
            user_priv,
            filewrite=False)

        assert site.content_manager.verifyFile(
            "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json",
            io.BytesIO(json.dumps(signed_content).encode()),
            ignore_same=False)

        # Test invalid cert_user_id
        user_content["cert_user_id"] = "nodomain"
        user_content["signs"] = {
            "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT":
            CryptBitcoin.sign(json.dumps(user_content, sort_keys=True),
                              user_priv)
        }
        signed_content = site.content_manager.sign(
            "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json",
            user_priv,
            filewrite=False)
        with pytest.raises(VerifyError) as err:
            site.content_manager.verifyFile(
                "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json",
                io.BytesIO(json.dumps(signed_content).encode()),
                ignore_same=False)
        assert "Invalid domain in cert_user_id" in str(err.value)

        # Test removed cert
        del user_content["cert_user_id"]
        del user_content["cert_auth_type"]
        del user_content["signs"]  # Remove signs before signing
        user_content["signs"] = {
            "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT":
            CryptBitcoin.sign(json.dumps(user_content, sort_keys=True),
                              user_priv)
        }
        signed_content = site.content_manager.sign(
            "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json",
            user_priv,
            filewrite=False)
        with pytest.raises(VerifyError) as err:
            site.content_manager.verifyFile(
                "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json",
                io.BytesIO(json.dumps(signed_content).encode()),
                ignore_same=False)
        assert "Missing cert_user_id" in str(err.value)
    def sign(self,
             inner_path="content.json",
             privatekey=None,
             filewrite=True,
             update_changed_files=False):
        content = self.contents.get(inner_path)
        if not content:  # Content not exits yet, load default one
            self.log.info("File %s not exits yet, loading default values..." %
                          inner_path)
            content = {"files": {}, "signs": {}}  # Default content.json
            if inner_path == "content.json":  # Its the root content.json, add some more fields
                content["title"] = "%s - ZeroNet_" % self.site.address
                content["description"] = ""
                content["signs_required"] = 1
                content["ignore"] = ""

        directory = self.toDir(self.site.storage.getPath(inner_path))
        self.log.info("Opening site data directory: %s..." % directory)

        hashed_files = {}
        changed_files = [inner_path]
        for root, dirs, files in os.walk(directory):
            for file_name in files:
                file_path = self.site.storage.getPath(
                    "%s/%s" % (root.strip("/"), file_name))
                file_inner_path = re.sub(re.escape(directory), "", file_path)

                if file_name == "content.json" or (
                        content.get("ignore")
                        and re.match(content["ignore"], file_inner_path)
                ) or file_name.startswith(
                        "."
                ):  # Ignore content.json, definied regexp and files starting with .
                    self.log.info("- [SKIPPED] %s" % file_inner_path)
                else:
                    sha512sum = CryptHash.sha512sum(
                        file_path)  # Calculate sha512 sum of file
                    self.log.info("- %s (SHA512: %s)" %
                                  (file_inner_path, sha512sum))
                    hashed_files[file_inner_path] = {
                        "sha512": sha512sum,
                        "size": os.path.getsize(file_path)
                    }
                    if file_inner_path in content["files"].keys(
                    ) and hashed_files[file_inner_path]["sha512"] != content[
                            "files"][file_inner_path].get("sha512"):
                        changed_files.append(file_path)

        self.log.debug("Changed files: %s" % changed_files)
        if update_changed_files:
            for file_path in changed_files:
                self.site.storage.onUpdated(file_path)

        # Generate new content.json
        self.log.info("Adding timestamp and sha512sums to new content.json...")

        new_content = content.copy()  # Create a copy of current content.json
        new_content["files"] = hashed_files  # Add files sha512 hash
        new_content["modified"] = time.time()  # Add timestamp
        if inner_path == "content.json":
            new_content["address"] = self.site.address
            new_content["zeronet_version"] = config.version
            new_content["signs_required"] = content.get("signs_required", 1)

        from Crypt import CryptBitcoin
        self.log.info("Verifying private key...")
        privatekey_address = CryptBitcoin.privatekeyToAddress(privatekey)
        valid_signers = self.getValidSigners(inner_path)
        if privatekey_address not in valid_signers:
            return self.log.error(
                "Private key invalid! Valid signers: %s, Private key address: %s"
                % (valid_signers, privatekey_address))
        self.log.info("Correct %s in valid signers: %s" %
                      (privatekey_address, valid_signers))

        if inner_path == "content.json" and privatekey_address == self.site.address:  # If signing using the root key sign the valid signers
            new_content["signers_sign"] = CryptBitcoin.sign(
                "%s:%s" %
                (new_content["signs_required"], ",".join(valid_signers)),
                privatekey)
            if not new_content["signers_sign"]:
                self.log.info("Old style address, signers_sign is none")

        self.log.info("Signing %s..." % inner_path)

        if "signs" in new_content:
            del (new_content["signs"])  # Delete old signs
        if "sign" in new_content:
            del (new_content["sign"]
                 )  # Delete old sign (backward compatibility)

        sign_content = json.dumps(new_content, sort_keys=True)
        sign = CryptBitcoin.sign(sign_content, privatekey)
        #new_content["signs"] = content.get("signs", {}) # TODO: Multisig
        if sign:  # If signing is successful (not an old address)
            new_content["signs"] = {}
            new_content["signs"][privatekey_address] = sign

        if inner_path == "content.json":  # To root content.json add old format sign for backward compatibility
            oldsign_content = json.dumps(new_content, sort_keys=True)
            new_content["sign"] = CryptBitcoin.signOld(oldsign_content,
                                                       privatekey)

        if not self.validContent(inner_path, new_content):
            self.log.error("Sign failed: Invalid content")
            return False

        if filewrite:
            self.log.info("Saving to %s..." % inner_path)
            json.dump(new_content,
                      open(self.site.storage.getPath(inner_path), "w"),
                      indent=2,
                      sort_keys=True)

        self.log.info("File %s signed!" % inner_path)

        if filewrite:  # Written to file
            return True
        else:  # Return the new content
            return new_content
Exemple #48
0
 def cryptSign(self, message, privatekey):
     from Crypt import CryptBitcoin
     print CryptBitcoin.sign(message, privatekey)
Exemple #49
0
    def testVerify(self, site):
        privatekey = "5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv"  # For 1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT
        user_inner_path = "data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/content.json"
        data_dict = site.storage.loadJson(user_inner_path)
        users_content = site.content_manager.contents[
            "data/users/content.json"]

        data = io.BytesIO(json.dumps(data_dict).encode())
        assert site.content_manager.verifyFile(user_inner_path,
                                               data,
                                               ignore_same=False)

        # Test max size exception by setting allowed to 0
        rules = site.content_manager.getRules(user_inner_path, data_dict)
        assert rules["max_size"] == 10000
        assert users_content["user_contents"]["permission_rules"][".*"][
            "max_size"] == 10000

        users_content["user_contents"]["permission_rules"][".*"][
            "max_size"] = 0
        rules = site.content_manager.getRules(user_inner_path, data_dict)
        assert rules["max_size"] == 0
        data = io.BytesIO(json.dumps(data_dict).encode())

        with pytest.raises(VerifyError) as err:
            site.content_manager.verifyFile(user_inner_path,
                                            data,
                                            ignore_same=False)
        assert "Include too large" in str(err.value)
        users_content["user_contents"]["permission_rules"][".*"][
            "max_size"] = 10000  # Reset

        # Test max optional size exception
        # 1 MB gif = Allowed
        data_dict["files_optional"]["peanut-butter-jelly-time.gif"][
            "size"] = 1024 * 1024
        del data_dict["signs"]  # Remove signs before signing
        data_dict["signs"] = {
            "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT":
            CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True),
                              privatekey)
        }
        data = io.BytesIO(json.dumps(data_dict).encode())
        assert site.content_manager.verifyFile(user_inner_path,
                                               data,
                                               ignore_same=False)

        # 100 MB gif = Not allowed
        data_dict["files_optional"]["peanut-butter-jelly-time.gif"][
            "size"] = 100 * 1024 * 1024
        del data_dict["signs"]  # Remove signs before signing
        data_dict["signs"] = {
            "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT":
            CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True),
                              privatekey)
        }
        data = io.BytesIO(json.dumps(data_dict).encode())
        with pytest.raises(VerifyError) as err:
            site.content_manager.verifyFile(user_inner_path,
                                            data,
                                            ignore_same=False)
        assert "Include optional files too large" in str(err.value)
        data_dict["files_optional"]["peanut-butter-jelly-time.gif"][
            "size"] = 1024 * 1024  # Reset

        # hello.exe = Not allowed
        data_dict["files_optional"]["hello.exe"] = data_dict["files_optional"][
            "peanut-butter-jelly-time.gif"]
        del data_dict["signs"]  # Remove signs before signing
        data_dict["signs"] = {
            "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT":
            CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True),
                              privatekey)
        }
        data = io.BytesIO(json.dumps(data_dict).encode())
        with pytest.raises(VerifyError) as err:
            site.content_manager.verifyFile(user_inner_path,
                                            data,
                                            ignore_same=False)
        assert "Optional file not allowed" in str(err.value)
        del data_dict["files_optional"]["hello.exe"]  # Reset

        # Includes not allowed in user content
        data_dict["includes"] = {"other.json": {}}
        del data_dict["signs"]  # Remove signs before signing
        data_dict["signs"] = {
            "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT":
            CryptBitcoin.sign(json.dumps(data_dict, sort_keys=True),
                              privatekey)
        }
        data = io.BytesIO(json.dumps(data_dict).encode())
        with pytest.raises(VerifyError) as err:
            site.content_manager.verifyFile(user_inner_path,
                                            data,
                                            ignore_same=False)
        assert "Includes not allowed" in str(err.value)
    def sign(self, inner_path="content.json", privatekey=None, filewrite=True, update_changed_files=False, extend=None):
        content = self.contents.get(inner_path)
        if not content:  # Content not exist yet, load default one
            self.log.info("File %s not exist yet, loading default values..." % inner_path)
            content = {"files": {}, "signs": {}}  # Default content.json
            if inner_path == "content.json":  # It's the root content.json, add some more fields
                content["title"] = "%s - ZeroNet_" % self.site.address
                content["description"] = ""
                content["signs_required"] = 1
                content["ignore"] = ""
            if extend:
                content.update(extend)  # Add custom fields

        directory = helper.getDirname(self.site.storage.getPath(inner_path))
        inner_directory = helper.getDirname(inner_path)
        self.log.info("Opening site data directory: %s..." % directory)

        changed_files = [inner_path]
        files_node, files_optional_node = self.hashFiles(
            helper.getDirname(inner_path), content.get("ignore"), content.get("optional")
        )

        # Find changed files
        files_merged = files_node.copy()
        files_merged.update(files_optional_node)
        for file_relative_path, file_details in files_merged.iteritems():
            old_hash = content["files"].get(file_relative_path, {}).get("sha512")
            new_hash = files_merged[file_relative_path]["sha512"]
            if old_hash != new_hash:
                changed_files.append(inner_directory + file_relative_path)

        self.log.debug("Changed files: %s" % changed_files)
        if update_changed_files:
            for file_path in changed_files:
                self.site.storage.onUpdated(file_path)

        # Generate new content.json
        self.log.info("Adding timestamp and sha512sums to new content.json...")

        new_content = content.copy()  # Create a copy of current content.json
        new_content["files"] = files_node  # Add files sha512 hash
        if files_optional_node:
            new_content["files_optional"] = files_optional_node
        elif "files_optional" in new_content:
            del new_content["files_optional"]

        new_content["modified"] = time.time()  # Add timestamp
        if inner_path == "content.json":
            new_content["zeronet_version"] = config.version
            new_content["signs_required"] = content.get("signs_required", 1)

        new_content["address"] = self.site.address
        new_content["inner_path"] = inner_path

        # Verify private key
        from Crypt import CryptBitcoin
        self.log.info("Verifying private key...")
        privatekey_address = CryptBitcoin.privatekeyToAddress(privatekey)
        valid_signers = self.getValidSigners(inner_path, new_content)
        if privatekey_address not in valid_signers:
            return self.log.error(
                "Private key invalid! Valid signers: %s, Private key address: %s" %
                (valid_signers, privatekey_address)
            )
        self.log.info("Correct %s in valid signers: %s" % (privatekey_address, valid_signers))

        if inner_path == "content.json" and privatekey_address == self.site.address:
            # If signing using the root key, then sign the valid signers
            new_content["signers_sign"] = CryptBitcoin.sign(
                "%s:%s" % (new_content["signs_required"], ",".join(valid_signers)), privatekey
            )
            if not new_content["signers_sign"]:
                self.log.info("Old style address, signers_sign is none")

        self.log.info("Signing %s..." % inner_path)

        if "signs" in new_content:
            del(new_content["signs"])  # Delete old signs
        if "sign" in new_content:
            del(new_content["sign"])  # Delete old sign (backward compatibility)

        sign_content = json.dumps(new_content, sort_keys=True)
        sign = CryptBitcoin.sign(sign_content, privatekey)
        # new_content["signs"] = content.get("signs", {}) # TODO: Multisig
        if sign:  # If signing is successful (not an old address)
            new_content["signs"] = {}
            new_content["signs"][privatekey_address] = sign

        if inner_path == "content.json":  # To root content.json add old format sign for backward compatibility
            oldsign_content = json.dumps(new_content, sort_keys=True)
            new_content["sign"] = CryptBitcoin.signOld(oldsign_content, privatekey)

        if not self.verifyContent(inner_path, new_content):
            self.log.error("Sign failed: Invalid content")
            return False

        if filewrite:
            self.log.info("Saving to %s..." % inner_path)
            self.site.storage.writeJson(inner_path, new_content)
            self.contents[inner_path] = new_content

        self.log.info("File %s signed!" % inner_path)

        if filewrite:  # Written to file
            return True
        else:  # Return the new content
            return new_content
Exemple #51
0
 def cryptGetPrivatekey(self, master_seed, site_address_index=None):
     from Crypt import CryptBitcoin
     print CryptBitcoin.hdPrivatekey(master_seed, site_address_index)
    def verifyFile(self, inner_path, file, ignore_same=True):
        if inner_path.endswith("content.json"):  # content.json: Check using sign
            from Crypt import CryptBitcoin
            try:
                new_content = json.load(file)
                if inner_path in self.contents:
                    old_content = self.contents.get(inner_path)
                    # Checks if its newer the ours
                    if old_content["modified"] == new_content["modified"] and ignore_same:  # Ignore, have the same content.json
                        return None
                    elif old_content["modified"] > new_content["modified"]:  # We have newer
                        self.log.debug(
                            "We have newer %s (Our: %s, Sent: %s)" %
                            (inner_path, old_content["modified"], new_content["modified"])
                        )
                        # gevent.spawn(self.site.publish, inner_path=inner_path)  # Try to fix the broken peers
                        return False
                if new_content["modified"] > time.time() + 60 * 60 * 24:  # Content modified in the far future (allow 1 day+)
                    self.log.error("%s modify is in the future!" % inner_path)
                    return False
                # Check sign
                sign = new_content.get("sign")
                signs = new_content.get("signs", {})
                if "sign" in new_content:
                    del(new_content["sign"])  # The file signed without the sign
                if "signs" in new_content:
                    del(new_content["signs"])  # The file signed without the signs
                sign_content = json.dumps(new_content, sort_keys=True)  # Dump the json to string to remove whitepsace

                if not self.verifyContent(inner_path, new_content):
                    return False  # Content not valid (files too large, invalid files)

                if signs:  # New style signing
                    valid_signers = self.getValidSigners(inner_path, new_content)
                    signs_required = self.getSignsRequired(inner_path, new_content)

                    if inner_path == "content.json" and len(valid_signers) > 1:  # Check signers_sign on root content.json
                        if not CryptBitcoin.verify(
                            "%s:%s" % (signs_required, ",".join(valid_signers)), self.site.address, new_content["signers_sign"]
                        ):
                            self.log.error("%s invalid signers_sign!" % inner_path)
                            return False

                    if inner_path != "content.json" and not self.verifyCert(inner_path, new_content):  # Check if cert valid
                        self.log.error("%s invalid cert!" % inner_path)
                        return False

                    valid_signs = 0
                    for address in valid_signers:
                        if address in signs:
                            valid_signs += CryptBitcoin.verify(sign_content, address, signs[address])
                        if valid_signs >= signs_required:
                            break  # Break if we has enough signs
                    self.log.debug("%s: Valid signs: %s/%s" % (inner_path, valid_signs, signs_required))
                    return valid_signs >= signs_required
                else:  # Old style signing
                    return CryptBitcoin.verify(sign_content, self.site.address, sign)

            except Exception, err:
                self.log.error("Verify sign error: %s" % Debug.formatException(err))
                return False
Exemple #53
0
    def signContent(self, privatekey=None):
        if not self.content:  # New site
            self.log.info(
                "Site not exits yet, loading default content.json values...")
            self.content = {
                "files": {},
                "title": "%s - ZeroNet_" % self.address,
                "sign": "",
                "modified": 0.0,
                "description": "",
                "address": self.address,
                "ignore": "",
                "zeronet_version": config.version
            }  # Default content.json

        self.log.info("Opening site data directory: %s..." % self.directory)

        hashed_files = {}

        for root, dirs, files in os.walk(self.directory):
            for file_name in files:
                file_path = self.getPath("%s/%s" % (root, file_name))

                if file_name == "content.json" or (
                        self.content["ignore"] and re.match(
                            self.content["ignore"],
                            file_path.replace(self.directory + "/", ""))
                ):  # Dont add content.json and ignore regexp pattern definied in content.json
                    self.log.info("- [SKIPPED] %s" % file_path)
                else:
                    sha1sum = CryptHash.sha1sum(
                        file_path)  # Calculate sha1 sum of file
                    sha512sum = CryptHash.sha512sum(
                        file_path)  # Calculate sha512 sum of file
                    inner_path = re.sub("^%s/" % re.escape(self.directory), "",
                                        file_path)
                    self.log.info("- %s (SHA512: %s)" % (file_path, sha512sum))
                    hashed_files[inner_path] = {
                        "sha1": sha1sum,
                        "sha512": sha512sum,
                        "size": os.path.getsize(file_path)
                    }

        # Generate new content.json
        self.log.info("Adding timestamp and sha512sums to new content.json...")

        content = self.content.copy()  # Create a copy of current content.json
        content["address"] = self.address
        content["files"] = hashed_files  # Add files sha512 hash
        content["modified"] = time.time()  # Add timestamp
        content["zeronet_version"] = config.version  # Signer's zeronet version
        del (content["sign"])  # Delete old sign

        # Signing content
        from Crypt import CryptBitcoin

        self.log.info("Verifying private key...")
        privatekey_address = CryptBitcoin.privatekeyToAddress(privatekey)
        if self.address != privatekey_address:
            return self.log.error(
                "Private key invalid! Site address: %s, Private key address: %s"
                % (self.address, privatekey_address))

        self.log.info("Signing modified content.json...")
        sign_content = json.dumps(content, sort_keys=True)
        sign = CryptBitcoin.sign(sign_content, privatekey)
        content["sign"] = sign

        # Saving modified content.json
        self.log.info("Saving to %s/content.json..." % self.directory)
        open("%s/content.json" % self.directory,
             "w").write(json.dumps(content, indent=4, sort_keys=True))

        self.log.info("Site signed!")
        return True