def create(a, M, A=None, N=None, m=None, a_packed=None, M_packed=None, A_packed=None, N_packed=None): if A==None: A = ed25519.Point.B_times(a) if N==None: N = M * a if A_packed==None: A_packed = A.pack() if N_packed==None: N_packed = N.pack() if M_packed==None: M_packed = M.pack() if a_packed==None: a_packed = ed25519.scalar_pack(a) r = ed25519.scalar_unpack(common.sha256( b"DHTProof" + a_packed + M_packed)) R_B = ed25519.Point.B_times(r) if m==None: R_M = M * r else: R_M = ed25519.Point.B_times(m*r) R_M_packed = R_M.pack() R_B_packed = R_B.pack() h = ed25519.scalar_unpack(common.sha256( A_packed + M_packed + N_packed + R_M_packed + R_B_packed)) s = (r + h * a) % ed25519.l return DHTProof(R_M, R_B, s, R_M_packed=R_M_packed, R_B_packed=R_B_packed)
def benchmark_depseudonymize(self, args): ip = os.urandom(16) # manually compute investigator-local pseudonym pseudonym_secrets = {} for peer_secrets in self.args.secrets.peers.values(): for shard, shard_secrets in peer_secrets.by_shard.items(): pseudonym_secrets[shard] \ = shard_secrets.pseudonym_component_secret s = 1 e = ed25519.scalar_unpack(common.sha256(b"PEP3 investigator")) for secret in pseudonym_secrets.values(): s *= pow(ed25519.scalar_unpack(secret),e,ed25519.l) s %= ed25519.l investigator_local_ip = ( ed25519.Point.lizard(ip)*s ).pack() # manually create warrant warrant = pep3_pb2.DepseudonymizationRequest.Warrant() warrant.act.actor = b"PEP3 investigator" warrant.act.name.state = pep3_pb2.Pseudonymizable.UNENCRYPTED_PSEUDONYM warrant.act.name.data = investigator_local_ip self.investigator.encrypt([ warrant.act.name ], cheats.public_key(self.args.secrets, b"PEP3 investigator", 'pseudonym')) warrant.signature = crypto.sign( crypto.load_privatekey(crypto.FILETYPE_PEM, self.args.secrets.root_certificate_keys.warrants), warrant.act.SerializeToString(), 'sha256') result = self.investigator.connect_to("investigator")\ .Depseudonymize(warrant)
def test_localization_of_pseudonym(self): name = b" a 16 byte name " target = b"PEP3 storage_facility" pp = pep3_pb2.Pseudonymizable( data=name, state=pep3_pb2.Pseudonymizable.UNENCRYPTED_NAME) self.collector.pseudonymize([pp]) self.collector.relocalize([pp], self.config.collector.warrants.to_sf) sfp = elgamal.Triple.unpack(pp.data)\ .decrypt(self.sf.private_keys['pseudonym']) pseudonym_secrets = {} for peer_secrets in self.secrets.peers.values(): for shard, shard_secrets in peer_secrets.by_shard.items(): pseudonym_secrets[shard] \ = shard_secrets.pseudonym_component_secret s = 1 e = ed25519.scalar_unpack(common.sha256(target)) for secret in pseudonym_secrets.values(): s *= pow(ed25519.scalar_unpack(secret), e, ed25519.l) s %= ed25519.l self.assertEqual(sfp * ed25519.scalar_inv(s), ed25519.Point.lizard(name))
def test_sha256(self): for i in range(10): n = random.randint(10,200) data = os.urandom(n) cbuf = ristretto.ffi.new("unsigned char[]", 32) buf = ristretto.ffi.buffer(cbuf) ristretto.lib.sha256(data, n, cbuf) self.assertEqual(buf[:], common.sha256(data))
def lizard_without_elligator(payload, N=16): assert (N <= 30 and N % 2 == 0) assert (len(payload) == N) data = bytearray(common.sha256(payload)) data[16 - N // 2:16 + N // 2] = payload data[0] &= 0b11111110 data[31] &= 0b00111111 return bytes(data)
def Enroll(self, request, context): common_name = common.authenticate(context) # TODO: have a smarter check return_components = (common_name == b"PEP3 investigator" or common_name == b"PEP3 researcher") response = pep3_pb2.EnrollmentResponse() # 1. [ REMOVED ] # 2. set response.by_shard[...].private_local_keys # and response.components[...].keys e = ed25519.scalar_unpack(common.sha256(common_name)) # "e" is the exponent used to compute the key/pseudonym components for shard, shard_secrets in self.pep.secrets.by_shard.items(): for domain, domain_secrets in shard_secrets.by_domain.items(): x = ed25519.scalar_unpack(domain_secrets.private_master_key) k = ed25519.scalar_unpack(domain_secrets.key_component_secret) k_local = pow(k, e, ed25519.l) if return_components: self.pep._cryptopu.certified_component_create( response.components[shard].keys[domain], self.pep.global_config.\ components[shard].keys[domain].\ base_times_two_to_the_power_of, k, e) x_local = (k_local * x) % ed25519.l response.by_shard[shard].private_local_keys[domain]\ = ed25519.scalar_pack(x_local) # 3. set response.components[...].pseudonym if return_components: for shard in self.pep.config.shards: s = ed25519.scalar_unpack( self.pep.secrets.by_shard[shard]\ .pseudonym_component_secret) self.pep._cryptopu.certified_component_create( response.components[shard].pseudonym, self.pep.global_config.\ components[shard].pseudonym.\ base_times_two_to_the_power_of, s, e) return response
def private_key(secrets, common_name, domain): e = ed25519.scalar_unpack(common.sha256(common_name)) x = 1 for shard, shard_secrets in secrets_by_shard(secrets).items(): x *= ed25519.scalar_unpack( shard_secrets.by_domain[domain]\ .private_master_key ) x %= ed25519.l x *= pow(ed25519.scalar_unpack( shard_secrets.by_domain[domain].key_component_secret), e, ed25519.l) x %= ed25519.l return x
def lizard_inv(self, N=16): assert (N <= 30 and N % 2 == 0) for x in self.elligator2_inv(): data = fe_pack(x) payload = data[16 - N // 2:16 + N // 2] data_hash = bytearray(common.sha256(payload)) data_hash[0] &= 0b11111110 data_hash[31] &= 0b00111111 if data[:16-N//2]==data_hash[:16-N//2] \ and data[16+N//2:]==data_hash[16+N//2:]: return payload raise NoPreimage()
def server(port, secret, out=None): logger.info("Starting server ...") server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if out: out = pycommons.open_file(output, 'wb') generic_logging.add_file_handler(out, logger) else: out = sys.stdout try: server_socket.bind(('', port)) except socket.error as e: logger.error('Bind failed! :' + e[1]) sys.exit(-1) server_socket.listen(10) while 1: sock, addr = server_socket.accept() # print str(addr) length = struct.unpack('>Q', common.sock_read(sock, 8))[0] logger.info("Request length: %d" % (length)) msg_buf = common.sock_read(sock, length) request = protocol_pb2.Request() request.ParseFromString(msg_buf) if request.secret != common.sha256(secret): response = protocol_pb2.Response() response.type = protocol_pb2.Response.GENERIC response.status = protocol_pb2.ERROR response.error = "Invalid secret" send_response(sock, response) sock.close() continue response = protocol_pb2.Response() if request.type == protocol_pb2.Request.KEY_REQUEST: handle_key_request(sock, request.keyRequest, response) send_response(sock, response) sock.close()
def RegisterComponents(self, request, context): common_name = common.authenticate(context) e = ed25519.scalar_unpack(common.sha256(common_name)) response = pep3_pb2.ComponentsRegistrationResponse() for shard, by_shard in request.components.items(): if not self.pep._cryptopu.certified_component_is_valid_for( by_shard.pseudonym, self.pep.global_config.components[shard].pseudonym.\ base_times_two_to_the_power_of, e): context.abort( grpc.StatusCode.INVALID_ARGUMENT, f'pseudonym component for shard {shard} ' 'could not be verified') reminder = response.reminders.add() reminder.component = by_shard.pseudonym.component reminder.shard = shard reminder.pseudonym.SetInParent() # this sets reminder.pseudonym common.sign_protobuf(reminder, self.pep.secrets.reminders_hmac_secret) for domain, cc_msg in by_shard.keys.items(): if not self.pep._cryptopu.certified_component_is_valid_for( cc_msg, self.pep.global_config.components[shard].\ keys[domain].base_times_two_to_the_power_of, e): context.abort( grpc.StatusCode.INVALID_ARGUMENT, f'{domain} key component for shard {shard} ' 'could not be verified') reminder = response.reminders.add() reminder.component = cc_msg.component reminder.shard = shard reminder.key.domain = domain common.sign_protobuf(reminder, self.pep.secrets.reminders_hmac_secret) return response
def client(host, port, secret, command, **kwargs): try: client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) client_socket.connect((host, port)) except Exception: logger.critical('Could not connect to server!') logger.critical(traceback.format_exc()) return request = protocol_pb2.Request() request.secret = common.sha256(secret) if command == 'key': key_request(request, **kwargs) msg = request.SerializeToString() length = struct.pack('>Q', len(msg)) logger.debug("Request length: %d" % (len(msg))) client_socket.sendall(length + msg) logger.info("Request sent") handle_response(client_socket, **kwargs)
def is_valid_proof_for(self, A, M, N, A_packed=None, M_packed=None, N_packed=None): R_M,R_B,s = self._R_M, self._R_B, self._s R_M_packed = self._R_M_packed R_B_packed = self._R_B_packed if A_packed==None: A_packed = A.pack() if M_packed==None: M_packed = M.pack() if N_packed==None: N_packed = N.pack() h = ed25519.scalar_unpack(common.sha256( A_packed + M_packed + N_packed + R_M_packed + R_B_packed)) if ed25519.Point.B_times(s) != R_B + A * h: return False if M*s != R_M + N * h: return False return True
def Relocalize(self, request, context): common_name = common.authenticate(context) self._dispatch_message( pep3_pb2.Message(text="Relocalizing", code=pep3_pb2.Message.OK)) # catch trivial errors if len(request.warrant.signature) == 0: context.abort(grpc.StatusCode.PERMISSION_DENIED, "warrant has empty signature") if len(request.which_shards) == 0: context.abort(grpc.StatusCode.INVALID_ARGUMENT, "request.which_shards is empty") if len(request.names) == 0: context.abort(grpc.StatusCode.INVALID_ARGUMENT, "request.names is empty") # verify warrant if common_name != request.warrant.act.actor: context.abort( grpc.StatusCode.PERMISSION_DENIED, f"you, {common_name}, presented a warrant that " f"was issued to {request.warrant.act.actor}") try: crypto.verify( crypto.load_certificate( crypto.FILETYPE_PEM, self.pep.global_config.root_certificates.warrants), request.warrant.signature, request.warrant.act.SerializeToString(), 'sha256') except crypto.Error as e: context.abort(grpc.StatusCode.PERMISSION_DENIED, "the warrant's signature appears to be invalid") response = pep3_pb2.RelocalizationResponse() act = request.warrant.act k = s = 1 for shard in request.which_shards: s *= pow(ed25519.scalar_unpack(self.pep.secrets.by_shard[shard]\ .pseudonym_component_secret), ed25519.scalar_unpack(common.sha256(act.target)), ed25519.l) s %= ed25519.l if act.encrypt_for != b"": for shard in request.which_shards: k *= pow(ed25519.scalar_unpack(self.pep.secrets.by_shard[shard]\ .by_domain["pseudonym"].key_component_secret), ed25519.scalar_unpack(common.sha256(act.encrypt_for)), ed25519.l) k %= ed25519.l k_inv_comp = 1 encrypt_from = act.source if encrypt_from == b"plaintext": encrypt_from = act.actor for shard in request.which_shards: k_inv_comp *= \ pow(ed25519.scalar_unpack(self.pep.secrets.by_shard[shard]\ .by_domain["pseudonym"].key_component_secret), ed25519.scalar_unpack(common.sha256(encrypt_from)), ed25519.l) k_inv_comp %= ed25519.l k = (k * ed25519.scalar_inv(k_inv_comp)) % ed25519.l if act.source != b"plaintext": for shard in request.which_shards: s *= ed25519.scalar_inv(pow(ed25519.scalar_unpack( self.pep.secrets.by_shard[shard]\ .pseudonym_component_secret), ed25519.scalar_unpack(common.sha256(act.source)), ed25519.l)) s %= ed25519.l names = request.names # reshuffle, rekey, and rerandomize try: self.pep._cryptopu.rsk( names, k, s, [ed25519.scalar_random() for i in range(len(names))]) except cryptopu.InvalidArgument as e: context.abort(grpc.StatusCode.INVALID_ARGUMENT, str(e)) # change the state of the names if we rekeyed if act.encrypt_for != b"": for i in range(len(names)): names[i].state \ = pep3_pb2.Pseudonymizable.ENCRYPTED_PSEUDONYM response.names.extend(names) return response
def Depseudonymize(self, request, context): common_name = common.authenticate(context) e = ed25519.scalar_unpack(common.sha256(common_name)) self._dispatch_message( pep3_pb2.Message(text="Depseudonymizing", code=pep3_pb2.Message.OK)) # catch trivial errors if len(request.warrant.signature) == 0: context.abort(grpc.StatusCode.PERMISSION_DENIED, "warrant has empty signature") if len(request.which_shards) == 0: context.abort(grpc.StatusCode.INVALID_ARGUMENT, "request.which_shards is empty") # verify warrant if common_name != request.warrant.act.actor: context.abort( grpc.StatusCode.PERMISSION_DENIED, f"you, {common_name}, presented a warrant that " f"was issued to {request.warrant.act.actor}") try: crypto.verify( crypto.load_certificate( crypto.FILETYPE_PEM, self.pep.global_config.root_certificates.warrants), request.warrant.signature, request.warrant.act.SerializeToString(), 'sha256') except crypto.Error as e: context.abort(grpc.StatusCode.PERMISSION_DENIED, "the warrant's signature appears to be invalid") # verify reminders for i, reminder in enumerate(request.reminders): if not common.verify_protobuf_signature( reminder, self.pep.secrets.reminders_hmac_secret): context.abort(grpc.StatusCode.PERMISSION_DENIED, f"could not verify reminder #{i}.") # verify chain name = request.warrant.act.name name_unpacked = elgamal.Triple.unpack(name.data) for i, link in enumerate(request.chain): if link.peer not in self.pep.global_config.peers: context.abort(grpc.StatusCode.INVALID_ARGUMENT, f"unknown peer '{link.peer}' in link #{i}") for shard in link.which_shards: if shard not in self.pep.global_config\ .peers[link.peer].shards: context.abort( grpc.StatusCode.PERMISSION_DENIED, f"the peer {link.peer} of link #{i} " f"doesn't hold the shard {shard}!") rs_p = schnorr.RSProof.unpack(link.peer_response.rs_proof) sB_p = schnorr.DHTProof.unpack(link.peer_response.sB_proof) sB = ed25519.Point.unpack(link.peer_response.sB) kB = ed25519.Point.B_times(1) s_inv_B = ed25519.Point.unpack(link.peer_response.s_inv_B) link_name_unpacked = elgamal.Triple.unpack( link.peer_response.name.data) # check that s_inv_B is the inverse of sB if not sB_p.is_valid_proof_for(sB, s_inv_B, ed25519.Point.B_times(1)): context.abort(grpc.StatusCode.PERMISSION_DENIED, f"could not verify the sB proof of link #{i}.") # check the rs-operation was performed correctly if not rs_p.is_valid_proof_for(name_unpacked, sB, link_name_unpacked): context.abort(grpc.StatusCode.PERMISSION_DENIED, f"could not verify the rs proof of link #{i}.") # check that s_inv_B is indeed the product of their factors s_inv_B_factors = [ ed25519.Point.unpack(pp) for pp in link.peer_response.s_inv_B_factors ] s_inv_B_p = schnorr.ProductProof.from_protobuf( link.peer_response.s_inv_B_proof) if not s_inv_B_p.is_valid_proof_for(s_inv_B, s_inv_B_factors): context.abort( grpc.StatusCode.PERMISSION_DENIED, "could not verify the s_inv_B product proof " f"for link #{i}.") # make a lookup dictionary for the reminders reminders = {} for reminder in request.reminders: component = reminder.component if component in reminders: context.abort(grpc.StatusCode.INVALID_ARGUMENT, "double reminder") reminders[component] = reminder # check that the provided factors are valid for j, shard in enumerate(link.which_shards): # check s_inv_B factor s_inv_B_factor_packed = link.peer_response.s_inv_B_factors[j] if shard in self.pep.config.shards: # we can check s_inv_B by computing it ourselves s_ = ed25519.scalar_unpack(self.pep.secrets\ .by_shard[shard].pseudonym_component_secret) s_B = ed25519.Point.B_times(pow(s_, e, ed25519.l)) if s_B.pack() != s_inv_B_factor_packed: context.abort( grpc.StatusCode.PERMISSION_DENIED, f"s_inv_B factor #{j} (for shard {shard}, " f"and common name {common_name}, e={e})" f" of link #{i} is not correct: it should be " f"{s_B.pack()}, but {s_inv_B_factor_packed} " "was given.") else: # we need a reminder that s_inv_B is correct if s_inv_B_factor_packed not in reminders: context.abort( grpc.StatusCode.PERMISSION_DENIED, f"missing reminder that s_inv_B factor #{j} " f"of link #{i} is correct.") rem = reminders[s_inv_B_factor_packed] assert (rem.component == s_inv_B_factor_packed) error_message = None if not rem.HasField("pseudonym"): error_message = "reminder is for a key component"\ f" instead of a pseudonym component" elif rem.shard != shard: error_message = "reminder is for "\ f"the shard {rem.shard}"\ f" instead of the shard {shard}" if error_message != None: context.abort( grpc.StatusCode.PERMISSION_DENIED, f"s_inv_B factor #{j} of link #{i} " "is not correct: " + error_message) name = link.peer_response.name name_unpacked = link_name_unpacked # the provided request seems to be in order; # let us prepare our response. response = pep3_pb2.DepseudonymizationResponse() # compute rekey and reshuffle components k = s_inv = 1 s_inv_factors = [] for shard in request.which_shards: s_inv_factor = pow(ed25519.scalar_unpack(self.pep.secrets\ .by_shard[shard].pseudonym_component_secret), e, ed25519.l) s_inv_factors.append(s_inv_factor) s_inv *= s_inv_factor s_inv %= ed25519.l s = ed25519.scalar_inv(s_inv) r = ed25519.scalar_random() rs_proof, name_out = schnorr.RSProof.create(name_unpacked, s, r) response.rs_proof = rs_proof.pack() response.name.data = name_out.pack() response.name.state \ = pep3_pb2.Pseudonymizable.ENCRYPTED_PSEUDONYM # compute proofs for the reshuffle components s_inv_B_proof, s_inv_B_factors, s_inv_B \ = schnorr.ProductProof.create(s_inv_factors) s_inv_B_proof.to_protobuf(response.s_inv_B_proof) for s_inv_B_factor in s_inv_B_factors: response.s_inv_B_factors.append(s_inv_B_factor.pack()) response.s_inv_B = s_inv_B.pack() sB = ed25519.Point.B_times(s) sB_proof = schnorr.DHTProof.create(s, s_inv_B, A=sB, N=ed25519.Point.B_times(1)) response.sB = sB.pack() response.sB_proof = sB_proof.pack() return response
def test_store_and_retrieve(self): # first store a record with random source and target ip addresses, # and see if we can recover it. col_request = pep3_pb2.StoreRequest() col_request.id = os.urandom(16) flowrecord = col_request.records.add() flowrecord.source_ip.data = os.urandom(16) flowrecord.source_ip.state = pep3_pb2.Pseudonymizable.UNENCRYPTED_NAME flowrecord.destination_ip.data = os.urandom(16) flowrecord.destination_ip.state = \ pep3_pb2.Pseudonymizable.UNENCRYPTED_NAME flowrecord.anonymous_part.number_of_bytes = 123 flowrecord.anonymous_part.number_of_packets = 456 updates = list( self.collector.connect_to('collector').Store(iter([col_request]))) self.assertEqual(len(updates), 1) self.assertEqual(updates[0].stored_id, col_request.id) # store the same flowrecord twice, to see if that causes troubles col_request.id = os.urandom(16) updates = list( self.collector.connect_to('collector').Store(iter([col_request]))) self.assertEqual(len(updates), 1) self.assertEqual(updates[0].stored_id, col_request.id) query = pep3_pb2.SqlQuery() # manually compute storage_facility-local pseudonyms for query sf_name = b"PEP3 storage_facility" pseudonym_secrets = {} for peer_secrets in self.secrets.peers.values(): for shard, shard_secrets in peer_secrets.by_shard.items(): pseudonym_secrets[shard] \ = shard_secrets.pseudonym_component_secret s = 1 e = ed25519.scalar_unpack(common.sha256(sf_name)) for secret in pseudonym_secrets.values(): s *= pow(ed25519.scalar_unpack(secret), e, ed25519.l) s %= ed25519.l # see if the record was stored correctly by querying the # database directly. query.query = """SELECT peped_flows.p_dst_ip FROM peped_flows WHERE peped_flows.p_src_ip=:ip""" ip = query.parameters['ip'].pseudonymizable_value ip.data = (ed25519.Point.lizard(flowrecord.source_ip.data) * s).pack() ip.state = pep3_pb2.Pseudonymizable.UNENCRYPTED_PSEUDONYM row = self.sf.connect_to('database')\ .Query(query).next().rows[0] self.assertEqual( row.cells[0].pseudonymizable_value.data, (ed25519.Point.lizard(flowrecord.destination_ip.data) * s).pack()) # manually compute researcher-local pseudonyms for query researcher_name = b"PEP3 researcher" pseudonym_secrets = {} for peer_secrets in self.secrets.peers.values(): for shard, shard_secrets in peer_secrets.by_shard.items(): pseudonym_secrets[shard] \ = shard_secrets.pseudonym_component_secret s = 1 e = ed25519.scalar_unpack(common.sha256(researcher_name)) for secret in pseudonym_secrets.values(): s *= pow(ed25519.scalar_unpack(secret), e, ed25519.l) s %= ed25519.l # now query via the researcher query.parameters['ip'].pseudonymizable_value.data \ = ( ed25519.Point.lizard(flowrecord.source_ip.data)*s ).pack() row = self.researcher.connect_to('researcher')\ .Query(query).next().rows[0] self.assertEqual( row.cells[0].pseudonymizable_value.data, (ed25519.Point.lizard(flowrecord.destination_ip.data) * s).pack())
def hash(data): return ReferencePoint.elligator2(fe_unpack(common.sha256(data)))