def benchmark_depseudonymize(self, args): ip = os.urandom(16) # manually compute investigator-local pseudonym pseudonym_secrets = {} for peer_secrets in self.args.secrets.peers.values(): for shard, shard_secrets in peer_secrets.by_shard.items(): pseudonym_secrets[shard] \ = shard_secrets.pseudonym_component_secret s = 1 e = ed25519.scalar_unpack(common.sha256(b"PEP3 investigator")) for secret in pseudonym_secrets.values(): s *= pow(ed25519.scalar_unpack(secret),e,ed25519.l) s %= ed25519.l investigator_local_ip = ( ed25519.Point.lizard(ip)*s ).pack() # manually create warrant warrant = pep3_pb2.DepseudonymizationRequest.Warrant() warrant.act.actor = b"PEP3 investigator" warrant.act.name.state = pep3_pb2.Pseudonymizable.UNENCRYPTED_PSEUDONYM warrant.act.name.data = investigator_local_ip self.investigator.encrypt([ warrant.act.name ], cheats.public_key(self.args.secrets, b"PEP3 investigator", 'pseudonym')) warrant.signature = crypto.sign( crypto.load_privatekey(crypto.FILETYPE_PEM, self.args.secrets.root_certificate_keys.warrants), warrant.act.SerializeToString(), 'sha256') result = self.investigator.connect_to("investigator")\ .Depseudonymize(warrant)
def test_localization_of_pseudonym(self): name = b" a 16 byte name " target = b"PEP3 storage_facility" pp = pep3_pb2.Pseudonymizable( data=name, state=pep3_pb2.Pseudonymizable.UNENCRYPTED_NAME) self.collector.pseudonymize([pp]) self.collector.relocalize([pp], self.config.collector.warrants.to_sf) sfp = elgamal.Triple.unpack(pp.data)\ .decrypt(self.sf.private_keys['pseudonym']) pseudonym_secrets = {} for peer_secrets in self.secrets.peers.values(): for shard, shard_secrets in peer_secrets.by_shard.items(): pseudonym_secrets[shard] \ = shard_secrets.pseudonym_component_secret s = 1 e = ed25519.scalar_unpack(common.sha256(target)) for secret in pseudonym_secrets.values(): s *= pow(ed25519.scalar_unpack(secret), e, ed25519.l) s %= ed25519.l self.assertEqual(sfp * ed25519.scalar_inv(s), ed25519.Point.lizard(name))
def create(a, M, A=None, N=None, m=None, a_packed=None, M_packed=None, A_packed=None, N_packed=None): if A==None: A = ed25519.Point.B_times(a) if N==None: N = M * a if A_packed==None: A_packed = A.pack() if N_packed==None: N_packed = N.pack() if M_packed==None: M_packed = M.pack() if a_packed==None: a_packed = ed25519.scalar_pack(a) r = ed25519.scalar_unpack(common.sha256( b"DHTProof" + a_packed + M_packed)) R_B = ed25519.Point.B_times(r) if m==None: R_M = M * r else: R_M = ed25519.Point.B_times(m*r) R_M_packed = R_M.pack() R_B_packed = R_B.pack() h = ed25519.scalar_unpack(common.sha256( A_packed + M_packed + N_packed + R_M_packed + R_B_packed)) s = (r + h * a) % ed25519.l return DHTProof(R_M, R_B, s, R_M_packed=R_M_packed, R_B_packed=R_B_packed)
def Enroll(self, request, context): common_name = common.authenticate(context) # TODO: have a smarter check return_components = (common_name == b"PEP3 investigator" or common_name == b"PEP3 researcher") response = pep3_pb2.EnrollmentResponse() # 1. [ REMOVED ] # 2. set response.by_shard[...].private_local_keys # and response.components[...].keys e = ed25519.scalar_unpack(common.sha256(common_name)) # "e" is the exponent used to compute the key/pseudonym components for shard, shard_secrets in self.pep.secrets.by_shard.items(): for domain, domain_secrets in shard_secrets.by_domain.items(): x = ed25519.scalar_unpack(domain_secrets.private_master_key) k = ed25519.scalar_unpack(domain_secrets.key_component_secret) k_local = pow(k, e, ed25519.l) if return_components: self.pep._cryptopu.certified_component_create( response.components[shard].keys[domain], self.pep.global_config.\ components[shard].keys[domain].\ base_times_two_to_the_power_of, k, e) x_local = (k_local * x) % ed25519.l response.by_shard[shard].private_local_keys[domain]\ = ed25519.scalar_pack(x_local) # 3. set response.components[...].pseudonym if return_components: for shard in self.pep.config.shards: s = ed25519.scalar_unpack( self.pep.secrets.by_shard[shard]\ .pseudonym_component_secret) self.pep._cryptopu.certified_component_create( response.components[shard].pseudonym, self.pep.global_config.\ components[shard].pseudonym.\ base_times_two_to_the_power_of, s, e) return response
def test_scalar_unpack(self): for i in range(10): data = os.urandom(32) cscalar = ristretto.ffi.new("group_scalar*") ristretto.lib.group_scalar_unpack(cscalar, data) cbuf = ristretto.ffi.new("unsigned char[]", 32) ristretto.lib.group_scalar_pack(cbuf, cscalar) buf = ristretto.ffi.buffer(cbuf) self.assertEqual(ed25519.scalar_unpack(data), ed25519.scalar_unpack(buf[:]))
def private_key(secrets, common_name, domain): e = ed25519.scalar_unpack(common.sha256(common_name)) x = 1 for shard, shard_secrets in secrets_by_shard(secrets).items(): x *= ed25519.scalar_unpack( shard_secrets.by_domain[domain]\ .private_master_key ) x %= ed25519.l x *= pow(ed25519.scalar_unpack( shard_secrets.by_domain[domain].key_component_secret), e, ed25519.l) x %= ed25519.l return x
def unpack(data): assert(len(data)==3*32) R_M_packed = data[0:32] R_B_packed = data[32:64] s_packed = data[64:96] return DHTProof( ed25519.Point.unpack(R_M_packed), ed25519.Point.unpack(R_B_packed), ed25519.scalar_unpack(data[64:96]), R_M_packed=R_M_packed, R_B_packed=R_B_packed)
def RegisterComponents(self, request, context): common_name = common.authenticate(context) e = ed25519.scalar_unpack(common.sha256(common_name)) response = pep3_pb2.ComponentsRegistrationResponse() for shard, by_shard in request.components.items(): if not self.pep._cryptopu.certified_component_is_valid_for( by_shard.pseudonym, self.pep.global_config.components[shard].pseudonym.\ base_times_two_to_the_power_of, e): context.abort( grpc.StatusCode.INVALID_ARGUMENT, f'pseudonym component for shard {shard} ' 'could not be verified') reminder = response.reminders.add() reminder.component = by_shard.pseudonym.component reminder.shard = shard reminder.pseudonym.SetInParent() # this sets reminder.pseudonym common.sign_protobuf(reminder, self.pep.secrets.reminders_hmac_secret) for domain, cc_msg in by_shard.keys.items(): if not self.pep._cryptopu.certified_component_is_valid_for( cc_msg, self.pep.global_config.components[shard].\ keys[domain].base_times_two_to_the_power_of, e): context.abort( grpc.StatusCode.INVALID_ARGUMENT, f'{domain} key component for shard {shard} ' 'could not be verified') reminder = response.reminders.add() reminder.component = cc_msg.component reminder.shard = shard reminder.key.domain = domain common.sign_protobuf(reminder, self.pep.secrets.reminders_hmac_secret) return response
def is_valid_proof_for(self, A, M, N, A_packed=None, M_packed=None, N_packed=None): R_M,R_B,s = self._R_M, self._R_B, self._s R_M_packed = self._R_M_packed R_B_packed = self._R_B_packed if A_packed==None: A_packed = A.pack() if M_packed==None: M_packed = M.pack() if N_packed==None: N_packed = N.pack() h = ed25519.scalar_unpack(common.sha256( A_packed + M_packed + N_packed + R_M_packed + R_B_packed)) if ed25519.Point.B_times(s) != R_B + A * h: return False if M*s != R_M + N * h: return False return True
def test_store_and_retrieve(self): # first store a record with random source and target ip addresses, # and see if we can recover it. col_request = pep3_pb2.StoreRequest() col_request.id = os.urandom(16) flowrecord = col_request.records.add() flowrecord.source_ip.data = os.urandom(16) flowrecord.source_ip.state = pep3_pb2.Pseudonymizable.UNENCRYPTED_NAME flowrecord.destination_ip.data = os.urandom(16) flowrecord.destination_ip.state = \ pep3_pb2.Pseudonymizable.UNENCRYPTED_NAME flowrecord.anonymous_part.number_of_bytes = 123 flowrecord.anonymous_part.number_of_packets = 456 updates = list( self.collector.connect_to('collector').Store(iter([col_request]))) self.assertEqual(len(updates), 1) self.assertEqual(updates[0].stored_id, col_request.id) # store the same flowrecord twice, to see if that causes troubles col_request.id = os.urandom(16) updates = list( self.collector.connect_to('collector').Store(iter([col_request]))) self.assertEqual(len(updates), 1) self.assertEqual(updates[0].stored_id, col_request.id) query = pep3_pb2.SqlQuery() # manually compute storage_facility-local pseudonyms for query sf_name = b"PEP3 storage_facility" pseudonym_secrets = {} for peer_secrets in self.secrets.peers.values(): for shard, shard_secrets in peer_secrets.by_shard.items(): pseudonym_secrets[shard] \ = shard_secrets.pseudonym_component_secret s = 1 e = ed25519.scalar_unpack(common.sha256(sf_name)) for secret in pseudonym_secrets.values(): s *= pow(ed25519.scalar_unpack(secret), e, ed25519.l) s %= ed25519.l # see if the record was stored correctly by querying the # database directly. query.query = """SELECT peped_flows.p_dst_ip FROM peped_flows WHERE peped_flows.p_src_ip=:ip""" ip = query.parameters['ip'].pseudonymizable_value ip.data = (ed25519.Point.lizard(flowrecord.source_ip.data) * s).pack() ip.state = pep3_pb2.Pseudonymizable.UNENCRYPTED_PSEUDONYM row = self.sf.connect_to('database')\ .Query(query).next().rows[0] self.assertEqual( row.cells[0].pseudonymizable_value.data, (ed25519.Point.lizard(flowrecord.destination_ip.data) * s).pack()) # manually compute researcher-local pseudonyms for query researcher_name = b"PEP3 researcher" pseudonym_secrets = {} for peer_secrets in self.secrets.peers.values(): for shard, shard_secrets in peer_secrets.by_shard.items(): pseudonym_secrets[shard] \ = shard_secrets.pseudonym_component_secret s = 1 e = ed25519.scalar_unpack(common.sha256(researcher_name)) for secret in pseudonym_secrets.values(): s *= pow(ed25519.scalar_unpack(secret), e, ed25519.l) s %= ed25519.l # now query via the researcher query.parameters['ip'].pseudonymizable_value.data \ = ( ed25519.Point.lizard(flowrecord.source_ip.data)*s ).pack() row = self.researcher.connect_to('researcher')\ .Query(query).next().rows[0] self.assertEqual( row.cells[0].pseudonymizable_value.data, (ed25519.Point.lizard(flowrecord.destination_ip.data) * s).pack())
def Relocalize(self, request, context): common_name = common.authenticate(context) self._dispatch_message( pep3_pb2.Message(text="Relocalizing", code=pep3_pb2.Message.OK)) # catch trivial errors if len(request.warrant.signature) == 0: context.abort(grpc.StatusCode.PERMISSION_DENIED, "warrant has empty signature") if len(request.which_shards) == 0: context.abort(grpc.StatusCode.INVALID_ARGUMENT, "request.which_shards is empty") if len(request.names) == 0: context.abort(grpc.StatusCode.INVALID_ARGUMENT, "request.names is empty") # verify warrant if common_name != request.warrant.act.actor: context.abort( grpc.StatusCode.PERMISSION_DENIED, f"you, {common_name}, presented a warrant that " f"was issued to {request.warrant.act.actor}") try: crypto.verify( crypto.load_certificate( crypto.FILETYPE_PEM, self.pep.global_config.root_certificates.warrants), request.warrant.signature, request.warrant.act.SerializeToString(), 'sha256') except crypto.Error as e: context.abort(grpc.StatusCode.PERMISSION_DENIED, "the warrant's signature appears to be invalid") response = pep3_pb2.RelocalizationResponse() act = request.warrant.act k = s = 1 for shard in request.which_shards: s *= pow(ed25519.scalar_unpack(self.pep.secrets.by_shard[shard]\ .pseudonym_component_secret), ed25519.scalar_unpack(common.sha256(act.target)), ed25519.l) s %= ed25519.l if act.encrypt_for != b"": for shard in request.which_shards: k *= pow(ed25519.scalar_unpack(self.pep.secrets.by_shard[shard]\ .by_domain["pseudonym"].key_component_secret), ed25519.scalar_unpack(common.sha256(act.encrypt_for)), ed25519.l) k %= ed25519.l k_inv_comp = 1 encrypt_from = act.source if encrypt_from == b"plaintext": encrypt_from = act.actor for shard in request.which_shards: k_inv_comp *= \ pow(ed25519.scalar_unpack(self.pep.secrets.by_shard[shard]\ .by_domain["pseudonym"].key_component_secret), ed25519.scalar_unpack(common.sha256(encrypt_from)), ed25519.l) k_inv_comp %= ed25519.l k = (k * ed25519.scalar_inv(k_inv_comp)) % ed25519.l if act.source != b"plaintext": for shard in request.which_shards: s *= ed25519.scalar_inv(pow(ed25519.scalar_unpack( self.pep.secrets.by_shard[shard]\ .pseudonym_component_secret), ed25519.scalar_unpack(common.sha256(act.source)), ed25519.l)) s %= ed25519.l names = request.names # reshuffle, rekey, and rerandomize try: self.pep._cryptopu.rsk( names, k, s, [ed25519.scalar_random() for i in range(len(names))]) except cryptopu.InvalidArgument as e: context.abort(grpc.StatusCode.INVALID_ARGUMENT, str(e)) # change the state of the names if we rekeyed if act.encrypt_for != b"": for i in range(len(names)): names[i].state \ = pep3_pb2.Pseudonymizable.ENCRYPTED_PSEUDONYM response.names.extend(names) return response
def Depseudonymize(self, request, context): common_name = common.authenticate(context) e = ed25519.scalar_unpack(common.sha256(common_name)) self._dispatch_message( pep3_pb2.Message(text="Depseudonymizing", code=pep3_pb2.Message.OK)) # catch trivial errors if len(request.warrant.signature) == 0: context.abort(grpc.StatusCode.PERMISSION_DENIED, "warrant has empty signature") if len(request.which_shards) == 0: context.abort(grpc.StatusCode.INVALID_ARGUMENT, "request.which_shards is empty") # verify warrant if common_name != request.warrant.act.actor: context.abort( grpc.StatusCode.PERMISSION_DENIED, f"you, {common_name}, presented a warrant that " f"was issued to {request.warrant.act.actor}") try: crypto.verify( crypto.load_certificate( crypto.FILETYPE_PEM, self.pep.global_config.root_certificates.warrants), request.warrant.signature, request.warrant.act.SerializeToString(), 'sha256') except crypto.Error as e: context.abort(grpc.StatusCode.PERMISSION_DENIED, "the warrant's signature appears to be invalid") # verify reminders for i, reminder in enumerate(request.reminders): if not common.verify_protobuf_signature( reminder, self.pep.secrets.reminders_hmac_secret): context.abort(grpc.StatusCode.PERMISSION_DENIED, f"could not verify reminder #{i}.") # verify chain name = request.warrant.act.name name_unpacked = elgamal.Triple.unpack(name.data) for i, link in enumerate(request.chain): if link.peer not in self.pep.global_config.peers: context.abort(grpc.StatusCode.INVALID_ARGUMENT, f"unknown peer '{link.peer}' in link #{i}") for shard in link.which_shards: if shard not in self.pep.global_config\ .peers[link.peer].shards: context.abort( grpc.StatusCode.PERMISSION_DENIED, f"the peer {link.peer} of link #{i} " f"doesn't hold the shard {shard}!") rs_p = schnorr.RSProof.unpack(link.peer_response.rs_proof) sB_p = schnorr.DHTProof.unpack(link.peer_response.sB_proof) sB = ed25519.Point.unpack(link.peer_response.sB) kB = ed25519.Point.B_times(1) s_inv_B = ed25519.Point.unpack(link.peer_response.s_inv_B) link_name_unpacked = elgamal.Triple.unpack( link.peer_response.name.data) # check that s_inv_B is the inverse of sB if not sB_p.is_valid_proof_for(sB, s_inv_B, ed25519.Point.B_times(1)): context.abort(grpc.StatusCode.PERMISSION_DENIED, f"could not verify the sB proof of link #{i}.") # check the rs-operation was performed correctly if not rs_p.is_valid_proof_for(name_unpacked, sB, link_name_unpacked): context.abort(grpc.StatusCode.PERMISSION_DENIED, f"could not verify the rs proof of link #{i}.") # check that s_inv_B is indeed the product of their factors s_inv_B_factors = [ ed25519.Point.unpack(pp) for pp in link.peer_response.s_inv_B_factors ] s_inv_B_p = schnorr.ProductProof.from_protobuf( link.peer_response.s_inv_B_proof) if not s_inv_B_p.is_valid_proof_for(s_inv_B, s_inv_B_factors): context.abort( grpc.StatusCode.PERMISSION_DENIED, "could not verify the s_inv_B product proof " f"for link #{i}.") # make a lookup dictionary for the reminders reminders = {} for reminder in request.reminders: component = reminder.component if component in reminders: context.abort(grpc.StatusCode.INVALID_ARGUMENT, "double reminder") reminders[component] = reminder # check that the provided factors are valid for j, shard in enumerate(link.which_shards): # check s_inv_B factor s_inv_B_factor_packed = link.peer_response.s_inv_B_factors[j] if shard in self.pep.config.shards: # we can check s_inv_B by computing it ourselves s_ = ed25519.scalar_unpack(self.pep.secrets\ .by_shard[shard].pseudonym_component_secret) s_B = ed25519.Point.B_times(pow(s_, e, ed25519.l)) if s_B.pack() != s_inv_B_factor_packed: context.abort( grpc.StatusCode.PERMISSION_DENIED, f"s_inv_B factor #{j} (for shard {shard}, " f"and common name {common_name}, e={e})" f" of link #{i} is not correct: it should be " f"{s_B.pack()}, but {s_inv_B_factor_packed} " "was given.") else: # we need a reminder that s_inv_B is correct if s_inv_B_factor_packed not in reminders: context.abort( grpc.StatusCode.PERMISSION_DENIED, f"missing reminder that s_inv_B factor #{j} " f"of link #{i} is correct.") rem = reminders[s_inv_B_factor_packed] assert (rem.component == s_inv_B_factor_packed) error_message = None if not rem.HasField("pseudonym"): error_message = "reminder is for a key component"\ f" instead of a pseudonym component" elif rem.shard != shard: error_message = "reminder is for "\ f"the shard {rem.shard}"\ f" instead of the shard {shard}" if error_message != None: context.abort( grpc.StatusCode.PERMISSION_DENIED, f"s_inv_B factor #{j} of link #{i} " "is not correct: " + error_message) name = link.peer_response.name name_unpacked = link_name_unpacked # the provided request seems to be in order; # let us prepare our response. response = pep3_pb2.DepseudonymizationResponse() # compute rekey and reshuffle components k = s_inv = 1 s_inv_factors = [] for shard in request.which_shards: s_inv_factor = pow(ed25519.scalar_unpack(self.pep.secrets\ .by_shard[shard].pseudonym_component_secret), e, ed25519.l) s_inv_factors.append(s_inv_factor) s_inv *= s_inv_factor s_inv %= ed25519.l s = ed25519.scalar_inv(s_inv) r = ed25519.scalar_random() rs_proof, name_out = schnorr.RSProof.create(name_unpacked, s, r) response.rs_proof = rs_proof.pack() response.name.data = name_out.pack() response.name.state \ = pep3_pb2.Pseudonymizable.ENCRYPTED_PSEUDONYM # compute proofs for the reshuffle components s_inv_B_proof, s_inv_B_factors, s_inv_B \ = schnorr.ProductProof.create(s_inv_factors) s_inv_B_proof.to_protobuf(response.s_inv_B_proof) for s_inv_B_factor in s_inv_B_factors: response.s_inv_B_factors.append(s_inv_B_factor.pack()) response.s_inv_B = s_inv_B.pack() sB = ed25519.Point.B_times(s) sB_proof = schnorr.DHTProof.create(s, s_inv_B, A=sB, N=ed25519.Point.B_times(1)) response.sB = sB.pack() response.sB_proof = sB_proof.pack() return response
def _enroll(self): void = pep3_pb2.Void() key_parts = {} components = {} # by shard for domain in self.global_config.domains: key_parts[domain] = {} futs = [] for peer_name in self.global_config.peers: futs.append(self.connect_to("peer", peer_name).Enroll.future(void)) for fut in futs: resp = fut.result() for shard, by_shard in resp.by_shard.items(): for domain, private_key_part in \ by_shard.private_local_keys.items(): if shard not in key_parts[domain]: key_parts[domain][shard] = private_key_part else: # inconsistency? assert(key_parts[domain][shard] == private_key_part) for shard, by_shard in resp.components.items(): assert(set(by_shard.keys.keys()) \ == set(self.global_config.domains)) if shard not in components: components[shard] = by_shard else: other_by_shard = components[shard] # check for consistency assert(other_by_shard.pseudonym.component == by_shard.pseudonym.component) for domain in self.global_config.domains: assert(other_by_shard.keys[domain].component == by_shard.keys[domain].component) # check if we have received everything if len(components)>0: for shard in self.global_config.shards: assert(shard in components) # compute private local keys private_local_keys = {} for domain in self.global_config.domains: private_local_keys[domain] = 1 for shard in self.global_config.shards: private_local_keys[domain] *= ed25519.scalar_unpack( key_parts[domain][shard]) private_local_keys[domain] %= ed25519.l # compute public keys public_keys = {} for domain in self.global_config.domains: public_keys[domain] = ed25519.Point.B_times( private_local_keys[domain]) # register components if we got them reminders = None if len(components)>0: reminders = self._register_components(components) return (public_keys, private_local_keys, components, reminders)
def fill_local_config_messages(config, secrets): config.domains.append("pseudonym") pu = cryptopu.CryptoPU() # configure peers peer_names = ('A', 'B', 'C', 'D', 'E') for letter in peer_names: config.peers.get_or_create(letter) secrets.peers.get_or_create(letter) secrets.peers[letter].reminders_hmac_secret = os.urandom(32) private_keys = {} for domain in config.domains: private_keys[domain] = 1 # will be set below peer_triples = tuple(itertools.combinations(peer_names,3)) for name1, name2, name3 in peer_triples: shard = name1 + name2 + name3 # ABC, ADE, ... config.shards.append(shard) config.peers[name1].shards.append(shard) config.peers[name2].shards.append(shard) config.peers[name3].shards.append(shard) secrets.peers[name1].by_shard[shard].pseudonym_component_secret\ = secrets.peers[name2].by_shard[shard]\ .pseudonym_component_secret\ = secrets.peers[name3].by_shard[shard]\ .pseudonym_component_secret\ = s_packed = os.urandom(32) s = ed25519.scalar_unpack(s_packed) config.components[shard].pseudonym.base_times_two_to_the_power_of\ .extend(pu.component_public_part(s)) for domain in config.domains: private_key_part = ed25519.scalar_random() secrets.peers[name1].by_shard[shard].by_domain[domain]\ .private_master_key\ = secrets.peers[name2].by_shard[shard].by_domain[domain]\ .private_master_key\ = secrets.peers[name3].by_shard[shard].by_domain[domain]\ .private_master_key\ = ed25519.scalar_pack(private_key_part) private_keys[domain] *= private_key_part private_keys[domain] %= ed25519.l secrets.peers[name1].by_shard[shard].by_domain[domain]\ .key_component_secret\ = secrets.peers[name2].by_shard[shard].by_domain[domain]\ .key_component_secret\ = secrets.peers[name3].by_shard[shard].by_domain[domain]\ .key_component_secret\ = k_packed = os.urandom(32) k = ed25519.scalar_unpack(k_packed) config.components[shard].keys[domain]\ .base_times_two_to_the_power_of\ .extend(pu.component_public_part(k)) # generate certificates and port numbers for the servers root_key = crypto.PKey() root_key.generate_key(crypto.TYPE_RSA, 1024) secrets.root_certificate_keys.tls = crypto.dump_privatekey( crypto.FILETYPE_PEM, root_key) root_crt = crypto.X509() root_crt.get_subject().CN = "PEP3 TLS Root" root_crt.set_serial_number(1) root_crt.gmtime_adj_notBefore(0) root_crt.gmtime_adj_notAfter(356*24*60*60) root_crt.set_issuer(root_crt.get_subject()) root_crt.set_pubkey(root_key) root_crt.sign(root_key, 'sha256') config.root_certificates.tls = crypto.dump_certificate( crypto.FILETYPE_PEM, root_crt) port = 1234 number_of_cpus = multiprocessing.cpu_count() for server_type_name, server_type in SERVER_TYPES.items(): if server_type.is_singleton: server_configs = { None: getattr(config,server_type_name) } else: server_configs = getattr(config, server_type_name+"s") for name, server_config in server_configs.items(): server_config.number_of_threads = number_of_cpus # set address and port server_config.location.address = \ server_config.location.listen_address = f"localhost:{port}" port += 1 # set tls certificate server_key = crypto.PKey() server_key.generate_key(crypto.TYPE_RSA, 1024) server_crt = crypto.X509() server_crt.get_subject().CN = "PEP3 " + server_type_name server_crt.set_serial_number(1) server_crt.gmtime_adj_notBefore(0) server_crt.gmtime_adj_notAfter(356*24*60*60) server_crt.set_issuer(root_crt.get_subject()) ext = crypto.X509Extension(b"subjectAltName",False, b"DNS:localhost") server_crt.add_extensions([ext]) server_crt.set_pubkey(server_key) server_crt.sign(root_key, 'sha256') server_config.location.tls_certificate = crypto.dump_certificate( crypto.FILETYPE_PEM, server_crt) if name==None: server_secrets = getattr(secrets,server_type_name) else: server_secrets = getattr(secrets,server_type_name+"s")[name] server_secrets.tls_certificate_key = crypto.dump_privatekey( crypto.FILETYPE_PEM, server_key) # The following uri makes sqlalchemy.create_engine use sqlite's :memory: # in-memory database. config.database.engine.uri = "sqlite://" config.database.engine.connect_args['check_same_thread'] = False config.database.engine.poolclass = 'StaticPool' config.database.engine.create_tables = True config.database.number_of_threads = 1 # generate keys for warrants warrant_key = crypto.PKey() warrant_key.generate_key(crypto.TYPE_RSA, 1024) secrets.root_certificate_keys.warrants = crypto.dump_privatekey( crypto.FILETYPE_PEM, warrant_key) warrant_crt = crypto.X509() warrant_crt.get_subject().CN = "PEP3 Warrant Root" warrant_crt.set_serial_number(1) warrant_crt.gmtime_adj_notBefore(0) warrant_crt.gmtime_adj_notAfter(356*24*60*60) warrant_crt.set_issuer(warrant_crt.get_subject()) warrant_crt.set_pubkey(warrant_key) warrant_crt.sign(warrant_key, 'sha256') warrant_crt_data = crypto.dump_certificate(crypto.FILETYPE_PEM, warrant_crt) config.root_certificates.warrants = warrant_crt_data # for Collector.Store warrant = config.collector.warrants.to_sf act = warrant.act act.target = b"PEP3 storage_facility" act.encrypt_for = b"PEP3 storage_facility" act.source = b"plaintext" act.actor = b"PEP3 collector" warrant.signature = crypto.sign( warrant_key, act.SerializeToString(), 'sha256') # for Researcher.Query warrant = config.researcher.warrants.from_me_to_sf act = warrant.act act.target = b"PEP3 storage_facility" act.encrypt_for = b"PEP3 storage_facility" act.source = b"PEP3 researcher" act.actor = b"PEP3 researcher" warrant.signature = crypto.sign( warrant_key, act.SerializeToString(), 'sha256') warrant = config.researcher.warrants.from_sf_to_me act = warrant.act act.target = b"PEP3 researcher" act.encrypt_for = b"PEP3 researcher" act.source = b"PEP3 storage_facility" act.actor = b"PEP3 researcher" warrant.signature = crypto.sign( warrant_key, act.SerializeToString(), 'sha256') # for Investigator.Query warrant = config.investigator.warrants.from_me_to_sf act = warrant.act act.target = b"PEP3 storage_facility" act.encrypt_for = b"PEP3 storage_facility" act.source = b"PEP3 investigator" act.actor = b"PEP3 investigator" warrant.signature = crypto.sign( warrant_key, act.SerializeToString(), 'sha256') warrant = config.investigator.warrants.from_sf_to_me act = warrant.act act.target = b"PEP3 investigator" act.encrypt_for = b"PEP3 investigator" act.source = b"PEP3 storage_facility" act.actor = b"PEP3 investigator" warrant.signature = crypto.sign( warrant_key, act.SerializeToString(), 'sha256') # describe tables used by the database columns = config.db_desc['peped_flows'].columns columns['p_src_ip'] = 'pseudonymized' columns['p_dst_ip'] = 'pseudonymized' for name in ('start_time', 'end_time', 'src_port', 'dst_port', 'protocol', 'packets', 'bytes'): columns[name] = 'plain' # config.batchsize = 1024