def test_k_21(self):
challenge: bytes = bytes([i for i in range(0, 32)])
plot_seed: bytes = bytes([5, 104, 52, 4, 51, 55, 23, 84, 91, 10, 111, 12, 13,
222, 151, 16, 228, 211, 254, 45, 92, 198, 204, 10, 9,
10, 11, 129, 139, 171, 15, 23])
pl = DiskPlotter()
pl.create_plot_disk(".", ".", ".", "myplot.dat", 21, bytes([1, 2, 3, 4, 5]), plot_seed, 2*1024)
pl = None
pr = DiskProver(str(Path("myplot.dat")))
total_proofs: int = 0
iterations: int = 5000
v = Verifier()
for i in range(iterations):
if i % 100 == 0:
print(i)
challenge = sha256(i.to_bytes(4, "big")).digest()
for index, quality in enumerate(pr.get_qualities_for_challenge(challenge)):
proof = pr.get_full_proof(challenge, index)
assert len(proof) == 8*pr.get_size()
computed_quality = v.validate_proof(plot_seed, pr.get_size(), challenge, proof)
assert computed_quality == quality
total_proofs += 1
print(f"total proofs {total_proofs} out of {iterations}\
{total_proofs / iterations}")
assert total_proofs == 4647
pr = None
Path("myplot.dat").unlink()
def test_faulty_plot_doesnt_crash(self):
if Path("myplot.dat").exists():
Path("myplot.dat").unlink()
if Path("myplotbad.dat").exists():
Path("myplotbad.dat").unlink()
plot_id: bytes = bytes([i for i in range(32, 64)])
pl = DiskPlotter()
pl.create_plot_disk(
".",
".",
".",
"myplot.dat",
21,
bytes([1, 2, 3, 4, 5]),
plot_id,
300,
32,
8192,
8,
False,
)
f = open("myplot.dat", "rb")
all_data = bytearray(f.read())
f.close()
assert len(all_data) > 20000000
all_data_bad = all_data[:20000000] + bytearray(
token_bytes(10000)) + all_data[20100000:]
f_bad = open("myplotbad.dat", "wb")
f_bad.write(all_data_bad)
f_bad.close()
pr = DiskProver(str(Path("myplotbad.dat")))
iterations: int = 50000
v = Verifier()
successes = 0
failures = 0
for i in range(iterations):
if i % 100 == 0:
print(i)
challenge = sha256(i.to_bytes(4, "big")).digest()
try:
for index, quality in enumerate(
pr.get_qualities_for_challenge(challenge)):
proof = pr.get_full_proof(challenge, index)
computed_quality = v.validate_proof(
plot_id, pr.get_size(), challenge, proof)
if computed_quality == quality:
successes += 1
else:
print("Did not validate")
failures += 1
except Exception as e:
print(f"Exception: {e}")
failures += 1
print(f"Successes: {successes}")
print(f"Failures: {failures}")
def main():
"""
Script for checking all plots in the plots.yaml file. Specify a number of challenge to test for each plot.
"""
parser = argparse.ArgumentParser(description="Chia plot checking script.")
parser.add_argument("-n",
"--num",
help="Number of challenges",
type=int,
default=1000)
args = parser.parse_args()
v = Verifier()
if os.path.isfile(plot_config_filename):
plot_config = safe_load(open(plot_config_filename, "r"))
for plot_filename, plot_info in plot_config["plots"].items():
plot_seed: bytes32 = ProofOfSpace.calculate_plot_seed(
PublicKey.from_bytes(bytes.fromhex(plot_info["pool_pk"])),
PrivateKey.from_bytes(bytes.fromhex(
plot_info["sk"])).get_public_key(),
)
if not os.path.isfile(plot_filename):
# Tries relative path
full_path: str = os.path.join(plot_root, plot_filename)
if not os.path.isfile(full_path):
# Tries absolute path
full_path: str = plot_filename
if not os.path.isfile(full_path):
print(f"Plot file {full_path} not found.")
continue
pr = DiskProver(full_path)
else:
pr = DiskProver(plot_filename)
total_proofs = 0
try:
for i in range(args.num):
challenge = sha256(i.to_bytes(32, "big")).digest()
for index, quality in enumerate(
pr.get_qualities_for_challenge(challenge)):
proof = pr.get_full_proof(challenge, index)
total_proofs += 1
ver_quality = v.validate_proof(plot_seed,
pr.get_size(),
challenge, proof)
assert quality == ver_quality
except BaseException as e:
print(
f"{type(e)}: {e} error in proving/verifying for plot {plot_filename}"
)
print(
f"{plot_filename}: Proofs {total_proofs} / {args.num}, {round(total_proofs/float(args.num), 4)}"
)
else:
print(f"Not plot file found at {plot_config_filename}")
def blocking_lookup(filename: Path, prover: DiskProver) -> Optional[List]:
# Uses the DiskProver object to lookup qualities. This is a blocking call,
# so it should be run in a threadpool.
try:
quality_strings = prover.get_qualities_for_challenge(
new_challenge.challenge_hash
)
except RuntimeError:
log.error("Error using prover object. Reinitializing prover object.")
try:
self.provers[filename] = DiskProver(str(filename))
quality_strings = prover.get_qualities_for_challenge(
new_challenge.challenge_hash
)
except RuntimeError:
log.error(
f"Retry-Error using prover object on {filename}. Giving up."
)
quality_strings = None
return quality_strings
class Harvester:
config: Dict
provers: Dict[Path, PlotInfo]
failed_to_open_filenames: Set[Path]
no_key_filenames: Set[Path]
farmer_public_keys: List[G1Element]
pool_public_keys: List[G1Element]
cached_challenges: List[harvester_protocol.NewChallenge]
root_path: Path
_is_shutdown: bool
executor: concurrent.futures.ThreadPoolExecutor
state_changed_callback: Optional[Callable]
constants: Dict
_refresh_lock: asyncio.Lock
def __init__(self, root_path: Path, override_constants={}):
self.root_path = root_path
# From filename to prover
self.provers = {}
self.failed_to_open_filenames = set()
self.no_key_filenames = set()
self._is_shutdown = False
self.global_connections: Optional[PeerConnections] = None
self.farmer_public_keys = []
self.pool_public_keys = []
self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=10)
self.state_changed_callback = None
self.server = None
self.constants = consensus_constants.copy()
self.cached_challenges = []
for key, value in override_constants.items():
self.constants[key] = value
async def _start(self):
self._refresh_lock = asyncio.Lock()
def _close(self):
self._is_shutdown = True
self.executor.shutdown(wait=True)
async def _await_closed(self):
pass
def _set_state_changed_callback(self, callback: Callable):
self.state_changed_callback = callback
if self.global_connections is not None:
self.global_connections.set_state_changed_callback(callback)
def _state_changed(self, change: str):
if self.state_changed_callback is not None:
self.state_changed_callback(change)
def _get_plots(self) -> Tuple[List[Dict], List[str], List[str]]:
response_plots: List[Dict] = []
for path, plot_info in self.provers.items():
prover = plot_info.prover
response_plots.append({
"filename": str(path),
"size": prover.get_size(),
"plot-seed": prover.get_id(),
"pool_public_key": plot_info.pool_public_key,
"farmer_public_key": plot_info.farmer_public_key,
"plot_public_key": plot_info.plot_public_key,
"local_sk": plot_info.local_sk,
"file_size": plot_info.file_size,
"time_modified": plot_info.time_modified,
})
return (
response_plots,
[str(s) for s in self.failed_to_open_filenames],
[str(s) for s in self.no_key_filenames],
)
async def _refresh_plots(self):
async with self._refresh_lock:
(
changed,
self.provers,
self.failed_to_open_filenames,
self.no_key_filenames,
) = load_plots(
self.provers,
self.failed_to_open_filenames,
self.farmer_public_keys,
self.pool_public_keys,
self.root_path,
)
if changed:
self._state_changed("plots")
def _delete_plot(self, str_path: str):
path = Path(str_path).resolve()
if path in self.provers:
del self.provers[path]
# Remove absolute and relative paths
if path.exists():
path.unlink()
self._state_changed("plots")
return True
async def _add_plot_directory(self, str_path: str) -> bool:
config = load_config(self.root_path, "config.yaml")
if str(Path(str_path).resolve()
) not in config["harvester"]["plot_directories"]:
config["harvester"]["plot_directories"].append(
str(Path(str_path).resolve()))
save_config(self.root_path, "config.yaml", config)
await self._refresh_plots()
return True
def _set_global_connections(self,
global_connections: Optional[PeerConnections]):
self.global_connections = global_connections
def _set_server(self, server):
self.server = server
@api_request
async def harvester_handshake(
self, harvester_handshake: harvester_protocol.HarvesterHandshake):
"""
Handshake between the harvester and farmer. The harvester receives the pool public keys,
as well as the farmer pks, which must be put into the plots, before the plotting process begins.
We cannot use any plots which have different keys in them.
"""
self.farmer_public_keys = harvester_handshake.farmer_public_keys
self.pool_public_keys = harvester_handshake.pool_public_keys
await self._refresh_plots()
if len(self.provers) == 0:
log.warning(
"Not farming any plots on this harvester. Check your configuration."
)
return
for new_challenge in self.cached_challenges:
async for msg in self.new_challenge(new_challenge):
yield msg
self.cached_challenges = []
self._state_changed("plots")
@api_request
async def new_challenge(self,
new_challenge: harvester_protocol.NewChallenge):
"""
The harvester receives a new challenge from the farmer, and looks up the quality string
for any proofs of space that are are found in the plots. If proofs are found, a
ChallengeResponse message is sent for each of the proofs found.
"""
if len(self.pool_public_keys) == 0 or len(
self.farmer_public_keys) == 0:
self.cached_challenges = self.cached_challenges[:5]
self.cached_challenges.insert(0, new_challenge)
return
start = time.time()
assert len(new_challenge.challenge_hash) == 32
# Refresh plots to see if there are any new ones
await self._refresh_plots()
loop = asyncio.get_running_loop()
def blocking_lookup(filename: Path,
prover: DiskProver) -> Optional[List]:
# Uses the DiskProver object to lookup qualities. This is a blocking call,
# so it should be run in a threadpool.
try:
quality_strings = prover.get_qualities_for_challenge(
new_challenge.challenge_hash)
except RuntimeError:
log.error(
"Error using prover object. Reinitializing prover object.")
try:
self.prover = DiskProver(str(filename))
quality_strings = self.prover.get_qualities_for_challenge(
new_challenge.challenge_hash)
except RuntimeError:
log.error(
f"Retry-Error using prover object on {filename}. Giving up."
)
quality_strings = None
return quality_strings
async def lookup_challenge(
filename: Path, prover: DiskProver
) -> List[harvester_protocol.ChallengeResponse]:
# Exectures a DiskProverLookup in a threadpool, and returns responses
all_responses: List[harvester_protocol.ChallengeResponse] = []
quality_strings = await loop.run_in_executor(
self.executor, blocking_lookup, filename, prover)
if quality_strings is not None:
for index, quality_str in enumerate(quality_strings):
response: harvester_protocol.ChallengeResponse = harvester_protocol.ChallengeResponse(
new_challenge.challenge_hash,
str(filename),
uint8(index),
quality_str,
prover.get_size(),
)
all_responses.append(response)
return all_responses
awaitables = []
for filename, plot_info in self.provers.items():
if ProofOfSpace.can_create_proof(
plot_info.prover.get_id(),
new_challenge.challenge_hash,
self.constants["NUMBER_ZERO_BITS_CHALLENGE_SIG"],
):
awaitables.append(lookup_challenge(filename, plot_info.prover))
# Concurrently executes all lookups on disk, to take advantage of multiple disk parallelism
total_proofs_found = 0
for sublist_awaitable in asyncio.as_completed(awaitables):
for response in await sublist_awaitable:
total_proofs_found += 1
yield OutboundMessage(
NodeType.FARMER,
Message("challenge_response", response),
Delivery.RESPOND,
)
log.info(
f"{len(awaitables)} plots were eligible for farming {new_challenge.challenge_hash.hex()[:10]}..."
f" Found {total_proofs_found} proofs. Time: {time.time() - start}. "
f"Total {len(self.provers)} plots")
@api_request
async def request_proof_of_space(
self, request: harvester_protocol.RequestProofOfSpace):
"""
The farmer requests a proof of space, for one of the plots.
We look up the correct plot based on the plot id and response number, lookup the proof,
and return it.
"""
response: Optional[harvester_protocol.RespondProofOfSpace] = None
challenge_hash = request.challenge_hash
filename = Path(request.plot_id).resolve()
index = request.response_number
proof_xs: bytes
plot_info = self.provers[filename]
try:
try:
proof_xs = plot_info.prover.get_full_proof(
challenge_hash, index)
except RuntimeError:
prover = DiskProver(str(filename))
self.provers[filename] = PlotInfo(
prover,
plot_info.pool_public_key,
plot_info.farmer_public_key,
plot_info.plot_public_key,
plot_info.local_sk,
plot_info.file_size,
plot_info.time_modified,
)
proof_xs = self.provers[filename].prover.get_full_proof(
challenge_hash, index)
except KeyError:
log.warning(f"KeyError plot {filename} does not exist.")
plot_info = self.provers[filename]
plot_public_key = ProofOfSpace.generate_plot_public_key(
plot_info.local_sk.get_g1(), plot_info.farmer_public_key)
proof_of_space: ProofOfSpace = ProofOfSpace(
challenge_hash,
plot_info.pool_public_key,
plot_public_key,
uint8(self.provers[filename].prover.get_size()),
proof_xs,
)
response = harvester_protocol.RespondProofOfSpace(
request.plot_id,
request.response_number,
proof_of_space,
)
if response:
yield OutboundMessage(
NodeType.FARMER,
Message("respond_proof_of_space", response),
Delivery.RESPOND,
)
@api_request
async def request_signature(self,
request: harvester_protocol.RequestSignature):
"""
The farmer requests a signature on the header hash, for one of the proofs that we found.
A signature is created on the header hash using the harvester private key. This can also
be used for pooling.
"""
plot_info = self.provers[Path(request.plot_id).resolve()]
local_sk = plot_info.local_sk
agg_pk = ProofOfSpace.generate_plot_public_key(
local_sk.get_g1(), plot_info.farmer_public_key)
# This is only a partial signature. When combined with the farmer's half, it will
# form a complete PrependSignature.
signature: G2Element = AugSchemeMPL.sign(local_sk, request.message,
agg_pk)
response: harvester_protocol.RespondSignature = harvester_protocol.RespondSignature(
request.plot_id,
request.message,
local_sk.get_g1(),
plot_info.farmer_public_key,
signature,
)
yield OutboundMessage(
NodeType.FARMER,
Message("respond_signature", response),
Delivery.RESPOND,
)
def test_k_21(self):
challenge: bytes = bytes([i for i in range(0, 32)])
plot_seed: bytes = bytes(
[
5,
104,
52,
4,
51,
55,
23,
84,
91,
10,
111,
12,
13,
222,
151,
16,
228,
211,
254,
45,
92,
198,
204,
10,
9,
10,
11,
129,
139,
171,
15,
23,
]
)
pl = DiskPlotter()
pl.create_plot_disk(
".", ".", ".", "myplot.dat", 21, bytes([1, 2, 3, 4, 5]), plot_seed, 300, 32, 8192, 8
)
pl = None
pr = DiskProver(str(Path("myplot.dat")))
total_proofs: int = 0
iterations: int = 5000
v = Verifier()
for i in range(iterations):
if i % 100 == 0:
print(i)
challenge = sha256(i.to_bytes(4, "big")).digest()
for index, quality in enumerate(pr.get_qualities_for_challenge(challenge)):
proof = pr.get_full_proof(challenge, index)
assert len(proof) == 8 * pr.get_size()
computed_quality = v.validate_proof(
plot_seed, pr.get_size(), challenge, proof
)
assert computed_quality == quality
total_proofs += 1
print(
f"total proofs {total_proofs} out of {iterations}\
{total_proofs / iterations}"
)
assert total_proofs > 4000
assert total_proofs < 6000
pr = None
sha256_plot_hash = sha256()
with open("myplot.dat", "rb") as f:
# Read and update hash string value in blocks of 4K
for byte_block in iter(lambda: f.read(4096), b""):
sha256_plot_hash.update(byte_block)
plot_hash = str(sha256_plot_hash.hexdigest())
assert (
plot_hash
== "80e32f560f3a4347760d6baae8d16fbaf484948088bff05c51bdcc24b7bc40d9"
)
print(f"\nPlotfile asserted sha256: {plot_hash}\n")
Path("myplot.dat").unlink()
class Harvester:
config: Dict
plot_config: Dict
provers: Dict[Path, DiskProver]
challenge_hashes: Dict[bytes32, Tuple[bytes32, Path, uint8]]
_plot_notification_task: asyncio.Task
_is_shutdown: bool
executor: concurrent.futures.ThreadPoolExecutor
@staticmethod
async def create(config: Dict, plot_config: Dict):
self = Harvester()
self.config = config
self.plot_config = plot_config
# From filename to prover
self.provers = {}
# From quality string to (challenge_hash, filename, index)
self.challenge_hashes = {}
self._plot_notification_task = asyncio.create_task(
self._plot_notification())
self._is_shutdown = False
self.server = None
self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=10)
return self
async def _plot_notification(self):
"""
Log the plot filenames to console periodically
"""
counter = 1
while not self._is_shutdown:
if counter % 600 == 0:
found = False
for filename, prover in self.provers.items():
log.info(
f"Farming plot {filename} of size {prover.get_size()}")
found = True
if not found:
log.warning(
"Not farming any plots on this harvester. Check your configuration."
)
await asyncio.sleep(1)
counter += 1
def set_server(self, server):
self.server = server
def _start_bg_tasks(self):
"""
Start a background task that checks connection and reconnects periodically to the farmer.
"""
farmer_peer = PeerInfo(self.config["farmer_peer"]["host"],
self.config["farmer_peer"]["port"])
async def connection_check():
while not self._is_shutdown:
if self.server is not None:
farmer_retry = True
for connection in self.server.global_connections.get_connections(
):
if connection.get_peer_info() == farmer_peer:
farmer_retry = False
if farmer_retry:
log.info(f"Reconnecting to farmer {farmer_retry}")
if not await self.server.start_client(
farmer_peer, None, auth=True):
await asyncio.sleep(1)
await asyncio.sleep(30)
self.reconnect_task = asyncio.create_task(connection_check())
def _shutdown(self):
self._is_shutdown = True
self.executor.shutdown(wait=True)
async def _await_shutdown(self):
await self._plot_notification_task
@api_request
async def harvester_handshake(
self, harvester_handshake: harvester_protocol.HarvesterHandshake):
"""
Handshake between the harvester and farmer. The harvester receives the pool public keys,
which must be put into the plots, before the plotting process begins. We cannot
use any plots which don't have one of the pool keys.
"""
self.provers = load_plots(self.config, self.plot_config,
harvester_handshake.pool_pubkeys)
if len(self.provers) == 0:
log.warning(
"Not farming any plots on this harvester. Check your configuration."
)
@api_request
async def new_challenge(self,
new_challenge: harvester_protocol.NewChallenge):
"""
The harvester receives a new challenge from the farmer, and looks up the quality string
for any proofs of space that are are found in the plots. If proofs are found, a
ChallengeResponse message is sent for each of the proofs found.
"""
start = time.time()
challenge_size = len(new_challenge.challenge_hash)
if challenge_size != 32:
raise ValueError(
f"Invalid challenge size {challenge_size}, 32 was expected")
loop = asyncio.get_running_loop()
def blocking_lookup(filename: Path,
prover: DiskProver) -> Optional[List]:
# Uses the DiskProver object to lookup qualities. This is a blocking call,
# so it should be run in a threadpool.
try:
quality_strings = prover.get_qualities_for_challenge(
new_challenge.challenge_hash)
except RuntimeError:
log.error(
"Error using prover object. Reinitializing prover object.")
try:
self.prover = DiskProver(str(filename))
quality_strings = self.prover.get_qualities_for_challenge(
new_challenge.challenge_hash)
except RuntimeError:
log.error(
f"Retry-Error using prover object on {filename}. Giving up."
)
quality_strings = None
return quality_strings
async def lookup_challenge(
filename: Path, prover: DiskProver
) -> List[harvester_protocol.ChallengeResponse]:
# Exectures a DiskProverLookup in a threadpool, and returns responses
all_responses: List[harvester_protocol.ChallengeResponse] = []
quality_strings = await loop.run_in_executor(
self.executor, blocking_lookup, filename, prover)
if quality_strings is not None:
for index, quality_str in enumerate(quality_strings):
self.challenge_hashes[quality_str] = (
new_challenge.challenge_hash,
filename,
uint8(index),
)
response: harvester_protocol.ChallengeResponse = harvester_protocol.ChallengeResponse(
new_challenge.challenge_hash, quality_str,
prover.get_size())
all_responses.append(response)
return all_responses
awaitables = [
lookup_challenge(filename, prover)
for filename, prover in self.provers.items()
]
# Concurrently executes all lookups on disk, to take advantage of multiple disk parallelism
for sublist_awaitable in asyncio.as_completed(awaitables):
for response in await sublist_awaitable:
yield OutboundMessage(
NodeType.FARMER,
Message("challenge_response", response),
Delivery.RESPOND,
)
log.info(
f"Time taken to lookup qualities in {len(self.provers)} plots: {time.time() - start}"
)
@api_request
async def request_proof_of_space(
self, request: harvester_protocol.RequestProofOfSpace):
"""
The farmer requests a signature on the header hash, for one of the proofs that we found.
We look up the correct plot based on the quality, lookup the proof, and return it.
"""
response: Optional[harvester_protocol.RespondProofOfSpace] = None
try:
# Using the quality string, find the right plot and index from our solutions
challenge_hash, filename, index = self.challenge_hashes[
request.quality_string]
except KeyError:
log.warning(f"Quality string {request.quality_string} not found")
return
if index is not None:
proof_xs: bytes
try:
proof_xs = self.provers[filename].get_full_proof(
challenge_hash, index)
except RuntimeError:
self.provers[filename] = DiskProver(str(filename))
proof_xs = self.provers[filename].get_full_proof(
challenge_hash, index)
pool_pubkey = PublicKey.from_bytes(
bytes.fromhex(self.plot_config["plots"][filename]["pool_pk"]))
plot_pubkey = PrivateKey.from_bytes(
bytes.fromhex(self.plot_config["plots"][filename]
["sk"])).get_public_key()
proof_of_space: ProofOfSpace = ProofOfSpace(
challenge_hash,
pool_pubkey,
plot_pubkey,
uint8(self.provers[filename].get_size()),
proof_xs,
)
response = harvester_protocol.RespondProofOfSpace(
request.quality_string, proof_of_space)
if response:
yield OutboundMessage(
NodeType.FARMER,
Message("respond_proof_of_space", response),
Delivery.RESPOND,
)
@api_request
async def request_header_signature(
self, request: harvester_protocol.RequestHeaderSignature):
"""
The farmer requests a signature on the header hash, for one of the proofs that we found.
A signature is created on the header hash using the plot private key.
"""
if request.quality_string not in self.challenge_hashes:
return
_, filename, _ = self.challenge_hashes[request.quality_string]
plot_sk = PrivateKey.from_bytes(
bytes.fromhex(self.plot_config["plots"][filename]["sk"]))
header_hash_signature: PrependSignature = plot_sk.sign_prepend(
request.header_hash)
assert header_hash_signature.verify(
[Util.hash256(request.header_hash)], [plot_sk.get_public_key()])
response: harvester_protocol.RespondHeaderSignature = harvester_protocol.RespondHeaderSignature(
request.quality_string,
header_hash_signature,
)
yield OutboundMessage(
NodeType.FARMER,
Message("respond_header_signature", response),
Delivery.RESPOND,
)
@api_request
async def request_partial_proof(
self, request: harvester_protocol.RequestPartialProof):
"""
The farmer requests a signature on the farmer_target, for one of the proofs that we found.
We look up the correct plot based on the quality, lookup the proof, and sign
the farmer target hash using the plot private key. This will be used as a pool share.
"""
_, filename, _ = self.challenge_hashes[request.quality_string]
plot_sk = PrivateKey.from_bytes(
bytes.fromhex(self.plot_config["plots"][filename]["sk"]))
farmer_target_signature: PrependSignature = plot_sk.sign_prepend(
request.farmer_target_hash)
response: harvester_protocol.RespondPartialProof = harvester_protocol.RespondPartialProof(
request.quality_string, farmer_target_signature)
yield OutboundMessage(
NodeType.FARMER,
Message("respond_partial_proof", response),
Delivery.RESPOND,
)
def _create_block(
self,
test_constants: Dict,
challenge_hash: bytes32,
height: uint32,
prev_header_hash: bytes32,
prev_iters: uint64,
prev_weight: uint128,
timestamp: uint64,
difficulty: uint64,
min_iters: uint64,
seed: bytes,
genesis: bool = False,
reward_puzzlehash: bytes32 = None,
transactions: Program = None,
aggsig: BLSSignature = None,
fees: uint64 = uint64(0),
) -> FullBlock:
"""
Creates a block with the specified details. Uses the stored plots to create a proof of space,
and also evaluates the VDF for the proof of time.
"""
selected_prover = None
selected_plot_sk = None
selected_pool_sk = None
selected_proof_index = 0
plots = list(self.plot_config["plots"].items())
selected_quality: Optional[bytes] = None
best_quality = 0
if self.use_any_pos:
for i in range(len(plots) * 3):
# Allow passing in seed, to create reorgs and different chains
random.seed(seed + i.to_bytes(4, "big"))
seeded_pn = random.randint(0, len(plots) - 1)
pool_sk = PrivateKey.from_bytes(
bytes.fromhex(plots[seeded_pn][1]["pool_sk"])
)
plot_sk = PrivateKey.from_bytes(
bytes.fromhex(plots[seeded_pn][1]["sk"])
)
prover = DiskProver(plots[seeded_pn][0])
qualities = prover.get_qualities_for_challenge(challenge_hash)
if len(qualities) > 0:
if self.use_any_pos:
selected_quality = qualities[0]
selected_prover = prover
selected_pool_sk = pool_sk
selected_plot_sk = plot_sk
break
else:
for i in range(len(plots)):
pool_sk = PrivateKey.from_bytes(bytes.fromhex(plots[i][1]["pool_sk"]))
plot_sk = PrivateKey.from_bytes(bytes.fromhex(plots[i][1]["sk"]))
prover = DiskProver(plots[i][0])
qualities = prover.get_qualities_for_challenge(challenge_hash)
j = 0
for quality in qualities:
qual_int = int.from_bytes(quality, "big", signed=False)
if qual_int > best_quality:
best_quality = qual_int
selected_quality = quality
selected_prover = prover
selected_pool_sk = pool_sk
selected_plot_sk = plot_sk
selected_proof_index = j
j += 1
assert selected_prover
assert selected_pool_sk
assert selected_plot_sk
pool_pk = selected_pool_sk.get_public_key()
plot_pk = selected_plot_sk.get_public_key()
if selected_quality is None:
raise RuntimeError("No proofs for this challenge")
proof_xs: bytes = selected_prover.get_full_proof(
challenge_hash, selected_proof_index
)
proof_of_space: ProofOfSpace = ProofOfSpace(
challenge_hash, pool_pk, plot_pk, selected_prover.get_size(), proof_xs
)
number_iters: uint64 = pot_iterations.calculate_iterations(
proof_of_space, difficulty, min_iters
)
# print("Doing iters", number_iters)
int_size = (test_constants["DISCRIMINANT_SIZE_BITS"] + 16) >> 4
result = prove(
challenge_hash, test_constants["DISCRIMINANT_SIZE_BITS"], number_iters
)
output = ClassgroupElement(
int512(int.from_bytes(result[0:int_size], "big", signed=True,)),
int512(
int.from_bytes(result[int_size : 2 * int_size], "big", signed=True,)
),
)
proof_bytes = result[2 * int_size : 4 * int_size]
proof_of_time = ProofOfTime(
challenge_hash, number_iters, output, self.n_wesolowski, proof_bytes,
)
if not reward_puzzlehash:
reward_puzzlehash = self.fee_target
# Use the extension data to create different blocks based on header hash
extension_data: bytes32 = bytes32([random.randint(0, 255) for _ in range(32)])
cost = uint64(0)
coinbase_reward = block_rewards.calculate_block_reward(height)
fee_reward = uint64(block_rewards.calculate_base_fee(height) + fees)
coinbase_coin, coinbase_signature = create_coinbase_coin_and_signature(
height, reward_puzzlehash, coinbase_reward, selected_pool_sk
)
parent_coin_name = std_hash(std_hash(height))
fees_coin = Coin(parent_coin_name, reward_puzzlehash, uint64(fee_reward))
# Create filter
byte_array_tx: List[bytes32] = []
tx_additions: List[Coin] = []
tx_removals: List[bytes32] = []
encoded = None
if transactions:
error, npc_list, _ = get_name_puzzle_conditions(transactions)
additions: List[Coin] = additions_for_npc(npc_list)
for coin in additions:
tx_additions.append(coin)
byte_array_tx.append(bytearray(coin.puzzle_hash))
for npc in npc_list:
tx_removals.append(npc.coin_name)
byte_array_tx.append(bytearray(npc.coin_name))
bip158: PyBIP158 = PyBIP158(byte_array_tx)
encoded = bytes(bip158.GetEncoded())
removal_merkle_set = MerkleSet()
addition_merkle_set = MerkleSet()
# Create removal Merkle set
for coin_name in tx_removals:
removal_merkle_set.add_already_hashed(coin_name)
# Create addition Merkle set
puzzlehash_coin_map: Dict[bytes32, List[Coin]] = {}
for coin in tx_additions:
if coin.puzzle_hash in puzzlehash_coin_map:
puzzlehash_coin_map[coin.puzzle_hash].append(coin)
else:
puzzlehash_coin_map[coin.puzzle_hash] = [coin]
# Addition Merkle set contains puzzlehash and hash of all coins with that puzzlehash
for puzzle, coins in puzzlehash_coin_map.items():
addition_merkle_set.add_already_hashed(puzzle)
addition_merkle_set.add_already_hashed(hash_coin_list(coins))
additions_root = addition_merkle_set.get_root()
removal_root = removal_merkle_set.get_root()
generator_hash = (
transactions.get_tree_hash()
if transactions is not None
else bytes32([0] * 32)
)
filter_hash = std_hash(encoded) if encoded is not None else bytes32([0] * 32)
header_data: HeaderData = HeaderData(
height,
prev_header_hash,
timestamp,
filter_hash,
proof_of_space.get_hash(),
uint128(prev_weight + difficulty),
uint64(prev_iters + number_iters),
additions_root,
removal_root,
coinbase_coin,
coinbase_signature,
fees_coin,
aggsig,
cost,
extension_data,
generator_hash,
)
header_hash_sig: PrependSignature = selected_plot_sk.sign_prepend(
header_data.get_hash()
)
header: Header = Header(header_data, header_hash_sig)
full_block: FullBlock = FullBlock(
proof_of_space, proof_of_time, header, transactions, encoded
)
return full_block
import secrets
import os
challenge: bytes = bytes([i for i in range(0, 32)])
plot_id: bytes = bytes([
5, 104, 52, 4, 51, 55, 23, 84, 91, 10, 111, 12, 13, 222, 151, 16, 228, 211,
254, 45, 92, 198, 204, 10, 9, 10, 11, 129, 139, 171, 15, 23
])
filename = "./myplot.dat"
pl = DiskPlotter()
pl.create_plot_disk(filename, 21, bytes([1, 2, 3, 4, 5]), plot_id)
pr = DiskProver(filename)
total_proofs: int = 0
iterations: int = 5000
v = Verifier()
for i in range(iterations):
challenge = sha256(i.to_bytes(4, "big")).digest()
for index, quality in enumerate(pr.get_qualities_for_challenge(challenge)):
proof = pr.get_full_proof(challenge, index)
total_proofs += 1
ver_quality = v.validate_proof(plot_id, 21, challenge, proof)
assert (quality == ver_quality)
os.remove(filename)
print(f"total proofs {total_proofs} out of {iterations}\
{total_proofs / iterations}")
def _create_block(
self,
test_constants: Dict,
challenge_hash: bytes32,
height: uint32,
prev_header_hash: bytes32,
prev_iters: uint64,
prev_weight: uint64,
timestamp: uint64,
difficulty: uint64,
ips: uint64,
seed: bytes,
) -> FullBlock:
"""
Creates a block with the specified details. Uses the stored plots to create a proof of space,
and also evaluates the VDF for the proof of time.
"""
prover = None
plot_pk = None
plot_sk = None
qualities: List[bytes] = []
for pn in range(num_plots):
# Allow passing in seed, to create reorgs and different chains
seeded_pn = (pn + 17 * int.from_bytes(seed, "big")) % num_plots
filename = self.filenames[seeded_pn]
plot_pk = plot_pks[seeded_pn]
plot_sk = plot_sks[seeded_pn]
prover = DiskProver(os.path.join(self.plot_dir, filename))
qualities = prover.get_qualities_for_challenge(challenge_hash)
if len(qualities) > 0:
break
assert prover
assert plot_pk
assert plot_sk
if len(qualities) == 0:
raise NoProofsOfSpaceFound("No proofs for this challenge")
proof_xs: bytes = prover.get_full_proof(challenge_hash, 0)
proof_of_space: ProofOfSpace = ProofOfSpace(
challenge_hash, pool_pk, plot_pk, k, [uint8(b) for b in proof_xs])
number_iters: uint64 = pot_iterations.calculate_iterations(
proof_of_space, difficulty, ips, test_constants["MIN_BLOCK_TIME"])
disc: int = create_discriminant(
challenge_hash, test_constants["DISCRIMINANT_SIZE_BITS"])
start_x: ClassGroup = ClassGroup.from_ab_discriminant(2, 1, disc)
y_cl, proof_bytes = create_proof_of_time_nwesolowski(
disc, start_x, number_iters, disc, n_wesolowski)
output = ClassgroupElement(y_cl[0], y_cl[1])
proof_of_time = ProofOfTime(
challenge_hash,
number_iters,
output,
n_wesolowski,
[uint8(b) for b in proof_bytes],
)
coinbase: CoinbaseInfo = CoinbaseInfo(
height,
block_rewards.calculate_block_reward(uint32(height)),
coinbase_target,
)
coinbase_sig: PrependSignature = pool_sk.sign_prepend(bytes(coinbase))
fees_target: FeesTarget = FeesTarget(fee_target, uint64(0))
solutions_generator: bytes32 = sha256(seed).digest()
cost = uint64(0)
body: Body = Body(coinbase, coinbase_sig, fees_target, None,
solutions_generator, cost)
header_data: HeaderData = HeaderData(
prev_header_hash,
timestamp,
bytes([0] * 32),
proof_of_space.get_hash(),
body.get_hash(),
bytes([0] * 32),
)
header_hash_sig: PrependSignature = plot_sk.sign_prepend(
header_data.get_hash())
header: Header = Header(header_data, header_hash_sig)
challenge = Challenge(
challenge_hash,
proof_of_space.get_hash(),
proof_of_time.get_hash(),
height,
uint64(prev_weight + difficulty),
uint64(prev_iters + number_iters),
)
header_block = HeaderBlock(proof_of_space, proof_of_time, challenge,
header)
full_block: FullBlock = FullBlock(header_block, body)
return full_block
class Harvester:
config: Dict
plot_config: Dict
provers: Dict[str, DiskProver]
failed_to_open_filenames: List[str]
not_found_filenames: List[str]
challenge_hashes: Dict[bytes32, Tuple[bytes32, str, uint8]]
pool_pubkeys: List[PublicKey]
root_path: Path
_plot_notification_task: asyncio.Future
_is_shutdown: bool
executor: concurrent.futures.ThreadPoolExecutor
state_changed_callback: Optional[Callable]
def __init__(self, config: Dict, plot_config: Dict, root_path: Path):
self.config = config
self.plot_config = plot_config
self.root_path = root_path
# From filename to prover
self.provers = {}
self.failed_to_open_filenames = []
self.not_found_filenames = []
# From quality string to (challenge_hash, filename, index)
self.challenge_hashes = {}
self._plot_notification_task = asyncio.ensure_future(self._plot_notification())
self._is_shutdown = False
self.global_connections: Optional[PeerConnections] = None
self.pool_pubkeys = []
self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=10)
self.state_changed_callback = None
self.server = None
def _set_state_changed_callback(self, callback: Callable):
self.state_changed_callback = callback
if self.global_connections is not None:
self.global_connections.set_state_changed_callback(callback)
def _state_changed(self, change: str):
if self.state_changed_callback is not None:
self.state_changed_callback(change)
async def _plot_notification(self):
"""
Log the plot filenames to console periodically
"""
counter = 1
while not self._is_shutdown:
if counter % 600 == 0:
found = False
for filename, prover in self.provers.items():
log.info(f"Farming plot {filename} of size {prover.get_size()}")
found = True
if not found:
log.warning(
"Not farming any plots on this harvester. Check your configuration."
)
await asyncio.sleep(1)
counter += 1
def _get_plots(self) -> Tuple[List[Dict], List[str], List[str]]:
response_plots: List[Dict] = []
for path, prover in self.provers.items():
plot_pk = PrivateKey.from_bytes(
bytes.fromhex(self.plot_config["plots"][path]["sk"])
).get_public_key()
pool_pk = PublicKey.from_bytes(
bytes.fromhex(self.plot_config["plots"][path]["pool_pk"])
)
response_plots.append(
{
"filename": str(path),
"size": prover.get_size(),
"plot-seed": prover.get_id(),
"memo": prover.get_memo(),
"plot_pk": bytes(plot_pk),
"pool_pk": bytes(pool_pk),
}
)
return (response_plots, self.failed_to_open_filenames, self.not_found_filenames)
def _refresh_plots(self, reload_config_file=True):
if reload_config_file:
self.plot_config = load_config(self.root_path, "plots.yaml")
(
self.provers,
self.failed_to_open_filenames,
self.not_found_filenames,
) = load_plots(self.config, self.plot_config, self.pool_pubkeys, self.root_path)
self._state_changed("plots")
def _delete_plot(self, str_path: str):
if str_path in self.provers:
del self.provers[str_path]
plot_root = path_from_root(self.root_path, self.config.get("plot_root", "."))
# Remove absolute and relative paths
if Path(str_path).exists():
Path(str_path).unlink()
if (plot_root / Path(str_path)).exists():
(plot_root / Path(str_path)).unlink()
try:
# Removes the plot from config.yaml
plot_config = load_config(self.root_path, "plots.yaml")
if str_path in plot_config["plots"]:
del plot_config["plots"][str_path]
save_config(self.root_path, "plots.yaml", plot_config)
self.plot_config = plot_config
except (FileNotFoundError, KeyError) as e:
log.warning(f"Could not remove {str_path} {e}")
return False
self._state_changed("plots")
return True
def _add_plot(
self, str_path: str, plot_sk: PrivateKey, pool_pk: Optional[PublicKey]
) -> bool:
plot_config = load_config(self.root_path, "plots.yaml")
if pool_pk is None:
for pool_pk_cand in self.pool_pubkeys:
pr = DiskProver(str_path)
if (
ProofOfSpace.calculate_plot_seed(
pool_pk_cand, plot_sk.get_public_key()
)
== pr.get_id()
):
pool_pk = pool_pk_cand
break
if pool_pk is None:
return False
plot_config["plots"][str_path] = {
"sk": bytes(plot_sk).hex(),
"pool_pk": bytes(pool_pk).hex(),
}
save_config(self.root_path, "plots.yaml", plot_config)
self._refresh_plots()
return True
def set_global_connections(self, global_connections: Optional[PeerConnections]):
self.global_connections = global_connections
def set_server(self, server):
self.server = server
def _shutdown(self):
self._is_shutdown = True
self.executor.shutdown(wait=True)
async def _await_shutdown(self):
await self._plot_notification_task
@api_request
async def harvester_handshake(
self, harvester_handshake: harvester_protocol.HarvesterHandshake
):
"""
Handshake between the harvester and farmer. The harvester receives the pool public keys,
which must be put into the plots, before the plotting process begins. We cannot
use any plots which don't have one of the pool keys.
"""
self.pool_pubkeys = harvester_handshake.pool_pubkeys
self._refresh_plots(reload_config_file=False)
if len(self.provers) == 0:
log.warning(
"Not farming any plots on this harvester. Check your configuration."
)
@api_request
async def new_challenge(self, new_challenge: harvester_protocol.NewChallenge):
"""
The harvester receives a new challenge from the farmer, and looks up the quality string
for any proofs of space that are are found in the plots. If proofs are found, a
ChallengeResponse message is sent for each of the proofs found.
"""
start = time.time()
challenge_size = len(new_challenge.challenge_hash)
if challenge_size != 32:
raise ValueError(
f"Invalid challenge size {challenge_size}, 32 was expected"
)
loop = asyncio.get_running_loop()
def blocking_lookup(filename: Path, prover: DiskProver) -> Optional[List]:
# Uses the DiskProver object to lookup qualities. This is a blocking call,
# so it should be run in a threadpool.
try:
quality_strings = prover.get_qualities_for_challenge(
new_challenge.challenge_hash
)
except RuntimeError:
log.error("Error using prover object. Reinitializing prover object.")
try:
self.prover = DiskProver(str(filename))
quality_strings = self.prover.get_qualities_for_challenge(
new_challenge.challenge_hash
)
except RuntimeError:
log.error(
f"Retry-Error using prover object on {filename}. Giving up."
)
quality_strings = None
return quality_strings
async def lookup_challenge(
filename: str, prover: DiskProver
) -> List[harvester_protocol.ChallengeResponse]:
# Exectures a DiskProverLookup in a threadpool, and returns responses
all_responses: List[harvester_protocol.ChallengeResponse] = []
quality_strings = await loop.run_in_executor(
self.executor, blocking_lookup, filename, prover
)
if quality_strings is not None:
for index, quality_str in enumerate(quality_strings):
self.challenge_hashes[quality_str] = (
new_challenge.challenge_hash,
filename,
uint8(index),
)
response: harvester_protocol.ChallengeResponse = harvester_protocol.ChallengeResponse(
new_challenge.challenge_hash, quality_str, prover.get_size()
)
all_responses.append(response)
return all_responses
awaitables = [
lookup_challenge(filename, prover)
for filename, prover in self.provers.items()
]
# Concurrently executes all lookups on disk, to take advantage of multiple disk parallelism
for sublist_awaitable in asyncio.as_completed(awaitables):
for response in await sublist_awaitable:
yield OutboundMessage(
NodeType.FARMER,
Message("challenge_response", response),
Delivery.RESPOND,
)
log.info(
f"Time taken to lookup qualities in {len(self.provers)} plots: {time.time() - start}"
)
@api_request
async def request_proof_of_space(
self, request: harvester_protocol.RequestProofOfSpace
):
"""
The farmer requests a signature on the header hash, for one of the proofs that we found.
We look up the correct plot based on the quality, lookup the proof, and return it.
"""
response: Optional[harvester_protocol.RespondProofOfSpace] = None
try:
# Using the quality string, find the right plot and index from our solutions
challenge_hash, filename, index = self.challenge_hashes[
request.quality_string
]
except KeyError:
log.warning(f"Quality string {request.quality_string} not found")
return
if index is not None:
proof_xs: bytes
try:
try:
proof_xs = self.provers[filename].get_full_proof(
challenge_hash, index
)
except RuntimeError:
self.provers[filename] = DiskProver(str(filename))
proof_xs = self.provers[filename].get_full_proof(
challenge_hash, index
)
except KeyError:
log.warning(f"KeyError plot {filename} does not exist.")
pool_pubkey = PublicKey.from_bytes(
bytes.fromhex(self.plot_config["plots"][filename]["pool_pk"])
)
plot_pubkey = PrivateKey.from_bytes(
bytes.fromhex(self.plot_config["plots"][filename]["sk"])
).get_public_key()
proof_of_space: ProofOfSpace = ProofOfSpace(
challenge_hash,
pool_pubkey,
plot_pubkey,
uint8(self.provers[filename].get_size()),
proof_xs,
)
response = harvester_protocol.RespondProofOfSpace(
request.quality_string, proof_of_space
)
if response:
yield OutboundMessage(
NodeType.FARMER,
Message("respond_proof_of_space", response),
Delivery.RESPOND,
)
@api_request
async def request_header_signature(
self, request: harvester_protocol.RequestHeaderSignature
):
"""
The farmer requests a signature on the header hash, for one of the proofs that we found.
A signature is created on the header hash using the plot private key.
"""
if request.quality_string not in self.challenge_hashes:
return
_, filename, _ = self.challenge_hashes[request.quality_string]
plot_sk = PrivateKey.from_bytes(
bytes.fromhex(self.plot_config["plots"][filename]["sk"])
)
header_hash_signature: PrependSignature = plot_sk.sign_prepend(
request.header_hash
)
assert header_hash_signature.verify(
[Util.hash256(request.header_hash)], [plot_sk.get_public_key()]
)
response: harvester_protocol.RespondHeaderSignature = harvester_protocol.RespondHeaderSignature(
request.quality_string, header_hash_signature,
)
yield OutboundMessage(
NodeType.FARMER,
Message("respond_header_signature", response),
Delivery.RESPOND,
)
@api_request
async def request_partial_proof(
self, request: harvester_protocol.RequestPartialProof
):
"""
The farmer requests a signature on the farmer_target, for one of the proofs that we found.
We look up the correct plot based on the quality, lookup the proof, and sign
the farmer target hash using the plot private key. This will be used as a pool share.
"""
_, filename, _ = self.challenge_hashes[request.quality_string]
plot_sk = PrivateKey.from_bytes(
bytes.fromhex(self.plot_config["plots"][filename]["sk"])
)
farmer_target_signature: PrependSignature = plot_sk.sign_prepend(
request.farmer_target_hash
)
response: harvester_protocol.RespondPartialProof = harvester_protocol.RespondPartialProof(
request.quality_string, farmer_target_signature
)
yield OutboundMessage(
NodeType.FARMER,
Message("respond_partial_proof", response),
Delivery.RESPOND,
)
