def test_k_21(self):
challenge: bytes = bytes([i for i in range(0, 32)])
plot_seed: bytes = bytes([5, 104, 52, 4, 51, 55, 23, 84, 91, 10, 111, 12, 13,
222, 151, 16, 228, 211, 254, 45, 92, 198, 204, 10, 9,
10, 11, 129, 139, 171, 15, 23])
pl = DiskPlotter()
pl.create_plot_disk(".", ".", ".", "myplot.dat", 21, bytes([1, 2, 3, 4, 5]), plot_seed, 2*1024)
pl = None
pr = DiskProver(str(Path("myplot.dat")))
total_proofs: int = 0
iterations: int = 5000
v = Verifier()
for i in range(iterations):
if i % 100 == 0:
print(i)
challenge = sha256(i.to_bytes(4, "big")).digest()
for index, quality in enumerate(pr.get_qualities_for_challenge(challenge)):
proof = pr.get_full_proof(challenge, index)
assert len(proof) == 8*pr.get_size()
computed_quality = v.validate_proof(plot_seed, pr.get_size(), challenge, proof)
assert computed_quality == quality
total_proofs += 1
print(f"total proofs {total_proofs} out of {iterations}\
{total_proofs / iterations}")
assert total_proofs == 4647
pr = None
Path("myplot.dat").unlink()
def test_faulty_plot_doesnt_crash(self):
if Path("myplot.dat").exists():
Path("myplot.dat").unlink()
if Path("myplotbad.dat").exists():
Path("myplotbad.dat").unlink()
plot_id: bytes = bytes([i for i in range(32, 64)])
pl = DiskPlotter()
pl.create_plot_disk(
".",
".",
".",
"myplot.dat",
21,
bytes([1, 2, 3, 4, 5]),
plot_id,
300,
32,
8192,
8,
False,
)
f = open("myplot.dat", "rb")
all_data = bytearray(f.read())
f.close()
assert len(all_data) > 20000000
all_data_bad = all_data[:20000000] + bytearray(
token_bytes(10000)) + all_data[20100000:]
f_bad = open("myplotbad.dat", "wb")
f_bad.write(all_data_bad)
f_bad.close()
pr = DiskProver(str(Path("myplotbad.dat")))
iterations: int = 50000
v = Verifier()
successes = 0
failures = 0
for i in range(iterations):
if i % 100 == 0:
print(i)
challenge = sha256(i.to_bytes(4, "big")).digest()
try:
for index, quality in enumerate(
pr.get_qualities_for_challenge(challenge)):
proof = pr.get_full_proof(challenge, index)
computed_quality = v.validate_proof(
plot_id, pr.get_size(), challenge, proof)
if computed_quality == quality:
successes += 1
else:
print("Did not validate")
failures += 1
except Exception as e:
print(f"Exception: {e}")
failures += 1
print(f"Successes: {successes}")
print(f"Failures: {failures}")
def main():
"""
Script for checking all plots in the plots.yaml file. Specify a number of challenge to test for each plot.
"""
parser = argparse.ArgumentParser(description="Chia plot checking script.")
parser.add_argument("-n",
"--num",
help="Number of challenges",
type=int,
default=1000)
args = parser.parse_args()
v = Verifier()
if os.path.isfile(plot_config_filename):
plot_config = safe_load(open(plot_config_filename, "r"))
for plot_filename, plot_info in plot_config["plots"].items():
plot_seed: bytes32 = ProofOfSpace.calculate_plot_seed(
PublicKey.from_bytes(bytes.fromhex(plot_info["pool_pk"])),
PrivateKey.from_bytes(bytes.fromhex(
plot_info["sk"])).get_public_key(),
)
if not os.path.isfile(plot_filename):
# Tries relative path
full_path: str = os.path.join(plot_root, plot_filename)
if not os.path.isfile(full_path):
# Tries absolute path
full_path: str = plot_filename
if not os.path.isfile(full_path):
print(f"Plot file {full_path} not found.")
continue
pr = DiskProver(full_path)
else:
pr = DiskProver(plot_filename)
total_proofs = 0
try:
for i in range(args.num):
challenge = sha256(i.to_bytes(32, "big")).digest()
for index, quality in enumerate(
pr.get_qualities_for_challenge(challenge)):
proof = pr.get_full_proof(challenge, index)
total_proofs += 1
ver_quality = v.validate_proof(plot_seed,
pr.get_size(),
challenge, proof)
assert quality == ver_quality
except BaseException as e:
print(
f"{type(e)}: {e} error in proving/verifying for plot {plot_filename}"
)
print(
f"{plot_filename}: Proofs {total_proofs} / {args.num}, {round(total_proofs/float(args.num), 4)}"
)
else:
print(f"Not plot file found at {plot_config_filename}")
async def lookup_challenge(
filename: Path, prover: DiskProver
) -> List[harvester_protocol.ChallengeResponse]:
# Exectures a DiskProverLookup in a threadpool, and returns responses
all_responses: List[harvester_protocol.ChallengeResponse] = []
quality_strings = await loop.run_in_executor(
self.executor, blocking_lookup, filename, prover)
if quality_strings is not None:
for index, quality_str in enumerate(quality_strings):
response: harvester_protocol.ChallengeResponse = harvester_protocol.ChallengeResponse(
new_challenge.challenge_hash,
str(filename),
uint8(index),
quality_str,
prover.get_size(),
)
all_responses.append(response)
return all_responses
def process_file(file_path: Path) -> Optional[PlotInfo]:
if not self._refreshing_enabled:
return None
filename_str = str(file_path)
if self.match_str is not None and self.match_str not in filename_str:
return None
if (
file_path in self.failed_to_open_filenames
and (time.time() - self.failed_to_open_filenames[file_path])
< self.refresh_parameter.retry_invalid_seconds
):
# Try once every `refresh_parameter.retry_invalid_seconds` seconds to open the file
return None
if file_path in self.plots:
return self.plots[file_path]
entry: Optional[Tuple[str, Set[str]]] = self.plot_filename_paths.get(file_path.name)
if entry is not None:
loaded_parent, duplicates = entry
if str(file_path.parent) in duplicates:
log.debug(f"Skip duplicated plot {str(file_path)}")
return None
try:
if not file_path.exists():
return None
prover = DiskProver(str(file_path))
log.debug(f"process_file {str(file_path)}")
expected_size = _expected_plot_size(prover.get_size()) * UI_ACTUAL_SPACE_CONSTANT_FACTOR
stat_info = file_path.stat()
# TODO: consider checking if the file was just written to (which would mean that the file is still
# being copied). A segfault might happen in this edge case.
if prover.get_size() >= 30 and stat_info.st_size < 0.98 * expected_size:
log.warning(
f"Not farming plot {file_path}. Size is {stat_info.st_size / (1024**3)} GiB, but expected"
f" at least: {expected_size / (1024 ** 3)} GiB. We assume the file is being copied."
)
return None
cache_entry = self.cache.get(prover.get_id())
if cache_entry is None:
(
pool_public_key_or_puzzle_hash,
farmer_public_key,
local_master_sk,
) = parse_plot_info(prover.get_memo())
# Only use plots that correct keys associated with them
if farmer_public_key not in self.farmer_public_keys:
log.warning(f"Plot {file_path} has a farmer public key that is not in the farmer's pk list.")
self.no_key_filenames.add(file_path)
if not self.open_no_key_filenames:
return None
pool_public_key: Optional[G1Element] = None
pool_contract_puzzle_hash: Optional[bytes32] = None
if isinstance(pool_public_key_or_puzzle_hash, G1Element):
pool_public_key = pool_public_key_or_puzzle_hash
else:
assert isinstance(pool_public_key_or_puzzle_hash, bytes32)
pool_contract_puzzle_hash = pool_public_key_or_puzzle_hash
if pool_public_key is not None and pool_public_key not in self.pool_public_keys:
log.warning(f"Plot {file_path} has a pool public key that is not in the farmer's pool pk list.")
self.no_key_filenames.add(file_path)
if not self.open_no_key_filenames:
return None
# If a plot is in `no_key_filenames` the keys were missing in earlier refresh cycles. We can remove
# the current plot from that list if its in there since we passed the key checks above.
if file_path in self.no_key_filenames:
self.no_key_filenames.remove(file_path)
local_sk = master_sk_to_local_sk(local_master_sk)
plot_public_key: G1Element = ProofOfSpace.generate_plot_public_key(
local_sk.get_g1(), farmer_public_key, pool_contract_puzzle_hash is not None
)
cache_entry = CacheEntry(pool_public_key, pool_contract_puzzle_hash, plot_public_key)
self.cache.update(prover.get_id(), cache_entry)
with self.plot_filename_paths_lock:
paths: Optional[Tuple[str, Set[str]]] = self.plot_filename_paths.get(file_path.name)
if paths is None:
paths = (str(Path(prover.get_filename()).parent), set())
self.plot_filename_paths[file_path.name] = paths
else:
paths[1].add(str(Path(prover.get_filename()).parent))
log.warning(f"Have multiple copies of the plot {file_path.name} in {[paths[0], *paths[1]]}.")
return None
new_plot_info: PlotInfo = PlotInfo(
prover,
cache_entry.pool_public_key,
cache_entry.pool_contract_puzzle_hash,
cache_entry.plot_public_key,
stat_info.st_size,
stat_info.st_mtime,
)
with counter_lock:
result.loaded.append(new_plot_info)
if file_path in self.failed_to_open_filenames:
del self.failed_to_open_filenames[file_path]
except Exception as e:
tb = traceback.format_exc()
log.error(f"Failed to open file {file_path}. {e} {tb}")
self.failed_to_open_filenames[file_path] = int(time.time())
return None
log.info(f"Found plot {file_path} of size {new_plot_info.prover.get_size()}")
if self.show_memo:
plot_memo: bytes32
if pool_contract_puzzle_hash is None:
plot_memo = stream_plot_info_pk(pool_public_key, farmer_public_key, local_master_sk)
else:
plot_memo = stream_plot_info_ph(pool_contract_puzzle_hash, farmer_public_key, local_master_sk)
plot_memo_str: str = plot_memo.hex()
log.info(f"Memo: {plot_memo_str}")
return new_plot_info
def process_file(directory: Path, filename: Path) -> Tuple[int, Dict]:
new_provers: Dict[Path, PlotInfo] = {}
nonlocal changed
filename_str = str(filename)
if match_str is not None and match_str not in filename_str:
return 0, new_provers
if filename.exists():
if filename in failed_to_open_filenames and (
time.time() - failed_to_open_filenames[filename]) < 1200:
# Try once every 20 minutes to open the file
return 0, new_provers
if filename in provers:
try:
stat_info = filename.stat()
except Exception as e:
log.error(f"Failed to open file {filename}. {e}")
return 0, new_provers
if stat_info.st_mtime == provers[filename].time_modified:
with plot_ids_lock:
if provers[filename].prover.get_id() in plot_ids:
log.warning(
f"Have multiple copies of the plot {filename}, not adding it."
)
return 0, new_provers
plot_ids.add(provers[filename].prover.get_id())
new_provers[filename] = provers[filename]
return stat_info.st_size, new_provers
try:
prover = DiskProver(str(filename))
expected_size = _expected_plot_size(
prover.get_size()) * UI_ACTUAL_SPACE_CONSTANT_FACTOR
stat_info = filename.stat()
# TODO: consider checking if the file was just written to (which would mean that the file is still
# being copied). A segfault might happen in this edge case.
if prover.get_size(
) >= 30 and stat_info.st_size < 0.98 * expected_size:
log.warning(
f"Not farming plot {filename}. Size is {stat_info.st_size / (1024**3)} GiB, but expected"
f" at least: {expected_size / (1024 ** 3)} GiB. We assume the file is being copied."
)
return 0, new_provers
(
pool_public_key_or_puzzle_hash,
farmer_public_key,
local_master_sk,
) = parse_plot_info(prover.get_memo())
# Only use plots that correct keys associated with them
if farmer_public_keys is not None and farmer_public_key not in farmer_public_keys:
log.warning(
f"Plot {filename} has a farmer public key that is not in the farmer's pk list."
)
no_key_filenames.add(filename)
if not open_no_key_filenames:
return 0, new_provers
if isinstance(pool_public_key_or_puzzle_hash, G1Element):
pool_public_key = pool_public_key_or_puzzle_hash
pool_contract_puzzle_hash = None
else:
assert isinstance(pool_public_key_or_puzzle_hash, bytes32)
pool_public_key = None
pool_contract_puzzle_hash = pool_public_key_or_puzzle_hash
if (pool_public_keys is not None
and pool_public_key is not None
and pool_public_key not in pool_public_keys):
log.warning(
f"Plot {filename} has a pool public key that is not in the farmer's pool pk list."
)
no_key_filenames.add(filename)
if not open_no_key_filenames:
return 0, new_provers
stat_info = filename.stat()
local_sk = master_sk_to_local_sk(local_master_sk)
plot_public_key: G1Element = ProofOfSpace.generate_plot_public_key(
local_sk.get_g1(), farmer_public_key)
with plot_ids_lock:
if prover.get_id() in plot_ids:
log.warning(
f"Have multiple copies of the plot {filename}, not adding it."
)
return 0, new_provers
plot_ids.add(prover.get_id())
new_provers[filename] = PlotInfo(
prover,
pool_public_key,
pool_contract_puzzle_hash,
plot_public_key,
stat_info.st_size,
stat_info.st_mtime,
directory,
)
changed = True
except Exception as e:
tb = traceback.format_exc()
log.error(f"Failed to open file {filename}. {e} {tb}")
failed_to_open_filenames[filename] = int(time.time())
return 0, new_provers
log.info(
f"Found plot {filename} of size {new_provers[filename].prover.get_size()}"
)
if show_memo:
plot_memo: bytes32
if pool_contract_puzzle_hash is None:
plot_memo = stream_plot_info_pk(pool_public_key,
farmer_public_key,
local_master_sk)
else:
plot_memo = stream_plot_info_ph(pool_contract_puzzle_hash,
farmer_public_key,
local_master_sk)
plot_memo_str: str = plot_memo.hex()
log.info(f"Memo: {plot_memo_str}")
return stat_info.st_size, new_provers
return 0, new_provers
def load_plots(
provers: Dict[Path, PlotInfo],
failed_to_open_filenames: Dict[Path, int],
farmer_public_keys: Optional[List[G1Element]],
pool_public_keys: Optional[List[G1Element]],
match_str: Optional[str],
root_path: Path,
open_no_key_filenames=False,
) -> Tuple[bool, Dict[Path, PlotInfo], Dict[Path, int], Set[Path]]:
start_time = time.time()
config_file = load_config(root_path, "config.yaml", "harvester")
changed = False
no_key_filenames: Set[Path] = set()
log.info(f'Searching directories {config_file["plot_directories"]}')
plot_filenames: Dict[Path, List[Path]] = get_plot_filenames(config_file)
all_filenames: List[Path] = []
for paths in plot_filenames.values():
all_filenames += paths
total_size = 0
new_provers: Dict[Path, PlotInfo] = {}
if match_str is not None:
log.info(
f'Only loading plots that contain "{match_str}" in the file or directory name'
)
for filename in all_filenames:
filename_str = str(filename)
if match_str is not None and match_str not in filename_str:
continue
if filename.exists():
if filename in failed_to_open_filenames and (
time.time() - failed_to_open_filenames[filename]) < 1200:
# Try once every 20 minutes to open the file
continue
if filename in provers:
stat_info = filename.stat()
if stat_info.st_mtime == provers[filename].time_modified:
total_size += stat_info.st_size
new_provers[filename] = provers[filename]
continue
try:
prover = DiskProver(str(filename))
expected_size = _expected_plot_size(
prover.get_size()) * UI_ACTUAL_SPACE_CONSTANT_FACTOR / 2.0
stat_info = filename.stat()
# TODO: consider checking if the file was just written to (which would mean that the file is still
# being copied). A segfault might happen in this edge case.
if prover.get_size(
) >= 30 and stat_info.st_size < 0.98 * expected_size:
log.warning(
f"Not farming plot {filename}. Size is {stat_info.st_size / (1024**3)} GiB, but expected"
f" at least: {expected_size / (1024 ** 3)} GiB. We assume the file is being copied."
)
continue
(
pool_public_key,
farmer_public_key,
local_master_sk,
) = parse_plot_info(prover.get_memo())
# Only use plots that correct keys associated with them
if farmer_public_keys is not None and farmer_public_key not in farmer_public_keys:
log.warning(
f"Plot {filename} has a farmer public key that is not in the farmer's pk list."
)
no_key_filenames.add(filename)
if not open_no_key_filenames:
continue
if pool_public_keys is not None and pool_public_key not in pool_public_keys:
log.warning(
f"Plot {filename} has a pool public key that is not in the farmer's pool pk list."
)
no_key_filenames.add(filename)
if not open_no_key_filenames:
continue
stat_info = filename.stat()
local_sk = master_sk_to_local_sk(local_master_sk)
plot_public_key: G1Element = ProofOfSpace.generate_plot_public_key(
local_sk.get_g1(), farmer_public_key)
new_provers[filename] = PlotInfo(
prover,
pool_public_key,
farmer_public_key,
plot_public_key,
local_sk,
stat_info.st_size,
stat_info.st_mtime,
)
total_size += stat_info.st_size
changed = True
except Exception as e:
tb = traceback.format_exc()
log.error(f"Failed to open file {filename}. {e} {tb}")
failed_to_open_filenames[filename] = int(time.time())
continue
log.info(
f"Found plot {filename} of size {new_provers[filename].prover.get_size()}"
)
log.info(
f"Loaded a total of {len(new_provers)} plots of size {total_size / (1024 ** 4)} TiB, in"
f" {time.time()-start_time} seconds")
return changed, new_provers, failed_to_open_filenames, no_key_filenames
def test_k_21(self):
challenge: bytes = bytes([i for i in range(0, 32)])
plot_seed: bytes = bytes(
[
5,
104,
52,
4,
51,
55,
23,
84,
91,
10,
111,
12,
13,
222,
151,
16,
228,
211,
254,
45,
92,
198,
204,
10,
9,
10,
11,
129,
139,
171,
15,
23,
]
)
pl = DiskPlotter()
pl.create_plot_disk(
".", ".", ".", "myplot.dat", 21, bytes([1, 2, 3, 4, 5]), plot_seed, 300, 32, 8192, 8
)
pl = None
pr = DiskProver(str(Path("myplot.dat")))
total_proofs: int = 0
iterations: int = 5000
v = Verifier()
for i in range(iterations):
if i % 100 == 0:
print(i)
challenge = sha256(i.to_bytes(4, "big")).digest()
for index, quality in enumerate(pr.get_qualities_for_challenge(challenge)):
proof = pr.get_full_proof(challenge, index)
assert len(proof) == 8 * pr.get_size()
computed_quality = v.validate_proof(
plot_seed, pr.get_size(), challenge, proof
)
assert computed_quality == quality
total_proofs += 1
print(
f"total proofs {total_proofs} out of {iterations}\
{total_proofs / iterations}"
)
assert total_proofs > 4000
assert total_proofs < 6000
pr = None
sha256_plot_hash = sha256()
with open("myplot.dat", "rb") as f:
# Read and update hash string value in blocks of 4K
for byte_block in iter(lambda: f.read(4096), b""):
sha256_plot_hash.update(byte_block)
plot_hash = str(sha256_plot_hash.hexdigest())
assert (
plot_hash
== "80e32f560f3a4347760d6baae8d16fbaf484948088bff05c51bdcc24b7bc40d9"
)
print(f"\nPlotfile asserted sha256: {plot_hash}\n")
Path("myplot.dat").unlink()
def process_file(file_path: Path) -> Dict:
new_provers: Dict[Path, PlotInfo] = {}
filename_str = str(file_path)
if self.match_str is not None and self.match_str not in filename_str:
return new_provers
if file_path.exists():
if (
file_path in self.failed_to_open_filenames
and (time.time() - self.failed_to_open_filenames[file_path]) > 1200
):
# Try once every 20 minutes to open the file
return new_provers
if file_path in self.plots:
try:
stat_info = file_path.stat()
except Exception as e:
log.error(f"Failed to open file {file_path}. {e}")
return new_provers
if stat_info.st_mtime == self.plots[file_path].time_modified:
new_provers[file_path] = self.plots[file_path]
return new_provers
entry: Optional[Tuple[str, Set[str]]] = self.plot_filename_paths.get(file_path.name)
if entry is not None:
loaded_parent, duplicates = entry
if str(file_path.parent) in duplicates:
log.debug(f"Skip duplicated plot {str(file_path)}")
return new_provers
try:
with counter_lock:
if result.processed_files >= self.refresh_parameter.batch_size:
result.remaining_files += 1
return new_provers
result.processed_files += 1
prover = DiskProver(str(file_path))
log.debug(f"process_file {str(file_path)}")
expected_size = _expected_plot_size(prover.get_size()) * UI_ACTUAL_SPACE_CONSTANT_FACTOR
stat_info = file_path.stat()
# TODO: consider checking if the file was just written to (which would mean that the file is still
# being copied). A segfault might happen in this edge case.
if prover.get_size() >= 30 and stat_info.st_size < 0.98 * expected_size:
log.warning(
f"Not farming plot {file_path}. Size is {stat_info.st_size / (1024**3)} GiB, but expected"
f" at least: {expected_size / (1024 ** 3)} GiB. We assume the file is being copied."
)
return new_provers
(
pool_public_key_or_puzzle_hash,
farmer_public_key,
local_master_sk,
) = parse_plot_info(prover.get_memo())
# Only use plots that correct keys associated with them
if self.farmer_public_keys is not None and farmer_public_key not in self.farmer_public_keys:
log.warning(f"Plot {file_path} has a farmer public key that is not in the farmer's pk list.")
self.no_key_filenames.add(file_path)
if not self.open_no_key_filenames:
return new_provers
if isinstance(pool_public_key_or_puzzle_hash, G1Element):
pool_public_key = pool_public_key_or_puzzle_hash
pool_contract_puzzle_hash = None
else:
assert isinstance(pool_public_key_or_puzzle_hash, bytes32)
pool_public_key = None
pool_contract_puzzle_hash = pool_public_key_or_puzzle_hash
if (
self.pool_public_keys is not None
and pool_public_key is not None
and pool_public_key not in self.pool_public_keys
):
log.warning(f"Plot {file_path} has a pool public key that is not in the farmer's pool pk list.")
self.no_key_filenames.add(file_path)
if not self.open_no_key_filenames:
return new_provers
stat_info = file_path.stat()
local_sk = master_sk_to_local_sk(local_master_sk)
plot_public_key: G1Element = ProofOfSpace.generate_plot_public_key(
local_sk.get_g1(), farmer_public_key, pool_contract_puzzle_hash is not None
)
with self.plot_filename_paths_lock:
if file_path.name not in self.plot_filename_paths:
self.plot_filename_paths[file_path.name] = (str(Path(prover.get_filename()).parent), set())
else:
self.plot_filename_paths[file_path.name][1].add(str(Path(prover.get_filename()).parent))
if len(self.plot_filename_paths[file_path.name][1]) > 0:
log.warning(
f"Have multiple copies of the plot {file_path} in "
f"{self.plot_filename_paths[file_path.name][1]}."
)
return new_provers
new_provers[file_path] = PlotInfo(
prover,
pool_public_key,
pool_contract_puzzle_hash,
plot_public_key,
stat_info.st_size,
stat_info.st_mtime,
)
with counter_lock:
result.loaded_plots += 1
result.loaded_size += stat_info.st_size
except Exception as e:
tb = traceback.format_exc()
log.error(f"Failed to open file {file_path}. {e} {tb}")
self.failed_to_open_filenames[file_path] = int(time.time())
return new_provers
log.info(f"Found plot {file_path} of size {new_provers[file_path].prover.get_size()}")
if self.show_memo:
plot_memo: bytes32
if pool_contract_puzzle_hash is None:
plot_memo = stream_plot_info_pk(pool_public_key, farmer_public_key, local_master_sk)
else:
plot_memo = stream_plot_info_ph(pool_contract_puzzle_hash, farmer_public_key, local_master_sk)
plot_memo_str: str = plot_memo.hex()
log.info(f"Memo: {plot_memo_str}")
return new_provers
return new_provers
