def __init__(self, root_path: Path):
self.plot_manager = PlotManager(root_path, self.refresh_callback)
# Set a very high refresh interval here to avoid unintentional refresh cycles
self.plot_manager.refresh_parameter.interval_seconds = 10000
# Set to the current time to avoid automated refresh after we start below.
self.plot_manager.last_refresh_time = time.time()
self.plot_manager.start_refreshing()
def __init__(self, root_path: Path, config: Dict,
constants: ConsensusConstants):
self.log = log
self.root_path = root_path
# TODO, remove checks below later after some versions / time
refresh_parameter: PlotsRefreshParameter = PlotsRefreshParameter()
if "plot_loading_frequency_seconds" in config:
self.log.info(
"`harvester.plot_loading_frequency_seconds` is deprecated. Consider replacing it with the new section "
"`harvester.plots_refresh_parameter`. See `initial-config.yaml`."
)
if "plots_refresh_parameter" in config:
refresh_parameter = dataclass_from_dict(
PlotsRefreshParameter, config["plots_refresh_parameter"])
self.plot_manager = PlotManager(
root_path,
refresh_parameter=refresh_parameter,
refresh_callback=self._plot_refresh_callback)
self._is_shutdown = False
self.executor = concurrent.futures.ThreadPoolExecutor(
max_workers=config["num_threads"])
self.state_changed_callback = None
self.server = None
self.constants = constants
self.cached_challenges = []
self.state_changed_callback: Optional[Callable] = None
self.parallel_read: bool = config.get("parallel_read", True)
class PlotRefreshTester:
plot_manager: PlotManager
expected_result: PlotRefreshResult
expected_result_matched: bool
def __init__(self, root_path: Path):
self.plot_manager = PlotManager(root_path, self.refresh_callback)
# Set a very high refresh interval here to avoid unintentional refresh cycles
self.plot_manager.refresh_parameter.interval_seconds = 10000
# Set to the current time to avoid automated refresh after we start below.
self.plot_manager.last_refresh_time = time.time()
self.plot_manager.start_refreshing()
def refresh_callback(self, event: PlotRefreshEvents, refresh_result: PlotRefreshResult):
if event != PlotRefreshEvents.done:
# Only validate the final results for this tests
return
for name in ["loaded", "removed", "processed", "remaining"]:
try:
actual_value = refresh_result.__getattribute__(name)
if type(actual_value) == list:
expected_list = self.expected_result.__getattribute__(name)
if len(expected_list) != len(actual_value):
return
values_found = 0
for value in actual_value:
if type(value) == PlotInfo:
for plot_info in expected_list:
if plot_info.prover.get_filename() == value.prover.get_filename():
values_found += 1
continue
else:
if value in expected_list:
values_found += 1
continue
if values_found != len(expected_list):
log.error(f"{name} invalid: values_found {values_found} expected {len(expected_list)}")
return
else:
expected_value = self.expected_result.__getattribute__(name)
if actual_value != expected_value:
log.error(f"{name} invalid: actual {actual_value} expected {expected_value}")
return
except AttributeError as error:
log.error(f"{error}")
return
self.expected_result_matched = True
async def run(self, expected_result: PlotRefreshResult):
self.expected_result = expected_result
self.expected_result_matched = False
self.plot_manager.trigger_refresh()
await time_out_assert(5, self.plot_manager.needs_refresh, value=False)
assert self.expected_result_matched
async def test1(self, simulation):
test_rpc_port = uint16(21522)
test_rpc_port_2 = uint16(21523)
harvester, farmer_api = simulation
def stop_node_cb():
pass
def stop_node_cb_2():
pass
config = bt.config
hostname = config["self_hostname"]
daemon_port = config["daemon_port"]
farmer_rpc_api = FarmerRpcApi(farmer_api.farmer)
harvester_rpc_api = HarvesterRpcApi(harvester)
rpc_cleanup = await start_rpc_server(
farmer_rpc_api,
hostname,
daemon_port,
test_rpc_port,
stop_node_cb,
bt.root_path,
config,
connect_to_daemon=False,
)
rpc_cleanup_2 = await start_rpc_server(
harvester_rpc_api,
hostname,
daemon_port,
test_rpc_port_2,
stop_node_cb_2,
bt.root_path,
config,
connect_to_daemon=False,
)
try:
client = await FarmerRpcClient.create(self_hostname, test_rpc_port, bt.root_path, config)
client_2 = await HarvesterRpcClient.create(self_hostname, test_rpc_port_2, bt.root_path, config)
async def have_connections():
return len(await client.get_connections()) > 0
await time_out_assert(15, have_connections, True)
assert (await client.get_signage_point(std_hash(b"2"))) is None
assert len(await client.get_signage_points()) == 0
async def have_signage_points():
return len(await client.get_signage_points()) > 0
sp = farmer_protocol.NewSignagePoint(
std_hash(b"1"), std_hash(b"2"), std_hash(b"3"), uint64(1), uint64(1000000), uint8(2)
)
await farmer_api.new_signage_point(sp)
await time_out_assert(5, have_signage_points, True)
assert (await client.get_signage_point(std_hash(b"2"))) is not None
async def have_plots():
return len((await client_2.get_plots())["plots"]) > 0
await time_out_assert(5, have_plots, True)
res = await client_2.get_plots()
num_plots = len(res["plots"])
assert num_plots > 0
plot_dir = get_plot_dir() / "subdir"
plot_dir.mkdir(parents=True, exist_ok=True)
plot_dir_sub = get_plot_dir() / "subdir" / "subsubdir"
plot_dir_sub.mkdir(parents=True, exist_ok=True)
plotter = DiskPlotter()
filename = "test_farmer_harvester_rpc_plot.plot"
filename_2 = "test_farmer_harvester_rpc_plot2.plot"
plotter.create_plot_disk(
str(plot_dir),
str(plot_dir),
str(plot_dir),
filename,
18,
stream_plot_info_pk(bt.pool_pk, bt.farmer_pk, AugSchemeMPL.key_gen(bytes([4] * 32))),
token_bytes(32),
128,
0,
2000,
0,
False,
)
# Making a plot with a puzzle hash encoded into it instead of pk
plot_id_2 = token_bytes(32)
plotter.create_plot_disk(
str(plot_dir),
str(plot_dir),
str(plot_dir),
filename_2,
18,
stream_plot_info_ph(std_hash(b"random ph"), bt.farmer_pk, AugSchemeMPL.key_gen(bytes([5] * 32))),
plot_id_2,
128,
0,
2000,
0,
False,
)
# Making the same plot, in a different dir. This should not be farmed
plotter.create_plot_disk(
str(plot_dir_sub),
str(plot_dir_sub),
str(plot_dir_sub),
filename_2,
18,
stream_plot_info_ph(std_hash(b"random ph"), bt.farmer_pk, AugSchemeMPL.key_gen(bytes([5] * 32))),
plot_id_2,
128,
0,
2000,
0,
False,
)
res_2 = await client_2.get_plots()
assert len(res_2["plots"]) == num_plots
# Test farmer get_harvesters
async def test_get_harvesters():
farmer_res = await client.get_harvesters()
if len(list(farmer_res["harvesters"])) != 1:
return False
if len(list(farmer_res["harvesters"][0]["plots"])) != num_plots:
return False
return True
await time_out_assert(30, test_get_harvesters)
expected_result: PlotRefreshResult = PlotRefreshResult()
def test_refresh_callback(refresh_result: PlotRefreshResult):
assert refresh_result.loaded_plots == expected_result.loaded_plots
assert refresh_result.removed_plots == expected_result.removed_plots
assert refresh_result.processed_files == expected_result.processed_files
assert refresh_result.remaining_files == expected_result.remaining_files
harvester.plot_manager.set_refresh_callback(test_refresh_callback)
async def test_case(
trigger, expect_loaded, expect_removed, expect_processed, expected_directories, expect_total_plots
):
expected_result.loaded_plots = expect_loaded
expected_result.removed_plots = expect_removed
expected_result.processed_files = expect_processed
await trigger
harvester.plot_manager.trigger_refresh()
assert len(await client_2.get_plot_directories()) == expected_directories
await time_out_assert(5, harvester.plot_manager.needs_refresh, value=False)
result = await client_2.get_plots()
assert len(result["plots"]) == expect_total_plots
assert len(harvester.plot_manager.cache) == expect_total_plots
assert len(harvester.plot_manager.failed_to_open_filenames) == 0
# Add plot_dir with two new plots
await test_case(
client_2.add_plot_directory(str(plot_dir)),
expect_loaded=2,
expect_removed=0,
expect_processed=2,
expected_directories=2,
expect_total_plots=num_plots + 2,
)
# Add plot_dir_sub with one duplicate
await test_case(
client_2.add_plot_directory(str(plot_dir_sub)),
expect_loaded=0,
expect_removed=0,
expect_processed=1,
expected_directories=3,
expect_total_plots=num_plots + 2,
)
# Delete one plot
await test_case(
client_2.delete_plot(str(plot_dir / filename)),
expect_loaded=0,
expect_removed=1,
expect_processed=0,
expected_directories=3,
expect_total_plots=num_plots + 1,
)
# Remove directory with the duplicate
await test_case(
client_2.remove_plot_directory(str(plot_dir_sub)),
expect_loaded=0,
expect_removed=1,
expect_processed=0,
expected_directories=2,
expect_total_plots=num_plots + 1,
)
# Re-add the directory with the duplicate for other tests
await test_case(
client_2.add_plot_directory(str(plot_dir_sub)),
expect_loaded=0,
expect_removed=0,
expect_processed=1,
expected_directories=3,
expect_total_plots=num_plots + 1,
)
# Remove the directory which has the duplicated plot loaded. This removes the duplicated plot from plot_dir
# and in the same run loads the plot from plot_dir_sub which is not longer seen as duplicate.
await test_case(
client_2.remove_plot_directory(str(plot_dir)),
expect_loaded=1,
expect_removed=1,
expect_processed=1,
expected_directories=2,
expect_total_plots=num_plots + 1,
)
# Re-add the directory now the plot seen as duplicate is from plot_dir, not from plot_dir_sub like before
await test_case(
client_2.add_plot_directory(str(plot_dir)),
expect_loaded=0,
expect_removed=0,
expect_processed=1,
expected_directories=3,
expect_total_plots=num_plots + 1,
)
# Remove the duplicated plot
await test_case(
client_2.delete_plot(str(plot_dir / filename_2)),
expect_loaded=0,
expect_removed=1,
expect_processed=0,
expected_directories=3,
expect_total_plots=num_plots + 1,
)
# Remove the directory with the loaded plot which is not longer a duplicate
await test_case(
client_2.remove_plot_directory(str(plot_dir_sub)),
expect_loaded=0,
expect_removed=1,
expect_processed=0,
expected_directories=2,
expect_total_plots=num_plots,
)
# Remove the directory which contains all other plots
await test_case(
client_2.remove_plot_directory(str(get_plot_dir())),
expect_loaded=0,
expect_removed=20,
expect_processed=0,
expected_directories=1,
expect_total_plots=0,
)
# Recover the plots to test caching
# First make sure cache gets written if required and new plots are loaded
await test_case(
client_2.add_plot_directory(str(get_plot_dir())),
expect_loaded=20,
expect_removed=0,
expect_processed=20,
expected_directories=2,
expect_total_plots=20,
)
assert harvester.plot_manager.cache.path().exists()
unlink(harvester.plot_manager.cache.path())
# Should not write the cache again on shutdown because it didn't change
assert not harvester.plot_manager.cache.path().exists()
harvester.plot_manager.stop_refreshing()
assert not harvester.plot_manager.cache.path().exists()
# Manually trigger `save_cache` and make sure it creates a new cache file
harvester.plot_manager.cache.save()
assert harvester.plot_manager.cache.path().exists()
expected_result.loaded_plots = 20
expected_result.removed_plots = 0
expected_result.processed_files = 20
expected_result.remaining_files = 0
plot_manager: PlotManager = PlotManager(harvester.root_path, test_refresh_callback)
plot_manager.start_refreshing()
assert len(harvester.plot_manager.cache) == len(plot_manager.cache)
await time_out_assert(5, plot_manager.needs_refresh, value=False)
for path, plot_info in harvester.plot_manager.plots.items():
assert path in plot_manager.plots
assert plot_manager.plots[path].prover.get_filename() == plot_info.prover.get_filename()
assert plot_manager.plots[path].prover.get_id() == plot_info.prover.get_id()
assert plot_manager.plots[path].prover.get_memo() == plot_info.prover.get_memo()
assert plot_manager.plots[path].prover.get_size() == plot_info.prover.get_size()
assert plot_manager.plots[path].pool_public_key == plot_info.pool_public_key
assert plot_manager.plots[path].pool_contract_puzzle_hash == plot_info.pool_contract_puzzle_hash
assert plot_manager.plots[path].plot_public_key == plot_info.plot_public_key
assert plot_manager.plots[path].file_size == plot_info.file_size
assert plot_manager.plots[path].time_modified == plot_info.time_modified
assert harvester.plot_manager.plot_filename_paths == plot_manager.plot_filename_paths
assert harvester.plot_manager.failed_to_open_filenames == plot_manager.failed_to_open_filenames
assert harvester.plot_manager.no_key_filenames == plot_manager.no_key_filenames
plot_manager.stop_refreshing()
# Modify the content of the plot_manager.dat
with open(harvester.plot_manager.cache.path(), "r+b") as file:
file.write(b"\xff\xff") # Sets Cache.version to 65535
# Make sure it just loads the plots normally if it fails to load the cache
plot_manager = PlotManager(harvester.root_path, test_refresh_callback)
plot_manager.cache.load()
assert len(plot_manager.cache) == 0
plot_manager.set_public_keys(
harvester.plot_manager.farmer_public_keys, harvester.plot_manager.pool_public_keys
)
expected_result.loaded_plots = 20
expected_result.removed_plots = 0
expected_result.processed_files = 20
expected_result.remaining_files = 0
plot_manager.start_refreshing()
await time_out_assert(5, plot_manager.needs_refresh, value=False)
assert len(plot_manager.plots) == len(harvester.plot_manager.plots)
plot_manager.stop_refreshing()
# Test re-trying if processing a plot failed
# First save the plot
retry_test_plot = Path(plot_dir_sub / filename_2).resolve()
retry_test_plot_save = Path(plot_dir_sub / "save").resolve()
copy(retry_test_plot, retry_test_plot_save)
# Invalidate the plot
with open(plot_dir_sub / filename_2, "r+b") as file:
file.write(bytes(100))
# Add it and validate it fails to load
await harvester.add_plot_directory(str(plot_dir_sub))
expected_result.loaded_plots = 0
expected_result.removed_plots = 0
expected_result.processed_files = 1
expected_result.remaining_files = 0
harvester.plot_manager.start_refreshing()
await time_out_assert(5, harvester.plot_manager.needs_refresh, value=False)
assert retry_test_plot in harvester.plot_manager.failed_to_open_filenames
# Make sure the file stays in `failed_to_open_filenames` and doesn't get loaded or processed in the next
# update round
expected_result.loaded_plots = 0
expected_result.processed_files = 0
harvester.plot_manager.trigger_refresh()
await time_out_assert(5, harvester.plot_manager.needs_refresh, value=False)
assert retry_test_plot in harvester.plot_manager.failed_to_open_filenames
# Now decrease the re-try timeout, restore the valid plot file and make sure it properly loads now
harvester.plot_manager.refresh_parameter.retry_invalid_seconds = 0
move(retry_test_plot_save, retry_test_plot)
expected_result.loaded_plots = 1
expected_result.processed_files = 1
harvester.plot_manager.trigger_refresh()
await time_out_assert(5, harvester.plot_manager.needs_refresh, value=False)
assert retry_test_plot not in harvester.plot_manager.failed_to_open_filenames
targets_1 = await client.get_reward_targets(False)
assert "have_pool_sk" not in targets_1
assert "have_farmer_sk" not in targets_1
targets_2 = await client.get_reward_targets(True)
assert targets_2["have_pool_sk"] and targets_2["have_farmer_sk"]
new_ph: bytes32 = create_puzzlehash_for_pk(master_sk_to_wallet_sk(bt.farmer_master_sk, uint32(10)).get_g1())
new_ph_2: bytes32 = create_puzzlehash_for_pk(
master_sk_to_wallet_sk(bt.pool_master_sk, uint32(472)).get_g1()
)
await client.set_reward_targets(encode_puzzle_hash(new_ph, "xch"), encode_puzzle_hash(new_ph_2, "xch"))
targets_3 = await client.get_reward_targets(True)
assert decode_puzzle_hash(targets_3["farmer_target"]) == new_ph
assert decode_puzzle_hash(targets_3["pool_target"]) == new_ph_2
assert targets_3["have_pool_sk"] and targets_3["have_farmer_sk"]
new_ph_3: bytes32 = create_puzzlehash_for_pk(
master_sk_to_wallet_sk(bt.pool_master_sk, uint32(1888)).get_g1()
)
await client.set_reward_targets(None, encode_puzzle_hash(new_ph_3, "xch"))
targets_4 = await client.get_reward_targets(True)
assert decode_puzzle_hash(targets_4["farmer_target"]) == new_ph
assert decode_puzzle_hash(targets_4["pool_target"]) == new_ph_3
assert not targets_4["have_pool_sk"] and targets_3["have_farmer_sk"]
root_path = farmer_api.farmer._root_path
config = load_config(root_path, "config.yaml")
assert config["farmer"]["xch_target_address"] == encode_puzzle_hash(new_ph, "xch")
assert config["pool"]["xch_target_address"] == encode_puzzle_hash(new_ph_3, "xch")
new_ph_3_encoded = encode_puzzle_hash(new_ph_3, "xch")
added_char = new_ph_3_encoded + "a"
with pytest.raises(ValueError):
await client.set_reward_targets(None, added_char)
replaced_char = new_ph_3_encoded[0:-1] + "a"
with pytest.raises(ValueError):
await client.set_reward_targets(None, replaced_char)
assert len((await client.get_pool_state())["pool_state"]) == 0
all_sks = farmer_api.farmer.local_keychain.get_all_private_keys()
auth_sk = master_sk_to_pooling_authentication_sk(all_sks[0][0], 2, 1)
pool_list = [
{
"launcher_id": "ae4ef3b9bfe68949691281a015a9c16630fc8f66d48c19ca548fb80768791afa",
"authentication_public_key": bytes(auth_sk.get_g1()).hex(),
"owner_public_key": "84c3fcf9d5581c1ddc702cb0f3b4a06043303b334dd993ab42b2c320ebfa98e5ce558448615b3f69638ba92cf7f43da5",
"payout_instructions": "c2b08e41d766da4116e388357ed957d04ad754623a915f3fd65188a8746cf3e8",
"pool_url": "localhost",
"p2_singleton_puzzle_hash": "16e4bac26558d315cded63d4c5860e98deb447cc59146dd4de06ce7394b14f17",
"target_puzzle_hash": "344587cf06a39db471d2cc027504e8688a0a67cce961253500c956c73603fd58",
}
]
config["pool"]["pool_list"] = pool_list
save_config(root_path, "config.yaml", config)
await farmer_api.farmer.update_pool_state()
pool_state = (await client.get_pool_state())["pool_state"]
assert len(pool_state) == 1
assert (
pool_state[0]["pool_config"]["payout_instructions"]
== "c2b08e41d766da4116e388357ed957d04ad754623a915f3fd65188a8746cf3e8"
)
await client.set_payout_instructions(hexstr_to_bytes(pool_state[0]["pool_config"]["launcher_id"]), "1234vy")
await farmer_api.farmer.update_pool_state()
pool_state = (await client.get_pool_state())["pool_state"]
assert pool_state[0]["pool_config"]["payout_instructions"] == "1234vy"
finally:
# Checks that the RPC manages to stop the node
client.close()
client_2.close()
await client.await_closed()
await client_2.await_closed()
await rpc_cleanup()
await rpc_cleanup_2()
def check_plots(root_path, num, challenge_start, grep_string, list_duplicates, debug_show_memo):
config = load_config(root_path, "config.yaml")
plot_refresh_parameter: PlotsRefreshParameter = PlotsRefreshParameter(100, 100, 1)
plot_manager: PlotManager = PlotManager(
root_path,
match_str=grep_string,
show_memo=debug_show_memo,
open_no_key_filenames=True,
refresh_parameter=plot_refresh_parameter,
refresh_callback=plot_refresh_callback,
)
if num is not None:
if num == 0:
log.warning("Not opening plot files")
else:
if num < 5:
log.warning(f"{num} challenges is too low, setting it to the minimum of 5")
num = 5
if num < 30:
log.warning("Use 30 challenges (our default) for balance of speed and accurate results")
else:
num = 30
if challenge_start is not None:
num_start = challenge_start
num_end = num_start + num
else:
num_start = 0
num_end = num
challenges = num_end - num_start
if list_duplicates:
log.warning("Checking for duplicate Plot IDs")
log.info("Plot filenames expected to end with -[64 char plot ID].plot")
if list_duplicates:
all_filenames: List[Path] = []
for paths in get_plot_filenames(root_path).values():
all_filenames += paths
find_duplicate_plot_IDs(all_filenames)
if num == 0:
return None
parallel_read: bool = config["harvester"].get("parallel_read", True)
v = Verifier()
log.info(f"Loading plots in config.yaml using plot_manager loading code (parallel read: {parallel_read})\n")
# Prompts interactively if the keyring is protected by a master passphrase. To use the daemon
# for keychain access, KeychainProxy/connect_to_keychain should be used instead of Keychain.
kc: Keychain = Keychain()
plot_manager.set_public_keys(
[master_sk_to_farmer_sk(sk).get_g1() for sk, _ in kc.get_all_private_keys()],
[G1Element.from_bytes(bytes.fromhex(pk)) for pk in config["farmer"]["pool_public_keys"]],
)
plot_manager.start_refreshing()
while plot_manager.needs_refresh():
sleep(1)
plot_manager.stop_refreshing()
if plot_manager.plot_count() > 0:
log.info("")
log.info("")
log.info(f"Starting to test each plot with {num} challenges each\n")
total_good_plots: Counter = Counter()
total_bad_plots = 0
total_size = 0
bad_plots_list: List[Path] = []
with plot_manager:
for plot_path, plot_info in plot_manager.plots.items():
pr = plot_info.prover
log.info(f"Testing plot {plot_path} k={pr.get_size()}")
log.info(f"\tPool public key: {plot_info.pool_public_key}")
# Look up local_sk from plot to save locked memory
(
pool_public_key_or_puzzle_hash,
farmer_public_key,
local_master_sk,
) = parse_plot_info(pr.get_memo())
local_sk = master_sk_to_local_sk(local_master_sk)
log.info(f"\tFarmer public key: {farmer_public_key}")
log.info(f"\tLocal sk: {local_sk}")
total_proofs = 0
caught_exception: bool = False
for i in range(num_start, num_end):
challenge = std_hash(i.to_bytes(32, "big"))
# Some plot errors cause get_qualities_for_challenge to throw a RuntimeError
try:
quality_start_time = int(round(time() * 1000))
for index, quality_str in enumerate(pr.get_qualities_for_challenge(challenge)):
quality_spent_time = int(round(time() * 1000)) - quality_start_time
if quality_spent_time > 5000:
log.warning(
f"\tLooking up qualities took: {quality_spent_time} ms. This should be below 5 seconds "
f"to minimize risk of losing rewards."
)
else:
log.info(f"\tLooking up qualities took: {quality_spent_time} ms.")
# Other plot errors cause get_full_proof or validate_proof to throw an AssertionError
try:
proof_start_time = int(round(time() * 1000))
proof = pr.get_full_proof(challenge, index, parallel_read)
proof_spent_time = int(round(time() * 1000)) - proof_start_time
if proof_spent_time > 15000:
log.warning(
f"\tFinding proof took: {proof_spent_time} ms. This should be below 15 seconds "
f"to minimize risk of losing rewards."
)
else:
log.info(f"\tFinding proof took: {proof_spent_time} ms")
total_proofs += 1
ver_quality_str = v.validate_proof(pr.get_id(), pr.get_size(), challenge, proof)
assert quality_str == ver_quality_str
except AssertionError as e:
log.error(f"{type(e)}: {e} error in proving/verifying for plot {plot_path}")
caught_exception = True
quality_start_time = int(round(time() * 1000))
except KeyboardInterrupt:
log.warning("Interrupted, closing")
return None
except SystemExit:
log.warning("System is shutting down.")
return None
except Exception as e:
log.error(f"{type(e)}: {e} error in getting challenge qualities for plot {plot_path}")
caught_exception = True
if caught_exception is True:
break
if total_proofs > 0 and caught_exception is False:
log.info(f"\tProofs {total_proofs} / {challenges}, {round(total_proofs/float(challenges), 4)}")
total_good_plots[pr.get_size()] += 1
total_size += plot_path.stat().st_size
else:
total_bad_plots += 1
log.error(f"\tProofs {total_proofs} / {challenges}, {round(total_proofs/float(challenges), 4)}")
bad_plots_list.append(plot_path)
log.info("")
log.info("")
log.info("Summary")
total_plots: int = sum(list(total_good_plots.values()))
log.info(f"Found {total_plots} valid plots, total size {total_size / (1024 * 1024 * 1024 * 1024):.5f} TiB")
for (k, count) in sorted(dict(total_good_plots).items()):
log.info(f"{count} plots of size {k}")
grand_total_bad = total_bad_plots + len(plot_manager.failed_to_open_filenames)
if grand_total_bad > 0:
log.warning(f"{grand_total_bad} invalid plots found:")
for bad_plot_path in bad_plots_list:
log.warning(f"{bad_plot_path}")
if len(plot_manager.no_key_filenames) > 0:
log.warning(
f"There are {len(plot_manager.no_key_filenames)} plots with a farmer or pool public key that "
f"is not on this machine. The farmer private key must be in the keychain in order to "
f"farm them, use 'chia keys' to transfer keys. The pool public keys must be in the config.yaml"
)
class Harvester:
plot_manager: PlotManager
root_path: Path
_is_shutdown: bool
executor: ThreadPoolExecutor
state_changed_callback: Optional[Callable]
cached_challenges: List
constants: ConsensusConstants
_refresh_lock: asyncio.Lock
event_loop: asyncio.events.AbstractEventLoop
def __init__(self, root_path: Path, config: Dict,
constants: ConsensusConstants):
self.log = log
self.root_path = root_path
# TODO, remove checks below later after some versions / time
refresh_parameter: PlotsRefreshParameter = PlotsRefreshParameter()
if "plot_loading_frequency_seconds" in config:
self.log.info(
"`harvester.plot_loading_frequency_seconds` is deprecated. Consider replacing it with the new section "
"`harvester.plots_refresh_parameter`. See `initial-config.yaml`."
)
if "plots_refresh_parameter" in config:
refresh_parameter = dataclass_from_dict(
PlotsRefreshParameter, config["plots_refresh_parameter"])
self.plot_manager = PlotManager(
root_path,
refresh_parameter=refresh_parameter,
refresh_callback=self._plot_refresh_callback)
self._is_shutdown = False
self.executor = concurrent.futures.ThreadPoolExecutor(
max_workers=config["num_threads"])
self.state_changed_callback = None
self.server = None
self.constants = constants
self.cached_challenges = []
self.state_changed_callback: Optional[Callable] = None
self.parallel_read: bool = config.get("parallel_read", True)
async def _start(self):
self._refresh_lock = asyncio.Lock()
self.event_loop = asyncio.get_event_loop()
def _close(self):
self._is_shutdown = True
self.executor.shutdown(wait=True)
self.plot_manager.stop_refreshing()
async def _await_closed(self):
pass
def _set_state_changed_callback(self, callback: Callable):
self.state_changed_callback = callback
def _state_changed(self, change: str):
if self.state_changed_callback is not None:
self.state_changed_callback(change)
def _plot_refresh_callback(self, update_result: PlotRefreshResult):
self.log.info(
f"refresh_batch: loaded_plots {update_result.loaded_plots}, "
f"loaded_size {update_result.loaded_size / (1024 ** 4):.2f} TiB, "
f"removed_plots {update_result.removed_plots}, processed_plots {update_result.processed_files}, "
f"remaining_plots {update_result.remaining_files}, "
f"duration: {update_result.duration:.2f} seconds")
if update_result.loaded_plots > 0:
self.event_loop.call_soon_threadsafe(self._state_changed, "plots")
def on_disconnect(self, connection: ws.WSChiaConnection):
self.log.info(f"peer disconnected {connection.get_peer_logging()}")
self._state_changed("close_connection")
def get_plots(self) -> Tuple[List[Dict], List[str], List[str]]:
self.log.debug(
f"get_plots prover items: {self.plot_manager.plot_count()}")
response_plots: List[Dict] = []
with self.plot_manager:
for path, plot_info in self.plot_manager.plots.items():
prover = plot_info.prover
response_plots.append({
"filename": str(path),
"size": prover.get_size(),
"plot-seed": prover.get_id(), # Deprecated
"plot_id": prover.get_id(),
"pool_public_key": plot_info.pool_public_key,
"pool_contract_puzzle_hash":
plot_info.pool_contract_puzzle_hash,
"plot_public_key": plot_info.plot_public_key,
"file_size": plot_info.file_size,
"time_modified": plot_info.time_modified,
})
self.log.debug(
f"get_plots response: plots: {len(response_plots)}, "
f"failed_to_open_filenames: {len(self.plot_manager.failed_to_open_filenames)}, "
f"no_key_filenames: {len(self.plot_manager.no_key_filenames)}")
return (
response_plots,
[
str(s) for s, _ in
self.plot_manager.failed_to_open_filenames.items()
],
[str(s) for s in self.plot_manager.no_key_filenames],
)
def delete_plot(self, str_path: str):
remove_plot(Path(str_path))
self.plot_manager.trigger_refresh()
self._state_changed("plots")
return True
async def add_plot_directory(self, str_path: str) -> bool:
add_plot_directory(self.root_path, str_path)
self.plot_manager.trigger_refresh()
return True
async def get_plot_directories(self) -> List[str]:
return get_plot_directories(self.root_path)
async def remove_plot_directory(self, str_path: str) -> bool:
remove_plot_directory(self.root_path, str_path)
self.plot_manager.trigger_refresh()
return True
def set_server(self, server):
self.server = server