async def test_keys_missing(test_environment: TestEnvironment) -> None:
env: TestEnvironment = test_environment
not_in_keychain_plots: List[Path] = get_test_plots("not_in_keychain")
dir_not_in_keychain: TestDirectory = TestDirectory(
env.root_path / "plots" / "not_in_keychain", not_in_keychain_plots
)
expected_result = PlotRefreshResult()
# The plots in "not_in_keychain" directory have infinity g1 elements as farmer/pool key so they should be plots
# with missing keys for now
add_plot_directory(env.root_path, str(dir_not_in_keychain.path))
expected_result.loaded = []
expected_result.removed = []
expected_result.processed = len(dir_not_in_keychain)
expected_result.remaining = 0
for i in range(2):
await env.refresh_tester.run(expected_result)
assert len(env.refresh_tester.plot_manager.no_key_filenames) == len(dir_not_in_keychain)
for path in env.refresh_tester.plot_manager.no_key_filenames:
assert path in dir_not_in_keychain.plots
# Delete one of the plots and make sure it gets dropped from the no key filenames list
drop_plot = dir_not_in_keychain.path_list()[0]
dir_not_in_keychain.drop(drop_plot)
drop_plot.unlink()
assert drop_plot in env.refresh_tester.plot_manager.no_key_filenames
expected_result.processed -= 1
await env.refresh_tester.run(expected_result)
assert drop_plot not in env.refresh_tester.plot_manager.no_key_filenames
# Now add the missing keys to the plot manager's key lists and make sure the plots are getting loaded
env.refresh_tester.plot_manager.farmer_public_keys.append(G1Element())
env.refresh_tester.plot_manager.pool_public_keys.append(G1Element())
expected_result.loaded = dir_not_in_keychain.plot_info_list() # type: ignore[assignment]
expected_result.processed = len(dir_not_in_keychain)
await env.refresh_tester.run(expected_result)
# And make sure they are dropped from the list of plots with missing keys
assert len(env.refresh_tester.plot_manager.no_key_filenames) == 0
async def test_callback_event_raises(test_environment,
event_to_raise: PlotRefreshEvents):
last_event_fired: Optional[PlotRefreshEvents] = None
def raising_callback(event: PlotRefreshEvents, _: PlotRefreshResult):
nonlocal last_event_fired
last_event_fired = event
if event == event_to_raise:
raise Exception(f"run_raise_in_callback {event_to_raise}")
env: TestEnvironment = test_environment
expected_result = PlotRefreshResult()
# Load dir_1
add_plot_directory(env.root_path, str(env.dir_1.path))
expected_result.loaded = env.dir_1.plot_info_list(
) # type: ignore[assignment]
expected_result.removed = []
expected_result.processed = len(env.dir_1)
expected_result.remaining = 0
await env.refresh_tester.run(expected_result)
# Load dir_2
add_plot_directory(env.root_path, str(env.dir_2.path))
expected_result.loaded = env.dir_2.plot_info_list(
) # type: ignore[assignment]
expected_result.removed = []
expected_result.processed = len(env.dir_1) + len(env.dir_2)
expected_result.remaining = 0
await env.refresh_tester.run(expected_result)
# Now raise the exception in the callback
default_callback = env.refresh_tester.plot_manager._refresh_callback
env.refresh_tester.plot_manager.set_refresh_callback(raising_callback)
env.refresh_tester.plot_manager.start_refreshing()
env.refresh_tester.plot_manager.trigger_refresh()
await time_out_assert(5,
env.refresh_tester.plot_manager.needs_refresh,
value=False)
# And make sure the follow-up evens aren't fired
assert last_event_fired == event_to_raise
# The exception should trigger `PlotManager.reset()` and clear the plots
assert len(env.refresh_tester.plot_manager.plots) == 0
assert len(env.refresh_tester.plot_manager.plot_filename_paths) == 0
assert len(env.refresh_tester.plot_manager.failed_to_open_filenames) == 0
assert len(env.refresh_tester.plot_manager.no_key_filenames) == 0
# The next run without the valid callback should lead to re-loading of all plot
env.refresh_tester.plot_manager.set_refresh_callback(default_callback)
expected_result.loaded = env.dir_1.plot_info_list(
) + env.dir_2.plot_info_list() # type: ignore[assignment]
expected_result.removed = []
expected_result.processed = len(env.dir_1) + len(env.dir_2)
expected_result.remaining = 0
await env.refresh_tester.run(expected_result)
async def test_plot_info_caching(test_environment):
env: TestEnvironment = test_environment
expected_result = PlotRefreshResult()
add_plot_directory(env.root_path, str(env.dir_1.path))
expected_result.loaded = env.dir_1.plot_info_list()
expected_result.removed = []
expected_result.processed = len(env.dir_1)
expected_result.remaining = 0
await env.refresh_tester.run(expected_result)
assert env.refresh_tester.plot_manager.cache.path().exists()
unlink(env.refresh_tester.plot_manager.cache.path())
# Should not write the cache again on shutdown because it didn't change
assert not env.refresh_tester.plot_manager.cache.path().exists()
env.refresh_tester.plot_manager.stop_refreshing()
assert not env.refresh_tester.plot_manager.cache.path().exists()
# Manually trigger `save_cache` and make sure it creates a new cache file
env.refresh_tester.plot_manager.cache.save()
assert env.refresh_tester.plot_manager.cache.path().exists()
refresh_tester: PlotRefreshTester = PlotRefreshTester(env.root_path)
plot_manager = refresh_tester.plot_manager
plot_manager.cache.load()
assert len(plot_manager.cache) == len(plot_manager.cache)
await refresh_tester.run(expected_result)
for path, plot_info in plot_manager.plots.items():
assert path in plot_manager.plots
assert plot_manager.plots[path].prover.get_filename() == plot_info.prover.get_filename()
assert plot_manager.plots[path].prover.get_id() == plot_info.prover.get_id()
assert plot_manager.plots[path].prover.get_memo() == plot_info.prover.get_memo()
assert plot_manager.plots[path].prover.get_size() == plot_info.prover.get_size()
assert plot_manager.plots[path].pool_public_key == plot_info.pool_public_key
assert plot_manager.plots[path].pool_contract_puzzle_hash == plot_info.pool_contract_puzzle_hash
assert plot_manager.plots[path].plot_public_key == plot_info.plot_public_key
assert plot_manager.plots[path].file_size == plot_info.file_size
assert plot_manager.plots[path].time_modified == plot_info.time_modified
assert plot_manager.plot_filename_paths == plot_manager.plot_filename_paths
assert plot_manager.failed_to_open_filenames == plot_manager.failed_to_open_filenames
assert plot_manager.no_key_filenames == plot_manager.no_key_filenames
plot_manager.stop_refreshing()
# Modify the content of the plot_manager.dat
with open(plot_manager.cache.path(), "r+b") as file:
file.write(b"\xff\xff") # Sets Cache.version to 65535
# Make sure it just loads the plots normally if it fails to load the cache
refresh_tester: PlotRefreshTester = PlotRefreshTester(env.root_path)
plot_manager = refresh_tester.plot_manager
plot_manager.cache.load()
assert len(plot_manager.cache) == 0
plot_manager.set_public_keys(bt.plot_manager.farmer_public_keys, bt.plot_manager.pool_public_keys)
await refresh_tester.run(expected_result)
assert len(plot_manager.plots) == len(plot_manager.plots)
plot_manager.stop_refreshing()
async def test_invalid_plots(test_environment):
env: TestEnvironment = test_environment
expected_result = PlotRefreshResult()
# Test re-trying if processing a plot failed
# First create a backup of the plot
retry_test_plot = env.dir_1.path_list()[0].resolve()
retry_test_plot_save = Path(env.dir_1.path / ".backup").resolve()
copy(retry_test_plot, retry_test_plot_save)
# Invalidate the plot
with open(retry_test_plot, "r+b") as file:
file.write(bytes(100))
# Add it and validate it fails to load
add_plot_directory(env.root_path, str(env.dir_1.path))
expected_result.loaded = env.dir_1.plot_info_list()[1:]
expected_result.removed = []
expected_result.processed = len(env.dir_1)
expected_result.remaining = 0
await env.refresh_tester.run(expected_result)
assert len(env.refresh_tester.plot_manager.failed_to_open_filenames) == 1
assert retry_test_plot in env.refresh_tester.plot_manager.failed_to_open_filenames
# Give it a non .plot ending and make sure it gets removed from the invalid list on the next refresh
retry_test_plot_unload = Path(env.dir_1.path / ".unload").resolve()
move(retry_test_plot, retry_test_plot_unload)
expected_result.processed -= 1
expected_result.loaded = []
await env.refresh_tester.run(expected_result)
assert len(env.refresh_tester.plot_manager.failed_to_open_filenames) == 0
assert retry_test_plot not in env.refresh_tester.plot_manager.failed_to_open_filenames
# Recover the name and make sure it reappears in the invalid list
move(retry_test_plot_unload, retry_test_plot)
expected_result.processed += 1
await env.refresh_tester.run(expected_result)
assert len(env.refresh_tester.plot_manager.failed_to_open_filenames) == 1
assert retry_test_plot in env.refresh_tester.plot_manager.failed_to_open_filenames
# Make sure the file stays in `failed_to_open_filenames` and doesn't get loaded in the next refresh cycle
expected_result.loaded = []
expected_result.processed = len(env.dir_1)
await env.refresh_tester.run(expected_result)
assert len(env.refresh_tester.plot_manager.failed_to_open_filenames) == 1
assert retry_test_plot in env.refresh_tester.plot_manager.failed_to_open_filenames
# Now decrease the re-try timeout, restore the valid plot file and make sure it properly loads now
env.refresh_tester.plot_manager.refresh_parameter.retry_invalid_seconds = 0
move(retry_test_plot_save, retry_test_plot)
expected_result.loaded = env.dir_1.plot_info_list()[0:1]
expected_result.processed = len(env.dir_1)
await env.refresh_tester.run(expected_result)
assert len(env.refresh_tester.plot_manager.failed_to_open_filenames) == 0
assert retry_test_plot not in env.refresh_tester.plot_manager.failed_to_open_filenames
def add_cmd(ctx: click.Context, final_dir: str):
from chia.plotting.util import add_plot_directory
add_plot_directory(ctx.obj["root_path"], final_dir)
async def add_plot_directory(self, str_path: str) -> bool:
add_plot_directory(self.root_path, str_path)
self.plot_manager.trigger_refresh()
return True
async def create_plots(args, keys: PlotKeys, root_path, use_datetime=True, test_private_keys: Optional[List] = None):
config_filename = config_path_for_filename(root_path, "config.yaml")
config = load_config(root_path, config_filename)
if args.tmp2_dir is None:
args.tmp2_dir = args.tmp_dir
assert (keys.pool_public_key is None) != (keys.pool_contract_puzzle_hash is None)
num = args.num
if args.size < config["min_mainnet_k_size"] and test_private_keys is None:
log.warning(f"Creating plots with size k={args.size}, which is less than the minimum required for mainnet")
if args.size < 22:
log.warning("k under 22 is not supported. Increasing k to 22")
args.size = 22
if keys.pool_public_key is not None:
log.info(
f"Creating {num} plots of size {args.size}, pool public key: "
f"{bytes(keys.pool_public_key).hex()} farmer public key: {bytes(keys.farmer_public_key).hex()}"
)
else:
assert keys.pool_contract_puzzle_hash is not None
log.info(
f"Creating {num} plots of size {args.size}, pool contract address: "
f"{keys.pool_contract_address} farmer public key: {bytes(keys.farmer_public_key).hex()}"
)
tmp_dir_created = False
if not args.tmp_dir.exists():
mkdir(args.tmp_dir)
tmp_dir_created = True
tmp2_dir_created = False
if not args.tmp2_dir.exists():
mkdir(args.tmp2_dir)
tmp2_dir_created = True
mkdir(args.final_dir)
finished_filenames = []
for i in range(num):
# Generate a random master secret key
if test_private_keys is not None:
assert len(test_private_keys) == num
sk: PrivateKey = test_private_keys[i]
else:
sk = AugSchemeMPL.key_gen(token_bytes(32))
# The plot public key is the combination of the harvester and farmer keys
# New plots will also include a taproot of the keys, for extensibility
include_taproot: bool = keys.pool_contract_puzzle_hash is not None
plot_public_key = ProofOfSpace.generate_plot_public_key(
master_sk_to_local_sk(sk).get_g1(), keys.farmer_public_key, include_taproot
)
# The plot id is based on the harvester, farmer, and pool keys
if keys.pool_public_key is not None:
plot_id: bytes32 = ProofOfSpace.calculate_plot_id_pk(keys.pool_public_key, plot_public_key)
plot_memo: bytes32 = stream_plot_info_pk(keys.pool_public_key, keys.farmer_public_key, sk)
else:
assert keys.pool_contract_puzzle_hash is not None
plot_id = ProofOfSpace.calculate_plot_id_ph(keys.pool_contract_puzzle_hash, plot_public_key)
plot_memo = stream_plot_info_ph(keys.pool_contract_puzzle_hash, keys.farmer_public_key, sk)
if args.plotid is not None:
log.info(f"Debug plot ID: {args.plotid}")
plot_id = bytes32(bytes.fromhex(args.plotid))
if args.memo is not None:
log.info(f"Debug memo: {args.memo}")
plot_memo = bytes.fromhex(args.memo)
# Uncomment next two lines if memo is needed for dev debug
plot_memo_str: str = plot_memo.hex()
log.info(f"Memo: {plot_memo_str}")
dt_string = datetime.now().strftime("%Y-%m-%d-%H-%M")
if use_datetime:
filename: str = f"plot-k{args.size}-{dt_string}-{plot_id}.plot"
else:
filename = f"plot-k{args.size}-{plot_id}.plot"
full_path: Path = args.final_dir / filename
resolved_final_dir: str = str(Path(args.final_dir).resolve())
plot_directories_list: str = config["harvester"]["plot_directories"]
if args.exclude_final_dir:
log.info(f"NOT adding directory {resolved_final_dir} to harvester for farming")
if resolved_final_dir in plot_directories_list:
log.warning(f"Directory {resolved_final_dir} already exists for harvester, please remove it manually")
else:
if resolved_final_dir not in plot_directories_list:
# Adds the directory to the plot directories if it is not present
log.info(f"Adding directory {resolved_final_dir} to harvester for farming")
config = add_plot_directory(root_path, resolved_final_dir)
if not full_path.exists():
log.info(f"Starting plot {i + 1}/{num}")
# Creates the plot. This will take a long time for larger plots.
plotter: DiskPlotter = DiskPlotter()
plotter.create_plot_disk(
str(args.tmp_dir),
str(args.tmp2_dir),
str(args.final_dir),
filename,
args.size,
plot_memo,
plot_id,
args.buffer,
args.buckets,
args.stripe_size,
args.num_threads,
args.nobitfield,
)
finished_filenames.append(filename)
else:
log.info(f"Plot {filename} already exists")
log.info("Summary:")
if tmp_dir_created:
try:
args.tmp_dir.rmdir()
except Exception:
log.info(f"warning: did not remove primary temporary folder {args.tmp_dir}, it may not be empty.")
if tmp2_dir_created:
try:
args.tmp2_dir.rmdir()
except Exception:
log.info(f"warning: did not remove secondary temporary folder {args.tmp2_dir}, it may not be empty.")
log.info(f"Created a total of {len(finished_filenames)} new plots")
for filename in finished_filenames:
log.info(filename)