def test_scraper_imports_existing_ethpmcli_dir(log, log_2, test_assets_dir,
w3):
release(
log,
w3,
"owned",
"1.0.0",
"ipfs://QmbeVyFLSuEUxiXKwSsEjef6icpdTdA4kGG9BcrJXKNKUW",
)
w3.testing.mine(3)
release(
log_2,
w3,
"owned-dupe",
"1.0.0",
"ipfs://QmbeVyFLSuEUxiXKwSsEjef6icpdTdA4kGG9BcrJXKNKUW",
)
ethpmcli_dir = get_xdg_ethpmcli_root()
# First scrape
scrape(w3, ethpmcli_dir, 1)
w3.testing.mine(3)
release(
log,
w3,
"wallet",
"1.0.0",
"ipfs://QmRMSm4k37mr2T3A2MGxAj2eAHGR5veibVt1t9Leh5waV1",
)
w3.testing.mine(3)
# Second scrape
scrape(w3, ethpmcli_dir, 1)
assert check_dir_trees_equal(ethpmcli_dir,
(test_assets_dir.parent / "ipfs"))
def test_scraper_imports_existing_ethpmcli_dir(log, log_2, test_assets_dir,
w3):
release(
log,
w3,
"owned",
"1.0.0",
"ipfs://QmcxvhkJJVpbxEAa6cgW3B6XwPJb79w9GpNUv2P2THUzZR",
)
w3.testing.mine(3)
release(
log_2,
w3,
"owned-dupe",
"1.0.0",
"ipfs://QmcxvhkJJVpbxEAa6cgW3B6XwPJb79w9GpNUv2P2THUzZR",
)
ethpmcli_dir = get_xdg_ethpmcli_root()
# First scrape
scrape(w3, ethpmcli_dir, 1)
w3.testing.mine(3)
release(
log,
w3,
"wallet",
"1.0.0",
"ipfs://QmRALeFkttSr6DLmPiNtAqLcMJYXu4BK3SjZGVgW8VASnm",
)
w3.testing.mine(3)
# Second scrape
scrape(w3, ethpmcli_dir, 1)
assert check_dir_trees_equal(ethpmcli_dir,
(test_assets_dir.parent / "ipfs"))
def scrape_action(args: argparse.Namespace) -> None:
config = Config(args)
xdg_ethpmcli_root = get_xdg_ethpmcli_root()
chain_data_path = xdg_ethpmcli_root / IPFS_CHAIN_DATA
validate_chain_data_store(chain_data_path, config.w3)
cli_logger.info("Loading IPFS scraper...")
start_block = args.start_block if args.start_block else 0
last_scraped_block = scrape(config.w3, xdg_ethpmcli_root, start_block)
last_scraped_block_hash = config.w3.eth.getBlock(last_scraped_block)["hash"]
cli_logger.info(
"All blocks scraped up to # %d: %s.",
last_scraped_block,
humanize_hash(last_scraped_block_hash),
)
cli_logger.debug(
"All blocks scraped up to # %d: %s.",
last_scraped_block,
last_scraped_block_hash,
)
def test_scraper_logs_scraped_block_ranges(log, w3):
ethpmcli_dir = get_xdg_ethpmcli_root()
# validate tmpdir
assert "pytest" in str(ethpmcli_dir)
# Initial scrape
w3.testing.mine(6)
scrape(w3, ethpmcli_dir, 1)
expected_1 = {"chain_id": 1, "scraped_blocks": [{"min": "0", "max": "6"}]}
actual_1 = json.loads((ethpmcli_dir / "chain_data.json").read_text())
assert actual_1 == expected_1
# Scrape from custom start block
w3.testing.mine(4)
scrape(w3, ethpmcli_dir, 9)
expected_2 = {
"chain_id": 1,
"scraped_blocks": [{
"min": "0",
"max": "6"
}, {
"min": "9",
"max": "10"
}],
}
actual_2 = json.loads((ethpmcli_dir / "chain_data.json").read_text())
assert actual_2 == expected_2
# Complex scrape from custom start block
w3.testing.mine(4)
expected_3 = {
"chain_id":
1,
"scraped_blocks": [
{
"min": "0",
"max": "6"
},
{
"min": "9",
"max": "10"
},
{
"min": "13",
"max": "14"
},
],
}
scrape(w3, ethpmcli_dir, 13)
actual_3 = json.loads((ethpmcli_dir / "chain_data.json").read_text())
assert actual_3 == expected_3
# Test ranges partially collapse
scrape(w3, ethpmcli_dir, 10)
expected_4 = {
"chain_id": 1,
"scraped_blocks": [{
"min": "0",
"max": "6"
}, {
"min": "9",
"max": "14"
}],
}
actual_4 = json.loads((ethpmcli_dir / "chain_data.json").read_text())
assert actual_4 == expected_4
# Test ranges fully collapse
scrape(w3, ethpmcli_dir, 1)
expected_5 = {"chain_id": 1, "scraped_blocks": [{"min": "0", "max": "14"}]}
actual_5 = json.loads((ethpmcli_dir / "chain_data.json").read_text())
assert actual_5 == expected_5