def test_scraper_imports_existing_ethpmcli_dir(log, log_2, test_assets_dir,
w3):
release(
log,
w3,
"owned",
"1.0.0",
"ipfs://QmcxvhkJJVpbxEAa6cgW3B6XwPJb79w9GpNUv2P2THUzZR",
)
w3.testing.mine(3)
release(
log_2,
w3,
"owned-dupe",
"1.0.0",
"ipfs://QmcxvhkJJVpbxEAa6cgW3B6XwPJb79w9GpNUv2P2THUzZR",
)
ethpmcli_dir = get_xdg_ethpmcli_root()
# First scrape
scrape(w3, ethpmcli_dir, 1)
w3.testing.mine(3)
release(
log,
w3,
"wallet",
"1.0.0",
"ipfs://QmRALeFkttSr6DLmPiNtAqLcMJYXu4BK3SjZGVgW8VASnm",
)
w3.testing.mine(3)
# Second scrape
scrape(w3, ethpmcli_dir, 1)
assert check_dir_trees_equal(ethpmcli_dir,
(test_assets_dir.parent / "ipfs"))
def import_keyfile(keyfile_path: Path) -> None:
validate_keyfile(keyfile_path)
ethpm_xdg_root = get_xdg_ethpmcli_root()
ethpm_cli_keyfile_path = ethpm_xdg_root / KEYFILE_PATH
tmp_keyfile = Path(tempfile.NamedTemporaryFile().name)
tmp_keyfile.write_text(keyfile_path.read_text())
tmp_keyfile.replace(ethpm_cli_keyfile_path)
def test_scraper_imports_existing_ethpmcli_dir(log, log_2, test_assets_dir,
w3):
release(
log,
w3,
"owned",
"1.0.0",
"ipfs://QmbeVyFLSuEUxiXKwSsEjef6icpdTdA4kGG9BcrJXKNKUW",
)
w3.testing.mine(3)
release(
log_2,
w3,
"owned-dupe",
"1.0.0",
"ipfs://QmbeVyFLSuEUxiXKwSsEjef6icpdTdA4kGG9BcrJXKNKUW",
)
ethpmcli_dir = get_xdg_ethpmcli_root()
# First scrape
scrape(w3, ethpmcli_dir, 1)
w3.testing.mine(3)
release(
log,
w3,
"wallet",
"1.0.0",
"ipfs://QmRMSm4k37mr2T3A2MGxAj2eAHGR5veibVt1t9Leh5waV1",
)
w3.testing.mine(3)
# Second scrape
scrape(w3, ethpmcli_dir, 1)
assert check_dir_trees_equal(ethpmcli_dir,
(test_assets_dir.parent / "ipfs"))
def keyfile(keyfile_auth):
private_key, _, password = keyfile_auth
xdg_ethpm_dir = get_xdg_ethpmcli_root()
assert "pytest" in str(xdg_ethpm_dir)
tmp_keyfile = xdg_ethpm_dir / KEYFILE_PATH
keyfile_json = eth_keyfile.create_keyfile_json(private_key, password)
tmp_keyfile.write_text(json.dumps(keyfile_json))
return tmp_keyfile
def get_keyfile_path() -> Path:
ethpm_xdg_root = get_xdg_ethpmcli_root()
keyfile_path = ethpm_xdg_root / KEYFILE_PATH
if not keyfile_path.is_file():
raise AuthorizationError(f"No keyfile located at {keyfile_path}.")
if not keyfile_path.read_text():
raise AuthorizationError(f"Empty keyfile located at {keyfile_path}.")
return keyfile_path
def __init__(self, args: Namespace) -> None:
# Setup IPFS backend
if "local_ipfs" in args and args.local_ipfs:
self.ipfs_backend = get_ipfs_backend(args.local_ipfs)
else:
self.ipfs_backend = get_ipfs_backend()
# Setup _ethpm_packages dir
if "ethpm_dir" in args and args.ethpm_dir:
self.ethpm_dir = args.ethpm_dir
elif ETHPM_DIR_ENV_VAR in os.environ:
self.ethpm_dir = Path(os.environ[ETHPM_DIR_ENV_VAR])
else:
self.ethpm_dir = Path.cwd() / ETHPM_PACKAGES_DIR
if not self.ethpm_dir.is_dir():
self.ethpm_dir.mkdir()
validate_ethpm_dir(self.ethpm_dir)
# Setup w3
if "chain_id" in args and args.chain_id:
chain_id = args.chain_id
else:
chain_id = 1
if "keyfile_path" in args and args.keyfile_path:
import_keyfile(args.keyfile_path)
if "keyfile_password" in args and args.keyfile_password:
self.private_key = get_authorized_private_key(
args.keyfile_password)
self.w3 = setup_w3(chain_id, self.private_key)
# Setup xdg ethpm dir
self.xdg_ethpmcli_root = get_xdg_ethpmcli_root()
setup_xdg_ethpm_dir(self.xdg_ethpmcli_root, self.w3)
# Setup projects dir
if "project_dir" in args and args.project_dir:
validate_project_directory(args.project_dir)
self.project_dir = args.project_dir
else:
self.project_dir = None
if "manifest_path" in args and args.manifest_path:
if not args.manifest_path.is_file():
raise ConfigurationError(
f"Provided manifest path: {args.manifest_path} is not a file."
)
self.manifest_path = args.manifest_path
else:
self.manifest_path = None
def scrape_action(args: argparse.Namespace) -> None:
config = Config(args)
xdg_ethpmcli_root = get_xdg_ethpmcli_root()
chain_data_path = xdg_ethpmcli_root / IPFS_CHAIN_DATA
validate_chain_data_store(chain_data_path, config.w3)
cli_logger.info("Loading IPFS scraper...")
start_block = args.start_block if args.start_block else 0
last_scraped_block = scrape(config.w3, xdg_ethpmcli_root, start_block)
last_scraped_block_hash = config.w3.eth.getBlock(last_scraped_block)["hash"]
cli_logger.info(
"All blocks scraped up to # %d: %s.",
last_scraped_block,
humanize_hash(last_scraped_block_hash),
)
cli_logger.debug(
"All blocks scraped up to # %d: %s.",
last_scraped_block,
last_scraped_block_hash,
)
def test_import_keyfile(keyfile):
ethpmcli_dir = get_xdg_ethpmcli_root()
import_keyfile(keyfile)
assert filecmp.cmp(ethpmcli_dir / KEYFILE_PATH, keyfile)
def test_scraper_logs_scraped_block_ranges(log, w3):
ethpmcli_dir = get_xdg_ethpmcli_root()
# validate tmpdir
assert "pytest" in str(ethpmcli_dir)
# Initial scrape
w3.testing.mine(6)
scrape(w3, ethpmcli_dir, 1)
expected_1 = {"chain_id": 1, "scraped_blocks": [{"min": "0", "max": "6"}]}
actual_1 = json.loads((ethpmcli_dir / "chain_data.json").read_text())
assert actual_1 == expected_1
# Scrape from custom start block
w3.testing.mine(4)
scrape(w3, ethpmcli_dir, 9)
expected_2 = {
"chain_id": 1,
"scraped_blocks": [{
"min": "0",
"max": "6"
}, {
"min": "9",
"max": "10"
}],
}
actual_2 = json.loads((ethpmcli_dir / "chain_data.json").read_text())
assert actual_2 == expected_2
# Complex scrape from custom start block
w3.testing.mine(4)
expected_3 = {
"chain_id":
1,
"scraped_blocks": [
{
"min": "0",
"max": "6"
},
{
"min": "9",
"max": "10"
},
{
"min": "13",
"max": "14"
},
],
}
scrape(w3, ethpmcli_dir, 13)
actual_3 = json.loads((ethpmcli_dir / "chain_data.json").read_text())
assert actual_3 == expected_3
# Test ranges partially collapse
scrape(w3, ethpmcli_dir, 10)
expected_4 = {
"chain_id": 1,
"scraped_blocks": [{
"min": "0",
"max": "6"
}, {
"min": "9",
"max": "14"
}],
}
actual_4 = json.loads((ethpmcli_dir / "chain_data.json").read_text())
assert actual_4 == expected_4
# Test ranges fully collapse
scrape(w3, ethpmcli_dir, 1)
expected_5 = {"chain_id": 1, "scraped_blocks": [{"min": "0", "max": "14"}]}
actual_5 = json.loads((ethpmcli_dir / "chain_data.json").read_text())
assert actual_5 == expected_5
def test_config_initializes_xdg_dir(config):
xdg_ethpm_dir = get_xdg_ethpmcli_root()
assert (xdg_ethpm_dir / KEYFILE_PATH).is_file()
assert (xdg_ethpm_dir / IPFS_CHAIN_DATA).is_file()
