def test_scraper_imports_existing_ethpmcli_dir(log, log_2, test_assets_dir, w3): release( log, w3, "owned", "1.0.0", "ipfs://QmcxvhkJJVpbxEAa6cgW3B6XwPJb79w9GpNUv2P2THUzZR", ) w3.testing.mine(3) release( log_2, w3, "owned-dupe", "1.0.0", "ipfs://QmcxvhkJJVpbxEAa6cgW3B6XwPJb79w9GpNUv2P2THUzZR", ) ethpmcli_dir = get_xdg_ethpmcli_root() # First scrape scrape(w3, ethpmcli_dir, 1) w3.testing.mine(3) release( log, w3, "wallet", "1.0.0", "ipfs://QmRALeFkttSr6DLmPiNtAqLcMJYXu4BK3SjZGVgW8VASnm", ) w3.testing.mine(3) # Second scrape scrape(w3, ethpmcli_dir, 1) assert check_dir_trees_equal(ethpmcli_dir, (test_assets_dir.parent / "ipfs"))
def import_keyfile(keyfile_path: Path) -> None: validate_keyfile(keyfile_path) ethpm_xdg_root = get_xdg_ethpmcli_root() ethpm_cli_keyfile_path = ethpm_xdg_root / KEYFILE_PATH tmp_keyfile = Path(tempfile.NamedTemporaryFile().name) tmp_keyfile.write_text(keyfile_path.read_text()) tmp_keyfile.replace(ethpm_cli_keyfile_path)
def test_scraper_imports_existing_ethpmcli_dir(log, log_2, test_assets_dir, w3): release( log, w3, "owned", "1.0.0", "ipfs://QmbeVyFLSuEUxiXKwSsEjef6icpdTdA4kGG9BcrJXKNKUW", ) w3.testing.mine(3) release( log_2, w3, "owned-dupe", "1.0.0", "ipfs://QmbeVyFLSuEUxiXKwSsEjef6icpdTdA4kGG9BcrJXKNKUW", ) ethpmcli_dir = get_xdg_ethpmcli_root() # First scrape scrape(w3, ethpmcli_dir, 1) w3.testing.mine(3) release( log, w3, "wallet", "1.0.0", "ipfs://QmRMSm4k37mr2T3A2MGxAj2eAHGR5veibVt1t9Leh5waV1", ) w3.testing.mine(3) # Second scrape scrape(w3, ethpmcli_dir, 1) assert check_dir_trees_equal(ethpmcli_dir, (test_assets_dir.parent / "ipfs"))
def keyfile(keyfile_auth): private_key, _, password = keyfile_auth xdg_ethpm_dir = get_xdg_ethpmcli_root() assert "pytest" in str(xdg_ethpm_dir) tmp_keyfile = xdg_ethpm_dir / KEYFILE_PATH keyfile_json = eth_keyfile.create_keyfile_json(private_key, password) tmp_keyfile.write_text(json.dumps(keyfile_json)) return tmp_keyfile
def get_keyfile_path() -> Path: ethpm_xdg_root = get_xdg_ethpmcli_root() keyfile_path = ethpm_xdg_root / KEYFILE_PATH if not keyfile_path.is_file(): raise AuthorizationError(f"No keyfile located at {keyfile_path}.") if not keyfile_path.read_text(): raise AuthorizationError(f"Empty keyfile located at {keyfile_path}.") return keyfile_path
def __init__(self, args: Namespace) -> None: # Setup IPFS backend if "local_ipfs" in args and args.local_ipfs: self.ipfs_backend = get_ipfs_backend(args.local_ipfs) else: self.ipfs_backend = get_ipfs_backend() # Setup _ethpm_packages dir if "ethpm_dir" in args and args.ethpm_dir: self.ethpm_dir = args.ethpm_dir elif ETHPM_DIR_ENV_VAR in os.environ: self.ethpm_dir = Path(os.environ[ETHPM_DIR_ENV_VAR]) else: self.ethpm_dir = Path.cwd() / ETHPM_PACKAGES_DIR if not self.ethpm_dir.is_dir(): self.ethpm_dir.mkdir() validate_ethpm_dir(self.ethpm_dir) # Setup w3 if "chain_id" in args and args.chain_id: chain_id = args.chain_id else: chain_id = 1 if "keyfile_path" in args and args.keyfile_path: import_keyfile(args.keyfile_path) if "keyfile_password" in args and args.keyfile_password: self.private_key = get_authorized_private_key( args.keyfile_password) self.w3 = setup_w3(chain_id, self.private_key) # Setup xdg ethpm dir self.xdg_ethpmcli_root = get_xdg_ethpmcli_root() setup_xdg_ethpm_dir(self.xdg_ethpmcli_root, self.w3) # Setup projects dir if "project_dir" in args and args.project_dir: validate_project_directory(args.project_dir) self.project_dir = args.project_dir else: self.project_dir = None if "manifest_path" in args and args.manifest_path: if not args.manifest_path.is_file(): raise ConfigurationError( f"Provided manifest path: {args.manifest_path} is not a file." ) self.manifest_path = args.manifest_path else: self.manifest_path = None
def scrape_action(args: argparse.Namespace) -> None: config = Config(args) xdg_ethpmcli_root = get_xdg_ethpmcli_root() chain_data_path = xdg_ethpmcli_root / IPFS_CHAIN_DATA validate_chain_data_store(chain_data_path, config.w3) cli_logger.info("Loading IPFS scraper...") start_block = args.start_block if args.start_block else 0 last_scraped_block = scrape(config.w3, xdg_ethpmcli_root, start_block) last_scraped_block_hash = config.w3.eth.getBlock(last_scraped_block)["hash"] cli_logger.info( "All blocks scraped up to # %d: %s.", last_scraped_block, humanize_hash(last_scraped_block_hash), ) cli_logger.debug( "All blocks scraped up to # %d: %s.", last_scraped_block, last_scraped_block_hash, )
def test_import_keyfile(keyfile): ethpmcli_dir = get_xdg_ethpmcli_root() import_keyfile(keyfile) assert filecmp.cmp(ethpmcli_dir / KEYFILE_PATH, keyfile)
def test_scraper_logs_scraped_block_ranges(log, w3): ethpmcli_dir = get_xdg_ethpmcli_root() # validate tmpdir assert "pytest" in str(ethpmcli_dir) # Initial scrape w3.testing.mine(6) scrape(w3, ethpmcli_dir, 1) expected_1 = {"chain_id": 1, "scraped_blocks": [{"min": "0", "max": "6"}]} actual_1 = json.loads((ethpmcli_dir / "chain_data.json").read_text()) assert actual_1 == expected_1 # Scrape from custom start block w3.testing.mine(4) scrape(w3, ethpmcli_dir, 9) expected_2 = { "chain_id": 1, "scraped_blocks": [{ "min": "0", "max": "6" }, { "min": "9", "max": "10" }], } actual_2 = json.loads((ethpmcli_dir / "chain_data.json").read_text()) assert actual_2 == expected_2 # Complex scrape from custom start block w3.testing.mine(4) expected_3 = { "chain_id": 1, "scraped_blocks": [ { "min": "0", "max": "6" }, { "min": "9", "max": "10" }, { "min": "13", "max": "14" }, ], } scrape(w3, ethpmcli_dir, 13) actual_3 = json.loads((ethpmcli_dir / "chain_data.json").read_text()) assert actual_3 == expected_3 # Test ranges partially collapse scrape(w3, ethpmcli_dir, 10) expected_4 = { "chain_id": 1, "scraped_blocks": [{ "min": "0", "max": "6" }, { "min": "9", "max": "14" }], } actual_4 = json.loads((ethpmcli_dir / "chain_data.json").read_text()) assert actual_4 == expected_4 # Test ranges fully collapse scrape(w3, ethpmcli_dir, 1) expected_5 = {"chain_id": 1, "scraped_blocks": [{"min": "0", "max": "14"}]} actual_5 = json.loads((ethpmcli_dir / "chain_data.json").read_text()) assert actual_5 == expected_5
def test_config_initializes_xdg_dir(config): xdg_ethpm_dir = get_xdg_ethpmcli_root() assert (xdg_ethpm_dir / KEYFILE_PATH).is_file() assert (xdg_ethpm_dir / IPFS_CHAIN_DATA).is_file()