def setup_network(config_file=None):
config = Config(filename=config_file) if config_file else get_config()
network_url = config.network_url
artifacts_path = get_artifacts_path(config)
ContractHandler.set_artifacts_path(artifacts_path)
if network_url.startswith('http'):
provider = CustomHTTPProvider
elif network_url.startswith('wss'):
provider = WebsocketProvider
else:
raise AssertionError(f'Unsupported network url {network_url}. Must start with http or wss.')
Web3Provider.init_web3(provider=provider(network_url))
if network_url.startswith('wss'):
from web3.middleware import geth_poa_middleware
Web3Provider.get_web3().middleware_stack.inject(geth_poa_middleware, layer=0)
init_account_envvars()
wallet = get_provider_wallet()
if wallet is None:
raise AssertionError(f'Ocean Provider cannot run without a valid '
f'ethereum account. `PROVIDER_PRIVATE_KEY` was not found in the environment '
f'variables. \nENV WAS: {sorted(os.environ.items())}')
if not wallet.private_key:
raise AssertionError(f'Ocean Provider cannot run without a valid '
f'ethereum private key..')
def get_tx_receipt(tx_hash: str, timeout=20):
"""
Get the receipt of a tx.
:param tx_hash: hash of the transaction
:param timeout: int in seconds to wait for transaction receipt
:return: Tx receipt
"""
try:
Web3Provider.get_web3().eth.waitForTransactionReceipt(
tx_hash, timeout=timeout)
except ValueError as e:
logger.error(f'Waiting for transaction receipt failed: {e}')
return None
except Timeout as e:
logger.info(
f'Waiting for transaction receipt may have timed out: {e}.')
return None
except ConnectionClosed as e:
logger.info(
f'ConnectionClosed error waiting for transaction receipt failed: {e}.'
)
raise
except Exception as e:
logger.info(f'Unknown error waiting for transaction receipt: {e}.')
raise
return Web3Provider.get_web3().eth.getTransactionReceipt(tx_hash)
def run_events_monitor():
setup_logging()
logger.info('EventsMonitor: preparing')
required_env_vars = ['EVENTS_RPC', 'CONFIG_FILE']
for envvar in required_env_vars:
if not os.getenv(envvar):
raise AssertionError(
f'env var {envvar} is missing, make sure to set the following '
f'environment variables before starting the events monitor: {required_env_vars}'
)
network_rpc = os.environ.get('EVENTS_RPC', 'http:127.0.0.1:8545')
config_file = os.getenv('CONFIG_FILE', 'config.ini')
logger.info(
f'EventsMonitor: starting with the following values: rpc={network_rpc}'
)
ConfigProvider.set_config(Config(config_file))
from ocean_lib.ocean.util import get_web3_connection_provider
Web3Provider.init_web3(provider=get_web3_connection_provider(network_rpc))
ContractHandler.set_artifacts_path(get_artifacts_path())
if get_network_name().lower() == 'rinkeby':
from web3.middleware import geth_poa_middleware
Web3Provider.get_web3().middleware_stack.inject(geth_poa_middleware,
layer=0)
monitor = EventsMonitor(Web3Provider.get_web3(), config_file)
monitor.start_events_monitor()
logger.info(f'EventsMonitor: started')
while True:
time.sleep(5)
def setup_network(config_file=None):
config = Config(filename=config_file) if config_file else get_config()
network_url = config.network_url
artifacts_path = get_artifacts_path(config)
ContractHandler.set_artifacts_path(artifacts_path)
w3_connection_provider = get_web3_connection_provider(network_url)
Web3Provider.init_web3(provider=w3_connection_provider)
if network_url.startswith("wss"):
from web3.middleware import geth_poa_middleware
Web3Provider.get_web3().middleware_stack.inject(geth_poa_middleware,
layer=0)
init_account_envvars()
wallet = get_provider_wallet()
if wallet is None:
raise AssertionError(
f"Ocean Provider cannot run without a valid "
f"ethereum account. `PROVIDER_PRIVATE_KEY` was not found in the environment "
f"variables. \nENV WAS: {sorted(os.environ.items())}")
if not wallet.private_key:
raise AssertionError(
"Ocean Provider cannot run without a valid ethereum private key.")
def init_components(config=None):
if config is None:
config = Config(os.getenv(ENV_CONFIG_FILE))
ConfigProvider.set_config(config)
Web3Provider.init_web3(provider=get_web3_connection_provider(config.network_url))
ContractHandler.set_artifacts_path(config.artifacts_path)
def test1():
# ocean instance
config = ExampleConfig.get_config()
ConfigProvider.set_config(config)
Web3Provider.init_web3(
provider=get_web3_connection_provider(config.network_url))
ContractHandler.set_artifacts_path(config.artifacts_path)
ocean = Ocean(config)
OCEAN_address_before = ocean.OCEAN_address
# deploy, distribute, etc
deploy_fake_OCEAN()
# test: OCEAN address should have changed
OCEAN_address_after = ocean.OCEAN_address
assert OCEAN_address_before != OCEAN_address_after
# test: TEST_PRIVATE_KEY{1,2} should each hold OCEAN
wallet1 = Wallet(ocean.web3, private_key=os.getenv("TEST_PRIVATE_KEY1"))
wallet2 = Wallet(ocean.web3, private_key=os.getenv("TEST_PRIVATE_KEY2"))
OCEAN_after = BToken(ocean.OCEAN_address)
assert OCEAN_after.balanceOf(wallet1.address) > 0
assert OCEAN_after.balanceOf(wallet2.address) > 0
def setup_all():
config = ExampleConfig.get_config()
ConfigProvider.set_config(config)
Web3Provider.init_web3(
provider=get_web3_connection_provider(config.network_url))
ContractHandler.set_artifacts_path(config.artifacts_path)
network = Web3Helper.get_network_name()
wallet = get_ganache_wallet()
if network in ["ganache", "development"] and wallet:
print(
f"sender: {wallet.key}, {wallet.address}, {wallet.password}, {wallet.keysStr()}"
)
print(
f"sender balance: {Web3Helper.from_wei(Web3Helper.get_ether_balance(wallet.address))}"
)
assert Web3Helper.from_wei(Web3Helper.get_ether_balance(
wallet.address)) > 10
from ocean_lib.models.data_token import DataToken
OCEAN_token = DataToken(get_ocean_token_address(network))
amt_distribute = 1000
amt_distribute_base = to_base_18(float(amt_distribute))
for w in (get_publisher_wallet(), get_consumer_wallet()):
if Web3Helper.from_wei(Web3Helper.get_ether_balance(
w.address)) < 2:
Web3Helper.send_ether(wallet, w.address, 4)
if OCEAN_token.token_balance(w.address) < 100:
OCEAN_token.transfer(w.address,
amt_distribute_base,
from_wallet=wallet)
def test_main(dtfactory_address):
with pytest.raises(AssertionError):
Web3Provider.init_web3(None, None)
Web3Provider.set_web3(None)
assert Web3Provider._web3 is None
assert isinstance(Web3Provider.get_web3(network_url="http://test.test"),
Web3)
assert Web3Provider._web3 is not None
def initialize(private_key):
load_dotenv(".env")
config = Config(os.getenv('config.ini'))
config = Config(os.getenv('config.ini'))
print(config.network_url)
# config.network_url="https://rinkeby.infura.io/v3/31d95be121a545b688a0e07e4de4d256"
ConfigProvider.set_config(config)
Web3Provider.init_web3(
provider=get_web3_connection_provider(config.network_url))
ContractHandler.set_artifacts_path(config.artifacts_path)
ocean = Ocean()
wallet = Wallet(ocean.web3, private_key=private_key)
return ocean, wallet
def get_transfer_events_in_range(self, from_block, to_block):
name = "Transfer"
event = getattr(self.events, name)
return self.getLogs(
event, Web3Provider.get_web3(), fromBlock=from_block, toBlock=to_block
)
def _load(contract_name, address=None):
"""Retrieve the contract instance for `contract_name` that represent the smart
contract in the ethereum network.
:param contract_name: str name of the solidity smart contract.
:param address: hex str -- address of smart contract
:return: web3.eth.Contract instance
"""
assert (ContractHandler.artifacts_path
is not None), "artifacts_path should be already set."
contract_definition = ContractHandler.read_abi_from_file(
contract_name, ContractHandler.artifacts_path)
if not address and "address" in contract_definition:
address = contract_definition.get("address")
assert address, "Cannot find contract address in the abi file."
address = Web3.toChecksumAddress(address)
abi = contract_definition["abi"]
bytecode = contract_definition["bytecode"]
contract = Web3Provider.get_web3().eth.contract(address=address,
abi=abi,
bytecode=bytecode)
ContractHandler._set(contract_name, contract)
return ContractHandler._contracts[(contract_name, address)]
def cancel_or_replace_transaction(from_wallet,
nonce_value,
gas_price=None,
gas_limit=None):
w3 = Web3Provider.get_web3()
tx = {
"from": from_wallet.address,
"to": from_wallet.address,
"value": 0
}
gas = gas_limit if gas_limit is not None else w3.eth.estimateGas(tx)
tx = {
"from": from_wallet.address,
"to": from_wallet.address,
"value": 0,
"gas": gas + 1,
}
wallet = Wallet(w3,
private_key=from_wallet.key,
address=from_wallet.address)
raw_tx = wallet.sign_tx(tx,
fixed_nonce=nonce_value,
gas_price=gas_price)
tx_hash = w3.eth.sendRawTransaction(raw_tx)
receipt = w3.eth.waitForTransactionReceipt(tx_hash, timeout=30)
return receipt
def delist_ddo(did):
assert request.json and isinstance(request.json,
dict), 'invalid payload format.'
data = request.json
address = data.get('adminAddress', None)
if not address or not has_update_request_permission(address):
return jsonify(error=f'Unauthorized.'), 401
_address = None
signature = data.get('signature', None)
if signature:
_address = get_signer_address(address, signature, logger)
if not _address or _address.lower() != address.lower():
return jsonify(error=f'Unauthorized.'), 401
try:
asset_record = dao.get(did)
if not asset_record:
return jsonify(error=f'Asset {did} not found.'), 404
updater = MetadataUpdater(oceandb=dao.oceandb,
web3=Web3Provider.get_web3(),
config=ConfigProvider.get_config())
updater.do_single_update(asset_record)
return jsonify('acknowledged.'), 200
except Exception as e:
logger.error(f'get_metadata: {str(e)}')
return f'{did} asset DID is not in OceanDB', 404
def send_order(client, ddo, datatoken, service, cons_wallet):
web3 = Web3Provider.get_web3()
init_endpoint = BaseURLs.ASSETS_URL + '/initialize'
# initialize the service
payload = dict({
'documentId': ddo.did,
'serviceId': service.index,
'serviceType': service.type,
'dataToken': datatoken.address,
'consumerAddress': cons_wallet.address
})
request_url = init_endpoint + '?' + '&'.join(
[f'{k}={v}' for k, v in payload.items()])
response = client.get(request_url)
assert response.status == '200 OK'
tx_params = response.json
num_tokens = tx_params['numTokens']
nonce = tx_params.get('nonce')
receiver = tx_params['to']
assert tx_params['from'] == cons_wallet.address
assert receiver == get_datatoken_minter(ddo, datatoken.address)
assert tx_params['dataToken'] == ddo.as_dictionary()['dataToken']
assert nonce is not None, f'expecting a `nonce` value in the response, got {nonce}'
# Transfer tokens to provider account
amount = to_base_18(num_tokens)
tx_id = datatoken.startOrder(cons_wallet.address, amount, service.index,
'0xF9f2DB837b3db03Be72252fAeD2f6E0b73E428b9',
cons_wallet)
datatoken.verify_order_tx(web3, tx_id, ddo.asset_id, service.index, amount,
cons_wallet.address)
return tx_id
def get_datatoken_minter(asset, datatoken_address):
publisher = Web3Provider.get_web3().toChecksumAddress(asset.publisher)
dt = DataToken(datatoken_address)
if not dt.contract_concise.isMinter(publisher):
raise AssertionError(f'ddo publisher {publisher} is not the current '
f'minter for the DataToken contract at {datatoken_address}.')
return publisher
def get_wallet(index):
name = "PARITY_ADDRESS" if not index else f"PARITY_ADDRESS{index}"
pswrd_name = "PARITY_PASSWORD" if not index else f"PARITY_PASSWORD{index}"
key_name = "PARITY_KEY" if not index else f"PARITY_KEY{index}"
encrypted_key_name = ("PARITY_ENCRYPTED_KEY"
if not index else f"PARITY_ENCRYPTED_KEY{index}")
keyfile_name = "PARITY_KEYFILE" if not index else f"PARITY_KEYFILE{index}"
address = os.getenv(name)
if not address:
return None
pswrd = os.getenv(pswrd_name)
key = os.getenv(key_name)
encr_key = os.getenv(encrypted_key_name)
key_file = os.getenv(keyfile_name)
if key_file and not encr_key:
with open(key_file) as _file:
encr_key = json.loads(_file.read())
from ocean_lib.web3_internal.wallet import Wallet
return Wallet(
Web3Provider.get_web3(),
private_key=key,
encrypted_key=encr_key,
address=Web3.toChecksumAddress(address),
password=pswrd,
)
def read_token(self, address):
"""
Retrieve stored signed token for the given ethereum address
:param address: hex str the ethereum address that signed the token
:return: tuple (signed_token, created_at)
"""
try:
checksumAddress = Web3Provider.get_web3().toChecksumAddress(
address)
rows = [
row for row in self._run_query(
f"""SELECT signed_token, created
FROM {self.AUTH_TOKENS_TABLE}
WHERE address=?;""",
(checksumAddress, ),
)
]
token, timestamp = rows[0] if rows else (None, None)
logger.debug(f"Read auth token from `auth_tokens` storage: "
f"account={address}, token={token}")
return token, timestamp
except Exception as e:
logging.error(f"Error reading token: {e}")
return None, None
def get_event_logs(
self, event_name, from_block, to_block, filters, web3=None, chunk_size=1000
):
event = getattr(self.events, event_name)
if not web3:
web3 = Web3Provider.get_web3()
chunk = chunk_size
_from = from_block
_to = _from + chunk - 1
all_logs = []
error_count = 0
_to = min(_to, to_block)
while _from <= to_block:
try:
logs = self.getLogs(
event, web3, argument_filters=filters, fromBlock=_from, toBlock=_to
)
all_logs.extend(logs)
_from = _to + 1
_to = min(_from + chunk - 1, to_block)
error_count = 0
if (_from - from_block) % 1000 == 0:
print(
f" So far processed {len(all_logs)} Transfer events from {_from-from_block} blocks."
)
except requests.exceptions.ReadTimeout as err:
print(f"ReadTimeout ({_from}, {_to}): {err}")
error_count += 1
if error_count > 1:
break
return all_logs
def get_all_pools(self,
from_block=0,
chunk_size=1000,
include_balance=False):
web3 = Web3Provider.get_web3()
current_block = web3.eth.blockNumber
bfactory = BFactory(self.bfactory_address)
logs = bfactory.get_event_logs(
"BPoolRegistered",
from_block,
current_block,
{},
web3=web3,
chunk_size=chunk_size,
)
if include_balance:
pools = sorted(
[(
lg.args.bpoolAddress,
from_base_18(
BPool(lg.args.bpoolAddress).getBalance(
self.ocean_address)),
) for lg in logs],
key=lambda x: x[1],
reverse=True,
)
else:
pools = {lg.args.bpoolAddress for lg in logs}
return pools
def get_network_id():
"""
Return the ethereum network id calling the `web3.version.network` method.
:return: Network id, int
"""
return int(Web3Provider.get_web3().version.network)
def setup_all(request):
# a test can skip setup_all() via decorator "@pytest.mark.nosetup_all"
if "nosetup_all" in request.keywords:
return
config = ExampleConfig.get_config()
ConfigProvider.set_config(config)
Web3Provider.init_web3(
provider=get_web3_connection_provider(config.network_url))
ContractHandler.set_artifacts_path(config.artifacts_path)
wallet = get_ganache_wallet()
if not wallet:
return
addresses_file = config.address_file
if not os.path.exists(addresses_file):
return
with open(addresses_file) as f:
network_addresses = json.load(f)
print(
f"sender: {wallet.key}, {wallet.address}, {wallet.password}, {wallet.keysStr()}"
)
print(f"sender balance: {from_wei(get_ether_balance(wallet.address))}")
assert (from_wei(get_ether_balance(wallet.address)) >
10), "Ether balance less than 10."
from ocean_lib.models.data_token import DataToken
OCEAN_token = DataToken(address=network_addresses["development"]["Ocean"])
amt_distribute = 1000
amt_distribute_base = to_base_18(float(amt_distribute))
for w in (get_publisher_wallet(), get_consumer_wallet()):
if from_wei(get_ether_balance(w.address)) < 2:
send_ether(wallet, w.address, 4)
if OCEAN_token.token_balance(w.address) < 100:
OCEAN_token.mint(wallet.address,
amt_distribute_base,
from_wallet=wallet)
OCEAN_token.transfer(w.address,
amt_distribute_base,
from_wallet=wallet)
def pay_for_service(
amount: float,
token_address: str,
did: str,
service_id: int,
fee_receiver: str,
from_wallet: Wallet,
consumer: str,
) -> str:
"""
Submits the payment for chosen service in DataTokens.
:param amount:
:param token_address:
:param did:
:param service_id:
:param fee_receiver:
:param from_wallet: Wallet instance
:param consumer: str the address of consumer of the service, defaults to the payer (the `from_wallet` address)
:return: hex str id of transfer transaction
"""
amount_base = to_base_18(amount)
dt = DataToken(token_address)
balance = dt.balanceOf(from_wallet.address)
if balance < amount_base:
raise AssertionError(
f"Your token balance {balance} is not sufficient "
f"to execute the requested service. This service "
f"requires {amount_base} number of tokens.")
if did.startswith("did:"):
did = add_0x_prefix(did_to_id(did))
if fee_receiver is None:
fee_receiver = ZERO_ADDRESS
if consumer is None:
consumer = from_wallet.address
tx_hash = dt.startOrder(consumer, amount_base, service_id,
fee_receiver, from_wallet)
try:
dt.verify_order_tx(
Web3Provider.get_web3(),
tx_hash,
did,
service_id,
amount_base,
from_wallet.address,
)
return tx_hash
except (AssertionError, Exception) as e:
msg = (
f"Downloading asset files failed. The problem is related to "
f"the transfer of the data tokens required for the download "
f"service: {e}")
logger.error(msg)
raise AssertionError(msg)
def mint_fake_OCEAN():
"""
Does the following:
1. Mints tokens
2. Distributes tokens to TEST_PRIVATE_KEY1 and TEST_PRIVATE_KEY2
"""
config = ExampleConfig.get_config()
ConfigProvider.set_config(config)
Web3Provider.init_web3(
provider=get_web3_connection_provider(config.network_url))
ContractHandler.set_artifacts_path(config.artifacts_path)
addresses_file = config.address_file
ocean = get_publisher_ocean_instance()
web3 = ocean.web3
with open(addresses_file) as f:
network_addresses = json.load(f)
network = "development"
deployer_wallet = get_ganache_wallet()
OCEAN_token = DataToken(address=network_addresses[network]["Ocean"])
amt_distribute = 1000
amt_distribute_base = to_base_18(float(amt_distribute))
OCEAN_token.mint(deployer_wallet.address,
2 * amt_distribute_base,
from_wallet=deployer_wallet)
for key_label in ["TEST_PRIVATE_KEY1", "TEST_PRIVATE_KEY2"]:
key = os.environ.get(key_label)
if not key:
continue
w = Wallet(web3, private_key=key)
if OCEAN_token.token_balance(w.address) < 1000:
OCEAN_token.transfer(w.address,
amt_distribute_base,
from_wallet=deployer_wallet)
if from_wei(get_ether_balance(w.address)) < 2:
send_ether(deployer_wallet, w.address, 4)
def events_object():
global EVENTS_INSTANCE
if not EVENTS_INSTANCE:
config_file = os.getenv('CONFIG_FILE', 'config.ini')
network_rpc = os.environ.get('EVENTS_RPC', 'http://127.0.0.1:8545')
ConfigProvider.set_config(Config(config_file))
from ocean_lib.ocean.util import get_web3_connection_provider
Web3Provider.init_web3(
provider=get_web3_connection_provider(network_rpc))
ContractHandler.set_artifacts_path(get_artifacts_path())
EVENTS_INSTANCE = EventsMonitor(Web3Provider.get_web3(),
app.config['CONFIG_FILE'])
EVENTS_INSTANCE.store_last_processed_block(0)
return EVENTS_INSTANCE
def get_ether_balance(address: str) -> int:
"""
Get balance of an ethereum address.
:param address: address, bytes32
:return: balance, int
"""
return Web3Provider.get_web3().eth.getBalance(address, block_identifier="latest")
def add_ethereum_prefix_and_hash_msg(text):
"""
This method of adding the ethereum prefix seems to be used in web3.personal.sign/ecRecover.
:param text: str any str to be signed / used in recovering address from a signature
:return: hash of prefixed text according to the recommended ethereum prefix
"""
prefixed_msg = f"\x19Ethereum Signed Message:\n{len(text)}{text}"
return Web3Provider.get_web3().sha3(text=prefixed_msg)
def validate_order(sender, token_address, num_tokens, tx_id, did, service_id):
dt_contract = DataToken(token_address)
try:
amount = to_base_18(num_tokens)
tx, order_event, transfer_event = dt_contract.verify_order_tx(
Web3Provider.get_web3(), tx_id, did, service_id, amount, sender)
return tx, order_event, transfer_event
except AssertionError:
raise
def get_ganache_wallet():
web3 = Web3Provider.get_web3()
if web3.eth.accounts and web3.eth.accounts[0].lower(
) == '0xe2DD09d719Da89e5a3D0F2549c7E24566e947260'.lower():
return Wallet(
web3,
private_key=
'0xc594c6e5def4bab63ac29eed19a134c130388f74f019bc74b8f4389df2837a58'
)
return None
def get_ganache_wallet():
web3 = Web3Provider.get_web3()
if (web3.eth.accounts and web3.eth.accounts[0].lower()
== "0xe2DD09d719Da89e5a3D0F2549c7E24566e947260".lower()):
return Wallet(
web3,
private_key=
"0xfd5c1ccea015b6d663618850824154a3b3fb2882c46cefb05b9a93fea8c3d215",
)
return None
def new_factory_contract():
web3 = Web3Provider.get_web3()
deployer_wallet = get_ganache_wallet()
dt_address = DataToken.deploy(
web3, deployer_wallet, ContractHandler.artifacts_path,
'Template Contract', 'TEMPLATE', deployer_wallet.address,
DataToken.DEFAULT_CAP_BASE, DTFactory.FIRST_BLOB,
deployer_wallet.address)
return DTFactory(
DTFactory.deploy(web3, deployer_wallet, ContractHandler.artifacts_path,
dt_address, deployer_wallet.address))
