def setup_network(config_file=None):
config = Config(filename=config_file) if config_file else get_config()
network_url = config.network_url
artifacts_path = get_artifacts_path(config)
ContractHandler.set_artifacts_path(artifacts_path)
if network_url.startswith('http'):
provider = CustomHTTPProvider
elif network_url.startswith('wss'):
provider = WebsocketProvider
else:
raise AssertionError(f'Unsupported network url {network_url}. Must start with http or wss.')
Web3Provider.init_web3(provider=provider(network_url))
if network_url.startswith('wss'):
from web3.middleware import geth_poa_middleware
Web3Provider.get_web3().middleware_stack.inject(geth_poa_middleware, layer=0)
init_account_envvars()
wallet = get_provider_wallet()
if wallet is None:
raise AssertionError(f'Ocean Provider cannot run without a valid '
f'ethereum account. `PROVIDER_PRIVATE_KEY` was not found in the environment '
f'variables. \nENV WAS: {sorted(os.environ.items())}')
if not wallet.private_key:
raise AssertionError(f'Ocean Provider cannot run without a valid '
f'ethereum private key..')
def setup_network(config_file=None):
config = Config(filename=config_file) if config_file else get_config()
network_url = config.network_url
artifacts_path = get_artifacts_path(config)
ContractHandler.set_artifacts_path(artifacts_path)
w3_connection_provider = get_web3_connection_provider(network_url)
Web3Provider.init_web3(provider=w3_connection_provider)
if network_url.startswith("wss"):
from web3.middleware import geth_poa_middleware
Web3Provider.get_web3().middleware_stack.inject(geth_poa_middleware,
layer=0)
init_account_envvars()
wallet = get_provider_wallet()
if wallet is None:
raise AssertionError(
f"Ocean Provider cannot run without a valid "
f"ethereum account. `PROVIDER_PRIVATE_KEY` was not found in the environment "
f"variables. \nENV WAS: {sorted(os.environ.items())}")
if not wallet.private_key:
raise AssertionError(
"Ocean Provider cannot run without a valid ethereum private key.")
def run_events_monitor():
setup_logging()
logger.info('EventsMonitor: preparing')
required_env_vars = ['EVENTS_RPC', 'CONFIG_FILE']
for envvar in required_env_vars:
if not os.getenv(envvar):
raise AssertionError(
f'env var {envvar} is missing, make sure to set the following '
f'environment variables before starting the events monitor: {required_env_vars}'
)
network_rpc = os.environ.get('EVENTS_RPC', 'http:127.0.0.1:8545')
config_file = os.getenv('CONFIG_FILE', 'config.ini')
logger.info(
f'EventsMonitor: starting with the following values: rpc={network_rpc}'
)
ConfigProvider.set_config(Config(config_file))
from ocean_lib.ocean.util import get_web3_connection_provider
Web3Provider.init_web3(provider=get_web3_connection_provider(network_rpc))
ContractHandler.set_artifacts_path(get_artifacts_path())
if get_network_name().lower() == 'rinkeby':
from web3.middleware import geth_poa_middleware
Web3Provider.get_web3().middleware_stack.inject(geth_poa_middleware,
layer=0)
monitor = EventsMonitor(Web3Provider.get_web3(), config_file)
monitor.start_events_monitor()
logger.info(f'EventsMonitor: started')
while True:
time.sleep(5)
def get_tx_receipt(tx_hash: str, timeout=20):
"""
Get the receipt of a tx.
:param tx_hash: hash of the transaction
:param timeout: int in seconds to wait for transaction receipt
:return: Tx receipt
"""
try:
Web3Provider.get_web3().eth.waitForTransactionReceipt(
tx_hash, timeout=timeout)
except ValueError as e:
logger.error(f'Waiting for transaction receipt failed: {e}')
return None
except Timeout as e:
logger.info(
f'Waiting for transaction receipt may have timed out: {e}.')
return None
except ConnectionClosed as e:
logger.info(
f'ConnectionClosed error waiting for transaction receipt failed: {e}.'
)
raise
except Exception as e:
logger.info(f'Unknown error waiting for transaction receipt: {e}.')
raise
return Web3Provider.get_web3().eth.getTransactionReceipt(tx_hash)
def send_order(client, ddo, datatoken, service, cons_wallet):
web3 = Web3Provider.get_web3()
init_endpoint = BaseURLs.ASSETS_URL + '/initialize'
# initialize the service
payload = dict({
'documentId': ddo.did,
'serviceId': service.index,
'serviceType': service.type,
'dataToken': datatoken.address,
'consumerAddress': cons_wallet.address
})
request_url = init_endpoint + '?' + '&'.join(
[f'{k}={v}' for k, v in payload.items()])
response = client.get(request_url)
assert response.status == '200 OK'
tx_params = response.json
num_tokens = tx_params['numTokens']
nonce = tx_params.get('nonce')
receiver = tx_params['to']
assert tx_params['from'] == cons_wallet.address
assert receiver == get_datatoken_minter(ddo, datatoken.address)
assert tx_params['dataToken'] == ddo.as_dictionary()['dataToken']
assert nonce is not None, f'expecting a `nonce` value in the response, got {nonce}'
# Transfer tokens to provider account
amount = to_base_18(num_tokens)
tx_id = datatoken.startOrder(cons_wallet.address, amount, service.index,
'0xF9f2DB837b3db03Be72252fAeD2f6E0b73E428b9',
cons_wallet)
datatoken.verify_order_tx(web3, tx_id, ddo.asset_id, service.index, amount,
cons_wallet.address)
return tx_id
def read_token(self, address):
"""
Retrieve stored signed token for the given ethereum address
:param address: hex str the ethereum address that signed the token
:return: tuple (signed_token, created_at)
"""
try:
checksumAddress = Web3Provider.get_web3().toChecksumAddress(
address)
rows = [
row for row in self._run_query(
f"""SELECT signed_token, created
FROM {self.AUTH_TOKENS_TABLE}
WHERE address=?;""",
(checksumAddress, ),
)
]
token, timestamp = rows[0] if rows else (None, None)
logger.debug(f"Read auth token from `auth_tokens` storage: "
f"account={address}, token={token}")
return token, timestamp
except Exception as e:
logging.error(f"Error reading token: {e}")
return None, None
def get_all_pools(self,
from_block=0,
chunk_size=1000,
include_balance=False):
web3 = Web3Provider.get_web3()
current_block = web3.eth.blockNumber
bfactory = BFactory(self.bfactory_address)
logs = bfactory.get_event_logs(
"BPoolRegistered",
from_block,
current_block,
{},
web3=web3,
chunk_size=chunk_size,
)
if include_balance:
pools = sorted(
[(
lg.args.bpoolAddress,
from_base_18(
BPool(lg.args.bpoolAddress).getBalance(
self.ocean_address)),
) for lg in logs],
key=lambda x: x[1],
reverse=True,
)
else:
pools = {lg.args.bpoolAddress for lg in logs}
return pools
def get_datatoken_minter(asset, datatoken_address):
publisher = Web3Provider.get_web3().toChecksumAddress(asset.publisher)
dt = DataToken(datatoken_address)
if not dt.contract_concise.isMinter(publisher):
raise AssertionError(f'ddo publisher {publisher} is not the current '
f'minter for the DataToken contract at {datatoken_address}.')
return publisher
def get_network_id():
"""
Return the ethereum network id calling the `web3.version.network` method.
:return: Network id, int
"""
return int(Web3Provider.get_web3().version.network)
def cancel_or_replace_transaction(from_wallet,
nonce_value,
gas_price=None,
gas_limit=None):
w3 = Web3Provider.get_web3()
tx = {
"from": from_wallet.address,
"to": from_wallet.address,
"value": 0
}
gas = gas_limit if gas_limit is not None else w3.eth.estimateGas(tx)
tx = {
"from": from_wallet.address,
"to": from_wallet.address,
"value": 0,
"gas": gas + 1,
}
wallet = Wallet(w3,
private_key=from_wallet.key,
address=from_wallet.address)
raw_tx = wallet.sign_tx(tx,
fixed_nonce=nonce_value,
gas_price=gas_price)
tx_hash = w3.eth.sendRawTransaction(raw_tx)
receipt = w3.eth.waitForTransactionReceipt(tx_hash, timeout=30)
return receipt
def get_transfer_events_in_range(self, from_block, to_block):
name = "Transfer"
event = getattr(self.events, name)
return self.getLogs(
event, Web3Provider.get_web3(), fromBlock=from_block, toBlock=to_block
)
def get_event_logs(
self, event_name, from_block, to_block, filters, web3=None, chunk_size=1000
):
event = getattr(self.events, event_name)
if not web3:
web3 = Web3Provider.get_web3()
chunk = chunk_size
_from = from_block
_to = _from + chunk - 1
all_logs = []
error_count = 0
_to = min(_to, to_block)
while _from <= to_block:
try:
logs = self.getLogs(
event, web3, argument_filters=filters, fromBlock=_from, toBlock=_to
)
all_logs.extend(logs)
_from = _to + 1
_to = min(_from + chunk - 1, to_block)
error_count = 0
if (_from - from_block) % 1000 == 0:
print(
f" So far processed {len(all_logs)} Transfer events from {_from-from_block} blocks."
)
except requests.exceptions.ReadTimeout as err:
print(f"ReadTimeout ({_from}, {_to}): {err}")
error_count += 1
if error_count > 1:
break
return all_logs
def _load(contract_name, address=None):
"""Retrieve the contract instance for `contract_name` that represent the smart
contract in the ethereum network.
:param contract_name: str name of the solidity smart contract.
:param address: hex str -- address of smart contract
:return: web3.eth.Contract instance
"""
assert (ContractHandler.artifacts_path
is not None), "artifacts_path should be already set."
contract_definition = ContractHandler.read_abi_from_file(
contract_name, ContractHandler.artifacts_path)
if not address and "address" in contract_definition:
address = contract_definition.get("address")
assert address, "Cannot find contract address in the abi file."
address = Web3.toChecksumAddress(address)
abi = contract_definition["abi"]
bytecode = contract_definition["bytecode"]
contract = Web3Provider.get_web3().eth.contract(address=address,
abi=abi,
bytecode=bytecode)
ContractHandler._set(contract_name, contract)
return ContractHandler._contracts[(contract_name, address)]
def get_wallet(index):
name = "PARITY_ADDRESS" if not index else f"PARITY_ADDRESS{index}"
pswrd_name = "PARITY_PASSWORD" if not index else f"PARITY_PASSWORD{index}"
key_name = "PARITY_KEY" if not index else f"PARITY_KEY{index}"
encrypted_key_name = ("PARITY_ENCRYPTED_KEY"
if not index else f"PARITY_ENCRYPTED_KEY{index}")
keyfile_name = "PARITY_KEYFILE" if not index else f"PARITY_KEYFILE{index}"
address = os.getenv(name)
if not address:
return None
pswrd = os.getenv(pswrd_name)
key = os.getenv(key_name)
encr_key = os.getenv(encrypted_key_name)
key_file = os.getenv(keyfile_name)
if key_file and not encr_key:
with open(key_file) as _file:
encr_key = json.loads(_file.read())
from ocean_lib.web3_internal.wallet import Wallet
return Wallet(
Web3Provider.get_web3(),
private_key=key,
encrypted_key=encr_key,
address=Web3.toChecksumAddress(address),
password=pswrd,
)
def delist_ddo(did):
assert request.json and isinstance(request.json,
dict), 'invalid payload format.'
data = request.json
address = data.get('adminAddress', None)
if not address or not has_update_request_permission(address):
return jsonify(error=f'Unauthorized.'), 401
_address = None
signature = data.get('signature', None)
if signature:
_address = get_signer_address(address, signature, logger)
if not _address or _address.lower() != address.lower():
return jsonify(error=f'Unauthorized.'), 401
try:
asset_record = dao.get(did)
if not asset_record:
return jsonify(error=f'Asset {did} not found.'), 404
updater = MetadataUpdater(oceandb=dao.oceandb,
web3=Web3Provider.get_web3(),
config=ConfigProvider.get_config())
updater.do_single_update(asset_record)
return jsonify('acknowledged.'), 200
except Exception as e:
logger.error(f'get_metadata: {str(e)}')
return f'{did} asset DID is not in OceanDB', 404
def pay_for_service(
amount: float,
token_address: str,
did: str,
service_id: int,
fee_receiver: str,
from_wallet: Wallet,
consumer: str,
) -> str:
"""
Submits the payment for chosen service in DataTokens.
:param amount:
:param token_address:
:param did:
:param service_id:
:param fee_receiver:
:param from_wallet: Wallet instance
:param consumer: str the address of consumer of the service, defaults to the payer (the `from_wallet` address)
:return: hex str id of transfer transaction
"""
amount_base = to_base_18(amount)
dt = DataToken(token_address)
balance = dt.balanceOf(from_wallet.address)
if balance < amount_base:
raise AssertionError(
f"Your token balance {balance} is not sufficient "
f"to execute the requested service. This service "
f"requires {amount_base} number of tokens.")
if did.startswith("did:"):
did = add_0x_prefix(did_to_id(did))
if fee_receiver is None:
fee_receiver = ZERO_ADDRESS
if consumer is None:
consumer = from_wallet.address
tx_hash = dt.startOrder(consumer, amount_base, service_id,
fee_receiver, from_wallet)
try:
dt.verify_order_tx(
Web3Provider.get_web3(),
tx_hash,
did,
service_id,
amount_base,
from_wallet.address,
)
return tx_hash
except (AssertionError, Exception) as e:
msg = (
f"Downloading asset files failed. The problem is related to "
f"the transfer of the data tokens required for the download "
f"service: {e}")
logger.error(msg)
raise AssertionError(msg)
def get_ether_balance(address: str) -> int:
"""
Get balance of an ethereum address.
:param address: address, bytes32
:return: balance, int
"""
return Web3Provider.get_web3().eth.getBalance(address, block_identifier="latest")
def add_ethereum_prefix_and_hash_msg(text):
"""
This method of adding the ethereum prefix seems to be used in web3.personal.sign/ecRecover.
:param text: str any str to be signed / used in recovering address from a signature
:return: hash of prefixed text according to the recommended ethereum prefix
"""
prefixed_msg = f"\x19Ethereum Signed Message:\n{len(text)}{text}"
return Web3Provider.get_web3().sha3(text=prefixed_msg)
def test_main(dtfactory_address):
with pytest.raises(AssertionError):
Web3Provider.init_web3(None, None)
Web3Provider.set_web3(None)
assert Web3Provider._web3 is None
assert isinstance(Web3Provider.get_web3(network_url="http://test.test"),
Web3)
assert Web3Provider._web3 is not None
def validate_order(sender, token_address, num_tokens, tx_id, did, service_id):
dt_contract = DataToken(token_address)
try:
amount = to_base_18(num_tokens)
tx, order_event, transfer_event = dt_contract.verify_order_tx(
Web3Provider.get_web3(), tx_id, did, service_id, amount, sender)
return tx, order_event, transfer_event
except AssertionError:
raise
def get_ganache_wallet():
web3 = Web3Provider.get_web3()
if (web3.eth.accounts and web3.eth.accounts[0].lower()
== "0xe2DD09d719Da89e5a3D0F2549c7E24566e947260".lower()):
return Wallet(
web3,
private_key=
"0xfd5c1ccea015b6d663618850824154a3b3fb2882c46cefb05b9a93fea8c3d215",
)
return None
def get_ganache_wallet():
web3 = Web3Provider.get_web3()
if web3.eth.accounts and web3.eth.accounts[0].lower(
) == '0xe2DD09d719Da89e5a3D0F2549c7E24566e947260'.lower():
return Wallet(
web3,
private_key=
'0xc594c6e5def4bab63ac29eed19a134c130388f74f019bc74b8f4389df2837a58'
)
return None
def new_factory_contract():
web3 = Web3Provider.get_web3()
deployer_wallet = get_ganache_wallet()
dt_address = DataToken.deploy(
web3, deployer_wallet, ContractHandler.artifacts_path,
'Template Contract', 'TEMPLATE', deployer_wallet.address,
DataToken.DEFAULT_CAP_BASE, DTFactory.FIRST_BLOB,
deployer_wallet.address)
return DTFactory(
DTFactory.deploy(web3, deployer_wallet, ContractHandler.artifacts_path,
dt_address, deployer_wallet.address))
def generate_multi_value_hash(types, values):
"""
Return the hash of the given list of values.
This is equivalent to packing and hashing values in a solidity smart contract
hence the use of `soliditySha3`.
:param types: list of solidity types expressed as strings
:param values: list of values matching the `types` list
:return: bytes
"""
assert len(types) == len(values)
return Web3Provider.get_web3().soliditySha3(types, values)
def get_creation_block(self, pool_address):
web3 = Web3Provider.get_web3()
bfactory = BFactory(self.bfactory_address)
current_block = web3.eth.blockNumber
logs = bfactory.get_event_logs('BPoolCreated',
0,
current_block,
{'newBPoolAddress': pool_address},
web3=web3,
chunk_size=current_block)
if not logs:
return {}
assert len(logs) == 1, 'cannot happen'
return logs[0].blockNumber
def send_order(client,
ddo,
datatoken,
service,
cons_wallet,
expect_failure=False):
web3 = Web3Provider.get_web3()
init_endpoint = BaseURLs.ASSETS_URL + "/initialize"
# initialize the service
payload = dict({
"documentId": ddo.did,
"serviceId": service.index,
"serviceType": service.type,
"dataToken": datatoken.address,
"consumerAddress": cons_wallet.address,
})
request_url = (init_endpoint + "?" +
"&".join([f"{k}={v}" for k, v in payload.items()]))
response = client.get(request_url)
if expect_failure:
assert response.status == "400 BAD REQUEST"
return
assert response.status == "200 OK"
tx_params = response.json
num_tokens = tx_params["numTokens"]
nonce = tx_params.get("nonce")
receiver = tx_params["to"]
assert tx_params["from"] == cons_wallet.address
assert receiver == get_datatoken_minter(ddo, datatoken.address)
assert tx_params["dataToken"] == ddo.as_dictionary()["dataToken"]
assert nonce is not None, f"expecting a `nonce` value in the response, got {nonce}"
# Transfer tokens to provider account
amount = to_base_18(num_tokens)
tx_id = datatoken.startOrder(
cons_wallet.address,
amount,
service.index,
"0xF9f2DB837b3db03Be72252fAeD2f6E0b73E428b9",
cons_wallet,
)
datatoken.verify_order_tx(web3, tx_id, ddo.asset_id, service.index, amount,
cons_wallet.address)
return tx_id
def get_user_balances(self, user_address, from_block):
web3 = Web3Provider.get_web3()
current_block = web3.eth.blockNumber
pool = BPool(None)
pools = self.get_all_pools(from_block, chunk_size=5000, include_balance=False)
join_logs = pool.get_join_logs(
web3, from_block, current_block, user_address, this_pool_only=False
)
join_logs = [l for l in join_logs if l.address in pools]
balances = {
l.address: DataToken(l.address).token_balance(user_address)
for l in join_logs
}
return balances
def ec_recover(message, signed_message):
"""
This method does not prepend the message with the prefix `\x19Ethereum Signed Message:\n32`.
The caller should add the prefix to the msg/hash before calling this if the signature was
produced for an ethereum-prefixed message.
:param message:
:param signed_message:
:return:
"""
w3 = Web3Provider.get_web3()
v, r, s = split_signature(w3, w3.toBytes(hexstr=signed_message))
signature_object = SignatureFix(vrs=(v, big_endian_to_int(r),
big_endian_to_int(s)))
return w3.eth.account.recoverHash(
message, signature=signature_object.to_hex_v_hacked())
def validate_order(sender, token_address, num_tokens, tx_id, did, service_id):
dt_contract = DataToken(token_address)
amount = to_base_18(num_tokens)
num_tries = 3
i = 0
while i < num_tries:
i += 1
try:
tx, order_event, transfer_event = dt_contract.verify_order_tx(
Web3Provider.get_web3(), tx_id, did, service_id, amount,
sender)
return tx, order_event, transfer_event
except ConnectionClosed:
if i == num_tries:
raise
def check(self, token):
"""
:param token: hex str consist of signature and timestamp
:return: hex str ethereum address
"""
parts = token.split('-')
if len(parts) < 2:
return '0x0'
sig, timestamp = parts
if self._get_timestamp() > (int(timestamp) + self._get_expiration()):
return '0x0'
message = self._get_message(timestamp)
address = Web3Helper.personal_ec_recover(message, sig)
return Web3Provider.get_web3().toChecksumAddress(address)