def test_faucet(self, get):
response = Mock(spec=requests.Response)
response.status_code = 200
pp = PaymentProcessor(self.client, self.privkey, faucet=True)
pp.get_ether_from_faucet()
assert get.call_count == 1
self.addr.encode('hex') in get.call_args[0][0]
def test_synchronized(self):
I = PaymentProcessor.SYNC_CHECK_INTERVAL
PaymentProcessor.SYNC_CHECK_INTERVAL = SYNC_TEST_INTERVAL
pp = PaymentProcessor(self.client, self.privkey, faucet=False)
syncing_status = {
'startingBlock': '0x384',
'currentBlock': '0x386',
'highestBlock': '0x454'
}
combinations = ((0, False), (0, syncing_status), (1, False),
(1, syncing_status), (65, syncing_status), (65, False))
for c in combinations:
print("Subtest {}".format(c))
# Allow reseting the status.
time.sleep(1.5 * PaymentProcessor.SYNC_CHECK_INTERVAL)
self.client.get_peer_count.return_value = 0
self.client.is_syncing.return_value = False
assert not pp.synchronized()
time.sleep(1.5 * PaymentProcessor.SYNC_CHECK_INTERVAL)
self.client.get_peer_count.return_value = c[0]
self.client.is_syncing.return_value = c[1]
assert not pp.synchronized() # First time is always no.
time.sleep(1.5 * PaymentProcessor.SYNC_CHECK_INTERVAL)
assert pp.synchronized() == (c[0] and not c[1])
PaymentProcessor.SYNC_CHECK_INTERVAL = I
def test_gnt_faucet(self):
self.client.call.return_value = '0x00'
pp = PaymentProcessor(self.client, self.privkey, faucet=True)
pp.get_gnt_from_faucet()
assert self.client.send.call_count == 1
tx = self.client.send.call_args[0][0]
assert tx.nonce == self.nonce
assert len(tx.data) == 4
def setUp(self):
DatabaseFixture.setUp(self)
self.privkey = urandom(32)
self.addr = privtoaddr(self.privkey)
self.client = mock.MagicMock(spec=Client)
self.client.get_balance.return_value = 0
self.client.send.side_effect = lambda tx: "0x" + tx.hash.encode('hex')
self.nonce = random.randint(0, 9999)
self.client.get_transaction_count.return_value = self.nonce
# FIXME: PaymentProcessor should be started and stopped!
self.pp = PaymentProcessor(self.client, self.privkey)
self.pp._loopingCall.clock = Clock() # Disable looping call.
def setUp(self):
DatabaseFixture.setUp(self)
self.state = tester.state()
gnt_addr = self.state.evm(decode_hex(TestGNT.INIT_HEX))
self.state.mine()
self.gnt = tester.ABIContract(self.state, TestGNT.ABI, gnt_addr)
PaymentProcessor.TESTGNT_ADDR = gnt_addr
self.privkey = tester.k1
self.client = mock.MagicMock(spec=Client)
self.client.get_peer_count.return_value = 0
self.client.is_syncing.return_value = False
self.client.get_transaction_count.side_effect = \
lambda addr: self.state.block.get_nonce(decode_hex(addr[2:]))
self.client.get_balance.side_effect = \
lambda addr: self.state.block.get_balance(decode_hex(addr[2:]))
def call(_from, to, data, **kw):
# pyethereum does not have direct support for non-mutating calls.
# The implemenation just copies the state and discards it after.
# Here we apply real transaction, but with gas price 0.
# We assume the transaction does not modify the state, but nonce
# will be bumped no matter what.
_from = _from[2:].decode('hex')
data = data[2:].decode('hex')
nonce = self.state.block.get_nonce(_from)
value = kw.get('value', 0)
tx = Transaction(nonce, 0, 100000, to, value, data)
assert _from == tester.a1
tx.sign(tester.k1)
block = kw['block']
assert block == 'pending'
success, output = apply_transaction(self.state.block, tx)
assert success
return '0x' + output.encode('hex')
def send(tx):
success, _ = apply_transaction(self.state.block, tx)
assert success # What happens in real RPC eth_send?
return '0x' + tx.hash.encode('hex')
self.client.call.side_effect = call
self.client.send.side_effect = send
self.pp = PaymentProcessor(self.client, self.privkey)
self.clock = Clock()
self.pp._loopingCall.clock = self.clock
def setUp(self):
DatabaseFixture.setUp(self)
self.addr = encode_hex(privtoaddr(urandom(32)))
self.sci = mock.Mock()
self.sci.GAS_PRICE = 20
self.sci.GAS_PER_PAYMENT = 300
self.sci.GAS_BATCH_PAYMENT_BASE = 30
self.sci.get_eth_balance.return_value = 0
self.sci.get_gntb_balance.return_value = 0
self.sci.get_eth_address.return_value = self.addr
self.sci.get_current_gas_price.return_value = self.sci.GAS_PRICE
latest_block = mock.Mock()
latest_block.gas_limit = 10**10
self.sci.get_latest_block.return_value = latest_block
self.pp = PaymentProcessor(self.sci)
self.pp._gnt_converter = mock.Mock()
self.pp._gnt_converter.is_converting.return_value = False
self.pp._gnt_converter.get_gate_balance.return_value = 0
def __init__(self, datadir, node_priv_key):
""" Create new transaction system instance for node with given id
:param node_priv_key str: node's private key for Ethereum account (32b)
"""
super(EthereumTransactionSystem, self).__init__()
# FIXME: Passing private key all around might be a security issue.
# Proper account managment is needed.
if not isinstance(node_priv_key, basestring)\
or len(node_priv_key) != 32:
raise ValueError("Invalid private key: {}".format(node_priv_key))
self.__node_address = keys.privtoaddr(node_priv_key)
log.info("Node Ethereum address: " + self.get_payment_address())
self.__eth_node = Client(datadir)
self.__proc = PaymentProcessor(self.__eth_node,
node_priv_key,
faucet=True)
self.__proc.start()
def setUp(self):
DatabaseFixture.setUp(self)
self.sci = mock.Mock()
self.sci.GAS_BATCH_PAYMENT_BASE = 10
self.sci.GAS_PER_PAYMENT = 1
self.sci.GAS_PRICE = 20
self.sci.get_gate_address.return_value = None
self.sci.get_current_gas_price.return_value = self.sci.GAS_PRICE
latest_block = mock.Mock()
latest_block.gas_limit = 10**10
self.sci.get_latest_block.return_value = latest_block
self.tx_hash = '0xdead'
self.sci.batch_transfer.return_value = self.tx_hash
self.pp = PaymentProcessor(self.sci)
self.pp._gnt_converter = mock.Mock()
self.pp._gnt_converter.is_converting.return_value = False
self.pp._gnt_converter.get_gate_balance.return_value = 0
def _init(self) -> None:
if len(self._privkey) != 32:
raise Exception(
"Invalid private key. Did you forget to set password?")
eth_addr = privkeytoaddr(self._privkey)
log.info("Node Ethereum address: %s", eth_addr)
self._node.start()
self._sci = new_sci(
self._node.web3,
eth_addr,
self._config.CHAIN,
JsonTransactionsStorage(self._datadir / self.TX_FILENAME),
self._config.CONTRACT_ADDRESSES,
lambda tx: tx.sign(self._privkey),
)
gate_address = self._sci.get_gate_address()
if gate_address is not None:
if self._sci.get_gnt_balance(gate_address):
self._gnt_conversion_status = ConversionStatus.UNFINISHED
self._payment_processor = PaymentProcessor(self._sci)
self._eth_per_payment = self._current_eth_per_payment()
recipients_count = self._payment_processor.recipients_count
if recipients_count > 0:
required_eth = recipients_count * self._eth_per_payment
if required_eth > self._eth_balance:
self._eth_per_payment = self._eth_balance // recipients_count
self._subscribe_to_events()
self._refresh_balances()
log.info(
"Initial balances: %f GNTB, %f GNT, %f ETH",
self._gntb_balance / denoms.ether,
self._gnt_balance / denoms.ether,
self._eth_balance / denoms.ether,
)
class TransactionSystem(LoopingCallService):
""" Transaction system connected with Ethereum """
TX_FILENAME: ClassVar[str] = 'transactions.json'
KEYSTORE_FILENAME: ClassVar[str] = 'wallet.json'
BLOCK_NUMBER_DB_KEY: ClassVar[str] = 'ets_subscriptions_block_number'
LOOP_INTERVAL: ClassVar[int] = 13
def __init__(self, datadir: Path, config) -> None:
super().__init__(self.LOOP_INTERVAL)
datadir.mkdir(exist_ok=True)
self._datadir = datadir
self._config = config
self._privkey = b''
node_list = config.NODE_LIST.copy()
random.shuffle(node_list)
node_list += config.FALLBACK_NODE_LIST
self._node = NodeProcess(node_list)
self._sci: Optional[SmartContractsInterface] = None
self._payments_keeper = PaymentsKeeper()
self._incomes_keeper = IncomesKeeper()
self._payment_processor: Optional[PaymentProcessor] = None
self._gnt_faucet_requested = False
self._gnt_conversion_status = ConversionStatus.NONE
self._concent_withdraw_requested = False
self._eth_balance: int = 0
self._gnt_balance: int = 0
self._gntb_balance: int = 0
self._last_eth_update: Optional[float] = None
self._last_gnt_update: Optional[float] = None
self._payments_locked: int = 0
self._gntb_locked: int = 0
self._gntb_withdrawn: int = 0
# Amortized gas cost per payment used when dealing with locks
self._eth_per_payment: int = 0
@property # type: ignore
@sci_required()
def gas_price(self):
return self._sci.get_current_gas_price()
def backwards_compatibility_tx_storage(self, old_datadir: Path) -> None:
if self.running:
raise Exception(
"Service already started, can't do backwards compatibility")
# Filename is the same as TX_FILENAME, but the constant shouldn't be
# used here as if it ever changes this value below should stay the same.
old_storage_path = old_datadir / 'transactions.json'
if not old_storage_path.exists():
return
log.info(
"Initializing transaction storage from old path: %s",
old_storage_path,
)
new_storage_path = self._datadir / self.TX_FILENAME
if new_storage_path.exists():
raise Exception("Storage already exists, can't override")
with open(old_storage_path, 'r') as f:
json_content = json.load(f)
with open(new_storage_path, 'w') as f:
json.dump(json_content, f)
os.remove(old_storage_path)
def backwards_compatibility_privkey(
self,
privkey: bytes,
password: str) -> None:
keystore_path = self._datadir / self.KEYSTORE_FILENAME
# Sanity check that this is in fact still the same key
if keystore_path.exists():
self.set_password(password)
try:
if privkey != self._privkey:
raise Exception("Private key is not backward compatible")
finally:
self._privkey = b''
return
log.info("Initializing keystore with backward compatible value")
keystore = create_keyfile_json(
privkey,
password.encode('utf-8'),
iterations=1024,
)
with open(keystore_path, 'w') as f:
json.dump(keystore, f)
def _init(self) -> None:
if len(self._privkey) != 32:
raise Exception(
"Invalid private key. Did you forget to set password?")
eth_addr = privkeytoaddr(self._privkey)
log.info("Node Ethereum address: %s", eth_addr)
self._node.start()
self._sci = new_sci(
self._node.web3,
eth_addr,
self._config.CHAIN,
JsonTransactionsStorage(self._datadir / self.TX_FILENAME),
self._config.CONTRACT_ADDRESSES,
lambda tx: tx.sign(self._privkey),
)
gate_address = self._sci.get_gate_address()
if gate_address is not None:
if self._sci.get_gnt_balance(gate_address):
self._gnt_conversion_status = ConversionStatus.UNFINISHED
self._payment_processor = PaymentProcessor(self._sci)
self._eth_per_payment = self._current_eth_per_payment()
recipients_count = self._payment_processor.recipients_count
if recipients_count > 0:
required_eth = recipients_count * self._eth_per_payment
if required_eth > self._eth_balance:
self._eth_per_payment = self._eth_balance // recipients_count
self._subscribe_to_events()
self._refresh_balances()
log.info(
"Initial balances: %f GNTB, %f GNT, %f ETH",
self._gntb_balance / denoms.ether,
self._gnt_balance / denoms.ether,
self._eth_balance / denoms.ether,
)
def start(self, now: bool = True) -> None:
self._init()
super().start(now)
def set_password(self, password: str) -> None:
keystore_path = self._datadir / self.KEYSTORE_FILENAME
if keystore_path.exists():
self._privkey = extract_key_from_keyfile(
str(keystore_path),
password.encode('utf-8'),
)
else:
log.info("Generating new Ethereum private key")
self._privkey = os.urandom(32)
keystore = create_keyfile_json(
self._privkey,
password.encode('utf-8'),
iterations=1024,
)
with open(keystore_path, 'w') as f:
json.dump(keystore, f)
@sci_required()
def _subscribe_to_events(self) -> None:
values = model.GenericKeyValue.select().where(
model.GenericKeyValue.key == self.BLOCK_NUMBER_DB_KEY)
from_block = int(values.get().value) if values.count() == 1 else 0
ik = self._incomes_keeper
self._sci.subscribe_to_batch_transfers(
None,
self._sci.get_eth_address(),
from_block,
lambda event: ik.received_batch_transfer(
event.tx_hash,
event.sender,
event.amount,
event.closure_time,
)
)
# Temporary try-catch block, until GNTDeposit is deployed on mainnet.
# Remove it after that.
try:
self._sci.subscribe_to_forced_subtask_payments(
None,
self._sci.get_eth_address(),
from_block,
lambda event: ik.received_forced_subtask_payment(
event.tx_hash,
event.requestor,
str(bytes32_to_uuid(event.subtask_id)),
event.amount,
)
)
self._sci.subscribe_to_forced_payments(
requestor_address=None,
provider_address=self._sci.get_eth_address(),
from_block=from_block,
cb=lambda event: ik.received_forced_payment(
tx_hash=event.tx_hash,
sender=event.requestor,
amount=event.amount,
closure_time=event.closure_time,
),
)
self._schedule_concent_withdraw()
except AttributeError as e:
log.info("Can't use GNTDeposit on mainnet yet: %r", e)
@sci_required()
def _save_subscription_block_number(self) -> None:
block_number = self._sci.get_block_number() - self._sci.REQUIRED_CONFS
kv, _ = model.GenericKeyValue.get_or_create(
key=self.BLOCK_NUMBER_DB_KEY,
)
kv.value = block_number - 1
kv.save()
def stop(self):
self._payment_processor.sendout(0)
self._save_subscription_block_number()
self._sci.stop()
super().stop()
def add_payment_info(
self,
subtask_id: str,
value: int,
eth_address: str) -> int:
if not self._payment_processor:
raise Exception('Start was not called')
payee = decode_hex(eth_address)
if len(payee) != 20:
raise ValueError(
"Incorrect 'payee' length: {}. Should be 20".format(len(payee)))
payment = model.Payment.create(
subtask=subtask_id,
payee=payee,
value=value,
)
return self._payment_processor.add(payment)
@sci_required()
def get_payment_address(self):
""" Human readable Ethereum address for incoming payments."""
return self._sci.get_eth_address()
def get_payments_list(self):
""" Return list of all planned and made payments
:return list: list of dictionaries describing payments
"""
return self._payments_keeper.get_list_of_all_payments()
@classmethod
def get_deposit_payments_list(cls, limit: int = 1000, offset: int = 0) \
-> List[model.DepositPayment]:
query = model.DepositPayment.select() \
.order_by('id') \
.limit(limit) \
.offset(offset)
return list(query)
def get_subtasks_payments(
self,
subtask_ids: Iterable[str]) -> List[model.Payment]:
return self._payments_keeper.get_subtasks_payments(subtask_ids)
def get_incomes_list(self):
""" Return list of all expected and received incomes
:return list: list of dictionaries describing incomes
"""
return self._incomes_keeper.get_list_of_all_incomes()
def get_available_eth(self) -> int:
return self._eth_balance - self.get_locked_eth()
def get_locked_eth(self) -> int:
if not self._payment_processor:
raise Exception('Start was not called')
payments_num = self._payments_locked + \
self._payment_processor.recipients_count
if payments_num == 0:
return 0
return payments_num * self._eth_per_payment
@sci_required()
def get_available_gnt(self, account_address: Optional[str] = None) -> int:
if (account_address is None) \
or (account_address == self._sci.get_eth_address()):
return self._gntb_balance - self.get_locked_gnt() - \
self._gntb_withdrawn
return self._sci.get_gntb_balance(address=account_address)
def get_locked_gnt(self) -> int:
if not self._payment_processor:
raise Exception('Start was not called')
return self._gntb_locked + self._payment_processor.reserved_gntb
@sci_required()
def get_balance(self) -> Dict[str, Any]:
return {
'gnt_available': self.get_available_gnt(),
'gnt_locked': self.get_locked_gnt(),
'gnt_nonconverted': self._gnt_balance,
'eth_available': self.get_available_eth(),
'eth_locked': self.get_locked_eth(),
'block_number': self._sci.get_block_number(),
'gnt_update_time': self._last_gnt_update,
'eth_update_time': self._last_eth_update,
}
def lock_funds_for_payments(self, price: int, num: int) -> None:
if not self._payment_processor:
raise Exception('Start was not called')
gnt = price * num
if gnt > self.get_available_gnt():
raise exceptions.NotEnoughFunds(
gnt,
self.get_available_gnt(), 'GNT',
)
eth = self.eth_for_batch_payment(num)
eth_available = self.get_available_eth()
if eth > eth_available:
raise exceptions.NotEnoughFunds(eth, eth_available, 'ETH')
log.info(
"Locking %f GNT and ETH for %d payments",
gnt / denoms.ether,
num,
)
locked_eth = self.get_locked_eth()
self._gntb_locked += gnt
self._payments_locked += num
self._eth_per_payment = (eth + locked_eth) // \
(self._payments_locked + self._payment_processor.recipients_count)
def unlock_funds_for_payments(self, price: int, num: int) -> None:
gnt = price * num
if gnt > self._gntb_locked:
raise Exception("Can't unlock {} GNT, locked: {}".format(
gnt / denoms.ether,
self._gntb_locked / denoms.ether,
))
if num > self._payments_locked:
raise Exception("Can't unlock {} payments, locked: {}".format(
num,
self._payments_locked,
))
log.info(
"Unlocking %f GNT and ETH for %d payments",
gnt / denoms.ether,
num,
)
self._gntb_locked -= gnt
self._payments_locked -= num
def expect_income(
self,
sender_node: str,
subtask_id: str,
payer_address: str,
value: int,
accepted_ts: int) -> None:
self._incomes_keeper.expect(
sender_node,
subtask_id,
payer_address,
value,
accepted_ts,
)
def settle_income(
self,
sender_node: str,
subtask_id: str,
settled_ts: int) -> None:
self._incomes_keeper.settled(sender_node, subtask_id, settled_ts)
def eth_for_batch_payment(self, num_payments: int) -> int:
if not self._payment_processor:
raise Exception('Start was not called')
num_payments += self._payments_locked + \
self._payment_processor.recipients_count
required = self._current_eth_per_payment() * num_payments + \
self._eth_base_for_batch_payment()
return required - self.get_locked_eth()
@sci_required()
def _eth_base_for_batch_payment(self) -> int:
return self._sci.GAS_BATCH_PAYMENT_BASE * self._sci.GAS_PRICE
@sci_required()
def _current_eth_per_payment(self) -> int:
gas_price = \
min(self._sci.GAS_PRICE, 2 * self.gas_price) # type: ignore
return gas_price * self._sci.GAS_PER_PAYMENT
@sci_required()
def get_withdraw_gas_cost(
self,
amount: int,
destination: str,
currency: str) -> int:
gas_price = self.gas_price
if currency == 'ETH':
return self._sci.estimate_transfer_eth_gas(destination, amount) * \
gas_price
if currency == 'GNT':
return self._sci.GAS_WITHDRAW * gas_price
raise ValueError('Unknown currency {}'.format(currency))
@sci_required()
def withdraw(
self,
amount: int,
destination: str,
currency: str) -> str:
if not self._config.WITHDRAWALS_ENABLED:
raise Exception("Withdrawals are disabled")
if not is_address(destination):
raise ValueError("{} is not valid ETH address".format(destination))
if currency == 'ETH':
if amount > self.get_available_eth():
raise exceptions.NotEnoughFunds(
amount,
self.get_available_eth(),
currency,
)
log.info(
"Withdrawing %f ETH to %s",
amount / denoms.ether,
destination,
)
return self._sci.transfer_eth(destination, amount)
if currency == 'GNT':
if amount > self.get_available_gnt():
raise exceptions.NotEnoughFunds(
amount,
self.get_available_gnt(),
currency,
)
log.info(
"Withdrawing %f GNT to %s",
amount / denoms.ether,
destination,
)
tx_hash = self._sci.convert_gntb_to_gnt(destination, amount)
def on_receipt(receipt) -> None:
self._gntb_withdrawn -= amount
if not receipt.status:
log.error("Failed GNTB withdrawal: %r", receipt)
self._sci.on_transaction_confirmed(tx_hash, on_receipt)
self._gntb_withdrawn += amount
return tx_hash
raise ValueError('Unknown currency {}'.format(currency))
@sci_required()
def concent_balance(self, account_address: Optional[str] = None) -> int:
if account_address is None:
account_address = self._sci.get_eth_address()
return self._sci.get_deposit_value(
account_address=account_address,
)
@sci_required()
def concent_timelock(self, account_address: Optional[str] = None) -> int:
# FIXME Use decorator to DRY #3190
# possible lock values:
# 0 - locked
# > now - unlocking
# < now - unlocked
if account_address is None:
account_address = self._sci.get_eth_address()
return self._sci.get_deposit_locked_until(
account_address=account_address,
)
@defer.inlineCallbacks
@sci_required()
def concent_deposit(
self,
required: int,
expected: int,
force: bool = False) \
-> Generator[defer.Deferred, TransactionReceipt, Optional[str]]:
current = self.concent_balance()
if current >= required:
return None
required -= current
expected -= current
gntb_balance = self.get_available_gnt()
if gntb_balance < required:
raise exceptions.NotEnoughFunds(required, gntb_balance, 'GNTB')
if self.gas_price >= self._sci.GAS_PRICE: # type: ignore
if not force:
raise exceptions.LongTransactionTime("Gas price too high")
log.warning(
'Gas price is high. It can take some time to mine deposit.',
)
max_possible_amount = min(expected, gntb_balance)
tx_hash = self._sci.deposit_payment(max_possible_amount)
log.info(
"Requested concent deposit of %.6fGNT (tx: %r)",
max_possible_amount / denoms.ether,
tx_hash,
)
dpayment = model.DepositPayment.create(
status=model.PaymentStatus.sent,
value=max_possible_amount,
tx=tx_hash,
)
log.debug('DEPOSIT PAYMENT %s', dpayment)
transaction_receipt = defer.Deferred()
self._sci.on_transaction_confirmed(
tx_hash=tx_hash,
cb=transaction_receipt.callback,
)
receipt = yield transaction_receipt
if not receipt.status:
dpayment.delete_instance()
raise exceptions.DepositError(
"Deposit failed",
transaction_receipt=receipt,
)
tx_gas_price = self._sci.get_transaction_gas_price( # type: ignore
receipt.tx_hash,
)
dpayment.fee = receipt.gas_used * tx_gas_price
dpayment.status = model.PaymentStatus.confirmed
dpayment.save()
return dpayment.tx
@rpc_utils.expose('pay.deposit.relock')
def concent_relock(self):
if self.concent_balance() == 0:
return
self._sci.lock_deposit()
@rpc_utils.expose('pay.deposit.unlock')
def concent_unlock(self):
if self.concent_balance() == 0:
return
tx_hash = self._sci.unlock_deposit()
log.info("Unlocking concent deposit, tx: %s", tx_hash)
def _on_receipt(receipt):
if not receipt.status:
log.error("Transaction failed, %r", receipt)
return
self._schedule_concent_withdraw()
self._sci.on_transaction_confirmed(tx_hash, _on_receipt)
def _schedule_concent_withdraw(self) -> None:
timelock = self.concent_timelock()
if timelock == 0:
return
delay = max(0, timelock - int(time.time()))
call_later(delay, self.concent_withdraw)
def concent_withdraw(self):
if self._concent_withdraw_requested:
return
timelock = self.concent_timelock()
if timelock == 0 or timelock > time.time():
return
tx_hash = self._sci.withdraw_deposit()
self._concent_withdraw_requested = True
def on_confirmed(_receipt) -> None:
self._concent_withdraw_requested = False
self._sci.on_transaction_confirmed(tx_hash, on_confirmed)
log.info("Withdrawing concent deposit, tx: %s", tx_hash)
@sci_required()
def _get_funds_from_faucet(self) -> None:
if not self._config.FAUCET_ENABLED:
return
if self._eth_balance < 0.01 * denoms.ether:
log.info("Requesting tETH from faucet")
tETH_faucet_donate(self._sci.get_eth_address())
return
if self._gnt_balance + self._gntb_balance < 100 * denoms.ether:
if not self._gnt_faucet_requested:
log.info("Requesting GNT from faucet")
self._sci.request_gnt_from_faucet()
self._gnt_faucet_requested = True
else:
self._gnt_faucet_requested = False
@sci_required()
def _refresh_balances(self) -> None:
now = time.mktime(datetime.today().timetuple())
addr = self._sci.get_eth_address()
# Sometimes web3 may throw but it's fine here, we'll just update the
# balances next time
try:
self._eth_balance = self._sci.get_eth_balance(addr)
self._last_eth_update = now
self._gnt_balance = self._sci.get_gnt_balance(addr)
self._gntb_balance = self._sci.get_gntb_balance(addr)
self._last_gnt_update = now
except Exception as e: # pylint: disable=broad-except
log.warning('Failed to update balances: %r', e)
@sci_required()
def _try_convert_gnt(self) -> None: # pylint: disable=too-many-branches
if self._gnt_conversion_status == ConversionStatus.UNFINISHED:
if self._gnt_balance > 0:
self._gnt_conversion_status = ConversionStatus.NONE
else:
gas_cost = self.gas_price * \
self._sci.GAS_TRANSFER_FROM_GATE
if self._eth_balance >= gas_cost:
tx_hash = self._sci.transfer_from_gate()
log.info(
"Finishing previously started GNT conversion %s",
tx_hash,
)
self._gnt_conversion_status = ConversionStatus.TRANSFERRING
else:
log.info(
"Not enough gas to finish GNT conversion, has %.6f,"
" needed: %.6f",
self._eth_balance / denoms.ether,
gas_cost / denoms.ether,
)
return
if self._gnt_balance == 0:
self._gnt_conversion_status = ConversionStatus.NONE
return
gas_price = self.gas_price
gate_address = self._sci.get_gate_address()
if gate_address is None:
gas_cost = gas_price * self._sci.GAS_OPEN_GATE
if self._gnt_conversion_status != ConversionStatus.OPENING_GATE:
if self._eth_balance >= gas_cost:
tx_hash = self._sci.open_gate()
log.info("Opening GNT-GNTB conversion gate %s", tx_hash)
self._gnt_conversion_status = ConversionStatus.OPENING_GATE
else:
log.info(
"Not enough gas for opening conversion gate, has: %.6f,"
" needed: %.6f",
self._eth_balance / denoms.ether,
gas_cost / denoms.ether,
)
return
# This is extra safety check, shouldn't ever happen
if int(gate_address, 16) == 0:
log.critical('Gate address should not equal to %s', gate_address)
return
if self._gnt_conversion_status == ConversionStatus.OPENING_GATE:
self._gnt_conversion_status = ConversionStatus.NONE
gas_cost = gas_price * \
(self._sci.GAS_GNT_TRANSFER + self._sci.GAS_TRANSFER_FROM_GATE)
if self._gnt_conversion_status != ConversionStatus.TRANSFERRING:
if self._eth_balance >= gas_cost:
tx_hash1 = \
self._sci.transfer_gnt(gate_address, self._gnt_balance)
tx_hash2 = self._sci.transfer_from_gate()
log.info(
"Converting %.6f GNT to GNTB %s %s",
self._gnt_balance / denoms.ether,
tx_hash1,
tx_hash2,
)
self._gnt_conversion_status = ConversionStatus.TRANSFERRING
else:
log.info(
"Not enough gas for GNT conversion, has: %.6f,"
" needed: %.6f",
self._eth_balance / denoms.ether,
gas_cost / denoms.ether,
)
def _run(self) -> None:
if not self._payment_processor:
raise Exception('Start was not called')
self._refresh_balances()
self._get_funds_from_faucet()
self._try_convert_gnt()
self._payment_processor.sendout()
self._incomes_keeper.update_overdue_incomes()
class PaymentProcessorInternalTest(DatabaseFixture):
""" In this suite we test internal logic of PaymentProcessor. The final
Ethereum transactions are not inspected.
"""
def setUp(self):
DatabaseFixture.setUp(self)
self.privkey = urandom(32)
self.addr = privtoaddr(self.privkey)
self.client = mock.MagicMock(spec=Client)
self.client.get_balance.return_value = 0
self.client.send.side_effect = lambda tx: "0x" + tx.hash.encode('hex')
self.nonce = random.randint(0, 9999)
self.client.get_transaction_count.return_value = self.nonce
# FIXME: PaymentProcessor should be started and stopped!
self.pp = PaymentProcessor(self.client, self.privkey)
self.pp._loopingCall.clock = Clock() # Disable looping call.
def test_eth_balance(self):
expected_balance = random.randint(0, 2**128 - 1)
self.client.get_balance.return_value = expected_balance
b = self.pp.eth_balance()
assert b == expected_balance
b = self.pp.eth_balance()
assert b == expected_balance
addr_hex = '0x' + self.addr.encode('hex')
self.client.get_balance.assert_called_once_with(addr_hex)
def test_gnt_balance(self):
expected_balance = random.randint(0, 2**128 - 1)
v = '0x{:x}'.format(expected_balance)
self.client.call.return_value = v
b = self.pp.gnt_balance()
assert b == expected_balance
self.client.call.return_value = '0xaa'
b = self.pp.gnt_balance()
assert b == expected_balance
self.client.call.assert_called_once()
def test_eth_balance_refresh(self):
expected_balance = random.randint(0, 2**128 - 1)
self.client.get_balance.return_value = expected_balance
b = self.pp.eth_balance()
assert b == expected_balance
addr_hex = '0x' + self.addr.encode('hex')
self.client.get_balance.assert_called_once_with(addr_hex)
b = self.pp.eth_balance(refresh=True)
assert b == expected_balance
assert self.client.get_balance.call_count == 2
def test_eth_balance_refresh_increase(self):
expected_balance = random.randint(0, 2**127 - 1)
self.client.get_balance.return_value = expected_balance
b = self.pp.eth_balance(refresh=True)
assert b == expected_balance
addr_hex = '0x' + self.addr.encode('hex')
self.client.get_balance.assert_called_once_with(addr_hex)
expected_balance += random.randint(0, 2**127 - 1)
self.client.get_balance.return_value = expected_balance
assert self.pp.eth_balance() == b
b = self.pp.eth_balance(refresh=True)
assert b == expected_balance
assert self.client.get_balance.call_count == 2
def test_balance_refresh_decrease(self):
expected_balance = random.randint(0, 2**127 - 1)
self.client.get_balance.return_value = expected_balance
b = self.pp.eth_balance(refresh=True)
assert b == expected_balance
addr_hex = '0x' + self.addr.encode('hex')
self.client.get_balance.assert_called_once_with(addr_hex)
expected_balance -= random.randint(0, expected_balance)
assert expected_balance >= 0
self.client.get_balance.return_value = expected_balance
b = self.pp.eth_balance(refresh=True)
assert b == expected_balance
assert self.client.get_balance.call_count == 2
def test_available_eth_zero(self):
assert self.pp._eth_available() == 0
def test_available_eth_nonzero(self):
eth = random.randint(0, 10 * denoms.ether)
self.client.get_balance.return_value = eth
assert self.pp._eth_available() == eth
def test_add_failure(self):
a1 = urandom(20)
a2 = urandom(20)
p1 = Payment.create(subtask="p1", payee=a1, value=1)
p2 = Payment.create(subtask="p2", payee=a2, value=2)
assert p1.status is PaymentStatus.awaiting
assert p2.status is PaymentStatus.awaiting
self.client.get_balance.return_value = 0
assert self.pp.add(p1) is False
assert self.pp.add(p2) is False
self.client.get_balance.assert_called_once_with(
'0x' + self.addr.encode('hex'))
assert p1.status is PaymentStatus.awaiting
assert p2.status is PaymentStatus.awaiting
def test_add_invalid_payment_status(self):
a1 = urandom(20)
p1 = Payment.create(subtask="p1",
payee=a1,
value=1,
status=PaymentStatus.confirmed)
assert p1.status is PaymentStatus.confirmed
with self.assertRaises(RuntimeError):
self.pp.add(p1)
@patch('requests.get')
def test_faucet(self, get):
response = Mock(spec=requests.Response)
response.status_code = 200
pp = PaymentProcessor(self.client, self.privkey, faucet=True)
pp.get_ether_from_faucet()
assert get.call_count == 1
self.addr.encode('hex') in get.call_args[0][0]
def test_gnt_faucet(self):
self.client.call.return_value = '0x00'
pp = PaymentProcessor(self.client, self.privkey, faucet=True)
pp.get_gnt_from_faucet()
assert self.client.send.call_count == 1
tx = self.client.send.call_args[0][0]
assert tx.nonce == self.nonce
assert len(tx.data) == 4
def test_faucet_gimme_money(self):
assert self.pp.eth_balance() == 0
value = 12 * denoms.ether
Faucet.gimme_money(self.client, self.addr, value)
def test_payment_aggregation(self):
a1 = urandom(20)
a2 = urandom(20)
a3 = urandom(20)
self.client.get_balance.return_value = 100 * denoms.ether
self.client.call.return_value = hex(100 * denoms.ether)[:-1]
assert self.pp.add(Payment.create(subtask="p1", payee=a1, value=1))
assert self.pp.add(Payment.create(subtask="p2", payee=a2, value=1))
assert self.pp.add(Payment.create(subtask="p3", payee=a2, value=1))
assert self.pp.add(Payment.create(subtask="p4", payee=a3, value=1))
assert self.pp.add(Payment.create(subtask="p5", payee=a3, value=1))
assert self.pp.add(Payment.create(subtask="p6", payee=a3, value=1))
self.pp.deadline = int(time.time())
assert self.pp.sendout()
assert self.client.send.call_count == 1
tx = self.client.send.call_args[0][0]
assert tx.value == 0
assert len(
tx.data) == 4 + 2 * 32 + 3 * 32 # Id + array abi + bytes32[3]
def test_payment_deadline(self):
a1 = urandom(20)
a2 = urandom(20)
a3 = urandom(20)
self.client.get_balance.return_value = 100 * denoms.ether
self.client.call.return_value = hex(100 * denoms.ether)[:-1]
now = int(time.time())
assert self.pp.add(Payment.create(subtask="p1", payee=a1, value=1))
assert check_deadline(self.pp.deadline, now + self.pp.DEFAULT_DEADLINE)
assert self.pp.add(Payment.create(subtask="p2", payee=a2, value=1),
deadline=20000)
assert check_deadline(self.pp.deadline, now + self.pp.DEFAULT_DEADLINE)
assert self.pp.add(Payment.create(subtask="p3", payee=a2, value=1),
deadline=1)
assert check_deadline(self.pp.deadline, now + 1)
assert self.pp.add(Payment.create(subtask="p4", payee=a3, value=1))
assert check_deadline(self.pp.deadline, now + 1)
assert self.pp.add(Payment.create(subtask="p5", payee=a3, value=1),
deadline=1)
assert check_deadline(self.pp.deadline, now + 1)
assert self.pp.add(Payment.create(subtask="p6", payee=a3, value=1),
deadline=0)
assert check_deadline(self.pp.deadline, now)
assert self.pp.add(Payment.create(subtask="p7", payee=a3, value=1),
deadline=-1)
assert check_deadline(self.pp.deadline, now - 1)
def test_payment_deadline_not_reached(self):
a1 = urandom(20)
self.client.get_balance.return_value = 100 * denoms.ether
self.client.call.return_value = hex(100 * denoms.ether)[:-1]
now = int(time.time())
inf = now + 12 * 30 * 24 * 60 * 60
deadline = self.pp.deadline
assert self.pp.deadline > inf
assert not self.pp.sendout()
assert self.pp.deadline == deadline
p = Payment.create(subtask="p1", payee=a1, value=1111)
assert self.pp.add(p, deadline=1111)
assert check_deadline(self.pp.deadline, now + 1111)
assert not self.pp.sendout()
assert check_deadline(self.pp.deadline, now + 1111)
def test_synchronized(self):
I = PaymentProcessor.SYNC_CHECK_INTERVAL
PaymentProcessor.SYNC_CHECK_INTERVAL = SYNC_TEST_INTERVAL
pp = PaymentProcessor(self.client, self.privkey, faucet=False)
syncing_status = {
'startingBlock': '0x384',
'currentBlock': '0x386',
'highestBlock': '0x454'
}
combinations = ((0, False), (0, syncing_status), (1, False),
(1, syncing_status), (65, syncing_status), (65, False))
for c in combinations:
print("Subtest {}".format(c))
# Allow reseting the status.
time.sleep(1.5 * PaymentProcessor.SYNC_CHECK_INTERVAL)
self.client.get_peer_count.return_value = 0
self.client.is_syncing.return_value = False
assert not pp.synchronized()
time.sleep(1.5 * PaymentProcessor.SYNC_CHECK_INTERVAL)
self.client.get_peer_count.return_value = c[0]
self.client.is_syncing.return_value = c[1]
assert not pp.synchronized() # First time is always no.
time.sleep(1.5 * PaymentProcessor.SYNC_CHECK_INTERVAL)
assert pp.synchronized() == (c[0] and not c[1])
PaymentProcessor.SYNC_CHECK_INTERVAL = I
def test_synchronized_unstable(self):
I = PaymentProcessor.SYNC_CHECK_INTERVAL
PaymentProcessor.SYNC_CHECK_INTERVAL = SYNC_TEST_INTERVAL
pp = PaymentProcessor(self.client, self.privkey, faucet=False)
syncing_status = {
'startingBlock': '0x0',
'currentBlock': '0x1',
'highestBlock': '0x4096'
}
self.client.get_peer_count.return_value = 1
self.client.is_syncing.return_value = False
assert not pp.synchronized()
time.sleep(1.5 * PaymentProcessor.SYNC_CHECK_INTERVAL)
self.client.get_peer_count.return_value = 1
self.client.is_syncing.return_value = syncing_status
assert not pp.synchronized()
time.sleep(1.5 * PaymentProcessor.SYNC_CHECK_INTERVAL)
assert not pp.synchronized()
self.client.get_peer_count.return_value = 1
self.client.is_syncing.return_value = False
assert not pp.synchronized()
time.sleep(1.5 * PaymentProcessor.SYNC_CHECK_INTERVAL)
assert not pp.synchronized()
time.sleep(1.5 * PaymentProcessor.SYNC_CHECK_INTERVAL)
assert pp.synchronized()
time.sleep(1.5 * PaymentProcessor.SYNC_CHECK_INTERVAL)
self.client.get_peer_count.return_value = 0
self.client.is_syncing.return_value = False
assert not pp.synchronized()
time.sleep(1.5 * PaymentProcessor.SYNC_CHECK_INTERVAL)
self.client.get_peer_count.return_value = 2
self.client.is_syncing.return_value = False
assert not pp.synchronized()
time.sleep(1.5 * PaymentProcessor.SYNC_CHECK_INTERVAL)
assert pp.synchronized()
time.sleep(1.5 * PaymentProcessor.SYNC_CHECK_INTERVAL)
self.client.get_peer_count.return_value = 2
self.client.is_syncing.return_value = syncing_status
assert not pp.synchronized()
PaymentProcessor.SYNC_CHECK_INTERVAL = I
def test_monitor_progress(self):
a1 = urandom(20)
inprogress = self.pp._inprogress
# Give 1 ETH and 99 GNT
balance_eth = 1 * denoms.ether
balance_gnt = 99 * denoms.ether
self.client.get_balance.return_value = balance_eth
self.client.call.return_value = hex(balance_gnt)[:-1] # Skip L suffix.
assert self.pp._gnt_reserved() == 0
assert self.pp._gnt_available() == balance_gnt
assert self.pp._eth_reserved() == 0
assert self.pp._eth_available() == balance_eth
gnt_value = 10**17
p = Payment.create(subtask="p1", payee=a1, value=gnt_value)
assert self.pp.add(p)
assert self.pp._gnt_reserved() == gnt_value
assert self.pp._gnt_available() == balance_gnt - gnt_value
assert self.pp._eth_reserved(
) == PaymentProcessor.SINGLE_PAYMENT_ETH_COST
assert self.pp._eth_available(
) == balance_eth - PaymentProcessor.SINGLE_PAYMENT_ETH_COST
self.pp.deadline = int(time.time())
assert self.pp.sendout()
assert self.client.send.call_count == 1
tx = self.client.send.call_args[0][0]
assert tx.value == 0
assert len(tx.data) == 4 + 2 * 32 + 32 # Id + array abi + bytes32[1]
assert len(inprogress) == 1
assert tx.hash in inprogress
assert inprogress[tx.hash] == [p]
# Check payment status in the Blockchain
self.client.get_transaction_receipt.return_value = None
self.client.call.return_value = hex(balance_gnt -
gnt_value)[:-1] # Skip L suffix.
self.pp.monitor_progress()
assert len(inprogress) == 1
assert tx.hash in inprogress
assert inprogress[tx.hash] == [p]
assert self.pp.gnt_balance(True) == balance_gnt - gnt_value
assert self.pp._gnt_reserved() == 0
assert self.pp._gnt_available() == balance_gnt - gnt_value
assert self.pp._eth_reserved(
) == PaymentProcessor.SINGLE_PAYMENT_ETH_COST
assert self.pp._eth_available(
) == balance_eth - PaymentProcessor.SINGLE_PAYMENT_ETH_COST
self.pp.monitor_progress()
assert len(inprogress) == 1
assert self.pp._gnt_reserved() == 0
assert self.pp._gnt_available() == balance_gnt - gnt_value
assert self.pp._eth_reserved(
) == PaymentProcessor.SINGLE_PAYMENT_ETH_COST
assert self.pp._eth_available(
) == balance_eth - PaymentProcessor.SINGLE_PAYMENT_ETH_COST
receipt = {
'blockNumber': 8214,
'blockHash': '0x' + 64 * 'f',
'gasUsed': 55001
}
self.client.get_transaction_receipt.return_value = receipt
self.pp.monitor_progress()
assert len(inprogress) == 0
assert p.status == PaymentStatus.confirmed
assert p.details['block_number'] == 8214
assert p.details['block_hash'] == 64 * 'f'
assert p.details['fee'] == 55001 * self.pp.GAS_PRICE
assert self.pp._gnt_reserved() == 0
class PaymentProcessorFunctionalTest(DatabaseFixture):
""" In this suite we test Ethereum state changes done by PaymentProcessor.
"""
def setUp(self):
DatabaseFixture.setUp(self)
self.state = tester.state()
gnt_addr = self.state.evm(decode_hex(TestGNT.INIT_HEX))
self.state.mine()
self.gnt = tester.ABIContract(self.state, TestGNT.ABI, gnt_addr)
PaymentProcessor.TESTGNT_ADDR = gnt_addr
self.privkey = tester.k1
self.client = mock.MagicMock(spec=Client)
self.client.get_peer_count.return_value = 0
self.client.is_syncing.return_value = False
self.client.get_transaction_count.side_effect = \
lambda addr: self.state.block.get_nonce(decode_hex(addr[2:]))
self.client.get_balance.side_effect = \
lambda addr: self.state.block.get_balance(decode_hex(addr[2:]))
def call(_from, to, data, **kw):
# pyethereum does not have direct support for non-mutating calls.
# The implemenation just copies the state and discards it after.
# Here we apply real transaction, but with gas price 0.
# We assume the transaction does not modify the state, but nonce
# will be bumped no matter what.
_from = _from[2:].decode('hex')
data = data[2:].decode('hex')
nonce = self.state.block.get_nonce(_from)
value = kw.get('value', 0)
tx = Transaction(nonce, 0, 100000, to, value, data)
assert _from == tester.a1
tx.sign(tester.k1)
block = kw['block']
assert block == 'pending'
success, output = apply_transaction(self.state.block, tx)
assert success
return '0x' + output.encode('hex')
def send(tx):
success, _ = apply_transaction(self.state.block, tx)
assert success # What happens in real RPC eth_send?
return '0x' + tx.hash.encode('hex')
self.client.call.side_effect = call
self.client.send.side_effect = send
self.pp = PaymentProcessor(self.client, self.privkey)
self.clock = Clock()
self.pp._loopingCall.clock = self.clock
def test_initial_eth_balance(self):
# ethereum.tester assigns this amount to predefined accounts.
assert self.pp.eth_balance() == 1000000000000000000000000
def check_synchronized(self):
assert not self.pp.synchronized()
self.client.get_peer_count.return_value = 1
assert not self.pp.synchronized()
I = PaymentProcessor.SYNC_CHECK_INTERVAL = SYNC_TEST_INTERVAL
assert self.pp.SYNC_CHECK_INTERVAL == SYNC_TEST_INTERVAL
time.sleep(1.5 * PaymentProcessor.SYNC_CHECK_INTERVAL)
assert not self.pp.synchronized()
time.sleep(1.5 * PaymentProcessor.SYNC_CHECK_INTERVAL)
assert self.pp.synchronized()
PaymentProcessor.SYNC_CHECK_INTERVAL = I
def test_synchronized(self):
self.pp.start()
self.check_synchronized()
self.pp.stop()
def test_gnt_faucet(self, *_):
self.pp._PaymentProcessor__faucet = True
self.pp._run()
assert self.pp.eth_balance() > 0
assert self.pp.gnt_balance() == 0
self.check_synchronized()
self.state.mine()
self.clock.advance(60)
self.pp._run()
assert self.pp.gnt_balance(True) == 1000 * denoms.ether
def test_single_payment(self, *_):
self.pp._run()
self.gnt.create(sender=self.privkey)
self.state.mine()
self.check_synchronized()
assert self.pp.gnt_balance() == 1000 * denoms.ether
payee = urandom(20)
b = self.pp.gnt_balance()
# FIXME: Big values does not fit into the database
value = random.randint(0, b / 1000)
p1 = Payment.create(subtask="p1", payee=payee, value=value)
assert self.pp._gnt_available() == b
assert self.pp._gnt_reserved() == 0
self.pp.add(p1)
assert self.pp._gnt_available() == b - value
assert self.pp._gnt_reserved() == value
# Sendout.
self.pp.deadline = int(time.time())
self.pp._run()
assert self.pp.gnt_balance(True) == b - value
assert self.pp._gnt_available() == b - value
assert self.pp._gnt_reserved() == 0
assert self.gnt.balanceOf(payee) == value
assert self.gnt.balanceOf(tester.a1) == self.pp._gnt_available()
# Confirm.
assert self.pp.gnt_balance(True) == b - value
assert self.pp._gnt_reserved() == 0
def test_get_ether(self, *_):
def exception(*_):
raise Exception
def failure(*_):
return False
def success(*_):
return True
self.pp.monitor_progress = Mock()
self.pp.synchronized = lambda *_: True
self.pp.sendout = Mock()
self.pp.get_gnt_from_faucet = failure
self.pp.get_ether_from_faucet = failure
self.pp._run()
assert not self.pp._waiting_for_faucet
assert not self.pp.monitor_progress.called
assert not self.pp.sendout.called
self.pp.get_ether_from_faucet = success
self.pp._run()
assert not self.pp._waiting_for_faucet
assert not self.pp.monitor_progress.called
assert not self.pp.sendout.called
self.pp.get_gnt_from_faucet = success
self.pp._run()
assert not self.pp._waiting_for_faucet
assert self.pp.monitor_progress.called
assert self.pp.sendout.called
def test_no_gnt_available(self):
self.pp.start()
self.gnt.create(sender=self.privkey)
self.state.mine()
self.check_synchronized()
assert self.pp.gnt_balance() == 1000 * denoms.ether
payee = urandom(20)
b = self.pp.gnt_balance()
value = b / 5 - 100
for i in range(5):
subtask_id = 's{}'.format(i)
p = Payment.create(subtask=subtask_id, payee=payee, value=value)
assert self.pp.add(p)
q = Payment.create(subtask='F', payee=payee, value=value)
assert not self.pp.add(q)
def test_synchronized_unstable(self):
I = PaymentProcessor.SYNC_CHECK_INTERVAL
PaymentProcessor.SYNC_CHECK_INTERVAL = SYNC_TEST_INTERVAL
pp = PaymentProcessor(self.client, self.privkey, faucet=False)
syncing_status = {
'startingBlock': '0x0',
'currentBlock': '0x1',
'highestBlock': '0x4096'
}
self.client.get_peer_count.return_value = 1
self.client.is_syncing.return_value = False
assert not pp.synchronized()
time.sleep(1.5 * PaymentProcessor.SYNC_CHECK_INTERVAL)
self.client.get_peer_count.return_value = 1
self.client.is_syncing.return_value = syncing_status
assert not pp.synchronized()
time.sleep(1.5 * PaymentProcessor.SYNC_CHECK_INTERVAL)
assert not pp.synchronized()
self.client.get_peer_count.return_value = 1
self.client.is_syncing.return_value = False
assert not pp.synchronized()
time.sleep(1.5 * PaymentProcessor.SYNC_CHECK_INTERVAL)
assert not pp.synchronized()
time.sleep(1.5 * PaymentProcessor.SYNC_CHECK_INTERVAL)
assert pp.synchronized()
time.sleep(1.5 * PaymentProcessor.SYNC_CHECK_INTERVAL)
self.client.get_peer_count.return_value = 0
self.client.is_syncing.return_value = False
assert not pp.synchronized()
time.sleep(1.5 * PaymentProcessor.SYNC_CHECK_INTERVAL)
self.client.get_peer_count.return_value = 2
self.client.is_syncing.return_value = False
assert not pp.synchronized()
time.sleep(1.5 * PaymentProcessor.SYNC_CHECK_INTERVAL)
assert pp.synchronized()
time.sleep(1.5 * PaymentProcessor.SYNC_CHECK_INTERVAL)
self.client.get_peer_count.return_value = 2
self.client.is_syncing.return_value = syncing_status
assert not pp.synchronized()
PaymentProcessor.SYNC_CHECK_INTERVAL = I
class EthereumTransactionSystem(TransactionSystem):
""" Transaction system connected with Ethereum """
def __init__(self, datadir, node_priv_key):
""" Create new transaction system instance for node with given id
:param node_priv_key str: node's private key for Ethereum account (32b)
"""
super(EthereumTransactionSystem, self).__init__()
# FIXME: Passing private key all around might be a security issue.
# Proper account managment is needed.
if not isinstance(node_priv_key, basestring)\
or len(node_priv_key) != 32:
raise ValueError("Invalid private key: {}".format(node_priv_key))
self.__node_address = keys.privtoaddr(node_priv_key)
log.info("Node Ethereum address: " + self.get_payment_address())
self.__eth_node = Client(datadir)
self.__proc = PaymentProcessor(self.__eth_node,
node_priv_key,
faucet=True)
self.__proc.start()
def stop(self):
if self.__proc.running:
self.__proc.stop()
if self.__eth_node.node is not None:
self.__eth_node.node.stop()
def add_payment_info(self, *args, **kwargs):
payment = super(EthereumTransactionSystem,
self).add_payment_info(*args, **kwargs)
self.__proc.add(payment)
return payment
def get_payment_address(self):
""" Human readable Ethereum address for incoming payments."""
return '0x' + self.__node_address.encode('hex')
def get_balance(self):
if not self.__proc.balance_known():
return None, None, None
gnt = self.__proc.gnt_balance()
av_gnt = self.__proc._gnt_available()
eth = self.__proc.eth_balance()
return gnt, av_gnt, eth
def pay_for_task(self, task_id, payments):
""" Pay for task using Ethereum connector
:param task_id: pay for task with given id
:param dict payments: all payments group by ethereum address
"""
pass
def sync(self):
syncing = True
while syncing:
try:
syncing = self.__eth_node.is_syncing()
except Exception as e:
log.error("IPC error: {}".format(e))
syncing = False
else:
sleep(0.5)
class PaymentProcessorInternalTest(DatabaseFixture):
""" In this suite we test internal logic of PaymentProcessor. The final
Ethereum transactions are not inspected.
"""
def setUp(self):
DatabaseFixture.setUp(self)
self.addr = encode_hex(privtoaddr(urandom(32)))
self.sci = mock.Mock()
self.sci.GAS_PRICE = 20
self.sci.GAS_PER_PAYMENT = 300
self.sci.GAS_BATCH_PAYMENT_BASE = 30
self.sci.get_eth_balance.return_value = 0
self.sci.get_gntb_balance.return_value = 0
self.sci.get_eth_address.return_value = self.addr
self.sci.get_current_gas_price.return_value = self.sci.GAS_PRICE
latest_block = mock.Mock()
latest_block.gas_limit = 10**10
self.sci.get_latest_block.return_value = latest_block
self.pp = PaymentProcessor(self.sci)
self.pp._gnt_converter = mock.Mock()
self.pp._gnt_converter.is_converting.return_value = False
self.pp._gnt_converter.get_gate_balance.return_value = 0
def test_load_from_db_awaiting(self):
self.assertEqual([], self.pp._awaiting)
value = 10
payment = Payment.create(
subtask=str(uuid.uuid4()),
payee=urandom(20),
value=value,
)
self.pp.load_from_db()
expected = [payment]
self.assertEqual(expected, self.pp._awaiting)
self.assertEqual(value, self.pp.reserved_gntb)
self.assertLess(0, self.pp.recipients_count)
def test_load_from_db_sent(self):
tx_hash1 = encode_hex(urandom(32))
tx_hash2 = encode_hex(urandom(32))
value = 10
payee = urandom(20)
sent_payment11 = Payment.create(
subtask=str(uuid.uuid4()),
payee=payee,
value=value,
details=PaymentDetails(tx=tx_hash1[2:]),
status=PaymentStatus.sent)
sent_payment12 = Payment.create(
subtask=str(uuid.uuid4()),
payee=payee,
value=value,
details=PaymentDetails(tx=tx_hash1[2:]),
status=PaymentStatus.sent)
sent_payment21 = Payment.create(
subtask=str(uuid.uuid4()),
payee=payee,
value=value,
details=PaymentDetails(tx=tx_hash2[2:]),
status=PaymentStatus.sent)
self.pp.load_from_db()
self.assertEqual(3 * value, self.pp.reserved_gntb)
self.assertEqual(0, self.pp.recipients_count)
assert self.sci.on_transaction_confirmed.call_count == 2
assert self.sci.on_transaction_confirmed.call_args_list[0][0][0] == \
tx_hash1
assert self.sci.on_transaction_confirmed.call_args_list[1][0][0] == \
tx_hash2
with mock.patch('golem.ethereum.paymentprocessor.threads') as threads:
self.sci.on_transaction_confirmed.call_args_list[0][0][1](
mock.Mock())
threads.deferToThread.assert_called_once_with(
self.pp._on_batch_confirmed,
[sent_payment11, sent_payment12],
mock.ANY,
)
threads.reset_mock()
self.sci.on_transaction_confirmed.call_args_list[1][0][1](
mock.Mock())
threads.deferToThread.assert_called_once_with(
self.pp._on_batch_confirmed,
[sent_payment21],
mock.ANY,
)
def test_recipients_count(self):
assert self.pp.recipients_count == 0
def test_add_invalid_payment_status(self):
a1 = urandom(20)
p1 = Payment.create(subtask="p1",
payee=a1,
value=1,
status=PaymentStatus.confirmed)
assert p1.status is PaymentStatus.confirmed
with self.assertRaises(RuntimeError):
self.pp.add(p1)
def test_monitor_progress(self):
balance_eth = 1 * denoms.ether
balance_gntb = 99 * denoms.ether
gas_price = 10**9
self.sci.get_eth_balance.return_value = balance_eth
self.sci.get_gntb_balance.return_value = balance_gntb
self.sci.get_transaction_gas_price.return_value = gas_price
self.pp.CLOSURE_TIME_DELAY = 0
assert self.pp.reserved_gntb == 0
assert self.pp.recipients_count == 0
gnt_value = 10**17
p = Payment.create(subtask="p1", payee=urandom(20), value=gnt_value)
self.pp.add(p)
assert self.pp.reserved_gntb == gnt_value
assert self.pp.recipients_count == 1
tx_hash = '0xdead'
self.sci.batch_transfer.return_value = tx_hash
assert self.pp.sendout(0)
assert self.sci.batch_transfer.call_count == 1
self.sci.on_transaction_confirmed.assert_called_once_with(
tx_hash,
mock.ANY,
)
tx_block_number = 1337
self.sci.get_block_number.return_value = tx_block_number
receipt = TransactionReceipt({
'transactionHash': HexBytes(tx_hash),
'blockNumber': tx_block_number,
'blockHash': HexBytes('0x' + 64 * 'f'),
'gasUsed': 55001,
'status': 1,
})
with mock.patch('golem.ethereum.paymentprocessor.threads') as threads:
self.sci.on_transaction_confirmed.call_args[0][1](receipt)
threads.deferToThread.assert_called_once_with(
self.pp._on_batch_confirmed,
[p],
receipt,
)
self.pp._on_batch_confirmed([p], receipt)
self.assertEqual(p.status, PaymentStatus.confirmed)
self.assertEqual(p.details.block_number, tx_block_number)
self.assertEqual(p.details.block_hash, 64 * 'f')
self.assertEqual(p.details.fee, 55001 * gas_price)
self.assertEqual(self.pp.reserved_gntb, 0)
def test_failed_transaction(self):
balance_eth = 1 * denoms.ether
balance_gntb = 99 * denoms.ether
self.sci.get_eth_balance.return_value = balance_eth
self.sci.get_gntb_balance.return_value = balance_gntb
gnt_value = 10**17
p = Payment.create(subtask="p1", payee=urandom(20), value=gnt_value)
self.pp.add(p)
self.pp.CLOSURE_TIME_DELAY = 0
tx_hash = '0xdead'
self.sci.batch_transfer.return_value = tx_hash
assert self.pp.sendout(0)
tx_block_number = 1337
receipt = TransactionReceipt({
'transactionHash': HexBytes(tx_hash),
'blockNumber': tx_block_number,
'blockHash': HexBytes('0x' + 64 * 'f'),
'gasUsed': 55001,
'status': 0,
})
with mock.patch('golem.ethereum.paymentprocessor.threads') as threads:
self.sci.on_transaction_confirmed.call_args[0][1](receipt)
threads.deferToThread.assert_called_once_with(
self.pp._on_batch_confirmed,
[p],
receipt,
)
self.pp._on_batch_confirmed([p], receipt)
self.assertEqual(p.status, PaymentStatus.awaiting)
assert len(self.pp._awaiting) == 1
def test_payment_timestamp(self):
self.sci.get_eth_balance.return_value = denoms.ether
ts = 7000000
p = Payment.create(subtask="p1", payee=urandom(20), value=1)
with freeze_time(timestamp_to_datetime(ts)):
self.pp.add(p)
self.assertEqual(ts, p.processed_ts)
new_ts = 900000
with freeze_time(timestamp_to_datetime(new_ts)):
self.pp.add(p)
self.assertEqual(ts, p.processed_ts)
class InteractionWithSmartContractInterfaceTest(DatabaseFixture):
def setUp(self):
DatabaseFixture.setUp(self)
self.sci = mock.Mock()
self.sci.GAS_BATCH_PAYMENT_BASE = 10
self.sci.GAS_PER_PAYMENT = 1
self.sci.GAS_PRICE = 20
self.sci.get_gate_address.return_value = None
self.sci.get_current_gas_price.return_value = self.sci.GAS_PRICE
latest_block = mock.Mock()
latest_block.gas_limit = 10**10
self.sci.get_latest_block.return_value = latest_block
self.tx_hash = '0xdead'
self.sci.batch_transfer.return_value = self.tx_hash
self.pp = PaymentProcessor(self.sci)
self.pp._gnt_converter = mock.Mock()
self.pp._gnt_converter.is_converting.return_value = False
self.pp._gnt_converter.get_gate_balance.return_value = 0
def _assert_batch_transfer_called_with(self, payments,
closure_time: int) -> None:
self.sci.batch_transfer.assert_called_with(mock.ANY, closure_time)
called_payments = self.sci.batch_transfer.call_args[0][0]
assert len(called_payments) == len(payments)
for expected, actual in zip(payments, called_payments):
assert expected.payee == actual.payee
assert expected.amount == actual.amount
def test_batch_transfer(self):
deadline = PAYMENT_MAX_DELAY
self.pp.CLOSURE_TIME_DELAY = 0
self.sci.get_eth_balance.return_value = denoms.ether
self.sci.get_gnt_balance.return_value = 0
self.sci.get_gntb_balance.return_value = 1000 * denoms.ether
assert not self.pp.sendout()
self.sci.batch_transfer.assert_not_called()
ts1 = 1230000
ts2 = ts1 + 2 * deadline
p1, scip1 = make_awaiting_payment(ts=ts1)
p2, scip2 = make_awaiting_payment(ts=ts2)
self.pp.add(p1)
self.pp.add(p2)
with freeze_time(timestamp_to_datetime(ts1 + deadline - 1)):
assert not self.pp.sendout()
self.sci.batch_transfer.assert_not_called()
with freeze_time(timestamp_to_datetime(ts1 + deadline + 1)):
assert self.pp.sendout()
self._assert_batch_transfer_called_with(
[scip1],
ts1,
)
self.sci.batch_transfer.reset_mock()
with freeze_time(timestamp_to_datetime(ts2 + deadline - 1)):
assert not self.pp.sendout()
self.sci.batch_transfer.assert_not_called()
with freeze_time(timestamp_to_datetime(ts2 + deadline + 1)):
assert self.pp.sendout()
self._assert_batch_transfer_called_with(
[scip2],
ts2,
)
self.sci.batch_transfer.reset_mock()
def test_closure_time(self):
self.sci.get_eth_balance.return_value = denoms.ether
self.sci.get_gnt_balance.return_value = 0
self.sci.get_gntb_balance.return_value = 1000 * denoms.ether
p1, scip1 = make_awaiting_payment()
p2, scip2 = make_awaiting_payment()
p5, scip5 = make_awaiting_payment()
with freeze_time(timestamp_to_datetime(1000000)):
self.pp.add(p1)
with freeze_time(timestamp_to_datetime(2000000)):
self.pp.add(p2)
with freeze_time(timestamp_to_datetime(5000000)):
self.pp.add(p5)
closure_time = 2000000
time_value = closure_time + self.pp.CLOSURE_TIME_DELAY
with freeze_time(timestamp_to_datetime(time_value)):
self.pp.sendout(0)
self._assert_batch_transfer_called_with([scip1, scip2],
closure_time)
self.sci.batch_transfer.reset_mock()
closure_time = 4000000
time_value = closure_time + self.pp.CLOSURE_TIME_DELAY
with freeze_time(timestamp_to_datetime(time_value)):
self.pp.sendout(0)
self.sci.batch_transfer.assert_not_called()
self.sci.batch_transfer.reset_mock()
closure_time = 5000000
time_value = closure_time + self.pp.CLOSURE_TIME_DELAY
with freeze_time(timestamp_to_datetime(time_value)):
self.pp.sendout(0)
self._assert_batch_transfer_called_with([scip5], closure_time)
self.sci.batch_transfer.reset_mock()
def test_short_on_gnt(self):
self.sci.get_eth_balance.return_value = denoms.ether
self.sci.get_gnt_balance.return_value = 0
self.sci.get_gntb_balance.return_value = 4 * denoms.ether
self.pp.CLOSURE_TIME_DELAY = 0
p1, scip1 = make_awaiting_payment(value=1 * denoms.ether, ts=1)
p2, scip2 = make_awaiting_payment(value=2 * denoms.ether, ts=2)
p5, scip5 = make_awaiting_payment(value=5 * denoms.ether, ts=3)
self.pp.add(p1)
self.pp.add(p2)
self.pp.add(p5)
with freeze_time(timestamp_to_datetime(10000)):
self.pp.sendout(0)
self._assert_batch_transfer_called_with([scip1, scip2], 2)
self.sci.batch_transfer.reset_mock()
self.sci.get_gntb_balance.return_value = 5 * denoms.ether
with freeze_time(timestamp_to_datetime(10000)):
self.pp.sendout(0)
self._assert_batch_transfer_called_with([scip5], 3)
self.sci.batch_transfer.reset_mock()
def test_short_on_gnt_closure_time(self):
self.sci.get_eth_balance.return_value = denoms.ether
self.sci.get_gnt_balance.return_value = 0
self.sci.get_gntb_balance.return_value = 4 * denoms.ether
self.pp.CLOSURE_TIME_DELAY = 0
ts1 = 1000
ts2 = 2000
p1, scip1 = make_awaiting_payment(value=1 * denoms.ether, ts=ts1)
p2, scip2 = make_awaiting_payment(value=2 * denoms.ether, ts=ts2)
p5, scip5 = make_awaiting_payment(value=5 * denoms.ether, ts=ts2)
self.pp.add(p1)
self.pp.add(p2)
self.pp.add(p5)
with freeze_time(timestamp_to_datetime(10000)):
self.pp.sendout(0)
self._assert_batch_transfer_called_with([scip1], ts1)
self.sci.batch_transfer.reset_mock()
self.sci.get_gntb_balance.return_value = 10 * denoms.ether
with freeze_time(timestamp_to_datetime(10000)):
self.pp.sendout(0)
self._assert_batch_transfer_called_with([scip2, scip5], ts2)
self.sci.batch_transfer.reset_mock()
def test_short_on_eth(self):
self.sci.get_eth_balance.return_value = self.sci.GAS_PRICE * \
(self.sci.GAS_BATCH_PAYMENT_BASE + 2 * self.sci.GAS_PER_PAYMENT)
self.sci.get_gnt_balance.return_value = 0
self.sci.get_gntb_balance.return_value = 1000 * denoms.ether
self.pp.CLOSURE_TIME_DELAY = 0
p1, scip1 = make_awaiting_payment(value=1, ts=1)
p2, scip2 = make_awaiting_payment(value=2, ts=2)
p5, scip5 = make_awaiting_payment(value=5, ts=3)
self.pp.add(p1)
self.pp.add(p2)
self.pp.add(p5)
with freeze_time(timestamp_to_datetime(10000)):
self.pp.sendout(0)
self._assert_batch_transfer_called_with([scip1, scip2], 2)
self.sci.batch_transfer.reset_mock()
self.sci.get_eth_balance.return_value = denoms.ether
with freeze_time(timestamp_to_datetime(10000)):
self.pp.sendout(0)
self._assert_batch_transfer_called_with([scip5], 3)
self.sci.batch_transfer.reset_mock()
def test_sorted_payments(self):
self.sci.get_eth_balance.return_value = 1000 * denoms.ether
self.sci.get_gnt_balance.return_value = 0
self.sci.get_gntb_balance.return_value = 1000 * denoms.ether
self.pp.CLOSURE_TIME_DELAY = 0
p1, _ = make_awaiting_payment(value=1, ts=300000)
p2, scip2 = make_awaiting_payment(value=2, ts=200000)
p3, scip3 = make_awaiting_payment(value=3, ts=100000)
self.pp.add(p1)
self.pp.add(p2)
self.pp.add(p3)
with freeze_time(timestamp_to_datetime(200000)):
self.pp.sendout(0)
self._assert_batch_transfer_called_with([scip3, scip2], 200000)
def test_batch_transfer_throws(self):
self.sci.get_eth_balance.return_value = 1000 * denoms.ether
self.sci.get_gnt_balance.return_value = 0
self.sci.get_gntb_balance.return_value = 1000 * denoms.ether
self.pp.CLOSURE_TIME_DELAY = 0
ts = 100000
p, scip = make_awaiting_payment(value=1, ts=ts)
self.pp.add(p)
self.sci.batch_transfer.side_effect = Exception
with freeze_time(timestamp_to_datetime(ts)):
with self.assertRaises(Exception):
self.pp.sendout(0)
self._assert_batch_transfer_called_with([scip], ts)
self.sci.batch_transfer.reset_mock()
self.sci.batch_transfer.side_effect = None
with freeze_time(timestamp_to_datetime(ts)):
self.pp.sendout(0)
self._assert_batch_transfer_called_with([scip], ts)
def test_block_gas_limit(self):
self.sci.get_eth_balance.return_value = denoms.ether
self.sci.get_gnt_balance.return_value = 0
self.sci.get_gntb_balance.return_value = 1000 * denoms.ether
self.sci.get_latest_block.return_value.gas_limit = \
(self.sci.GAS_BATCH_PAYMENT_BASE + self.sci.GAS_PER_PAYMENT) /\
self.pp.BLOCK_GAS_LIMIT_RATIO
self.pp.CLOSURE_TIME_DELAY = 0
p1, scip1 = make_awaiting_payment(value=1, ts=1)
p2, _ = make_awaiting_payment(value=2, ts=2)
self.pp.add(p1)
self.pp.add(p2)
with freeze_time(timestamp_to_datetime(10000)):
self.pp.sendout(0)
self._assert_batch_transfer_called_with([scip1], 1)
self.sci.batch_transfer.reset_mock()