def test_attempt_single_batch_tz(sign, request_url, request_url_post):
network_config = {"BLOCK_TIME_IN_SEC": 60, "MINIMAL_BLOCK_DELAY": 30}
batch_payer = BatchPayer(
node_url="node_addr",
pymnt_addr=TEST_TZ_ADDRESS,
clnt_mngr=ClientManager(
node_endpoint=PUBLIC_NODE_URL[CURRENT_TESTNET],
signer_endpoint=PRIVATE_SIGNER_URL,
),
delegator_pays_ra_fee=True,
delegator_pays_xfer_fee=True,
network_config=network_config,
plugins_manager=MagicMock(),
dry_run=False,
)
batch_payer.base_counter = 0
reward_log = RewardLog(
address=TEST_TZ_ADDRESS,
type="D",
staking_balance=80,
current_balance=100,
)
reward_log.adjusted_amount = 15577803
reward_log.skipped = False
opt_counter = OpCounter()
status, operation_hash, _ = batch_payer.attempt_single_batch([reward_log],
opt_counter,
dry_run=True)
assert status == PaymentStatus.DONE
assert operation_hash is None
assert reward_log.delegator_transaction_fee == TZTX_FEE
assert opt_counter.counter == 3209358
def test_simulate_single_operation():
default_fee = TZTX_FEE
network_config = {"BLOCK_TIME_IN_SEC": 60, "MINIMAL_BLOCK_DELAY": 30}
batch_payer = BatchPayer(
node_url="node_addr",
pymnt_addr="tz1234567890123456789012345678901234",
clnt_mngr=ClientManager(
node_endpoint=PUBLIC_NODE_URL[CURRENT_TESTNET],
signer_endpoint=PRIVATE_SIGNER_URL,
),
delegator_pays_ra_fee=True,
delegator_pays_xfer_fee=True,
network_config=network_config,
plugins_manager=MagicMock(),
dry_run=False,
)
batch_payer.base_counter = 0
reward_log = RewardLog(
address="KT1P3Y1mkGASzuJqLh7uGuQEvHatztGuQRgC",
type="D",
staking_balance=0,
current_balance=0,
)
reward_log.amount = 15577803
reward_log.skipped = False
simulation_status, simulation_results = batch_payer.simulate_single_operation(
reward_log, reward_log.amount, "hash", "unittest")
assert PaymentStatus.DONE == simulation_status
consumed_gas, tx_fee, storage = simulation_results
assert 150 == consumed_gas
assert 410 == default_fee + consumed_gas * MUTEZ_PER_GAS_UNIT
assert int == type(storage) # type of storage should be int
assert 24 == storage
def run(self):
while True:
try:
# 1- wait until a reward is present
payment_items = self.payments_queue.get(True)
if payment_items[0].type == EXIT_PAYMENT_TYPE:
logger.debug("Exit signal received. Killing the thread...")
break
# each log in the batch belongs to the same cycle
pymnt_cycle = payment_items[0].cycle
logger.info(
"Starting payments for cycle {}".format(pymnt_cycle))
# 2- select suitable payment script
# if len(payment_items) == 1:
# regular_payer = RegularPayer(self.client_path, self.key_name)
# payment_log = regular_payer.pay(payment_items[0], self.verbose, dry_run=self.dry_run)
# payment_logs = [payment_log]
batch_payer = BatchPayer(self.node_addr, self.key_name,
self.wllt_clnt_mngr,
self.delegator_pays_xfer_fee)
# 3- do the payment
payment_logs = batch_payer.pay(payment_items,
self.verbose,
dry_run=self.dry_run)
# 4- count failed payments
nb_failed = count_and_log_failed(payment_logs, pymnt_cycle)
# 5- create payment report file
report_file = self.create_payment_report(
nb_failed, payment_logs, pymnt_cycle)
# 6- upon successful payment, clean failure reports
# note that failed payment reports are cleaned after creation of successful payment report
if nb_failed == 0:
self.clean_failed_payment_reports(pymnt_cycle)
# 7- send email
if not self.dry_run:
self.mm.send_payment_mail(pymnt_cycle, report_file,
nb_failed)
except Exception:
logger.error("Error at reward payment", exc_info=True)
logger.info("Consumer returning ...")
return
def test_simulate_single_operation():
config = configparser.ConfigParser()
assert os.path.isfile(FEE_INI) is True
config.read(FEE_INI)
default_fee = int(config["KTTX"]["fee"])
network_config = {"BLOCK_TIME_IN_SEC": 64}
batch_payer = BatchPayer(
node_url="node_addr",
pymnt_addr="tz1234567890123456789012345678901234",
clnt_mngr=ClientManager(
node_endpoint="https://testnet-tezos.giganode.io:443",
signer_endpoint="http://127.0.0.1:6732",
),
delegator_pays_ra_fee=True,
delegator_pays_xfer_fee=True,
network_config=network_config,
plugins_manager=MagicMock(),
dry_run=False,
)
batch_payer.base_counter = 0
reward_log = RewardLog(
address="KT1P3Y1mkGASzuJqLh7uGuQEvHatztGuQRgC",
type="D",
staking_balance=0,
current_balance=0,
)
reward_log.amount = 15577803
reward_log.skipped = False
simulation_status, simulation_results = batch_payer.simulate_single_operation(
reward_log, reward_log.amount, "hash", "unittest")
assert PaymentStatus.DONE == simulation_status
consumed_gas, tx_fee, storage = simulation_results
assert 150 == consumed_gas
assert 589 == default_fee + consumed_gas * MUTEZ_PER_GAS_UNIT
assert int == type(storage) # type of storage should be int
assert 24 == storage
def run(self):
while True:
try:
# 1 - wait until a reward is present
payment_batch = self.payments_queue.get(True)
payment_items = payment_batch.batch
if len(payment_items) == 0:
logger.debug("Batch is empty, ignoring ...")
continue
if payment_items[0].type == EXIT_PAYMENT_TYPE:
logger.warn("Exit signal received. Terminating...")
break
time.sleep(1)
pymnt_cycle = payment_batch.cycle
logger.info("Starting payments for cycle {}".format(pymnt_cycle))
# Handle remapping of payment to alternate address
phase5 = CalculatePhase5(self.dest_map)
payment_items, _ = phase5.calculate(payment_items, None)
# Merge payments to same address
phase6 = CalculatePhase6(addr_dest_dict=self.dest_map)
payment_items, _ = phase6.calculate(payment_items, None)
# Filter zero-balance addresses based on config
phase7 = CalculatePhase7(self.reactivate_zeroed)
payment_items = phase7.calculate(payment_items)
# Filter out non-payable items
payment_items = [pi for pi in payment_items if pi.payable]
payment_items.sort(key=functools.cmp_to_key(cmp_by_type_balance))
batch_payer = BatchPayer(self.node_addr, self.key_name, self.wllt_clnt_mngr,
self.delegator_pays_ra_fee, self.delegator_pays_xfer_fee,
self.network_config, self.mm,
self.dry_run)
# 3- do the payment
payment_logs, total_attempts, number_future_payable_cycles = batch_payer.pay(payment_items, self.verbose, dry_run=self.dry_run)
# override batch data
payment_batch.batch = payment_logs
# 4- count failed payments
nb_failed, nb_injected = count_and_log_failed(payment_logs)
# 5- create payment report file
report_file = self.create_payment_report(nb_failed, payment_logs, pymnt_cycle)
# 6- Clean failure reports
self.clean_failed_payment_reports(pymnt_cycle, nb_failed == 0)
# 7- notify batch producer
if nb_failed == 0:
if payment_batch.producer_ref:
payment_batch.producer_ref.on_success(payment_batch)
else:
if payment_batch.producer_ref:
payment_batch.producer_ref.on_fail(payment_batch)
# 8- send email
if not self.dry_run and total_attempts > 0:
self.mm.send_payment_mail(pymnt_cycle, report_file, nb_failed, nb_injected, number_future_payable_cycles)
# 9- publish anonymous stats
if self.publish_stats and self.args and not self.dry_run:
stats_dict = self.create_stats_dict(nb_failed, nb_injected, pymnt_cycle,
payment_logs, total_attempts)
stats_publisher(stats_dict)
except Exception:
logger.error("Error at reward payment", exc_info=True)
logger.info("Consumer returning ...")
return
def _consume_batch(self, payment_batch):
try:
payment_items = payment_batch.batch
if len(payment_items) == 0:
logger.debug("Batch is empty, ignoring...")
return True
if payment_items[0].type == EXIT_PAYMENT_TYPE:
logger.warn("Exit signal received. Terminating...")
return False
sleep(1)
pymnt_cycle = payment_batch.cycle
logger.info("Starting payments for cycle {}".format(pymnt_cycle))
# Filter out non-payable items
payment_items = [pi for pi in payment_items if pi.payable]
already_paid_items = [
pi for pi in payment_items if pi.paid.is_processed()
]
payment_items = [
pi for pi in payment_items if not pi.paid.is_processed()
]
# Handle remapping of payment to alternate address
phaseMapping = CalculatePhaseMapping()
payment_items = phaseMapping.calculate(payment_items,
self.dest_map)
# Merge payments to same address
phaseMerge = CalculatePhaseMerge()
payment_items = phaseMerge.calculate(payment_items)
# Filter zero-balance addresses based on config
phaseZeroBalance = CalculatePhaseZeroBalance()
payment_items = phaseZeroBalance.calculate(payment_items,
self.reactivate_zeroed)
payment_items.sort(key=functools.cmp_to_key(cmp_by_type_balance))
batch_payer = BatchPayer(
self.node_addr,
self.key_name,
self.client_manager,
self.delegator_pays_ra_fee,
self.delegator_pays_xfer_fee,
self.network_config,
self.plugins_manager,
self.dry_run,
)
# 3- do the payment
(
payment_logs,
total_attempts,
total_payout_amount,
number_future_payable_cycles,
) = batch_payer.pay(payment_items, dry_run=self.dry_run)
# override batch data
payment_batch.batch = payment_logs
# 4- count failed payments
nb_paid, nb_failed, nb_unknown = count_and_log_failed(payment_logs)
# 5- create payment report file
report_file = self.create_payment_report(nb_failed, payment_logs,
pymnt_cycle,
already_paid_items)
# 5.1- modify calculations report
if total_attempts > 0:
self.add_transaction_fees_to_calculation_report(
payment_logs, pymnt_cycle)
# 6- Clean failure reports
self.clean_failed_payment_reports(pymnt_cycle, nb_failed == 0)
# 7- notify batch producer
if nb_failed == 0:
if payment_batch.producer_ref:
payment_batch.producer_ref.on_success(payment_batch)
else:
if payment_batch.producer_ref:
payment_batch.producer_ref.on_fail(payment_batch)
# 8- send notification via plugins
if total_attempts > 0:
subject = "Reward Payouts for Cycle {:d}".format(pymnt_cycle)
status = ""
if nb_failed == 0 and nb_unknown == 0:
status = status + "Completed Successfully!"
else:
status = status + "attempted"
if nb_failed > 0:
status = status + ", {:d} failed".format(nb_failed)
if nb_unknown > 0:
status = (status +
", {:d} injected but final state not known".
format(nb_unknown))
subject = subject + " " + status
admin_message = "The current payout account balance is expected to last for the next {:d} cycle(s)!".format(
number_future_payable_cycles)
# Payout notification receives cycle, rewards total, number of delegators
self.plugins_manager.send_payout_notification(
pymnt_cycle, total_payout_amount,
(nb_paid + nb_failed + nb_unknown))
# Admin notification receives subject, message, CSV report, raw log objects
self.plugins_manager.send_admin_notification(
subject, admin_message, [report_file], payment_logs)
# 9- publish anonymous stats
if self.publish_stats and self.args and not self.dry_run:
stats_dict = self.create_stats_dict(
self.key_name,
nb_failed,
nb_unknown,
pymnt_cycle,
payment_logs,
total_attempts,
)
stats_publisher(stats_dict)
else:
logger.info("Anonymous statistics disabled{:s}".format(
", (Dry run)" if self.dry_run else ""))
except Exception:
logger.error("Error at reward payment", exc_info=True)
return True
def run(self):
while True:
try:
# 1- wait until a reward is present
payment_batch = self.payments_queue.get(True)
payment_items = payment_batch.batch
if len(payment_items) == 0:
logger.debug("Batch is empty, ignoring ...")
continue
if payment_items[0].type == EXIT_PAYMENT_TYPE:
logger.warn("Exit signal received. Terminating...")
break
time.sleep(1)
pymnt_cycle = payment_batch.cycle
logger.info(
"Starting payments for cycle {}".format(pymnt_cycle))
phase5 = CalculatePhase5(self.dest_map)
payment_items, _ = phase5.calculate(payment_items, None)
phase6 = CalculatePhase6(addr_dest_dict=self.dest_map)
payment_items, _ = phase6.calculate(payment_items, None)
# filter out non-payable items
payment_items = [pi for pi in payment_items if pi.payable]
payment_items.sort(
key=functools.cmp_to_key(cmp_by_type_balance))
batch_payer = BatchPayer(self.node_addr, self.key_name,
self.wllt_clnt_mngr,
self.delegator_pays_xfer_fee,
self.network_config)
# 3- do the payment
payment_logs, total_attempts = batch_payer.pay(
payment_items, self.verbose, dry_run=self.dry_run)
# override batch data
payment_batch.batch = payment_logs
# total_attempts = 1
# payment_logs = []
# for pl in payment_items:
# pl.paid=True
# pl.hash='132'
# payment_logs.append(pl)
# 4- count failed payments
nb_failed, nb_injected = count_and_log_failed(payment_logs)
# 5- create payment report file
report_file = self.create_payment_report(
nb_failed, nb_injected, payment_logs, pymnt_cycle,
total_attempts)
# 6- Clean failure reports
self.clean_failed_payment_reports(pymnt_cycle, nb_failed == 0)
# 7- notify back producer
if nb_failed == 0:
if payment_batch.producer_ref:
payment_batch.producer_ref.on_success(payment_batch)
else:
if payment_batch.producer_ref:
payment_batch.producer_ref.on_fail(payment_batch)
# 8- send email
if not self.dry_run:
self.mm.send_payment_mail(pymnt_cycle, report_file,
nb_failed, nb_injected)
except Exception:
logger.error("Error at reward payment", exc_info=True)
logger.info("Consumer returning ...")
return
def test_batch_payer_total_payout_amount():
factory = ProviderFactory(provider="prpc")
parser = BakingYamlConfParser(baking_config,
None,
None,
None,
None,
block_api=factory,
api_base_url=None)
parser.parse()
parser.process()
cfg_dict = parser.get_conf_obj()
baking_cfg = BakingConf(cfg_dict)
srvc_fee_calc = ServiceFeeCalculator(
baking_cfg.get_full_supporters_set(),
baking_cfg.get_specials_map(),
baking_cfg.get_service_fee(),
)
rules_model = RulesModel(
baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map(),
)
payment_calc = PhasedPaymentCalculator(
baking_cfg.get_founders_map(),
baking_cfg.get_owners_map(),
srvc_fee_calc,
baking_cfg.get_min_delegation_amount() * MUTEZ,
rules_model,
)
rewardApi = factory.newRewardApi(
default_network_config_map[CURRENT_TESTNET],
baking_cfg.get_baking_address(), "")
# Simulate logic in payment_producer
reward_logs = []
attempts = 0
exiting = False
while not exiting and attempts < 2:
attempts += 1
# Reward data
# Fetch cycle 51 of granadanet for tz1gtHbmBF3TSebsgJfJPvUB2e9x8EDeNm6V
reward_model = rewardApi.get_rewards_for_cycle_map(
PAYOUT_CYCLE, RewardsType.ACTUAL)
# Calculate rewards - payment_producer.py
reward_model.computed_reward_amount = reward_model.total_reward_amount
reward_logs, total_amount = payment_calc.calculate(reward_model)
# Check total reward amount matches sums of records
assert total_amount == sum(
[rl.amount for rl in reward_logs if rl.payable])
exiting = True
# Merge payments to same address
phaseMerge = CalculatePhaseMerge()
reward_logs = phaseMerge.calculate(reward_logs)
# Handle remapping of payment to alternate address
phaseMapping = CalculatePhaseMapping()
reward_logs = phaseMapping.calculate(reward_logs,
baking_cfg.get_dest_map())
# Filter zero-balance addresses based on config
phaseZeroBalance = CalculatePhaseZeroBalance()
reward_logs = phaseZeroBalance.calculate(
reward_logs, baking_cfg.get_reactivate_zeroed())
# Filter out non-payable items
reward_logs = [
payment_item for payment_item in reward_logs if payment_item.payable
]
reward_logs.sort(key=cmp_to_key(cmp_by_type_balance))
batch_payer = BatchPayer(
node_url=node_endpoint,
pymnt_addr="tz1gtHbmBF3TSebsgJfJPvUB2e9x8EDeNm6V",
clnt_mngr=ClientManager(node_endpoint, PRIVATE_SIGNER_URL),
delegator_pays_ra_fee=True,
delegator_pays_xfer_fee=True,
network_config=network,
plugins_manager=PluginManager(baking_cfg.get_plugins_conf(),
dry_run=True),
dry_run=True,
)
# Fix the endpoint auto port assignment because
# https://mainnet-tezos.giganode.io:8732 cannot be reached
batch_payer.clnt_mngr.node_endpoint = node_endpoint
# Do the payment
(
payment_logs,
total_attempts,
total_payout_amount,
number_future_payable_cycles,
) = batch_payer.pay(reward_logs, dry_run=True)
assert total_attempts == 3
assert total_payout_amount == 238211030
assert (PAYMENT_ADDRESS_BALANCE //
total_payout_amount) - 1 == number_future_payable_cycles
def test_batch_payer_total_payout_amount():
factory = ProviderFactory(provider="prpc")
parser = BakingYamlConfParser(
baking_config, None, None, None, None, block_api=factory, api_base_url=None
)
parser.parse()
parser.process()
cfg_dict = parser.get_conf_obj()
baking_cfg = BakingConf(cfg_dict)
srvc_fee_calc = ServiceFeeCalculator(
baking_cfg.get_full_supporters_set(),
baking_cfg.get_specials_map(),
baking_cfg.get_service_fee(),
)
rules_model = RulesModel(
baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map(),
)
payment_calc = PhasedPaymentCalculator(
baking_cfg.get_founders_map(),
baking_cfg.get_owners_map(),
srvc_fee_calc,
int(baking_cfg.get_min_delegation_amount() * MUTEZ_PER_TEZ),
rules_model,
)
rewardApi = factory.newRewardApi(
default_network_config_map[CURRENT_TESTNET], baking_cfg.get_baking_address(), ""
)
# Simulate logic in payment_producer
reward_logs = []
attempts = 0
exiting = False
while not exiting and attempts < 2:
attempts += 1
# Reward data
# Fetch cycle 51 of granadanet for tz1gtHbmBF3TSebsgJfJPvUB2e9x8EDeNm6V
reward_model = rewardApi.get_rewards_for_cycle_map(
PAYOUT_CYCLE, RewardsType.ACTUAL
)
# Calculate rewards - payment_producer.py
reward_model.computed_reward_amount = reward_model.total_reward_amount
reward_logs, total_amount = payment_calc.calculate(reward_model)
# Check total reward amount matches sums of records
# diff of 1 expected due to floating point arithmetic
assert (
total_amount - sum([rl.adjusted_amount for rl in reward_logs if rl.payable])
<= 1
)
exiting = True
# Merge payments to same address
phaseMerge = CalculatePhaseMerge()
reward_logs = phaseMerge.calculate(reward_logs)
# Handle remapping of payment to alternate address
phaseMapping = CalculatePhaseMapping()
reward_logs = phaseMapping.calculate(reward_logs, baking_cfg.get_dest_map())
# Filter zero-balance addresses based on config
phaseZeroBalance = CalculatePhaseZeroBalance()
reward_logs = phaseZeroBalance.calculate(
reward_logs, baking_cfg.get_reactivate_zeroed()
)
# Filter out non-payable items
reward_logs = [payment_item for payment_item in reward_logs if payment_item.payable]
reward_logs.sort(key=cmp_to_key(cmp_by_type_balance))
batch_payer = BatchPayer(
node_url=node_endpoint,
pymnt_addr="tz1N4UfQCahHkRShBanv9QP9TnmXNgCaqCyZ",
clnt_mngr=ClientManager(node_endpoint, PRIVATE_SIGNER_URL),
delegator_pays_ra_fee=True,
delegator_pays_xfer_fee=True,
network_config=network,
plugins_manager=PluginManager(baking_cfg.get_plugins_conf(), dry_run=True),
dry_run=True,
)
# Do the payment
(
_,
total_attempts,
total_payout_amount,
number_future_payable_cycles,
) = batch_payer.pay(reward_logs, dry_run=True)
# Payment does not have status done, paid or injected thus the total payout amount is zero
assert total_payout_amount == 0
assert number_future_payable_cycles == 2
assert total_attempts == 3
# Check the adjusted amount
assert reward_logs[0].adjusted_amount == 40418486
assert reward_logs[1].adjusted_amount == 10581272
assert reward_logs[2].adjusted_amount == 109732835
assert reward_logs[3].adjusted_amount == 48362127
assert reward_logs[4].adjusted_amount == 29116310