def __init__(self, name, initial_payment_cycle, network_config, payments_dir, calculations_dir, run_mode,
service_fee_calc, release_override, payment_offset, baking_cfg, payments_queue, life_cycle,
dry_run, wllt_clnt_mngr, node_url, provider_factory, node_url_public='', verbose=False,
api_base_url=None):
super(PaymentProducer, self).__init__()
self.rules_model = RulesModel(baking_cfg.get_excluded_set_tob(), baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(), baking_cfg.get_dest_map())
self.baking_address = baking_cfg.get_baking_address()
self.owners_map = baking_cfg.get_owners_map()
self.founders_map = baking_cfg.get_founders_map()
self.min_delegation_amt_in_mutez = baking_cfg.get_min_delegation_amount() * MUTEZ
self.delegator_pays_xfer_fee = baking_cfg.get_delegator_pays_xfer_fee()
self.name = name
self.reward_api = provider_factory.newRewardApi(
network_config, self.baking_address, node_url, node_url_public, api_base_url)
self.block_api = provider_factory.newBlockApi(network_config, node_url, api_base_url)
self.fee_calc = service_fee_calc
self.initial_payment_cycle = initial_payment_cycle
self.nw_config = network_config
self.payments_root = payments_dir
self.calculations_dir = calculations_dir
self.run_mode = run_mode
self.exiting = False
self.release_override = release_override
self.payment_offset = payment_offset
self.verbose = verbose
self.payments_queue = payments_queue
self.life_cycle = life_cycle
self.dry_run = dry_run
self.payment_calc = PhasedPaymentCalculator(self.founders_map, self.owners_map, self.fee_calc,
self.min_delegation_amt_in_mutez, self.rules_model)
self.retry_fail_thread = threading.Thread(target=self.retry_fail_run, name=self.name + "_retry_fail")
self.retry_fail_event = threading.Event()
logger.info('Producer "{}" started'.format(self.name))
def test_process_payouts(self):
logger.debug("") # Console formatting
factory = ProviderFactory(provider='prpc')
parser = BakingYamlConfParser(self.baking_config,
None,
None,
None,
None,
block_api=factory,
api_base_url=None)
parser.parse()
parser.process()
cfg_dict = parser.get_conf_obj()
baking_cfg = BakingConf(cfg_dict)
srvc_fee_calc = ServiceFeeCalculator(
baking_cfg.get_full_supporters_set(),
baking_cfg.get_specials_map(), baking_cfg.get_service_fee())
rules_model = RulesModel(baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map())
payment_calc = PhasedPaymentCalculator(
baking_cfg.get_founders_map(), baking_cfg.get_owners_map(),
srvc_fee_calc,
baking_cfg.get_min_delegation_amount() * MUTEZ, rules_model)
rewardApi = factory.newRewardApi(
default_network_config_map[CURRENT_TESTNET],
baking_cfg.get_baking_address(), "")
# Simulate logic in payment_producer
reward_logs = []
attempts = 0
exiting = False
while not exiting and attempts < 2:
attempts += 1
try:
# Reward data
# Fetch cycle 90 of delphinet for tz1gtHbmBF3TSebsgJfJPvUB2e9x8EDeNm6V
reward_model = rewardApi.get_rewards_for_cycle_map(
PAYOUT_CYCLE)
# Calculate rewards - payment_producer.py
reward_logs, total_amount = payment_calc.calculate(
reward_model)
# Check total reward amount matches sums of records
self.assertTrue(
total_amount,
sum([rl.amount for rl in reward_logs if rl.payable]))
exiting = True
except ApiProviderException as e:
logger.error(
"{:s} error at payment producer loop: '{:s}', will try again."
.format("RPC", str(e)))
sleep(5)
#
# The next 3 phases happen in payment_consumer.py
#
# Merge payments to same address
phaseMerge = CalculatePhaseMerge()
reward_logs = phaseMerge.calculate(reward_logs)
# Handle remapping of payment to alternate address
phaseMapping = CalculatePhaseMapping()
reward_logs = phaseMapping.calculate(reward_logs,
baking_cfg.get_dest_map())
# Filter zero-balance addresses based on config
phaseZeroBalance = CalculatePhaseZeroBalance()
reward_logs = phaseZeroBalance.calculate(
reward_logs, baking_cfg.get_reactivate_zeroed())
# Filter out non-payable items
reward_logs = [pi for pi in reward_logs if pi.payable]
reward_logs.sort(key=cmp_to_key(cmp_by_type_balance))
# TRD Calculated Results
# tz1V9SpwXaGFiYdDfGJtWjA61EumAH3DwSyT type: D, stake bal: 62657.83, cur bal: 62657.83, ratio: 0.327420, fee_ratio: 0.000000, amount: 0.000000, fee_amount: 0.000000, fee_rate: 0.00, payable: N, skipped: Y, at-phase: 1, desc: Excluded by configuration, pay_addr: tz1V9SpwXaGFiYdDfGJtWjA61EumAH3DwSyT
# tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace type: D, stake bal: 55646.70, cur bal: 55646.70, ratio: 0.432340, fee_ratio: 0.000000, amount: 102.988160, fee_amount: 0.000000, fee_rate: 0.00, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace
# tz1T5woJN3r7SV5v2HGDyA5kurhbD9Y8ZKHZ type: D, stake bal: 25689.88, cur bal: 25689.88, ratio: 0.179635, fee_ratio: 0.019959, amount: 42.791010, fee_amount: 4.754557, fee_rate: 0.10, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1T5woJN3r7SV5v2HGDyA5kurhbD9Y8ZKHZ
# tz1fgX6oRWQb4HYHUT6eRjW8diNFrqjEfgq7 type: D, stake bal: 24916.33, cur bal: 24916.33, ratio: 0.193584, fee_ratio: 0.000000, amount: 46.113902, fee_amount: 0.000000, fee_rate: 0.00, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1fgX6oRWQb4HYHUT6eRjW8diNFrqjEfgq7
# tz1RRzfechTs3gWdM58y6xLeByta3JWaPqwP type: D, stake bal: 6725.43, cur bal: 6725.43, ratio: 0.047027, fee_ratio: 0.005225, amount: 11.202382, fee_amount: 1.244709, fee_rate: 0.10, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1RMmSzPSWPSSaKU193Voh4PosWSZx1C7Hs
# tz1L1XQWKxG38wk1Ain1foGaEZj8zeposcbk type: D, stake bal: 981.64, cur bal: 981.64, ratio: 0.007627, fee_ratio: 0.000000, amount: 1.816762, fee_amount: 0.000000, fee_rate: 0.00, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1L1XQWKxG38wk1Ain1foGaEZj8zeposcbk
# tz1L1XQWKxG38wk1Ain1foGaEZj8zeposcbk type: O, stake bal: 14750.53, cur bal: 0.00, ratio: 0.114602, fee_ratio: 0.000000, amount: 27.299548, fee_amount: 0.000000, fee_rate: 0.00, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1L1XQWKxG38wk1Ain1foGaEZj8zeposcbk
# tz1fgX6oRWQb4HYHUT6eRjW8diNFrqjEfgq7 type: F, stake bal: 0.00, cur bal: 0.00, ratio: 0.006296, fee_ratio: 0.000000, amount: 1.499816, fee_amount: 0.000000, fee_rate: 0.00, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1fgX6oRWQb4HYHUT6eRjW8diNFrqjEfgq7
# tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace type: F, stake bal: 0.00, cur bal: 0.00, ratio: 0.018889, fee_ratio: 0.000000, amount: 4.499450, fee_amount: 0.000000, fee_rate: 0.00, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace
# Final records before creating transactions
# These values are known to be correct
cr = {}
cr["tz1T5woJN3r7SV5v2HGDyA5kurhbD9Y8ZKHZ"] = {
"type": "D",
"amount": 42791010,
"pay_addr": "tz1T5woJN3r7SV5v2HGDyA5kurhbD9Y8ZKHZ"
}
cr["tz1RRzfechTs3gWdM58y6xLeByta3JWaPqwP"] = {
"type": "D",
"amount": 11202382,
"pay_addr": "tz1RMmSzPSWPSSaKU193Voh4PosWSZx1C7Hs"
}
cr["tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace"] = {
"type": "M",
"amount": 107487610,
"pay_addr": "tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace"
}
cr["tz1fgX6oRWQb4HYHUT6eRjW8diNFrqjEfgq7"] = {
"type": "M",
"amount": 47613718,
"pay_addr": "tz1fgX6oRWQb4HYHUT6eRjW8diNFrqjEfgq7"
}
cr["tz1L1XQWKxG38wk1Ain1foGaEZj8zeposcbk"] = {
"type": "M",
"amount": 29116310,
"pay_addr": "tz1L1XQWKxG38wk1Ain1foGaEZj8zeposcbk"
}
# Verify that TRD calculated matches known values
for r in reward_logs:
# We know this address should be skipped
if r.address == "tz1V9SpwXaGFiYdDfGJtWjA61EumAH3DwSyT":
self.assertEqual(r.skipped, 1)
self.assertEqual(r.amount, 0)
continue
# All others we can compare normally
cmp = cr[r.address]
self.assertEqual(r.type, cmp["type"])
self.assertEqual(r.amount, (cmp["amount"]))
self.assertEqual(r.paymentaddress, cmp["pay_addr"])
def __init__(self,
name,
initial_payment_cycle,
network_config,
payments_dir,
calculations_dir,
run_mode,
service_fee_calc,
release_override,
payment_offset,
baking_cfg,
payments_queue,
life_cycle,
dry_run,
client_manager,
node_url,
reward_data_provider,
node_url_public='',
api_base_url=None,
retry_injected=False):
super(PaymentProducer, self).__init__()
self.rules_model = RulesModel(baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map())
self.baking_address = baking_cfg.get_baking_address()
self.owners_map = baking_cfg.get_owners_map()
self.founders_map = baking_cfg.get_founders_map()
self.min_delegation_amt_in_mutez = baking_cfg.get_min_delegation_amount(
) * MUTEZ
self.delegator_pays_xfer_fee = baking_cfg.get_delegator_pays_xfer_fee()
self.provider_factory = ProviderFactory(reward_data_provider)
self.name = name
self.node_url = node_url
self.client_manager = client_manager
self.reward_api = self.provider_factory.newRewardApi(
network_config, self.baking_address, self.node_url,
node_url_public, api_base_url)
self.block_api = self.provider_factory.newBlockApi(
network_config, self.node_url, api_base_url)
dexter_contracts_set = baking_cfg.get_contracts_set()
if len(dexter_contracts_set) > 0 and not (self.reward_api.name
== 'tzstats'):
logger.warning(
"The Dexter functionality is currently only supported using tzstats."
"The contract address will be treated as a normal delegator.")
else:
self.reward_api.set_dexter_contracts_set(dexter_contracts_set)
self.rewards_type = baking_cfg.get_rewards_type()
self.fee_calc = service_fee_calc
self.initial_payment_cycle = initial_payment_cycle
if self.initial_payment_cycle is None:
recent = get_latest_report_file(payments_dir)
# if payment logs exists set initial cycle to following cycle
# if payment logs does not exists, set initial cycle to 0, so that payment starts from last released rewards
self.initial_payment_cycle = 0 if recent is None else int(
recent) + 1
logger.info("initial_cycle set to {}".format(
self.initial_payment_cycle))
self.nw_config = network_config
self.payments_root = payments_dir
self.calculations_dir = calculations_dir
self.run_mode = run_mode
self.exiting = False
self.release_override = release_override
self.payment_offset = payment_offset
self.payments_queue = payments_queue
self.life_cycle = life_cycle
self.dry_run = dry_run
self.payment_calc = PhasedPaymentCalculator(
self.founders_map, self.owners_map, self.fee_calc,
self.min_delegation_amt_in_mutez, self.rules_model)
self.retry_fail_thread = threading.Thread(target=self.retry_fail_run,
name=self.name +
"_retry_fail")
self.retry_fail_event = threading.Event()
self.retry_injected = retry_injected
self.retry_producer = RetryProducer(self.payments_queue,
self.reward_api, self,
self.payments_root,
self.retry_injected)
logger.info('Producer "{}" started'.format(self.name))
def __init__(self,
name,
initial_payment_cycle,
network_config,
payments_dir,
calculations_dir,
run_mode,
service_fee_calc,
release_override,
payment_offset,
baking_cfg,
payments_queue,
life_cycle,
dry_run,
wllt_clnt_mngr,
node_url,
provider_factory,
node_url_public='',
api_base_url=None,
retry_injected=False):
super(PaymentProducer, self).__init__()
self.rules_model = RulesModel(baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map())
self.baking_address = baking_cfg.get_baking_address()
self.owners_map = baking_cfg.get_owners_map()
self.founders_map = baking_cfg.get_founders_map()
self.min_delegation_amt_in_mutez = baking_cfg.get_min_delegation_amount(
) * MUTEZ
self.delegator_pays_xfer_fee = baking_cfg.get_delegator_pays_xfer_fee()
self.name = name
self.node_url = node_url
self.wllt_clnt_mngr = wllt_clnt_mngr
self.reward_api = provider_factory.newRewardApi(
network_config, self.baking_address, self.node_url,
node_url_public, api_base_url)
self.block_api = provider_factory.newBlockApi(network_config,
self.node_url,
api_base_url)
dexter_contracts_set = baking_cfg.get_contracts_set()
if len(dexter_contracts_set) > 0 and not (self.reward_api.name
== 'tzstats'):
logger.warning(
"The Dexter functionality is currently only supported using tzstats."
"The contract address will be treated as a normal delegator.")
else:
self.reward_api.set_dexter_contracts_set(dexter_contracts_set)
self.fee_calc = service_fee_calc
self.initial_payment_cycle = initial_payment_cycle
self.nw_config = network_config
self.payments_root = payments_dir
self.calculations_dir = calculations_dir
self.run_mode = run_mode
self.exiting = False
self.release_override = release_override
self.payment_offset = payment_offset
self.payments_queue = payments_queue
self.life_cycle = life_cycle
self.dry_run = dry_run
self.payment_calc = PhasedPaymentCalculator(
self.founders_map, self.owners_map, self.fee_calc,
self.min_delegation_amt_in_mutez, self.rules_model)
self.retry_fail_thread = threading.Thread(target=self.retry_fail_run,
name=self.name +
"_retry_fail")
self.retry_fail_event = threading.Event()
self.retry_injected = retry_injected
logger.info('Producer "{}" started'.format(self.name))
def test_batch_payer_total_payout_amount():
factory = ProviderFactory(provider="prpc")
parser = BakingYamlConfParser(baking_config,
None,
None,
None,
None,
block_api=factory,
api_base_url=None)
parser.parse()
parser.process()
cfg_dict = parser.get_conf_obj()
baking_cfg = BakingConf(cfg_dict)
srvc_fee_calc = ServiceFeeCalculator(
baking_cfg.get_full_supporters_set(),
baking_cfg.get_specials_map(),
baking_cfg.get_service_fee(),
)
rules_model = RulesModel(
baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map(),
)
payment_calc = PhasedPaymentCalculator(
baking_cfg.get_founders_map(),
baking_cfg.get_owners_map(),
srvc_fee_calc,
baking_cfg.get_min_delegation_amount() * MUTEZ,
rules_model,
)
rewardApi = factory.newRewardApi(
default_network_config_map[CURRENT_TESTNET],
baking_cfg.get_baking_address(), "")
# Simulate logic in payment_producer
reward_logs = []
attempts = 0
exiting = False
while not exiting and attempts < 2:
attempts += 1
# Reward data
# Fetch cycle 51 of granadanet for tz1gtHbmBF3TSebsgJfJPvUB2e9x8EDeNm6V
reward_model = rewardApi.get_rewards_for_cycle_map(
PAYOUT_CYCLE, RewardsType.ACTUAL)
# Calculate rewards - payment_producer.py
reward_model.computed_reward_amount = reward_model.total_reward_amount
reward_logs, total_amount = payment_calc.calculate(reward_model)
# Check total reward amount matches sums of records
assert total_amount == sum(
[rl.amount for rl in reward_logs if rl.payable])
exiting = True
# Merge payments to same address
phaseMerge = CalculatePhaseMerge()
reward_logs = phaseMerge.calculate(reward_logs)
# Handle remapping of payment to alternate address
phaseMapping = CalculatePhaseMapping()
reward_logs = phaseMapping.calculate(reward_logs,
baking_cfg.get_dest_map())
# Filter zero-balance addresses based on config
phaseZeroBalance = CalculatePhaseZeroBalance()
reward_logs = phaseZeroBalance.calculate(
reward_logs, baking_cfg.get_reactivate_zeroed())
# Filter out non-payable items
reward_logs = [
payment_item for payment_item in reward_logs if payment_item.payable
]
reward_logs.sort(key=cmp_to_key(cmp_by_type_balance))
batch_payer = BatchPayer(
node_url=node_endpoint,
pymnt_addr="tz1gtHbmBF3TSebsgJfJPvUB2e9x8EDeNm6V",
clnt_mngr=ClientManager(node_endpoint, PRIVATE_SIGNER_URL),
delegator_pays_ra_fee=True,
delegator_pays_xfer_fee=True,
network_config=network,
plugins_manager=PluginManager(baking_cfg.get_plugins_conf(),
dry_run=True),
dry_run=True,
)
# Fix the endpoint auto port assignment because
# https://mainnet-tezos.giganode.io:8732 cannot be reached
batch_payer.clnt_mngr.node_endpoint = node_endpoint
# Do the payment
(
payment_logs,
total_attempts,
total_payout_amount,
number_future_payable_cycles,
) = batch_payer.pay(reward_logs, dry_run=True)
assert total_attempts == 3
assert total_payout_amount == 238211030
assert (PAYMENT_ADDRESS_BALANCE //
total_payout_amount) - 1 == number_future_payable_cycles
def test_batch_payer_total_payout_amount():
factory = ProviderFactory(provider="prpc")
parser = BakingYamlConfParser(
baking_config, None, None, None, None, block_api=factory, api_base_url=None
)
parser.parse()
parser.process()
cfg_dict = parser.get_conf_obj()
baking_cfg = BakingConf(cfg_dict)
srvc_fee_calc = ServiceFeeCalculator(
baking_cfg.get_full_supporters_set(),
baking_cfg.get_specials_map(),
baking_cfg.get_service_fee(),
)
rules_model = RulesModel(
baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map(),
)
payment_calc = PhasedPaymentCalculator(
baking_cfg.get_founders_map(),
baking_cfg.get_owners_map(),
srvc_fee_calc,
int(baking_cfg.get_min_delegation_amount() * MUTEZ_PER_TEZ),
rules_model,
)
rewardApi = factory.newRewardApi(
default_network_config_map[CURRENT_TESTNET], baking_cfg.get_baking_address(), ""
)
# Simulate logic in payment_producer
reward_logs = []
attempts = 0
exiting = False
while not exiting and attempts < 2:
attempts += 1
# Reward data
# Fetch cycle 51 of granadanet for tz1gtHbmBF3TSebsgJfJPvUB2e9x8EDeNm6V
reward_model = rewardApi.get_rewards_for_cycle_map(
PAYOUT_CYCLE, RewardsType.ACTUAL
)
# Calculate rewards - payment_producer.py
reward_model.computed_reward_amount = reward_model.total_reward_amount
reward_logs, total_amount = payment_calc.calculate(reward_model)
# Check total reward amount matches sums of records
# diff of 1 expected due to floating point arithmetic
assert (
total_amount - sum([rl.adjusted_amount for rl in reward_logs if rl.payable])
<= 1
)
exiting = True
# Merge payments to same address
phaseMerge = CalculatePhaseMerge()
reward_logs = phaseMerge.calculate(reward_logs)
# Handle remapping of payment to alternate address
phaseMapping = CalculatePhaseMapping()
reward_logs = phaseMapping.calculate(reward_logs, baking_cfg.get_dest_map())
# Filter zero-balance addresses based on config
phaseZeroBalance = CalculatePhaseZeroBalance()
reward_logs = phaseZeroBalance.calculate(
reward_logs, baking_cfg.get_reactivate_zeroed()
)
# Filter out non-payable items
reward_logs = [payment_item for payment_item in reward_logs if payment_item.payable]
reward_logs.sort(key=cmp_to_key(cmp_by_type_balance))
batch_payer = BatchPayer(
node_url=node_endpoint,
pymnt_addr="tz1N4UfQCahHkRShBanv9QP9TnmXNgCaqCyZ",
clnt_mngr=ClientManager(node_endpoint, PRIVATE_SIGNER_URL),
delegator_pays_ra_fee=True,
delegator_pays_xfer_fee=True,
network_config=network,
plugins_manager=PluginManager(baking_cfg.get_plugins_conf(), dry_run=True),
dry_run=True,
)
# Do the payment
(
_,
total_attempts,
total_payout_amount,
number_future_payable_cycles,
) = batch_payer.pay(reward_logs, dry_run=True)
# Payment does not have status done, paid or injected thus the total payout amount is zero
assert total_payout_amount == 0
assert number_future_payable_cycles == 2
assert total_attempts == 3
# Check the adjusted amount
assert reward_logs[0].adjusted_amount == 40418486
assert reward_logs[1].adjusted_amount == 10581272
assert reward_logs[2].adjusted_amount == 109732835
assert reward_logs[3].adjusted_amount == 48362127
assert reward_logs[4].adjusted_amount == 29116310