def test_calculate(self):
nw = {
'NAME': 'MAINNET',
'NB_FREEZE_CYCLE': 5,
'BLOCK_TIME_IN_SEC': 60,
'BLOCKS_PER_CYCLE': 4096,
'BLOCKS_PER_ROLL_SNAPSHOT': 256
}
api = ProviderFactory(provider='prpc').newRewardApi(nw, BAKING_ADDRESS, '')
model = api.get_rewards_for_cycle_map(153)
phase0 = CalculatePhase0(model)
reward_data, total_rewards = phase0.calculate()
delegate_staking_balance = int(model.delegate_staking_balance)
# total reward ratio is 1
self.assertTrue(1.0, sum(r.ratio0 for r in reward_data))
# check that ratio calculations are correct
delegators_balances_dict = model.delegator_balance_dict
# check ratios
for (address, delegator_info), reward in zip(delegators_balances_dict.items(), reward_data):
# ratio must be equal to stake/total staking balance
delegator_staking_balance = int(delegator_info["staking_balance"])
self.assertEqual(delegator_staking_balance / delegate_staking_balance, reward.ratio0)
# last one is owners record
self.assertTrue(reward_data[-1].type == reward_log.TYPE_OWNERS_PARENT)
def test_calculate(self):
nw = DEFAULT_NETWORK_CONFIG_MAP["MAINNET"]
api = ProviderFactory(provider="tzkt").newRewardApi(
nw, BAKING_ADDRESS, "")
model = api.get_rewards_for_cycle_map(CYCLE, REWARDS_TYPE)
phase0 = CalculatePhase0(model)
reward_data = phase0.calculate()
delegate_staking_balance = int(model.delegate_staking_balance)
# total reward ratio is 1
self.assertEqual(1.0, sum(r.ratio0 for r in reward_data))
# check that ratio calculations are correct
delegators_balances_dict = model.delegator_balance_dict
# check ratios
for (address,
delegator_info), reward in zip(delegators_balances_dict.items(),
reward_data):
# ratio must be equal to stake/total staking balance
delegator_staking_balance = int(delegator_info["staking_balance"])
self.assertEqual(
delegator_staking_balance / delegate_staking_balance,
reward.ratio0)
# last one is owners record
self.assertTrue(reward_data[-1].type == TYPE_OWNERS_PARENT)
def __init__(self,
name,
initial_payment_cycle,
network_config,
payments_dir,
calculations_dir,
run_mode,
service_fee_calc,
release_override,
payment_offset,
baking_cfg,
payments_queue,
life_cycle,
dry_run,
wllt_clnt_mngr,
node_url,
provider,
verbose=False):
super(PaymentProducer, self).__init__()
self.baking_address = baking_cfg.get_baking_address()
self.owners_map = baking_cfg.get_owners_map()
self.founders_map = baking_cfg.get_founders_map()
self.excluded_delegators_set = baking_cfg.get_excluded_delegators_set()
self.min_delegation_amt = baking_cfg.get_min_delegation_amount()
self.pymnt_scale = baking_cfg.get_payment_scale()
self.prcnt_scale = baking_cfg.get_percentage_scale()
self.name = name
rc = RoundingCommand(self.prcnt_scale)
provider_factory = ProviderFactory(provider)
self.reward_api = provider_factory.newRewardApi(
network_config, self.baking_address, wllt_clnt_mngr, node_url)
self.block_api = provider_factory.newBlockApi(network_config,
wllt_clnt_mngr, node_url)
self.reward_calculator_api = provider_factory.newCalcApi(
self.founders_map, self.min_delegation_amt,
self.excluded_delegators_set, rc)
self.fee_calc = service_fee_calc
self.initial_payment_cycle = initial_payment_cycle
self.nw_config = network_config
self.payments_root = payments_dir
self.calculations_dir = calculations_dir
self.run_mode = run_mode
self.exiting = False
self.release_override = release_override
self.payment_offset = payment_offset
self.verbose = verbose
self.payments_queue = payments_queue
self.life_cycle = life_cycle
self.dry_run = dry_run
logger.debug('Producer started')
def load_config_file(wllt_clnt_mngr, network_config, master_cfg):
provider_factory = ProviderFactory(args.reward_data_provider)
parser = BakingYamlConfParser(None,
wllt_clnt_mngr,
provider_factory,
network_config,
args.node_addr,
api_base_url=args.api_base_url)
parser.parse()
parser.validate()
parser.process()
cfg_dict = parser.get_conf_obj()
# dictionary to BakingConf object, for a bit of type safety
cfg = BakingConf(cfg_dict, master_cfg)
logger.info("Baking Configuration {}".format(cfg))
baking_address = cfg.get_baking_address()
payment_address = cfg.get_payment_address()
logger.info(LINER)
logger.info("BAKING ADDRESS is {}".format(baking_address))
logger.info("PAYMENT ADDRESS is {}".format(payment_address))
logger.info(LINER)
# 7- get reporting directories
reports_dir = os.path.expanduser(args.reports_base)
reports_dir = os.path.join(reports_dir, baking_address)
payments_root = get_payment_root(reports_dir, create=True)
get_successful_payments_dir(payments_root, create=True)
get_failed_payments_dir(payments_root, create=True)
def do_build_parser(self, e):
provider_factory = ProviderFactory(self.args.reward_data_provider)
self.__parser = BakingYamlConfParser(
yaml_text=self.__config_text,
clnt_mngr=self.__node_client,
provider_factory=provider_factory,
network_config=self.__nw_cfg,
node_url=self.args.node_endpoint,
api_base_url=self.args.api_base_url)
def test_disk_full_payment_producer(args, caplog):
# Issue: https://github.com/tezos-reward-distributor-organization/tezos-reward-distributor/issues/504
client_manager = ClientManager(args.node_endpoint, args.signer_endpoint)
network_config_map = init_network_config(args.network, client_manager)
factory = ProviderFactory(provider="prpc")
parser = BakingYamlConfParser(
baking_config, None, None, None, None, block_api=factory, api_base_url=None
)
parser.parse()
parser.process()
cfg_dict = parser.get_conf_obj()
baking_cfg = BakingConf(cfg_dict)
baking_dirs = BakingDirs(args, baking_cfg.get_baking_address())
srvc_fee_calc = ServiceFeeCalculator(
baking_cfg.get_full_supporters_set(),
baking_cfg.get_specials_map(),
baking_cfg.get_service_fee(),
)
payments_queue = queue.Queue(50)
plc = ProcessLifeCycle(None)
pp = PaymentProducer(
name="producer",
network_config=network_config_map[args.network],
payments_dir=baking_dirs.payments_root,
calculations_dir=baking_dirs.calculations_root,
run_mode=RunMode(args.run_mode),
service_fee_calc=srvc_fee_calc,
release_override=args.release_override,
payment_offset=args.payment_offset,
baking_cfg=baking_cfg,
life_cycle=plc,
payments_queue=payments_queue,
dry_run=args.dry_run,
client_manager=client_manager,
node_url=args.node_endpoint,
reward_data_provider=args.reward_data_provider,
node_url_public=args.node_addr_public,
api_base_url=args.api_base_url,
retry_injected=args.retry_injected,
initial_payment_cycle=args.initial_cycle,
)
assert disk_is_full()
try:
pp.daemon = True
pp.start()
finally:
pp.stop()
assert (
"Disk is becoming full. Only 0.50 Gb left from 10.00 Gb. Please clean up disk to continue saving logs and reports."
in caplog.text
)
def test_disk_full_payment_consumer(args, caplog):
# Issue: https://github.com/tezos-reward-distributor-organization/tezos-reward-distributor/issues/504
client_manager = ClientManager(args.node_endpoint, args.signer_endpoint)
network_config_map = init_network_config(args.network, client_manager)
factory = ProviderFactory(provider="prpc")
parser = BakingYamlConfParser(
baking_config, None, None, None, None, block_api=factory, api_base_url=None
)
parser.parse()
parser.process()
cfg_dict = parser.get_conf_obj()
baking_cfg = BakingConf(cfg_dict)
baking_dirs = BakingDirs(args, baking_cfg.get_baking_address())
payments_queue = queue.Queue(50)
plugins_manager = plugins.PluginManager(baking_cfg.get_plugins_conf(), args.dry_run)
pc = PaymentConsumer(
name="consumer0",
payments_dir=baking_dirs.payments_root,
key_name=baking_cfg.get_payment_address(),
payments_queue=payments_queue,
node_addr=args.node_endpoint,
client_manager=client_manager,
plugins_manager=plugins_manager,
rewards_type=baking_cfg.get_rewards_type(),
args=args,
dry_run=args.dry_run,
reactivate_zeroed=baking_cfg.get_reactivate_zeroed(),
delegator_pays_ra_fee=baking_cfg.get_delegator_pays_ra_fee(),
delegator_pays_xfer_fee=baking_cfg.get_delegator_pays_xfer_fee(),
dest_map=baking_cfg.get_dest_map(),
network_config=network_config_map[args.network],
publish_stats=not args.do_not_publish_stats,
)
assert disk_is_full()
try:
pc.daemon = True
pc.start()
finally:
pc.stop()
assert (
"Disk is becoming full. Only 0.30 Gb left from 11.00 Gb. Please clean up disk to continue saving logs and reports."
in caplog.text
)
def onbakingaddress(input):
try:
AddressValidator("bakingaddress").validate(input)
except Exception as e:
printe("Invalid baking address: " + traceback.format_exc())
return
if not input.startswith("tz"):
printe("Only tz addresses are allowed")
return
provider_factory = ProviderFactory(args.reward_data_provider)
global parser
parser = BakingYamlConfParser(None, wllt_clnt_mngr, provider_factory, network_config, args.node_addr)
parser.set(BAKING_ADDRESS, input)
messages['paymentaddress']=messages['paymentaddress'].format([v['alias'] for k,v in wllt_clnt_mngr.get_addr_dict().items() if v['sk']])+" (without quotes)"
fsm.go()
def onbakingaddress(input):
try:
AddressValidator("baking address").validate(input)
except Exception as e:
printe(f"Invalid baking address: {str(e)}")
return
if not input.startswith("tz"):
printe("Only tz addresses are allowed")
return
provider_factory = ProviderFactory(args.reward_data_provider)
global parser
parser = BakingYamlConfParser(None, client_manager, provider_factory, network_config, args.node_endpoint,
api_base_url=args.api_base_url)
parser.set(BAKING_ADDRESS, input)
fsm.go()
def main(args):
logger.info("TRD version {} is running in {} mode.".format(
version.version,
"daemon" if args.background_service else "interactive"))
logger.info("Arguments Configuration = {}".format(
json.dumps(args.__dict__, indent=1)))
# 1- find where configuration is
config_dir = os.path.expanduser(args.config_dir)
# create configuration directory if it is not present
# so that user can easily put his configuration there
if config_dir and not os.path.exists(config_dir):
os.makedirs(config_dir)
# 2- Load master configuration file if it is present
master_config_file_path = os.path.join(config_dir, "master.yaml")
master_cfg = {}
if os.path.isfile(master_config_file_path):
logger.debug("Loading master configuration file {}".format(
master_config_file_path))
master_parser = YamlConfParser(
ConfigParser.load_file(master_config_file_path))
master_cfg = master_parser.parse()
else:
logger.debug("master configuration file not present.")
managers = None
contracts_by_alias = None
addresses_by_pkh = None
if 'managers' in master_cfg:
managers = master_cfg['managers']
if 'contracts_by_alias' in master_cfg:
contracts_by_alias = master_cfg['contracts_by_alias']
if 'addresses_by_pkh' in master_cfg:
addresses_by_pkh = master_cfg['addresses_by_pkh']
# 3- get client path
client_path = get_client_path(
[x.strip() for x in args.executable_dirs.split(',')], args.docker,
args.network, args.verbose)
logger.debug("Dune client path is {}".format(client_path))
# 4. get network config
config_client_manager = SimpleClientManager(client_path)
network_config_map = init_network_config(args.network,
config_client_manager,
args.node_addr)
network_config = network_config_map[args.network]
logger.debug("Network config {}".format(network_config))
# 5- load baking configuration file
config_file_path = get_baking_configuration_file(config_dir)
logger.info(
"Loading baking configuration file {}".format(config_file_path))
wllt_clnt_mngr = WalletClientManager(client_path,
contracts_by_alias,
addresses_by_pkh,
managers,
verbose=args.verbose)
provider_factory = ProviderFactory(args.reward_data_provider,
verbose=args.verbose)
parser = BakingYamlConfParser(ConfigParser.load_file(config_file_path),
wllt_clnt_mngr,
provider_factory,
network_config,
args.node_addr,
verbose=args.verbose)
parser.parse()
parser.validate()
parser.process()
cfg_dict = parser.get_conf_obj()
# dictionary to BakingConf object, for a bit of type safety
cfg = BakingConf(cfg_dict, master_cfg)
logger.info("Baking Configuration {}".format(cfg))
baking_address = cfg.get_baking_address()
payment_address = cfg.get_payment_address()
logger.info(LINER)
logger.info("BAKING ADDRESS is {}".format(baking_address))
logger.info("PAYMENT ADDRESS is {}".format(payment_address))
logger.info(LINER)
# 6- is it a reports run
dry_run = args.dry_run_no_consumers or args.dry_run
if args.dry_run_no_consumers:
global NB_CONSUMERS
NB_CONSUMERS = 0
# 7- get reporting directories
reports_base = os.path.expanduser(args.reports_base)
# if in reports run mode, do not create consumers
# create reports in reports directory
if dry_run:
reports_base = os.path.expanduser("./reports")
reports_dir = os.path.join(reports_base, baking_address)
payments_root = get_payment_root(reports_dir, create=True)
calculations_root = get_calculations_root(reports_dir, create=True)
get_successful_payments_dir(payments_root, create=True)
get_failed_payments_dir(payments_root, create=True)
# 8- start the life cycle
life_cycle.start(not dry_run)
# 9- service fee calculator
srvc_fee_calc = ServiceFeeCalculator(cfg.get_full_supporters_set(),
cfg.get_specials_map(),
cfg.get_service_fee())
if args.initial_cycle is None:
recent = get_latest_report_file(payments_root)
# if payment logs exists set initial cycle to following cycle
# if payment logs does not exists, set initial cycle to 0, so that payment starts from last released rewards
args.initial_cycle = 0 if recent is None else int(recent) + 1
logger.info("initial_cycle set to {}".format(args.initial_cycle))
p = PaymentProducer(name='producer',
initial_payment_cycle=args.initial_cycle,
network_config=network_config,
payments_dir=payments_root,
calculations_dir=calculations_root,
run_mode=RunMode(args.run_mode),
service_fee_calc=srvc_fee_calc,
release_override=args.release_override,
payment_offset=args.payment_offset,
baking_cfg=cfg,
life_cycle=life_cycle,
payments_queue=payments_queue,
dry_run=dry_run,
wllt_clnt_mngr=wllt_clnt_mngr,
node_url=args.node_addr,
provider_factory=provider_factory,
verbose=args.verbose)
p.start()
publish_stats = not args.do_not_publish_stats
for i in range(NB_CONSUMERS):
c = PaymentConsumer(
name='consumer' + str(i),
payments_dir=payments_root,
key_name=payment_address,
client_path=client_path,
payments_queue=payments_queue,
node_addr=args.node_addr,
wllt_clnt_mngr=wllt_clnt_mngr,
args=args,
verbose=args.verbose,
dry_run=dry_run,
delegator_pays_xfer_fee=cfg.get_delegator_pays_xfer_fee(),
dest_map=cfg.get_dest_map(),
network_config=network_config,
publish_stats=publish_stats)
time.sleep(1)
c.start()
logger.info("Application start completed")
logger.info(LINER)
try:
while life_cycle.is_running():
time.sleep(10)
except KeyboardInterrupt:
logger.info("Interrupted.")
life_cycle.stop()
def main(args):
logger.info("Arguments Configuration = {}".format(
json.dumps(args.__dict__, indent=1)))
# 1- find where configuration is
config_dir = os.path.expanduser(args.config_dir)
# create configuration directory if it is not present
# so that user can easily put his configuration there
if config_dir and not os.path.exists(config_dir):
os.makedirs(config_dir)
# 2- Load master configuration file if it is present
master_config_file_path = os.path.join(config_dir, "master.yaml")
master_cfg = {}
if os.path.isfile(master_config_file_path):
logger.info("Loading master configuration file {}".format(
master_config_file_path))
master_parser = YamlConfParser(
ConfigParser.load_file(master_config_file_path))
master_cfg = master_parser.parse()
else:
logger.info("master configuration file not present.")
managers = None
contracts_by_alias = None
addresses_by_pkh = None
if 'managers' in master_cfg:
managers = master_cfg['managers']
if 'contracts_by_alias' in master_cfg:
contracts_by_alias = master_cfg['contracts_by_alias']
if 'addresses_by_pkh' in master_cfg:
addresses_by_pkh = master_cfg['addresses_by_pkh']
# 3- get client path
client_path = get_client_path(
[x.strip() for x in args.executable_dirs.split(',')], args.docker,
args.network, args.verbose)
logger.debug("Tezos client path is {}".format(client_path))
# 4. get network config
config_client_manager = SimpleClientManager(client_path, args.node_addr)
network_config_map = init_network_config(args.network,
config_client_manager,
args.node_addr)
network_config = network_config_map[args.network]
# 5- load baking configuration file
config_file_path = get_baking_configuration_file(config_dir)
logger.info(
"Loading baking configuration file {}".format(config_file_path))
wllt_clnt_mngr = WalletClientManager(client_path,
contracts_by_alias,
addresses_by_pkh,
managers,
verbose=args.verbose)
provider_factory = ProviderFactory(args.reward_data_provider,
verbose=args.verbose)
parser = BakingYamlConfParser(ConfigParser.load_file(config_file_path),
wllt_clnt_mngr,
provider_factory,
network_config,
args.node_addr,
verbose=args.verbose,
api_base_url=args.api_base_url)
parser.parse()
parser.validate()
parser.process()
cfg_dict = parser.get_conf_obj()
# dictionary to BakingConf object, for a bit of type safety
cfg = BakingConf(cfg_dict, master_cfg)
logger.info("Baking Configuration {}".format(cfg))
baking_address = cfg.get_baking_address()
payment_address = cfg.get_payment_address()
logger.info(LINER)
logger.info("BAKING ADDRESS is {}".format(baking_address))
logger.info("PAYMENT ADDRESS is {}".format(payment_address))
logger.info(LINER)
# 6- is it a reports run
dry_run = args.dry_run_no_consumers or args.dry_run
if args.dry_run_no_consumers:
global NB_CONSUMERS
NB_CONSUMERS = 0
# 7- get reporting directories
reports_dir = os.path.expanduser(args.reports_base)
# if in reports run mode, do not create consumers
# create reports in reports directory
if dry_run:
reports_dir = os.path.expanduser("./reports")
reports_dir = os.path.join(reports_dir, baking_address)
payments_root = get_payment_root(reports_dir, create=True)
calculations_root = get_calculations_root(reports_dir, create=True)
get_successful_payments_dir(payments_root, create=True)
get_failed_payments_dir(payments_root, create=True)
# 8- start the life cycle
life_cycle.start(False)
# 9- service fee calculator
srvc_fee_calc = ServiceFeeCalculator(cfg.get_full_supporters_set(),
cfg.get_specials_map(),
cfg.get_service_fee())
try:
p = PaymentProducer(name='producer',
initial_payment_cycle=None,
network_config=network_config,
payments_dir=payments_root,
calculations_dir=calculations_root,
run_mode=RunMode.ONETIME,
service_fee_calc=srvc_fee_calc,
release_override=0,
payment_offset=0,
baking_cfg=cfg,
life_cycle=life_cycle,
payments_queue=payments_queue,
dry_run=dry_run,
wllt_clnt_mngr=wllt_clnt_mngr,
node_url=args.node_addr,
provider_factory=provider_factory,
verbose=args.verbose,
api_base_url=args.api_base_url)
p.retry_failed_payments(args.retry_injected)
c = PaymentConsumer(
name='consumer_retry_failed',
payments_dir=payments_root,
key_name=payment_address,
client_path=client_path,
payments_queue=payments_queue,
node_addr=args.node_addr,
wllt_clnt_mngr=wllt_clnt_mngr,
verbose=args.verbose,
dry_run=dry_run,
delegator_pays_xfer_fee=cfg.get_delegator_pays_xfer_fee(),
network_config=network_config)
time.sleep(1)
c.start()
p.exit()
c.join()
logger.info("Application start completed")
logger.info(LINER)
sleep(5)
except KeyboardInterrupt:
logger.info("Interrupted.")
def test_batch_payer_total_payout_amount():
factory = ProviderFactory(provider="prpc")
parser = BakingYamlConfParser(
baking_config, None, None, None, None, block_api=factory, api_base_url=None
)
parser.parse()
parser.process()
cfg_dict = parser.get_conf_obj()
baking_cfg = BakingConf(cfg_dict)
srvc_fee_calc = ServiceFeeCalculator(
baking_cfg.get_full_supporters_set(),
baking_cfg.get_specials_map(),
baking_cfg.get_service_fee(),
)
rules_model = RulesModel(
baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map(),
)
payment_calc = PhasedPaymentCalculator(
baking_cfg.get_founders_map(),
baking_cfg.get_owners_map(),
srvc_fee_calc,
int(baking_cfg.get_min_delegation_amount() * MUTEZ_PER_TEZ),
rules_model,
)
rewardApi = factory.newRewardApi(
default_network_config_map[CURRENT_TESTNET], baking_cfg.get_baking_address(), ""
)
# Simulate logic in payment_producer
reward_logs = []
attempts = 0
exiting = False
while not exiting and attempts < 2:
attempts += 1
# Reward data
# Fetch cycle 51 of granadanet for tz1gtHbmBF3TSebsgJfJPvUB2e9x8EDeNm6V
reward_model = rewardApi.get_rewards_for_cycle_map(
PAYOUT_CYCLE, RewardsType.ACTUAL
)
# Calculate rewards - payment_producer.py
reward_model.computed_reward_amount = reward_model.total_reward_amount
reward_logs, total_amount = payment_calc.calculate(reward_model)
# Check total reward amount matches sums of records
# diff of 1 expected due to floating point arithmetic
assert (
total_amount - sum([rl.adjusted_amount for rl in reward_logs if rl.payable])
<= 1
)
exiting = True
# Merge payments to same address
phaseMerge = CalculatePhaseMerge()
reward_logs = phaseMerge.calculate(reward_logs)
# Handle remapping of payment to alternate address
phaseMapping = CalculatePhaseMapping()
reward_logs = phaseMapping.calculate(reward_logs, baking_cfg.get_dest_map())
# Filter zero-balance addresses based on config
phaseZeroBalance = CalculatePhaseZeroBalance()
reward_logs = phaseZeroBalance.calculate(
reward_logs, baking_cfg.get_reactivate_zeroed()
)
# Filter out non-payable items
reward_logs = [payment_item for payment_item in reward_logs if payment_item.payable]
reward_logs.sort(key=cmp_to_key(cmp_by_type_balance))
batch_payer = BatchPayer(
node_url=node_endpoint,
pymnt_addr="tz1N4UfQCahHkRShBanv9QP9TnmXNgCaqCyZ",
clnt_mngr=ClientManager(node_endpoint, PRIVATE_SIGNER_URL),
delegator_pays_ra_fee=True,
delegator_pays_xfer_fee=True,
network_config=network,
plugins_manager=PluginManager(baking_cfg.get_plugins_conf(), dry_run=True),
dry_run=True,
)
# Do the payment
(
_,
total_attempts,
total_payout_amount,
number_future_payable_cycles,
) = batch_payer.pay(reward_logs, dry_run=True)
# Payment does not have status done, paid or injected thus the total payout amount is zero
assert total_payout_amount == 0
assert number_future_payable_cycles == 2
assert total_attempts == 3
# Check the adjusted amount
assert reward_logs[0].adjusted_amount == 40418486
assert reward_logs[1].adjusted_amount == 10581272
assert reward_logs[2].adjusted_amount == 109732835
assert reward_logs[3].adjusted_amount == 48362127
assert reward_logs[4].adjusted_amount == 29116310
def __init__(self,
name,
initial_payment_cycle,
network_config,
payments_dir,
calculations_dir,
run_mode,
service_fee_calc,
release_override,
payment_offset,
baking_cfg,
payments_queue,
life_cycle,
dry_run,
client_manager,
node_url,
reward_data_provider,
node_url_public='',
api_base_url=None,
retry_injected=False):
super(PaymentProducer, self).__init__()
self.rules_model = RulesModel(baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map())
self.baking_address = baking_cfg.get_baking_address()
self.owners_map = baking_cfg.get_owners_map()
self.founders_map = baking_cfg.get_founders_map()
self.min_delegation_amt_in_mutez = baking_cfg.get_min_delegation_amount(
) * MUTEZ
self.delegator_pays_xfer_fee = baking_cfg.get_delegator_pays_xfer_fee()
self.provider_factory = ProviderFactory(reward_data_provider)
self.name = name
self.node_url = node_url
self.client_manager = client_manager
self.reward_api = self.provider_factory.newRewardApi(
network_config, self.baking_address, self.node_url,
node_url_public, api_base_url)
self.block_api = self.provider_factory.newBlockApi(
network_config, self.node_url, api_base_url)
dexter_contracts_set = baking_cfg.get_contracts_set()
if len(dexter_contracts_set) > 0 and not (self.reward_api.name
== 'tzstats'):
logger.warning(
"The Dexter functionality is currently only supported using tzstats."
"The contract address will be treated as a normal delegator.")
else:
self.reward_api.set_dexter_contracts_set(dexter_contracts_set)
self.rewards_type = baking_cfg.get_rewards_type()
self.fee_calc = service_fee_calc
self.initial_payment_cycle = initial_payment_cycle
if self.initial_payment_cycle is None:
recent = get_latest_report_file(payments_dir)
# if payment logs exists set initial cycle to following cycle
# if payment logs does not exists, set initial cycle to 0, so that payment starts from last released rewards
self.initial_payment_cycle = 0 if recent is None else int(
recent) + 1
logger.info("initial_cycle set to {}".format(
self.initial_payment_cycle))
self.nw_config = network_config
self.payments_root = payments_dir
self.calculations_dir = calculations_dir
self.run_mode = run_mode
self.exiting = False
self.release_override = release_override
self.payment_offset = payment_offset
self.payments_queue = payments_queue
self.life_cycle = life_cycle
self.dry_run = dry_run
self.payment_calc = PhasedPaymentCalculator(
self.founders_map, self.owners_map, self.fee_calc,
self.min_delegation_amt_in_mutez, self.rules_model)
self.retry_fail_thread = threading.Thread(target=self.retry_fail_run,
name=self.name +
"_retry_fail")
self.retry_fail_event = threading.Event()
self.retry_injected = retry_injected
self.retry_producer = RetryProducer(self.payments_queue,
self.reward_api, self,
self.payments_root,
self.retry_injected)
logger.info('Producer "{}" started'.format(self.name))
class PaymentProducer(threading.Thread, PaymentProducerABC):
def __init__(self,
name,
initial_payment_cycle,
network_config,
payments_dir,
calculations_dir,
run_mode,
service_fee_calc,
release_override,
payment_offset,
baking_cfg,
payments_queue,
life_cycle,
dry_run,
client_manager,
node_url,
reward_data_provider,
node_url_public='',
api_base_url=None,
retry_injected=False):
super(PaymentProducer, self).__init__()
self.rules_model = RulesModel(baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map())
self.baking_address = baking_cfg.get_baking_address()
self.owners_map = baking_cfg.get_owners_map()
self.founders_map = baking_cfg.get_founders_map()
self.min_delegation_amt_in_mutez = baking_cfg.get_min_delegation_amount(
) * MUTEZ
self.delegator_pays_xfer_fee = baking_cfg.get_delegator_pays_xfer_fee()
self.provider_factory = ProviderFactory(reward_data_provider)
self.name = name
self.node_url = node_url
self.client_manager = client_manager
self.reward_api = self.provider_factory.newRewardApi(
network_config, self.baking_address, self.node_url,
node_url_public, api_base_url)
self.block_api = self.provider_factory.newBlockApi(
network_config, self.node_url, api_base_url)
dexter_contracts_set = baking_cfg.get_contracts_set()
if len(dexter_contracts_set) > 0 and not (self.reward_api.name
== 'tzstats'):
logger.warning(
"The Dexter functionality is currently only supported using tzstats."
"The contract address will be treated as a normal delegator.")
else:
self.reward_api.set_dexter_contracts_set(dexter_contracts_set)
self.rewards_type = baking_cfg.get_rewards_type()
self.fee_calc = service_fee_calc
self.initial_payment_cycle = initial_payment_cycle
if self.initial_payment_cycle is None:
recent = get_latest_report_file(payments_dir)
# if payment logs exists set initial cycle to following cycle
# if payment logs does not exists, set initial cycle to 0, so that payment starts from last released rewards
self.initial_payment_cycle = 0 if recent is None else int(
recent) + 1
logger.info("initial_cycle set to {}".format(
self.initial_payment_cycle))
self.nw_config = network_config
self.payments_root = payments_dir
self.calculations_dir = calculations_dir
self.run_mode = run_mode
self.exiting = False
self.release_override = release_override
self.payment_offset = payment_offset
self.payments_queue = payments_queue
self.life_cycle = life_cycle
self.dry_run = dry_run
self.payment_calc = PhasedPaymentCalculator(
self.founders_map, self.owners_map, self.fee_calc,
self.min_delegation_amt_in_mutez, self.rules_model)
self.retry_fail_thread = threading.Thread(target=self.retry_fail_run,
name=self.name +
"_retry_fail")
self.retry_fail_event = threading.Event()
self.retry_injected = retry_injected
self.retry_producer = RetryProducer(self.payments_queue,
self.reward_api, self,
self.payments_root,
self.retry_injected)
logger.info('Producer "{}" started'.format(self.name))
def exit(self):
if not self.exiting:
self.payments_queue.put(
PaymentBatch(self, 0, [self.create_exit_payment()]))
self.exiting = True
if self.life_cycle.is_running() and threading.current_thread(
) is not threading.main_thread():
_thread.interrupt_main()
if self.retry_fail_event:
self.retry_fail_event.set()
def retry_fail_run(self):
logger.info('Retry Fail thread "{}" started'.format(
self.retry_fail_thread.name))
sleep(60) # producer thread already tried once, wait for next try
while not self.exiting and self.life_cycle.is_running():
self.retry_producer.retry_failed_payments()
try:
# prepare to wait on event
self.retry_fail_event.clear()
# this will either return with timeout or set from parent producer thread
self.retry_fail_event.wait(60 * 60) # 1 hour
except RuntimeError:
pass
def run(self):
# call first retry if not in onetime mode.
# retry_failed script is more suitable for one time cases.
if not self.run_mode == RunMode.ONETIME:
self.retry_producer.retry_failed_payments()
if self.run_mode == RunMode.RETRY_FAILED:
sleep(5)
self.exit()
return
# first retry is done by producer thread, start retry thread for further retries
if self.run_mode == RunMode.FOREVER:
self.retry_fail_thread.start()
try:
current_cycle = self.block_api.get_current_cycle()
pymnt_cycle = self.initial_payment_cycle
except ApiProviderException as a:
logger.error(
"Unable to fetch current cycle, {:s}. Exiting.".format(str(a)))
self.exit()
return
# if non-positive initial_payment_cycle, set initial_payment_cycle to
# 'current cycle - abs(initial_cycle) - (NB_FREEZE_CYCLE+1)'
if self.initial_payment_cycle <= 0:
pymnt_cycle = current_cycle - abs(self.initial_payment_cycle) - (
self.nw_config['NB_FREEZE_CYCLE'] + 1)
logger.debug("Payment cycle is set to {}".format(pymnt_cycle))
get_verbose_log_helper().reset(pymnt_cycle)
while not self.exiting and self.life_cycle.is_running():
# take a breath
sleep(5)
try:
# Check if local node is bootstrapped; sleep if needed; restart loop
if not self.node_is_bootstrapped():
logger.info(
"Local node, {}, is not in sync with the Tezos network. Will sleep for {} blocks and check again."
.format(self.node_url, BOOTSTRAP_SLEEP))
self.wait_for_blocks(BOOTSTRAP_SLEEP)
continue
# Local node is ready
current_level = self.block_api.get_current_level()
current_cycle = self.block_api.level_to_cycle(current_level)
level_in_cycle = self.block_api.level_in_cycle(current_level)
# create reports dir
if self.calculations_dir and not os.path.exists(
self.calculations_dir):
os.makedirs(self.calculations_dir)
logger.debug(
"Checking for pending payments : payment_cycle <= current_cycle - (self.nw_config['NB_FREEZE_CYCLE'] + 1) - self.release_override"
)
logger.info(
"Checking for pending payments : checking {} <= {} - ({} + 1) - {}"
.format(pymnt_cycle, current_cycle,
self.nw_config['NB_FREEZE_CYCLE'],
self.release_override))
# payments should not pass beyond last released reward cycle
if pymnt_cycle <= current_cycle - (
self.nw_config['NB_FREEZE_CYCLE'] +
1) - self.release_override:
if not self.payments_queue.full():
# Paying upcoming cycles (-R in [-6, -11] )
if pymnt_cycle >= current_cycle:
logger.warn(
"Please note that you are doing payouts for future rewards!!! These rewards are not earned yet, they are an estimation."
)
if not self.rewards_type.isIdeal():
logger.error(
"For future rewards payout, you must configure the payout type to 'Ideal', see documentation"
)
self.exit()
break
# Paying cycles with frozen rewards (-R in [-1, -5] )
elif pymnt_cycle >= current_cycle - self.nw_config[
'NB_FREEZE_CYCLE']:
logger.warn(
"Please note that you are doing payouts for frozen rewards!!!"
)
if (not self.rewards_type.isIdeal()
) and self.reward_api.name == 'RPC':
logger.error(
"Paying out frozen rewards with Node RPC API and rewards type 'Actual' is unsupported, you must use TzKT or tzstats API"
)
self.exit()
break
# If user wants to offset payments within a cycle, check here
if level_in_cycle < self.payment_offset:
wait_offset_blocks = self.payment_offset - level_in_cycle
logger.info(
"Current level within the cycle is {}; Requested offset is {}; Waiting for {} more blocks."
.format(level_in_cycle, self.payment_offset,
wait_offset_blocks))
self.wait_for_blocks(wait_offset_blocks)
continue # Break/Repeat loop
else:
result = self.try_to_pay(
pymnt_cycle,
expected_rewards=self.rewards_type.isIdeal())
if result:
# single run is done. Do not continue.
if self.run_mode == RunMode.ONETIME:
logger.info(
"Run mode ONETIME satisfied. Terminating ..."
)
self.exit()
break
else:
pymnt_cycle = pymnt_cycle + 1
get_verbose_log_helper().reset(pymnt_cycle)
# end of queue size check
else:
logger.debug("Wait a few minutes, queue is full")
# wait a few minutes to let payments finish
sleep(60 * 3)
# end of payment cycle check
else:
logger.info(
"No pending payments for cycle {}, current cycle is {}"
.format(pymnt_cycle, current_cycle))
# pending payments done. Do not wait any more.
if self.run_mode == RunMode.PENDING:
logger.info(
"Run mode PENDING satisfied. Terminating ...")
self.exit()
break
sleep(10)
# calculate number of blocks until end of current cycle
nb_blocks_remaining = (
current_cycle +
1) * self.nw_config['BLOCKS_PER_CYCLE'] - current_level
# plus offset. cycle beginnings may be busy, move payments forward
nb_blocks_remaining = nb_blocks_remaining + self.payment_offset
logger.debug(
"Waiting until next cycle; {} blocks remaining".format(
nb_blocks_remaining))
# wait until current cycle ends
self.wait_for_blocks(nb_blocks_remaining)
except (ApiProviderException, ReadTimeout, ConnectTimeout) as e:
logger.debug(
"{:s} error at payment producer loop: '{:s}'".format(
self.reward_api.name, str(e)),
exc_info=True)
logger.error(
"{:s} error at payment producer loop: '{:s}', will try again."
.format(self.reward_api.name, str(e)))
except Exception as e:
logger.debug(
"Unknown error in payment producer loop: {:s}".format(
str(e)),
exc_info=True)
logger.error(
"Unknown error in payment producer loop: {:s}, will try again."
.format(str(e)))
# end of endless loop
logger.info("Producer returning ...")
# ensure consumer exits
self.exit()
return
def try_to_pay(self, pymnt_cycle, expected_rewards=False):
try:
logger.info("Payment cycle is " + str(pymnt_cycle))
# 0- check for past payment evidence for current cycle
past_payment_state = check_past_payment(self.payments_root,
pymnt_cycle)
if not self.dry_run and past_payment_state:
logger.warn(past_payment_state)
return True
# 1- get reward data
if expected_rewards:
logger.info(
"Using expected/ideal rewards for payouts calculations")
else:
logger.info("Using actual rewards for payouts calculations")
reward_model = self.reward_api.get_rewards_for_cycle_map(
pymnt_cycle, expected_rewards)
# 2- calculate rewards
reward_logs, total_amount = self.payment_calc.calculate(
reward_model)
# 3- set cycle info
for rl in reward_logs:
rl.cycle = pymnt_cycle
total_amount_to_pay = sum(
[rl.amount for rl in reward_logs if rl.payable])
# 4- if total_rewards > 0, proceed with payment
if total_amount_to_pay > 0:
report_file_path = get_calculation_report_file(
self.calculations_dir, pymnt_cycle)
# 5- send to payment consumer
self.payments_queue.put(
PaymentBatch(self, pymnt_cycle, reward_logs))
# logger.info("Total payment amount is {:,} mutez. %s".format(total_amount_to_pay),
# "" if self.delegator_pays_xfer_fee else "(Transfer fee is not included)")
logger.debug("Creating calculation report (%s)",
report_file_path)
sleep(5.0)
# 6- create calculations report file. This file contains calculations details
self.create_calculations_report(reward_logs, report_file_path,
total_amount, expected_rewards)
# 7- processing of cycle is done
logger.info(
"Reward creation is done for cycle {}, created {} rewards."
.format(pymnt_cycle, len(reward_logs)))
elif total_amount_to_pay == 0:
logger.info("Total payment amount is 0. Nothing to pay!")
except ApiProviderException as a:
logger.error("[try_to_pay] API provider error {:s}".format(str(a)))
raise a from a
except Exception as e:
logger.error("[try_to_pay] Generic exception {:s}".format(str(e)))
raise e from e
# Either succeeded or raised exception
return True
def wait_for_blocks(self, nb_blocks_remaining):
for x in range(nb_blocks_remaining):
sleep(self.nw_config['BLOCK_TIME_IN_SEC'])
# if shutting down, exit
if not self.life_cycle.is_running():
self.exit()
break
def node_is_bootstrapped(self):
# Get RPC node's (-A) bootstrap time. If bootstrap time + 2 minutes is
# before local time, node is not bootstrapped.
#
# clnt_mngr is a super class of SimpleClientManager which interfaces
# with the tezos-node used for txn forging/signing/injection. This is the
# node which we need to determine bootstrapped state
try:
boot_time = self.client_manager.get_bootstrapped()
utc_time = datetime.utcnow()
if (boot_time + timedelta(minutes=2)) < utc_time:
logger.info(
"Current time is '{}', latest block of local node is '{}'."
.format(utc_time, boot_time))
return False
except ValueError:
logger.error(
"Unable to determine local node's bootstrap status. Continuing..."
)
return True
def create_calculations_report(self, payment_logs, report_file_path,
total_rewards, expected_rewards):
rt = "I" if expected_rewards else "A"
# Open reports file and write; auto-closes file
with open(report_file_path, 'w', newline='') as f:
writer = csv.writer(f,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
# write headers and total rewards
writer.writerow([
"address", "type", "staked_balance", "current_balance",
"ratio", "fee_ratio", "amount", "fee_amount", "fee_rate",
"payable", "skipped", "atphase", "desc", "payment_address",
"rewards_type"
])
# First row is for the baker
writer.writerow([
self.baking_address, "B",
sum([pl.staking_balance for pl in payment_logs]),
"{0:f}".format(1.0), "{0:f}".format(1.0), "{0:f}".format(0.0),
"{0:f}".format(total_rewards), "{0:f}".format(0.0),
"{0:f}".format(0.0), "0", "0", "-1", "Baker", "None", rt
])
for pymnt_log in payment_logs:
# write row to csv file
array = [
pymnt_log.address, pymnt_log.type,
pymnt_log.staking_balance, pymnt_log.current_balance,
"{0:.10f}".format(pymnt_log.ratio),
"{0:.10f}".format(pymnt_log.service_fee_ratio),
"{0:f}".format(pymnt_log.amount),
"{0:f}".format(pymnt_log.service_fee_amount),
"{0:f}".format(pymnt_log.service_fee_rate),
"1" if pymnt_log.payable else "0",
"1" if pymnt_log.skipped else "0",
pymnt_log.skippedatphase if pymnt_log.skipped else "-1",
pymnt_log.desc if pymnt_log.desc else "None",
pymnt_log.paymentaddress, rt
]
writer.writerow(array)
logger.debug(
"Reward created for {:s} type: {:s}, stake bal: {:>10.2f}, cur bal: {:>10.2f}, ratio: {:.6f}, fee_ratio: {:.6f}, "
"amount: {:>10.6f}, fee_amount: {:>4.6f}, fee_rate: {:.2f}, payable: {:s}, skipped: {:s}, at-phase: {:d}, "
"desc: {:s}, pay_addr: {:s}, type: {:s}".format(
pymnt_log.address, pymnt_log.type,
pymnt_log.staking_balance / MUTEZ,
pymnt_log.current_balance / MUTEZ, pymnt_log.ratio,
pymnt_log.service_fee_ratio, pymnt_log.amount / MUTEZ,
pymnt_log.service_fee_amount / MUTEZ,
pymnt_log.service_fee_rate,
"Y" if pymnt_log.payable else "N",
"Y" if pymnt_log.skipped else "N",
pymnt_log.skippedatphase, pymnt_log.desc,
pymnt_log.paymentaddress, rt))
logger.info(
"Calculation report is created at '{}'".format(report_file_path))
@staticmethod
def create_exit_payment():
return RewardLog.ExitInstance()
def notify_retry_fail_thread(self):
self.retry_fail_event.set()
# upon success retry failed payments if present
# success may indicate what went wrong in past is fixed.
def on_success(self, pymnt_batch):
self.notify_retry_fail_thread()
def on_fail(self, pymnt_batch):
pass
class PaymentProducer(threading.Thread, PaymentProducerABC):
def __init__(
self,
name,
initial_payment_cycle,
network_config,
payments_dir,
calculations_dir,
run_mode,
service_fee_calc,
release_override,
payment_offset,
baking_cfg,
payments_queue,
life_cycle,
dry_run,
client_manager,
node_url,
reward_data_provider,
node_url_public="",
api_base_url=None,
retry_injected=False,
):
super(PaymentProducer, self).__init__()
self.event = threading.Event()
self.rules_model = RulesModel(
baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map(),
)
self.baking_address = baking_cfg.get_baking_address()
self.owners_map = baking_cfg.get_owners_map()
self.founders_map = baking_cfg.get_founders_map()
self.min_delegation_amt_in_mutez = (
baking_cfg.get_min_delegation_amount() * MUTEZ)
self.delegator_pays_xfer_fee = baking_cfg.get_delegator_pays_xfer_fee()
self.provider_factory = ProviderFactory(reward_data_provider)
self.name = name
self.node_url = node_url
self.client_manager = client_manager
self.reward_api = self.provider_factory.newRewardApi(
network_config,
self.baking_address,
self.node_url,
node_url_public,
api_base_url,
)
self.block_api = self.provider_factory.newBlockApi(
network_config, self.node_url, api_base_url)
dexter_contracts_set = baking_cfg.get_contracts_set()
if len(dexter_contracts_set) > 0 and not (self.reward_api.name
== "tzstats"):
logger.warning(
"The Dexter functionality is currently only supported using tzstats."
"The contract address will be treated as a normal delegator.")
else:
self.reward_api.set_dexter_contracts_set(dexter_contracts_set)
self.rewards_type = baking_cfg.get_rewards_type()
self.pay_denunciation_rewards = baking_cfg.get_pay_denunciation_rewards(
)
self.fee_calc = service_fee_calc
self.initial_payment_cycle = initial_payment_cycle
logger.info("Initial cycle set to {}".format(
self.initial_payment_cycle))
self.nw_config = network_config
self.payments_root = payments_dir
self.calculations_dir = calculations_dir
self.run_mode = run_mode
self.exiting = False
self.release_override = release_override
self.payment_offset = payment_offset
self.payments_queue = payments_queue
self.life_cycle = life_cycle
self.dry_run = dry_run
self.payment_calc = PhasedPaymentCalculator(
self.founders_map,
self.owners_map,
self.fee_calc,
self.min_delegation_amt_in_mutez,
self.rules_model,
)
self.retry_fail_thread = threading.Thread(target=self.retry_fail_run,
name=self.name +
"_retry_fail")
self.retry_fail_event = threading.Event()
self.retry_injected = retry_injected
self.retry_producer = RetryProducer(
self.payments_queue,
self.reward_api,
self,
self.payments_root,
self.initial_payment_cycle,
self.retry_injected,
)
logger.debug('Producer "{}" started'.format(self.name))
def exit(self):
if not self.exiting:
self.payments_queue.put(
PaymentBatch(self, 0, [self.create_exit_payment()]))
self.exiting = True
if (self.life_cycle.is_running() and threading.current_thread()
is not threading.main_thread()):
_thread.interrupt_main()
if self.retry_fail_event:
self.retry_fail_event.set()
def retry_fail_run(self):
logger.debug('Retry Fail thread "{}" started'.format(
self.retry_fail_thread.name))
sleep(60) # producer thread already tried once, wait for next try
while not self.exiting and self.life_cycle.is_running():
self.retry_producer.retry_failed_payments()
try:
# prepare to wait on event
self.retry_fail_event.clear()
# this will either return with timeout or set from parent producer thread
self.retry_fail_event.wait(60 * 60) # 1 hour
except RuntimeError:
pass
def run(self):
# call first retry if not in onetime mode.
# retry_failed script is more suitable for one time cases.
if not self.run_mode == RunMode.ONETIME:
self.retry_producer.retry_failed_payments()
if self.run_mode == RunMode.RETRY_FAILED:
sleep(5)
self.exit()
return
# first retry is done by producer thread, start retry thread for further retries
if self.run_mode == RunMode.FOREVER:
self.retry_fail_thread.start()
try:
(
current_cycle,
current_level,
) = self.block_api.get_current_cycle_and_level()
except ApiProviderException as a:
logger.error(
"Unable to fetch current cycle, {:s}. Exiting.".format(str(a)))
self.exit()
return
# if initial_payment_cycle has the default value of -1 resulting in the last released cycle
if self.initial_payment_cycle == -1:
pymnt_cycle = (current_cycle -
(self.nw_config["NB_FREEZE_CYCLE"] + 1) -
self.release_override)
if pymnt_cycle < 0:
logger.error(
"Payment cycle cannot be < 0 but configuration results to {}"
.format(pymnt_cycle))
else:
logger.debug(
"Payment cycle is set to last released cycle {}".format(
pymnt_cycle))
else:
pymnt_cycle = self.initial_payment_cycle
get_verbose_log_helper().reset(pymnt_cycle)
while not self.exiting and self.life_cycle.is_running():
# take a breath
sleep(5)
try:
# Exit if disk is full
# https://github.com/tezos-reward-distributor-organization/tezos-reward-distributor/issues/504
if disk_is_full():
self.exit()
break
# Check if local node is bootstrapped; sleep if needed; restart loop
if not self.node_is_bootstrapped():
logger.info(
"Local node {} is not in sync with the Tezos network. Will sleep for {} blocks and check again."
.format(self.node_url, BOOTSTRAP_SLEEP))
self.wait_for_blocks(BOOTSTRAP_SLEEP)
continue
# Local node is ready
(
current_cycle,
current_level,
) = self.block_api.get_current_cycle_and_level()
level_in_cycle = self.block_api.level_in_cycle(current_level)
# create reports dir
if self.calculations_dir and not os.path.exists(
self.calculations_dir):
os.makedirs(self.calculations_dir)
logger.debug(
"Checking for pending payments: payment_cycle <= current_cycle - (self.nw_config['NB_FREEZE_CYCLE'] + 1) - self.release_override"
)
logger.info(
"Checking for pending payments: checking {} <= {} - ({} + 1) - {}"
.format(
pymnt_cycle,
current_cycle,
self.nw_config["NB_FREEZE_CYCLE"],
self.release_override,
))
# payments should not pass beyond last released reward cycle
if (pymnt_cycle <= current_cycle -
(self.nw_config["NB_FREEZE_CYCLE"] + 1) -
self.release_override):
if not self.payments_queue.full():
if (not self.pay_denunciation_rewards
) and self.reward_api.name == "RPC":
logger.info(
"Error: pay_denunciation_rewards=False requires an indexer since it is not possible to distinguish reward source using RPC"
)
e = "You must set 'pay_denunciation_rewards' to True when using RPC provider."
logger.error(e)
self.exit()
break
# Paying upcoming cycles (-R in [-6, -11] )
if pymnt_cycle >= current_cycle:
logger.warn(
"Please note that you are doing payouts for future rewards!!! These rewards are not earned yet, they are an estimation."
)
if not self.rewards_type.isEstimated():
logger.error(
"For future rewards payout, you must configure the payout type to 'Estimated', see documentation"
)
self.exit()
break
# Paying cycles with frozen rewards (-R in [-1, -5] )
elif (pymnt_cycle >= current_cycle -
self.nw_config["NB_FREEZE_CYCLE"]):
logger.warn(
"Please note that you are doing payouts for frozen rewards!!!"
)
# If user wants to offset payments within a cycle, check here
if level_in_cycle < self.payment_offset:
wait_offset_blocks = self.payment_offset - level_in_cycle
wait_offset_minutes = (
wait_offset_blocks *
self.nw_config["MINIMAL_BLOCK_DELAY"]) / 60
logger.info(
"Current level within the cycle is {}; Requested offset is {}; Waiting for {} more blocks (~{} minutes)"
.format(
level_in_cycle,
self.payment_offset,
wait_offset_blocks,
wait_offset_minutes,
))
self.wait_for_blocks(wait_offset_blocks)
continue # Break/Repeat loop
else:
result = self.try_to_pay(pymnt_cycle,
self.rewards_type,
self.nw_config)
if result:
# single run is done. Do not continue.
if self.run_mode == RunMode.ONETIME:
logger.info(
"Run mode ONETIME satisfied. Terminating..."
)
self.exit()
break
else:
pymnt_cycle = pymnt_cycle + 1
get_verbose_log_helper().reset(pymnt_cycle)
# end of queue size check
else:
logger.debug("Wait a few minutes, queue is full")
# wait a few minutes to let payments finish
sleep(60 * 3)
# end of payment cycle check
else:
logger.info(
"No pending payments for cycle {}, current cycle is {}"
.format(pymnt_cycle, current_cycle))
# pending payments done. Do not wait any more.
if self.run_mode == RunMode.PENDING:
logger.info(
"Run mode PENDING satisfied. Terminating...")
self.exit()
break
sleep(10)
# calculate number of blocks until end of current cycle plus user-defined offset
nb_blocks_remaining = (self.nw_config["BLOCKS_PER_CYCLE"] -
level_in_cycle +
self.payment_offset)
logger.debug(
"Waiting until next cycle; {} blocks remaining".format(
nb_blocks_remaining))
# wait until current cycle ends
self.wait_for_blocks(nb_blocks_remaining)
except (ApiProviderException, ReadTimeout, ConnectTimeout) as e:
logger.debug(
"{:s} error at payment producer loop: '{:s}'".format(
self.reward_api.name, str(e)),
exc_info=True,
)
logger.error(
"{:s} error at payment producer loop: '{:s}', will try again."
.format(self.reward_api.name, str(e)))
except Exception as e:
logger.debug(
"Unknown error in payment producer loop: {:s}".format(
str(e)),
exc_info=True,
)
logger.error(
"Unknown error in payment producer loop: {:s}, will try again."
.format(str(e)))
# end of endless loop
logger.debug("Producer returning...")
# ensure consumer exits
self.exit()
return
def stop(self):
self.exit()
self.event.set()
def compute_rewards(self, reward_model, rewards_type, network_config):
if rewards_type.isEstimated():
logger.info("Using estimated rewards for payouts calculations")
block_reward = network_config["BLOCK_REWARD"]
endorsement_reward = network_config["ENDORSEMENT_REWARD"]
total_estimated_block_reward = reward_model.num_baking_rights * block_reward
total_estimated_endorsement_reward = (
reward_model.num_endorsing_rights * endorsement_reward)
computed_reward_amount = (total_estimated_block_reward +
total_estimated_endorsement_reward)
elif rewards_type.isActual():
logger.info("Using actual rewards for payouts calculations")
if self.pay_denunciation_rewards:
computed_reward_amount = reward_model.total_reward_amount
else:
# omit denunciation rewards
computed_reward_amount = (reward_model.rewards_and_fees -
reward_model.equivocation_losses)
elif rewards_type.isIdeal():
logger.info("Using ideal rewards for payouts calculations")
if self.pay_denunciation_rewards:
computed_reward_amount = (reward_model.total_reward_amount +
reward_model.offline_losses)
else:
# omit denunciation rewards and double baking loss
computed_reward_amount = (reward_model.rewards_and_fees +
reward_model.offline_losses)
return computed_reward_amount
def try_to_pay(self, pymnt_cycle, rewards_type, network_config):
try:
logger.info("Payment cycle is {:s}".format(str(pymnt_cycle)))
# 0- check for past payment evidence for current cycle
past_payment_state = check_past_payment(self.payments_root,
pymnt_cycle)
if past_payment_state:
logger.warn(past_payment_state)
return True
# 1- get reward data
reward_model = self.reward_api.get_rewards_for_cycle_map(
pymnt_cycle, rewards_type)
# 2- compute reward amount to distribute based on configuration
reward_model.computed_reward_amount = self.compute_rewards(
reward_model, rewards_type, network_config)
# 3- calculate rewards for delegators
reward_logs, total_amount = self.payment_calc.calculate(
reward_model)
# 4- set cycle info
for rl in reward_logs:
rl.cycle = pymnt_cycle
total_amount_to_pay = sum(
[rl.amount for rl in reward_logs if rl.payable])
# 5- if total_rewards > 0, proceed with payment
if total_amount_to_pay > 0:
# 6- send to payment consumer
self.payments_queue.put(
PaymentBatch(self, pymnt_cycle, reward_logs))
sleep(5.0)
# 7- create calculations report file. This file contains calculations details
report_file_path = get_calculation_report_file(
self.calculations_dir, pymnt_cycle)
logger.debug("Creating calculation report (%s)",
report_file_path)
self.create_calculations_report(reward_logs, report_file_path,
total_amount, rewards_type)
# 8- processing of cycle is done
logger.info(
"Reward creation is done for cycle {}, created {} rewards."
.format(pymnt_cycle, len(reward_logs)))
elif total_amount_to_pay == 0:
logger.info("Total payment amount is 0. Nothing to pay!")
except ApiProviderException as a:
logger.error("[try_to_pay] API provider error {:s}".format(str(a)))
raise a from a
except Exception as e:
logger.error("[try_to_pay] Generic exception {:s}".format(str(e)))
raise e from e
# Either succeeded or raised exception
return True
def wait_for_blocks(self, nb_blocks_remaining):
for x in range(nb_blocks_remaining):
sleep(self.nw_config["MINIMAL_BLOCK_DELAY"])
# if shutting down, exit
if not self.life_cycle.is_running():
self.exit()
break
def node_is_bootstrapped(self):
# Get RPC node's (-A) bootstrap time. If bootstrap time + 2 minutes is
# before local time, node is not bootstrapped.
#
# clnt_mngr is a super class of SimpleClientManager which interfaces
# with the tezos-node used for txn forging/signing/injection. This is the
# node which we need to determine bootstrapped state
try:
boot_time = self.client_manager.get_bootstrapped()
utc_time = datetime.utcnow()
if (boot_time + timedelta(minutes=2)) < utc_time:
logger.debug(
"Current time is '{}', latest block of local node is '{}'."
.format(utc_time, boot_time))
return False
except ValueError:
logger.error(
"Unable to determine local node's bootstrap status. Continuing..."
)
return True
def create_calculations_report(self, payment_logs, report_file_path,
total_rewards, rewards_type):
if rewards_type.isEstimated():
rt = "E"
elif rewards_type.isActual():
rt = "A"
elif rewards_type.isIdeal():
rt = "I"
try:
# Open reports file and write; auto-closes file
with open(report_file_path, "w", newline="") as f:
writer = csv.writer(f,
delimiter=",",
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
# write headers and total rewards
writer.writerow([
"address",
"type",
"staked_balance",
"current_balance",
"ratio",
"fee_ratio",
"amount",
"fee_amount",
"fee_rate",
"payable",
"skipped",
"atphase",
"desc",
"payment_address",
"rewards_type",
])
# First row is for the baker
writer.writerow([
self.baking_address,
"B",
sum([pl.staking_balance for pl in payment_logs]),
"{0:f}".format(1.0),
"{0:f}".format(1.0),
"{0:f}".format(0.0),
"{0:f}".format(total_rewards),
"{0:f}".format(0.0),
"{0:f}".format(0.0),
"0",
"0",
"-1",
"Baker",
"None",
rt,
])
for pymnt_log in payment_logs:
# write row to csv file
array = [
pymnt_log.address,
pymnt_log.type,
pymnt_log.staking_balance,
pymnt_log.current_balance,
"{0:.10f}".format(pymnt_log.ratio),
"{0:.10f}".format(pymnt_log.service_fee_ratio),
"{0:f}".format(pymnt_log.amount),
"{0:f}".format(pymnt_log.service_fee_amount),
"{0:f}".format(pymnt_log.service_fee_rate),
"1" if pymnt_log.payable else "0",
"1" if pymnt_log.skipped else "0",
pymnt_log.skippedatphase
if pymnt_log.skipped else "-1",
pymnt_log.desc if pymnt_log.desc else "None",
pymnt_log.paymentaddress,
rt,
]
writer.writerow(array)
logger.debug(
"Reward created for {:s} type: {:s}, stake bal: {:>10.2f}, cur bal: {:>10.2f}, ratio: {:.6f}, fee_ratio: {:.6f}, "
"amount: {:>10.6f}, fee_amount: {:>4.6f}, fee_rate: {:.2f}, payable: {:s}, skipped: {:s}, at-phase: {:d}, "
"desc: {:s}, pay_addr: {:s}, type: {:s}".format(
pymnt_log.address,
pymnt_log.type,
pymnt_log.staking_balance / MUTEZ,
pymnt_log.current_balance / MUTEZ,
pymnt_log.ratio,
pymnt_log.service_fee_ratio,
pymnt_log.amount / MUTEZ,
pymnt_log.service_fee_amount / MUTEZ,
pymnt_log.service_fee_rate,
"Y" if pymnt_log.payable else "N",
"Y" if pymnt_log.skipped else "N",
pymnt_log.skippedatphase,
pymnt_log.desc,
pymnt_log.paymentaddress,
rt,
))
except Exception as e:
import errno
print("Exception during write operation invoked: {}".format(e))
if e.errno == errno.ENOSPC:
print("Not enough space on device!")
exit()
logger.info(
"Calculation report is created at '{}'".format(report_file_path))
@staticmethod
def create_exit_payment():
return RewardLog.ExitInstance()
def notify_retry_fail_thread(self):
self.retry_fail_event.set()
# upon success retry failed payments if present
# success may indicate what went wrong in past is fixed.
def on_success(self, pymnt_batch):
self.notify_retry_fail_thread()
def on_fail(self, pymnt_batch):
pass
def test_batch_payer_total_payout_amount():
factory = ProviderFactory(provider="prpc")
parser = BakingYamlConfParser(baking_config,
None,
None,
None,
None,
block_api=factory,
api_base_url=None)
parser.parse()
parser.process()
cfg_dict = parser.get_conf_obj()
baking_cfg = BakingConf(cfg_dict)
srvc_fee_calc = ServiceFeeCalculator(
baking_cfg.get_full_supporters_set(),
baking_cfg.get_specials_map(),
baking_cfg.get_service_fee(),
)
rules_model = RulesModel(
baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map(),
)
payment_calc = PhasedPaymentCalculator(
baking_cfg.get_founders_map(),
baking_cfg.get_owners_map(),
srvc_fee_calc,
baking_cfg.get_min_delegation_amount() * MUTEZ,
rules_model,
)
rewardApi = factory.newRewardApi(
default_network_config_map[CURRENT_TESTNET],
baking_cfg.get_baking_address(), "")
# Simulate logic in payment_producer
reward_logs = []
attempts = 0
exiting = False
while not exiting and attempts < 2:
attempts += 1
# Reward data
# Fetch cycle 51 of granadanet for tz1gtHbmBF3TSebsgJfJPvUB2e9x8EDeNm6V
reward_model = rewardApi.get_rewards_for_cycle_map(
PAYOUT_CYCLE, RewardsType.ACTUAL)
# Calculate rewards - payment_producer.py
reward_model.computed_reward_amount = reward_model.total_reward_amount
reward_logs, total_amount = payment_calc.calculate(reward_model)
# Check total reward amount matches sums of records
assert total_amount == sum(
[rl.amount for rl in reward_logs if rl.payable])
exiting = True
# Merge payments to same address
phaseMerge = CalculatePhaseMerge()
reward_logs = phaseMerge.calculate(reward_logs)
# Handle remapping of payment to alternate address
phaseMapping = CalculatePhaseMapping()
reward_logs = phaseMapping.calculate(reward_logs,
baking_cfg.get_dest_map())
# Filter zero-balance addresses based on config
phaseZeroBalance = CalculatePhaseZeroBalance()
reward_logs = phaseZeroBalance.calculate(
reward_logs, baking_cfg.get_reactivate_zeroed())
# Filter out non-payable items
reward_logs = [
payment_item for payment_item in reward_logs if payment_item.payable
]
reward_logs.sort(key=cmp_to_key(cmp_by_type_balance))
batch_payer = BatchPayer(
node_url=node_endpoint,
pymnt_addr="tz1gtHbmBF3TSebsgJfJPvUB2e9x8EDeNm6V",
clnt_mngr=ClientManager(node_endpoint, PRIVATE_SIGNER_URL),
delegator_pays_ra_fee=True,
delegator_pays_xfer_fee=True,
network_config=network,
plugins_manager=PluginManager(baking_cfg.get_plugins_conf(),
dry_run=True),
dry_run=True,
)
# Fix the endpoint auto port assignment because
# https://mainnet-tezos.giganode.io:8732 cannot be reached
batch_payer.clnt_mngr.node_endpoint = node_endpoint
# Do the payment
(
payment_logs,
total_attempts,
total_payout_amount,
number_future_payable_cycles,
) = batch_payer.pay(reward_logs, dry_run=True)
assert total_attempts == 3
assert total_payout_amount == 238211030
assert (PAYMENT_ADDRESS_BALANCE //
total_payout_amount) - 1 == number_future_payable_cycles
def main(args):
logger.info("TRD version {} is running in {} mode.".format(
VERSION, "daemon" if args.background_service else "interactive"))
logger.info("Arguments Configuration = {}".format(
json.dumps(args.__dict__, indent=1)))
publish_stats = not args.do_not_publish_stats
logger.info(
"Anonymous statistics {} be collected. See docs/statistics.rst for more information."
.format("will" if publish_stats else "will not"))
# 1- find where configuration is
config_dir = os.path.expanduser(args.config_dir)
# create configuration directory if it is not present
# so that user can easily put his configuration there
if config_dir and not os.path.exists(config_dir):
os.makedirs(config_dir)
# 4. get network config
client_manager = ClientManager(node_endpoint=args.node_endpoint,
signer_endpoint=args.signer_endpoint)
network_config_map = init_network_config(args.network, client_manager)
network_config = network_config_map[args.network]
logger.debug("Network config {}".format(network_config))
# Setup provider to fetch RPCs
provider_factory = ProviderFactory(args.reward_data_provider)
# 5- load and verify baking configuration file
config_file_path = None
try:
config_file_path = get_baking_configuration_file(config_dir)
logger.info(
"Loading baking configuration file {}".format(config_file_path))
parser = BakingYamlConfParser(
yaml_text=ConfigParser.load_file(config_file_path),
clnt_mngr=client_manager,
provider_factory=provider_factory,
network_config=network_config,
node_url=args.node_endpoint,
api_base_url=args.api_base_url)
parser.parse()
parser.validate()
parser.process()
# dictionary to BakingConf object, for a bit of type safety
cfg_dict = parser.get_conf_obj()
cfg = BakingConf(cfg_dict)
except ConfigurationException as e:
logger.info(
"Unable to parse '{}' config file.".format(config_file_path))
logger.info(e)
sys.exit(1)
logger.info("Baking Configuration {}".format(cfg))
baking_address = cfg.get_baking_address()
payment_address = cfg.get_payment_address()
logger.info(LINER)
logger.info("BAKING ADDRESS is {}".format(baking_address))
logger.info("PAYMENT ADDRESS is {}".format(payment_address))
logger.info(LINER)
# 6- is it a reports run
dry_run = args.dry_run_no_consumers or args.dry_run
if args.dry_run_no_consumers:
global NB_CONSUMERS
NB_CONSUMERS = 0
# 7- get reporting directories
reports_base = os.path.expanduser(args.reports_base)
# if in reports run mode, do not create consumers
# create reports in reports directory
if dry_run:
reports_base = os.path.expanduser("./reports")
reports_dir = os.path.join(reports_base, baking_address)
payments_root = get_payment_root(reports_dir, create=True)
calculations_root = get_calculations_root(reports_dir, create=True)
get_successful_payments_dir(payments_root, create=True)
get_failed_payments_dir(payments_root, create=True)
# 8- start the life cycle
life_cycle.start(not dry_run)
# 9- service fee calculator
srvc_fee_calc = ServiceFeeCalculator(cfg.get_full_supporters_set(),
cfg.get_specials_map(),
cfg.get_service_fee())
if args.initial_cycle is None:
recent = get_latest_report_file(payments_root)
# if payment logs exists set initial cycle to following cycle
# if payment logs does not exists, set initial cycle to 0, so that payment starts from last released rewards
args.initial_cycle = 0 if recent is None else int(recent) + 1
logger.info("initial_cycle set to {}".format(args.initial_cycle))
# 10- load plugins
plugins_manager = plugins.PluginManager(cfg.get_plugins_conf(), dry_run)
# 11- Start producer and consumer
p = PaymentProducer(name='producer',
initial_payment_cycle=args.initial_cycle,
network_config=network_config,
payments_dir=payments_root,
calculations_dir=calculations_root,
run_mode=RunMode(args.run_mode),
service_fee_calc=srvc_fee_calc,
release_override=args.release_override,
payment_offset=args.payment_offset,
baking_cfg=cfg,
life_cycle=life_cycle,
payments_queue=payments_queue,
dry_run=dry_run,
client_manager=client_manager,
node_url=args.node_endpoint,
provider_factory=provider_factory,
node_url_public=args.node_addr_public,
api_base_url=args.api_base_url,
retry_injected=args.retry_injected)
p.start()
for i in range(NB_CONSUMERS):
c = PaymentConsumer(
name='consumer' + str(i),
payments_dir=payments_root,
key_name=payment_address,
payments_queue=payments_queue,
node_addr=args.node_endpoint,
client_manager=client_manager,
plugins_manager=plugins_manager,
rewards_type=cfg.get_rewards_type(),
args=args,
dry_run=dry_run,
reactivate_zeroed=cfg.get_reactivate_zeroed(),
delegator_pays_ra_fee=cfg.get_delegator_pays_ra_fee(),
delegator_pays_xfer_fee=cfg.get_delegator_pays_xfer_fee(),
dest_map=cfg.get_dest_map(),
network_config=network_config,
publish_stats=publish_stats)
sleep(1)
c.start()
logger.info("Application start completed")
logger.info(LINER)
# Run forever
try:
while life_cycle.is_running():
sleep(10)
except KeyboardInterrupt:
logger.info("Interrupted.")
life_cycle.stop()
def test_process_payouts(self):
logger.debug("") # Console formatting
factory = ProviderFactory(provider='prpc')
parser = BakingYamlConfParser(self.baking_config,
None,
None,
None,
None,
block_api=factory,
api_base_url=None)
parser.parse()
parser.process()
cfg_dict = parser.get_conf_obj()
baking_cfg = BakingConf(cfg_dict)
srvc_fee_calc = ServiceFeeCalculator(
baking_cfg.get_full_supporters_set(),
baking_cfg.get_specials_map(), baking_cfg.get_service_fee())
rules_model = RulesModel(baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map())
payment_calc = PhasedPaymentCalculator(
baking_cfg.get_founders_map(), baking_cfg.get_owners_map(),
srvc_fee_calc,
baking_cfg.get_min_delegation_amount() * MUTEZ, rules_model)
rewardApi = factory.newRewardApi(
default_network_config_map[CURRENT_TESTNET],
baking_cfg.get_baking_address(), "")
# Simulate logic in payment_producer
reward_logs = []
attempts = 0
exiting = False
while not exiting and attempts < 2:
attempts += 1
try:
# Reward data
# Fetch cycle 90 of delphinet for tz1gtHbmBF3TSebsgJfJPvUB2e9x8EDeNm6V
reward_model = rewardApi.get_rewards_for_cycle_map(
PAYOUT_CYCLE)
# Calculate rewards - payment_producer.py
reward_logs, total_amount = payment_calc.calculate(
reward_model)
# Check total reward amount matches sums of records
self.assertTrue(
total_amount,
sum([rl.amount for rl in reward_logs if rl.payable]))
exiting = True
except ApiProviderException as e:
logger.error(
"{:s} error at payment producer loop: '{:s}', will try again."
.format("RPC", str(e)))
sleep(5)
#
# The next 3 phases happen in payment_consumer.py
#
# Merge payments to same address
phaseMerge = CalculatePhaseMerge()
reward_logs = phaseMerge.calculate(reward_logs)
# Handle remapping of payment to alternate address
phaseMapping = CalculatePhaseMapping()
reward_logs = phaseMapping.calculate(reward_logs,
baking_cfg.get_dest_map())
# Filter zero-balance addresses based on config
phaseZeroBalance = CalculatePhaseZeroBalance()
reward_logs = phaseZeroBalance.calculate(
reward_logs, baking_cfg.get_reactivate_zeroed())
# Filter out non-payable items
reward_logs = [pi for pi in reward_logs if pi.payable]
reward_logs.sort(key=cmp_to_key(cmp_by_type_balance))
# TRD Calculated Results
# tz1V9SpwXaGFiYdDfGJtWjA61EumAH3DwSyT type: D, stake bal: 62657.83, cur bal: 62657.83, ratio: 0.327420, fee_ratio: 0.000000, amount: 0.000000, fee_amount: 0.000000, fee_rate: 0.00, payable: N, skipped: Y, at-phase: 1, desc: Excluded by configuration, pay_addr: tz1V9SpwXaGFiYdDfGJtWjA61EumAH3DwSyT
# tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace type: D, stake bal: 55646.70, cur bal: 55646.70, ratio: 0.432340, fee_ratio: 0.000000, amount: 102.988160, fee_amount: 0.000000, fee_rate: 0.00, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace
# tz1T5woJN3r7SV5v2HGDyA5kurhbD9Y8ZKHZ type: D, stake bal: 25689.88, cur bal: 25689.88, ratio: 0.179635, fee_ratio: 0.019959, amount: 42.791010, fee_amount: 4.754557, fee_rate: 0.10, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1T5woJN3r7SV5v2HGDyA5kurhbD9Y8ZKHZ
# tz1fgX6oRWQb4HYHUT6eRjW8diNFrqjEfgq7 type: D, stake bal: 24916.33, cur bal: 24916.33, ratio: 0.193584, fee_ratio: 0.000000, amount: 46.113902, fee_amount: 0.000000, fee_rate: 0.00, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1fgX6oRWQb4HYHUT6eRjW8diNFrqjEfgq7
# tz1RRzfechTs3gWdM58y6xLeByta3JWaPqwP type: D, stake bal: 6725.43, cur bal: 6725.43, ratio: 0.047027, fee_ratio: 0.005225, amount: 11.202382, fee_amount: 1.244709, fee_rate: 0.10, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1RMmSzPSWPSSaKU193Voh4PosWSZx1C7Hs
# tz1L1XQWKxG38wk1Ain1foGaEZj8zeposcbk type: D, stake bal: 981.64, cur bal: 981.64, ratio: 0.007627, fee_ratio: 0.000000, amount: 1.816762, fee_amount: 0.000000, fee_rate: 0.00, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1L1XQWKxG38wk1Ain1foGaEZj8zeposcbk
# tz1L1XQWKxG38wk1Ain1foGaEZj8zeposcbk type: O, stake bal: 14750.53, cur bal: 0.00, ratio: 0.114602, fee_ratio: 0.000000, amount: 27.299548, fee_amount: 0.000000, fee_rate: 0.00, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1L1XQWKxG38wk1Ain1foGaEZj8zeposcbk
# tz1fgX6oRWQb4HYHUT6eRjW8diNFrqjEfgq7 type: F, stake bal: 0.00, cur bal: 0.00, ratio: 0.006296, fee_ratio: 0.000000, amount: 1.499816, fee_amount: 0.000000, fee_rate: 0.00, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1fgX6oRWQb4HYHUT6eRjW8diNFrqjEfgq7
# tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace type: F, stake bal: 0.00, cur bal: 0.00, ratio: 0.018889, fee_ratio: 0.000000, amount: 4.499450, fee_amount: 0.000000, fee_rate: 0.00, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace
# Final records before creating transactions
# These values are known to be correct
cr = {}
cr["tz1T5woJN3r7SV5v2HGDyA5kurhbD9Y8ZKHZ"] = {
"type": "D",
"amount": 42791010,
"pay_addr": "tz1T5woJN3r7SV5v2HGDyA5kurhbD9Y8ZKHZ"
}
cr["tz1RRzfechTs3gWdM58y6xLeByta3JWaPqwP"] = {
"type": "D",
"amount": 11202382,
"pay_addr": "tz1RMmSzPSWPSSaKU193Voh4PosWSZx1C7Hs"
}
cr["tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace"] = {
"type": "M",
"amount": 107487610,
"pay_addr": "tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace"
}
cr["tz1fgX6oRWQb4HYHUT6eRjW8diNFrqjEfgq7"] = {
"type": "M",
"amount": 47613718,
"pay_addr": "tz1fgX6oRWQb4HYHUT6eRjW8diNFrqjEfgq7"
}
cr["tz1L1XQWKxG38wk1Ain1foGaEZj8zeposcbk"] = {
"type": "M",
"amount": 29116310,
"pay_addr": "tz1L1XQWKxG38wk1Ain1foGaEZj8zeposcbk"
}
# Verify that TRD calculated matches known values
for r in reward_logs:
# We know this address should be skipped
if r.address == "tz1V9SpwXaGFiYdDfGJtWjA61EumAH3DwSyT":
self.assertEqual(r.skipped, 1)
self.assertEqual(r.amount, 0)
continue
# All others we can compare normally
cmp = cr[r.address]
self.assertEqual(r.type, cmp["type"])
self.assertEqual(r.amount, (cmp["amount"]))
self.assertEqual(r.paymentaddress, cmp["pay_addr"])
class PaymentProducer(threading.Thread, PaymentProducerABC):
def __init__(
self,
name,
initial_payment_cycle,
network_config,
payments_dir,
calculations_dir,
run_mode,
service_fee_calc,
release_override,
payment_offset,
baking_cfg,
payments_queue,
life_cycle,
dry_run,
client_manager,
node_url,
reward_data_provider,
node_url_public="",
api_base_url=None,
retry_injected=False,
):
super(PaymentProducer, self).__init__()
self.event = threading.Event()
self.rules_model = RulesModel(
baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map(),
)
self.baking_address = baking_cfg.get_baking_address()
self.owners_map = baking_cfg.get_owners_map()
self.founders_map = baking_cfg.get_founders_map()
self.min_delegation_amt_in_mutez = int(
baking_cfg.get_min_delegation_amount() * MUTEZ_PER_TEZ)
self.delegator_pays_xfer_fee = baking_cfg.get_delegator_pays_xfer_fee()
self.provider_factory = ProviderFactory(reward_data_provider)
self.name = name
self.node_url = node_url
self.client_manager = client_manager
self.reward_api = self.provider_factory.newRewardApi(
network_config,
self.baking_address,
self.node_url,
node_url_public,
api_base_url,
)
self.block_api = self.provider_factory.newBlockApi(
network_config, self.node_url, api_base_url)
dexter_contracts_set = baking_cfg.get_contracts_set()
if len(dexter_contracts_set) > 0 and not (self.reward_api.name
== "tzstats"):
logger.warning(
"The Dexter functionality is currently only supported using tzstats."
"The contract address will be treated as a normal delegator.")
else:
self.reward_api.set_dexter_contracts_set(dexter_contracts_set)
self.rewards_type = baking_cfg.get_rewards_type()
self.pay_denunciation_rewards = baking_cfg.get_pay_denunciation_rewards(
)
self.fee_calc = service_fee_calc
self.initial_payment_cycle = initial_payment_cycle
logger.info("Initial cycle set to {}".format(
self.initial_payment_cycle))
self.nw_config = network_config
self.payments_root = payments_dir
self.calculations_dir = calculations_dir
self.run_mode = run_mode
self.exiting = False
self.release_override = release_override
self.payment_offset = payment_offset
self.payments_queue = payments_queue
self.life_cycle = life_cycle
self.dry_run = dry_run
self.payment_calc = PhasedPaymentCalculator(
self.founders_map,
self.owners_map,
self.fee_calc,
self.min_delegation_amt_in_mutez,
self.rules_model,
)
self.retry_fail_thread = threading.Thread(target=self.retry_fail_run,
name=self.name +
"_retry_fail")
self.retry_fail_event = threading.Event()
self.retry_injected = retry_injected
self.retry_producer = RetryProducer(
self.payments_queue,
self.reward_api,
self,
self.payments_root,
self.initial_payment_cycle,
self.retry_injected,
)
logger.debug('Producer "{}" started'.format(self.name))
def exit(self):
if not self.exiting:
self.payments_queue.put(
PaymentBatch(self, 0, [self.create_exit_payment()]))
self.exiting = True
if (self.life_cycle.is_running() and threading.current_thread()
is not threading.main_thread()):
_thread.interrupt_main()
logger.info("Sending KeyboardInterrupt signal.")
if self.retry_fail_event:
self.retry_fail_event.set()
def retry_fail_run(self):
logger.debug('Retry Fail thread "{}" started'.format(
self.retry_fail_thread.name))
sleep(60) # producer thread already tried once, wait for next try
while not self.exiting and self.life_cycle.is_running():
self.retry_producer.retry_failed_payments()
try:
# prepare to wait on event
self.retry_fail_event.clear()
# this will either return with timeout or set from parent producer thread
self.retry_fail_event.wait(60 * 60) # 1 hour
except RuntimeError:
pass
def run(self):
# call first retry if not in onetime mode.
# retry_failed script is more suitable for one time cases.
if not self.run_mode == RunMode.ONETIME:
self.retry_producer.retry_failed_payments()
if self.run_mode == RunMode.RETRY_FAILED:
sleep(5)
self.exit()
return
# first retry is done by producer thread, start retry thread for further retries
if self.run_mode == RunMode.FOREVER:
self.retry_fail_thread.start()
try:
(
current_cycle,
current_level,
) = self.block_api.get_current_cycle_and_level()
except ApiProviderException as a:
logger.error(
"Unable to fetch current cycle, {:s}. Exiting.".format(str(a)))
self.exit()
return
# if initial_payment_cycle has the default value of -1 resulting in the last released cycle
if self.initial_payment_cycle == -1:
pymnt_cycle = (current_cycle -
(self.nw_config["NB_FREEZE_CYCLE"] + 1) -
self.release_override)
if pymnt_cycle < 0:
logger.error(
"Payment cycle cannot be < 0 but configuration results to {}"
.format(pymnt_cycle))
else:
logger.debug(
"Payment cycle is set to last released cycle {}".format(
pymnt_cycle))
else:
pymnt_cycle = self.initial_payment_cycle
get_verbose_log_helper().reset(pymnt_cycle)
while not self.exiting and self.life_cycle.is_running():
# take a breath
sleep(5)
try:
# Exit if disk is full
# https://github.com/tezos-reward-distributor-organization/tezos-reward-distributor/issues/504
if disk_is_full():
self.exit()
break
# Check if local node is bootstrapped; sleep if needed; restart loop
if not self.node_is_bootstrapped():
logger.info(
"Local node {} is not in sync with the Tezos network. Will sleep for {} blocks and check again."
.format(self.node_url, BOOTSTRAP_SLEEP))
self.wait_for_blocks(BOOTSTRAP_SLEEP)
continue
# Local node is ready
(
current_cycle,
current_level,
) = self.block_api.get_current_cycle_and_level()
level_in_cycle = self.block_api.level_in_cycle(current_level)
# create reports dir
if self.calculations_dir and not os.path.exists(
self.calculations_dir):
os.makedirs(self.calculations_dir)
logger.debug(
"Checking for pending payments: payment_cycle <= current_cycle - (self.nw_config['NB_FREEZE_CYCLE'] + 1) - self.release_override"
)
logger.info(
"Checking for pending payments: checking {} <= {} - ({} + 1) - {}"
.format(
pymnt_cycle,
current_cycle,
self.nw_config["NB_FREEZE_CYCLE"],
self.release_override,
))
# payments should not pass beyond last released reward cycle
if (pymnt_cycle <= current_cycle -
(self.nw_config["NB_FREEZE_CYCLE"] + 1) -
self.release_override):
if not self.payments_queue.full():
if (not self.pay_denunciation_rewards
) and self.reward_api.name == "RPC":
logger.info(
"Error: pay_denunciation_rewards=False requires an indexer since it is not possible to distinguish reward source using RPC"
)
e = "You must set 'pay_denunciation_rewards' to True when using RPC provider."
logger.error(e)
self.exit()
break
# Paying upcoming cycles (-R set to -11 )
if pymnt_cycle >= current_cycle:
logger.warn(
"Please note that you are doing payouts for future rewards!!! These rewards are not earned yet, they are an estimation."
)
logger.warn(
"TRD will attempt to adjust the amount after the cycle runs, but it may not work."
)
# Paying cycles with frozen rewards (-R set to -5 )
elif (pymnt_cycle >= current_cycle -
self.nw_config["NB_FREEZE_CYCLE"]):
logger.warn(
"Please note that you are doing payouts for frozen rewards!!!"
)
# If user wants to offset payments within a cycle, check here
if level_in_cycle < self.payment_offset:
wait_offset_blocks = self.payment_offset - level_in_cycle
wait_offset_minutes = (
wait_offset_blocks *
self.nw_config["MINIMAL_BLOCK_DELAY"]) / 60
logger.info(
"Current level within the cycle is {}; Requested offset is {}; Waiting for {} more blocks (~{} minutes)"
.format(
level_in_cycle,
self.payment_offset,
wait_offset_blocks,
wait_offset_minutes,
))
self.wait_for_blocks(wait_offset_blocks)
continue # Break/Repeat loop
else:
result = self.try_to_pay(
pymnt_cycle,
self.rewards_type,
self.nw_config,
current_cycle,
)
if result:
# single run is done. Do not continue.
if self.run_mode == RunMode.ONETIME:
logger.info(
"Run mode ONETIME satisfied. Terminating..."
)
self.exit()
break
else:
pymnt_cycle = pymnt_cycle + 1
get_verbose_log_helper().reset(pymnt_cycle)
# end of queue size check
else:
logger.debug("Wait a few minutes, queue is full")
# wait a few minutes to let payments finish
sleep(60 * 3)
# end of payment cycle check
else:
logger.info(
"No pending payments for cycle {}, current cycle is {}"
.format(pymnt_cycle, current_cycle))
# pending payments done. Do not wait any more.
if self.run_mode == RunMode.PENDING:
logger.info(
"Run mode PENDING satisfied. Terminating...")
self.exit()
break
sleep(10)
# calculate number of blocks until end of current cycle plus user-defined offset
nb_blocks_remaining = (self.nw_config["BLOCKS_PER_CYCLE"] -
level_in_cycle +
self.payment_offset)
logger.debug(
"Waiting until next cycle; {} blocks remaining".format(
nb_blocks_remaining))
# wait until current cycle ends
self.wait_for_blocks(nb_blocks_remaining)
except (ApiProviderException, ReadTimeout, ConnectTimeout) as e:
logger.debug(
"{:s} error at payment producer loop: '{:s}'".format(
self.reward_api.name, str(e)),
exc_info=True,
)
logger.error(
"{:s} error at payment producer loop: '{:s}', will try again."
.format(self.reward_api.name, str(e)))
except Exception as e:
logger.debug(
"Unknown error in payment producer loop: {:s}".format(
str(e)),
exc_info=True,
)
logger.error(
"Unknown error in payment producer loop: {:s}, will try again."
.format(str(e)))
# end of endless loop
logger.debug("Producer returning...")
# ensure consumer exits
self.exit()
return
def stop(self):
self.exit()
self.event.set()
def compute_rewards(self,
pymnt_cycle,
computation_type,
network_config,
adjustments={}):
"""
Compute total rewards based on computation type and policy, then
calls payment_call.calculate to calculate rewards per delegator.
:param pymnt_cycle: the cycle for which rewards are being calculated
:param computation_type: calculate estimated, actual or ideal rewards
:param network_config: configuration of the current tezos network, needed to calc rewards.
:param adjustments: a map of adjustments per address. We add to amount to compute adjusted_amount
:return: rewards per delegator (reward logs) and total amount
"""
# does calculation report file already exist for this cycle?
logger.info("Computing rewards for cycle {:s}.".format(
str(pymnt_cycle)))
reward_model = self.reward_api.get_rewards_for_cycle_map(
pymnt_cycle, computation_type)
if computation_type.isEstimated():
logger.info("Using estimated rewards for payouts calculations")
block_reward = network_config["BLOCK_REWARD"]
endorsement_reward = network_config["ENDORSEMENT_REWARD"]
total_estimated_block_reward = reward_model.num_baking_rights * block_reward
total_estimated_endorsement_reward = (
reward_model.num_endorsing_rights * endorsement_reward)
reward_model.computed_reward_amount = (
total_estimated_block_reward +
total_estimated_endorsement_reward)
elif computation_type.isActual():
logger.info("Using actual rewards for payouts calculations")
if self.pay_denunciation_rewards:
reward_model.computed_reward_amount = reward_model.total_reward_amount
else:
# omit denunciation rewards
reward_model.computed_reward_amount = (
reward_model.rewards_and_fees -
reward_model.equivocation_losses)
elif computation_type.isIdeal():
logger.info("Using ideal rewards for payouts calculations")
if self.pay_denunciation_rewards:
reward_model.computed_reward_amount = (
reward_model.total_reward_amount +
reward_model.offline_losses)
else:
# omit denunciation rewards and double baking loss
reward_model.computed_reward_amount = (
reward_model.rewards_and_fees +
reward_model.offline_losses)
# 3- calculate rewards for delegators
return self.payment_calc.calculate(reward_model, adjustments)
def recompute_rewards(self, completed_cycle, computation_type,
network_config):
"""
In case of early payout, the payout is already done when the cycle runs.
After a cycle has run, we redo the computations. If we find overpayemnt
(overestimate) or underpayment (negative overestimate), we record it in the
calculation report csv file and return an adjustment map.
:param completed_cycle: the cycle for which rewards are being recalculated
:param computation_type: calculate estimated, actual or ideal rewards
:param network_config: configuration of the current tezos network, needed to calc rewards.
:return: adjustments map showing negative of overestimates per delegator
"""
logger.info(
"Checking for potential adjustment for recently completed cycle {:s}."
.format(str(completed_cycle)))
completed_cycle_report_file_path = get_calculation_report_file_path(
self.calculations_dir, completed_cycle)
adjustments = {}
if os.path.isfile(completed_cycle_report_file_path):
logger.info(
"TRD ran for cycle: {:s}, calculating adjustments.".format(
str(completed_cycle)))
(
reward_logs_from_report,
total_amount_from_report,
rewards_type_from_report,
_,
) = CsvCalculationFileParser().parse(
completed_cycle_report_file_path, self.baking_address)
# check that the overestimate has not been computed yet
if sum([rl.overestimate or 0
for rl in reward_logs_from_report]) > 0:
logger.info(
"Overestimate has already been calculated for cycle {:s}, not calculating it again."
.format(str(completed_cycle)))
completed_cycle_reward_logs, completed_cycle_total_amount = (
reward_logs_from_report,
total_amount_from_report,
)
else:
(
completed_cycle_reward_logs,
completed_cycle_total_amount,
) = self.compute_rewards(completed_cycle, computation_type,
network_config)
overestimate = int(total_amount_from_report -
completed_cycle_total_amount)
logger.info(
"We {:s}estimated payout for cycle {:s} by {:<,d} mutez, will attempt to adjust."
.format(
("over" if overestimate > 0 else "under"),
str(completed_cycle),
abs(overestimate),
))
for rl in reward_logs_from_report:
# overwrite only overestimate in report csv file, leave the rest alone
rl.overestimate = int(
Decimal(rl.ratio * overestimate).to_integral_value(
rounding=ROUND_HALF_DOWN))
# we adjust the cycle we are paying out with the overestimate of the
# just completed cycle
adjustments[rl.address] = rl.overestimate
logger.debug(
f"Will try to recover {adjustments[rl.address]} mutez for {rl.address} based on past overpayment"
)
CsvCalculationFileParser().write(
reward_logs_from_report,
completed_cycle_report_file_path,
total_amount_from_report,
rewards_type_from_report,
self.baking_address,
False,
)
return adjustments
def try_to_pay(self, pymnt_cycle, rewards_type, network_config,
current_cycle):
try:
logger.info("Payment cycle is {:s}".format(str(pymnt_cycle)))
# 0- check for past payment evidence for current cycle
past_payment_state = check_past_payment(self.payments_root,
pymnt_cycle)
if past_payment_state:
logger.warn(past_payment_state)
return True
adjustments = {}
early_payout = False
current_cycle_rewards_type = rewards_type
# 1- adjust past cycle if necessary
if self.release_override == -11 and pymnt_cycle >= current_cycle:
early_payout = True
completed_cycle = pymnt_cycle - 6
adjustments = self.recompute_rewards(completed_cycle,
rewards_type,
network_config)
# payout for current cycle will be estimated since we don't know actual rewards yet
current_cycle_rewards_type = RewardsType.ESTIMATED
# 2- get reward data and compute how to distribute them
reward_logs, total_amount = self.compute_rewards(
pymnt_cycle, current_cycle_rewards_type, network_config,
adjustments)
total_recovered_adjustments = int(
sum([rl.adjustment for rl in reward_logs]))
total_adjustments_to_recover = int(sum(adjustments.values()))
if total_adjustments_to_recover > 0:
logger.debug(
"Total adjustments to recover is {:<,d} mutez, total recovered adjustment is {:<,d} mutez."
.format(total_adjustments_to_recover,
total_recovered_adjustments))
logger.info(
"After early payout of cycle {:s}, {:<,d} mutez were not recovered."
.format(
str(completed_cycle),
total_adjustments_to_recover +
total_recovered_adjustments,
))
# 3- create calculations report file. This file contains calculations details
report_file_path = get_calculation_report_file_path(
self.calculations_dir, pymnt_cycle)
logger.debug("Creating calculation report (%s)", report_file_path)
CsvCalculationFileParser().write(
reward_logs,
report_file_path,
total_amount,
current_cycle_rewards_type,
self.baking_address,
early_payout,
)
# 4- set cycle info
for reward_log in reward_logs:
reward_log.cycle = pymnt_cycle
total_amount_to_pay = int(
sum([
reward_log.adjusted_amount for reward_log in reward_logs
if reward_log.payable
]))
# 5- if total_rewards > 0, proceed with payment
if total_amount_to_pay > 0:
self.payments_queue.put(
PaymentBatch(self, pymnt_cycle, reward_logs))
sleep(5.0)
# 6- processing of cycle is done
logger.info(
"Reward creation is done for cycle {}, created {} rewards."
.format(pymnt_cycle, len(reward_logs)))
elif total_amount_to_pay == 0:
logger.info("Total payment amount is 0. Nothing to pay!")
except ApiProviderException as a:
logger.error("[try_to_pay] API provider error {:s}".format(str(a)))
raise a from a
except Exception as e:
logger.error("[try_to_pay] Generic exception {:s}".format(str(e)))
raise e from e
# Either succeeded or raised exception
return True
def wait_for_blocks(self, nb_blocks_remaining):
for x in range(nb_blocks_remaining):
sleep(self.nw_config["MINIMAL_BLOCK_DELAY"])
# if shutting down, exit
if not self.life_cycle.is_running():
self.exit()
break
def node_is_bootstrapped(self):
# Get RPC node's (-A) bootstrap time. If bootstrap time + 2 minutes is
# before local time, node is not bootstrapped.
#
# clnt_mngr is a super class of SimpleClientManager which interfaces
# with the tezos-node used for txn forging/signing/injection. This is the
# node which we need to determine bootstrapped state
try:
boot_time = self.client_manager.get_bootstrapped()
utc_time = datetime.utcnow()
if (boot_time + timedelta(minutes=2)) < utc_time:
logger.debug(
"Current time is '{}', latest block of local node is '{}'."
.format(utc_time, boot_time))
return False
except ValueError:
logger.error(
"Unable to determine local node's bootstrap status. Continuing..."
)
return True
@staticmethod
def create_exit_payment():
return RewardLog.ExitInstance()
def notify_retry_fail_thread(self):
self.retry_fail_event.set()
# upon success retry failed payments if present
# success may indicate what went wrong in past is fixed.
def on_success(self, pymnt_batch):
self.notify_retry_fail_thread()
def on_fail(self, pymnt_batch):
pass
