def __init__(self, name, initial_payment_cycle, network_config, payments_dir, calculations_dir, run_mode,
service_fee_calc, release_override, payment_offset, baking_cfg, payments_queue, life_cycle,
dry_run, wllt_clnt_mngr, node_url, provider_factory, node_url_public='', verbose=False,
api_base_url=None):
super(PaymentProducer, self).__init__()
self.rules_model = RulesModel(baking_cfg.get_excluded_set_tob(), baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(), baking_cfg.get_dest_map())
self.baking_address = baking_cfg.get_baking_address()
self.owners_map = baking_cfg.get_owners_map()
self.founders_map = baking_cfg.get_founders_map()
self.min_delegation_amt_in_mutez = baking_cfg.get_min_delegation_amount() * MUTEZ
self.delegator_pays_xfer_fee = baking_cfg.get_delegator_pays_xfer_fee()
self.name = name
self.reward_api = provider_factory.newRewardApi(
network_config, self.baking_address, node_url, node_url_public, api_base_url)
self.block_api = provider_factory.newBlockApi(network_config, node_url, api_base_url)
self.fee_calc = service_fee_calc
self.initial_payment_cycle = initial_payment_cycle
self.nw_config = network_config
self.payments_root = payments_dir
self.calculations_dir = calculations_dir
self.run_mode = run_mode
self.exiting = False
self.release_override = release_override
self.payment_offset = payment_offset
self.verbose = verbose
self.payments_queue = payments_queue
self.life_cycle = life_cycle
self.dry_run = dry_run
self.payment_calc = PhasedPaymentCalculator(self.founders_map, self.owners_map, self.fee_calc,
self.min_delegation_amt_in_mutez, self.rules_model)
self.retry_fail_thread = threading.Thread(target=self.retry_fail_run, name=self.name + "_retry_fail")
self.retry_fail_event = threading.Event()
logger.info('Producer "{}" started'.format(self.name))
def test_process_payouts(self):
logger.debug("") # Console formatting
factory = ProviderFactory(provider='prpc')
parser = BakingYamlConfParser(self.baking_config,
None,
None,
None,
None,
block_api=factory,
api_base_url=None)
parser.parse()
parser.process()
cfg_dict = parser.get_conf_obj()
baking_cfg = BakingConf(cfg_dict)
srvc_fee_calc = ServiceFeeCalculator(
baking_cfg.get_full_supporters_set(),
baking_cfg.get_specials_map(), baking_cfg.get_service_fee())
rules_model = RulesModel(baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map())
payment_calc = PhasedPaymentCalculator(
baking_cfg.get_founders_map(), baking_cfg.get_owners_map(),
srvc_fee_calc,
baking_cfg.get_min_delegation_amount() * MUTEZ, rules_model)
rewardApi = factory.newRewardApi(
default_network_config_map[CURRENT_TESTNET],
baking_cfg.get_baking_address(), "")
# Simulate logic in payment_producer
reward_logs = []
attempts = 0
exiting = False
while not exiting and attempts < 2:
attempts += 1
try:
# Reward data
# Fetch cycle 90 of delphinet for tz1gtHbmBF3TSebsgJfJPvUB2e9x8EDeNm6V
reward_model = rewardApi.get_rewards_for_cycle_map(
PAYOUT_CYCLE)
# Calculate rewards - payment_producer.py
reward_logs, total_amount = payment_calc.calculate(
reward_model)
# Check total reward amount matches sums of records
self.assertTrue(
total_amount,
sum([rl.amount for rl in reward_logs if rl.payable]))
exiting = True
except ApiProviderException as e:
logger.error(
"{:s} error at payment producer loop: '{:s}', will try again."
.format("RPC", str(e)))
sleep(5)
#
# The next 3 phases happen in payment_consumer.py
#
# Merge payments to same address
phaseMerge = CalculatePhaseMerge()
reward_logs = phaseMerge.calculate(reward_logs)
# Handle remapping of payment to alternate address
phaseMapping = CalculatePhaseMapping()
reward_logs = phaseMapping.calculate(reward_logs,
baking_cfg.get_dest_map())
# Filter zero-balance addresses based on config
phaseZeroBalance = CalculatePhaseZeroBalance()
reward_logs = phaseZeroBalance.calculate(
reward_logs, baking_cfg.get_reactivate_zeroed())
# Filter out non-payable items
reward_logs = [pi for pi in reward_logs if pi.payable]
reward_logs.sort(key=cmp_to_key(cmp_by_type_balance))
# TRD Calculated Results
# tz1V9SpwXaGFiYdDfGJtWjA61EumAH3DwSyT type: D, stake bal: 62657.83, cur bal: 62657.83, ratio: 0.327420, fee_ratio: 0.000000, amount: 0.000000, fee_amount: 0.000000, fee_rate: 0.00, payable: N, skipped: Y, at-phase: 1, desc: Excluded by configuration, pay_addr: tz1V9SpwXaGFiYdDfGJtWjA61EumAH3DwSyT
# tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace type: D, stake bal: 55646.70, cur bal: 55646.70, ratio: 0.432340, fee_ratio: 0.000000, amount: 102.988160, fee_amount: 0.000000, fee_rate: 0.00, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace
# tz1T5woJN3r7SV5v2HGDyA5kurhbD9Y8ZKHZ type: D, stake bal: 25689.88, cur bal: 25689.88, ratio: 0.179635, fee_ratio: 0.019959, amount: 42.791010, fee_amount: 4.754557, fee_rate: 0.10, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1T5woJN3r7SV5v2HGDyA5kurhbD9Y8ZKHZ
# tz1fgX6oRWQb4HYHUT6eRjW8diNFrqjEfgq7 type: D, stake bal: 24916.33, cur bal: 24916.33, ratio: 0.193584, fee_ratio: 0.000000, amount: 46.113902, fee_amount: 0.000000, fee_rate: 0.00, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1fgX6oRWQb4HYHUT6eRjW8diNFrqjEfgq7
# tz1RRzfechTs3gWdM58y6xLeByta3JWaPqwP type: D, stake bal: 6725.43, cur bal: 6725.43, ratio: 0.047027, fee_ratio: 0.005225, amount: 11.202382, fee_amount: 1.244709, fee_rate: 0.10, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1RMmSzPSWPSSaKU193Voh4PosWSZx1C7Hs
# tz1L1XQWKxG38wk1Ain1foGaEZj8zeposcbk type: D, stake bal: 981.64, cur bal: 981.64, ratio: 0.007627, fee_ratio: 0.000000, amount: 1.816762, fee_amount: 0.000000, fee_rate: 0.00, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1L1XQWKxG38wk1Ain1foGaEZj8zeposcbk
# tz1L1XQWKxG38wk1Ain1foGaEZj8zeposcbk type: O, stake bal: 14750.53, cur bal: 0.00, ratio: 0.114602, fee_ratio: 0.000000, amount: 27.299548, fee_amount: 0.000000, fee_rate: 0.00, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1L1XQWKxG38wk1Ain1foGaEZj8zeposcbk
# tz1fgX6oRWQb4HYHUT6eRjW8diNFrqjEfgq7 type: F, stake bal: 0.00, cur bal: 0.00, ratio: 0.006296, fee_ratio: 0.000000, amount: 1.499816, fee_amount: 0.000000, fee_rate: 0.00, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1fgX6oRWQb4HYHUT6eRjW8diNFrqjEfgq7
# tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace type: F, stake bal: 0.00, cur bal: 0.00, ratio: 0.018889, fee_ratio: 0.000000, amount: 4.499450, fee_amount: 0.000000, fee_rate: 0.00, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace
# Final records before creating transactions
# These values are known to be correct
cr = {}
cr["tz1T5woJN3r7SV5v2HGDyA5kurhbD9Y8ZKHZ"] = {
"type": "D",
"amount": 42791010,
"pay_addr": "tz1T5woJN3r7SV5v2HGDyA5kurhbD9Y8ZKHZ"
}
cr["tz1RRzfechTs3gWdM58y6xLeByta3JWaPqwP"] = {
"type": "D",
"amount": 11202382,
"pay_addr": "tz1RMmSzPSWPSSaKU193Voh4PosWSZx1C7Hs"
}
cr["tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace"] = {
"type": "M",
"amount": 107487610,
"pay_addr": "tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace"
}
cr["tz1fgX6oRWQb4HYHUT6eRjW8diNFrqjEfgq7"] = {
"type": "M",
"amount": 47613718,
"pay_addr": "tz1fgX6oRWQb4HYHUT6eRjW8diNFrqjEfgq7"
}
cr["tz1L1XQWKxG38wk1Ain1foGaEZj8zeposcbk"] = {
"type": "M",
"amount": 29116310,
"pay_addr": "tz1L1XQWKxG38wk1Ain1foGaEZj8zeposcbk"
}
# Verify that TRD calculated matches known values
for r in reward_logs:
# We know this address should be skipped
if r.address == "tz1V9SpwXaGFiYdDfGJtWjA61EumAH3DwSyT":
self.assertEqual(r.skipped, 1)
self.assertEqual(r.amount, 0)
continue
# All others we can compare normally
cmp = cr[r.address]
self.assertEqual(r.type, cmp["type"])
self.assertEqual(r.amount, (cmp["amount"]))
self.assertEqual(r.paymentaddress, cmp["pay_addr"])
class PaymentProducer(threading.Thread, PaymentProducerABC):
def __init__(self,
name,
initial_payment_cycle,
network_config,
payments_dir,
calculations_dir,
run_mode,
service_fee_calc,
release_override,
payment_offset,
baking_cfg,
payments_queue,
life_cycle,
dry_run,
wllt_clnt_mngr,
node_url,
provider_factory,
node_url_public='',
verbose=False):
super(PaymentProducer, self).__init__()
self.rules_model = RulesModel(baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map())
self.baking_address = baking_cfg.get_baking_address()
self.owners_map = baking_cfg.get_owners_map()
self.founders_map = baking_cfg.get_founders_map()
self.min_delegation_amt_in_mutez = baking_cfg.get_min_delegation_amount(
) * MUTEZ
self.delegator_pays_xfer_fee = baking_cfg.get_delegator_pays_xfer_fee()
self.name = name
self.reward_api = provider_factory.newRewardApi(
network_config, self.baking_address, node_url, node_url_public)
self.block_api = provider_factory.newBlockApi(network_config, node_url)
self.fee_calc = service_fee_calc
self.initial_payment_cycle = initial_payment_cycle
self.nw_config = network_config
self.payments_root = payments_dir
self.calculations_dir = calculations_dir
self.run_mode = run_mode
self.exiting = False
self.release_override = release_override
self.payment_offset = payment_offset
self.verbose = verbose
self.payments_queue = payments_queue
self.life_cycle = life_cycle
self.dry_run = dry_run
self.payment_calc = PhasedPaymentCalculator(
self.founders_map, self.owners_map, self.fee_calc,
self.min_delegation_amt_in_mutez, self.rules_model)
self.retry_fail_thread = threading.Thread(target=self.retry_fail_run,
name=self.name +
"_retry_fail")
self.retry_fail_event = threading.Event()
logger.info('Producer "{}" started'.format(self.name))
def exit(self):
if not self.exiting:
self.payments_queue.put(
PaymentBatch(self, 0, [self.create_exit_payment()]))
self.exiting = True
if self.life_cycle.is_running() and threading.current_thread(
) is not threading.main_thread():
_thread.interrupt_main()
if self.retry_fail_event:
self.retry_fail_event.set()
def retry_fail_run(self):
logger.info('Retry Fail thread "{}" started'.format(
self.retry_fail_thread.name))
sleep(60) # producer thread already tried once, wait for next try
while not self.exiting and self.life_cycle.is_running():
self.retry_failed_payments()
try:
# prepare to wait on event
self.retry_fail_event.clear()
# this will either return with timeout or set from parent producer thread
self.retry_fail_event.wait(60 * 60) # 1 hour
except RuntimeError:
pass
pass
def run(self):
# call first retry if not in onetime mode.
# retry_failed script is more suitable for one time cases.
if not self.run_mode == RunMode.ONETIME:
self.retry_failed_payments()
# first retry is done by producer thread, start retry thread for further retries
if self.run_mode == RunMode.FOREVER:
self.retry_fail_thread.start()
crrnt_cycle = self.block_api.get_current_cycle()
pymnt_cycle = self.initial_payment_cycle
# if non-positive initial_payment_cycle, set initial_payment_cycle to
# 'current cycle - abs(initial_cycle) - (NB_FREEZE_CYCLE+1)'
if self.initial_payment_cycle <= 0:
pymnt_cycle = crrnt_cycle - abs(self.initial_payment_cycle) - (
self.nw_config['NB_FREEZE_CYCLE'] + 1)
logger.debug("Payment cycle is set to {}".format(pymnt_cycle))
while not self.exiting and self.life_cycle.is_running():
# take a breath
time.sleep(5)
try:
current_level = self.block_api.get_current_level(
verbose=self.verbose)
crrnt_cycle = self.block_api.level_to_cycle(current_level)
# create reports dir
if self.calculations_dir and not os.path.exists(
self.calculations_dir):
os.makedirs(self.calculations_dir)
logger.debug(
"Checking for pending payments : payment_cycle <= "
"current_cycle - (self.nw_config['NB_FREEZE_CYCLE'] + 1) - self.release_override"
)
logger.info(
"Checking for pending payments : checking {} <= {} - ({} + 1) - {}"
.format(pymnt_cycle, crrnt_cycle,
self.nw_config['NB_FREEZE_CYCLE'],
self.release_override))
# payments should not pass beyond last released reward cycle
if pymnt_cycle <= crrnt_cycle - (
self.nw_config['NB_FREEZE_CYCLE'] +
1) - self.release_override:
if not self.payments_queue.full():
# Paying upcoming cycles (-R in [-6, -11] )
if pymnt_cycle >= crrnt_cycle:
if self.reward_api.name == 'tzstats':
logger.warn(
"Please note that you are doing payouts for future rewards!!! These rewards are not earned yet, they are an estimation given by tzstats."
)
result = self.try_to_pay(pymnt_cycle,
expected_reward=True)
else:
logger.error(
"This feature is only possible using tzstats. Please consider changing the provider using the -P flag."
)
self.exit()
break
# Paying cycles with frozen rewards (-R in [-1, -5] )
elif pymnt_cycle >= crrnt_cycle - self.nw_config[
'NB_FREEZE_CYCLE']:
if self.reward_api.name == 'tzstats':
logger.warn(
"Please note that you are doing payouts for frozen rewards!!!"
)
result = self.try_to_pay(pymnt_cycle)
else:
logger.error(
"This feature is only possible using tzstats. Please consider changing the provider using the -P flag or wait until the rewards are unfrozen."
)
self.exit()
break
else:
result = self.try_to_pay(pymnt_cycle)
if result:
# single run is done. Do not continue.
if self.run_mode == RunMode.ONETIME:
logger.info(
"Run mode ONETIME satisfied. Terminating ..."
)
self.exit()
break
else:
pymnt_cycle = pymnt_cycle + 1
# end of queue size check
else:
logger.debug("Wait a few minutes, queue is full")
# wait a few minutes to let payments done
time.sleep(60 * 3)
# end of payment cycle check
else:
logger.info(
"No pending payments for cycle {}, current cycle is {}"
.format(pymnt_cycle, crrnt_cycle))
# pending payments done. Do not wait any more.
if self.run_mode == RunMode.PENDING:
logger.info(
"Run mode PENDING satisfied. Terminating ...")
self.exit()
break
time.sleep(10)
# calculate number of blocks until end of current cycle
nb_blocks_remaining = (
crrnt_cycle +
1) * self.nw_config['BLOCKS_PER_CYCLE'] - current_level
# plus offset. cycle beginnings may be busy, move payments forward
nb_blocks_remaining = nb_blocks_remaining + self.payment_offset
logger.debug("Wait until next cycle, for {} blocks".format(
nb_blocks_remaining))
# wait until current cycle ends
self.wait_until_next_cycle(nb_blocks_remaining)
except ApiProviderException:
logger.debug("{} API error at reward loop".format(
self.reward_api.name),
exc_info=True)
logger.info(
"{} API error at reward loop, will try again.".format(
self.reward_api.name))
except Exception:
logger.error("Error at payment producer loop", exc_info=True)
# end of endless loop
logger.info("Producer returning ...")
# ensure consumer exits
self.exit()
return
def try_to_pay(self, pymnt_cycle, expected_reward=False):
try:
logger.info("Payment cycle is " + str(pymnt_cycle))
# 0- check for past payment evidence for current cycle
past_payment_state = check_past_payment(self.payments_root,
pymnt_cycle)
if not self.dry_run and past_payment_state:
logger.warn(past_payment_state)
return True
# 1- get reward data
if expected_reward:
reward_model = self.reward_api.get_rewards_for_cycle_map(
pymnt_cycle, expected_reward)
else:
reward_model = self.reward_api.get_rewards_for_cycle_map(
pymnt_cycle)
# 2- calculate rewards
reward_logs, total_amount = self.payment_calc.calculate(
reward_model)
# set cycle info
for rl in reward_logs:
rl.cycle = pymnt_cycle
total_amount_to_pay = sum(
[rl.amount for rl in reward_logs if rl.payable])
# 4- if total_rewards > 0, proceed with payment
if total_amount_to_pay > 0:
report_file_path = get_calculation_report_file(
self.calculations_dir, pymnt_cycle)
# 5- send to payment consumer
self.payments_queue.put(
PaymentBatch(self, pymnt_cycle, reward_logs))
# logger.info("Total payment amount is {:,} mutez. %s".format(total_amount_to_pay),
# "" if self.delegator_pays_xfer_fee else "(Transfer fee is not included)")
logger.debug("Creating calculation report (%s)",
report_file_path)
sleep(5.0)
# 6- create calculations report file. This file contains calculations details
self.create_calculations_report(reward_logs, report_file_path,
total_amount)
# 7- processing of cycle is done
logger.info(
"Reward creation is done for cycle {}, created {} rewards."
.format(pymnt_cycle, len(reward_logs)))
elif total_amount_to_pay == 0:
logger.info("Total payment amount is 0. Nothing to pay!")
return True
except ReadTimeout:
logger.info("API provider call failed, will try again.")
logger.debug("API provider call failed", exc_info=False)
return False
except ConnectTimeout:
logger.info("API provider connection failed, will try again.")
logger.debug("API provider connection failed", exc_info=False)
return False
except ApiProviderException:
logger.info("API provider error at reward loop, will try again.")
logger.debug("API provider error at reward loop", exc_info=False)
return False
except Exception:
logger.error("Error at payment producer loop, will try again.",
exc_info=True)
return False
finally:
sleep(10)
def wait_until_next_cycle(self, nb_blocks_remaining):
for x in range(nb_blocks_remaining):
time.sleep(self.nw_config['BLOCK_TIME_IN_SEC'])
# if shutting down, exit
if not self.life_cycle.is_running():
self.exit()
break
def create_calculations_report(self, payment_logs, report_file_path,
total_rewards):
with open(report_file_path, 'w', newline='') as f:
writer = csv.writer(f,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
# write headers and total rewards
writer.writerow([
"address", "type", "staked_balance", "current_balance",
"ratio", "fee_ratio", "amount", "fee_amount", "fee_rate",
"payable", "skipped", "atphase", "desc", "payment_address"
])
writer.writerow([
self.baking_address, "B",
sum([pl.staking_balance for pl in payment_logs]),
"{0:f}".format(1.0), "{0:f}".format(1.0), "{0:f}".format(0.0),
"{0:f}".format(total_rewards), "{0:f}".format(0.0),
"{0:f}".format(0.0), "0", "0", "-1", "Baker", "None"
])
for pymnt_log in payment_logs:
# write row to csv file
array = [
pymnt_log.address, pymnt_log.type,
pymnt_log.staking_balance, pymnt_log.current_balance,
"{0:.10f}".format(pymnt_log.ratio),
"{0:.10f}".format(pymnt_log.service_fee_ratio),
"{0:f}".format(pymnt_log.amount),
"{0:f}".format(pymnt_log.service_fee_amount),
"{0:f}".format(pymnt_log.service_fee_rate),
"1" if pymnt_log.payable else "0",
"1" if pymnt_log.skipped else "0",
pymnt_log.skippedatphase if pymnt_log.skipped else "-1",
pymnt_log.desc if pymnt_log.desc else "None",
pymnt_log.paymentaddress
]
writer.writerow(array)
logger.debug(
"Reward created for %s type: %s, stake bal: {:>10.2f}, cur bal: {:>10.2f}, ratio: {:.6f}, fee_ratio: {:.6f}, "
"amount: {:>10.6f}, fee_amount: {:>4.6f}, fee_rate: {:.2f}, payable: %s, skipped: %s, at-phase: %s, "
"desc: %s, pay_addr: %s".format(
pymnt_log.staking_balance / MUTEZ,
pymnt_log.current_balance / MUTEZ, pymnt_log.ratio,
pymnt_log.service_fee_ratio, pymnt_log.amount / MUTEZ,
pymnt_log.service_fee_amount / MUTEZ,
pymnt_log.service_fee_rate), pymnt_log.address,
pymnt_log.type, pymnt_log.payable, pymnt_log.skipped,
pymnt_log.skippedatphase, pymnt_log.desc,
pymnt_log.paymentaddress)
logger.info(
"Calculation report is created at '{}'".format(report_file_path))
@staticmethod
def create_exit_payment():
return RewardLog.ExitInstance()
def retry_failed_payments(self, retry_injected=False):
logger.debug("retry_failed_payments started")
# 1 - list csv files under payments/failed directory
# absolute path of csv files found under payments_root/failed directory
failed_payments_dir = get_failed_payments_dir(self.payments_root)
payment_reports_failed = [
os.path.join(failed_payments_dir, x)
for x in os.listdir(failed_payments_dir) if x.endswith('.csv')
]
if payment_reports_failed:
payment_reports_failed = sorted(
payment_reports_failed,
key=lambda x: int(os.path.splitext(os.path.basename(x))[0]))
logger.debug("Failed payment files found are: '{}'".format(
",".join(payment_reports_failed)))
else:
logger.debug(
"No failed payment files found under directory '{}'".format(
failed_payments_dir))
# 2- for each csv file with name csv_report.csv
for payment_failed_report_file in payment_reports_failed:
logger.info("Working on failed payment file {}".format(
payment_failed_report_file))
# 2.1 - if there is a file csv_report.csv under payments/done, it means payment is already done
if os.path.isfile(
payment_failed_report_file.replace(PAYMENT_FAILED_DIR,
PAYMENT_DONE_DIR)):
# remove payments/failed/csv_report.csv
os.remove(payment_failed_report_file)
logger.info(
"Payment for failed payment {} is already done. Removing.".
format(payment_failed_report_file))
# remove payments/failed/csv_report.csv.BUSY
# if there is a busy failed payment report file, remove it.
remove_busy_file(payment_failed_report_file)
# do not double pay
continue
# 2.2 - if queue is full, wait for sometime
# make sure the queue is not full
while self.payments_queue.full():
logger.debug("Payments queue is full. Wait a few minutes.")
time.sleep(60 * 3)
cycle = int(
os.path.splitext(
os.path.basename(payment_failed_report_file))[0])
# 2.3 read payments/failed/csv_report.csv file into a list of dictionaries
batch = CsvPaymentFileParser().parse(payment_failed_report_file,
cycle)
nb_paid = len(
list(filter(lambda f: f.paid == PaymentStatus.PAID, batch)))
nb_done = len(
list(filter(lambda f: f.paid == PaymentStatus.DONE, batch)))
nb_injected = len(
list(filter(lambda f: f.paid == PaymentStatus.INJECTED,
batch)))
nb_failed = len(
list(filter(lambda f: f.paid == PaymentStatus.FAIL, batch)))
logger.info(
"Summary {} paid, {} done, {} injected, {} fail".format(
nb_paid, nb_done, nb_injected, nb_failed))
if retry_injected:
nb_converted = 0
for pl in batch:
if pl.paid == PaymentStatus.INJECTED:
pl.paid = PaymentStatus.FAIL
nb_converted += 1
logger.debug(
"Reward converted from %s to fail for cycle %s, address %s, amount %f, tz type %s",
pl.paid, pl.cycle, pl.address, pl.amount, pl.type)
if nb_converted:
logger.info(
"{} rewards converted from injected to fail.".format(
nb_converted))
# 2.4 - Filter batch to only include those which failed. No need to mess with PAID/DONE
batch = list(filter(lambda f: f.paid == PaymentStatus.FAIL, batch))
# 2.5 - Need to fetch current balance for addresses of any failed payments
self.reward_api.update_current_balances(batch)
# 2.6 - put records into payment_queue. payment_consumer will make payments
self.payments_queue.put(PaymentBatch(self, cycle, batch))
# 2.7 - rename payments/failed/csv_report.csv to payments/failed/csv_report.csv.BUSY
# mark the files as in use. we do not want it to be read again
# BUSY file will be removed, if successful payment is done
os.rename(payment_failed_report_file,
payment_failed_report_file + BUSY_FILE)
def notify_retry_fail_thread(self):
self.retry_fail_event.set()
# upon success retry failed payments if present
# success may indicate what went wrong in past is fixed.
def on_success(self, pymnt_batch):
self.notify_retry_fail_thread()
def on_fail(self, pymnt_batch):
pass
class PaymentProducer(threading.Thread, PaymentProducerABC):
def __init__(self,
name,
initial_payment_cycle,
network_config,
payments_dir,
calculations_dir,
run_mode,
service_fee_calc,
release_override,
payment_offset,
baking_cfg,
payments_queue,
life_cycle,
dry_run,
client_manager,
node_url,
reward_data_provider,
node_url_public='',
api_base_url=None,
retry_injected=False):
super(PaymentProducer, self).__init__()
self.rules_model = RulesModel(baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map())
self.baking_address = baking_cfg.get_baking_address()
self.owners_map = baking_cfg.get_owners_map()
self.founders_map = baking_cfg.get_founders_map()
self.min_delegation_amt_in_mutez = baking_cfg.get_min_delegation_amount(
) * MUTEZ
self.delegator_pays_xfer_fee = baking_cfg.get_delegator_pays_xfer_fee()
self.provider_factory = ProviderFactory(reward_data_provider)
self.name = name
self.node_url = node_url
self.client_manager = client_manager
self.reward_api = self.provider_factory.newRewardApi(
network_config, self.baking_address, self.node_url,
node_url_public, api_base_url)
self.block_api = self.provider_factory.newBlockApi(
network_config, self.node_url, api_base_url)
dexter_contracts_set = baking_cfg.get_contracts_set()
if len(dexter_contracts_set) > 0 and not (self.reward_api.name
== 'tzstats'):
logger.warning(
"The Dexter functionality is currently only supported using tzstats."
"The contract address will be treated as a normal delegator.")
else:
self.reward_api.set_dexter_contracts_set(dexter_contracts_set)
self.rewards_type = baking_cfg.get_rewards_type()
self.fee_calc = service_fee_calc
self.initial_payment_cycle = initial_payment_cycle
if self.initial_payment_cycle is None:
recent = get_latest_report_file(payments_dir)
# if payment logs exists set initial cycle to following cycle
# if payment logs does not exists, set initial cycle to 0, so that payment starts from last released rewards
self.initial_payment_cycle = 0 if recent is None else int(
recent) + 1
logger.info("initial_cycle set to {}".format(
self.initial_payment_cycle))
self.nw_config = network_config
self.payments_root = payments_dir
self.calculations_dir = calculations_dir
self.run_mode = run_mode
self.exiting = False
self.release_override = release_override
self.payment_offset = payment_offset
self.payments_queue = payments_queue
self.life_cycle = life_cycle
self.dry_run = dry_run
self.payment_calc = PhasedPaymentCalculator(
self.founders_map, self.owners_map, self.fee_calc,
self.min_delegation_amt_in_mutez, self.rules_model)
self.retry_fail_thread = threading.Thread(target=self.retry_fail_run,
name=self.name +
"_retry_fail")
self.retry_fail_event = threading.Event()
self.retry_injected = retry_injected
self.retry_producer = RetryProducer(self.payments_queue,
self.reward_api, self,
self.payments_root,
self.retry_injected)
logger.info('Producer "{}" started'.format(self.name))
def exit(self):
if not self.exiting:
self.payments_queue.put(
PaymentBatch(self, 0, [self.create_exit_payment()]))
self.exiting = True
if self.life_cycle.is_running() and threading.current_thread(
) is not threading.main_thread():
_thread.interrupt_main()
if self.retry_fail_event:
self.retry_fail_event.set()
def retry_fail_run(self):
logger.info('Retry Fail thread "{}" started'.format(
self.retry_fail_thread.name))
sleep(60) # producer thread already tried once, wait for next try
while not self.exiting and self.life_cycle.is_running():
self.retry_producer.retry_failed_payments()
try:
# prepare to wait on event
self.retry_fail_event.clear()
# this will either return with timeout or set from parent producer thread
self.retry_fail_event.wait(60 * 60) # 1 hour
except RuntimeError:
pass
def run(self):
# call first retry if not in onetime mode.
# retry_failed script is more suitable for one time cases.
if not self.run_mode == RunMode.ONETIME:
self.retry_producer.retry_failed_payments()
if self.run_mode == RunMode.RETRY_FAILED:
sleep(5)
self.exit()
return
# first retry is done by producer thread, start retry thread for further retries
if self.run_mode == RunMode.FOREVER:
self.retry_fail_thread.start()
try:
current_cycle = self.block_api.get_current_cycle()
pymnt_cycle = self.initial_payment_cycle
except ApiProviderException as a:
logger.error(
"Unable to fetch current cycle, {:s}. Exiting.".format(str(a)))
self.exit()
return
# if non-positive initial_payment_cycle, set initial_payment_cycle to
# 'current cycle - abs(initial_cycle) - (NB_FREEZE_CYCLE+1)'
if self.initial_payment_cycle <= 0:
pymnt_cycle = current_cycle - abs(self.initial_payment_cycle) - (
self.nw_config['NB_FREEZE_CYCLE'] + 1)
logger.debug("Payment cycle is set to {}".format(pymnt_cycle))
get_verbose_log_helper().reset(pymnt_cycle)
while not self.exiting and self.life_cycle.is_running():
# take a breath
sleep(5)
try:
# Check if local node is bootstrapped; sleep if needed; restart loop
if not self.node_is_bootstrapped():
logger.info(
"Local node, {}, is not in sync with the Tezos network. Will sleep for {} blocks and check again."
.format(self.node_url, BOOTSTRAP_SLEEP))
self.wait_for_blocks(BOOTSTRAP_SLEEP)
continue
# Local node is ready
current_level = self.block_api.get_current_level()
current_cycle = self.block_api.level_to_cycle(current_level)
level_in_cycle = self.block_api.level_in_cycle(current_level)
# create reports dir
if self.calculations_dir and not os.path.exists(
self.calculations_dir):
os.makedirs(self.calculations_dir)
logger.debug(
"Checking for pending payments : payment_cycle <= current_cycle - (self.nw_config['NB_FREEZE_CYCLE'] + 1) - self.release_override"
)
logger.info(
"Checking for pending payments : checking {} <= {} - ({} + 1) - {}"
.format(pymnt_cycle, current_cycle,
self.nw_config['NB_FREEZE_CYCLE'],
self.release_override))
# payments should not pass beyond last released reward cycle
if pymnt_cycle <= current_cycle - (
self.nw_config['NB_FREEZE_CYCLE'] +
1) - self.release_override:
if not self.payments_queue.full():
# Paying upcoming cycles (-R in [-6, -11] )
if pymnt_cycle >= current_cycle:
logger.warn(
"Please note that you are doing payouts for future rewards!!! These rewards are not earned yet, they are an estimation."
)
if not self.rewards_type.isIdeal():
logger.error(
"For future rewards payout, you must configure the payout type to 'Ideal', see documentation"
)
self.exit()
break
# Paying cycles with frozen rewards (-R in [-1, -5] )
elif pymnt_cycle >= current_cycle - self.nw_config[
'NB_FREEZE_CYCLE']:
logger.warn(
"Please note that you are doing payouts for frozen rewards!!!"
)
if (not self.rewards_type.isIdeal()
) and self.reward_api.name == 'RPC':
logger.error(
"Paying out frozen rewards with Node RPC API and rewards type 'Actual' is unsupported, you must use TzKT or tzstats API"
)
self.exit()
break
# If user wants to offset payments within a cycle, check here
if level_in_cycle < self.payment_offset:
wait_offset_blocks = self.payment_offset - level_in_cycle
logger.info(
"Current level within the cycle is {}; Requested offset is {}; Waiting for {} more blocks."
.format(level_in_cycle, self.payment_offset,
wait_offset_blocks))
self.wait_for_blocks(wait_offset_blocks)
continue # Break/Repeat loop
else:
result = self.try_to_pay(
pymnt_cycle,
expected_rewards=self.rewards_type.isIdeal())
if result:
# single run is done. Do not continue.
if self.run_mode == RunMode.ONETIME:
logger.info(
"Run mode ONETIME satisfied. Terminating ..."
)
self.exit()
break
else:
pymnt_cycle = pymnt_cycle + 1
get_verbose_log_helper().reset(pymnt_cycle)
# end of queue size check
else:
logger.debug("Wait a few minutes, queue is full")
# wait a few minutes to let payments finish
sleep(60 * 3)
# end of payment cycle check
else:
logger.info(
"No pending payments for cycle {}, current cycle is {}"
.format(pymnt_cycle, current_cycle))
# pending payments done. Do not wait any more.
if self.run_mode == RunMode.PENDING:
logger.info(
"Run mode PENDING satisfied. Terminating ...")
self.exit()
break
sleep(10)
# calculate number of blocks until end of current cycle
nb_blocks_remaining = (
current_cycle +
1) * self.nw_config['BLOCKS_PER_CYCLE'] - current_level
# plus offset. cycle beginnings may be busy, move payments forward
nb_blocks_remaining = nb_blocks_remaining + self.payment_offset
logger.debug(
"Waiting until next cycle; {} blocks remaining".format(
nb_blocks_remaining))
# wait until current cycle ends
self.wait_for_blocks(nb_blocks_remaining)
except (ApiProviderException, ReadTimeout, ConnectTimeout) as e:
logger.debug(
"{:s} error at payment producer loop: '{:s}'".format(
self.reward_api.name, str(e)),
exc_info=True)
logger.error(
"{:s} error at payment producer loop: '{:s}', will try again."
.format(self.reward_api.name, str(e)))
except Exception as e:
logger.debug(
"Unknown error in payment producer loop: {:s}".format(
str(e)),
exc_info=True)
logger.error(
"Unknown error in payment producer loop: {:s}, will try again."
.format(str(e)))
# end of endless loop
logger.info("Producer returning ...")
# ensure consumer exits
self.exit()
return
def try_to_pay(self, pymnt_cycle, expected_rewards=False):
try:
logger.info("Payment cycle is " + str(pymnt_cycle))
# 0- check for past payment evidence for current cycle
past_payment_state = check_past_payment(self.payments_root,
pymnt_cycle)
if not self.dry_run and past_payment_state:
logger.warn(past_payment_state)
return True
# 1- get reward data
if expected_rewards:
logger.info(
"Using expected/ideal rewards for payouts calculations")
else:
logger.info("Using actual rewards for payouts calculations")
reward_model = self.reward_api.get_rewards_for_cycle_map(
pymnt_cycle, expected_rewards)
# 2- calculate rewards
reward_logs, total_amount = self.payment_calc.calculate(
reward_model)
# 3- set cycle info
for rl in reward_logs:
rl.cycle = pymnt_cycle
total_amount_to_pay = sum(
[rl.amount for rl in reward_logs if rl.payable])
# 4- if total_rewards > 0, proceed with payment
if total_amount_to_pay > 0:
report_file_path = get_calculation_report_file(
self.calculations_dir, pymnt_cycle)
# 5- send to payment consumer
self.payments_queue.put(
PaymentBatch(self, pymnt_cycle, reward_logs))
# logger.info("Total payment amount is {:,} mutez. %s".format(total_amount_to_pay),
# "" if self.delegator_pays_xfer_fee else "(Transfer fee is not included)")
logger.debug("Creating calculation report (%s)",
report_file_path)
sleep(5.0)
# 6- create calculations report file. This file contains calculations details
self.create_calculations_report(reward_logs, report_file_path,
total_amount, expected_rewards)
# 7- processing of cycle is done
logger.info(
"Reward creation is done for cycle {}, created {} rewards."
.format(pymnt_cycle, len(reward_logs)))
elif total_amount_to_pay == 0:
logger.info("Total payment amount is 0. Nothing to pay!")
except ApiProviderException as a:
logger.error("[try_to_pay] API provider error {:s}".format(str(a)))
raise a from a
except Exception as e:
logger.error("[try_to_pay] Generic exception {:s}".format(str(e)))
raise e from e
# Either succeeded or raised exception
return True
def wait_for_blocks(self, nb_blocks_remaining):
for x in range(nb_blocks_remaining):
sleep(self.nw_config['BLOCK_TIME_IN_SEC'])
# if shutting down, exit
if not self.life_cycle.is_running():
self.exit()
break
def node_is_bootstrapped(self):
# Get RPC node's (-A) bootstrap time. If bootstrap time + 2 minutes is
# before local time, node is not bootstrapped.
#
# clnt_mngr is a super class of SimpleClientManager which interfaces
# with the tezos-node used for txn forging/signing/injection. This is the
# node which we need to determine bootstrapped state
try:
boot_time = self.client_manager.get_bootstrapped()
utc_time = datetime.utcnow()
if (boot_time + timedelta(minutes=2)) < utc_time:
logger.info(
"Current time is '{}', latest block of local node is '{}'."
.format(utc_time, boot_time))
return False
except ValueError:
logger.error(
"Unable to determine local node's bootstrap status. Continuing..."
)
return True
def create_calculations_report(self, payment_logs, report_file_path,
total_rewards, expected_rewards):
rt = "I" if expected_rewards else "A"
# Open reports file and write; auto-closes file
with open(report_file_path, 'w', newline='') as f:
writer = csv.writer(f,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
# write headers and total rewards
writer.writerow([
"address", "type", "staked_balance", "current_balance",
"ratio", "fee_ratio", "amount", "fee_amount", "fee_rate",
"payable", "skipped", "atphase", "desc", "payment_address",
"rewards_type"
])
# First row is for the baker
writer.writerow([
self.baking_address, "B",
sum([pl.staking_balance for pl in payment_logs]),
"{0:f}".format(1.0), "{0:f}".format(1.0), "{0:f}".format(0.0),
"{0:f}".format(total_rewards), "{0:f}".format(0.0),
"{0:f}".format(0.0), "0", "0", "-1", "Baker", "None", rt
])
for pymnt_log in payment_logs:
# write row to csv file
array = [
pymnt_log.address, pymnt_log.type,
pymnt_log.staking_balance, pymnt_log.current_balance,
"{0:.10f}".format(pymnt_log.ratio),
"{0:.10f}".format(pymnt_log.service_fee_ratio),
"{0:f}".format(pymnt_log.amount),
"{0:f}".format(pymnt_log.service_fee_amount),
"{0:f}".format(pymnt_log.service_fee_rate),
"1" if pymnt_log.payable else "0",
"1" if pymnt_log.skipped else "0",
pymnt_log.skippedatphase if pymnt_log.skipped else "-1",
pymnt_log.desc if pymnt_log.desc else "None",
pymnt_log.paymentaddress, rt
]
writer.writerow(array)
logger.debug(
"Reward created for {:s} type: {:s}, stake bal: {:>10.2f}, cur bal: {:>10.2f}, ratio: {:.6f}, fee_ratio: {:.6f}, "
"amount: {:>10.6f}, fee_amount: {:>4.6f}, fee_rate: {:.2f}, payable: {:s}, skipped: {:s}, at-phase: {:d}, "
"desc: {:s}, pay_addr: {:s}, type: {:s}".format(
pymnt_log.address, pymnt_log.type,
pymnt_log.staking_balance / MUTEZ,
pymnt_log.current_balance / MUTEZ, pymnt_log.ratio,
pymnt_log.service_fee_ratio, pymnt_log.amount / MUTEZ,
pymnt_log.service_fee_amount / MUTEZ,
pymnt_log.service_fee_rate,
"Y" if pymnt_log.payable else "N",
"Y" if pymnt_log.skipped else "N",
pymnt_log.skippedatphase, pymnt_log.desc,
pymnt_log.paymentaddress, rt))
logger.info(
"Calculation report is created at '{}'".format(report_file_path))
@staticmethod
def create_exit_payment():
return RewardLog.ExitInstance()
def notify_retry_fail_thread(self):
self.retry_fail_event.set()
# upon success retry failed payments if present
# success may indicate what went wrong in past is fixed.
def on_success(self, pymnt_batch):
self.notify_retry_fail_thread()
def on_fail(self, pymnt_batch):
pass
def __init__(self,
name,
initial_payment_cycle,
network_config,
payments_dir,
calculations_dir,
run_mode,
service_fee_calc,
release_override,
payment_offset,
baking_cfg,
payments_queue,
life_cycle,
dry_run,
client_manager,
node_url,
reward_data_provider,
node_url_public='',
api_base_url=None,
retry_injected=False):
super(PaymentProducer, self).__init__()
self.rules_model = RulesModel(baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map())
self.baking_address = baking_cfg.get_baking_address()
self.owners_map = baking_cfg.get_owners_map()
self.founders_map = baking_cfg.get_founders_map()
self.min_delegation_amt_in_mutez = baking_cfg.get_min_delegation_amount(
) * MUTEZ
self.delegator_pays_xfer_fee = baking_cfg.get_delegator_pays_xfer_fee()
self.provider_factory = ProviderFactory(reward_data_provider)
self.name = name
self.node_url = node_url
self.client_manager = client_manager
self.reward_api = self.provider_factory.newRewardApi(
network_config, self.baking_address, self.node_url,
node_url_public, api_base_url)
self.block_api = self.provider_factory.newBlockApi(
network_config, self.node_url, api_base_url)
dexter_contracts_set = baking_cfg.get_contracts_set()
if len(dexter_contracts_set) > 0 and not (self.reward_api.name
== 'tzstats'):
logger.warning(
"The Dexter functionality is currently only supported using tzstats."
"The contract address will be treated as a normal delegator.")
else:
self.reward_api.set_dexter_contracts_set(dexter_contracts_set)
self.rewards_type = baking_cfg.get_rewards_type()
self.fee_calc = service_fee_calc
self.initial_payment_cycle = initial_payment_cycle
if self.initial_payment_cycle is None:
recent = get_latest_report_file(payments_dir)
# if payment logs exists set initial cycle to following cycle
# if payment logs does not exists, set initial cycle to 0, so that payment starts from last released rewards
self.initial_payment_cycle = 0 if recent is None else int(
recent) + 1
logger.info("initial_cycle set to {}".format(
self.initial_payment_cycle))
self.nw_config = network_config
self.payments_root = payments_dir
self.calculations_dir = calculations_dir
self.run_mode = run_mode
self.exiting = False
self.release_override = release_override
self.payment_offset = payment_offset
self.payments_queue = payments_queue
self.life_cycle = life_cycle
self.dry_run = dry_run
self.payment_calc = PhasedPaymentCalculator(
self.founders_map, self.owners_map, self.fee_calc,
self.min_delegation_amt_in_mutez, self.rules_model)
self.retry_fail_thread = threading.Thread(target=self.retry_fail_run,
name=self.name +
"_retry_fail")
self.retry_fail_event = threading.Event()
self.retry_injected = retry_injected
self.retry_producer = RetryProducer(self.payments_queue,
self.reward_api, self,
self.payments_root,
self.retry_injected)
logger.info('Producer "{}" started'.format(self.name))
class PaymentProducer(threading.Thread, PaymentProducerABC):
def __init__(
self,
name,
initial_payment_cycle,
network_config,
payments_dir,
calculations_dir,
run_mode,
service_fee_calc,
release_override,
payment_offset,
baking_cfg,
payments_queue,
life_cycle,
dry_run,
client_manager,
node_url,
reward_data_provider,
node_url_public="",
api_base_url=None,
retry_injected=False,
):
super(PaymentProducer, self).__init__()
self.event = threading.Event()
self.rules_model = RulesModel(
baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map(),
)
self.baking_address = baking_cfg.get_baking_address()
self.owners_map = baking_cfg.get_owners_map()
self.founders_map = baking_cfg.get_founders_map()
self.min_delegation_amt_in_mutez = (
baking_cfg.get_min_delegation_amount() * MUTEZ)
self.delegator_pays_xfer_fee = baking_cfg.get_delegator_pays_xfer_fee()
self.provider_factory = ProviderFactory(reward_data_provider)
self.name = name
self.node_url = node_url
self.client_manager = client_manager
self.reward_api = self.provider_factory.newRewardApi(
network_config,
self.baking_address,
self.node_url,
node_url_public,
api_base_url,
)
self.block_api = self.provider_factory.newBlockApi(
network_config, self.node_url, api_base_url)
dexter_contracts_set = baking_cfg.get_contracts_set()
if len(dexter_contracts_set) > 0 and not (self.reward_api.name
== "tzstats"):
logger.warning(
"The Dexter functionality is currently only supported using tzstats."
"The contract address will be treated as a normal delegator.")
else:
self.reward_api.set_dexter_contracts_set(dexter_contracts_set)
self.rewards_type = baking_cfg.get_rewards_type()
self.pay_denunciation_rewards = baking_cfg.get_pay_denunciation_rewards(
)
self.fee_calc = service_fee_calc
self.initial_payment_cycle = initial_payment_cycle
logger.info("Initial cycle set to {}".format(
self.initial_payment_cycle))
self.nw_config = network_config
self.payments_root = payments_dir
self.calculations_dir = calculations_dir
self.run_mode = run_mode
self.exiting = False
self.release_override = release_override
self.payment_offset = payment_offset
self.payments_queue = payments_queue
self.life_cycle = life_cycle
self.dry_run = dry_run
self.payment_calc = PhasedPaymentCalculator(
self.founders_map,
self.owners_map,
self.fee_calc,
self.min_delegation_amt_in_mutez,
self.rules_model,
)
self.retry_fail_thread = threading.Thread(target=self.retry_fail_run,
name=self.name +
"_retry_fail")
self.retry_fail_event = threading.Event()
self.retry_injected = retry_injected
self.retry_producer = RetryProducer(
self.payments_queue,
self.reward_api,
self,
self.payments_root,
self.initial_payment_cycle,
self.retry_injected,
)
logger.debug('Producer "{}" started'.format(self.name))
def exit(self):
if not self.exiting:
self.payments_queue.put(
PaymentBatch(self, 0, [self.create_exit_payment()]))
self.exiting = True
if (self.life_cycle.is_running() and threading.current_thread()
is not threading.main_thread()):
_thread.interrupt_main()
if self.retry_fail_event:
self.retry_fail_event.set()
def retry_fail_run(self):
logger.debug('Retry Fail thread "{}" started'.format(
self.retry_fail_thread.name))
sleep(60) # producer thread already tried once, wait for next try
while not self.exiting and self.life_cycle.is_running():
self.retry_producer.retry_failed_payments()
try:
# prepare to wait on event
self.retry_fail_event.clear()
# this will either return with timeout or set from parent producer thread
self.retry_fail_event.wait(60 * 60) # 1 hour
except RuntimeError:
pass
def run(self):
# call first retry if not in onetime mode.
# retry_failed script is more suitable for one time cases.
if not self.run_mode == RunMode.ONETIME:
self.retry_producer.retry_failed_payments()
if self.run_mode == RunMode.RETRY_FAILED:
sleep(5)
self.exit()
return
# first retry is done by producer thread, start retry thread for further retries
if self.run_mode == RunMode.FOREVER:
self.retry_fail_thread.start()
try:
(
current_cycle,
current_level,
) = self.block_api.get_current_cycle_and_level()
except ApiProviderException as a:
logger.error(
"Unable to fetch current cycle, {:s}. Exiting.".format(str(a)))
self.exit()
return
# if initial_payment_cycle has the default value of -1 resulting in the last released cycle
if self.initial_payment_cycle == -1:
pymnt_cycle = (current_cycle -
(self.nw_config["NB_FREEZE_CYCLE"] + 1) -
self.release_override)
if pymnt_cycle < 0:
logger.error(
"Payment cycle cannot be < 0 but configuration results to {}"
.format(pymnt_cycle))
else:
logger.debug(
"Payment cycle is set to last released cycle {}".format(
pymnt_cycle))
else:
pymnt_cycle = self.initial_payment_cycle
get_verbose_log_helper().reset(pymnt_cycle)
while not self.exiting and self.life_cycle.is_running():
# take a breath
sleep(5)
try:
# Exit if disk is full
# https://github.com/tezos-reward-distributor-organization/tezos-reward-distributor/issues/504
if disk_is_full():
self.exit()
break
# Check if local node is bootstrapped; sleep if needed; restart loop
if not self.node_is_bootstrapped():
logger.info(
"Local node {} is not in sync with the Tezos network. Will sleep for {} blocks and check again."
.format(self.node_url, BOOTSTRAP_SLEEP))
self.wait_for_blocks(BOOTSTRAP_SLEEP)
continue
# Local node is ready
(
current_cycle,
current_level,
) = self.block_api.get_current_cycle_and_level()
level_in_cycle = self.block_api.level_in_cycle(current_level)
# create reports dir
if self.calculations_dir and not os.path.exists(
self.calculations_dir):
os.makedirs(self.calculations_dir)
logger.debug(
"Checking for pending payments: payment_cycle <= current_cycle - (self.nw_config['NB_FREEZE_CYCLE'] + 1) - self.release_override"
)
logger.info(
"Checking for pending payments: checking {} <= {} - ({} + 1) - {}"
.format(
pymnt_cycle,
current_cycle,
self.nw_config["NB_FREEZE_CYCLE"],
self.release_override,
))
# payments should not pass beyond last released reward cycle
if (pymnt_cycle <= current_cycle -
(self.nw_config["NB_FREEZE_CYCLE"] + 1) -
self.release_override):
if not self.payments_queue.full():
if (not self.pay_denunciation_rewards
) and self.reward_api.name == "RPC":
logger.info(
"Error: pay_denunciation_rewards=False requires an indexer since it is not possible to distinguish reward source using RPC"
)
e = "You must set 'pay_denunciation_rewards' to True when using RPC provider."
logger.error(e)
self.exit()
break
# Paying upcoming cycles (-R in [-6, -11] )
if pymnt_cycle >= current_cycle:
logger.warn(
"Please note that you are doing payouts for future rewards!!! These rewards are not earned yet, they are an estimation."
)
if not self.rewards_type.isEstimated():
logger.error(
"For future rewards payout, you must configure the payout type to 'Estimated', see documentation"
)
self.exit()
break
# Paying cycles with frozen rewards (-R in [-1, -5] )
elif (pymnt_cycle >= current_cycle -
self.nw_config["NB_FREEZE_CYCLE"]):
logger.warn(
"Please note that you are doing payouts for frozen rewards!!!"
)
# If user wants to offset payments within a cycle, check here
if level_in_cycle < self.payment_offset:
wait_offset_blocks = self.payment_offset - level_in_cycle
wait_offset_minutes = (
wait_offset_blocks *
self.nw_config["MINIMAL_BLOCK_DELAY"]) / 60
logger.info(
"Current level within the cycle is {}; Requested offset is {}; Waiting for {} more blocks (~{} minutes)"
.format(
level_in_cycle,
self.payment_offset,
wait_offset_blocks,
wait_offset_minutes,
))
self.wait_for_blocks(wait_offset_blocks)
continue # Break/Repeat loop
else:
result = self.try_to_pay(pymnt_cycle,
self.rewards_type,
self.nw_config)
if result:
# single run is done. Do not continue.
if self.run_mode == RunMode.ONETIME:
logger.info(
"Run mode ONETIME satisfied. Terminating..."
)
self.exit()
break
else:
pymnt_cycle = pymnt_cycle + 1
get_verbose_log_helper().reset(pymnt_cycle)
# end of queue size check
else:
logger.debug("Wait a few minutes, queue is full")
# wait a few minutes to let payments finish
sleep(60 * 3)
# end of payment cycle check
else:
logger.info(
"No pending payments for cycle {}, current cycle is {}"
.format(pymnt_cycle, current_cycle))
# pending payments done. Do not wait any more.
if self.run_mode == RunMode.PENDING:
logger.info(
"Run mode PENDING satisfied. Terminating...")
self.exit()
break
sleep(10)
# calculate number of blocks until end of current cycle plus user-defined offset
nb_blocks_remaining = (self.nw_config["BLOCKS_PER_CYCLE"] -
level_in_cycle +
self.payment_offset)
logger.debug(
"Waiting until next cycle; {} blocks remaining".format(
nb_blocks_remaining))
# wait until current cycle ends
self.wait_for_blocks(nb_blocks_remaining)
except (ApiProviderException, ReadTimeout, ConnectTimeout) as e:
logger.debug(
"{:s} error at payment producer loop: '{:s}'".format(
self.reward_api.name, str(e)),
exc_info=True,
)
logger.error(
"{:s} error at payment producer loop: '{:s}', will try again."
.format(self.reward_api.name, str(e)))
except Exception as e:
logger.debug(
"Unknown error in payment producer loop: {:s}".format(
str(e)),
exc_info=True,
)
logger.error(
"Unknown error in payment producer loop: {:s}, will try again."
.format(str(e)))
# end of endless loop
logger.debug("Producer returning...")
# ensure consumer exits
self.exit()
return
def stop(self):
self.exit()
self.event.set()
def compute_rewards(self, reward_model, rewards_type, network_config):
if rewards_type.isEstimated():
logger.info("Using estimated rewards for payouts calculations")
block_reward = network_config["BLOCK_REWARD"]
endorsement_reward = network_config["ENDORSEMENT_REWARD"]
total_estimated_block_reward = reward_model.num_baking_rights * block_reward
total_estimated_endorsement_reward = (
reward_model.num_endorsing_rights * endorsement_reward)
computed_reward_amount = (total_estimated_block_reward +
total_estimated_endorsement_reward)
elif rewards_type.isActual():
logger.info("Using actual rewards for payouts calculations")
if self.pay_denunciation_rewards:
computed_reward_amount = reward_model.total_reward_amount
else:
# omit denunciation rewards
computed_reward_amount = (reward_model.rewards_and_fees -
reward_model.equivocation_losses)
elif rewards_type.isIdeal():
logger.info("Using ideal rewards for payouts calculations")
if self.pay_denunciation_rewards:
computed_reward_amount = (reward_model.total_reward_amount +
reward_model.offline_losses)
else:
# omit denunciation rewards and double baking loss
computed_reward_amount = (reward_model.rewards_and_fees +
reward_model.offline_losses)
return computed_reward_amount
def try_to_pay(self, pymnt_cycle, rewards_type, network_config):
try:
logger.info("Payment cycle is {:s}".format(str(pymnt_cycle)))
# 0- check for past payment evidence for current cycle
past_payment_state = check_past_payment(self.payments_root,
pymnt_cycle)
if past_payment_state:
logger.warn(past_payment_state)
return True
# 1- get reward data
reward_model = self.reward_api.get_rewards_for_cycle_map(
pymnt_cycle, rewards_type)
# 2- compute reward amount to distribute based on configuration
reward_model.computed_reward_amount = self.compute_rewards(
reward_model, rewards_type, network_config)
# 3- calculate rewards for delegators
reward_logs, total_amount = self.payment_calc.calculate(
reward_model)
# 4- set cycle info
for rl in reward_logs:
rl.cycle = pymnt_cycle
total_amount_to_pay = sum(
[rl.amount for rl in reward_logs if rl.payable])
# 5- if total_rewards > 0, proceed with payment
if total_amount_to_pay > 0:
# 6- send to payment consumer
self.payments_queue.put(
PaymentBatch(self, pymnt_cycle, reward_logs))
sleep(5.0)
# 7- create calculations report file. This file contains calculations details
report_file_path = get_calculation_report_file(
self.calculations_dir, pymnt_cycle)
logger.debug("Creating calculation report (%s)",
report_file_path)
self.create_calculations_report(reward_logs, report_file_path,
total_amount, rewards_type)
# 8- processing of cycle is done
logger.info(
"Reward creation is done for cycle {}, created {} rewards."
.format(pymnt_cycle, len(reward_logs)))
elif total_amount_to_pay == 0:
logger.info("Total payment amount is 0. Nothing to pay!")
except ApiProviderException as a:
logger.error("[try_to_pay] API provider error {:s}".format(str(a)))
raise a from a
except Exception as e:
logger.error("[try_to_pay] Generic exception {:s}".format(str(e)))
raise e from e
# Either succeeded or raised exception
return True
def wait_for_blocks(self, nb_blocks_remaining):
for x in range(nb_blocks_remaining):
sleep(self.nw_config["MINIMAL_BLOCK_DELAY"])
# if shutting down, exit
if not self.life_cycle.is_running():
self.exit()
break
def node_is_bootstrapped(self):
# Get RPC node's (-A) bootstrap time. If bootstrap time + 2 minutes is
# before local time, node is not bootstrapped.
#
# clnt_mngr is a super class of SimpleClientManager which interfaces
# with the tezos-node used for txn forging/signing/injection. This is the
# node which we need to determine bootstrapped state
try:
boot_time = self.client_manager.get_bootstrapped()
utc_time = datetime.utcnow()
if (boot_time + timedelta(minutes=2)) < utc_time:
logger.debug(
"Current time is '{}', latest block of local node is '{}'."
.format(utc_time, boot_time))
return False
except ValueError:
logger.error(
"Unable to determine local node's bootstrap status. Continuing..."
)
return True
def create_calculations_report(self, payment_logs, report_file_path,
total_rewards, rewards_type):
if rewards_type.isEstimated():
rt = "E"
elif rewards_type.isActual():
rt = "A"
elif rewards_type.isIdeal():
rt = "I"
try:
# Open reports file and write; auto-closes file
with open(report_file_path, "w", newline="") as f:
writer = csv.writer(f,
delimiter=",",
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
# write headers and total rewards
writer.writerow([
"address",
"type",
"staked_balance",
"current_balance",
"ratio",
"fee_ratio",
"amount",
"fee_amount",
"fee_rate",
"payable",
"skipped",
"atphase",
"desc",
"payment_address",
"rewards_type",
])
# First row is for the baker
writer.writerow([
self.baking_address,
"B",
sum([pl.staking_balance for pl in payment_logs]),
"{0:f}".format(1.0),
"{0:f}".format(1.0),
"{0:f}".format(0.0),
"{0:f}".format(total_rewards),
"{0:f}".format(0.0),
"{0:f}".format(0.0),
"0",
"0",
"-1",
"Baker",
"None",
rt,
])
for pymnt_log in payment_logs:
# write row to csv file
array = [
pymnt_log.address,
pymnt_log.type,
pymnt_log.staking_balance,
pymnt_log.current_balance,
"{0:.10f}".format(pymnt_log.ratio),
"{0:.10f}".format(pymnt_log.service_fee_ratio),
"{0:f}".format(pymnt_log.amount),
"{0:f}".format(pymnt_log.service_fee_amount),
"{0:f}".format(pymnt_log.service_fee_rate),
"1" if pymnt_log.payable else "0",
"1" if pymnt_log.skipped else "0",
pymnt_log.skippedatphase
if pymnt_log.skipped else "-1",
pymnt_log.desc if pymnt_log.desc else "None",
pymnt_log.paymentaddress,
rt,
]
writer.writerow(array)
logger.debug(
"Reward created for {:s} type: {:s}, stake bal: {:>10.2f}, cur bal: {:>10.2f}, ratio: {:.6f}, fee_ratio: {:.6f}, "
"amount: {:>10.6f}, fee_amount: {:>4.6f}, fee_rate: {:.2f}, payable: {:s}, skipped: {:s}, at-phase: {:d}, "
"desc: {:s}, pay_addr: {:s}, type: {:s}".format(
pymnt_log.address,
pymnt_log.type,
pymnt_log.staking_balance / MUTEZ,
pymnt_log.current_balance / MUTEZ,
pymnt_log.ratio,
pymnt_log.service_fee_ratio,
pymnt_log.amount / MUTEZ,
pymnt_log.service_fee_amount / MUTEZ,
pymnt_log.service_fee_rate,
"Y" if pymnt_log.payable else "N",
"Y" if pymnt_log.skipped else "N",
pymnt_log.skippedatphase,
pymnt_log.desc,
pymnt_log.paymentaddress,
rt,
))
except Exception as e:
import errno
print("Exception during write operation invoked: {}".format(e))
if e.errno == errno.ENOSPC:
print("Not enough space on device!")
exit()
logger.info(
"Calculation report is created at '{}'".format(report_file_path))
@staticmethod
def create_exit_payment():
return RewardLog.ExitInstance()
def notify_retry_fail_thread(self):
self.retry_fail_event.set()
# upon success retry failed payments if present
# success may indicate what went wrong in past is fixed.
def on_success(self, pymnt_batch):
self.notify_retry_fail_thread()
def on_fail(self, pymnt_batch):
pass
def __init__(self,
name,
initial_payment_cycle,
network_config,
payments_dir,
calculations_dir,
run_mode,
service_fee_calc,
release_override,
payment_offset,
baking_cfg,
payments_queue,
life_cycle,
dry_run,
wllt_clnt_mngr,
node_url,
provider_factory,
node_url_public='',
api_base_url=None,
retry_injected=False):
super(PaymentProducer, self).__init__()
self.rules_model = RulesModel(baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map())
self.baking_address = baking_cfg.get_baking_address()
self.owners_map = baking_cfg.get_owners_map()
self.founders_map = baking_cfg.get_founders_map()
self.min_delegation_amt_in_mutez = baking_cfg.get_min_delegation_amount(
) * MUTEZ
self.delegator_pays_xfer_fee = baking_cfg.get_delegator_pays_xfer_fee()
self.name = name
self.node_url = node_url
self.wllt_clnt_mngr = wllt_clnt_mngr
self.reward_api = provider_factory.newRewardApi(
network_config, self.baking_address, self.node_url,
node_url_public, api_base_url)
self.block_api = provider_factory.newBlockApi(network_config,
self.node_url,
api_base_url)
dexter_contracts_set = baking_cfg.get_contracts_set()
if len(dexter_contracts_set) > 0 and not (self.reward_api.name
== 'tzstats'):
logger.warning(
"The Dexter functionality is currently only supported using tzstats."
"The contract address will be treated as a normal delegator.")
else:
self.reward_api.set_dexter_contracts_set(dexter_contracts_set)
self.fee_calc = service_fee_calc
self.initial_payment_cycle = initial_payment_cycle
self.nw_config = network_config
self.payments_root = payments_dir
self.calculations_dir = calculations_dir
self.run_mode = run_mode
self.exiting = False
self.release_override = release_override
self.payment_offset = payment_offset
self.payments_queue = payments_queue
self.life_cycle = life_cycle
self.dry_run = dry_run
self.payment_calc = PhasedPaymentCalculator(
self.founders_map, self.owners_map, self.fee_calc,
self.min_delegation_amt_in_mutez, self.rules_model)
self.retry_fail_thread = threading.Thread(target=self.retry_fail_run,
name=self.name +
"_retry_fail")
self.retry_fail_event = threading.Event()
self.retry_injected = retry_injected
logger.info('Producer "{}" started'.format(self.name))
def test_batch_payer_total_payout_amount():
factory = ProviderFactory(provider="prpc")
parser = BakingYamlConfParser(baking_config,
None,
None,
None,
None,
block_api=factory,
api_base_url=None)
parser.parse()
parser.process()
cfg_dict = parser.get_conf_obj()
baking_cfg = BakingConf(cfg_dict)
srvc_fee_calc = ServiceFeeCalculator(
baking_cfg.get_full_supporters_set(),
baking_cfg.get_specials_map(),
baking_cfg.get_service_fee(),
)
rules_model = RulesModel(
baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map(),
)
payment_calc = PhasedPaymentCalculator(
baking_cfg.get_founders_map(),
baking_cfg.get_owners_map(),
srvc_fee_calc,
baking_cfg.get_min_delegation_amount() * MUTEZ,
rules_model,
)
rewardApi = factory.newRewardApi(
default_network_config_map[CURRENT_TESTNET],
baking_cfg.get_baking_address(), "")
# Simulate logic in payment_producer
reward_logs = []
attempts = 0
exiting = False
while not exiting and attempts < 2:
attempts += 1
# Reward data
# Fetch cycle 51 of granadanet for tz1gtHbmBF3TSebsgJfJPvUB2e9x8EDeNm6V
reward_model = rewardApi.get_rewards_for_cycle_map(
PAYOUT_CYCLE, RewardsType.ACTUAL)
# Calculate rewards - payment_producer.py
reward_model.computed_reward_amount = reward_model.total_reward_amount
reward_logs, total_amount = payment_calc.calculate(reward_model)
# Check total reward amount matches sums of records
assert total_amount == sum(
[rl.amount for rl in reward_logs if rl.payable])
exiting = True
# Merge payments to same address
phaseMerge = CalculatePhaseMerge()
reward_logs = phaseMerge.calculate(reward_logs)
# Handle remapping of payment to alternate address
phaseMapping = CalculatePhaseMapping()
reward_logs = phaseMapping.calculate(reward_logs,
baking_cfg.get_dest_map())
# Filter zero-balance addresses based on config
phaseZeroBalance = CalculatePhaseZeroBalance()
reward_logs = phaseZeroBalance.calculate(
reward_logs, baking_cfg.get_reactivate_zeroed())
# Filter out non-payable items
reward_logs = [
payment_item for payment_item in reward_logs if payment_item.payable
]
reward_logs.sort(key=cmp_to_key(cmp_by_type_balance))
batch_payer = BatchPayer(
node_url=node_endpoint,
pymnt_addr="tz1gtHbmBF3TSebsgJfJPvUB2e9x8EDeNm6V",
clnt_mngr=ClientManager(node_endpoint, PRIVATE_SIGNER_URL),
delegator_pays_ra_fee=True,
delegator_pays_xfer_fee=True,
network_config=network,
plugins_manager=PluginManager(baking_cfg.get_plugins_conf(),
dry_run=True),
dry_run=True,
)
# Fix the endpoint auto port assignment because
# https://mainnet-tezos.giganode.io:8732 cannot be reached
batch_payer.clnt_mngr.node_endpoint = node_endpoint
# Do the payment
(
payment_logs,
total_attempts,
total_payout_amount,
number_future_payable_cycles,
) = batch_payer.pay(reward_logs, dry_run=True)
assert total_attempts == 3
assert total_payout_amount == 238211030
assert (PAYMENT_ADDRESS_BALANCE //
total_payout_amount) - 1 == number_future_payable_cycles
def test_batch_payer_total_payout_amount():
factory = ProviderFactory(provider="prpc")
parser = BakingYamlConfParser(
baking_config, None, None, None, None, block_api=factory, api_base_url=None
)
parser.parse()
parser.process()
cfg_dict = parser.get_conf_obj()
baking_cfg = BakingConf(cfg_dict)
srvc_fee_calc = ServiceFeeCalculator(
baking_cfg.get_full_supporters_set(),
baking_cfg.get_specials_map(),
baking_cfg.get_service_fee(),
)
rules_model = RulesModel(
baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map(),
)
payment_calc = PhasedPaymentCalculator(
baking_cfg.get_founders_map(),
baking_cfg.get_owners_map(),
srvc_fee_calc,
int(baking_cfg.get_min_delegation_amount() * MUTEZ_PER_TEZ),
rules_model,
)
rewardApi = factory.newRewardApi(
default_network_config_map[CURRENT_TESTNET], baking_cfg.get_baking_address(), ""
)
# Simulate logic in payment_producer
reward_logs = []
attempts = 0
exiting = False
while not exiting and attempts < 2:
attempts += 1
# Reward data
# Fetch cycle 51 of granadanet for tz1gtHbmBF3TSebsgJfJPvUB2e9x8EDeNm6V
reward_model = rewardApi.get_rewards_for_cycle_map(
PAYOUT_CYCLE, RewardsType.ACTUAL
)
# Calculate rewards - payment_producer.py
reward_model.computed_reward_amount = reward_model.total_reward_amount
reward_logs, total_amount = payment_calc.calculate(reward_model)
# Check total reward amount matches sums of records
# diff of 1 expected due to floating point arithmetic
assert (
total_amount - sum([rl.adjusted_amount for rl in reward_logs if rl.payable])
<= 1
)
exiting = True
# Merge payments to same address
phaseMerge = CalculatePhaseMerge()
reward_logs = phaseMerge.calculate(reward_logs)
# Handle remapping of payment to alternate address
phaseMapping = CalculatePhaseMapping()
reward_logs = phaseMapping.calculate(reward_logs, baking_cfg.get_dest_map())
# Filter zero-balance addresses based on config
phaseZeroBalance = CalculatePhaseZeroBalance()
reward_logs = phaseZeroBalance.calculate(
reward_logs, baking_cfg.get_reactivate_zeroed()
)
# Filter out non-payable items
reward_logs = [payment_item for payment_item in reward_logs if payment_item.payable]
reward_logs.sort(key=cmp_to_key(cmp_by_type_balance))
batch_payer = BatchPayer(
node_url=node_endpoint,
pymnt_addr="tz1N4UfQCahHkRShBanv9QP9TnmXNgCaqCyZ",
clnt_mngr=ClientManager(node_endpoint, PRIVATE_SIGNER_URL),
delegator_pays_ra_fee=True,
delegator_pays_xfer_fee=True,
network_config=network,
plugins_manager=PluginManager(baking_cfg.get_plugins_conf(), dry_run=True),
dry_run=True,
)
# Do the payment
(
_,
total_attempts,
total_payout_amount,
number_future_payable_cycles,
) = batch_payer.pay(reward_logs, dry_run=True)
# Payment does not have status done, paid or injected thus the total payout amount is zero
assert total_payout_amount == 0
assert number_future_payable_cycles == 2
assert total_attempts == 3
# Check the adjusted amount
assert reward_logs[0].adjusted_amount == 40418486
assert reward_logs[1].adjusted_amount == 10581272
assert reward_logs[2].adjusted_amount == 109732835
assert reward_logs[3].adjusted_amount == 48362127
assert reward_logs[4].adjusted_amount == 29116310
class PaymentProducer(threading.Thread, PaymentProducerABC):
def __init__(
self,
name,
initial_payment_cycle,
network_config,
payments_dir,
calculations_dir,
run_mode,
service_fee_calc,
release_override,
payment_offset,
baking_cfg,
payments_queue,
life_cycle,
dry_run,
client_manager,
node_url,
reward_data_provider,
node_url_public="",
api_base_url=None,
retry_injected=False,
):
super(PaymentProducer, self).__init__()
self.event = threading.Event()
self.rules_model = RulesModel(
baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map(),
)
self.baking_address = baking_cfg.get_baking_address()
self.owners_map = baking_cfg.get_owners_map()
self.founders_map = baking_cfg.get_founders_map()
self.min_delegation_amt_in_mutez = int(
baking_cfg.get_min_delegation_amount() * MUTEZ_PER_TEZ)
self.delegator_pays_xfer_fee = baking_cfg.get_delegator_pays_xfer_fee()
self.provider_factory = ProviderFactory(reward_data_provider)
self.name = name
self.node_url = node_url
self.client_manager = client_manager
self.reward_api = self.provider_factory.newRewardApi(
network_config,
self.baking_address,
self.node_url,
node_url_public,
api_base_url,
)
self.block_api = self.provider_factory.newBlockApi(
network_config, self.node_url, api_base_url)
dexter_contracts_set = baking_cfg.get_contracts_set()
if len(dexter_contracts_set) > 0 and not (self.reward_api.name
== "tzstats"):
logger.warning(
"The Dexter functionality is currently only supported using tzstats."
"The contract address will be treated as a normal delegator.")
else:
self.reward_api.set_dexter_contracts_set(dexter_contracts_set)
self.rewards_type = baking_cfg.get_rewards_type()
self.pay_denunciation_rewards = baking_cfg.get_pay_denunciation_rewards(
)
self.fee_calc = service_fee_calc
self.initial_payment_cycle = initial_payment_cycle
logger.info("Initial cycle set to {}".format(
self.initial_payment_cycle))
self.nw_config = network_config
self.payments_root = payments_dir
self.calculations_dir = calculations_dir
self.run_mode = run_mode
self.exiting = False
self.release_override = release_override
self.payment_offset = payment_offset
self.payments_queue = payments_queue
self.life_cycle = life_cycle
self.dry_run = dry_run
self.payment_calc = PhasedPaymentCalculator(
self.founders_map,
self.owners_map,
self.fee_calc,
self.min_delegation_amt_in_mutez,
self.rules_model,
)
self.retry_fail_thread = threading.Thread(target=self.retry_fail_run,
name=self.name +
"_retry_fail")
self.retry_fail_event = threading.Event()
self.retry_injected = retry_injected
self.retry_producer = RetryProducer(
self.payments_queue,
self.reward_api,
self,
self.payments_root,
self.initial_payment_cycle,
self.retry_injected,
)
logger.debug('Producer "{}" started'.format(self.name))
def exit(self):
if not self.exiting:
self.payments_queue.put(
PaymentBatch(self, 0, [self.create_exit_payment()]))
self.exiting = True
if (self.life_cycle.is_running() and threading.current_thread()
is not threading.main_thread()):
_thread.interrupt_main()
logger.info("Sending KeyboardInterrupt signal.")
if self.retry_fail_event:
self.retry_fail_event.set()
def retry_fail_run(self):
logger.debug('Retry Fail thread "{}" started'.format(
self.retry_fail_thread.name))
sleep(60) # producer thread already tried once, wait for next try
while not self.exiting and self.life_cycle.is_running():
self.retry_producer.retry_failed_payments()
try:
# prepare to wait on event
self.retry_fail_event.clear()
# this will either return with timeout or set from parent producer thread
self.retry_fail_event.wait(60 * 60) # 1 hour
except RuntimeError:
pass
def run(self):
# call first retry if not in onetime mode.
# retry_failed script is more suitable for one time cases.
if not self.run_mode == RunMode.ONETIME:
self.retry_producer.retry_failed_payments()
if self.run_mode == RunMode.RETRY_FAILED:
sleep(5)
self.exit()
return
# first retry is done by producer thread, start retry thread for further retries
if self.run_mode == RunMode.FOREVER:
self.retry_fail_thread.start()
try:
(
current_cycle,
current_level,
) = self.block_api.get_current_cycle_and_level()
except ApiProviderException as a:
logger.error(
"Unable to fetch current cycle, {:s}. Exiting.".format(str(a)))
self.exit()
return
# if initial_payment_cycle has the default value of -1 resulting in the last released cycle
if self.initial_payment_cycle == -1:
pymnt_cycle = (current_cycle -
(self.nw_config["NB_FREEZE_CYCLE"] + 1) -
self.release_override)
if pymnt_cycle < 0:
logger.error(
"Payment cycle cannot be < 0 but configuration results to {}"
.format(pymnt_cycle))
else:
logger.debug(
"Payment cycle is set to last released cycle {}".format(
pymnt_cycle))
else:
pymnt_cycle = self.initial_payment_cycle
get_verbose_log_helper().reset(pymnt_cycle)
while not self.exiting and self.life_cycle.is_running():
# take a breath
sleep(5)
try:
# Exit if disk is full
# https://github.com/tezos-reward-distributor-organization/tezos-reward-distributor/issues/504
if disk_is_full():
self.exit()
break
# Check if local node is bootstrapped; sleep if needed; restart loop
if not self.node_is_bootstrapped():
logger.info(
"Local node {} is not in sync with the Tezos network. Will sleep for {} blocks and check again."
.format(self.node_url, BOOTSTRAP_SLEEP))
self.wait_for_blocks(BOOTSTRAP_SLEEP)
continue
# Local node is ready
(
current_cycle,
current_level,
) = self.block_api.get_current_cycle_and_level()
level_in_cycle = self.block_api.level_in_cycle(current_level)
# create reports dir
if self.calculations_dir and not os.path.exists(
self.calculations_dir):
os.makedirs(self.calculations_dir)
logger.debug(
"Checking for pending payments: payment_cycle <= current_cycle - (self.nw_config['NB_FREEZE_CYCLE'] + 1) - self.release_override"
)
logger.info(
"Checking for pending payments: checking {} <= {} - ({} + 1) - {}"
.format(
pymnt_cycle,
current_cycle,
self.nw_config["NB_FREEZE_CYCLE"],
self.release_override,
))
# payments should not pass beyond last released reward cycle
if (pymnt_cycle <= current_cycle -
(self.nw_config["NB_FREEZE_CYCLE"] + 1) -
self.release_override):
if not self.payments_queue.full():
if (not self.pay_denunciation_rewards
) and self.reward_api.name == "RPC":
logger.info(
"Error: pay_denunciation_rewards=False requires an indexer since it is not possible to distinguish reward source using RPC"
)
e = "You must set 'pay_denunciation_rewards' to True when using RPC provider."
logger.error(e)
self.exit()
break
# Paying upcoming cycles (-R set to -11 )
if pymnt_cycle >= current_cycle:
logger.warn(
"Please note that you are doing payouts for future rewards!!! These rewards are not earned yet, they are an estimation."
)
logger.warn(
"TRD will attempt to adjust the amount after the cycle runs, but it may not work."
)
# Paying cycles with frozen rewards (-R set to -5 )
elif (pymnt_cycle >= current_cycle -
self.nw_config["NB_FREEZE_CYCLE"]):
logger.warn(
"Please note that you are doing payouts for frozen rewards!!!"
)
# If user wants to offset payments within a cycle, check here
if level_in_cycle < self.payment_offset:
wait_offset_blocks = self.payment_offset - level_in_cycle
wait_offset_minutes = (
wait_offset_blocks *
self.nw_config["MINIMAL_BLOCK_DELAY"]) / 60
logger.info(
"Current level within the cycle is {}; Requested offset is {}; Waiting for {} more blocks (~{} minutes)"
.format(
level_in_cycle,
self.payment_offset,
wait_offset_blocks,
wait_offset_minutes,
))
self.wait_for_blocks(wait_offset_blocks)
continue # Break/Repeat loop
else:
result = self.try_to_pay(
pymnt_cycle,
self.rewards_type,
self.nw_config,
current_cycle,
)
if result:
# single run is done. Do not continue.
if self.run_mode == RunMode.ONETIME:
logger.info(
"Run mode ONETIME satisfied. Terminating..."
)
self.exit()
break
else:
pymnt_cycle = pymnt_cycle + 1
get_verbose_log_helper().reset(pymnt_cycle)
# end of queue size check
else:
logger.debug("Wait a few minutes, queue is full")
# wait a few minutes to let payments finish
sleep(60 * 3)
# end of payment cycle check
else:
logger.info(
"No pending payments for cycle {}, current cycle is {}"
.format(pymnt_cycle, current_cycle))
# pending payments done. Do not wait any more.
if self.run_mode == RunMode.PENDING:
logger.info(
"Run mode PENDING satisfied. Terminating...")
self.exit()
break
sleep(10)
# calculate number of blocks until end of current cycle plus user-defined offset
nb_blocks_remaining = (self.nw_config["BLOCKS_PER_CYCLE"] -
level_in_cycle +
self.payment_offset)
logger.debug(
"Waiting until next cycle; {} blocks remaining".format(
nb_blocks_remaining))
# wait until current cycle ends
self.wait_for_blocks(nb_blocks_remaining)
except (ApiProviderException, ReadTimeout, ConnectTimeout) as e:
logger.debug(
"{:s} error at payment producer loop: '{:s}'".format(
self.reward_api.name, str(e)),
exc_info=True,
)
logger.error(
"{:s} error at payment producer loop: '{:s}', will try again."
.format(self.reward_api.name, str(e)))
except Exception as e:
logger.debug(
"Unknown error in payment producer loop: {:s}".format(
str(e)),
exc_info=True,
)
logger.error(
"Unknown error in payment producer loop: {:s}, will try again."
.format(str(e)))
# end of endless loop
logger.debug("Producer returning...")
# ensure consumer exits
self.exit()
return
def stop(self):
self.exit()
self.event.set()
def compute_rewards(self,
pymnt_cycle,
computation_type,
network_config,
adjustments={}):
"""
Compute total rewards based on computation type and policy, then
calls payment_call.calculate to calculate rewards per delegator.
:param pymnt_cycle: the cycle for which rewards are being calculated
:param computation_type: calculate estimated, actual or ideal rewards
:param network_config: configuration of the current tezos network, needed to calc rewards.
:param adjustments: a map of adjustments per address. We add to amount to compute adjusted_amount
:return: rewards per delegator (reward logs) and total amount
"""
# does calculation report file already exist for this cycle?
logger.info("Computing rewards for cycle {:s}.".format(
str(pymnt_cycle)))
reward_model = self.reward_api.get_rewards_for_cycle_map(
pymnt_cycle, computation_type)
if computation_type.isEstimated():
logger.info("Using estimated rewards for payouts calculations")
block_reward = network_config["BLOCK_REWARD"]
endorsement_reward = network_config["ENDORSEMENT_REWARD"]
total_estimated_block_reward = reward_model.num_baking_rights * block_reward
total_estimated_endorsement_reward = (
reward_model.num_endorsing_rights * endorsement_reward)
reward_model.computed_reward_amount = (
total_estimated_block_reward +
total_estimated_endorsement_reward)
elif computation_type.isActual():
logger.info("Using actual rewards for payouts calculations")
if self.pay_denunciation_rewards:
reward_model.computed_reward_amount = reward_model.total_reward_amount
else:
# omit denunciation rewards
reward_model.computed_reward_amount = (
reward_model.rewards_and_fees -
reward_model.equivocation_losses)
elif computation_type.isIdeal():
logger.info("Using ideal rewards for payouts calculations")
if self.pay_denunciation_rewards:
reward_model.computed_reward_amount = (
reward_model.total_reward_amount +
reward_model.offline_losses)
else:
# omit denunciation rewards and double baking loss
reward_model.computed_reward_amount = (
reward_model.rewards_and_fees +
reward_model.offline_losses)
# 3- calculate rewards for delegators
return self.payment_calc.calculate(reward_model, adjustments)
def recompute_rewards(self, completed_cycle, computation_type,
network_config):
"""
In case of early payout, the payout is already done when the cycle runs.
After a cycle has run, we redo the computations. If we find overpayemnt
(overestimate) or underpayment (negative overestimate), we record it in the
calculation report csv file and return an adjustment map.
:param completed_cycle: the cycle for which rewards are being recalculated
:param computation_type: calculate estimated, actual or ideal rewards
:param network_config: configuration of the current tezos network, needed to calc rewards.
:return: adjustments map showing negative of overestimates per delegator
"""
logger.info(
"Checking for potential adjustment for recently completed cycle {:s}."
.format(str(completed_cycle)))
completed_cycle_report_file_path = get_calculation_report_file_path(
self.calculations_dir, completed_cycle)
adjustments = {}
if os.path.isfile(completed_cycle_report_file_path):
logger.info(
"TRD ran for cycle: {:s}, calculating adjustments.".format(
str(completed_cycle)))
(
reward_logs_from_report,
total_amount_from_report,
rewards_type_from_report,
_,
) = CsvCalculationFileParser().parse(
completed_cycle_report_file_path, self.baking_address)
# check that the overestimate has not been computed yet
if sum([rl.overestimate or 0
for rl in reward_logs_from_report]) > 0:
logger.info(
"Overestimate has already been calculated for cycle {:s}, not calculating it again."
.format(str(completed_cycle)))
completed_cycle_reward_logs, completed_cycle_total_amount = (
reward_logs_from_report,
total_amount_from_report,
)
else:
(
completed_cycle_reward_logs,
completed_cycle_total_amount,
) = self.compute_rewards(completed_cycle, computation_type,
network_config)
overestimate = int(total_amount_from_report -
completed_cycle_total_amount)
logger.info(
"We {:s}estimated payout for cycle {:s} by {:<,d} mutez, will attempt to adjust."
.format(
("over" if overestimate > 0 else "under"),
str(completed_cycle),
abs(overestimate),
))
for rl in reward_logs_from_report:
# overwrite only overestimate in report csv file, leave the rest alone
rl.overestimate = int(
Decimal(rl.ratio * overestimate).to_integral_value(
rounding=ROUND_HALF_DOWN))
# we adjust the cycle we are paying out with the overestimate of the
# just completed cycle
adjustments[rl.address] = rl.overestimate
logger.debug(
f"Will try to recover {adjustments[rl.address]} mutez for {rl.address} based on past overpayment"
)
CsvCalculationFileParser().write(
reward_logs_from_report,
completed_cycle_report_file_path,
total_amount_from_report,
rewards_type_from_report,
self.baking_address,
False,
)
return adjustments
def try_to_pay(self, pymnt_cycle, rewards_type, network_config,
current_cycle):
try:
logger.info("Payment cycle is {:s}".format(str(pymnt_cycle)))
# 0- check for past payment evidence for current cycle
past_payment_state = check_past_payment(self.payments_root,
pymnt_cycle)
if past_payment_state:
logger.warn(past_payment_state)
return True
adjustments = {}
early_payout = False
current_cycle_rewards_type = rewards_type
# 1- adjust past cycle if necessary
if self.release_override == -11 and pymnt_cycle >= current_cycle:
early_payout = True
completed_cycle = pymnt_cycle - 6
adjustments = self.recompute_rewards(completed_cycle,
rewards_type,
network_config)
# payout for current cycle will be estimated since we don't know actual rewards yet
current_cycle_rewards_type = RewardsType.ESTIMATED
# 2- get reward data and compute how to distribute them
reward_logs, total_amount = self.compute_rewards(
pymnt_cycle, current_cycle_rewards_type, network_config,
adjustments)
total_recovered_adjustments = int(
sum([rl.adjustment for rl in reward_logs]))
total_adjustments_to_recover = int(sum(adjustments.values()))
if total_adjustments_to_recover > 0:
logger.debug(
"Total adjustments to recover is {:<,d} mutez, total recovered adjustment is {:<,d} mutez."
.format(total_adjustments_to_recover,
total_recovered_adjustments))
logger.info(
"After early payout of cycle {:s}, {:<,d} mutez were not recovered."
.format(
str(completed_cycle),
total_adjustments_to_recover +
total_recovered_adjustments,
))
# 3- create calculations report file. This file contains calculations details
report_file_path = get_calculation_report_file_path(
self.calculations_dir, pymnt_cycle)
logger.debug("Creating calculation report (%s)", report_file_path)
CsvCalculationFileParser().write(
reward_logs,
report_file_path,
total_amount,
current_cycle_rewards_type,
self.baking_address,
early_payout,
)
# 4- set cycle info
for reward_log in reward_logs:
reward_log.cycle = pymnt_cycle
total_amount_to_pay = int(
sum([
reward_log.adjusted_amount for reward_log in reward_logs
if reward_log.payable
]))
# 5- if total_rewards > 0, proceed with payment
if total_amount_to_pay > 0:
self.payments_queue.put(
PaymentBatch(self, pymnt_cycle, reward_logs))
sleep(5.0)
# 6- processing of cycle is done
logger.info(
"Reward creation is done for cycle {}, created {} rewards."
.format(pymnt_cycle, len(reward_logs)))
elif total_amount_to_pay == 0:
logger.info("Total payment amount is 0. Nothing to pay!")
except ApiProviderException as a:
logger.error("[try_to_pay] API provider error {:s}".format(str(a)))
raise a from a
except Exception as e:
logger.error("[try_to_pay] Generic exception {:s}".format(str(e)))
raise e from e
# Either succeeded or raised exception
return True
def wait_for_blocks(self, nb_blocks_remaining):
for x in range(nb_blocks_remaining):
sleep(self.nw_config["MINIMAL_BLOCK_DELAY"])
# if shutting down, exit
if not self.life_cycle.is_running():
self.exit()
break
def node_is_bootstrapped(self):
# Get RPC node's (-A) bootstrap time. If bootstrap time + 2 minutes is
# before local time, node is not bootstrapped.
#
# clnt_mngr is a super class of SimpleClientManager which interfaces
# with the tezos-node used for txn forging/signing/injection. This is the
# node which we need to determine bootstrapped state
try:
boot_time = self.client_manager.get_bootstrapped()
utc_time = datetime.utcnow()
if (boot_time + timedelta(minutes=2)) < utc_time:
logger.debug(
"Current time is '{}', latest block of local node is '{}'."
.format(utc_time, boot_time))
return False
except ValueError:
logger.error(
"Unable to determine local node's bootstrap status. Continuing..."
)
return True
@staticmethod
def create_exit_payment():
return RewardLog.ExitInstance()
def notify_retry_fail_thread(self):
self.retry_fail_event.set()
# upon success retry failed payments if present
# success may indicate what went wrong in past is fixed.
def on_success(self, pymnt_batch):
self.notify_retry_fail_thread()
def on_fail(self, pymnt_batch):
pass