def test_disk_full_payment_producer(args, caplog):
# Issue: https://github.com/tezos-reward-distributor-organization/tezos-reward-distributor/issues/504
client_manager = ClientManager(args.node_endpoint, args.signer_endpoint)
network_config_map = init_network_config(args.network, client_manager)
factory = ProviderFactory(provider="prpc")
parser = BakingYamlConfParser(
baking_config, None, None, None, None, block_api=factory, api_base_url=None
)
parser.parse()
parser.process()
cfg_dict = parser.get_conf_obj()
baking_cfg = BakingConf(cfg_dict)
baking_dirs = BakingDirs(args, baking_cfg.get_baking_address())
srvc_fee_calc = ServiceFeeCalculator(
baking_cfg.get_full_supporters_set(),
baking_cfg.get_specials_map(),
baking_cfg.get_service_fee(),
)
payments_queue = queue.Queue(50)
plc = ProcessLifeCycle(None)
pp = PaymentProducer(
name="producer",
network_config=network_config_map[args.network],
payments_dir=baking_dirs.payments_root,
calculations_dir=baking_dirs.calculations_root,
run_mode=RunMode(args.run_mode),
service_fee_calc=srvc_fee_calc,
release_override=args.release_override,
payment_offset=args.payment_offset,
baking_cfg=baking_cfg,
life_cycle=plc,
payments_queue=payments_queue,
dry_run=args.dry_run,
client_manager=client_manager,
node_url=args.node_endpoint,
reward_data_provider=args.reward_data_provider,
node_url_public=args.node_addr_public,
api_base_url=args.api_base_url,
retry_injected=args.retry_injected,
initial_payment_cycle=args.initial_cycle,
)
assert disk_is_full()
try:
pp.daemon = True
pp.start()
finally:
pp.stop()
assert (
"Disk is becoming full. Only 0.50 Gb left from 10.00 Gb. Please clean up disk to continue saving logs and reports."
in caplog.text
)
def test_disk_full_payment_consumer(args, caplog):
# Issue: https://github.com/tezos-reward-distributor-organization/tezos-reward-distributor/issues/504
client_manager = ClientManager(args.node_endpoint, args.signer_endpoint)
network_config_map = init_network_config(args.network, client_manager)
factory = ProviderFactory(provider="prpc")
parser = BakingYamlConfParser(
baking_config, None, None, None, None, block_api=factory, api_base_url=None
)
parser.parse()
parser.process()
cfg_dict = parser.get_conf_obj()
baking_cfg = BakingConf(cfg_dict)
baking_dirs = BakingDirs(args, baking_cfg.get_baking_address())
payments_queue = queue.Queue(50)
plugins_manager = plugins.PluginManager(baking_cfg.get_plugins_conf(), args.dry_run)
pc = PaymentConsumer(
name="consumer0",
payments_dir=baking_dirs.payments_root,
key_name=baking_cfg.get_payment_address(),
payments_queue=payments_queue,
node_addr=args.node_endpoint,
client_manager=client_manager,
plugins_manager=plugins_manager,
rewards_type=baking_cfg.get_rewards_type(),
args=args,
dry_run=args.dry_run,
reactivate_zeroed=baking_cfg.get_reactivate_zeroed(),
delegator_pays_ra_fee=baking_cfg.get_delegator_pays_ra_fee(),
delegator_pays_xfer_fee=baking_cfg.get_delegator_pays_xfer_fee(),
dest_map=baking_cfg.get_dest_map(),
network_config=network_config_map[args.network],
publish_stats=not args.do_not_publish_stats,
)
assert disk_is_full()
try:
pc.daemon = True
pc.start()
finally:
pc.stop()
assert (
"Disk is becoming full. Only 0.30 Gb left from 11.00 Gb. Please clean up disk to continue saving logs and reports."
in caplog.text
)
def run(self):
running = True
while running:
# Exit if disk is full
# https://github.com/tezos-reward-distributor-organization/tezos-reward-distributor/issues/504
if disk_is_full():
running = False
break
# Wait until a reward is present
payment_batch = self.payments_queue.get(True)
running = self._consume_batch(payment_batch)
logger.debug("Consumer returning...")
return
def main(args):
logger.info(
"Arguments Configuration = {}".format(json.dumps(args.__dict__, indent=1))
)
# Load payments file
payments_file = os.path.expanduser(os.path.normpath(args.payments_file))
if not os.path.isfile(payments_file):
raise Exception("payments_file ({}) does not exist.".format(payments_file))
with open(payments_file, "r") as file:
payment_lines = file.readlines()
payments_dict = {}
for line in payment_lines:
pkh, amt = line.split(":")
pkh = pkh.strip()
amt = float(amt.strip())
payments_dict[pkh] = amt
if not payments_dict:
raise Exception("No payments to process")
# Check if dry-run
dry_run = args.dry_run
# Get reporting directories
reports_dir = os.path.expanduser(os.path.normpath(args.base_directory))
# Check the disk size at the reports dir location
if disk_is_full(reports_dir):
raise Exception(
"Disk is full at {}. Please free space to continue saving reports.".format(
reports_dir
)
)
# if in reports run mode, do not create consumers
# create reports in reports directory
if dry_run:
reports_dir = os.path.join(reports_dir, SIMULATIONS_DIR, "")
else:
reports_dir = os.path.join(reports_dir, REPORTS_DIR, "")
reports_dir = os.path.join(reports_dir, "manual", "")
payments_root = get_payment_root(reports_dir, create=True)
get_successful_payments_dir(payments_root, create=True)
get_failed_payments_dir(payments_root, create=True)
client_manager = ClientManager(
node_endpoint=args.node_endpoint, signer_endpoint=args.signer_endpoint
)
for i in range(NB_CONSUMERS):
c = PaymentConsumer(
name="manual_payment_consumer",
payments_dir=payments_root,
key_name=args.paymentaddress,
payments_queue=payments_queue,
node_addr=args.node_endpoint,
client_manager=client_manager,
dry_run=dry_run,
reactivate_zeroed=False,
delegator_pays_ra_fee=False,
delegator_pays_xfer_fee=False,
)
time.sleep(1)
c.start()
base_name_no_ext = os.path.basename(payments_file)
base_name_no_ext = os.path.splitext(base_name_no_ext)[0]
now = datetime.now()
now_str = now.strftime("%Y%m%d%H%M%S")
file_name = base_name_no_ext + "_" + now_str
payment_items = []
for key, value in payments_dict.items():
pi = RewardLog.ExternalInstance(file_name, key, value)
pi.payment = pi.payment * MUTEZ
payment_items.append(pi)
logger.info(
"Reward created for cycle %s address %s amount %f fee %f tz type %s",
pi.cycle,
pi.address,
pi.payment,
pi.fee,
pi.type,
)
payments_queue.put(PaymentBatch(None, 0, payment_items))
payments_queue.put(PaymentBatch(None, 0, [RewardLog.ExitInstance()]))
def run(self):
# call first retry if not in onetime mode.
# retry_failed script is more suitable for one time cases.
if not self.run_mode == RunMode.ONETIME:
self.retry_producer.retry_failed_payments()
if self.run_mode == RunMode.RETRY_FAILED:
sleep(5)
self.exit()
return
# first retry is done by producer thread, start retry thread for further retries
if self.run_mode == RunMode.FOREVER:
self.retry_fail_thread.start()
try:
(
current_cycle,
current_level,
) = self.block_api.get_current_cycle_and_level()
except ApiProviderException as a:
logger.error(
"Unable to fetch current cycle, {:s}. Exiting.".format(str(a)))
self.exit()
return
# if initial_payment_cycle has the default value of -1 resulting in the last released cycle
if self.initial_payment_cycle == -1:
pymnt_cycle = (current_cycle -
(self.nw_config["NB_FREEZE_CYCLE"] + 1) -
self.release_override)
if pymnt_cycle < 0:
logger.error(
"Payment cycle cannot be < 0 but configuration results to {}"
.format(pymnt_cycle))
else:
logger.debug(
"Payment cycle is set to last released cycle {}".format(
pymnt_cycle))
else:
pymnt_cycle = self.initial_payment_cycle
get_verbose_log_helper().reset(pymnt_cycle)
while not self.exiting and self.life_cycle.is_running():
# take a breath
sleep(5)
try:
# Exit if disk is full
# https://github.com/tezos-reward-distributor-organization/tezos-reward-distributor/issues/504
if disk_is_full():
self.exit()
break
# Check if local node is bootstrapped; sleep if needed; restart loop
if not self.node_is_bootstrapped():
logger.info(
"Local node {} is not in sync with the Tezos network. Will sleep for {} blocks and check again."
.format(self.node_url, BOOTSTRAP_SLEEP))
self.wait_for_blocks(BOOTSTRAP_SLEEP)
continue
# Local node is ready
(
current_cycle,
current_level,
) = self.block_api.get_current_cycle_and_level()
level_in_cycle = self.block_api.level_in_cycle(current_level)
# create reports dir
if self.calculations_dir and not os.path.exists(
self.calculations_dir):
os.makedirs(self.calculations_dir)
logger.debug(
"Checking for pending payments: payment_cycle <= current_cycle - (self.nw_config['NB_FREEZE_CYCLE'] + 1) - self.release_override"
)
logger.info(
"Checking for pending payments: checking {} <= {} - ({} + 1) - {}"
.format(
pymnt_cycle,
current_cycle,
self.nw_config["NB_FREEZE_CYCLE"],
self.release_override,
))
# payments should not pass beyond last released reward cycle
if (pymnt_cycle <= current_cycle -
(self.nw_config["NB_FREEZE_CYCLE"] + 1) -
self.release_override):
if not self.payments_queue.full():
if (not self.pay_denunciation_rewards
) and self.reward_api.name == "RPC":
logger.info(
"Error: pay_denunciation_rewards=False requires an indexer since it is not possible to distinguish reward source using RPC"
)
e = "You must set 'pay_denunciation_rewards' to True when using RPC provider."
logger.error(e)
self.exit()
break
# Paying upcoming cycles (-R in [-6, -11] )
if pymnt_cycle >= current_cycle:
logger.warn(
"Please note that you are doing payouts for future rewards!!! These rewards are not earned yet, they are an estimation."
)
if not self.rewards_type.isEstimated():
logger.error(
"For future rewards payout, you must configure the payout type to 'Estimated', see documentation"
)
self.exit()
break
# Paying cycles with frozen rewards (-R in [-1, -5] )
elif (pymnt_cycle >= current_cycle -
self.nw_config["NB_FREEZE_CYCLE"]):
logger.warn(
"Please note that you are doing payouts for frozen rewards!!!"
)
# If user wants to offset payments within a cycle, check here
if level_in_cycle < self.payment_offset:
wait_offset_blocks = self.payment_offset - level_in_cycle
wait_offset_minutes = (
wait_offset_blocks *
self.nw_config["MINIMAL_BLOCK_DELAY"]) / 60
logger.info(
"Current level within the cycle is {}; Requested offset is {}; Waiting for {} more blocks (~{} minutes)"
.format(
level_in_cycle,
self.payment_offset,
wait_offset_blocks,
wait_offset_minutes,
))
self.wait_for_blocks(wait_offset_blocks)
continue # Break/Repeat loop
else:
result = self.try_to_pay(pymnt_cycle,
self.rewards_type,
self.nw_config)
if result:
# single run is done. Do not continue.
if self.run_mode == RunMode.ONETIME:
logger.info(
"Run mode ONETIME satisfied. Terminating..."
)
self.exit()
break
else:
pymnt_cycle = pymnt_cycle + 1
get_verbose_log_helper().reset(pymnt_cycle)
# end of queue size check
else:
logger.debug("Wait a few minutes, queue is full")
# wait a few minutes to let payments finish
sleep(60 * 3)
# end of payment cycle check
else:
logger.info(
"No pending payments for cycle {}, current cycle is {}"
.format(pymnt_cycle, current_cycle))
# pending payments done. Do not wait any more.
if self.run_mode == RunMode.PENDING:
logger.info(
"Run mode PENDING satisfied. Terminating...")
self.exit()
break
sleep(10)
# calculate number of blocks until end of current cycle plus user-defined offset
nb_blocks_remaining = (self.nw_config["BLOCKS_PER_CYCLE"] -
level_in_cycle +
self.payment_offset)
logger.debug(
"Waiting until next cycle; {} blocks remaining".format(
nb_blocks_remaining))
# wait until current cycle ends
self.wait_for_blocks(nb_blocks_remaining)
except (ApiProviderException, ReadTimeout, ConnectTimeout) as e:
logger.debug(
"{:s} error at payment producer loop: '{:s}'".format(
self.reward_api.name, str(e)),
exc_info=True,
)
logger.error(
"{:s} error at payment producer loop: '{:s}', will try again."
.format(self.reward_api.name, str(e)))
except Exception as e:
logger.debug(
"Unknown error in payment producer loop: {:s}".format(
str(e)),
exc_info=True,
)
logger.error(
"Unknown error in payment producer loop: {:s}, will try again."
.format(str(e)))
# end of endless loop
logger.debug("Producer returning...")
# ensure consumer exits
self.exit()
return