def test_disk_full_payment_producer(args, caplog):
# Issue: https://github.com/tezos-reward-distributor-organization/tezos-reward-distributor/issues/504
client_manager = ClientManager(args.node_endpoint, args.signer_endpoint)
network_config_map = init_network_config(args.network, client_manager)
factory = ProviderFactory(provider="prpc")
parser = BakingYamlConfParser(
baking_config, None, None, None, None, block_api=factory, api_base_url=None
)
parser.parse()
parser.process()
cfg_dict = parser.get_conf_obj()
baking_cfg = BakingConf(cfg_dict)
baking_dirs = BakingDirs(args, baking_cfg.get_baking_address())
srvc_fee_calc = ServiceFeeCalculator(
baking_cfg.get_full_supporters_set(),
baking_cfg.get_specials_map(),
baking_cfg.get_service_fee(),
)
payments_queue = queue.Queue(50)
plc = ProcessLifeCycle(None)
pp = PaymentProducer(
name="producer",
network_config=network_config_map[args.network],
payments_dir=baking_dirs.payments_root,
calculations_dir=baking_dirs.calculations_root,
run_mode=RunMode(args.run_mode),
service_fee_calc=srvc_fee_calc,
release_override=args.release_override,
payment_offset=args.payment_offset,
baking_cfg=baking_cfg,
life_cycle=plc,
payments_queue=payments_queue,
dry_run=args.dry_run,
client_manager=client_manager,
node_url=args.node_endpoint,
reward_data_provider=args.reward_data_provider,
node_url_public=args.node_addr_public,
api_base_url=args.api_base_url,
retry_injected=args.retry_injected,
initial_payment_cycle=args.initial_cycle,
)
assert disk_is_full()
try:
pp.daemon = True
pp.start()
finally:
pp.stop()
assert (
"Disk is becoming full. Only 0.50 Gb left from 10.00 Gb. Please clean up disk to continue saving logs and reports."
in caplog.text
)
def do_launch_producers(self, e):
PaymentProducer(name='producer',
initial_payment_cycle=self.args.initial_cycle,
network_config=self.__nw_config,
payments_dir=self.__baking_dirs.payments_root,
calculations_dir=self.__baking_dirs.calculations_root,
run_mode=RunMode(self.args.run_mode),
service_fee_calc=self.__srvc_fee_calc,
release_override=self.args.release_override,
payment_offset=self.args.payment_offset,
baking_cfg=self.__cfg,
life_cycle=self,
payments_queue=self.__payments_queue,
dry_run=self.args.dry_run,
client_manager=self.__node_client,
node_url=self.args.node_endpoint,
reward_data_provider=self.args.reward_data_provider,
node_url_public=self.args.node_addr_public,
api_base_url=self.args.api_base_url,
retry_injected=self.args.retry_injected).start()
def main(args):
logger.info("TRD version {} is running in {} mode.".format(
version.version,
"daemon" if args.background_service else "interactive"))
logger.info("Arguments Configuration = {}".format(
json.dumps(args.__dict__, indent=1)))
# 1- find where configuration is
config_dir = os.path.expanduser(args.config_dir)
# create configuration directory if it is not present
# so that user can easily put his configuration there
if config_dir and not os.path.exists(config_dir):
os.makedirs(config_dir)
# 2- Load master configuration file if it is present
master_config_file_path = os.path.join(config_dir, "master.yaml")
master_cfg = {}
if os.path.isfile(master_config_file_path):
logger.debug("Loading master configuration file {}".format(
master_config_file_path))
master_parser = YamlConfParser(
ConfigParser.load_file(master_config_file_path))
master_cfg = master_parser.parse()
else:
logger.debug("master configuration file not present.")
managers = None
contracts_by_alias = None
addresses_by_pkh = None
if 'managers' in master_cfg:
managers = master_cfg['managers']
if 'contracts_by_alias' in master_cfg:
contracts_by_alias = master_cfg['contracts_by_alias']
if 'addresses_by_pkh' in master_cfg:
addresses_by_pkh = master_cfg['addresses_by_pkh']
# 3- get client path
client_path = get_client_path(
[x.strip() for x in args.executable_dirs.split(',')], args.docker,
args.network, args.verbose)
logger.debug("Dune client path is {}".format(client_path))
# 4. get network config
config_client_manager = SimpleClientManager(client_path)
network_config_map = init_network_config(args.network,
config_client_manager,
args.node_addr)
network_config = network_config_map[args.network]
logger.debug("Network config {}".format(network_config))
# 5- load baking configuration file
config_file_path = get_baking_configuration_file(config_dir)
logger.info(
"Loading baking configuration file {}".format(config_file_path))
wllt_clnt_mngr = WalletClientManager(client_path,
contracts_by_alias,
addresses_by_pkh,
managers,
verbose=args.verbose)
provider_factory = ProviderFactory(args.reward_data_provider,
verbose=args.verbose)
parser = BakingYamlConfParser(ConfigParser.load_file(config_file_path),
wllt_clnt_mngr,
provider_factory,
network_config,
args.node_addr,
verbose=args.verbose)
parser.parse()
parser.validate()
parser.process()
cfg_dict = parser.get_conf_obj()
# dictionary to BakingConf object, for a bit of type safety
cfg = BakingConf(cfg_dict, master_cfg)
logger.info("Baking Configuration {}".format(cfg))
baking_address = cfg.get_baking_address()
payment_address = cfg.get_payment_address()
logger.info(LINER)
logger.info("BAKING ADDRESS is {}".format(baking_address))
logger.info("PAYMENT ADDRESS is {}".format(payment_address))
logger.info(LINER)
# 6- is it a reports run
dry_run = args.dry_run_no_consumers or args.dry_run
if args.dry_run_no_consumers:
global NB_CONSUMERS
NB_CONSUMERS = 0
# 7- get reporting directories
reports_base = os.path.expanduser(args.reports_base)
# if in reports run mode, do not create consumers
# create reports in reports directory
if dry_run:
reports_base = os.path.expanduser("./reports")
reports_dir = os.path.join(reports_base, baking_address)
payments_root = get_payment_root(reports_dir, create=True)
calculations_root = get_calculations_root(reports_dir, create=True)
get_successful_payments_dir(payments_root, create=True)
get_failed_payments_dir(payments_root, create=True)
# 8- start the life cycle
life_cycle.start(not dry_run)
# 9- service fee calculator
srvc_fee_calc = ServiceFeeCalculator(cfg.get_full_supporters_set(),
cfg.get_specials_map(),
cfg.get_service_fee())
if args.initial_cycle is None:
recent = get_latest_report_file(payments_root)
# if payment logs exists set initial cycle to following cycle
# if payment logs does not exists, set initial cycle to 0, so that payment starts from last released rewards
args.initial_cycle = 0 if recent is None else int(recent) + 1
logger.info("initial_cycle set to {}".format(args.initial_cycle))
p = PaymentProducer(name='producer',
initial_payment_cycle=args.initial_cycle,
network_config=network_config,
payments_dir=payments_root,
calculations_dir=calculations_root,
run_mode=RunMode(args.run_mode),
service_fee_calc=srvc_fee_calc,
release_override=args.release_override,
payment_offset=args.payment_offset,
baking_cfg=cfg,
life_cycle=life_cycle,
payments_queue=payments_queue,
dry_run=dry_run,
wllt_clnt_mngr=wllt_clnt_mngr,
node_url=args.node_addr,
provider_factory=provider_factory,
verbose=args.verbose)
p.start()
publish_stats = not args.do_not_publish_stats
for i in range(NB_CONSUMERS):
c = PaymentConsumer(
name='consumer' + str(i),
payments_dir=payments_root,
key_name=payment_address,
client_path=client_path,
payments_queue=payments_queue,
node_addr=args.node_addr,
wllt_clnt_mngr=wllt_clnt_mngr,
args=args,
verbose=args.verbose,
dry_run=dry_run,
delegator_pays_xfer_fee=cfg.get_delegator_pays_xfer_fee(),
dest_map=cfg.get_dest_map(),
network_config=network_config,
publish_stats=publish_stats)
time.sleep(1)
c.start()
logger.info("Application start completed")
logger.info(LINER)
try:
while life_cycle.is_running():
time.sleep(10)
except KeyboardInterrupt:
logger.info("Interrupted.")
life_cycle.stop()
def main(args):
logger.info("Arguments Configuration = {}".format(
json.dumps(args.__dict__, indent=1)))
# 1- find where configuration is
config_dir = os.path.expanduser(args.config_dir)
# create configuration directory if it is not present
# so that user can easily put his configuration there
if config_dir and not os.path.exists(config_dir):
os.makedirs(config_dir)
# 2- Load master configuration file if it is present
master_config_file_path = os.path.join(config_dir, "master.yaml")
master_cfg = {}
if os.path.isfile(master_config_file_path):
logger.info("Loading master configuration file {}".format(
master_config_file_path))
master_parser = YamlConfParser(
ConfigParser.load_file(master_config_file_path))
master_cfg = master_parser.parse()
else:
logger.info("master configuration file not present.")
managers = None
contracts_by_alias = None
addresses_by_pkh = None
if 'managers' in master_cfg:
managers = master_cfg['managers']
if 'contracts_by_alias' in master_cfg:
contracts_by_alias = master_cfg['contracts_by_alias']
if 'addresses_by_pkh' in master_cfg:
addresses_by_pkh = master_cfg['addresses_by_pkh']
# 3- get client path
client_path = get_client_path(
[x.strip() for x in args.executable_dirs.split(',')], args.docker,
args.network, args.verbose)
logger.debug("Tezos client path is {}".format(client_path))
# 4. get network config
config_client_manager = SimpleClientManager(client_path, args.node_addr)
network_config_map = init_network_config(args.network,
config_client_manager,
args.node_addr)
network_config = network_config_map[args.network]
# 5- load baking configuration file
config_file_path = get_baking_configuration_file(config_dir)
logger.info(
"Loading baking configuration file {}".format(config_file_path))
wllt_clnt_mngr = WalletClientManager(client_path,
contracts_by_alias,
addresses_by_pkh,
managers,
verbose=args.verbose)
provider_factory = ProviderFactory(args.reward_data_provider,
verbose=args.verbose)
parser = BakingYamlConfParser(ConfigParser.load_file(config_file_path),
wllt_clnt_mngr,
provider_factory,
network_config,
args.node_addr,
verbose=args.verbose,
api_base_url=args.api_base_url)
parser.parse()
parser.validate()
parser.process()
cfg_dict = parser.get_conf_obj()
# dictionary to BakingConf object, for a bit of type safety
cfg = BakingConf(cfg_dict, master_cfg)
logger.info("Baking Configuration {}".format(cfg))
baking_address = cfg.get_baking_address()
payment_address = cfg.get_payment_address()
logger.info(LINER)
logger.info("BAKING ADDRESS is {}".format(baking_address))
logger.info("PAYMENT ADDRESS is {}".format(payment_address))
logger.info(LINER)
# 6- is it a reports run
dry_run = args.dry_run_no_consumers or args.dry_run
if args.dry_run_no_consumers:
global NB_CONSUMERS
NB_CONSUMERS = 0
# 7- get reporting directories
reports_dir = os.path.expanduser(args.reports_base)
# if in reports run mode, do not create consumers
# create reports in reports directory
if dry_run:
reports_dir = os.path.expanduser("./reports")
reports_dir = os.path.join(reports_dir, baking_address)
payments_root = get_payment_root(reports_dir, create=True)
calculations_root = get_calculations_root(reports_dir, create=True)
get_successful_payments_dir(payments_root, create=True)
get_failed_payments_dir(payments_root, create=True)
# 8- start the life cycle
life_cycle.start(False)
# 9- service fee calculator
srvc_fee_calc = ServiceFeeCalculator(cfg.get_full_supporters_set(),
cfg.get_specials_map(),
cfg.get_service_fee())
try:
p = PaymentProducer(name='producer',
initial_payment_cycle=None,
network_config=network_config,
payments_dir=payments_root,
calculations_dir=calculations_root,
run_mode=RunMode.ONETIME,
service_fee_calc=srvc_fee_calc,
release_override=0,
payment_offset=0,
baking_cfg=cfg,
life_cycle=life_cycle,
payments_queue=payments_queue,
dry_run=dry_run,
wllt_clnt_mngr=wllt_clnt_mngr,
node_url=args.node_addr,
provider_factory=provider_factory,
verbose=args.verbose,
api_base_url=args.api_base_url)
p.retry_failed_payments(args.retry_injected)
c = PaymentConsumer(
name='consumer_retry_failed',
payments_dir=payments_root,
key_name=payment_address,
client_path=client_path,
payments_queue=payments_queue,
node_addr=args.node_addr,
wllt_clnt_mngr=wllt_clnt_mngr,
verbose=args.verbose,
dry_run=dry_run,
delegator_pays_xfer_fee=cfg.get_delegator_pays_xfer_fee(),
network_config=network_config)
time.sleep(1)
c.start()
p.exit()
c.join()
logger.info("Application start completed")
logger.info(LINER)
sleep(5)
except KeyboardInterrupt:
logger.info("Interrupted.")
def main(args):
logger.info("TRD version {} is running in {} mode.".format(
VERSION, "daemon" if args.background_service else "interactive"))
logger.info("Arguments Configuration = {}".format(
json.dumps(args.__dict__, indent=1)))
publish_stats = not args.do_not_publish_stats
logger.info(
"Anonymous statistics {} be collected. See docs/statistics.rst for more information."
.format("will" if publish_stats else "will not"))
# 1- find where configuration is
config_dir = os.path.expanduser(args.config_dir)
# create configuration directory if it is not present
# so that user can easily put his configuration there
if config_dir and not os.path.exists(config_dir):
os.makedirs(config_dir)
# 4. get network config
client_manager = ClientManager(node_endpoint=args.node_endpoint,
signer_endpoint=args.signer_endpoint)
network_config_map = init_network_config(args.network, client_manager)
network_config = network_config_map[args.network]
logger.debug("Network config {}".format(network_config))
# Setup provider to fetch RPCs
provider_factory = ProviderFactory(args.reward_data_provider)
# 5- load and verify baking configuration file
config_file_path = None
try:
config_file_path = get_baking_configuration_file(config_dir)
logger.info(
"Loading baking configuration file {}".format(config_file_path))
parser = BakingYamlConfParser(
yaml_text=ConfigParser.load_file(config_file_path),
clnt_mngr=client_manager,
provider_factory=provider_factory,
network_config=network_config,
node_url=args.node_endpoint,
api_base_url=args.api_base_url)
parser.parse()
parser.validate()
parser.process()
# dictionary to BakingConf object, for a bit of type safety
cfg_dict = parser.get_conf_obj()
cfg = BakingConf(cfg_dict)
except ConfigurationException as e:
logger.info(
"Unable to parse '{}' config file.".format(config_file_path))
logger.info(e)
sys.exit(1)
logger.info("Baking Configuration {}".format(cfg))
baking_address = cfg.get_baking_address()
payment_address = cfg.get_payment_address()
logger.info(LINER)
logger.info("BAKING ADDRESS is {}".format(baking_address))
logger.info("PAYMENT ADDRESS is {}".format(payment_address))
logger.info(LINER)
# 6- is it a reports run
dry_run = args.dry_run_no_consumers or args.dry_run
if args.dry_run_no_consumers:
global NB_CONSUMERS
NB_CONSUMERS = 0
# 7- get reporting directories
reports_base = os.path.expanduser(args.reports_base)
# if in reports run mode, do not create consumers
# create reports in reports directory
if dry_run:
reports_base = os.path.expanduser("./reports")
reports_dir = os.path.join(reports_base, baking_address)
payments_root = get_payment_root(reports_dir, create=True)
calculations_root = get_calculations_root(reports_dir, create=True)
get_successful_payments_dir(payments_root, create=True)
get_failed_payments_dir(payments_root, create=True)
# 8- start the life cycle
life_cycle.start(not dry_run)
# 9- service fee calculator
srvc_fee_calc = ServiceFeeCalculator(cfg.get_full_supporters_set(),
cfg.get_specials_map(),
cfg.get_service_fee())
if args.initial_cycle is None:
recent = get_latest_report_file(payments_root)
# if payment logs exists set initial cycle to following cycle
# if payment logs does not exists, set initial cycle to 0, so that payment starts from last released rewards
args.initial_cycle = 0 if recent is None else int(recent) + 1
logger.info("initial_cycle set to {}".format(args.initial_cycle))
# 10- load plugins
plugins_manager = plugins.PluginManager(cfg.get_plugins_conf(), dry_run)
# 11- Start producer and consumer
p = PaymentProducer(name='producer',
initial_payment_cycle=args.initial_cycle,
network_config=network_config,
payments_dir=payments_root,
calculations_dir=calculations_root,
run_mode=RunMode(args.run_mode),
service_fee_calc=srvc_fee_calc,
release_override=args.release_override,
payment_offset=args.payment_offset,
baking_cfg=cfg,
life_cycle=life_cycle,
payments_queue=payments_queue,
dry_run=dry_run,
client_manager=client_manager,
node_url=args.node_endpoint,
provider_factory=provider_factory,
node_url_public=args.node_addr_public,
api_base_url=args.api_base_url,
retry_injected=args.retry_injected)
p.start()
for i in range(NB_CONSUMERS):
c = PaymentConsumer(
name='consumer' + str(i),
payments_dir=payments_root,
key_name=payment_address,
payments_queue=payments_queue,
node_addr=args.node_endpoint,
client_manager=client_manager,
plugins_manager=plugins_manager,
rewards_type=cfg.get_rewards_type(),
args=args,
dry_run=dry_run,
reactivate_zeroed=cfg.get_reactivate_zeroed(),
delegator_pays_ra_fee=cfg.get_delegator_pays_ra_fee(),
delegator_pays_xfer_fee=cfg.get_delegator_pays_xfer_fee(),
dest_map=cfg.get_dest_map(),
network_config=network_config,
publish_stats=publish_stats)
sleep(1)
c.start()
logger.info("Application start completed")
logger.info(LINER)
# Run forever
try:
while life_cycle.is_running():
sleep(10)
except KeyboardInterrupt:
logger.info("Interrupted.")
life_cycle.stop()
