def do_build_parser(self, e):
provider_factory = ProviderFactory(self.args.reward_data_provider)
self.__parser = BakingYamlConfParser(
yaml_text=self.__config_text,
clnt_mngr=self.__node_client,
provider_factory=provider_factory,
network_config=self.__nw_cfg,
node_url=self.args.node_endpoint,
api_base_url=self.args.api_base_url)
def load_config_file(wllt_clnt_mngr, network_config, master_cfg):
provider_factory = ProviderFactory(args.reward_data_provider)
parser = BakingYamlConfParser(None,
wllt_clnt_mngr,
provider_factory,
network_config,
args.node_addr,
api_base_url=args.api_base_url)
parser.parse()
parser.validate()
parser.process()
cfg_dict = parser.get_conf_obj()
# dictionary to BakingConf object, for a bit of type safety
cfg = BakingConf(cfg_dict, master_cfg)
logger.info("Baking Configuration {}".format(cfg))
baking_address = cfg.get_baking_address()
payment_address = cfg.get_payment_address()
logger.info(LINER)
logger.info("BAKING ADDRESS is {}".format(baking_address))
logger.info("PAYMENT ADDRESS is {}".format(payment_address))
logger.info(LINER)
# 7- get reporting directories
reports_dir = os.path.expanduser(args.reports_base)
reports_dir = os.path.join(reports_dir, baking_address)
payments_root = get_payment_root(reports_dir, create=True)
get_successful_payments_dir(payments_root, create=True)
get_failed_payments_dir(payments_root, create=True)
def test_validate_plugins(self):
data = """
baking_address: tz1g8vkmcde6sWKaG2NN9WKzCkDM6Rziq194
plugins:
enabled:
- plug1
- plug2
"""
block_api = RpcBlockApiImpl(network, self.mainnet_public_node_url)
cnf_prsr = BakingYamlConfParser(
data,
clnt_mngr=None,
provider_factory=None,
network_config=None,
node_url="",
block_api=block_api,
)
cnf_prsr.parse()
cnf_prsr.validate_plugins(cnf_prsr.get_conf_obj())
plugins = cnf_prsr.get_conf_obj_attr("plugins")
self.assertIsInstance(plugins, dict)
self.assertIsInstance(plugins["enabled"], list)
self.assertEqual(len(plugins["enabled"]), 2)
def onbakingaddress(input):
try:
AddressValidator("bakingaddress").validate(input)
except Exception as e:
printe("Invalid baking address: " + traceback.format_exc())
return
if not input.startswith("tz"):
printe("Only tz addresses are allowed")
return
provider_factory = ProviderFactory(args.reward_data_provider)
global parser
parser = BakingYamlConfParser(None, wllt_clnt_mngr, provider_factory, network_config, args.node_addr)
parser.set(BAKING_ADDRESS, input)
messages['paymentaddress']=messages['paymentaddress'].format([v['alias'] for k,v in wllt_clnt_mngr.get_addr_dict().items() if v['sk']])+" (without quotes)"
fsm.go()
def onbakingaddress(input):
try:
AddressValidator("baking address").validate(input)
except Exception as e:
printe(f"Invalid baking address: {str(e)}")
return
if not input.startswith("tz"):
printe("Only tz addresses are allowed")
return
provider_factory = ProviderFactory(args.reward_data_provider)
global parser
parser = BakingYamlConfParser(None, client_manager, provider_factory, network_config, args.node_endpoint,
api_base_url=args.api_base_url)
parser.set(BAKING_ADDRESS, input)
fsm.go()
def test_address_is_baker_address(block_api):
data_fine = """
version: 1.0
baking_address: {0}
""".format(Constants.STAKENOW_ADDRESS)
wallet_client_manager = ClientManager(node_endpoint, PRIVATE_SIGNER_URL)
cnf_prsr = BakingYamlConfParser(
data_fine,
wallet_client_manager,
provider_factory=None,
network_config=network,
node_url=node_endpoint,
block_api=block_api,
)
cnf_prsr.parse()
assert cnf_prsr.validate_baking_address(cnf_prsr.conf_obj) is None
def test_disk_full_payment_producer(args, caplog):
# Issue: https://github.com/tezos-reward-distributor-organization/tezos-reward-distributor/issues/504
client_manager = ClientManager(args.node_endpoint, args.signer_endpoint)
network_config_map = init_network_config(args.network, client_manager)
factory = ProviderFactory(provider="prpc")
parser = BakingYamlConfParser(
baking_config, None, None, None, None, block_api=factory, api_base_url=None
)
parser.parse()
parser.process()
cfg_dict = parser.get_conf_obj()
baking_cfg = BakingConf(cfg_dict)
baking_dirs = BakingDirs(args, baking_cfg.get_baking_address())
srvc_fee_calc = ServiceFeeCalculator(
baking_cfg.get_full_supporters_set(),
baking_cfg.get_specials_map(),
baking_cfg.get_service_fee(),
)
payments_queue = queue.Queue(50)
plc = ProcessLifeCycle(None)
pp = PaymentProducer(
name="producer",
network_config=network_config_map[args.network],
payments_dir=baking_dirs.payments_root,
calculations_dir=baking_dirs.calculations_root,
run_mode=RunMode(args.run_mode),
service_fee_calc=srvc_fee_calc,
release_override=args.release_override,
payment_offset=args.payment_offset,
baking_cfg=baking_cfg,
life_cycle=plc,
payments_queue=payments_queue,
dry_run=args.dry_run,
client_manager=client_manager,
node_url=args.node_endpoint,
reward_data_provider=args.reward_data_provider,
node_url_public=args.node_addr_public,
api_base_url=args.api_base_url,
retry_injected=args.retry_injected,
initial_payment_cycle=args.initial_cycle,
)
assert disk_is_full()
try:
pp.daemon = True
pp.start()
finally:
pp.stop()
assert (
"Disk is becoming full. Only 0.50 Gb left from 10.00 Gb. Please clean up disk to continue saving logs and reports."
in caplog.text
)
def test_disk_full_payment_consumer(args, caplog):
# Issue: https://github.com/tezos-reward-distributor-organization/tezos-reward-distributor/issues/504
client_manager = ClientManager(args.node_endpoint, args.signer_endpoint)
network_config_map = init_network_config(args.network, client_manager)
factory = ProviderFactory(provider="prpc")
parser = BakingYamlConfParser(
baking_config, None, None, None, None, block_api=factory, api_base_url=None
)
parser.parse()
parser.process()
cfg_dict = parser.get_conf_obj()
baking_cfg = BakingConf(cfg_dict)
baking_dirs = BakingDirs(args, baking_cfg.get_baking_address())
payments_queue = queue.Queue(50)
plugins_manager = plugins.PluginManager(baking_cfg.get_plugins_conf(), args.dry_run)
pc = PaymentConsumer(
name="consumer0",
payments_dir=baking_dirs.payments_root,
key_name=baking_cfg.get_payment_address(),
payments_queue=payments_queue,
node_addr=args.node_endpoint,
client_manager=client_manager,
plugins_manager=plugins_manager,
rewards_type=baking_cfg.get_rewards_type(),
args=args,
dry_run=args.dry_run,
reactivate_zeroed=baking_cfg.get_reactivate_zeroed(),
delegator_pays_ra_fee=baking_cfg.get_delegator_pays_ra_fee(),
delegator_pays_xfer_fee=baking_cfg.get_delegator_pays_xfer_fee(),
dest_map=baking_cfg.get_dest_map(),
network_config=network_config_map[args.network],
publish_stats=not args.do_not_publish_stats,
)
assert disk_is_full()
try:
pc.daemon = True
pc.start()
finally:
pc.stop()
assert (
"Disk is becoming full. Only 0.30 Gb left from 11.00 Gb. Please clean up disk to continue saving logs and reports."
in caplog.text
)
def test_validate_plugins(self):
data = """
baking_address: tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj
plugins:
enabled:
"""
block_api = RpcBlockApiImpl(network, self.mainnet_public_node_url)
cnf_prsr = BakingYamlConfParser(data,
wllt_clnt_mngr=None,
provider_factory=None,
network_config=None,
node_url="",
block_api=block_api)
cnf_prsr.parse()
cnf_prsr.validate_plugins(cnf_prsr.get_conf_obj())
def test_validate_plugins(self):
data = """
baking_address: tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj
plugins:
enabled:
- plug1
- plug2
"""
block_api = RpcBlockApiImpl(network, self.mainnet_public_node_url)
cnf_prsr = BakingYamlConfParser(data,
clnt_mngr=None,
provider_factory=None,
network_config=None,
node_url="",
block_api=block_api)
cnf_prsr.parse()
cnf_prsr.validate_plugins(cnf_prsr.get_conf_obj())
plugins = cnf_prsr.get_conf_obj_attr('plugins')
self.assertIsInstance(plugins, dict)
self.assertIsInstance(plugins['enabled'], list)
self.assertEqual(len(plugins['enabled']), 2)
def test_invalid_baking_address(block_api):
data_fine = """
version: 1.0
baking_address: tz123
"""
wallet_client_manager = ClientManager(node_endpoint, PRIVATE_SIGNER_URL)
cnf_prsr = BakingYamlConfParser(
data_fine,
wallet_client_manager,
provider_factory=None,
network_config=network,
node_url=node_endpoint,
block_api=block_api,
)
cnf_prsr.parse()
with pytest.raises(ConfigurationException,
match="Baking address length must be 36"):
cnf_prsr.validate_baking_address(cnf_prsr.conf_obj)
def test_address_is_not_baker_address(block_api):
data_fine = """
version: 1.0
baking_address: tz1N4UfQCahHkRShBanv9QP9TnmXNgCaqCyZ
"""
wallet_client_manager = ClientManager(node_endpoint, PRIVATE_SIGNER_URL)
cnf_prsr = BakingYamlConfParser(
data_fine,
wallet_client_manager,
provider_factory=None,
network_config=network,
node_url=node_endpoint,
block_api=block_api,
)
cnf_prsr.parse()
with pytest.raises(
ConfigurationException,
match=
"Baking address tz1N4UfQCahHkRShBanv9QP9TnmXNgCaqCyZ is not enabled for delegation",
):
cnf_prsr.validate_baking_address(cnf_prsr.conf_obj)
def main(args):
logger.info("Arguments Configuration = {}".format(
json.dumps(args.__dict__, indent=1)))
# 1- find where configuration is
config_dir = os.path.expanduser(args.config_dir)
# create configuration directory if it is not present
# so that user can easily put his configuration there
if config_dir and not os.path.exists(config_dir):
os.makedirs(config_dir)
# 2- Load master configuration file if it is present
master_config_file_path = os.path.join(config_dir, "master.yaml")
master_cfg = {}
if os.path.isfile(master_config_file_path):
logger.info("Loading master configuration file {}".format(
master_config_file_path))
master_parser = YamlConfParser(
ConfigParser.load_file(master_config_file_path))
master_cfg = master_parser.parse()
else:
logger.info("master configuration file not present.")
managers = None
contracts_by_alias = None
addresses_by_pkh = None
if 'managers' in master_cfg:
managers = master_cfg['managers']
if 'contracts_by_alias' in master_cfg:
contracts_by_alias = master_cfg['contracts_by_alias']
if 'addresses_by_pkh' in master_cfg:
addresses_by_pkh = master_cfg['addresses_by_pkh']
# 3- get client path
client_path = get_client_path(
[x.strip() for x in args.executable_dirs.split(',')], args.docker,
args.network, args.verbose)
logger.debug("Tezos client path is {}".format(client_path))
# 4. get network config
config_client_manager = SimpleClientManager(client_path, args.node_addr)
network_config_map = init_network_config(args.network,
config_client_manager,
args.node_addr)
network_config = network_config_map[args.network]
# 5- load baking configuration file
config_file_path = get_baking_configuration_file(config_dir)
logger.info(
"Loading baking configuration file {}".format(config_file_path))
wllt_clnt_mngr = WalletClientManager(client_path,
contracts_by_alias,
addresses_by_pkh,
managers,
verbose=args.verbose)
provider_factory = ProviderFactory(args.reward_data_provider,
verbose=args.verbose)
parser = BakingYamlConfParser(ConfigParser.load_file(config_file_path),
wllt_clnt_mngr,
provider_factory,
network_config,
args.node_addr,
verbose=args.verbose,
api_base_url=args.api_base_url)
parser.parse()
parser.validate()
parser.process()
cfg_dict = parser.get_conf_obj()
# dictionary to BakingConf object, for a bit of type safety
cfg = BakingConf(cfg_dict, master_cfg)
logger.info("Baking Configuration {}".format(cfg))
baking_address = cfg.get_baking_address()
payment_address = cfg.get_payment_address()
logger.info(LINER)
logger.info("BAKING ADDRESS is {}".format(baking_address))
logger.info("PAYMENT ADDRESS is {}".format(payment_address))
logger.info(LINER)
# 6- is it a reports run
dry_run = args.dry_run_no_consumers or args.dry_run
if args.dry_run_no_consumers:
global NB_CONSUMERS
NB_CONSUMERS = 0
# 7- get reporting directories
reports_dir = os.path.expanduser(args.reports_base)
# if in reports run mode, do not create consumers
# create reports in reports directory
if dry_run:
reports_dir = os.path.expanduser("./reports")
reports_dir = os.path.join(reports_dir, baking_address)
payments_root = get_payment_root(reports_dir, create=True)
calculations_root = get_calculations_root(reports_dir, create=True)
get_successful_payments_dir(payments_root, create=True)
get_failed_payments_dir(payments_root, create=True)
# 8- start the life cycle
life_cycle.start(False)
# 9- service fee calculator
srvc_fee_calc = ServiceFeeCalculator(cfg.get_full_supporters_set(),
cfg.get_specials_map(),
cfg.get_service_fee())
try:
p = PaymentProducer(name='producer',
initial_payment_cycle=None,
network_config=network_config,
payments_dir=payments_root,
calculations_dir=calculations_root,
run_mode=RunMode.ONETIME,
service_fee_calc=srvc_fee_calc,
release_override=0,
payment_offset=0,
baking_cfg=cfg,
life_cycle=life_cycle,
payments_queue=payments_queue,
dry_run=dry_run,
wllt_clnt_mngr=wllt_clnt_mngr,
node_url=args.node_addr,
provider_factory=provider_factory,
verbose=args.verbose,
api_base_url=args.api_base_url)
p.retry_failed_payments(args.retry_injected)
c = PaymentConsumer(
name='consumer_retry_failed',
payments_dir=payments_root,
key_name=payment_address,
client_path=client_path,
payments_queue=payments_queue,
node_addr=args.node_addr,
wllt_clnt_mngr=wllt_clnt_mngr,
verbose=args.verbose,
dry_run=dry_run,
delegator_pays_xfer_fee=cfg.get_delegator_pays_xfer_fee(),
network_config=network_config)
time.sleep(1)
c.start()
p.exit()
c.join()
logger.info("Application start completed")
logger.info(LINER)
sleep(5)
except KeyboardInterrupt:
logger.info("Interrupted.")
def test_validate_no_founders_map(self):
data_no_founders = """
version: 1.0
baking_address: tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj
payment_address: tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj
owners_map: {'KT2Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj': 0.5,'KT3Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj': 0.5}
supporters_set: None
service_fee: 4.5
reactivate_zeroed: False
delegator_pays_ra_fee: True
plugins:
enabled:
"""
managers_map = {
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj':
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj'
}
contr_dict_by_alias = {}
addr_dict_by_pkh = {
"tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj": {
"pkh": "tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj",
"originated": False,
"alias": "main1",
"sk": True,
"manager": "tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj"
}
}
wallet_client_manager = WalletClientManager(
client_path=None,
node_addr=None,
addr_dict_by_pkh=addr_dict_by_pkh,
contr_dict_by_alias=contr_dict_by_alias,
managers=managers_map)
block_api = RpcBlockApiImpl(network, self.mainnet_public_node_url)
cnf_prsr = BakingYamlConfParser(data_no_founders,
wallet_client_manager,
provider_factory=None,
network_config=network,
node_url=self.mainnet_public_node_url,
block_api=block_api)
cnf_prsr.parse()
cnf_prsr.validate()
self.assertEqual(cnf_prsr.get_conf_obj_attr('baking_address'),
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj')
self.assertEqual(cnf_prsr.get_conf_obj_attr('payment_address'),
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj')
self.assertEqual(cnf_prsr.get_conf_obj_attr('__payment_address_pkh'),
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj')
self.assertEqual(
cnf_prsr.get_conf_obj_attr('__payment_address_manager'),
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj')
self.assertEqual(cnf_prsr.get_conf_obj_attr('__payment_address_type'),
AddrType.TZ)
self.assertEqual(cnf_prsr.get_conf_obj_attr('founders_map'), dict())
self.assertEqual(cnf_prsr.get_conf_obj_attr('specials_map'), dict())
self.assertEqual(cnf_prsr.get_conf_obj_attr('supporters_set'), set())
self.assertEqual(cnf_prsr.get_conf_obj_attr('min_delegation_amt'), 0)
self.assertEqual(cnf_prsr.get_conf_obj_attr('reactivate_zeroed'),
False)
self.assertEqual(cnf_prsr.get_conf_obj_attr('delegator_pays_ra_fee'),
True)
def test_validate(self):
data_fine = """
version : 1.0
baking_address : tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj
payment_address : tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj
founders_map : {'KT2Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj':0.5,'KT3Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj':0.5}
owners_map : {'KT2Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj':0.5,'KT3Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj':0.5}
service_fee : 4.53
"""
managers = {
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj':
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj',
'KT1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj':
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj'
}
contr_dict_by_alias = {}
addr_dict_by_pkh = {
"tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj": {
"pkh": "tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj",
"originated": False,
"alias": "main1",
"sk": True,
"manager": "tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj"
}
}
wallet_client_manager = WalletClientManager(
client_path=None,
addr_dict_by_pkh=addr_dict_by_pkh,
contr_dict_by_alias=contr_dict_by_alias,
managers=managers)
cnf_prsr = BakingYamlConfParser(data_fine, wallet_client_manager)
cnf_prsr.parse()
cnf_prsr.validate()
self.assertEqual(cnf_prsr.get_conf_obj_attr('baking_address'),
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj')
self.assertEqual(cnf_prsr.get_conf_obj_attr('payment_address'),
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj')
self.assertEqual(cnf_prsr.get_conf_obj_attr('payment_address_pkh'),
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj')
self.assertEqual(cnf_prsr.get_conf_obj_attr('payment_address_manager'),
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj')
self.assertEqual(cnf_prsr.get_conf_obj_attr('payment_address_type'),
AddrType.TZ)
self.assertEqual(0, cnf_prsr.get_conf_obj_attr('min_delegation_amt'))
self.assertEqual(cnf_prsr.get_conf_obj_attr('excluded_delegators_set'),
set())
self.assertEqual(cnf_prsr.get_conf_obj_attr('prcnt_scale'), None)
self.assertEqual(cnf_prsr.get_conf_obj_attr('pymnt_scale'), None)
def test_validate_no_founders_map(self):
data_no_founders = """
version: 1.0
baking_address: tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj
payment_address: tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj
owners_map: {'KT2Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj': 0.5,'KT3Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj': 0.5}
supporters_set: None
service_fee: 4.5
reactivate_zeroed: False
delegator_pays_ra_fee: True
rewards_type:
plugins:
enabled:
"""
wallet_client_manager = ClientManager('', '')
block_api = RpcBlockApiImpl(network, self.mainnet_public_node_url)
cnf_prsr = BakingYamlConfParser(data_no_founders,
wallet_client_manager,
provider_factory=None,
network_config=network,
node_url=self.mainnet_public_node_url,
block_api=block_api)
cnf_prsr.parse()
cnf_prsr.validate()
self.assertEqual(cnf_prsr.get_conf_obj_attr('baking_address'),
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj')
self.assertEqual(cnf_prsr.get_conf_obj_attr('payment_address'),
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj')
self.assertEqual(cnf_prsr.get_conf_obj_attr('__payment_address_pkh'),
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj')
self.assertEqual(
cnf_prsr.get_conf_obj_attr('__payment_address_manager'),
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj')
self.assertEqual(cnf_prsr.get_conf_obj_attr('__payment_address_type'),
AddrType.TZ)
self.assertEqual(cnf_prsr.get_conf_obj_attr('founders_map'), dict())
self.assertEqual(cnf_prsr.get_conf_obj_attr('specials_map'), dict())
self.assertEqual(cnf_prsr.get_conf_obj_attr('supporters_set'), set())
self.assertEqual(cnf_prsr.get_conf_obj_attr('min_delegation_amt'), 0)
self.assertEqual(cnf_prsr.get_conf_obj_attr('reactivate_zeroed'),
False)
self.assertEqual(cnf_prsr.get_conf_obj_attr('delegator_pays_ra_fee'),
True)
self.assertEqual(cnf_prsr.get_conf_obj_attr('rewards_type'),
RewardsType.ACTUAL)
plugins = cnf_prsr.get_conf_obj_attr('plugins')
self.assertIsInstance(plugins, dict)
self.assertIsNone(plugins['enabled'], None)
def test_validate_pymnt_alias(self):
data_no_founders = """
version : 1.0
baking_address : tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj
payment_address : tzPay
owners_map : {'KT2Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj':0.5,'KT3Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj':0.5}
service_fee : 4.5
min_delegation_amt : 100
"""
managers_map = {
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj':
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj',
'KT1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj':
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj'
}
contr_dict_by_alias = {
'kt': 'KT1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj',
'tzPay': 'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj'
}
addr_dict_by_pkh = {
"tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj": {
"pkh": "tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj",
"originated": False,
"alias": "tz1",
"sk": True,
"manager": "tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj"
},
"KT1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj": {
"pkh": "tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj",
"originated": False,
"alias": "ktPay",
"sk": True,
"revealed": True,
"manager": "tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj"
}
}
wallet_client_manager = WalletClientManager(
client_path=None,
node_addr=None,
addr_dict_by_pkh=addr_dict_by_pkh,
contr_dict_by_alias=contr_dict_by_alias,
managers=managers_map)
block_api = RpcBlockApiImpl(network, mainnet_public_node_url)
cnf_prsr = BakingYamlConfParser(data_no_founders,
wallet_client_manager,
provider_factory=None,
network_config=network,
node_url=mainnet_public_node_url,
block_api=block_api)
cnf_prsr.parse()
cnf_prsr.validate()
self.assertEqual(cnf_prsr.get_conf_obj_attr('baking_address'),
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj')
self.assertEqual(cnf_prsr.get_conf_obj_attr('payment_address'),
'tzPay')
self.assertEqual(cnf_prsr.get_conf_obj_attr('__payment_address_pkh'),
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj')
self.assertEqual(
cnf_prsr.get_conf_obj_attr('__payment_address_manager'),
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj')
self.assertEqual(cnf_prsr.get_conf_obj_attr('__payment_address_type'),
AddrType.TZALS)
self.assertEqual(cnf_prsr.get_conf_obj_attr('founders_map'), dict())
self.assertEqual(cnf_prsr.get_conf_obj_attr('specials_map'), dict())
self.assertEqual(cnf_prsr.get_conf_obj_attr('supporters_set'), set())
self.assertEqual(100, cnf_prsr.get_conf_obj_attr('min_delegation_amt'))
def main(args):
logger.info("TRD version {} is running in {} mode.".format(
version.version,
"daemon" if args.background_service else "interactive"))
logger.info("Arguments Configuration = {}".format(
json.dumps(args.__dict__, indent=1)))
# 1- find where configuration is
config_dir = os.path.expanduser(args.config_dir)
# create configuration directory if it is not present
# so that user can easily put his configuration there
if config_dir and not os.path.exists(config_dir):
os.makedirs(config_dir)
# 2- Load master configuration file if it is present
master_config_file_path = os.path.join(config_dir, "master.yaml")
master_cfg = {}
if os.path.isfile(master_config_file_path):
logger.debug("Loading master configuration file {}".format(
master_config_file_path))
master_parser = YamlConfParser(
ConfigParser.load_file(master_config_file_path))
master_cfg = master_parser.parse()
else:
logger.debug("master configuration file not present.")
managers = None
contracts_by_alias = None
addresses_by_pkh = None
if 'managers' in master_cfg:
managers = master_cfg['managers']
if 'contracts_by_alias' in master_cfg:
contracts_by_alias = master_cfg['contracts_by_alias']
if 'addresses_by_pkh' in master_cfg:
addresses_by_pkh = master_cfg['addresses_by_pkh']
# 3- get client path
client_path = get_client_path(
[x.strip() for x in args.executable_dirs.split(',')], args.docker,
args.network, args.verbose)
logger.debug("Dune client path is {}".format(client_path))
# 4. get network config
config_client_manager = SimpleClientManager(client_path)
network_config_map = init_network_config(args.network,
config_client_manager,
args.node_addr)
network_config = network_config_map[args.network]
logger.debug("Network config {}".format(network_config))
# 5- load baking configuration file
config_file_path = get_baking_configuration_file(config_dir)
logger.info(
"Loading baking configuration file {}".format(config_file_path))
wllt_clnt_mngr = WalletClientManager(client_path,
contracts_by_alias,
addresses_by_pkh,
managers,
verbose=args.verbose)
provider_factory = ProviderFactory(args.reward_data_provider,
verbose=args.verbose)
parser = BakingYamlConfParser(ConfigParser.load_file(config_file_path),
wllt_clnt_mngr,
provider_factory,
network_config,
args.node_addr,
verbose=args.verbose)
parser.parse()
parser.validate()
parser.process()
cfg_dict = parser.get_conf_obj()
# dictionary to BakingConf object, for a bit of type safety
cfg = BakingConf(cfg_dict, master_cfg)
logger.info("Baking Configuration {}".format(cfg))
baking_address = cfg.get_baking_address()
payment_address = cfg.get_payment_address()
logger.info(LINER)
logger.info("BAKING ADDRESS is {}".format(baking_address))
logger.info("PAYMENT ADDRESS is {}".format(payment_address))
logger.info(LINER)
# 6- is it a reports run
dry_run = args.dry_run_no_consumers or args.dry_run
if args.dry_run_no_consumers:
global NB_CONSUMERS
NB_CONSUMERS = 0
# 7- get reporting directories
reports_base = os.path.expanduser(args.reports_base)
# if in reports run mode, do not create consumers
# create reports in reports directory
if dry_run:
reports_base = os.path.expanduser("./reports")
reports_dir = os.path.join(reports_base, baking_address)
payments_root = get_payment_root(reports_dir, create=True)
calculations_root = get_calculations_root(reports_dir, create=True)
get_successful_payments_dir(payments_root, create=True)
get_failed_payments_dir(payments_root, create=True)
# 8- start the life cycle
life_cycle.start(not dry_run)
# 9- service fee calculator
srvc_fee_calc = ServiceFeeCalculator(cfg.get_full_supporters_set(),
cfg.get_specials_map(),
cfg.get_service_fee())
if args.initial_cycle is None:
recent = get_latest_report_file(payments_root)
# if payment logs exists set initial cycle to following cycle
# if payment logs does not exists, set initial cycle to 0, so that payment starts from last released rewards
args.initial_cycle = 0 if recent is None else int(recent) + 1
logger.info("initial_cycle set to {}".format(args.initial_cycle))
p = PaymentProducer(name='producer',
initial_payment_cycle=args.initial_cycle,
network_config=network_config,
payments_dir=payments_root,
calculations_dir=calculations_root,
run_mode=RunMode(args.run_mode),
service_fee_calc=srvc_fee_calc,
release_override=args.release_override,
payment_offset=args.payment_offset,
baking_cfg=cfg,
life_cycle=life_cycle,
payments_queue=payments_queue,
dry_run=dry_run,
wllt_clnt_mngr=wllt_clnt_mngr,
node_url=args.node_addr,
provider_factory=provider_factory,
verbose=args.verbose)
p.start()
publish_stats = not args.do_not_publish_stats
for i in range(NB_CONSUMERS):
c = PaymentConsumer(
name='consumer' + str(i),
payments_dir=payments_root,
key_name=payment_address,
client_path=client_path,
payments_queue=payments_queue,
node_addr=args.node_addr,
wllt_clnt_mngr=wllt_clnt_mngr,
args=args,
verbose=args.verbose,
dry_run=dry_run,
delegator_pays_xfer_fee=cfg.get_delegator_pays_xfer_fee(),
dest_map=cfg.get_dest_map(),
network_config=network_config,
publish_stats=publish_stats)
time.sleep(1)
c.start()
logger.info("Application start completed")
logger.info(LINER)
try:
while life_cycle.is_running():
time.sleep(10)
except KeyboardInterrupt:
logger.info("Interrupted.")
life_cycle.stop()
class ConfigLifeCycle:
def __init__(self, args, nw_cfg, node_client, callback):
self.__args = args
self.__nw_cfg = nw_cfg
self.__node_client = node_client
self.__config_text = None
self.__parser = None
self.__callback = callback
self.fsm = self.get_fsm_builder().build()
def get_fsm_builder(self):
fsm_builder = TransitionsFsmBuilder()
fsm_builder.add_initial_state(
ConfigState.INITIAL,
on_leave=lambda e: logger.debug(
"Loading baking configuration file ..."),
)
fsm_builder.add_state(ConfigState.READ,
on_enter=self.do_read_configuration_file)
fsm_builder.add_state(ConfigState.BUILT, on_enter=self.do_build_parser)
fsm_builder.add_state(ConfigState.PARSED, on_enter=self.do_parse_cfg)
fsm_builder.add_state(ConfigState.VALIDATED,
on_enter=self.do_validate_cfg)
fsm_builder.add_state(ConfigState.PROCESSED,
on_enter=self.do_process_cfg)
fsm_builder.add_final_state(
ConfigState.COMPLETED,
on_enter=lambda e: self.__callback(self.get_conf()))
fsm_builder.add_transition(ConfigEvent.READ, ConfigState.INITIAL,
ConfigState.READ)
fsm_builder.add_transition(ConfigEvent.BUILD, ConfigState.READ,
ConfigState.BUILT)
fsm_builder.add_transition(ConfigEvent.PARSE, ConfigState.BUILT,
ConfigState.PARSED)
fsm_builder.add_transition(ConfigEvent.VALIDATE, ConfigState.PARSED,
ConfigState.VALIDATED)
fsm_builder.add_transition(ConfigEvent.PROCESS, ConfigState.VALIDATED,
ConfigState.PROCESSED)
fsm_builder.add_transition(ConfigEvent.COMPLETE, ConfigState.PROCESSED,
ConfigState.COMPLETED)
return fsm_builder
def start(self):
self.fsm.trigger_event(ConfigEvent.READ)
self.fsm.trigger_event(ConfigEvent.BUILD)
self.fsm.trigger_event(ConfigEvent.PARSE)
self.fsm.trigger_event(ConfigEvent.VALIDATE)
self.fsm.trigger_event(ConfigEvent.PROCESS)
self.fsm.trigger_event(ConfigEvent.COMPLETE)
def do_read_configuration_file(self, e):
config_dir = os.path.join(
os.path.expanduser(os.path.normpath(self.args.base_directory)),
CONFIG_DIR,
"",
)
# create configuration directory if it is not present
# so that user can easily put his configuration there
if config_dir and not os.path.exists(config_dir):
logger.debug("Creating path '{}'".format(config_dir))
os.makedirs(config_dir)
config_file_path = self.get_baking_cfg_file(config_dir)
logger.info(
"Loading baking configuration file {}".format(config_file_path))
self.__config_text = ConfigParser.load_file(config_file_path)
def do_build_parser(self, e):
provider_factory = ProviderFactory(self.args.reward_data_provider)
self.__parser = BakingYamlConfParser(
yaml_text=self.__config_text,
clnt_mngr=self.__node_client,
provider_factory=provider_factory,
network_config=self.__nw_cfg,
node_url=self.args.node_endpoint,
api_base_url=self.args.api_base_url,
)
def do_parse_cfg(self, e):
self.__parser.parse()
def do_validate_cfg(self, e):
self.__parser.validate()
def do_process_cfg(self, e):
self.__parser.process()
def get_conf(self):
cfg_dict = self.__parser.get_conf_obj()
return BakingConf(cfg_dict)
@property
def args(self):
return self.__args
@staticmethod
def get_baking_cfg_file(cfg_dir):
cfg_file = None
for file in os.listdir(cfg_dir):
if file.endswith(".yaml") and not file.startswith("master"):
if cfg_file:
raise Exception(
"Application only supports one baking configuration file. Found at least 2 {}, {}"
.format(cfg_file, file))
cfg_file = file
if cfg_file is None:
raise Exception(
"Unable to find any '.yaml' configuration files inside configuration directory({})"
.format(cfg_dir))
return os.path.join(cfg_dir, cfg_file)
def test_batch_payer_total_payout_amount():
factory = ProviderFactory(provider="prpc")
parser = BakingYamlConfParser(
baking_config, None, None, None, None, block_api=factory, api_base_url=None
)
parser.parse()
parser.process()
cfg_dict = parser.get_conf_obj()
baking_cfg = BakingConf(cfg_dict)
srvc_fee_calc = ServiceFeeCalculator(
baking_cfg.get_full_supporters_set(),
baking_cfg.get_specials_map(),
baking_cfg.get_service_fee(),
)
rules_model = RulesModel(
baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map(),
)
payment_calc = PhasedPaymentCalculator(
baking_cfg.get_founders_map(),
baking_cfg.get_owners_map(),
srvc_fee_calc,
int(baking_cfg.get_min_delegation_amount() * MUTEZ_PER_TEZ),
rules_model,
)
rewardApi = factory.newRewardApi(
default_network_config_map[CURRENT_TESTNET], baking_cfg.get_baking_address(), ""
)
# Simulate logic in payment_producer
reward_logs = []
attempts = 0
exiting = False
while not exiting and attempts < 2:
attempts += 1
# Reward data
# Fetch cycle 51 of granadanet for tz1gtHbmBF3TSebsgJfJPvUB2e9x8EDeNm6V
reward_model = rewardApi.get_rewards_for_cycle_map(
PAYOUT_CYCLE, RewardsType.ACTUAL
)
# Calculate rewards - payment_producer.py
reward_model.computed_reward_amount = reward_model.total_reward_amount
reward_logs, total_amount = payment_calc.calculate(reward_model)
# Check total reward amount matches sums of records
# diff of 1 expected due to floating point arithmetic
assert (
total_amount - sum([rl.adjusted_amount for rl in reward_logs if rl.payable])
<= 1
)
exiting = True
# Merge payments to same address
phaseMerge = CalculatePhaseMerge()
reward_logs = phaseMerge.calculate(reward_logs)
# Handle remapping of payment to alternate address
phaseMapping = CalculatePhaseMapping()
reward_logs = phaseMapping.calculate(reward_logs, baking_cfg.get_dest_map())
# Filter zero-balance addresses based on config
phaseZeroBalance = CalculatePhaseZeroBalance()
reward_logs = phaseZeroBalance.calculate(
reward_logs, baking_cfg.get_reactivate_zeroed()
)
# Filter out non-payable items
reward_logs = [payment_item for payment_item in reward_logs if payment_item.payable]
reward_logs.sort(key=cmp_to_key(cmp_by_type_balance))
batch_payer = BatchPayer(
node_url=node_endpoint,
pymnt_addr="tz1N4UfQCahHkRShBanv9QP9TnmXNgCaqCyZ",
clnt_mngr=ClientManager(node_endpoint, PRIVATE_SIGNER_URL),
delegator_pays_ra_fee=True,
delegator_pays_xfer_fee=True,
network_config=network,
plugins_manager=PluginManager(baking_cfg.get_plugins_conf(), dry_run=True),
dry_run=True,
)
# Do the payment
(
_,
total_attempts,
total_payout_amount,
number_future_payable_cycles,
) = batch_payer.pay(reward_logs, dry_run=True)
# Payment does not have status done, paid or injected thus the total payout amount is zero
assert total_payout_amount == 0
assert number_future_payable_cycles == 2
assert total_attempts == 3
# Check the adjusted amount
assert reward_logs[0].adjusted_amount == 40418486
assert reward_logs[1].adjusted_amount == 10581272
assert reward_logs[2].adjusted_amount == 109732835
assert reward_logs[3].adjusted_amount == 48362127
assert reward_logs[4].adjusted_amount == 29116310
def test_validate_no_founders_map(self):
data_no_founders = """
version : 1.0
baking_address : tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj
payment_address : tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj
owners_map : {'KT2Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj':0.5,'KT3Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj':0.5}
service_fee : 4.5
"""
managers_map = {
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj':
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj'
}
contr_dict_by_alias = {}
addr_dict_by_pkh = {
"tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj": {
"pkh": "tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj",
"originated": False,
"alias": "main1",
"sk": True,
"manager": "tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj"
}
}
wallet_client_manager = WalletClientManager(
client_path=None,
addr_dict_by_pkh=addr_dict_by_pkh,
contr_dict_by_alias=contr_dict_by_alias,
managers=managers_map)
cnf_prsr = BakingYamlConfParser(data_no_founders,
wallet_client_manager,
network_config=network)
cnf_prsr.parse()
cnf_prsr.validate()
self.assertEqual(cnf_prsr.get_conf_obj_attr('baking_address'),
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj')
self.assertEqual(cnf_prsr.get_conf_obj_attr('payment_address'),
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj')
self.assertEqual(cnf_prsr.get_conf_obj_attr('payment_address_pkh'),
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj')
self.assertEqual(cnf_prsr.get_conf_obj_attr('payment_address_manager'),
'tz1Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj')
self.assertEqual(cnf_prsr.get_conf_obj_attr('payment_address_type'),
AddrType.TZ)
self.assertEqual(cnf_prsr.get_conf_obj_attr('founders_map'), dict())
self.assertEqual(cnf_prsr.get_conf_obj_attr('specials_map'), dict())
self.assertEqual(cnf_prsr.get_conf_obj_attr('supporters_set'), set())
self.assertEqual(0, cnf_prsr.get_conf_obj_attr('min_delegation_amt'))
def test_validate_no_founders_map(self):
data_no_founders = """
version : 1.0
baking_address : dn1VWnJRnF3vsa3tkYp3PqoCbV1eBRc5vJ2x
payment_address : dn1VWnJRnF3vsa3tkYp3PqoCbV1eBRc5vJ2x
owners_map : {'KT2Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj':0.5,'KT3Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj':0.5}
service_fee : 4.5
"""
managers_map = {
'dn1VWnJRnF3vsa3tkYp3PqoCbV1eBRc5vJ2x':
'dn1VWnJRnF3vsa3tkYp3PqoCbV1eBRc5vJ2x'
}
contr_dict_by_alias = {}
addr_dict_by_pkh = {
"dn1VWnJRnF3vsa3tkYp3PqoCbV1eBRc5vJ2x": {
"pkh": "dn1VWnJRnF3vsa3tkYp3PqoCbV1eBRc5vJ2x",
"originated": False,
"alias": "main1",
"sk": True,
"manager": "dn1VWnJRnF3vsa3tkYp3PqoCbV1eBRc5vJ2x"
}
}
wallet_client_manager = WalletClientManager(
client_path=None,
addr_dict_by_pkh=addr_dict_by_pkh,
contr_dict_by_alias=contr_dict_by_alias,
managers=managers_map)
mirror_selector = DunScanMirrorSelector(network)
mirror_selector.initialize()
block_api = DunScanBlockApiImpl(network, mirror_selector)
cnf_prsr = BakingYamlConfParser(data_no_founders,
wallet_client_manager,
provider_factory=None,
network_config=network,
node_url=mainnet_public_node_url,
block_api=block_api)
cnf_prsr.parse()
cnf_prsr.validate()
self.assertEqual(cnf_prsr.get_conf_obj_attr('baking_address'),
'dn1VWnJRnF3vsa3tkYp3PqoCbV1eBRc5vJ2x')
self.assertEqual(cnf_prsr.get_conf_obj_attr('payment_address'),
'dn1VWnJRnF3vsa3tkYp3PqoCbV1eBRc5vJ2x')
self.assertEqual(cnf_prsr.get_conf_obj_attr('__payment_address_pkh'),
'dn1VWnJRnF3vsa3tkYp3PqoCbV1eBRc5vJ2x')
self.assertEqual(
cnf_prsr.get_conf_obj_attr('__payment_address_manager'),
'dn1VWnJRnF3vsa3tkYp3PqoCbV1eBRc5vJ2x')
self.assertEqual(cnf_prsr.get_conf_obj_attr('__payment_address_type'),
AddrType.DN)
self.assertEqual(cnf_prsr.get_conf_obj_attr('founders_map'), dict())
self.assertEqual(cnf_prsr.get_conf_obj_attr('specials_map'), dict())
self.assertEqual(cnf_prsr.get_conf_obj_attr('supporters_set'), set())
self.assertEqual(0, cnf_prsr.get_conf_obj_attr('min_delegation_amt'))
def main(args):
logger.info("TRD version {} is running in {} mode.".format(
VERSION, "daemon" if args.background_service else "interactive"))
logger.info("Arguments Configuration = {}".format(
json.dumps(args.__dict__, indent=1)))
publish_stats = not args.do_not_publish_stats
logger.info(
"Anonymous statistics {} be collected. See docs/statistics.rst for more information."
.format("will" if publish_stats else "will not"))
# 1- find where configuration is
config_dir = os.path.expanduser(args.config_dir)
# create configuration directory if it is not present
# so that user can easily put his configuration there
if config_dir and not os.path.exists(config_dir):
os.makedirs(config_dir)
# 4. get network config
client_manager = ClientManager(node_endpoint=args.node_endpoint,
signer_endpoint=args.signer_endpoint)
network_config_map = init_network_config(args.network, client_manager)
network_config = network_config_map[args.network]
logger.debug("Network config {}".format(network_config))
# Setup provider to fetch RPCs
provider_factory = ProviderFactory(args.reward_data_provider)
# 5- load and verify baking configuration file
config_file_path = None
try:
config_file_path = get_baking_configuration_file(config_dir)
logger.info(
"Loading baking configuration file {}".format(config_file_path))
parser = BakingYamlConfParser(
yaml_text=ConfigParser.load_file(config_file_path),
clnt_mngr=client_manager,
provider_factory=provider_factory,
network_config=network_config,
node_url=args.node_endpoint,
api_base_url=args.api_base_url)
parser.parse()
parser.validate()
parser.process()
# dictionary to BakingConf object, for a bit of type safety
cfg_dict = parser.get_conf_obj()
cfg = BakingConf(cfg_dict)
except ConfigurationException as e:
logger.info(
"Unable to parse '{}' config file.".format(config_file_path))
logger.info(e)
sys.exit(1)
logger.info("Baking Configuration {}".format(cfg))
baking_address = cfg.get_baking_address()
payment_address = cfg.get_payment_address()
logger.info(LINER)
logger.info("BAKING ADDRESS is {}".format(baking_address))
logger.info("PAYMENT ADDRESS is {}".format(payment_address))
logger.info(LINER)
# 6- is it a reports run
dry_run = args.dry_run_no_consumers or args.dry_run
if args.dry_run_no_consumers:
global NB_CONSUMERS
NB_CONSUMERS = 0
# 7- get reporting directories
reports_base = os.path.expanduser(args.reports_base)
# if in reports run mode, do not create consumers
# create reports in reports directory
if dry_run:
reports_base = os.path.expanduser("./reports")
reports_dir = os.path.join(reports_base, baking_address)
payments_root = get_payment_root(reports_dir, create=True)
calculations_root = get_calculations_root(reports_dir, create=True)
get_successful_payments_dir(payments_root, create=True)
get_failed_payments_dir(payments_root, create=True)
# 8- start the life cycle
life_cycle.start(not dry_run)
# 9- service fee calculator
srvc_fee_calc = ServiceFeeCalculator(cfg.get_full_supporters_set(),
cfg.get_specials_map(),
cfg.get_service_fee())
if args.initial_cycle is None:
recent = get_latest_report_file(payments_root)
# if payment logs exists set initial cycle to following cycle
# if payment logs does not exists, set initial cycle to 0, so that payment starts from last released rewards
args.initial_cycle = 0 if recent is None else int(recent) + 1
logger.info("initial_cycle set to {}".format(args.initial_cycle))
# 10- load plugins
plugins_manager = plugins.PluginManager(cfg.get_plugins_conf(), dry_run)
# 11- Start producer and consumer
p = PaymentProducer(name='producer',
initial_payment_cycle=args.initial_cycle,
network_config=network_config,
payments_dir=payments_root,
calculations_dir=calculations_root,
run_mode=RunMode(args.run_mode),
service_fee_calc=srvc_fee_calc,
release_override=args.release_override,
payment_offset=args.payment_offset,
baking_cfg=cfg,
life_cycle=life_cycle,
payments_queue=payments_queue,
dry_run=dry_run,
client_manager=client_manager,
node_url=args.node_endpoint,
provider_factory=provider_factory,
node_url_public=args.node_addr_public,
api_base_url=args.api_base_url,
retry_injected=args.retry_injected)
p.start()
for i in range(NB_CONSUMERS):
c = PaymentConsumer(
name='consumer' + str(i),
payments_dir=payments_root,
key_name=payment_address,
payments_queue=payments_queue,
node_addr=args.node_endpoint,
client_manager=client_manager,
plugins_manager=plugins_manager,
rewards_type=cfg.get_rewards_type(),
args=args,
dry_run=dry_run,
reactivate_zeroed=cfg.get_reactivate_zeroed(),
delegator_pays_ra_fee=cfg.get_delegator_pays_ra_fee(),
delegator_pays_xfer_fee=cfg.get_delegator_pays_xfer_fee(),
dest_map=cfg.get_dest_map(),
network_config=network_config,
publish_stats=publish_stats)
sleep(1)
c.start()
logger.info("Application start completed")
logger.info(LINER)
# Run forever
try:
while life_cycle.is_running():
sleep(10)
except KeyboardInterrupt:
logger.info("Interrupted.")
life_cycle.stop()
def test_batch_payer_total_payout_amount():
factory = ProviderFactory(provider="prpc")
parser = BakingYamlConfParser(baking_config,
None,
None,
None,
None,
block_api=factory,
api_base_url=None)
parser.parse()
parser.process()
cfg_dict = parser.get_conf_obj()
baking_cfg = BakingConf(cfg_dict)
srvc_fee_calc = ServiceFeeCalculator(
baking_cfg.get_full_supporters_set(),
baking_cfg.get_specials_map(),
baking_cfg.get_service_fee(),
)
rules_model = RulesModel(
baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map(),
)
payment_calc = PhasedPaymentCalculator(
baking_cfg.get_founders_map(),
baking_cfg.get_owners_map(),
srvc_fee_calc,
baking_cfg.get_min_delegation_amount() * MUTEZ,
rules_model,
)
rewardApi = factory.newRewardApi(
default_network_config_map[CURRENT_TESTNET],
baking_cfg.get_baking_address(), "")
# Simulate logic in payment_producer
reward_logs = []
attempts = 0
exiting = False
while not exiting and attempts < 2:
attempts += 1
# Reward data
# Fetch cycle 51 of granadanet for tz1gtHbmBF3TSebsgJfJPvUB2e9x8EDeNm6V
reward_model = rewardApi.get_rewards_for_cycle_map(
PAYOUT_CYCLE, RewardsType.ACTUAL)
# Calculate rewards - payment_producer.py
reward_model.computed_reward_amount = reward_model.total_reward_amount
reward_logs, total_amount = payment_calc.calculate(reward_model)
# Check total reward amount matches sums of records
assert total_amount == sum(
[rl.amount for rl in reward_logs if rl.payable])
exiting = True
# Merge payments to same address
phaseMerge = CalculatePhaseMerge()
reward_logs = phaseMerge.calculate(reward_logs)
# Handle remapping of payment to alternate address
phaseMapping = CalculatePhaseMapping()
reward_logs = phaseMapping.calculate(reward_logs,
baking_cfg.get_dest_map())
# Filter zero-balance addresses based on config
phaseZeroBalance = CalculatePhaseZeroBalance()
reward_logs = phaseZeroBalance.calculate(
reward_logs, baking_cfg.get_reactivate_zeroed())
# Filter out non-payable items
reward_logs = [
payment_item for payment_item in reward_logs if payment_item.payable
]
reward_logs.sort(key=cmp_to_key(cmp_by_type_balance))
batch_payer = BatchPayer(
node_url=node_endpoint,
pymnt_addr="tz1gtHbmBF3TSebsgJfJPvUB2e9x8EDeNm6V",
clnt_mngr=ClientManager(node_endpoint, PRIVATE_SIGNER_URL),
delegator_pays_ra_fee=True,
delegator_pays_xfer_fee=True,
network_config=network,
plugins_manager=PluginManager(baking_cfg.get_plugins_conf(),
dry_run=True),
dry_run=True,
)
# Fix the endpoint auto port assignment because
# https://mainnet-tezos.giganode.io:8732 cannot be reached
batch_payer.clnt_mngr.node_endpoint = node_endpoint
# Do the payment
(
payment_logs,
total_attempts,
total_payout_amount,
number_future_payable_cycles,
) = batch_payer.pay(reward_logs, dry_run=True)
assert total_attempts == 3
assert total_payout_amount == 238211030
assert (PAYMENT_ADDRESS_BALANCE //
total_payout_amount) - 1 == number_future_payable_cycles
def test_process_payouts(self):
logger.debug("") # Console formatting
factory = ProviderFactory(provider='prpc')
parser = BakingYamlConfParser(self.baking_config,
None,
None,
None,
None,
block_api=factory,
api_base_url=None)
parser.parse()
parser.process()
cfg_dict = parser.get_conf_obj()
baking_cfg = BakingConf(cfg_dict)
srvc_fee_calc = ServiceFeeCalculator(
baking_cfg.get_full_supporters_set(),
baking_cfg.get_specials_map(), baking_cfg.get_service_fee())
rules_model = RulesModel(baking_cfg.get_excluded_set_tob(),
baking_cfg.get_excluded_set_toe(),
baking_cfg.get_excluded_set_tof(),
baking_cfg.get_dest_map())
payment_calc = PhasedPaymentCalculator(
baking_cfg.get_founders_map(), baking_cfg.get_owners_map(),
srvc_fee_calc,
baking_cfg.get_min_delegation_amount() * MUTEZ, rules_model)
rewardApi = factory.newRewardApi(
default_network_config_map[CURRENT_TESTNET],
baking_cfg.get_baking_address(), "")
# Simulate logic in payment_producer
reward_logs = []
attempts = 0
exiting = False
while not exiting and attempts < 2:
attempts += 1
try:
# Reward data
# Fetch cycle 90 of delphinet for tz1gtHbmBF3TSebsgJfJPvUB2e9x8EDeNm6V
reward_model = rewardApi.get_rewards_for_cycle_map(
PAYOUT_CYCLE)
# Calculate rewards - payment_producer.py
reward_logs, total_amount = payment_calc.calculate(
reward_model)
# Check total reward amount matches sums of records
self.assertTrue(
total_amount,
sum([rl.amount for rl in reward_logs if rl.payable]))
exiting = True
except ApiProviderException as e:
logger.error(
"{:s} error at payment producer loop: '{:s}', will try again."
.format("RPC", str(e)))
sleep(5)
#
# The next 3 phases happen in payment_consumer.py
#
# Merge payments to same address
phaseMerge = CalculatePhaseMerge()
reward_logs = phaseMerge.calculate(reward_logs)
# Handle remapping of payment to alternate address
phaseMapping = CalculatePhaseMapping()
reward_logs = phaseMapping.calculate(reward_logs,
baking_cfg.get_dest_map())
# Filter zero-balance addresses based on config
phaseZeroBalance = CalculatePhaseZeroBalance()
reward_logs = phaseZeroBalance.calculate(
reward_logs, baking_cfg.get_reactivate_zeroed())
# Filter out non-payable items
reward_logs = [pi for pi in reward_logs if pi.payable]
reward_logs.sort(key=cmp_to_key(cmp_by_type_balance))
# TRD Calculated Results
# tz1V9SpwXaGFiYdDfGJtWjA61EumAH3DwSyT type: D, stake bal: 62657.83, cur bal: 62657.83, ratio: 0.327420, fee_ratio: 0.000000, amount: 0.000000, fee_amount: 0.000000, fee_rate: 0.00, payable: N, skipped: Y, at-phase: 1, desc: Excluded by configuration, pay_addr: tz1V9SpwXaGFiYdDfGJtWjA61EumAH3DwSyT
# tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace type: D, stake bal: 55646.70, cur bal: 55646.70, ratio: 0.432340, fee_ratio: 0.000000, amount: 102.988160, fee_amount: 0.000000, fee_rate: 0.00, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace
# tz1T5woJN3r7SV5v2HGDyA5kurhbD9Y8ZKHZ type: D, stake bal: 25689.88, cur bal: 25689.88, ratio: 0.179635, fee_ratio: 0.019959, amount: 42.791010, fee_amount: 4.754557, fee_rate: 0.10, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1T5woJN3r7SV5v2HGDyA5kurhbD9Y8ZKHZ
# tz1fgX6oRWQb4HYHUT6eRjW8diNFrqjEfgq7 type: D, stake bal: 24916.33, cur bal: 24916.33, ratio: 0.193584, fee_ratio: 0.000000, amount: 46.113902, fee_amount: 0.000000, fee_rate: 0.00, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1fgX6oRWQb4HYHUT6eRjW8diNFrqjEfgq7
# tz1RRzfechTs3gWdM58y6xLeByta3JWaPqwP type: D, stake bal: 6725.43, cur bal: 6725.43, ratio: 0.047027, fee_ratio: 0.005225, amount: 11.202382, fee_amount: 1.244709, fee_rate: 0.10, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1RMmSzPSWPSSaKU193Voh4PosWSZx1C7Hs
# tz1L1XQWKxG38wk1Ain1foGaEZj8zeposcbk type: D, stake bal: 981.64, cur bal: 981.64, ratio: 0.007627, fee_ratio: 0.000000, amount: 1.816762, fee_amount: 0.000000, fee_rate: 0.00, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1L1XQWKxG38wk1Ain1foGaEZj8zeposcbk
# tz1L1XQWKxG38wk1Ain1foGaEZj8zeposcbk type: O, stake bal: 14750.53, cur bal: 0.00, ratio: 0.114602, fee_ratio: 0.000000, amount: 27.299548, fee_amount: 0.000000, fee_rate: 0.00, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1L1XQWKxG38wk1Ain1foGaEZj8zeposcbk
# tz1fgX6oRWQb4HYHUT6eRjW8diNFrqjEfgq7 type: F, stake bal: 0.00, cur bal: 0.00, ratio: 0.006296, fee_ratio: 0.000000, amount: 1.499816, fee_amount: 0.000000, fee_rate: 0.00, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1fgX6oRWQb4HYHUT6eRjW8diNFrqjEfgq7
# tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace type: F, stake bal: 0.00, cur bal: 0.00, ratio: 0.018889, fee_ratio: 0.000000, amount: 4.499450, fee_amount: 0.000000, fee_rate: 0.00, payable: Y, skipped: N, at-phase: 0, desc: , pay_addr: tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace
# Final records before creating transactions
# These values are known to be correct
cr = {}
cr["tz1T5woJN3r7SV5v2HGDyA5kurhbD9Y8ZKHZ"] = {
"type": "D",
"amount": 42791010,
"pay_addr": "tz1T5woJN3r7SV5v2HGDyA5kurhbD9Y8ZKHZ"
}
cr["tz1RRzfechTs3gWdM58y6xLeByta3JWaPqwP"] = {
"type": "D",
"amount": 11202382,
"pay_addr": "tz1RMmSzPSWPSSaKU193Voh4PosWSZx1C7Hs"
}
cr["tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace"] = {
"type": "M",
"amount": 107487610,
"pay_addr": "tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace"
}
cr["tz1fgX6oRWQb4HYHUT6eRjW8diNFrqjEfgq7"] = {
"type": "M",
"amount": 47613718,
"pay_addr": "tz1fgX6oRWQb4HYHUT6eRjW8diNFrqjEfgq7"
}
cr["tz1L1XQWKxG38wk1Ain1foGaEZj8zeposcbk"] = {
"type": "M",
"amount": 29116310,
"pay_addr": "tz1L1XQWKxG38wk1Ain1foGaEZj8zeposcbk"
}
# Verify that TRD calculated matches known values
for r in reward_logs:
# We know this address should be skipped
if r.address == "tz1V9SpwXaGFiYdDfGJtWjA61EumAH3DwSyT":
self.assertEqual(r.skipped, 1)
self.assertEqual(r.amount, 0)
continue
# All others we can compare normally
cmp = cr[r.address]
self.assertEqual(r.type, cmp["type"])
self.assertEqual(r.amount, (cmp["amount"]))
self.assertEqual(r.paymentaddress, cmp["pay_addr"])
def test_validate_no_founders_map(self):
data_no_founders = """
version: 1.0
baking_address: tz1g8vkmcde6sWKaG2NN9WKzCkDM6Rziq194
payment_address: tz1g8vkmcde6sWKaG2NN9WKzCkDM6Rziq194
owners_map: {'KT2Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj': 0.5,'KT3Z1tMai15JWUWeN2PKL9faXXVPMuWamzJj': 0.5}
supporters_set: None
service_fee: 4.5
reactivate_zeroed: False
delegator_pays_ra_fee: True
rewards_type:
plugins:
enabled:
"""
wallet_client_manager = ClientManager(self.mainnet_public_node_url,
self.signer_endpoint)
block_api = RpcBlockApiImpl(network, self.mainnet_public_node_url)
cnf_prsr = BakingYamlConfParser(
data_no_founders,
wallet_client_manager,
provider_factory=None,
network_config=network,
node_url=self.mainnet_public_node_url,
block_api=block_api,
)
cnf_prsr.parse()
cnf_prsr.validate()
self.assertEqual(
cnf_prsr.get_conf_obj_attr("baking_address"),
"tz1g8vkmcde6sWKaG2NN9WKzCkDM6Rziq194",
)
self.assertEqual(
cnf_prsr.get_conf_obj_attr("payment_address"),
"tz1g8vkmcde6sWKaG2NN9WKzCkDM6Rziq194",
)
self.assertEqual(
cnf_prsr.get_conf_obj_attr("__payment_address_pkh"),
"tz1g8vkmcde6sWKaG2NN9WKzCkDM6Rziq194",
)
self.assertEqual(
cnf_prsr.get_conf_obj_attr("__payment_address_manager"),
"tz1g8vkmcde6sWKaG2NN9WKzCkDM6Rziq194",
)
self.assertEqual(cnf_prsr.get_conf_obj_attr("__payment_address_type"),
AddrType.TZ)
self.assertEqual(cnf_prsr.get_conf_obj_attr("founders_map"), dict())
self.assertEqual(cnf_prsr.get_conf_obj_attr("specials_map"), dict())
self.assertEqual(cnf_prsr.get_conf_obj_attr("supporters_set"), set())
self.assertEqual(cnf_prsr.get_conf_obj_attr("min_delegation_amt"), 0)
self.assertEqual(cnf_prsr.get_conf_obj_attr("reactivate_zeroed"),
False)
self.assertEqual(cnf_prsr.get_conf_obj_attr("delegator_pays_ra_fee"),
True)
self.assertEqual(cnf_prsr.get_conf_obj_attr("rewards_type"),
RewardsType.ACTUAL)
plugins = cnf_prsr.get_conf_obj_attr("plugins")
self.assertIsInstance(plugins, dict)
self.assertIsNone(plugins["enabled"], None)
