def load_config_file(wllt_clnt_mngr, network_config, master_cfg):
provider_factory = ProviderFactory(args.reward_data_provider)
parser = BakingYamlConfParser(None,
wllt_clnt_mngr,
provider_factory,
network_config,
args.node_addr,
api_base_url=args.api_base_url)
parser.parse()
parser.validate()
parser.process()
cfg_dict = parser.get_conf_obj()
# dictionary to BakingConf object, for a bit of type safety
cfg = BakingConf(cfg_dict, master_cfg)
logger.info("Baking Configuration {}".format(cfg))
baking_address = cfg.get_baking_address()
payment_address = cfg.get_payment_address()
logger.info(LINER)
logger.info("BAKING ADDRESS is {}".format(baking_address))
logger.info("PAYMENT ADDRESS is {}".format(payment_address))
logger.info(LINER)
# 7- get reporting directories
reports_dir = os.path.expanduser(args.reports_base)
reports_dir = os.path.join(reports_dir, baking_address)
payments_root = get_payment_root(reports_dir, create=True)
get_successful_payments_dir(payments_root, create=True)
get_failed_payments_dir(payments_root, create=True)
def __init__(self, args, baking_address) -> None:
super().__init__()
# Get reporting directories
if args.dry_run:
base_directory = os.path.join(
os.path.expanduser(os.path.normpath(args.base_directory)),
SIMULATIONS_DIR,
"",
)
else:
base_directory = os.path.join(
os.path.expanduser(os.path.normpath(args.base_directory)),
REPORTS_DIR,
"",
)
self.reports_dir = os.path.join(base_directory, baking_address, "")
self.payments_root = get_payment_root(self.reports_dir, create=True)
self.calculations_root = get_calculations_root(self.reports_dir,
create=True)
self.successful_payments_dir = get_successful_payments_dir(
self.payments_root, create=True)
self.failed_payments_dir = get_failed_payments_dir(self.payments_root,
create=True)
def __init__(self, args, baking_address) -> None:
super().__init__()
# 7- get reporting directories
reports_base = os.path.expanduser(args.reports_base)
# if in reports run mode, do not create consumers
# create reports in reports directory
if args.dry_run:
reports_base = os.path.expanduser("./reports")
self.reports_dir = os.path.join(reports_base, baking_address)
self.payments_root = get_payment_root(self.reports_dir, create=True)
self.calculations_root = get_calculations_root(self.reports_dir,
create=True)
self.successful_payments_dir = get_successful_payments_dir(
self.payments_root, create=True)
self.failed_payments_dir = get_failed_payments_dir(self.payments_root,
create=True)
def main(args):
logger.info(
"Arguments Configuration = {}".format(json.dumps(args.__dict__, indent=1))
)
# Load payments file
payments_file = os.path.expanduser(os.path.normpath(args.payments_file))
if not os.path.isfile(payments_file):
raise Exception("payments_file ({}) does not exist.".format(payments_file))
with open(payments_file, "r") as file:
payment_lines = file.readlines()
payments_dict = {}
for line in payment_lines:
pkh, amt = line.split(":")
pkh = pkh.strip()
amt = float(amt.strip())
payments_dict[pkh] = amt
if not payments_dict:
raise Exception("No payments to process")
# Check if dry-run
dry_run = args.dry_run
# Get reporting directories
reports_dir = os.path.expanduser(os.path.normpath(args.base_directory))
# Check the disk size at the reports dir location
if disk_is_full(reports_dir):
raise Exception(
"Disk is full at {}. Please free space to continue saving reports.".format(
reports_dir
)
)
# if in reports run mode, do not create consumers
# create reports in reports directory
if dry_run:
reports_dir = os.path.join(reports_dir, SIMULATIONS_DIR, "")
else:
reports_dir = os.path.join(reports_dir, REPORTS_DIR, "")
reports_dir = os.path.join(reports_dir, "manual", "")
payments_root = get_payment_root(reports_dir, create=True)
get_successful_payments_dir(payments_root, create=True)
get_failed_payments_dir(payments_root, create=True)
client_manager = ClientManager(
node_endpoint=args.node_endpoint, signer_endpoint=args.signer_endpoint
)
for i in range(NB_CONSUMERS):
c = PaymentConsumer(
name="manual_payment_consumer",
payments_dir=payments_root,
key_name=args.paymentaddress,
payments_queue=payments_queue,
node_addr=args.node_endpoint,
client_manager=client_manager,
dry_run=dry_run,
reactivate_zeroed=False,
delegator_pays_ra_fee=False,
delegator_pays_xfer_fee=False,
)
time.sleep(1)
c.start()
base_name_no_ext = os.path.basename(payments_file)
base_name_no_ext = os.path.splitext(base_name_no_ext)[0]
now = datetime.now()
now_str = now.strftime("%Y%m%d%H%M%S")
file_name = base_name_no_ext + "_" + now_str
payment_items = []
for key, value in payments_dict.items():
pi = RewardLog.ExternalInstance(file_name, key, value)
pi.payment = pi.payment * MUTEZ
payment_items.append(pi)
logger.info(
"Reward created for cycle %s address %s amount %f fee %f tz type %s",
pi.cycle,
pi.address,
pi.payment,
pi.fee,
pi.type,
)
payments_queue.put(PaymentBatch(None, 0, payment_items))
payments_queue.put(PaymentBatch(None, 0, [RewardLog.ExitInstance()]))
def main(args):
logger.info("Arguments Configuration = {}".format(
json.dumps(args.__dict__, indent=1)))
# 1- find where configuration is
config_dir = os.path.expanduser(args.config_dir)
# create configuration directory if it is not present
# so that user can easily put his configuration there
if config_dir and not os.path.exists(config_dir):
os.makedirs(config_dir)
# 3- load payments file
payments_file = os.path.expanduser(args.payments_file)
if not os.path.isfile(payments_file):
raise Exception(
"payments_file ({}) does not exist.".format(payments_file))
with open(payments_file, 'r') as file:
payment_lines = file.readlines()
payments_dict = {}
for line in payment_lines:
pkh, amt = line.split(":")
pkh = pkh.strip()
amt = float(amt.strip())
payments_dict[pkh] = amt
if not payments_dict:
raise Exception("No payments to process")
# 6- is it a reports run
dry_run = args.dry_run
# 7- get reporting directories
reports_dir = os.path.expanduser(args.reports_dir)
# if in reports run mode, do not create consumers
# create reports in reports directory
if dry_run:
reports_dir = os.path.expanduser("./reports")
reports_dir = os.path.join(reports_dir, "manual")
payments_root = get_payment_root(reports_dir, create=True)
get_successful_payments_dir(payments_root, create=True)
get_failed_payments_dir(payments_root, create=True)
client_manager = ClientManager(node_endpoint=args.node_endpoint,
signer_endpoint=args.signer_endpoint)
for i in range(NB_CONSUMERS):
c = PaymentConsumer(name='manual_payment_consumer',
payments_dir=payments_root,
key_name=args.paymentaddress,
payments_queue=payments_queue,
node_addr=args.node_endpoint,
client_manager=client_manager,
dry_run=dry_run,
reactivate_zeroed=False,
delegator_pays_ra_fee=False,
delegator_pays_xfer_fee=False)
time.sleep(1)
c.start()
base_name_no_ext = os.path.basename(payments_file)
base_name_no_ext = os.path.splitext(base_name_no_ext)[0]
now = datetime.now()
now_str = now.strftime("%Y%m%d%H%M%S")
file_name = base_name_no_ext + "_" + now_str
payment_items = []
for key, value in payments_dict.items():
pi = RewardLog.ExternalInstance(file_name, key, value)
pi.payment = pi.payment * MUTEZ
payment_items.append(pi)
logger.info(
"Reward created for cycle %s address %s amount %f fee %f tz type %s",
pi.cycle, pi.address, pi.payment, pi.fee, pi.type)
payments_queue.put(PaymentBatch(None, 0, payment_items))
payments_queue.put(PaymentBatch(None, 0, [RewardLog.ExitInstance()]))
def retry_failed_payments(self, retry_injected=False):
logger.debug("retry_failed_payments started")
# 1 - list csv files under payments/failed directory
# absolute path of csv files found under payments_root/failed directory
failed_payments_dir = get_failed_payments_dir(self.payments_root)
payment_reports_failed = [
os.path.join(failed_payments_dir, x)
for x in os.listdir(failed_payments_dir) if x.endswith('.csv')
]
if payment_reports_failed:
payment_reports_failed = sorted(
payment_reports_failed,
key=lambda x: int(os.path.splitext(os.path.basename(x))[0]))
logger.debug("Failed payment files found are: '{}'".format(
",".join(payment_reports_failed)))
else:
logger.debug(
"No failed payment files found under directory '{}'".format(
failed_payments_dir))
# 2- for each csv file with name csv_report.csv
for payment_failed_report_file in payment_reports_failed:
logger.info("Working on failed payment file {}".format(
payment_failed_report_file))
# 2.1 - if there is a file csv_report.csv under payments/done, it means payment is already done
if os.path.isfile(
payment_failed_report_file.replace(PAYMENT_FAILED_DIR,
PAYMENT_DONE_DIR)):
# remove payments/failed/csv_report.csv
os.remove(payment_failed_report_file)
logger.info(
"Payment for failed payment {} is already done. Removing.".
format(payment_failed_report_file))
# remove payments/failed/csv_report.csv.BUSY
# if there is a busy failed payment report file, remove it.
remove_busy_file(payment_failed_report_file)
# do not double pay
continue
# 2.2 - if queue is full, wait for sometime
# make sure the queue is not full
while self.payments_queue.full():
logger.debug("Payments queue is full. Wait a few minutes.")
time.sleep(60 * 3)
cycle = int(
os.path.splitext(
os.path.basename(payment_failed_report_file))[0])
# 2.3 read payments/failed/csv_report.csv file into a list of dictionaries
batch = CsvPaymentFileParser().parse(payment_failed_report_file,
cycle)
nb_paid = len(
list(filter(lambda f: f.paid == PaymentStatus.PAID, batch)))
nb_done = len(
list(filter(lambda f: f.paid == PaymentStatus.DONE, batch)))
nb_injected = len(
list(filter(lambda f: f.paid == PaymentStatus.INJECTED,
batch)))
nb_failed = len(
list(filter(lambda f: f.paid == PaymentStatus.FAIL, batch)))
logger.info(
"Summary {} paid, {} done, {} injected, {} fail".format(
nb_paid, nb_done, nb_injected, nb_failed))
if retry_injected:
nb_converted = 0
for pl in batch:
if pl.paid == PaymentStatus.INJECTED:
pl.paid = PaymentStatus.FAIL
nb_converted += 1
logger.debug(
"Reward converted from %s to fail for cycle %s, address %s, amount %f, tz type %s",
pl.paid, pl.cycle, pl.address, pl.amount, pl.type)
if nb_converted:
logger.info(
"{} rewards converted from injected to fail.".format(
nb_converted))
# 2.4 - Filter batch to only include those which failed. No need to mess with PAID/DONE
batch = list(filter(lambda f: f.paid == PaymentStatus.FAIL, batch))
# 2.5 - Need to fetch current balance for addresses of any failed payments
self.reward_api.update_current_balances(batch)
# 2.6 - put records into payment_queue. payment_consumer will make payments
self.payments_queue.put(PaymentBatch(self, cycle, batch))
# 2.7 - rename payments/failed/csv_report.csv to payments/failed/csv_report.csv.BUSY
# mark the files as in use. we do not want it to be read again
# BUSY file will be removed, if successful payment is done
os.rename(payment_failed_report_file,
payment_failed_report_file + BUSY_FILE)
def main(args):
logger.info("TRD version {} is running in {} mode.".format(
version.version,
"daemon" if args.background_service else "interactive"))
logger.info("Arguments Configuration = {}".format(
json.dumps(args.__dict__, indent=1)))
# 1- find where configuration is
config_dir = os.path.expanduser(args.config_dir)
# create configuration directory if it is not present
# so that user can easily put his configuration there
if config_dir and not os.path.exists(config_dir):
os.makedirs(config_dir)
# 2- Load master configuration file if it is present
master_config_file_path = os.path.join(config_dir, "master.yaml")
master_cfg = {}
if os.path.isfile(master_config_file_path):
logger.debug("Loading master configuration file {}".format(
master_config_file_path))
master_parser = YamlConfParser(
ConfigParser.load_file(master_config_file_path))
master_cfg = master_parser.parse()
else:
logger.debug("master configuration file not present.")
managers = None
contracts_by_alias = None
addresses_by_pkh = None
if 'managers' in master_cfg:
managers = master_cfg['managers']
if 'contracts_by_alias' in master_cfg:
contracts_by_alias = master_cfg['contracts_by_alias']
if 'addresses_by_pkh' in master_cfg:
addresses_by_pkh = master_cfg['addresses_by_pkh']
# 3- get client path
client_path = get_client_path(
[x.strip() for x in args.executable_dirs.split(',')], args.docker,
args.network, args.verbose)
logger.debug("Dune client path is {}".format(client_path))
# 4. get network config
config_client_manager = SimpleClientManager(client_path)
network_config_map = init_network_config(args.network,
config_client_manager,
args.node_addr)
network_config = network_config_map[args.network]
logger.debug("Network config {}".format(network_config))
# 5- load baking configuration file
config_file_path = get_baking_configuration_file(config_dir)
logger.info(
"Loading baking configuration file {}".format(config_file_path))
wllt_clnt_mngr = WalletClientManager(client_path,
contracts_by_alias,
addresses_by_pkh,
managers,
verbose=args.verbose)
provider_factory = ProviderFactory(args.reward_data_provider,
verbose=args.verbose)
parser = BakingYamlConfParser(ConfigParser.load_file(config_file_path),
wllt_clnt_mngr,
provider_factory,
network_config,
args.node_addr,
verbose=args.verbose)
parser.parse()
parser.validate()
parser.process()
cfg_dict = parser.get_conf_obj()
# dictionary to BakingConf object, for a bit of type safety
cfg = BakingConf(cfg_dict, master_cfg)
logger.info("Baking Configuration {}".format(cfg))
baking_address = cfg.get_baking_address()
payment_address = cfg.get_payment_address()
logger.info(LINER)
logger.info("BAKING ADDRESS is {}".format(baking_address))
logger.info("PAYMENT ADDRESS is {}".format(payment_address))
logger.info(LINER)
# 6- is it a reports run
dry_run = args.dry_run_no_consumers or args.dry_run
if args.dry_run_no_consumers:
global NB_CONSUMERS
NB_CONSUMERS = 0
# 7- get reporting directories
reports_base = os.path.expanduser(args.reports_base)
# if in reports run mode, do not create consumers
# create reports in reports directory
if dry_run:
reports_base = os.path.expanduser("./reports")
reports_dir = os.path.join(reports_base, baking_address)
payments_root = get_payment_root(reports_dir, create=True)
calculations_root = get_calculations_root(reports_dir, create=True)
get_successful_payments_dir(payments_root, create=True)
get_failed_payments_dir(payments_root, create=True)
# 8- start the life cycle
life_cycle.start(not dry_run)
# 9- service fee calculator
srvc_fee_calc = ServiceFeeCalculator(cfg.get_full_supporters_set(),
cfg.get_specials_map(),
cfg.get_service_fee())
if args.initial_cycle is None:
recent = get_latest_report_file(payments_root)
# if payment logs exists set initial cycle to following cycle
# if payment logs does not exists, set initial cycle to 0, so that payment starts from last released rewards
args.initial_cycle = 0 if recent is None else int(recent) + 1
logger.info("initial_cycle set to {}".format(args.initial_cycle))
p = PaymentProducer(name='producer',
initial_payment_cycle=args.initial_cycle,
network_config=network_config,
payments_dir=payments_root,
calculations_dir=calculations_root,
run_mode=RunMode(args.run_mode),
service_fee_calc=srvc_fee_calc,
release_override=args.release_override,
payment_offset=args.payment_offset,
baking_cfg=cfg,
life_cycle=life_cycle,
payments_queue=payments_queue,
dry_run=dry_run,
wllt_clnt_mngr=wllt_clnt_mngr,
node_url=args.node_addr,
provider_factory=provider_factory,
verbose=args.verbose)
p.start()
publish_stats = not args.do_not_publish_stats
for i in range(NB_CONSUMERS):
c = PaymentConsumer(
name='consumer' + str(i),
payments_dir=payments_root,
key_name=payment_address,
client_path=client_path,
payments_queue=payments_queue,
node_addr=args.node_addr,
wllt_clnt_mngr=wllt_clnt_mngr,
args=args,
verbose=args.verbose,
dry_run=dry_run,
delegator_pays_xfer_fee=cfg.get_delegator_pays_xfer_fee(),
dest_map=cfg.get_dest_map(),
network_config=network_config,
publish_stats=publish_stats)
time.sleep(1)
c.start()
logger.info("Application start completed")
logger.info(LINER)
try:
while life_cycle.is_running():
time.sleep(10)
except KeyboardInterrupt:
logger.info("Interrupted.")
life_cycle.stop()
def main(args):
logger.info("Arguments Configuration = {}".format(
json.dumps(args.__dict__, indent=1)))
# 1- find where configuration is
config_dir = os.path.expanduser(args.config_dir)
# create configuration directory if it is not present
# so that user can easily put his configuration there
if config_dir and not os.path.exists(config_dir):
os.makedirs(config_dir)
# 2- Load master configuration file if it is present
master_config_file_path = os.path.join(config_dir, "master.yaml")
master_cfg = {}
if os.path.isfile(master_config_file_path):
logger.info("Loading master configuration file {}".format(
master_config_file_path))
master_parser = YamlConfParser(
ConfigParser.load_file(master_config_file_path))
master_cfg = master_parser.parse()
else:
logger.info("master configuration file not present.")
managers = None
contracts_by_alias = None
addresses_by_pkh = None
if 'managers' in master_cfg:
managers = master_cfg['managers']
if 'contracts_by_alias' in master_cfg:
contracts_by_alias = master_cfg['contracts_by_alias']
if 'addresses_by_pkh' in master_cfg:
addresses_by_pkh = master_cfg['addresses_by_pkh']
# 3- load payments file
payments_file = os.path.expanduser(args.payments_file)
if not os.path.isfile(payments_file):
raise Exception(
"payments_file ({}) does not exist.".format(payments_file))
with open(payments_file, 'r') as file:
payment_lines = file.readlines()
payments_dict = {}
for line in payment_lines:
pkh, amt = line.split(":")
pkh = pkh.strip()
amt = float(amt.strip())
payments_dict[pkh] = amt
if not payments_dict:
raise Exception("No payments to process")
# 4- get client path
network_config = network_config_map[args.network]
client_path = get_client_path(
[x.strip() for x in args.executable_dirs.split(',')], args.docker,
network_config, args.verbose)
logger.debug("Tezos client path is {}".format(client_path))
# 6- is it a reports run
dry_run = args.dry_run
# 7- get reporting directories
reports_dir = os.path.expanduser(args.reports_dir)
# if in reports run mode, do not create consumers
# create reports in reports directory
if dry_run:
reports_dir = os.path.expanduser("./reports")
reports_dir = os.path.join(reports_dir, "manual")
payments_root = get_payment_root(reports_dir, create=True)
calculations_root = get_calculations_root(reports_dir, create=True)
get_successful_payments_dir(payments_root, create=True)
get_failed_payments_dir(payments_root, create=True)
wllt_clnt_mngr = WalletClientManager(client_path, contracts_by_alias,
addresses_by_pkh, managers)
for i in range(NB_CONSUMERS):
c = PaymentConsumer(name='manual_payment_consumer',
payments_dir=payments_root,
key_name=args.paymentaddress,
client_path=client_path,
payments_queue=payments_queue,
node_addr=args.node_addr,
wllt_clnt_mngr=wllt_clnt_mngr,
verbose=args.verbose,
dry_run=dry_run,
delegator_pays_xfer_fee=False)
time.sleep(1)
c.start()
base_name_no_ext = os.path.basename(payments_file)
base_name_no_ext = os.path.splitext(base_name_no_ext)[0]
now = datetime.now()
now_str = now.strftime("%Y%m%d%H%M%S")
file_name = base_name_no_ext + "_" + now_str
payment_items = []
for key, value in payments_dict.items():
payment_items.append(
PaymentRecord.ManualInstance(file_name, key, value))
payments_queue.put(payment_items)
payments_queue.put([PaymentRecord.ExitInstance()])
def main(args):
logger.info("Arguments Configuration = {}".format(
json.dumps(args.__dict__, indent=1)))
# 1- find where configuration is
config_dir = os.path.expanduser(args.config_dir)
# create configuration directory if it is not present
# so that user can easily put his configuration there
if config_dir and not os.path.exists(config_dir):
os.makedirs(config_dir)
# 2- Load master configuration file if it is present
master_config_file_path = os.path.join(config_dir, "master.yaml")
master_cfg = {}
if os.path.isfile(master_config_file_path):
logger.info("Loading master configuration file {}".format(
master_config_file_path))
master_parser = YamlConfParser(
ConfigParser.load_file(master_config_file_path))
master_cfg = master_parser.parse()
else:
logger.info("master configuration file not present.")
managers = None
contracts_by_alias = None
addresses_by_pkh = None
if 'managers' in master_cfg:
managers = master_cfg['managers']
if 'contracts_by_alias' in master_cfg:
contracts_by_alias = master_cfg['contracts_by_alias']
if 'addresses_by_pkh' in master_cfg:
addresses_by_pkh = master_cfg['addresses_by_pkh']
# 3- get client path
client_path = get_client_path(
[x.strip() for x in args.executable_dirs.split(',')], args.docker,
args.network, args.verbose)
logger.debug("Tezos client path is {}".format(client_path))
# 4. get network config
config_client_manager = SimpleClientManager(client_path, args.node_addr)
network_config_map = init_network_config(args.network,
config_client_manager,
args.node_addr)
network_config = network_config_map[args.network]
# 5- load baking configuration file
config_file_path = get_baking_configuration_file(config_dir)
logger.info(
"Loading baking configuration file {}".format(config_file_path))
wllt_clnt_mngr = WalletClientManager(client_path,
contracts_by_alias,
addresses_by_pkh,
managers,
verbose=args.verbose)
provider_factory = ProviderFactory(args.reward_data_provider,
verbose=args.verbose)
parser = BakingYamlConfParser(ConfigParser.load_file(config_file_path),
wllt_clnt_mngr,
provider_factory,
network_config,
args.node_addr,
verbose=args.verbose,
api_base_url=args.api_base_url)
parser.parse()
parser.validate()
parser.process()
cfg_dict = parser.get_conf_obj()
# dictionary to BakingConf object, for a bit of type safety
cfg = BakingConf(cfg_dict, master_cfg)
logger.info("Baking Configuration {}".format(cfg))
baking_address = cfg.get_baking_address()
payment_address = cfg.get_payment_address()
logger.info(LINER)
logger.info("BAKING ADDRESS is {}".format(baking_address))
logger.info("PAYMENT ADDRESS is {}".format(payment_address))
logger.info(LINER)
# 6- is it a reports run
dry_run = args.dry_run_no_consumers or args.dry_run
if args.dry_run_no_consumers:
global NB_CONSUMERS
NB_CONSUMERS = 0
# 7- get reporting directories
reports_dir = os.path.expanduser(args.reports_base)
# if in reports run mode, do not create consumers
# create reports in reports directory
if dry_run:
reports_dir = os.path.expanduser("./reports")
reports_dir = os.path.join(reports_dir, baking_address)
payments_root = get_payment_root(reports_dir, create=True)
calculations_root = get_calculations_root(reports_dir, create=True)
get_successful_payments_dir(payments_root, create=True)
get_failed_payments_dir(payments_root, create=True)
# 8- start the life cycle
life_cycle.start(False)
# 9- service fee calculator
srvc_fee_calc = ServiceFeeCalculator(cfg.get_full_supporters_set(),
cfg.get_specials_map(),
cfg.get_service_fee())
try:
p = PaymentProducer(name='producer',
initial_payment_cycle=None,
network_config=network_config,
payments_dir=payments_root,
calculations_dir=calculations_root,
run_mode=RunMode.ONETIME,
service_fee_calc=srvc_fee_calc,
release_override=0,
payment_offset=0,
baking_cfg=cfg,
life_cycle=life_cycle,
payments_queue=payments_queue,
dry_run=dry_run,
wllt_clnt_mngr=wllt_clnt_mngr,
node_url=args.node_addr,
provider_factory=provider_factory,
verbose=args.verbose,
api_base_url=args.api_base_url)
p.retry_failed_payments(args.retry_injected)
c = PaymentConsumer(
name='consumer_retry_failed',
payments_dir=payments_root,
key_name=payment_address,
client_path=client_path,
payments_queue=payments_queue,
node_addr=args.node_addr,
wllt_clnt_mngr=wllt_clnt_mngr,
verbose=args.verbose,
dry_run=dry_run,
delegator_pays_xfer_fee=cfg.get_delegator_pays_xfer_fee(),
network_config=network_config)
time.sleep(1)
c.start()
p.exit()
c.join()
logger.info("Application start completed")
logger.info(LINER)
sleep(5)
except KeyboardInterrupt:
logger.info("Interrupted.")
def main(args):
logger.info("Arguments Configuration = {}".format(
json.dumps(args.__dict__, indent=1)))
# find where configuration is
config_dir = os.path.expanduser(args.config_dir)
# create configuration directory if it is not present
# so that user can easily put his configuration there
if config_dir and not os.path.exists(config_dir):
os.makedirs(config_dir)
# load baking configuration file
config_file_path = os.path.join(config_dir, BAKING_ADDRESS + ".yaml")
logger.info(
"Creating baking configuration file {}".format(config_file_path))
bkg_cfg_dict = {}
bkg_cfg_dict['version'] = '1.0'
bkg_cfg_dict['baking_address'] = BAKING_ADDRESS
bkg_cfg_dict['payment_address'] = args.paymentaddress
bkg_cfg_dict['service_fee'] = STANDARD_FEE * 100
bkg_cfg_dict['founders_map'] = founders_map
bkg_cfg_dict['owners_map'] = owners_map
bkg_cfg_dict['prcnt_scale'] = prcnt_scale
bkg_cfg_dict['pymnt_scale'] = pymnt_scale
for key, value in specials_map.items():
specials_map[key] = value * 100
bkg_cfg_dict['specials_map'] = specials_map
bkg_cfg_dict['supporters_set'] = supporters_set
bkg_cfg_dict['min_delegation_amt'] = MIN_DELEGATION_AMT
bkg_cfg_dict['excluded_delegators_set'] = excluded_delegators_set
if args.verbose:
dump = yaml.dump(bkg_cfg_dict, default_flow_style=True)
logger.info("Generated yaml configuration {}".format(dump))
with open(config_file_path, 'w') as outfile:
yaml.dump(bkg_cfg_dict, outfile, default_flow_style=True)
legacy_reports_dir = os.path.expanduser(args.legacy_reports_dir)
legacy_payments_root = get_payment_root(legacy_reports_dir, create=False)
legacy_successful_payments_dir = get_successful_payments_dir(
legacy_payments_root, create=False)
legacy_failed_payments_dir = get_failed_payments_dir(legacy_payments_root,
create=False)
legacy_calculations_root = get_calculations_root(legacy_reports_dir,
create=False)
# 7- get reporting directories
reports_dir = os.path.expanduser(args.reports_dir)
reports_dir = os.path.join(reports_dir, BAKING_ADDRESS)
payments_root = get_payment_root(reports_dir, create=True)
calculations_root = get_calculations_root(reports_dir, create=True)
successful_payments_dir = get_successful_payments_dir(payments_root,
create=True)
failed_payments_dir = get_failed_payments_dir(payments_root, create=True)
if os.path.isdir(legacy_successful_payments_dir):
logger.info("Copy success logs")
copy_files(legacy_successful_payments_dir, successful_payments_dir,
args.verbose)
if os.path.isdir(legacy_failed_payments_dir):
logger.info("Copy fail logs")
copy_files(legacy_failed_payments_dir, failed_payments_dir,
args.verbose)
if os.path.isdir(legacy_calculations_root):
logger.info("Copy calculation logs")
copy_files(legacy_calculations_root, calculations_root, args.verbose)
def retry_failed_payments(self):
logger.info("retry_failed_payments started")
# 1 - list csv files under payments/failed directory
# absolute path of csv files found under payments_root/failed directory
failed_payments_dir = get_failed_payments_dir(self.payments_root)
payment_reports_failed = [
os.path.join(failed_payments_dir, x)
for x in os.listdir(failed_payments_dir) if x.endswith('.csv')
]
logger.debug("Trying failed payments : '{}'".format(
",".join(payment_reports_failed)))
# 2- for each csv file with name csv_report.csv
for payment_failed_report_file in payment_reports_failed:
logger.debug("Working on failed payment file {}".format(
payment_failed_report_file))
# 2.1 - if there is a file csv_report.csv under payments/done, it means payment is already done
if os.path.isfile(
payment_failed_report_file.replace(PAYMENT_FAILED_DIR,
PAYMENT_DONE_DIR)):
# remove payments/failed/csv_report.csv
os.remove(payment_failed_report_file)
logger.debug(
"Payment for failed payment {} is already done. Removing.".
format(payment_failed_report_file))
# remove payments/failed/csv_report.csv.BUSY
# if there is a busy failed payment report file, remove it.
remove_busy_file(payment_failed_report_file)
# do not double pay
continue
# 2.2 - if queue is full, wait for sometime
# make sure the queue is not full
while self.payments_queue.full():
logger.info("Payments queue is full. Wait for 30 seconds.")
time.sleep(30)
cycle = int(
os.path.splitext(
os.path.basename(payment_failed_report_file))[0])
# 2.3 read payments/failed/csv_report.csv file into a list of dictionaries
with open(payment_failed_report_file) as f:
# read csv into list of dictionaries
dict_rows = [{key: value
for key, value in row.items()}
for row in csv.DictReader(
f, delimiter='\t', skipinitialspace=True)]
batch = PaymentRecord.FromPaymentCSVDictRows(dict_rows, cycle)
# 2.4 put records into payment_queue. payment_consumer will make payments
if batch:
self.payments_queue.put(batch)
else:
logger.info("Nothing to pay.")
# 2.5 rename payments/failed/csv_report.csv to payments/failed/csv_report.csv.BUSY
# mark the files as in use. we do not want it to be read again
# BUSY file will be removed, if successful payment is done
os.rename(payment_failed_report_file,
get_busy_file(payment_failed_report_file))
def main(args):
logger.info("TRD version {} is running in {} mode.".format(
VERSION, "daemon" if args.background_service else "interactive"))
logger.info("Arguments Configuration = {}".format(
json.dumps(args.__dict__, indent=1)))
publish_stats = not args.do_not_publish_stats
logger.info(
"Anonymous statistics {} be collected. See docs/statistics.rst for more information."
.format("will" if publish_stats else "will not"))
# 1- find where configuration is
config_dir = os.path.expanduser(args.config_dir)
# create configuration directory if it is not present
# so that user can easily put his configuration there
if config_dir and not os.path.exists(config_dir):
os.makedirs(config_dir)
# 4. get network config
client_manager = ClientManager(node_endpoint=args.node_endpoint,
signer_endpoint=args.signer_endpoint)
network_config_map = init_network_config(args.network, client_manager)
network_config = network_config_map[args.network]
logger.debug("Network config {}".format(network_config))
# Setup provider to fetch RPCs
provider_factory = ProviderFactory(args.reward_data_provider)
# 5- load and verify baking configuration file
config_file_path = None
try:
config_file_path = get_baking_configuration_file(config_dir)
logger.info(
"Loading baking configuration file {}".format(config_file_path))
parser = BakingYamlConfParser(
yaml_text=ConfigParser.load_file(config_file_path),
clnt_mngr=client_manager,
provider_factory=provider_factory,
network_config=network_config,
node_url=args.node_endpoint,
api_base_url=args.api_base_url)
parser.parse()
parser.validate()
parser.process()
# dictionary to BakingConf object, for a bit of type safety
cfg_dict = parser.get_conf_obj()
cfg = BakingConf(cfg_dict)
except ConfigurationException as e:
logger.info(
"Unable to parse '{}' config file.".format(config_file_path))
logger.info(e)
sys.exit(1)
logger.info("Baking Configuration {}".format(cfg))
baking_address = cfg.get_baking_address()
payment_address = cfg.get_payment_address()
logger.info(LINER)
logger.info("BAKING ADDRESS is {}".format(baking_address))
logger.info("PAYMENT ADDRESS is {}".format(payment_address))
logger.info(LINER)
# 6- is it a reports run
dry_run = args.dry_run_no_consumers or args.dry_run
if args.dry_run_no_consumers:
global NB_CONSUMERS
NB_CONSUMERS = 0
# 7- get reporting directories
reports_base = os.path.expanduser(args.reports_base)
# if in reports run mode, do not create consumers
# create reports in reports directory
if dry_run:
reports_base = os.path.expanduser("./reports")
reports_dir = os.path.join(reports_base, baking_address)
payments_root = get_payment_root(reports_dir, create=True)
calculations_root = get_calculations_root(reports_dir, create=True)
get_successful_payments_dir(payments_root, create=True)
get_failed_payments_dir(payments_root, create=True)
# 8- start the life cycle
life_cycle.start(not dry_run)
# 9- service fee calculator
srvc_fee_calc = ServiceFeeCalculator(cfg.get_full_supporters_set(),
cfg.get_specials_map(),
cfg.get_service_fee())
if args.initial_cycle is None:
recent = get_latest_report_file(payments_root)
# if payment logs exists set initial cycle to following cycle
# if payment logs does not exists, set initial cycle to 0, so that payment starts from last released rewards
args.initial_cycle = 0 if recent is None else int(recent) + 1
logger.info("initial_cycle set to {}".format(args.initial_cycle))
# 10- load plugins
plugins_manager = plugins.PluginManager(cfg.get_plugins_conf(), dry_run)
# 11- Start producer and consumer
p = PaymentProducer(name='producer',
initial_payment_cycle=args.initial_cycle,
network_config=network_config,
payments_dir=payments_root,
calculations_dir=calculations_root,
run_mode=RunMode(args.run_mode),
service_fee_calc=srvc_fee_calc,
release_override=args.release_override,
payment_offset=args.payment_offset,
baking_cfg=cfg,
life_cycle=life_cycle,
payments_queue=payments_queue,
dry_run=dry_run,
client_manager=client_manager,
node_url=args.node_endpoint,
provider_factory=provider_factory,
node_url_public=args.node_addr_public,
api_base_url=args.api_base_url,
retry_injected=args.retry_injected)
p.start()
for i in range(NB_CONSUMERS):
c = PaymentConsumer(
name='consumer' + str(i),
payments_dir=payments_root,
key_name=payment_address,
payments_queue=payments_queue,
node_addr=args.node_endpoint,
client_manager=client_manager,
plugins_manager=plugins_manager,
rewards_type=cfg.get_rewards_type(),
args=args,
dry_run=dry_run,
reactivate_zeroed=cfg.get_reactivate_zeroed(),
delegator_pays_ra_fee=cfg.get_delegator_pays_ra_fee(),
delegator_pays_xfer_fee=cfg.get_delegator_pays_xfer_fee(),
dest_map=cfg.get_dest_map(),
network_config=network_config,
publish_stats=publish_stats)
sleep(1)
c.start()
logger.info("Application start completed")
logger.info(LINER)
# Run forever
try:
while life_cycle.is_running():
sleep(10)
except KeyboardInterrupt:
logger.info("Interrupted.")
life_cycle.stop()
def retry_failed_payments(self):
logger.debug("retry_failed_payments started")
# 1 - list csv files under payments/failed directory
# absolute path of csv files found under payments_root/failed directory
failed_payments_dir = get_failed_payments_dir(self.payments_root)
payment_reports_failed = [
join(failed_payments_dir, x) for x in listdir(failed_payments_dir)
if x.endswith(".csv")
and int(x.split(".csv")[0]) >= self.initial_payment_cycle
]
if payment_reports_failed:
payment_reports_failed = sorted(payment_reports_failed,
key=self.get_basename)
logger.debug("Failed payment files found are: '{}'".format(
",".join(payment_reports_failed)))
else:
logger.info(
"No failed payment files found under directory '{}' on or after cycle '{}'"
.format(failed_payments_dir, self.initial_payment_cycle))
# 2- for each csv file with name csv_report.csv
for payment_failed_report_file in payment_reports_failed:
logger.info("Working on failed payment file {}".format(
payment_failed_report_file))
# 2.1 - Read csv_report.csv under payments/failed and -if existing- under payments/done
cycle = int(
os.path.splitext(
os.path.basename(payment_failed_report_file))[0])
batch = CsvPaymentFileParser().parse(payment_failed_report_file,
cycle)
payment_successful_report_file = payment_failed_report_file.replace(
PAYMENT_FAILED_DIR, PAYMENT_DONE_DIR)
if os.path.isfile(payment_successful_report_file):
batch += CsvPaymentFileParser().parse(
payment_successful_report_file, cycle)
# 2.2 Translate batch into a list of dictionaries
nb_paid = len(
list(filter(lambda f: f.paid == PaymentStatus.PAID, batch)))
nb_done = len(
list(filter(lambda f: f.paid == PaymentStatus.DONE, batch)))
nb_injected = len(
list(filter(lambda f: f.paid == PaymentStatus.INJECTED,
batch)))
nb_failed = len(
list(filter(lambda f: f.paid == PaymentStatus.FAIL, batch)))
nb_avoided = len(
list(filter(lambda f: f.paid == PaymentStatus.AVOIDED, batch)))
logger.info(
"Summary {} paid, {} done, {} injected, {} fail, {} avoided".
format(nb_paid, nb_done, nb_injected, nb_failed, nb_avoided))
if self.retry_injected:
self.convert_injected_to_fail(batch)
# 2.3 - if queue is full, wait for sometime
# make sure the queue is not full
while self.payments_queue.full():
logger.debug(
"Payments queue is full. Please wait three minutes.")
sleep(60 * 3)
# 2.5 - Need to fetch current balance for addresses of any failed payments
self.reward_api.update_current_balances(batch)
# 2.6 - put records into payment_queue. payment_consumer will make payments
self.payments_queue.put(
PaymentBatch(self.payment_producer, cycle, batch))
# 2.7 - rename payments/failed/csv_report.csv to payments/failed/csv_report.csv.BUSY
# mark the files as in use. we do not want it to be read again
# BUSY file will be removed, if successful payment is done
os.rename(payment_failed_report_file,
payment_failed_report_file + BUSY_FILE)
return
