def load_config_file(wllt_clnt_mngr, network_config, master_cfg): provider_factory = ProviderFactory(args.reward_data_provider) parser = BakingYamlConfParser(None, wllt_clnt_mngr, provider_factory, network_config, args.node_addr, api_base_url=args.api_base_url) parser.parse() parser.validate() parser.process() cfg_dict = parser.get_conf_obj() # dictionary to BakingConf object, for a bit of type safety cfg = BakingConf(cfg_dict, master_cfg) logger.info("Baking Configuration {}".format(cfg)) baking_address = cfg.get_baking_address() payment_address = cfg.get_payment_address() logger.info(LINER) logger.info("BAKING ADDRESS is {}".format(baking_address)) logger.info("PAYMENT ADDRESS is {}".format(payment_address)) logger.info(LINER) # 7- get reporting directories reports_dir = os.path.expanduser(args.reports_base) reports_dir = os.path.join(reports_dir, baking_address) payments_root = get_payment_root(reports_dir, create=True) get_successful_payments_dir(payments_root, create=True) get_failed_payments_dir(payments_root, create=True)
def __init__(self, args, baking_address) -> None: super().__init__() # Get reporting directories if args.dry_run: base_directory = os.path.join( os.path.expanduser(os.path.normpath(args.base_directory)), SIMULATIONS_DIR, "", ) else: base_directory = os.path.join( os.path.expanduser(os.path.normpath(args.base_directory)), REPORTS_DIR, "", ) self.reports_dir = os.path.join(base_directory, baking_address, "") self.payments_root = get_payment_root(self.reports_dir, create=True) self.calculations_root = get_calculations_root(self.reports_dir, create=True) self.successful_payments_dir = get_successful_payments_dir( self.payments_root, create=True) self.failed_payments_dir = get_failed_payments_dir(self.payments_root, create=True)
def __init__(self, args, baking_address) -> None: super().__init__() # 7- get reporting directories reports_base = os.path.expanduser(args.reports_base) # if in reports run mode, do not create consumers # create reports in reports directory if args.dry_run: reports_base = os.path.expanduser("./reports") self.reports_dir = os.path.join(reports_base, baking_address) self.payments_root = get_payment_root(self.reports_dir, create=True) self.calculations_root = get_calculations_root(self.reports_dir, create=True) self.successful_payments_dir = get_successful_payments_dir( self.payments_root, create=True) self.failed_payments_dir = get_failed_payments_dir(self.payments_root, create=True)
def main(args): logger.info( "Arguments Configuration = {}".format(json.dumps(args.__dict__, indent=1)) ) # Load payments file payments_file = os.path.expanduser(os.path.normpath(args.payments_file)) if not os.path.isfile(payments_file): raise Exception("payments_file ({}) does not exist.".format(payments_file)) with open(payments_file, "r") as file: payment_lines = file.readlines() payments_dict = {} for line in payment_lines: pkh, amt = line.split(":") pkh = pkh.strip() amt = float(amt.strip()) payments_dict[pkh] = amt if not payments_dict: raise Exception("No payments to process") # Check if dry-run dry_run = args.dry_run # Get reporting directories reports_dir = os.path.expanduser(os.path.normpath(args.base_directory)) # Check the disk size at the reports dir location if disk_is_full(reports_dir): raise Exception( "Disk is full at {}. Please free space to continue saving reports.".format( reports_dir ) ) # if in reports run mode, do not create consumers # create reports in reports directory if dry_run: reports_dir = os.path.join(reports_dir, SIMULATIONS_DIR, "") else: reports_dir = os.path.join(reports_dir, REPORTS_DIR, "") reports_dir = os.path.join(reports_dir, "manual", "") payments_root = get_payment_root(reports_dir, create=True) get_successful_payments_dir(payments_root, create=True) get_failed_payments_dir(payments_root, create=True) client_manager = ClientManager( node_endpoint=args.node_endpoint, signer_endpoint=args.signer_endpoint ) for i in range(NB_CONSUMERS): c = PaymentConsumer( name="manual_payment_consumer", payments_dir=payments_root, key_name=args.paymentaddress, payments_queue=payments_queue, node_addr=args.node_endpoint, client_manager=client_manager, dry_run=dry_run, reactivate_zeroed=False, delegator_pays_ra_fee=False, delegator_pays_xfer_fee=False, ) time.sleep(1) c.start() base_name_no_ext = os.path.basename(payments_file) base_name_no_ext = os.path.splitext(base_name_no_ext)[0] now = datetime.now() now_str = now.strftime("%Y%m%d%H%M%S") file_name = base_name_no_ext + "_" + now_str payment_items = [] for key, value in payments_dict.items(): pi = RewardLog.ExternalInstance(file_name, key, value) pi.payment = pi.payment * MUTEZ payment_items.append(pi) logger.info( "Reward created for cycle %s address %s amount %f fee %f tz type %s", pi.cycle, pi.address, pi.payment, pi.fee, pi.type, ) payments_queue.put(PaymentBatch(None, 0, payment_items)) payments_queue.put(PaymentBatch(None, 0, [RewardLog.ExitInstance()]))
def main(args): logger.info("Arguments Configuration = {}".format( json.dumps(args.__dict__, indent=1))) # 1- find where configuration is config_dir = os.path.expanduser(args.config_dir) # create configuration directory if it is not present # so that user can easily put his configuration there if config_dir and not os.path.exists(config_dir): os.makedirs(config_dir) # 3- load payments file payments_file = os.path.expanduser(args.payments_file) if not os.path.isfile(payments_file): raise Exception( "payments_file ({}) does not exist.".format(payments_file)) with open(payments_file, 'r') as file: payment_lines = file.readlines() payments_dict = {} for line in payment_lines: pkh, amt = line.split(":") pkh = pkh.strip() amt = float(amt.strip()) payments_dict[pkh] = amt if not payments_dict: raise Exception("No payments to process") # 6- is it a reports run dry_run = args.dry_run # 7- get reporting directories reports_dir = os.path.expanduser(args.reports_dir) # if in reports run mode, do not create consumers # create reports in reports directory if dry_run: reports_dir = os.path.expanduser("./reports") reports_dir = os.path.join(reports_dir, "manual") payments_root = get_payment_root(reports_dir, create=True) get_successful_payments_dir(payments_root, create=True) get_failed_payments_dir(payments_root, create=True) client_manager = ClientManager(node_endpoint=args.node_endpoint, signer_endpoint=args.signer_endpoint) for i in range(NB_CONSUMERS): c = PaymentConsumer(name='manual_payment_consumer', payments_dir=payments_root, key_name=args.paymentaddress, payments_queue=payments_queue, node_addr=args.node_endpoint, client_manager=client_manager, dry_run=dry_run, reactivate_zeroed=False, delegator_pays_ra_fee=False, delegator_pays_xfer_fee=False) time.sleep(1) c.start() base_name_no_ext = os.path.basename(payments_file) base_name_no_ext = os.path.splitext(base_name_no_ext)[0] now = datetime.now() now_str = now.strftime("%Y%m%d%H%M%S") file_name = base_name_no_ext + "_" + now_str payment_items = [] for key, value in payments_dict.items(): pi = RewardLog.ExternalInstance(file_name, key, value) pi.payment = pi.payment * MUTEZ payment_items.append(pi) logger.info( "Reward created for cycle %s address %s amount %f fee %f tz type %s", pi.cycle, pi.address, pi.payment, pi.fee, pi.type) payments_queue.put(PaymentBatch(None, 0, payment_items)) payments_queue.put(PaymentBatch(None, 0, [RewardLog.ExitInstance()]))
def retry_failed_payments(self, retry_injected=False): logger.debug("retry_failed_payments started") # 1 - list csv files under payments/failed directory # absolute path of csv files found under payments_root/failed directory failed_payments_dir = get_failed_payments_dir(self.payments_root) payment_reports_failed = [ os.path.join(failed_payments_dir, x) for x in os.listdir(failed_payments_dir) if x.endswith('.csv') ] if payment_reports_failed: payment_reports_failed = sorted( payment_reports_failed, key=lambda x: int(os.path.splitext(os.path.basename(x))[0])) logger.debug("Failed payment files found are: '{}'".format( ",".join(payment_reports_failed))) else: logger.debug( "No failed payment files found under directory '{}'".format( failed_payments_dir)) # 2- for each csv file with name csv_report.csv for payment_failed_report_file in payment_reports_failed: logger.info("Working on failed payment file {}".format( payment_failed_report_file)) # 2.1 - if there is a file csv_report.csv under payments/done, it means payment is already done if os.path.isfile( payment_failed_report_file.replace(PAYMENT_FAILED_DIR, PAYMENT_DONE_DIR)): # remove payments/failed/csv_report.csv os.remove(payment_failed_report_file) logger.info( "Payment for failed payment {} is already done. Removing.". format(payment_failed_report_file)) # remove payments/failed/csv_report.csv.BUSY # if there is a busy failed payment report file, remove it. remove_busy_file(payment_failed_report_file) # do not double pay continue # 2.2 - if queue is full, wait for sometime # make sure the queue is not full while self.payments_queue.full(): logger.debug("Payments queue is full. Wait a few minutes.") time.sleep(60 * 3) cycle = int( os.path.splitext( os.path.basename(payment_failed_report_file))[0]) # 2.3 read payments/failed/csv_report.csv file into a list of dictionaries batch = CsvPaymentFileParser().parse(payment_failed_report_file, cycle) nb_paid = len( list(filter(lambda f: f.paid == PaymentStatus.PAID, batch))) nb_done = len( list(filter(lambda f: f.paid == PaymentStatus.DONE, batch))) nb_injected = len( list(filter(lambda f: f.paid == PaymentStatus.INJECTED, batch))) nb_failed = len( list(filter(lambda f: f.paid == PaymentStatus.FAIL, batch))) logger.info( "Summary {} paid, {} done, {} injected, {} fail".format( nb_paid, nb_done, nb_injected, nb_failed)) if retry_injected: nb_converted = 0 for pl in batch: if pl.paid == PaymentStatus.INJECTED: pl.paid = PaymentStatus.FAIL nb_converted += 1 logger.debug( "Reward converted from %s to fail for cycle %s, address %s, amount %f, tz type %s", pl.paid, pl.cycle, pl.address, pl.amount, pl.type) if nb_converted: logger.info( "{} rewards converted from injected to fail.".format( nb_converted)) # 2.4 - Filter batch to only include those which failed. No need to mess with PAID/DONE batch = list(filter(lambda f: f.paid == PaymentStatus.FAIL, batch)) # 2.5 - Need to fetch current balance for addresses of any failed payments self.reward_api.update_current_balances(batch) # 2.6 - put records into payment_queue. payment_consumer will make payments self.payments_queue.put(PaymentBatch(self, cycle, batch)) # 2.7 - rename payments/failed/csv_report.csv to payments/failed/csv_report.csv.BUSY # mark the files as in use. we do not want it to be read again # BUSY file will be removed, if successful payment is done os.rename(payment_failed_report_file, payment_failed_report_file + BUSY_FILE)
def main(args): logger.info("TRD version {} is running in {} mode.".format( version.version, "daemon" if args.background_service else "interactive")) logger.info("Arguments Configuration = {}".format( json.dumps(args.__dict__, indent=1))) # 1- find where configuration is config_dir = os.path.expanduser(args.config_dir) # create configuration directory if it is not present # so that user can easily put his configuration there if config_dir and not os.path.exists(config_dir): os.makedirs(config_dir) # 2- Load master configuration file if it is present master_config_file_path = os.path.join(config_dir, "master.yaml") master_cfg = {} if os.path.isfile(master_config_file_path): logger.debug("Loading master configuration file {}".format( master_config_file_path)) master_parser = YamlConfParser( ConfigParser.load_file(master_config_file_path)) master_cfg = master_parser.parse() else: logger.debug("master configuration file not present.") managers = None contracts_by_alias = None addresses_by_pkh = None if 'managers' in master_cfg: managers = master_cfg['managers'] if 'contracts_by_alias' in master_cfg: contracts_by_alias = master_cfg['contracts_by_alias'] if 'addresses_by_pkh' in master_cfg: addresses_by_pkh = master_cfg['addresses_by_pkh'] # 3- get client path client_path = get_client_path( [x.strip() for x in args.executable_dirs.split(',')], args.docker, args.network, args.verbose) logger.debug("Dune client path is {}".format(client_path)) # 4. get network config config_client_manager = SimpleClientManager(client_path) network_config_map = init_network_config(args.network, config_client_manager, args.node_addr) network_config = network_config_map[args.network] logger.debug("Network config {}".format(network_config)) # 5- load baking configuration file config_file_path = get_baking_configuration_file(config_dir) logger.info( "Loading baking configuration file {}".format(config_file_path)) wllt_clnt_mngr = WalletClientManager(client_path, contracts_by_alias, addresses_by_pkh, managers, verbose=args.verbose) provider_factory = ProviderFactory(args.reward_data_provider, verbose=args.verbose) parser = BakingYamlConfParser(ConfigParser.load_file(config_file_path), wllt_clnt_mngr, provider_factory, network_config, args.node_addr, verbose=args.verbose) parser.parse() parser.validate() parser.process() cfg_dict = parser.get_conf_obj() # dictionary to BakingConf object, for a bit of type safety cfg = BakingConf(cfg_dict, master_cfg) logger.info("Baking Configuration {}".format(cfg)) baking_address = cfg.get_baking_address() payment_address = cfg.get_payment_address() logger.info(LINER) logger.info("BAKING ADDRESS is {}".format(baking_address)) logger.info("PAYMENT ADDRESS is {}".format(payment_address)) logger.info(LINER) # 6- is it a reports run dry_run = args.dry_run_no_consumers or args.dry_run if args.dry_run_no_consumers: global NB_CONSUMERS NB_CONSUMERS = 0 # 7- get reporting directories reports_base = os.path.expanduser(args.reports_base) # if in reports run mode, do not create consumers # create reports in reports directory if dry_run: reports_base = os.path.expanduser("./reports") reports_dir = os.path.join(reports_base, baking_address) payments_root = get_payment_root(reports_dir, create=True) calculations_root = get_calculations_root(reports_dir, create=True) get_successful_payments_dir(payments_root, create=True) get_failed_payments_dir(payments_root, create=True) # 8- start the life cycle life_cycle.start(not dry_run) # 9- service fee calculator srvc_fee_calc = ServiceFeeCalculator(cfg.get_full_supporters_set(), cfg.get_specials_map(), cfg.get_service_fee()) if args.initial_cycle is None: recent = get_latest_report_file(payments_root) # if payment logs exists set initial cycle to following cycle # if payment logs does not exists, set initial cycle to 0, so that payment starts from last released rewards args.initial_cycle = 0 if recent is None else int(recent) + 1 logger.info("initial_cycle set to {}".format(args.initial_cycle)) p = PaymentProducer(name='producer', initial_payment_cycle=args.initial_cycle, network_config=network_config, payments_dir=payments_root, calculations_dir=calculations_root, run_mode=RunMode(args.run_mode), service_fee_calc=srvc_fee_calc, release_override=args.release_override, payment_offset=args.payment_offset, baking_cfg=cfg, life_cycle=life_cycle, payments_queue=payments_queue, dry_run=dry_run, wllt_clnt_mngr=wllt_clnt_mngr, node_url=args.node_addr, provider_factory=provider_factory, verbose=args.verbose) p.start() publish_stats = not args.do_not_publish_stats for i in range(NB_CONSUMERS): c = PaymentConsumer( name='consumer' + str(i), payments_dir=payments_root, key_name=payment_address, client_path=client_path, payments_queue=payments_queue, node_addr=args.node_addr, wllt_clnt_mngr=wllt_clnt_mngr, args=args, verbose=args.verbose, dry_run=dry_run, delegator_pays_xfer_fee=cfg.get_delegator_pays_xfer_fee(), dest_map=cfg.get_dest_map(), network_config=network_config, publish_stats=publish_stats) time.sleep(1) c.start() logger.info("Application start completed") logger.info(LINER) try: while life_cycle.is_running(): time.sleep(10) except KeyboardInterrupt: logger.info("Interrupted.") life_cycle.stop()
def main(args): logger.info("Arguments Configuration = {}".format( json.dumps(args.__dict__, indent=1))) # 1- find where configuration is config_dir = os.path.expanduser(args.config_dir) # create configuration directory if it is not present # so that user can easily put his configuration there if config_dir and not os.path.exists(config_dir): os.makedirs(config_dir) # 2- Load master configuration file if it is present master_config_file_path = os.path.join(config_dir, "master.yaml") master_cfg = {} if os.path.isfile(master_config_file_path): logger.info("Loading master configuration file {}".format( master_config_file_path)) master_parser = YamlConfParser( ConfigParser.load_file(master_config_file_path)) master_cfg = master_parser.parse() else: logger.info("master configuration file not present.") managers = None contracts_by_alias = None addresses_by_pkh = None if 'managers' in master_cfg: managers = master_cfg['managers'] if 'contracts_by_alias' in master_cfg: contracts_by_alias = master_cfg['contracts_by_alias'] if 'addresses_by_pkh' in master_cfg: addresses_by_pkh = master_cfg['addresses_by_pkh'] # 3- load payments file payments_file = os.path.expanduser(args.payments_file) if not os.path.isfile(payments_file): raise Exception( "payments_file ({}) does not exist.".format(payments_file)) with open(payments_file, 'r') as file: payment_lines = file.readlines() payments_dict = {} for line in payment_lines: pkh, amt = line.split(":") pkh = pkh.strip() amt = float(amt.strip()) payments_dict[pkh] = amt if not payments_dict: raise Exception("No payments to process") # 4- get client path network_config = network_config_map[args.network] client_path = get_client_path( [x.strip() for x in args.executable_dirs.split(',')], args.docker, network_config, args.verbose) logger.debug("Tezos client path is {}".format(client_path)) # 6- is it a reports run dry_run = args.dry_run # 7- get reporting directories reports_dir = os.path.expanduser(args.reports_dir) # if in reports run mode, do not create consumers # create reports in reports directory if dry_run: reports_dir = os.path.expanduser("./reports") reports_dir = os.path.join(reports_dir, "manual") payments_root = get_payment_root(reports_dir, create=True) calculations_root = get_calculations_root(reports_dir, create=True) get_successful_payments_dir(payments_root, create=True) get_failed_payments_dir(payments_root, create=True) wllt_clnt_mngr = WalletClientManager(client_path, contracts_by_alias, addresses_by_pkh, managers) for i in range(NB_CONSUMERS): c = PaymentConsumer(name='manual_payment_consumer', payments_dir=payments_root, key_name=args.paymentaddress, client_path=client_path, payments_queue=payments_queue, node_addr=args.node_addr, wllt_clnt_mngr=wllt_clnt_mngr, verbose=args.verbose, dry_run=dry_run, delegator_pays_xfer_fee=False) time.sleep(1) c.start() base_name_no_ext = os.path.basename(payments_file) base_name_no_ext = os.path.splitext(base_name_no_ext)[0] now = datetime.now() now_str = now.strftime("%Y%m%d%H%M%S") file_name = base_name_no_ext + "_" + now_str payment_items = [] for key, value in payments_dict.items(): payment_items.append( PaymentRecord.ManualInstance(file_name, key, value)) payments_queue.put(payment_items) payments_queue.put([PaymentRecord.ExitInstance()])
def main(args): logger.info("Arguments Configuration = {}".format( json.dumps(args.__dict__, indent=1))) # 1- find where configuration is config_dir = os.path.expanduser(args.config_dir) # create configuration directory if it is not present # so that user can easily put his configuration there if config_dir and not os.path.exists(config_dir): os.makedirs(config_dir) # 2- Load master configuration file if it is present master_config_file_path = os.path.join(config_dir, "master.yaml") master_cfg = {} if os.path.isfile(master_config_file_path): logger.info("Loading master configuration file {}".format( master_config_file_path)) master_parser = YamlConfParser( ConfigParser.load_file(master_config_file_path)) master_cfg = master_parser.parse() else: logger.info("master configuration file not present.") managers = None contracts_by_alias = None addresses_by_pkh = None if 'managers' in master_cfg: managers = master_cfg['managers'] if 'contracts_by_alias' in master_cfg: contracts_by_alias = master_cfg['contracts_by_alias'] if 'addresses_by_pkh' in master_cfg: addresses_by_pkh = master_cfg['addresses_by_pkh'] # 3- get client path client_path = get_client_path( [x.strip() for x in args.executable_dirs.split(',')], args.docker, args.network, args.verbose) logger.debug("Tezos client path is {}".format(client_path)) # 4. get network config config_client_manager = SimpleClientManager(client_path, args.node_addr) network_config_map = init_network_config(args.network, config_client_manager, args.node_addr) network_config = network_config_map[args.network] # 5- load baking configuration file config_file_path = get_baking_configuration_file(config_dir) logger.info( "Loading baking configuration file {}".format(config_file_path)) wllt_clnt_mngr = WalletClientManager(client_path, contracts_by_alias, addresses_by_pkh, managers, verbose=args.verbose) provider_factory = ProviderFactory(args.reward_data_provider, verbose=args.verbose) parser = BakingYamlConfParser(ConfigParser.load_file(config_file_path), wllt_clnt_mngr, provider_factory, network_config, args.node_addr, verbose=args.verbose, api_base_url=args.api_base_url) parser.parse() parser.validate() parser.process() cfg_dict = parser.get_conf_obj() # dictionary to BakingConf object, for a bit of type safety cfg = BakingConf(cfg_dict, master_cfg) logger.info("Baking Configuration {}".format(cfg)) baking_address = cfg.get_baking_address() payment_address = cfg.get_payment_address() logger.info(LINER) logger.info("BAKING ADDRESS is {}".format(baking_address)) logger.info("PAYMENT ADDRESS is {}".format(payment_address)) logger.info(LINER) # 6- is it a reports run dry_run = args.dry_run_no_consumers or args.dry_run if args.dry_run_no_consumers: global NB_CONSUMERS NB_CONSUMERS = 0 # 7- get reporting directories reports_dir = os.path.expanduser(args.reports_base) # if in reports run mode, do not create consumers # create reports in reports directory if dry_run: reports_dir = os.path.expanduser("./reports") reports_dir = os.path.join(reports_dir, baking_address) payments_root = get_payment_root(reports_dir, create=True) calculations_root = get_calculations_root(reports_dir, create=True) get_successful_payments_dir(payments_root, create=True) get_failed_payments_dir(payments_root, create=True) # 8- start the life cycle life_cycle.start(False) # 9- service fee calculator srvc_fee_calc = ServiceFeeCalculator(cfg.get_full_supporters_set(), cfg.get_specials_map(), cfg.get_service_fee()) try: p = PaymentProducer(name='producer', initial_payment_cycle=None, network_config=network_config, payments_dir=payments_root, calculations_dir=calculations_root, run_mode=RunMode.ONETIME, service_fee_calc=srvc_fee_calc, release_override=0, payment_offset=0, baking_cfg=cfg, life_cycle=life_cycle, payments_queue=payments_queue, dry_run=dry_run, wllt_clnt_mngr=wllt_clnt_mngr, node_url=args.node_addr, provider_factory=provider_factory, verbose=args.verbose, api_base_url=args.api_base_url) p.retry_failed_payments(args.retry_injected) c = PaymentConsumer( name='consumer_retry_failed', payments_dir=payments_root, key_name=payment_address, client_path=client_path, payments_queue=payments_queue, node_addr=args.node_addr, wllt_clnt_mngr=wllt_clnt_mngr, verbose=args.verbose, dry_run=dry_run, delegator_pays_xfer_fee=cfg.get_delegator_pays_xfer_fee(), network_config=network_config) time.sleep(1) c.start() p.exit() c.join() logger.info("Application start completed") logger.info(LINER) sleep(5) except KeyboardInterrupt: logger.info("Interrupted.")
def main(args): logger.info("Arguments Configuration = {}".format( json.dumps(args.__dict__, indent=1))) # find where configuration is config_dir = os.path.expanduser(args.config_dir) # create configuration directory if it is not present # so that user can easily put his configuration there if config_dir and not os.path.exists(config_dir): os.makedirs(config_dir) # load baking configuration file config_file_path = os.path.join(config_dir, BAKING_ADDRESS + ".yaml") logger.info( "Creating baking configuration file {}".format(config_file_path)) bkg_cfg_dict = {} bkg_cfg_dict['version'] = '1.0' bkg_cfg_dict['baking_address'] = BAKING_ADDRESS bkg_cfg_dict['payment_address'] = args.paymentaddress bkg_cfg_dict['service_fee'] = STANDARD_FEE * 100 bkg_cfg_dict['founders_map'] = founders_map bkg_cfg_dict['owners_map'] = owners_map bkg_cfg_dict['prcnt_scale'] = prcnt_scale bkg_cfg_dict['pymnt_scale'] = pymnt_scale for key, value in specials_map.items(): specials_map[key] = value * 100 bkg_cfg_dict['specials_map'] = specials_map bkg_cfg_dict['supporters_set'] = supporters_set bkg_cfg_dict['min_delegation_amt'] = MIN_DELEGATION_AMT bkg_cfg_dict['excluded_delegators_set'] = excluded_delegators_set if args.verbose: dump = yaml.dump(bkg_cfg_dict, default_flow_style=True) logger.info("Generated yaml configuration {}".format(dump)) with open(config_file_path, 'w') as outfile: yaml.dump(bkg_cfg_dict, outfile, default_flow_style=True) legacy_reports_dir = os.path.expanduser(args.legacy_reports_dir) legacy_payments_root = get_payment_root(legacy_reports_dir, create=False) legacy_successful_payments_dir = get_successful_payments_dir( legacy_payments_root, create=False) legacy_failed_payments_dir = get_failed_payments_dir(legacy_payments_root, create=False) legacy_calculations_root = get_calculations_root(legacy_reports_dir, create=False) # 7- get reporting directories reports_dir = os.path.expanduser(args.reports_dir) reports_dir = os.path.join(reports_dir, BAKING_ADDRESS) payments_root = get_payment_root(reports_dir, create=True) calculations_root = get_calculations_root(reports_dir, create=True) successful_payments_dir = get_successful_payments_dir(payments_root, create=True) failed_payments_dir = get_failed_payments_dir(payments_root, create=True) if os.path.isdir(legacy_successful_payments_dir): logger.info("Copy success logs") copy_files(legacy_successful_payments_dir, successful_payments_dir, args.verbose) if os.path.isdir(legacy_failed_payments_dir): logger.info("Copy fail logs") copy_files(legacy_failed_payments_dir, failed_payments_dir, args.verbose) if os.path.isdir(legacy_calculations_root): logger.info("Copy calculation logs") copy_files(legacy_calculations_root, calculations_root, args.verbose)
def retry_failed_payments(self): logger.info("retry_failed_payments started") # 1 - list csv files under payments/failed directory # absolute path of csv files found under payments_root/failed directory failed_payments_dir = get_failed_payments_dir(self.payments_root) payment_reports_failed = [ os.path.join(failed_payments_dir, x) for x in os.listdir(failed_payments_dir) if x.endswith('.csv') ] logger.debug("Trying failed payments : '{}'".format( ",".join(payment_reports_failed))) # 2- for each csv file with name csv_report.csv for payment_failed_report_file in payment_reports_failed: logger.debug("Working on failed payment file {}".format( payment_failed_report_file)) # 2.1 - if there is a file csv_report.csv under payments/done, it means payment is already done if os.path.isfile( payment_failed_report_file.replace(PAYMENT_FAILED_DIR, PAYMENT_DONE_DIR)): # remove payments/failed/csv_report.csv os.remove(payment_failed_report_file) logger.debug( "Payment for failed payment {} is already done. Removing.". format(payment_failed_report_file)) # remove payments/failed/csv_report.csv.BUSY # if there is a busy failed payment report file, remove it. remove_busy_file(payment_failed_report_file) # do not double pay continue # 2.2 - if queue is full, wait for sometime # make sure the queue is not full while self.payments_queue.full(): logger.info("Payments queue is full. Wait for 30 seconds.") time.sleep(30) cycle = int( os.path.splitext( os.path.basename(payment_failed_report_file))[0]) # 2.3 read payments/failed/csv_report.csv file into a list of dictionaries with open(payment_failed_report_file) as f: # read csv into list of dictionaries dict_rows = [{key: value for key, value in row.items()} for row in csv.DictReader( f, delimiter='\t', skipinitialspace=True)] batch = PaymentRecord.FromPaymentCSVDictRows(dict_rows, cycle) # 2.4 put records into payment_queue. payment_consumer will make payments if batch: self.payments_queue.put(batch) else: logger.info("Nothing to pay.") # 2.5 rename payments/failed/csv_report.csv to payments/failed/csv_report.csv.BUSY # mark the files as in use. we do not want it to be read again # BUSY file will be removed, if successful payment is done os.rename(payment_failed_report_file, get_busy_file(payment_failed_report_file))
def main(args): logger.info("TRD version {} is running in {} mode.".format( VERSION, "daemon" if args.background_service else "interactive")) logger.info("Arguments Configuration = {}".format( json.dumps(args.__dict__, indent=1))) publish_stats = not args.do_not_publish_stats logger.info( "Anonymous statistics {} be collected. See docs/statistics.rst for more information." .format("will" if publish_stats else "will not")) # 1- find where configuration is config_dir = os.path.expanduser(args.config_dir) # create configuration directory if it is not present # so that user can easily put his configuration there if config_dir and not os.path.exists(config_dir): os.makedirs(config_dir) # 4. get network config client_manager = ClientManager(node_endpoint=args.node_endpoint, signer_endpoint=args.signer_endpoint) network_config_map = init_network_config(args.network, client_manager) network_config = network_config_map[args.network] logger.debug("Network config {}".format(network_config)) # Setup provider to fetch RPCs provider_factory = ProviderFactory(args.reward_data_provider) # 5- load and verify baking configuration file config_file_path = None try: config_file_path = get_baking_configuration_file(config_dir) logger.info( "Loading baking configuration file {}".format(config_file_path)) parser = BakingYamlConfParser( yaml_text=ConfigParser.load_file(config_file_path), clnt_mngr=client_manager, provider_factory=provider_factory, network_config=network_config, node_url=args.node_endpoint, api_base_url=args.api_base_url) parser.parse() parser.validate() parser.process() # dictionary to BakingConf object, for a bit of type safety cfg_dict = parser.get_conf_obj() cfg = BakingConf(cfg_dict) except ConfigurationException as e: logger.info( "Unable to parse '{}' config file.".format(config_file_path)) logger.info(e) sys.exit(1) logger.info("Baking Configuration {}".format(cfg)) baking_address = cfg.get_baking_address() payment_address = cfg.get_payment_address() logger.info(LINER) logger.info("BAKING ADDRESS is {}".format(baking_address)) logger.info("PAYMENT ADDRESS is {}".format(payment_address)) logger.info(LINER) # 6- is it a reports run dry_run = args.dry_run_no_consumers or args.dry_run if args.dry_run_no_consumers: global NB_CONSUMERS NB_CONSUMERS = 0 # 7- get reporting directories reports_base = os.path.expanduser(args.reports_base) # if in reports run mode, do not create consumers # create reports in reports directory if dry_run: reports_base = os.path.expanduser("./reports") reports_dir = os.path.join(reports_base, baking_address) payments_root = get_payment_root(reports_dir, create=True) calculations_root = get_calculations_root(reports_dir, create=True) get_successful_payments_dir(payments_root, create=True) get_failed_payments_dir(payments_root, create=True) # 8- start the life cycle life_cycle.start(not dry_run) # 9- service fee calculator srvc_fee_calc = ServiceFeeCalculator(cfg.get_full_supporters_set(), cfg.get_specials_map(), cfg.get_service_fee()) if args.initial_cycle is None: recent = get_latest_report_file(payments_root) # if payment logs exists set initial cycle to following cycle # if payment logs does not exists, set initial cycle to 0, so that payment starts from last released rewards args.initial_cycle = 0 if recent is None else int(recent) + 1 logger.info("initial_cycle set to {}".format(args.initial_cycle)) # 10- load plugins plugins_manager = plugins.PluginManager(cfg.get_plugins_conf(), dry_run) # 11- Start producer and consumer p = PaymentProducer(name='producer', initial_payment_cycle=args.initial_cycle, network_config=network_config, payments_dir=payments_root, calculations_dir=calculations_root, run_mode=RunMode(args.run_mode), service_fee_calc=srvc_fee_calc, release_override=args.release_override, payment_offset=args.payment_offset, baking_cfg=cfg, life_cycle=life_cycle, payments_queue=payments_queue, dry_run=dry_run, client_manager=client_manager, node_url=args.node_endpoint, provider_factory=provider_factory, node_url_public=args.node_addr_public, api_base_url=args.api_base_url, retry_injected=args.retry_injected) p.start() for i in range(NB_CONSUMERS): c = PaymentConsumer( name='consumer' + str(i), payments_dir=payments_root, key_name=payment_address, payments_queue=payments_queue, node_addr=args.node_endpoint, client_manager=client_manager, plugins_manager=plugins_manager, rewards_type=cfg.get_rewards_type(), args=args, dry_run=dry_run, reactivate_zeroed=cfg.get_reactivate_zeroed(), delegator_pays_ra_fee=cfg.get_delegator_pays_ra_fee(), delegator_pays_xfer_fee=cfg.get_delegator_pays_xfer_fee(), dest_map=cfg.get_dest_map(), network_config=network_config, publish_stats=publish_stats) sleep(1) c.start() logger.info("Application start completed") logger.info(LINER) # Run forever try: while life_cycle.is_running(): sleep(10) except KeyboardInterrupt: logger.info("Interrupted.") life_cycle.stop()
def retry_failed_payments(self): logger.debug("retry_failed_payments started") # 1 - list csv files under payments/failed directory # absolute path of csv files found under payments_root/failed directory failed_payments_dir = get_failed_payments_dir(self.payments_root) payment_reports_failed = [ join(failed_payments_dir, x) for x in listdir(failed_payments_dir) if x.endswith(".csv") and int(x.split(".csv")[0]) >= self.initial_payment_cycle ] if payment_reports_failed: payment_reports_failed = sorted(payment_reports_failed, key=self.get_basename) logger.debug("Failed payment files found are: '{}'".format( ",".join(payment_reports_failed))) else: logger.info( "No failed payment files found under directory '{}' on or after cycle '{}'" .format(failed_payments_dir, self.initial_payment_cycle)) # 2- for each csv file with name csv_report.csv for payment_failed_report_file in payment_reports_failed: logger.info("Working on failed payment file {}".format( payment_failed_report_file)) # 2.1 - Read csv_report.csv under payments/failed and -if existing- under payments/done cycle = int( os.path.splitext( os.path.basename(payment_failed_report_file))[0]) batch = CsvPaymentFileParser().parse(payment_failed_report_file, cycle) payment_successful_report_file = payment_failed_report_file.replace( PAYMENT_FAILED_DIR, PAYMENT_DONE_DIR) if os.path.isfile(payment_successful_report_file): batch += CsvPaymentFileParser().parse( payment_successful_report_file, cycle) # 2.2 Translate batch into a list of dictionaries nb_paid = len( list(filter(lambda f: f.paid == PaymentStatus.PAID, batch))) nb_done = len( list(filter(lambda f: f.paid == PaymentStatus.DONE, batch))) nb_injected = len( list(filter(lambda f: f.paid == PaymentStatus.INJECTED, batch))) nb_failed = len( list(filter(lambda f: f.paid == PaymentStatus.FAIL, batch))) nb_avoided = len( list(filter(lambda f: f.paid == PaymentStatus.AVOIDED, batch))) logger.info( "Summary {} paid, {} done, {} injected, {} fail, {} avoided". format(nb_paid, nb_done, nb_injected, nb_failed, nb_avoided)) if self.retry_injected: self.convert_injected_to_fail(batch) # 2.3 - if queue is full, wait for sometime # make sure the queue is not full while self.payments_queue.full(): logger.debug( "Payments queue is full. Please wait three minutes.") sleep(60 * 3) # 2.5 - Need to fetch current balance for addresses of any failed payments self.reward_api.update_current_balances(batch) # 2.6 - put records into payment_queue. payment_consumer will make payments self.payments_queue.put( PaymentBatch(self.payment_producer, cycle, batch)) # 2.7 - rename payments/failed/csv_report.csv to payments/failed/csv_report.csv.BUSY # mark the files as in use. we do not want it to be read again # BUSY file will be removed, if successful payment is done os.rename(payment_failed_report_file, payment_failed_report_file + BUSY_FILE) return