def load_config_file(wllt_clnt_mngr, network_config, master_cfg): provider_factory = ProviderFactory("dunscan") parser = BakingYamlConfParser(None, wllt_clnt_mngr, provider_factory, network_config, args.node_addr) parser.parse() parser.validate() parser.process() cfg_dict = parser.get_conf_obj() # dictionary to BakingConf object, for a bit of type safety cfg = BakingConf(cfg_dict, master_cfg) logger.info("Baking Configuration {}".format(cfg)) baking_address = cfg.get_baking_address() payment_address = cfg.get_payment_address() logger.info(LINER) logger.info("BAKING ADDRESS is {}".format(baking_address)) logger.info("PAYMENT ADDRESS is {}".format(payment_address)) logger.info(LINER) # 7- get reporting directories reports_dir = os.path.expanduser(args.reports_base) reports_dir = os.path.join(reports_dir, baking_address) payments_root = get_payment_root(reports_dir, create=True) calculations_root = get_calculations_root(reports_dir, create=True) get_successful_payments_dir(payments_root, create=True) get_failed_payments_dir(payments_root, create=True)
def __init__(self, args, baking_address) -> None: super().__init__() # Get reporting directories if args.dry_run: base_directory = os.path.join( os.path.expanduser(os.path.normpath(args.base_directory)), SIMULATIONS_DIR, "", ) else: base_directory = os.path.join( os.path.expanduser(os.path.normpath(args.base_directory)), REPORTS_DIR, "", ) self.reports_dir = os.path.join(base_directory, baking_address, "") self.payments_root = get_payment_root(self.reports_dir, create=True) self.calculations_root = get_calculations_root(self.reports_dir, create=True) self.successful_payments_dir = get_successful_payments_dir( self.payments_root, create=True) self.failed_payments_dir = get_failed_payments_dir(self.payments_root, create=True)
def __init__(self, args, baking_address) -> None: super().__init__() # 7- get reporting directories reports_base = os.path.expanduser(args.reports_base) # if in reports run mode, do not create consumers # create reports in reports directory if args.dry_run: reports_base = os.path.expanduser("./reports") self.reports_dir = os.path.join(reports_base, baking_address) self.payments_root = get_payment_root(self.reports_dir, create=True) self.calculations_root = get_calculations_root(self.reports_dir, create=True) self.successful_payments_dir = get_successful_payments_dir( self.payments_root, create=True) self.failed_payments_dir = get_failed_payments_dir(self.payments_root, create=True)
def main(args): logger.info("TRD version {} is running in {} mode.".format( version.version, "daemon" if args.background_service else "interactive")) logger.info("Arguments Configuration = {}".format( json.dumps(args.__dict__, indent=1))) # 1- find where configuration is config_dir = os.path.expanduser(args.config_dir) # create configuration directory if it is not present # so that user can easily put his configuration there if config_dir and not os.path.exists(config_dir): os.makedirs(config_dir) # 2- Load master configuration file if it is present master_config_file_path = os.path.join(config_dir, "master.yaml") master_cfg = {} if os.path.isfile(master_config_file_path): logger.debug("Loading master configuration file {}".format( master_config_file_path)) master_parser = YamlConfParser( ConfigParser.load_file(master_config_file_path)) master_cfg = master_parser.parse() else: logger.debug("master configuration file not present.") managers = None contracts_by_alias = None addresses_by_pkh = None if 'managers' in master_cfg: managers = master_cfg['managers'] if 'contracts_by_alias' in master_cfg: contracts_by_alias = master_cfg['contracts_by_alias'] if 'addresses_by_pkh' in master_cfg: addresses_by_pkh = master_cfg['addresses_by_pkh'] # 3- get client path client_path = get_client_path( [x.strip() for x in args.executable_dirs.split(',')], args.docker, args.network, args.verbose) logger.debug("Dune client path is {}".format(client_path)) # 4. get network config config_client_manager = SimpleClientManager(client_path) network_config_map = init_network_config(args.network, config_client_manager, args.node_addr) network_config = network_config_map[args.network] logger.debug("Network config {}".format(network_config)) # 5- load baking configuration file config_file_path = get_baking_configuration_file(config_dir) logger.info( "Loading baking configuration file {}".format(config_file_path)) wllt_clnt_mngr = WalletClientManager(client_path, contracts_by_alias, addresses_by_pkh, managers, verbose=args.verbose) provider_factory = ProviderFactory(args.reward_data_provider, verbose=args.verbose) parser = BakingYamlConfParser(ConfigParser.load_file(config_file_path), wllt_clnt_mngr, provider_factory, network_config, args.node_addr, verbose=args.verbose) parser.parse() parser.validate() parser.process() cfg_dict = parser.get_conf_obj() # dictionary to BakingConf object, for a bit of type safety cfg = BakingConf(cfg_dict, master_cfg) logger.info("Baking Configuration {}".format(cfg)) baking_address = cfg.get_baking_address() payment_address = cfg.get_payment_address() logger.info(LINER) logger.info("BAKING ADDRESS is {}".format(baking_address)) logger.info("PAYMENT ADDRESS is {}".format(payment_address)) logger.info(LINER) # 6- is it a reports run dry_run = args.dry_run_no_consumers or args.dry_run if args.dry_run_no_consumers: global NB_CONSUMERS NB_CONSUMERS = 0 # 7- get reporting directories reports_base = os.path.expanduser(args.reports_base) # if in reports run mode, do not create consumers # create reports in reports directory if dry_run: reports_base = os.path.expanduser("./reports") reports_dir = os.path.join(reports_base, baking_address) payments_root = get_payment_root(reports_dir, create=True) calculations_root = get_calculations_root(reports_dir, create=True) get_successful_payments_dir(payments_root, create=True) get_failed_payments_dir(payments_root, create=True) # 8- start the life cycle life_cycle.start(not dry_run) # 9- service fee calculator srvc_fee_calc = ServiceFeeCalculator(cfg.get_full_supporters_set(), cfg.get_specials_map(), cfg.get_service_fee()) if args.initial_cycle is None: recent = get_latest_report_file(payments_root) # if payment logs exists set initial cycle to following cycle # if payment logs does not exists, set initial cycle to 0, so that payment starts from last released rewards args.initial_cycle = 0 if recent is None else int(recent) + 1 logger.info("initial_cycle set to {}".format(args.initial_cycle)) p = PaymentProducer(name='producer', initial_payment_cycle=args.initial_cycle, network_config=network_config, payments_dir=payments_root, calculations_dir=calculations_root, run_mode=RunMode(args.run_mode), service_fee_calc=srvc_fee_calc, release_override=args.release_override, payment_offset=args.payment_offset, baking_cfg=cfg, life_cycle=life_cycle, payments_queue=payments_queue, dry_run=dry_run, wllt_clnt_mngr=wllt_clnt_mngr, node_url=args.node_addr, provider_factory=provider_factory, verbose=args.verbose) p.start() publish_stats = not args.do_not_publish_stats for i in range(NB_CONSUMERS): c = PaymentConsumer( name='consumer' + str(i), payments_dir=payments_root, key_name=payment_address, client_path=client_path, payments_queue=payments_queue, node_addr=args.node_addr, wllt_clnt_mngr=wllt_clnt_mngr, args=args, verbose=args.verbose, dry_run=dry_run, delegator_pays_xfer_fee=cfg.get_delegator_pays_xfer_fee(), dest_map=cfg.get_dest_map(), network_config=network_config, publish_stats=publish_stats) time.sleep(1) c.start() logger.info("Application start completed") logger.info(LINER) try: while life_cycle.is_running(): time.sleep(10) except KeyboardInterrupt: logger.info("Interrupted.") life_cycle.stop()
def main(args): logger.info("Arguments Configuration = {}".format( json.dumps(args.__dict__, indent=1))) # 1- find where configuration is config_dir = os.path.expanduser(args.config_dir) # create configuration directory if it is not present # so that user can easily put his configuration there if config_dir and not os.path.exists(config_dir): os.makedirs(config_dir) # 2- Load master configuration file if it is present master_config_file_path = os.path.join(config_dir, "master.yaml") master_cfg = {} if os.path.isfile(master_config_file_path): logger.info("Loading master configuration file {}".format( master_config_file_path)) master_parser = YamlConfParser( ConfigParser.load_file(master_config_file_path)) master_cfg = master_parser.parse() else: logger.info("master configuration file not present.") managers = None contracts_by_alias = None addresses_by_pkh = None if 'managers' in master_cfg: managers = master_cfg['managers'] if 'contracts_by_alias' in master_cfg: contracts_by_alias = master_cfg['contracts_by_alias'] if 'addresses_by_pkh' in master_cfg: addresses_by_pkh = master_cfg['addresses_by_pkh'] # 3- load payments file payments_file = os.path.expanduser(args.payments_file) if not os.path.isfile(payments_file): raise Exception( "payments_file ({}) does not exist.".format(payments_file)) with open(payments_file, 'r') as file: payment_lines = file.readlines() payments_dict = {} for line in payment_lines: pkh, amt = line.split(":") pkh = pkh.strip() amt = float(amt.strip()) payments_dict[pkh] = amt if not payments_dict: raise Exception("No payments to process") # 4- get client path network_config = network_config_map[args.network] client_path = get_client_path( [x.strip() for x in args.executable_dirs.split(',')], args.docker, network_config, args.verbose) logger.debug("Tezos client path is {}".format(client_path)) # 6- is it a reports run dry_run = args.dry_run # 7- get reporting directories reports_dir = os.path.expanduser(args.reports_dir) # if in reports run mode, do not create consumers # create reports in reports directory if dry_run: reports_dir = os.path.expanduser("./reports") reports_dir = os.path.join(reports_dir, "manual") payments_root = get_payment_root(reports_dir, create=True) calculations_root = get_calculations_root(reports_dir, create=True) get_successful_payments_dir(payments_root, create=True) get_failed_payments_dir(payments_root, create=True) wllt_clnt_mngr = WalletClientManager(client_path, contracts_by_alias, addresses_by_pkh, managers) for i in range(NB_CONSUMERS): c = PaymentConsumer(name='manual_payment_consumer', payments_dir=payments_root, key_name=args.paymentaddress, client_path=client_path, payments_queue=payments_queue, node_addr=args.node_addr, wllt_clnt_mngr=wllt_clnt_mngr, verbose=args.verbose, dry_run=dry_run, delegator_pays_xfer_fee=False) time.sleep(1) c.start() base_name_no_ext = os.path.basename(payments_file) base_name_no_ext = os.path.splitext(base_name_no_ext)[0] now = datetime.now() now_str = now.strftime("%Y%m%d%H%M%S") file_name = base_name_no_ext + "_" + now_str payment_items = [] for key, value in payments_dict.items(): payment_items.append( PaymentRecord.ManualInstance(file_name, key, value)) payments_queue.put(payment_items) payments_queue.put([PaymentRecord.ExitInstance()])
def main(args): logger.info("Arguments Configuration = {}".format( json.dumps(args.__dict__, indent=1))) # 1- find where configuration is config_dir = os.path.expanduser(args.config_dir) # create configuration directory if it is not present # so that user can easily put his configuration there if config_dir and not os.path.exists(config_dir): os.makedirs(config_dir) # 2- Load master configuration file if it is present master_config_file_path = os.path.join(config_dir, "master.yaml") master_cfg = {} if os.path.isfile(master_config_file_path): logger.info("Loading master configuration file {}".format( master_config_file_path)) master_parser = YamlConfParser( ConfigParser.load_file(master_config_file_path)) master_cfg = master_parser.parse() else: logger.info("master configuration file not present.") managers = None contracts_by_alias = None addresses_by_pkh = None if 'managers' in master_cfg: managers = master_cfg['managers'] if 'contracts_by_alias' in master_cfg: contracts_by_alias = master_cfg['contracts_by_alias'] if 'addresses_by_pkh' in master_cfg: addresses_by_pkh = master_cfg['addresses_by_pkh'] # 3- get client path client_path = get_client_path( [x.strip() for x in args.executable_dirs.split(',')], args.docker, args.network, args.verbose) logger.debug("Tezos client path is {}".format(client_path)) # 4. get network config config_client_manager = SimpleClientManager(client_path, args.node_addr) network_config_map = init_network_config(args.network, config_client_manager, args.node_addr) network_config = network_config_map[args.network] # 5- load baking configuration file config_file_path = get_baking_configuration_file(config_dir) logger.info( "Loading baking configuration file {}".format(config_file_path)) wllt_clnt_mngr = WalletClientManager(client_path, contracts_by_alias, addresses_by_pkh, managers, verbose=args.verbose) provider_factory = ProviderFactory(args.reward_data_provider, verbose=args.verbose) parser = BakingYamlConfParser(ConfigParser.load_file(config_file_path), wllt_clnt_mngr, provider_factory, network_config, args.node_addr, verbose=args.verbose, api_base_url=args.api_base_url) parser.parse() parser.validate() parser.process() cfg_dict = parser.get_conf_obj() # dictionary to BakingConf object, for a bit of type safety cfg = BakingConf(cfg_dict, master_cfg) logger.info("Baking Configuration {}".format(cfg)) baking_address = cfg.get_baking_address() payment_address = cfg.get_payment_address() logger.info(LINER) logger.info("BAKING ADDRESS is {}".format(baking_address)) logger.info("PAYMENT ADDRESS is {}".format(payment_address)) logger.info(LINER) # 6- is it a reports run dry_run = args.dry_run_no_consumers or args.dry_run if args.dry_run_no_consumers: global NB_CONSUMERS NB_CONSUMERS = 0 # 7- get reporting directories reports_dir = os.path.expanduser(args.reports_base) # if in reports run mode, do not create consumers # create reports in reports directory if dry_run: reports_dir = os.path.expanduser("./reports") reports_dir = os.path.join(reports_dir, baking_address) payments_root = get_payment_root(reports_dir, create=True) calculations_root = get_calculations_root(reports_dir, create=True) get_successful_payments_dir(payments_root, create=True) get_failed_payments_dir(payments_root, create=True) # 8- start the life cycle life_cycle.start(False) # 9- service fee calculator srvc_fee_calc = ServiceFeeCalculator(cfg.get_full_supporters_set(), cfg.get_specials_map(), cfg.get_service_fee()) try: p = PaymentProducer(name='producer', initial_payment_cycle=None, network_config=network_config, payments_dir=payments_root, calculations_dir=calculations_root, run_mode=RunMode.ONETIME, service_fee_calc=srvc_fee_calc, release_override=0, payment_offset=0, baking_cfg=cfg, life_cycle=life_cycle, payments_queue=payments_queue, dry_run=dry_run, wllt_clnt_mngr=wllt_clnt_mngr, node_url=args.node_addr, provider_factory=provider_factory, verbose=args.verbose, api_base_url=args.api_base_url) p.retry_failed_payments(args.retry_injected) c = PaymentConsumer( name='consumer_retry_failed', payments_dir=payments_root, key_name=payment_address, client_path=client_path, payments_queue=payments_queue, node_addr=args.node_addr, wllt_clnt_mngr=wllt_clnt_mngr, verbose=args.verbose, dry_run=dry_run, delegator_pays_xfer_fee=cfg.get_delegator_pays_xfer_fee(), network_config=network_config) time.sleep(1) c.start() p.exit() c.join() logger.info("Application start completed") logger.info(LINER) sleep(5) except KeyboardInterrupt: logger.info("Interrupted.")
def main(args): logger.info("Arguments Configuration = {}".format( json.dumps(args.__dict__, indent=1))) # find where configuration is config_dir = os.path.expanduser(args.config_dir) # create configuration directory if it is not present # so that user can easily put his configuration there if config_dir and not os.path.exists(config_dir): os.makedirs(config_dir) # load baking configuration file config_file_path = os.path.join(config_dir, BAKING_ADDRESS + ".yaml") logger.info( "Creating baking configuration file {}".format(config_file_path)) bkg_cfg_dict = {} bkg_cfg_dict['version'] = '1.0' bkg_cfg_dict['baking_address'] = BAKING_ADDRESS bkg_cfg_dict['payment_address'] = args.paymentaddress bkg_cfg_dict['service_fee'] = STANDARD_FEE * 100 bkg_cfg_dict['founders_map'] = founders_map bkg_cfg_dict['owners_map'] = owners_map bkg_cfg_dict['prcnt_scale'] = prcnt_scale bkg_cfg_dict['pymnt_scale'] = pymnt_scale for key, value in specials_map.items(): specials_map[key] = value * 100 bkg_cfg_dict['specials_map'] = specials_map bkg_cfg_dict['supporters_set'] = supporters_set bkg_cfg_dict['min_delegation_amt'] = MIN_DELEGATION_AMT bkg_cfg_dict['excluded_delegators_set'] = excluded_delegators_set if args.verbose: dump = yaml.dump(bkg_cfg_dict, default_flow_style=True) logger.info("Generated yaml configuration {}".format(dump)) with open(config_file_path, 'w') as outfile: yaml.dump(bkg_cfg_dict, outfile, default_flow_style=True) legacy_reports_dir = os.path.expanduser(args.legacy_reports_dir) legacy_payments_root = get_payment_root(legacy_reports_dir, create=False) legacy_successful_payments_dir = get_successful_payments_dir( legacy_payments_root, create=False) legacy_failed_payments_dir = get_failed_payments_dir(legacy_payments_root, create=False) legacy_calculations_root = get_calculations_root(legacy_reports_dir, create=False) # 7- get reporting directories reports_dir = os.path.expanduser(args.reports_dir) reports_dir = os.path.join(reports_dir, BAKING_ADDRESS) payments_root = get_payment_root(reports_dir, create=True) calculations_root = get_calculations_root(reports_dir, create=True) successful_payments_dir = get_successful_payments_dir(payments_root, create=True) failed_payments_dir = get_failed_payments_dir(payments_root, create=True) if os.path.isdir(legacy_successful_payments_dir): logger.info("Copy success logs") copy_files(legacy_successful_payments_dir, successful_payments_dir, args.verbose) if os.path.isdir(legacy_failed_payments_dir): logger.info("Copy fail logs") copy_files(legacy_failed_payments_dir, failed_payments_dir, args.verbose) if os.path.isdir(legacy_calculations_root): logger.info("Copy calculation logs") copy_files(legacy_calculations_root, calculations_root, args.verbose)
def main(args): logger.info("TRD version {} is running in {} mode.".format( VERSION, "daemon" if args.background_service else "interactive")) logger.info("Arguments Configuration = {}".format( json.dumps(args.__dict__, indent=1))) publish_stats = not args.do_not_publish_stats logger.info( "Anonymous statistics {} be collected. See docs/statistics.rst for more information." .format("will" if publish_stats else "will not")) # 1- find where configuration is config_dir = os.path.expanduser(args.config_dir) # create configuration directory if it is not present # so that user can easily put his configuration there if config_dir and not os.path.exists(config_dir): os.makedirs(config_dir) # 4. get network config client_manager = ClientManager(node_endpoint=args.node_endpoint, signer_endpoint=args.signer_endpoint) network_config_map = init_network_config(args.network, client_manager) network_config = network_config_map[args.network] logger.debug("Network config {}".format(network_config)) # Setup provider to fetch RPCs provider_factory = ProviderFactory(args.reward_data_provider) # 5- load and verify baking configuration file config_file_path = None try: config_file_path = get_baking_configuration_file(config_dir) logger.info( "Loading baking configuration file {}".format(config_file_path)) parser = BakingYamlConfParser( yaml_text=ConfigParser.load_file(config_file_path), clnt_mngr=client_manager, provider_factory=provider_factory, network_config=network_config, node_url=args.node_endpoint, api_base_url=args.api_base_url) parser.parse() parser.validate() parser.process() # dictionary to BakingConf object, for a bit of type safety cfg_dict = parser.get_conf_obj() cfg = BakingConf(cfg_dict) except ConfigurationException as e: logger.info( "Unable to parse '{}' config file.".format(config_file_path)) logger.info(e) sys.exit(1) logger.info("Baking Configuration {}".format(cfg)) baking_address = cfg.get_baking_address() payment_address = cfg.get_payment_address() logger.info(LINER) logger.info("BAKING ADDRESS is {}".format(baking_address)) logger.info("PAYMENT ADDRESS is {}".format(payment_address)) logger.info(LINER) # 6- is it a reports run dry_run = args.dry_run_no_consumers or args.dry_run if args.dry_run_no_consumers: global NB_CONSUMERS NB_CONSUMERS = 0 # 7- get reporting directories reports_base = os.path.expanduser(args.reports_base) # if in reports run mode, do not create consumers # create reports in reports directory if dry_run: reports_base = os.path.expanduser("./reports") reports_dir = os.path.join(reports_base, baking_address) payments_root = get_payment_root(reports_dir, create=True) calculations_root = get_calculations_root(reports_dir, create=True) get_successful_payments_dir(payments_root, create=True) get_failed_payments_dir(payments_root, create=True) # 8- start the life cycle life_cycle.start(not dry_run) # 9- service fee calculator srvc_fee_calc = ServiceFeeCalculator(cfg.get_full_supporters_set(), cfg.get_specials_map(), cfg.get_service_fee()) if args.initial_cycle is None: recent = get_latest_report_file(payments_root) # if payment logs exists set initial cycle to following cycle # if payment logs does not exists, set initial cycle to 0, so that payment starts from last released rewards args.initial_cycle = 0 if recent is None else int(recent) + 1 logger.info("initial_cycle set to {}".format(args.initial_cycle)) # 10- load plugins plugins_manager = plugins.PluginManager(cfg.get_plugins_conf(), dry_run) # 11- Start producer and consumer p = PaymentProducer(name='producer', initial_payment_cycle=args.initial_cycle, network_config=network_config, payments_dir=payments_root, calculations_dir=calculations_root, run_mode=RunMode(args.run_mode), service_fee_calc=srvc_fee_calc, release_override=args.release_override, payment_offset=args.payment_offset, baking_cfg=cfg, life_cycle=life_cycle, payments_queue=payments_queue, dry_run=dry_run, client_manager=client_manager, node_url=args.node_endpoint, provider_factory=provider_factory, node_url_public=args.node_addr_public, api_base_url=args.api_base_url, retry_injected=args.retry_injected) p.start() for i in range(NB_CONSUMERS): c = PaymentConsumer( name='consumer' + str(i), payments_dir=payments_root, key_name=payment_address, payments_queue=payments_queue, node_addr=args.node_endpoint, client_manager=client_manager, plugins_manager=plugins_manager, rewards_type=cfg.get_rewards_type(), args=args, dry_run=dry_run, reactivate_zeroed=cfg.get_reactivate_zeroed(), delegator_pays_ra_fee=cfg.get_delegator_pays_ra_fee(), delegator_pays_xfer_fee=cfg.get_delegator_pays_xfer_fee(), dest_map=cfg.get_dest_map(), network_config=network_config, publish_stats=publish_stats) sleep(1) c.start() logger.info("Application start completed") logger.info(LINER) # Run forever try: while life_cycle.is_running(): sleep(10) except KeyboardInterrupt: logger.info("Interrupted.") life_cycle.stop()