def try_to_pay(self, pymnt_cycle): try: logger.info("Payment cycle is " + str(pymnt_cycle)) # 0- check for past payment evidence for current cycle past_payment_state = check_past_payment(self.payments_root, pymnt_cycle) if not self.dry_run and past_payment_state: logger.warn(past_payment_state) return True # 1- get reward data reward_model = self.reward_api.get_rewards_for_cycle_map(pymnt_cycle) # 2- calculate rewards reward_logs, total_amount = self.payment_calc.calculate(reward_model) # set cycle info for rl in reward_logs: rl.cycle = pymnt_cycle total_amount_to_pay = sum([rl.amount for rl in reward_logs if rl.payable]) # 4- if total_rewards > 0, proceed with payment if total_amount_to_pay > 0: report_file_path = get_calculation_report_file(self.calculations_dir, pymnt_cycle) # 5- send to payment consumer self.payments_queue.put(PaymentBatch(self, pymnt_cycle, reward_logs)) # logger.info("Total payment amount is {:,} mutez. %s".format(total_amount_to_pay), # "" if self.delegator_pays_xfer_fee else "(Transfer fee is not included)") logger.debug("Creating calculation report (%s)", report_file_path) # 6- create calculations report file. This file contains calculations details self.create_calculations_report(reward_logs, report_file_path, total_amount) # 7- next cycle # processing of cycle is done logger.info( "Reward creation is done for cycle {}, created {} rewards.".format(pymnt_cycle, len(reward_logs))) elif total_amount_to_pay == 0: logger.info("Total payment amount is 0. Nothing to pay!") return True except ReadTimeout: logger.info("Tzscan call failed, will try again.") logger.debug("Tzscan call failed", exc_info=False) return False except ConnectTimeout: logger.info("Tzscan connection failed, will try again.") logger.debug("Tzscan connection failed", exc_info=False) return False except TzScanException: logger.info("Tzscan error at reward loop, will try again.") logger.debug("Tzscan error at reward loop", exc_info=False) return False except Exception: logger.error("Error at payment producer loop, will try again.", exc_info=True) return False finally: sleep(10)
def exit(self): if not self.exiting: self.payments_queue.put(PaymentBatch(self, 0, [self.create_exit_payment()])) self.exiting = True if self.life_cycle.is_running() and threading.current_thread() is not threading.main_thread(): _thread.interrupt_main() if self.retry_fail_event: self.retry_fail_event.set()
def main(args): logger.info( "Arguments Configuration = {}".format(json.dumps(args.__dict__, indent=1)) ) # Load payments file payments_file = os.path.expanduser(os.path.normpath(args.payments_file)) if not os.path.isfile(payments_file): raise Exception("payments_file ({}) does not exist.".format(payments_file)) with open(payments_file, "r") as file: payment_lines = file.readlines() payments_dict = {} for line in payment_lines: pkh, amt = line.split(":") pkh = pkh.strip() amt = float(amt.strip()) payments_dict[pkh] = amt if not payments_dict: raise Exception("No payments to process") # Check if dry-run dry_run = args.dry_run # Get reporting directories reports_dir = os.path.expanduser(os.path.normpath(args.base_directory)) # Check the disk size at the reports dir location if disk_is_full(reports_dir): raise Exception( "Disk is full at {}. Please free space to continue saving reports.".format( reports_dir ) ) # if in reports run mode, do not create consumers # create reports in reports directory if dry_run: reports_dir = os.path.join(reports_dir, SIMULATIONS_DIR, "") else: reports_dir = os.path.join(reports_dir, REPORTS_DIR, "") reports_dir = os.path.join(reports_dir, "manual", "") payments_root = get_payment_root(reports_dir, create=True) get_successful_payments_dir(payments_root, create=True) get_failed_payments_dir(payments_root, create=True) client_manager = ClientManager( node_endpoint=args.node_endpoint, signer_endpoint=args.signer_endpoint ) for i in range(NB_CONSUMERS): c = PaymentConsumer( name="manual_payment_consumer", payments_dir=payments_root, key_name=args.paymentaddress, payments_queue=payments_queue, node_addr=args.node_endpoint, client_manager=client_manager, dry_run=dry_run, reactivate_zeroed=False, delegator_pays_ra_fee=False, delegator_pays_xfer_fee=False, ) time.sleep(1) c.start() base_name_no_ext = os.path.basename(payments_file) base_name_no_ext = os.path.splitext(base_name_no_ext)[0] now = datetime.now() now_str = now.strftime("%Y%m%d%H%M%S") file_name = base_name_no_ext + "_" + now_str payment_items = [] for key, value in payments_dict.items(): pi = RewardLog.ExternalInstance(file_name, key, value) pi.payment = pi.payment * MUTEZ payment_items.append(pi) logger.info( "Reward created for cycle %s address %s amount %f fee %f tz type %s", pi.cycle, pi.address, pi.payment, pi.fee, pi.type, ) payments_queue.put(PaymentBatch(None, 0, payment_items)) payments_queue.put(PaymentBatch(None, 0, [RewardLog.ExitInstance()]))
def main(args): logger.info("Arguments Configuration = {}".format( json.dumps(args.__dict__, indent=1))) # 1- find where configuration is config_dir = os.path.expanduser(args.config_dir) # create configuration directory if it is not present # so that user can easily put his configuration there if config_dir and not os.path.exists(config_dir): os.makedirs(config_dir) # 3- load payments file payments_file = os.path.expanduser(args.payments_file) if not os.path.isfile(payments_file): raise Exception( "payments_file ({}) does not exist.".format(payments_file)) with open(payments_file, 'r') as file: payment_lines = file.readlines() payments_dict = {} for line in payment_lines: pkh, amt = line.split(":") pkh = pkh.strip() amt = float(amt.strip()) payments_dict[pkh] = amt if not payments_dict: raise Exception("No payments to process") # 6- is it a reports run dry_run = args.dry_run # 7- get reporting directories reports_dir = os.path.expanduser(args.reports_dir) # if in reports run mode, do not create consumers # create reports in reports directory if dry_run: reports_dir = os.path.expanduser("./reports") reports_dir = os.path.join(reports_dir, "manual") payments_root = get_payment_root(reports_dir, create=True) get_successful_payments_dir(payments_root, create=True) get_failed_payments_dir(payments_root, create=True) client_manager = ClientManager(node_endpoint=args.node_endpoint, signer_endpoint=args.signer_endpoint) for i in range(NB_CONSUMERS): c = PaymentConsumer(name='manual_payment_consumer', payments_dir=payments_root, key_name=args.paymentaddress, payments_queue=payments_queue, node_addr=args.node_endpoint, client_manager=client_manager, dry_run=dry_run, reactivate_zeroed=False, delegator_pays_ra_fee=False, delegator_pays_xfer_fee=False) time.sleep(1) c.start() base_name_no_ext = os.path.basename(payments_file) base_name_no_ext = os.path.splitext(base_name_no_ext)[0] now = datetime.now() now_str = now.strftime("%Y%m%d%H%M%S") file_name = base_name_no_ext + "_" + now_str payment_items = [] for key, value in payments_dict.items(): pi = RewardLog.ExternalInstance(file_name, key, value) pi.payment = pi.payment * MUTEZ payment_items.append(pi) logger.info( "Reward created for cycle %s address %s amount %f fee %f tz type %s", pi.cycle, pi.address, pi.payment, pi.fee, pi.type) payments_queue.put(PaymentBatch(None, 0, payment_items)) payments_queue.put(PaymentBatch(None, 0, [RewardLog.ExitInstance()]))
def retry_failed_payments(self, retry_injected=False): logger.debug("retry_failed_payments started") # 1 - list csv files under payments/failed directory # absolute path of csv files found under payments_root/failed directory failed_payments_dir = get_failed_payments_dir(self.payments_root) payment_reports_failed = [ os.path.join(failed_payments_dir, x) for x in os.listdir(failed_payments_dir) if x.endswith('.csv') ] if payment_reports_failed: payment_reports_failed = sorted( payment_reports_failed, key=lambda x: int(os.path.splitext(os.path.basename(x))[0])) logger.debug("Failed payment files found are: '{}'".format( ",".join(payment_reports_failed))) else: logger.debug( "No failed payment files found under directory '{}'".format( failed_payments_dir)) # 2- for each csv file with name csv_report.csv for payment_failed_report_file in payment_reports_failed: logger.info("Working on failed payment file {}".format( payment_failed_report_file)) # 2.1 - if there is a file csv_report.csv under payments/done, it means payment is already done if os.path.isfile( payment_failed_report_file.replace(PAYMENT_FAILED_DIR, PAYMENT_DONE_DIR)): # remove payments/failed/csv_report.csv os.remove(payment_failed_report_file) logger.info( "Payment for failed payment {} is already done. Removing.". format(payment_failed_report_file)) # remove payments/failed/csv_report.csv.BUSY # if there is a busy failed payment report file, remove it. remove_busy_file(payment_failed_report_file) # do not double pay continue # 2.2 - if queue is full, wait for sometime # make sure the queue is not full while self.payments_queue.full(): logger.debug("Payments queue is full. Wait a few minutes.") time.sleep(60 * 3) cycle = int( os.path.splitext( os.path.basename(payment_failed_report_file))[0]) # 2.3 read payments/failed/csv_report.csv file into a list of dictionaries batch = CsvPaymentFileParser().parse(payment_failed_report_file, cycle) nb_paid = len( list(filter(lambda f: f.paid == PaymentStatus.PAID, batch))) nb_done = len( list(filter(lambda f: f.paid == PaymentStatus.DONE, batch))) nb_injected = len( list(filter(lambda f: f.paid == PaymentStatus.INJECTED, batch))) nb_failed = len( list(filter(lambda f: f.paid == PaymentStatus.FAIL, batch))) logger.info( "Summary {} paid, {} done, {} injected, {} fail".format( nb_paid, nb_done, nb_injected, nb_failed)) if retry_injected: nb_converted = 0 for pl in batch: if pl.paid == PaymentStatus.INJECTED: pl.paid = PaymentStatus.FAIL nb_converted += 1 logger.debug( "Reward converted from %s to fail for cycle %s, address %s, amount %f, tz type %s", pl.paid, pl.cycle, pl.address, pl.amount, pl.type) if nb_converted: logger.info( "{} rewards converted from injected to fail.".format( nb_converted)) # 2.4 - Filter batch to only include those which failed. No need to mess with PAID/DONE batch = list(filter(lambda f: f.paid == PaymentStatus.FAIL, batch)) # 2.5 - Need to fetch current balance for addresses of any failed payments self.reward_api.update_current_balances(batch) # 2.6 - put records into payment_queue. payment_consumer will make payments self.payments_queue.put(PaymentBatch(self, cycle, batch)) # 2.7 - rename payments/failed/csv_report.csv to payments/failed/csv_report.csv.BUSY # mark the files as in use. we do not want it to be read again # BUSY file will be removed, if successful payment is done os.rename(payment_failed_report_file, payment_failed_report_file + BUSY_FILE)
def try_to_pay(self, pymnt_cycle, expected_rewards=False): try: logger.info("Payment cycle is " + str(pymnt_cycle)) # 0- check for past payment evidence for current cycle past_payment_state = check_past_payment(self.payments_root, pymnt_cycle) if not self.dry_run and past_payment_state: logger.warn(past_payment_state) return True # 1- get reward data if expected_rewards: logger.info( "Using expected/ideal rewards for payouts calculations") else: logger.info("Using actual rewards for payouts calculations") reward_model = self.reward_api.get_rewards_for_cycle_map( pymnt_cycle, expected_rewards) # 2- calculate rewards reward_logs, total_amount = self.payment_calc.calculate( reward_model) # 3- set cycle info for rl in reward_logs: rl.cycle = pymnt_cycle total_amount_to_pay = sum( [rl.amount for rl in reward_logs if rl.payable]) # 4- if total_rewards > 0, proceed with payment if total_amount_to_pay > 0: report_file_path = get_calculation_report_file( self.calculations_dir, pymnt_cycle) # 5- send to payment consumer self.payments_queue.put( PaymentBatch(self, pymnt_cycle, reward_logs)) # logger.info("Total payment amount is {:,} mutez. %s".format(total_amount_to_pay), # "" if self.delegator_pays_xfer_fee else "(Transfer fee is not included)") logger.debug("Creating calculation report (%s)", report_file_path) sleep(5.0) # 6- create calculations report file. This file contains calculations details self.create_calculations_report(reward_logs, report_file_path, total_amount, expected_rewards) # 7- processing of cycle is done logger.info( "Reward creation is done for cycle {}, created {} rewards." .format(pymnt_cycle, len(reward_logs))) elif total_amount_to_pay == 0: logger.info("Total payment amount is 0. Nothing to pay!") except ApiProviderException as a: logger.error("[try_to_pay] API provider error {:s}".format(str(a))) raise a from a except Exception as e: logger.error("[try_to_pay] Generic exception {:s}".format(str(e))) raise e from e # Either succeeded or raised exception return True
def try_to_pay(self, pymnt_cycle, rewards_type, network_config): try: logger.info("Payment cycle is {:s}".format(str(pymnt_cycle))) # 0- check for past payment evidence for current cycle past_payment_state = check_past_payment(self.payments_root, pymnt_cycle) if past_payment_state: logger.warn(past_payment_state) return True # 1- get reward data reward_model = self.reward_api.get_rewards_for_cycle_map( pymnt_cycle, rewards_type) # 2- compute reward amount to distribute based on configuration reward_model.computed_reward_amount = self.compute_rewards( reward_model, rewards_type, network_config) # 3- calculate rewards for delegators reward_logs, total_amount = self.payment_calc.calculate( reward_model) # 4- set cycle info for rl in reward_logs: rl.cycle = pymnt_cycle total_amount_to_pay = sum( [rl.amount for rl in reward_logs if rl.payable]) # 5- if total_rewards > 0, proceed with payment if total_amount_to_pay > 0: # 6- send to payment consumer self.payments_queue.put( PaymentBatch(self, pymnt_cycle, reward_logs)) sleep(5.0) # 7- create calculations report file. This file contains calculations details report_file_path = get_calculation_report_file( self.calculations_dir, pymnt_cycle) logger.debug("Creating calculation report (%s)", report_file_path) self.create_calculations_report(reward_logs, report_file_path, total_amount, rewards_type) # 8- processing of cycle is done logger.info( "Reward creation is done for cycle {}, created {} rewards." .format(pymnt_cycle, len(reward_logs))) elif total_amount_to_pay == 0: logger.info("Total payment amount is 0. Nothing to pay!") except ApiProviderException as a: logger.error("[try_to_pay] API provider error {:s}".format(str(a))) raise a from a except Exception as e: logger.error("[try_to_pay] Generic exception {:s}".format(str(e))) raise e from e # Either succeeded or raised exception return True
def main(args): logger.info("Arguments Configuration = {}".format(json.dumps(args.__dict__, indent=1))) # 1- find where configuration is config_dir = os.path.expanduser(args.config_dir) # create configuration directory if it is not present # so that user can easily put his configuration there if config_dir and not os.path.exists(config_dir): os.makedirs(config_dir) # 2- Load master configuration file if it is present master_config_file_path = os.path.join(config_dir, "master.yaml") master_cfg = {} if os.path.isfile(master_config_file_path): logger.info("Loading master configuration file {}".format(master_config_file_path)) master_parser = YamlConfParser(ConfigParser.load_file(master_config_file_path)) master_cfg = master_parser.parse() else: logger.info("master configuration file not present.") managers = None contracts_by_alias = None addresses_by_pkh = None if 'managers' in master_cfg: managers = master_cfg['managers'] if 'contracts_by_alias' in master_cfg: contracts_by_alias = master_cfg['contracts_by_alias'] if 'addresses_by_pkh' in master_cfg: addresses_by_pkh = master_cfg['addresses_by_pkh'] # 3- load payments file payments_file = os.path.expanduser(args.payments_file) if not os.path.isfile(payments_file): raise Exception("payments_file ({}) does not exist.".format(payments_file)) with open(payments_file, 'r') as file: payment_lines = file.readlines() payments_dict = {} for line in payment_lines: pkh, amt = line.split(":") pkh = pkh.strip() amt = float(amt.strip()) payments_dict[pkh] = amt if not payments_dict: raise Exception("No payments to process") # 3- get client path client_path = get_client_path([x.strip() for x in args.executable_dirs.split(',')], args.docker, args.network, args.verbose) logger.debug("Dune client path is {}".format(client_path)) # 4- get client path client_path = get_client_path([x.strip() for x in args.executable_dirs.split(',')], args.docker, args.network, args.verbose) logger.debug("Dune client path is {}".format(client_path)) # 6- is it a reports run dry_run = args.dry_run # 7- get reporting directories reports_dir = os.path.expanduser(args.reports_dir) # if in reports run mode, do not create consumers # create reports in reports directory if dry_run: reports_dir = os.path.expanduser("./reports") reports_dir = os.path.join(reports_dir, "manual") payments_root = get_payment_root(reports_dir, create=True) get_successful_payments_dir(payments_root, create=True) get_failed_payments_dir(payments_root, create=True) wllt_clnt_mngr = WalletClientManager(client_path, contracts_by_alias, addresses_by_pkh, managers) for i in range(NB_CONSUMERS): c = PaymentConsumer(name='manual_payment_consumer', payments_dir=payments_root, key_name=args.paymentaddress, client_path=client_path, payments_queue=payments_queue, node_addr=args.node_addr, wllt_clnt_mngr=wllt_clnt_mngr, verbose=args.verbose, dry_run=dry_run, delegator_pays_xfer_fee=False) time.sleep(1) c.start() base_name_no_ext = os.path.basename(payments_file) base_name_no_ext = os.path.splitext(base_name_no_ext)[0] now = datetime.now() now_str = now.strftime("%Y%m%d%H%M%S") file_name = base_name_no_ext + "_" + now_str payment_items = [] for key, value in payments_dict.items(): pi = RewardLog.ExternalInstance(file_name, key, value) pi.payment = pi.payment * MUTEZ payment_items.append(pi) logger.info("Reward created for cycle %s address %s amount %f fee %f dn type %s", pi.cycle, pi.address, pi.payment, pi.fee, pi.type) payments_queue.put(PaymentBatch(None, 0, payment_items)) payments_queue.put(PaymentBatch(None, 0, [RewardLog.ExitInstance()]))
def retry_failed_payments(self): logger.debug("retry_failed_payments started") # 1 - list csv files under payments/failed directory # absolute path of csv files found under payments_root/failed directory failed_payments_dir = get_failed_payments_dir(self.payments_root) payment_reports_failed = [ join(failed_payments_dir, x) for x in listdir(failed_payments_dir) if x.endswith(".csv") and int(x.split(".csv")[0]) >= self.initial_payment_cycle ] if payment_reports_failed: payment_reports_failed = sorted(payment_reports_failed, key=self.get_basename) logger.debug("Failed payment files found are: '{}'".format( ",".join(payment_reports_failed))) else: logger.info( "No failed payment files found under directory '{}' on or after cycle '{}'" .format(failed_payments_dir, self.initial_payment_cycle)) # 2- for each csv file with name csv_report.csv for payment_failed_report_file in payment_reports_failed: logger.info("Working on failed payment file {}".format( payment_failed_report_file)) # 2.1 - Read csv_report.csv under payments/failed and -if existing- under payments/done cycle = int( os.path.splitext( os.path.basename(payment_failed_report_file))[0]) batch = CsvPaymentFileParser().parse(payment_failed_report_file, cycle) payment_successful_report_file = payment_failed_report_file.replace( PAYMENT_FAILED_DIR, PAYMENT_DONE_DIR) if os.path.isfile(payment_successful_report_file): batch += CsvPaymentFileParser().parse( payment_successful_report_file, cycle) # 2.2 Translate batch into a list of dictionaries nb_paid = len( list(filter(lambda f: f.paid == PaymentStatus.PAID, batch))) nb_done = len( list(filter(lambda f: f.paid == PaymentStatus.DONE, batch))) nb_injected = len( list(filter(lambda f: f.paid == PaymentStatus.INJECTED, batch))) nb_failed = len( list(filter(lambda f: f.paid == PaymentStatus.FAIL, batch))) nb_avoided = len( list(filter(lambda f: f.paid == PaymentStatus.AVOIDED, batch))) logger.info( "Summary {} paid, {} done, {} injected, {} fail, {} avoided". format(nb_paid, nb_done, nb_injected, nb_failed, nb_avoided)) if self.retry_injected: self.convert_injected_to_fail(batch) # 2.3 - if queue is full, wait for sometime # make sure the queue is not full while self.payments_queue.full(): logger.debug( "Payments queue is full. Please wait three minutes.") sleep(60 * 3) # 2.5 - Need to fetch current balance for addresses of any failed payments self.reward_api.update_current_balances(batch) # 2.6 - put records into payment_queue. payment_consumer will make payments self.payments_queue.put( PaymentBatch(self.payment_producer, cycle, batch)) # 2.7 - rename payments/failed/csv_report.csv to payments/failed/csv_report.csv.BUSY # mark the files as in use. we do not want it to be read again # BUSY file will be removed, if successful payment is done os.rename(payment_failed_report_file, payment_failed_report_file + BUSY_FILE) return
def try_to_pay(self, pymnt_cycle, rewards_type, network_config, current_cycle): try: logger.info("Payment cycle is {:s}".format(str(pymnt_cycle))) # 0- check for past payment evidence for current cycle past_payment_state = check_past_payment(self.payments_root, pymnt_cycle) if past_payment_state: logger.warn(past_payment_state) return True adjustments = {} early_payout = False current_cycle_rewards_type = rewards_type # 1- adjust past cycle if necessary if self.release_override == -11 and pymnt_cycle >= current_cycle: early_payout = True completed_cycle = pymnt_cycle - 6 adjustments = self.recompute_rewards(completed_cycle, rewards_type, network_config) # payout for current cycle will be estimated since we don't know actual rewards yet current_cycle_rewards_type = RewardsType.ESTIMATED # 2- get reward data and compute how to distribute them reward_logs, total_amount = self.compute_rewards( pymnt_cycle, current_cycle_rewards_type, network_config, adjustments) total_recovered_adjustments = int( sum([rl.adjustment for rl in reward_logs])) total_adjustments_to_recover = int(sum(adjustments.values())) if total_adjustments_to_recover > 0: logger.debug( "Total adjustments to recover is {:<,d} mutez, total recovered adjustment is {:<,d} mutez." .format(total_adjustments_to_recover, total_recovered_adjustments)) logger.info( "After early payout of cycle {:s}, {:<,d} mutez were not recovered." .format( str(completed_cycle), total_adjustments_to_recover + total_recovered_adjustments, )) # 3- create calculations report file. This file contains calculations details report_file_path = get_calculation_report_file_path( self.calculations_dir, pymnt_cycle) logger.debug("Creating calculation report (%s)", report_file_path) CsvCalculationFileParser().write( reward_logs, report_file_path, total_amount, current_cycle_rewards_type, self.baking_address, early_payout, ) # 4- set cycle info for reward_log in reward_logs: reward_log.cycle = pymnt_cycle total_amount_to_pay = int( sum([ reward_log.adjusted_amount for reward_log in reward_logs if reward_log.payable ])) # 5- if total_rewards > 0, proceed with payment if total_amount_to_pay > 0: self.payments_queue.put( PaymentBatch(self, pymnt_cycle, reward_logs)) sleep(5.0) # 6- processing of cycle is done logger.info( "Reward creation is done for cycle {}, created {} rewards." .format(pymnt_cycle, len(reward_logs))) elif total_amount_to_pay == 0: logger.info("Total payment amount is 0. Nothing to pay!") except ApiProviderException as a: logger.error("[try_to_pay] API provider error {:s}".format(str(a))) raise a from a except Exception as e: logger.error("[try_to_pay] Generic exception {:s}".format(str(e))) raise e from e # Either succeeded or raised exception return True