args.sort_run_merger_read_batch_size, batch_processor_options=dj_pb.BatchProcessorOptions( batch_size=args.process_batch_size, max_flying_item=-1 ), input_raw_data=dj_pb.RawDataOptions( raw_data_iter=args.raw_data_iter, compressed_type=args.compressed_type, read_ahead_size=args.read_ahead_size, read_batch_size=args.read_batch_size ), writer_options=dj_pb.WriterOptions( output_writer=args.output_builder, compressed_type=args.builder_compressed_type, ) ) if args.psi_role.upper() == 'LEADER': preprocessor_options.role = common_pb.FLRole.Leader else: assert args.psi_role.upper() == 'FOLLOWER' preprocessor_options.role = common_pb.FLRole.Follower db_database, db_addr, db_username, db_password, db_base_dir = \ get_kvstore_config(args.kvstore_type) preprocessor = RsaPsiPreProcessor(preprocessor_options, db_database, db_base_dir, db_addr, db_username, db_password) preprocessor.start_process() logging.info("PreProcessor launched for %s of RSA PSI", args.psi_role) preprocessor.wait_for_finished() logging.info("PreProcessor finished for %s of RSA PSI", args.psi_role)
sort_run_merger_read_ahead_buffer=\ args.sort_run_merger_read_ahead_buffer, sort_run_merger_read_batch_size=\ args.sort_run_merger_read_batch_size, batch_processor_options=dj_pb.BatchProcessorOptions( batch_size=args.process_batch_size, max_flying_item=-1 ), input_raw_data=dj_pb.RawDataOptions( raw_data_iter=args.raw_data_iter, compressed_type=args.compressed_type, read_ahead_size=args.read_ahead_size, read_batch_size=args.read_batch_size ), writer_options=dj_pb.WriterOptions( output_writer=args.output_builder, compressed_type=args.builder_compressed_type, ) ) if args.psi_role.upper() == 'LEADER': preprocessor_options.role = common_pb.FLRole.Leader else: assert args.psi_role.upper() == 'FOLLOWER' preprocessor_options.role = common_pb.FLRole.Follower preprocessor = RsaPsiPreProcessor(preprocessor_options, args.etcd_name, args.etcd_addrs, args.etcd_base_dir) preprocessor.start_process() logging.info("PreProcessor launched for %s of RSA PSI", args.psi_role) preprocessor.wait_for_finished() logging.info("PreProcessor finished for %s of RSA PSI", args.psi_role)
slow_sign_threshold=args.slow_sign_threshold, sort_run_merger_read_ahead_buffer=\ args.sort_run_merger_read_ahead_buffer, sort_run_merger_read_batch_size=\ args.sort_run_merger_read_batch_size, batch_processor_options=dj_pb.BatchProcessorOptions( batch_size=args.process_batch_size, max_flying_item=-1 ), input_raw_data=dj_pb.RawDataOptions( raw_data_iter=args.raw_data_iter, compressed_type=args.compressed_type, read_ahead_size=args.read_ahead_size, read_batch_size=args.read_batch_size ), writer_options=dj_pb.WriterOptions( output_writer=args.output_builder, compressed_type=args.builder_compressed_type, ) ) if args.psi_role.upper() == 'LEADER': preprocessor_options.role = common_pb.FLRole.Leader else: assert args.psi_role.upper() == 'FOLLOWER' preprocessor_options.role = common_pb.FLRole.Follower preprocessor = RsaPsiPreProcessor(preprocessor_options, args.kvstore_type) preprocessor.start_process() logging.info("PreProcessor launched for %s of RSA PSI", args.psi_role) preprocessor.wait_for_finished() logging.info("PreProcessor finished for %s of RSA PSI", args.psi_role)