args.sort_run_merger_read_batch_size,
batch_processor_options=dj_pb.BatchProcessorOptions(
batch_size=args.process_batch_size,
max_flying_item=-1
),
input_raw_data=dj_pb.RawDataOptions(
raw_data_iter=args.raw_data_iter,
compressed_type=args.compressed_type,
read_ahead_size=args.read_ahead_size,
read_batch_size=args.read_batch_size
),
writer_options=dj_pb.WriterOptions(
output_writer=args.output_builder,
compressed_type=args.builder_compressed_type,
)
)
if args.psi_role.upper() == 'LEADER':
preprocessor_options.role = common_pb.FLRole.Leader
else:
assert args.psi_role.upper() == 'FOLLOWER'
preprocessor_options.role = common_pb.FLRole.Follower
db_database, db_addr, db_username, db_password, db_base_dir = \
get_kvstore_config(args.kvstore_type)
preprocessor = RsaPsiPreProcessor(preprocessor_options, db_database,
db_base_dir, db_addr, db_username,
db_password)
preprocessor.start_process()
logging.info("PreProcessor launched for %s of RSA PSI", args.psi_role)
preprocessor.wait_for_finished()
logging.info("PreProcessor finished for %s of RSA PSI", args.psi_role)
sort_run_merger_read_ahead_buffer=\
args.sort_run_merger_read_ahead_buffer,
sort_run_merger_read_batch_size=\
args.sort_run_merger_read_batch_size,
batch_processor_options=dj_pb.BatchProcessorOptions(
batch_size=args.process_batch_size,
max_flying_item=-1
),
input_raw_data=dj_pb.RawDataOptions(
raw_data_iter=args.raw_data_iter,
compressed_type=args.compressed_type,
read_ahead_size=args.read_ahead_size,
read_batch_size=args.read_batch_size
),
writer_options=dj_pb.WriterOptions(
output_writer=args.output_builder,
compressed_type=args.builder_compressed_type,
)
)
if args.psi_role.upper() == 'LEADER':
preprocessor_options.role = common_pb.FLRole.Leader
else:
assert args.psi_role.upper() == 'FOLLOWER'
preprocessor_options.role = common_pb.FLRole.Follower
preprocessor = RsaPsiPreProcessor(preprocessor_options, args.etcd_name,
args.etcd_addrs, args.etcd_base_dir)
preprocessor.start_process()
logging.info("PreProcessor launched for %s of RSA PSI", args.psi_role)
preprocessor.wait_for_finished()
logging.info("PreProcessor finished for %s of RSA PSI", args.psi_role)
slow_sign_threshold=args.slow_sign_threshold,
sort_run_merger_read_ahead_buffer=\
args.sort_run_merger_read_ahead_buffer,
sort_run_merger_read_batch_size=\
args.sort_run_merger_read_batch_size,
batch_processor_options=dj_pb.BatchProcessorOptions(
batch_size=args.process_batch_size,
max_flying_item=-1
),
input_raw_data=dj_pb.RawDataOptions(
raw_data_iter=args.raw_data_iter,
compressed_type=args.compressed_type,
read_ahead_size=args.read_ahead_size,
read_batch_size=args.read_batch_size
),
writer_options=dj_pb.WriterOptions(
output_writer=args.output_builder,
compressed_type=args.builder_compressed_type,
)
)
if args.psi_role.upper() == 'LEADER':
preprocessor_options.role = common_pb.FLRole.Leader
else:
assert args.psi_role.upper() == 'FOLLOWER'
preprocessor_options.role = common_pb.FLRole.Follower
preprocessor = RsaPsiPreProcessor(preprocessor_options, args.kvstore_type)
preprocessor.start_process()
logging.info("PreProcessor launched for %s of RSA PSI", args.psi_role)
preprocessor.wait_for_finished()
logging.info("PreProcessor finished for %s of RSA PSI", args.psi_role)