def main():
max_passage_length = 256
encoder = PassageSampling(max_passage_length)
max_seq_length = 512
worker_factory = partial(RobustWorker, RobustPointwiseTrainGenEx(encoder, max_seq_length))
runner = JobRunner(job_man_dir, 4, "robust_train_sampling_256", worker_factory)
runner.start()
def generate_robust_sero_for_train():
total_sequence_length = 512 * 4
src_window_size = 512
encoder = MultiWindow(src_window_size, total_sequence_length)
worker_factory = partial(RobustWorker, RobustPointwiseTrainGenEx(encoder, total_sequence_length, "desc"))
runner = JobRunner(sydney_working_dir, 4, "RobustSero5", worker_factory)
runner.start()
def generate_robust_sero_for_train():
total_sequence_length = 128 * 32
src_window_size = 128
encoder = MultiWindowOverlap(src_window_size, total_sequence_length)
worker_factory = partial(RobustWorker, RobustPointwiseTrainGenEx(encoder, total_sequence_length, "desc"))
runner = JobRunner(job_man_dir, 4, "RobustSero_128_32_overlap", worker_factory)
runner.start()
def main():
max_seq_length = 512
encoder = AllSegmentAsDoc(max_seq_length)
worker_factory = partial(
RobustWorker, RobustPointwiseTrainGenEx(encoder, max_seq_length))
runner = JobRunner(job_man_dir, 4, "robust_all_passage_pointwise_ex",
worker_factory)
runner.start()
def main():
max_passage_length = 128
num_segment = 1
encoder = LeadingN(max_passage_length, num_segment)
max_seq_length = max_passage_length
worker_factory = partial(RobustWorker, RobustPointwiseTrainGenEx(encoder, max_seq_length, "desc"))
runner = JobRunner(job_man_dir, 4, "first_128_desc", worker_factory)
runner.start()
def main():
max_passage_length = 128
g = 0.5
encoder = GeoSampler(max_passage_length, g)
max_seq_length = max_passage_length
worker_factory = partial(RobustWorker, RobustPointwiseTrainGenEx(encoder, max_seq_length, "desc"))
runner = JobRunner(job_man_dir, 4, "robust_geo05", worker_factory)
runner.start()
def main():
max_passage_length = 512
encoder = FirstEquiSero(max_passage_length, 128, 4)
max_seq_length = max_passage_length
worker_factory = partial(
RobustWorker, RobustPointwiseTrainGenEx(encoder, max_seq_length,
"desc"))
runner = JobRunner(job_man_dir, 4, "first_512_equi_sero", worker_factory)
runner.start()
def main():
max_passage_length = 128
num_segment = 4
encoder = FirstAndRandom(max_passage_length, num_segment)
max_seq_length = max_passage_length
worker_factory = partial(
RobustWorker, RobustPointwiseTrainGenEx(encoder, max_seq_length))
runner = JobRunner(job_man_dir, 4, "leading_segments", worker_factory)
runner.start()
def main():
max_passage_length = 128
num_segment = 4
encoder = LeadingSegmentsCombined(max_passage_length, num_segment)
max_seq_length = max_passage_length
worker_factory = partial(
RobustWorker, RobustPointwiseTrainGenEx(encoder, max_seq_length,
"desc"))
runner = JobRunner(job_man_dir, 4, "leading_segments_combined_desc",
worker_factory)
runner.start()
def generate_robust_sero_for_train():
window_size = int(sys.argv[1])
n_window = int(sys.argv[2])
total_sequence_length = window_size * n_window
src_window_size = window_size
encoder = MultiWindow(src_window_size, total_sequence_length)
worker_factory = partial(
RobustWorker,
RobustPointwiseTrainGenEx(encoder, total_sequence_length, "desc"))
runner = JobRunner(job_man_dir, 4,
"RobustSero_{}_{}".format(window_size,
n_window), worker_factory)
runner.auto_runner()
def main():
max_seq_length = 512
encoder = PassageSampling(max_seq_length)
worker_factory = partial(RobustWorker, RobustPointwiseTrainGenEx(encoder, max_seq_length, "title", 100))
runner = JobRunner(job_man_dir, 4, "robust_train_sampling_w_100neg", worker_factory)
runner.start()