def checkpoint_stage(stage, result):
checkpoint = utils.get_checkpoint().get(stage, {})
checkpoint.update({'result': result})
utils.update_checkpoint({
'stage': stages.index(stage),
stage: checkpoint,
})
def checkpoint_stage(stage, result):
checkpoint = utils.get_checkpoint().get(stage, {})
checkpoint.update({'result': result})
utils.update_checkpoint({
'stage': stages.index(stage),
stage: checkpoint,
})
def do_sieve(parameters, polysel_result):
result = {}
name = parameters.myparams({"name": str}, ['tasks'])["name"]
workdir = parameters.myparams({"workdir": str}, ['tasks'])["workdir"]
# msieve_filter is the last checkpointed stage in this block
if stage_required('sieve'):
sieve_start = time.time()
sieving_offset = utils.slurm_cpu_time_start()
s = sieving.Sieve(parameters, polysel_result['polyfile'])
relation_files = s.run()
result['relation_files'] = relation_files
sieve_finish = time.time()
result['duration'] = sieve_finish - sieve_start
logger.info("\tSieving in %s", utils.str_time(result['duration']))
if parameters.myparams({
"collect_cputime": [str]
}, ['tasks', 'sieve']).get("collect_cputime"):
logger.info("--- Collecting sieving cumulative CPUTime ---")
sieving_slurm_outfile = str(
os.path.join(workdir, name + ".slurm_sieving"))
result['cputime'] = utils.slurm_cpu_time_end(
sieving_offset, sieving_slurm_outfile)
logger.info("\tSieving cumulative CPUTime %s" %
utils.str_time(result['cputime']))
else:
result['cputime'] = 0
relation_files_file = str(
os.path.join(workdir, name + '.relation_files'))
logger.info("Saving used relation files from this run to %s",
relation_files_file)
with open(relation_files_file, 'w') as f:
for line in relation_files:
f.write(line + '\n')
if s.completed_factorization:
logger.info(
"Completed factorization using trial division. Check log file for factors."
)
utils.update_checkpoint({'trail_division': True})
checkpoint_stage('sieve', result)
else:
result = load_stage('sieve')
post_sieve = parameters.myparams({
'post_sieve': None
}, ['commands']).get('post_sieve')
if post_sieve != None:
logger.info('Post-sieve command %s', post_sieve)
utils.run_command(post_sieve, logger=logger)
return result
def do_sieve(parameters, polysel_result):
result = {}
name = parameters.myparams({"name": str}, ['tasks'])["name"]
workdir = parameters.myparams({"workdir": str}, ['tasks'])["workdir"]
# msieve_filter is the last checkpointed stage in this block
if stage_required('sieve'):
sieve_start = time.time()
sieving_offset = utils.slurm_cpu_time_start()
s = sieving.Sieve(parameters, polysel_result['polyfile'])
relation_files = s.run()
result['relation_files'] = relation_files
sieve_finish = time.time()
result['duration'] = sieve_finish - sieve_start
logger.info("\tSieving in %s", utils.str_time(result['duration']))
if parameters.myparams({"collect_cputime": [str]}, ['tasks', 'sieve']).get("collect_cputime"):
logger.info("--- Collecting sieving cumulative CPUTime ---")
sieving_slurm_outfile = str(os.path.join(workdir, name + ".slurm_sieving"))
result['cputime'] = utils.slurm_cpu_time_end(sieving_offset, sieving_slurm_outfile)
logger.info("\tSieving cumulative CPUTime %s" % utils.str_time(result['cputime']))
else:
result['cputime'] = 0
relation_files_file = str(os.path.join(workdir, name + '.relation_files'))
logger.info("Saving used relation files from this run to %s", relation_files_file)
with open(relation_files_file, 'w') as f:
for line in relation_files:
f.write(line+'\n')
if s.completed_factorization:
logger.info("Completed factorization using trial division. Check log file for factors.")
utils.update_checkpoint({
'trail_division': True})
checkpoint_stage('sieve', result)
else:
result = load_stage('sieve')
post_sieve = parameters.myparams({'post_sieve': None}, ['commands']).get('post_sieve')
if post_sieve != None:
logger.info('Post-sieve command %s', post_sieve)
utils.run_command(post_sieve, logger=logger)
return result
def main():
signal.signal(signal.SIGINT, signal_handler)
parser = argparse.ArgumentParser(description="Integer Factorization with "
"the Number Field Sieve")
parser.add_argument("parameters", help="A file with the parameters to use")
parser.add_argument("options", metavar="OPTION", help="An option as in "
"parameter file (format: key=value)", nargs="*")
parser.add_argument('--resume','-r', help="checkpoint file to resume from")
parser.add_argument('--stage','-s', action='append', help="stage to complete ('start','polysel','sieving','linalg','complete'), add + to run all subsequent stages")
args = parser.parse_args()
parameters = utils.get_params(args.parameters, args.options)
name = parameters.myparams({"name": str}, ['tasks'])["name"]
workdir = parameters.myparams({"workdir": str}, ['tasks'])["workdir"]
if not os.path.exists(workdir):
logger.info("Creating work directory %s", workdir)
os.makedirs(workdir)
setup_logging(workdir, name)
# Load or create initial checkpoint
checkpoint_file = args.resume
if not checkpoint_file:
checkpoint_file = os.path.join(workdir, "checkpoint.dat")
utils.init_checkpoint(checkpoint_file)
# set parameters that are unlikely to change from run to run, such as filenames and directories
parameters = set_static_parameters(parameters)
# check that all required parameters are present
params = check_parameters(parameters)
utils.update_checkpoint({'params': params})
# set parameters that will likely change from run to run
parameters = set_dynamic_parameters(parameters)
# Write a snapshot of the parameters to a file
snapshot_filename = "%s/%s.parameters_snapshot" % (workdir, name)
with open(snapshot_filename, "w") as snapshot_file:
logger.debug("Writing parameter snapshot to %s", snapshot_filename)
snapshot_file.write(str(parameters))
snapshot_file.write("\n")
start_time = time.time()
# For each checkpointed stage, check if the stage should be run again.
# A stage should be run again under the following circumstances:
# - The user manually requested to run the stage
# - No checkpoint exists for the stage
# - A stage on which this stage depends will be re-run
# - Parameters on which the stage depends have been changed since the last run
if args.stage:
for stage in args.stage:
if stage.endswith('+'):
stage = stage[:-1]
if stage not in stages:
continue
stage_required.manual_stages = range(stages.index(stage), len(stages))
break
if stage not in stages:
args.stage.pop(stage)
else:
stage_required.manual_stages.append(stages.index(stage))
else:
# since no stage were specified to run manually, choose the first stage based on the checkpoint file
stage_required.manual_stages = set_manual_stages(params)
# Run polynomial selection
polysel_result = do_polysel(parameters)
# Run sieving
sieve_result = do_sieve(parameters, polysel_result)
# Run linalg
linalg_result = do_linalg(parameters, sieve_result)
# Run square root
sqrt_result = do_sqrt(parameters, linalg_result)
factoring_duration = polysel_result['duration'] + sieve_result['duration'] + linalg_result['duration'] + sqrt_result['duration']
logger.info('Factoring completed in %s', utils.str_time(factoring_duration))
logger.info('\tPolysel in real/cpu %s/%s', utils.str_time(polysel_result['duration']), utils.str_time(polysel_result['cputime']))
logger.info("\tSieving in real/cpu %s/%s", utils.str_time(sieve_result['duration']), utils.str_time(sieve_result['cputime']))
logger.info("\tLinalg in %s", utils.str_time(linalg_result['duration']))
logger.info("\tSqrt in %s", utils.str_time(sqrt_result['duration']))
logger.info("\tFactors %s", ','.join(sqrt_result['factors']))
post_factor = parameters.myparams({'post_factor': None}, ['commands']).get('post_factor')
if post_factor != None:
logger.info('Post-factor command %s', post_factor)
utils.run_command(post_factor, logger=logger)
def main():
signal.signal(signal.SIGINT, signal_handler)
parser = argparse.ArgumentParser(description="Integer Factorization with "
"the Number Field Sieve")
parser.add_argument("parameters", help="A file with the parameters to use")
parser.add_argument("options",
metavar="OPTION",
help="An option as in "
"parameter file (format: key=value)",
nargs="*")
parser.add_argument('--resume',
'-r',
help="checkpoint file to resume from")
parser.add_argument(
'--stage',
'-s',
action='append',
help=
"stage to complete ('start','polysel','sieving','linalg','complete'), add + to run all subsequent stages"
)
args = parser.parse_args()
parameters = utils.get_params(args.parameters, args.options)
name = parameters.myparams({"name": str}, ['tasks'])["name"]
workdir = parameters.myparams({"workdir": str}, ['tasks'])["workdir"]
if not os.path.exists(workdir):
logger.info("Creating work directory %s", workdir)
os.makedirs(workdir)
setup_logging(workdir, name)
# Load or create initial checkpoint
checkpoint_file = args.resume
if not checkpoint_file:
checkpoint_file = os.path.join(workdir, "checkpoint.dat")
utils.init_checkpoint(checkpoint_file)
# set parameters that are unlikely to change from run to run, such as filenames and directories
parameters = set_static_parameters(parameters)
# check that all required parameters are present
params = check_parameters(parameters)
utils.update_checkpoint({'params': params})
# set parameters that will likely change from run to run
parameters = set_dynamic_parameters(parameters)
# Write a snapshot of the parameters to a file
snapshot_filename = "%s/%s.parameters_snapshot" % (workdir, name)
with open(snapshot_filename, "w") as snapshot_file:
logger.debug("Writing parameter snapshot to %s", snapshot_filename)
snapshot_file.write(str(parameters))
snapshot_file.write("\n")
start_time = time.time()
# For each checkpointed stage, check if the stage should be run again.
# A stage should be run again under the following circumstances:
# - The user manually requested to run the stage
# - No checkpoint exists for the stage
# - A stage on which this stage depends will be re-run
# - Parameters on which the stage depends have been changed since the last run
if args.stage:
for stage in args.stage:
if stage.endswith('+'):
stage = stage[:-1]
if stage not in stages:
continue
stage_required.manual_stages = range(stages.index(stage),
len(stages))
break
if stage not in stages:
args.stage.pop(stage)
else:
stage_required.manual_stages.append(stages.index(stage))
else:
# since no stage were specified to run manually, choose the first stage based on the checkpoint file
stage_required.manual_stages = set_manual_stages(params)
# Run polynomial selection
polysel_result = do_polysel(parameters)
# Run sieving
sieve_result = do_sieve(parameters, polysel_result)
# Run linalg
linalg_result = do_linalg(parameters, sieve_result)
# Run square root
sqrt_result = do_sqrt(parameters, linalg_result)
factoring_duration = polysel_result['duration'] + sieve_result[
'duration'] + linalg_result['duration'] + sqrt_result['duration']
logger.info('Factoring completed in %s',
utils.str_time(factoring_duration))
logger.info('\tPolysel in real/cpu %s/%s',
utils.str_time(polysel_result['duration']),
utils.str_time(polysel_result['cputime']))
logger.info("\tSieving in real/cpu %s/%s",
utils.str_time(sieve_result['duration']),
utils.str_time(sieve_result['cputime']))
logger.info("\tLinalg in %s", utils.str_time(linalg_result['duration']))
logger.info("\tSqrt in %s", utils.str_time(sqrt_result['duration']))
logger.info("\tFactors %s", ','.join(sqrt_result['factors']))
post_factor = parameters.myparams({
'post_factor': None
}, ['commands']).get('post_factor')
if post_factor != None:
logger.info('Post-factor command %s', post_factor)
utils.run_command(post_factor, logger=logger)
