def checkpoint_stage(stage, result): checkpoint = utils.get_checkpoint().get(stage, {}) checkpoint.update({'result': result}) utils.update_checkpoint({ 'stage': stages.index(stage), stage: checkpoint, })
def do_sieve(parameters, polysel_result): result = {} name = parameters.myparams({"name": str}, ['tasks'])["name"] workdir = parameters.myparams({"workdir": str}, ['tasks'])["workdir"] # msieve_filter is the last checkpointed stage in this block if stage_required('sieve'): sieve_start = time.time() sieving_offset = utils.slurm_cpu_time_start() s = sieving.Sieve(parameters, polysel_result['polyfile']) relation_files = s.run() result['relation_files'] = relation_files sieve_finish = time.time() result['duration'] = sieve_finish - sieve_start logger.info("\tSieving in %s", utils.str_time(result['duration'])) if parameters.myparams({ "collect_cputime": [str] }, ['tasks', 'sieve']).get("collect_cputime"): logger.info("--- Collecting sieving cumulative CPUTime ---") sieving_slurm_outfile = str( os.path.join(workdir, name + ".slurm_sieving")) result['cputime'] = utils.slurm_cpu_time_end( sieving_offset, sieving_slurm_outfile) logger.info("\tSieving cumulative CPUTime %s" % utils.str_time(result['cputime'])) else: result['cputime'] = 0 relation_files_file = str( os.path.join(workdir, name + '.relation_files')) logger.info("Saving used relation files from this run to %s", relation_files_file) with open(relation_files_file, 'w') as f: for line in relation_files: f.write(line + '\n') if s.completed_factorization: logger.info( "Completed factorization using trial division. Check log file for factors." ) utils.update_checkpoint({'trail_division': True}) checkpoint_stage('sieve', result) else: result = load_stage('sieve') post_sieve = parameters.myparams({ 'post_sieve': None }, ['commands']).get('post_sieve') if post_sieve != None: logger.info('Post-sieve command %s', post_sieve) utils.run_command(post_sieve, logger=logger) return result
def do_sieve(parameters, polysel_result): result = {} name = parameters.myparams({"name": str}, ['tasks'])["name"] workdir = parameters.myparams({"workdir": str}, ['tasks'])["workdir"] # msieve_filter is the last checkpointed stage in this block if stage_required('sieve'): sieve_start = time.time() sieving_offset = utils.slurm_cpu_time_start() s = sieving.Sieve(parameters, polysel_result['polyfile']) relation_files = s.run() result['relation_files'] = relation_files sieve_finish = time.time() result['duration'] = sieve_finish - sieve_start logger.info("\tSieving in %s", utils.str_time(result['duration'])) if parameters.myparams({"collect_cputime": [str]}, ['tasks', 'sieve']).get("collect_cputime"): logger.info("--- Collecting sieving cumulative CPUTime ---") sieving_slurm_outfile = str(os.path.join(workdir, name + ".slurm_sieving")) result['cputime'] = utils.slurm_cpu_time_end(sieving_offset, sieving_slurm_outfile) logger.info("\tSieving cumulative CPUTime %s" % utils.str_time(result['cputime'])) else: result['cputime'] = 0 relation_files_file = str(os.path.join(workdir, name + '.relation_files')) logger.info("Saving used relation files from this run to %s", relation_files_file) with open(relation_files_file, 'w') as f: for line in relation_files: f.write(line+'\n') if s.completed_factorization: logger.info("Completed factorization using trial division. Check log file for factors.") utils.update_checkpoint({ 'trail_division': True}) checkpoint_stage('sieve', result) else: result = load_stage('sieve') post_sieve = parameters.myparams({'post_sieve': None}, ['commands']).get('post_sieve') if post_sieve != None: logger.info('Post-sieve command %s', post_sieve) utils.run_command(post_sieve, logger=logger) return result
def main(): signal.signal(signal.SIGINT, signal_handler) parser = argparse.ArgumentParser(description="Integer Factorization with " "the Number Field Sieve") parser.add_argument("parameters", help="A file with the parameters to use") parser.add_argument("options", metavar="OPTION", help="An option as in " "parameter file (format: key=value)", nargs="*") parser.add_argument('--resume','-r', help="checkpoint file to resume from") parser.add_argument('--stage','-s', action='append', help="stage to complete ('start','polysel','sieving','linalg','complete'), add + to run all subsequent stages") args = parser.parse_args() parameters = utils.get_params(args.parameters, args.options) name = parameters.myparams({"name": str}, ['tasks'])["name"] workdir = parameters.myparams({"workdir": str}, ['tasks'])["workdir"] if not os.path.exists(workdir): logger.info("Creating work directory %s", workdir) os.makedirs(workdir) setup_logging(workdir, name) # Load or create initial checkpoint checkpoint_file = args.resume if not checkpoint_file: checkpoint_file = os.path.join(workdir, "checkpoint.dat") utils.init_checkpoint(checkpoint_file) # set parameters that are unlikely to change from run to run, such as filenames and directories parameters = set_static_parameters(parameters) # check that all required parameters are present params = check_parameters(parameters) utils.update_checkpoint({'params': params}) # set parameters that will likely change from run to run parameters = set_dynamic_parameters(parameters) # Write a snapshot of the parameters to a file snapshot_filename = "%s/%s.parameters_snapshot" % (workdir, name) with open(snapshot_filename, "w") as snapshot_file: logger.debug("Writing parameter snapshot to %s", snapshot_filename) snapshot_file.write(str(parameters)) snapshot_file.write("\n") start_time = time.time() # For each checkpointed stage, check if the stage should be run again. # A stage should be run again under the following circumstances: # - The user manually requested to run the stage # - No checkpoint exists for the stage # - A stage on which this stage depends will be re-run # - Parameters on which the stage depends have been changed since the last run if args.stage: for stage in args.stage: if stage.endswith('+'): stage = stage[:-1] if stage not in stages: continue stage_required.manual_stages = range(stages.index(stage), len(stages)) break if stage not in stages: args.stage.pop(stage) else: stage_required.manual_stages.append(stages.index(stage)) else: # since no stage were specified to run manually, choose the first stage based on the checkpoint file stage_required.manual_stages = set_manual_stages(params) # Run polynomial selection polysel_result = do_polysel(parameters) # Run sieving sieve_result = do_sieve(parameters, polysel_result) # Run linalg linalg_result = do_linalg(parameters, sieve_result) # Run square root sqrt_result = do_sqrt(parameters, linalg_result) factoring_duration = polysel_result['duration'] + sieve_result['duration'] + linalg_result['duration'] + sqrt_result['duration'] logger.info('Factoring completed in %s', utils.str_time(factoring_duration)) logger.info('\tPolysel in real/cpu %s/%s', utils.str_time(polysel_result['duration']), utils.str_time(polysel_result['cputime'])) logger.info("\tSieving in real/cpu %s/%s", utils.str_time(sieve_result['duration']), utils.str_time(sieve_result['cputime'])) logger.info("\tLinalg in %s", utils.str_time(linalg_result['duration'])) logger.info("\tSqrt in %s", utils.str_time(sqrt_result['duration'])) logger.info("\tFactors %s", ','.join(sqrt_result['factors'])) post_factor = parameters.myparams({'post_factor': None}, ['commands']).get('post_factor') if post_factor != None: logger.info('Post-factor command %s', post_factor) utils.run_command(post_factor, logger=logger)
def main(): signal.signal(signal.SIGINT, signal_handler) parser = argparse.ArgumentParser(description="Integer Factorization with " "the Number Field Sieve") parser.add_argument("parameters", help="A file with the parameters to use") parser.add_argument("options", metavar="OPTION", help="An option as in " "parameter file (format: key=value)", nargs="*") parser.add_argument('--resume', '-r', help="checkpoint file to resume from") parser.add_argument( '--stage', '-s', action='append', help= "stage to complete ('start','polysel','sieving','linalg','complete'), add + to run all subsequent stages" ) args = parser.parse_args() parameters = utils.get_params(args.parameters, args.options) name = parameters.myparams({"name": str}, ['tasks'])["name"] workdir = parameters.myparams({"workdir": str}, ['tasks'])["workdir"] if not os.path.exists(workdir): logger.info("Creating work directory %s", workdir) os.makedirs(workdir) setup_logging(workdir, name) # Load or create initial checkpoint checkpoint_file = args.resume if not checkpoint_file: checkpoint_file = os.path.join(workdir, "checkpoint.dat") utils.init_checkpoint(checkpoint_file) # set parameters that are unlikely to change from run to run, such as filenames and directories parameters = set_static_parameters(parameters) # check that all required parameters are present params = check_parameters(parameters) utils.update_checkpoint({'params': params}) # set parameters that will likely change from run to run parameters = set_dynamic_parameters(parameters) # Write a snapshot of the parameters to a file snapshot_filename = "%s/%s.parameters_snapshot" % (workdir, name) with open(snapshot_filename, "w") as snapshot_file: logger.debug("Writing parameter snapshot to %s", snapshot_filename) snapshot_file.write(str(parameters)) snapshot_file.write("\n") start_time = time.time() # For each checkpointed stage, check if the stage should be run again. # A stage should be run again under the following circumstances: # - The user manually requested to run the stage # - No checkpoint exists for the stage # - A stage on which this stage depends will be re-run # - Parameters on which the stage depends have been changed since the last run if args.stage: for stage in args.stage: if stage.endswith('+'): stage = stage[:-1] if stage not in stages: continue stage_required.manual_stages = range(stages.index(stage), len(stages)) break if stage not in stages: args.stage.pop(stage) else: stage_required.manual_stages.append(stages.index(stage)) else: # since no stage were specified to run manually, choose the first stage based on the checkpoint file stage_required.manual_stages = set_manual_stages(params) # Run polynomial selection polysel_result = do_polysel(parameters) # Run sieving sieve_result = do_sieve(parameters, polysel_result) # Run linalg linalg_result = do_linalg(parameters, sieve_result) # Run square root sqrt_result = do_sqrt(parameters, linalg_result) factoring_duration = polysel_result['duration'] + sieve_result[ 'duration'] + linalg_result['duration'] + sqrt_result['duration'] logger.info('Factoring completed in %s', utils.str_time(factoring_duration)) logger.info('\tPolysel in real/cpu %s/%s', utils.str_time(polysel_result['duration']), utils.str_time(polysel_result['cputime'])) logger.info("\tSieving in real/cpu %s/%s", utils.str_time(sieve_result['duration']), utils.str_time(sieve_result['cputime'])) logger.info("\tLinalg in %s", utils.str_time(linalg_result['duration'])) logger.info("\tSqrt in %s", utils.str_time(sqrt_result['duration'])) logger.info("\tFactors %s", ','.join(sqrt_result['factors'])) post_factor = parameters.myparams({ 'post_factor': None }, ['commands']).get('post_factor') if post_factor != None: logger.info('Post-factor command %s', post_factor) utils.run_command(post_factor, logger=logger)