def testrun(args, parser): """TESTRUN Reserve pages, run pages and optionally upload pages """ emop_submit = EmopSubmit(args.config_path) # Do not run testrun subcommand if not in a valid cluster job environment # This prevents accidentally running resource intensive program on login nodes if not emop_submit.scheduler.is_job_environment(): print("Can only use testrun subcommand from within a cluster job environment") sys.exit(1) # Reserve pages equal to --num-pages proc_id = emop_submit.reserve(num_pages=args.testrun_num_pages, r_filter=args.filter) if not proc_id: print("Failed to reserve pages") sys.exit(1) # Run reserved pages emop_run = EmopRun(args.config_path, proc_id) run_status = emop_run.run(force=True) if not run_status: sys.exit(1) # Exit if --no-upload if args.testrun_no_upload: sys.exit(0) # Upload results emop_upload = EmopUpload(args.config_path) upload_status = emop_upload.upload_proc_id(proc_id=proc_id) if not upload_status: sys.exit(1) sys.exit(0)
def submit(args, parser): """ submit command This command will submit jobs based on the various arguments. Arguments used: --num-jobs: Number of jobs to submit. The default value is determined based on optimization logic. --pages-per-job: Number of pages per job. The default value if determined based on optimization logic. --no-schedule: Currently unused --sim: Simulate and print the job optimization and commands to use but do not actually submit jobs --filter: Filter to use when querying pending pages The default logic of this function is to determine optimal number of jobs and pages per job. Once this is determined the necessary number of pages are reserved via Dashboard API. The returned API results are saved to a PROC_ID input file. All the reserved job's PROC_IDs are then processed and a Globus transfer is initiated and a transfer job is submitted to wait for the transfer to complete. All the reserved page PROC_IDs are submitted as batch jobs and will depend on the transfer job to complete before they start. """ # Ensure --num-jobs and --pages-per-job are both present # if either is used if (args.num_jobs and not args.pages_per_job or not args.num_jobs and args.pages_per_job): print("--num-jobs and --pages-per-job must be used together") parser.print_help() sys.exit(1) emop_submit = EmopSubmit(args.config_path) emop_query = EmopQuery(args.config_path) pending_pages = emop_query.pending_pages_count(q_filter=args.filter) # Exit if no pages to run if pending_pages == 0: print("No work to be done") sys.exit(0) if not pending_pages: print("Error querying pending pages") sys.exit(1) # Exit if the number of submitted jobs has reached the limit if args.schedule: current_job_count = emop_submit.scheduler.current_job_count() if current_job_count >= emop_submit.settings.max_jobs: print("Job limit of %s reached." % emop_submit.settings.max_jobs) sys.exit(0) # Optimize job submission if --pages-per-job and --num-jobs was not set if not args.pages_per_job and not args.num_jobs: num_jobs, pages_per_job = emop_submit.optimize_submit(pending_pages, current_job_count, sim=args.submit_simulate) else: num_jobs = args.num_jobs pages_per_job = args.pages_per_job if args.submit_simulate: sys.exit(0) # Verify transfers are possible emop_transfer = EmopTransfer(args.config_path) endpoint_check = emop_transfer.check_endpoints(fail_on_warn=True) if not endpoint_check: print("ERROR: Not all endpoints are activated or activation expires soon.") sys.exit(1) # Loop that performs the actual submission proc_ids = [] for i in xrange(num_jobs): proc_id = emop_submit.reserve(num_pages=pages_per_job, r_filter=args.filter) if not proc_id: print("ERROR: Failed to reserve page") continue proc_ids.append(proc_id) if proc_ids: if args.transfer: task_id = emop_transfer.stage_in_proc_ids(proc_ids=proc_ids, wait=False) transfer_job_id = emop_submit.scheduler.submit_transfer_job(task_id=task_id) else: transfer_job_id = None for proc_id in proc_ids: job_id = emop_submit.scheduler.submit_job(proc_id=proc_id, num_pages=pages_per_job, dependency=transfer_job_id) emop_submit.set_job_id(proc_id=proc_id, job_id=job_id) sys.exit(0)