def test_add_job_int_fail(): with pytest.raises(TypeError) as excinfo: dag = pycondor.Dagman('dagname') dag.add_job(50) error = 'Expecting a Job or Dagman. ' + \ 'Got an object of type {}'.format(type(50)) assert error == str(excinfo.value)
def apply_all( datadir, arrname, outdir, workspace, fold_results=None, single_results=None, and_submit=False, ): """Generate BDT response arrays for all ROOT files in DATADIR.""" import glob import shutil import pycondor if len(single_results) > 0 and len(fold_results) > 0: raise ValueError("Cannot use -f and -s together with apply-single") results_flags = None if len(fold_results) > 0: results_flags = "-f {}".format(" -f ".join(fold_results)) elif len(single_results) > 0: results_flags = "-s {}".format(" -s ".join(single_results)) else: raise ValueError("-f or -s required") ws = PosixPath(workspace).resolve() outpath = PosixPath(outdir).resolve() outpath.mkdir(exist_ok=True) datapath = PosixPath(datadir).resolve(strict=True) all_files = glob.glob(f"{datapath}/*.root") arglist = [f"{f} {arrname} {outpath} {results_flags}" for f in all_files] condor_dag = pycondor.Dagman(name="dag_train_scan", submit=str(ws / "sub")) condor_job_scan = pycondor.Job( name="job_apply_all", universe="vanilla", getenv=True, notification="Error", extra_lines=["notify_user = [email protected]"], executable=shutil.which("tdub"), submit=str(ws / "sub"), error=str(ws / "err"), output=str(ws / "out"), log=str(ws / "log"), dag=condor_dag, ) for run in arglist: condor_job_scan.add_arg(f"apply single {run}") if and_submit: condor_dag.build_submit() else: condor_dag.build()
def create_python_script_jobs( main_job_name: str, python_script: str, job_args_list: List[Dict], job_names_list: List[str], request_memory: Optional[str] = None, extra_lines: Optional[List[str]] = [], ): """Creates a set of parallel jobs for a python script :param request_memory: mem of job (eg '16 GB') :param main_job_name: name of main job (and dir of where the job will run) :param python_script: the abspath to the python script :param job_args_list: list of args-dicts for the python script [{job_arg: arg_val}] :param job_names_list: list of the individual job names :return: None """ run_dir = os.path.abspath(main_job_name) python_script = os.path.abspath(python_script) print(f"Making jobs for {python_script}.") subdir = os.path.join(run_dir, "submit") logdir = os.path.join(run_dir, f"logs") dag = pycondor.Dagman(name=main_job_name, submit=subdir) jobs = [] bash_commands = [] for job_name, job_args in zip(job_names_list, job_args_list): jobs.append( create_python_script_job( python_script, job_name, job_args, logdir, subdir, dag, request_memory, extra_lines, )) bash_commands.append( f"{python_script} {convert_args_dict_to_str(job_args)}") dag.build(makedirs=True, fancyname=False) command_line = "$ condor_submit_dag {}".format( os.path.relpath(dag.submit_file)) open(f"{main_job_name}/bash.sh", 'w').write("\n".join(bash_commands)) print(f"Created {len(jobs)} jobs. Run:\n{command_line}\n")
# submit background tests to cluster for stacking analysis # specify number of jobs, trials per job, and the time-window of analysis t100 = 1000.00 N_trials = 1000 N_jobs = 100 error = '/home/sfahey/condor/error' output = '/home/sfahey/condor/output' log = '/home/sfahey/condor/log' submit = '/home/sfahey/condor/submit' job = pycondor.Job('FRB_stacking_bg', 'frb_bg_tsd_stacking_noRepeater.py', error=error, output=output, log=log, submit=submit, verbose=2, request_memory=4000) for seed in range(N_jobs): job.add_arg('-t %.2f -r %i -n %i' % (t100, seed, N_trials)) dagman = pycondor.Dagman('FRB_stacking_bg_dT%.2f' % t100, submit=submit, verbose=2) dagman.add_job(job) dagman.build_submit()
import pycondor from glob import glob import numpy as np error = '/scratch/apizzuto/novae/condor/error' output = '/scratch/apizzuto/novae/condor/output' log = '/scratch/apizzuto/novae/condor/log' submit = '/scratch/apizzuto/novae/condor/submit' dagman = pycondor.Dagman('csky_Novae_stacking_jobs', submit=submit, verbose=2) background_jobs = pycondor.Job( 'sensitivity_stacking_novae_greco', '/home/apizzuto/Nova/scripts/stacking/stacking_background.py', error=error, output=output, log=log, submit=submit, getenv=True, universe='vanilla', verbose=2, request_memory=8000, request_cpus=5, extra_lines=[ 'should_transfer_files = YES', 'when_to_transfer_output = ON_EXIT' ], dag=dagman) low_mem_signal = pycondor.Job( 'sensitivity_signal_novae_greco_low_mem', '/home/apizzuto/Nova/scripts/stacking/stacking_signal_trials.py',
import numpy as np import sys import glob path = "/home/jlazar/condor_logs/conventional_flux_interpolation/" error = "%s/error" % path output = "%s/output" % path log = "%s/log" % path submit = "%s/submit" % path xlines = ["request_memory = (NumJobStarts is undefined) ? 2 * pow(2, 10) : 2048 * pow(2, NumJobStarts + 1)", "periodic_release = (HoldReasonCode =?= 21 && HoldReasonSubCode =?= 1001) || HoldReasonCode =?= 21", "periodic_remove = (JobStatus =?= 5 && (HoldReasonCode =!= 34 && HoldReasonCode =!= 21)) || (RequestMemory > 13192)" ] dagman = pycondor.Dagman("conventional_flux_interpolation", submit=submit, verbose=2) run = pycondor.Job("conv_interp", "/data/user/jlazar/solar_WIMP/conventional_flux_interpolation.sh", error=error, output=output, log=log, submit=submit, universe="vanilla", notification="never", dag=dagman, verbose=2, extra_lines=xlines ) for mcf in np.genfromtxt("/data/user/jlazar/solar_WIMP/mc_paths.txt", dtype=str):
# Define path to executables make_maps_ex = os.path.join(comp.paths.project_home, 'processing/anisotropy/ks_test_multipart', 'make_maps.py') merge_maps_ex = os.path.join(comp.paths.project_home, 'processing/anisotropy/ks_test_multipart', 'merge_maps.py') save_pvals_ex = os.path.join(comp.paths.project_home, 'processing/anisotropy/ks_test_multipart', 'save_pvals.py') # Create Dagman instance dag_name = 'anisotropy_kstest_{}'.format(args.config) if args.test: dag_name += '_test' dagman = pycondor.Dagman(dag_name, submit=submit, verbose=1) # Create Job for saving ks-test p-values for each trial save_pvals_name = 'save_pvals_{}'.format(args.config) if args.low_energy: save_pvals_name += '_lowenergy' save_pvals_job = pycondor.Job(save_pvals_name, save_pvals_ex, error=error, output=output, log=log, submit=submit, verbose=1) save_pvals_infiles_0 = [] save_pvals_infiles_1 = []
import pycondor from glob import glob import numpy as np error = '/scratch/apizzuto/novae/condor/error' output = '/scratch/apizzuto/novae/condor/output' log = '/scratch/apizzuto/novae/condor/log' submit = '/scratch/apizzuto/novae/condor/submit' dagman = pycondor.Dagman( 'Novae_stacking_differential_jobs', submit=submit, verbose=2 ) low_mem_signal = pycondor.Job( 'differential_novae_greco_low_mem', '/home/apizzuto/Nova/scripts/stacking_differential_sensitivity.py', error=error, output=output, log=log, submit=submit, getenv=True, universe='vanilla', verbose=2, request_memory=8000, request_cpus=5, extra_lines=[ 'should_transfer_files = YES', 'when_to_transfer_output = ON_EXIT'], dag=dagman) high_mem_signal = pycondor.Job(
# submit signal tests to cluster for stacking analysis # scan each time-window of analysis, specify a spectral gamma times = [ '0.01', '0.03', '0.10', '0.32', '1.00', '3.16', '10.00', '31.60', '100.00', '316.00' ] gamma = 3.0 error = '/home/sfahey/condor/error' output = '/home/sfahey/condor/output' log = '/home/sfahey/condor/log' submit = '/home/sfahey/condor/submit' job = pycondor.Job('FRB_stacking_sig', 'frb_sig_tsd_stacking_noRepeater.py', error=error, output=output, log=log, submit=submit, verbose=2, request_memory=7000) for time in times: job.add_arg('-t %.2f -g %.2f' % (float(time), gamma)) dagman = pycondor.Dagman('FRB_stacking_sig', submit=submit, verbose=2) dagman.add_job(job) dagman.build_submit()
def simulation_processing_dag(config='IC86.2012', batch_size=1000, test=False, snow_lambda=None, dom_eff=None): if all([snow_lambda, dom_eff]): raise NotImplementedError('Specifying a value for both snow_lambda ' 'and dom_eff is not currently supported.') base_dir = os.path.join(comp.paths.comp_data_dir, config, 'sim', 'test' if test else '') if snow_lambda is not None: # snow_lambda_str = str(snow_lambda).replace('.', '-') i3_hdf_outdir = os.path.join(base_dir, 'i3_hdf', 'snow_lambda_{}'.format(snow_lambda)) df_hdf_outdir = os.path.join(base_dir, 'processed_hdf', 'snow_lambda_{}'.format(snow_lambda)) elif dom_eff is not None: i3_hdf_outdir = os.path.join(base_dir, 'i3_hdf', 'dom_eff_{}'.format(dom_eff)) df_hdf_outdir = os.path.join(base_dir, 'processed_hdf', 'dom_eff_{}'.format(dom_eff)) else: i3_hdf_outdir = os.path.join(base_dir, 'i3_hdf', 'nominal') df_hdf_outdir = os.path.join(base_dir, 'processed_hdf', 'nominal') # Create data processing Jobs / Dagman dag_name = 'sim_processing_{}'.format(args.config.replace('.', '-')) dag = pycondor.Dagman(dag_name, submit=PYCONDOR_SUBMIT, verbose=1) sims = comp.simfunctions.config_to_sim(config) for sim in sims: process_i3_job = pycondor.Job( name='process_i3_{}'.format(sim), executable=WRAPPER_EX, error=PYCONDOR_ERROR, output=PYCONDOR_OUTPUT, log=PYCONDOR_LOG, submit=PYCONDOR_SUBMIT, getenv=False, request_memory='3GB' if dom_eff else None, dag=dag) save_df_job = pycondor.Job(name='save_dataframe_{}'.format(sim), executable=WRAPPER_EX, error=PYCONDOR_ERROR, output=PYCONDOR_OUTPUT, log=PYCONDOR_LOG, submit=PYCONDOR_SUBMIT, getenv=False, dag=dag) # Ensure that save_df_job doesn't begin until process_i3_job completes save_df_job.add_parent(process_i3_job) # Split file list into smaller batches for submission if test: batch_size = 2 max_batches = 2 else: max_batches = None sim_file_batches = comp.level3_sim_file_batches( sim, size=batch_size, max_batches=max_batches) gcd = comp.level3_sim_GCD_file(sim) for idx, files in enumerate(sim_file_batches): # Set up process_i3_job arguments outfile_basename = 'sim_{}_part_{:02d}.hdf'.format(sim, idx) process_i3_outfile = os.path.join(i3_hdf_outdir, outfile_basename) # Don't forget to insert GCD file at beginning of FileNameList files.insert(0, gcd) files_str = ' '.join(files) process_i3_arg_template = '{ex} --type sim --files {i3_files} --outfile {outfile}' process_i3_arg = process_i3_arg_template.format( ex=PROCESS_I3_EX, i3_files=files_str, outfile=process_i3_outfile) if snow_lambda is not None: process_i3_arg += ' --snow_lambda {}'.format(snow_lambda) if dom_eff is not None: process_i3_arg += ' --dom_eff {}'.format(dom_eff) process_i3_job.add_arg(process_i3_arg) # Set up save_df_job arguments save_df_outfile = os.path.join(df_hdf_outdir, outfile_basename) save_df_arg_template = '{ex} --input {input} --output {output} --type sim --sim {sim} --config {config}' save_df_arg = save_df_arg_template.format(ex=SAVE_DF_EX, input=process_i3_outfile, output=save_df_outfile, sim=sim, config=config) save_df_job.add_arg(save_df_arg) return dag
def train_scan( datadir, workspace, pre_exec, early_stop, test_size, overwrite, and_submit, ): """Perform a parameter scan via condor jobs. DATADIR points to the intput ROOT files, training is performed on the REGION and all output is saved to WORKSPACE. $ tdub train scan /data/path 2j2b scan_2j2b """ if pre_exec is not None: exec(PosixPath(pre_exec).read_text()) from tdub.batch import create_condor_workspace import tdub.config import itertools ws = create_condor_workspace(workspace, overwrite=overwrite) (ws / "res").mkdir() runs = [] i = 0 if pre_exec is None: pre_exec = "_NONE" else: pre_exec = str(PosixPath(pre_exec).resolve()) pd = tdub.config.DEFAULT_SCAN_PARAMETERS itr = itertools.product( pd.get("max_depth"), pd.get("num_leaves"), pd.get("learning_rate"), pd.get("min_child_samples"), pd.get("reg_lambda"), ) log.info("Scan grid:") log.info(" - max_depth: {}".format(pd.get("max_depth"))) log.info(" - num_leaves: {}".format(pd.get("num_leaves"))) log.info(" - learning_rate: {}".format(pd.get("learning_rate"))) log.info(" - min_child_samples: {}".format(pd.get("min_child_samples"))) log.info(" - reg_lambda: {}".format(pd.get("reg_lambda"))) for (max_depth, num_leaves, learning_rate, min_child_samples, reg_lambda) in itr: suffix = "{}-{}-{}-{}-{}".format( max_depth, num_leaves, learning_rate, min_child_samples, reg_lambda, ) outdir = ws / "res" / f"{i:04d}_{suffix}" arglist = ( "{} {} -p {} -s {} " "--learning-rate {} " "--num-leaves {} " "--min-child-samples {} " "--max-depth {} " "--reg-lambda {} " "--early-stop {} " ).format( datadir, outdir, pre_exec, test_size, learning_rate, num_leaves, min_child_samples, max_depth, reg_lambda, early_stop, ) arglist = arglist.replace("-p _NONE ", "") runs.append(arglist) i += 1 with (ws / "run.sh").open("w") as outscript: print("#!/bin/bash\n\n", file=outscript) for run in runs: print(f"tdub train single {run}\n", file=outscript) os.chmod(ws / "run.sh", 0o755) import pycondor condor_dag = pycondor.Dagman(name="dag_train_scan", submit=str(ws / "sub")) condor_job_scan = pycondor.Job( name="job_train_scan", universe="vanilla", getenv=True, notification="Error", extra_lines=["notify_user = [email protected]"], executable=shutil.which("tdub"), submit=str(ws / "sub"), error=str(ws / "err"), output=str(ws / "out"), log=str(ws / "log"), dag=condor_dag, ) for run in runs: condor_job_scan.add_arg(f"train single {run}") condor_job_check = pycondor.Job( name="job_train_check", universe="vanilla", getenv=True, notification="Error", extra_lines=["notify_user = [email protected]"], executable=shutil.which("tdub"), submit=str(ws / "sub"), error=str(ws / "err"), output=str(ws / "out"), log=str(ws / "log"), dag=condor_dag, ) condor_job_check.add_arg(f"train check {ws}") condor_job_check.add_parent(condor_job_scan) if and_submit: condor_dag.build_submit() else: condor_dag.build() # log.info(f"prepared {len(runs)} jobs for submission") # with (ws / "condor.sub").open("w") as f: # condor_preamble(ws, shutil.which("tdub"), memory="2GB", GetEnv=True, to_file=f) # for run in runs: # add_condor_arguments(f"train-single {run}", f) # if and_submit: # condor_submit(workspace) return 0
ms = [500, 800, 1000, 2000, 3000, 5000, 10000] chs = [5, 8, 11] path = "/home/jlazar/condor_logs/dn_dz_interpolation/" error = "%s/error" % path output = "%s/output" % path log = "%s/log" % path submit = "%s/submit" % path xlines = [ "request_memory = (NumJobStarts is undefined) ? 2 * pow(2, 10) : 2048 * pow(2, NumJobStarts + 1)", "periodic_release = (HoldReasonCode =?= 21 && HoldReasonSubCode =?= 1001) || HoldReasonCode =?= 21", "periodic_remove = (JobStatus =?= 5 && (HoldReasonCode =!= 34 && HoldReasonCode =!= 21)) || (RequestMemory > 13192)" ] dagman = pycondor.Dagman("dn_dz_interpolation", submit=submit, verbose=2) run = pycondor.Job("interp", "/data/user/jlazar/solar_WIMP/dn_dz_interpolation.sh", error=error, output=output, log=log, submit=submit, universe="vanilla", notification="never", dag=dagman, verbose=2, extra_lines=xlines) for m in ms: fluxfiles = [
job = pycondor.Job( 'sensitivity_fastresponse_alerts', f_path + 'transient_scripts/alert_event_fits.py', error=error, output=output, log=log, submit=submit, getenv=True, universe='vanilla', verbose=2, request_memory=8000, request_cpus=5, extra_lines=[ 'should_transfer_files = YES', 'when_to_transfer_output = ON_EXIT', 'Requirements = (Machine != "node128.icecube.wisc.edu")' ]) skymap_files = glob( '/data/ana/realtime/alert_catalog_v2/fits_files/Run1*.fits.gz') for index in range(len(skymap_files)): for deltaT in np.array([1000.0, 2. * 86400.]): for smear_str in [' --smear']: #, '']: add_str = ' --ntrials=100' if deltaT > 100000. else '' job.add_arg('--deltaT={} --index={}'.format(deltaT, index) + add_str + smear_str) dagman = pycondor.Dagman('alert_event_fra_fits', submit=submit, verbose=2) dagman.add_job(job) dagman.build_submit()
f_path + 'transient_scripts/alert_event_sensitivity.py', error=error, output=output, log=log, submit=submit, getenv=True, universe='vanilla', verbose=2, request_memory=8000, request_cpus=5, extra_lines=[ 'should_transfer_files = YES', 'when_to_transfer_output = ON_EXIT', 'Requirements = (Machine != "node128.icecube.wisc.edu")' ]) skymap_files = glob( '/data/ana/realtime/alert_catalog_v2/fits_files/Run1*.fits.gz') for index in range(len(skymap_files)): for deltaT in np.array([1000., 2. * 86400.]): for smear_str in [' --smear']: #, '']: add_str = ' --ntrials=100' if deltaT > 100000. else '' job.add_arg('--deltaT={} --index={}'.format(deltaT, index) + add_str + smear_str) dagman = pycondor.Dagman('alert_event_fra_sensitivity', submit=submit, verbose=2) dagman.add_job(job) dagman.build_submit()
batch_size=args.batch_size_sim, test=args.test, snow_lambda=args.snow_lambda, dom_eff=args.dom_eff) dags.append(sim_dag) if args.data: data_dag = data_processing_dag(config=args.config, batch_size=args.batch_size_data, test=args.test) dags.append(data_dag) # TODO: Consider adding more processing dags. E.g. # if args.efficiencies: # if args.models: # if args.livetimes: # Create Dagman to manage processing workflow if not dags: raise ValueError('No processing tasks were specified') elif len(dags) == 1: dag = dags[0] else: dag_name = 'processing_{}'.format(args.config.replace('.', '-')) dag = pycondor.Dagman(dag_name, submit=PYCONDOR_SUBMIT, verbose=1) for subdag in dags: dag.add_subdag(subdag) # Build and submit processing dagman dag.build_submit(fancyname=True, submit_options='-maxjobs 3000')
def setup_pycondor_dag(self): self.pycondor_dag = pycondor.Dagman( name=self.dag_name, submit=self.submit_directory )
def data_processing_dag(config='IC86.2012', batch_size=1000, test=False): base_dir = os.path.join(comp.paths.comp_data_dir, config, 'data', 'test' if test else '') i3_hdf_outdir = os.path.join(base_dir, 'i3_hdf') df_hdf_outdir = os.path.join(base_dir, 'processed_hdf') # Create data processing Jobs / Dagman dag_name = 'data_processing_{}'.format(args.config.replace('.', '-')) dag = pycondor.Dagman(dag_name, submit=PYCONDOR_SUBMIT, verbose=1) process_i3_job = pycondor.Job(name='process_i3', executable=WRAPPER_EX, error=PYCONDOR_ERROR, output=PYCONDOR_OUTPUT, log=PYCONDOR_LOG, submit=PYCONDOR_SUBMIT, getenv=False, dag=dag) save_df_job = pycondor.Job(name='save_dataframe', executable=WRAPPER_EX, error=PYCONDOR_ERROR, output=PYCONDOR_OUTPUT, log=PYCONDOR_LOG, submit=PYCONDOR_SUBMIT, getenv=False, dag=dag) # Ensure that save_df_job doesn't begin until process_i3_job completes save_df_job.add_parent(process_i3_job) run_gen = comp.datafunctions.run_generator(config) if test: run_gen = islice(run_gen, 2) batch_size = 2 max_batches = 2 else: max_batches = None for run in run_gen: # Get files associated with this run gcd = comp.level3_data_GCD_file(config, run) data_file_batches = comp.level3_data_file_batches( config=config, run=run, size=batch_size, max_batches=max_batches) # Process run files in batches for idx, files in enumerate(data_file_batches): # Set up process_i3_job arguments outfile_basename = 'run_{}_part_{:02d}.hdf'.format(run, idx) process_i3_outfile = os.path.join(i3_hdf_outdir, outfile_basename) # Don't forget to insert GCD file at beginning of FileNameList files.insert(0, gcd) files_str = ' '.join(files) process_i3_arg_template = '{ex} --type data --files {i3_files} --outfile {outfile}' process_i3_arg = process_i3_arg_template.format( ex=PROCESS_I3_EX, i3_files=files_str, outfile=process_i3_outfile) process_i3_job.add_arg(process_i3_arg, retry=3) # Set up save_df_job arguments save_df_outfile = os.path.join(df_hdf_outdir, outfile_basename) save_df_arg_template = '{ex} --input {input} --output {output} --type data --config {config}' save_df_arg = save_df_arg_template.format(ex=SAVE_DF_EX, input=process_i3_outfile, output=save_df_outfile, config=config) save_df_job.add_arg(save_df_arg) return dag
error=error, output=output, log=log, submit=submit, getenv=True, universe='vanilla', verbose=1, request_memory=8000, #request_cpus=5, extra_lines=[ 'should_transfer_files = YES', 'when_to_transfer_output = ON_EXIT' ]) sky_files = glob( '/data/ana/realtime/alert_catalog_v2/fits_files/Run1*.fits.gz') for rng in range(10): for index in range(len(sky_files)): for gamma in [2.5]: for fit in [True, False]: for smear in [' --smear']: add_str = ' --fit' if fit else '' job.add_arg('--rng={} --i={} --ntrials=10 --g={}{}'.format( rng, index, gamma, smear) + add_str) dagman = pycondor.Dagman('alert_event_fra_signal_steady', submit=submit, verbose=1) dagman.add_job(job) dagman.build_submit()
from sys import argv as args from controls import chs, ms, mcfiles path = "/home/jlazar/condor_logs/energy_delta_theta_hist_signal" error = "%s/error" % path output = "%s/output" % path log = "%s/log" % path submit = "%s/submit" % path xlines = [ "request_memory = (NumJobStarts is undefined) ? 2 * pow(2, 10) : 2048 * pow(2, NumJobStarts + 1)", "periodic_release = (HoldReasonCode =?= 21 && HoldReasonSubCode =?= 1001) || HoldReasonCode =?= 21", "periodic_remove = (JobStatus =?= 5 && (HoldReasonCode =!= 34 && HoldReasonCode =!= 21)) || (RequestMemory > 13192)" ] dagman = pycondor.Dagman("e_d_theta", submit=submit, verbose=2) run = pycondor.Job( "energy_delta_theta_hist_signal", "/data/user/jlazar/solar_WIMP/calc_energy_delta_theta_hist_signal.sh", error=error, output=output, log=log, submit=submit, universe="vanilla", notification="never", dag=dagman, verbose=2, extra_lines=xlines) fluxes = [f"ch{ch}-m{m}" for ch in chs for m in ms] print(fluxes)
import pandas as pd error = '/scratch/runzeli/condor/error' output = '/scratch/runzeli/condor/output' log = '/scratch/runzeli/condor/log' submit = '/scratch/runzeli/condor/submit' job = pycondor.Job( 'Runze_muons', './F_B_test.py', error=error, output=output, log=log, submit=submit, getenv=True, universe='vanilla', verbose=2, request_memory=4000, extra_lines=[ 'should_transfer_files = YES', 'when_to_transfer_output = ON_EXIT', 'Requirements = (Machine != "node128.icecube.wisc.edu")' ]) for index in range(500): for index_2 in [1, 2, 3, 4]: job.add_arg(str(index) + " " + str(index_2)) dagman = pycondor.Dagman('lee_muon_test_100B', submit=submit, verbose=2) dagman.add_job(job) dagman.build_submit()
error = '/scratch/apizzuto/fast_response/condor/error' output = '/scratch/apizzuto/fast_response/condor/output' log = '/scratch/apizzuto/fast_response/condor/log' submit = '/scratch/apizzuto/fast_response/condor/submit' job = pycondor.Job('background_fastresponse_alerts_steady', f_path + 'time_integrated_scripts/steady_background_trials.py', error=error, output=output, log=log, submit=submit, getenv=True, universe='vanilla', verbose=2, request_memory=8000, #request_cpus=5, extra_lines= ['should_transfer_files = YES', 'when_to_transfer_output = ON_EXIT', 'Requirements = (Machine != "node128.icecube.wisc.edu")'] ) sky_files = glob('/data/ana/realtime/alert_catalog_v2/fits_files/Run1*.fits.gz') for rng in range(5): for index in range(len(sky_files)): for smear in [' --smear']: job.add_arg('--rng={} --i={} --ntrials=200{}'.format(rng, index, smear)) dagman = pycondor.Dagman('alert_event_fra_background_steady', submit=submit, verbose=2) dagman.add_job(job) dagman.build_submit()
p.add_argument('--overwrite', dest='overwrite', default=False, action='store_true', help='Overwrite existing merged files') args = p.parse_args() # Define output directories for condor jobs mypaths = comp.get_paths() error = mypaths.condor_data_dir + '/error' output = mypaths.condor_data_dir + '/output' log = mypaths.condor_scratch_dir + '/log' submit = mypaths.condor_scratch_dir + '/submit' # Submit this Job under a Dagman, even though there aren't many arguments dagman = pycondor.Dagman('validation-dagman', submit=submit, verbose=2) validation_ex = os.path.join(os.getcwd(), 'validation-curves.py') merge_ex = os.path.join(os.getcwd(), 'merge_dataframe.py') # # Maximum tree depth # dagman = add_hyperparameter(dagman, 'max_depth', range(1, 11), 'int', # validation_ex, merge_ex, **vars(args)) # # # Learning rate # dagman = add_hyperparameter(dagman, 'learning_rate', np.arange(0.1, 1.1, 0.1), 'float', # validation_ex, merge_ex, **vars(args)) # Number of estimators dagman = add_hyperparameter(dagman, 'n_estimators', range(10, 500, 50), 'int', validation_ex, merge_ex, **vars(args))
job1.add_arg('--length {}'.format(i), retry=7) # Setting up second PyCondor Job job2 = pycondor.Job('examplejob2', 'savelist.py', error=error, output=output, log=log, submit=submit, verbose=2) # Adding arguments to job1 job2.add_arg('--length 200', name='200jobname') job2.add_arg('--length 400', name='400jobname', retry=3) # Setting up a PyCondor Dagman subdag = pycondor.Dagman('example_subdag', submit=submit, verbose=2) # Add job1 to dagman subdag.add_job(job1) subdag.add_job(job2) # Setting up third PyCondor Job job3 = pycondor.Job('examplejob3', 'savelist.py', error=error, output=output, log=log, submit=submit, verbose=2) # Adding arguments to job1 for length in range(210, 220): job3.add_arg('--length {}'.format(length))
import pycondor, argparse, sys, os.path from glob import glob import numpy as np import pandas as pd error = '/scratch/apizzuto/fast_response/condor/error' output = '/scratch/apizzuto/fast_response/condor/output' log = '/scratch/apizzuto/fast_response/condor/log' submit = '/scratch/apizzuto/fast_response/condor/submit' script_base = '/data/user/apizzuto/fast_response_skylab/alert_event_followup/cascade_scripts/' dagman_short_timescale = pycondor.Dagman('FRA_cascade_short_timescale_sens', submit=submit, verbose=2) dagman_long_timescale = pycondor.Dagman('FRA_cascade_long_timescale_sens', submit=submit, verbose=2) dagman_fits = pycondor.Dagman('FRA_cascade_fitting_bias', submit=submit, verbose=2) background_short_time_jobs = pycondor.Job( 'fra_background_short_time', script_base + 'cascade_alert_background.py', error=error, output=output, log=log, submit=submit, getenv=True, universe='vanilla',
# submit signal tests to cluster for max-burst analysis # scan each time-window of analysis, specify a spectral gamma times = [ '0.01', '0.03', '0.10', '0.32', '1.00', '3.16', '10.00', '31.60', '100.00', '316.00' ] #times = ['100.00', '316.00', '1000.00'] # try large dT with fewer trials, less memory gamma = 3.0 error = 'condor/error' output = 'condor/output' log = 'condor/log' submit = 'condor/submit' job = pycondor.Job('FRB_maxburst_sig', 'frb_sig_tsd_maxburst_noRepeater.py', error=error, output=output, log=log, submit=submit, verbose=2, request_memory=7000) for time in times: job.add_arg('-t %.2f -g %.2f' % (float(time), gamma)) dagman = pycondor.Dagman('FRB_maxburst_sig', submit=submit, verbose=2) dagman.add_job(job) dagman.build_submit()
log = mypaths.condor_scratch_dir + '/log' submit = mypaths.condor_scratch_dir + '/submit' # Set up pycondor Job ex = os.getcwd() + '/sequential-feature-selection.py' job = pycondor.Job('SFS', ex, error=error, output=output, log=log, submit=submit, request_memory='5GB', extra_lines=['request_cpus = {}'.format(args.n_jobs)], verbose=2) method = ['forward', 'backward'] floating = [True, False] base_arg = '--config {} --pipeline {} --cv {} --scoring {} --n_jobs {}'.format( args.config, args.pipeline, args.cv, args.scoring, args.n_jobs) for method, floating in itertools.product(method, floating): arg = base_arg + ' --method {}'.format(method) if floating: arg += ' --floating' job.add_arg(arg) # Submit this Job under a Dagman, even though there aren't many arguments dagman = pycondor.Dagman('SFS_dagman', submit=submit, verbose=2) dagman.add_job(job) dagman.build_submit()
from glob import glob import numpy as np # submit background tests to cluster for max-burst analysis # specify number of jobs, trials per job, and the time-window of analysis t100 = 1000.00 N_trials = 1000 N_jobs = 100 error='/home/sfahey/condor/error' output='/home/sfahey/condor/output' log='/home/sfahey/condor/log' submit='/home/sfahey/condor/submit' job = pycondor.Job('FRB_maxburst_bg','frb_bg_tsd_maxburst_noRepeater.py', error=error, output=output, log=log, submit=submit, verbose=2, request_memory=4000 ) for seed in range(N_jobs): job.add_arg('-t %.2f -r %i -n %i'%(t100, seed, N_trials)) dagman = pycondor.Dagman('FRB_maxburst_bg_dT%.2f'%t100, submit=submit, verbose=2) dagman.add_job(job) dagman.build_submit()
]) for delta_t in [1000., 2. * 86400.]: for manual_lumi in np.logspace(49, 60, 45): for density in np.logspace(-11., -6., 21)[:]: for lumi in ['SC']: # , 'LG']: for evol in ['MD2014SFR']: if density * manual_lumi * delta_t > 3e54: continue elif density * manual_lumi * delta_t < 1e47: continue else: N = 500 if density < 1e-7 else 150 if density > 3e-8: high_mem_job.add_arg( '--delta_t={} --density={} --LF={} --n={} --evol={} --manual_lumi={}' .format(delta_t, density, lumi, N, evol, manual_lumi)) else: low_mem_job.add_arg( '--delta_t={} --density={} --LF={} --n={} --evol={} --manual_lumi={}' .format(delta_t, density, lumi, N, evol, manual_lumi)) dagman = pycondor.Dagman('ts_distributions', submit=submit, verbose=2) dagman.add_job(low_mem_job) dagman.add_job(high_mem_job) dagman.build_submit()
num_groups = args.num_groups pipeline = args.pipeline n_jobs = args.n_jobs executable = os.path.abspath('feature_scan.py') # Define pycondor Job/Dagman directories error = os.path.join(comp.paths.condor_data_dir, 'error') output = os.path.join(comp.paths.condor_data_dir, 'output') log = os.path.join(comp.paths.condor_scratch_dir, 'log') submit = os.path.join(comp.paths.condor_scratch_dir, 'submit') outdir = os.path.join(os.path.dirname(__file__), 'feature_scan_results') dag_name = 'feature_scan_{}_num_groups-{}'.format(pipeline, num_groups) dag = pycondor.Dagman(name=dag_name, submit=submit) for idx, (features, random_feature) in enumerate(product(scan_features, [True, False])): feature_str = '-'.join(features) job = pycondor.Job(name='feature_scan_num_groups-{}_{}'.format(num_groups, idx), executable=executable, submit=submit, error=error, output=output, log=log, request_cpus=n_jobs, request_memory='3GB', verbose=1, dag=dag) argument = '--features {} '.format(' '.join(features)) for arg_name in ['config', 'num_groups', 'pipeline', 'n_jobs']: argument += '--{} {} '.format(arg_name, getattr(args, arg_name))
job = pycondor.Job( 'sensitivity_fastresponse_alerts', f_path + 'transient_scripts/alert_event_unblind.py', error=error, output=output, log=log, submit=submit, getenv=True, universe='vanilla', verbose=2, request_memory=8000, request_cpus=5, extra_lines=[ 'should_transfer_files = YES', 'when_to_transfer_output = ON_EXIT', 'Requirements = (Machine != "node128.icecube.wisc.edu")' ]) skymap_files = glob( '/data/ana/realtime/alert_catalog_v2/fits_files/Run1*.fits.gz') for index in range(len(skymap_files)): for deltaT in np.array([1000.0, 2. * 86400.]): for smear_str in [' --smear']: job.add_arg('--deltaT={} --index={}'.format(deltaT, index) + smear_str) dagman = pycondor.Dagman('alert_event_fra_unblind', submit=submit, verbose=2) dagman.add_job(job) dagman.build_submit()