def main(): # dask cluster and client number_processes = 1 number_jobs = 35 number_workers = number_processes * number_jobs cluster = SGECluster( interface="ib0", walltime="04:00:00", memory=f"2 G", resource_spec=f"h_vmem=2G", scheduler_options={ "dashboard_address": ":2727", }, job_extra=[ "-cwd", "-V", f"-pe smp {number_processes}", f"-l disk=1G", ], local_directory=os.sep.join( [os.environ.get("PWD"), "dask-worker-space"]), ) client = Client(cluster) cluster.scale(jobs=number_jobs) # main processing print("processing ...") results = [] bag = db.from_sequence(nums, npartitions=number_workers) results = bag.map(weird_function).compute() print("saving ...") joblib.dump(results, f"/nobackup/${USER}/results.joblib") client.close() cluster.close()
def init_cluster(name, args): resource_spec = "h_vmem={}G,mem_req={}G".format(args.h_vmem, args.mem_req) exclude_nodes = "&".join(["!" + x for x in args.exclude_nodes]) if len(exclude_nodes) > 0: exclude_nodes = "#$ -l h=" + exclude_nodes env_extra = [ "#$ -e {}".format(args.log_dir or "/dev/null"), "#$ -o {}".format(args.log_dir or "/dev/null"), "#$ -pe serial {}".format(args.ngpus if args.ngpus > 0 else args.ncpus), exclude_nodes, "source " + args.to_source if args.to_source is not None else "", "export LANG=en_US.UTF-8", "export LC_ALL=en_US.UTF-8", "export MKL_NUM_THREADS=1", "export NUMEXPR_NUM_THREADS=1", "export OMP_NUM_THREADS=1", "export DISABLE_MP_CACHE=1", "export TORCH_HOME=/sequoia/data1/rriochet/.torch", ] for var in args.export_var: env_extra.append(f'export {var}="{os.environ[var]}"') cluster = SGECluster( queue=args.queue, resource_spec=resource_spec, walltime="720:00:00", name=name, cores=args.ncpus, memory="{}G".format(args.mem_req), processes=1, interface="ib0", local_directory=args.log_dir, env_extra=env_extra, spill_dir=args.spill_dir, extra=["--no-nanny"], ) # cluster.adapt(maximum_jobs=args.jobs) cluster.scale(args.jobs) return cluster
def test_complex_cancel_command(loop): with SGECluster(walltime="00:02:00", cores=1, processes=1, memory="2GB", loop=loop) as cluster: with Client(cluster) as client: username = "******" cluster.cancel_command = "qdel -u {}".format(username) cluster.scale(2) start = time() while not client.scheduler_info()["workers"]: sleep(0.100) assert time() < start + QUEUE_WAIT cluster.scale(0) start = time() while client.scheduler_info()["workers"]: sleep(0.100) assert time() < start + QUEUE_WAIT
def main(): # dask cluster and client number_processes = 1 number_jobs = 35 number_workers = number_processes * number_jobs cluster = SGECluster( interface="ib0", walltime="04:00:00", memory=f"12 G", resource_spec=f"h_vmem=12G", scheduler_options={ "dashboard_address": ":2727", }, job_extra=[ "-cwd", "-V", f"-pe smp {number_processes}", f"-l disk=1G", ], local_directory=os.sep.join( [os.environ.get("PWD"), "dask-worker-space"]), ) client = Client(cluster) cluster.scale(jobs=number_jobs) # main processing print("processing ...") results = [] bag = db.from_sequence(sims, npartitions=number_workers) results = bag.map(create_ozone_metric).compute() print("complete") client.close() cluster.close()
def run_JK_distributed_massboosted(df, param): '''Receives the pandas dataframe with the objects containing the temperature decrements and the parameter object and run the kSZ statistic and generate Jack Knifes. Everything runs in the cluster, so current terminal does not need to request many cpus. df: dataframe object containing the variables for the calculation params: param file for this calculation NJK: how many subgroups we will make to run the calculation''' Ncores = envVars.Ncores NWorkers = envVars.NWorkers Ngroups = param.JK_NGROUPS #setup cluster cluster = SGECluster( walltime='172800', processes=1, cores=1, env_extra=[ '#$-pe sge_pe %i' % Ncores, '-l m_core=%i' % Ncores, 'mkdir -p /tmp/pag227/dask/dask-scratch', 'export NUMBA_NUM_THREADS=%i' % Ncores, 'export OMP_NUM_THREADS=%i' % Ncores # 'export OMP_NUM_THREADS=1', # noqa ]) cluster.scale(NWorkers) client = Client(cluster) time.sleep(30) #end setting up cluster #send full dataset to the cluster future_fullDataset = client.scatter(df) future_params = client.scatter(param) res_fullDataset = client.submit(get_pairwise_ksz_massboosted, future_fullDataset, future_params, multithreading=True) #done with the full dataset jk_results = [] futureData = [] #data to be sent in jk or bootstrap in galaxy space for j in range(Ngroups): df_bs = df.copy() choose = np.random.choice(len(df), len(df)) df_bs['dT'] = df.dT.values[choose] futureData.append(client.scatter(df_bs)) if param.JK_RESAMPLING_METHOD.lower() == "bs_dt_mass_boosted_est": get_pw_func = get_pairwise_ksz_massboosted elif param.JK_RESAMPLING_METHOD.lower( ) == 'bs_dt_mass_boosted_est_debiased': # noqa get_pw_func = get_pairwise_ksz_massboosted_debiased for j in range(Ngroups): jk_results.append( client.submit(get_pw_func, futureData[j], future_params, multithreading=True)) # extract results fullDataset_results = res_fullDataset.result() jk_results = client.gather(jk_results) client.close() # cluster.close() return fullDataset_results, jk_results
#client = Client(cluster) # start local workers as threads #import ipdb; ipdb.set_trace() ### SGE #""" from dask_jobqueue import PBSCluster, SGECluster q1d_resource_spec = "q_1day=TRUE,io_big=TRUE" gpu_resource_spec = "q_gpu=TRUE" cluster = SGECluster( queue="q_1day", memory='4GB', cores=2, log_directory="/idiap/user/tpereira/github/snakemaking/dask/bob_bio/logs", local_directory="/idiap/user/tpereira/github/snakemaking/dask/bob_bio/logs", resource_spec=q1d_resource_spec) cluster.scale_up(NODES) client = Client(cluster) # start local workers as threads #""" ############# # HERE I'M TESTING HETEROGENEOUS JOBS # ### NODES q1d #for i in range(NODES-1): # GETTING THE SPEC TEMPLATE#
def main(): # dask cluster and client n_processes = 1 n_jobs = 35 n_workers = n_processes * n_jobs cluster = SGECluster( interface="ib0", walltime="01:00:00", memory=f"64 G", resource_spec=f"h_vmem=64G", scheduler_options={ "dashboard_address": ":5757", }, job_extra=[ "-cwd", "-V", f"-pe smp {n_processes}", f"-l disk=32G", ], local_directory=os.sep.join( [os.environ.get("PWD"), "dask-worker-scale-space"]), ) client = Client(cluster) cluster.scale(jobs=n_jobs) time_start = time.time() # scale custom outputs if normal: emission_configs = np.array( np.meshgrid( np.linspace(0.0, 1.4, 8), np.linspace(0.0, 1.4, 8), np.linspace(0.0, 1.4, 8), np.linspace(0.0, 1.4, 8), np.linspace(0.0, 1.4, 8), )).T.reshape(-1, 5) emission_configs_20percentintervals = [] for emission_config in emission_configs: emission_configs_20percentintervals.append( f'RES{round(emission_config[0], 1)}_IND{round(emission_config[1], 1)}_TRA{round(emission_config[2], 1)}_AGR{round(emission_config[3], 1)}_ENE{round(emission_config[4], 1)}' ) if extra: custom_inputs_main = [ np.array([[1.15, 1.27, 0.98, 0.98, 1.36]]), # bottom-up 2010 np.array([[1.19, 1.30, 1.01, 1.01, 1.46]]), # bottom-up 2011 np.array([[1.20, 1.30, 1.01, 1.02, 1.39]]), # bottom-up 2012 np.array([[1.13, 1.29, 1.02, 1.01, 1.29]]), # bottom-up 2013 np.array([[1.06, 1.12, 0.99, 1.01, 1.12]]), # bottom-up 2014 np.array([[0.92, 0.84, 0.97, 0.99, 0.94]]), # bottom-up 2016 np.array([[0.84, 0.81, 0.99, 0.99, 0.89]]), # bottom-up 2017 np.array([[0.76, 0.934, 0.735, 0.683, 0.708]]), np.array([[0.704, 0.786, 0.73, 0.659, 0.6]]), np.array([[0.712, 0.703, 0.725, 0.676, 0.649]]), np.array([[0.739, 0.668, 0.701, 0.686, 0.682]]), np.array([[0.67, 0.609, 0.709, 0.621, 0.661]]), np.array([[0.744, 0.904, 0.778, 0.678, 0.716]]), np.array([[0.771, 0.835, 0.711, 0.685, 0.544]]), np.array([[0.647, 0.945, 0.746, 0.588, 0.473]]), np.array([[0.657, 0.745, 0.714, 0.613, 0.591]]), np.array([[0.582, 0.7, 0.672, 0.5, 0.492]]), np.array([[0.803, 0.835, 0.742, 0.71, 0.717]]), np.array([[0.721, 0.863, 0.712, 0.74, 0.709]]), np.array([[0.661, 0.674, 0.694, 0.742, 0.715]]), np.array([[0.701, 0.642, 0.669, 0.681, 0.679]]), np.array([[0.604, 0.399, 0.659, 0.613, 0.724]]), np.array([[0.769, 1.009, 0.697, 0.69, 0.72]]), np.array([[0.824, 0.759, 0.767, 0.641, 0.429]]), np.array([[0.858, 1.092, 0.794, 0.604, 0.475]]), np.array([[0.8, 0.987, 0.648, 0.57, 0.493]]), np.array([[0.867, 0.957, 0.677, 0.558, 0.477]]) ] custom_inputs = [] for custom_input in custom_inputs_main: custom_input_res = np.copy(custom_input) custom_input_ind = np.copy(custom_input) custom_input_tra = np.copy(custom_input) custom_input_agr = np.copy(custom_input) custom_input_ene = np.copy(custom_input) custom_input_nores = np.copy(custom_input) custom_input_noind = np.copy(custom_input) custom_input_notra = np.copy(custom_input) custom_input_noagr = np.copy(custom_input) custom_input_noene = np.copy(custom_input) custom_input_resonly = np.copy(custom_input) custom_input_indonly = np.copy(custom_input) custom_input_traonly = np.copy(custom_input) custom_input_agronly = np.copy(custom_input) custom_input_eneonly = np.copy(custom_input) custom_input_res[0][1:] = 1.0 custom_input_ind[0][0] = 1.0 custom_input_ind[0][2:] = 1.0 custom_input_tra[0][:2] = 1.0 custom_input_tra[0][3:] = 1.0 custom_input_agr[0][:3] = 1.0 custom_input_agr[0][4:] = 1.0 custom_input_ene[0][:4] = 1.0 custom_input_nores[0][0] = 0.0 custom_input_noind[0][1] = 0.0 custom_input_notra[0][2] = 0.0 custom_input_noagr[0][3] = 0.0 custom_input_noene[0][4] = 0.0 custom_input_resonly[0][1:] = 0.0 custom_input_indonly[0][0] = 0.0 custom_input_indonly[0][2:] = 0.0 custom_input_traonly[0][:2] = 0.0 custom_input_traonly[0][3:] = 0.0 custom_input_agronly[0][:3] = 0.0 custom_input_agronly[0][4:] = 0.0 custom_input_eneonly[0][:4] = 0.0 custom_inputs.append(custom_input) custom_inputs.append(custom_input_res) custom_inputs.append(custom_input_ind) custom_inputs.append(custom_input_tra) custom_inputs.append(custom_input_agr) custom_inputs.append(custom_input_ene) custom_inputs.append(custom_input_nores) custom_inputs.append(custom_input_noind) custom_inputs.append(custom_input_notra) custom_inputs.append(custom_input_noagr) custom_inputs.append(custom_input_noene) custom_inputs.append(custom_input_resonly) custom_inputs.append(custom_input_indonly) custom_inputs.append(custom_input_traonly) custom_inputs.append(custom_input_agronly) custom_inputs.append(custom_input_eneonly) emission_configs_20percentintervals = [] for custom_input in custom_inputs: emission_config = f'RES{custom_input[0][0]:0.3f}_IND{custom_input[0][1]:0.3f}_TRA{custom_input[0][2]:0.3f}_AGR{custom_input[0][3]:0.3f}_ENE{custom_input[0][4]:0.3f}' emission_configs_20percentintervals.append(emission_config) if climate_cobenefits: custom_inputs_main = [ np.array([[0.91, 0.95, 0.85, 1.05, 0.96]]), # Base_CLE_2020 np.array([[0.91, 0.95, 0.85, 1.05, 0.96]]), # Base_MFR_2020 np.array([[0.91, 0.95, 0.85, 1.05, 0.96]]), # SDS_MFR_2020 np.array([[0.68, 0.84, 0.71, 1.16, 0.93]]), # Base_CLE_2030 np.array([[0.33, 0.47, 0.48, 0.81, 0.69]]), # Base_MFR_2030 np.array([[0.27, 0.45, 0.41, 0.81, 0.55]]), # SDS_MFR_2030 np.array([[0.57, 0.75, 0.69, 1.2, 0.94]]), # Base_CLE_2040 np.array([[0.24, 0.41, 0.31, 0.83, 0.73]]), # Base_MFR_2040 np.array([[0.19, 0.38, 0.22, 0.83, 0.5]]), # SDS_MFR_2040 np.array([[0.52, 0.72, 0.65, 1.24, 0.91]]), # Base_CLE_2050 np.array([[0.2, 0.38, 0.29, 0.86, 0.72]]), # Base_MFR_2050 np.array([[0.18, 0.35, 0.2, 0.86, 0.46]]), # SDS_MFR_2050 ] custom_inputs = [] for custom_input in custom_inputs_main: custom_input_res = np.copy(custom_input) custom_input_ind = np.copy(custom_input) custom_input_tra = np.copy(custom_input) custom_input_agr = np.copy(custom_input) custom_input_ene = np.copy(custom_input) custom_input_nores = np.copy(custom_input) custom_input_noind = np.copy(custom_input) custom_input_notra = np.copy(custom_input) custom_input_noagr = np.copy(custom_input) custom_input_noene = np.copy(custom_input) custom_input_resonly = np.copy(custom_input) custom_input_indonly = np.copy(custom_input) custom_input_traonly = np.copy(custom_input) custom_input_agronly = np.copy(custom_input) custom_input_eneonly = np.copy(custom_input) custom_input_res[0][1:] = 1.0 custom_input_ind[0][0] = 1.0 custom_input_ind[0][2:] = 1.0 custom_input_tra[0][:2] = 1.0 custom_input_tra[0][3:] = 1.0 custom_input_agr[0][:3] = 1.0 custom_input_agr[0][4:] = 1.0 custom_input_ene[0][:4] = 1.0 custom_input_nores[0][0] = 0.0 custom_input_noind[0][1] = 0.0 custom_input_notra[0][2] = 0.0 custom_input_noagr[0][3] = 0.0 custom_input_noene[0][4] = 0.0 custom_input_resonly[0][1:] = 0.0 custom_input_indonly[0][0] = 0.0 custom_input_indonly[0][2:] = 0.0 custom_input_traonly[0][:2] = 0.0 custom_input_traonly[0][3:] = 0.0 custom_input_agronly[0][:3] = 0.0 custom_input_agronly[0][4:] = 0.0 custom_input_eneonly[0][:4] = 0.0 custom_inputs.append(custom_input) custom_inputs.append(custom_input_res) custom_inputs.append(custom_input_ind) custom_inputs.append(custom_input_tra) custom_inputs.append(custom_input_agr) custom_inputs.append(custom_input_ene) custom_inputs.append(custom_input_nores) custom_inputs.append(custom_input_noind) custom_inputs.append(custom_input_notra) custom_inputs.append(custom_input_noagr) custom_inputs.append(custom_input_noene) custom_inputs.append(custom_input_resonly) custom_inputs.append(custom_input_indonly) custom_inputs.append(custom_input_traonly) custom_inputs.append(custom_input_agronly) custom_inputs.append(custom_input_eneonly) emission_configs_20percentintervals = [] for custom_input in custom_inputs: emission_config = f'RES{custom_input[0][0]:0.3f}_IND{custom_input[0][1]:0.3f}_TRA{custom_input[0][2]:0.3f}_AGR{custom_input[0][3]:0.3f}_ENE{custom_input[0][4]:0.3f}' emission_configs_20percentintervals.append(emission_config) if top_down_2020_baseline: emission_config_2020_baseline = np.array( [0.604, 0.399, 0.659, 0.613, 0.724]) # matching to PM2.5 only, top 1,000 emission_configs = np.array( np.meshgrid( np.linspace( emission_config_2020_baseline[0] * 0.50, emission_config_2020_baseline[0], 6 ), # 10% reduction increments from 2020 baseline up to 50% np.linspace(emission_config_2020_baseline[1] * 0.50, emission_config_2020_baseline[1], 6), np.linspace(emission_config_2020_baseline[2] * 0.50, emission_config_2020_baseline[2], 6), np.linspace(emission_config_2020_baseline[3] * 0.50, emission_config_2020_baseline[3], 6), np.linspace(emission_config_2020_baseline[4] * 0.50, emission_config_2020_baseline[4], 6), )).T.reshape(-1, 5) # add a couple more for larger reductions in RES and IND to reach WHO-IT2 emission_configs = list(emission_configs) emission_configs.append(np.array([0.242, 0.160, 0.659, 0.613, 0.724])) emission_configs.append(np.array([0.181, 0.120, 0.659, 0.613, 0.724])) emission_configs.append(np.array([0.121, 0.080, 0.659, 0.613, 0.724])) emission_configs.append(np.array([0.060, 0.040, 0.659, 0.613, 0.724])) emission_configs_20percentintervals = [] for emission_config in emission_configs: emission_configs_20percentintervals.append( f'RES{round(emission_config[0], 3):.3f}_IND{round(emission_config[1], 3):.3f}_TRA{round(emission_config[2], 3):.3f}_AGR{round(emission_config[3], 3):.3f}_ENE{round(emission_config[4], 3):.3f}' ) emission_configs_completed = glob.glob( f"/nobackup/earlacoa/machinelearning/data_annual/predictions/{output}_adjusted_scaled/ds*{output}_popgrid_0.25deg_adjusted_scaled.nc" ) emission_configs_completed = [ f"{item[88:-45]}" for item in emission_configs_completed ] emission_configs_20percentintervals_remaining_set = set( emission_configs_20percentintervals) - set(emission_configs_completed) emission_configs_remaining = [ item for item in emission_configs_20percentintervals_remaining_set ] print( f"custom outputs remaining for {output}: {len(emission_configs_remaining)} - 20% intervals with {int(100 * len(emission_configs_20percentintervals_remaining_set) / len(emission_configs_20percentintervals))}% remaining" ) # dask bag and process emission_configs_remaining = emission_configs_remaining[:35000] print( f"predicting for {len(emission_configs_remaining)} custom outputs ...") bag_emission_configs = db.from_sequence(emission_configs_remaining, npartitions=n_workers) bag_emission_configs.map(scale).compute() time_end = time.time() - time_start print( f"completed in {time_end:0.2f} seconds, or {time_end / 60:0.2f} minutes, or {time_end / 3600:0.2f} hours" ) print( f"average time per custom output is {time_end / len(emission_configs_remaining):0.2f} seconds" ) client.close() cluster.close()
def run_JK_distributed(df, param, randomize=True): '''Receives the pandas dataframe with the objects containing the temperature decrements and the parameter object and run the kSZ statistic and generate Jack Knifes. Everything runs in the cluster, so current terminal does not need to request many cpus. df: dataframe object containing the variables for the calculation params: param file for this calculation NJK: how many subgroups we will make to run the calculation randomize: shuffle data before running the JK''' Ncores = envVars.Ncores NWorkers = envVars.NWorkers Ngroups = param.JK_NGROUPS resampling_method = param.JK_RESAMPLING_METHOD.lower() #setup cluster cluster = SGECluster(walltime='172800', processes=1, cores=1, env_extra=['#$-pe sge_pe %i' % Ncores, '-l m_core=%i' % Ncores, 'mkdir -p /tmp/pag227/dask/dask-scratch', 'export NUMBA_NUM_THREADS=%i' % Ncores, 'export OMP_NUM_THREADS=%i' % Ncores # 'export OMP_NUM_THREADS=1', # noqa ]) cluster.scale(NWorkers) client = Client(cluster) time.sleep(30) #end setting up cluster #send full dataset to the cluster future_fullDataset = client.scatter(df) future_params = client.scatter(param) res_fullDataset = client.submit(pairwiser.get_pairwise_ksz, future_fullDataset, future_params, multithreading=True) #done with the full dataset #iterate over partial dataset for the JK if JK == resampling_method: indices_toDrop = JK_tools.indicesToDrop(df, Ngroups, randomize=randomize) jk_results = [] futureData = [] #data to be sent in jk or bootstrap in galaxy space if (JK == resampling_method) or (BS == resampling_method): for j in range(Ngroups): # submit data to the cluster if JK in resampling_method: # if method jk dataJK = df.drop(indices_toDrop[j], inplace=False) futureData.append(client.scatter(dataJK)) elif BS in resampling_method: dataBS = df.sample(len(df), replace=True) futureData.append(client.scatter(dataBS)) #Now do the JK calculation for j in range(Ngroups): jk_results.append(client.submit(pairwiser.get_pairwise_ksz, futureData[j], future_params, multithreading=True)) if BS_PW == resampling_method: # submit the same dataset futureData = client.scatter(df, broadcast=True) for j in range(Ngroups): jk_results.append(client.submit(bs_pw.get_bootstrap_pairwise, futureData, future_params, multithreading=True, pure=False)) if resampling_method == BS_DT: for j in range(Ngroups): df_bs = df.copy() choose = np.random.choice(len(df), len(df)) df_bs['dT'] = df.dT.values[choose] futureData.append(client.scatter(df_bs)) for j in range(Ngroups): jk_results.append(client.submit(pairwiser.get_pairwise_ksz, futureData[j], future_params, multithreading=True)) if resampling_method == TL_JK: tiled_JK.classify_grid(df) df = tiled_JK.remove_edge_galaxies(df, tol_sigma=1.5) Ntiles = tiled_JK.how_many_tiles(df) for j in range(Ntiles): df_tosubmit = tiled_JK.remove_tile(df, j) futureData.append(client.scatter(df_tosubmit)) for j in range(Ntiles): jk_results.append(client.submit(pairwiser.get_pairwise_ksz, futureData[j], future_params, multithreading=True)) #extract results fullDataset_results = res_fullDataset.result() jk_results = client.gather(jk_results) client.close() # cluster.close() return fullDataset_results, jk_results
if __name__ == '__main__': data = "./clustering_Biden/" N = 10000000 #n_workers = int(os.getenv('NSLOTS')) #print("n_workers: ", n_workers) #from dask.distributed import LocalCluster #cluster = LocalCluster() from dask_jobqueue import SGECluster cluster = SGECluster(cores=28, memory='200GB', job_extra=['-P scv', '-pe mpi_28_tasks_per_node 112']) #cluster.scale(4) from dask.distributed import Client, progress #client = Client("tcp://192.168.19.195:8786") client = Client(cluster) print(client) entries = os.listdir(data) #from multiprocessing import Process, Lock, Manager # About 1min/file, may need parallization later. if (not path.exists("./dict_A.pickle")): t1 = time.time() # A dictionary that holds all records in A
def execute(self, pipeline_context, execution_plan): check.inst_param(pipeline_context, "pipeline_context", SystemPipelineExecutionContext) check.inst_param(execution_plan, "execution_plan", ExecutionPlan) check.param_invariant( isinstance(pipeline_context.executor, DaskExecutor), "pipeline_context", "Expected executor to be DaskExecutor got {}".format( pipeline_context.executor), ) check.invariant( pipeline_context.instance.is_persistent, "Dask execution requires a persistent DagsterInstance", ) step_levels = execution_plan.execution_step_levels() pipeline_name = pipeline_context.pipeline_def.name instance = pipeline_context.instance cluster_type = self.cluster_type if cluster_type == "local": from dask.distributed import LocalCluster cluster = LocalCluster(**self.build_dict(pipeline_name)) elif cluster_type == "yarn": from dask_yarn import YarnCluster cluster = YarnCluster(**self.build_dict(pipeline_name)) elif cluster_type == "ssh": from dask.distributed import SSHCluster cluster = SSHCluster(**self.build_dict(pipeline_name)) elif cluster_type == "pbs": from dask_jobqueue import PBSCluster cluster = PBSCluster(**self.build_dict(pipeline_name)) elif cluster_type == "moab": from dask_jobqueue import MoabCluster cluster = MoabCluster(**self.build_dict(pipeline_name)) elif cluster_type == "sge": from dask_jobqueue import SGECluster cluster = SGECluster(**self.build_dict(pipeline_name)) elif cluster_type == "lsf": from dask_jobqueue import LSFCluster cluster = LSFCluster(**self.build_dict(pipeline_name)) elif cluster_type == "slurm": from dask_jobqueue import SLURMCluster cluster = SLURMCluster(**self.build_dict(pipeline_name)) elif cluster_type == "oar": from dask_jobqueue import OARCluster cluster = OARCluster(**self.build_dict(pipeline_name)) elif cluster_type == "kube": from dask_kubernetes import KubeCluster cluster = KubeCluster(**self.build_dict(pipeline_name)) else: raise ValueError( f"Must be providing one of the following ('local', 'yarn', 'ssh', 'pbs', 'moab', 'sge', 'lsf', 'slurm', 'oar', 'kube') not {cluster_type}" ) with dask.distributed.Client(cluster) as client: execution_futures = [] execution_futures_dict = {} for step_level in step_levels: for step in step_level: # We ensure correctness in sequencing by letting Dask schedule futures and # awaiting dependencies within each step. dependencies = [] for step_input in step.step_inputs: for key in step_input.dependency_keys: dependencies.append(execution_futures_dict[key]) run_config = dict(pipeline_context.run_config, execution={"in_process": {}}) recon_repo = pipeline_context.pipeline.get_reconstructable_repository( ) dask_task_name = "%s.%s" % (pipeline_name, step.key) recon_pipeline = recon_repo.get_reconstructable_pipeline( pipeline_name) future = client.submit( query_on_dask_worker, dependencies, recon_pipeline, pipeline_context.pipeline_run, run_config, [step.key], pipeline_context.mode_def.name, instance.get_ref(), key=dask_task_name, resources=get_dask_resource_requirements(step.tags), ) execution_futures.append(future) execution_futures_dict[step.key] = future # This tells Dask to awaits the step executions and retrieve their results to the # master futures = dask.distributed.as_completed(execution_futures, with_results=True) # Allow interrupts while waiting for the results from Dask for future, result in iterate_with_context( raise_interrupts_immediately, futures): for step_event in result: check.inst(step_event, DagsterEvent) yield step_event
def run_error_estimation_distributed(df1, df2, param): Ncores = envVars.Ncores NWorkers = envVars.NWorkers Ngroups = param.JK_NGROUPS #setup cluster cluster = SGECluster( walltime='172800', processes=1, cores=1, env_extra=[ '#$-pe sge_pe %i' % Ncores, '-l m_core=%i' % Ncores, 'mkdir -p /tmp/pag227/dask/dask-scratch', 'export NUMBA_NUM_THREADS=%i' % Ncores, 'export OMP_NUM_THREADS=%i' % Ncores # 'export OMP_NUM_THREADS=1', # noqa ]) cluster.scale(NWorkers) client = Client(cluster) time.sleep(10) #end setting up cluster #send full dataset to the cluster future_df1 = client.scatter(df1) future_df2 = client.scatter(df2) future_params = client.scatter(param) res_fullDataset_11 = client.submit(cpw.get_cross_pairwise_ksz, future_df1, future_df1, future_params) res_fullDataset_12 = client.submit(cpw.get_cross_pairwise_ksz, future_df1, future_df2, future_params) res_fullDataset_22 = client.submit(cpw.get_cross_pairwise_ksz, future_df2, future_df2, future_params) #done with the full dataset #iterate over partial dataset for the JK replicants1 = [] #data to be sent replicants2 = [] if 'jk' in param.JK_RESAMPLING_METHOD.lower(): all_indx = np.arange(len(df1)) np.random.shuffle(all_indx) indx_to_drop = np.array_split(all_indx, param.JK_NGROUPS) for j in range(Ngroups): # submit data to the cluster if 'jk' in param.JK_RESAMPLING_METHOD.lower(): # if method jk todrop = indx_to_drop[j] replicant1 = df1.drop(df1.index[todrop], inplace=False) replicant2 = df2.drop(df2.index[todrop], inplace=False) replicants1.append(client.scatter(replicant1)) replicants2.append(client.scatter(replicant2)) elif 'bootstrap' in param.JK_RESAMPLING_METHOD.lower(): indxs = np.random.randint(low=0, high=len(df1), size=len(df1)) replicant1 = df1.iloc[indxs] replicant2 = df2.iloc[indxs] replicants1.append(client.scatter(replicant1)) replicants2.append(client.scatter(replicant2)) #Now do the JK calculation realizations11 = [] realizations12 = [] realizations22 = [] for j in range(Ngroups): realizations11.append( client.submit(cpw.get_cross_pairwise_ksz, replicants1[j], replicants1[j], future_params)) realizations12.append( client.submit(cpw.get_cross_pairwise_ksz, replicants1[j], replicants2[j], future_params)) realizations22.append( client.submit(cpw.get_cross_pairwise_ksz, replicants2[j], replicants2[j], future_params)) #extract results fullDataset_result11 = res_fullDataset_11.result() fullDataset_result12 = res_fullDataset_12.result() fullDataset_result22 = res_fullDataset_22.result() resampling_result11 = client.gather(realizations11) resampling_result12 = client.gather(realizations12) resampling_result22 = client.gather(realizations22) client.close() # cluster.close() results = { 'full11': fullDataset_result11, 'full12': fullDataset_result12, 'full22': fullDataset_result22, 'resampled11': resampling_result11, 'resampled12': resampling_result12, 'resampled22': resampling_result22 } return results
def execute(self, pipeline_context, execution_plan): check.inst_param(pipeline_context, 'pipeline_context', SystemPipelineExecutionContext) check.inst_param(execution_plan, 'execution_plan', ExecutionPlan) check.param_invariant( isinstance(pipeline_context.executor, DaskExecutor), 'pipeline_context', 'Expected executor to be DaskExecutor got {}'.format( pipeline_context.executor), ) # Checks to ensure storage is compatible with Dask configuration storage = pipeline_context.run_config.get('storage') check.invariant(storage.keys(), 'Must specify storage to use Dask execution') check.invariant( pipeline_context.instance.is_persistent, 'Dask execution requires a persistent DagsterInstance', ) # https://github.com/dagster-io/dagster/issues/2440 check.invariant( pipeline_context.system_storage_def.is_persistent, 'Cannot use in-memory storage with Dask, use filesystem, S3, or GCS', ) step_levels = execution_plan.execution_step_levels() pipeline_name = pipeline_context.pipeline_def.name instance = pipeline_context.instance cluster_type = self.cluster_type if cluster_type == 'local': from dask.distributed import LocalCluster cluster = LocalCluster(**self.build_dict(pipeline_name)) elif cluster_type == 'yarn': from dask_yarn import YarnCluster cluster = YarnCluster(**self.build_dict(pipeline_name)) elif cluster_type == 'ssh': from dask.distributed import SSHCluster cluster = SSHCluster(**self.build_dict(pipeline_name)) elif cluster_type == 'pbs': from dask_jobqueue import PBSCluster cluster = PBSCluster(**self.build_dict(pipeline_name)) elif cluster_type == 'moab': from dask_jobqueue import MoabCluster cluster = MoabCluster(**self.build_dict(pipeline_name)) elif cluster_type == 'sge': from dask_jobqueue import SGECluster cluster = SGECluster(**self.build_dict(pipeline_name)) elif cluster_type == 'lsf': from dask_jobqueue import LSFCluster cluster = LSFCluster(**self.build_dict(pipeline_name)) elif cluster_type == 'slurm': from dask_jobqueue import SLURMCluster cluster = SLURMCluster(**self.build_dict(pipeline_name)) elif cluster_type == 'oar': from dask_jobqueue import OARCluster cluster = OARCluster(**self.build_dict(pipeline_name)) elif cluster_type == 'kube': from dask_kubernetes import KubeCluster cluster = KubeCluster(**self.build_dict(pipeline_name)) else: raise ValueError( f"Must be providing one of the following ('local', 'yarn', 'ssh', 'pbs', 'moab', 'sge', 'lsf', 'slurm', 'oar', 'kube') not {cluster_type}" ) with dask.distributed.Client(cluster) as client: execution_futures = [] execution_futures_dict = {} for step_level in step_levels: for step in step_level: # We ensure correctness in sequencing by letting Dask schedule futures and # awaiting dependencies within each step. dependencies = [] for step_input in step.step_inputs: for key in step_input.dependency_keys: dependencies.append(execution_futures_dict[key]) run_config = dict(pipeline_context.run_config, execution={'in_process': {}}) recon_repo = pipeline_context.pipeline.get_reconstructable_repository( ) variables = { 'executionParams': { 'selector': { 'pipelineName': pipeline_name, 'repositoryName': recon_repo.get_definition().name, 'repositoryLocationName': '<<in_process>>', }, 'runConfigData': run_config, 'mode': pipeline_context.mode_def.name, 'executionMetadata': { 'runId': pipeline_context.pipeline_run.run_id }, 'stepKeys': [step.key], } } dask_task_name = '%s.%s' % (pipeline_name, step.key) workspace = create_in_process_ephemeral_workspace( pointer=pipeline_context.pipeline. get_reconstructable_repository().pointer) future = client.submit( query_on_dask_worker, workspace, variables, dependencies, instance.get_ref(), key=dask_task_name, resources=get_dask_resource_requirements(step.tags), ) execution_futures.append(future) execution_futures_dict[step.key] = future # This tells Dask to awaits the step executions and retrieve their results to the # master for future in dask.distributed.as_completed(execution_futures): for step_event in future.result(): check.inst(step_event, DagsterEvent) yield step_event
def run_dask(options: dict, docker_username: str = None, docker_password: str = None, docker: bool = False, slurm_job_array: bool = False): try: if 'jobqueue' not in options: cluster = LocalCluster() else: jobqueue = options['jobqueue'] gpus = options['gpus'] if 'gpus' in options else 0 if 'slurm' in jobqueue: print("Requesting SLURM cluster:") pprint(jobqueue['slurm']) cluster = SLURMCluster(job_extra=[f"--gres=gpu:{gpus}"], **jobqueue['slurm']) if gpus else SLURMCluster(**jobqueue['slurm']) elif 'pbs' in jobqueue: print("Requesting PBS cluster:") pprint(jobqueue['pbs']) cluster = PBSCluster(job_extra=[f"--gres=gpu:{gpus}"], **jobqueue['pbs']) if gpus else PBSCluster(**jobqueue['pbs']) elif 'moab' in jobqueue: print("Requesting MOAB cluster:") pprint(jobqueue['moab']) cluster = MoabCluster(job_extra=[f"--gres=gpu:{gpus}"], **jobqueue['moab']) if gpus else MoabCluster(**jobqueue['moab']) elif 'sge' in jobqueue: print("Requesting SGE cluster:") pprint(jobqueue['sge']) cluster = SGECluster(job_extra=[f"--gres=gpu:{gpus}"], **jobqueue['sge']) if gpus else SGECluster(**jobqueue['sge']) elif 'lsf' in jobqueue: print("Requesting LSF cluster:") pprint(jobqueue['lsf']) cluster = LSFCluster(job_extra=[f"--gres=gpu:{gpus}"], **jobqueue['lsf']) if gpus else LSFCluster(**jobqueue['lsf']) elif 'oar' in jobqueue: print("Requesting OAR cluster:") pprint(jobqueue['oar']) cluster = OARCluster(job_extra=[f"--gres=gpu:{gpus}"], **jobqueue['oar']) if gpus else OARCluster(**jobqueue['oar']) else: raise ValueError(f"Unsupported jobqueue configuration: {jobqueue}") print(f"Cluster job script: {cluster.job_script()}") if 'output' in options and 'from' in options['output']: output_path = options['output']['from'] else: output_path = '.' if 'input' not in options: env = options['env'] if 'env' in options else [] params = options['parameters'] if 'parameters' in options else [] bind_mounts = options['bind_mounts'] if 'bind_mounts' in options else [] no_cache = options['no_cache'] if 'no_cache' in options else False gpus = options['gpus'] if 'gpus' in options else 0 if 'jobqueue' in options: cluster.scale(1) with Client(cluster) as client: command = prep_command( work_dir=options['workdir'], image=options['image'], command=options['command'], env=env + [{'key': 'INDEX', 'value': 1}], parameters=params + [{'key': 'OUTPUT', 'value': output_path}], bind_mounts=bind_mounts, no_cache=no_cache, gpus=gpus, docker_username=docker_username, docker_password=docker_password, docker=docker) logger.info(f"Submitting container") future = submit_command(client, command, options['log_file'] if 'log_file' in options else None, 3) future.result() if future.status != 'finished': logger.error(f"Container failed: {future.exception}") else: logger.info(f"Container completed") elif options['input']['kind'] == InputKind.DIRECTORY: input_path = options['input']['path'] env = options['env'] if 'env' in options else [] params = options['parameters'] if 'parameters' in options else [] bind_mounts = options['bind_mounts'] if 'bind_mounts' in options else [] no_cache = options['no_cache'] if 'no_cache' in options else False gpus = options['gpus'] if 'gpus' in options else 0 if 'jobqueue' in options: cluster.scale(1) with Client(cluster) as client: command = prep_command( work_dir=options['workdir'], image=options['image'], command=options['command'], env=env + [{'key': 'INDEX', 'value': 1}], parameters=params + [{'key': 'INPUT', 'value': input_path}, {'key': 'OUTPUT', 'value': output_path}], bind_mounts=bind_mounts, no_cache=no_cache, gpus=gpus, docker_username=docker_username, docker_password=docker_password, docker=docker) logger.info(f"Submitting container for directory '{input_path}'") future = submit_command(client, command, options['log_file'] if 'log_file' in options else None, 3) future.result() if future.status != 'finished': logger.error(f"Container failed for directory '{input_path}': {future.exception}") else: logger.info(f"Container completed for directory '{input_path}'") elif options['input']['kind'] == InputKind.FILES: input_path = options['input']['path'] if slurm_job_array: files = os.listdir(input_path) file_id = int(os.environ.get('SLURM_ARRAY_TASK_ID')) current_file = files[file_id] env = options['env'] if 'env' in options else [] params = options['parameters'] if 'parameters' in options else [] patterns = options['input']['patterns'] if 'patterns' in options['input'] else [] bind_mounts = options['bind_mounts'] if 'bind_mounts' in options else [] no_cache = options['no_cache'] if 'no_cache' in options else False gpus = options['gpus'] if 'gpus' in options else 0 if 'jobqueue' in options: cluster.scale(1) with Client(cluster) as client: command = prep_command( work_dir=options['workdir'], image=options['image'], command=options['command'], env=env + [{'key': 'INDEX', 'value': file_id}] + [{'key': 'PATTERNS', 'value': ','.join(patterns)}], parameters=params + [{'key': 'INPUT', 'value': join(input_path, current_file)}, {'key': 'OUTPUT', 'value': output_path}], bind_mounts=bind_mounts, no_cache=no_cache, gpus=gpus, docker_username=docker_username, docker_password=docker_password, docker=docker) logger.info(f"Submitting container for file '{input_path}'") future = submit_command(client, command, options['log_file'] if 'log_file' in options else None, 3) future.result() if future.status != 'finished': logger.error(f"Container failed for file '{input_path}': {future.exception}") else: logger.info(f"Container completed for file '{input_path}'") logger.info(f"Run succeeded") else: files = os.listdir(input_path) count = len(files) futures = [] if 'jobqueue' not in options: logger.info(f"Processing {count} files in '{input_path}'") else: logger.info(f"Requesting {count} nodes to process {count} files in '{input_path}' with job script:\n{cluster.job_script()}") cluster.scale(count) env = options['env'] if 'env' in options else [] params = deepcopy(options['parameters']) if 'parameters' in options else [] patterns = options['input']['patterns'] if 'patterns' in options['input'] else [] bind_mounts = options['bind_mounts'] if 'bind_mounts' in options else [] no_cache = options['no_cache'] if 'no_cache' in options else False gpus = options['gpus'] if 'gpus' in options else 0 with Client(cluster) as client: num_files = len(files) for i, current_file in tqdm.tqdm(enumerate(files), total=num_files): command = prep_command( work_dir=options['workdir'], image=options['image'], command=options['command'], env=env + [{'key': 'INDEX', 'value': i}] + [{'key': 'PATTERNS', 'value': ','.join(patterns)}], parameters=params + [{'key': 'INPUT', 'value': join(input_path, current_file)}, {'key': 'OUTPUT', 'value': output_path}], bind_mounts=bind_mounts, no_cache=no_cache, gpus=gpus, docker_username=docker_username, docker_password=docker_password, docker=docker) logger.info(f"Submitting container for file {i}") futures.append(submit_command(client, command, options['log_file'] if 'log_file' in options else None, 3)) finished = 0 for future in tqdm.tqdm(as_completed(futures), total=num_files): finished += 1 if future.status != 'finished': logger.error(f"Container failed for file {finished}: {future.exception}") else: logger.info(f"Container completed for file {finished}") elif options['input']['kind'] == InputKind.FILE: input_path = options['input']['path'] env = options['env'] if 'env' in options else [] params = options['parameters'] if 'parameters' in options else [] patterns = options['input']['patterns'] if 'patterns' in options['input'] else [] bind_mounts = options['bind_mounts'] if 'bind_mounts' in options else [] no_cache = options['no_cache'] if 'no_cache' in options else False gpus = options['gpus'] if 'gpus' in options else 0 if 'jobqueue' in options: cluster.scale(1) with Client(cluster) as client: command = prep_command( work_dir=options['workdir'], image=options['image'], command=options['command'], env=env + [{'key': 'INDEX', 'value': 1}] + [{'key': 'PATTERNS', 'value': ','.join(patterns)}], parameters=params + [{'key': 'INPUT', 'value': input_path}, {'key': 'OUTPUT', 'value': output_path}], bind_mounts=bind_mounts, no_cache=no_cache, gpus=gpus, docker_username=docker_username, docker_password=docker_password, docker=docker) logger.info(f"Submitting container for file 1") future = submit_command(client, command, options['log_file'] if 'log_file' in options else None, 3) future.result() if future.status != 'finished': logger.error(f"Container failed for file 1") logger.error(future.exception) else: logger.info(f"Container completed for file 1") logger.info(f"Run succeeded") except: logger.error(f"Run failed: {traceback.format_exc()}") raise
def main(): # dask cluster and client n_processes = 1 n_jobs = 35 n_workers = n_processes * n_jobs cluster = SGECluster( interface="ib0", walltime="01:00:00", memory=f"2 G", resource_spec=f"h_vmem=2G", scheduler_options={ "dashboard_address": ":5757", }, job_extra=[ "-cwd", "-V", f"-pe smp {n_processes}", f"-l disk=1G", ], local_directory=os.sep.join( [os.environ.get("PWD"), "dask-worker-space"]), ) client = Client(cluster) cluster.scale(jobs=n_jobs) time_start = time.time() # custom inputs if normal: matrix_stacked = np.array( np.meshgrid( np.linspace( 0, 1.5, 16 ), # 1.5 and 16 for 0.1, 1.5 and 6 for 0.3, 1.4 and 8 for 0.2 np.linspace(0, 1.5, 16), np.linspace(0, 1.5, 16), np.linspace(0, 1.5, 16), np.linspace(0, 1.5, 16), )).T.reshape(-1, 5) custom_inputs_set = set( tuple(map(float, map("{:.1f}".format, item))) for item in matrix_stacked) custom_inputs_completed_filenames = glob.glob( f"/nobackup/earlacoa/machinelearning/data_annual/predictions/{output}/ds*{output}*" ) custom_inputs_completed_list = [] for custom_inputs_completed_filename in custom_inputs_completed_filenames: custom_inputs_completed_list.append([ float(item) for item in re.findall( r"\d+\.\d+", custom_inputs_completed_filename) ]) custom_inputs_completed_set = set( tuple(item) for item in custom_inputs_completed_list) custom_inputs_remaining_set = custom_inputs_set - custom_inputs_completed_set custom_inputs = [ np.array(item).reshape(1, -1) for item in custom_inputs_remaining_set ] print(f"custom inputs remaining for {output}: {len(custom_inputs)}") if extra: custom_inputs_main = [ np.array([[1.15, 1.27, 0.98, 0.98, 1.36]]), # bottom-up 2010 np.array([[1.19, 1.30, 1.01, 1.01, 1.46]]), # bottom-up 2011 np.array([[1.20, 1.30, 1.01, 1.02, 1.39]]), # bottom-up 2012 np.array([[1.13, 1.29, 1.02, 1.01, 1.29]]), # bottom-up 2013 np.array([[1.06, 1.12, 0.99, 1.01, 1.12]]), # bottom-up 2014 np.array([[0.92, 0.84, 0.97, 0.99, 0.94]]), # bottom-up 2016 np.array([[0.84, 0.81, 0.99, 0.99, 0.89]]), # bottom-up 2017 np.array([[0.76, 0.934, 0.735, 0.683, 0.708]]), np.array([[0.704, 0.786, 0.73, 0.659, 0.6]]), np.array([[0.712, 0.703, 0.725, 0.676, 0.649]]), np.array([[0.739, 0.668, 0.701, 0.686, 0.682]]), np.array([[0.67, 0.609, 0.709, 0.621, 0.661]]), np.array([[0.744, 0.904, 0.778, 0.678, 0.716]]), np.array([[0.771, 0.835, 0.711, 0.685, 0.544]]), np.array([[0.647, 0.945, 0.746, 0.588, 0.473]]), np.array([[0.657, 0.745, 0.714, 0.613, 0.591]]), np.array([[0.582, 0.7, 0.672, 0.5, 0.492]]), np.array([[0.803, 0.835, 0.742, 0.71, 0.717]]), np.array([[0.721, 0.863, 0.712, 0.74, 0.709]]), np.array([[0.661, 0.674, 0.694, 0.742, 0.715]]), np.array([[0.701, 0.642, 0.669, 0.681, 0.679]]), np.array([[0.604, 0.399, 0.659, 0.613, 0.724]]), np.array([[0.769, 1.009, 0.697, 0.69, 0.72]]), np.array([[0.824, 0.759, 0.767, 0.641, 0.429]]), np.array([[0.858, 1.092, 0.794, 0.604, 0.475]]), np.array([[0.8, 0.987, 0.648, 0.57, 0.493]]), np.array([[0.867, 0.957, 0.677, 0.558, 0.477]]) ] custom_inputs = [] for custom_input in custom_inputs_main: custom_input_res = np.copy(custom_input) custom_input_ind = np.copy(custom_input) custom_input_tra = np.copy(custom_input) custom_input_agr = np.copy(custom_input) custom_input_ene = np.copy(custom_input) custom_input_nores = np.copy(custom_input) custom_input_noind = np.copy(custom_input) custom_input_notra = np.copy(custom_input) custom_input_noagr = np.copy(custom_input) custom_input_noene = np.copy(custom_input) custom_input_resonly = np.copy(custom_input) custom_input_indonly = np.copy(custom_input) custom_input_traonly = np.copy(custom_input) custom_input_agronly = np.copy(custom_input) custom_input_eneonly = np.copy(custom_input) custom_input_res[0][1:] = 1.0 custom_input_ind[0][0] = 1.0 custom_input_ind[0][2:] = 1.0 custom_input_tra[0][:2] = 1.0 custom_input_tra[0][3:] = 1.0 custom_input_agr[0][:3] = 1.0 custom_input_agr[0][4:] = 1.0 custom_input_ene[0][:4] = 1.0 custom_input_nores[0][0] = 0.0 custom_input_noind[0][1] = 0.0 custom_input_notra[0][2] = 0.0 custom_input_noagr[0][3] = 0.0 custom_input_noene[0][4] = 0.0 custom_input_resonly[0][1:] = 0.0 custom_input_indonly[0][0] = 0.0 custom_input_indonly[0][2:] = 0.0 custom_input_traonly[0][:2] = 0.0 custom_input_traonly[0][3:] = 0.0 custom_input_agronly[0][:3] = 0.0 custom_input_agronly[0][4:] = 0.0 custom_input_eneonly[0][:4] = 0.0 custom_inputs.append(custom_input) custom_inputs.append(custom_input_res) custom_inputs.append(custom_input_ind) custom_inputs.append(custom_input_tra) custom_inputs.append(custom_input_agr) custom_inputs.append(custom_input_ene) custom_inputs.append(custom_input_nores) custom_inputs.append(custom_input_noind) custom_inputs.append(custom_input_notra) custom_inputs.append(custom_input_noagr) custom_inputs.append(custom_input_noene) custom_inputs.append(custom_input_resonly) custom_inputs.append(custom_input_indonly) custom_inputs.append(custom_input_traonly) custom_inputs.append(custom_input_agronly) custom_inputs.append(custom_input_eneonly) # just for emulator_predictions.py as this is required in order to adjust for double emissions custom_inputs_temp = custom_inputs.copy() for custom_input in custom_inputs_temp: custom_input_resonly = np.copy(custom_input) custom_input_indonly = np.copy(custom_input) custom_input_traonly = np.copy(custom_input) custom_input_agronly = np.copy(custom_input) custom_input_eneonly = np.copy(custom_input) custom_input_resonly[0][1:] = 0.0 custom_input_indonly[0][0] = 0.0 custom_input_indonly[0][2:] = 0.0 custom_input_traonly[0][:2] = 0.0 custom_input_traonly[0][3:] = 0.0 custom_input_agronly[0][:3] = 0.0 custom_input_agronly[0][4:] = 0.0 custom_input_eneonly[0][:4] = 0.0 custom_inputs.append(custom_input_resonly) custom_inputs.append(custom_input_indonly) custom_inputs.append(custom_input_traonly) custom_inputs.append(custom_input_agronly) custom_inputs.append(custom_input_eneonly) emission_configs_20percentintervals = [] for custom_input in custom_inputs: emission_config = f'RES{custom_input[0][0]:0.3f}_IND{custom_input[0][1]:0.3f}_TRA{custom_input[0][2]:0.3f}_AGR{custom_input[0][3]:0.3f}_ENE{custom_input[0][4]:0.3f}' emission_configs_20percentintervals.append(emission_config) emission_configs_20percentintervals = list( set(emission_configs_20percentintervals)) custom_inputs = [] for emission_config in emission_configs_20percentintervals: custom_input = np.array([ float(num) for num in re.findall(r'\d.\d+', emission_config) ]).reshape(1, -1) custom_inputs.append(custom_input) if climate_cobenefits: custom_inputs_main = [ np.array([[0.91, 0.95, 0.85, 1.05, 0.96]]), # Base_CLE_2020 np.array([[0.91, 0.95, 0.85, 1.05, 0.96]]), # Base_MFR_2020 np.array([[0.91, 0.95, 0.85, 1.05, 0.96]]), # SDS_MFR_2020 np.array([[0.68, 0.84, 0.71, 1.16, 0.93]]), # Base_CLE_2030 np.array([[0.33, 0.47, 0.48, 0.81, 0.69]]), # Base_MFR_2030 np.array([[0.27, 0.45, 0.41, 0.81, 0.55]]), # SDS_MFR_2030 np.array([[0.57, 0.75, 0.69, 1.2, 0.94]]), # Base_CLE_2040 np.array([[0.24, 0.41, 0.31, 0.83, 0.73]]), # Base_MFR_2040 np.array([[0.19, 0.38, 0.22, 0.83, 0.5]]), # SDS_MFR_2040 np.array([[0.52, 0.72, 0.65, 1.24, 0.91]]), # Base_CLE_2050 np.array([[0.2, 0.38, 0.29, 0.86, 0.72]]), # Base_MFR_2050 np.array([[0.18, 0.35, 0.2, 0.86, 0.46]]), # SDS_MFR_2050 ] custom_inputs = [] for custom_input in custom_inputs_main: custom_input_res = np.copy(custom_input) custom_input_ind = np.copy(custom_input) custom_input_tra = np.copy(custom_input) custom_input_agr = np.copy(custom_input) custom_input_ene = np.copy(custom_input) custom_input_nores = np.copy(custom_input) custom_input_noind = np.copy(custom_input) custom_input_notra = np.copy(custom_input) custom_input_noagr = np.copy(custom_input) custom_input_noene = np.copy(custom_input) custom_input_resonly = np.copy(custom_input) custom_input_indonly = np.copy(custom_input) custom_input_traonly = np.copy(custom_input) custom_input_agronly = np.copy(custom_input) custom_input_eneonly = np.copy(custom_input) custom_input_res[0][1:] = 1.0 custom_input_ind[0][0] = 1.0 custom_input_ind[0][2:] = 1.0 custom_input_tra[0][:2] = 1.0 custom_input_tra[0][3:] = 1.0 custom_input_agr[0][:3] = 1.0 custom_input_agr[0][4:] = 1.0 custom_input_ene[0][:4] = 1.0 custom_input_nores[0][0] = 0.0 custom_input_noind[0][1] = 0.0 custom_input_notra[0][2] = 0.0 custom_input_noagr[0][3] = 0.0 custom_input_noene[0][4] = 0.0 custom_input_resonly[0][1:] = 0.0 custom_input_indonly[0][0] = 0.0 custom_input_indonly[0][2:] = 0.0 custom_input_traonly[0][:2] = 0.0 custom_input_traonly[0][3:] = 0.0 custom_input_agronly[0][:3] = 0.0 custom_input_agronly[0][4:] = 0.0 custom_input_eneonly[0][:4] = 0.0 custom_inputs.append(custom_input) custom_inputs.append(custom_input_res) custom_inputs.append(custom_input_ind) custom_inputs.append(custom_input_tra) custom_inputs.append(custom_input_agr) custom_inputs.append(custom_input_ene) custom_inputs.append(custom_input_nores) custom_inputs.append(custom_input_noind) custom_inputs.append(custom_input_notra) custom_inputs.append(custom_input_noagr) custom_inputs.append(custom_input_noene) custom_inputs.append(custom_input_resonly) custom_inputs.append(custom_input_indonly) custom_inputs.append(custom_input_traonly) custom_inputs.append(custom_input_agronly) custom_inputs.append(custom_input_eneonly) # just for emulator_predictions.py as this is required in order to adjust for double emissions custom_inputs_temp = custom_inputs.copy() for custom_input in custom_inputs_temp: custom_input_resonly = np.copy(custom_input) custom_input_indonly = np.copy(custom_input) custom_input_traonly = np.copy(custom_input) custom_input_agronly = np.copy(custom_input) custom_input_eneonly = np.copy(custom_input) custom_input_resonly[0][1:] = 0.0 custom_input_indonly[0][0] = 0.0 custom_input_indonly[0][2:] = 0.0 custom_input_traonly[0][:2] = 0.0 custom_input_traonly[0][3:] = 0.0 custom_input_agronly[0][:3] = 0.0 custom_input_agronly[0][4:] = 0.0 custom_input_eneonly[0][:4] = 0.0 custom_inputs.append(custom_input_resonly) custom_inputs.append(custom_input_indonly) custom_inputs.append(custom_input_traonly) custom_inputs.append(custom_input_agronly) custom_inputs.append(custom_input_eneonly) emission_configs_20percentintervals = [] for custom_input in custom_inputs: emission_config = f'RES{custom_input[0][0]:0.3f}_IND{custom_input[0][1]:0.3f}_TRA{custom_input[0][2]:0.3f}_AGR{custom_input[0][3]:0.3f}_ENE{custom_input[0][4]:0.3f}' emission_configs_20percentintervals.append(emission_config) emission_configs_20percentintervals = list( set(emission_configs_20percentintervals)) custom_inputs = [] for emission_config in emission_configs_20percentintervals: custom_input = np.array([ float(num) for num in re.findall(r'\d.\d+', emission_config) ]).reshape(1, -1) custom_inputs.append(custom_input) if top_down_2020_baseline: emission_config_2020_baseline = np.array( [0.604, 0.399, 0.659, 0.613, 0.724]) # matching to PM2.5 only, top 1,000 emission_configs = np.array( np.meshgrid( np.linspace( emission_config_2020_baseline[0] * 0.50, emission_config_2020_baseline[0], 6 ), # 10% reduction increments from 2020 baseline up to 50% np.linspace(emission_config_2020_baseline[1] * 0.50, emission_config_2020_baseline[1], 6), np.linspace(emission_config_2020_baseline[2] * 0.50, emission_config_2020_baseline[2], 6), np.linspace(emission_config_2020_baseline[3] * 0.50, emission_config_2020_baseline[3], 6), np.linspace(emission_config_2020_baseline[4] * 0.50, emission_config_2020_baseline[4], 6), )).T.reshape(-1, 5) custom_inputs = [ np.array(item).reshape(1, -1) for item in emission_configs ] # add a couple more for larger reductions in RES and IND to reach WHO-IT2 custom_inputs.append(np.array([[0.242, 0.160, 0.659, 0.613, 0.724]])) custom_inputs.append(np.array([[0.181, 0.120, 0.659, 0.613, 0.724]])) custom_inputs.append(np.array([[0.121, 0.080, 0.659, 0.613, 0.724]])) custom_inputs.append(np.array([[0.060, 0.040, 0.659, 0.613, 0.724]])) # just for emulator_predictions.py as this is required in order to adjust for double emissions custom_inputs_temp = custom_inputs.copy() for custom_input in custom_inputs_temp: custom_input_resonly = np.copy(custom_input) custom_input_indonly = np.copy(custom_input) custom_input_traonly = np.copy(custom_input) custom_input_agronly = np.copy(custom_input) custom_input_eneonly = np.copy(custom_input) custom_input_resonly[0][1:] = 0.0 custom_input_indonly[0][0] = 0.0 custom_input_indonly[0][2:] = 0.0 custom_input_traonly[0][:2] = 0.0 custom_input_traonly[0][3:] = 0.0 custom_input_agronly[0][:3] = 0.0 custom_input_agronly[0][4:] = 0.0 custom_input_eneonly[0][:4] = 0.0 custom_inputs.append(custom_input_resonly) custom_inputs.append(custom_input_indonly) custom_inputs.append(custom_input_traonly) custom_inputs.append(custom_input_agronly) custom_inputs.append(custom_input_eneonly) emission_configs_20percentintervals = [] for custom_input in custom_inputs: emission_config = f'RES{custom_input[0][0]:0.3f}_IND{custom_input[0][1]:0.3f}_TRA{custom_input[0][2]:0.3f}_AGR{custom_input[0][3]:0.3f}_ENE{custom_input[0][4]:0.3f}' emission_configs_20percentintervals.append(emission_config) emission_configs_20percentintervals = set( emission_configs_20percentintervals) custom_inputs_completed_filenames = glob.glob( f"/nobackup/earlacoa/machinelearning/data_annual/predictions/{output}/ds*{output}.nc" ) custom_inputs_completed_list = [] for custom_inputs_completed_filename in custom_inputs_completed_filenames: emission_config = re.findall( r"RES\d+\.\d+_IND\d+\.\d+_TRA\d+\.\d+_AGR\d+\.\d+_ENE\d+\.\d+", custom_inputs_completed_filename) if len(emission_config) > 0: custom_inputs_completed_list.append(emission_config) custom_inputs_completed_set = set( item[0] for item in custom_inputs_completed_list) custom_inputs_remaining_set = emission_configs_20percentintervals - custom_inputs_completed_set custom_inputs = [ np.array([float(n) for n in re.findall(r'\d+.\d+', item)]).reshape(1, -1) for item in custom_inputs_remaining_set ] # dask bag and process custom_inputs = custom_inputs[:5000] #custom_inputs = custom_inputs[5000:] print(f"predicting for {len(custom_inputs)} custom inputs ...") bag_custom_inputs = db.from_sequence(custom_inputs, npartitions=n_workers) bag_custom_inputs.map(custom_predicts).compute() time_end = time.time() - time_start print( f"completed in {time_end:0.2f} seconds, or {time_end / 60:0.2f} minutes, or {time_end / 3600:0.2f} hours" ) print( f"average time per custom input is {time_end / len(custom_inputs):0.2f} seconds" ) client.close() cluster.close()
def main(): # dask cluster and client if output == 'PM2_5_DRY': n_jobs = 20 n_outputs = 1000 elif output == 'o3_6mDM8h': n_jobs = 20 n_outputs = 2000 n_processes = 1 n_workers = n_processes * n_jobs cluster = SGECluster( interface="ib0", walltime="02:00:00", memory=f"48 G", resource_spec=f"h_vmem=48G", scheduler_options={ "dashboard_address": ":7777", }, job_extra=[ "-cwd", "-V", f"-pe smp {n_processes}", f"-l disk=48G", ], local_directory=os.sep.join( [os.environ.get("PWD"), "dask-hia-ozone-space"]), ) client = Client(cluster) cluster.scale(jobs=n_jobs) time_start = time.time() # find remaining inputs if normal: custom_outputs = glob.glob( f"/nobackup/earlacoa/machinelearning/data_annual/predictions/{output}_adjusted_scaled/ds*{output}_popgrid_0.25deg_adjusted_scaled.nc" ) custom_outputs_completed = glob.glob( f"/nobackup/earlacoa/machinelearning/data_annual/health_impact_assessments/{output}_adjusted_scaled/df_country_hia_*.csv" ) custom_outputs_remaining_set = set([ item.split("/")[-1][3:-1 - len(output) - 19 - 7] for item in custom_outputs ]) - set([ item.split("/")[-1][15 + len(output) + 1:-4 - 7] for item in custom_outputs_completed ]) custom_outputs_remaining = [ item for item in custom_outputs_remaining_set ] print( f"custom outputs remaining for {output}: {len(custom_outputs_remaining)} - 10% intervals with {int(100 * len(custom_outputs_remaining_set) / 16**5)}% remaining" ) reduce_to_20percent_intervals = True if reduce_to_20percent_intervals: emission_configs = np.array( np.meshgrid( np.linspace(0.0, 1.4, 8), np.linspace(0.0, 1.4, 8), np.linspace(0.0, 1.4, 8), np.linspace(0.0, 1.4, 8), np.linspace(0.0, 1.4, 8), )).T.reshape(-1, 5) emission_configs_20percentintervals = [] for emission_config in emission_configs: emission_configs_20percentintervals.append( f'RES{round(emission_config[0], 1)}_IND{round(emission_config[1], 1)}_TRA{round(emission_config[2], 1)}_AGR{round(emission_config[3], 1)}_ENE{round(emission_config[4], 1)}' ) emission_configs_completed = [] for custom_output_completed in custom_outputs_completed: emission_configs_completed.append( re.findall( r'RES\d+.\d+_IND\d+.\d+_TRA\d+.\d+_AGR\d+.\d+_ENE\d+.\d+', custom_output_completed)[0]) emission_configs_20percentintervals_remaining_set = set( emission_configs_20percentintervals) - set( emission_configs_completed) custom_outputs_remaining = [ item for item in emission_configs_20percentintervals_remaining_set ] print( f"custom outputs remaining for {output}: {len(custom_outputs_remaining)} - 20% intervals with {int(100 * len(emission_configs_20percentintervals_remaining_set) / len(emission_configs_20percentintervals))}% remaining" ) if extra: if year == '2010': custom_inputs_main = [ np.array([[1.15, 1.27, 0.98, 0.98, 1.36]]), # bottom-up 2010 ] elif year == '2011': custom_inputs_main = [ np.array([[1.19, 1.30, 1.01, 1.01, 1.46]]), # bottom-up 2011 ] elif year == '2012': custom_inputs_main = [ np.array([[1.20, 1.30, 1.01, 1.02, 1.39]]), # bottom-up 2012 ] elif year == '2013': custom_inputs_main = [ np.array([[1.13, 1.29, 1.02, 1.01, 1.29]]), # bottom-up 2013 ] elif year == '2014': custom_inputs_main = [ np.array([[1.06, 1.12, 0.99, 1.01, 1.12]]), # bottom-up 2014 ] elif year == '2015': custom_inputs_main = [ np.array([[1.0, 1.0, 1.0, 1.0, 1.0]]), # control ] elif year == '2016': custom_inputs_main = [ np.array([[0.92, 0.84, 0.97, 0.99, 0.94]]), # bottom-up 2016 np.array([[0.76, 0.934, 0.735, 0.683, 0.708]]), # top-down 2016 - both np.array([[0.744, 0.904, 0.778, 0.678, 0.716]]), # top-down 2016 - either np.array([[0.803, 0.835, 0.742, 0.71, 0.717]]), # top-down 2016 - pm25 only np.array([[0.769, 1.009, 0.697, 0.69, 0.72]]), # top-down 2016 - o3 only ] elif year == '2017': custom_inputs_main = [ np.array([[0.84, 0.81, 0.99, 0.99, 0.89]]), # bottom-up 2017 np.array([[0.704, 0.786, 0.73, 0.659, 0.6]]), # top-down 2017 - both np.array([[0.771, 0.835, 0.711, 0.685, 0.544]]), # top-down 2017 - either np.array([[0.721, 0.863, 0.712, 0.74, 0.709]]), # top-down 2017 - pm25 only np.array([[0.824, 0.759, 0.767, 0.641, 0.429]]), # top-down 2017 - o3 only ] elif year == '2018': custom_inputs_main = [ np.array([[0.712, 0.703, 0.725, 0.676, 0.649]]), # top-down 2018 - both np.array([[0.647, 0.945, 0.746, 0.588, 0.473]]), # top-down 2018 - either np.array([[0.661, 0.674, 0.694, 0.742, 0.715]]), # top-down 2018 - pm25 only np.array([[0.858, 1.092, 0.794, 0.604, 0.475]]), # top-down 2018 - o3 only ] elif year == '2019': custom_inputs_main = [ np.array([[0.739, 0.668, 0.701, 0.686, 0.682]]), # top-down 2019 - both np.array([[0.657, 0.745, 0.714, 0.613, 0.591]]), # top-down 2019 - either np.array([[0.701, 0.642, 0.669, 0.681, 0.679]]), # top-down 2019 - pm25 only np.array([[0.8, 0.987, 0.648, 0.57, 0.493]]), # top-down 2019 - o3 only ] elif year == '2020': custom_inputs_main = [ np.array([[0.67, 0.609, 0.709, 0.621, 0.661]]), # top-down 2020 - both np.array([[0.582, 0.7, 0.672, 0.5, 0.492]]), # top-down 2020 - either np.array([[0.604, 0.399, 0.659, 0.613, 0.724]]), # top-down 2020 - pm25 only np.array([[0.867, 0.957, 0.677, 0.558, 0.477]]), # top-down 2020 - o3 only ] custom_inputs = [] for custom_input in custom_inputs_main: custom_input_res = np.copy(custom_input) custom_input_ind = np.copy(custom_input) custom_input_tra = np.copy(custom_input) custom_input_agr = np.copy(custom_input) custom_input_ene = np.copy(custom_input) custom_input_nores = np.copy(custom_input) custom_input_noind = np.copy(custom_input) custom_input_notra = np.copy(custom_input) custom_input_noagr = np.copy(custom_input) custom_input_noene = np.copy(custom_input) custom_input_res[0][1:] = 1.0 custom_input_ind[0][0] = 1.0 custom_input_ind[0][2:] = 1.0 custom_input_tra[0][:2] = 1.0 custom_input_tra[0][3:] = 1.0 custom_input_agr[0][:3] = 1.0 custom_input_agr[0][4:] = 1.0 custom_input_ene[0][:4] = 1.0 custom_input_nores[0][0] = 0.0 custom_input_noind[0][1] = 0.0 custom_input_notra[0][2] = 0.0 custom_input_noagr[0][3] = 0.0 custom_input_noene[0][4] = 0.0 custom_inputs.append(custom_input) custom_inputs.append(custom_input_res) custom_inputs.append(custom_input_ind) custom_inputs.append(custom_input_tra) custom_inputs.append(custom_input_agr) custom_inputs.append(custom_input_ene) custom_inputs.append(custom_input_nores) custom_inputs.append(custom_input_noind) custom_inputs.append(custom_input_notra) custom_inputs.append(custom_input_noagr) custom_inputs.append(custom_input_noene) custom_outputs_remaining = [] for custom_input in custom_inputs: emission_config = f'RES{custom_input[0][0]:0.3f}_IND{custom_input[0][1]:0.3f}_TRA{custom_input[0][2]:0.3f}_AGR{custom_input[0][3]:0.3f}_ENE{custom_input[0][4]:0.3f}' custom_outputs_remaining.append(emission_config) if climate_cobenefits: custom_inputs_main = [ np.array([[0.91, 0.95, 0.85, 1.05, 0.96]]), # Base_CLE_2020 np.array([[0.91, 0.95, 0.85, 1.05, 0.96]]), # Base_MFR_2020 np.array([[0.91, 0.95, 0.85, 1.05, 0.96]]), # SDS_MFR_2020 np.array([[0.68, 0.84, 0.71, 1.16, 0.93]]), # Base_CLE_2030 np.array([[0.33, 0.47, 0.48, 0.81, 0.69]]), # Base_MFR_2030 np.array([[0.27, 0.45, 0.41, 0.81, 0.55]]), # SDS_MFR_2030 np.array([[0.57, 0.75, 0.69, 1.2, 0.94]]), # Base_CLE_2040 np.array([[0.24, 0.41, 0.31, 0.83, 0.73]]), # Base_MFR_2040 np.array([[0.19, 0.38, 0.22, 0.83, 0.5]]), # SDS_MFR_2040 np.array([[0.52, 0.72, 0.65, 1.24, 0.91]]), # Base_CLE_2050 np.array([[0.2, 0.38, 0.29, 0.86, 0.72]]), # Base_MFR_2050 np.array([[0.18, 0.35, 0.2, 0.86, 0.46]]), # SDS_MFR_2050 ] custom_inputs = [] for custom_input in custom_inputs_main: custom_input_res = np.copy(custom_input) custom_input_ind = np.copy(custom_input) custom_input_tra = np.copy(custom_input) custom_input_agr = np.copy(custom_input) custom_input_ene = np.copy(custom_input) custom_input_nores = np.copy(custom_input) custom_input_noind = np.copy(custom_input) custom_input_notra = np.copy(custom_input) custom_input_noagr = np.copy(custom_input) custom_input_noene = np.copy(custom_input) custom_input_resonly = np.copy(custom_input) custom_input_indonly = np.copy(custom_input) custom_input_traonly = np.copy(custom_input) custom_input_agronly = np.copy(custom_input) custom_input_eneonly = np.copy(custom_input) custom_input_res[0][1:] = 1.0 custom_input_ind[0][0] = 1.0 custom_input_ind[0][2:] = 1.0 custom_input_tra[0][:2] = 1.0 custom_input_tra[0][3:] = 1.0 custom_input_agr[0][:3] = 1.0 custom_input_agr[0][4:] = 1.0 custom_input_ene[0][:4] = 1.0 custom_input_nores[0][0] = 0.0 custom_input_noind[0][1] = 0.0 custom_input_notra[0][2] = 0.0 custom_input_noagr[0][3] = 0.0 custom_input_noene[0][4] = 0.0 custom_input_resonly[0][1:] = 0.0 custom_input_indonly[0][0] = 0.0 custom_input_indonly[0][2:] = 0.0 custom_input_traonly[0][:2] = 0.0 custom_input_traonly[0][3:] = 0.0 custom_input_agronly[0][:3] = 0.0 custom_input_agronly[0][4:] = 0.0 custom_input_eneonly[0][:4] = 0.0 custom_inputs.append(custom_input) custom_inputs.append(custom_input_res) custom_inputs.append(custom_input_ind) custom_inputs.append(custom_input_tra) custom_inputs.append(custom_input_agr) custom_inputs.append(custom_input_ene) custom_inputs.append(custom_input_nores) custom_inputs.append(custom_input_noind) custom_inputs.append(custom_input_notra) custom_inputs.append(custom_input_noagr) custom_inputs.append(custom_input_noene) custom_inputs.append(custom_input_resonly) custom_inputs.append(custom_input_indonly) custom_inputs.append(custom_input_traonly) custom_inputs.append(custom_input_agronly) custom_inputs.append(custom_input_eneonly) custom_outputs_remaining = [] for custom_input in custom_inputs: emission_config = f'RES{custom_input[0][0]:0.3f}_IND{custom_input[0][1]:0.3f}_TRA{custom_input[0][2]:0.3f}_AGR{custom_input[0][3]:0.3f}_ENE{custom_input[0][4]:0.3f}' custom_outputs_remaining.append(emission_config) if top_down_2020_baseline: emission_config_2020_baseline = np.array( [0.604, 0.399, 0.659, 0.613, 0.724]) # matching to PM2.5 only, top 1,000 emission_configs = np.array( np.meshgrid( np.linspace( emission_config_2020_baseline[0] * 0.50, emission_config_2020_baseline[0], 6 ), # 10% reduction increments from 2020 baseline up to 50% np.linspace(emission_config_2020_baseline[1] * 0.50, emission_config_2020_baseline[1], 6), np.linspace(emission_config_2020_baseline[2] * 0.50, emission_config_2020_baseline[2], 6), np.linspace(emission_config_2020_baseline[3] * 0.50, emission_config_2020_baseline[3], 6), np.linspace(emission_config_2020_baseline[4] * 0.50, emission_config_2020_baseline[4], 6), )).T.reshape(-1, 5) # add a couple more for larger reductions in RES and IND to reach WHO-IT2 emission_configs = list(emission_configs) emission_configs.append(np.array([[0.242, 0.160, 0.659, 0.613, 0.724]])) emission_configs.append(np.array([[0.181, 0.120, 0.659, 0.613, 0.724]])) emission_configs.append(np.array([[0.121, 0.080, 0.659, 0.613, 0.724]])) emission_configs.append(np.array([[0.060, 0.040, 0.659, 0.613, 0.724]])) emission_configs_total = [] for emission_config in emission_configs: emission_configs_total.append( f'RES{round(emission_config[0], 3):.3f}_IND{round(emission_config[1], 3):.3f}_TRA{round(emission_config[2], 3):.3f}_AGR{round(emission_config[3], 3):.3f}_ENE{round(emission_config[4], 3):.3f}' ) custom_outputs_completed = glob.glob( f"/nobackup/earlacoa/machinelearning/data_annual/health_impact_assessments/{output}_adjusted_scaled/df_country_hia_*.csv" ) emission_configs_completed = [] for custom_output_completed in custom_outputs_completed: emission_configs_completed.append( re.findall( r'RES\d+.\d+_IND\d+.\d+_TRA\d+.\d+_AGR\d+.\d+_ENE\d+.\d+', custom_output_completed)[0]) emission_configs_remaining_set = set(emission_configs_total) - set( emission_configs_completed) custom_outputs_remaining = [ item for item in emission_configs_remaining_set ] print( f"custom outputs remaining: {len(custom_outputs_remaining)}, {int(100 * len(emission_configs_remaining_set) / len(emission_configs_total))}%" ) # -------------------------------------------------- # dask bag and process # run in 10 chunks over 10 cores, each chunk taking 2 minutes custom_outputs_remaining = custom_outputs_remaining[0:n_outputs] print(f"predicting for {len(custom_outputs_remaining)} custom outputs ...") bag_custom_outputs = db.from_sequence(custom_outputs_remaining, npartitions=n_workers) if output == "PM2_5_DRY": bag_custom_outputs.map(health_impact_assessment_pm25).compute() elif output == "o3_6mDM8h": bag_custom_outputs.map(health_impact_assessment_o3).compute() time_end = time.time() - time_start print( f"completed in {time_end:0.2f} seconds, or {time_end / 60:0.2f} minutes, or {time_end / 3600:0.2f} hours" ) client.close() cluster.close()
def main(): # dask cluster and client n_processes = 1 n_jobs = 35 n_workers = n_processes * n_jobs cluster = SGECluster( interface="ib0", walltime="01:00:00", memory=f"64 G", resource_spec=f"h_vmem=64G", scheduler_options={ "dashboard_address": ":5757", }, job_extra=[ "-cwd", "-V", f"-pe smp {n_processes}", f"-l disk=32G", ], local_directory=os.sep.join([os.environ.get("PWD"), "dask-worker-space"]), ) client = Client(cluster) cluster.scale(jobs=n_jobs) time_start = time.time() # regrid custom outputs to pop grid custom_outputs = glob.glob( f"/nobackup/earlacoa/machinelearning/data_annual/predictions/{output}/ds*{output}.nc" ) custom_outputs_completed = glob.glob( f"/nobackup/earlacoa/machinelearning/data_annual/predictions/{output}/ds*{output}_popgrid_0.25deg.nc" ) custom_outputs_completed = [ f"{item[0:-19]}.nc" for item in custom_outputs_completed ] custom_outputs_remaining_set = set(custom_outputs) - set(custom_outputs_completed) custom_outputs_remaining = [item for item in custom_outputs_remaining_set] print(f"custom outputs remaining for {output}: {len(custom_outputs_remaining)}") # dask bag and process custom_outputs_remaining = custom_outputs_remaining[ 0:5000 ] # run in 5,000 chunks over 30 cores, each chunk taking 2 minutes print(f"predicting for {len(custom_outputs_remaining)} custom outputs ...") bag_custom_outputs = db.from_sequence( custom_outputs_remaining, npartitions=n_workers ) bag_custom_outputs.map(regrid_to_pop).compute() time_end = time.time() - time_start print( f"completed in {time_end:0.2f} seconds, or {time_end / 60:0.2f} minutes, or {time_end / 3600:0.2f} hours" ) print( f"average time per custom output is {time_end / len(custom_outputs_remaining):0.2f} seconds" ) client.close() cluster.close()
def _init_dask(self): """ Starts a dask cluster, according to the cluster type specified in the constructor. Sets self.client. Also writes useful URLs to graph-links.txt. If the 'cluster-type' is 'synchronous', then the cluster will be a special stub class (DebugCluster), which provides dummy implementations of a few functions from the DistributedCluster API. (Mostly just for convenient unit testing.) """ # Consider using client.register_worker_callbacks() to configure # - faulthandler (later) # - excepthook? # - (okay, maybe it's just best to put that stuff in __init__.py, like in DSS) load_and_overwrite_dask_config(self.cluster_type, 'dask-config.yaml', True) self._write_driver_graph_urls() if self.cluster_type in JOBQUEUE_CLUSTERS: update_jobqueue_config_with_defaults(self.cluster_type) if self.cluster_type == "lsf": from dask_jobqueue import LSFCluster cluster = LSFCluster() #ip='0.0.0.0') elif self.cluster_type == "sge": from dask_jobqueue import SGECluster cluster = SGECluster(ip='0.0.0.0') elif self.cluster_type == "slurm": from dask_jobqueue import SLURMCluster cluster = SLURMCluster(ip='0.0.0.0') else: raise AssertionError("Unimplemented jobqueue cluster") cluster.scale(self.num_workers) elif self.cluster_type == "local-cluster": cluster = LocalCluster(self.num_workers, threads_per_worker=1, processes=True, ip='0.0.0.0') elif self.cluster_type in ("synchronous", "processes"): cluster = None # synchronous/processes mode is for testing and debugging only assert dask.config.get('scheduler', self.cluster_type) == self.cluster_type, \ "Inconsistency between the dask-config and the scheduler you chose." dask.config.set(scheduler=self.cluster_type) self.client = DebugClient(self.cluster_type) else: raise AssertionError("Unknown cluster type") dump_dask_config('full-dask-config.yaml') if cluster: dashboard = cluster.dashboard_link logger.info(f"Dashboard running on {dashboard}") dashboard_ip = extract_ip_from_link(dashboard) dashboard = dashboard.replace(dashboard_ip, socket.gethostname()) logger.info(f" a.k.a. {dashboard}") # Note: Overrides config value: distributed.comm.timeouts.connect self.client = Client(cluster, timeout='60s') # Wait for the workers to spin up. with Timer(f"Waiting for {self.num_workers} workers to launch", logger) as wait_timer: while (self.wait_for_workers and self.client.status == "running" and len(self.client.cluster.scheduler.workers) < self.num_workers): if wait_timer.seconds > (60 * self.cluster_max_wait): msg = ( f"Not all cluster workers could be launched within the " "allotted time ({self.cluster_max_wait} minutes).\n" "Try again or adjust the 'cluster-max-wait' setting.\n" ) raise RuntimeError(msg) time.sleep(0.1) if self.wait_for_workers and self.cluster_type == "lsf": self._write_worker_graph_urls('graph-links.txt')
def main(): # dask cluster and client n_processes = 1 n_workers = n_processes * n_jobs cluster = SGECluster( interface="ib0", walltime=walltime, memory=f"32 G", resource_spec=f"h_vmem=32G", scheduler_options={ "dashboard_address": ":5761", }, job_extra=[ "-cwd", "-V", f"-pe smp {n_processes}", f"-l disk=32G", ], local_directory=os.sep.join( [os.environ.get("PWD"), "dask-find-emis-pm-space"]), ) client = Client(cluster) cluster.scale(jobs=n_jobs) time_start = time.time() # dask bag over emission_configs print( f"predicting over {len(emission_configs)} emission configs for {station_id} ..." ) bag_emission_configs = db.from_sequence(emission_configs, npartitions=n_workers) results = bag_emission_configs.map(filter_emission_configs).compute() station_diffs_abs = [result[0] for result in results] station_diffs_per = [result[1] for result in results] key = [key for key in baselines.keys()][0] station_diffs_abs = [ station_diff_abs for station_diff_abs in station_diffs_abs if len(station_diff_abs[key]) > 0 ] station_diffs_per = [ station_diff_per for station_diff_per in station_diffs_per if len(station_diff_per[key]) > 0 ] merged_per = {} for station_diff_per in station_diffs_per: merged_per = {**merged_per, **station_diff_per[key]} merged_abs = {} for station_diff_abs in station_diffs_abs: merged_abs = {**merged_abs, **station_diff_abs[key]} station_diffs_per = {key: merged_per} station_diffs_abs = {key: merged_abs} joblib.dump( obs_change_abs, f"/nobackup/earlacoa/machinelearning/data_annual/find_emissions_that_match_change_air_quality/{sub_folder}_adjusted_scaled/obs_change_abs_{output}_{station_id}.joblib" ) joblib.dump( obs_change_per, f"/nobackup/earlacoa/machinelearning/data_annual/find_emissions_that_match_change_air_quality/{sub_folder}_adjusted_scaled/obs_change_per_{output}_{station_id}.joblib" ) joblib.dump( baselines, f"/nobackup/earlacoa/machinelearning/data_annual/find_emissions_that_match_change_air_quality/{sub_folder}_adjusted_scaled/baselines_{output}_{station_id}.joblib" ) joblib.dump( targets, f"/nobackup/earlacoa/machinelearning/data_annual/find_emissions_that_match_change_air_quality/{sub_folder}_adjusted_scaled/targets_{output}_{station_id}.joblib" ) joblib.dump( target_diffs, f"/nobackup/earlacoa/machinelearning/data_annual/find_emissions_that_match_change_air_quality/{sub_folder}_adjusted_scaled/target_diffs_{output}_{station_id}.joblib" ) joblib.dump( station_diffs_abs, f"/nobackup/earlacoa/machinelearning/data_annual/find_emissions_that_match_change_air_quality/{sub_folder}_adjusted_scaled/station_diffs_abs_{output}_{station_id}.joblib" ) joblib.dump( station_diffs_per, f"/nobackup/earlacoa/machinelearning/data_annual/find_emissions_that_match_change_air_quality/{sub_folder}_adjusted_scaled/station_diffs_per_{output}_{station_id}.joblib" ) time_end = time.time() - time_start print( f"completed in {time_end:0.2f} seconds, or {time_end / 60:0.2f} minutes, or {time_end / 3600:0.2f} hours" ) client.close() cluster.close()
def main(): # dask cluster and client n_processes = 1 n_jobs = 35 n_workers = n_processes * n_jobs cluster = SGECluster(interface='ib0', walltime='01:00:00', memory=f'2 G', resource_spec=f'h_vmem=2G', scheduler_options={ 'dashboard_address': ':5757', }, project='admiralty', job_extra=[ '-cwd', '-V', f'-pe smp {n_processes}', f'-l disk=1G', ], local_directory=os.sep.join( [os.environ.get('PWD'), 'dask-worker-space'])) client = Client(cluster) cluster.scale(jobs=n_jobs) time_start = time.time() # custom inputs matrix_stacked = np.array( np.meshgrid( np.linspace( 0, 1.5, 16 ), # 1.5 and 16 for 0.1, 1.5 and 6 for 0.3, 1.4 and 8 for 0.2 np.linspace(0, 1.5, 16), np.linspace(0, 1.5, 16), np.linspace(0, 1.5, 16), np.linspace(0, 1.5, 16))).T.reshape(-1, 5) custom_inputs_set = set( tuple(map(float, map("{:.1f}".format, item))) for item in matrix_stacked) custom_inputs_completed_filenames = glob.glob( '/nobackup/earlacoa/machinelearning/data/summary/ds*' + output + '*') custom_inputs_completed_list = [] for custom_inputs_completed_filename in custom_inputs_completed_filenames: custom_inputs_completed_list.append([ float(item) for item in re.findall( r'\d+\.\d+', custom_inputs_completed_filename) ]) custom_inputs_completed_set = set( tuple(item) for item in custom_inputs_completed_list) custom_inputs_remaining_set = custom_inputs_set - custom_inputs_completed_set custom_inputs = [ np.array(item).reshape(1, -1) for item in custom_inputs_remaining_set ] print(f'custom inputs remaining for {output}: {len(custom_inputs)}') # dask bag and process custom_inputs = custom_inputs[ 0:5000] # run in 1,000 chunks over 30 cores, each chunk taking 1 hour print(f'predicting for {len(custom_inputs)} custom inputs ...') bag_custom_inputs = db.from_sequence(custom_inputs, npartitions=n_workers) bag_custom_inputs.map(custom_predicts).compute() time_end = time.time() - time_start print( f'completed in {time_end:0.2f} seconds, or {time_end / 60:0.2f} minutes, or {time_end / 3600:0.2f} hours' ) print( f'average time per custom input is {time_end / len(custom_inputs):0.2f} seconds' ) client.close() cluster.close()
def main(): start_logging() #------------------------------------------ # Create input and output dataset #------------------------------------------ IO = IOClass() DATA = IO.create_data_file() # Create global result and restart datasets RESULT = IO.create_result_file() RESTART = IO.create_restart_file() #---------------------------------------------- # Calculation - Multithreading using all cores #---------------------------------------------- # Auxiliary variables for futures futures = [] # Measure time start_time = datetime.now() #----------------------------------------------- # Create a client for distributed calculations #----------------------------------------------- if (slurm_use): with SLURMCluster(scheduler_port=port, cores=cores, processes=processes, memory=memory, shebang=shebang, name=name, job_extra=slurm_parameters, local_directory='logs/dask-worker-space') as cluster: cluster.scale(processes * nodes) print(cluster.job_script()) print("You are using SLURM!\n") print(cluster) run_cosipy(cluster, IO, DATA, RESULT, RESTART, futures) elif sge_use: with SGECluster(interface=interface, cores=cores, processes=processes, memory=memory, shebang=shebang, walltime=walltime, resource_spec=resource_spec, job_extra=sge_parameters, local_directory='logs/dask-worker-space') as cluster: cluster.scale(cores=total_cores) print(cluster.job_script()) print("You are using SGE!\n") print(cluster) run_cosipy(cluster, IO, DATA, RESULT, RESTART, futures) else: with LocalCluster(scheduler_port=local_port, n_workers=workers, threads_per_worker=1, silence_logs=True) as cluster: print(cluster) run_cosipy(cluster, IO, DATA, RESULT, RESTART, futures) print('\n') print('--------------------------------------------------------------') print('Write results ...') print('-------------------------------------------------------------- \n') start_writing = datetime.now() #----------------------------------------------- # Write results and restart files #----------------------------------------------- timestamp = pd.to_datetime(str( IO.get_restart().time.values)).strftime('%Y-%m-%dT%H-%M') encoding = dict() for var in IO.get_result().data_vars: dataMin = IO.get_result()[var].min(skipna=True).values dataMax = IO.get_result()[var].max(skipna=True).values dtype = 'int16' FillValue = -9999 scale_factor, add_offset = compute_scale_and_offset( dataMin, dataMax, 16) #encoding[var] = dict(zlib=True, complevel=compression_level, dtype=dtype, scale_factor=scale_factor, add_offset=add_offset, _FillValue=FillValue) encoding[var] = dict(zlib=True, complevel=compression_level) IO.get_result().to_netcdf(os.path.join(data_path, 'output', output_netcdf), encoding=encoding, mode='w') encoding = dict() for var in IO.get_restart().data_vars: dataMin = IO.get_restart()[var].min(skipna=True).values dataMax = IO.get_restart()[var].max(skipna=True).values dtype = 'int16' FillValue = -9999 scale_factor, add_offset = compute_scale_and_offset( dataMin, dataMax, 16) #encoding[var] = dict(zlib=True, complevel=compression_level, dtype=dtype, scale_factor=scale_factor, add_offset=add_offset, _FillValue=FillValue) encoding[var] = dict(zlib=True, complevel=compression_level) IO.get_restart().to_netcdf(os.path.join(data_path, 'restart', 'restart_' + timestamp + '.nc'), encoding=encoding) #----------------------------------------------- # Stop time measurement #----------------------------------------------- duration_run = datetime.now() - start_time duration_run_writing = datetime.now() - start_writing #----------------------------------------------- # Print out some information #----------------------------------------------- print( "\t Time required tor write restart and output files: %4g minutes %2g seconds \n" % (duration_run_writing.total_seconds() // 60.0, duration_run_writing.total_seconds() % 60.0)) print("\t Total run duration: %4g minutes %2g seconds \n" % (duration_run.total_seconds() // 60.0, duration_run.total_seconds() % 60.0)) print('--------------------------------------------------------------') print('\t SIMULATION WAS SUCCESSFUL') print('--------------------------------------------------------------')