def cluster(self, num_nodes): from distributed import Client from dask_jobqueue import PBSCluster cluster_ = PBSCluster(walltime='00:15:00', cores=36, memory='60GB', processes=1) self.client = Client(cluster_) cluster_.scale(num_nodes)
from dask_jobqueue import PBSCluster from dask.distributed import Client import dask.dataframe as dd import dask cluster = PBSCluster(cores=1, memory="120GB", project='pangeo', processes=1, walltime='04:00:00', local_directory='$TMPDIR') cluster cluster.scale(40) client = Client(cluster) # scheduler_file="/home/ad/briolf/scheduler.json") client print(len(client.scheduler_info()["workers"])) import zarr import netCDF4 import pytide import xarray as xr import os import glob import numpy as np root = "/work/ALT/odatis/eNATL60/BLBT02/gridT-2D/" files = sorted(glob.glob(root + '/*.nc'))
i"""A minimal example of connecting to a PBS cluster and submitting a job.""" from dask_jobqueue import PBSCluster from dask.distributed import Client import dask.array as da cluster = PBSCluster() # defaults to values set in ~/.config/dask/jobqueue.yml cluster.scale(10) # ask for ten workers client = Client(cluster) # connect this local process to remote workers cluster.scale(1) # ask for one more workers # run a simple computation - TODO replace this with hyperparameter search x = da.random.random((10000, 10000), chunks=(1000, 1000)) # create data x.persist() # keep in memory z = (x - x.mean(axis=0)) / x.std(axis=0) z.persist()
# i=i+1 # #检查结果不是all true?multi_copy_gene_Vector中是否有异常值,blast结果中检查只用一个hsp行不行 # # best_blast_value_list=list(R_parse_blast_result.parse_blast_result( # # single_copy_gene_list, # # multi_copy_gene_list, # # R_blast_vlaue_df, # # str(blast_general/(head_id+"_single_err.txt")) # # )) # # extract_gene(best_blast_value_list,all_row_gene_fasta_file,blast_general/(head_id+"_single_copy.fasta")) # R_blast_vlaue_df=base.do_call("rbind",R_blast_vlaue_list) # utils.write_table(R_blast_vlaue_df,**{'file': str(general_out_path/"all.txt")},**{'append': False},**{'quote': False},**{'sep': "\t"},**{'row.names': False},**{'col.names': True}) if __name__ == '__main__': # client = Client(n_workers=12, threads_per_worker=1) cluster = PBSCluster( job_extra=["-l nodes=1:ppn=24","-l mem=5000MB"], header_skip=["select"], processes=24, walltime='25:00:00' ) cluster.scale(jobs=10) from dask.distributed import Client client = Client(cluster) print(cluster.job_script()) extract_ortholog_gene( Path("../Pan_genome_data_2/contig_gff3_gffutils_db"), Path("../Pan_genome_data_2/156_contig"), Path("../Pan_genome_data_2/ortho/joined_df.tsv"), directory_creater("/gpfshome/home/Baojd/wangzhe/ortholog_blast_mgg_key"), )
parallel=False) pysp2.io.write_dat(my_binary, out_path + base + '.particle.dat') # Get all of the unique dates all_sp2_files = glob(sp2b_path + '*.sp2b') sp2_date_list = [x.split(".")[3] for x in all_sp2_files] sp2_date_list = sorted(list(set(sp2_date_list))) #sp2_date_list = ['20200218'] #process_day('20200218') print(sp2_date_list) cluster = PBSCluster(processes=6, cores=36, walltime='5:00:00', memory='270GB', name='dask-worker', queue='arm_high_mem', project='arm', job_extra=['-W group_list=cades-arm'], interface='ib0', extra=['--no-dashboard']) cluster.scale(36 * 6) client = Client(cluster) print("Waiting for workers before starting processing...") client.wait_for_workers(9) print(client) results = client.map(process_day, sp2_date_list) wait(results) #del client
from dask_jobqueue import PBSCluster cluster = PBSCluster() cluster.scale(10) from dask.distributed import Client client = Client(clsuter) # Do compute # This is the simplest setup # ----------------------------------------------------------------------------------------------------------------------- # A bit more pcluster = PBSCluster(cores=36, memory='100GB', project='SOMETHING', queue='smp', interface='ib0', walltime='03:00:00') pcluster.scale(100) client = Client(pcluster) # ----------------------------------------------------------------------------------------------------------------------- #
import zarr import xarray as xr import time import dask import glob ask_workers=20 memory='180GB' from dask_jobqueue import PBSCluster from dask.distributed import Client import dask.dataframe as dd cluster = PBSCluster(cores=1, memory=memory, project='PerfTestPangeo', walltime='06:00:00') cluster.scale(ask_workers) c = Client(cluster) c from dask.utils import ensure_dict, format_bytes wk = c.scheduler_info()["workers"] text="Workers= " + str(len(wk)) memory = [w["memory_limit"] for w in wk.values()] if all(memory): text += ", Memory=" + format_bytes(sum(memory)) print(text) files=sorted(glob.glob('/work/ALT/odatis/eNATL60/BLBT02/gridT-2D/eNATL60*gridT-2D*.nc'))
''' Author: your name Date: 2020-10-27 08:56:03 LastEditTime: 2020-10-28 21:01:19 LastEditors: Please set LastEditors Description: In User Settings Edit FilePath: /Pan_genome_github/210_mafft.py ''' from dask_jobqueue import PBSCluster from dask import delayed import dask cluster = PBSCluster(job_extra=["-l nodes=1:ppn=24", "-l mem=5000MB"], header_skip=["select"], processes=24, walltime='25:00:00') cluster.scale(jobs=2) from dask.distributed import Client client = Client(cluster) print(cluster.job_script()) import subprocess from pathlib import Path def mafft_popen(gene_file_path, out_dir_path, out_stderr_dir_path): out_file = out_dir_path / gene_file_path.stem if out_file.exists() is True: return ma_popen = subprocess.Popen( [ "mafft", "--genafpair", "--maxiterate", "16", "--reorder", gene_file_path.resolve()
from dask.distributed import Client import string from random import randint from time import sleep ### LOCAL EXECUTION #client = Client(processes=False) # start local workers as threads ### SGE from dask_jobqueue import PBSCluster cluster = PBSCluster(cores=2, memory="4GB", queue="all.q") cluster.scale(10) client = Client(cluster) # Creating my tasks def get_jobid(): job_id = "" if "JOB_ID" in os.environ: job_id = str(os.environ["HOSTNAME"]) return job_id def taskA (inputs=[]):
args = parser.parse_args() print(args.input_file) print(args.output_file) ########################### Dask Cluster ############################### cluster = None if args.cluster_mode == "cluster": cluster = PBSCluster(processes=1, cores=4, memory="20GB", local_directory='$TMPDIR', project='DaskTest', walltime='01:00:00', interface='ib0') cluster.scale(8) else: print("Local mode") cluster = LocalCluster() cluster.scale(2) client = Client(cluster) print(client) ########################### Read Input File ########################### # Create a Dask DataFrame from input_file # 6 colunms defined : Id t0 x0 v0 tmax and dt df_in = dd.read_csv(args.input_file, sep=" ", names=['Id', 'x0', 'v0', 't0', 'tmax', 'dt'])
f'Unknown fields in pbs_spec: {args.pbs_spec.keys()}') from dask.distributed import Client from dask_jobqueue import PBSCluster USER = os.environ['USER'] cluster = PBSCluster(queue=queue, cores=36, processes=9, memory='100GB', project=project, walltime=walltime, local_directory=f'/glade/scratch/{USER}/dask-tmp') client = Client(cluster) cluster.scale(9 * n_nodes) #--------------------------------------------------------------------------- #-- read the input dataset ds = xr.open_mfdataset(args.file_in, **xr_open_ds) if isel: ds = ds.isel(**isel) if args.variable_list: if args.invert_var_selction: drop_vars = [v for v in ds.variables if v in args.variable_list] else: drop_vars = [ v for v in ds.variables if v not in args.variable_list ] ds = ds.drop(drop_vars)
def start_cluster(nb_workers, walltime, out_dir, timeout=600): """ This function create a dask cluster. Each worker has nb_cpus cpus and only one python process is started on each worker. start_cluster will use OMP_NUM_THREADS environment variable to determine how many threads might be used by a worker when running C/C++ code (default to 1) start_cluster will use CARS_NB_WORKERS_PER_PBS_JOB environment variable to determine how many workers should be started by a single PBS job (default to 1) start_cluster will use CARS_PBS_QUEUE to determine in which queue worker jobs should be posted. :param nb_workers: Number of dask workers :type nb_workers: int :param walltime: Walltime for each dask worker :type walltime: string :param out_dir: Output directory :type out_dir: string :return: Dask cluster and dask client :rtype: (dask_jobqueue.PBSCluster, dask.distributed.Client) tuple """ # Retrieve multi-threading factor for C/C++ code if available omp_num_threads = 1 if os.environ.get('OMP_NUM_THREADS'): omp_num_threads = int(os.environ['OMP_NUM_THREADS']) # Retrieve number of workers per PBS job nb_workers_per_job = 1 if os.environ.get('CARS_NB_WORKERS_PER_PBS_JOB'): nb_workers_per_job = int(os.environ['CARS_NB_WORKERS_PER_PBS_JOB']) # Retrieve PBS queue pbs_queue = None if os.environ.get('CARS_PBS_QUEUE'): pbs_queue = os.environ['CARS_PBS_QUEUE'] # Total number of cpus is multi-threading factor times size of batch # (number of workers per PBS job) nb_cpus = nb_workers_per_job * omp_num_threads # Cluster nodes have 5GB per core memory = nb_cpus * 5000 # Ressource string for PBS resource = "select=1:ncpus={}:mem={}mb".format(nb_cpus, memory) nb_jobs = int(math.ceil(nb_workers / nb_workers_per_job)) logging.info( "Starting Dask PBS cluster with {} workers ({} workers with {} cores each per PSB job)" .format(nb_workers, nb_workers_per_job, omp_num_threads)) logging.info( "Submitting {} PBS jobs with configuration cpu={},mem={},walltime={}". format(nb_jobs, nb_cpus, memory, walltime)) names = [ 'PATH', 'PYTHONPATH', 'CARS_STATIC_CONFIGURATION', 'LD_LIBRARY_PATH', 'OTB_APPLICATION_PATH', 'OTB_GEOID_FILE', 'OMP_NUM_THREADS', 'NUMBA_NUM_THREADS', 'OPJ_NUM_THREADS', 'GDAL_NUM_THREADS', 'OTB_MAX_RAM_HINT', 'VIRTUAL_ENV', 'ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS', 'GDAL_CACHEMAX' ] names = [name for name in names if os.environ.get(name)] envs = ["export {}={}".format(name, os.environ[name]) for name in names] log_directory = os.path.join(os.path.abspath(out_dir), "dask_log") local_directory = '$TMPDIR' cluster = PBSCluster(processes=nb_workers_per_job, cores=nb_workers_per_job, resource_spec=resource, memory="{}MB".format(memory), local_directory=local_directory, project='dask-test', walltime=walltime, interface='ib0', queue=pbs_queue, env_extra=envs, log_directory=log_directory) logging.info("Dask cluster started") cluster.scale(nb_workers) client = Client(cluster, timeout=timeout) logging.info("Dashboard started at {}".format(get_dashboard_link(cluster))) return cluster, client
from bluesky import RunEngine from rapidz import Stream from shed_streaming.translation import FromEventStream, ToEventStream, DBFriendly from databroker import Broker import bluesky.plans as bp from ophyd.sim import hw hw = hw() from dask.distributed import Client from dask_jobqueue import PBSCluster from pprint import pprint cluster = PBSCluster() cluster.scale(10) # Ask for ten workers client = Client(cluster) db = Broker.named('temp') # Now this pipeline runs using HPC resources source = Stream() (FromEventStream('event', 'motor1', upstream=source).scatter().map( op.add, 1).buffer(8).gather().ToEventStream('result').DBFriendly().starsink( db.insert)) RE = RunEngine() RE.subscribe(lambda *x: source.emit(x))
if __name__ == '__main__': # Get all of the unique dates all_sp2_files = glob(sp2b_path + '*' + sys.argv[1] + '*.sp2b') sp2_date_list = [x.split(".")[3] for x in all_sp2_files] sp2_date_list = sorted(list(set(sp2_date_list))) cluster = PBSCluster(processes=9, cores=36, walltime='4:00:00', memory='270GB', name='dask-worker', queue='arm_high_mem', project='arm', job_extra=['-W group_list=cades-arm'], interface='ib0', extra=['--no-dashboard']) cluster.scale(72) client = Client(cluster) print("Waiting for workers before starting processing...") #client.wait_for_workers(9) #for day in sp2_date_list: # print("Processing %s" % day) # process_day(day) #print(sp2_date_list) process_day('20191216') #results = client.map(process_day, sp2_date_list) #wait(results) del client
def silly_function(i): import socket host = socket.gethostname() print(host) time.sleep(10 + i / 10) print(i) return hostname if __name__ == "__main__": oly_cluster = PBSCluster( shebang='#!/bin/tcsh', name='MODIS_dl', cores=8, memory='4GB', processes=8, #doesn't work with scale(jobs) for some reason local_directory='/home/disk/eos9/jkcm/temp', resource_spec='nodes=1:ppn=8', walltime='00:05:00', job_extra=['-m ae', '-M [email protected]'], interface='ib0') oly_cluster.scale(4) print(oly_cluster.job_script()) client = Client(oly_cluster) L = client.map(silly_function, range(20)) client.gather(L) print(L) # g = # print(g)