def cluster(self, num_nodes):
from distributed import Client
from dask_jobqueue import PBSCluster
cluster_ = PBSCluster(walltime='00:15:00', cores=36, memory='60GB', processes=1)
self.client = Client(cluster_)
cluster_.scale(num_nodes)
from dask_jobqueue import PBSCluster
from dask.distributed import Client
import dask.dataframe as dd
import dask
cluster = PBSCluster(cores=1,
memory="120GB",
project='pangeo',
processes=1,
walltime='04:00:00',
local_directory='$TMPDIR')
cluster
cluster.scale(40)
client = Client(cluster) # scheduler_file="/home/ad/briolf/scheduler.json")
client
print(len(client.scheduler_info()["workers"]))
import zarr
import netCDF4
import pytide
import xarray as xr
import os
import glob
import numpy as np
root = "/work/ALT/odatis/eNATL60/BLBT02/gridT-2D/"
files = sorted(glob.glob(root + '/*.nc'))
i"""A minimal example of connecting to a PBS cluster and submitting a job.""" from dask_jobqueue import PBSCluster from dask.distributed import Client import dask.array as da cluster = PBSCluster() # defaults to values set in ~/.config/dask/jobqueue.yml cluster.scale(10) # ask for ten workers client = Client(cluster) # connect this local process to remote workers cluster.scale(1) # ask for one more workers # run a simple computation - TODO replace this with hyperparameter search x = da.random.random((10000, 10000), chunks=(1000, 1000)) # create data x.persist() # keep in memory z = (x - x.mean(axis=0)) / x.std(axis=0) z.persist()
# i=i+1
# #检查结果不是all true？multi_copy_gene_Vector中是否有异常值，blast结果中检查只用一个hsp行不行
# # best_blast_value_list=list(R_parse_blast_result.parse_blast_result(
# # single_copy_gene_list,
# # multi_copy_gene_list,
# # R_blast_vlaue_df,
# # str(blast_general/(head_id+"_single_err.txt"))
# # ))
# # extract_gene(best_blast_value_list,all_row_gene_fasta_file,blast_general/(head_id+"_single_copy.fasta"))
# R_blast_vlaue_df=base.do_call("rbind",R_blast_vlaue_list)
# utils.write_table(R_blast_vlaue_df,**{'file': str(general_out_path/"all.txt")},**{'append': False},**{'quote': False},**{'sep': "\t"},**{'row.names': False},**{'col.names': True})
if __name__ == '__main__':
# client = Client(n_workers=12, threads_per_worker=1)
cluster = PBSCluster(
job_extra=["-l nodes=1:ppn=24","-l mem=5000MB"],
header_skip=["select"],
processes=24,
walltime='25:00:00'
)
cluster.scale(jobs=10)
from dask.distributed import Client
client = Client(cluster)
print(cluster.job_script())
extract_ortholog_gene(
Path("../Pan_genome_data_2/contig_gff3_gffutils_db"),
Path("../Pan_genome_data_2/156_contig"),
Path("../Pan_genome_data_2/ortho/joined_df.tsv"),
directory_creater("/gpfshome/home/Baojd/wangzhe/ortholog_blast_mgg_key"),
)
parallel=False)
pysp2.io.write_dat(my_binary, out_path + base + '.particle.dat')
# Get all of the unique dates
all_sp2_files = glob(sp2b_path + '*.sp2b')
sp2_date_list = [x.split(".")[3] for x in all_sp2_files]
sp2_date_list = sorted(list(set(sp2_date_list)))
#sp2_date_list = ['20200218']
#process_day('20200218')
print(sp2_date_list)
cluster = PBSCluster(processes=6,
cores=36,
walltime='5:00:00',
memory='270GB',
name='dask-worker',
queue='arm_high_mem',
project='arm',
job_extra=['-W group_list=cades-arm'],
interface='ib0',
extra=['--no-dashboard'])
cluster.scale(36 * 6)
client = Client(cluster)
print("Waiting for workers before starting processing...")
client.wait_for_workers(9)
print(client)
results = client.map(process_day, sp2_date_list)
wait(results)
#del client
from dask_jobqueue import PBSCluster
cluster = PBSCluster()
cluster.scale(10)
from dask.distributed import Client
client = Client(clsuter)
# Do compute
# This is the simplest setup
# -----------------------------------------------------------------------------------------------------------------------
# A bit more
pcluster = PBSCluster(cores=36,
memory='100GB',
project='SOMETHING',
queue='smp',
interface='ib0',
walltime='03:00:00')
pcluster.scale(100)
client = Client(pcluster)
# -----------------------------------------------------------------------------------------------------------------------
#
import zarr
import xarray as xr
import time
import dask
import glob
ask_workers=20
memory='180GB'
from dask_jobqueue import PBSCluster
from dask.distributed import Client
import dask.dataframe as dd
cluster = PBSCluster(cores=1, memory=memory, project='PerfTestPangeo', walltime='06:00:00')
cluster.scale(ask_workers)
c = Client(cluster)
c
from dask.utils import ensure_dict, format_bytes
wk = c.scheduler_info()["workers"]
text="Workers= " + str(len(wk))
memory = [w["memory_limit"] for w in wk.values()]
if all(memory):
text += ", Memory=" + format_bytes(sum(memory))
print(text)
files=sorted(glob.glob('/work/ALT/odatis/eNATL60/BLBT02/gridT-2D/eNATL60*gridT-2D*.nc'))
'''
Author: your name
Date: 2020-10-27 08:56:03
LastEditTime: 2020-10-28 21:01:19
LastEditors: Please set LastEditors
Description: In User Settings Edit
FilePath: /Pan_genome_github/210_mafft.py
'''
from dask_jobqueue import PBSCluster
from dask import delayed
import dask
cluster = PBSCluster(job_extra=["-l nodes=1:ppn=24", "-l mem=5000MB"],
header_skip=["select"],
processes=24,
walltime='25:00:00')
cluster.scale(jobs=2)
from dask.distributed import Client
client = Client(cluster)
print(cluster.job_script())
import subprocess
from pathlib import Path
def mafft_popen(gene_file_path, out_dir_path, out_stderr_dir_path):
out_file = out_dir_path / gene_file_path.stem
if out_file.exists() is True: return
ma_popen = subprocess.Popen(
[
"mafft", "--genafpair", "--maxiterate", "16", "--reorder",
gene_file_path.resolve()
from dask.distributed import Client
import string
from random import randint
from time import sleep
### LOCAL EXECUTION
#client = Client(processes=False) # start local workers as threads
### SGE
from dask_jobqueue import PBSCluster
cluster = PBSCluster(cores=2, memory="4GB", queue="all.q")
cluster.scale(10)
client = Client(cluster)
# Creating my tasks
def get_jobid():
job_id = ""
if "JOB_ID" in os.environ:
job_id = str(os.environ["HOSTNAME"])
return job_id
def taskA (inputs=[]):
args = parser.parse_args()
print(args.input_file)
print(args.output_file)
########################### Dask Cluster ###############################
cluster = None
if args.cluster_mode == "cluster":
cluster = PBSCluster(processes=1,
cores=4,
memory="20GB",
local_directory='$TMPDIR',
project='DaskTest',
walltime='01:00:00',
interface='ib0')
cluster.scale(8)
else:
print("Local mode")
cluster = LocalCluster()
cluster.scale(2)
client = Client(cluster)
print(client)
########################### Read Input File ###########################
# Create a Dask DataFrame from input_file
# 6 colunms defined : Id t0 x0 v0 tmax and dt
df_in = dd.read_csv(args.input_file,
sep=" ",
names=['Id', 'x0', 'v0', 't0', 'tmax', 'dt'])
f'Unknown fields in pbs_spec: {args.pbs_spec.keys()}')
from dask.distributed import Client
from dask_jobqueue import PBSCluster
USER = os.environ['USER']
cluster = PBSCluster(queue=queue,
cores=36,
processes=9,
memory='100GB',
project=project,
walltime=walltime,
local_directory=f'/glade/scratch/{USER}/dask-tmp')
client = Client(cluster)
cluster.scale(9 * n_nodes)
#---------------------------------------------------------------------------
#-- read the input dataset
ds = xr.open_mfdataset(args.file_in, **xr_open_ds)
if isel:
ds = ds.isel(**isel)
if args.variable_list:
if args.invert_var_selction:
drop_vars = [v for v in ds.variables if v in args.variable_list]
else:
drop_vars = [
v for v in ds.variables if v not in args.variable_list
]
ds = ds.drop(drop_vars)
def start_cluster(nb_workers, walltime, out_dir, timeout=600):
"""
This function create a dask cluster. Each worker has nb_cpus cpus and only one
python process is started on each worker.
start_cluster will use OMP_NUM_THREADS environment variable to determine how many threads
might be used by a worker when running C/C++ code (default to 1)
start_cluster will use CARS_NB_WORKERS_PER_PBS_JOB environment variable to determine
how many workers should be started by a single PBS job (default to 1)
start_cluster will use CARS_PBS_QUEUE to determine in which queue worker jobs should
be posted.
:param nb_workers: Number of dask workers
:type nb_workers: int
:param walltime: Walltime for each dask worker
:type walltime: string
:param out_dir: Output directory
:type out_dir: string
:return: Dask cluster and dask client
:rtype: (dask_jobqueue.PBSCluster, dask.distributed.Client) tuple
"""
# Retrieve multi-threading factor for C/C++ code if available
omp_num_threads = 1
if os.environ.get('OMP_NUM_THREADS'):
omp_num_threads = int(os.environ['OMP_NUM_THREADS'])
# Retrieve number of workers per PBS job
nb_workers_per_job = 1
if os.environ.get('CARS_NB_WORKERS_PER_PBS_JOB'):
nb_workers_per_job = int(os.environ['CARS_NB_WORKERS_PER_PBS_JOB'])
# Retrieve PBS queue
pbs_queue = None
if os.environ.get('CARS_PBS_QUEUE'):
pbs_queue = os.environ['CARS_PBS_QUEUE']
# Total number of cpus is multi-threading factor times size of batch
# (number of workers per PBS job)
nb_cpus = nb_workers_per_job * omp_num_threads
# Cluster nodes have 5GB per core
memory = nb_cpus * 5000
# Ressource string for PBS
resource = "select=1:ncpus={}:mem={}mb".format(nb_cpus, memory)
nb_jobs = int(math.ceil(nb_workers / nb_workers_per_job))
logging.info(
"Starting Dask PBS cluster with {} workers ({} workers with {} cores each per PSB job)"
.format(nb_workers, nb_workers_per_job, omp_num_threads))
logging.info(
"Submitting {} PBS jobs with configuration cpu={},mem={},walltime={}".
format(nb_jobs, nb_cpus, memory, walltime))
names = [
'PATH', 'PYTHONPATH', 'CARS_STATIC_CONFIGURATION', 'LD_LIBRARY_PATH',
'OTB_APPLICATION_PATH', 'OTB_GEOID_FILE', 'OMP_NUM_THREADS',
'NUMBA_NUM_THREADS', 'OPJ_NUM_THREADS', 'GDAL_NUM_THREADS',
'OTB_MAX_RAM_HINT', 'VIRTUAL_ENV',
'ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS', 'GDAL_CACHEMAX'
]
names = [name for name in names if os.environ.get(name)]
envs = ["export {}={}".format(name, os.environ[name]) for name in names]
log_directory = os.path.join(os.path.abspath(out_dir), "dask_log")
local_directory = '$TMPDIR'
cluster = PBSCluster(processes=nb_workers_per_job,
cores=nb_workers_per_job,
resource_spec=resource,
memory="{}MB".format(memory),
local_directory=local_directory,
project='dask-test',
walltime=walltime,
interface='ib0',
queue=pbs_queue,
env_extra=envs,
log_directory=log_directory)
logging.info("Dask cluster started")
cluster.scale(nb_workers)
client = Client(cluster, timeout=timeout)
logging.info("Dashboard started at {}".format(get_dashboard_link(cluster)))
return cluster, client
from bluesky import RunEngine
from rapidz import Stream
from shed_streaming.translation import FromEventStream, ToEventStream, DBFriendly
from databroker import Broker
import bluesky.plans as bp
from ophyd.sim import hw
hw = hw()
from dask.distributed import Client
from dask_jobqueue import PBSCluster
from pprint import pprint
cluster = PBSCluster()
cluster.scale(10) # Ask for ten workers
client = Client(cluster)
db = Broker.named('temp')
# Now this pipeline runs using HPC resources
source = Stream()
(FromEventStream('event', 'motor1', upstream=source).scatter().map(
op.add,
1).buffer(8).gather().ToEventStream('result').DBFriendly().starsink(
db.insert))
RE = RunEngine()
RE.subscribe(lambda *x: source.emit(x))
if __name__ == '__main__':
# Get all of the unique dates
all_sp2_files = glob(sp2b_path + '*' + sys.argv[1] + '*.sp2b')
sp2_date_list = [x.split(".")[3] for x in all_sp2_files]
sp2_date_list = sorted(list(set(sp2_date_list)))
cluster = PBSCluster(processes=9,
cores=36,
walltime='4:00:00',
memory='270GB',
name='dask-worker',
queue='arm_high_mem',
project='arm',
job_extra=['-W group_list=cades-arm'],
interface='ib0',
extra=['--no-dashboard'])
cluster.scale(72)
client = Client(cluster)
print("Waiting for workers before starting processing...")
#client.wait_for_workers(9)
#for day in sp2_date_list:
# print("Processing %s" % day)
# process_day(day)
#print(sp2_date_list)
process_day('20191216')
#results = client.map(process_day, sp2_date_list)
#wait(results)
del client
def silly_function(i):
import socket
host = socket.gethostname()
print(host)
time.sleep(10 + i / 10)
print(i)
return hostname
if __name__ == "__main__":
oly_cluster = PBSCluster(
shebang='#!/bin/tcsh',
name='MODIS_dl',
cores=8,
memory='4GB',
processes=8, #doesn't work with scale(jobs) for some reason
local_directory='/home/disk/eos9/jkcm/temp',
resource_spec='nodes=1:ppn=8',
walltime='00:05:00',
job_extra=['-m ae', '-M [email protected]'],
interface='ib0')
oly_cluster.scale(4)
print(oly_cluster.job_script())
client = Client(oly_cluster)
L = client.map(silly_function, range(20))
client.gather(L)
print(L)
# g =
# print(g)