def condor_config(cores_per_job=4, mem_per_core=2048,
total_workers=24, max_workers=200,
pyenv_dir='%s/.local' % (os.environ['HOME'], ),
grid_proxy_dir='/tmp',
htex_label='coffea_parsl_condor_htex',
wrk_init=None,
condor_cfg=None):
pyenv_relpath = pyenv_dir.split('/')[-1]
if wrk_init is None:
wrk_init = '''
source /cvmfs/sft.cern.ch/lcg/views/LCG_95apython3/x86_64-centos7-gcc7-opt/setup.sh
export PATH=`pwd`/%s:$PATH
export PYTHONPATH=`pwd`/%s:$PYTHONPATH
export X509_USER_PROXY=`pwd`/%s
mkdir -p ./%s
''' % ('%s/bin' % pyenv_relpath,
'%s/lib/python3.6/site-packages' % pyenv_relpath,
x509_proxy,
htex_label)
if condor_cfg in None:
condor_cfg = '''
transfer_output_files = %s
RequestMemory = %d
RequestCpus = %d
''' % (htex_label, mem_per_core * cores_per_job, cores_per_job)
xfer_files = [pyenv_dir, osp.join(grid_proxy_dir, x509_proxy)]
condor_htex = Config(
executors=[
HighThroughputExecutor(
label=htex_label,
address=address_by_hostname(),
prefetch_capacity=0,
cores_per_worker=1,
max_workers=cores_per_job,
worker_logdir_root='./',
provider=CondorProvider(
channel=LocalChannel(),
init_blocks=total_workers,
max_blocks=max_workers,
nodes_per_block=1,
worker_init=wrk_init,
transfer_input_files=xfer_files,
scheduler_options=condor_cfg
),
)
],
strategy=None,
)
return condor_htex
from parsl.executors import HighThroughputExecutor
from parsl.utils import get_all_checkpoints
cores_per_slot = 8
worker_init = """
source ~/.bashrc
conda activate parsl
"""
scheduler_options = """
RequestMemory={}
""".format(5000 * cores_per_slot)
config = Config(executors=[
HighThroughputExecutor(
cores_per_worker=1,
heartbeat_threshold=120,
heartbeat_period=30,
provider=CondorProvider(
scheduler_options=scheduler_options,
cores_per_slot=cores_per_slot,
init_blocks=1,
max_blocks=1,
worker_init=worker_init,
),
)
],
checkpoint_mode='task_exit',
checkpoint_files=get_all_checkpoints())
tmpdir=$_CONDOR_SCRATCH_DIR
export TMPDIR=$tmpdir
else
# otherwise use the TMPDIR
tmpdir='.'
mkdir -p $tmpdir
fi
source activate bnl
echo `which python`
"""
WALLTIME = "48:00:00"
config = Config(executors=[
HighThroughputExecutor(worker_debug=True,
max_workers=1,
address='astro0010.rcf.bnl.gov',
provider=CondorProvider(
cores_per_slot=1,
mem_per_slot=2,
nodes_per_block=1,
init_blocks=100,
parallelism=0.01,
max_blocks=10000,
scheduler_options=SCHED_OPTS,
worker_init=WORKER_INIT,
walltime=WALLTIME))
], )
worker_init = '''
# source /cvmfs/sft.cern.ch/lcg/views/LCG_95apython3/x86_64-centos7-gcc7-opt/setup.sh
wrapper.sh bash
export PATH=~/.local/bin:$PATH
export PYTHONPATH=~/.local/lib/python3.6/site-packages:$PYTHONPATH
export FUNCX_TMPDIR=/tmp/{user}
export WORKER_TMPDIR=/tmp/{user}/workers
export X509_USER_PROXY=`pwd`/{proxy}
mkdir -p $FUNCX_TMPDIR
'''.format(user=os.environ['USER'], proxy=os.path.basename(proxy))
config = Config(
scaling_enabled=False,
worker_debug=True,
cores_per_worker=1,
strategy=SimpleStrategy(max_idletime=60000), # TOTAL HACK FIXME
provider=CondorProvider(
scheduler_options=
'stream_error=TRUE\nstream_output=TRUE\nTransferOut=TRUE\nTransferErr=TRUE',
cores_per_slot=8,
init_blocks=5,
max_blocks=5,
worker_init=worker_init,
transfer_input_files=[proxy, siteconf, wrapper]),
)
from parsl.config import Config
from parsl.providers import CondorProvider
from parsl.executors import HighThroughputExecutor
config = Config(
executors=[
HighThroughputExecutor(
label='OSG_HTEX',
max_workers=1,
provider=CondorProvider(
nodes_per_block=1,
init_blocks=4,
max_blocks=4,
# This scheduler option string ensures that the compute nodes provisioned
# will have modules
scheduler_options="""
+ProjectName = "MyProject"
Requirements = HAS_MODULES=?=TRUE
""",
# Command to be run before starting a worker, such as:
# 'module load Anaconda; source activate parsl_env'.
worker_init='''unset HOME; unset PYTHONPATH; module load python/3.7.0;
python3 -m venv parsl_env; source parsl_env/bin/activate; python3 -m pip install parsl''',
walltime="00:20:00",
),
worker_logdir_root='$OSG_WN_TMP',
worker_ports=(31000, 31001)
)
]
)
worker_init="\n".join(env_extra),
walltime='00:120:00'),
)
],
retries=20,
)
elif 'condor' in args.executor:
htex_config = Config(executors=[
HighThroughputExecutor(
label='coffea_parsl_condor',
address=address_by_query(),
# max_workers=1,
provider=CondorProvider(
nodes_per_block=1,
init_blocks=1,
max_blocks=1,
worker_init="\n".join(env_extra + condor_extra),
walltime="00:20:00",
),
)
])
else:
raise NotImplementedError
dfk = parsl.load(htex_config)
output = processor.run_uproot_job(
sample_dict,
treename='Events',
processor_instance=processor_instance,
executor=processor.parsl_executor,
max_workers=1,
poll_period=10,
prefetch_capacity=0,
provider=CondorProvider(
channel=OAuthSSHChannel(
'spce01.sdcc.bnl.gov',
envs={},
port=2222,
script_dir='/sdcc/u/dcde1000006/parsl_scripts',
username='******'),
environment={},
init_blocks=1,
# launcher=SingleNodeLauncher(),
max_blocks=1,
min_blocks=1,
nodes_per_block=1,
#parallelism=1,
parallelism=0,
project='',
#Trying this Requirements directive per Dong's instructions:
#requirements='regexp("^sp[oa]", machine)',
scheduler_options=
'accounting_group = group_sdcc.main \nRequirements = (regexp("^sp[oa]", machine))',
transfer_input_files=[],
walltime='00:30:00',
#worker_init='source /sdcc/u/dcde1000001/dcdesetup.sh'
worker_init=
'source /hpcgpfs01/work/dcde/setup.sh; source activate dcdemaster20191008'
),
storage_access=[],
suppress_failure=False,
worker_debug=True,
managed=True,
max_workers=1,
mem_per_worker=None,
poll_period=10,
prefetch_capacity=0,
provider=CondorProvider(
channel=OAuthSSHChannel(
'spce01.sdcc.bnl.gov',
envs={},
port=2222,
script_dir='/sdcc/u/dcde1000006/parsl_scripts',
username='******'
),
environment={},
init_blocks=1,
# launcher=SingleNodeLauncher(),
max_blocks=1,
min_blocks=0,
nodes_per_block=1,
parallelism=1,
project='',
requirements='',
scheduler_options='accounting_group = group_sdcc.main',
transfer_input_files=[],
walltime='00:10:00',
worker_init='source /sdcc/u/dcde1000001/dcdesetup.sh'
),
storage_access=[],
suppress_failure=False,
worker_debug=True,
worker_logdir_root='/sdcc/u/dcde1000006/parsl_scripts/logs',
worker_port_range=(50000, 51000),
def parsl_condor_config(workers=1):
x509_proxy = f'x509up_u{UID}'
grid_proxy_dir = '/tmp'
cores_per_job = 1
mem_per_core = 2000
mem_request = mem_per_core * cores_per_job
init_blocks = 1
min_blocks = 1
max_blocks = workers
htex_label='coffea_parsl_condor_htex'
log_dir = 'parsl_logs'
log_dir_full = os.path.join('/nfs_scratch/dntaylor',log_dir)
worker_init = f'''
echo "Setting up environment"
tar -zxf columnar.tar.gz
source columnar/bin/activate
export PATH=columnar/bin:$PATH
export PYTHONPATH=columnar/lib/python3.6/site-packages:$PYTHONPATH
export X509_USER_PROXY={x509_proxy}
mkdir -p {log_dir}/{htex_label}
echo "Environment ready"
'''
# requirements for T2_US_Wisconsin (HAS_CMS_HDFS forces to run a T2 node not CHTC)
# Removing for now:
scheduler_options = f'''
transfer_output_files = {log_dir}/{htex_label}
RequestMemory = {mem_request}
RequestCpus = {cores_per_job}
+RequiresCVMFS = True
Requirements = TARGET.HAS_CMS_HDFS && TARGET.Arch == "X86_64"
priority = 10
'''
transfer_input_files = [os.path.join(os.path.dirname(os.path.abspath(__file__)),'columnar.tar.gz'), os.path.join(grid_proxy_dir, x509_proxy)]
htex = Config(
executors=[
HighThroughputExecutor(
label=htex_label,
address=address_by_hostname(),
prefetch_capacity=0,
cores_per_worker=1,
max_workers=cores_per_job,
worker_logdir_root=log_dir,
provider=CondorProvider(
channel=LocalChannel(
userhome='/nfs_scratch/dntaylor',
),
init_blocks=init_blocks,
min_blocks=min_blocks,
max_blocks=max_blocks,
nodes_per_block=1,
worker_init=worker_init,
transfer_input_files=transfer_input_files,
scheduler_options=scheduler_options,
),
),
# TODO: works, but really isn't helpful since half of the tasks get shipped to the condor
# executor and don't flock back when the local executor is empty
# an alternative could be to preprocess locally and process on the grid
# add a local executor so stuff starts fast
#HighThroughputExecutor(
# label="coffea_parsl_default",
# cores_per_worker=1,
# max_workers=1, # TODO: multicore local?
# worker_logdir_root=log_dir,
# provider=LocalProvider(
# channel=LocalChannel(),
# init_blocks=1,
# max_blocks=1,
# ),
#),
],
strategy='simple',
run_dir=os.path.join(log_dir_full,'runinfo'),
retries = 2, # retry all failures, xrootd failures are retried then skipped via coffea executor itself
)
return htex
from parsl.config import Config
from parsl.executors import WorkQueueExecutor
from parsl.providers import CondorProvider
config = Config(executors=[
WorkQueueExecutor(
port=50055,
source=True,
provider=CondorProvider(),
# init_command='source /home/yadu/src/wq_parsl/setup_parsl_env.sh;
# echo "Ran at $date" > /home/yadu/src/wq_parsl/parsl/tests/workqueue_tests/ran.log',
)
])
import parsl
from parsl.channels import LocalChannel
from parsl.providers import CondorProvider
from parsl.config import Config
from parsl.executors.ipp import IPyParallelExecutor
from parsl.app.app import python_app, bash_app
config = Config(
executors=[
IPyParallelExecutor(
label='htcondor',
provider=CondorProvider(
channel=LocalChannel(),
nodes_per_block=1,
init_blocks=1,
max_blocks=1,
scheduler_options='accounting_group = group_sdcc.main',
worker_init='', # Input your worker_init if needed
requirements='',
),
engine_debug_level='DEBUG',
)
], )
parsl.clear()
parsl.load(config)
@python_app()
def hello():
return 'Hello World'
from parsl.channels import LocalChannel
path_str = os.environ["PATH"]
# TODO: find a way to compute this.
#worker_script_path = "/afs/crc.nd.edu/user/d/dsmith47/.local/bin"
#worker_script_path = "/afs/crc.nd.edu/user/d/dsmith47/Documents/Parsl/parsl/parsl"
worker_script_path = "/afs/crc.nd.edu/user/d/dsmith47/Documents/Parsl/parsl/build/lib/parsl/executors/high_throughput/"
# TODO: find a way to determine this
address_str = "condorfe.crc.nd.edu"
config = Config(executors=[
HighThroughputExecutor(
label='condor_hte',
provider=CondorProvider(
project="condor_hte",
channel=LocalChannel(),
worker_init="export HOME=$PWD",
environment={"PATH": path_str + ":" + worker_script_path},
requirements="process_worker_pool.py"),
address=address_str,
working_dir=os.getcwd(),
)
])
parsl.load(config)
@python_app
def generate(limit):
from random import randint
return randint(1, limit)
j.result()
end_time = time.time()
return end_time - start_time
local_config = Config(
executors=[ThreadPoolExecutor(max_threads=8, label='local_threads')])
address_str = "condorfe.crc.nd.edu"
path_str = os.environ["PATH"]
condor_config = Config(executors=[
HighThroughputExecutor(
label='condor_hte',
provider=CondorProvider(project="condor_hte",
channel=LocalChannel(),
worker_init="export HOME=$PWD",
environment={"PATH": path_str}),
address=address_str,
working_dir=os.getcwd(),
),
])
#TODO: This port number needs to be configurable so that a script can specify
# the port here and when spinning up workers.
wq_config = Config(executors=[
WorkQueueExecutor(label='wq_exec',
project_name='wq_benchmark',
env={"PATH": path_str},
init_command="export HOME=$PWD",
port=9000)
])
proxy = '/tmp/x509up_u{}'.format(os.getuid())
if not os.path.isfile(proxy):
raise RuntimeError(
'No valid proxy found-- please run `voms-proxy-init -voms cms`')
worker_init = '''
source /cvmfs/sft.cern.ch/lcg/views/LCG_95apython3/x86_64-centos7-gcc7-opt/setup.sh
export PATH=~/.local/bin:$PATH
export PYTHONPATH=~/.local/lib/python3.6/site-packages:$PYTHONPATH
export X509_USER_PROXY=`pwd`/{}
'''.format(os.path.basename(proxy))
config = Config(executors=[
HighThroughputExecutor(
address=address_by_hostname(),
cores_per_worker=1,
worker_debug=True,
provider=CondorProvider(
cores_per_slot=8,
init_blocks=90,
max_blocks=90,
worker_init=worker_init,
transfer_input_files=[proxy],
),
)
],
retries=5,
strategy=None)
This produces an error in stdout from parsl:
parsl.parsl.auto.1561593082.171704.script: line 7: process_worker_pool.py: command not found
corresponding line in the submitted script:
process_worker_pool.py -c 1.0 --poll 10 --task_url=tcp://130.199.185.13:54772 --result_url=tcp://130.199.185.13:54295 --l
"""
bnl_sdcc_condor = Config(executors=[
HighThroughputExecutor(
label='sdcc_condor',
provider=CondorProvider(
channel=LocalChannel(script_dir='parsl_scriptdir'),
nodes_per_block=1,
init_blocks=1,
min_blocks=1,
max_blocks=1,
parallelism=0,
scheduler_options='accounting_group = group_sdcc.main'),
address='130.199.185.13',
managed=False)
], )
parsl.load(bnl_sdcc_condor)
"""
try doing something more explicit with stderr and stdout (based on Ketan's
Cori/relion example):
"""
#condorinfo = sdcc_wninfo(executors=['sdcc_condor'])
condorinfo = sdcc_wninfo(stdout='relion_condor.out',
# If you are a developer running tests, make sure to update parsl/tests/configs/user_opts.py
# If you are a user copying-and-pasting this as an example, make sure to either
# 1) create a local `user_opts.py`, or
# 2) delete the user_opts import below and replace all appearances of `user_opts` with the literal value
# (i.e., user_opts['swan']['username'] -> 'your_username')
from .user_opts import user_opts
config = Config(
executors=[
HighThroughputExecutor(
label='OSG_HTEX',
address=address_by_query(),
max_workers=1,
provider=CondorProvider(
nodes_per_block=1,
init_blocks=4,
max_blocks=4,
# This scheduler option string ensures that the compute nodes provisioned
# will have modules
scheduler_options=
'Requirements = OSGVO_OS_STRING == "RHEL 6" && Arch == "X86_64" && HAS_MODULES == True',
# Command to be run before starting a worker, such as:
# 'module load Anaconda; source activate parsl_env'.
worker_init=user_opts['osg']['worker_init'],
walltime="00:20:00",
),
)
],
run_dir=get_rundir())
docker_image = "opensciencegrid/osgvo-el6"
transfer_output_files = coffea_parsl_condor
''' #% (nproc, ) # twoGB*nproc,
#RequestMemory = %d
#RequestCpus = %d
#RequestDisk = 1048576
xfer_files = ['%s/.local' % (os.environ['HOME'], ), '/tmp/%s' % (x509_proxy, )]
config = Config(
executors=[
HighThroughputExecutor(
label="coffea_parsl_condor",
address=address_by_hostname(),
prefetch_capacity=0,
cores_per_worker=1,
max_workers=nproc,
worker_logdir_root='./',
provider=CondorProvider(init_blocks=8,
max_blocks=200,
nodes_per_block=1,
worker_init=wrk_init,
transfer_input_files=xfer_files,
scheduler_options=condor_cfg),
)
],
strategy=None,
)