def parslConfigFromCompute(compute): """Given a Compute instance, return a setup parsl configuration""" if isinstance(compute, EC2Compute): # NOTE: Assumes the paropt is being run on an EC2 instance with access to metadata service try: public_ip = getAWSPublicIP() # get the required environment variables required_env_vars = [ "PAROPT_AWS_REGION", "PAROPT_AWS_KEY_NAME", "PAROPT_AWS_STATE_FILE", "PAROPT_AWS_IAM_INSTANCE_PROFILE_ARN" ] env_vars = { varname.replace('PAROPT_AWS_', '').lower(): os.getenv(varname) for varname in required_env_vars } missing_vars = [ varname for varname, value in env_vars.items() if value == None ] if missing_vars: raise Exception( "Missing required environment variables for running parsl with AWS:\n{}" .format(missing_vars)) parsl_config = Config( executors=[ HighThroughputExecutor( label='htex_local', address=public_ip, worker_port_range=(54000, 54050), interchange_port_range=(54051, 54100), cores_per_worker=1, max_workers=1, provider=AWSProvider( image_id=compute.ami, instance_type=compute.instance_model, worker_init= 'pip3 install git+https://[email protected]/globus-labs/ParaOpt@Chaofeng_modification', # git+https://[email protected]/chaofengwu/paropt',#git+https://[email protected]/macintoshpie/paropt', nodes_per_block=1, init_blocks=1, max_blocks=1, min_blocks=0, walltime='24:00:00', spot_max_bid=2.0, **env_vars), ) ], strategy=None, ) return parsl_config except KeyError as e: logger.error('Failed initializing aws config: {}'.format(e)) raise e except (HTTPError, URLError, OSError) as e: logger.error('Request to metadata service failed: {}'.format(e)) raise e elif isinstance(compute, LocalCompute): return Config(executors=[ ThreadPoolExecutor(max_threads=8, label='local_threads') ]) else: raise Exception('Unknown Compute type')
nproc = 36 sched_opts = ''' #SBATCH --cpus-per-task=%d ''' % (nproc) slurm_htex = Config( executors=[ HighThroughputExecutor( label="coffea_parsl_slurm", address=address_by_hostname(), prefetch_capacity=0, max_workers=nproc, provider=SlurmProvider( channel=LocalChannel(script_dir='parsl_slurm'), launcher=SrunLauncher(), max_blocks=(args.ncpu)+5, init_blocks=args.ncpu, partition='all', scheduler_options=sched_opts, # Enter scheduler_options if needed worker_init=wrk_init, # Enter worker_init if needed walltime='00:120:00' ), ) ], retries=20, ) dfk = parsl.load(slurm_htex) else: config = Config(executors=[ThreadPoolExecutor(max_threads=args.ncpu)]) parsl.load(config)
from parsl import App, DataFlowKernel # from parsl.monitoring.db_logger import MonitoringConfig from parsl.monitoring.monitoring import MonitoringHub from parsl.config import Config from parsl.executors import ThreadPoolExecutor import logging # parsl.set_stream_logger() threads_config = Config( executors=[ThreadPoolExecutor( label='threads', max_threads=4) ], monitoring=MonitoringHub( hub_address="127.0.0.1", hub_port=55055, logging_level=logging.INFO, resource_monitoring_interval=10, ) ) dfk = DataFlowKernel(config=threads_config) @App('python', dfk) def sleeper(dur=25): import time time.sleep(dur)