def __init__(self,
label='turbine',
storage_access=None,
working_dir=None,
managed=True):
"""Initialize the thread pool.
Trying to implement the emews model.
"""
NoStatusHandlingExecutor.__init__(self)
logger.debug("Initializing TurbineExecutor")
self.label = label
self.storage_access = storage_access
self.working_dir = working_dir
self.managed = managed
def __init__(self,
label: str = "WorkQueueExecutor",
provider: ExecutionProvider = LocalProvider(),
working_dir: str = ".",
managed: bool = True,
project_name: Optional[str] = None,
project_password_file: Optional[str] = None,
address: Optional[str] = None,
port: int = WORK_QUEUE_DEFAULT_PORT,
env: Optional[Dict] = None,
shared_fs: bool = False,
storage_access: Optional[List[Staging]] = None,
use_cache: bool = False,
source: bool = False,
pack: bool = False,
autolabel: bool = False,
autolabel_window: int = 1,
autocategory: bool = False,
init_command: str = "",
full_debug: bool = True):
NoStatusHandlingExecutor.__init__(self)
self._provider = provider
self._scaling_enabled = True
if not _work_queue_enabled:
raise OptionalModuleMissing(
['work_queue'],
"WorkQueueExecutor requires the work_queue module.")
self.label = label
self.managed = managed
self.task_queue = multiprocessing.Queue(
) # type: multiprocessing.Queue
self.collector_queue = multiprocessing.Queue(
) # type: multiprocessing.Queue
self.blocks = {} # type: Dict[str, str]
self.address = address
self.port = port
self.task_counter = -1
self.project_name = project_name
self.project_password_file = project_password_file
self.env = env
self.init_command = init_command
self.shared_fs = shared_fs
self.storage_access = storage_access
self.use_cache = use_cache
self.working_dir = working_dir
self.registered_files = set() # type: Set[str]
self.full = full_debug
self.source = True if pack else source
self.pack = pack
self.autolabel = autolabel
self.autolabel_window = autolabel_window
self.autocategory = autocategory
self.should_stop = multiprocessing.Value(c_bool, False)
self.cached_envs = {} # type: Dict[int, str]
if not self.address:
self.address = socket.gethostname()
if self.project_password_file is not None and not os.path.exists(
self.project_password_file):
raise WorkQueueFailure('Could not find password file: {}'.format(
self.project_password_file))
if self.project_password_file is not None:
if os.path.exists(self.project_password_file) is False:
logger.debug("Password File does not exist, no file used")
self.project_password_file = None
# Build foundations of the launch command
self.launch_cmd = (
"{package_prefix}python3 exec_parsl_function.py {mapping} {function} {result}"
)
if self.init_command != "":
self.launch_cmd = self.init_command + "; " + self.launch_cmd
def __init__(self,
label="WorkQueueExecutor",
working_dir=".",
managed=True,
project_name=None,
project_password=None,
project_password_file=None,
port=WORK_QUEUE_DEFAULT_PORT,
env=None,
shared_fs=False,
source=False,
init_command="",
full_debug=True,
see_worker_output=False):
NoStatusHandlingExecutor.__init__(self)
if not _work_queue_enabled:
raise OptionalModuleMissing(
['work_queue'],
"WorkQueueExecutor requires the work_queue module.")
self.label = label
self.managed = managed
self.task_queue = multiprocessing.Queue()
self.collector_queue = multiprocessing.Queue()
self.port = port
self.task_counter = -1
self.scaling_enabled = False
self.project_name = project_name
self.project_password = project_password
self.project_password_file = project_password_file
self.env = env
self.init_command = init_command
self.shared_fs = shared_fs
self.working_dir = working_dir
self.used_names = {}
self.shared_files = set()
self.registered_files = set()
self.worker_output = see_worker_output
self.full = full_debug
self.source = source
self.cancel_value = multiprocessing.Value('i', 1)
# Resolve ambiguity when password and password_file are both specified
if self.project_password is not None and self.project_password_file is not None:
logger.warning(
"Password File and Password text specified for WorkQueue Executor, only Password Text will be used"
)
self.project_password_file = None
if self.project_password_file is not None:
if os.path.exists(self.project_password_file) is False:
logger.debug("Password File does not exist, no file used")
self.project_password_file = None
# Build foundations of the launch command
self.launch_cmd = (
"python3 workqueue_worker.py -i {input_file} -o {output_file} {remapping_string}"
)
if self.shared_fs is True:
self.launch_cmd += " --shared-fs"
if self.source is True:
self.launch_cmd += " --source"
if self.init_command != "":
self.launch_cmd = self.init_command + "; " + self.launch_cmd