def __init__(self, label: str = 'threads', max_threads: int = 2, thread_name_prefix: str = '', storage_access: List[Any] = None, working_dir: Optional[str] = None, managed: bool = True): NoStatusHandlingExecutor.__init__(self) self.label = label self._scaling_enabled = False self.max_threads = max_threads self.thread_name_prefix = thread_name_prefix # we allow storage_access to be None now, which means something else to [] now # None now means that a default storage access list will be used, while # [] is a list with no storage access in it at all self.storage_access = storage_access self.working_dir = working_dir self.managed = managed
def __init__(self, label='turbine', storage_access=None, working_dir=None, managed=True): """Initialize the thread pool. Trying to implement the emews model. """ NoStatusHandlingExecutor.__init__(self) logger.debug("Initializing TurbineExecutor") self.label = label self.storage_access = storage_access self.working_dir = working_dir self.managed = managed
def __init__(self, label: str = "WorkQueueExecutor", provider: ExecutionProvider = LocalProvider(), working_dir: str = ".", managed: bool = True, project_name: Optional[str] = None, project_password_file: Optional[str] = None, address: Optional[str] = None, port: int = WORK_QUEUE_DEFAULT_PORT, env: Optional[Dict] = None, shared_fs: bool = False, storage_access: Optional[List[Staging]] = None, use_cache: bool = False, source: bool = False, pack: bool = False, autolabel: bool = False, autolabel_window: int = 1, autocategory: bool = False, init_command: str = "", full_debug: bool = True): NoStatusHandlingExecutor.__init__(self) self._provider = provider self._scaling_enabled = True if not _work_queue_enabled: raise OptionalModuleMissing( ['work_queue'], "WorkQueueExecutor requires the work_queue module.") self.label = label self.managed = managed self.task_queue = multiprocessing.Queue( ) # type: multiprocessing.Queue self.collector_queue = multiprocessing.Queue( ) # type: multiprocessing.Queue self.blocks = {} # type: Dict[str, str] self.address = address self.port = port self.task_counter = -1 self.project_name = project_name self.project_password_file = project_password_file self.env = env self.init_command = init_command self.shared_fs = shared_fs self.storage_access = storage_access self.use_cache = use_cache self.working_dir = working_dir self.registered_files = set() # type: Set[str] self.full = full_debug self.source = True if pack else source self.pack = pack self.autolabel = autolabel self.autolabel_window = autolabel_window self.autocategory = autocategory self.should_stop = multiprocessing.Value(c_bool, False) self.cached_envs = {} # type: Dict[int, str] if not self.address: self.address = socket.gethostname() if self.project_password_file is not None and not os.path.exists( self.project_password_file): raise WorkQueueFailure('Could not find password file: {}'.format( self.project_password_file)) if self.project_password_file is not None: if os.path.exists(self.project_password_file) is False: logger.debug("Password File does not exist, no file used") self.project_password_file = None # Build foundations of the launch command self.launch_cmd = ( "{package_prefix}python3 exec_parsl_function.py {mapping} {function} {result}" ) if self.init_command != "": self.launch_cmd = self.init_command + "; " + self.launch_cmd
def __init__(self, label="WorkQueueExecutor", working_dir=".", managed=True, project_name=None, project_password=None, project_password_file=None, port=WORK_QUEUE_DEFAULT_PORT, env=None, shared_fs=False, source=False, init_command="", full_debug=True, see_worker_output=False): NoStatusHandlingExecutor.__init__(self) if not _work_queue_enabled: raise OptionalModuleMissing( ['work_queue'], "WorkQueueExecutor requires the work_queue module.") self.label = label self.managed = managed self.task_queue = multiprocessing.Queue() self.collector_queue = multiprocessing.Queue() self.port = port self.task_counter = -1 self.scaling_enabled = False self.project_name = project_name self.project_password = project_password self.project_password_file = project_password_file self.env = env self.init_command = init_command self.shared_fs = shared_fs self.working_dir = working_dir self.used_names = {} self.shared_files = set() self.registered_files = set() self.worker_output = see_worker_output self.full = full_debug self.source = source self.cancel_value = multiprocessing.Value('i', 1) # Resolve ambiguity when password and password_file are both specified if self.project_password is not None and self.project_password_file is not None: logger.warning( "Password File and Password text specified for WorkQueue Executor, only Password Text will be used" ) self.project_password_file = None if self.project_password_file is not None: if os.path.exists(self.project_password_file) is False: logger.debug("Password File does not exist, no file used") self.project_password_file = None # Build foundations of the launch command self.launch_cmd = ( "python3 workqueue_worker.py -i {input_file} -o {output_file} {remapping_string}" ) if self.shared_fs is True: self.launch_cmd += " --shared-fs" if self.source is True: self.launch_cmd += " --source" if self.init_command != "": self.launch_cmd = self.init_command + "; " + self.launch_cmd