def __init__(self, name, reason, details=None): # NB: this is late imported else we have a circular dependency that's hard to fix from simpleflow.format import decode self.name = name self.reason = decode(reason, parse_json=False, use_proxy=False) self.details = decode(details, parse_json=False, use_proxy=False) super(TaskFailed, self).__init__(name, self.reason, self.details)
def test_jumbo_fields_in_task_failed_is_decoded(self): # prepare execution self.register_activity_type( "tests.test_simpleflow.swf.test_executor.print_me_n_times", "default" ) # start execution self.start_workflow_execution( input='{"args": ["012345679", 10000], "kwargs": {"raises": true}}', ) # decider part result = self.build_decisions(ExampleJumboWorkflow) assert len(result.decisions) == 1 self.take_decisions(result.decisions, result.execution_context) # worker part self.process_activity_task() # now check the history events = self.get_workflow_execution_history()["events"] activity_result_evt = events[-2] assert activity_result_evt["eventType"] == "ActivityTaskFailed" attrs = activity_result_evt["activityTaskFailedEventAttributes"] expect(attrs["reason"]).to.match( r"simpleflow\+s3://jumbo-bucket/[a-z0-9-]+ 9\d{4}" ) expect(attrs["details"]).to.match( r"simpleflow\+s3://jumbo-bucket/[a-z0-9-]+ 9\d{4}" ) details = format.decode(attrs["details"]) expect(details["error"]).to.equal("ValueError") expect(len(details["message"])).to.be.greater_than(9 * 10000) # decide again (should lead to workflow failure) result = self.build_decisions(ExampleJumboWorkflow) assert len(result.decisions) == 1 assert result.decisions[0]["decisionType"] == "FailWorkflowExecution" self.take_decisions(result.decisions, result.execution_context) # now check history again events = self.get_workflow_execution_history()["events"] event = events[-1] assert event["eventType"] == "WorkflowExecutionFailed" attrs = event["workflowExecutionFailedEventAttributes"] details = format.decode(attrs["details"], use_proxy=False) expect(details).to.be.a("dict") expect(details["message"]).to.match(r"^Number: 012345.*") reason = format.decode(attrs["reason"], use_proxy=False) expect(reason).to.match( r"^Workflow execution error in activity-tests.test_simpleflow.swf." r'test_executor.print_me_n_times: "ValueError: Number: 012345679\d+"$' )
def test_jumbo_fields_in_task_failed_is_decoded(self): # prepare execution self.register_activity_type( "tests.test_simpleflow.swf.test_executor.print_me_n_times", "default" ) # start execution self.start_workflow_execution( input='{"args": ["012345679", 10000], "kwargs": {"raises": true}}', ) # decider part result = self.build_decisions(ExampleJumboWorkflow) assert len(result.decisions) == 1 self.take_decisions(result.decisions, result.execution_context) # worker part self.process_activity_task() # now check the history events = self.get_workflow_execution_history()["events"] activity_result_evt = events[-2] assert activity_result_evt["eventType"] == "ActivityTaskFailed" attrs = activity_result_evt["activityTaskFailedEventAttributes"] expect(attrs["reason"]).to.match(r"simpleflow\+s3://jumbo-bucket/[a-z0-9-]+ 9\d{4}") expect(attrs["details"]).to.match(r"simpleflow\+s3://jumbo-bucket/[a-z0-9-]+ 9\d{4}") details = format.decode(attrs["details"]) expect(details["error"]).to.equal("ValueError") expect(len(details["message"])).to.be.greater_than(9*10000) # decide again (should lead to workflow failure) result = self.build_decisions(ExampleJumboWorkflow) assert len(result.decisions) == 1 assert result.decisions[0]["decisionType"] == "FailWorkflowExecution" self.take_decisions(result.decisions, result.execution_context) # now check history again events = self.get_workflow_execution_history()["events"] event = events[-1] assert event["eventType"] == "WorkflowExecutionFailed" attrs = event["workflowExecutionFailedEventAttributes"] details = format.decode(attrs["details"], use_proxy=False) expect(details).to.be.a("dict") expect(details["message"]).to.match(r"^Number: 012345.*") reason = format.decode(attrs["reason"], use_proxy=False) expect(reason).to.match( r'^Workflow execution error in activity-tests.test_simpleflow.swf.' r'test_executor.print_me_n_times: "ValueError: Number: 012345679\d+"$' )
def list_markers(self, all=False): if all: return [ Marker(m['name'], format.decode(m['details'])) for ml in self._history.markers.values() for m in ml ] rc = [] for ml in self._history.markers.values(): m = ml[-1] if m['state'] == 'recorded': rc.append(Marker(m['name'], format.decode(m['details']))) return rc
def _extract_reason(err): if hasattr(err.exception, 'reason'): raw = err.exception.reason # don't parse eventual json object here, since we will cast # the result to a string anyway, better keep a json representation return format.decode(raw, parse_json=False, use_proxy=False) return repr(err.exception)
def get_event_details(self, event_type, event_name): if event_type == 'signal': return self._history.signals.get(event_name) elif event_type == 'marker': marker_list = self._history.markers.get(event_name) if not marker_list: return None marker_list = list( filter( lambda m: m['state'] == 'recorded', marker_list ) ) if not marker_list: return None # Make pleasing details marker = copy.copy(marker_list[-1]) marker['details'] = format.decode(marker['details']) return marker elif event_type == 'timer': return self._history.timers.get(event_name) else: raise ValueError('Unimplemented type {!r} for get_event_details'.format( event_type ))
def _get_future_from_child_workflow_event(self, event): """Maps a child workflow event to a Future with the corresponding state. :param event: child workflow event :type event: dict[str, Any] """ future = futures.Future() state = event['state'] if state == 'start_initiated': pass # future._state = futures.PENDING elif state == 'start_failed': if event['cause'] == 'WORKFLOW_TYPE_DOES_NOT_EXIST': workflow_type = swf.models.WorkflowType( self.domain, name=event['name'], version=event['version'], ) logger.info('Creating workflow type {} in domain {}'.format( workflow_type.name, self.domain.name, )) try: workflow_type.save() except swf.exceptions.AlreadyExistsError: # Could have be created by a concurrent workflow execution. pass return None future.set_exception(exceptions.TaskFailed( name=event['id'], reason=event['cause'], details=event.get('details'), )) elif state == 'started': future.set_running() elif state == 'completed': future.set_finished(format.decode(event['result'])) elif state == 'failed': future.set_exception(exceptions.TaskFailed( name=event['id'], reason=event['reason'], details=event.get('details'), )) elif state == 'timed_out': future.set_exception(exceptions.TimeoutError( event['timeout_type'], None, )) elif state == 'canceled': future.set_exception(exceptions.TaskCanceled( event.get('details'), )) elif state == 'terminated': future.set_exception(exceptions.TaskTerminated()) return future
def process(self, poller, token, task): """ :param poller: :type poller: ActivityPoller :param token: :type token: str :param task: :type task: swf.models.ActivityTask """ logger.debug('ActivityWorker.process() pid={}'.format(os.getpid())) try: activity = self.dispatch(task) input = format.decode(task.input) args = input.get('args', ()) kwargs = input.get('kwargs', {}) context = sanitize_activity_context(task.context) context['domain_name'] = poller.domain.name if input.get('meta', {}).get('binaries'): download_binaries(input['meta']['binaries']) result = ActivityTask(activity, *args, context=context, **kwargs).execute() except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() logger.exception("process error: {}".format(str(exc_value))) if isinstance(exc_value, ExecutionError) and len(exc_value.args): details = exc_value.args[0] reason = format_exc(exc_value) # FIXME json.loads and rebuild? else: tb = traceback.format_tb(exc_traceback) reason = format_exc(exc_value) details = json_dumps( { 'error': exc_type.__name__, 'message': str(exc_value), 'traceback': tb, }, default=repr ) return poller.fail_with_retry( token, task, reason=reason, details=details ) try: logger.info('completing activity') poller.complete_with_retry(token, result) except Exception as err: logger.exception("complete error") reason = 'cannot complete task {}: {} {}'.format( task.activity_id, err.__class__.__name__, err, ) poller.fail_with_retry(token, task, reason)
def _get_future_from_activity_event(self, event): """Maps an activity event to a Future with the corresponding state. :param event: activity event :type event: dict[str, Any] :rtype: futures.Future """ future = futures.Future() # state is PENDING. state = event['state'] if state == 'scheduled': pass elif state == 'schedule_failed': if event['cause'] == 'ACTIVITY_TYPE_DOES_NOT_EXIST': activity_type = swf.models.ActivityType( self.domain, name=event['activity_type']['name'], version=event['activity_type']['version']) logger.info('creating activity type {} in domain {}'.format( activity_type.name, self.domain.name)) try: activity_type.save() except swf.exceptions.AlreadyExistsError: logger.info( 'oops: Activity type {} in domain {} already exists, creation failed, continuing...'.format( activity_type.name, self.domain.name)) return None logger.info('failed to schedule {}: {}'.format( event['activity_type']['name'], event['cause'], )) return None elif state == 'started': future.set_running() elif state == 'completed': result = event['result'] future.set_finished(format.decode(result)) elif state == 'canceled': future.set_cancelled() elif state == 'failed': exception = exceptions.TaskFailed( name=event['id'], reason=event['reason'], details=event.get('details')) future.set_exception(exception) elif state == 'timed_out': exception = exceptions.TimeoutError( event['timeout_type'], event['timeout_value']) future.set_exception(exception) return future
def test_decode(self): self.setup_jumbo_fields("jumbo-bucket") push_content("jumbo-bucket", "abc", "decoded jumbo field yay!") cases = [ [None, None], ["foo bar baz", "foo bar baz"], ['"a string"', "a string"], ['[1, 2]', [1, 2]], ["simpleflow+s3://jumbo-bucket/abc 24", "decoded jumbo field yay!"], ] for case in cases: self.assertEqual(case[1], format.decode(case[0]))
def activity_rerun(domain, workflow_id, run_id, input, scheduled_id, activity_id): # handle params if not activity_id and not scheduled_id: logger.error("Please supply --scheduled-id or --activity-id.") sys.exit(1) input_override = None if input: input_override = format.decode(input) # find workflow execution try: wfe = helpers.get_workflow_execution(domain, workflow_id, run_id) except (swf.exceptions.DoesNotExistError, IndexError): logger.error("Couldn't find execution, exiting.") sys.exit(1) logger.info("Found execution: workflowId={} runId={}".format( wfe.workflow_id, wfe.run_id)) # now rerun the specified activity history = History(wfe.history()) history.parse() task, args, kwargs, meta, params = helpers.find_activity( history, scheduled_id=scheduled_id, activity_id=activity_id, input=input_override, ) kwargs["context"].update({ "workflow_id": wfe.workflow_id, "run_id": wfe.run_id, }) logger.debug("Found activity. Last execution:") for line in json_dumps(params, pretty=True).split("\n"): logger.debug(line) if input_override: logger.info("NB: input will be overriden with the passed one!") logger.info("Will re-run: {}(*{}, **{}) [+meta={}]".format( task, args, kwargs, meta)) # download binaries if needed download_binaries(meta.get("binaries", {})) # execute the activity task with the correct arguments instance = ActivityTask(task, *args, **kwargs) result = instance.execute() if hasattr(instance, "post_execute"): instance.post_execute() logger.info("Result (JSON): {}".format(json_dumps(result, compact=False)))
def activity_rerun(domain, workflow_id, run_id, input, scheduled_id, activity_id): # handle params if not activity_id and not scheduled_id: logger.error("Please supply --scheduled-id or --activity-id.") sys.exit(1) input_override = None if input: input_override = format.decode(input) # find workflow execution try: wfe = helpers.get_workflow_execution(domain, workflow_id, run_id) except (swf.exceptions.DoesNotExistError, IndexError): logger.error("Couldn't find execution, exiting.") sys.exit(1) logger.info("Found execution: workflowId={} runId={}".format(wfe.workflow_id, wfe.run_id)) # now rerun the specified activity history = History(wfe.history()) history.parse() task, args, kwargs, meta, params = helpers.find_activity( history, scheduled_id=scheduled_id, activity_id=activity_id, input=input_override, ) logger.debug("Found activity. Last execution:") for line in json_dumps(params, pretty=True).split("\n"): logger.debug(line) if input_override: logger.info("NB: input will be overriden with the passed one!") logger.info("Will re-run: {}(*{}, **{}) [+meta={}]".format(task, args, kwargs, meta)) # download binaries if needed download_binaries(meta.get("binaries", {})) # execute the activity task with the correct arguments instance = ActivityTask(task, *args, **kwargs) result = instance.execute() if hasattr(instance, 'post_execute'): instance.post_execute() logger.info("Result (JSON): {}".format(json_dumps(result, compact=False)))
def execute(*args, **kwargs): logger = logging.getLogger(logger_name) command = 'simpleflow.execute' # name of a module. sys.stdout.flush() sys.stderr.flush() result_str = None # useless context = kwargs.pop('context', {}) with tempfile.TemporaryFile() as result_fd, tempfile.TemporaryFile( ) as error_fd: dup_result_fd = os.dup(result_fd.fileno()) # remove FD_CLOEXEC dup_error_fd = os.dup(error_fd.fileno()) # remove FD_CLOEXEC # print('error_fd: {}'.format(dup_error_fd)) full_command = [ interpreter, '-m', command, # execute module a script. get_name(func), format_arguments_json(*args, **kwargs), '--logger-name={}'.format(logger_name), '--result-fd={}'.format(dup_result_fd), '--error-fd={}'.format(dup_error_fd), '--context={}'.format(json_dumps(context)), ] if kill_children: full_command.append('--kill-children') if compat.PY2: # close_fds doesn't work with python2 (using its C _posixsubprocess helper) close_fds = False pass_fds = () else: close_fds = True pass_fds = (dup_result_fd, dup_error_fd) process = subprocess.Popen( full_command, bufsize=-1, close_fds=close_fds, pass_fds=pass_fds, ) rc = wait_subprocess(process, timeout=timeout, command_info=full_command) os.close(dup_result_fd) os.close(dup_error_fd) if rc: error_fd.seek(0) err_output = error_fd.read() if err_output: if not compat.PY2: err_output = err_output.decode('utf-8', errors='replace') raise ExecutionError(err_output) result_fd.seek(0) result_str = result_fd.read() if not result_str: return None try: if not compat.PY2: result_str = result_str.decode('utf-8', errors='replace') result = format.decode(result_str) return result except BaseException as ex: logger.exception('Exception in python.execute: {} {}'.format( ex.__class__.__name__, ex)) logger.warning('%r', result_str)
def input(self, value): self._input = format.decode(value)
def control(self, value): self._control = format.decode(value)
def get_input(wf_input): if not wf_input: wf_input = sys.stdin.read() wf_input = format.decode(wf_input) return transform_input(wf_input)
def main(): """ When executed as a script, this module expects the name of a callable as its first argument and the arguments of the callable encoded in a JSON string as its second argument. It then executes the callable with the arguments after decoding them into Python objects. It finally encodes the value returned by the callable into a JSON string and prints it on stdout. the arguments of the callable are stored in a dict with the following format: :: {'args': [...], 'kwargs': { ..., } } Synopsis -------- :: usage: execute.py [-h] funcname funcargs positional arguments: funcname name of the callable to execute funcargs callable arguments in JSON optional arguments: -h, --help show this help message and exit Examples -------- :: $ python -m simpleflow.execute "os.path.exists" '{"args": ["/tmp"]}' true """ import argparse parser = argparse.ArgumentParser() parser.add_argument( 'funcname', help='name of the callable to execute', ) parser.add_argument( 'funcargs', help='callable arguments in JSON', ) parser.add_argument( '--context', help='Activity Context', ) parser.add_argument( '--logger-name', help='logger name', ) parser.add_argument( '--result-fd', type=int, default=1, metavar='N', help='result file descriptor', ) parser.add_argument( '--error-fd', type=int, default=2, metavar='N', help='error file descriptor', ) parser.add_argument( '--arguments-json-fd', type=int, default=None, metavar='N', help='JSON input file descriptor', ) parser.add_argument( '--kill-children', action='store_true', help='kill child processes on exit', ) cmd_arguments = parser.parse_args() def kill_child_processes(): process = psutil.Process(os.getpid()) children = process.children(recursive=True) for child in children: try: child.terminate() except psutil.NoSuchProcess: pass _, still_alive = psutil.wait_procs(children, timeout=0.3) for child in still_alive: try: child.kill() except psutil.NoSuchProcess: pass funcname = cmd_arguments.funcname if cmd_arguments.arguments_json_fd is None: content = cmd_arguments.funcargs if content is None: parser.error('the following arguments are required: funcargs') else: with os.fdopen(cmd_arguments.arguments_json_fd) as arguments_json_file: content = arguments_json_file.read() try: arguments = format.decode(content) except Exception: raise ValueError('cannot load arguments from {}'.format( content)) if cmd_arguments.logger_name: logger = logging.getLogger(cmd_arguments.logger_name) else: logger = simpleflow_logger callable_ = make_callable(funcname) if hasattr(callable_, '__wrapped__'): callable_ = callable_.__wrapped__ args = arguments.get('args', ()) kwargs = arguments.get('kwargs', {}) context = json.loads(cmd_arguments.context) if cmd_arguments.context is not None else None try: if hasattr(callable_, 'execute'): inst = callable_(*args, **kwargs) if context is not None: inst.context = context result = inst.execute() if hasattr(inst, 'post_execute'): inst.post_execute() else: if context is not None: callable_.context = context result = callable_(*args, **kwargs) except Exception as err: logger.error('Exception: {}'.format(err)) exc_type, exc_value, exc_traceback = sys.exc_info() tb = traceback.format_tb(exc_traceback) details = json_dumps( { 'error': exc_type.__name__, 'message': str(exc_value), 'traceback': tb, }, default=repr, ) if cmd_arguments.error_fd == 2: sys.stderr.flush() if not compat.PY2: details = details.encode('utf-8') os.write(cmd_arguments.error_fd, details) if cmd_arguments.kill_children: kill_child_processes() sys.exit(1) if cmd_arguments.result_fd == 1: # stdout (legacy) sys.stdout.flush() # may have print's in flight os.write(cmd_arguments.result_fd, b'\n') result = json_dumps(result) if not compat.PY2: result = result.encode('utf-8') os.write(cmd_arguments.result_fd, result) if cmd_arguments.kill_children: kill_child_processes()
def execute(*args, **kwargs): logger = logging.getLogger(logger_name) command = 'simpleflow.execute' # name of a module. sys.stdout.flush() sys.stderr.flush() result_str = None # useless context = kwargs.pop('context', {}) with tempfile.TemporaryFile() as result_fd, tempfile.TemporaryFile( ) as error_fd: dup_result_fd = os.dup(result_fd.fileno()) # remove FD_CLOEXEC dup_error_fd = os.dup(error_fd.fileno()) # remove FD_CLOEXEC arguments_json = format_arguments_json(*args, **kwargs) full_command = [ interpreter, '-m', command, # execute module a script. get_name(func), '--logger-name={}'.format(logger_name), '--result-fd={}'.format(dup_result_fd), '--error-fd={}'.format(dup_error_fd), '--context={}'.format(json_dumps(context)), ] if len( arguments_json ) < MAX_ARGUMENTS_JSON_LENGTH: # command-line limit on Linux: 128K full_command.append(arguments_json) arg_file = None arg_fd = None else: arg_file = tempfile.TemporaryFile() arg_file.write(arguments_json.encode('utf-8')) arg_file.flush() arg_file.seek(0) arg_fd = os.dup(arg_file.fileno()) full_command.append( '--arguments-json-fd={}'.format(arg_fd)) full_command.append('foo') # dummy funcarg if kill_children: full_command.append('--kill-children') if is_buggy_subprocess32( ): # close_fds doesn't work with subprocess32 < 3.5.0 close_fds = False pass_fds = [] else: close_fds = True pass_fds = [dup_result_fd, dup_error_fd] if arg_file: pass_fds.append(arg_fd) process = subprocess.Popen( full_command, bufsize=-1, close_fds=close_fds, pass_fds=pass_fds, ) rc = wait_subprocess(process, timeout=timeout, command_info=full_command) os.close(dup_result_fd) os.close(dup_error_fd) if arg_file: arg_file.close() if rc: error_fd.seek(0) err_output = error_fd.read() if err_output: if not compat.PY2: err_output = err_output.decode('utf-8', errors='replace') raise ExecutionError(err_output) result_fd.seek(0) result_str = result_fd.read() if not result_str: return None try: if not compat.PY2: result_str = result_str.decode('utf-8', errors='replace') result = format.decode(result_str) return result except BaseException as ex: logger.exception('Exception in python.execute: {} {}'.format( ex.__class__.__name__, ex)) logger.warning('%r', result_str)
def main(): """ When executed as a script, this module expects the name of a callable as its first argument and the arguments of the callable encoded in a JSON string as its second argument. It then executes the callable with the arguments after decoding them into Python objects. It finally encodes the value returned by the callable into a JSON string and prints it on stdout. the arguments of the callable are stored in a dict with the following format: :: {'args': [...], 'kwargs': { ..., } } Synopsis -------- :: usage: execute.py [-h] funcname funcargs positional arguments: funcname name of the callable to execute funcargs callable arguments in JSON optional arguments: -h, --help show this help message and exit Examples -------- :: $ python -m simpleflow.execute "os.path.exists" '{"args": ["/tmp"]}' true """ import argparse parser = argparse.ArgumentParser() parser.add_argument( 'funcname', help='name of the callable to execute', ) parser.add_argument( 'funcargs', help='callable arguments in JSON', ) parser.add_argument( '--context', help='Activity Context', ) parser.add_argument( '--logger-name', help='logger name', ) parser.add_argument( '--result-fd', type=int, default=1, metavar='N', help='result file descriptor', ) parser.add_argument( '--error-fd', type=int, default=2, metavar='N', help='error file descriptor', ) parser.add_argument( '--arguments-json-fd', type=int, default=None, metavar='N', help='JSON input file descriptor', ) parser.add_argument( '--kill-children', action='store_true', help='kill child processes on exit', ) cmd_arguments = parser.parse_args() def kill_child_processes(): process = psutil.Process(os.getpid()) children = process.children(recursive=True) for child in children: try: child.terminate() except psutil.NoSuchProcess: pass _, still_alive = psutil.wait_procs(children, timeout=0.3) for child in still_alive: try: child.kill() except psutil.NoSuchProcess: pass funcname = cmd_arguments.funcname if cmd_arguments.arguments_json_fd is None: content = cmd_arguments.funcargs if content is None: parser.error('the following arguments are required: funcargs') else: with os.fdopen(cmd_arguments.arguments_json_fd) as arguments_json_file: content = arguments_json_file.read() try: arguments = format.decode(content) except Exception: raise ValueError('cannot load arguments from {}'.format(content)) if cmd_arguments.logger_name: logger = logging.getLogger(cmd_arguments.logger_name) else: logger = simpleflow_logger callable_ = make_callable(funcname) if hasattr(callable_, '__wrapped__'): callable_ = callable_.__wrapped__ args = arguments.get('args', ()) kwargs = arguments.get('kwargs', {}) context = json.loads( cmd_arguments.context) if cmd_arguments.context is not None else None try: if hasattr(callable_, 'execute'): inst = callable_(*args, **kwargs) if context is not None: inst.context = context result = inst.execute() if hasattr(inst, 'post_execute'): inst.post_execute() else: if context is not None: callable_.context = context result = callable_(*args, **kwargs) except Exception as err: logger.error('Exception: {}'.format(err)) exc_type, exc_value, exc_traceback = sys.exc_info() tb = traceback.format_tb(exc_traceback) details = json_dumps( { 'error': exc_type.__name__, 'message': str(exc_value), 'traceback': tb, }, default=repr, ) if cmd_arguments.error_fd == 2: sys.stderr.flush() if not compat.PY2: details = details.encode('utf-8') os.write(cmd_arguments.error_fd, details) if cmd_arguments.kill_children: kill_child_processes() sys.exit(1) if cmd_arguments.result_fd == 1: # stdout (legacy) sys.stdout.flush() # may have print's in flight os.write(cmd_arguments.result_fd, b'\n') result = json_dumps(result) if not compat.PY2: result = result.encode('utf-8') os.write(cmd_arguments.result_fd, result) if cmd_arguments.kill_children: kill_child_processes()
def load_input(input_fp): if input_fp is None: input_fp = sys.stdin input = format.decode(input_fp) return transform_input(input)
def execute(*args, **kwargs): logger = logging.getLogger(logger_name) command = 'simpleflow.execute' # name of a module. sys.stdout.flush() sys.stderr.flush() result_str = None # useless context = kwargs.pop('context', {}) with tempfile.TemporaryFile() as result_fd, tempfile.TemporaryFile() as error_fd: dup_result_fd = os.dup(result_fd.fileno()) # remove FD_CLOEXEC dup_error_fd = os.dup(error_fd.fileno()) # remove FD_CLOEXEC arguments_json = format_arguments_json(*args, **kwargs) full_command = [ interpreter, '-m', command, # execute module a script. get_name(func), '--logger-name={}'.format(logger_name), '--result-fd={}'.format(dup_result_fd), '--error-fd={}'.format(dup_error_fd), '--context={}'.format(json_dumps(context)), ] if len(arguments_json) < MAX_ARGUMENTS_JSON_LENGTH: # command-line limit on Linux: 128K full_command.append(arguments_json) arg_file = None arg_fd = None else: arg_file = tempfile.TemporaryFile() arg_file.write(arguments_json.encode('utf-8')) arg_file.flush() arg_file.seek(0) arg_fd = os.dup(arg_file.fileno()) full_command.append('--arguments-json-fd={}'.format(arg_fd)) full_command.append('foo') # dummy funcarg if kill_children: full_command.append('--kill-children') if is_buggy_subprocess32(): # close_fds doesn't work with subprocess32 < 3.5.0 close_fds = False pass_fds = [] else: close_fds = True pass_fds = [dup_result_fd, dup_error_fd] if arg_file: pass_fds.append(arg_fd) process = subprocess.Popen( full_command, bufsize=-1, close_fds=close_fds, pass_fds=pass_fds, ) rc = wait_subprocess(process, timeout=timeout, command_info=full_command) os.close(dup_result_fd) os.close(dup_error_fd) if arg_file: arg_file.close() if rc: error_fd.seek(0) err_output = error_fd.read() if err_output: if not compat.PY2: err_output = err_output.decode('utf-8', errors='replace') raise ExecutionError(err_output) result_fd.seek(0) result_str = result_fd.read() if not result_str: return None try: if not compat.PY2: result_str = result_str.decode('utf-8', errors='replace') result = format.decode(result_str) return result except BaseException as ex: logger.exception('Exception in python.execute: {} {}'.format(ex.__class__.__name__, ex)) logger.warning('%r', result_str)