def test_traceback(mode, entry_point): tb_tool = yc.build_path('library/python/runtime/test/traceback/traceback') stdout_path = yc.test_output_path('stdout_raw.txt') stderr_path = yc.test_output_path('stderr_raw.txt') filtered_stdout_path = yc.test_output_path('stdout.txt') filtered_stderr_path = yc.test_output_path('stderr.txt') env = os.environ.copy() if entry_point == 'custom': env['Y_PYTHON_ENTRY_POINT'] = 'library.python.runtime.test.traceback.crash:main' proc = yc.execute( command=[tb_tool, mode], env=env, stdout=stdout_path, stderr=stderr_path, check_exit_code=False, ) with open(filtered_stdout_path, 'w') as f: f.write(clean_traceback(proc.std_out)) with open(filtered_stderr_path, 'w') as f: f.write(clean_traceback(proc.std_err)) return { 'stdout': yc.canonical_file( filtered_stdout_path, local=True, ), 'stderr': yc.canonical_file( filtered_stderr_path, local=True, ), }
def get_test_output_path(pattern): assert '{}' in pattern result = yc.test_output_path(pattern.format('')) attempt = 0 while os.path.exists(result): result = yc.test_output_path(pattern.format(attempt)) attempt += 1 return result
def execute( command, check_exit_code=True, shell=False, timeout=None, cwd=None, env=None, stdin=None, stdout=None, stderr=None, creationflags=0, wait=True, process_progress_listener=None, close_fds=False, collect_cores=True, check_sanitizer=True, preexec_fn=None, on_timeout=None, # YT specific input_data=None, output_data=None, data_mine_strategy=None, env_mine_strategy=None, operation_spec=None, task_spec=None, yt_proxy=None, output_result_path=None, init_func=None, fini_func=None, # Service args spec_filename=None, test_tool_bin=None, executor=_YtExecutor, runner_log_path=None, runner_log2stderr=False, runner_meta_path=None, target_stdout_path=None, target_stderr_path=None, operation_log_path=None, operation_description=None, ): """ Executes a command on the YT. Listed below are options whose behavior is different from yatest.common.execute :param command: can be a list of arguments or a string (all paths matched prefixes yatest.common.*_path will be fixed) :param timeout: timeout for command executed on the YT (doesn't take into account the time spent for execution preparation - uploading/downloading data, etc) :param cwd: ignored :param env: all paths matched prefixes yatest.common.*_path will be fixed :param stdin: stdin will be fully read before execution and uploaded to the YT :param stdout: stdout will be available after the execution of the command on the YT. Set to False to skip downloading :param stderr: same as stdout :param process_progress_listener: ignored :param preexec_fn: ignored :param on_timeout: ignored :param input_data: map of input files/dirs required for command run which will be uploaded to YT (local path -> YT sandbox path) :param output_data: map of output files/dirs which will be downloaded from YT after command execution (YT sandbox path -> local path) Take into account that runner will call os.path.dirname(YT sandbox path) to create intermediate directories for every entry :param data_mine_strategy: allows to provide own function to mine input data and fix cmd. For more info take a look at *_mine_strategy() :param env_mine_strategy: allows to provide own function to mine input data and fix env. For more info take a look at *_mine_strategy() :param operation_spec: YT operation spec :param task_spec: YT task spec :param output_result_path: specify path to output archive. Used for test purposes :param init_func: Function which will be executed before target program. See note below :param fini_func: Function which will be executed after target program. See note below :return: Execution object .. note:: init_func and fini_func have some limitations: * every used module should be imported inside this functions, because functions will be called in a different environment and required modules may be not imported or available * you can only use built-in python modules (because test_tool uploads itself and runs init/fini func inside itself) """ test_tool_bin = test_tool_bin or _get_test_tool_bin() data_mine_strategy = data_mine_strategy or default_mine_strategy env_mine_strategy = env_mine_strategy or default_env_mine_strategy if not wait: raise NotImplementedError() orig_command = command command, env, to_upload, to_download = _fix_user_data(command, shell, env, input_data, output_data, data_mine_strategy, env_mine_strategy) command_name = ytc.process.get_command_name(command) exec_spec = { 'env': env, 'command': command, 'timeout': timeout, 'input_data': to_upload, 'output_data': to_download, 'description': operation_description, } if stdin: if isinstance(stdin, basestring): stdin_path = stdin else: logger.deubg('Reading stdin') with tempfile.NamedTemporaryFile(delete=False) as afile: afile.write(stdin.read()) stdin_path = afile.name to_upload[stdin_path] = get_yt_sandbox_path('env/stdin') exec_spec['stdin'] = get_yt_sandbox_path('env/stdin') for stream, name, filename in [ (True, 'meta', runner_meta_path), (stdout, 'stdout', target_stdout_path), (stderr, 'stderr', target_stderr_path), ]: if stream is not False: path = get_yt_sandbox_path("env/{}".format(name)) exec_spec[name] = path to_download[path] = filename or ytc.get_unique_file_path(ytc.work_path(), 'yt_vanilla_{}_{}'.format(command_name, name)) runner_log_dst = get_yt_sandbox_path('env/runner_log') exec_spec['runner_log'] = runner_log_dst to_download[runner_log_dst] = runner_log_path or ytc.path.get_unique_file_path(ytc.test_output_path(), 'yt_vanilla_wrapper_{}.log'.format(command_name)) exec_spec['op_spec'] = _get_spec( default={ 'max_failed_job_count': 2, # Preventing dangling operations in case when test is get killed - see https://st.yandex-team.ru/DEVTOOLS-4753#1539181402000 'time_limit': int(1000 * 60 * 60 * 1.5) # 1.5h (milliseconds) }, user=operation_spec, ) exec_spec['task_spec'] = _get_spec( default={'memory_limit': 3 * (1024 ** 3)}, user=task_spec, mandatory={'job_count': 1}, ) if init_func: exec_spec['init_func'] = _dump_func(init_func) if fini_func: exec_spec['fini_func'] = _dump_func(fini_func) exec_spec_path = _dump_spec(spec_filename, exec_spec) executor_cmd = [ test_tool_bin, 'yt_vanilla_execute', '--spec-file', exec_spec_path, '--log-path', operation_log_path or ytc.path.get_unique_file_path(ytc.test_output_path(), 'yt_vanilla_op_{}.log'.format(command_name)), ] if yt_proxy: executor_cmd += ['--yt-proxy', yt_proxy] if output_result_path: executor_cmd += ['--output-path', output_result_path] if runner_log2stderr: executor_cmd += ['--log2stderr'] executor_stderr = sys.stderr else: executor_stderr = None res = ytc.execute( executor_cmd, stderr=executor_stderr, collect_cores=collect_cores, wait=False, check_sanitizer=check_sanitizer, executor=executor, ) if wait: res.wait(exec_spec, orig_command, stdout, stderr, check_exit_code, timeout) return res
def execute( command, check_exit_code=True, shell=False, timeout=None, cwd=None, env=None, stdin=None, stdout=None, stderr=None, creationflags=0, wait=True, process_progress_listener=None, close_fds=False, collect_cores=True, check_sanitizer=True, preexec_fn=None, on_timeout=None, # YT specific input_data=None, output_data=None, data_mine_strategy=None, operation_spec=None, task_spec=None, yt_proxy=None, output_result_path=None, ): """ Executes a command on the YT. Listed below are options whose behavior is different from yatest.common.execute :param command: can be a list of arguments or a string (all paths matched prefixes yatest.common.*_path will be fixed) :param timeout: timeout for command executed on the YT (doesn't take into account the time spent for execution preparation - uploading/downloading data, etc) :param cwd: ignored :param env: all paths matched prefixes yatest.common.*_path will be fixed :param stdin: stdin will be fully read before execution and uploaded to the YT :param stdout: stdout will be available after the execution of the command on the YT. Set to False to skip downloading :param stderr: same as stdout :param process_progress_listener: ignored :param preexec_fn: ignored :param on_timeout: ignored :param input_data: map of input files/dirs required for command run which will be uploaded to YT (local path -> YT sandbox path) :param output_data: map of output files/dirs which will be downloaded from YT after command execution (YT sandbox path -> local path) Take into account that runner will call os.path.dirname(YT sandbox path) to create intermediate directories for every entry :param data_mine_strategy: allows to provide own function to mine input data and fix cmd. For more info take a look at *_mine_strategy() :param operation_spec: YT operation spec :param task_spec: YT task spec :param output_result_path: specify path to output archive. Used for test purposes :return: Execution object """ test_tool_bin = _get_test_tool_bin() data_mine_strategy = data_mine_strategy or default_mine_strategy if not wait: raise NotImplementedError() env = _get_fixed_env(env, data_mine_strategy) orig_command = command command, to_upload, to_download = _fix_user_data(command, shell, input_data, output_data, data_mine_strategy) command_name = ytc.process.get_command_name(command) exec_spec = { 'env': env, 'command': command, 'timeout': timeout, 'input_data': to_upload, 'output_data': to_download, } if stdin: if isinstance(stdin, basestring): stdin_path = stdin else: logger.deubg('Reading stdin') with tempfile.NamedTemporaryFile(delete=False) as afile: afile.write(stdin.read()) stdin_path = afile.name to_upload[stdin_path] = 'env/stdin' exec_spec['stdin'] = 'env/stdin' for stream, name in [ (True, 'meta'), (stdout, 'stdout'), (stderr, 'stderr'), ]: if stream is not False: path = 'env/{}'.format(name) exec_spec[name] = path to_download[path] = ytc.get_unique_file_path( ytc.work_path(), 'yt_vanilla_{}_{}'.format(command_name, name)) runner_log_dst = 'env/runner_log' exec_spec['runner_log'] = runner_log_dst to_download[runner_log_dst] = ytc.path.get_unique_file_path( ytc.test_output_path(), 'yt_vanilla_wrapper_{}.log'.format(command_name)) exec_spec['op_spec'] = _get_spec( default={'max_failed_job_count': 2}, user=operation_spec, ) exec_spec['task_spec'] = _get_spec( default={'memory_limit': 3 * (1024**3)}, user=task_spec, mandatory={'job_count': 1}, ) exec_spec_path = _dump_spec(exec_spec) executor_cmd = [ test_tool_bin, 'yt_vanilla_execute', '--spec-file', exec_spec_path, '--log-path', ytc.path.get_unique_file_path( ytc.test_output_path(), 'yt_vanilla_op_{}.log'.format(command_name)), ] if yt_proxy: executor_cmd += ['--yt-proxy', yt_proxy] if output_result_path: executor_cmd += ['--output-path', output_result_path] res = ytc.execute(executor_cmd, collect_cores=collect_cores, wait=False, check_sanitizer=check_sanitizer) if wait: res.wait(check_exit_code=True) _patch_result(res, exec_spec, orig_command, stdout, stderr, check_exit_code, timeout) return res