def call_pyinstaller(clazz, args, build_dir = None, replace_env = None): check.check_string_seq(args) check.check_string(build_dir, allow_none = True) check.check_dict(replace_env, check.STRING_TYPES, check.STRING_TYPES) cmd = command_line.parse_args(args) replace_env = replace_env or {} env = os_env.clone_current_env(d = {}) env.update(replace_env) clazz._log.log_d('using env: {}'.format(pprint.pformat(env))) clazz._log.log_d('calling pyinstaller: {}'.format(' '.join(cmd))) if build_dir: file_util.mkdir(build_dir) dist_dir = path.join(build_dir, 'dist') work_dir = path.join(build_dir, 'work') spec_dir = path.join(build_dir, 'spec') args = args[:] args.extend([ '--distpath', dist_dir ]) args.extend([ '--workpath', work_dir ]) args.extend([ '--specpath', spec_dir ]) try: with env_override(env = env) as _: PyInstaller_run(pyi_args = args, pyi_config = None) except Exception as ex: raise pyinstaller_error(str(ex))
def call_handle(clazz, args, raise_error=True): command_line.check_args_type(args) check.check_bool(raise_error) if isinstance(args, (list, tuple)): parsed_args = list(args) else: parsed_args = command_line.parse_args(args) handle_exe = clazz._find_handle_exe() cmd = [handle_exe, '-nobanner'] + args return execute.execute(cmd, raise_error=raise_error)
def _call_ssh_keyscan(clazz, args, cwd=None): exe = which.which('ssh-keyscan') if not exe: raise ssh_config_error('ssh-keyscan executable not found') cmd = [exe] + command_line.parse_args(args) try: rv = execute.execute(cmd, env=os_env.make_clean_env(), cwd=cwd, stderr_to_stdout=True) return rv.stdout.strip() except Exception as ex: raise ssh_config_error('Failed to run: "{}" - '.format( ' '.join(cmd), str(ex)))
def call_docker(clazz, args, cwd=None, non_blocking=True, shell=False): exe = which.which('docker') if not exe: raise docker_error('docker not found') cmd = [exe] + command_line.parse_args(args) env = os_env.clone_current_env(d={}) rv = execute.execute(cmd, env=env, shell=shell, cwd=cwd, stderr_to_stdout=True, non_blocking=non_blocking, raise_error=False) if rv.exit_code != 0: cmd_flag = ' '.join(cmd) msg = 'docker command failed: {}\n{}'.format(cmd_flag, rv.stdout) raise docker_error(msg) return rv
def call_program(self, args, **kargs): 'Call a program with the right environment' command_line.check_args_type(args) kargs = copy.deepcopy(kargs) self._log.log_method_d() self._log.log_d('call_program: args={}'.format(args)) parsed_args = command_line.parse_args(args) self._log.log_d('call_program: parsed_args={}'.format(parsed_args)) env = os_env.clone_current_env() env.update(self.env) PATH = env_var(env, 'PATH') PYTHONPATH = env_var(env, 'PYTHONPATH') if 'env' in kargs: kargs_env = kargs['env'] del kargs['env'] if 'PATH' in kargs_env: PATH.append(kargs_env['PATH']) del kargs_env['PATH'] if 'PYTHONPATH' in kargs_env: PYTHONPATH.append(kargs_env['PYTHONPATH']) del kargs_env['PYTHONPATH'] env.update(kargs_env) PATH.prepend(self.PATH) PYTHONPATH.prepend(self.PYTHONPATH) kargs['env'] = env self._log.log_d('call_program: env={}'.format(env)) kargs['shell'] = True kargs['check_python_script'] = False for key, value in sorted(env.items()): self._log.log_d('call_program({}): ENV: {}={}'.format(args[0], key, value)) return execute.execute(parsed_args, **kargs)
def call_git(clazz, root, args, raise_error=True, extra_env=None, num_tries=None, retry_wait_seconds=None): check.check_string(root) command_line.check_args_type(args) check.check_bool(raise_error) check.check_dict(extra_env, check.STRING_TYPES, check.STRING_TYPES, allow_none=True) check.check_int(num_tries, allow_none=True) check.check_float(retry_wait_seconds, allow_none=True) #print('args={}'.format(args)) if isinstance(args, (list, tuple)): parsed_args = list(args) else: parsed_args = command_line.parse_args(args) num_tries = num_tries if num_tries != None else clazz._DEFAULT_NUM_TRIES retry_wait_seconds = retry_wait_seconds if retry_wait_seconds != None else clazz._DEFAULT_RETRY_WAIT_SECONDS if num_tries < 1 or num_tries >= clazz._MAX_NUM_TRIES: raise git_error( 'num_tries should be between 1 and {} instead of "{}"'.format( clazz._DEFAULT_NUM_TRIES, num_tries)) if retry_wait_seconds < clazz._MIN_RETRY_WAIT_SECONDS or retry_wait_seconds >= clazz._MAX_RETRY_WAIT_SECONDS: raise git_error( 'retry_wait_seconds should be between {} and {} instead of "{}"' .format(clazz._MIN_RETRY_WAIT_SECONDS, clazz._MAX_RETRY_WAIT_SECONDS, retry_wait_seconds)) if not hasattr(clazz, '_git_exe'): git_exe = clazz.find_git_exe() if not git_exe: raise git_error('git exe not found in: {}'.format(' '.join( os.environ['PATH'].split(os.pathsep)))) setattr(clazz, '_git_exe', git_exe) git_exe = getattr(clazz, '_git_exe') cmd = [git_exe] + parsed_args clazz.log.log_d('root=%s; cmd=%s' % (root, ' '.join(cmd))) extra_env = extra_env or {} env = os_env.clone_current_env(d=extra_env, prepend=True) last_try_exception = None num_failed_attempts = 0 rv = None for i in range(0, num_tries): try: clazz.log.log_d('call_git: attempt {} of {}: {}'.format( i + 1, num_tries, ' '.join(cmd))) rv = execute.execute(cmd, cwd=root, raise_error=False, env=env) if rv.exit_code == 0: clazz.log.log_i('call_git: success {} of {}: {}'.format( i + 1, num_tries, ' '.join(cmd))) break else: clazz.log.log_i('call_git: failed {} of {}: {}'.format( i + 1, num_tries, ' '.join(cmd))) except Exception as ex: num_failed_attempts += 1 clazz.log.log_w('call_git: failed {} of {}: {}'.format( i + 1, num_tries, ' '.join(cmd))) clazz.log.log_d('call_git: exception: {}'.format(str(ex))) clazz.log.log_d( 'call_git: sleeping {} seconds'.format(retry_wait_seconds)) time.sleep(retry_wait_seconds) last_try_exception = ex if not rv: message = 'git command attempt failed {} times: {} in {}'.format( num_tries, ' '.join(cmd), root) clazz.log.log_w('call_git: {}'.format(message)) raise git_error(message, execute_result=None) # first handle the retry failure if num_tries > 1 and num_failed_attempts == num_tries and last_try_exception: message = 'git command attempt failed {} times: {} in {}\n{}\n{}\n{}'.format( num_tries, ' '.join(cmd), root, str(last_try_exception), rv.stderr, rv.stdout) clazz.log.log_w('call_git: {}'.format(message)) raise git_error(message, execute_result=rv) # handle raise_error if needed if rv.exit_code != 0 and raise_error: message = 'git command failed: {} in {}\n{}\n{}\n{}'.format( ' '.join(cmd), root, str(last_try_exception), rv.stderr, rv.stdout) clazz.log.log_w('call_git: {}'.format(message)) raise git_error(message, execute_result=rv) #print('rv={}'.format(rv)) return rv
def vm_run_package(self, vm_id, package_dir, entry_command, entry_command_args, run_program_options): check.check_string(vm_id) check.check_string(package_dir) check.check_string(entry_command) check.check_string_seq(entry_command_args) check.check_vmware_run_program_options(run_program_options) self._log.log_method_d() vm = self._resolve_vmx_to_local_vm(vm_id) if not path.isdir(package_dir): raise vmware_error( 'package_dir not found or not a dir: "{}"'.format(package_dir)) tmp_dir_local = temp_file.make_temp_dir(suffix='-run_package.dir', delete=not self._options.debug) if self._options.debug: print('tmp_dir_local={}'.format(tmp_dir_local)) tmp_remote_dir = path.join('/tmp', path.basename(tmp_dir_local)) self._log.log_d( 'vm_run_package: tmp_remote_dir={}'.format(tmp_remote_dir)) tmp_package = self._make_tmp_file_pair(tmp_dir_local, 'package.tar.gz') tmp_caller_script = self._make_tmp_file_pair(tmp_dir_local, 'caller_script.py') tmp_output_log = self._make_tmp_file_pair(tmp_dir_local, 'output.log') archiver.create(tmp_package.local, package_dir) file_util.save(tmp_caller_script.local, content=vmware_guest_scripts.RUN_PACKAGE_CALLER, mode=0o0755) self._log.log_d('vm_run_package: tmp_package={}'.format(tmp_package)) self._log.log_d( 'vm_run_package: tmp_caller_script={}'.format(tmp_caller_script)) self._log.log_d( 'vm_run_package: tmp_output_log={}'.format(tmp_output_log)) parsed_entry_command_args = command_line.parse_args(entry_command_args) debug_args = [] if self._options.debug: debug_args.append('--debug') if self._options.tty: debug_args.extend(['--tty', tty]) caller_args = debug_args + [ tmp_package.remote, entry_command, tmp_output_log.remote, ] + parsed_entry_command_args with vmware_restore_vm_running_state(self) as _: with vmware_run_operation(vm, run_program_options) as target_vm: target_vm.dir_create(tmp_remote_dir) target_vm.file_copy_to(tmp_package.local, tmp_package.remote) target_vm.file_copy_to(tmp_caller_script.local, tmp_caller_script.remote) rv = target_vm.run_program( tmp_caller_script.remote, caller_args, run_program_options=run_program_options) target_vm.file_copy_from(tmp_output_log.remote, tmp_output_log.local) target_vm.dir_delete(tmp_remote_dir) with file_util.open_with_default( filename=run_program_options.output_filename) as f: log_content = file_util.read(tmp_output_log.local, codec='utf-8') f.write(log_content) f.flush() return rv
def _parse_script(clazz, cmd): args = command_line.parse_args(cmd) return git_util.script(args[0], args[1:])
def run(clazz, image_id, command, run_files=None, run_files_substitutions=None, run_label=None, volumes=None, env=None, name=None, restart=False, detach=False, tty=False, interactive=False, expose=None, debug=False, non_blocking=True, remove=True, tmp_dir=None): check.check_string(image_id) check.check_string(command) check.check_string_seq(run_files, allow_none=True) check.check_dict(run_files_substitutions, check.STRING_TYPES, check.STRING_TYPES, allow_none=True) check.check_string(run_label, allow_none=True) check.check_string(name, allow_none=True) check.check_dict(env, check.STRING_TYPES, check.STRING_TYPES, allow_none=True) check.check_dict(volumes, check.STRING_TYPES, check.STRING_TYPES, allow_none=True) check.check_bool(restart) check.check_bool(detach) check.check_bool(tty) check.check_bool(interactive) check.check_int(expose, allow_none=True) check.check_bool(debug) check.check_bool(non_blocking) check.check_bool(remove) check.check_string(tmp_dir, allow_none=True) cli = command_line.parse_args(command) env = env or {} volumes = volumes or {} command_args = cli[1:] run_files = run_files or [] run_files_substitutions = run_files_substitutions or {} run_label = run_label or 'docker.run' tmp_dir = tmp_dir or os.getcwd() tmp_run_dir = temp_file.make_temp_dir(suffix='-{}'.format(run_label), dir=tmp_dir, delete=not debug) input_dir = path.join(tmp_run_dir, 'input') output_dir = path.join(tmp_run_dir, 'output') for cf in run_files: src_file = path.join(os.getcwd(), cf) dst_file = path.join(input_dir, cf) if not path.isfile(src_file): raise docker_error('run file not found: "{}"'.format(src_file)) if run_files_substitutions and file_mime.is_text(src_file): file_replace.copy_with_substitute(src_file, dst_file, run_files_substitutions, backup=False) else: file_util.copy(src_file, dst_file) docker_run_args = ['run'] if detach: docker_run_args.append('--detach') if tty: docker_run_args.append('--tty') if interactive: docker_run_args.append('--interactive') if remove: docker_run_args.append('--rm') docker_run_args.extend(clazz._make_env_args(env)) volumes[input_dir] = '/input' volumes[output_dir] = '/output' docker_run_args.extend(clazz._make_volume_args(volumes)) if expose: docker_run_args.extend(['--expose', str(expose)]) if restart: docker_run_args.extend(['--restart', restart]) if name: docker_run_args.extend(['--name', name]) docker_run_args.append(image_id) docker_run_args.append(cli[0]) docker_run_args.extend(cli[1:]) clazz.log.log_d('running docker: {}'.format(' '.join(docker_run_args))) rv = docker_exe.call_docker(docker_run_args, non_blocking=non_blocking) container_id = docker_container.last_container() return clazz._run_result(container_id, rv.exit_code, rv.stdout, input_dir, output_dir)