def call_pyinstaller(clazz, args, build_dir = None, replace_env = None):
check.check_string_seq(args)
check.check_string(build_dir, allow_none = True)
check.check_dict(replace_env, check.STRING_TYPES, check.STRING_TYPES)
cmd = command_line.parse_args(args)
replace_env = replace_env or {}
env = os_env.clone_current_env(d = {})
env.update(replace_env)
clazz._log.log_d('using env: {}'.format(pprint.pformat(env)))
clazz._log.log_d('calling pyinstaller: {}'.format(' '.join(cmd)))
if build_dir:
file_util.mkdir(build_dir)
dist_dir = path.join(build_dir, 'dist')
work_dir = path.join(build_dir, 'work')
spec_dir = path.join(build_dir, 'spec')
args = args[:]
args.extend([ '--distpath', dist_dir ])
args.extend([ '--workpath', work_dir ])
args.extend([ '--specpath', spec_dir ])
try:
with env_override(env = env) as _:
PyInstaller_run(pyi_args = args, pyi_config = None)
except Exception as ex:
raise pyinstaller_error(str(ex))
def call_handle(clazz, args, raise_error=True):
command_line.check_args_type(args)
check.check_bool(raise_error)
if isinstance(args, (list, tuple)):
parsed_args = list(args)
else:
parsed_args = command_line.parse_args(args)
handle_exe = clazz._find_handle_exe()
cmd = [handle_exe, '-nobanner'] + args
return execute.execute(cmd, raise_error=raise_error)
def _call_ssh_keyscan(clazz, args, cwd=None):
exe = which.which('ssh-keyscan')
if not exe:
raise ssh_config_error('ssh-keyscan executable not found')
cmd = [exe] + command_line.parse_args(args)
try:
rv = execute.execute(cmd,
env=os_env.make_clean_env(),
cwd=cwd,
stderr_to_stdout=True)
return rv.stdout.strip()
except Exception as ex:
raise ssh_config_error('Failed to run: "{}" - '.format(
' '.join(cmd), str(ex)))
def call_docker(clazz, args, cwd=None, non_blocking=True, shell=False):
exe = which.which('docker')
if not exe:
raise docker_error('docker not found')
cmd = [exe] + command_line.parse_args(args)
env = os_env.clone_current_env(d={})
rv = execute.execute(cmd,
env=env,
shell=shell,
cwd=cwd,
stderr_to_stdout=True,
non_blocking=non_blocking,
raise_error=False)
if rv.exit_code != 0:
cmd_flag = ' '.join(cmd)
msg = 'docker command failed: {}\n{}'.format(cmd_flag, rv.stdout)
raise docker_error(msg)
return rv
def call_program(self, args, **kargs):
'Call a program with the right environment'
command_line.check_args_type(args)
kargs = copy.deepcopy(kargs)
self._log.log_method_d()
self._log.log_d('call_program: args={}'.format(args))
parsed_args = command_line.parse_args(args)
self._log.log_d('call_program: parsed_args={}'.format(parsed_args))
env = os_env.clone_current_env()
env.update(self.env)
PATH = env_var(env, 'PATH')
PYTHONPATH = env_var(env, 'PYTHONPATH')
if 'env' in kargs:
kargs_env = kargs['env']
del kargs['env']
if 'PATH' in kargs_env:
PATH.append(kargs_env['PATH'])
del kargs_env['PATH']
if 'PYTHONPATH' in kargs_env:
PYTHONPATH.append(kargs_env['PYTHONPATH'])
del kargs_env['PYTHONPATH']
env.update(kargs_env)
PATH.prepend(self.PATH)
PYTHONPATH.prepend(self.PYTHONPATH)
kargs['env'] = env
self._log.log_d('call_program: env={}'.format(env))
kargs['shell'] = True
kargs['check_python_script'] = False
for key, value in sorted(env.items()):
self._log.log_d('call_program({}): ENV: {}={}'.format(args[0], key, value))
return execute.execute(parsed_args, **kargs)
def call_git(clazz,
root,
args,
raise_error=True,
extra_env=None,
num_tries=None,
retry_wait_seconds=None):
check.check_string(root)
command_line.check_args_type(args)
check.check_bool(raise_error)
check.check_dict(extra_env,
check.STRING_TYPES,
check.STRING_TYPES,
allow_none=True)
check.check_int(num_tries, allow_none=True)
check.check_float(retry_wait_seconds, allow_none=True)
#print('args={}'.format(args))
if isinstance(args, (list, tuple)):
parsed_args = list(args)
else:
parsed_args = command_line.parse_args(args)
num_tries = num_tries if num_tries != None else clazz._DEFAULT_NUM_TRIES
retry_wait_seconds = retry_wait_seconds if retry_wait_seconds != None else clazz._DEFAULT_RETRY_WAIT_SECONDS
if num_tries < 1 or num_tries >= clazz._MAX_NUM_TRIES:
raise git_error(
'num_tries should be between 1 and {} instead of "{}"'.format(
clazz._DEFAULT_NUM_TRIES, num_tries))
if retry_wait_seconds < clazz._MIN_RETRY_WAIT_SECONDS or retry_wait_seconds >= clazz._MAX_RETRY_WAIT_SECONDS:
raise git_error(
'retry_wait_seconds should be between {} and {} instead of "{}"'
.format(clazz._MIN_RETRY_WAIT_SECONDS,
clazz._MAX_RETRY_WAIT_SECONDS, retry_wait_seconds))
if not hasattr(clazz, '_git_exe'):
git_exe = clazz.find_git_exe()
if not git_exe:
raise git_error('git exe not found in: {}'.format(' '.join(
os.environ['PATH'].split(os.pathsep))))
setattr(clazz, '_git_exe', git_exe)
git_exe = getattr(clazz, '_git_exe')
cmd = [git_exe] + parsed_args
clazz.log.log_d('root=%s; cmd=%s' % (root, ' '.join(cmd)))
extra_env = extra_env or {}
env = os_env.clone_current_env(d=extra_env, prepend=True)
last_try_exception = None
num_failed_attempts = 0
rv = None
for i in range(0, num_tries):
try:
clazz.log.log_d('call_git: attempt {} of {}: {}'.format(
i + 1, num_tries, ' '.join(cmd)))
rv = execute.execute(cmd, cwd=root, raise_error=False, env=env)
if rv.exit_code == 0:
clazz.log.log_i('call_git: success {} of {}: {}'.format(
i + 1, num_tries, ' '.join(cmd)))
break
else:
clazz.log.log_i('call_git: failed {} of {}: {}'.format(
i + 1, num_tries, ' '.join(cmd)))
except Exception as ex:
num_failed_attempts += 1
clazz.log.log_w('call_git: failed {} of {}: {}'.format(
i + 1, num_tries, ' '.join(cmd)))
clazz.log.log_d('call_git: exception: {}'.format(str(ex)))
clazz.log.log_d(
'call_git: sleeping {} seconds'.format(retry_wait_seconds))
time.sleep(retry_wait_seconds)
last_try_exception = ex
if not rv:
message = 'git command attempt failed {} times: {} in {}'.format(
num_tries, ' '.join(cmd), root)
clazz.log.log_w('call_git: {}'.format(message))
raise git_error(message, execute_result=None)
# first handle the retry failure
if num_tries > 1 and num_failed_attempts == num_tries and last_try_exception:
message = 'git command attempt failed {} times: {} in {}\n{}\n{}\n{}'.format(
num_tries, ' '.join(cmd), root, str(last_try_exception),
rv.stderr, rv.stdout)
clazz.log.log_w('call_git: {}'.format(message))
raise git_error(message, execute_result=rv)
# handle raise_error if needed
if rv.exit_code != 0 and raise_error:
message = 'git command failed: {} in {}\n{}\n{}\n{}'.format(
' '.join(cmd), root, str(last_try_exception), rv.stderr,
rv.stdout)
clazz.log.log_w('call_git: {}'.format(message))
raise git_error(message, execute_result=rv)
#print('rv={}'.format(rv))
return rv
def vm_run_package(self, vm_id, package_dir, entry_command,
entry_command_args, run_program_options):
check.check_string(vm_id)
check.check_string(package_dir)
check.check_string(entry_command)
check.check_string_seq(entry_command_args)
check.check_vmware_run_program_options(run_program_options)
self._log.log_method_d()
vm = self._resolve_vmx_to_local_vm(vm_id)
if not path.isdir(package_dir):
raise vmware_error(
'package_dir not found or not a dir: "{}"'.format(package_dir))
tmp_dir_local = temp_file.make_temp_dir(suffix='-run_package.dir',
delete=not self._options.debug)
if self._options.debug:
print('tmp_dir_local={}'.format(tmp_dir_local))
tmp_remote_dir = path.join('/tmp', path.basename(tmp_dir_local))
self._log.log_d(
'vm_run_package: tmp_remote_dir={}'.format(tmp_remote_dir))
tmp_package = self._make_tmp_file_pair(tmp_dir_local, 'package.tar.gz')
tmp_caller_script = self._make_tmp_file_pair(tmp_dir_local,
'caller_script.py')
tmp_output_log = self._make_tmp_file_pair(tmp_dir_local, 'output.log')
archiver.create(tmp_package.local, package_dir)
file_util.save(tmp_caller_script.local,
content=vmware_guest_scripts.RUN_PACKAGE_CALLER,
mode=0o0755)
self._log.log_d('vm_run_package: tmp_package={}'.format(tmp_package))
self._log.log_d(
'vm_run_package: tmp_caller_script={}'.format(tmp_caller_script))
self._log.log_d(
'vm_run_package: tmp_output_log={}'.format(tmp_output_log))
parsed_entry_command_args = command_line.parse_args(entry_command_args)
debug_args = []
if self._options.debug:
debug_args.append('--debug')
if self._options.tty:
debug_args.extend(['--tty', tty])
caller_args = debug_args + [
tmp_package.remote,
entry_command,
tmp_output_log.remote,
] + parsed_entry_command_args
with vmware_restore_vm_running_state(self) as _:
with vmware_run_operation(vm, run_program_options) as target_vm:
target_vm.dir_create(tmp_remote_dir)
target_vm.file_copy_to(tmp_package.local, tmp_package.remote)
target_vm.file_copy_to(tmp_caller_script.local,
tmp_caller_script.remote)
rv = target_vm.run_program(
tmp_caller_script.remote,
caller_args,
run_program_options=run_program_options)
target_vm.file_copy_from(tmp_output_log.remote,
tmp_output_log.local)
target_vm.dir_delete(tmp_remote_dir)
with file_util.open_with_default(
filename=run_program_options.output_filename) as f:
log_content = file_util.read(tmp_output_log.local,
codec='utf-8')
f.write(log_content)
f.flush()
return rv
def _parse_script(clazz, cmd):
args = command_line.parse_args(cmd)
return git_util.script(args[0], args[1:])
def run(clazz,
image_id,
command,
run_files=None,
run_files_substitutions=None,
run_label=None,
volumes=None,
env=None,
name=None,
restart=False,
detach=False,
tty=False,
interactive=False,
expose=None,
debug=False,
non_blocking=True,
remove=True,
tmp_dir=None):
check.check_string(image_id)
check.check_string(command)
check.check_string_seq(run_files, allow_none=True)
check.check_dict(run_files_substitutions,
check.STRING_TYPES,
check.STRING_TYPES,
allow_none=True)
check.check_string(run_label, allow_none=True)
check.check_string(name, allow_none=True)
check.check_dict(env,
check.STRING_TYPES,
check.STRING_TYPES,
allow_none=True)
check.check_dict(volumes,
check.STRING_TYPES,
check.STRING_TYPES,
allow_none=True)
check.check_bool(restart)
check.check_bool(detach)
check.check_bool(tty)
check.check_bool(interactive)
check.check_int(expose, allow_none=True)
check.check_bool(debug)
check.check_bool(non_blocking)
check.check_bool(remove)
check.check_string(tmp_dir, allow_none=True)
cli = command_line.parse_args(command)
env = env or {}
volumes = volumes or {}
command_args = cli[1:]
run_files = run_files or []
run_files_substitutions = run_files_substitutions or {}
run_label = run_label or 'docker.run'
tmp_dir = tmp_dir or os.getcwd()
tmp_run_dir = temp_file.make_temp_dir(suffix='-{}'.format(run_label),
dir=tmp_dir,
delete=not debug)
input_dir = path.join(tmp_run_dir, 'input')
output_dir = path.join(tmp_run_dir, 'output')
for cf in run_files:
src_file = path.join(os.getcwd(), cf)
dst_file = path.join(input_dir, cf)
if not path.isfile(src_file):
raise docker_error('run file not found: "{}"'.format(src_file))
if run_files_substitutions and file_mime.is_text(src_file):
file_replace.copy_with_substitute(src_file,
dst_file,
run_files_substitutions,
backup=False)
else:
file_util.copy(src_file, dst_file)
docker_run_args = ['run']
if detach:
docker_run_args.append('--detach')
if tty:
docker_run_args.append('--tty')
if interactive:
docker_run_args.append('--interactive')
if remove:
docker_run_args.append('--rm')
docker_run_args.extend(clazz._make_env_args(env))
volumes[input_dir] = '/input'
volumes[output_dir] = '/output'
docker_run_args.extend(clazz._make_volume_args(volumes))
if expose:
docker_run_args.extend(['--expose', str(expose)])
if restart:
docker_run_args.extend(['--restart', restart])
if name:
docker_run_args.extend(['--name', name])
docker_run_args.append(image_id)
docker_run_args.append(cli[0])
docker_run_args.extend(cli[1:])
clazz.log.log_d('running docker: {}'.format(' '.join(docker_run_args)))
rv = docker_exe.call_docker(docker_run_args, non_blocking=non_blocking)
container_id = docker_container.last_container()
return clazz._run_result(container_id, rv.exit_code, rv.stdout,
input_dir, output_dir)