def sshoperator(ssh_conn_id,
ssh_hook=None,
remote_host=None,
command=None,
timeout=10,
do_xcom_push=None,
**kwargs):
"""
:param ssh_conn_id:
:param ssh_hook:
:param remote_host:
:param command:
:param timeout:
:param do_xcom_push:
:param kwargs:
:return:
"""
try:
if ssh_conn_id and not ssh_hook:
ssh_hook = SSHHook(ssh_conn_id=ssh_conn_id)
if not ssh_hook:
raise AirflowException(
"can not operate without ssh_hook or ssh_conn_id")
if remote_host is not None:
ssh_hook.remote_host = remote_host
if not command:
raise AirflowException(
"no command specified so nothing to execute here.")
#access the task
if kwargs['task_id']:
task = 'init_task_{}'.format(kwargs['task_id'])
else:
task = 'boot_task'
ti = kwargs['ti']
#get all the parameters
xcom_keys = kwargs['params']
xcom_values = pullXcom(ti, task, xcom_keys)
LOG_PATH = xcom_values['hdfs_path']
SOURCE_PATH = xcom_values['fs_path']
log.info(str(command))
tmp_cmd = str(command).replace("{{ params.LOG_PATH }}", LOG_PATH)
tmp_cmd = tmp_cmd.replace("{{ params.SOURCE_PATH }}", SOURCE_PATH)
log.info(tmp_cmd)
command = tmp_cmd
# Auto apply tty when its required in case of sudo
get_pty = False
if command.startswith('sudo'):
get_pty = True
ssh_client = ssh_hook.get_conn()
# set timeout taken as params
stdin, stdout, stderr = ssh_client.exec_command(command=command,
get_pty=get_pty,
timeout=timeout)
# get channels
channel = stdout.channel
# closing stdin
stdin.close()
channel.shutdown_write()
agg_stdout = b''
agg_stderr = b''
# capture any initial output in case channel is closed already
stdout_buffer_length = len(stdout.channel.in_buffer)
if stdout_buffer_length > 0:
agg_stdout += stdout.channel.recv(stdout_buffer_length)
# read from both stdout and stderr
while not channel.closed or \
channel.recv_ready() or \
channel.recv_stderr_ready():
readq, _, _ = select([channel], [], [], timeout)
for c in readq:
if c.recv_ready():
line = stdout.channel.recv(len(c.in_buffer))
line = line
agg_stdout += line
log.info(line.decode('utf-8').strip('\n'))
if c.recv_stderr_ready():
line = stderr.channel.recv_stderr(len(c.in_stderr_buffer))
line = line
agg_stderr += line
log.warning(line.decode('utf-8').strip('\n'))
if stdout.channel.exit_status_ready() \
and not stderr.channel.recv_stderr_ready() \
and not stdout.channel.recv_ready():
stdout.channel.shutdown_read()
stdout.channel.close()
break
stdout.close()
stderr.close()
exit_status = stdout.channel.recv_exit_status()
if exit_status is 0:
# returning output if do_xcom_push is set
if do_xcom_push:
enable_pickling = configuration.conf.getboolean(
'core', 'enable_xcom_pickling')
if enable_pickling:
return agg_stdout
else:
return b64encode(agg_stdout).decode('utf-8')
else:
error_msg = agg_stderr.decode('utf-8')
raise AirflowException("error running cmd: {0}, error: {1}".format(
command, error_msg))
except Exception as e:
raise AirflowException("SSH operator error: {0}".format(str(e)))
return True
print("ssh call")
return True
