def test_get_conn_from_connection(self, mock_get_connection,
mock_protocol):
connection = mock_get_connection.return_value
winrm_hook = WinRMHook(ssh_conn_id='conn_id')
winrm_hook.get_conn()
mock_get_connection.assert_called_once_with(winrm_hook.ssh_conn_id)
mock_protocol.assert_called_once_with(
endpoint=str(connection.extra_dejson['endpoint']),
transport=str(connection.extra_dejson['transport']),
username=connection.login,
password=connection.password,
service=str(connection.extra_dejson['service']),
keytab=str(connection.extra_dejson['keytab']),
ca_trust_path=str(connection.extra_dejson['ca_trust_path']),
cert_pem=str(connection.extra_dejson['cert_pem']),
cert_key_pem=str(connection.extra_dejson['cert_key_pem']),
server_cert_validation=str(
connection.extra_dejson['server_cert_validation']),
kerberos_delegation=str(
connection.extra_dejson['kerberos_delegation']).lower() ==
'true',
read_timeout_sec=int(connection.extra_dejson['read_timeout_sec']),
operation_timeout_sec=int(
connection.extra_dejson['operation_timeout_sec']),
kerberos_hostname_override=str(
connection.extra_dejson['kerberos_hostname_override']),
message_encryption=str(
connection.extra_dejson['message_encryption']),
credssp_disable_tlsv1_2=str(
connection.extra_dejson['credssp_disable_tlsv1_2']).lower() ==
'true',
send_cbt=str(
connection.extra_dejson['send_cbt']).lower() == 'true')
def test_get_conn_no_endpoint(self, mock_protocol):
winrm_hook = WinRMHook(remote_host='host', password='******')
winrm_hook.get_conn()
self.assertEqual('http://{0}:{1}/wsman'.format(winrm_hook.remote_host, winrm_hook.remote_port),
winrm_hook.endpoint)
def test_get_conn_exists(self, mock_protocol):
winrm_hook = WinRMHook()
winrm_hook.client = mock_protocol.return_value.open_shell.return_value
conn = winrm_hook.get_conn()
self.assertEqual(conn, winrm_hook.client)
def test_get_conn_from_connection(self, mock_get_connection, mock_protocol):
connection = mock_get_connection.return_value
winrm_hook = WinRMHook(ssh_conn_id='conn_id')
winrm_hook.get_conn()
mock_get_connection.assert_called_once_with(winrm_hook.ssh_conn_id)
mock_protocol.assert_called_once_with(
endpoint=str(connection.extra_dejson['endpoint']),
transport=str(connection.extra_dejson['transport']),
username=connection.login,
password=connection.password,
service=str(connection.extra_dejson['service']),
keytab=str(connection.extra_dejson['keytab']),
ca_trust_path=str(connection.extra_dejson['ca_trust_path']),
cert_pem=str(connection.extra_dejson['cert_pem']),
cert_key_pem=str(connection.extra_dejson['cert_key_pem']),
server_cert_validation=str(connection.extra_dejson['server_cert_validation']),
kerberos_delegation=str(connection.extra_dejson['kerberos_delegation']).lower() == 'true',
read_timeout_sec=int(connection.extra_dejson['read_timeout_sec']),
operation_timeout_sec=int(connection.extra_dejson['operation_timeout_sec']),
kerberos_hostname_override=str(connection.extra_dejson['kerberos_hostname_override']),
message_encryption=str(connection.extra_dejson['message_encryption']),
credssp_disable_tlsv1_2=str(connection.extra_dejson['credssp_disable_tlsv1_2']).lower() == 'true',
send_cbt=str(connection.extra_dejson['send_cbt']).lower() == 'true'
)
def test_get_conn_exists(self, mock_protocol):
winrm_hook = WinRMHook()
winrm_hook.client = mock_protocol.return_value.open_shell.return_value
conn = winrm_hook.get_conn()
self.assertEqual(conn, winrm_hook.client)
def test_get_conn_no_endpoint(self, mock_protocol):
winrm_hook = WinRMHook(remote_host='host', password='******')
winrm_hook.get_conn()
self.assertEqual('http://{0}:{1}/wsman'.format(winrm_hook.remote_host, winrm_hook.remote_port),
winrm_hook.endpoint)
def execute(self, context):
try:
if self.ssh_conn_id and not self.winrm_hook:
self.log.info("hook not found, creating")
self.winrm_hook = WinRMHook(ssh_conn_id=self.ssh_conn_id)
if not self.winrm_hook:
raise AirflowException("can not operate without ssh_hook or ssh_conn_id")
if self.remote_host is not None:
self.winrm_hook.remote_host = self.remote_host
winrm_client = self.winrm_hook.get_conn()
self.log.info("Established WinRM connection")
if not self.command:
raise AirflowException("no command specified so nothing to execute here.")
self.log.info(
"Starting command: '{command}' on remote host: {remotehost}".
format(command=self.command, remotehost=self.winrm_hook.remote_host)
)
command_id = self.winrm_hook.winrm_protocol. \
run_command(winrm_client, self.command)
std_out, std_err, status_code = self.winrm_hook.winrm_protocol. \
get_command_output(winrm_client, command_id)
self.log.info("std out: " + std_out.decode())
self.log.info("std err: " + std_err.decode())
self.log.info("exit code: " + str(status_code))
self.log.info("Cleaning up WinRM command")
self.winrm_hook.winrm_protocol.cleanup_command(winrm_client, command_id)
self.log.info("Cleaning up WinRM protocol shell")
self.winrm_hook.winrm_protocol.close_shell(winrm_client)
if status_code is 0:
return std_out.decode()
else:
error_msg = std_err.decode()
raise AirflowException("error running cmd: {0}, error: {1}"
.format(self.command, error_msg))
except Exception as e:
raise AirflowException("WinRM operator error: {0}".format(str(e)))
return True
def execute(self, context):
if self.ssh_conn_id and not self.winrm_hook:
self.log.info("Hook not found, creating...")
self.winrm_hook = WinRMHook(ssh_conn_id=self.ssh_conn_id)
if not self.winrm_hook:
raise AirflowException("Cannot operate without winrm_hook or ssh_conn_id.")
if self.remote_host is not None:
self.winrm_hook.remote_host = self.remote_host
if not self.command:
raise AirflowException("No command specified so nothing to execute here.")
winrm_client = self.winrm_hook.get_conn()
try:
self.log.info("Running command: '{command}'...".format(command=self.command))
command_id = self.winrm_hook.winrm_protocol.run_command(
winrm_client,
self.command
)
# See: https://github.com/diyan/pywinrm/blob/master/winrm/protocol.py
stdout_buffer = []
stderr_buffer = []
command_done = False
while not command_done:
try:
stdout, stderr, return_code, command_done = \
self.winrm_hook.winrm_protocol._raw_get_command_output(
winrm_client,
command_id
)
# Only buffer stdout if we need to so that we minimize memory usage.
if self.do_xcom_push:
stdout_buffer.append(stdout)
stderr_buffer.append(stderr)
for line in stdout.decode('utf-8').splitlines():
self.log.info(line)
for line in stderr.decode('utf-8').splitlines():
self.log.warning(line)
except WinRMOperationTimeoutError as e:
# this is an expected error when waiting for a
# long-running process, just silently retry
pass
self.winrm_hook.winrm_protocol.cleanup_command(winrm_client, command_id)
self.winrm_hook.winrm_protocol.close_shell(winrm_client)
except Exception as e:
raise AirflowException("WinRM operator error: {0}".format(str(e)))
if return_code is 0:
# returning output if do_xcom_push is set
if self.do_xcom_push:
enable_pickling = configuration.conf.getboolean(
'core', 'enable_xcom_pickling'
)
if enable_pickling:
return stdout_buffer
else:
return b64encode(b''.join(stdout_buffer)).decode('utf-8')
else:
error_msg = "Error running cmd: {0}, return code: {1}, error: {2}".format(
self.command,
return_code,
b''.join(stderr_buffer).decode('utf-8')
)
raise AirflowException(error_msg)
self.log.info("Finished!")
return True
class WinRMOperator(BaseOperator):
"""
WinRMOperator to execute commands on given remote host using the winrm_hook.
:param winrm_hook: predefined ssh_hook to use for remote execution
:type winrm_hook: :class:`WinRMHook`
:param ssh_conn_id: connection id from airflow Connections
:type ssh_conn_id: str
:param remote_host: remote host to connect
:type remote_host: str
:param command: command to execute on remote host. (templated)
:type command: str
:param timeout: timeout for executing the command.
:type timeout: int
:param do_xcom_push: return the stdout which also get set in xcom by airflow platform
:type do_xcom_push: bool
"""
template_fields = ('command',)
@apply_defaults
def __init__(self,
winrm_hook=None,
ssh_conn_id=None,
remote_host=None,
command=None,
timeout=10,
do_xcom_push=False,
*args,
**kwargs):
super(WinRMOperator, self).__init__(*args, **kwargs)
self.winrm_hook = winrm_hook
self.ssh_conn_id = ssh_conn_id
self.remote_host = remote_host
self.command = command
self.timeout = timeout
self.do_xcom_push = do_xcom_push
def execute(self, context):
if self.ssh_conn_id and not self.winrm_hook:
self.log.info("Hook not found, creating...")
self.winrm_hook = WinRMHook(ssh_conn_id=self.ssh_conn_id)
if not self.winrm_hook:
raise AirflowException("Cannot operate without winrm_hook or ssh_conn_id.")
if self.remote_host is not None:
self.winrm_hook.remote_host = self.remote_host
if not self.command:
raise AirflowException("No command specified so nothing to execute here.")
winrm_client = self.winrm_hook.get_conn()
try:
self.log.info("Running command: '{command}'...".format(command=self.command))
command_id = self.winrm_hook.winrm_protocol.run_command(
winrm_client,
self.command
)
# See: https://github.com/diyan/pywinrm/blob/master/winrm/protocol.py
stdout_buffer = []
stderr_buffer = []
command_done = False
while not command_done:
try:
stdout, stderr, return_code, command_done = \
self.winrm_hook.winrm_protocol._raw_get_command_output(
winrm_client,
command_id
)
# Only buffer stdout if we need to so that we minimize memory usage.
if self.do_xcom_push:
stdout_buffer.append(stdout)
stderr_buffer.append(stderr)
for line in stdout.decode('utf-8').splitlines():
self.log.info(line)
for line in stderr.decode('utf-8').splitlines():
self.log.warning(line)
except WinRMOperationTimeoutError as e:
# this is an expected error when waiting for a
# long-running process, just silently retry
pass
self.winrm_hook.winrm_protocol.cleanup_command(winrm_client, command_id)
self.winrm_hook.winrm_protocol.close_shell(winrm_client)
except Exception as e:
raise AirflowException("WinRM operator error: {0}".format(str(e)))
if return_code is 0:
# returning output if do_xcom_push is set
if self.do_xcom_push:
enable_pickling = configuration.conf.getboolean(
'core', 'enable_xcom_pickling'
)
if enable_pickling:
return stdout_buffer
else:
return b64encode(b''.join(stdout_buffer)).decode('utf-8')
else:
error_msg = "Error running cmd: {0}, return code: {1}, error: {2}".format(
self.command,
return_code,
b''.join(stderr_buffer).decode('utf-8')
)
raise AirflowException(error_msg)
self.log.info("Finished!")
return True
from airflow.operators.dummy_operator import DummyOperator
default_args = {
'owner': 'airflow',
'start_date': airflow.utils.dates.days_ago(2)
}
with DAG(dag_id='POC_winrm_parallel',
default_args=default_args,
schedule_interval='0 0 * * *',
dagrun_timeout=timedelta(minutes=60)) as dag:
cmd = 'ls -l'
run_this_last = DummyOperator(task_id='run_this_last')
winRMHook = WinRMHook(ssh_conn_id='ssh_POC1')
t1 = WinRMOperator(task_id="wintask1",
command='ls -altr',
winrm_hook=winRMHook)
t2 = WinRMOperator(task_id="wintask2",
command='sleep 60',
winrm_hook=winRMHook)
t3 = WinRMOperator(task_id="wintask3",
command='echo \'luke test\' ',
winrm_hook=winRMHook)
[t1, t2, t3] >> run_this_last
def test_get_conn_error(self, mock_protocol):
mock_protocol.side_effect = Exception('Error')
with self.assertRaises(AirflowException):
WinRMHook(remote_host='host').get_conn()
def test_get_conn_missing_remote_host(self):
with self.assertRaises(AirflowException):
WinRMHook().get_conn()
def test_get_conn_no_username(self, mock_protocol, mock_getuser):
winrm_hook = WinRMHook(remote_host='host', password='******')
winrm_hook.get_conn()
self.assertEqual(mock_getuser.return_value, winrm_hook.username)
class WinRMOperator(BaseOperator):
"""
WinRMOperator to execute commands on given remote host using the winrm_hook.
:param winrm_hook: predefined ssh_hook to use for remote execution
:type winrm_hook: :class:`WinRMHook`
:param ssh_conn_id: connection id from airflow Connections
:type ssh_conn_id: str
:param remote_host: remote host to connect
:type remote_host: str
:param command: command to execute on remote host. (templated)
:type command: str
:param timeout: timeout for executing the command.
:type timeout: int
:param do_xcom_push: return the stdout which also get set in xcom by airflow platform
:type do_xcom_push: bool
"""
template_fields = ('command',)
@apply_defaults
def __init__(self,
winrm_hook=None,
ssh_conn_id=None,
remote_host=None,
command=None,
timeout=10,
do_xcom_push=False,
*args,
**kwargs):
super(WinRMOperator, self).__init__(*args, **kwargs)
self.winrm_hook = winrm_hook
self.ssh_conn_id = ssh_conn_id
self.remote_host = remote_host
self.command = command
self.timeout = timeout
self.do_xcom_push = do_xcom_push
def execute(self, context):
try:
if self.ssh_conn_id and not self.winrm_hook:
self.log.info("hook not found, creating")
self.winrm_hook = WinRMHook(ssh_conn_id=self.ssh_conn_id)
if not self.winrm_hook:
raise AirflowException("can not operate without ssh_hook or ssh_conn_id")
if self.remote_host is not None:
self.winrm_hook.remote_host = self.remote_host
winrm_client = self.winrm_hook.get_conn()
self.log.info("Established WinRM connection")
if not self.command:
raise AirflowException("no command specified so nothing to execute here.")
self.log.info(
"Starting command: '{command}' on remote host: {remotehost}".
format(command=self.command, remotehost=self.winrm_hook.remote_host)
)
command_id = self.winrm_hook.winrm_protocol. \
run_command(winrm_client, self.command)
std_out, std_err, status_code = self.winrm_hook.winrm_protocol. \
get_command_output(winrm_client, command_id)
self.log.info("std out: " + std_out.decode())
self.log.info("std err: " + std_err.decode())
self.log.info("exit code: " + str(status_code))
self.log.info("Cleaning up WinRM command")
self.winrm_hook.winrm_protocol.cleanup_command(winrm_client, command_id)
self.log.info("Cleaning up WinRM protocol shell")
self.winrm_hook.winrm_protocol.close_shell(winrm_client)
if status_code is 0:
return std_out.decode()
else:
error_msg = std_err.decode()
raise AirflowException("error running cmd: {0}, error: {1}"
.format(self.command, error_msg))
except Exception as e:
raise AirflowException("WinRM operator error: {0}".format(str(e)))
return True
def test_get_conn_no_username(self, mock_protocol, mock_getuser):
winrm_hook = WinRMHook(remote_host='host', password='******')
winrm_hook.get_conn()
self.assertEqual(mock_getuser.return_value, winrm_hook.username)