def fetch(self, src, dest, configuration_name=DEFAULT_CONFIGURATION_NAME):
"""
Will fetch a single file from the remote Windows host and create a
local copy. Like copy(), this can be slow when it comes to fetching
large files due to the limitation of WinRM.
This method will first store the file in a temporary location before
creating or replacing the file at dest if the checksum is correct.
:param src: The path to the file on the remote host to fetch
:param dest: The path on the localhost host to store the file as
:param configuration_name: The PowerShell configuration endpoint to
use when fetching the file.
"""
dest = os.path.expanduser(os.path.expandvars(dest))
log.info("Fetching '%s' to '%s'" % (src, dest))
with RunspacePool(self.wsman, configuration_name=configuration_name) as pool:
script = get_pwsh_script('fetch.ps1')
powershell = PowerShell(pool)
powershell.add_script(script).add_argument(src)
log.debug("Starting remote process to output file data")
powershell.invoke()
log.debug("Finished remote process to output file data")
if powershell.had_errors:
errors = powershell.streams.error
error = "\n".join([str(err) for err in errors])
raise WinRMError("Failed to fetch file %s: %s" % (src, error))
expected_hash = powershell.output[-1]
temp_file, path = tempfile.mkstemp()
try:
file_bytes = base64.b64decode(powershell.output[0])
os.write(temp_file, file_bytes)
sha1 = hashlib.sha1()
sha1.update(file_bytes)
actual_hash = sha1.hexdigest()
log.debug("Remote Hash: %s, Local Hash: %s"
% (expected_hash, actual_hash))
if actual_hash != expected_hash:
raise WinRMError("Failed to fetch file %s, hash mismatch\n"
"Source: %s\nFetched: %s"
% (src, expected_hash, actual_hash))
shutil.copy(path, dest)
finally:
os.close(temp_file)
os.remove(path)
def unwrap_message(self, message, boundary):
log.debug("Unwrapped message")
# Talking to Exchange endpoints gives a non-compliant boundary that has a space between the -- {boundary}, not
# ideal but we just need to handle it.
parts = re.compile(to_bytes(r"--\s*%s\r\n" % re.escape(boundary))).split(message)
parts = list(filter(None, parts))
message = b""
for i in range(0, len(parts), 2):
header = parts[i].strip()
payload = parts[i + 1]
expected_length = int(header.split(b"Length=")[1])
# remove the end MIME block if it exists
payload = re.sub(to_bytes(r'--\s*%s--\r\n$') % to_bytes(boundary), b'', payload)
wrapped_data = payload.replace(b"\tContent-Type: application/octet-stream\r\n", b"")
unwrapped_data = self._unwrap(wrapped_data)
actual_length = len(unwrapped_data)
log.debug("Actual unwrapped length: %d, expected unwrapped length:"
" %d" % (actual_length, expected_length))
if actual_length != expected_length:
raise WinRMError("The encrypted length from the server does "
"not match the expected length, decryption "
"failed, actual: %d != expected: %d"
% (actual_length, expected_length))
message += unwrapped_data
return message
def unwrap_message(self, message, hostname):
log.debug("Unwrapped message for host: %s" % hostname)
parts = message.split(to_bytes("%s\r\n" % self.MIME_BOUNDARY))
parts = list(filter(None, parts))
message = b""
for i in range(0, len(parts), 2):
header = parts[i].strip()
payload = parts[i + 1]
expected_length = int(header.split(b"Length=")[1])
# remove the end MIME block if it exists
if payload.endswith(to_bytes("%s--\r\n" % self.MIME_BOUNDARY)):
payload = payload[:len(payload) - 24]
wrapped_data = payload.replace(
b"\tContent-Type: application/octet-stream\r\n", b"")
unwrapped_data = self._unwrap(wrapped_data, hostname)
actual_length = len(unwrapped_data)
log.debug("Actual unwrapped length: %d, expected unwrapped length:"
" %d" % (actual_length, expected_length))
if actual_length != expected_length:
raise WinRMError("The encrypted length from the server does "
"not match the expected length, decryption "
"failed, actual: %d != expected: %d" %
(actual_length, expected_length))
message += unwrapped_data
return message
def invoke(self,
action,
resource_uri,
resource,
option_set=None,
selector_set=None,
timeout=None):
"""
Send a generic WSMan request to the host.
:param action: The action to run, this relates to the wsa:Action header
field.
:param resource_uri: The resource URI that the action relates to, this
relates to the wsman:ResourceURI header field.
:param resource: This is an optional xml.etree.ElementTree Element to
be added to the s:Body section.
:param option_set: a wsman.OptionSet to add to the request
:param selector_set: a wsman.SelectorSet to add to the request
:param timeout: Override the default wsman:OperationTimeout value for
the request, this should be an int in seconds.
:return: The ET Element of the response XML from the server
"""
s = NAMESPACES['s']
envelope = ET.Element("{%s}Envelope" % s)
header = self._create_header(action, resource_uri, option_set,
selector_set, timeout)
envelope.append(header)
body = ET.SubElement(envelope, "{%s}Body" % s)
if resource is not None:
body.append(resource)
message_id = header.find("wsa:MessageID", namespaces=NAMESPACES).text
xml = ET.tostring(envelope, encoding='utf-8', method='xml')
try:
response = self.transport.send(xml)
except WinRMTransportError as err:
try:
# try and parse the XML and get the WSManFault
raise self._parse_wsman_fault(err.response_text)
except ET.ParseError:
# no XML message is present so not a WSManFault error
log.error("Failed to parse WSManFault message on WinRM error"
" response, raising original WinRMTransportError")
raise err
response_xml = ET.fromstring(response)
relates_to = response_xml.find("s:Header/wsa:RelatesTo",
namespaces=NAMESPACES).text
if message_id != relates_to:
raise WinRMError("Received related id does not match related "
"expected message id: Sent: %s, Received: %s" %
(message_id, relates_to))
return response_xml
def _invoke(self,
action,
resource_uri,
resource,
option_set=None,
selector_set=None,
timeout=None):
s = NAMESPACES['s']
envelope = ET.Element("{%s}Envelope" % s)
header = self._create_header(action, resource_uri, option_set,
selector_set, timeout)
envelope.append(header)
body = ET.SubElement(envelope, "{%s}Body" % s)
if resource is not None:
body.append(resource)
message_id = header.find("wsa:MessageID", namespaces=NAMESPACES).text
xml = ET.tostring(envelope, encoding='utf-8', method='xml')
try:
response = self.transport.send(xml)
except WinRMTransportError as err:
try:
# try and parse the XML and get the WSManFault
raise self._parse_wsman_fault(err.response_text)
except ET.ParseError:
# no XML message is present so not a WSManFault error
log.warning("Failed to parse WSManFault message on WinRM error"
" response, raising original WinRMTransportError")
raise err
response_xml = ET.fromstring(response)
relates_to = response_xml.find("s:Header/wsa:RelatesTo",
namespaces=NAMESPACES).text
if message_id != relates_to:
raise WinRMError("Received related id does not match related "
"expected message id: Sent: %s, Received: %s" %
(message_id, relates_to))
response_body = response_xml.find("s:Body", namespaces=NAMESPACES)
return response_body
def test_winrm_error():
with pytest.raises(WinRMError) as exc:
raise WinRMError("error msg")
assert str(exc.value) == "error msg"
def _handle_powershell_error(powershell, message):
if message and powershell.had_errors:
errors = powershell.streams.error
error = "\n".join([str(err) for err in errors])
raise WinRMError("%s: %s" % (message, error))
def copy(self, src, dest, configuration_name=DEFAULT_CONFIGURATION_NAME):
"""
Copies a single file from the current host to the remote Windows host.
This can be quite slow when it comes to large files due to the
limitations of WinRM but it is designed to be as fast as it can be.
During the copy process, the bytes will be stored in a temporary file
before being copied.
When copying it will replace the file at dest if one already exists. It
also will verify the checksum of the copied file is the same as the
actual file locally before copying the file to the path at dest.
:param src: The path to the local file
:param dest: The path to the destionation file on the Windows host
:param configuration_name: The PowerShell configuration endpoint to
use when copying the file.
:return: The absolute path of the file on the Windows host
"""
def read_buffer(b_path, total_size, buffer_size):
offset = 0
sha1 = hashlib.sha1()
with open(b_path, 'rb') as src_file:
for data in iter((lambda: src_file.read(buffer_size)), b""):
log.debug("Reading data of file at offset=%d with size=%d"
% (offset, buffer_size))
offset += len(data)
sha1.update(data)
b64_data = base64.b64encode(data)
result = [to_unicode(b64_data)]
if offset == total_size:
result.append(to_unicode(base64.b64encode(to_bytes(sha1.hexdigest()))))
yield result
# the file was empty, return empty buffer
if offset == 0:
yield [u"", to_unicode(base64.b64encode(to_bytes(sha1.hexdigest())))]
src = os.path.expanduser(os.path.expandvars(src))
b_src = to_bytes(src)
src_size = os.path.getsize(b_src)
log.info("Copying '%s' to '%s' with a total size of %d"
% (src, dest, src_size))
with RunspacePool(self.wsman, configuration_name=configuration_name) as pool:
# Get the buffer size of each fragment to send, subtract. Adjust to size of the base64 encoded bytes. Also
# subtract 82 for the fragment, message, and other header info that PSRP adds.
buffer_size = int((self.wsman.max_payload_size - 82) / 4 * 3)
log.info("Creating file reader with a buffer size of %d" % buffer_size)
read_gen = read_buffer(b_src, src_size, buffer_size)
command = get_pwsh_script('copy.ps1')
log.debug("Starting to send file data to remote process")
powershell = PowerShell(pool)
powershell.add_script(command).add_argument(dest)
powershell.invoke(input=read_gen)
log.debug("Finished sending file data to remote process")
for warning in powershell.streams.warning:
warnings.warn(str(warning))
if powershell.had_errors:
errors = powershell.streams.error
error = "\n".join([str(err) for err in errors])
raise WinRMError("Failed to copy file: %s" % error)
output_file = to_unicode(powershell.output[-1]).strip()
log.info("Completed file transfer of '%s' to '%s'" % (src, output_file))
return output_file
def copy(self, src, dest):
"""
Copies a single file from the current host to the remote Windows host.
This can be quite slow when it comes to large files due to the
limitations of WinRM but it is designed to be as fast as it can be.
During the copy process, the bytes will be stored in a temporary file
before being copied.
When copying it will replace the file at dest if one already exists. It
also will verify the checksum of the copied file is the same as the
actual file locally before copying the file to the path at dest.
:param src: The path to the local file
:param dest: The path to the destionation file on the Windows host
:return: The absolute path of the file on the Windows host
"""
def read_buffer(b_path, buffer_size):
offset = 0
sha1 = hashlib.sha1()
with open(b_path, 'rb') as src_file:
for data in iter((lambda: src_file.read(buffer_size)), b""):
log.debug("Reading data of file at offset=%d with size=%d"
% (offset, buffer_size))
offset += len(data)
sha1.update(data)
b64_data = base64.b64encode(data) + b"\r\n"
yield b64_data, False
# the file was empty, return empty buffer
if offset == 0:
yield b"", False
# the last input is the actual file hash used to verify the
# transfer was ok
actual_hash = b"\x00\xffHash: " + to_bytes(sha1.hexdigest())
yield base64.b64encode(actual_hash), True
src = os.path.expanduser(os.path.expandvars(src))
b_src = to_bytes(src)
src_size = os.path.getsize(b_src)
log.info("Copying '%s' to '%s' with a total size of %d"
% (src, dest, src_size))
# check if the src size is twice as large as the max payload and fetch
# the max size from the server, we only check in this case to save on a
# round trip if the file is small enough to fit in 2 msg's, otherwise
# we want to get the largest size possible
buffer_size = int(self.wsman.max_payload_size / 4 * 3)
if src_size > (buffer_size * 2) and \
self.wsman.max_envelope_size == 153600:
log.debug("Updating the max WSMan envelope size")
self.wsman.update_max_payload_size()
buffer_size = int(self.wsman.max_payload_size / 4 * 3)
log.info("Creating file reader with a buffer size of %d" % buffer_size)
read_gen = read_buffer(b_src, buffer_size)
command = u'''begin {
$ErrorActionPreference = "Stop"
$path = [System.IO.Path]::GetTempFileName()
$fd = [System.IO.File]::Create($path)
$algo = [System.Security.Cryptography.SHA1CryptoServiceProvider]::Create()
$bytes = @()
$expected_hash = ""
} process {
$base64_string = $input
$bytes = [System.Convert]::FromBase64String($base64_string)
if ($bytes.Count -eq 48 -and $bytes[0] -eq 0 -and $bytes[1] -eq 255) {
$hash_bytes = $bytes[-40..-1]
$expected_hash = [System.Text.Encoding]::UTF8.GetString($hash_bytes)
} else {
$algo.TransformBlock($bytes, 0, $bytes.Length, $bytes, 0) > $null
$fd.Write($bytes, 0, $bytes.Length)
}
} end {
$output_path = "%s"
$dest = New-Object -TypeName System.IO.FileInfo -ArgumentList $output_path
$fd.Close()
try {
$algo.TransformFinalBlock($bytes, 0, 0) > $null
$actual_hash = [System.BitConverter]::ToString($algo.Hash)
$actual_hash = $actual_hash.Replace("-", "").ToLowerInvariant()
if ($actual_hash -ne $expected_hash) {
$msg = "Transport failure, hash mistmatch"
$msg += "`r`nActual: $actual_hash"
$msg += "`r`nExpected: $expected_hash"
throw $msg
}
[System.IO.File]::Copy($path, $output_path, $true)
$dest.FullName
} finally {
[System.IO.File]::Delete($path)
}
}''' % to_unicode(dest)
encoded_command = to_string(base64.b64encode(to_bytes(command,
'utf-16-le')))
with WinRS(self.wsman) as shell:
process = Process(shell, "powershell.exe",
["-NoProfile", "-NonInteractive",
"-EncodedCommand", encoded_command])
process.begin_invoke()
log.info("Starting to send file data to remote process")
for input_data, end in read_gen:
process.send(input_data, end)
log.info("Finished sending file data to remote process")
process.end_invoke()
stderr = self.sanitise_clixml(process.stderr)
if process.rc != 0:
raise WinRMError("Failed to copy file: %s" % stderr)
output_file = to_unicode(process.stdout).strip()
log.info("Completed file transfer of '%s' to '%s'"
% (src, output_file))
return output_file
def fetch(self, src, dest):
"""
Will fetch a single file from the remote Windows host and create a
local copy. Like copy(), this can be slow when it comes to fetching
large files due to the limitation of WinRM.
This method will first store the file in a temporary location before
creating or replacing the file at dest if the checksum is correct.
:param src: The path to the file on the remote host to fetch
:param dest: The path on the localhost host to store the file as
"""
dest = os.path.expanduser(os.path.expandvars(dest))
log.info("Fetching '%s' to '%s'" % (src, dest))
self.wsman.update_max_payload_size()
# Need to output as a base64 string as PS Runspaces will create an
# individual byte objects for each byte in a byte array which has way
# more overhead than a single base64 string.
# I also wanted to output in chunks and have the local side process
# the output in parallel for large files but it seems like the base64
# stream is getting sent in one chunk when in a loop so scratch that
# idea
script = '''$ErrorActionPreference = 'Stop'
$algo = [System.Security.Cryptography.SHA1CryptoServiceProvider]::Create()
$src = New-Object -TypeName System.IO.FileInfo -ArgumentList '%s'
if ("Directory" -in $src.Attributes.ToString()) {
throw "The path at '$($src.FullName)' is a directory, src must be a file"
} elseif (-not $src.Exists) {
throw "The path at '$($src.FullName)' does not exist"
}
$buffer_size = 4096
$offset = 0
$fs = $src.OpenRead()
$total_bytes = $fs.Length
$bytes_to_read = $total_bytes - $offset
try {
while ($bytes_to_read -ne 0) {
$bytes = New-Object -TypeName byte[] -ArgumentList $bytes_to_read
$read = $fs.Read($bytes, $offset, $bytes_to_read)
Write-Output -InputObject ([System.Convert]::ToBase64String($bytes))
$bytes_to_read -= $read
$offset += $read
$algo.TransformBlock($bytes, 0, $bytes.Length, $bytes, 0) > $null
}
} finally {
$fs.Dispose()
}
$algo.TransformFinalBlock($bytes, 0, 0) > $Null
$hash = [System.BitConverter]::ToString($algo.Hash)
$hash.Replace("-", "").ToLowerInvariant()''' % src
with RunspacePool(self.wsman) as pool:
powershell = PowerShell(pool)
powershell.add_script(script)
log.info("Starting remote process to output file data")
powershell.invoke()
log.info("Finished remote process to output file data")
if powershell.had_errors:
errors = powershell.streams.error
error = "\n".join([str(err) for err in errors])
raise WinRMError("Failed to fetch file %s: %s" % (src, error))
expected_hash = powershell.output[-1]
temp_file, path = tempfile.mkstemp()
try:
file_bytes = base64.b64decode(powershell.output[0])
os.write(temp_file, file_bytes)
sha1 = hashlib.sha1()
sha1.update(file_bytes)
actual_hash = sha1.hexdigest()
log.debug("Remote Hash: %s, Local Hash: %s"
% (expected_hash, actual_hash))
if actual_hash != expected_hash:
raise WinRMError("Failed to fetch file %s, hash mismatch\n"
"Source: %s\nFetched: %s"
% (src, expected_hash, actual_hash))
shutil.copy(path, dest)
finally:
os.close(temp_file)
os.remove(path)
