def unwrap_message(self, message, boundary):
log.debug("Unwrapped message")
# Talking to Exchange endpoints gives a non-compliant boundary that has a space between the -- {boundary}, not
# ideal but we just need to handle it.
parts = re.compile(to_bytes(r"--\s*%s\r\n" % re.escape(boundary))).split(message)
parts = list(filter(None, parts))
message = b""
for i in range(0, len(parts), 2):
header = parts[i].strip()
payload = parts[i + 1]
expected_length = int(header.split(b"Length=")[1])
# remove the end MIME block if it exists
payload = re.sub(to_bytes(r'--\s*%s--\r\n$') % to_bytes(boundary), b'', payload)
wrapped_data = payload.replace(b"\tContent-Type: application/octet-stream\r\n", b"")
unwrapped_data = self._unwrap(wrapped_data)
actual_length = len(unwrapped_data)
log.debug("Actual unwrapped length: %d, expected unwrapped length:"
" %d" % (actual_length, expected_length))
if actual_length != expected_length:
raise WinRMError("The encrypted length from the server does "
"not match the expected length, decryption "
"failed, actual: %d != expected: %d"
% (actual_length, expected_length))
message += unwrapped_data
return message
def read_buffer(b_path, total_size, buffer_size):
offset = 0
sha1 = hashlib.sha1()
with open(b_path, 'rb') as src_file:
for data in iter((lambda: src_file.read(buffer_size)), b""):
log.debug(
"Reading data of file at offset=%d with size=%d" %
(offset, buffer_size))
offset += len(data)
sha1.update(data)
b64_data = base64.b64encode(data)
result = [to_unicode(b64_data)]
if offset == total_size:
result.append(
to_unicode(
base64.b64encode(to_bytes(sha1.hexdigest()))))
yield result
# the file was empty, return empty buffer
if offset == 0:
yield [
u"",
to_unicode(base64.b64encode(to_bytes(
sha1.hexdigest())))
]
def unwrap_message(self, message, hostname):
log.debug("Unwrapped message for host: %s" % hostname)
parts = message.split(to_bytes("%s\r\n" % self.MIME_BOUNDARY))
parts = list(filter(None, parts))
message = b""
for i in range(0, len(parts), 2):
header = parts[i].strip()
payload = parts[i + 1]
expected_length = int(header.split(b"Length=")[1])
# remove the end MIME block if it exists
if payload.endswith(to_bytes("%s--\r\n" % self.MIME_BOUNDARY)):
payload = payload[:len(payload) - 24]
wrapped_data = payload.replace(
b"\tContent-Type: application/octet-stream\r\n", b"")
unwrapped_data = self._unwrap(wrapped_data, hostname)
actual_length = len(unwrapped_data)
log.debug("Actual unwrapped length: %d, expected unwrapped length:"
" %d" % (actual_length, expected_length))
if actual_length != expected_length:
raise WinRMError("The encrypted length from the server does "
"not match the expected length, decryption "
"failed, actual: %d != expected: %d" %
(actual_length, expected_length))
message += unwrapped_data
return message
def _get_security_context(name_type,
mech,
spn,
username,
password,
delegate,
wrap_required,
channel_bindings=None):
if username is not None:
username = gssapi.Name(base=username, name_type=name_type)
server_name = gssapi.Name(spn,
name_type=gssapi.NameType.hostbased_service)
# first try and get the cred from the existing cache, if that fails
# then get a new ticket with the password (if specified). The cache
# can only be used for Kerberos, NTLM/SPNEGO must have acquire the
# cred with a pass
cred = None
kerb_oid = GSSAPIContext._AUTH_PROVIDERS['kerberos']
kerb_mech = gssapi.OID.from_int_seq(kerb_oid)
if mech == kerb_mech:
try:
cred = gssapi.Credentials(name=username,
usage='initiate',
mechs=[mech])
except gssapi.exceptions.GSSError as err:
# we can't acquire the cred if no password was supplied
if password is None:
raise err
pass
elif username is None or password is None:
raise ValueError("Can only use implicit credentials with kerberos "
"authentication")
if cred is None:
# error when trying to access the existing cache, get our own
# credentials with the password specified
b_password = to_bytes(password)
cred = gssapi.raw.acquire_cred_with_password(username,
b_password,
usage='initiate',
mechs=[mech])
cred = cred.creds
flags = gssapi.RequirementFlag.mutual_authentication | \
gssapi.RequirementFlag.out_of_sequence_detection
if delegate:
flags |= gssapi.RequirementFlag.delegate_to_peer
if wrap_required:
flags |= gssapi.RequirementFlag.confidentiality
context = gssapi.SecurityContext(name=server_name,
creds=cred,
usage='initiate',
mech=mech,
flags=flags,
channel_bindings=channel_bindings)
return context
def _deserialize_string(
self,
value: typing.Optional[str],
) -> str:
if value is None:
return ""
def rplcr(matchobj):
# The matched object is the UTF-16 byte representation of the UTF-8
# hex string value. We need to decode the byte str to unicode and
# then unhexlify that hex string to get the actual bytes of the
# _x****_ value, e.g.
# group(0) == b"\x00_\x00x\x000\x000\x000\x00A\x00_"
# group(1) == b"\x000\x000\x000\x00A"
# unicode (from utf-16-be) == u"000A"
# returns b"\x00\x0A"
match_hex = matchobj.group(1)
hex_string = to_unicode(match_hex, encoding="utf-16-be")
return binascii.unhexlify(hex_string)
# need to ensure we start with a unicode representation of the string
# so that we can get the actual UTF-16 bytes value from that string
unicode_value = to_unicode(value)
unicode_bytes = to_bytes(unicode_value, encoding="utf-16-be")
bytes_value = re.sub(self._deserial_str, rplcr, unicode_bytes)
return to_unicode(bytes_value, encoding="utf-16-be")
def test_str_to_bytes():
# Python 3 the default string type is unicode so the expected value will
# be "abc" in UTF-16 form while Python 2 "abc" is the bytes representation
# already
expected = b"\x61\x00\x62\x00\x63\x00"
actual = to_bytes("abc", encoding="utf-16-le")
assert actual == expected
def rplcr(matchobj):
surrogate_char = matchobj.group(0)
byte_char = to_bytes(surrogate_char, encoding='utf-16-be')
hex_char = to_unicode(binascii.hexlify(byte_char)).upper()
hex_split = [hex_char[i:i + 4] for i in range(0, len(hex_char), 4)]
return u"".join([u"_x%s_" % i for i in hex_split])
def _wrap_message(self, message, hostname):
msg_length = str(len(message))
wrapped_data = self._wrap(message, hostname)
payload = "\r\n".join([
self.MIME_BOUNDARY,
"\tContent-Type: %s" % self.protocol,
"\tOriginalContent: type=application/soap+xml;charset=UTF-8;"
"Length=%s" % msg_length, self.MIME_BOUNDARY,
"\tContent-Type: application/octet-stream", ""
])
payload = to_bytes(payload) + wrapped_data
return payload
def _wrap_message(self, message: bytes) -> bytes:
header, wrapped_data, padding_length = self.context.wrap_winrm(message)
wrapped_data = struct.pack("<i", len(header)) + header + wrapped_data
msg_length = str(len(message) + padding_length)
payload = "\r\n".join([
self.MIME_BOUNDARY,
"\tContent-Type: %s" % self.protocol,
"\tOriginalContent: type=application/soap+xml;charset=UTF-8;Length=%s"
% msg_length,
self.MIME_BOUNDARY,
"\tContent-Type: application/octet-stream",
"",
])
return to_bytes(payload) + wrapped_data
def wrap_message(self, message):
log.debug("Wrapping message")
if self.protocol == self.CREDSSP and len(message) > self.SIXTEEN_KB:
content_type = "multipart/x-multi-encrypted"
encrypted_msg = b""
chunks = [message[i:i + self.SIXTEEN_KB] for i in
range(0, len(message), self.SIXTEEN_KB)]
for chunk in chunks:
encrypted_chunk = self._wrap_message(chunk)
encrypted_msg += encrypted_chunk
else:
content_type = "multipart/encrypted"
encrypted_msg = self._wrap_message(message)
encrypted_msg += to_bytes("%s--\r\n" % self.MIME_BOUNDARY)
log.debug("Created wrapped message of content type %s" % content_type)
return content_type, encrypted_msg
def _serialize_secure_string(self, value):
if self.cipher is None:
raise SerializationError("Cannot generate secure string as cipher "
"is not initialised")
# convert the string to a UTF-16 byte string as that is what is
# expected in Windows. If a byte string (native string in Python 2) was
# passed in, the sender must make sure it is a valid UTF-16
# representation and not UTF-8 or else the server will fail to decrypt
# the secure string in most cases
string_bytes = to_bytes(value, encoding='utf-16-le')
padder = PKCS7(self.cipher.algorithm.block_size).padder()
padded_data = padder.update(string_bytes) + padder.finalize()
encryptor = self.cipher.encryptor()
ss_value = encryptor.update(padded_data) + encryptor.finalize()
ss_string = to_string(base64.b64encode(ss_value))
return ss_string
def read_buffer(b_path, buffer_size):
offset = 0
sha1 = hashlib.sha1()
with open(b_path, 'rb') as src_file:
for data in iter((lambda: src_file.read(buffer_size)), b""):
log.debug("Reading data of file at offset=%d with size=%d"
% (offset, buffer_size))
offset += len(data)
sha1.update(data)
b64_data = base64.b64encode(data) + b"\r\n"
yield b64_data, False
# the file was empty, return empty buffer
if offset == 0:
yield b"", False
# the last input is the actual file hash used to verify the
# transfer was ok
actual_hash = b"\x00\xffHash: " + to_bytes(sha1.hexdigest())
yield base64.b64encode(actual_hash), True
def test_unicode_to_bytes_diff_encoding():
expected = b"\x61\x00\x62\x00\x63\x00"
actual = to_bytes(u"abc", encoding='utf-16-le')
assert actual == expected
def copy(self,
src,
dest,
configuration_name=DEFAULT_CONFIGURATION_NAME,
expand_variables=False):
"""
Copies a single file from the current host to the remote Windows host.
This can be quite slow when it comes to large files due to the
limitations of WinRM but it is designed to be as fast as it can be.
During the copy process, the bytes will be stored in a temporary file
before being copied.
When copying it will replace the file at dest if one already exists. It
also will verify the checksum of the copied file is the same as the
actual file locally before copying the file to the path at dest.
:param src: The path to the local file
:param dest: The path to the destination file on the Windows host
:param configuration_name: The PowerShell configuration endpoint to
use when copying the file.
:param expand_variables: Expand variables in path. Disabled by default
Enable for cmd like expansion (for example %TMP% in path)
:return: The absolute path of the file on the Windows host
"""
def read_buffer(b_path, total_size, buffer_size):
offset = 0
sha1 = hashlib.sha1()
with open(b_path, 'rb') as src_file:
for data in iter((lambda: src_file.read(buffer_size)), b""):
log.debug(
"Reading data of file at offset=%d with size=%d" %
(offset, buffer_size))
offset += len(data)
sha1.update(data)
b64_data = base64.b64encode(data)
result = [to_unicode(b64_data)]
if offset == total_size:
result.append(
to_unicode(
base64.b64encode(to_bytes(sha1.hexdigest()))))
yield result
# the file was empty, return empty buffer
if offset == 0:
yield [
u"",
to_unicode(base64.b64encode(to_bytes(
sha1.hexdigest())))
]
if expand_variables:
src = os.path.expanduser(os.path.expandvars(src))
b_src = to_bytes(src)
src_size = os.path.getsize(b_src)
log.info("Copying '%s' to '%s' with a total size of %d" %
(src, dest, src_size))
with RunspacePool(self.wsman,
configuration_name=configuration_name) as pool:
# Get the buffer size of each fragment to send, subtract. Adjust to size of the base64 encoded bytes. Also
# subtract 82 for the fragment, message, and other header info that PSRP adds.
buffer_size = int((self.wsman.max_payload_size - 82) / 4 * 3)
log.info("Creating file reader with a buffer size of %d" %
buffer_size)
read_gen = read_buffer(b_src, src_size, buffer_size)
command = get_pwsh_script('copy.ps1')
log.debug("Starting to send file data to remote process")
powershell = PowerShell(pool)
powershell.add_script(command) \
.add_argument(dest) \
.add_argument(expand_variables)
powershell.invoke(input=read_gen)
_handle_powershell_error(powershell, "Failed to copy file")
log.debug("Finished sending file data to remote process")
for warning in powershell.streams.warning:
warnings.warn(str(warning))
output_file = to_unicode(powershell.output[-1]).strip()
log.info("Completed file transfer of '%s' to '%s'" %
(src, output_file))
return output_file
def test_bytes_to_bytes():
expected = b"\x01\x02\x03\x04"
actual = to_bytes(b"\x01\x02\x03\x04")
assert actual == expected
def copy(self, src, dest):
"""
Copies a single file from the current host to the remote Windows host.
This can be quite slow when it comes to large files due to the
limitations of WinRM but it is designed to be as fast as it can be.
During the copy process, the bytes will be stored in a temporary file
before being copied.
When copying it will replace the file at dest if one already exists. It
also will verify the checksum of the copied file is the same as the
actual file locally before copying the file to the path at dest.
:param src: The path to the local file
:param dest: The path to the destionation file on the Windows host
:return: The absolute path of the file on the Windows host
"""
def read_buffer(b_path, buffer_size):
offset = 0
sha1 = hashlib.sha1()
with open(b_path, 'rb') as src_file:
for data in iter((lambda: src_file.read(buffer_size)), b""):
log.debug("Reading data of file at offset=%d with size=%d"
% (offset, buffer_size))
offset += len(data)
sha1.update(data)
b64_data = base64.b64encode(data) + b"\r\n"
yield b64_data, False
# the file was empty, return empty buffer
if offset == 0:
yield b"", False
# the last input is the actual file hash used to verify the
# transfer was ok
actual_hash = b"\x00\xffHash: " + to_bytes(sha1.hexdigest())
yield base64.b64encode(actual_hash), True
src = os.path.expanduser(os.path.expandvars(src))
b_src = to_bytes(src)
src_size = os.path.getsize(b_src)
log.info("Copying '%s' to '%s' with a total size of %d"
% (src, dest, src_size))
# check if the src size is twice as large as the max payload and fetch
# the max size from the server, we only check in this case to save on a
# round trip if the file is small enough to fit in 2 msg's, otherwise
# we want to get the largest size possible
buffer_size = int(self.wsman.max_payload_size / 4 * 3)
if src_size > (buffer_size * 2) and \
self.wsman.max_envelope_size == 153600:
log.debug("Updating the max WSMan envelope size")
self.wsman.update_max_payload_size()
buffer_size = int(self.wsman.max_payload_size / 4 * 3)
log.info("Creating file reader with a buffer size of %d" % buffer_size)
read_gen = read_buffer(b_src, buffer_size)
command = u'''begin {
$ErrorActionPreference = "Stop"
$path = [System.IO.Path]::GetTempFileName()
$fd = [System.IO.File]::Create($path)
$algo = [System.Security.Cryptography.SHA1CryptoServiceProvider]::Create()
$bytes = @()
$expected_hash = ""
} process {
$base64_string = $input
$bytes = [System.Convert]::FromBase64String($base64_string)
if ($bytes.Count -eq 48 -and $bytes[0] -eq 0 -and $bytes[1] -eq 255) {
$hash_bytes = $bytes[-40..-1]
$expected_hash = [System.Text.Encoding]::UTF8.GetString($hash_bytes)
} else {
$algo.TransformBlock($bytes, 0, $bytes.Length, $bytes, 0) > $null
$fd.Write($bytes, 0, $bytes.Length)
}
} end {
$output_path = "%s"
$dest = New-Object -TypeName System.IO.FileInfo -ArgumentList $output_path
$fd.Close()
try {
$algo.TransformFinalBlock($bytes, 0, 0) > $null
$actual_hash = [System.BitConverter]::ToString($algo.Hash)
$actual_hash = $actual_hash.Replace("-", "").ToLowerInvariant()
if ($actual_hash -ne $expected_hash) {
$msg = "Transport failure, hash mistmatch"
$msg += "`r`nActual: $actual_hash"
$msg += "`r`nExpected: $expected_hash"
throw $msg
}
[System.IO.File]::Copy($path, $output_path, $true)
$dest.FullName
} finally {
[System.IO.File]::Delete($path)
}
}''' % to_unicode(dest)
encoded_command = to_string(base64.b64encode(to_bytes(command,
'utf-16-le')))
with WinRS(self.wsman) as shell:
process = Process(shell, "powershell.exe",
["-NoProfile", "-NonInteractive",
"-EncodedCommand", encoded_command])
process.begin_invoke()
log.info("Starting to send file data to remote process")
for input_data, end in read_gen:
process.send(input_data, end)
log.info("Finished sending file data to remote process")
process.end_invoke()
stderr = self.sanitise_clixml(process.stderr)
if process.rc != 0:
raise WinRMError("Failed to copy file: %s" % stderr)
output_file = to_unicode(process.stdout).strip()
log.info("Completed file transfer of '%s' to '%s'"
% (src, output_file))
return output_file
def test_unicode_to_bytes_default():
expected = b"\x61\x62\x63"
actual = to_bytes(u"abc")
assert actual == expected
def _normalise_xml(self, xml, generify=True, overrides=None):
overrides = overrides if overrides is not None else []
if generify:
# convert all UUID values to the blank UUID
xml = re.sub(self._uuid_pattern,
"00000000-0000-0000-0000-000000000000", xml)
xml_obj = ET.fromstring(to_bytes(xml))
# convert the To hostname in the headers to the generic one
to_field = xml_obj.find("s:Header/wsa:To", NAMESPACES)
if to_field is not None:
to_field.text = self.endpoint
for override in overrides:
override_element = xml_obj.find(override['path'], NAMESPACES)
if override.get('text'):
override_element.text = override['text']
attributes = override.get('attributes', {})
for attr_key, attr_value in attributes.items():
override_element.attrib[attr_key] = attr_value
# PSRP messages contain the runspace and pipeline ID that is
# base64 encoded, this needs to be converted back to \x00 * 16
# for mock test
creation_xml = xml_obj.find(
"s:Body/rsp:Shell/{http://schemas."
"microsoft.com/powershell}creationXml", NAMESPACES)
if creation_xml is not None:
self._generify_fragment(creation_xml)
connect_xml = xml_obj.find("s:Body/rsp:Connect/pwsh:connectXml",
NAMESPACES)
if connect_xml is not None:
self._generify_fragment(connect_xml)
# when resource uri is PowerShell we know the Send/Command messages
# contain PSRP fragments and we need to generify them
exp_res_uri = "http://schemas.microsoft.com/powershell/"
res_uri = \
xml_obj.find("s:Header/wsman:ResourceURI", NAMESPACES)
if res_uri is not None:
res_uri = res_uri.text
streams = xml_obj.findall("s:Body/rsp:Send/rsp:Stream", NAMESPACES)
if res_uri is not None and res_uri.startswith(exp_res_uri) and \
len(streams) > 0:
for stream in streams:
self._generify_fragment(stream)
command = xml_obj.find("s:Body/rsp:CommandLine/rsp:Arguments",
NAMESPACES)
if res_uri is not None and res_uri.startswith(exp_res_uri) and \
command is not None:
self._generify_fragment(command)
else:
xml_obj = ET.fromstring(to_bytes(xml))
# convert the string to an XML object, for Python 2.6 (lxml) we need
# to change the namespace handling to mimic the ElementTree way of
# working so the string compare works
if sys.version_info[0] == 2 and sys.version_info[1] < 7:
namespaces = {}
new_xml_obj = self._simplify_namespaces(namespaces, xml_obj)
for key, value in namespaces.items():
new_xml_obj.attrib["xmlns:%s" % key] = value
xml_obj = new_xml_obj
return to_string(ETNew.tostring(xml_obj, encoding='utf-8'))
def _set_auth_token(request, token, auth_provider):
encoded_token = base64.b64encode(token)
auth_header = to_bytes("%s " % auth_provider) + encoded_token
request.headers['Authorization'] = auth_header
def _normalise_xml(self,
xml,
generify=True,
overrides=None,
psrp_fragment_type=None):
if not xml:
return xml
overrides = overrides if overrides is not None else []
if generify:
# convert all UUID values to the blank UUID
xml = re.sub(self._uuid_pattern,
"00000000-0000-0000-0000-000000000000", xml)
xml_obj = ET.fromstring(to_bytes(xml))
# convert the To hostname in the headers to the generic one
to_field = xml_obj.find("s:Header/wsa:To", NAMESPACES)
if to_field is not None:
to_field.text = self.endpoint
for override in overrides:
override_element = xml_obj.find(override["path"], NAMESPACES)
if override.get("text"):
override_element.text = override["text"]
attributes = override.get("attributes", {})
for attr_key, attr_value in attributes.items():
override_element.attrib[attr_key] = attr_value
else:
xml_obj = ET.fromstring(to_bytes(xml))
# PSRP message contain another set of XML messages that have been
# base64 encoded. We need to strip these out and compare them
# separately once all the fragments have been received.
if psrp_fragment_type:
creation_xml = xml_obj.find(
"s:Body/rsp:Shell/{http://schemas.microsoft.com/powershell}creationXml",
NAMESPACES)
if creation_xml is not None:
creation_xml.text = self._generify_fragment(
creation_xml.text, psrp_fragment_type)
connect_xml = xml_obj.find("s:Body/rsp:Connect/pwsh:connectXml",
NAMESPACES)
if connect_xml is not None:
connect_xml.text = self._generify_fragment(
connect_xml.text, psrp_fragment_type)
# when resource uri is PowerShell we know the Send/Command messages
# contain PSRP fragments and we need to generify them
exp_res_uri = "http://schemas.microsoft.com/powershell/"
res_uri = xml_obj.find("s:Header/wsman:ResourceURI", NAMESPACES)
if res_uri is not None:
res_uri = res_uri.text
streams = xml_obj.findall("s:Body/rsp:Send/rsp:Stream", NAMESPACES)
if res_uri is not None and res_uri.startswith(
exp_res_uri) and len(streams) > 0:
for stream in streams:
stream.text = self._generify_fragment(
stream.text, psrp_fragment_type)
command = xml_obj.find("s:Body/rsp:CommandLine/rsp:Arguments",
NAMESPACES)
if res_uri is not None and res_uri.startswith(
exp_res_uri) and command is not None:
command.text = self._generify_fragment(command.text,
psrp_fragment_type)
return to_string(ET.tostring(xml_obj, encoding="utf-8"))
