def test_serialize_byte_string_py3(self):
serialzier = Serializer()
expected = "<BA>YWJj</BA>"
actual = serialzier.serialize(b"abc")
actual_xml = to_unicode(ET.tostring(actual))
expected_xml = to_unicode(expected)
assert actual_xml == expected_xml
def execute_cmd(self, command, encoding='437', environment=None):
"""
Executes a command in a cmd shell and returns the stdout/stderr/rc of
that process. This uses the raw WinRS layer and can be used to execute
a traditional process.
:param command: The command to execute
:param encoding: The encoding of the output std buffers, this
correlates to the codepage of the host and traditionally en-US
is 437. This probably doesn't need to be modified unless you are
running a different codepage on your host
:param environment: A dictionary containing environment keys and
values to set on the executing process.
:return: A tuple of
stdout: A unicode string of the stdout
stderr: A unicode string of the stderr
rc: The return code of the process
Both stdout and stderr are returned from the server as a byte string,
they are converted to a unicode string based on the encoding variable
set
"""
log.info("Executing cmd process '%s'" % command)
with WinRS(self.wsman, environment=environment) as shell:
process = Process(shell, command)
process.invoke()
process.signal(SignalCode.CTRL_C)
return to_unicode(process.stdout, encoding), \
to_unicode(process.stderr, encoding), process.rc
def read_buffer(b_path, total_size, buffer_size):
offset = 0
sha1 = hashlib.sha1()
with open(b_path, 'rb') as src_file:
for data in iter((lambda: src_file.read(buffer_size)), b""):
log.debug(
"Reading data of file at offset=%d with size=%d" %
(offset, buffer_size))
offset += len(data)
sha1.update(data)
b64_data = base64.b64encode(data)
result = [to_unicode(b64_data)]
if offset == total_size:
result.append(
to_unicode(
base64.b64encode(to_bytes(sha1.hexdigest()))))
yield result
# the file was empty, return empty buffer
if offset == 0:
yield [
u"",
to_unicode(base64.b64encode(to_bytes(
sha1.hexdigest())))
]
def _deserialize_string(
self,
value: typing.Optional[str],
) -> str:
if value is None:
return ""
def rplcr(matchobj):
# The matched object is the UTF-16 byte representation of the UTF-8
# hex string value. We need to decode the byte str to unicode and
# then unhexlify that hex string to get the actual bytes of the
# _x****_ value, e.g.
# group(0) == b"\x00_\x00x\x000\x000\x000\x00A\x00_"
# group(1) == b"\x000\x000\x000\x00A"
# unicode (from utf-16-be) == u"000A"
# returns b"\x00\x0A"
match_hex = matchobj.group(1)
hex_string = to_unicode(match_hex, encoding="utf-16-be")
return binascii.unhexlify(hex_string)
# need to ensure we start with a unicode representation of the string
# so that we can get the actual UTF-16 bytes value from that string
unicode_value = to_unicode(value)
unicode_bytes = to_bytes(unicode_value, encoding="utf-16-be")
bytes_value = re.sub(self._deserial_str, rplcr, unicode_bytes)
return to_unicode(bytes_value, encoding="utf-16-be")
def test_serialize_primitives(self, data, expected):
serializer = Serializer()
actual = serializer.serialize(data)
actual_xml = to_unicode(ET.tostring(actual))
expected_xml = to_unicode(expected)
assert actual_xml == expected_xml
deserial_actual = serializer.deserialize(actual)
assert deserial_actual == data
def test_serialize_primitives(self, data, expected):
serializer = Serializer()
actual = serializer.serialize(data)
actual_xml = to_unicode(ET.tostring(actual))
expected_xml = to_unicode(expected)
assert actual_xml == expected_xml
deserial_actual = serializer.deserialize(actual)
if isinstance(data, TaggedValue):
data = data.value
assert deserial_actual == data
def rplcr(matchobj):
surrogate_char = matchobj.group(0)
byte_char = to_bytes(surrogate_char, encoding='utf-16-be')
hex_char = to_unicode(binascii.hexlify(byte_char)).upper()
hex_split = [hex_char[i:i + 4] for i in range(0, len(hex_char), 4)]
return u"".join([u"_x%s_" % i for i in hex_split])
def _send_request(self, request: requests.PreparedRequest) -> bytes:
response = self.session.send(request, timeout=(self.connection_timeout, self.read_timeout)) # type: ignore[union-attr] # This should not happen
content_type = response.headers.get("content-type", "")
if content_type.startswith("multipart/encrypted;") or content_type.startswith("multipart/x-multi-encrypted;"):
boundary = re.search("boundary=[" '|\\"](.*)[' '|\\"]', response.headers["content-type"]).group(1) # type: ignore[union-attr] # This should not happen
response_content = self.encryption.unwrap_message(response.content, to_unicode(boundary)) # type: ignore[union-attr] # This should not happen
response_text = to_string(response_content)
else:
response_content = response.content
response_text = response.text if response_content else ""
log.debug("Received message: %s" % response_text)
# for testing, keep commented out
# self._test_messages[-1]['response'] = response_text
try:
response.raise_for_status()
except requests.HTTPError as err:
response = err.response
if response.status_code == 401:
raise AuthenticationError("Failed to authenticate the user %s with %s" % (self.username, self.auth))
else:
code = response.status_code
raise WinRMTransportError("http", code, response_text)
return response_content
def test_client_copy_expand_vars(self, wsman_conn):
client = self._get_client(wsman_conn)
test_string = b"abcdefghijklmnopqrstuvwxyz"
temp_file, path = tempfile.mkstemp()
try:
os.write(temp_file, test_string)
actual = client.copy(path,
"%TEMP%\\test_file",
expand_variables=True)
# run it a 2nd time to ensure it doesn't fail
actual = client.copy(path, actual)
finally:
os.close(temp_file)
os.remove(path)
try:
# verify the returned object is the full path
assert actual == "C:\\Users\\vagrant\\AppData\\Local\\Temp\\test_file"
actual_content = client.execute_cmd(
"powershell.exe Get-Content %s" % actual)[0].strip()
assert actual_content == to_unicode(test_string)
finally:
client.execute_cmd("powershell Remove-Item -Path '%s'" % actual)
def _serialize_string(
self,
value: typing.Optional[str],
) -> typing.Optional[str]:
if value is None:
return None
def rplcr(matchobj):
surrogate_char = matchobj.group(0)
byte_char = to_bytes(surrogate_char, encoding="utf-16-be")
hex_char = to_unicode(binascii.hexlify(byte_char)).upper()
hex_split = [hex_char[i:i + 4] for i in range(0, len(hex_char), 4)]
return "".join(["_x%s_" % i for i in hex_split])
# before running the translation we need to make sure _ before x is
# encoded, normally _ isn't encoded except when preceding x
string_value = to_unicode(value)
# The MS-PSRP docs don't state this but the _x0000_ matcher is case insensitive so we need to make sure we
# escape _X as well as _x.
string_value = re.sub("(?i)_(x)", "_x005F_\\1", string_value)
string_value = re.sub(self._serial_str, rplcr, string_value)
return string_value
def test_str_to_unicode():
if PY3:
expected = u"a\x00b\x00c\x00"
else:
expected = u"abc"
actual = to_unicode("a\x00b\x00c\x00", encoding='utf-16-le')
assert actual == expected
def sanitise_clixml(clixml):
"""
When running a powershell script in execute_cmd (WinRS), the stderr
stream may contain some clixml. This method will clear it up and
replace it with the error string it would represent. This isn't done
by default on execute_cmd for various reasons but people can call it
manually here if they like.
:param clixml: The clixml to parse
:return: A unicode code string of the decoded output
"""
output = to_unicode(clixml)
if output.startswith("#< CLIXML"):
# Strip off the '#< CLIXML\r\n' by finding the 2nd index of '<'
output = output[clixml.index('<', 2):]
element = ET.fromstring(output)
namespace = element.tag.replace("Objs", "")[1:-1]
errors = []
for error in element.findall("{%s}S[@S='Error']" % namespace):
errors.append(error.text)
output = Serializer()._deserialize_string("".join(errors))
return output
def rplcr(matchobj):
# The matched object is the UTF-16 byte representation of the UTF-8
# hex string value. We need to decode the byte str to unicode and
# then unhexlify that hex string to get the actual bytes of the
# _x****_ value, e.g.
# group(0) == b"\x00_\x00x\x000\x000\x000\x00A\x00_"
# group(1) == b"\x000\x000\x000\x00A"
# unicode (from utf-16-be) == u"000A"
# returns b"\x00\x0A"
match_hex = matchobj.group(1)
hex_string = to_unicode(match_hex, encoding='utf-16-be')
return binascii.unhexlify(hex_string)
def _create_endpoint(ssl, server, port, path):
scheme = "https" if ssl else "http"
# Check if the server is an IPv6 Address, enclose in [] if it is
try:
address = ipaddress.IPv6Address(to_unicode(server))
except ipaddress.AddressValueError:
pass
else:
server = "[%s]" % address.compressed
endpoint = "%s://%s:%s/%s" % (scheme, server, port, path)
return endpoint
def _deserialize_secure_string(self, value):
if self.cipher is None:
# cipher is not set up so we can't decrypt the string, just return
# the raw element
return value
ss_string = base64.b64decode(value.text)
decryptor = self.cipher.decryptor()
decrypted_bytes = decryptor.update(ss_string) + decryptor.finalize()
unpadder = PKCS7(self.cipher.algorithm.block_size).unpadder()
unpadded_bytes = unpadder.update(decrypted_bytes) + unpadder.finalize()
decrypted_string = to_unicode(unpadded_bytes, 'utf-16-le')
return decrypted_string
def _serialize_string(self, value):
if value is None:
return None
def rplcr(matchobj):
surrogate_char = matchobj.group(0)
byte_char = to_bytes(surrogate_char, encoding='utf-16-be')
hex_char = to_unicode(binascii.hexlify(byte_char)).upper()
hex_split = [hex_char[i:i + 4] for i in range(0, len(hex_char), 4)]
return u"".join([u"_x%s_" % i for i in hex_split])
# before running the translation we need to make sure _ before x is
# encoded, normally _ isn't encoded except when preceding x
string_value = to_unicode(value)
string_value = string_value.replace(u"_x", u"_x005F_x")
string_value = re.sub(self._serial_str, rplcr, string_value)
return string_value
def test_client_copy_file(self, wsman_conn):
client = self._get_client(wsman_conn)
test_string = b"abcdefghijklmnopqrstuvwxyz"
temp_file, path = tempfile.mkstemp()
try:
os.write(temp_file, test_string)
actual = client.copy(path, "test_file")
# run it a 2nd time to ensure it doesn't fail
actual = client.copy(path, actual)
finally:
os.close(temp_file)
os.remove(path)
try:
# verify the returned object is the full path
assert actual.startswith("C:\\Users\\")
actual_content = client.execute_cmd(
"powershell.exe Get-Content %s" % actual)[0].strip()
assert actual_content == to_unicode(test_string)
finally:
client.execute_cmd("powershell Remove-Item -Path '%s'" % actual)
def test_unicode_to_unicode():
expected = u"abc"
actual = to_unicode(u"abc")
assert actual == expected
def test_str_to_unicode():
expected = "a\x00b\x00c\x00"
actual = to_unicode("a\x00b\x00c\x00", encoding="utf-16-le")
assert actual == expected
def test_byte_to_unicode():
expected = u"abc"
actual = to_unicode(b"\x61\x62\x63")
assert actual == expected
def copy(self,
src,
dest,
configuration_name=DEFAULT_CONFIGURATION_NAME,
expand_variables=False):
"""
Copies a single file from the current host to the remote Windows host.
This can be quite slow when it comes to large files due to the
limitations of WinRM but it is designed to be as fast as it can be.
During the copy process, the bytes will be stored in a temporary file
before being copied.
When copying it will replace the file at dest if one already exists. It
also will verify the checksum of the copied file is the same as the
actual file locally before copying the file to the path at dest.
:param src: The path to the local file
:param dest: The path to the destination file on the Windows host
:param configuration_name: The PowerShell configuration endpoint to
use when copying the file.
:param expand_variables: Expand variables in path. Disabled by default
Enable for cmd like expansion (for example %TMP% in path)
:return: The absolute path of the file on the Windows host
"""
def read_buffer(b_path, total_size, buffer_size):
offset = 0
sha1 = hashlib.sha1()
with open(b_path, 'rb') as src_file:
for data in iter((lambda: src_file.read(buffer_size)), b""):
log.debug(
"Reading data of file at offset=%d with size=%d" %
(offset, buffer_size))
offset += len(data)
sha1.update(data)
b64_data = base64.b64encode(data)
result = [to_unicode(b64_data)]
if offset == total_size:
result.append(
to_unicode(
base64.b64encode(to_bytes(sha1.hexdigest()))))
yield result
# the file was empty, return empty buffer
if offset == 0:
yield [
u"",
to_unicode(base64.b64encode(to_bytes(
sha1.hexdigest())))
]
if expand_variables:
src = os.path.expanduser(os.path.expandvars(src))
b_src = to_bytes(src)
src_size = os.path.getsize(b_src)
log.info("Copying '%s' to '%s' with a total size of %d" %
(src, dest, src_size))
with RunspacePool(self.wsman,
configuration_name=configuration_name) as pool:
# Get the buffer size of each fragment to send, subtract. Adjust to size of the base64 encoded bytes. Also
# subtract 82 for the fragment, message, and other header info that PSRP adds.
buffer_size = int((self.wsman.max_payload_size - 82) / 4 * 3)
log.info("Creating file reader with a buffer size of %d" %
buffer_size)
read_gen = read_buffer(b_src, src_size, buffer_size)
command = get_pwsh_script('copy.ps1')
log.debug("Starting to send file data to remote process")
powershell = PowerShell(pool)
powershell.add_script(command) \
.add_argument(dest) \
.add_argument(expand_variables)
powershell.invoke(input=read_gen)
_handle_powershell_error(powershell, "Failed to copy file")
log.debug("Finished sending file data to remote process")
for warning in powershell.streams.warning:
warnings.warn(str(warning))
output_file = to_unicode(powershell.output[-1]).strip()
log.info("Completed file transfer of '%s' to '%s'" %
(src, output_file))
return output_file
def test_byte_to_unicode_diff_encoding():
expected = u"abc"
actual = to_unicode(b"\x61\x00\x62\x00\x63\x00", encoding='utf-16-le')
assert actual == expected
def normalise_xml(xml_string):
xml = "".join([l.lstrip() for l in to_unicode(xml_string).splitlines()])
xml = ETInbuilt.fromstring(xml)
return to_unicode(ETInbuilt.tostring(xml))
def copy(self, src, dest):
"""
Copies a single file from the current host to the remote Windows host.
This can be quite slow when it comes to large files due to the
limitations of WinRM but it is designed to be as fast as it can be.
During the copy process, the bytes will be stored in a temporary file
before being copied.
When copying it will replace the file at dest if one already exists. It
also will verify the checksum of the copied file is the same as the
actual file locally before copying the file to the path at dest.
:param src: The path to the local file
:param dest: The path to the destionation file on the Windows host
:return: The absolute path of the file on the Windows host
"""
def read_buffer(b_path, buffer_size):
offset = 0
sha1 = hashlib.sha1()
with open(b_path, 'rb') as src_file:
for data in iter((lambda: src_file.read(buffer_size)), b""):
log.debug("Reading data of file at offset=%d with size=%d"
% (offset, buffer_size))
offset += len(data)
sha1.update(data)
b64_data = base64.b64encode(data) + b"\r\n"
yield b64_data, False
# the file was empty, return empty buffer
if offset == 0:
yield b"", False
# the last input is the actual file hash used to verify the
# transfer was ok
actual_hash = b"\x00\xffHash: " + to_bytes(sha1.hexdigest())
yield base64.b64encode(actual_hash), True
src = os.path.expanduser(os.path.expandvars(src))
b_src = to_bytes(src)
src_size = os.path.getsize(b_src)
log.info("Copying '%s' to '%s' with a total size of %d"
% (src, dest, src_size))
# check if the src size is twice as large as the max payload and fetch
# the max size from the server, we only check in this case to save on a
# round trip if the file is small enough to fit in 2 msg's, otherwise
# we want to get the largest size possible
buffer_size = int(self.wsman.max_payload_size / 4 * 3)
if src_size > (buffer_size * 2) and \
self.wsman.max_envelope_size == 153600:
log.debug("Updating the max WSMan envelope size")
self.wsman.update_max_payload_size()
buffer_size = int(self.wsman.max_payload_size / 4 * 3)
log.info("Creating file reader with a buffer size of %d" % buffer_size)
read_gen = read_buffer(b_src, buffer_size)
command = u'''begin {
$ErrorActionPreference = "Stop"
$path = [System.IO.Path]::GetTempFileName()
$fd = [System.IO.File]::Create($path)
$algo = [System.Security.Cryptography.SHA1CryptoServiceProvider]::Create()
$bytes = @()
$expected_hash = ""
} process {
$base64_string = $input
$bytes = [System.Convert]::FromBase64String($base64_string)
if ($bytes.Count -eq 48 -and $bytes[0] -eq 0 -and $bytes[1] -eq 255) {
$hash_bytes = $bytes[-40..-1]
$expected_hash = [System.Text.Encoding]::UTF8.GetString($hash_bytes)
} else {
$algo.TransformBlock($bytes, 0, $bytes.Length, $bytes, 0) > $null
$fd.Write($bytes, 0, $bytes.Length)
}
} end {
$output_path = "%s"
$dest = New-Object -TypeName System.IO.FileInfo -ArgumentList $output_path
$fd.Close()
try {
$algo.TransformFinalBlock($bytes, 0, 0) > $null
$actual_hash = [System.BitConverter]::ToString($algo.Hash)
$actual_hash = $actual_hash.Replace("-", "").ToLowerInvariant()
if ($actual_hash -ne $expected_hash) {
$msg = "Transport failure, hash mistmatch"
$msg += "`r`nActual: $actual_hash"
$msg += "`r`nExpected: $expected_hash"
throw $msg
}
[System.IO.File]::Copy($path, $output_path, $true)
$dest.FullName
} finally {
[System.IO.File]::Delete($path)
}
}''' % to_unicode(dest)
encoded_command = to_string(base64.b64encode(to_bytes(command,
'utf-16-le')))
with WinRS(self.wsman) as shell:
process = Process(shell, "powershell.exe",
["-NoProfile", "-NonInteractive",
"-EncodedCommand", encoded_command])
process.begin_invoke()
log.info("Starting to send file data to remote process")
for input_data, end in read_gen:
process.send(input_data, end)
log.info("Finished sending file data to remote process")
process.end_invoke()
stderr = self.sanitise_clixml(process.stderr)
if process.rc != 0:
raise WinRMError("Failed to copy file: %s" % stderr)
output_file = to_unicode(process.stdout).strip()
log.info("Completed file transfer of '%s' to '%s'"
% (src, output_file))
return output_file