def test_foreign_json(self):
"""
Not all JSON that we decode will be encoded by this python thrift
protocol implementation. E.g. this encode implementation stuffs raw
unicode into the output, but we may use this implementation to decode
JSON from other implementations, which escape unicode (sometimes
incorrectly e.g. PHP). And we may use this implementation to decode
JSON that was not encoded by thrift at all, which may contain nulls.
"""
s = "a fancy e looks like \u00e9"
j = '{"aString": "a fancy e looks like \\u00e9", "anotherString": null, "anInteger": 10, "unknownField": null}'
stuff = Stuff()
Serializer.deserialize(
TSimpleJSONProtocol.TSimpleJSONProtocolFactory(), j, stuff
)
self.assertEqual(stuff.aString, s)
def should_throw():
j = '{"aString": "foo", "anotherString": nullcorrupt}'
stuff = Stuff()
Serializer.deserialize(
TSimpleJSONProtocol.TSimpleJSONProtocolFactory(), j, stuff
)
self.assertRaises(TProtocol.TProtocolException, should_throw)
def test_deserializer(self):
j = '{"aShort": 1, "anInteger": 2, "aLong": 3}'
stuff = Stuff()
Serializer.deserialize(
TSimpleJSONProtocol.TSimpleJSONProtocolFactory(), j, stuff)
self.assertEqual(stuff.aShort, 1)
self.assertEqual(stuff.anInteger, 2)
self.assertEqual(stuff.aLong, 3)
def test_deserializer(self):
j = '{"aShort": 1, "anInteger": 2, "aLong": 3}'
stuff = Stuff()
Serializer.deserialize(
TSimpleJSONProtocol.TSimpleJSONProtocolFactory(), j, stuff)
self.assertEqual(stuff.aShort, 1)
self.assertEqual(stuff.anInteger, 2)
self.assertEqual(stuff.aLong, 3)
def testNestedStructs(self):
nested = NestedStructs(bonk=Bonk(), bools=Bools())
json.loads(self._serialize(nested))
# Old protocol should be able to deserialize both valid and invalid
# JSON.
protocol_factory = TJSONProtocol.TJSONProtocolFactory(validJSON=False)
Serializer.deserialize(protocol_factory,
'{"1":{"rec":{}}"2":{"rec":{}}}', NestedStructs())
Serializer.deserialize(protocol_factory,
'{"1":{"rec":{}},"2":{"rec":{}}}', NestedStructs())
def testNestedStructs(self):
nested = NestedStructs(bonk=Bonk(), bools=Bools())
json.loads(self._serialize(nested))
# Old protocol should be able to deserialize both valid and invalid
# JSON.
protocol_factory = TJSONProtocol.TJSONProtocolFactory(validJSON=False)
Serializer.deserialize(protocol_factory,
'{"1":{"rec":{}}"2":{"rec":{}}}',
NestedStructs())
Serializer.deserialize(protocol_factory,
'{"1":{"rec":{}},"2":{"rec":{}}}',
NestedStructs())
def _load_overlay_tree(overlay_dir: str, inode_number: int) -> OverlayDir:
from thrift.util import Serializer
from thrift.protocol import TCompactProtocol
dir_name = '{:02x}'.format(inode_number % 256)
overlay_file_path = os.path.join(overlay_dir, dir_name, str(inode_number))
with open(overlay_file_path, 'rb') as f:
data = f.read()
assert data[0:4] == b'OVDR'
tree_data = OverlayDir()
protocol_factory = TCompactProtocol.TCompactProtocolFactory()
Serializer.deserialize(protocol_factory, data[64:], tree_data)
return tree_data
def test_roundtrip(self) -> None:
INPUTS = {
"empty": (Foo(), FooWithoutAdapters()),
"default_values": (
Foo(
structField={},
oStructField={},
mapField={},
),
FooWithoutAdapters(
structField=Bar(),
oStructField=Bar(),
mapField={},
),
),
"basic": (
Foo(
structField={"field": 42},
oStructField={"field": 43},
mapField={
1: {
"field": 44
},
2: {
"field": 45
},
},
),
FooWithoutAdapters(
structField=Bar(field=42),
oStructField=Bar(field=43),
mapField={
1: Bar(field=44),
2: Bar(field=45),
},
),
),
}
for protocol in PROTOCOLS:
for (name, (foo, foo_without_adapters)) in INPUTS.items():
with self.subTest(case=name, protocol=type(protocol).__name__):
serialized = Serializer.serialize(protocol, foo)
deserialized = Serializer.deserialize(
protocol, serialized, Foo())
self.assertEqual(deserialized, foo)
no_adapter = Serializer.deserialize(
protocol, serialized, FooWithoutAdapters())
self.assertEqual(no_adapter, foo_without_adapters)
def serialize_v2_event(event):
"""Serialize a Thrift struct to bytes for the V2 event protocol.
:param event: A Thrift struct from the event schemas.
"""
return Serializer.serialize(_V2_PROTOCOL_FACTORY, event)
def test_exception_safety(self) -> None:
for protocol in PROTOCOLS:
with self.subTest(protocol=type(protocol).__name__):
foo = Foo(structField={})
with patch(
"thrift.test.py.adapter_for_tests.AdapterTestStructToDict.to_thrift"
) as mock_to_thrift, self.assertRaises(RuntimeError):
mock_to_thrift.side_effect = RuntimeError()
Serializer.serialize(protocol, foo)
serialized = Serializer.serialize(
protocol,
FooWithoutAdapters(structField=Bar()),
)
with patch(
"thrift.test.py.adapter_for_tests.AdapterTestStructToDict.from_thrift"
) as mock_from_thrift, self.assertRaises(RuntimeError):
mock_from_thrift.side_effect = RuntimeError()
Serializer.deserialize(protocol, serialized, Foo())
def test_foreign_json(self):
"""
Not all JSON that we decode will be encoded by this python thrift
protocol implementation. E.g. this encode implementation stuffs raw
unicode into the output, but we may use this implementation to decode
JSON from other implementations, which escape unicode (sometimes
incorrectly e.g. PHP). And we may use this implementation to decode
JSON that was not encoded by thrift at all, which may contain nulls.
"""
s = "a fancy e looks like \u00e9"
j = '{"aString": "a fancy e looks like \\u00e9", "anotherString": null, "anInteger": 10, "unknownField": null}'
stuff = Stuff()
Serializer.deserialize(
TSimpleJSONProtocol.TSimpleJSONProtocolFactory(), j, stuff)
self.assertEqual(stuff.aString, s)
def should_throw():
j = '{"aString": "foo", "anotherString": nullcorrupt}'
stuff = Stuff()
Serializer.deserialize(
TSimpleJSONProtocol.TSimpleJSONProtocolFactory(), j, stuff)
self.assertRaises(TProtocol.TProtocolException, should_throw)
def _t_request(self): # pylint: disable=method-hidden
# Importing the Thrift models inline so that building them is not a
# hard, import-time dependency for tasks like building the docs.
from .thrift.ttypes import Loid as TLoid
from .thrift.ttypes import Request as TRequest
from .thrift.ttypes import Session as TSession
_t_request = TRequest()
_t_request.loid = TLoid()
_t_request.session = TSession()
if self._header:
try:
Serializer.deserialize(
self._HEADER_PROTOCOL_FACTORY,
self._header,
_t_request,
)
except Exception:
logger.debug(
"Invalid Edge-Request header. %s",
self._header,
)
return _t_request
def do_stats_general(instance: EdenInstance,
options: StatsGeneralOptions) -> None:
out, basic, json = options
with instance.get_thrift_client_legacy(timeout=20) as client:
statsMask = STATS_MOUNTS_STATS | STATS_RSS_BYTES if basic else STATS_ALL
stat_info = client.getStatInfo(GetStatInfoParams(statsMask=statsMask))
if json:
json_factory = TSimpleJSONProtocolFactory()
out.write(
ThriftSerializer.serialize(json_factory,
stat_info).decode("utf-8"))
else:
print_stats(stat_info, out)
def create(cls,
authentication_context=None,
loid_id=None,
loid_created_ms=None,
session_id=None):
"""Factory method to create a new EdgeRequestContext object.
Builds a new EdgeRequestContext object with the given parameters and
serializes the "Edge-Request" header.
:param baseplate.core.AuthenticationContext authentication_context:
(Optional) AuthenticationContext for the current request if it is
authenticated.
:param str loid_id: (Optional) ID for the current LoID in fullname
format.
:param int loid_created_ms: (Optional) Epoch milliseconds when the
current LoID cookie was created.
:param str session_id: (Optional) ID for the current session cookie.
"""
# Importing the Thrift models inline so that building them is not a
# hard, import-time dependency for tasks like building the docs.
from .thrift.ttypes import Loid as TLoid
from .thrift.ttypes import Request as TRequest
from .thrift.ttypes import Session as TSession
if loid_id is not None and not loid_id.startswith("t2_"):
raise ValueError(
"loid_id <%s> is not in a valid format, it should be in the "
"fullname format with the '0' padding removed: 't2_loid_id'",
loid_id)
loid = TLoid(id=loid_id, created_ms=loid_created_ms)
session = TSession(id=session_id)
request = TRequest(loid=loid, session=session)
header = Serializer.serialize(cls._HEADER_PROTOCOL_FACTORY, request)
if authentication_context is None:
authentication_context = AuthenticationContext()
request_context = cls(header, authentication_context)
# Set the _t_request property so we can skip the deserialization step
# since we already have the thrift object.
request_context._t_request = request
return request_context
def _serialize(self, obj):
return Serializer.serialize(self.serialize_factory, obj)
def _serialize(self, obj):
return Serializer.serialize(self.serialize_factory, obj)
def should_throw():
j = '{"aString": "foo", "anotherString": nullcorrupt}'
stuff = Stuff()
Serializer.deserialize(
TSimpleJSONProtocol.TSimpleJSONProtocolFactory(), j, stuff
)
def should_throw():
j = '{"aString": "foo", "anotherString": nullcorrupt}'
stuff = Stuff()
Serializer.deserialize(
TSimpleJSONProtocol.TSimpleJSONProtocolFactory(), j, stuff)
def writeToJSON(obj):
return Serializer.serialize(NomadJSONProtocolFactory(), obj)
def _serialize(self, obj):
return Serializer.serialize(self.protocol_factory, obj)
def fromJson(msg, spec):
return Serializer.deserialize(TSimpleJSONProtocolFactory(), msg, spec)
def testNestedStructs(self):
self._deserialize(NestedStructs, '{"1":{"rec":{}}"2":{"rec":{}}}')
self._deserialize(NestedStructs, '{"1":{"rec":{}},"2":{"rec":{}}}')
nested = NestedStructs(bonk=Bonk(), bools=Bools())
protocol_factory = TJSONProtocol.TJSONProtocolFactory(validJSON=True)
json.loads(Serializer.serialize(protocol_factory, nested))
def _deserialize(self, objtype, data):
return Serializer.deserialize(self.protocol_factory, data, objtype())
def new(self,
authentication_token=None,
loid_id=None,
loid_created_ms=None,
session_id=None):
"""Return a new EdgeRequestContext object made from scratch.
Services at the edge that communicate directly with clients should use
this to pass on the information they get to downstream services. They
can then use this information to check authentication, run experiments,
etc.
To use this, create and attach the context early in your request flow:
.. code-block:: python
auth_cookie = request.cookies["authentication"]
token = request.authentication_service.authenticate_cookie(cookie)
loid = parse_loid(request.cookies["loid"])
session = parse_session(request.cookies["session"])
edge_context = self.edgecontext_factory.new(
authentication_token=token,
loid_id=loid.id,
loid_created_ms=loid.created,
session_id=session.id,
)
edge_context.attach_context(request)
:param authentication_token: (Optional) A raw authentication token
as returned by the authentication service.
:param str loid_id: (Optional) ID for the current LoID in fullname
format.
:param int loid_created_ms: (Optional) Epoch milliseconds when the
current LoID cookie was created.
:param str session_id: (Optional) ID for the current session cookie.
"""
# Importing the Thrift models inline so that building them is not a
# hard, import-time dependency for tasks like building the docs.
from .thrift.ttypes import Loid as TLoid
from .thrift.ttypes import Request as TRequest
from .thrift.ttypes import Session as TSession
if loid_id is not None and not loid_id.startswith("t2_"):
raise ValueError(
"loid_id <%s> is not in a valid format, it should be in the "
"fullname format with the '0' padding removed: 't2_loid_id'",
loid_id)
t_request = TRequest(
loid=TLoid(id=loid_id, created_ms=loid_created_ms),
session=TSession(id=session_id),
authentication_token=authentication_token,
)
header = Serializer.serialize(
EdgeRequestContext._HEADER_PROTOCOL_FACTORY, t_request)
context = EdgeRequestContext(self.authn_token_validator, header)
# Set the _t_request property so we can skip the deserialization step
# since we already have the thrift object.
context._t_request = t_request
return context
