def test_msgpack(everything: Everything):
converter = msgpack_make_converter()
raw = msgpack_dumps(converter.unstructure(everything))
assert (converter.structure(
msgpack_loads(raw, strict_map_key=False),
Everything,
) == everything)
def test_msgpack(everything: Everything):
from msgpack import dumps as msgpack_dumps
from msgpack import loads as msgpack_loads
converter = msgpack_make_converter()
raw = msgpack_dumps(converter.unstructure(everything))
assert (converter.structure(msgpack_loads(raw, strict_map_key=False),
Everything) == everything)
def __init__(self, key, value):
# Performance note: except for the initial value unpacking, all
# expensive deserialization operations are deferred until
# requested.
unpacked = msgpack_loads(value, use_list=False)
# IP addresses
self.begin_ip_packed = unpacked[0]
self.end_ip_packed = key
# Actual data, without any expensive decoding applied
self.latest_json = unpacked[1]
self.latest_datetime = unpacked[2].decode('ascii')
self.history_msgpack = unpacked[3]
def __init__(self, key, value):
# Performance note: except for the initial value unpacking, all
# expensive deserialization operations are deferred until
# requested.
unpacked = msgpack_loads(value, use_list=False)
# IP addresses
self.begin_ip_packed = unpacked[0]
self.end_ip_packed = key
# Actual data, without any expensive decoding applied
self.latest_json = unpacked[1]
self.latest_datetime = unpacked[2].decode('ascii')
self.history_msgpack = unpacked[3]
def from_exported_config(config: str) -> 'ScrapeConfig':
try:
from msgpack import loads as msgpack_loads
except ImportError as e:
print(
'You must install msgpack package - run: pip install "scrapfly-sdk[seepdup] or pip install msgpack'
)
raise
data = msgpack_loads(base64.b64decode(config))
headers = {}
for name, value in data['headers'].items():
if isinstance(value, Iterable):
headers[name] = '; '.join(value)
else:
headers[name] = value
return ScrapeConfig(url=data['url'],
retry=data['retry'],
headers=headers,
session=data['session'],
session_sticky_proxy=data['session_sticky_proxy'],
cache=data['cache'],
cache_ttl=data['cache_ttl'],
cache_clear=data['cache_clear'],
render_js=data['render_js'],
method=data['method'],
asp=data['asp'],
body=data['body'],
ssl=data['ssl'],
dns=data['dns'],
country=data['country'],
debug=data['debug'],
correlation_id=data['correlation_id'],
tags=data['tags'],
graphql=data['graphql_query'],
js=data['js'],
rendering_wait=data['rendering_wait'],
screenshots=data['screenshots'] or {},
proxy_pool=data['proxy_pool'])
def loads(cls, data: bytes) -> list:
return msgpack_loads(data, ext_hook=cls._decode, raw=False)