def test_pickle_categorical_column(slices):
sr = Series(["a", "b", None, "a", "c", "b"]).astype("category")
sliced_sr = sr.iloc[slices]
input_col = sliced_sr._column
pickled = pickle.dumps(input_col)
out = pickle.loads(pickled)
assert_eq(Series(out), Series(input_col))
def deserialize(self, header: Dict, buffers: List, context: Dict):
obj_type = self.obj_type
if 'obj_type' in header:
obj_type = pickle.loads(header['obj_type'])
if hasattr(obj_type, '_fields'):
# namedtuple
return obj_type(*self._iter_deserial(header['headers'], buffers, context))
else:
return obj_type(self._iter_deserial(header['headers'], buffers, context))
def test_displacement_pickle(self):
displacement = sitk.Image((512, 512), sitk.sitkVectorFloat64, 2)
displacement.SetOrigin((6, 5.2))
tx = sitk.DisplacementFieldTransform(displacement)
dump = pickle.dumps(tx)
tx2 = pickle.loads(dump)
self.assertEqual(tx, tx2)
def test_affine_pickle(self):
tx = sitk.AffineTransform(3)
tx.SetCenter([2.3, 4.5, 6.7])
tx.SetMatrix([9, 8, 7, 6, 5, 4, 3, 2, 1])
dump = pickle.dumps(tx)
tx2 = pickle.loads(dump)
self.assertEqual(tx, tx2)
def test_rmm_device_buffer_pickle_roundtrip(hb):
db = rmm.DeviceBuffer.to_device(hb)
pb = pickle.dumps(db)
del db
db2 = pickle.loads(pb)
hb2 = db2.tobytes()
assert hb == hb2
# out-of-band
if pickle.HIGHEST_PROTOCOL >= 5:
db = rmm.DeviceBuffer.to_device(hb)
buffers = []
pb2 = pickle.dumps(db, protocol=5, buffer_callback=buffers.append)
del db
assert len(buffers) == 1
assert isinstance(buffers[0], pickle.PickleBuffer)
assert bytes(buffers[0]) == hb
db3 = pickle.loads(pb2, buffers=buffers)
hb3 = db3.tobytes()
assert hb3 == hb
def test_pickle_series(named):
np.random.seed(0)
if named:
ser = Series(np.random.random(10), name="a")
else:
ser = Series(np.random.random(10))
pickled = pickle.dumps(ser)
out = pickle.loads(pickled)
assert (ser == out).all()
def deserialize(self, header: Dict, buffers: List, context: Dict):
obj_type = self.obj_type
if "obj_type" in header:
obj_type = pickle.loads(header["obj_type"])
if hasattr(obj_type, "_fields"):
# namedtuple
return obj_type(
*(yield from self._list_deserial(header["headers"], buffers)))
else:
return obj_type((yield from
self._list_deserial(header["headers"], buffers)))
def from_bytes(bytes_graph: bytes, check_version: bool = True) -> BELGraph:
"""Read a graph from bytes (the result of pickling the graph).
:param bytes_graph: File or filename to write
:param check_version: Checks if the graph was produced by this version of PyBEL
"""
graph = pickle.loads(bytes_graph)
raise_for_not_bel(graph)
if check_version:
raise_for_old_graph(graph)
return graph
def deserialize(self, header: Dict, buffers: List):
key_buffers = buffers[:header['key_buf_num']]
value_buffers = buffers[header['key_buf_num']:]
obj_type = dict
if 'obj_type' in header:
obj_type = pickle.loads(header['obj_type'])
return obj_type(
zip(
self._iter_deserial(header['key_headers'], key_buffers),
self._iter_deserial(header['value_headers'], value_buffers),
))
def get_pkl(bucket_name, key):
s3 = boto3.resource('s3')
obj = s3.Object(
bucket_name=bucket_name,
key=key,
)
obj_body = obj.get()['Body'].read()
data = pickle.loads(obj_body)
return data
def process_holistic(s, indexes, start=None, end=None):
keypoints = pickle5.loads(lzma.decompress(s['sign']))
keypoints = keypoints[start:end]
# Holistic
keypoints = keypoints.reshape(
(-1, 543, 4)).astype('float32')[:, :, :3] # [0, 1, 3] for visibility
# Openpose
# keypoints = keypoints.reshape((-1, 135, 3)).astype('float32')
# shoulder_means = (keypoints[:, 5:6, :2] + keypoints[:, 6:7, :2]) / 2
# keypoints[:, :, :2] -= shoulder_means
# TODO Reconcile
#keypoints[:, :, 1] = - keypoints[:, :, 1]
keypoints = rearrange(keypoints, "T V C -> C T V ()")
# T V C -> C T V M
keypoints = keypoints[:, :, indexes]
return keypoints
def test_bspline_pickle(self):
M = [0, 1, 1, 0]
img1 = sitk.Image([10, 10], sitk.sitkFloat64)
img1.SetOrigin((.01, 5.2))
img1.SetDirection(M)
img1 += -.1
img2 = sitk.Image([10, 10], sitk.sitkFloat64)
img2.SetOrigin([.01, 5.2])
img2.SetDirection(M)
img2 -= 0.1
tx = sitk.BSplineTransform([img1, img2], 3)
dump = pickle.dumps(tx)
tx2 = pickle.loads(dump)
self.assertEqual(tx, tx2)
def check_serialization(df):
# basic
assert_frame_picklable(df)
# sliced
assert_frame_picklable(df[:-1])
assert_frame_picklable(df[1:])
assert_frame_picklable(df[2:-2])
# sorted
sortvaldf = df.sort_values("vals")
assert isinstance(sortvaldf.index, GenericIndex)
assert_frame_picklable(sortvaldf)
# out-of-band
if pickle.HIGHEST_PROTOCOL >= 5:
buffers = []
serialbytes = pickle.dumps(
df, protocol=5, buffer_callback=buffers.append
)
for b in buffers:
assert isinstance(b, pickle.PickleBuffer)
loaded = pickle.loads(serialbytes, buffers=buffers)
assert_eq(loaded, df)
def test_pickle(input_type, protocol):
if protocol > pickle.HIGHEST_PROTOCOL:
pytest.skip(
f"Trying to test with pickle protocol {protocol},"
f" but highest supported protocol is {pickle.HIGHEST_PROTOCOL}."
)
if input_type == 'series':
inp = create_input(input_type, np.float32, (10, 1), 'C')
else:
inp = create_input(input_type, np.float32, (10, 5), 'F')
ary = CumlArray(data=inp)
dumps_kwargs = {"protocol": protocol}
loads_kwargs = {}
f = []
len_f = 0
if protocol >= 5:
dumps_kwargs["buffer_callback"] = f.append
loads_kwargs["buffers"] = f
len_f = 1
a = pickle.dumps(ary, **dumps_kwargs)
b = pickle.loads(a, **loads_kwargs)
assert len(f) == len_f
if input_type == 'numpy':
assert np.all(inp == b.to_output('numpy'))
elif input_type == 'series':
assert np.all(inp == b.to_output('series'))
else:
assert cp.all(inp == cp.asarray(b))
assert ary.__cuda_array_interface__['shape'] == \
b.__cuda_array_interface__['shape']
assert ary.__cuda_array_interface__['strides'] == \
b.__cuda_array_interface__['strides']
assert ary.__cuda_array_interface__['typestr'] == \
b.__cuda_array_interface__['typestr']
if input_type != 'series':
# skipping one dimensional ary order test
assert ary.order == b.order
def retrieve_model(bucket_name, source_blob_name):
"""Import pickle model already trained form bucket.
Arguments
---------
bucket_name: str
Bucket name.
source_blob_name: str
Path to the file trained model in bucket.
Return
------
model: scikit-learn Pipeline
Fitted pipeline.
"""
# Google Storage Client
storage_client = storage.Client()
bucket = storage_client.bucket(bucket_name)
blob = bucket.blob(source_blob_name)
pickle_in = blob.download_as_bytes()
# Model as pickle
model = pickle.loads(pickle_in)
return model
def thread(s):
client_socket, address = s.accept()
print(f"[+] {address} is connected.")
received = pickle.loads(client_socket.recv(10240))
filename = received["filename"]
filesize = received["filesize"]
filetype = received["filetype"]
filehash = received["filehash"]
fileaddr = received["fileaddr"]
# filename, filesize, filetype, filehash, fileaddr = received.split(SEPARATOR)
print(filetype)
print(fileaddr)
if filetype == "temp":
filename = os.path.basename(settings.TEMP_STORAGE_PATH + filename)
else:
filename = os.path.basename(settings.STORAGE_PATH + filename)
filesize = int(filesize)
with open(filename, "wb") as f:
while True:
bytes_read = client_socket.recv(BUFFER_SIZE)
if not bytes_read:
break
f.write(bytes_read)
if filetype == "temp":
# split and broadcast
# No. of splits to be made --> n
print(fileaddr)
key = file_split(filename)[0]
val = file_split(filename)[1]
file_send(key, filehash, fileaddr, filename, val)
client_socket.close()
else:
client_socket.close()
def device_deserialize(cls, header, frames):
"""Perform device-side deserialization tasks.
The primary purpose of this method is the creation of device memory
buffers from host buffers where necessary.
Parameters
----------
header : dict
The metadata required to reconstruct the object.
frames : list
The Buffers or memoryviews that the object should contain.
Returns
-------
Serializable
A new instance of `cls` (a subclass of `Serializable`) equivalent
to the instance that was serialized to produce the header and
frames.
:meta private:
"""
typ = pickle.loads(header["type-serialized"])
frames = [
cudf.core.buffer.Buffer(f) if c else memoryview(f)
for c, f in zip(header["is-cuda"], frames)
]
assert all(
(type(f._owner) is rmm.DeviceBuffer)
if c
else (type(f) is memoryview)
for c, f in zip(header["is-cuda"], frames)
)
obj = typ.deserialize(header, frames)
return obj
def test_pickle_index():
nelem = 10
idx = GenericIndex(np.arange(nelem), name="a")
pickled = pickle.dumps(idx)
out = pickle.loads(pickled)
assert idx == out
def assert_frame_picklable(df):
serialbytes = pickle.dumps(df)
loaded = pickle.loads(serialbytes)
assert_eq(loaded, df)
# import pickle import pickle5 as pickle pb = pickle.PickleBuffer(b"foo") data = pickle.dumps(pb, protocol=5) assert pickle.loads(data) == b"foo"
import pickle5 as pickle
import socket
import struct
HEADER_SIZE = 4
HOST = "127.0.0.1"
PORT = 12001
def receive(nb_bytes, conn):
# Ensure that exactly the desired amount of bytes is received
received = bytearray()
while len(received) < nb_bytes:
received += conn.recv(nb_bytes - len(received))
return received
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(1)
connection, address = s.accept()
while True:
# receive header
header = receive(HEADER_SIZE, connection)
data_size = struct.unpack(">i", header)[0]
# receive data
data = receive(data_size, connection)
print(pickle.loads(data))
def read_vantage_pts_from_blob_container(): blob_client = azure.storage.blob.BlobClient.from_connection_string(conn_str="my_conn_str", container_name="vantage-pts-container", blob_name="vantage_pts_blob.pickle") blob_data = blob_client.download_blob() return pickle.loads( blob_data.readall() )
read_sockets, write_sockets, error_sockets = select.select(
CONNECTION_LIST, [], [])
for sock in read_sockets:
#new connection
if sock == server_socket:
sockfd, addr = server_socket.accept()
CONNECTION_LIST.append(sockfd)
print("Client (%s, %s) connected" % addr)
#incoming message from client
else:
try:
data = sock.recv(RECV_BUFFER)
data = pickle.loads(data)
print(data)
print(type(data))
#print('received data from [%s:%s]: ' % addr + data)
sock.send('connected'.encode())
sock.close()
CONNECTION_LIST.remove(sock)
modWiFiConfig(str(data['wifi_name']), str(data['wifi_pass']))
print("Wifi Added")
modAccessConfig(str(data['device_name']), str(data['wak']),
str(data['e']), str(data['p']))
print("Access Added")
config_wifi_dhcpcd = Popen(
"sudo cp /etc/dhcpcd_WiFi.conf /etc/dhcpcd.conf",
def loads(self, buf, **kwds):
return pickle.loads(buf, **kwds)
def recv(self):
self._check_closed()
self._check_readable()
buf = self.recv_bytes()
return pickle.loads(buf)
def to_obj(data: bytes):
return pickle.loads(data)
def deserialize(self, header: Dict, buffers: List):
obj_type = self.obj_type
if 'obj_type' in header:
obj_type = pickle.loads(header['obj_type'])
return obj_type(self._iter_deserial(header['headers'], buffers))
def unpickle_buffers(buffers):
return pickle.loads(buffers[0], buffers=buffers[1:])
