def test_serde_sign_verify(self):
base = lambda x, y: (x + y) / 2
dumped = serde.dumps(base)
digest = serde.sign("test-key".encode(), dumped)
self.assertTrue(isinstance(digest, bytes))
self.assertIsNone(serde.verifyhmac("test-key".encode(), digest,
dumped))
def test_serde_sign_verify_wrong_key(self):
base = lambda x, y: (x + y) / 2
dumped = serde.dumps(base)
digest = serde.sign("test-key".encode(), dumped)
self.assertTrue(isinstance(digest, bytes))
with self.assertRaises(SignatureNotValidException):
self.assertEqual(
digest, serde.verifyhmac("wrong-key".encode(), digest, dumped))
async def send_data(self, data, flags=0, signkey=None):
"""Serialize `data` with cloudpickle and compress it before sending it
asynchronously through the socket
"""
serialized = serde.dumps(data)
if signkey:
signed = serde.sign(signkey.encode(), serialized)
return await self.send_pyobj((signed, serialized), flags=flags)
return await self.send_pyobj(serialized, flags=flags)
def send_result_data(self, result, signkey=None):
zipped_result = serde.dumps(result)
if signkey:
signed = serde.sign(signkey.encode(), zipped_result)
frame = struct.pack(
f"!H{len(signed)}s{len(zipped_result)}s",
len(signed),
signed,
zipped_result,
)
return self._backend.put_result(frame)
return self._backend.put_result(zipped_result)
def send_data(self, data, signkey=None):
"""Serialize `data` with cloudpickle and compress it before sending
through the socket
"""
serialized = serde.dumps(data)
if signkey:
signed = serde.sign(signkey.encode(), serialized)
frame = struct.pack(
f"!H{len(signed)}s{len(serialized)}s",
len(signed),
signed,
serialized,
)
return self._backend.put_job(frame)
return self._backend.put_job(serialized)
def run(self):
while True:
# Need to decompress and unpickle data here cause the function
# contained in the job could be not defined in the __main__ module
# being the worker optionally run in a remote machine
self._done = False
self._last_job = None
zipped_job = self._job_queue.get()
# Poison pill check
if zipped_job is None:
break
job = serde.loads(zipped_job)
self._last_job = job
self._log.debug("Executing job %s", job.job_id)
if "eta" in job.kwargs:
eta = job.kwargs.pop("eta")
multiples = {"h": 60 * 60, "m": 60, "s": 1}
if isinstance(eta, int):
delay = eta
else:
try:
delay = int(eta)
except ValueError:
delay = multiples[eta[-1]] * int(eta[:-1])
job.add_delay(delay)
response = self.execute_job(job)
# Push the completed job in the result queue ready to be
# answered to the requesting client
self._completed_jobs.put((job.job_id, response))
# Re enter the job in the queue
job.kwargs["eta"] = str(job.delay) + "s"
self._job_queue.put(serde.dumps(job))
else:
response = self.execute_job(job)
# Push the completed job in the result queue ready to be
# answered to the requesting client
self._completed_jobs.put((job.job_id, response))
self._done = True
def test_serde_dumps_loads(self):
base = lambda x, y: (x + y) / 2
dump_base = serde.dumps(base)
self.assertTrue(isinstance(dump_base, bytes))
self.assertEqual(base(1, 2), serde.loads(dump_base)(1, 2))