def has_key(self, *args, **kwargs):
callargs = inspect.getcallargs(self.func, *args, **kwargs)
key = binfootprint.dump(callargs)
with self.PDS(name=self.db_name, path=self.db_path) as db:
if self.subdbkey is not None:
with db.getData(self.subdbkey, create_sub_data=True) as subdb:
return key in subdb
else:
return key in db
def process_new_result(self, arg, result):
args, kwargs = arg
callargs = inspect.getcallargs(self.func, *args, **kwargs)
key = binfootprint.dump(callargs)
with self.PDS(name=self.db_name, path=self.db_path) as db:
if self.subdbkey is not None:
with db.getData(self.subdbkey, create_sub_data=True) as subdb:
subdb[key] = result
else:
db[key] = result
def __init__(self, func):
self.func = func
try:
self.__name__ = func.__name__
except AttributeError: # for example a 'functools.partial' object has no attribute __name__
self.__name__ = None
try:
self.__doc__ = func.__doc__
except AttributeError:
self.__doc__ = None
self.verbose = verbose
if db_name is None:
if self.__name__ is None:
raise ValueError("can not get db_name from func.__name (__name__ is None)")
self.db_name = self.__name__
if self.verbose > 0:
print("db_name ", db_name, "set from func.__name__")
else:
self.db_name = db_name
if self.verbose > 0:
print("db_name ", db_name)
self.db_path = db_path
if verbose > 0:
print("db_path ", db_path)
if kind == 'sql':
self.PDS = PersistentDataStructure
elif kind == 'hdf5':
self.PDS = PersistentDataStructure_HDF5
else:
raise ValueError("unknown kind '{}'".format(kind))
if verbose > 0:
print("kind ", kind)
if subdbkey is None:
self.subdbkey = None
if self.verbose > 0:
print("subdbkey not set")
else:
if not isinstance(subdbkey, bytes):
self.subdbkey = binfootprint.dump(subdbkey)
if self.verbose > 0:
print("subdbkey", subdbkey, "(converted to bytes)")
else:
self.subdbkey = subdbkey
if self.verbose > 0:
print("subdbkey", subdbkey)
def put_arg(self, args, kwargs):
callargs = inspect.getcallargs(self.func, *args, **kwargs)
key = binfootprint.dump(callargs)
with self.PDS(name=self.db_name, path=self.db_path) as db:
if self.subdbkey is not None:
with db.getData(self.subdbkey, create_sub_data=True) as subdb:
if key in subdb:
return subdb[key]
else:
if key in db:
return db[key]
arg = (args, kwargs)
jm.JobManager_Server.put_arg(self, arg)
return None
def __call__(self, *args, **kwargs):
callargs = inspect.getcallargs(self.func, *args, **kwargs)
if self.verbose > 0:
print("call with", callargs)
key = binfootprint.dump(callargs)
with self.PDS(name=self.db_name, path=self.db_path) as db:
if self.subdbkey is not None:
with db.getData(self.subdbkey, create_sub_data=True) as subdb:
if key in subdb:
if self.verbose > 0:
print("key found in subdb")
return subdb[key]
else:
if key in db:
if self.verbose > 0:
print("key found in db")
return db[key]
if self.verbose > 0:
print("key not found, crunch ...")
# did not find any cached results
value = self.func(*args, **kwargs)
if self.verbose > 0:
print("done!")
with self.PDS(name=self.db_name, path=self.db_path) as db:
if self.subdbkey is not None:
with db.getData(self.subdbkey, create_sub_data=True) as subdb:
if self.verbose > 0:
print("save value to subdb")
subdb[key] = value
else:
if self.verbose > 0:
print("save value to db")
db[key] = value
if self.verbose > 0:
print("return value")
return value
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import division, print_function
import sys
from os.path import abspath, dirname, split
# Add parent directory to beginning of path variable
sys.path = [split(dirname(abspath(__file__)))[0]] + sys.path
import binfootprint as bf
atoms = [
12345678, # int32
3.141, # float
"hallo Welt", # ascii string
"öäüß", # utf8 string
True,
False,
None, # special Value
2**65,
-(3**65), # integer
b"\xff\fe\03",
] # bytes
binkey = bf.dump(atoms) # generate a unique byte sequence
atoms_prime = bf.load(binkey) # restore the original object
def test_shutdown_server_while_client_running():
"""
start server with 100 elements in queue
start client
stop server -> client should catch exception, but can't do anything,
writing to fail won't work, because server went down
except do emergency dump
check if the final_result and the args dump end up to include
all arguments given
"""
global PORT
n = 100
sigs = [('SIGTERM', signal.SIGTERM), ('SIGINT', signal.SIGINT)]
for signame, sig in sigs:
PORT += 1
p_server = None
p_client = None
try:
p_server = mp.Process(target=start_server, args=(n,False,0.1))
p_server.start()
time.sleep(0.5)
assert p_server.is_alive()
p_client = mp.Process(target=start_client)
p_client.start()
time.sleep(2)
assert p_client.is_alive()
print(" send {} to server".format(signame))
os.kill(p_server.pid, sig)
p_server.join(TIMEOUT)
assert not p_server.is_alive(), "server did not shut down on time"
assert p_server.exitcode == 0, "the server raised an exception"
p_client.join(TIMEOUT)
assert not p_client.is_alive(), "client did not shut down on time"
assert p_client.exitcode == 0, "the client raised an exception"
print("[+] server and client joined {}".format(datetime.datetime.now().isoformat()))
fname = 'jobmanager.dump'
with open(fname, 'rb') as f:
data = jobmanager.JobManager_Server.static_load(f)
ac = data['job_q']
args_set = {binfootprint.dump(ac.data['_'+str(id_)]) for id_ in ac._not_gotten_ids}
final_result = data['final_result']
final_res_args = {binfootprint.dump(a[0]) for a in final_result}
args_ref = range(1,n)
set_ref = set()
for a in args_ref:
set_ref.add(binfootprint.dump(a))
set_recover = set(args_set) | set(final_res_args)
intersec_set = set_ref-set_recover
if len(intersec_set) == 0:
print("[+] no arguments lost!")
assert len(intersec_set) == 0, "NOT all arguments found in dump!"
except:
if p_server is not None:
p_server.terminate()
if p_client is not None:
p_client.terminate()
raise
def _convkey(self, key):
dump = False
if not isinstance(key, (bytes, bytearray)):
key = bf.dump(key)
dump = True
return key, dump