def test_filters_order():
l = ListCollection([Bunch(a='b', b=5), Bunch(a='c', c='c')])
filterd_l = l.filtered(a='b').select(lambda o: o.b > 4)
assert len(filterd_l) == 1
with pytest.raises(AttributeError):
filterd_l = l.select(lambda o: o.b > 4)
def test_typed_struct_field_defaults():
class Foo(ts.TypedStruct):
x = int
class Bar(ts.TypedStruct):
a = ts.Field(int, default=1)
b = ts.Field([int], default=[2, 3])
c = ts.Field(Foo, default=Foo(x=4))
d = ts.Field(Foo, default=dict(x=5))
e = ts.Field([Foo], default=[Foo(x=6), dict(x=7)])
f = ts.Field({str: Foo}, default=dict(g=Foo(x=8)))
bar1 = Bar()
bar2 = Bar()
assert bar1.a == bar2.a == 1
assert bar1.b == bar2.b == [2, 3]
assert bar1.b is not bar2.b
assert bar1.c == bar2.c == Foo(x=4)
assert bar1.c is not bar2.c
assert bar1.d == bar2.d == Foo(x=5)
assert bar1.d is not bar2.d
assert bar1.e == bar2.e == [Foo(x=6), Foo(x=7)]
assert bar1.e is not bar2.e
assert bar1.e[0] is not bar2.e[0]
assert bar1.e[1] is not bar2.e[1]
assert bar1.f == bar2.f == Bunch(g=Foo(x=8))
assert bar1.f is not bar2.f
assert bar1.f.g is not bar2.f.g
def test_pickle_texception():
import pickle
t1 = T(what="happened", a=1, b=Bunch(x=[1, 2, 3], y=range(5)))
t2 = pickle.loads(pickle.dumps(t1))
assert t1.render() == t2.render()
assert t1._params == t2._params
def _extract_args(self, rpc, args, kwargs):
result = Bunch(**kwargs)
if self._spec is None:
self._populate_methods()
spec = self._spec[rpc]
arg_names = (arg.name for arg in spec.args)
for arg, arg_name in zip(args, arg_names):
result[arg_name] = arg
return result
def test_typed_struct_bunch_fields(use_full_syntax):
if use_full_syntax:
class Bar(ts.TypedStruct):
nums = ts.Field({str: int})
else:
class Bar(ts.TypedStruct):
nums = {str: int}
# NOTE: TypedBunch shares most things with TypedDict, so I'm only doing a quick sanity test
bar = Bar(nums=Bunch(a=0))
assert bar.nums.a == 0
bar.nums.a = 1
bar.nums['b'] = 2
bar.nums.update(c=3)
with pytest.raises(ts.FieldTypeMismatch):
bar.nums.d = '4'
with pytest.raises(ts.FieldKeyTypeMismatch):
bar.nums[5] = 6
assert bar.nums == Bunch(a=1, b=2, c=3)
def test_iter_wait_progress_total_timeout():
data = Bunch(a=1000)
def get():
data.a -= 1
return data.a
with pytest.raises(TimeoutException) as exc:
for state in iter_wait_progress(get,
advance_timeout=1,
sleep=.05,
total_timeout=.1):
pass
assert exc.value.message.startswith("advanced but failed to finish")
def test_timecache():
ts = 0
data = Bunch(a=0, b=0)
def get_ts():
return ts
@timecache(expiration=1, get_ts_func=get_ts, key_func=lambda k: k)
def inc(k, x):
x += 1
data[k] += 1
assert data.a == data.b == 0
inc('a', random.random())
assert (data.a, data.b) == (1, 0)
inc('a', x=random.random())
assert (data.a, data.b) == (1, 0)
ts += 1
inc('a', random.random())
assert (data.a, data.b) == (2, 0)
inc('b', x=random.random())
assert (data.a, data.b) == (2, 1)
inc('b', random.random())
assert (data.a, data.b) == (2, 1)
ts += 1
inc('b', x=random.random())
assert (data.a, data.b) == (2, 2)
inc.cache_clear()
inc('a', x=random.random())
assert (data.a, data.b) == (3, 2)
inc('b', x=random.random())
assert (data.a, data.b) == (3, 3)
inc.cache_clear()
inc('a', x=random.random())
inc('b', x=random.random())
inc('a', x=random.random())
inc('b', x=random.random())
assert (data.a, data.b) == (4, 4)
inc.cache_pop('a', x=random.random())
inc('a', x=random.random())
inc('b', x=random.random())
def generate():
for key in actual.keys() | expected.keys():
actual_count = actual.get(key, 0)
expected_count = expected.get(key, 0)
if actual_count == 0 and expected_count == 0:
continue
actual_percentage = 100 * actual_count / actual_total
expected_percentage = 100 * expected_count / expected_total
diff = actual_percentage - expected_percentage
abs_diff = abs(diff)
color = color_for(abs_diff)
if color:
yield Bunch(key=colorize(key, color),
actual=colorize('%.1f%%' % actual_percentage, color),
expected=colorize('%.1f%%' % expected_percentage, color),
diff=colorize('%.1f%%' % diff, color),
abs_diff=abs_diff)
def context(self, kw):
for v in kw.values():
assert (v is None) or (isinstance(
v, (str, int, float))), "Can't use %r as context vars" % v
ctx = self._get_context_data()
ctx.append(Bunch(kw))
try:
yield
except Exception as exc:
context = get_current_context()
if context and not getattr(exc, "context", None):
try:
exc.context = context
except:
logging.warning("could not attach context to exception")
raise
finally:
ctx.pop(-1)
def test_iter_wait_progress_inbetween_sleep():
data = Bunch(a=3)
def get():
data.a -= 1
return data.a
sleep = .07
g = iter_wait_progress(get, advance_timeout=10, sleep=sleep)
# first iteration should be immediate
t = Timer()
next(g)
assert t.duration < sleep
# subsequent iteration should be at least 'sleep' long
next(g)
assert t.duration >= sleep
for state in g:
pass
assert state.finished is True
def iter_wait_progress(state_getter, advance_timeout, total_timeout=float("inf"), state_threshold=0, sleep=0.5, throw=True,
allow_regression=True, advancer_name=None, progressbar=True):
ADVANCE_TIMEOUT_MESSAGE = "did not advance for {duration: .1f} seconds"
TOTAL_TIMEOUT_MESSAGE = "advanced but failed to finish in {duration: .1f} seconds"
state = state_getter() # state_getter should return a number, represent current state
# we need this bunch to avoid using "nonlocal" keyword, for compatability with python2
progress = Bunch(state=state, finished=False, changed=False)
progress.total_timer = Timer(expiration=total_timeout)
progress.advance_timer = Timer(expiration=advance_timeout)
def finished():
return progress.state <= state_threshold
def did_advance():
current_state = state_getter()
progress.advanced = progress.state > current_state
progress.changed = progress.state != current_state
if progress.advanced or allow_regression:
progress.state = current_state
return progress.advanced
while not finished():
progress.timeout, message = min(
(progress.total_timer.remain, TOTAL_TIMEOUT_MESSAGE),
(progress.advance_timer.remain, ADVANCE_TIMEOUT_MESSAGE))
if advancer_name:
message = advancer_name + ' ' + message
result = wait(progress.timeout, pred=did_advance, sleep=sleep, message=message, throw=throw, progressbar=progressbar)
if not result: # if wait times out without throwing
return
progress.advance_timer.reset()
yield progress
progress.finished = True
yield progress # indicate success
def test_rwlock():
main_ctrl = threading.Event()
reader_ctrl = threading.Event()
writer_ctrl = threading.Event()
lock = RWLock("test")
state = Bunch(reading=False, writing=False)
def read():
logging.info("Before read")
reader_ctrl.wait()
reader_ctrl.clear()
with lock:
logging.info("Reading...")
state.reading = True
main_ctrl.set()
reader_ctrl.wait()
reader_ctrl.clear()
state.reading = False
logging.info("Done reading")
logging.info("After read")
def write():
logging.info("Before write")
writer_ctrl.wait()
writer_ctrl.clear()
with lock.exclusive():
logging.info("Writing...")
state.writing = True
main_ctrl.set()
writer_ctrl.wait()
writer_ctrl.clear()
state.writing = False
logging.info("Done writing")
logging.info("After write")
main_ctrl.set()
reader = concurrent(read, threadname='read')
writer = concurrent(write, threadname='write')
with reader, writer:
assert not state.reading and not state.writing
reader_ctrl.set()
main_ctrl.wait()
logging.info("lock acquired non-exclusively")
main_ctrl.clear()
assert state.reading and not state.writing
writer_ctrl.set()
logging.info("writer awaits exclusivity")
with lock:
assert state.reading and not state.writing
reader_ctrl.set()
main_ctrl.wait()
main_ctrl.clear()
logging.info("read lock released")
assert not state.reading and state.writing
writer_ctrl.set()
main_ctrl.wait()
main_ctrl.clear()
logging.info("write lock released")
assert not state.reading and not state.writing
def test_bunch_recursion():
x = Bunch(a='a', b="b", d=Bunch(x="axe", y="why"))
x.d.x = x
x.d.y = x.b
print(x)
def make_request():
if not quiet:
_logger.debug("request >> #%04d: %s/%s", message_id, self.url,
method)
try:
try:
conn = self.get_connection(
timeout_override=timeout_override)
except:
typ, exc, tb = sys.exc_info()
raise_if_async_exception(exc)
raise_with_traceback(
ConnectionError(url=self.url,
message_id=message_id,
exc=exc), tb)
json_req = _make_request(message_id, method, params)
try:
conn.request('POST', self._path, json_req, self._headers)
except socket.timeout:
typ, exc, tb = sys.exc_info()
raise_if_async_exception(exc)
raise_with_traceback(ServerTimeout(**exc_params), tb)
except:
typ, exc, tb = sys.exc_info()
raise_if_async_exception(exc)
raise_with_traceback(RequestError(exc=exc, **exc_params),
tb)
try:
response = conn.getresponse()
except socket.timeout:
typ, exc, tb = sys.exc_info()
raise_if_async_exception(exc)
raise_with_traceback(ServerTimeout(**exc_params), tb)
except:
typ, exc, tb = sys.exc_info()
raise_if_async_exception(exc)
# To address leader temporarily being down ("BadStatusLine")
time.sleep(1)
raise_with_traceback(ReplyError(exc=exc, **exc_params), tb)
except:
self._conn = None # create a new connection next time
raise
if response.status == 408 or response.status == 400:
# AWS issues
_logger.debug("Got error code %s, on request #%04d, retrying",
response.status, message_id)
self._conn = None # create a new connection next time
time.sleep(0.5)
self.retry_last_rpc(
'Got error code %s - %s' %
(response.status, httpclient.responses[response.status]))
if response.status == 301:
# redirect
new_url = response.getheader("location")
parsed = urlparse(new_url)
_logger.debug("redirecting http://%s:%s/%s to %s",
self._conn_params.host, self._conn_params.port,
self._path, new_url)
# print("redirect to", new_url)
orig = Bunch(**self._conn_params)
if ":" in parsed.netloc:
host, port = parsed.netloc.split(":")
self._conn_params.host = host
self._conn_params.port = int(port)
else:
self._conn_params.host = parsed.netloc
self.expire_connection()
conn = self.get_connection(timeout_override=timeout_override)
conn.request('POST', self._path,
_make_request(message_id, method, params),
self._headers)
response = conn.getresponse()
self.expire_connection()
self._conn_params = orig
if response.status == 503:
# service not available, retry again later
time.sleep(0.5)
return self.retry_last_rpc("Service unavailable, retry later")
try:
response_text = response.read().decode('utf-8')
except socket.timeout:
self._conn = None # create a new connection next time
typ, exc, tb = sys.exc_info()
raise_with_traceback(ServerTimeout(**exc_params), tb)
except IncompleteRead:
self._conn = None # create a new connection next time
typ, exc, tb = sys.exc_info()
raise_with_traceback(ReadError(exc=exc, **exc_params), tb)
if response.status != 200:
# some other error
raise HTTPException(status=response.status,
reason=response.reason,
text=response_text,
**exc_params)
try:
response_object = json.loads(response_text)
except ValueError:
raise ResponseError("Could not parse json respone",
response_text=response_text,
**exc_params)
if not quiet:
_logger.debug("response << #%04d: %s -> %s",
response_object.get('id', -1), method,
DataSize(response.getheader("content-length")))
if message_id != response_object.get('id'):
raise ResponseIdMismatch(responded=response_object.get('id'),
response_text=response_text,
**exc_params)
if 'error' not in response_object:
result = response_object['result']
return bunchify(result) if should_bunchify else result
error_code = response_object['error'].pop('code', None)
error_data = response_object['error'].pop(
'data', "(no further information)")
remote_message = response_object['error'].pop(
'message', "(no message)")
server_node_id = response.getheader('server-node-id', None)
if error_code == -32601: # Method not Found
raise RemoteMethodNotFound(method=method)
# raise AttributeError('%s' % remote_message)
if error_code == -32602: # Invalid Params
raise TypeError('%s(%s)' % (remote_message, error_data))
if isinstance(error_data, dict):
if server_node_id:
error_data['served_by_node'] = server_node_id
if 'exceptionClass' in error_data:
exception_class = error_data.pop('exceptionClass')
if isinstance(exception_class, list):
exceptions._register_ancestry(exception_class)
exception_class = exception_class[0]
exception_name = exception_class.rpartition(".")[-1]
handler = self._exception_handlers.get(
exception_name) or getattr(exceptions, exception_name)
if hasattr(handler, '__bases__') and error_data.get(
"retry",
False) and RetrySignal not in handler.__bases__:
handler.__bases__ += (RetrySignal, )
exception_text = error_data.pop('exceptionText',
remote_message)
error_data['jrpc'] = exc_params
raise handler(exception_text, **error_data)
else:
exc_params.update(error_data)
raise RemoteException(remote_message, **exc_params)
else:
if error_data:
remote_message += "\n%s" % (error_data, )
exc_params.update(response_object['error'])
if server_node_id:
exc_params['served_by_node'] = server_node_id
raise RemoteException(remote_message, **exc_params)
