def test_recursion(self):
# Check that comparison for recursive objects fails gracefully
from collections import UserList
a = UserList()
b = UserList()
a.append(b)
b.append(a)
self.assertRaises(RecursionError, operator.eq, a, b)
self.assertRaises(RecursionError, operator.ne, a, b)
self.assertRaises(RecursionError, operator.lt, a, b)
self.assertRaises(RecursionError, operator.le, a, b)
self.assertRaises(RecursionError, operator.gt, a, b)
self.assertRaises(RecursionError, operator.ge, a, b)
b.append(17)
# Even recursive lists of different lengths are different,
# but they cannot be ordered
self.assertTrue(not (a == b))
self.assertTrue(a != b)
self.assertRaises(RecursionError, operator.lt, a, b)
self.assertRaises(RecursionError, operator.le, a, b)
self.assertRaises(RecursionError, operator.gt, a, b)
self.assertRaises(RecursionError, operator.ge, a, b)
a.append(17)
self.assertRaises(RecursionError, operator.eq, a, b)
self.assertRaises(RecursionError, operator.ne, a, b)
a.insert(0, 11)
b.insert(0, 12)
self.assertTrue(not (a == b))
self.assertTrue(a != b)
self.assertTrue(a < b)
def test_py_record_batch_reader():
def make_schema():
return pa.schema([('field', pa.int64())])
def make_batches():
schema = make_schema()
batch1 = pa.record_batch([[1, 2, 3]], schema=schema)
batch2 = pa.record_batch([[4, 5]], schema=schema)
return [batch1, batch2]
# With iterable
batches = UserList(make_batches()) # weakrefable
wr = weakref.ref(batches)
with pa.ipc.RecordBatchReader.from_batches(make_schema(),
batches) as reader:
batches = None
assert wr() is not None
assert list(reader) == make_batches()
assert wr() is None
# With iterator
batches = iter(UserList(make_batches())) # weakrefable
wr = weakref.ref(batches)
with pa.ipc.RecordBatchReader.from_batches(make_schema(),
batches) as reader:
batches = None
assert wr() is not None
assert list(reader) == make_batches()
assert wr() is None
def test_recursion(self):
from collections import UserList
a = UserList()
b = UserList()
a.append(b)
b.append(a)
self.assertRaises(RecursionError, operator.eq, a, b)
self.assertRaises(RecursionError, operator.ne, a, b)
self.assertRaises(RecursionError, operator.lt, a, b)
self.assertRaises(RecursionError, operator.le, a, b)
self.assertRaises(RecursionError, operator.gt, a, b)
self.assertRaises(RecursionError, operator.ge, a, b)
b.append(17)
self.assertTrue(not a == b)
self.assertTrue(a != b)
self.assertRaises(RecursionError, operator.lt, a, b)
self.assertRaises(RecursionError, operator.le, a, b)
self.assertRaises(RecursionError, operator.gt, a, b)
self.assertRaises(RecursionError, operator.ge, a, b)
a.append(17)
self.assertRaises(RecursionError, operator.eq, a, b)
self.assertRaises(RecursionError, operator.ne, a, b)
a.insert(0, 11)
b.insert(0, 12)
self.assertTrue(not a == b)
self.assertTrue(a != b)
self.assertTrue(a < b)
def test_collections_userlist():
# Create userlist with cycle
a = UserList()
a.append(a)
cases = [
(UserList(), "UserList([])"),
(
UserList(i for i in range(1000, 1020)),
"UserList([1000,\n"
" 1001,\n"
" 1002,\n"
" 1003,\n"
" 1004,\n"
" 1005,\n"
" 1006,\n"
" 1007,\n"
" 1008,\n"
" 1009,\n"
" 1010,\n"
" 1011,\n"
" 1012,\n"
" 1013,\n"
" 1014,\n"
" 1015,\n"
" 1016,\n"
" 1017,\n"
" 1018,\n"
" 1019])",
),
(a, "UserList([UserList(...)])"),
]
for obj, expected in cases:
assert pretty.pretty(obj) == expected
def train_model(cls, model, cooccurrence_matrix, epochs=5):
'''
Train the model on a given cooccurence_matrix.
'''
# Create Targes, Contexts and Labels
targets = UserList()
contexts = UserList()
labels = UserList()
# We only train on non-zero elements of X_ij
for i, column in cooccurrence_matrix.items():
for j, xij in column.items():
if xij > 0:
targets.append(i)
contexts.append(j)
labels.append(xij)
targets = np.asarray(targets)
contexts = np.asarray(contexts)
labels = np.asarray(labels)
# Run Training
start_time = time()
loss = model.fit([targets, contexts], labels, epochs=epochs)
end_time = time()
logging.info(
"Training finished: Iterations: %d, loss=%f, Run-Time: %d sec",
epochs, loss.history['loss'][-1], int(end_time - start_time))
def test_advanced_dumps(self):
sample = UserList([
False, 123,
UserDict({}), 5 + 5j,
UserDict({
'haha': 2 + 3j,
'toto': [False, {
'nini': 'xixi'
}]
}), True
])
self.assertEqual(
'[\r\n false,\r\n 123,\r\n {},\r\n {\r\n "real": 5.0,\r\n '
' "imag": 5.0\r\n },\r\n {\r\n "haha": {\r\n "real": 2.0,\r\n '
' "imag": 3.0\r\n },\r\n "toto": [\r\n false,\r\n '
' {\r\n "nini": "xixi"\r\n }\r\n ]\r\n },\r\n '
' true\r\n]', sj.dumps(sample, cls=sj.AdvancedEncoder, indent=4))
sample = UserList([
False,
datetime.now().date(), 123,
UserDict({}), 5 + 5j,
UserDict({
'haha': 2 + 3j,
'toto': [False, {
'nini': 'xixi'
}]
}), True
])
haha = sj.dumps(sample, cls=sj.AdvancedEncoder, indent=4)
print(haha)
hhh = sj.loads(haha, cls=sj.AdvancedDecoder)
print(hhh)
async def get_messages(self, *args, **kwargs):
"""
Same as :meth:`iter_messages`, but returns a list instead
with an additional ``.total`` attribute on the list.
If the `limit` is not set, it will be 1 by default unless both
`min_id` **and** `max_id` are set (as *named* arguments), in
which case the entire range will be returned.
This is so because any integer limit would be rather arbitrary and
it's common to only want to fetch one message, but if a range is
specified it makes sense that it should return the entirety of it.
If `ids` is present in the *named* arguments and is not a list,
a single :tl:`Message` will be returned for convenience instead
of a list.
"""
total = [0]
kwargs['_total'] = total
if len(args) == 1 and 'limit' not in kwargs:
if 'min_id' in kwargs and 'max_id' in kwargs:
kwargs['limit'] = None
else:
kwargs['limit'] = 1
msgs = UserList()
async for x in self.iter_messages(*args, **kwargs):
msgs.append(x)
msgs.total = total[0]
if 'ids' in kwargs and not utils.is_list_like(kwargs['ids']):
return msgs[0]
return msgs
def classPupils(self, klass, date=None):
"""Read the pupil data for the given school-class (possibly with
streams).
Return an ordered list of <PupilData> named tuples.
If a <date> is supplied, pupils who left the school before that
date will not be included.
<klass> is a <Klass> instance. If it has no streams, all pupils
are returned. If there are strems, only those pupils in one of
the given streams are returned.
To enable indexing on pupil-id, the result has an extra
attribute, <pidmap>: {pid-> <PupilData> instance}
"""
fetched = self.db.select('PUPILS', CLASS=klass.klass)
rows = UserList()
rows.pidmap = {}
slist = klass.streams
for row in fetched:
pdata = PupilData(row)
# Check exit date
if date:
exd = pdata['EXIT_D']
if exd and exd < date:
continue
# Check stream
if (not slist) or ((pdata['STREAM'] or '_') in slist):
rows.append(pdata)
rows.pidmap[pdata['PID']] = pdata
return rows
def testWritelinesUserList(self):
l = UserList([b'1', b'2'])
self.f.writelines(l)
self.f.close()
self.f = self.open(TESTFN, 'rb')
buf = self.f.read()
self.assertEqual(buf, b'12')
def testSequenceArrayConversionTypeChecking(self):
"""Test error handling for sequence conversion to array arguments."""
from Python.Test import ArrayConversionTest
from Python.Test import Spam
if six.PY3:
from collections import UserList
else:
from UserList import UserList
# This should work, because null / None is a valid value in an
# array of reference types.
items = UserList()
for i in range(10):
items.append(Spam(str(i)))
items[1] = None
result = ArrayConversionTest.EchoRange(items)
self.assertTrue(result[0].__class__ == Spam)
self.assertTrue(result[1] == None)
self.assertTrue(len(result) == 10)
def test(items=items):
items[1] = 1
result = ArrayConversionTest.EchoRange(items)
self.assertRaises(TypeError, test)
def test(items=items):
items[1] = "spam"
result = ArrayConversionTest.EchoRange(items)
self.assertRaises(TypeError, test)
def test_other_iterables_are_list_like(self):
for thing in [[], (),
set(), (range if PY3 else xrange)(1),
generator(),
array('i'),
UserList()]:
assert_equals(is_list_like(thing), True, thing)
def test_sequence_array_conversion_type_checking():
"""Test error handling for sequence conversion to array arguments."""
from Python.Test import ArrayConversionTest
from Python.Test import Spam
# This should work, because null / None is a valid value in an
# array of reference types.
items = UserList()
for i in range(10):
items.append(Spam(str(i)))
items[1] = None
result = ArrayConversionTest.EchoRange(items)
assert result[0].__class__ == Spam
assert result[1] is None
assert len(result) == 10
with pytest.raises(TypeError):
items[1] = 1
_ = ArrayConversionTest.EchoRange(items)
with pytest.raises(TypeError):
items[1] = "spam"
_ = ArrayConversionTest.EchoRange(items)
def testWritelinesUserList(self):
l = UserList([b'123', b'456'])
self.f.writelines(l)
self.f.close()
self.f = _FileIO(TESTFN, 'rb')
buf = self.f.read()
self.assertEqual(buf, b'123456')
def load_model(weights_label="imagenet"):
# load the pre-trained Keras model (here we are using a model
# pre-trained on ImageNet and provided by Keras, but you can
# substitute in your own networks just as easily)
global model
with tf.variable_scope('smallest_sample') as scope:
fn_produce_seq = lambda: [
np.empty(np.zeros(0)).tolist()
for _ in range(len(np.arange(3).tolist()))
]
fn_smallest_sample = lambda: UserList(np.array(fn_produce_seq())).data
fn_pad_sequence = lambda: [
np.array([fn_smallest_sample()]).tolist()
for _ in range(len(np.arange(4).tolist()))
]
fn_mapping_pad_sequences = lambda x=fn_pad_sequence, z=range(
len(np.arange(62 / 15).tolist())): map(lambda y: y(),
[x for _ in z])
smallest_sample_data = Variable(
[x for x in fn_mapping_pad_sequences()],
name='protected_user_list_input_tensor')
try:
model = ResNet50(weights=weights_label,
input_tensor=smallest_sample_data)
except Exception as e:
tf.logging.info(e)
finally:
model = ResNet50(weights=weights_label)
def test_advanced_complex(self):
sample = UserList([
False, 123,
UserDict({}), 5 + 5j,
UserDict({
'haha': 2 + 3j,
'toto': [False, {
'nini': 'xixi'
}]
}), True
])
output1 = sj.dumps(sample, cls=sj.AdvancedEncoder, indent=2)
sample = deque([
False, 123,
UserDict({}), 5 + 5j,
UserDict({
'haha': 2 + 3j,
'toto': [False, {
'nini': 'xixi'
}]
}), True
])
output2 = sj.dumps(sample, cls=sj.AdvancedEncoder, indent=2)
self.assertEqual(output1, output2)
output3 = sj.loads(output1, cls=sj.AdvancedDecoder)
print(output3)
self.assertEqual([
False, 123, {}, (5 + 5j), {
'haha': (2 + 3j),
'toto': [False, {
'nini': 'xixi'
}]
}, True
], output3)
self.assertTrue(isinstance(output3[3], complex))
def test_list_refcount(self):
a = UserList([1,2,3])
# temporary refcount fix until I understand why it incs by one.
inline_tools.inline("a[1] = 1234;",['a'])
before1 = sys.getrefcount(a)
after1 = sys.getrefcount(a)
assert_equal(after1,before1)
def test_iterables_in_general_are_list_like(self):
for thing in [[], (),
set(),
range(1),
generator(),
array('i'),
UserList()]:
assert_equal(is_list_like(thing), True, thing)
def testWritelinesUserList(self):
# verify writelines with instance sequence
l = UserList([b'1', b'2'])
self.f.writelines(l)
self.f.close()
self.f = self.open(TESTFN, 'rb')
buf = self.f.read()
self.assertEquals(buf, b'12')
def test_sequence_nested_array_conversion():
"""Test conversion of sequences to array-of-array arguments."""
from Python.Test import ArrayConversionTest
from Python.Test import Spam
items = UserList()
for i in range(10):
subs = UserList()
for _ in range(10):
subs.append(Spam(str(i)))
items.append(subs)
result = ArrayConversionTest.EchoRangeAA(items)
assert len(result) == 10
assert len(result[0]) == 10
assert result[0][0].__class__ == Spam
def test_json_01():
from datetime import datetime
from collections import UserDict, UserList
return serializer.to_json({
"key1": "val1",
"key2": datetime.now(),
"key3": UserDict({"key3.1": "val3.1"}),
"key4": UserList(["val4.1", "val4.2"])
})
def test_user_collections(self):
# Note: Not "UserDict" because UserDict doesn't implement
# __iter__ and hence isn't a collections.abc.Mapping, and doesn't
# implement enough API to implement the NSDictionary interface.
v = IterableUserDict()
self.assertIsSubclass(classOfProxy(v), NSMutableDictionary)
v = UserList()
self.assertIsSubclass(classOfProxy(v), NSMutableArray)
def __init__(self,
defaults=None,
dict_type=_default_dict,
allow_no_value=False,
delimiters=('=', ':'),
comment_prefixes=('#', ';'),
inline_comment_prefixes=None,
strict=True,
empty_lines_in_values=True,
default_section=DEFAULTSECT,
interpolation=_UNSET):
self._dict = dict_type
self._sections = self._dict()
self._defaults = UserList()
self._proxies = self._dict()
self._proxies[default_section] = UserList(
[SectionProxy(self, default_section, 0)])
if defaults:
for key, value in defaults.items():
self._defaults[self.optionxform(key)] = value
self._delimiters = tuple(delimiters)
if delimiters == ('=', ':'):
self._optcre = self.OPTCRE_NV if allow_no_value else self.OPTCRE
else:
d = "|".join(re.escape(d) for d in delimiters)
if allow_no_value:
self._optcre = re.compile(self._OPT_NV_TMPL.format(delim=d),
re.VERBOSE)
else:
self._optcre = re.compile(self._OPT_TMPL.format(delim=d),
re.VERBOSE)
self._comment_prefixes = tuple(comment_prefixes or ())
self._inline_comment_prefixes = tuple(inline_comment_prefixes or ())
self._strict = strict
self._allow_no_value = allow_no_value
self._empty_lines_in_values = empty_lines_in_values
self.default_section = default_section
self._interpolation = interpolation
if self._interpolation is _UNSET:
self._interpolation = self._DEFAULT_INTERPOLATION
if self._interpolation is None:
self._interpolation = Interpolation()
def test_delete_encoded_item(self, fake_app):
fake_storage = fake_app.storage
fake_storage_delete = fake_storage.delete
service = publish_queue.PublishQueueService(backend=MagicMock())
service.get_from_mongo = MagicMock()
cursor = UserList([{"_id": "4567", "encoded_item_id": "TEST ID"}])
cursor.sort = MagicMock()
cursor.sort.return_value = cursor
service.get_from_mongo.return_value = cursor
service.delete({"_id": "4567"})
assert fake_storage_delete.call_args == mock.call("TEST ID")
def test_vsBuiltinSort(self, n=500):
from random import choice
for insorted in (list(), UserList()):
for i in range(n):
digit = choice("0123456789")
if digit in "02468":
f = self.module.insort_left
else:
f = self.module.insort_right
f(insorted, digit)
self.assertEqual(sorted(insorted), insorted)
async def get_dialogs(self, *args, **kwargs):
"""
Same as :meth:`iter_dialogs`, but returns a list instead
with an additional ``.total`` attribute on the list.
"""
total = [0]
kwargs['_total'] = total
dialogs = UserList()
async for x in self.iter_dialogs(*args, **kwargs):
dialogs.append(x)
dialogs.total = total[0]
return dialogs
def test_sequence_array_conversion():
"""Test conversion of sequence-like obs to array arguments."""
from Python.Test import ArrayConversionTest
from Python.Test import Spam
items = UserList()
for i in range(10):
items.append(Spam(str(i)))
result = ArrayConversionTest.EchoRange(items)
assert result[0].__class__ == Spam
assert len(result) == 10
def testSequenceNestedArrayConversion(self):
"""Test conversion of sequences to array-of-array arguments."""
from Python.Test import ArrayConversionTest
from Python.Test import Spam
if six.PY3:
from collections import UserList
else:
from UserList import UserList
items = UserList()
for i in range(10):
subs = UserList()
for n in range(10):
subs.append(Spam(str(i)))
items.append(subs)
result = ArrayConversionTest.EchoRangeAA(items)
self.assertTrue(len(result) == 10)
self.assertTrue(len(result[0]) == 10)
self.assertTrue(result[0][0].__class__ == Spam)
async def get_participants(self, *args, **kwargs):
"""
Same as :meth:`iter_participants`, but returns a list instead
with an additional ``.total`` attribute on the list.
"""
total = [0]
kwargs['_total'] = total
participants = UserList()
async for x in self.iter_participants(*args, **kwargs):
participants.append(x)
participants.total = total[0]
return participants
def enable_history_trace(ql: Qiling, nrecords: int):
"""Enable instruction-level tracing in history mode.
To allow faster execution, the trace info collected throughout program execution is not
emitted and undergo as minimal post-processing as possible. When program crahses, the
last `nrecords` trace lines are shown.
Args:
ql: qiling instance
nrecords: number of last records to show
"""
# enable detailed disassembly info
md = ql.create_disassembler()
md.detail = True
# if available, use symbols map to resolve memory accesses
symsmap = getattr(ql.loader, 'symsmap', {})
# wrap the trace records list to allow it to be passed and modified by-ref
history: UserList[TraceRecord] = UserList()
def __trace_hook(ql: Qiling, address: int, size: int):
"""[internal] Trace hook callback.
"""
recent = list(__get_trace_records(ql, address, size, md))
history.data = (history + recent)[-nrecords:]
ql.hook_code(__trace_hook)
# replace the emulation error handler with our own so we can emit the trace
# records when program crashes. before we do that, we save the original one
# so we can call it.
orig_emu_error = ql.os.emu_error
def __emu_error(*args):
# first run the original emulation error handler
orig_emu_error(*args)
# then parse and emit the trace info we collected
ql.log.error(f'History:')
for record in history:
line = __to_trace_line(record, symsmap)
ql.log.error(line)
ql.log.error(f'')
ql.os.emu_error = __emu_error
def test_is_List(self):
assert is_List([])
assert is_List(UserList())
try:
class mylist(list):
pass
except TypeError:
pass
else:
assert is_List(mylist([]))
assert not is_List(())
assert not is_List({})
assert not is_List("")
