def test_serialize(self):
buf = self.dst.serialize()
res = struct.unpack_from(self.form, six.binary_type(buf))
eq_(self.nxt, res[0])
eq_(self.size, res[1])
offset = struct.calcsize(self.form)
opt1 = ipv6.option.parser(six.binary_type(buf[offset:]))
offset += len(opt1)
opt2 = ipv6.option.parser(six.binary_type(buf[offset:]))
offset += len(opt2)
opt3 = ipv6.option.parser(six.binary_type(buf[offset:]))
offset += len(opt3)
opt4 = ipv6.option.parser(six.binary_type(buf[offset:]))
eq_(5, opt1.type_)
eq_(2, opt1.len_)
eq_(b'\x00\x00', opt1.data)
eq_(1, opt2.type_)
eq_(0, opt2.len_)
eq_(None, opt2.data)
eq_(0xc2, opt3.type_)
eq_(4, opt3.len_)
eq_(b'\x00\x01\x00\x00', opt3.data)
eq_(1, opt4.type_)
eq_(0, opt4.len_)
eq_(None, opt4.data)
def _get_client_by_tier(self):
"""Get a client that uses ServiceRouter"""
config = self.config
serviceRouter = ServiceRouter()
overrides = ConnConfigs()
for key, val in six.iteritems(config.conn_configs):
key = six.binary_type(key)
val = six.binary_type(val)
overrides[key] = val
sr_options = ServiceOptions()
for key, val in six.iteritems(config.service_options):
key = six.binary_type(key)
if not isinstance(val, list):
raise TypeError("Service option %s expected list; got %s (%s)"
% (key, val, type(val)))
val = [six.binary_type(elem) for elem in val]
sr_options[key] = val
service_name = config.tier
# Obtain a normal client connection using SR2
client = serviceRouter.getClient2(self.client_class,
service_name, sr_options,
overrides, False)
if client is None:
raise NameError('Failed to lookup host for tier %s' % service_name)
return client
def combine_uuids(uuids, ordered=True, salt=''):
"""
Creates a uuid that specifies a group of UUIDS
Args:
uuids (list): list of uuid objects
ordered (bool): if False uuid order changes the resulting combined uuid
otherwise the uuids are considered an orderless set
salt (str): salts the resulting hash
Returns:
uuid.UUID: combined uuid
CommandLine:
python -m utool.util_hash --test-combine_uuids
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_hash import * # NOQA
>>> import utool as ut
>>> uuids = [hashable_to_uuid('one'), hashable_to_uuid('two'),
>>> hashable_to_uuid('three')]
>>> combo1 = combine_uuids(uuids, ordered=True)
>>> combo2 = combine_uuids(uuids[::-1], ordered=True)
>>> combo3 = combine_uuids(uuids, ordered=False)
>>> combo4 = combine_uuids(uuids[::-1], ordered=False)
>>> result = ut.repr4([combo1, combo2, combo3, combo4], nobr=True)
>>> print(result)
UUID('83ee781f-8646-ccba-0ed8-13842825c12a'),
UUID('52bbb33f-612e-2ab8-a62c-2f46e5b1edc8'),
UUID('945cadab-e834-e581-0f74-62f106d20d81'),
UUID('945cadab-e834-e581-0f74-62f106d20d81'),
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_hash import * # NOQA
>>> import utool as ut
>>> uuids = [uuid.UUID('5ff6b34e-7d8f-ef32-5fad-489266acd2ae'),
>>> uuid.UUID('f2400146-ec12-950b-1489-668228e155a8'),
>>> uuid.UUID('037d6f31-8c73-f961-1fe4-d616442a1e86'),
>>> uuid.UUID('ca45d6e2-e648-09cc-a49e-e71c6fa3b3f3')]
>>> ordered = True
>>> salt = u''
>>> result = combine_uuids(uuids, ordered, salt)
>>> print(result)
1dabc66b-b564-676a-99b4-5cae7a9e7294
"""
if len(uuids) == 0:
return get_zero_uuid()
elif len(uuids) == 1:
return uuids[0]
else:
if not ordered:
uuids = sorted(uuids)
sep_str = '-'
sep_byte = six.binary_type(six.b(sep_str))
pref = six.binary_type(six.b('{}{}{}'.format(salt, sep_str, len(uuids))))
combined_bytes = pref + sep_byte.join([u.bytes for u in uuids])
combined_uuid = hashable_to_uuid(combined_bytes)
return combined_uuid
def test_default_args(self):
ip = ipv6.ipv6()
buf = ip.serialize(bytearray(), None)
res = struct.unpack(ipv6.ipv6._PACK_STR, six.binary_type(buf))
eq_(res[0], 6 << 28)
eq_(res[1], 0)
eq_(res[2], 6)
eq_(res[3], 255)
eq_(res[4], addrconv.ipv6.text_to_bin('10::10'))
eq_(res[5], addrconv.ipv6.text_to_bin('20::20'))
# with extension header
ip = ipv6.ipv6(
nxt=0, ext_hdrs=[
ipv6.hop_opts(58, 0, [
ipv6.option(5, 2, b'\x00\x00'),
ipv6.option(1, 0, None)])])
buf = ip.serialize(bytearray(), None)
res = struct.unpack(ipv6.ipv6._PACK_STR + '8s', six.binary_type(buf))
eq_(res[0], 6 << 28)
eq_(res[1], 8)
eq_(res[2], 0)
eq_(res[3], 255)
eq_(res[4], addrconv.ipv6.text_to_bin('10::10'))
eq_(res[5], addrconv.ipv6.text_to_bin('20::20'))
eq_(res[6], b'\x3a\x00\x05\x02\x00\x00\x01\x00')
def test_retrieves_all_components_for_installed_apps(self, run):
response = self.client.get(self.url, format='json')
assert response.status_code == 200
assert self.component3.uuid not in [d['uuid'] for d in response.data]
assert response.data == [
{
'uuid': six.binary_type(self.component1.uuid),
'type': 'issue-link',
'schema': self.component1.schema,
'sentryApp': {
'uuid': self.sentry_app1.uuid,
'slug': self.sentry_app1.slug,
'name': self.sentry_app1.name,
},
},
{
'uuid': six.binary_type(self.component2.uuid),
'type': 'issue-link',
'schema': self.component2.schema,
'sentryApp': {
'uuid': self.sentry_app2.uuid,
'slug': self.sentry_app2.slug,
'name': self.sentry_app2.name,
},
},
]
def test_jsanitize(self):
# clean_json should have no effect on None types.
d = {"hello": 1, "world": None}
clean = jsanitize(d)
self.assertIsNone(clean["world"])
self.assertEqual(json.loads(json.dumps(d)), json.loads(json.dumps(
clean)))
d = {"hello": GoodMSONClass(1, 2, 3)}
self.assertRaises(TypeError, json.dumps, d)
clean = jsanitize(d)
self.assertIsInstance(clean["hello"], six.string_types)
clean_strict = jsanitize(d, strict=True)
self.assertEqual(clean_strict["hello"]["a"], 1)
self.assertEqual(clean_strict["hello"]["b"], 2)
d = {"dt": datetime.datetime.now()}
clean = jsanitize(d)
self.assertIsInstance(clean["dt"], six.string_types)
clean = jsanitize(d, allow_bson=True)
self.assertIsInstance(clean["dt"], datetime.datetime)
d = {"a": ["b", np.array([1, 2, 3])],
"b": ObjectId.from_datetime(datetime.datetime.now())}
clean = jsanitize(d)
self.assertEqual(clean["a"], ['b', [1, 2, 3]])
self.assertIsInstance(clean["b"], six.string_types)
rnd_bin = six.binary_type(np.random.rand(10))
d = {"a": six.binary_type(rnd_bin)}
clean = jsanitize(d, allow_bson=True)
self.assertEqual(clean["a"], six.binary_type(rnd_bin))
self.assertIsInstance(clean["a"], six.binary_type)
def test_relay_id_missmatch_response(self):
data = {
'public_key': six.binary_type(self.public_key),
'relay_id': self.relay_id,
}
raw_json, signature = self.private_key.pack(data)
resp = self.client.post(
self.path,
data=raw_json,
content_type='application/json',
HTTP_X_SENTRY_RELAY_ID=self.relay_id,
HTTP_X_SENTRY_RELAY_SIGNATURE=signature,
)
assert resp.status_code == 200, resp.content
result = json.loads(resp.content)
raw_json, signature = self.private_key.pack(result)
resp = self.client.post(
reverse(
'sentry-api-0-relay-register-response'
),
data=raw_json,
content_type='application/json',
HTTP_X_SENTRY_RELAY_ID=six.binary_type(uuid4()),
HTTP_X_SENTRY_RELAY_SIGNATURE=signature,
)
assert resp.status_code == 400, resp.content
def _get_connection(driver_info):
# NOTE: python-iboot wants username and password as strings (not unicode)
return iboot.iBootInterface(driver_info['address'],
six.binary_type(driver_info['username']),
six.binary_type(driver_info['password']),
port=driver_info['port'],
num_relays=driver_info['relay_id'])
def test_init_automatic_pad(self):
"""
Addresses are automatically padded to 81 trytes.
"""
addy = Address(
b'JVMTDGDPDFYHMZPMWEKKANBQSLSDTIIHAYQUMZOK'
b'HXXXGJHJDQPOMDOMNRDKYCZRUFZROZDADTHZC'
)
self.assertEqual(
binary_type(addy),
# Note the extra 9's added to the end.
b'JVMTDGDPDFYHMZPMWEKKANBQSLSDTIIHAYQUMZOK'
b'HXXXGJHJDQPOMDOMNRDKYCZRUFZROZDADTHZC9999',
)
# This attribute will make more sense once we start working with
# address checksums.
self.assertEqual(
binary_type(addy.address),
b'JVMTDGDPDFYHMZPMWEKKANBQSLSDTIIHAYQUMZOK'
b'HXXXGJHJDQPOMDOMNRDKYCZRUFZROZDADTHZC9999',
)
# Checksum is not generated automatically.
self.assertIsNone(addy.checksum)
def test_init_with_checksum(self):
"""
Creating an address with checksum already attached.
"""
addy = Address(
b'RVORZ9SIIP9RCYMREUIXXVPQIPHVCNPQ9HZWYKFWYWZRE'
b'9JQKG9REPKIASHUUECPSQO9JT9XNMVKWYGVAFOXM9MUBX'
)
self.assertEqual(
binary_type(addy),
b'RVORZ9SIIP9RCYMREUIXXVPQIPHVCNPQ9HZWYKFWYWZRE'
b'9JQKG9REPKIASHUUECPSQO9JT9XNMVKWYGVAFOXM9MUBX',
)
self.assertEqual(
binary_type(addy.address),
b'RVORZ9SIIP9RCYMREUIXXVPQIPHVCNPQ9HZWYKFWYWZRE'
b'9JQKG9REPKIASHUUECPSQO9JT9XNMVKWYGVA',
)
self.assertEqual(
binary_type(addy.checksum),
b'FOXM9MUBX',
)
def test_concatenation(self):
"""
Concatenating TryteStrings with TrytesCompatibles.
"""
trytes1 = TryteString(b'RBTC9D9DCDQA')
trytes2 = TryteString(b'EASBYBCCKBFA')
concat = trytes1 + trytes2
self.assertIsInstance(concat, TryteString)
self.assertEqual(binary_type(concat), b'RBTC9D9DCDQAEASBYBCCKBFA')
# You can also concatenate a TryteString with any TrytesCompatible.
self.assertEqual(
binary_type(trytes1 + b'EASBYBCCKBFA'),
b'RBTC9D9DCDQAEASBYBCCKBFA',
)
self.assertEqual(
binary_type(trytes1 + 'EASBYBCCKBFA'),
b'RBTC9D9DCDQAEASBYBCCKBFA',
)
self.assertEqual(
binary_type(trytes1 + bytearray(b'EASBYBCCKBFA')),
b'RBTC9D9DCDQAEASBYBCCKBFA',
)
def _get_output_online(self, proc, log_stdout, log_stderr,
expect_stderr=False, expect_fail=False):
stdout, stderr = binary_type(), binary_type()
while proc.poll() is None:
if log_stdout:
line = proc.stdout.readline()
if line:
stdout += line
self._log_out(line.decode())
# TODO: what level to log at? was: level=5
# Changes on that should be properly adapted in
# test.cmd.test_runner_log_stdout()
else:
pass
if log_stderr:
line = proc.stderr.readline()
if line:
stderr += line
self._log_err(line.decode(), expect_stderr or expect_fail)
# TODO: what's the proper log level here?
# Changes on that should be properly adapted in
# test.cmd.test_runner_log_stderr()
else:
pass
return stdout, stderr
def test_default_args(self):
prev = ipv4(proto=inet.IPPROTO_IGMP)
g = igmpv3_report()
prev.serialize(g, None)
buf = g.serialize(bytearray(), prev)
res = unpack_from(igmpv3_report._PACK_STR, six.binary_type(buf))
buf = bytearray(buf)
pack_into("!H", buf, 2, 0)
eq_(res[0], IGMP_TYPE_REPORT_V3)
eq_(res[1], checksum(buf))
eq_(res[2], 0)
# records without record_num
prev = ipv4(proto=inet.IPPROTO_IGMP)
record1 = igmpv3_report_group(MODE_IS_INCLUDE, 0, 0, "225.0.0.1")
record2 = igmpv3_report_group(MODE_IS_INCLUDE, 0, 2, "225.0.0.2", ["172.16.10.10", "172.16.10.27"])
record3 = igmpv3_report_group(MODE_IS_INCLUDE, 1, 0, "225.0.0.3", [], b"abc\x00")
record4 = igmpv3_report_group(MODE_IS_INCLUDE, 1, 2, "225.0.0.4", ["172.16.10.10", "172.16.10.27"], b"abc\x00")
records = [record1, record2, record3, record4]
g = igmpv3_report(records=records)
prev.serialize(g, None)
buf = g.serialize(bytearray(), prev)
res = unpack_from(igmpv3_report._PACK_STR, six.binary_type(buf))
buf = bytearray(buf)
pack_into("!H", buf, 2, 0)
eq_(res[0], IGMP_TYPE_REPORT_V3)
eq_(res[1], checksum(buf))
eq_(res[2], len(records))
def setUp(self):
super(RelayPublicKeysConfigTest, self).setUp()
self.key_pair = generate_key_pair()
self.public_key = self.key_pair[1]
self.private_key = self.key_pair[0]
self.relay_id = six.text_type(uuid4())
self.relay = Relay.objects.create(
relay_id=self.relay_id,
public_key=six.binary_type(self.public_key),
is_internal=True
)
self.relay_a = Relay.objects.create(
relay_id=six.text_type(uuid4()),
public_key=six.binary_type(self.public_key),
is_internal=False
)
self.relay_b = Relay.objects.create(
relay_id=six.text_type(uuid4()),
public_key=six.binary_type(self.public_key),
is_internal=False
)
self.project = self.create_project()
self.path = reverse(
'sentry-api-0-relay-publickeys'
)
def _test_set_vlan_vid_none(self):
header = ofproto.OXM_OF_VLAN_VID
match = OFPMatch()
match.set_vlan_vid_none()
value = ofproto.OFPVID_NONE
cls_ = OFPMatchField._FIELDS_HEADERS.get(header)
pack_str = cls_.pack_str.replace('!', '')
fmt = '!HHI' + pack_str
# serialize
buf = bytearray()
length = match.serialize(buf, 0)
eq_(length, len(buf))
res = list(unpack_from(fmt, six.binary_type(buf), 0)[3:])
res_value = res.pop(0)
eq_(res_value, value)
# parser
res = match.parser(six.binary_type(buf), 0)
eq_(res.type, ofproto.OFPMT_OXM)
eq_(res.fields[0].header, header)
eq_(res.fields[0].value, value)
# to_jsondict
jsondict = match.to_jsondict()
# from_jsondict
match2 = match.from_jsondict(jsondict["OFPMatch"])
buf2 = bytearray()
match2.serialize(buf2, 0)
eq_(str(match), str(match2))
eq_(buf, buf2)
def make_header_prefix(self, message_class, version=Connection.protocol_version, stream_id=0):
if Connection.protocol_version < 3:
return six.binary_type().join(
map(
uint8_pack,
[
0xFF & (HEADER_DIRECTION_TO_CLIENT | version),
0, # flags (compression)
stream_id,
message_class.opcode, # opcode
],
)
)
else:
return six.binary_type().join(
map(
uint8_pack,
[
0xFF & (HEADER_DIRECTION_TO_CLIENT | version),
0, # flags (compression)
0, # MSB for v3+ stream
stream_id,
message_class.opcode, # opcode
],
)
)
def encode_key(project_id, namespace, key):
""" Encodes a key for memcached.
Args:
project_id: A string specifying the project ID.
namespace: A string specifying the namespace.
key: A bytestring specifying the memcache key.
Returns:
A bytestring in the form of <project-id>\x01<namespace>\x01<encoded-key>
Raises:
ApplicationError if the key is too long.
"""
if len(key) > MAX_KEY_SIZE:
raise apiproxy_errors.ApplicationError(
INVALID_VALUE, 'The key is too long: {}'.format(key))
project_id = six.binary_type(project_id)
namespace = six.binary_type(namespace)
encoded_key = base64.b64encode(key)
full_key = KEY_DELIMETER.join(
[project_id, namespace, INTACT_MARKER + encoded_key])
if len(full_key) <= MAX_KEY_SIZE:
return full_key
# GAE only rejects requests when the key length is too long. Since this
# implementation's stored key includes a namespace prefix, the key is hashed
# if necessary to comply with the memcached limit. The length of the key's
# hex digest + the max project ID size + the max namespace size is still less
# than the memcached limit.
hashed_key = hashlib.sha1(key).hexdigest()
return KEY_DELIMETER.join(
[project_id, namespace, HASHED_MARKER + hashed_key])
def getBytesStream(self):
"Render request String, but as bytes, which may contain invalid stuff"
ustr = six.text_type(self)
# from there we might be able to extract it as full ascii
try:
out = ustr.encode('ascii')
except UnicodeEncodeError:
# If we try to use characters which transletes to multy bytes chars
# we'll need to fix the size computations.
# There's a system in place which allows perfect binary crap for
# multibytes using BYTES_SPECIAL_REPLACE & record_bytes_to_send,
# this allows strange multibytes that a regular .encode('utf-8')
# would not allow.
raise BadEncodingException('Our requests should only '
'use ascii or binary specific tricks.')
# Now replace the strange bytes
for sbyte in self._strange_bytes_records:
if six.PY2:
out = out.replace(six.binary_type(Tools.BYTES_SPECIAL_REPLACE),
sbyte,
1)
else:
out = out.replace(six.binary_type(Tools.BYTES_SPECIAL_REPLACE,
'ascii'),
sbyte,
1)
return out
def test_string_type():
assert is_value_of_type('', STRING)
if six.PY2:
assert is_value_of_type(six.binary_type('string'), STRING)
else:
assert is_value_of_type(six.binary_type('string', encoding='utf-8'), STRING)
assert is_value_of_type(six.text_type('string'), STRING)
def serialize_HarResponse_from_Exception(self, io, raw):
err = self.get_EnvironmentError_from_Exception(raw)
status = str(err.errno)
if isinstance(status, six.text_type):
status = status.encode('utf-8')
reason = err.strerror
if isinstance(reason, six.text_type):
reason = reason.encode('utf-8')
# usually 4 tabs, omitted
io.write(b'"response": {\n')
io.write(b'\t"status": ' + six.binary_type(err.errno) + b',\n')
io.write(b'\t"statusText": "' +
six.binary_type(err.strerror) + b'",\n')
io.write(b'\t"httpVersion": "HTTP/1.1",\n')
io.write(b'\t"cookies": [\n')
io.write(b'\t],\n')
io.write(b'\t"headers": [\n')
io.write(b'\t],\n')
io.write(b'\t"content": {\n')
io.write(b'\t\t"mimeType": "application/x-error",\n')
io.write(b'\t\t"size": -1,\n')
io.write(b'\t\t"text": ')
io.write(six.binary_type(json.dumps(repr(raw))))
io.write(b'\n\t},\n')
io.write(b'\t"redirectURL": "' +
six.binary_type(err.filename) + '",\n')
io.write(b'\t"headersSize": -1,\n')
io.write(b'\t"bodySize": -1\n')
io.write(b'},\n')
def test_delete_node_with_children(self):
self.test_dt.create_node("/a", six.binary_type(b"test_data_a"))
self.test_dt.create_node("/a/b", six.binary_type(b"test_data_ab"))
self.test_dt.create_node("/a/c", six.binary_type(b"test_data_ac"))
pytest.raises(datatree.NotEmptyException, self.test_dt.delete_node,
"/a")
def to_frame(self, data, state):
"""
Extract a single frame from the data buffer. The consumed
data should be removed from the buffer. If no complete frame
can be read, must raise a ``NoFrames`` exception.
:param data: A ``bytearray`` instance containing the data so
far read.
:param state: An instance of ``FramerState``. If the buffer
contains a partial frame, this object can be
used to store state information to allow the
remainder of the frame to be read.
:returns: A frame. The frame may be any object. The stock
framers always return bytes.
"""
# Find the next null byte
data_len = data.find(b'\0')
if data_len < 0:
# No full frame yet
raise exc.NoFrames()
# Track how much to exclude
frame_len = data_len + 1
# Decode the data
frame = six.binary_type(self.variant.decode(
six.binary_type(data[:data_len])))
del data[:frame_len]
# Return the frame
return frame
def confirm(request, secret=None):
if request.POST["transStatus"] != "Y":
raise ValueError("Transaction not authorised")
passed_data = dict(request.POST.items())
M_params = sorted(
(key, value) for (key, value) in passed_data.items() if key.startswith("M_") and key != "M_authenticator"
)
if secret is not None:
# Generate a HMAC to verify our data is untouched
if not isinstance(secret, binary_type):
raise ValueError("Secret must be a bytes object")
auth = hmac.new(secret, digestmod=hashlib.sha256)
auth.update(binary_type(passed_data["cartId"].encode("utf-8")))
auth.update(binary_type(passed_data["amount"].encode("utf-8")))
auth.update(binary_type(passed_data["currency"].encode("utf-8")))
params = urlencode(M_params)
auth.update(params.encode("utf-8"))
authenticator = auth.hexdigest()
if authenticator != passed_data["M_authenticator"]:
logger.warn(
"Got incorrect authenticator. Expected %s, got %s." % (authenticator, passed_data["M_authenticator"])
)
raise ValueError("Transaction details have been tampered with. Aborting.")
return passed_data
def test_manage_library_users(self):
"""
Simple test that the Library "User Access" view works.
Also tests that we can use the REST API to assign a user to a library.
"""
library = LibraryFactory.create()
extra_user, _ = self.create_non_staff_user()
manage_users_url = reverse_library_url('manage_library_users', text_type(library.location.library_key))
response = self.client.get(manage_users_url)
self.assertEqual(response.status_code, 200)
# extra_user has not been assigned to the library so should not show up in the list:
self.assertNotIn(binary_type(extra_user.username), response.content)
# Now add extra_user to the library:
user_details_url = reverse_course_url(
'course_team_handler',
library.location.library_key, kwargs={'email': extra_user.email}
)
edit_response = self.client.ajax_post(user_details_url, {"role": LibraryUserRole.ROLE})
self.assertIn(edit_response.status_code, (200, 204))
# Now extra_user should apear in the list:
response = self.client.get(manage_users_url)
self.assertEqual(response.status_code, 200)
self.assertIn(binary_type(extra_user.username), response.content)
def test_can_insert_udts_with_nulls(self):
"""
Test the insertion of UDTs with null and empty string fields
"""
c = Cluster(protocol_version=PROTOCOL_VERSION)
s = c.connect(self.keyspace_name, wait_for_all_pools=True)
s.execute("CREATE TYPE user (a text, b int, c uuid, d blob)")
User = namedtuple('user', ('a', 'b', 'c', 'd'))
c.register_user_type(self.keyspace_name, "user", User)
s.execute("CREATE TABLE mytable (a int PRIMARY KEY, b frozen<user>)")
insert = s.prepare("INSERT INTO mytable (a, b) VALUES (0, ?)")
s.execute(insert, [User(None, None, None, None)])
results = s.execute("SELECT b FROM mytable WHERE a=0")
self.assertEqual((None, None, None, None), results[0].b)
select = s.prepare("SELECT b FROM mytable WHERE a=0")
self.assertEqual((None, None, None, None), s.execute(select)[0].b)
# also test empty strings
s.execute(insert, [User('', None, None, six.binary_type())])
results = s.execute("SELECT b FROM mytable WHERE a=0")
self.assertEqual(('', None, None, six.binary_type()), results[0].b)
c.shutdown()
def _parse_binary(v, header_d):
""" Parses binary string.
Note:
<str> for py2 and <binary> for py3.
"""
# This is often a no-op, but it ocassionally converts numbers into strings
v = nullify(v)
if v is None:
return None
if six.PY2:
try:
return six.binary_type(v).strip()
except UnicodeEncodeError:
return six.text_type(v).strip()
else:
# py3
try:
return six.binary_type(v, 'utf-8').strip()
except UnicodeEncodeError:
return six.text_type(v).strip()
def test_barbican_cert(self):
# Certificate data
self.certificate = six.binary_type(sample.X509_CERT)
self.intermediates = sample.X509_IMDS_LIST
self.private_key = six.binary_type(sample.X509_CERT_KEY_ENCRYPTED)
self.private_key_passphrase = sample.X509_CERT_KEY_PASSPHRASE
self._prepare()
container = containers.CertificateContainer(
api=mock.MagicMock(),
certificate=self.certificate_secret,
intermediates=self.intermediates_secret,
private_key=self.private_key_secret,
private_key_passphrase=self.private_key_passphrase_secret
)
# Create a cert
cert = barbican_common.BarbicanCert(
cert_container=container
)
# Validate the cert functions
self.assertEqual(cert.get_certificate(), sample.X509_CERT)
self.assertEqual(cert.get_intermediates(), sample.X509_IMDS_LIST)
self.assertEqual(cert.get_private_key(),
sample.X509_CERT_KEY_ENCRYPTED)
self.assertEqual(cert.get_private_key_passphrase(),
six.b(sample.X509_CERT_KEY_PASSPHRASE))
def parse_driver_info(node):
"""Parses and creates AMT driver info
:param node: an Ironic node object.
:returns: AMT driver info.
:raises: MissingParameterValue if any required parameters are missing.
:raises: InvalidParameterValue if any parameters have invalid values.
"""
info = node.driver_info or {}
d_info = {}
missing_info = []
for param in REQUIRED_PROPERTIES:
value = info.get(param)
if value:
d_info[param[4:]] = six.binary_type(value)
else:
missing_info.append(param)
if missing_info:
raise exception.MissingParameterValue(
_("AMT driver requires the following to be set in " "node's driver_info: %s.") % missing_info
)
d_info["uuid"] = node.uuid
param = "amt_protocol"
protocol = info.get(param, CONF.amt.get(param[4:]))
if protocol not in AMT_PROTOCOL_PORT_MAP:
raise exception.InvalidParameterValue(_("Invalid " "protocol %s.") % protocol)
d_info[param[4:]] = six.binary_type(protocol)
return d_info
def _serialize_field(cls, val, type_):
if type_ == 'str':
return six.binary_type(val)
if type_ in cls._struct_type_map:
struct_type = six.binary_type(cls._struct_type_map[type_])
return struct.pack(struct_type, val)
else:
raise NotImplementedError("%s serializer not implemented" % type_)
def decrypt(self, password):
key = derive_key(password, six.binary_type(self.salt))
return decrypt(six.binary_type(self.iv),
key,
six.binary_type(self.associated_data),
six.binary_type(self.tag),
b64decode(self.ciphertext))
def test_record_binary_post_data(self):
post_format = "binary"
db = self.visit("/post_request_ajax.html?format=" + post_format)
post_body = self.get_post_request_body_from_db(db, True)
# Binary strings get put into the database as-if they were latin-1.
assert six.binary_type(bytearray(range(100))) == post_body
def _ready_read(self):
reply = self.sender()
self._raw_html = self._raw_html + six.binary_type(
reply.peek(reply.bytesAvailable()))
def fetch_popular_shows(self, page_url=None, trakt_list=None):
"""Get a list of popular shows from different Trakt lists based on a provided trakt_list.
:param page_url: the page url opened to the base api url, for retreiving a specific list
:param trakt_list: a description of the trakt list
:return: A list of RecommendedShow objects, an empty list of none returned
:throw: ``Exception`` if an Exception is thrown not handled by the libtrats exceptions
"""
trending_shows = []
removed_from_medusa = []
# Create a trakt settings dict
trakt_settings = {
'trakt_api_secret': app.TRAKT_API_SECRET,
'trakt_api_key': app.TRAKT_API_KEY,
'trakt_access_token': app.TRAKT_ACCESS_TOKEN,
'trakt_refresh_token': app.TRAKT_REFRESH_TOKEN
}
trakt_api = TraktApi(timeout=app.TRAKT_TIMEOUT,
ssl_verify=app.SSL_VERIFY,
**trakt_settings)
try:
not_liked_show = ''
if app.TRAKT_ACCESS_TOKEN != '':
library_shows = self.fetch_and_refresh_token(trakt_api, 'sync/watched/shows?extended=noseasons') + \
self.fetch_and_refresh_token(trakt_api, 'sync/collection/shows?extended=full')
medusa_shows = [
show.indexerid for show in app.showList if show.indexerid
]
removed_from_medusa = [
lshow['show']['ids']['tvdb'] for lshow in library_shows
if lshow['show']['ids']['tvdb'] not in medusa_shows
]
if app.TRAKT_BLACKLIST_NAME is not None and app.TRAKT_BLACKLIST_NAME:
not_liked_show = trakt_api.request(
'users/' + app.TRAKT_USERNAME + '/lists/' +
app.TRAKT_BLACKLIST_NAME + '/items') or []
else:
log.debug('Trakt blacklist name is empty')
if trakt_list not in ['recommended', 'newshow', 'newseason']:
limit_show = '?limit=' + text_type(100 +
len(not_liked_show)) + '&'
else:
limit_show = '?'
series = self.fetch_and_refresh_token(
trakt_api,
page_url + limit_show + 'extended=full,images') or []
# Let's trigger a cache cleanup.
missing_posters.clean()
for show in series:
try:
if 'show' not in show:
show['show'] = show
if not_liked_show:
if show['show']['ids']['tvdb'] in (
s['show']['ids']['tvdb']
for s in not_liked_show
if s['type'] == 'show'):
continue
else:
trending_shows.append(
self._create_recommended_show(
show,
storage_key=b'trakt_{0}'.format(
show['show']['ids']['trakt'])))
except MultipleShowObjectsException:
continue
# Update the dogpile index. This will allow us to retrieve all stored dogpile shows from the dbm.
update_recommended_series_cache_index(
'trakt', [binary_type(s.series_id) for s in trending_shows])
blacklist = app.TRAKT_BLACKLIST_NAME not in ''
except TraktException as error:
log.warning('Could not connect to Trakt service: {0}', error)
raise
return blacklist, trending_shows, removed_from_medusa
def wrapper(*args):
time_point = int(time.time()) - 5 * 60
g = fun(*args)
return g.has('updated', (binary_type('_t'), lt(time_point)))
def test_init_happy_path(self):
"""
Creating a valid address checksum.
"""
self.assertEqual(binary_type(AddressChecksum(b'FOXM9MUBX')),
b'FOXM9MUBX')
def relay_id():
return six.binary_type(uuid4())
def _bytes_feature(value):
"""Wrapper for inserting bytes features into Example proto."""
if isinstance(value, six.string_types):
value = six.binary_type(value, encoding='utf-8')
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
def relay_id():
return six.binary_type(six.text_type(uuid4()).encode("ascii"))
def _sendall(self, data):
data = six.binary_type(data) if data else data
return SocketIOHandler._sendall(self, data)
def serialize(val):
return six.binary_type(val)
def __init__(self):
self._first_try = True
self._data = binary_type()
self._obj = zlib.decompressobj()
def read(self, amt=None, decode_content=None, cache_content=False):
"""
Similar to :meth:`httplib.HTTPResponse.read`, but with two additional
parameters: ``decode_content`` and ``cache_content``.
:param amt:
How much of the content to read. If specified, caching is skipped
because it doesn't make sense to cache partial content as the full
response.
:param decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param cache_content:
If True, will save the returned data such that the same result is
returned despite of the state of the underlying file object. This
is useful if you want the ``.data`` property to continue working
after having ``.read()`` the file object. (Overridden if ``amt`` is
set.)
"""
# Note: content-encoding value should be case-insensitive, per RFC 2616
# Section 3.5
content_encoding = self.headers.get('content-encoding', '').lower()
if self._decoder is None:
if content_encoding in self.CONTENT_DECODERS:
self._decoder = _get_decoder(content_encoding)
if decode_content is None:
decode_content = self.decode_content
if self._fp is None:
return
flush_decoder = False
try:
if amt is None:
# cStringIO doesn't like amt=None
data = self._fp.read()
flush_decoder = True
else:
cache_content = False
data = self._fp.read(amt)
if amt != 0 and not data: # Platform-specific: Buggy versions of Python.
# Close the connection when no data is returned
#
# This is redundant to what httplib/http.client _should_
# already do. However, versions of python released before
# December 15, 2012 (http://bugs.python.org/issue16298) do not
# properly close the connection in all cases. There is no harm
# in redundantly calling close.
self._fp.close()
flush_decoder = True
try:
if decode_content and self._decoder:
data = self._decoder.decompress(data)
except (IOError, zlib.error):
raise DecodeError(
"Received response with content-encoding: %s, but "
"failed to decode it." % content_encoding)
if flush_decoder and self._decoder:
buf = self._decoder.decompress(binary_type())
data += buf + self._decoder.flush()
if cache_content:
self._body = data
return data
finally:
if self._original_response and self._original_response.isclosed():
self.release_conn()
from __future__ import absolute_import
import pytest
import six
from sentry.utils.hashlib import md5_text, sha1_text, hash_values
from unittest import TestCase
HASHLIB_VALUES_TESTS = (
("seed", None, "75a0ad233bd9a091d9d26bacbe2f377e"),
("seed", True, "1057fb936dc9056388c0b9b48dd0c7df"),
("seed", False, "07aae33053c0f3487882d61353780682"),
("seed", 42, "d1ce9a19d659ae70a6b76ef6029ae542"),
("seed", six.binary_type("test"), "334e3fd2f66966a5c785d825c5f03494"),
("seed", six.text_type("test"), "ce35c0ce0d38976f61a5ca951de74a16"),
("seed", (4, 2), "d03b32e798444249d726158594d370f6"),
("seed", {"test": 42}, "c545cf1c4ab09eff4a1e0fa5209f1645"),
)
@pytest.mark.parametrize("seed,value,hash", HASHLIB_VALUES_TESTS)
def test_hash_values(seed, value, hash):
assert hash_values([value], seed=seed) == hash
class HashlibTest(TestCase):
def test_simple(self):
md5_text("x").hexdigest() == "9dd4e461268c8034f5c8564e155c67a6"
sha1_text("x").hexdigest() == "11f6ad8ec52a2984abaafd7c3b516503785c2072"
def fn(data):
binary_packets.append(bytearray(b64encode(six.binary_type(data))))
return {'_placeholder': True, 'num': len(binary_packets) - 1}
def test_getting_contents_as_bytes_str(d, val, encoding):
assert binary_type(d) == val.encode(encoding)
def execute(command,
env=None,
cwd=None,
split_lines=False,
ignore_errors=False,
extra_ignore_errors=(),
with_errors=True,
none_on_ignored_error=False,
return_error_code=False,
log_output_on_error=True,
results_unicode=True,
return_errors=False):
"""Execute a command and return the output.
Args:
command (unicode or list of unicode):
The command to execute.
env (dict, optional):
Environment variables to pass to the called executable. These will
be added to the current environment.
cwd (unicode, optional):
An optional working directory to change to before executing the
process.
split_lines (bool, optional):
Whether to return the output as a list of lines or a single string.
ignore_errors (bool, optional):
Whether to ignore errors. If ``False``, this will raise an
exception.
extra_ignore_errors (tuple, optional):
A set of errors to ignore even when ``ignore_errors`` is False.
This is used because some commands (such as diff) use non-zero
return codes even when the command was successful.
with_errors (bool, optional):
Whether to combine the output and error streams of the command
together into a single return value. This argument is mutually
exclusive with the ``return_errors`` argument.
none_on_ignored_error (bool, optional):
Whether to return ``None`` in the case that an error was ignored
(instead of the output of the command).
return_error_code (bool, optional):
Whether to include the exit status of the executed command in
addition to the output
log_output_on_error (bool, optional):
If ``True``, the output from the command will be logged in the case
that the command returned a non-zero exit code.
results_unicode (bool, optional):
If ``True``, the output will be treated as text and returned as
unicode strings instead of bytes.
return_errors (bool, optional):
Whether to return the content of the stderr stream. This argument
is mutually exclusive with the ``with_errors`` argument.
Returns:
This returns a single value, 2-tuple, or 3-tuple depending on the
arguments.
If ``return_error_code`` is True, the error code of the process will be
returned as the first element of the tuple.
If ``return_errors`` is True, the process' standard error stream will
be returned as the last element of the tuple.
If both of ``return_error_code`` and ``return_errors`` are ``False``,
then the process' output will be returned. If either or both of them
are ``True``, then this is the other element of the returned tuple.
"""
assert not (with_errors and return_errors)
if isinstance(command, list):
log_command_line('Running: %s', command)
else:
logging.debug('Running: %s', command)
new_env = os.environ.copy()
if env:
new_env.update(env)
# TODO: This can break on systems that don't have the en_US locale
# installed (which isn't very many). Ideally in this case, we could
# put something in the config file, but that's not plumbed through to here.
new_env['LC_ALL'] = 'en_US.UTF-8'
new_env['LANGUAGE'] = 'en_US.UTF-8'
if with_errors:
errors_output = subprocess.STDOUT
else:
errors_output = subprocess.PIPE
popen_encoding_args = {}
if results_unicode:
# Popen on Python 2 doesn't support the ``encoding`` parameter, so we
# have to use ``universal_newlines``.
if six.PY3:
popen_encoding_args['encoding'] = 'utf-8'
else:
popen_encoding_args['universal_newlines'] = True
if sys.platform.startswith('win'):
# Convert all environment variables to byte strings, so that subprocess
# doesn't blow up on Windows.
new_env = dict((six.binary_type(key), six.binary_type(value))
for key, value in six.iteritems(new_env))
p = subprocess.Popen(command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=errors_output,
shell=False,
env=new_env,
cwd=cwd,
**popen_encoding_args)
else:
p = subprocess.Popen(command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=errors_output,
shell=False,
close_fds=True,
env=new_env,
cwd=cwd,
**popen_encoding_args)
data, errors = p.communicate()
if split_lines:
data = data.splitlines(True)
if return_errors:
if split_lines:
errors = errors.splitlines(True)
else:
errors = None
rc = p.wait()
if rc and not ignore_errors and rc not in extra_ignore_errors:
if log_output_on_error:
logging.debug('Command exited with rc %s: %s\n%s---', rc, command,
data)
raise Exception('Failed to execute command: %s' % command)
elif rc:
if log_output_on_error:
logging.debug('Command exited with rc %s: %s\n%s---', rc, command,
data)
else:
logging.debug('Command exited with rc %s: %s', rc, command)
if rc and none_on_ignored_error:
data = None
if return_error_code and return_errors:
return rc, data, errors
elif return_error_code:
return rc, data
elif return_errors:
return data, errors
else:
return data
def serialize(val, protocol_version):
return six.binary_type(val)
def calcCRC(data):
crc = CRCCCITT("FFFF").calculate(six.binary_type(data))
b = bytearray(struct.pack(">H", crc))
return b
def test_from_trits(self):
"""
Converting a sequence of trit values into a TryteString.
"""
trits = [
0,
0,
-1,
-1,
1,
0,
-1,
1,
-1,
0,
1,
0,
0,
0,
0,
1,
1,
0,
0,
0,
0,
1,
1,
0,
0,
1,
0,
1,
1,
0,
-1,
0,
-1,
1,
0,
0,
-1,
-1,
1,
1,
0,
0,
1,
0,
-1,
-1,
1,
0,
1,
-1,
0,
-1,
1,
0,
0,
1,
0,
0,
1,
0,
-1,
1,
1,
-1,
1,
0,
0,
-1,
1,
1,
0,
0,
]
self.assertEqual(
binary_type(TryteString.from_trits(trits)),
b'RBTC9D9DCDQAEASBYBCCKBFA',
)
def default_uuid():
return six.binary_type(uuid.uuid4())
def _build_response_headers(self, service_provider):
return [('Content-Type', 'text/xml'),
('X-sp-url', six.binary_type(service_provider['sp_url'])),
('X-auth-url', six.binary_type(service_provider['auth_url']))]
def _integer_randomizer_factory(name, ttype, n_bits):
_universe_size = 2**n_bits
_min = -(2**(n_bits - 1))
_max = (2**(n_bits - 1)) - 1
_name = name
_ttype = ttype
_n_bits = n_bits
_random_i32 = _random_int_factory(_n_bits)
class NBitIntegerRandomizer(ScalarTypeRandomizer):
name = _name
ttype = _ttype
default_constraints = dict(ScalarTypeRandomizer.default_constraints)
default_constraints.update({'range': [], 'fuzz_max_delta': 4})
def _randomize(self):
val = super(NBitIntegerRandomizer, self)._randomize()
if val is not None:
return val
range_ = self.constraints['range']
if range_:
min_, max_ = range_
return random.randint(min_, max_)
return _random_i32()
def _flip_bit(self, seed):
"""Fuzz seed by flipping one bit, excluding the sign bit"""
flipper = 1 << random.randint(0, _n_bits - 2)
return seed ^ flipper
def _add_delta(self, seed):
"""Fuzz seed by adding a small number"""
max_delta = self.constraints['fuzz_max_delta']
delta = random.randint(-max_delta, max_delta)
fuzzed = seed + delta
# Make sure fuzzed is in [_min, _max] to avoid overflow
return max(min(_max, fuzzed), _min)
def _fuzz(self, seed):
"""Apply a random fuzzer function"""
seed = self.eval_seed(seed)
fuzz_fn = random.choice([self._flip_bit, self._add_delta])
return fuzz_fn(seed)
@property
def universe_size(self):
return _universe_size
def eval_seed(self, seed):
if isinstance(seed, six.string_types):
return int(seed)
elif isinstance(seed, six.integer_types):
return seed
else:
raise TypeError("Invalid %s seed: %s" % (_name, seed))
NBitIntegerRandomizer.__name__ = six.binary_type("%sRandomizer" % _name)
return NBitIntegerRandomizer
def header(buf):
assert len(buf) >= ofproto_common.OFP_HEADER_SIZE
# LOG.debug('len %d bufsize %d', len(buf), ofproto.OFP_HEADER_SIZE)
return struct.unpack_from(ofproto_common.OFP_HEADER_PACK_STR,
six.binary_type(buf))
# -*- coding: utf-8 -*-
from __future__ import absolute_import
import pytest
import six
from sentry.utils.hashlib import md5_text, sha1_text, hash_values
from unittest import TestCase
HASHLIB_VALUES_TESTS = (
('seed', None, '75a0ad233bd9a091d9d26bacbe2f377e'),
('seed', True, '1057fb936dc9056388c0b9b48dd0c7df'),
('seed', False, '07aae33053c0f3487882d61353780682'),
('seed', 42, 'd1ce9a19d659ae70a6b76ef6029ae542'),
('seed', six.binary_type('test'), '334e3fd2f66966a5c785d825c5f03494'),
('seed', six.text_type('test'), 'ce35c0ce0d38976f61a5ca951de74a16'),
('seed', (4, 2), 'd03b32e798444249d726158594d370f6'),
('seed', {
'test': 42
}, 'c545cf1c4ab09eff4a1e0fa5209f1645'),
)
@pytest.mark.parametrize('seed,value,hash', HASHLIB_VALUES_TESTS)
def test_hash_values(seed, value, hash):
assert hash_values([value], seed=seed) == hash
class HashlibTest(TestCase):
def test_simple(self):
def test_serialize(self):
buf = self.fragment.serialize()
res = struct.unpack_from(self.form, six.binary_type(buf))
eq_(self.nxt, res[0])
eq_(self.off_m, res[1])
eq_(self.id_, res[2])
from trackpy.tests.common import StrictTestCase
from trackpy.tests.common import TrackpyImageSequence
# Quiet warnings about get_store being deprecated.
# These come from pandas.io and are caused by line 62:
# s = self.storage_class(STORE_NAME)
import warnings
warnings.filterwarnings("ignore", message="get_store is deprecated")
path, _ = os.path.split(os.path.abspath(__file__))
# This is six stuff here because pandas.HDFStore is fussy about the string type of one of
# its option args. There seems to be no good reason for that at all.
if six.PY2:
zlib = six.binary_type('zlib')
elif six.PY3:
zlib = 'zlib'
else:
raise ("six is confused")
def _random_hash():
return ''.join(map(str, np.random.randint(0, 10, 10)))
def _skip_if_no_pytables():
try:
import tables
except ImportError:
raise unittest.SkipTest('pytables not installed. Skipping.')
def relay(relay_id, public_key):
return Relay.objects.create(relay_id=relay_id,
public_key=six.binary_type(public_key),
is_internal=True)
def make_msg(self, header, body=six.binary_type()):
return header + uint32_pack(len(body)) + body
def toString(self):
return six.binary_type(self.data)
def _get_output_online(self,
proc,
log_stdout,
log_stderr,
outputstream,
errstream,
expect_stderr=False,
expect_fail=False):
"""
If log_stdout or log_stderr are callables, they will be given a read
line to be processed, and return processed result. So if they need to
'swallow' the line from being logged, should just return None
Parameters
----------
proc
log_stdout: bool or callable or 'online' or 'offline'
log_stderr: : bool or callable or 'online' or 'offline'
If any of those 'offline', we would call proc.communicate at the
end to grab possibly outstanding output from it
expect_stderr
expect_fail
Returns
-------
"""
stdout, stderr = binary_type(), binary_type()
log_stdout_ = _decide_to_log(log_stdout)
log_stderr_ = _decide_to_log(log_stderr)
log_stdout_is_callable = callable(log_stdout_)
log_stderr_is_callable = callable(log_stderr_)
# arguments to be passed into _process_one_line
stdout_args = ('stdout', proc, log_stdout_, log_stdout_is_callable)
stderr_args = ('stderr', proc, log_stderr_, log_stderr_is_callable,
expect_stderr or expect_fail)
while proc.poll() is None:
# see for a possibly useful approach to processing output
# in another thread http://codereview.stackexchange.com/a/17959
# current problem is that if there is no output on stderr
# it stalls
# Monitor if anything was output and if nothing, sleep a bit
stdout_, stderr_ = None, None
if log_stdout_:
stdout_ = self._process_one_line(*stdout_args)
stdout += stdout_
if log_stderr_:
stderr_ = self._process_one_line(*stderr_args)
stderr += stderr_
if not (stdout_ or stderr_):
# no output was generated, so sleep a tiny bit
time.sleep(0.001)
# Handle possible remaining output
stdout_, stderr_ = proc.communicate()
# ??? should we condition it on log_stdout in {'offline'} ???
stdout += self._process_remaining_output(outputstream, stdout_,
*stdout_args)
stderr += self._process_remaining_output(errstream, stderr_,
*stderr_args)
return stdout, stderr