def test_small_long_encode_decode(self):
encoded1 = BSON.encode({'x': 256})
decoded1 = BSON.decode(encoded1)['x']
self.assertEqual(256, decoded1)
self.assertEqual(type(256), type(decoded1))
encoded2 = BSON.encode({'x': 256L})
decoded2 = BSON.decode(encoded2)['x']
self.assertEqual(256L, decoded2)
self.assertEqual(type(256L), type(decoded2))
self.assertNotEqual(type(decoded1), type(decoded2))
def test_small_long_encode_decode(self):
encoded1 = BSON.encode({'x': 256})
decoded1 = BSON.decode(encoded1)['x']
self.assertEqual(256, decoded1)
self.assertEqual(type(256), type(decoded1))
encoded2 = BSON.encode({'x': 256L})
decoded2 = BSON.decode(encoded2)['x']
self.assertEqual(256L, decoded2)
self.assertEqual(type(256L), type(decoded2))
self.assertNotEqual(type(decoded1), type(decoded2))
def test_small_long_encode_decode(self):
encoded1 = BSON.encode({'x': 256})
decoded1 = BSON.decode(encoded1)['x']
self.assertEqual(256, decoded1)
self.assertEqual(type(256), type(decoded1))
encoded2 = BSON.encode({'x': Int64(256)})
decoded2 = BSON.decode(encoded2)['x']
expected = Int64(256)
self.assertEqual(expected, decoded2)
self.assertEqual(type(expected), type(decoded2))
self.assertNotEqual(type(decoded1), type(decoded2))
def test_small_long_encode_decode(self):
encoded1 = BSON.encode({'x': 256})
decoded1 = BSON.decode(encoded1)['x']
self.assertEqual(256, decoded1)
self.assertEqual(type(256), type(decoded1))
encoded2 = BSON.encode({'x': Int64(256)})
decoded2 = BSON.decode(encoded2)['x']
expected = Int64(256)
self.assertEqual(expected, decoded2)
self.assertEqual(type(expected), type(decoded2))
self.assertNotEqual(type(decoded1), type(decoded2))
def test_small_long_encode_decode(self):
if PY3:
raise SkipTest("No long type in Python 3.")
encoded1 = BSON.encode({'x': 256})
decoded1 = BSON.decode(encoded1)['x']
self.assertEqual(256, decoded1)
self.assertEqual(type(256), type(decoded1))
encoded2 = BSON.encode({'x': 256L})
decoded2 = BSON.decode(encoded2)['x']
self.assertEqual(256L, decoded2)
self.assertEqual(type(256L), type(decoded2))
self.assertNotEqual(type(decoded1), type(decoded2))
def test_small_long_encode_decode(self):
if PY3:
raise SkipTest("No long type in Python 3.")
encoded1 = BSON.encode({'x': 256})
decoded1 = BSON.decode(encoded1)['x']
self.assertEqual(256, decoded1)
self.assertEqual(type(256), type(decoded1))
encoded2 = BSON.encode({'x': 256L})
decoded2 = BSON.decode(encoded2)['x']
self.assertEqual(256L, decoded2)
self.assertEqual(type(256L), type(decoded2))
self.assertNotEqual(type(decoded1), type(decoded2))
def loadBsonObject(_hash):
fpath = join(objectDir, _hash[:2], _hash[2:])
bsonBytes = open(fpath, 'rb').read()
if _hash != sha1(bsonBytes).hexdigest():
raise IOError('sha1 diggest does not match for object file "%s"' %
fpath)
return BSON.decode(bsonBytes)
def test_get_all_features_including_nlp(self):
"""
Test `get_all_features` when NLP features should be included.
"""
prediction_label = 'total_game_hours'
nlp_features = True
expected_features = {}
features_list = list(self.review_doc.keys())
for key in ['achievement_progress', 'nlp_features']:
del features_list[features_list.index(key)]
if key == 'nlp_features':
(expected_features
.update(BSON.decode(self.review_doc['nlp_features'])))
else:
key_features = self.review_doc.get(key, {})
for feature in key_features:
if (key_features[feature]
and key_features[feature] != self.nan):
expected_features[feature] = key_features[feature]
for key in features_list:
if not key in chain(LABELS, ['id_string']): continue
if (not self.review_doc[key]
or self.review_doc[key] == self.nan): continue
expected_features[key] = self.review_doc[key]
assert_equal(get_all_features(self.review_doc,
prediction_label,
nlp_features=nlp_features),
expected_features)
def clean_message(body, msg):
## Sanity Checks
rk = msg.delivery_info['routing_key']
if not rk:
raise Exception("Invalid routing-key '%s' (%s)" % (rk, body))
#logger.debug("Event: %s" % rk)
#logger.info( body )
## Try to decode event
if isinstance(body, dict):
event = body
else:
logger.debug(" + Decode JSON")
try:
if isinstance(body, str) or isinstance(body, unicode):
try:
event = json.loads(body)
logger.debug(" + Ok")
except Exception, err:
try:
logger.debug(" + Try hack for windows string")
# Hack for windows FS -_-
event = json.loads(body.replace('\\', '\\\\'))
logger.debug(" + Ok")
except Exception, err:
try:
logger.debug(" + Decode BSON")
bson = BSON(body)
event = bson.decode()
logger.debug(" + Ok")
except Exception, err:
raise Exception(err)
def send(request):
rid = random.randint(0, 0xFFFFFF).to_bytes(4, byteorder='big')
data = BSON.encode(request)
s = rid + data
p = struct.pack('>BLH', 2, random.randint(0, 0xFFFFFFFF), len(s))
sock.sendall(p + s)
data = bytearray()
exp_len = 7
got_len = False
while True:
data += sock.recv(4096)
if len(data) < exp_len:
continue
if not got_len:
d, r, l = struct.unpack('>BLH', data[:7])
assert d == 2
data = data[7:]
got_len = True
if len(data) < l:
continue
assert l > 4
assert len(data) == l
got_rid = data[:4]
assert got_rid == rid
bs = BSON.decode(bytes(data[4:]), tz_aware=True)
if 'error' in bs:
raise RemoteException(bs['error'])
if len(bs) == 1 and 'data' in bs:
return bs['data']
return bs
def deserialize(cls, fobj: IO[bytes]) -> Iterable[dict]:
for line in fobj:
# noinspection PyCallByClass,PyTypeChecker
obj = BSON.decode(line.rstrip(b'\n'))
# noinspection PyTypeChecker
obj = unbyteify_attachments(obj)
yield obj
def test_task_xmr():
t = TestTask(aggdef)
socket = Socket()
result = t.start_xmr(socket, full=False)
assert_equal(result, {'job_id': 'foobar'})
sent = BSON.decode(socket.sent[0])
assert_equal(sent['$command'], 'xmapreduce')
def read_gcfile(path):
"""
Read a gc in_file and decode the encoded scene object.
Uses the decoder object specified in the gazer.settings.
Parameters
----------
path : in_file like stream
File that contains an encoded scene.
Returns
-------
gazer.scene.Scene
Scene object encoded in the in_file or None if no valid Scene
was encoded.
"""
logger.debug('Reading file as gcfile {}'.format(path))
with open(path, 'rb') as in_file:
try:
contents = in_file.read()
bson_obj = BSON(contents)
wrapper = bson_obj.decode()
from gazer.settings import DECODERS
wrapper_type = wrapper['type']
decoder = DECODERS.get(wrapper_type)
if decoder is None:
raise ValueError('Decoder {} not found'.format(wrapper_type))
body = wrapper['data']
if wrapper['compression'] == 'bz2':
body = bz2.decompress(body)
scene = decoder.scene_from_data(body)
return scene
except RuntimeError:
logger.exception('Failed to read file.')
def work(self, body, msg, *args, **kargs):
## Sanity Checks
rk = msg.delivery_info['routing_key']
if not rk:
raise Exception("Invalid routing-key '%s' (%s)" % (rk, body))
#self.logger.info( body )
## Try to decode event
if isinstance(body, dict):
event = body
else:
self.logger.debug(" + Decode JSON")
try:
if isinstance(body, str) or isinstance(body, unicode):
try:
event = json.loads(body)
self.logger.debug(" + Ok")
except Exception, err:
try:
self.logger.debug(" + Try hack for windows string")
# Hack for windows FS -_-
event = json.loads(body.replace('\\', '\\\\'))
self.logger.debug(" + Ok")
except Exception, err :
try:
self.logger.debug(" + Decode BSON")
bson = BSON (body)
event = bson.decode()
self.logger.debug(" + Ok")
except Exception, err:
raise Exception(err)
def __process_bson_control_message(self, data):
data_json = BSON.decode(BSON(data), codec_options=CodecOptions(uuid_representation=STANDARD))
print data_json
if data_json.get('ping', None):
t = data_json.get('ping').get('timestamp')
info('Ping request with timestamp %d' % t)
self.send(AgentServerCommandPong())
elif data_json.get('exc_cmd', None):
t = data_json.get('exc_cmd').get('cmdline')
info('exc_cmd request with timestamp %s' % t)
self.send(AgentServerCommandExc())
elif data_json.get('watch-rule', None):
self.control_monitors(data_json.get('watch-rule').get('str', ''))
elif data_json.get('sensor-plugin-start', None):
plugin_id = data_json.get('sensor-plugin-start').get('plugin_id', None)
if plugin_id is not None:
self.watchdog_start_process(int(plugin_id))
elif data_json.get('sensor-plugin-stop', None):
plugin_id = data_json.get('sensor-plugin-stop').get('plugin_id', None)
if plugin_id is not None:
self.watchdog_stop_process(int(plugin_id))
elif data_json.get('sensor-plugin-enable'):
plugin_id = data_json.get('sensor-plugin-enable').get('plugin_id', None)
if plugin_id is not None:
self.watchdog_enable_process(int(plugin_id))
elif data_json.get('sensor-plugin-disable'):
plugin_id = data_json.get('sensor-plugin-disable').get('plugin_id', None)
if plugin_id is not None:
self.watchdog_disable_process(int(plugin_id))
else:
warning("Unknown BSON command: '%s'" % str(data_json))
def clean_message(body, msg):
## Sanity Checks
rk = msg.delivery_info["routing_key"]
if not rk:
raise Exception("Invalid routing-key '%s' (%s)" % (rk, body))
# logger.debug("Event: %s" % rk)
# logger.info( body )
## Try to decode event
if isinstance(body, dict):
event = body
else:
logger.debug(" + Decode JSON")
try:
if isinstance(body, str) or isinstance(body, unicode):
try:
event = json.loads(body)
logger.debug(" + Ok")
except Exception, err:
try:
logger.debug(" + Try hack for windows string")
# Hack for windows FS -_-
event = json.loads(body.replace("\\", "\\\\"))
logger.debug(" + Ok")
except Exception, err:
try:
logger.debug(" + Decode BSON")
bson = BSON(body)
event = bson.decode()
logger.debug(" + Ok")
except Exception, err:
raise Exception(err)
def _get_profiles(profiles):
"""
Out of incoming string create a bson string and decode it
"""
bson_string = BSON(profiles, encoding='utf8')
return bson_string.decode()
def read_beacon_payload(beacon_bytes):
"""Returns a BeaconPayload namedtuple representing the given beacon bytes.
Decrypts the inner beacon data if necessary.
:param beacon_bytes: beacon payload (bytes).
:return: BeaconPayload namedtuple
"""
if len(beacon_bytes) < BEACON_HEADER_LENGTH_V1:
raise InvalidBeaconingPacket(
"Beaconing packet must be at least %d bytes."
% BEACON_HEADER_LENGTH_V1
)
header = beacon_bytes[:BEACON_HEADER_LENGTH_V1]
version, beacon_type_code, expected_payload_length = struct.unpack(
BEACON_HEADER_FORMAT_V1, header
)
actual_payload_length = len(beacon_bytes) - BEACON_HEADER_LENGTH_V1
if len(beacon_bytes) - BEACON_HEADER_LENGTH_V1 < expected_payload_length:
raise InvalidBeaconingPacket(
"Invalid payload length: expected %d bytes, got %d bytes."
% (expected_payload_length, actual_payload_length)
)
payload_start = BEACON_HEADER_LENGTH_V1
payload_end = BEACON_HEADER_LENGTH_V1 + expected_payload_length
payload_bytes = beacon_bytes[payload_start:payload_end]
payload = None
if version == 1:
if len(payload_bytes) == 0:
# No encrypted inner payload; nothing to do.
pass
else:
try:
decrypted_data = fernet_decrypt_psk(
payload_bytes, ttl=60, raw=True
)
except InvalidToken:
raise InvalidBeaconingPacket(
"Failed to decrypt inner payload: check MAAS secret key."
)
try:
decompressed_data = decompress(decrypted_data)
except OSError:
raise InvalidBeaconingPacket(
"Failed to decompress inner payload: %r" % decrypted_data
)
try:
# Replace the data in the dictionary with its decrypted form.
payload = BSON.decode(decompressed_data)
except BSONError:
raise InvalidBeaconingPacket(
"Inner beacon payload is not BSON: %r" % decompressed_data
)
else:
raise InvalidBeaconingPacket("Unknown beacon version: %d" % version)
beacon_type_code = payload["type"] if payload else beacon_type_code
return BeaconPayload(
beacon_bytes, version, BEACON_TYPE_VALUES[beacon_type_code], payload
)
def get_details(self):
url = self.interpolate_url()
resp = self.api_client.get(url, op='details')
if resp.status_code == 200:
detail_config = BSON.decode(resp.content)
return detail_config
def read_version(self, library_name, symbol, as_of=None):
#TODO handle as_of
version_path = self._make_version_path(library_name, symbol)
try:
encoded_version_doc = self.client.get_object(Bucket=self.bucket,
Key=version_path)
except self.client.exceptions.NoSuchKey:
return None
return BSON.decode(encoded_version_doc['Body'].read())
def test_task_mr():
t = TestTask(aggdef)
socket = Socket()
Collection = namedtuple('Collection', 'name')
result = t.start_mr(socket, full=False, collection=Collection('zagg.foo'))
assert_equal(result, {'job_id': 'foobar'})
sent = BSON.decode(socket.sent[0])
assert_equal(sent['$command'], 'mapreduce')
assert_equal(sent['output_type'], 'reduce')
assert_equal(sent['output_name'], 'zagg.foo')
def from_bson(cls, bson):
data = BSON.decode(bson)
if not cls._valid(data):
raise Exception('invalid message')
headers = data['headers']
body = data['body']
method = data['method']
path = data['path']
return cls(headers, body, path, method)
def scene_from_data(self, data):
bson_data = BSON(data)
data_dict = bson_data.decode()
decoded_array = self._decode_array(data_dict[u'lookup_table'])
lut = ArrayLookupTable(decoded_array)
frames = [self._decode_array(value) for key, value
in
sorted(data_dict['frames'].items(), key=lambda x: int(x[0]))]
image_manager = ArrayStackImageManager(frames)
scene = ImageStackScene(image_manager, lut)
return scene
def __process_bson_control_message(self, data):
data_json = BSON.decode(
BSON(data),
codec_options=CodecOptions(uuid_representation=STANDARD))
#print data_json
if data_json.get('exc_cmd', None):
self.__process_exc_cmd(data_json)
elif data_json.get('fileupload', None):
self.__process_file_upload(data_json)
else:
warning("Unknown BSON command: '%s'" % str(data_json))
def test_task_basic():
t = TestTask(aggdef)
socket = Socket()
def func():
return 'asdf'
result = t.start_basic_job(socket, func)
assert_equal(result, {'job_id': 'foobar'})
sent = BSON.decode(socket.sent[0])
assert_equal(sent['$command'], 'basic')
assert_equal(sent['func_name'], 'func')
assert "return 'asdf'" in sent['func_text']
def test_encode_decode_numpy_data(self):
data = {'array': np.array([1, 2, 3])}
type_registry = TypeRegistry([NumpyArrayCodec()])
codec_options = CodecOptions(type_registry=type_registry)
encoded = BSON.encode(data, codec_options=codec_options)
decoded = BSON.decode(encoded, codec_options=codec_options)
self.assertIsInstance(decoded, type(data))
self.assertIn('array', decoded)
np.testing.assert_array_equal(data['array'], decoded['array'])
def kafka_take_out(self):
"""
连接kafka获取数据 将数据装换成字符串
:return:
"""
for each in self.consumer:
# try:
kafa_str = BSON.decode(each.value)
if kafa_str:
self.kafka_data_processing(kafa_str)
def headlineform( self, k, b, x ):
"""Return the headline form (JSON) for a given object.
The default implementation just returns all the top-level objects.
There are three arguments, the object [mongo] id,
the BSON representation, and the MongoEngine object
"""
d = {}
for k, v in BSON.decode( b ).items():
if k[0] != '_':
d[k] = v
return d
def onDataReceived(self, interest, data):
# now we have to retrieve the key
dataContent = str(data.getContent())
dataContent = BSON(dataContent)
dataDict = dataContent.decode()
allData = dataDict['results']
totalCount = dataDict['count']
dataCount = len(allData)
skipPos = dataDict['skip']
print '---------'
print 'Got {}/{} result(s), starting at {}'.format(dataCount, totalCount, skipPos)
asyncio.async(self.collectResults(allData))
def test_task_xmr_more():
t = TestTask(aggdef)
socket = Socket()
def map_func():
pass
def reduce_func():
pass
def finalize_func():
pass
result = t.start_xmr(socket, full=True, map=map_func, reduce=reduce_func, finalize=finalize_func)
assert_equal(result, {'job_id': 'foobar'})
sent = BSON.decode(socket.sent[0])
assert_equal(sent['$command'], 'xmapreduce')
assert_equal(sent['output_type'], 'replace')
assert_equal(sent['output_name'], 'zagg.result')
def get_geo():
#to acces the data first we need to get the colletion in where the files are stored
col = db.fs.files.find_one()
if col == None:
print("GeoJSON is still loading please wait and reselect year")
data = {}
else:
# once we have the object storing the file information, we can get the data and read it
bsdata = fs.get(col["_id"]).read()
# since the data was encode, we need to decode it back
data = BSON.decode(bsdata)
return jsonify(data)
def post(self):
"""
Recieve Snapshot and either pass the result to queue or publish function
"""
data = BSON.decode(request.get_data())
user_id = data[consts.USER_ID]
date = datetime.fromtimestamp(int(data[consts.DATETIME]) / 1000)
context = Context(base_save_path, user_id, date)
dump_binary_data(data, context)
if queue is not None:
queue.publish(consts.PARSER_INPUT_EXCHANGE_NAME,
json.dumps(data))
else:
publish(json.dumps(data))
def test_task_mr_more():
t = TestTask(aggdef)
socket = Socket()
Collection = namedtuple('Collection', 'name')
def map_func():
pass
def reduce_func():
pass
def finalize_func():
pass
result = t.start_mr(socket, full=True, collection=Collection('zagg.foo'),
map=map_func, reduce=reduce_func, finalize=finalize_func)
assert_equal(result, {'job_id': 'foobar'})
sent = BSON.decode(socket.sent[0])
assert_equal(sent['$command'], 'mapreduce')
assert_equal(sent['output_type'], 'replace')
assert_equal(sent['output_name'], 'zagg.foo')
def messages(self):
"""
Iterate over all available messages.
Yields:
object: The next decoded message.
"""
doc_start = 0
while doc_start + 4 < len(self._buffer):
# http://bsonspec.org/spec.html
(doc_len, ) = struct.unpack(b'<l', self._buffer[doc_start:doc_start + 4])
if doc_start + doc_len > len(self._buffer):
break
doc = BSON(self._buffer[doc_start:doc_start + doc_len])
yield doc.decode()
doc_start += doc_len
self._buffer = self._buffer[doc_start:]
def control_loop(self, stop_event):
while not stop_event.is_set():
try:
data = self.recv_line()
if data is not None:
if self.bson_protocol:
data_json = BSON.decode(BSON(data), codec_options=CodecOptions(uuid_representation=STANDARD))
if data_json.get('ping', None):
t = data_json.get('ping').get('timestamp')
self.send(AgentServerCommandPong(timestamp=long(t)))
elif data.startswith('ping'):
self.send(AgentServerCommandPong())
else:
self.close()
except Exception as e:
if not stop_event.is_set():
import traceback
error('Unexpected exception receiving from IDM server: {0}' + str(e))
time.sleep(1)
info('Ends IDM control message thread..')
self.control_thread_running = False
def process(self, data):
rid = data[:4]
bs = BSON.decode(bytes(data[4:]), tz_aware=True)
try:
value = self.process_bson(bs)
except Exception as e:
import traceback
traceback.print_exc()
value = None
error = 'Exception occured: {} {}'.format(str(type(e)), str(e))
else:
error = None
response = {}
if isinstance(value, (dict, OrderedDict)):
response = value
else:
if value != None:
response = { 'data': value }
if error != None:
response = { 'error': error }
if len(response):
bv = BSON.encode(response)
self.sendfun(rid + bv)
def __process_connection_message_bson(self, response):
message_ok = False
try:
bson_msg = BSON.decode(BSON(response), codec_options=CodecOptions(uuid_representation=STANDARD))
print bson_msg
if bson_msg.get('ok') is not None:
if bson_msg['ok'].get('id', None):
message_ok = True
print bson_msg['ok'].get('id', None)
elif bson_msg.get('noack') is not None:
your_sensor_id = bson_msg['noack'].get('your_sensor_id', None)
if your_sensor_id is not None:
self.__sensor_id_change_request_received = True
debug('UUID change request from :%s ' % self.__nameServer)
lock_uuid.acquire()
self.__write_new_system_id(str(your_sensor_id))
self.__set_sensor_id(str(your_sensor_id))
lock_uuid.release()
else:
error('Bad response from server')
except InvalidBSON:
error('Bad response from server {0}'.format(response))
return message_ok
def test_config(upload_fixture):
upload_sample(_SAMPLE_PATH, _HOST, _PORT)
data = BSON.decode(upload_fixture.last_request.body)
assert 'pose' in data and 'feelings' in data and 'depthImage' not in data
def toJsonBody(self):
return bson.decode(self.Body)
#!/usr/bin/env python3
import sys
from os.path import splitext
import json
from bson import BSON
for fname_json in sys.argv[1:]:
fname, ext = splitext(fname_json)
bson_s = open(fname_json, 'rb').read()
data = BSON.decode(bson_s)
open(fname + '.json', 'w').write(json.dumps(data))
def test_unicode_decode_error_handler(self):
enc = BSON.encode({"keystr": "foobar"})
# Test handling of bad key value.
invalid_key = BSON(enc[:7] + b'\xe9' + enc[8:])
replaced_key = b'ke\xe9str'.decode('utf-8', 'replace')
ignored_key = b'ke\xe9str'.decode('utf-8', 'ignore')
dec = BSON.decode(invalid_key,
CodecOptions(unicode_decode_error_handler="replace"))
self.assertEqual(dec, {replaced_key: u"foobar"})
dec = BSON.decode(invalid_key,
CodecOptions(unicode_decode_error_handler="ignore"))
self.assertEqual(dec, {ignored_key: u"foobar"})
self.assertRaises(InvalidBSON, BSON.decode, invalid_key,
CodecOptions(unicode_decode_error_handler="strict"))
self.assertRaises(InvalidBSON, BSON.decode, invalid_key,
CodecOptions())
self.assertRaises(InvalidBSON, BSON.decode, invalid_key)
# Test handing of bad string value.
invalid_val = BSON(enc[:18] + b'\xe9' + enc[19:])
replaced_val = b'fo\xe9bar'.decode('utf-8', 'replace')
ignored_val = b'fo\xe9bar'.decode('utf-8', 'ignore')
dec = BSON.decode(invalid_val,
CodecOptions(unicode_decode_error_handler="replace"))
self.assertEqual(dec, {u"keystr": replaced_val})
dec = BSON.decode(invalid_val,
CodecOptions(unicode_decode_error_handler="ignore"))
self.assertEqual(dec, {u"keystr": ignored_val})
self.assertRaises(InvalidBSON, BSON.decode, invalid_val,
CodecOptions(unicode_decode_error_handler="strict"))
self.assertRaises(InvalidBSON, BSON.decode, invalid_val,
CodecOptions())
self.assertRaises(InvalidBSON, BSON.decode, invalid_val)
# Test handing bad key + bad value.
invalid_both = BSON(enc[:7] + b'\xe9' + enc[8:18] + b'\xe9' + enc[19:])
dec = BSON.decode(invalid_both,
CodecOptions(unicode_decode_error_handler="replace"))
self.assertEqual(dec, {replaced_key: replaced_val})
dec = BSON.decode(invalid_both,
CodecOptions(unicode_decode_error_handler="ignore"))
self.assertEqual(dec, {ignored_key: ignored_val})
self.assertRaises(InvalidBSON, BSON.decode, invalid_both,
CodecOptions(unicode_decode_error_handler="strict"))
self.assertRaises(InvalidBSON, BSON.decode, invalid_both,
CodecOptions())
self.assertRaises(InvalidBSON, BSON.decode, invalid_both)
# Test handling bad error mode.
dec = BSON.decode(enc,
CodecOptions(unicode_decode_error_handler="junk"))
self.assertEqual(dec, {"keystr": "foobar"})
self.assertRaises(InvalidBSON, BSON.decode, invalid_both,
CodecOptions(unicode_decode_error_handler="junk"))
def unmarshal(cls, bson: bytes):
document = BSON.decode(bson)
return cls(payload=document['payload'],
capsule=document['capsule'],
collection=document['collection'],
owner_anid=document['owner_anid'])
# loads() method returns a Python dict
print("json_docs TYPE:", type(json_docs))
# return a list of all of the JSON document keys
print("MongoDB collections:", list(json_docs.keys()))
except ValueError as error:
print("json.loads() ValueError for BSON object:", error)
quit()
for key, val in json_docs.items():
for i, doc in enumerate(json_docs[key]):
try:
# print the original JSON document
print("\ndoc:", doc)
# encode the document using the BSON library
data = BSON.encode(doc)
print("BSON encoded data:", type(data))
# print the result of the BSON encoding
print("data:", data)
# decode the BSON document back to a Python dict object
decode_doc = BSON.decode(data)
print("decode_doc:", decode_doc)
except Exception as error:
# catch any BSON encoding or decoding errors
print("enumerate() JSON documents ERROR:", error)
def now():
return BSON.decode(BSON.encode({'t': datetime.datetime.now(datetime.timezone.utc)}), tz_aware=True)['t']
def loadBsonObject(_hash):
fpath = join(objectDir, _hash[:2], _hash[2:])
bsonBytes = open(fpath, 'rb').read()
if _hash != sha1(bsonBytes).hexdigest():
raise IOError('sha1 diggest does not match for object file "%s"'%fpath)
return BSON.decode(bsonBytes)
import pybonjour
import zmq
import time
from bson import BSON
context = zmq.Context()
socket = context.socket(zmq.REP)
port = socket.bind_to_random_port("tcp://*")
print "Bound to port %d" % port
sdRef = pybonjour.DNSServiceRegister(name = 'Responder',
regtype = '_00._tcp',
port = port)
try:
try:
while True:
message = BSON.decode(BSON(socket.recv()))
print "Received request %s" % message
time.sleep (1) # Do some 'work'
data = {
'msg': message['msg'][::-1]
}
socket.send(BSON.encode(data))
except KeyboardInterrupt:
pass
finally:
sdRef.close()
def test_upload(upload_all_fixture):
upload_sample(_SAMPLE_PATH, _HOST, _PORT)
data = BSON.decode(upload_all_fixture.last_request.body)
with open(_SNAPSHOT_PATH, 'r') as f:
snapshot = json.load(f)
assert snapshot == data
def read_file(path):
serial = file_utils.load(path)
serial = BSON(serial)
serial = BSON.decode(serial)
srn = SystemResourceNames.deserialize(serial)
return srn
def decode_bson(data):
return BSON.decode(BSON(data))
#!/usr/bin/env python3
import sys
from os.path import splitext
import json
from bson import BSON
for fname_json in sys.argv[1:]:
fname, ext = splitext(fname_json)
bson_s = open(fname_json, 'rb').read()
data = BSON.decode(bson_s)
open(fname + '.json', 'w').write(json.dumps(data))
def test_unicode_decode_error_handler(self):
enc = BSON.encode({"keystr": "foobar"})
# Test handling of bad key value.
invalid_key = BSON(enc[:7] + b'\xe9' + enc[8:])
replaced_key = b'ke\xef\xbf\xbdstr'.decode('utf-8')
dec = BSON.decode(invalid_key, CodecOptions(
unicode_decode_error_handler="replace"))
self.assertEqual(dec, {replaced_key: u("foobar")})
dec = BSON.decode(invalid_key, CodecOptions(
unicode_decode_error_handler="ignore"))
self.assertEqual(dec, {"kestr": "foobar"})
self.assertRaises(InvalidBSON, BSON.decode, invalid_key, CodecOptions(
unicode_decode_error_handler="strict"))
self.assertRaises(InvalidBSON, BSON.decode, invalid_key,
CodecOptions())
self.assertRaises(InvalidBSON, BSON.decode, invalid_key)
# Test handing of bad string value.
invalid_val = BSON(enc[:18] + b'\xe9' + enc[19:])
replaced_val = b'fo\xef\xbf\xbdbar'.decode('utf-8')
dec = BSON.decode(invalid_val, CodecOptions(
unicode_decode_error_handler="replace"))
self.assertEqual(dec, {"keystr": replaced_val})
dec = BSON.decode(invalid_val, CodecOptions(
unicode_decode_error_handler="ignore"))
self.assertEqual(dec, {"keystr": "fobar"})
self.assertRaises(InvalidBSON, BSON.decode, invalid_val, CodecOptions(
unicode_decode_error_handler="strict"))
self.assertRaises(InvalidBSON, BSON.decode, invalid_val,
CodecOptions())
self.assertRaises(InvalidBSON, BSON.decode, invalid_val)
# Test handing bad key + bad value.
invalid_both = BSON(
enc[:7] + b'\xe9' + enc[8:18] + b'\xe9' + enc[19:])
dec = BSON.decode(invalid_both, CodecOptions(
unicode_decode_error_handler="replace"))
self.assertEqual(dec, {replaced_key: replaced_val})
dec = BSON.decode(invalid_both, CodecOptions(
unicode_decode_error_handler="ignore"))
self.assertEqual(dec, {"kestr": "fobar"})
self.assertRaises(InvalidBSON, BSON.decode, invalid_both, CodecOptions(
unicode_decode_error_handler="strict"))
self.assertRaises(InvalidBSON, BSON.decode, invalid_both,
CodecOptions())
self.assertRaises(InvalidBSON, BSON.decode, invalid_both)
# Test handling bad error mode.
dec = BSON.decode(enc, CodecOptions(
unicode_decode_error_handler="junk"))
self.assertEqual(dec, {"keystr": "foobar"})
self.assertRaises(InvalidBSON, BSON.decode, invalid_both,
CodecOptions(unicode_decode_error_handler="junk"))
def work(self, body, msg, *args, **kargs):
## Sanity Checks
rk = msg.delivery_info['routing_key']
if not rk:
raise Exception("Invalid routing-key '%s' (%s)" % (rk, body))
#self.logger.info( body )
## Try to decode event
if isinstance(body, dict):
event = body
# force utf8 only if python version is 2
if PYVER < '3':
event = forceUTF8(event)
else:
self.logger.debug(" + Decode JSON")
try:
if isinstance(body, basestring):
try:
event = loads(body)
self.logger.debug(" + Ok")
except Exception as err:
try:
self.logger.debug(" + Try hack for windows string")
# Hack for windows FS -_-
event = loads(body.replace('\\', '\\\\'))
self.logger.debug(" + Ok")
except Exception as err:
try:
self.logger.debug(" + Decode BSON")
bson = BSON(body)
event = bson.decode()
self.logger.debug(" + Ok")
except Exception as err:
raise Exception(err)
except Exception as err:
self.logger.error(" + Failed (%s)" % err)
self.logger.debug("RK: '%s', Body:" % rk)
self.logger.debug(body)
raise Exception("Impossible to parse event '%s'" % rk)
event['rk'] = ensure_unicode(rk)
if "resource" in event:
if not isinstance(event['resource'], basestring):
event['resource'] = ''
else:
event['resource'] = ensure_unicode(event['resource'])
if not event['resource']:
del event['resource']
# Clean tags field
event['tags'] = event.get('tags', [])
tags = event['tags']
if isinstance(tags, basestring) and tags != "":
event['tags'] = [event['tags']]
elif not isinstance(tags, list):
event['tags'] = []
event["timestamp"] = int(event.get("timestamp", time()))
event["state"] = event.get("state", 0)
event["state_type"] = event.get("state_type", 1)
event["event_type"] = event.get("event_type", "check")
default_status = 0 if not event["state"] else 1
event["status"] = event.get("status", default_status)
event['output'] = event.get('output', '')
# Get perfdata
perf_data = event.get('perf_data')
perf_data_array = event.get('perf_data_array')
if perf_data_array is None:
perf_data_array = []
# Parse perfdata
if perf_data:
self.logger.debug(u' + perf_data: {0}'.format(perf_data))
try:
perf_data_array += PerfDataParser(perf_data).perf_data_array
except Exception as err:
self.logger.error(
"Impossible to parse perfdata from: {0} ({1})".format(
event, err
)
)
event['perf_data_array'] = perf_data_array
return event
callBack = browse_callback)
try:
try:
while True:
ready = select.select([browse_sdRef], [], [])
if browse_sdRef in ready[0]:
pybonjour.DNSServiceProcessResult(browse_sdRef)
if "Responder" in services and not con:
print "Connecting to server %s:%d" % (services['Responder']['host'], services['Responder']['port'])
socket = context.socket(zmq.REQ)
socket.connect ("tcp://%s:%d" % (services['Responder']['host'], services['Responder']['port']))
con = True
for request in range (1,10):
print "Sending request ", request,"..."
data = {
'msg': 'Hello',
}
socket.send (BSON.encode(data))
# Get the reply.
message = BSON(socket.recv())
print "Received reply ", request, BSON.decode(message)
break;
except KeyboardInterrupt:
pass
finally:
browse_sdRef.close()
def read_file(path):
serial = file_utils.load(path)
serial = BSON(serial)
serial = BSON.decode(serial)
srn = SystemResourceNames.deserialize(serial)
return srn
def work(self, body, msg, *args, **kargs):
## Sanity Checks
rk = msg.delivery_info['routing_key']
if not rk:
raise Exception("Invalid routing-key '%s' (%s)" % (rk, body))
#self.logger.info( body )
## Try to decode event
if isinstance(body, dict):
event = body
# force utf8 only if python version is 2
if PYVER < '3':
event = forceUTF8(event)
else:
self.logger.debug(" + Decode JSON")
try:
if isinstance(body, basestring):
try:
event = loads(body)
self.logger.debug(" + Ok")
except Exception as err:
try:
self.logger.debug(" + Try hack for windows string")
# Hack for windows FS -_-
event = loads(body.replace('\\', '\\\\'))
self.logger.debug(" + Ok")
except Exception as err:
try:
self.logger.debug(" + Decode BSON")
bson = BSON(body)
event = bson.decode()
self.logger.debug(" + Ok")
except Exception as err:
raise Exception(err)
except Exception as err:
self.logger.error(" + Failed (%s)" % err)
self.logger.debug("RK: '%s', Body:" % rk)
self.logger.debug(body)
raise Exception("Impossible to parse event '%s'" % rk)
event['rk'] = ensure_unicode(rk)
if event.get("resource", ''):
if not isinstance(event['resource'], basestring):
event['resource'] = ''
else:
event['resource'] = ensure_unicode(event['resource'])
if not event['resource']:
del event['resource']
# Clean tags field
event['tags'] = event.get('tags', [])
tags = event['tags']
if isinstance(tags, basestring) and tags != "":
event['tags'] = [event['tags']]
elif not isinstance(tags, list):
event['tags'] = []
event['timestamp'] = int(event.get('timestamp', time()))
try:
event['state'] = int(event.get('state', 0))
except (ValueError, TypeError, AttributeError):
self.logger.error(
'state convertion failed: must be an int ({})'.format(event))
event['state'] = 0
event['state_type'] = event.get('state_type', 1)
event['event_type'] = event.get('event_type', 'check')
default_status = 0 if not event["state"] else 1
event['status'] = event.get("status", default_status)
event['output'] = event.get('output', '')
# Get perfdata
perf_data = event.get('perf_data')
perf_data_array = event.get('perf_data_array')
if perf_data_array is None:
perf_data_array = []
# Parse perfdata
if perf_data:
self.logger.debug(u' + perf_data: {0}'.format(perf_data))
try:
perf_data_array += PerfDataParser(perf_data).perf_data_array
except Exception as err:
self.logger.error(
"Impossible to parse perfdata from: {0} ({1})".format(
event, err))
event['perf_data_array'] = perf_data_array
return event
def from_binary(binary):
return DataSet.fromdict(BSON.decode(binary))
def test_bson() -> None:
doc = {"foo": "bar", "buzz": True}
data = BSON.encode(doc)
assert BSON.decode(data) == doc
def decode_bson(data):
return BSON.decode(BSON(data))
