def test_utf8(self):
w = {u("aéあ"): u("aéあ")}
self.assertEqual(w, BSON.encode(w).decode())
# b'a\xe9' == u"aé".encode("iso-8859-1")
iso8859_bytes = b'a\xe9'
y = {"hello": iso8859_bytes}
if PY3:
# Stored as BSON binary subtype 0.
out = BSON.encode(y).decode()
self.assertTrue(isinstance(out['hello'], bytes))
self.assertEqual(out['hello'], iso8859_bytes)
else:
# Python 2.
try:
BSON.encode(y)
except InvalidStringData as e:
self.assertTrue(repr(iso8859_bytes) in str(e))
# The next two tests only make sense in python 2.x since
# you can't use `bytes` type as document keys in python 3.x.
x = {u("aéあ").encode("utf-8"): u("aéあ").encode("utf-8")}
self.assertEqual(w, BSON.encode(x).decode())
z = {iso8859_bytes: "hello"}
self.assertRaises(InvalidStringData, BSON.encode, z)
def test_bson_regex(self):
# Invalid Python regex, though valid PCRE.
bson_re1 = Regex(r'[\w-\.]')
self.assertEqual(r'[\w-\.]', bson_re1.pattern)
self.assertEqual(0, bson_re1.flags)
doc1 = {'r': bson_re1}
doc1_bson = (
b'\x11\x00\x00\x00' # document length
b'\x0br\x00[\\w-\\.]\x00\x00' # r: regex
b'\x00') # document terminator
self.assertEqual(doc1_bson, BSON.encode(doc1))
self.assertEqual(doc1, BSON(doc1_bson).decode())
# Valid Python regex, with flags.
re2 = re.compile('.*', re.I | re.L | re.M | re.S | re.U | re.X)
bson_re2 = Regex('.*', re.I | re.L | re.M | re.S | re.U | re.X)
doc2_with_re = {'r': re2}
doc2_with_bson_re = {'r': bson_re2}
doc2_bson = (
b"\x12\x00\x00\x00" # document length
b"\x0br\x00.*\x00ilmsux\x00" # r: regex
b"\x00") # document terminator
self.assertEqual(doc2_bson, BSON.encode(doc2_with_re))
self.assertEqual(doc2_bson, BSON.encode(doc2_with_bson_re))
self.assertEqual(re2.pattern, BSON(doc2_bson).decode()['r'].pattern)
self.assertEqual(re2.flags, BSON(doc2_bson).decode()['r'].flags)
def test_bad_dbref(self):
ref_only = {'ref': {'$ref': 'collection'}}
id_only = {'ref': {'$id': ObjectId()}}
self.assertEqual(DBRef('collection', id=None),
BSON.encode(ref_only).decode()['ref'])
self.assertEqual(id_only, BSON.encode(id_only).decode())
def update(self, spec, document, upsert=False, multi=False, safe=True, **kwargs):
if not isinstance(spec, types.DictType):
raise TypeError("spec must be an instance of dict")
if not isinstance(document, types.DictType):
raise TypeError("document must be an instance of dict")
if not isinstance(upsert, types.BooleanType):
raise TypeError("upsert must be an instance of bool")
flags = kwargs.get("flags", 0)
if multi:
flags |= UPDATE_MULTI
if upsert:
flags |= UPDATE_UPSERT
spec = BSON.encode(spec)
document = BSON.encode(document)
update = Update(flags=flags, collection=str(self),
selector=spec, update=document)
proto = yield self._database.connection.getprotocol()
proto.send_UPDATE(update)
if safe:
ret = yield proto.get_last_error(str(self._database))
defer.returnValue(ret)
def db_collection_insert(res_id, collection_name):
# TODO: Ensure request.json is not None.
if 'document' in request.json:
document = request.json['document']
else:
error = '\'document\' argument not found in the insert request.'
raise MWSServerError(400, error)
# Check quota
size = get_collection_size(res_id, collection_name)
# Handle inserting both a list of docs or a single doc
if isinstance(document, list):
req_size = 0
for d in document:
req_size += len(BSON.encode(d))
else:
req_size = len(BSON.encode(document))
if size + req_size > current_app.config['QUOTA_COLLECTION_SIZE']:
raise MWSServerError(403, 'Collection size exceeded')
# Insert document
with UseResId(res_id):
get_db()[collection_name].insert(document)
return empty_success()
def tile_socket(ws):
tilestore = None
while True:
message = ws.receive()
# Wraps entire websocket response, any errors will be reported back
with app.app_context():
try:
req = BSON(message).decode()
if "init" in req:
"""
Initialization request
"""
tilestore = models.ImageStore.objects.get(id=req["init"]["db"])
if tilestore == None:
raise Exception("Tile Store %s not found"%(req["init"]["db"]))
resp = BSON.encode({"request" : req, "success" : True})
elif "tile" in req:
"""
Regular request
"""
if tilestore == None:
raise Exception("Tile Store not initialized")
imgdata = tilestore.get_tile(req["tile"]["image"], req["tile"]["name"])
resp = BSON.encode({"request" : req, "image" : Binary(imgdata), "success" : True})
# resp = BSON.encode({"request" : req, "image" : b64encode(imgdata), "success" : True})
else:
raise Exception("Unknown request")
ws.send(resp, True)
except Exception as e:
resp = BSON.encode({"request" : req, "error" : e.message})
ws.send(resp, True)
def split_bson(path):
bsonfile_path = os.path.abspath(path)
splitsfile_out = os.path.join(os.path.dirname(bsonfile_path), "." + os.path.basename(bsonfile_path) + ".splits")
bsonfile = open(bsonfile_path, 'r')
splitsfile = open(splitsfile_out, 'w')
file_position = 0
cur_split_start = 0
cur_split_size = 0
while True:
size_bits = bsonfile.read(4)
if len(size_bits) < 4:
if cur_split_size > 0:
#print {"start":cur_split_start, "length": bsonfile.tell() - cur_split_start}
splitsfile.write(BSON.encode({"s":long(cur_split_start), "l": long(bsonfile.tell() - cur_split_start)}))
break
size = struct.unpack("<i", size_bits)[0] - 4 # BSON size byte includes itself
file_position += 4
if cur_split_size + 4 + size > SPLIT_SIZE:
#print {"start":cur_split_start, "length": bsonfile.tell() - 4 - cur_split_start}
splitsfile.write(BSON.encode({"s":long(cur_split_start), "l": long(bsonfile.tell() - 4 - cur_split_start)}))
cur_split_start = bsonfile.tell() - 4
cur_split_size = 0
bsonfile.seek(file_position + size)
file_position += size
cur_split_size += 4 + size
def test_bson_regex(self):
# Invalid Python regex, though valid PCRE.
bson_re1 = Regex(r'[\w-\.]')
self.assertEqual(r'[\w-\.]', bson_re1.pattern)
self.assertEqual(0, bson_re1.flags)
doc1 = {'r': bson_re1}
doc1_bson = b(
'\x11\x00\x00\x00' # document length
'\x0br\x00[\\w-\\.]\x00\x00' # r: regex
'\x00') # document terminator
self.assertEqual(doc1_bson, BSON.encode(doc1))
self.assertEqual(doc1, BSON(doc1_bson).decode(compile_re=False))
# Valid Python regex, with flags.
re2 = re.compile('.*', re.I | re.L | re.M | re.S | re.U | re.X)
bson_re2 = Regex('.*', re.I | re.L | re.M | re.S | re.U | re.X)
doc2_with_re = {'r': re2}
doc2_with_bson_re = {'r': bson_re2}
doc2_bson = b(
"\x12\x00\x00\x00" # document length
"\x0br\x00.*\x00ilmsux\x00" # r: regex
"\x00") # document terminator
self.assertEqual(doc2_bson, BSON.encode(doc2_with_re))
self.assertEqual(doc2_bson, BSON.encode(doc2_with_bson_re))
# Built-in re objects don't support ==. Compare pattern and flags.
self.assertEqual(re2.pattern, BSON(doc2_bson).decode()['r'].pattern)
self.assertEqual(re2.flags, BSON(doc2_bson).decode()['r'].flags)
self.assertEqual(
doc2_with_bson_re, BSON(doc2_bson).decode(compile_re=False))
def update(self, spec, document, upsert=False, multi=False, safe=None, flags=0, **kwargs):
if not isinstance(spec, dict):
raise TypeError("TxMongo: spec must be an instance of dict.")
if not isinstance(document, dict):
raise TypeError("TxMongo: document must be an instance of dict.")
if not isinstance(upsert, bool):
raise TypeError("TxMongo: upsert must be an instance of bool.")
if multi:
flags |= UPDATE_MULTI
if upsert:
flags |= UPDATE_UPSERT
spec = BSON.encode(spec)
document = BSON.encode(document)
update = Update(flags=flags, collection=str(self),
selector=spec, update=document)
proto = yield self._database.connection.getprotocol()
check_deadline(kwargs.pop("_deadline", None))
proto.send_UPDATE(update)
write_concern = self._get_write_concern(safe, **kwargs)
if write_concern.acknowledged:
ret = yield proto.get_last_error(str(self._database), **write_concern.document)
defer.returnValue(ret)
def __setitem__(self, name, value):
if hasattr(value, 'mongo_data_manager'):
if value.has_key('_id'):
self.uncommitted[name] = DBRef(value.collection.name,
value['_id'])
else:
txn = transaction.get()
if value in txn._resources:
# this document is part of the current transaction and
# doesn't have a mongo _id yet queue it and trigger adding
# the reference at the end of the transaction
self.queued[name] = value
else:
# this document is not part of the current transaction,
# so treat it as a regular python dict and make it an
# embedded document inside the current doc
logger.warn('mongo document does not exist in mongodb and'
' is not part of current transaction - saving'
' as embedded instead of a reference')
self.uncommitted[name] = value.copy()
else:
try:
BSON.encode({name:value})
self.uncommitted[name] = value
except:
self.uncommitted[name] = jsonpickle.encode(value)
def test_backports(self):
doc = BSON.encode({"tuple": (1, 2)})
exp = {"tuple": [1, 2]}
options = CodecOptions(uuid_representation=ALL_UUID_REPRESENTATIONS[0],
tz_aware=False, document_class=dict)
self.assertEqual(
{"tuple": [1, 2]},
BSON.encode(
{"tuple": (1, 2)}, codec_options=options,
uuid_subtype=ALL_UUID_REPRESENTATIONS[1]).decode())
self.assertEqual(exp, doc.decode(
as_class=SON,
tz_aware=True,
uuid_subtype=ALL_UUID_REPRESENTATIONS[1],
codec_options=options))
self.assertEqual([exp], list(decode_iter(
doc,
as_class=SON,
tz_aware=True,
uuid_subtype=ALL_UUID_REPRESENTATIONS[1],
codec_options=options)))
self.assertEqual([exp], list(decode_file_iter(
StringIO(doc),
as_class=SON,
tz_aware=True,
uuid_subtype=ALL_UUID_REPRESENTATIONS[1],
codec_options=options)))
self.assertEqual([exp], decode_all(
doc, SON, True, ALL_UUID_REPRESENTATIONS[1], True, options))
def store_task_result(self,posted_task,result):
"""
Store result of a 'successful' task run.
The task is removed from the original collection (tasks), and put in the
results collection.
"""
posted_task.remove()
# TODO: Need special serialization?
try:
# Can the result be safely BSON-encoded?
if type(result) != dict:
safe_result = BSON.encode({'_':result})
else:
safe_result = BSON.encode(result)
except:
safe_result = str(result)
self.logger.warning("%s: result of %s cannot be BSON-encoded: %s: %s" %
(self.worker_name,posted_task.name,safe_result,
sys.exc_info()[1]))
result = safe_result
posted_task.doc['result'] = result
posted_task.doc['status'] = 'completed'
posted_task.doc['completed_at'] = datetime.datetime.utcnow()
posted_task.save_into(self.results_collection)
posted_task.notify_results(self.activity_log)
def test_infinite_loop_exceeds_max_recursion_depth(self):
codecopts = CodecOptions(type_registry=TypeRegistry(
[self.B2A()], fallback_encoder=self.fallback_encoder_A2B))
# Raises max recursion depth exceeded error
with self.assertRaises(RuntimeError):
BSON.encode({'x': self.TypeA(100)}, codec_options=codecopts)
def test_overflow(self):
self.assertTrue(BSON.encode({"x": 9223372036854775807L}))
self.assertRaises(OverflowError, BSON.encode,
{"x": 9223372036854775808L})
self.assertTrue(BSON.encode({"x": -9223372036854775808L}))
self.assertRaises(OverflowError, BSON.encode,
{"x": -9223372036854775809L})
def test_overflow(self):
self.assertTrue(BSON.encode({"x": long(9223372036854775807)}))
self.assertRaises(OverflowError, BSON.encode,
{"x": long(9223372036854775808)})
self.assertTrue(BSON.encode({"x": long(-9223372036854775808)}))
self.assertRaises(OverflowError, BSON.encode,
{"x": long(-9223372036854775809)})
def test_EncodeDecodeUpdate(self):
request = Update(
flags=UPDATE_MULTI | UPDATE_UPSERT,
collection="coll",
selector=BSON.encode({"x": 42}),
update=BSON.encode({"$set": {"y": 123}}),
)
self.__test_encode_decode(request)
def test_encode_custom_then_fallback(self):
codecopts = CodecOptions(type_registry=TypeRegistry(
[self.B2A()], fallback_encoder=self.fallback_encoder_A2BSON))
testdoc = {'x': self.TypeB(123)}
expected_bytes = BSON.encode({'x': 123})
self.assertEqual(BSON.encode(testdoc, codec_options=codecopts),
expected_bytes)
def _generate_batch_commands(collname, command, docs_field, documents, ordered,
write_concern, max_bson, max_count):
# Takes a list of documents and generates one or many `insert` commands
# with documents list in each command is less or equal to max_bson bytes
# and contains less or equal documents than max_count
# Manually composing command in BSON form because this way we can
# perform costly documents serialization only once
msg = SON([(command, collname),
("ordered", ordered),
("writeConcern", write_concern.document)])
buf = io.BytesIO()
buf.write(BSON.encode(msg))
buf.seek(-1, io.SEEK_END) # -1 because we don't need final NUL from partial command
buf.write(docs_field) # type, name and length placeholder of 'documents' array
docs_start = buf.tell() - 4
def prepare_command():
docs_end = buf.tell() + 1 # +1 for final NUL for 'documents'
buf.write(b'\x00\x00') # final NULs for 'documents' and the command itself
total_length = buf.tell()
# writing 'documents' length
buf.seek(docs_start)
buf.write(struct.pack('<i', docs_end - docs_start))
# writing total message length
buf.seek(0)
buf.write(struct.pack('<i', total_length))
return BSON(buf.getvalue())
idx = 0
idx_offset = 0
for doc in documents:
key = str(idx).encode('ascii')
value = BSON.encode(doc)
enough_size = buf.tell() + len(key)+2 + len(value) - docs_start > max_bson
enough_count = idx >= max_count
if enough_size or enough_count:
yield idx_offset, prepare_command()
buf.seek(docs_start + 4)
buf.truncate()
idx_offset += idx
idx = 0
key = b'0'
buf.write(b'\x03' + key + b'\x00') # type and key of document
buf.write(value)
idx += 1
yield idx_offset, prepare_command()
def test_custom_class(self):
self.assertTrue(isinstance(BSON.encode({}).decode(), dict))
self.assertFalse(isinstance(BSON.encode({}).decode(), SON))
self.assertTrue(isinstance(BSON.encode({}).decode(SON), SON))
self.assertEqual(1, BSON.encode({"x": 1}).decode(SON)["x"])
x = BSON.encode({"x": [{"y": 1}]})
self.assertTrue(isinstance(x.decode(SON)["x"][0], SON))
def test_EncodeDecodeQuery(self):
request = Query(
collection="coll",
n_to_skip=123,
n_to_return=456,
query=BSON.encode({"x": 42}),
fields=BSON.encode({"y": 1}),
)
self.__test_encode_decode(request)
def debug(req):
if req.method == 'GET':
print (BSON.encode({'hello': 'world'}))
return HttpResponse(BSON.encode({'hello': 'world'}))
else:
try:
return HttpResponse(dumps(BSON(req.body).decode()))
except:
return HttpResponse('{"error": "decode error"}')
def _thread(self,context,worker_id,import_class,pid,serverid,counters,methods,target,stype):
"""
Worker thread for zmqrpc server - binds to zmq socket (target) and works ZMQRPCServer import_class.
Instantiated by work() threading
Handles BSON in/out, zmq REP to zmq QUEUE or REQ
"""
socket = self.context.socket(zmq.REP)
job_count = 0
if stype == LISTEN:
socket.bind(target)
else:
socket.connect(target)
nuclass = import_class()
while True:
sockin = socket.recv()
message = BSON(sockin).decode()
result = None
fail = None
tb = None
method = str(message['method'])
args = message.get('args',[])
# Convert kwargs from unicode strings to 8bit strings
if method == '__threadstatus__':
x = threading.current_thread()
socket.send(BSON.encode({'runner':None,'traceback':None,'fail':False,'result':{'id':serverid+':'+str(pid)+':'+str(x.name),'alive':x.is_alive(),'job_count':counters.get(x.name,0),'last_method':methods.get(x.name,''),}}))
else:
try:
kwargs = {}
for (k,v) in message.get('kwargs',{}).iteritems():
kwargs[str(k)] = v
job_count+=1
counters[threading.currentThread().name] = job_count
methods[threading.currentThread().name] = method
runner = {'job_count':job_count,'thread':threading.currentThread().name,'method':import_class.__name__+'.'+method,}
# Find the method in the module, run it.
try:
if hasattr(nuclass,method):
result = getattr(nuclass,method)(*args,**kwargs)
fail = False
else:
fail = True
tb = "NameError: name '"+method+"' is not defined in ZMQRPC class '"+import_class.__name__+"'"
except:
etype, evalue, etb = sys.exc_info()
fail = True
tb = "\n".join(traceback.format_exception(etype, evalue, etb))
socket.send(BSON.encode({'fail':fail,'result':result,'runner':runner,'traceback':tb}))
except:
etype, evalue, etb = sys.exc_info()
fail = True
tb = "\n".join(traceback.format_exception(etype, evalue, etb))
socket.send(BSON.encode({'fail':fail,'result':None,'runner':None,'traceback':tb}))
def update(col, spec, doc, upsert, multi):
colname = _make_c_string(col)
flags = 0
if upsert:
flags |= 1 << 0
if multi:
flags |= 1 << 1
fmt = '<i%dsi' % len(colname)
part = struct.pack(fmt, 0, colname, flags)
return "%s%s%s" % (part, BSON.encode(spec), BSON.encode(doc))
def query(col, spec, fields, skip, limit):
data = [
_ZERO,
_make_c_string(col),
struct.pack('<ii', skip, limit),
BSON.encode(spec or {}),
]
if fields:
data.append(BSON.encode(dict.fromkeys(fields, 1)))
return "".join(data)
def get_size(self):
"""
return the size of the underlying bson object
"""
try:
size = len(BSON.encode(self))
except:
self._process_custom_type('bson', self, self.structure)
size = len(BSON.encode(self))
self._process_custom_type('python', self, self.structure)
return size
def test_null_character(self):
doc = {"a": b"\x00"}
self.assertEqual(doc, BSON.encode(doc).decode())
doc = {"a": "\x00"}
self.assertEqual(doc, BSON.encode(doc).decode())
self.assertRaises(InvalidDocument, BSON.encode, {b"\x00": "a"})
self.assertRaises(InvalidDocument, BSON.encode, {"\x00": "a"})
self.assertRaises(InvalidDocument, BSON.encode, {"a": re.compile(b"ab\x00c")})
self.assertRaises(InvalidDocument, BSON.encode, {"a": re.compile("ab\x00c")})
def test_move_id(self):
self.assertEqual(b"\x19\x00\x00\x00\x02_id\x00\x02\x00\x00\x00a\x00"
b"\x02a\x00\x02\x00\x00\x00a\x00\x00",
BSON.encode(SON([("a", "a"), ("_id", "a")])))
self.assertEqual(b"\x2c\x00\x00\x00"
b"\x02_id\x00\x02\x00\x00\x00b\x00"
b"\x03b\x00"
b"\x19\x00\x00\x00\x02a\x00\x02\x00\x00\x00a\x00"
b"\x02_id\x00\x02\x00\x00\x00a\x00\x00\x00",
BSON.encode(SON([("b", SON([("a", "a"), ("_id", "a")])),
("_id", "b")])))
def test_small_long_encode_decode(self):
encoded1 = BSON.encode({'x': 256})
decoded1 = BSON.decode(encoded1)['x']
self.assertEqual(256, decoded1)
self.assertEqual(type(256), type(decoded1))
encoded2 = BSON.encode({'x': 256L})
decoded2 = BSON.decode(encoded2)['x']
self.assertEqual(256L, decoded2)
self.assertEqual(type(256L), type(decoded2))
self.assertNotEqual(type(decoded1), type(decoded2))
def __new__(cls, len=0, request_id=0, response_to=0, opcode=OP_QUERY,
flags=0, collection='', n_to_skip=0, n_to_return=-1,
query=None, fields=None):
if query is None:
query = {}
if not isinstance(query, BSON):
query = BSON.encode(query)
if fields is not None and not isinstance(fields, BSON):
fields = BSON.encode(fields)
return super(Query, cls).__new__(cls, len, request_id, response_to,
opcode, flags, collection, n_to_skip,
n_to_return, query, fields)
def test_utf8(self):
w = {u"aéあ": u"aéあ"}
self.assertEqual(w, BSON.encode(w).decode())
x = {u"aéあ".encode("utf-8"): u"aéあ".encode("utf-8")}
self.assertEqual(w, BSON.encode(x).decode())
y = {"hello": u"aé".encode("iso-8859-1")}
self.assertRaises(InvalidStringData, BSON.encode, y)
z = {u"aé".encode("iso-8859-1"): "hello"}
self.assertRaises(InvalidStringData, BSON.encode, z)
def test_server_responds_correctly():
with multiserver_and_client('inproc://zrpc', REGISTRY, 4) as client:
client.send(
BSON.encode({
"id": "abc",
"method": "add",
"params": [3, 4]
}))
assert_equal(
BSON(client.recv()).decode(), {
"id": "abc",
"result": 7,
"error": None
})
def test_raw(self):
"""Test with RawBSONDocument."""
raw_coll = self.coll.with_options(
codec_options=DEFAULT_RAW_BSON_OPTIONS)
with raw_coll.watch() as change_stream:
raw_doc = RawBSONDocument(BSON.encode({'_id': 1}))
self.coll.insert_one(raw_doc)
change = next(change_stream)
self.assertIsInstance(change, RawBSONDocument)
self.assertEqual(change['operationType'], 'insert')
self.assertEqual(change['ns']['db'], self.coll.database.name)
self.assertEqual(change['ns']['coll'], self.coll.name)
self.assertEqual(change['fullDocument'], raw_doc)
self.assertEqual(change['_id'], change_stream._resume_token)
def add(self, doc, autoflush=True):
try:
self._oplog.write(BSON.encode(doc))
self._writes_unflushed += 1
self._count += 1
if not self._first_ts:
self._first_ts = doc['ts']
self._last_ts = doc['ts']
if autoflush:
self.autoflush()
except Exception, e:
logging.fatal("Cannot write to oplog file %s! Error: %s" %
(self.oplog_file, e))
raise OperationError(e)
def get_response(self, func, *args, **kwargs):
"""
Run a Python function, returning the result in BSON-serializable form.
The behaviour of this function is to capture either a successful return
value or exception in a BSON-serializable form (a dictionary with
`result` and `error` keys).
"""
result, error = None, None
try:
result = func(*args, **kwargs)
except Exception, exc:
exc_type = "%s.%s" % (type(exc).__module__, type(exc).__name__)
exc_message = traceback.format_exception_only(type(exc),
exc)[-1].strip()
error = {"type": exc_type, "message": exc_message}
try:
BSON.encode({'args': exc.args})
except TypeError:
pass
else:
error["args"] = exc.args
def test_aware_datetime(self):
aware = datetime.datetime(1993,
4,
4,
2,
tzinfo=FixedOffset(555, "SomeZone"))
as_utc = (aware - aware.utcoffset()).replace(tzinfo=utc)
self.assertEqual(datetime.datetime(1993, 4, 3, 16, 45, tzinfo=utc),
as_utc)
after = BSON.encode({
"date": aware
}).decode(CodecOptions(tz_aware=True))["date"]
self.assertEqual(utc, after.tzinfo)
self.assertEqual(as_utc, after)
def test_errors_raised_in_method_are_returned():
with server_and_client('inproc://zrpc', REGISTRY) as client:
client.send(BSON.encode({
"id": "abc",
"method": "raises_error",
"params": []}))
assert_equal(BSON(client.recv()).decode(),
{"id": "abc",
"result": None,
"error": {
"type": "exceptions.Exception",
"args": ["some error occurred"],
"message": "Exception: some error occurred"
}})
def test_bson_unserializable_objects_returned_raise_an_error():
with server_and_client('inproc://zrpc', REGISTRY) as client:
client.send(BSON.encode({
"id": "abc",
"method": "returns_bson_unserializable_obj",
"params": []}))
assert_equal(BSON(client.recv()).decode(),
{"id": "abc",
"result": None,
"error": {
"type": "bson.errors.InvalidDocument",
"args": ["Cannot encode object: <unserializable>"],
"message": "InvalidDocument: Cannot encode object: <unserializable>"
}})
def test_missing_method_returns_an_error():
with server_and_client('inproc://zrpc', REGISTRY) as client:
client.send(BSON.encode({
"id": "abc",
"method": "doesnotexist",
"params": [3, 4]}))
assert_equal(BSON(client.recv()).decode(),
{"id": "abc",
"result": None,
"error": {
"type": "zrpc.exceptions.MissingMethod",
"args": ["doesnotexist"],
"message": "MissingMethod: doesnotexist"
}})
def chaton(s, chatId):
print " [*] CHATON from " + str(chatId)
data = '\x07\x00\x00\x00' # Packet ID
data += '\x00\x00' # Status Code : when sending command -> 0
data += 'CHATON\x00\x00\x00\x00\x00' # Method
data += '\x00' # Body Type : when sending command -> 0
body = BSON.encode({u'chatId': chatId})
data += body[:4]
data += body
succ = command_send(s, data)
return succ
def leave(s, chatId):
print " [*] LEAVE from " + str(chatId)
data = '\x06\x00\x00\x00' # Packet ID
data += '\x00\x00' # Status Code : when sending command -> 0
data += 'LEAVE\x00\x00\x00\x00\x00\x00' # Method
data += '\x00' # Body Type : when sending command -> 0
body = BSON.encode({u'chatId': chatId})
data += body[:4]
data += body
succ = command_send(s, data)
return succ
def test_CursorClosing(self):
# Calculate number of objects in 4mb batch
obj_count_4mb = 4 * 1024**2 // len(
BSON.encode(self.__make_big_object())) + 1
first_batch = 5
yield self.coll.insert([
self.__make_big_object()
for _ in range(first_batch + obj_count_4mb)
])
result = yield self.coll.find(limit=first_batch)
self.assertEqual(len(result), 5)
yield self.__check_no_open_cursors()
def cwrite(s, memId = [], msg = u'test'):
print " [*] CWRITE to " + str(memId) + " : " + msg
data = '\x06\x00\x00\x00' # Packet ID
data += '\x00\x00' # Status Code : when sending command -> 0
data += 'CWRITE\x00\x00\x00\x00\x00' # Method
data += '\x00' # Body Type : when sending command -> 0
body = BSON.encode({u'memberIds': memId, u'msg': msg, u'extra': None, u'pushAlert': True})
data += body[:4]
data += body
succ = send(s, data)
return succ
def test_utils_montyimport(monty_client, tmp_monty_utils_repo):
database = "dump_db_JSON"
collection = "dump_col_JSON"
with open_repo(tmp_monty_utils_repo):
with open(JSON_DUMP, "w") as dump:
dump.write(SERIALIZED)
montyimport(database, collection, JSON_DUMP)
col = monty_client[database][collection]
for i, doc in enumerate(col.find(sort=[("_id", 1)])):
assert doc == BSON.encode(DOCUMENTS[i]).decode()
os.remove(JSON_DUMP)
def test_utils_montyrestore(monty_client, tmp_monty_utils_repo):
database = "dump_db_BSON"
collection = "dump_col_BSON"
with open_repo(tmp_monty_utils_repo):
with open(BSON_DUMP, "wb") as dump:
dump.write(base64.b64decode(BINARY))
montyrestore(database, collection, BSON_DUMP)
col = monty_client[database][collection]
for i, doc in enumerate(col.find(sort=[("_id", 1)])):
assert doc == BSON.encode(DOCUMENTS[i]).decode()
os.remove(BSON_DUMP)
def write(self, do_merge=False):
f = None
try:
self.lock.acquire()
if do_merge and os.path.isfile(self.state_file):
curr = self.load(True)
data = self.merge(self.state, curr)
f = open(self.state_file, 'w+')
logging.debug("Writing %s state file: %s" % (self.__class__.__name__, self.state_file))
self.state['updated_at'] = int(time())
f.write(BSON.encode(self.state))
finally:
if f:
f.close()
self.lock.release()
def test_small_long_encode_decode(self):
if PY3:
raise SkipTest("No long type in Python 3.")
encoded1 = BSON.encode({'x': 256})
decoded1 = BSON.decode(encoded1)['x']
self.assertEqual(256, decoded1)
self.assertEqual(type(256), type(decoded1))
encoded2 = BSON.encode({'x': 256L})
decoded2 = BSON.decode(encoded2)['x']
self.assertEqual(256L, decoded2)
self.assertEqual(type(256L), type(decoded2))
self.assertNotEqual(type(decoded1), type(decoded2))
def reload_geo():
# check to see if there is an exisitng file. If so than do not reload
col = db.fs.files.find_one()
if col == None:
# if the geojson file is not stored, call the API.
response = requests.get("https://opendata.arcgis.com/datasets/d192da4d0ac249fa9584109b1d626286_0.geojson")
# GridFS stored BSON binary files, the fucntion to do that is BSON.encode
geojson = BSON.encode(response.json())
# then we store it with the put()
fs.put(geojson)
return get_geo()
def test_round_trip_codec_options(self):
doc = {
'date': datetime.datetime(2015, 6, 3, 18, 40, 50, 826000),
'_id': uuid.UUID('026fab8f-975f-4965-9fbf-85ad874c60ff')
}
db = self.client.pymongo_test
coll = db.get_collection(
'test_raw',
codec_options=CodecOptions(uuid_representation=JAVA_LEGACY))
coll.insert_one(doc)
raw_java_legacy = CodecOptions(uuid_representation=JAVA_LEGACY,
document_class=RawBSONDocument)
coll = db.get_collection('test_raw', codec_options=raw_java_legacy)
self.assertEqual(
RawBSONDocument(BSON.encode(doc, codec_options=raw_java_legacy)),
coll.find_one())
def test_bson(self):
" Test BSON support. "
from bson import BSON
response = self.get_resource('bson')
test = BSON(response.content).decode()
self.assertEqual(test['counter'], 1)
bson = BSON.encode(dict(counter=4))
uri = self.reverse('bson')
response = self.client.post(uri,
data=bson,
content_type='application/bson')
test = BSON(response.content).decode()
self.assertEqual(test['counter'], 5)
def _PostRequest(self, oparms):
oparms['_asyncrequest'] = True
if self.Binary:
data = BSON.encode(oparms)
else:
data = json.dumps(oparms, sort_keys=True)
try:
self.MessagesSent += 1
self.Socket.sendto(data, (self.Address, self.Port))
except:
print 'send failed; ', sys.exc_info()[0]
return json.loads('{"_Success" : 0, "_Message" : "unknown error"}')
return json.loads('{"_Success" : 2}')
def test_CursorClosingWithCursor(self):
# Calculate number of objects in 4mb batch
obj_count_4mb = 4 * 1024**2 // len(BSON.encode(self.__make_big_object())) + 1
first_batch = 5
yield self.coll.insert(
[self.__make_big_object() for _ in range(first_batch + obj_count_4mb)])
result = []
docs, dfr = yield self.coll.find_with_cursor({}, limit=first_batch)
while docs:
result.extend(docs)
docs, dfr = yield dfr
self.assertEqual(len(result), 5)
yield self.__check_no_open_cursors()
def check_encode_then_decode(self, doc_class=dict):
# Work around http://bugs.jython.org/issue1728
if sys.platform.startswith('java'):
doc_class = SON
def helper(doc):
self.assertEqual(doc, (BSON.encode(doc_class(doc))).decode())
helper({})
helper({"test": u("hello")})
self.assertTrue(isinstance(BSON.encode({"hello": "world"})
.decode()["hello"],
text_type))
helper({"mike": -10120})
helper({"long": Int64(10)})
helper({"really big long": 2147483648})
helper({u("hello"): 0.0013109})
helper({"something": True})
helper({"false": False})
helper({"an array": [1, True, 3.8, u("world")]})
helper({"an object": doc_class({"test": u("something")})})
helper({"a binary": Binary(b"test", 100)})
helper({"a binary": Binary(b"test", 128)})
helper({"a binary": Binary(b"test", 254)})
helper({"another binary": Binary(b"test", 2)})
helper(SON([(u('test dst'), datetime.datetime(1993, 4, 4, 2))]))
helper(SON([(u('test negative dst'),
datetime.datetime(1, 1, 1, 1, 1, 1))]))
helper({"big float": float(10000000000)})
helper({"ref": DBRef("coll", 5)})
helper({"ref": DBRef("coll", 5, foo="bar", bar=4)})
helper({"ref": DBRef("coll", 5, "foo")})
helper({"ref": DBRef("coll", 5, "foo", foo="bar")})
helper({"ref": Timestamp(1, 2)})
helper({"foo": MinKey()})
helper({"foo": MaxKey()})
helper({"$field": Code("function(){ return true; }")})
helper({"$field": Code("return function(){ return x; }", scope={'x': False})})
def encode_then_decode(doc):
return doc_class(doc) == BSON.encode(doc).decode(
CodecOptions(document_class=doc_class))
qcheck.check_unittest(self, encode_then_decode,
qcheck.gen_mongo_dict(3))
def test_encode_then_decode(self):
def helper(dict):
self.assertEqual(dict, (BSON.encode(dict)).decode())
helper({})
helper({"test": u"hello"})
self.assertTrue(isinstance(BSON.encode({"hello": "world"})
.decode()["hello"],
unicode))
helper({"mike": -10120})
helper({"long": long(10)})
helper({"really big long": 2147483648})
helper({u"hello": 0.0013109})
helper({"something": True})
helper({"false": False})
helper({"an array": [1, True, 3.8, u"world"]})
helper({"an object": {"test": u"something"}})
helper({"a binary": Binary(b("test"), 100)})
helper({"a binary": Binary(b("test"), 128)})
helper({"a binary": Binary(b("test"), 254)})
helper({"another binary": Binary(b("test"), 2)})
helper(SON([(u'test dst', datetime.datetime(1993, 4, 4, 2))]))
helper(SON([(u'test negative dst', datetime.datetime(1, 1, 1, 1, 1, 1))]))
helper({"big float": float(10000000000)})
helper({"ref": DBRef("coll", 5)})
helper({"ref": DBRef("coll", 5, foo="bar", bar=4)})
helper({"ref": DBRef("coll", 5, "foo")})
helper({"ref": DBRef("coll", 5, "foo", foo="bar")})
helper({"ref": Timestamp(1, 2)})
helper({"foo": MinKey()})
helper({"foo": MaxKey()})
helper({"$field": Code("function(){ return true; }")})
helper({"$field": Code("return function(){ return x; }", scope={'x': False})})
doc_class = dict
# Work around http://bugs.jython.org/issue1728
if (sys.platform.startswith('java') and
sys.version_info[:3] >= (2, 5, 2)):
doc_class = SON
def encode_then_decode(doc):
return doc == (BSON.encode(doc)).decode(as_class=doc_class)
qcheck.check_unittest(self, encode_then_decode,
qcheck.gen_mongo_dict(3))
def dump_op(output, op):
copier = utils.DictionaryCopier(op)
copier.copy_fields("ts", "ns", "op")
op_type = op["op"]
# handpick some essential fields to execute.
if op_type == "query":
copier.copy_fields("query", "ntoskip", "ntoreturn")
elif op_type == "insert":
copier.copy_fields("o")
elif op_type == "update":
copier.copy_fields("updateobj", "query")
elif op_type == "remove":
copier.copy_fields("query")
elif op_type == "command":
copier.copy_fields("command")
output.write(BSON.encode(copier.dest))
def test_utils_montyrestore(tmp_monty_repo):
database = "dump_db_BSON"
collection = "dump_col_BSON"
if not os.path.isdir(tmp_monty_repo):
os.makedirs(tmp_monty_repo)
with open_repo(tmp_monty_repo):
with open(BSON_DUMP, "wb") as dump:
dump.write(base64.b64decode(BINARY))
montyrestore(database, collection, BSON_DUMP)
client = MontyClient()
for i, doc in enumerate(client[database][collection].find()):
assert doc == BSON.encode(DOCUMENTS[i]).decode()
os.remove(BSON_DUMP)
def test_utils_montyimport(tmp_monty_repo):
database = "dump_db_JSON"
collection = "dump_col_JSON"
if not os.path.isdir(tmp_monty_repo):
os.makedirs(tmp_monty_repo)
with open_repo(tmp_monty_repo):
with open(JSON_DUMP, "w") as dump:
dump.write(SERIALIZED)
montyimport(database, collection, JSON_DUMP)
client = MontyClient()
for i, doc in enumerate(client[database][collection].find()):
assert doc == BSON.encode(DOCUMENTS[i]).decode()
os.remove(JSON_DUMP)
def dump(self, dump_path):
""" 备份数据库 """
# print('mongodb dump')
if not path.exists(dump_path):
os.mkdir(dump_path)
names = self._db.collection_names()
print(names)
for n in names:
if n.startswith('system.'):
continue
table_name = path.join(dump_path, n)
print('table:' + n)
table = self.get_table(n)
docs = [doc for doc in table.find()]
indexs = table.index_information()
d = BSON.encode(dict(data=docs, indexs=indexs))
with open(table_name, 'wb') as f:
f.write(d)
def split_sample_bson(input_bson_filename, output_bson_dir, n=100, split=10):
data = bson.decode_file_iter(open(input_bson_filename, 'rb'))
no_example_per_split = int(n / split)
print("total number of sample per split: {}".format(no_example_per_split))
for c, d in tqdm(enumerate(data), total=n):
f_n = int(c % split)
output = open(
os.path.join(output_bson_dir,
"{}_{}.bson".format(f_n, no_example_per_split)), 'a+')
output.write(BSON.encode(d))
print("Finish convert tfrecords with {} records".format(c + 1))
def send_api(self, thing):
""" Sends a new API notification to the Cuckoo host """
pid = thing.pid
api = thing.api
# We're required to report results of tracing a target process to
# *its own* result server. So create a communication socket...
if pid not in self.sockets:
self.sockets[pid] = self._create_socket()
if not self.sockets[pid]:
raise Exception("CuckooHost error: could not create socket.")
# ... and don't forget to explain every single API call again
self.descriptions.setdefault(pid, ["__process__", "__thread__"])
self._send_new_process(thing)
try:
lookup_idx = self.descriptions[pid].index(api)
except ValueError:
self.descriptions[pid].append(api)
lookup_idx = len(self.descriptions[pid]) - 1
self._send_api_description(lookup_idx, thing)
# Here's an api object:
# {
# "I" : (int)<index in the API lookup table>,
# "T" : (int)<caller thread id>,
# "t" : (int)<time (in milliseconds) since a process launch>,
# "args" : [
# (int)<1 if this API call was successfull, 0 otherwise>,
# (int)<return value>,
# (any)<value the first argument>,
# (any)<value the second argument>,
# ...
# (any)<value the n-th argument>,
# ]
# }
time_offset_ms = int(1000 * thing.timestamp -
1000 * self.launch_times[pid])
self.sockets[pid].sendall(
BSON.encode({
"I": lookup_idx,
"T": thing.tid,
"t": time_offset_ms,
"args": self._prepare_args(thing)
}))
def test_local_datetime(self):
# Timezone -60 minutes of UTC, with DST between April and July.
tz = DSTAwareTimezone(60, "sixty-minutes", 4, 7)
# It's not DST.
local = datetime.datetime(year=2025,
month=12,
hour=2,
day=1,
tzinfo=tz)
options = CodecOptions(tz_aware=True, tzinfo=tz)
# Encode with this timezone, then decode to UTC.
encoded = BSON.encode({'date': local}, codec_options=options)
self.assertEqual(local.replace(hour=1, tzinfo=None),
encoded.decode()['date'])
# It's DST.
local = datetime.datetime(year=2025, month=4, hour=1, day=1, tzinfo=tz)
encoded = BSON.encode({'date': local}, codec_options=options)
self.assertEqual(local.replace(month=3, day=31, hour=23, tzinfo=None),
encoded.decode()['date'])
# Encode UTC, then decode in a different timezone.
encoded = BSON.encode({'date': local.replace(tzinfo=utc)})
decoded = encoded.decode(options)['date']
self.assertEqual(local.replace(hour=3), decoded)
self.assertEqual(tz, decoded.tzinfo)
# Test round-tripping.
self.assertEqual(local,
(BSON.encode({
'date': local
}, codec_options=options).decode(options)['date']))
# Test around the Unix Epoch.
epochs = (EPOCH_AWARE,
EPOCH_AWARE.astimezone(FixedOffset(120, 'one twenty')),
EPOCH_AWARE.astimezone(FixedOffset(-120,
'minus one twenty')))
utc_co = CodecOptions(tz_aware=True)
for epoch in epochs:
doc = {'epoch': epoch}
# We always retrieve datetimes in UTC unless told to do otherwise.
self.assertEqual(
EPOCH_AWARE,
BSON.encode(doc).decode(codec_options=utc_co)['epoch'])
# Round-trip the epoch.
local_co = CodecOptions(tz_aware=True, tzinfo=epoch.tzinfo)
self.assertEqual(
epoch,
BSON.encode(doc).decode(codec_options=local_co)['epoch'])