def test_subtype(self):
one = Binary(b("hello"))
self.assertEqual(one.subtype, 0)
two = Binary(b("hello"), 2)
self.assertEqual(two.subtype, 2)
three = Binary(b("hello"), 100)
self.assertEqual(three.subtype, 100)
def test_bson_regex(self):
# Invalid Python regex, though valid PCRE.
bson_re1 = Regex(r'[\w-\.]')
self.assertEqual(r'[\w-\.]', bson_re1.pattern)
self.assertEqual(0, bson_re1.flags)
doc1 = {'r': bson_re1}
doc1_bson = b(
'\x11\x00\x00\x00' # document length
'\x0br\x00[\\w-\\.]\x00\x00' # r: regex
'\x00') # document terminator
self.assertEqual(doc1_bson, BSON.encode(doc1))
self.assertEqual(doc1, BSON(doc1_bson).decode(compile_re=False))
# Valid Python regex, with flags.
re2 = re.compile('.*', re.I | re.L | re.M | re.S | re.U | re.X)
bson_re2 = Regex('.*', re.I | re.L | re.M | re.S | re.U | re.X)
doc2_with_re = {'r': re2}
doc2_with_bson_re = {'r': bson_re2}
doc2_bson = b(
"\x12\x00\x00\x00" # document length
"\x0br\x00.*\x00ilmsux\x00" # r: regex
"\x00") # document terminator
self.assertEqual(doc2_bson, BSON.encode(doc2_with_re))
self.assertEqual(doc2_bson, BSON.encode(doc2_with_bson_re))
# Built-in re objects don't support ==. Compare pattern and flags.
self.assertEqual(re2.pattern, BSON(doc2_bson).decode()['r'].pattern)
self.assertEqual(re2.flags, BSON(doc2_bson).decode()['r'].flags)
self.assertEqual(
doc2_with_bson_re, BSON(doc2_bson).decode(compile_re=False))
def test_binary(self):
bin_type_dict = {"bin": Binary(b("\x00\x01\x02\x03\x04"))}
md5_type_dict = {"md5": Binary(b(" n7\x18\xaf\t/\xd1\xd1/\x80\xca\xe7q\xcc\xac"), MD5_SUBTYPE)}
custom_type_dict = {"custom": Binary(b("hello"), USER_DEFINED_SUBTYPE)}
self.round_trip(bin_type_dict)
self.round_trip(md5_type_dict)
self.round_trip(custom_type_dict)
# PYTHON-443 ensure old type formats are supported
json_bin_dump = json_util.dumps(bin_type_dict)
self.assertTrue('"$type": "00"' in json_bin_dump)
self.assertEqual(bin_type_dict, json_util.loads('{"bin": {"$type": 0, "$binary": "AAECAwQ="}}'))
json_bin_dump = json_util.dumps(md5_type_dict)
self.assertTrue('"$type": "05"' in json_bin_dump)
self.assertEqual(
md5_type_dict, json_util.loads('{"md5": {"$type": 5, "$binary":' ' "IG43GK8JL9HRL4DK53HMrA=="}}')
)
json_bin_dump = json_util.dumps(custom_type_dict)
self.assertTrue('"$type": "80"' in json_bin_dump)
self.assertEqual(custom_type_dict, json_util.loads('{"custom": {"$type": 128, "$binary":' ' "aGVsbG8="}}'))
# Handle mongoexport where subtype >= 128
self.assertEqual(
128, json_util.loads('{"custom": {"$type": "ffffff80", "$binary":' ' "aGVsbG8="}}')["custom"].subtype
)
self.assertEqual(
255, json_util.loads('{"custom": {"$type": "ffffffff", "$binary":' ' "aGVsbG8="}}')["custom"].subtype
)
def test_put_filelike(self):
db = self.cx.pymongo_test
fs = yield motor.MotorGridFS(db).open()
oid = yield fs.put(StringIO(b("hello world")), chunk_size=1)
self.assertEqual(11, (yield db.fs.chunks.count()))
gridout = yield fs.get(oid)
self.assertEqual(b("hello world"), (yield gridout.read()))
def test_pickle(self):
b1 = Binary(b('123'), 2)
# For testing backwards compatibility with pre-2.4 pymongo
if PY3:
p = b("\x80\x03cbson.binary\nBinary\nq\x00C\x03123q\x01\x85q"
"\x02\x81q\x03}q\x04X\x10\x00\x00\x00_Binary__subtypeq"
"\x05K\x02sb.")
else:
p = b("ccopy_reg\n_reconstructor\np0\n(cbson.binary\nBinary\np1\nc"
"__builtin__\nstr\np2\nS'123'\np3\ntp4\nRp5\n(dp6\nS'_Binary"
"__subtype'\np7\nI2\nsb.")
if not sys.version.startswith('3.0'):
self.assertEqual(b1, pickle.loads(p))
for proto in xrange(pickle.HIGHEST_PROTOCOL + 1):
self.assertEqual(b1, pickle.loads(pickle.dumps(b1, proto)))
if should_test_uuid:
uu = uuid.uuid4()
uul = UUIDLegacy(uu)
self.assertEqual(uul, copy.copy(uul))
self.assertEqual(uul, copy.deepcopy(uul))
for proto in xrange(pickle.HIGHEST_PROTOCOL + 1):
self.assertEqual(uul, pickle.loads(pickle.dumps(uul, proto)))
def test_pickle_backwards_compatability(self):
# For a full discussion see http://bugs.python.org/issue6137
if sys.version.startswith('3.0'):
raise SkipTest("Python 3.0.x can't unpickle "
"objects pickled in Python 2.x.")
# This string was generated by pickling an ObjectId in pymongo
# version 1.9
pickled_with_1_9 = b(
"ccopy_reg\n_reconstructor\np0\n"
"(cbson.objectid\nObjectId\np1\nc__builtin__\n"
"object\np2\nNtp3\nRp4\n"
"(dp5\nS'_ObjectId__id'\np6\n"
"S'M\\x9afV\\x13v\\xc0\\x0b\\x88\\x00\\x00\\x00'\np7\nsb.")
# We also test against a hardcoded "New" pickle format so that we
# make sure we're backward compatible with the current version in
# the future as well.
pickled_with_1_10 = b(
"ccopy_reg\n_reconstructor\np0\n"
"(cbson.objectid\nObjectId\np1\nc__builtin__\n"
"object\np2\nNtp3\nRp4\n"
"S'M\\x9afV\\x13v\\xc0\\x0b\\x88\\x00\\x00\\x00'\np5\nb."
)
if PY3:
# Have to load using 'latin-1' since these were pickled in python2.x.
oid_1_9 = pickle.loads(pickled_with_1_9, encoding='latin-1')
oid_1_10 = pickle.loads(pickled_with_1_10, encoding='latin-1')
else:
oid_1_9 = pickle.loads(pickled_with_1_9)
oid_1_10 = pickle.loads(pickled_with_1_10)
self.assertEqual(oid_1_9, ObjectId("4d9a66561376c00b88000000"))
self.assertEqual(oid_1_9, oid_1_10)
def test_binary(self):
a_string = "hello world"
a_binary = Binary(b("hello world"))
self.assertTrue(a_binary.startswith(b("hello")))
self.assertTrue(a_binary.endswith(b("world")))
self.assertTrue(isinstance(a_binary, Binary))
self.assertFalse(isinstance(a_string, Binary))
def test_put_callback(self):
(oid, error), _ = yield gen.Task(self.fs.put, b("hello"))
self.assertTrue(isinstance(oid, ObjectId))
self.assertEqual(None, error)
(result, error), _ = yield gen.Task(self.fs.put, b("hello"), _id=oid)
self.assertEqual(None, result)
self.assertTrue(isinstance(error, FileExists))
def test_put_filelike(self, done):
db = self.motor_connection(host, port).open_sync().pymongo_test
fs = yield motor.Op(motor.MotorGridFS(db).open)
oid = yield motor.Op(fs.put, StringIO(b("hello world")), chunk_size=1)
self.assertEqual(11, (yield motor.Op(db.fs.chunks.count)))
gridout = yield motor.Op(fs.get, oid)
self.assertEqual(b("hello world"), (yield motor.Op(gridout.read)))
done()
def test_grid_in_callback(self):
f = motor.MotorGridIn(self.db.fs, filename="test")
yield self.check_optional_callback(partial(f.set, 'name', 'value'))
yield self.check_optional_callback(partial(f.write, b('a')))
yield self.check_optional_callback(partial(f.writelines, [b('a')]))
self.assertRaises(TypeError, f.close, callback='foo')
self.assertRaises(TypeError, f.close, callback=1)
f.close(callback=None) # No error
def test_basic_decode(self):
self.assertEqual({"test": u"hello world"},
BSON(b("\x1B\x00\x00\x00\x0E\x74\x65\x73\x74\x00\x0C"
"\x00\x00\x00\x68\x65\x6C\x6C\x6F\x20\x77\x6F"
"\x72\x6C\x64\x00\x00")).decode())
self.assertEqual([{"test": u"hello world"}, {}],
decode_all(b("\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
"\x05\x00\x00\x00\x00")))
def test_gridfs_replica_set(self):
rsc = self._get_client(w=self.w, wtimeout=5000, read_preference=ReadPreference.SECONDARY)
try:
fs = gridfs.GridFS(rsc.pymongo_test)
oid = fs.put(b("foo"))
content = fs.get(oid).read()
self.assertEqual(b("foo"), content)
finally:
rsc.close()
def test_grid_in_callback(self, done):
db = self.motor_connection(host, port).open_sync().pymongo_test
f = motor.MotorGridIn(db.fs, filename="test")
self.check_callback_handling(f.open, False)
f = yield motor.Op(motor.MotorGridIn(db.fs, filename="test").open)
self.check_callback_handling(partial(f.set, 'name', 'value'), False)
self.check_callback_handling(f.close, False)
self.check_callback_handling(partial(f.write, b('a')), False)
self.check_callback_handling(partial(f.writelines, [b('a')]), False)
done()
def test_gridfs_replica_set(self):
rsc = yield self.motor_rsc(
w=self.w, wtimeout=5000,
read_preference=ReadPreference.SECONDARY)
fs = yield motor.MotorGridFS(rsc.pymongo_test).open()
oid = yield fs.put(b('foo'))
gridout = yield fs.get(oid)
content = yield gridout.read()
self.assertEqual(b('foo'), content)
def test_repr_str(self):
self.assertEqual(repr(ObjectId("1234567890abcdef12345678")),
"ObjectId('1234567890abcdef12345678')")
self.assertEqual(str(ObjectId("1234567890abcdef12345678")),
"1234567890abcdef12345678")
self.assertEqual(str(ObjectId(b("123456789012"))),
"313233343536373839303132")
self.assertEqual(ObjectId("1234567890abcdef12345678").binary,
b('\x124Vx\x90\xab\xcd\xef\x124Vx'))
self.assertEqual(str(ObjectId(b('\x124Vx\x90\xab\xcd\xef\x124Vx'))),
"1234567890abcdef12345678")
def test_list(self):
self.assertEqual([], (yield self.fs.list()))
yield self.fs.put(b("hello world"))
self.assertEqual([], (yield self.fs.list()))
yield self.fs.put(b(""), filename="mike")
yield self.fs.put(b("foo"), filename="test")
yield self.fs.put(b(""), filename="hello world")
self.assertEqual(set(["mike", "test", "hello world"]),
set((yield self.fs.list())))
def test_regex_from_native(self):
self.assertEqual('.*', Regex.from_native(re.compile('.*')).pattern)
self.assertEqual(0, Regex.from_native(re.compile(b(''))).flags)
regex = re.compile(b(''), re.I | re.L | re.M | re.S | re.X)
self.assertEqual(
re.I | re.L | re.M | re.S | re.X,
Regex.from_native(regex).flags)
unicode_regex = re.compile('', re.U)
self.assertEqual(re.U, Regex.from_native(unicode_regex).flags)
def test_gridfs_callback(self):
yield self.check_optional_callback(self.fs.new_file)
yield self.check_optional_callback(partial(self.fs.put, b('a')))
yield self.fs.put(b('foo'), _id=1, filename='f')
yield self.check_optional_callback(self.fs.get, 1)
yield self.check_optional_callback(self.fs.get_version, 'f')
yield self.check_optional_callback(self.fs.get_last_version, 'f')
yield self.check_optional_callback(partial(self.fs.delete, 1))
yield self.check_optional_callback(self.fs.list)
yield self.check_optional_callback(self.fs.exists)
def test_repr(self):
one = Binary(b("hello world"))
self.assertEqual(repr(one), "Binary(%s, 0)" % (repr(b("hello world")),))
two = Binary(b("hello world"), 2)
self.assertEqual(repr(two), "Binary(%s, 2)" % (repr(b("hello world")),))
three = Binary(b("\x08\xFF"))
self.assertEqual(repr(three), "Binary(%s, 0)" % (repr(b("\x08\xFF")),))
four = Binary(b("\x08\xFF"), 2)
self.assertEqual(repr(four), "Binary(%s, 2)" % (repr(b("\x08\xFF")),))
five = Binary(b("test"), 100)
self.assertEqual(repr(five), "Binary(%s, 100)" % (repr(b("test")),))
def test_corrupt_chunk(self):
files_id = self.fs.put(b('foobar'))
self.db.fs.chunks.update({'files_id': files_id},
{'$set': {'data': Binary(b('foo'), 0)}})
try:
out = self.fs.get(files_id)
self.assertRaises(CorruptGridFile, out.read)
out = self.fs.get(files_id)
self.assertRaises(CorruptGridFile, out.readline)
finally:
self.fs.delete(files_id)
def test_empty_file(self):
oid = self.fs.put(b(""))
self.assertEqual(b(""), self.fs.get(oid).read())
self.assertEqual(1, self.db.fs.files.count())
self.assertEqual(0, self.db.fs.chunks.count())
raw = self.db.fs.files.find_one()
self.assertEqual(0, raw["length"])
self.assertEqual(oid, raw["_id"])
self.assertTrue(isinstance(raw["uploadDate"], datetime.datetime))
self.assertEqual(256 * 1024, raw["chunkSize"])
self.assertTrue(isinstance(raw["md5"], basestring))
def test_grid_out_file_document(self):
one = GridIn(self.db.fs)
one.write(b("foo bar"))
one.close()
two = GridOut(self.db.fs, file_document=self.db.fs.files.find_one())
self.assertEqual(b("foo bar"), two.read())
three = GridOut(self.db.fs, 5, file_document=self.db.fs.files.find_one())
self.assertEqual(b("foo bar"), three.read())
self.assertRaises(NoFile, GridOut, self.db.fs, file_document={})
def test_threaded_reads(self):
self.fs.put(b("hello"), _id="test")
threads = []
results = []
for i in range(10):
threads.append(JustRead(self.fs, 10, results))
threads[i].start()
joinall(threads)
self.assertEqual(100 * [b("hello")], results)
def test_write_file_like(self):
one = motor.MotorGridIn(self.db.fs)
yield one.write(b("hello world"))
yield one.close()
two = motor.MotorGridOut(self.db.fs, one._id)
three = motor.MotorGridIn(self.db.fs)
yield three.write(two)
yield three.close()
four = motor.MotorGridOut(self.db.fs, three._id)
self.assertEqual(b("hello world"), (yield four.read()))
def test_file_exists(self):
db = get_client(w=1).pymongo_test
fs = gridfs.GridFS(db)
oid = fs.put(b("hello"))
self.assertRaises(FileExists, fs.put, b("world"), _id=oid)
one = fs.new_file(_id=123)
one.write(b("some content"))
one.close()
two = fs.new_file(_id=123)
self.assertRaises(FileExists, two.write, b('x' * 262146))
def test_move_id(self):
self.assertEqual(b("\x19\x00\x00\x00\x02_id\x00\x02\x00\x00\x00a\x00"
"\x02a\x00\x02\x00\x00\x00a\x00\x00"),
BSON.encode(SON([("a", "a"), ("_id", "a")])))
self.assertEqual(b("\x2c\x00\x00\x00"
"\x02_id\x00\x02\x00\x00\x00b\x00"
"\x03b\x00"
"\x19\x00\x00\x00\x02a\x00\x02\x00\x00\x00a\x00"
"\x02_id\x00\x02\x00\x00\x00a\x00\x00\x00"),
BSON.encode(SON([("b",
SON([("a", "a"), ("_id", "a")])),
("_id", "b")])))
def test_list(self):
db = self.cx.pymongo_test
fs = yield motor.MotorGridFS(db).open()
self.assertEqual([], (yield fs.list()))
yield fs.put(b("hello world"))
self.assertEqual([], (yield fs.list()))
yield fs.put(b(""), filename="mike")
yield fs.put(b("foo"), filename="test")
yield fs.put(b(""), filename="hello world")
self.assertEqual(set(["mike", "test", "hello world"]),
set((yield fs.list())))
def test_list(self, done):
db = self.motor_connection(host, port).open_sync().pymongo_test
fs = yield motor.Op(motor.MotorGridFS(db).open)
self.assertEqual([], (yield motor.Op(fs.list)))
yield motor.Op(fs.put, b("hello world"))
self.assertEqual([], (yield motor.Op(fs.list)))
yield motor.Op(fs.put, b(""), filename="mike")
yield motor.Op(fs.put, b("foo"), filename="test")
yield motor.Op(fs.put, b(""), filename="hello world")
self.assertEqual(set(["mike", "test", "hello world"]), set((yield motor.Op(fs.list))))
done()
def test_write_file_like(self):
db = self.cx.pymongo_test
one = yield motor.MotorGridIn(db.fs).open()
yield one.write(b("hello world"))
yield one.close()
two = yield motor.MotorGridOut(db.fs, one._id).open()
three = yield motor.MotorGridIn(db.fs).open()
yield three.write(two)
yield three.close()
four = yield motor.MotorGridOut(db.fs, three._id).open()
self.assertEqual(b("hello world"), (yield four.read()))
def test_spec(self):
for path in glob.glob(os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"decimal",
"decimal128*")):
with codecs.open(path, "r", "utf-8-sig") as fp:
suite = json.load(fp)
for case in suite.get("valid", []):
B = unhexlify(b(case["bson"]))
E = case["extjson"].replace(" ", "")
if "canonical_bson" in case:
cB = unhexlify(b(case["canonical_bson"]))
else:
cB = B
if "canonical_extjson" in case:
cE = case["canonical_extjson"].replace(" ", "")
else:
cE = E
self.assertEqual(BSON().encode(BSON(B).decode()), cB)
if B != cB:
self.assertEqual(BSON().encode(BSON(cB).decode()), cB)
self.assertEqual(
dumps(BSON(B).decode()).replace(" ", ""), cE)
self.assertEqual(
dumps(loads(E)).replace(" ", ""), cE)
if B != cB:
self.assertEqual(
dumps(BSON(cB).decode()).replace(" ", ""), cE)
if E != cE:
self.assertEqual(
dumps(loads(cE)).replace(" ", ""), cE)
if "lossy" not in case:
self.assertEqual(BSON().encode(loads(E)), cB)
if E != cE:
self.assertEqual(BSON().encode(loads(cE)), cB)
for test in suite.get("parseErrors", []):
self.assertRaises(
DecimalException, Decimal128, test["string"])
def test_list(self):
self.assertEqual([], self.fs.list())
self.fs.put(b("hello world"))
self.assertEqual([], self.fs.list())
# PYTHON-598: in server versions before 2.5.x, creating an index on
# filename, uploadDate causes list() to include None.
self.fs.get_last_version()
self.assertEqual([], self.fs.list())
self.fs.put(b(""), filename="mike")
self.fs.put(b("foo"), filename="test")
self.fs.put(b(""), filename="hello world")
self.assertEqual(set(["mike", "test", "hello world"]),
set(self.fs.list()))
def test_alternate_collection(self):
self.db.alt.files.remove({})
self.db.alt.chunks.remove({})
f = GridIn(self.db.alt)
f.write(b("hello world"))
f.close()
self.assertEqual(1, self.db.alt.files.find().count())
self.assertEqual(1, self.db.alt.chunks.find().count())
g = GridOut(self.db.alt, f._id)
self.assertEqual(b("hello world"), g.read())
# test that md5 still works...
self.assertEqual("5eb63bbbe01eeed093cb22bb8f5acdc3", g.md5)
def test_context_manager(self):
if sys.version_info < (2, 6):
raise SkipTest("With statement requires Python >= 2.6")
contents = b("Imagine this is some important data...")
# Hack around python2.4 an 2.5 not supporting 'with' syntax
exec """
def test_read_unaligned_buffer_size(self):
in_data = b("This is a text that doesn't "
"quite fit in a single 16-byte chunk.")
f = GridIn(self.db.fs, chunkSize=16)
f.write(in_data)
f.close()
g = GridOut(self.db.fs, f._id)
out_data = b('')
while 1:
s = g.read(13)
if not s:
break
out_data += s
self.assertEqual(in_data, out_data)
def test_grid_out_custom_opts(self):
one = GridIn(self.db.fs,
_id=5,
filename="my_file",
contentType="text/html",
chunkSize=1000,
aliases=["foo"],
metadata={
"foo": 1,
"bar": 2
},
bar=3,
baz="hello")
one.write(b("hello world"))
one.close()
two = GridOut(self.db.fs, 5)
self.assertEqual("my_file", two.name)
self.assertEqual("my_file", two.filename)
self.assertEqual(5, two._id)
self.assertEqual(11, two.length)
self.assertEqual("text/html", two.content_type)
self.assertEqual(1000, two.chunk_size)
self.assertTrue(isinstance(two.upload_date, datetime.datetime))
self.assertEqual(["foo"], two.aliases)
self.assertEqual({"foo": 1, "bar": 2}, two.metadata)
self.assertEqual(3, two.bar)
self.assertEqual("5eb63bbbe01eeed093cb22bb8f5acdc3", two.md5)
for attr in [
"_id", "name", "content_type", "length", "chunk_size",
"upload_date", "aliases", "metadata", "md5"
]:
self.assertRaises(AttributeError, setattr, two, attr, 5)
def readline(self, size=-1):
"""Read one line or up to `size` bytes from the file.
:Parameters:
- `size` (optional): the maximum number of bytes to read
.. versionadded:: 1.9
"""
if size == 0:
return b('')
remainder = int(self.length) - self.__position
if size < 0 or size > remainder:
size = remainder
received = 0
data = StringIO()
while received < size:
chunk_data = self.readchunk()
pos = chunk_data.find(NEWLN, 0, size)
if pos != -1:
size = received + pos + 1
received += len(chunk_data)
data.write(chunk_data)
if pos != -1:
break
self.__position -= received - size
# Return 'size' bytes and store the rest.
data.seek(size)
self.__buffer = data.read()
data.seek(0)
return data.read(size)
def test_repr(self):
one = Binary(b("hello world"))
self.assertEqual(repr(one),
"Binary(%s, 0)" % (repr(b("hello world")),))
two = Binary(b("hello world"), 2)
self.assertEqual(repr(two),
"Binary(%s, 2)" % (repr(b("hello world")),))
three = Binary(b("\x08\xFF"))
self.assertEqual(repr(three),
"Binary(%s, 0)" % (repr(b("\x08\xFF")),))
four = Binary(b("\x08\xFF"), 2)
self.assertEqual(repr(four),
"Binary(%s, 2)" % (repr(b("\x08\xFF")),))
five = Binary(b("test"), 100)
self.assertEqual(repr(five),
"Binary(%s, 100)" % (repr(b("test")),))
def test_gridfs_callback(self):
db = self.cx.pymongo_test
fs = motor.MotorGridFS(db)
yield self.check_optional_callback(fs.open)
fs = yield motor.MotorGridFS(db).open()
yield self.check_optional_callback(fs.new_file)
yield self.check_optional_callback(partial(fs.put, b('a')))
yield fs.put(b('foo'), _id=1, filename='f')
yield self.check_optional_callback(fs.get, 1)
yield self.check_optional_callback(fs.get_version, 'f')
yield self.check_optional_callback(fs.get_last_version, 'f')
yield self.check_optional_callback(partial(fs.delete, 1))
yield self.check_optional_callback(fs.list)
yield self.check_optional_callback(fs.exists)
def test_put_unacknowledged(self):
client = yield self.motor_client(w=0)
fs = yield motor.MotorGridFS(client.pymongo_test).open()
with assert_raises(ConfigurationError):
yield fs.put(b("hello"))
client.close()
def test_gridfs_secondary(self):
primary_host, primary_port = self.primary
primary_connection = MongoClient(primary_host, primary_port)
secondary_host, secondary_port = self.secondaries[0]
ctx = catch_warnings()
try:
warnings.simplefilter("ignore", DeprecationWarning)
for secondary_connection in [
MongoClient(secondary_host,
secondary_port,
slave_okay=True),
MongoClient(secondary_host,
secondary_port,
read_preference=ReadPreference.SECONDARY),
]:
primary_connection.pymongo_test.drop_collection("fs.files")
primary_connection.pymongo_test.drop_collection("fs.chunks")
# Should detect it's connected to secondary and not attempt to
# create index
fs = gridfs.GridFS(secondary_connection.pymongo_test)
# This won't detect secondary, raises error
self.assertRaises(ConnectionFailure, fs.put, b('foo'))
finally:
ctx.exit()
def test_get_version_with_metadata(self):
one = self.fs.put(b("foo"), filename="test", author="author1")
time.sleep(0.01)
two = self.fs.put(b("bar"), filename="test", author="author1")
time.sleep(0.01)
three = self.fs.put(b("baz"), filename="test", author="author2")
self.assertEqual(
b("foo"),
self.fs.get_version(filename="test", author="author1",
version=-2).read())
self.assertEqual(
b("bar"),
self.fs.get_version(filename="test", author="author1",
version=-1).read())
self.assertEqual(
b("foo"),
self.fs.get_version(filename="test", author="author1",
version=0).read())
self.assertEqual(
b("bar"),
self.fs.get_version(filename="test", author="author1",
version=1).read())
self.assertEqual(
b("baz"),
self.fs.get_version(filename="test", author="author2",
version=0).read())
self.assertEqual(
b("baz"),
self.fs.get_version(filename="test", version=-1).read())
self.assertEqual(
b("baz"),
self.fs.get_version(filename="test", version=2).read())
self.assertRaises(NoFile,
self.fs.get_version,
filename="test",
author="author3")
self.assertRaises(NoFile,
self.fs.get_version,
filename="test",
author="author1",
version=2)
self.fs.delete(one)
self.fs.delete(two)
self.fs.delete(three)
def test_null_character(self):
doc = {"a": "\x00"}
self.assertEqual(doc, BSON.encode(doc).decode())
# This test doesn't make much sense in Python2
# since {'a': '\x00'} == {'a': u'\x00'}.
# Decoding here actually returns {'a': '\x00'}
doc = {"a": u"\x00"}
self.assertEqual(doc, BSON.encode(doc).decode())
self.assertRaises(InvalidDocument, BSON.encode, {b("\x00"): "a"})
self.assertRaises(InvalidDocument, BSON.encode, {u"\x00": "a"})
self.assertRaises(InvalidDocument, BSON.encode,
{"a": re.compile(b("ab\x00c"))})
self.assertRaises(InvalidDocument, BSON.encode,
{"a": re.compile(u"ab\x00c")})
def test_alternate_collection(self):
db = self.cx.pymongo_test
yield db.alt.files.remove()
yield db.alt.chunks.remove()
f = yield motor.MotorGridIn(db.alt).open()
yield f.write(b("hello world"))
yield f.close()
self.assertEqual(1, (yield db.alt.files.find().count()))
self.assertEqual(1, (yield db.alt.chunks.find().count()))
g = yield motor.MotorGridOut(db.alt, f._id).open()
self.assertEqual(b("hello world"), (yield g.read()))
# test that md5 still works...
self.assertEqual("5eb63bbbe01eeed093cb22bb8f5acdc3", g.md5)
def test_basic(self):
f = motor.MotorGridIn(self.db.fs, filename="test")
yield f.write(b("hello world"))
yield f.close()
self.assertEqual(1, (yield self.db.fs.files.find().count()))
self.assertEqual(1, (yield self.db.fs.chunks.find().count()))
g = motor.MotorGridOut(self.db.fs, f._id)
self.assertEqual(b("hello world"), (yield g.read()))
f = motor.MotorGridIn(self.db.fs, filename="test")
yield f.close()
self.assertEqual(2, (yield self.db.fs.files.find().count()))
self.assertEqual(1, (yield self.db.fs.chunks.find().count()))
g = motor.MotorGridOut(self.db.fs, f._id)
self.assertEqual(b(""), (yield g.read()))
def test_basic_validation(self):
self.assertRaises(TypeError, is_valid, 100)
self.assertRaises(TypeError, is_valid, u"test")
self.assertRaises(TypeError, is_valid, 10.4)
self.assertFalse(is_valid(b("test")))
# the simplest valid BSON document
self.assertTrue(is_valid(b("\x05\x00\x00\x00\x00")))
self.assertTrue(is_valid(BSON(b("\x05\x00\x00\x00\x00"))))
self.assertFalse(is_valid(b("\x04\x00\x00\x00\x00")))
self.assertFalse(is_valid(b("\x05\x00\x00\x00\x01")))
self.assertFalse(is_valid(b("\x05\x00\x00\x00")))
self.assertFalse(is_valid(b("\x05\x00\x00\x00\x00\x00")))
self.assertFalse(is_valid(b("\x07\x00\x00\x00\x02a\x00\x78\x56\x34\x12")))
self.assertFalse(is_valid(b("\x09\x00\x00\x00\x10a\x00\x05\x00")))
self.assertFalse(is_valid(b("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00")))
def test_alt_collection(self):
db = self.cx.pymongo_test
alt = yield motor.MotorGridFS(db, 'alt').open()
oid = yield alt.put(b("hello world"))
gridout = yield alt.get(oid)
self.assertEqual(b("hello world"), (yield gridout.read()))
self.assertEqual(1, (yield db.alt.files.count()))
self.assertEqual(1, (yield db.alt.chunks.count()))
yield alt.delete(oid)
with assert_raises(NoFile):
yield alt.get(oid)
self.assertEqual(0, (yield db.alt.files.count()))
self.assertEqual(0, (yield db.alt.chunks.count()))
with assert_raises(NoFile):
yield alt.get("foo")
oid = yield alt.put(b("hello world"), _id="foo")
self.assertEqual("foo", oid)
gridout = yield alt.get("foo")
self.assertEqual(b("hello world"), (yield gridout.read()))
yield alt.put(b(""), filename="mike")
yield alt.put(b("foo"), filename="test")
yield alt.put(b(""), filename="hello world")
self.assertEqual(set(["mike", "test", "hello world"]),
set((yield alt.list())))
def test_encode_then_decode(self):
def helper(dict):
self.assertEqual(dict, (BSON.encode(dict)).decode())
helper({})
helper({"test": u"hello"})
self.assertTrue(isinstance(BSON.encode({"hello": "world"})
.decode()["hello"],
unicode))
helper({"mike": -10120})
helper({"long": long(10)})
helper({"really big long": 2147483648})
helper({u"hello": 0.0013109})
helper({"something": True})
helper({"false": False})
helper({"an array": [1, True, 3.8, u"world"]})
helper({"an object": {"test": u"something"}})
helper({"a binary": Binary(b("test"), 100)})
helper({"a binary": Binary(b("test"), 128)})
helper({"a binary": Binary(b("test"), 254)})
helper({"another binary": Binary(b("test"), 2)})
helper(SON([(u'test dst', datetime.datetime(1993, 4, 4, 2))]))
helper(SON([(u'test negative dst', datetime.datetime(1, 1, 1, 1, 1, 1))]))
helper({"big float": float(10000000000)})
helper({"ref": DBRef("coll", 5)})
helper({"ref": DBRef("coll", 5, foo="bar", bar=4)})
helper({"ref": DBRef("coll", 5, "foo")})
helper({"ref": DBRef("coll", 5, "foo", foo="bar")})
helper({"ref": Timestamp(1, 2)})
helper({"foo": MinKey()})
helper({"foo": MaxKey()})
helper({"$field": Code("function(){ return true; }")})
helper({"$field": Code("return function(){ return x; }", scope={'x': False})})
doc_class = dict
# Work around http://bugs.jython.org/issue1728
if (sys.platform.startswith('java') and
sys.version_info[:3] >= (2, 5, 2)):
doc_class = SON
def encode_then_decode(doc):
return doc == (BSON.encode(doc)).decode(as_class=doc_class)
qcheck.check_unittest(self, encode_then_decode,
qcheck.gen_mongo_dict(3))
def test_grid_out_file_document(self):
db = self.cx.pymongo_test
one = yield motor.MotorGridIn(db.fs).open()
yield one.write(b("foo bar"))
yield one.close()
two = yield motor.MotorGridOut(
db.fs, file_document=(yield db.fs.files.find_one())).open()
self.assertEqual(b("foo bar"), (yield two.read()))
three = yield motor.MotorGridOut(
db.fs, 5, file_document=(yield db.fs.files.find_one())).open()
self.assertEqual(b("foo bar"), (yield three.read()))
with assert_raises(NoFile):
yield motor.MotorGridOut(db.fs, file_document={}).open()
def test_pickle_backwards_compatability(self):
# This string was generated by pickling a SON object in pymongo
# version 2.1.1
pickled_with_2_1_1 = b(
"ccopy_reg\n_reconstructor\np0\n(cbson.son\nSON\np1\n"
"c__builtin__\ndict\np2\n(dp3\ntp4\nRp5\n(dp6\n"
"S'_SON__keys'\np7\n(lp8\nsb.")
son_2_1_1 = pickle.loads(pickled_with_2_1_1)
self.assertEqual(son_2_1_1, SON([]))
def test_bytes_as_keys(self):
doc = {b("foo"): 'bar'}
# Since `bytes` are stored as Binary you can't use them
# as keys in python 3.x. Using binary data as a key makes
# no sense in BSON anyway and little sense in python.
if PY3:
self.assertRaises(InvalidDocument, BSON.encode, doc)
else:
self.assertTrue(BSON.encode(doc))
def test_binary(self):
bin_type_dict = {"bin": Binary(b("\x00\x01\x02\x03\x04"))}
md5_type_dict = {
"md5":
Binary(b(' n7\x18\xaf\t/\xd1\xd1/\x80\xca\xe7q\xcc\xac'),
MD5_SUBTYPE)
}
custom_type_dict = {"custom": Binary(b("hello"), USER_DEFINED_SUBTYPE)}
self.round_trip(bin_type_dict)
self.round_trip(md5_type_dict)
self.round_trip(custom_type_dict)
# PYTHON-443 ensure old type formats are supported
json_bin_dump = json_util.dumps(bin_type_dict)
self.assertTrue('"$type": "00"' in json_bin_dump)
self.assertEqual(
bin_type_dict,
json_util.loads('{"bin": {"$type": 0, "$binary": "AAECAwQ="}}'))
json_bin_dump = json_util.dumps(md5_type_dict)
self.assertTrue('"$type": "05"' in json_bin_dump)
self.assertEqual(
md5_type_dict,
json_util.loads('{"md5": {"$type": 5, "$binary":'
' "IG43GK8JL9HRL4DK53HMrA=="}}'))
json_bin_dump = json_util.dumps(custom_type_dict)
self.assertTrue('"$type": "80"' in json_bin_dump)
self.assertEqual(
custom_type_dict,
json_util.loads('{"custom": {"$type": 128, "$binary":'
' "aGVsbG8="}}'))
# Handle mongoexport where subtype >= 128
self.assertEqual(
128,
json_util.loads('{"custom": {"$type": "ffffff80", "$binary":'
' "aGVsbG8="}}')['custom'].subtype)
self.assertEqual(
255,
json_util.loads('{"custom": {"$type": "ffffffff", "$binary":'
' "aGVsbG8="}}')['custom'].subtype)
def test_equality(self):
two = Binary(b("hello"))
three = Binary(b("hello"), 100)
self.assertNotEqual(two, three)
self.assertEqual(three, Binary(b("hello"), 100))
self.assertEqual(two, Binary(b("hello")))
self.assertNotEqual(two, Binary(b("hello ")))
self.assertNotEqual(b("hello"), Binary(b("hello")))
def _authenticate_cram_md5(credentials, sock_info, cmd_func):
"""Authenticate using CRAM-MD5 (RFC 2195)
"""
source, username, password = credentials
# The password used as the mac key is the
# same as what we use for MONGODB-CR
passwd = _password_digest(username, password)
cmd = SON([('saslStart', 1), ('mechanism', 'CRAM-MD5'),
('payload', Binary(b(''))), ('autoAuthorize', 1)])
response, _ = cmd_func(sock_info, source, cmd)
# MD5 as implicit default digest for digestmod is deprecated
# in python 3.4
mac = hmac.HMAC(key=passwd.encode('utf-8'), digestmod=_DMOD)
mac.update(response['payload'])
challenge = username.encode('utf-8') + b(' ') + b(mac.hexdigest())
cmd = SON([('saslContinue', 1),
('conversationId', response['conversationId']),
('payload', Binary(challenge))])
cmd_func(sock_info, source, cmd)
def test_grid_in_non_int_chunksize(self):
# Lua, and perhaps other buggy GridFS clients, store size as a float.
data = b('data')
self.fs.put(data, filename='f')
self.db.fs.files.update({'filename': 'f'},
{'$set': {
'chunkSize': 100.0
}})
self.assertEqual(data, self.fs.get_version('f').read())
def test_put_unicode(self):
self.assertRaises(TypeError, self.fs.put, u"hello")
oid = self.fs.put(u"hello", encoding="utf-8")
self.assertEqual(b("hello"), self.fs.get(oid).read())
self.assertEqual("utf-8", self.fs.get(oid).encoding)
oid = self.fs.put(u"aé", encoding="iso-8859-1")
self.assertEqual(u"aé".encode("iso-8859-1"), self.fs.get(oid).read())
self.assertEqual("iso-8859-1", self.fs.get(oid).encoding)
def test_invalid_decodes(self):
# Invalid object size (not enough bytes in document for even
# an object size of first object.
# NOTE: decode_all and decode_iter don't care, not sure if they should?
self.assertRaises(InvalidBSON, list,
decode_file_iter(StringIO(b("\x1B"))))
# An object size that's too small to even include the object size,
# but is correctly encoded, along with a correct EOO (and no data).
data = b("\x01\x00\x00\x00\x00")
self.assertRaises(InvalidBSON, decode_all, data)
self.assertRaises(InvalidBSON, list, decode_iter(data))
self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data)))
# One object, but with object size listed smaller than it is in the
# data.
data = b("\x1A\x00\x00\x00\x0E\x74\x65\x73\x74"
"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
"\x05\x00\x00\x00\x00")
self.assertRaises(InvalidBSON, decode_all, data)
self.assertRaises(InvalidBSON, list, decode_iter(data))
self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data)))
# One object, missing the EOO at the end.
data = b("\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
"\x05\x00\x00\x00")
self.assertRaises(InvalidBSON, decode_all, data)
self.assertRaises(InvalidBSON, list, decode_iter(data))
self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data)))
# One object, sized correctly, with a spot for an EOO, but the EOO
# isn't 0x00.
data = b("\x1B\x00\x00\x00\x0E\x74\x65\x73\x74"
"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C"
"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00"
"\x05\x00\x00\x00\xFF")
self.assertRaises(InvalidBSON, decode_all, data)
self.assertRaises(InvalidBSON, list, decode_iter(data))
self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data)))
def test_multiple_reads(self):
f = GridIn(self.db.fs, chunkSize=3)
f.write(b("hello world"))
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual(b("he"), g.read(2))
self.assertEqual(b("ll"), g.read(2))
self.assertEqual(b("o "), g.read(2))
self.assertEqual(b("wo"), g.read(2))
self.assertEqual(b("rl"), g.read(2))
self.assertEqual(b("d"), g.read(2))
self.assertEqual(b(""), g.read(2))
def test_basic(self):
db = self.cx.pymongo_test
fs = yield motor.MotorGridFS(db).open()
oid = yield fs.put(b("hello world"))
out = yield fs.get(oid)
self.assertEqual(b("hello world"), (yield out.read()))
self.assertEqual(1, (yield db.fs.files.count()))
self.assertEqual(1, (yield db.fs.chunks.count()))
yield fs.delete(oid)
with assert_raises(NoFile):
yield fs.get(oid)
self.assertEqual(0, (yield db.fs.files.count()))
self.assertEqual(0, (yield db.fs.chunks.count()))
with assert_raises(NoFile):
yield fs.get("foo")
self.assertEqual("foo", (yield fs.put(b("hello world"), _id="foo")))
gridout = yield fs.get("foo")
self.assertEqual(b("hello world"), (yield gridout.read()))
def test_is_valid(self):
self.assertFalse(ObjectId.is_valid(4))
self.assertFalse(ObjectId.is_valid(175.0))
self.assertFalse(ObjectId.is_valid({"test": 4}))
self.assertFalse(ObjectId.is_valid(["something"]))
self.assertFalse(ObjectId.is_valid(""))
self.assertFalse(ObjectId.is_valid("12345678901"))
self.assertFalse(ObjectId.is_valid("1234567890123"))
self.assertTrue(ObjectId.is_valid(b("123456789012")))
self.assertTrue(ObjectId.is_valid("123456789012123456789012"))
def test_readline(self):
f = GridIn(self.db.fs, chunkSize=5)
f.write(b("""Hello world,
How are you?
Hope all is well.
Bye"""))
f.close()
g = GridOut(self.db.fs, f._id)
self.assertEqual(b("H"), g.read(1))
self.assertEqual(b("ello world,\n"), g.readline())
self.assertEqual(b("How a"), g.readline(5))
self.assertEqual(b(""), g.readline(0))
self.assertEqual(b("re you?\n"), g.readline())
self.assertEqual(b("Hope all is well.\n"), g.readline(1000))
self.assertEqual(b("Bye"), g.readline())
self.assertEqual(b(""), g.readline())
