def test_get_full_query2(self):
with self.assertRaisesRegexp(
ValueError,
"Invalid ordering: should be str or list of \(column, "
"direction\) pairs"):
with monary.Monary() as m:
m.query("test",
"collection", {}, ["x1", "x2", "x3", "x4", "x5"],
["float64"] * 5,
hint=monary.Monary())
def test_with_authenticate_from_uri():
monary_connection = monary.Monary(
"mongodb://*****:*****@"
"127.0.0.1:27017/monary_auth_test")
[col] = monary_connection.query("monary_auth_test", "junk", {}, ["route"],
["int32"])
assert col[0] == 66, "test value could not be retrieved"
def test_monary_count3(self):
with self.assertRaisesRegexp(
monary.monary.MonaryError,
"Failed to handshake and validate TLS certificate"):
with monary.Monary("mongodb://localhost:27017/?ssl=true") as m:
m.query("test", "collection", {},
["x1", "x2", "x3", "x4", "x5"], ["float64"] * 5)
def test_retrieve_nested(self):
arrays = [
self.bool_arr, self.int8_arr, self.int16_arr, self.int32_arr,
self.int64_arr, self.float32_arr, self.float64_arr,
self.string_arr, self.seq
]
with monary.Monary() as m:
m.insert(
"monary_test", "data",
monary.MonaryParam.from_lists(arrays, [
"a.b.c.d.e.f.g.h.x1", "a.b.c.d.e.f.g.h.x2",
"a.b.c.d.e.f.g.h.x3", "a.b.c.d.e.f.g.h.x4",
"a.b.c.d.e.f.g.h.x5", "a.b.c.d.e.f.g.h.x6",
"a.b.c.d.e.f.g.h.x7", "a.b.c.d.e.f.g.h.x8", "sequence"
], [
"bool", "int8", "int16", "int32", "int64", "float32",
"float64", "string:10", "int64"
]))
with pymongo.MongoClient() as c:
col = c.monary_test.data
for i, doc in enumerate(col.find().sort([("sequence",
pymongo.ASCENDING)])):
assert doc["sequence"] == i
for j in range(8):
if not arrays[j].mask[i]:
data = arrays[j][i]
exp = doc["a"]["b"]["c"]["d"]["e"]["f"]["g"]["h"]
exp = exp["x" + str(j + 1)]
if PY3 and isinstance(data, bytes):
data = data.decode("ascii")
with pymongo.MongoClient() as c:
c.drop_database("monary_test")
def test_get_monary_numpy_type4(self):
with self.assertRaisesRegexp(
ValueError,
"'string' must have an explicit typearg with nonzero length"):
with monary.Monary() as m:
m.query("test", "collection", {},
["x1", "x2", "x3", "x4", "x5"], ["string:0"] * 5)
def test_insert_bson(self):
docs = []
for i in range(NUM_TEST_RECORDS):
doc = {"subdoc": {"num": random.randint(0, 255)}}
if i % 2 == 0:
doc["subdoc"]["bool"] = bool(random.getrandbits(1))
if i % 3 == 0:
doc["float"] = random.uniform(-1e30, 1e30)
docs.append(doc)
encoded = [bson.BSON.encode(d) for d in docs]
max_len = max(map(len, encoded))
encoded = np.ma.masked_array(encoded, np.zeros(NUM_TEST_RECORDS),
"<V%d" % max_len)
with monary.Monary() as m:
m.insert(
"monary_test", "data",
monary.MonaryParam.from_lists([encoded, self.seq],
["doc", "sequence"],
["bson:%d" % max_len, "int64"]))
with pymongo.MongoClient() as c:
col = c.monary_test.data
for i, doc in enumerate(col.find().sort([("sequence",
pymongo.ASCENDING)])):
assert doc["sequence"] == i
assert doc["doc"] == docs[i]
with pymongo.MongoClient() as c:
c.drop_database("monary_test")
def test_ssl_false(self):
with self.assertRaisesRegexp(monary.monary.MonaryError,
"Failed to read 4 bytes from socket."):
with monary.Monary("mongodb://localhost:27017/?ssl=false",
pem_file=global_client_pem) as m:
arrays = m.query("test", "ssl", {}, ["x1"], ["float64"])
assert len(arrays) == 1 and arrays[0] == 0.0
def test_oid(self):
with monary.Monary() as m:
# Insert documents to generate ObjectIds.
ids = m.insert(
"monary_test", "data",
monary.MonaryParam.from_lists([self.bool_arr, self.seq],
["dummy", "sequence"]))
assert len(ids) == ids.count() == NUM_TEST_RECORDS
# Increment the sequence so sorting still works
seq2 = self.seq + NUM_TEST_RECORDS
ids2 = m.insert(
"monary_test", "data",
monary.MonaryParam.from_lists([ids, seq2], ["oid", "sequence"],
["id", "int64"]))
assert len(ids2) == ids.count() == NUM_TEST_RECORDS
# Get back the ids from the original insert (_id) and the ids that
# were manually inserted (oid)
retrieved = m.query("monary_test",
"data", {}, ["_id", "oid"], ["id", "id"],
sort="sequence")
# These should be equal to ``ids`` from the top of this test.
expected = retrieved[0][:NUM_TEST_RECORDS]
# This is what we get back when querying for the ObjectIds
# that were inserted manually above.
data = retrieved[1][NUM_TEST_RECORDS:]
assert len(expected) == len(data)
assert len(expected) == expected.count()
assert len(data) == data.count()
for d, e in zip(data, expected):
assert monary.mvoid_to_bson_id(d) == monary.mvoid_to_bson_id(e)
with pymongo.MongoClient() as c:
c.drop_database("monary_test")
def test_with_authenticate(self):
with monary.Monary(host="127.0.0.1",
database="admin",
username="******",
password="******") as m:
col, = m.query("admin", "junk", {}, ["route"], ["int32"])
assert col[0] == 66, "test value could not be retrieved"
def test_validate_server_cert(self):
with monary.Monary("mongodb://localhost:27017/?ssl=true",
pem_file=global_client_pem,
ca_file=global_ca_pem,
weak_cert_validation=False) as m:
arrays = m.query("test", "ssl", {}, ["x1"], ["float64"])
assert len(arrays) == 1 and arrays[0] == 0.0
def test_monary_query_bson(self):
# Test should fail in monary_init_query.
with self.assertRaisesRegexp(
bson.InvalidDocument,
"documents must have only string keys, key was 0"):
with monary.Monary() as m:
m.query("test", "collection", {0: 0},
["x1", "x2", "x3", "x4", "x5"], ["float64"] * 5)
def test_monary_connect2(self):
with self.assertRaisesRegexp(
monary.monary.MonaryError,
re.compile("(Failed to authenticate credentials|Authentication"
" failed)")):
with monary.Monary(username='******') as m:
m.query("test", "collection", {},
["x1", "x2", "x3", "x4", "x5"], ["float64"] * 5)
def test_monary_aggregate2(self):
# Test should fail in monary_load_query.
with self.assertRaisesRegexp(
monary.monary.MonaryError,
"exception: Unrecognized pipeline stage name:"):
with monary.Monary() as m:
m.aggregate("test", "collection", {"hi": "you"},
['x1', 'x2', 'x3', 'x4', 'x5'], ["float64"] * 5)
def test_bad_uri_no_ssl(self):
with monary.Monary("mongodb://localhost:27017",
pem_file=global_client_pem,
ca_file=global_ca_pem,
ca_dir=global_cert_path,
weak_cert_validation=True) as m:
arrays = m.query("test", "ssl", {}, ["x1"], ["float64"])
assert len(arrays) == 1 and arrays[0] == 0.0
def test_bad_authenticate(self):
with self.assertRaisesRegexp(monary.monary.MonaryError,
"Failed to authenticate credentials"):
with monary.Monary(host="127.0.0.1",
database="admin",
username="******",
password="******") as m:
m.count("admin", "junk", {})
def test_with_authenticate():
monary_connection = monary.Monary(host="127.0.0.1",
database="monary_auth_test",
username="******",
password="******")
col, = monary_connection.query("monary_auth_test", "junk", {}, ["route"],
["int32"])
assert col[0] == 66, "test value could not be retrieved"
def test_monary_aggregate(self):
# Test should fail in monary_load_query.
with self.assertRaisesRegexp(
monary.monary.MonaryError,
"A pipeline stage specification object must contain exactly "
"one field"):
with monary.Monary() as m:
m.aggregate("test", "collection", {},
['x1', 'x2', 'x3', 'x4', 'x5'], ["float64"] * 5)
def monary_func(use_large):
# Monary doesn't allow > 1024 keys, and it's too slow to benchmark anyway.
if use_large:
return
m = monary.Monary()
dtype = dtypes[use_large]
m.query(db.name, collection_names[use_large], {}, dtype.names,
["float64"] * len(dtype.names))
def collection_count(hoodlist):
with monary.Monary() as m:
for n in hoodlist.keys():
num_drops = m.count("taxi", "drop",
loc_query("pickup_loc", hoodlist[n]))
num_pickups = m.count("taxi", "pickup",
loc_query("drop_loc", hoodlist[n]))
print str(hoodlist[n]["boroughCode"]) + ", " + n + ', ' + \
str(num_drops) + ', ' + str(num_pickups)
def get_monary_column(self, colname, coltype):
with monary.Monary("127.0.0.1") as m:
[column] = m.query("monary_test",
"test_data",
{},
[colname],
[coltype],
sort="sequence")
return list(column)
def test_bad_authenticate():
try:
monary_connection = monary.Monary(host="127.0.0.1",
database="monary_auth_test",
username="******",
password="******")
monary_connection.count("monary_auth_test", "junk", {})
assert False, "authentication should not have succeeded with wrong password"
except RuntimeError:
pass # auth should have failed
def aggregate_monary_column(self, colname, coltype, pipeline, **kwargs):
with monary.Monary("127.0.0.1") as m:
result = None
for block, in m.block_aggregate("monary_test", "data", pipeline,
[colname], [coltype], **kwargs):
if result is None:
result = block
else:
result += block
return result
def test_bad_uri(self):
with self.assertRaisesRegexp(monary.monary.MonaryError,
"Failed to read 4 bytes from socket."):
with monary.Monary("mongodb://localhost:27017",
pem_file=global_client_pem,
ca_file=global_ca_pem,
ca_dir=global_cert_path,
weak_cert_validation=True) as m:
arrays = m.query("test", "ssl", {}, ["x1"], ["float64"])
assert len(arrays) == 1 and arrays[0] == 0.0
def get_timeVfreq(metric, collection, field):
metric_query = "$" + metric
id_field = "_id." + metric
field_query = "$" + field
with monary.Monary() as m:
agg_pipe = [{"$group": {
"_id": {metric: {metric_query: field_query}},
"count": {"$sum": 1}}},
{"$sort": {"_id.hour": 1}}]
drop_array = m.aggregate("taxi", collection, agg_pipe,
[id_field, "count"], ["int32", "int32"])
return drop_array
def count_borough(hoodlist, borough_code):
with monary.Monary() as m:
total_drops = 0
total_pickups = 0
for n in hoodlist.keys():
if hoodlist[n]["boroughCode"] == borough_code:
num_drops = m.count("taxi", "drop",
loc_query("pickup_loc", hoodlist[n]))
num_pickups = m.count("taxi", "pickup",
loc_query("drop_loc", hoodlist[n]))
total_drops += num_drops
total_pickups += num_pickups
print str(hoodlist[n]["boroughCode"]) + ", " + n + ', ' + \
str(num_drops) + ', ' + str(num_pickups)
def test_utf8():
with monary.Monary("127.0.0.1") as m:
data, lens, sizes = m.query("monary_test",
"data", {}, ["test", "test", "test"],
["string:12", "length", "size"],
sort="sequence")
assert (lens < sizes).all()
expected = ["aあ", "âéÇ", "αλΩ", "眥¨≠"]
for x, l, y, in zip(data, lens, expected):
if sys.version_info[0] >= 3:
# Python 3
x = x.decode('utf8')
assert x == y
assert l == len(y)
def test_reconnection(self):
with monary.Monary(host="127.0.0.1",
database="admin",
username="******",
password="******") as m:
with self.assertRaisesRegexp(
monary.monary.MonaryError, "Failed to authenticate"
" credentials"):
m.count("admin", "junk", {})
m.connect(host="127.0.0.1",
database="admin",
username="******",
password="******")
col, = m.query("admin", "junk", {}, ["route"], ["int32"])
assert col[0] == 66, "test value could not be retrieved"
def test_reconnection():
try:
monary_connection = monary.Monary(host="127.0.0.1",
database="monary_auth_test",
username="******",
password="******")
monary_connection.count("monary_auth_test", "junk", {})
assert False, "authentication should not have succeeded with wrong password"
except RuntimeError:
pass # auth should have failed
monary_connection.connect(host="127.0.0.1",
database="monary_auth_test",
username="******",
password="******")
[col] = monary_connection.query("monary_auth_test", "junk", {}, ["route"],
["int32"])
assert col[0] == 66, "test value could not be retrieved"
def test_utf8(self):
with monary.Monary("127.0.0.1") as m:
data, lens, sizes = m.query("monary_test",
"data", {}, ["test", "test", "test"],
["string:12", "length", "size"],
sort="sequence")
assert (lens < sizes).all()
for monary_bytes, monary_len, expected_str, in zip(
data, lens, self.expected):
monary_str = monary_bytes.decode('utf8')
# We got the same string out from Monary as we inserted w/ PyMongo.
assert monary_str == expected_str
# Monary's idea of "length" == len(string).
assert monary_len == len(expected_str)
def main(argv):
options, start, end = parse_options(argv)
td = end - start
total_hours = td.days * 24 + td.seconds / 3600
print '%d hours' % total_hours
if options.profile:
yappi.start(builtins=True)
if options.movie:
if os.path.exists('tmp'):
shutil.rmtree('tmp')
os.mkdir('tmp')
# Make orthographic basemap.
m = Basemap(resolution='c', projection='cyl', lat_0=60., lon_0=-60.)
monary_connection = monary.Monary(port=options.port)
dt = start
n_hours = 10
while dt <= end:
loop(options, monary_connection, m, dt, n_hours)
dt += datetime.timedelta(hours=n_hours)
if not options.movie:
break
monary_connection.close()
if options.movie:
if os.path.exists('out.mp4'):
os.remove('out.mp4')
make_movie('tmp/*.png', 'out.mp4')
os.system('open out.mp4')
if options.profile:
yappi.stop()
stats = yappi.get_func_stats()
stats.save('callgrind.out', type='callgrind')