def test_field_subset(self):
adapter = mongoadapter.MongoAdapter(TestMongoAdapter.hostname,
TestMongoAdapter.port,
TestMongoAdapter.db_name,
'test_uints')
array = adapter['f0'][:]
array.sort()
self.assertEqual(array.size, nrecords)
self.assertEqual(array.dtype, numpy.dtype([('f0', 'u8')]))
for i in range(nrecords):
self.assertEqual(array[i].item(), (i * 5, ))
array = adapter[['f0', 'f4']][:]
array.sort()
self.assertEqual(array.size, nrecords)
self.assertEqual(array.dtype, numpy.dtype([('f0', 'u8'),
('f4', 'u8')]))
for i in range(nrecords):
self.assertEqual(array[i].item(), (i * 5, i * 5 + 4))
array = adapter[('f0', 'f1', 'f2', 'f3', 'f4')][:]
array.sort()
self.assertEqual(array.size, nrecords)
self.assertEqual(array.dtype, numpy.dtype('u8,u8,u8,u8,u8'))
for i in range(nrecords):
self.assertEqual(
array[i].item(),
(i * 5, i * 5 + 1, i * 5 + 2, i * 5 + 3, i * 5 + 4))
def test_int_list(self):
adapter = mongoadapter.MongoAdapter(TestMongoAdapter.hostname,
TestMongoAdapter.port,
TestMongoAdapter.db_name,
'test_ints')
array = adapter['f0'][[0, 1, 2]]
self.assertEqual(array.size, 3)
def test_cast_to_uint_from_int(self):
adapter = mongoadapter.MongoAdapter(TestMongoAdapter.hostname,
TestMongoAdapter.port,
TestMongoAdapter.db_name,
'test_ints')
arr1 = adapter[['f0', 'f1', 'f2', 'f3', 'f4']][:]
adapter.set_field_types({
'f0': 'u8',
'f1': 'u8',
'f2': 'u8',
'f3': 'u8',
'f4': 'u8'
})
self.assertTrue(adapter.is_field_type_set('f1'))
for i in range(5):
self.assertTrue(adapter.is_field_type_set(i))
array = adapter[:]
self.assertEqual(array.size, nrecords)
self.assertEqual(array.dtype, numpy.dtype('u8,u8,u8,u8,u8'))
sorted = numpy.sort(array)
for i in range(nrecords):
self.assertEqual(sorted[-i].item(), ((i*-5+0) & 0xffffffffffffffff, \
(i*-5+1) & 0xffffffffffffffff, \
(i*-5+2) & 0xffffffffffffffff, \
(i*-5+3) & 0xffffffffffffffff, \
(i*-5+4) & 0xffffffffffffffff))
def test_slice_no_fields(self):
adapter = mongoadapter.MongoAdapter(TestMongoAdapter.hostname,
TestMongoAdapter.port,
TestMongoAdapter.db_name,
'test_ints')
array = adapter[:]
self.assertEqual(array.size, nrecords)
def test_further_slicing(self):
adapter = mongoadapter.MongoAdapter(TestMongoAdapter.hostname,
TestMongoAdapter.port,
TestMongoAdapter.db_name,
'test_ints')
array_list_all = adapter[['f0', 'f1', 'f2', 'f3', 'f4']][:]
self.assertEqual(array_list_all.size, nrecords)
array_str_1011 = adapter['f0'][10:11]
self.assertEqual(array_str_1011.size, 1)
array_str_int = adapter['f0'][10]
self.assertEqual(array_str_int.size, 1)
self.assertEqual(array_str_int, array_str_1011)
array_list_10 = adapter[['f0']][10]
self.assertEqual(array_str_int, array_list_10)
array_list_515 = adapter[['f0']][5:15]
self.assertEqual(array_list_515.size, 10)
self.assertEqual(array_list_515[5:6], array_str_int)
for i in range(5):
self.assertEqual(array_list_515[5 + i][0],
array_list_all[10 + i][0])
self.assertTrue(numpy.allclose(array_list_515['f0'][5:10], \
array_list_all['f0'][10:15]))
def test_behavior_of_floats_that_hold_uints(self):
adapter = mongoadapter.MongoAdapter(
TestMongoAdapter.hostname, TestMongoAdapter.port,
TestMongoAdapter.db_name,
'test_behavior_of_floats_that_hold_uints')
array = adapter[['f0', 'f1', 'f2', 'f3', 'f4']][:]
self.assertEqual(array.size, nrecords)
self.assertEqual(array.dtype, numpy.dtype('u8,u8,u8,u8,u8'))
sorted = numpy.sort(array)
for i in range(nrecords):
self.assertEqual(
sorted[i].item(),
(i * 5, i * 5 + 1, i * 5 + 2, i * 5 + 3, i * 5 + 4))
def test_floats(self):
adapter = mongoadapter.MongoAdapter(TestMongoAdapter.hostname,
TestMongoAdapter.port,
TestMongoAdapter.db_name,
'test_floats')
array = adapter[['f0', 'f1', 'f2', 'f3', 'f4']][:]
self.assertEqual(array.size, nrecords)
self.assertEqual(array.dtype, numpy.dtype('f8,f8,f8,f8,f8'))
sorted = numpy.sort(array)
for i in range(nrecords):
self.assertEqual(sorted[i].item(),
(i * 5 + 0.1, i * 5 + 0.1 + 1, i * 5 + 0.1 + 2,
i * 5 + 0.1 + 3, i * 5 + 0.1 + 4))
def test_ints(self):
adapter = mongoadapter.MongoAdapter(TestMongoAdapter.hostname,
TestMongoAdapter.port,
TestMongoAdapter.db_name,
'test_ints')
array = adapter[['f0', 'f1', 'f2', 'f3', 'f4']][:]
self.assertEqual(array.size, nrecords)
self.assertEqual(array.dtype, numpy.dtype('i8,i8,i8,i8,i8'))
sorted = numpy.sort(array)
for i in range(nrecords):
self.assertEqual(
sorted[nrecords - 1 - i].item(),
(i * -5, i * -5 + 1, i * -5 + 2, i * -5 + 3, i * -5 + 4))
def test_type_inference(self):
adapter = mongoadapter.MongoAdapter(TestMongoAdapter.hostname,
TestMongoAdapter.port,
TestMongoAdapter.db_name,
'test_type_inference')
array = adapter[['f0', 'f1', 'f2', 'f3', 'f4']][:]
self.assertEqual(array.size, nrecords)
self.assertEqual(array.dtype, numpy.dtype('f8,O,u8,u8,u8'))
sorted = numpy.sort(array)
for i in range(nrecords - 1):
self.assertEqual(sorted[i].item(), (float(
i * 5), str(i * 5 + 1), i * 5 + 2, i * 5 + 3, i * 5 + 4))
self.assertEqual(sorted[-1].item(),
(float(4995.5), '4996', 4997, 4998, 4999))
def test_set_field_types(self):
adapter = mongoadapter.MongoAdapter(TestMongoAdapter.hostname,
TestMongoAdapter.port,
TestMongoAdapter.db_name,
'test_ints')
adapter[['f0', 'f1', 'f2', 'f3', 'f4']]
adapter.set_field_types({'f0': 'f8', 'f4': 'O'})
array = adapter[:]
self.assertEqual(array.size, nrecords)
self.assertEqual(array.dtype, numpy.dtype('f8,i8,i8,i8,O'))
sorted = numpy.sort(array)
for i in range(nrecords):
self.assertEqual(sorted[nrecords - 1 - i].item(), (float(
i * -5), i * -5 + 1, i * -5 + 2, i * -5 + 3, str(i * -5 + 4)))
def test_slice_overflow(self):
"""
Test using a value for slice end that is larger than max uint32 size.
Expected behavior is to round slice end down to max uint32 value
(effectively reading as many records as possible).
"""
adapter = mongoadapter.MongoAdapter(TestMongoAdapter.hostname,
TestMongoAdapter.port,
TestMongoAdapter.db_name,
'test_uints')
adapter.set_field_names(['f0', 'f1', 'f2', 'f3', 'f4'])
array = adapter[0:999999999999999999]
self.assertEqual(array.size, nrecords)
self.assertEqual(array.dtype, numpy.dtype('u8,u8,u8,u8,u8'))
sorted = numpy.sort(array)
for i in range(nrecords):
self.assertEqual(
sorted[i].item(),
(i * 5, i * 5 + 1, i * 5 + 2, i * 5 + 3, i * 5 + 4))
def test_cast_to_int_from_float(self):
adapter = mongoadapter.MongoAdapter(TestMongoAdapter.hostname,
TestMongoAdapter.port,
TestMongoAdapter.db_name,
'test_floats')
adapter[['f0', 'f1', 'f2', 'f3', 'f4']]
adapter.set_field_types({
'f0': 'i8',
'f1': 'i8',
'f2': 'i8',
'f3': 'i8',
'f4': 'i8'
})
array = adapter[:]
self.assertEqual(array.size, nrecords)
self.assertEqual(array.dtype, numpy.dtype('i8,i8,i8,i8,i8'))
sorted = numpy.sort(array)
for i in range(nrecords):
self.assertEqual(
sorted[i].item(),
(i * 5, i * 5 + 1, i * 5 + 2, i * 5 + 3, i * 5 + 4))
def test_array_slicing(self):
adapter = mongoadapter.MongoAdapter(TestMongoAdapter.hostname,
TestMongoAdapter.port,
TestMongoAdapter.db_name,
'test_uints')
control = adapter[['f0', 'f1', 'f2', 'f3', 'f4']][:]
array = adapter[['f0', 'f1', 'f2', 'f3', 'f4']][0:nrecords]
self.assertEqual(array.size, nrecords)
self.assertEqual(array.dtype, numpy.dtype('u8,u8,u8,u8,u8'))
for i in range(nrecords):
self.assertEqual(array[i].item(), control[i].item())
array = adapter[['f0', 'f1', 'f2', 'f3', 'f4']][-1]
self.assertEqual(array.size, 1)
self.assertEqual(array.dtype, numpy.dtype('u8,u8,u8,u8,u8'))
self.assertEqual(array[0].item(), control[-1].item())
array = adapter[['f0', 'f1', 'f2', 'f3', 'f4']][::2]
self.assertEqual(array.size, nrecords // 2)
self.assertEqual(array.dtype, numpy.dtype('u8,u8,u8,u8,u8'))
for i in range(0, nrecords // 2, 2):
self.assertEqual(array[i].item(), control[i * 2].item())
def test_missing_fill_values(self):
adapter = mongoadapter.MongoAdapter(TestMongoAdapter.hostname,
TestMongoAdapter.port,
TestMongoAdapter.db_name,
'test_missing_ints',
infer_types=False)
# Force f1 type to integer
adapter.set_field_types({'f1': 'u8'})
adapter.set_missing_values({0: ['NA']})
adapter.set_fill_values({0: 99, 1: 999, 4: 9999}, loose=True)
array = adapter[['f0', 'f1', 'f2', 'f3', 'f4']][:]
self.assertEqual(array.size, nrecords)
self.assertEqual(array.dtype, numpy.dtype('u8,u8,u8,u8,u8'))
# Records 0,5,10... contains 'NA' and 'NaN' values in f0
# that should convert to 99.
# Records 1,6,11... contains values in f1 that can't be converted to int;
# these should convert to 999.
# Records 4,9,14... contains missing values in f4
# that should convert to 9999.
for i in range(nrecords):
if i % 5 == 0:
self.assertEqual(
array[i], (99, i * 5 + 1, i * 5 + 2, i * 5 + 3, i * 5 + 4))
elif i % 5 == 1:
self.assertEqual(array[i],
(i * 5, 999, i * 5 + 2, i * 5 + 3, i * 5 + 4))
elif i % 5 == 4:
self.assertEqual(
array[i], (i * 5, i * 5 + 1, i * 5 + 2, i * 5 + 3, 9999))
else:
self.assertEqual(
array[i],
(i * 5, i * 5 + 1, i * 5 + 2, i * 5 + 3, i * 5 + 4))
def run_adapter_vs_pymongo(self, col_name, cast=False):
adapter = mongoadapter.MongoAdapter(TestMongoAdapter.hostname,
TestMongoAdapter.port,
TestMongoAdapter.db_name, col_name)
array = adapter[['f0', 'f1', 'f2', 'f3', 'f4']][:]
try:
mongo_conn = pymongo.MongoClient(TestMongoAdapter.hostname,
TestMongoAdapter.port)
except:
warnings.warn(
'Could not run Mongo tests: could not connect to Mongo database.'
)
self.assertTrue(True)
return
mongo_db = mongo_conn['MongoAdapter_tests']
col = mongo_db[col_name]
i = 0
for rec in col.find():
self.check_field('f0', array, i, rec, cast)
self.check_field('f1', array, i, rec, cast)
self.check_field('f2', array, i, rec, cast)
self.check_field('f3', array, i, rec, cast)
self.check_field('f4', array, i, rec, cast)
i += 1
db_name = 'MongoAdapter_tests'
colxn_name = 'test_floats'
nrecords = 5
try:
mongo_conn = pymongo.MongoClient(hostname, port)
except:
raise Exception("Could not connect to Mongo database. You must have a mongo demon running on port 27017 for this to work. Have you run '$ mongod'?")
mongo_db = mongo_conn[db_name]
col = mongo_db[colxn_name]
if col.count() > 0:
col.remove()
for i in range(nrecords):
record = {'f0': float(i * 5 + 0.1),
'f1': float(i * 5 + 1 + 0.1),
'f2': float(i * 5 + 2 + 0.1),
'f3': float(i * 5 + 3 + 0.1),
'f4': float(i * 5 + 4 + 0.1)}
col.insert(record)
# test out MongoAdapter
adapter = mongoadapter.MongoAdapter(hostname, port, db_name, colxn_name)
adapter[['f0', 'f1', 'f2', 'f3', 'f4']] # this is equivalent to set_field_names(['f0', 'f1', 'f2', 'f3', 'f4'])
print('get_field_names(): ' + repr(adapter.get_field_names()))
adapter.set_field_types({'f1':'i8'})
print("get_field_types() = " + repr(adapter.get_field_types()))
print("adapter[:] = \n" + repr(adapter[:]))
