def test_params_data_type_conversion(self):
self.confs['test'].post(**{'field.data': 'number', 'accelerated_fields.data': '{"data": -1}'})
for x in range(50):
self.col.insert(json.dumps({'_key': str(x), 'data': str(x), 'ignore': x}))
data = self.col.query(sort='data:-1', limit=20, fields='data,_id:0', skip=10)
self.assertEqual(len(data), 20)
for x in range(20):
self.assertEqual(data[x]['data'], 39 - x)
self.assertTrue(not 'ignore' in data[x])
self.assertTrue(not '_key' in data[x])
def test_list_with_offset(self):
import random
for offset in [random.randint(3,50) for x in range(5)]:
for coll_name in collections:
coll = getattr(self.service, coll_name)
expected = [ent.name for ent in coll.list(count=offset+10)][offset:]
found = [ent.name for ent in coll.list(offset=offset, count=10)]
self.assertEqual(expected, found,
msg='on %s (expected %s, found %s)' % \
(coll_name, expected, found))
def test_insert_query_delete_data(self):
for x in range(50):
self.col.insert(json.dumps({'_key': str(x), 'data': '#' + str(x), 'num': x}))
self.assertEqual(len(self.col.query()), 50)
self.assertEqual(len(self.col.query(query='{"num": 10}')), 1)
self.assertEqual(self.col.query(query='{"num": 10}')[0]['data'], '#10')
self.col.delete(json.dumps({'num': {'$gt': 39}}))
self.assertEqual(len(self.col.query()), 40)
self.col.delete()
self.assertEqual(len(self.col.query()), 0)
def _run(self):
writer = RecordWriterV2(self.recorder.output, maxresultrows=10)
write_record = writer.write_record
names = self.fieldnames
for self._serial_number in range(0, 31):
record = OrderedDict(izip(names, self.row))
write_record(record)
return
def test_update_delete_data(self):
for x in range(50):
self.col.insert(json.dumps({'_key': str(x), 'data': '#' + str(x), 'num': x}))
self.assertEqual(len(self.col.query()), 50)
self.assertEqual(self.col.query(query='{"num": 49}')[0]['data'], '#49')
self.col.update(str(49), json.dumps({'data': '#50', 'num': 50}))
self.assertEqual(len(self.col.query()), 50)
self.assertEqual(self.col.query(query='{"num": 50}')[0]['data'], '#50')
self.assertEqual(len(self.col.query(query='{"num": 49}')), 0)
self.col.delete_by_id(49)
self.assertEqual(len(self.col.query(query='{"num": 50}')), 0)
def test_insert_find_update_data(self):
data = [{'_key': str(x), 'data': '#' + str(x), 'num': x} for x in range(1000)]
self.col.batch_save(*data)
testData = self.col.query(sort='num')
self.assertEqual(len(testData), 1000)
for x in range(1000):
self.assertEqual(testData[x]['_key'], str(x))
self.assertEqual(testData[x]['data'], '#' + str(x))
self.assertEqual(testData[x]['num'], x)
data = [{'_key': str(x), 'data': '#' + str(x + 1), 'num': x + 1} for x in range(1000)]
self.col.batch_save(*data)
testData = self.col.query(sort='num')
self.assertEqual(len(testData), 1000)
for x in range(1000):
self.assertEqual(testData[x]['_key'], str(x))
self.assertEqual(testData[x]['data'], '#' + str(x + 1))
self.assertEqual(testData[x]['num'], x + 1)
query = [{"query": {"num": x + 1}} for x in range(100)]
testData = self.col.batch_find(*query)
self.assertEqual(len(testData), 100)
testData.sort(key=lambda x: x[0]['num'])
for x in range(100):
self.assertEqual(testData[x][0]['_key'], str(x))
self.assertEqual(testData[x][0]['data'], '#' + str(x + 1))
self.assertEqual(testData[x][0]['num'], x + 1)
def test_query_data(self):
if ('test1' in self.confs):
self.confs['test1'].delete()
self.confs.create('test1')
self.col = self.confs['test1'].data
for x in range(10):
self.col.insert(json.dumps({'_key': str(x), 'data': '#' + str(x), 'num': x}))
data = self.col.query(sort='data:-1', skip=9)
self.assertEqual(len(data), 1)
self.assertEqual(data[0]['data'], '#0')
data = self.col.query(sort='data:1')
self.assertEqual(data[0]['data'], '#0')
data = self.col.query(limit=2, skip=9)
self.assertEqual(len(data), 1)
def test_command_line_parser_unquote(self):
parser = CommandLineParser
options = [
r'foo', # unquoted string with no escaped characters
r'fo\o\ b\"a\\r', # unquoted string with some escaped characters
r'"foo"', # quoted string with no special characters
r'"""foobar1"""', # quoted string with quotes escaped like this: ""
r'"\"foobar2\""', # quoted string with quotes escaped like this: \"
r'"foo ""x"" bar"', # quoted string with quotes escaped like this: ""
r'"foo \"x\" bar"', # quoted string with quotes escaped like this: \"
r'"\\foobar"', # quoted string with an escaped backslash
r'"foo \\ bar"', # quoted string with an escaped backslash
r'"foobar\\"', # quoted string with an escaped backslash
r'foo\\\bar', # quoted string with an escaped backslash and an escaped 'b'
r'""', # pair of quotes
r''] # empty string
expected = [
r'foo',
r'foo b"a\r',
r'foo',
r'"foobar1"',
r'"foobar2"',
r'foo "x" bar',
r'foo "x" bar',
'\\foobar',
r'foo \ bar',
'foobar\\',
r'foo\bar',
r'',
r'']
# Command line with an assortment of string values
self.assertEqual(expected[-4], parser.unquote(options[-4]))
for i in range(0, len(options)):
self.assertEqual(expected[i], parser.unquote(options[i]))
self.assertRaises(SyntaxError, parser.unquote, '"')
self.assertRaises(SyntaxError, parser.unquote, '"foo')
self.assertRaises(SyntaxError, parser.unquote, 'foo"')
self.assertRaises(SyntaxError, parser.unquote, 'foo\\')
def test_duration(self):
# Duration validator should parse and format time intervals of the form
# HH:MM:SS
validator = validators.Duration()
for seconds in range(0, 25 * 60 * 60, 59):
for value in six.text_type(seconds), bytes(seconds):
self.assertEqual(validator(value), seconds)
self.assertEqual(validator(validator.format(seconds)), seconds)
value = '%d:%02d' % (seconds / 60, seconds % 60)
self.assertEqual(validator(value), seconds)
self.assertEqual(validator(validator.format(seconds)), seconds)
value = '%d:%02d:%02d' % (seconds / 3600, (seconds / 60) % 60, seconds % 60)
self.assertEqual(validator(value), seconds)
self.assertEqual(validator(validator.format(seconds)), seconds)
self.assertEqual(validator('230:00:00'), 230 * 60 * 60)
self.assertEqual(validator('23:00:00'), 23 * 60 * 60)
self.assertEqual(validator('00:59:00'), 59 * 60)
self.assertEqual(validator('00:00:59'), 59)
self.assertEqual(validator.format(230 * 60 * 60), '230:00:00')
self.assertEqual(validator.format(23 * 60 * 60), '23:00:00')
self.assertEqual(validator.format(59 * 60), '00:59:00')
self.assertEqual(validator.format(59), '00:00:59')
self.assertRaises(ValueError, validator, '-1')
self.assertRaises(ValueError, validator, '00:-1')
self.assertRaises(ValueError, validator, '-1:00')
self.assertRaises(ValueError, validator, '00:00:-1')
self.assertRaises(ValueError, validator, '00:-1:00')
self.assertRaises(ValueError, validator, '-1:00:00')
self.assertRaises(ValueError, validator, '00:00:60')
self.assertRaises(ValueError, validator, '00:60:00')
return
def test_insert_find_update_data(self):
data = [{
'_key': str(x),
'data': '#' + str(x),
'num': x
} for x in range(1000)]
self.col.batch_save(*data)
testData = self.col.query(sort='num')
self.assertEqual(len(testData), 1000)
for x in range(1000):
self.assertEqual(testData[x]['_key'], str(x))
self.assertEqual(testData[x]['data'], '#' + str(x))
self.assertEqual(testData[x]['num'], x)
data = [{
'_key': str(x),
'data': '#' + str(x + 1),
'num': x + 1
} for x in range(1000)]
self.col.batch_save(*data)
testData = self.col.query(sort='num')
self.assertEqual(len(testData), 1000)
for x in range(1000):
self.assertEqual(testData[x]['_key'], str(x))
self.assertEqual(testData[x]['data'], '#' + str(x + 1))
self.assertEqual(testData[x]['num'], x + 1)
query = [{"query": {"num": x + 1}} for x in range(100)]
testData = self.col.batch_find(*query)
self.assertEqual(len(testData), 100)
testData.sort(key=lambda x: x[0]['num'])
for x in range(100):
self.assertEqual(testData[x][0]['_key'], str(x))
self.assertEqual(testData[x][0]['data'], '#' + str(x + 1))
self.assertEqual(testData[x][0]['num'], x + 1)
def test_integer(self):
# Point of interest:
#
# On all *nix operating systems an int is 32-bits long on 32-bit systems and 64-bits long on 64-bit systems so
# that you can count on this equality:
#
# sys.maxint == sys.maxsize
#
# On Windows an int is always 32-bits long and you cannot count on the same equality. Specifically, on 64-bit
# systems:
#
# sys.maxint != sys.maxsize
maxsize = sys.maxsize
minsize = -(sys.maxsize - 1)
# The Integer validator should convert values in the range of a Python long which has unlimited precision
# Anecdotal evidence: This portion of the test checks 5-10 K integer values and runs for less than 2-3 seconds
validator = validators.Integer()
def test(integer):
for s in str(integer), six.text_type(integer):
value = validator.__call__(s)
self.assertEqual(value, integer)
if six.PY2:
self.assertIsInstance(value, long)
else:
self.assertIsInstance(value, int)
self.assertEqual(validator.format(integer), six.text_type(integer))
test(2 * minsize)
test(minsize)
test(-1)
test(0)
test(1)
test(2 * maxsize)
for i in range(0, 10000):
test(randint(minsize, maxsize))
# The Integer validator can impose a range restriction
validator = validators.Integer(minimum=0)
self.assertEqual(validator.__call__(0), 0)
self.assertEqual(validator.__call__(2 * maxsize), 2 * maxsize)
self.assertRaises(ValueError, validator.__call__, -1)
validator = validators.Integer(minimum=1, maximum=maxsize)
self.assertEqual(validator.__call__(1), 1)
self.assertEqual(validator.__call__(maxsize), maxsize)
self.assertRaises(ValueError, validator.__call__, 0)
self.assertRaises(ValueError, validator.__call__, maxsize + 1)
validator = validators.Integer(minimum=minsize, maximum=maxsize)
self.assertEqual(validator.__call__(minsize), minsize)
self.assertEqual(validator.__call__(0), 0)
self.assertEqual(validator.__call__(maxsize), maxsize)
self.assertRaises(ValueError, validator.__call__, minsize - 1)
self.assertRaises(ValueError, validator.__call__, maxsize + 1)
return
def test_record_writer_with_random_data(self, save_recording=False):
# Confirmed: [minint, maxint) covers the full range of values that xrange allows
# RecordWriter writes apps in units of maxresultrows records. Default: 50,0000.
# Partial results are written when the record count reaches maxresultrows.
writer = RecordWriterV2(
StringIO(),
maxresultrows=10) # small for the purposes of this unit test
test_data = OrderedDict()
fieldnames = [
'_serial', '_time', 'random_bytes', 'random_dict',
'random_integers', 'random_unicode'
]
test_data['fieldnames'] = fieldnames
test_data['values'] = []
write_record = writer.write_record
for serial_number in range(0, 31):
values = [
serial_number,
time(),
random_bytes(),
random_dict(),
random_integers(),
random_unicode()
]
record = OrderedDict(izip(fieldnames, values))
#try:
write_record(record)
#except Exception as error:
# self.fail(error)
test_data['values'].append(values)
# RecordWriter accumulates inspector messages and metrics until maxresultrows are written, a partial result
# is produced or we're finished
messages = [('debug', random_unicode()), ('error', random_unicode()),
('fatal', random_unicode()), ('info', random_unicode()),
('warn', random_unicode())]
test_data['messages'] = messages
for message_type, message_text in messages:
writer.write_message(message_type, '{}', message_text)
metrics = {
'metric-1': SearchMetric(1, 2, 3, 4),
'metric-2': SearchMetric(5, 6, 7, 8)
}
test_data['metrics'] = metrics
for name, metric in six.iteritems(metrics):
writer.write_metric(name, metric)
self.assertEqual(writer._chunk_count, 3)
self.assertEqual(writer._record_count, 1)
self.assertGreater(writer._buffer.tell(), 0)
self.assertEqual(writer._total_record_count, 30)
self.assertListEqual(writer._fieldnames, fieldnames)
self.assertListEqual(writer._inspector['messages'], messages)
self.assertDictEqual(
dict(
ifilter(lambda k_v: k_v[0].startswith('metric.'),
six.iteritems(writer._inspector))),
dict(
imap(lambda k_v1: ('metric.' + k_v1[0], k_v1[1]),
six.iteritems(metrics))))
writer.flush(finished=True)
self.assertEqual(writer._chunk_count, 4)
self.assertEqual(writer._record_count, 0)
self.assertEqual(writer._buffer.tell(), 0)
self.assertEqual(writer._buffer.getvalue(), '')
self.assertEqual(writer._total_record_count, 31)
self.assertRaises(AssertionError,
writer.flush,
finished=True,
partial=True)
self.assertRaises(AssertionError, writer.flush, finished='non-boolean')
self.assertRaises(AssertionError, writer.flush, partial='non-boolean')
self.assertRaises(AssertionError, writer.flush)
self.assertRaises(RuntimeError, writer.write_record, {})
self.assertFalse(writer._ofile.closed)
self.assertIsNone(writer._fieldnames)
self.assertDictEqual(writer._inspector, OrderedDict())
# P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to write partial results by calling
# RecordWriter.flush(partial=True).
# P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to finish early by calling
# RecordWriter.flush(finish=True).
if save_recording:
cls = self.__class__
method = cls.test_record_writer_with_recordings
base_path = os.path.join(
self._recordings_path, '.'.join(
(cls.__name__, method.__name__, six.text_type(time()))))
with gzip.open(base_path + '.input.gz', 'wb') as f:
pickle.dump(test_data, f)
with open(base_path + '.output', 'wb') as f:
f.write(writer._ofile.getvalue())
return
def random_unicode():
return ''.join(
imap(lambda x: six.unichr(x),
random.sample(range(maxunicode), random.randint(0, max_length))))
def test_integer(self):
# Point of interest:
#
# On all *nix operating systems an int is 32-bits long on 32-bit systems and 64-bits long on 64-bit systems so
# that you can count on this equality:
#
# sys.maxint == sys.maxsize
#
# On Windows an int is always 32-bits long and you cannot count on the same equality. Specifically, on 64-bit
# systems:
#
# sys.maxint != sys.maxsize
maxsize = sys.maxsize
minsize = -(sys.maxsize - 1)
# The Integer validator should convert values in the range of a Python long which has unlimited precision
# Anecdotal evidence: This portion of the test checks 5-10 K integer values and runs for less than 2-3 seconds
validator = validators.Integer()
def test(integer):
for s in str(integer), six.text_type(integer):
value = validator.__call__(s)
self.assertEqual(value, integer)
if six.PY2:
self.assertIsInstance(value, long)
else:
self.assertIsInstance(value, int)
self.assertEqual(validator.format(integer), six.text_type(integer))
test(2 * minsize)
test(minsize)
test(-1)
test(0)
test(1)
test(2 * maxsize)
for i in range(0, 10000):
test(randint(minsize, maxsize))
# The Integer validator can impose a range restriction
validator = validators.Integer(minimum=0)
self.assertEqual(validator.__call__(0), 0)
self.assertEqual(validator.__call__(2 * maxsize), 2 * maxsize)
self.assertRaises(ValueError, validator.__call__, -1)
validator = validators.Integer(minimum=1, maximum=maxsize)
self.assertEqual(validator.__call__(1), 1)
self.assertEqual(validator.__call__(maxsize), maxsize)
self.assertRaises(ValueError, validator.__call__, 0)
self.assertRaises(ValueError, validator.__call__, maxsize + 1)
validator = validators.Integer(minimum=minsize, maximum=maxsize)
self.assertEqual(validator.__call__(minsize), minsize)
self.assertEqual(validator.__call__(0), 0)
self.assertEqual(validator.__call__(maxsize), maxsize)
self.assertRaises(ValueError, validator.__call__, minsize - 1)
self.assertRaises(ValueError, validator.__call__, maxsize + 1)
return
def random_unicode():
return ''.join(
imap(
lambda x: six.unichr(x),
random.sample(range(MAX_NARROW_UNICODE),
random.randint(0, max_length))))
def random_unicode():
return ''.join(imap(lambda x: six.unichr(x), random.sample(range(maxunicode), random.randint(0, max_length))))
def test_record_writer_with_random_data(self, save_recording=False):
# Confirmed: [minint, maxint) covers the full range of values that xrange allows
# RecordWriter writes apps in units of maxresultrows records. Default: 50,0000.
# Partial results are written when the record count reaches maxresultrows.
writer = RecordWriterV2(StringIO(), maxresultrows=10) # small for the purposes of this unit test
test_data = OrderedDict()
fieldnames = ['_serial', '_time', 'random_bytes', 'random_dict', 'random_integers', 'random_unicode']
test_data['fieldnames'] = fieldnames
test_data['values'] = []
write_record = writer.write_record
for serial_number in range(0, 31):
values = [serial_number, time(), random_bytes(), random_dict(), random_integers(), random_unicode()]
record = OrderedDict(izip(fieldnames, values))
#try:
write_record(record)
#except Exception as error:
# self.fail(error)
test_data['values'].append(values)
# RecordWriter accumulates inspector messages and metrics until maxresultrows are written, a partial result
# is produced or we're finished
messages = [
('debug', random_unicode()),
('error', random_unicode()),
('fatal', random_unicode()),
('info', random_unicode()),
('warn', random_unicode())]
test_data['messages'] = messages
for message_type, message_text in messages:
writer.write_message(message_type, '{}', message_text)
metrics = {
'metric-1': SearchMetric(1, 2, 3, 4),
'metric-2': SearchMetric(5, 6, 7, 8)
}
test_data['metrics'] = metrics
for name, metric in six.iteritems(metrics):
writer.write_metric(name, metric)
self.assertEqual(writer._chunk_count, 3)
self.assertEqual(writer._record_count, 1)
self.assertGreater(writer._buffer.tell(), 0)
self.assertEqual(writer._total_record_count, 30)
self.assertListEqual(writer._fieldnames, fieldnames)
self.assertListEqual(writer._inspector['messages'], messages)
self.assertDictEqual(
dict(ifilter(lambda k_v: k_v[0].startswith('metric.'), six.iteritems(writer._inspector))),
dict(imap(lambda k_v1: ('metric.' + k_v1[0], k_v1[1]), six.iteritems(metrics))))
writer.flush(finished=True)
self.assertEqual(writer._chunk_count, 4)
self.assertEqual(writer._record_count, 0)
self.assertEqual(writer._buffer.tell(), 0)
self.assertEqual(writer._buffer.getvalue(), '')
self.assertEqual(writer._total_record_count, 31)
self.assertRaises(AssertionError, writer.flush, finished=True, partial=True)
self.assertRaises(AssertionError, writer.flush, finished='non-boolean')
self.assertRaises(AssertionError, writer.flush, partial='non-boolean')
self.assertRaises(AssertionError, writer.flush)
self.assertRaises(RuntimeError, writer.write_record, {})
self.assertFalse(writer._ofile.closed)
self.assertIsNone(writer._fieldnames)
self.assertDictEqual(writer._inspector, OrderedDict())
# P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to write partial results by calling
# RecordWriter.flush(partial=True).
# P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to finish early by calling
# RecordWriter.flush(finish=True).
if save_recording:
cls = self.__class__
method = cls.test_record_writer_with_recordings
base_path = os.path.join(self._recordings_path, '.'.join((cls.__name__, method.__name__, six.text_type(time()))))
with gzip.open(base_path + '.input.gz', 'wb') as f:
pickle.dump(test_data, f)
with open(base_path + '.output', 'wb') as f:
f.write(writer._ofile.getvalue())
return
def generate(self):
text = self.text
for i in range(1, self.count + 1):
yield {'_serial': i, '_time': time.time(), '_raw': six.text_type(i) + '. ' + text}
def generate(self):
self.logger.debug("Generating %s events" % self.count)
for i in range(1, self.count + 1):
text = 'Hello World %d' % i
yield {'_time': time.time(), 'event_no': i, '_raw': text}
def generate(self):
text = self.text
self.logger.debug("Generating %d events with text %s" % (self.count, self.text))
for i in range(1, self.count + 1):
yield {'_serial': i, '_time': time.time(), '_raw': six.text_type(i) + '. ' + text}
