def test_1m_pseudo(self):
# Compresses a pseudo-random suite. Still compresses very well.
random.seed(0)
array = sorted(random.randint(0, 1000000) for _ in xrange(1000000))
data = large.pack(array)
self.assertGreater(302000, len(data))
self.assertEqual(array, large.unpack(data))
def assertResults(self, expected, result):
self.assertEqual(['shards'], result.keys())
self.assertEqual(1, len(result['shards']))
self.assertTrue(result['shards'][0])
result = result['shards'][0].copy()
# These are not deterministic (or I'm too lazy to calculate the value).
if expected.get('performance_stats'):
self.assertLess(0, result['performance_stats'].pop('bot_overhead'))
self.assertLess(
0, result['performance_stats']['isolated_download'].pop(
'duration'))
self.assertLess(
0,
result['performance_stats']['isolated_upload'].pop('duration'))
for k in ('isolated_download', 'isolated_upload'):
for j in ('items_cold', 'items_hot'):
result['performance_stats'][k][j] = large.unpack(
base64.b64decode(result['performance_stats'][k].get(
j, '')))
bot_version = result.pop('bot_version')
self.assertTrue(bot_version)
self.assertLess(0, result.pop('costs_usd'))
self.assertTrue(result.pop('created_ts'))
self.assertTrue(result.pop('completed_ts'))
self.assertLess(0, result.pop('durations'))
self.assertTrue(result.pop('id'))
self.assertTrue(result.pop('modified_ts'))
self.assertTrue(result.pop('started_ts'))
self.assertEqual(expected, result)
return bot_version
def assertResults(self, expected, result):
self.assertEqual(['shards'], result.keys())
self.assertEqual(1, len(result['shards']))
self.assertTrue(result['shards'][0])
result = result['shards'][0].copy()
# These are not deterministic (or I'm too lazy to calculate the value).
if expected.get('performance_stats'):
self.assertLess(
0, result['performance_stats'].pop('bot_overhead'))
self.assertLess(
0, result['performance_stats']['isolated_download'].pop('duration'))
self.assertLess(
0, result['performance_stats']['isolated_upload'].pop('duration'))
for k in ('isolated_download', 'isolated_upload'):
for j in ('items_cold', 'items_hot'):
result['performance_stats'][k][j] = large.unpack(
base64.b64decode(result['performance_stats'][k].get(j, '')))
bot_version = result.pop('bot_version')
self.assertTrue(bot_version)
self.assertLess(0, result.pop('costs_usd'))
self.assertTrue(result.pop('created_ts'))
self.assertTrue(result.pop('completed_ts'))
self.assertLess(0, result.pop('durations'))
self.assertTrue(result.pop('id'))
self.assertTrue(result.pop('modified_ts'))
self.assertTrue(result.pop('started_ts'))
self.assertEqual(expected, result)
return bot_version
def test_1m_pseudo(self): # Compresses a pseudo-random suite. Still compresses very well. random.seed(0) array = sorted(random.randint(0, 1000000) for _ in xrange(1000000)) data = large.pack(array) self.assertGreater(302000, len(data)) self.assertEqual(array, large.unpack(data))
def load_isolated_stats(stats_json_path, key):
actual = json.loads(read_content(stats_json_path))
stats = actual['stats']['isolated'].get(key)
for key in ['items_cold', 'items_hot']:
if not stats[key]:
continue
stats[key] = large.unpack(base64.b64decode(stats[key]))
return stats
def assertResults(self, expected, result):
self.assertEqual(['shards'], result.keys())
self.assertEqual(1, len(result['shards']))
self.assertTrue(result['shards'][0])
result = result['shards'][0].copy()
self.assertFalse(result.get('abandoned_ts'))
# These are not deterministic (or I'm too lazy to calculate the value).
if expected.get('performance_stats'):
self.assertLess(0, result['performance_stats'].pop('bot_overhead'))
self.assertLess(
0, result['performance_stats']['isolated_download'].pop(
'duration'))
self.assertLess(
0,
result['performance_stats']['isolated_upload'].pop('duration'))
for k in ('isolated_download', 'isolated_upload'):
for j in ('items_cold', 'items_hot'):
result['performance_stats'][k][j] = large.unpack(
base64.b64decode(result['performance_stats'][k].get(
j, '')))
else:
perf_stats = result.pop('performance_stats', None)
if perf_stats:
# Ignore bot_overhead, everything else should be empty.
perf_stats.pop('bot_overhead', None)
self.assertFalse(perf_stats.pop('isolated_download', None))
self.assertFalse(perf_stats.pop('isolated_upload', None))
self.assertFalse(perf_stats)
bot_version = result.pop('bot_version')
self.assertTrue(bot_version)
if result[u'costs_usd'] is not None:
expected.pop(u'costs_usd', None)
self.assertLess(0, result.pop(u'costs_usd'))
if result[u'cost_saved_usd'] is not None:
expected.pop(u'cost_saved_usd', None)
self.assertLess(0, result.pop(u'cost_saved_usd'))
self.assertTrue(result.pop('created_ts'))
self.assertTrue(result.pop('completed_ts'))
self.assertLess(0, result.pop('durations'))
self.assertTrue(result.pop('id'))
self.assertTrue(result.pop('modified_ts'))
self.assertTrue(result.pop('started_ts'))
if getattr(expected.get('outputs'), 'match', None):
expected_outputs = expected.pop('outputs')
outputs = '\n'.join(result.pop('outputs'))
self.assertTrue(
expected_outputs.match(outputs),
'%s does not match %s' % (outputs, expected_outputs.pattern))
self.assertEqual(expected, result)
return bot_version
def check_final(kwargs):
self.assertLess(self.SHORT_TIME_OUT, kwargs['data'].pop('cost_usd'))
self.assertLess(self.SHORT_TIME_OUT, kwargs['data'].pop('duration'))
self.assertLess(0., kwargs['data'].pop('bot_overhead'))
self.assertLess(
0., kwargs['data']['isolated_stats']['download'].pop('duration'))
self.assertLess(
0., kwargs['data']['isolated_stats']['upload'].pop('duration'))
# Makes the diffing easier.
for k in ('download', 'upload'):
for j in ('items_cold', 'items_hot'):
kwargs['data']['isolated_stats'][k][j] = large.unpack(
base64.b64decode(kwargs['data']['isolated_stats'][k][j]))
# The command print the pid of this child and grand-child processes, each
# on its line.
output = base64.b64decode(kwargs['data'].pop('output', ''))
for line in output.splitlines():
try:
to_kill.append(int(line))
except ValueError:
pass
self.assertEqual(
{
'data': {
'exit_code': exit_code,
'hard_timeout': False,
'id': u'localhost',
'io_timeout': True,
'isolated_stats': {
u'download': {
u'initial_number_items': 0,
u'initial_size': 0,
u'items_cold': [144, 150, 285, 307],
u'items_hot': [],
},
u'upload': {
u'items_cold': [],
u'items_hot': [],
},
},
'output_chunk_start': 0,
'task_id': 23,
},
'follow_redirects': False,
'timeout': 180,
'headers': {'Cookie': 'GOOGAPPUID=42'},
},
kwargs)
def check_final(kwargs):
# Warning: this modifies input arguments.
# Makes the diffing easier.
kwargs['data']['output'] = base64.b64decode(kwargs['data']['output'])
self.assertLess(0, kwargs['data'].pop('cost_usd'))
self.assertLess(
0, kwargs['data'].pop('bot_overhead', None), kwargs['data'])
self.assertLess(0, kwargs['data'].pop('duration'))
self.assertLess(
0., kwargs['data']['isolated_stats']['download'].pop('duration'))
# duration==0 can happen on Windows when the clock is in the default
# resolution, 15.6ms.
self.assertLessEqual(
0., kwargs['data']['isolated_stats']['upload'].pop('duration'))
for k in ('download', 'upload'):
for j in ('items_cold', 'items_hot'):
kwargs['data']['isolated_stats'][k][j] = large.unpack(
base64.b64decode(kwargs['data']['isolated_stats'][k][j]))
self.assertEqual(
{
'data': {
'exit_code': 0,
'hard_timeout': False,
'id': u'localhost',
'io_timeout': False,
'isolated_stats': {
u'download': {
u'initial_number_items': 0,
u'initial_size': 0,
u'items_cold': [10, 86, 94, 276],
u'items_hot': [],
},
u'upload': {
u'items_cold': [],
u'items_hot': [],
},
},
'output': 'hi\n',
'output_chunk_start': 0,
'task_id': 23,
},
'follow_redirects': False,
'timeout': 180,
'headers': {'Cookie': 'GOOGAPPUID=42'},
},
kwargs)
def check_final(kwargs):
self.assertLess(self.SHORT_TIME_OUT, kwargs['data'].pop('cost_usd'))
self.assertLess(self.SHORT_TIME_OUT, kwargs['data'].pop('duration'))
self.assertLess(0., kwargs['data'].pop('bot_overhead'))
self.assertLess(
0., kwargs['data']['isolated_stats']['download'].pop('duration'))
self.assertLess(
0., kwargs['data']['isolated_stats']['upload'].pop('duration'))
# Makes the diffing easier.
for k in ('download', 'upload'):
for j in ('items_cold', 'items_hot'):
kwargs['data']['isolated_stats'][k][j] = large.unpack(
base64.b64decode(kwargs['data']['isolated_stats'][k][j]))
# The command print the pid of this child and grand-child processes, each
# on its line.
output = base64.b64decode(kwargs['data'].pop('output', ''))
for line in output.splitlines():
try:
to_kill.append(int(line))
except ValueError:
pass
self.assertEqual(
{
'data': {
'exit_code': exit_code,
'hard_timeout': False,
'id': u'localhost',
'io_timeout': True,
'isolated_stats': {
u'download': {
u'initial_number_items': 0,
u'initial_size': 0,
u'items_cold': [144, 150, 285, 307],
u'items_hot': [],
},
u'upload': {
u'items_cold': [],
u'items_hot': [],
},
},
'output_chunk_start': 0,
'task_id': 23,
},
},
kwargs)
def check_final(kwargs):
# Warning: this modifies input arguments.
self.assertLess(0, kwargs['data'].pop('cost_usd'))
self.assertLess(0, kwargs['data'].pop('bot_overhead'))
self.assertLess(0, kwargs['data'].pop('duration'))
self.assertLess(
0., kwargs['data']['isolated_stats']['download'].pop('duration'))
# duration==0 can happen on Windows when the clock is in the default
# resolution, 15.6ms.
self.assertLessEqual(
0., kwargs['data']['isolated_stats']['upload'].pop('duration'))
# Makes the diffing easier.
kwargs['data']['output'] = base64.b64decode(kwargs['data']['output'])
for k in ('download', 'upload'):
for j in ('items_cold', 'items_hot'):
kwargs['data']['isolated_stats'][k][j] = large.unpack(
base64.b64decode(kwargs['data']['isolated_stats'][k][j]))
self.assertEqual(
{
'data': {
'exit_code': 0,
'hard_timeout': False,
'id': u'localhost',
'io_timeout': False,
'isolated_stats': {
u'download': {
u'initial_number_items': 0,
u'initial_size': 0,
u'items_cold': [10, 86, 94, 276],
u'items_hot': [],
},
u'upload': {
u'items_cold': [],
u'items_hot': [],
},
},
'output': 'hi\n',
'output_chunk_start': 0,
'task_id': 23,
},
},
kwargs)
def test_1m_1000(self):
array = [i * 1000 for i in xrange(1000000)]
data = large.pack(array)
self.assertGreater(2000, len(data))
self.assertEqual(array, large.unpack(data))
def test_1m_1(self):
array = range(1000000)
data = large.pack(array)
self.assertGreater(1000, len(data))
self.assertEqual(array, large.unpack(data))
u'items_cold': [len(isolated_out_json)],
u'items_hot': [15],
},
},
},
u'version': 5,
}
actual = tools.read_json(out)
# duration can be exactly 0 due to low timer resolution, especially but not
# exclusively on Windows.
self.assertLessEqual(0, actual.pop(u'duration'))
actual_isolated_stats = actual[u'stats'][u'isolated']
self.assertLessEqual(
0, actual_isolated_stats[u'download'].pop(u'duration'))
self.assertLessEqual(0,
actual_isolated_stats[u'upload'].pop(u'duration'))
for i in (u'download', u'upload'):
for j in (u'items_cold', u'items_hot'):
actual_isolated_stats[i][j] = large.unpack(
base64.b64decode(actual_isolated_stats[i][j]))
self.assertEqual(expected, actual)
if __name__ == '__main__':
fix_encoding.fix_encoding()
if '-v' in sys.argv:
unittest.TestCase.maxDiff = None
logging.basicConfig(
level=logging.DEBUG if '-v' in sys.argv else logging.ERROR)
unittest.main()
def test_1m_1000(self): array = [i*1000 for i in xrange(1000000)] data = large.pack(array) self.assertGreater(2000, len(data)) self.assertEqual(array, large.unpack(data))
def test_empty(self):
self.assertEqual('', large.pack([]))
self.assertEqual([], large.unpack(''))
def test_empty(self):
self.assertEqual('', large.pack([]))
self.assertEqual([], large.unpack(''))
u'items_cold': [len(isolated_in_json)],
u'items_hot': [],
},
u'upload': {
u'items_cold': [len(isolated_out_json)],
u'items_hot': [15],
},
},
u'version': 3,
}
actual = tools.read_json(out)
# duration can be exactly 0 due to low timer resolution, especially but not
# exclusively on Windows.
self.assertLessEqual(0, actual.pop(u'duration'))
self.assertLessEqual(0, actual[u'stats'][u'download'].pop(u'duration'))
self.assertLessEqual(0, actual[u'stats'][u'upload'].pop(u'duration'))
for i in (u'download', u'upload'):
for j in (u'items_cold', u'items_hot'):
actual[u'stats'][i][j] = large.unpack(
base64.b64decode(actual[u'stats'][i][j]))
self.assertEqual(expected, actual)
if __name__ == '__main__':
fix_encoding.fix_encoding()
if '-v' in sys.argv:
unittest.TestCase.maxDiff = None
logging.basicConfig(
level=logging.DEBUG if '-v' in sys.argv else logging.ERROR)
unittest.main()
def test_1m_1(self): array = range(1000000) data = large.pack(array) self.assertGreater(1000, len(data)) self.assertEqual(array, large.unpack(data))
