def test_1m_pseudo(self): # Compresses a pseudo-random suite. Still compresses very well. random.seed(0) array = sorted(random.randint(0, 1000000) for _ in xrange(1000000)) data = large.pack(array) self.assertGreater(302000, len(data)) self.assertEqual(array, large.unpack(data))
def assertResults(self, expected, result): self.assertEqual(['shards'], result.keys()) self.assertEqual(1, len(result['shards'])) self.assertTrue(result['shards'][0]) result = result['shards'][0].copy() # These are not deterministic (or I'm too lazy to calculate the value). if expected.get('performance_stats'): self.assertLess(0, result['performance_stats'].pop('bot_overhead')) self.assertLess( 0, result['performance_stats']['isolated_download'].pop( 'duration')) self.assertLess( 0, result['performance_stats']['isolated_upload'].pop('duration')) for k in ('isolated_download', 'isolated_upload'): for j in ('items_cold', 'items_hot'): result['performance_stats'][k][j] = large.unpack( base64.b64decode(result['performance_stats'][k].get( j, ''))) bot_version = result.pop('bot_version') self.assertTrue(bot_version) self.assertLess(0, result.pop('costs_usd')) self.assertTrue(result.pop('created_ts')) self.assertTrue(result.pop('completed_ts')) self.assertLess(0, result.pop('durations')) self.assertTrue(result.pop('id')) self.assertTrue(result.pop('modified_ts')) self.assertTrue(result.pop('started_ts')) self.assertEqual(expected, result) return bot_version
def assertResults(self, expected, result): self.assertEqual(['shards'], result.keys()) self.assertEqual(1, len(result['shards'])) self.assertTrue(result['shards'][0]) result = result['shards'][0].copy() # These are not deterministic (or I'm too lazy to calculate the value). if expected.get('performance_stats'): self.assertLess( 0, result['performance_stats'].pop('bot_overhead')) self.assertLess( 0, result['performance_stats']['isolated_download'].pop('duration')) self.assertLess( 0, result['performance_stats']['isolated_upload'].pop('duration')) for k in ('isolated_download', 'isolated_upload'): for j in ('items_cold', 'items_hot'): result['performance_stats'][k][j] = large.unpack( base64.b64decode(result['performance_stats'][k].get(j, ''))) bot_version = result.pop('bot_version') self.assertTrue(bot_version) self.assertLess(0, result.pop('costs_usd')) self.assertTrue(result.pop('created_ts')) self.assertTrue(result.pop('completed_ts')) self.assertLess(0, result.pop('durations')) self.assertTrue(result.pop('id')) self.assertTrue(result.pop('modified_ts')) self.assertTrue(result.pop('started_ts')) self.assertEqual(expected, result) return bot_version
def test_1m_pseudo(self): # Compresses a pseudo-random suite. Still compresses very well. random.seed(0) array = sorted(random.randint(0, 1000000) for _ in xrange(1000000)) data = large.pack(array) self.assertGreater(302000, len(data)) self.assertEqual(array, large.unpack(data))
def load_isolated_stats(stats_json_path, key): actual = json.loads(read_content(stats_json_path)) stats = actual['stats']['isolated'].get(key) for key in ['items_cold', 'items_hot']: if not stats[key]: continue stats[key] = large.unpack(base64.b64decode(stats[key])) return stats
def assertResults(self, expected, result): self.assertEqual(['shards'], result.keys()) self.assertEqual(1, len(result['shards'])) self.assertTrue(result['shards'][0]) result = result['shards'][0].copy() self.assertFalse(result.get('abandoned_ts')) # These are not deterministic (or I'm too lazy to calculate the value). if expected.get('performance_stats'): self.assertLess(0, result['performance_stats'].pop('bot_overhead')) self.assertLess( 0, result['performance_stats']['isolated_download'].pop( 'duration')) self.assertLess( 0, result['performance_stats']['isolated_upload'].pop('duration')) for k in ('isolated_download', 'isolated_upload'): for j in ('items_cold', 'items_hot'): result['performance_stats'][k][j] = large.unpack( base64.b64decode(result['performance_stats'][k].get( j, ''))) else: perf_stats = result.pop('performance_stats', None) if perf_stats: # Ignore bot_overhead, everything else should be empty. perf_stats.pop('bot_overhead', None) self.assertFalse(perf_stats.pop('isolated_download', None)) self.assertFalse(perf_stats.pop('isolated_upload', None)) self.assertFalse(perf_stats) bot_version = result.pop('bot_version') self.assertTrue(bot_version) if result[u'costs_usd'] is not None: expected.pop(u'costs_usd', None) self.assertLess(0, result.pop(u'costs_usd')) if result[u'cost_saved_usd'] is not None: expected.pop(u'cost_saved_usd', None) self.assertLess(0, result.pop(u'cost_saved_usd')) self.assertTrue(result.pop('created_ts')) self.assertTrue(result.pop('completed_ts')) self.assertLess(0, result.pop('durations')) self.assertTrue(result.pop('id')) self.assertTrue(result.pop('modified_ts')) self.assertTrue(result.pop('started_ts')) if getattr(expected.get('outputs'), 'match', None): expected_outputs = expected.pop('outputs') outputs = '\n'.join(result.pop('outputs')) self.assertTrue( expected_outputs.match(outputs), '%s does not match %s' % (outputs, expected_outputs.pattern)) self.assertEqual(expected, result) return bot_version
def check_final(kwargs): self.assertLess(self.SHORT_TIME_OUT, kwargs['data'].pop('cost_usd')) self.assertLess(self.SHORT_TIME_OUT, kwargs['data'].pop('duration')) self.assertLess(0., kwargs['data'].pop('bot_overhead')) self.assertLess( 0., kwargs['data']['isolated_stats']['download'].pop('duration')) self.assertLess( 0., kwargs['data']['isolated_stats']['upload'].pop('duration')) # Makes the diffing easier. for k in ('download', 'upload'): for j in ('items_cold', 'items_hot'): kwargs['data']['isolated_stats'][k][j] = large.unpack( base64.b64decode(kwargs['data']['isolated_stats'][k][j])) # The command print the pid of this child and grand-child processes, each # on its line. output = base64.b64decode(kwargs['data'].pop('output', '')) for line in output.splitlines(): try: to_kill.append(int(line)) except ValueError: pass self.assertEqual( { 'data': { 'exit_code': exit_code, 'hard_timeout': False, 'id': u'localhost', 'io_timeout': True, 'isolated_stats': { u'download': { u'initial_number_items': 0, u'initial_size': 0, u'items_cold': [144, 150, 285, 307], u'items_hot': [], }, u'upload': { u'items_cold': [], u'items_hot': [], }, }, 'output_chunk_start': 0, 'task_id': 23, }, 'follow_redirects': False, 'timeout': 180, 'headers': {'Cookie': 'GOOGAPPUID=42'}, }, kwargs)
def check_final(kwargs): # Warning: this modifies input arguments. # Makes the diffing easier. kwargs['data']['output'] = base64.b64decode(kwargs['data']['output']) self.assertLess(0, kwargs['data'].pop('cost_usd')) self.assertLess( 0, kwargs['data'].pop('bot_overhead', None), kwargs['data']) self.assertLess(0, kwargs['data'].pop('duration')) self.assertLess( 0., kwargs['data']['isolated_stats']['download'].pop('duration')) # duration==0 can happen on Windows when the clock is in the default # resolution, 15.6ms. self.assertLessEqual( 0., kwargs['data']['isolated_stats']['upload'].pop('duration')) for k in ('download', 'upload'): for j in ('items_cold', 'items_hot'): kwargs['data']['isolated_stats'][k][j] = large.unpack( base64.b64decode(kwargs['data']['isolated_stats'][k][j])) self.assertEqual( { 'data': { 'exit_code': 0, 'hard_timeout': False, 'id': u'localhost', 'io_timeout': False, 'isolated_stats': { u'download': { u'initial_number_items': 0, u'initial_size': 0, u'items_cold': [10, 86, 94, 276], u'items_hot': [], }, u'upload': { u'items_cold': [], u'items_hot': [], }, }, 'output': 'hi\n', 'output_chunk_start': 0, 'task_id': 23, }, 'follow_redirects': False, 'timeout': 180, 'headers': {'Cookie': 'GOOGAPPUID=42'}, }, kwargs)
def check_final(kwargs): self.assertLess(self.SHORT_TIME_OUT, kwargs['data'].pop('cost_usd')) self.assertLess(self.SHORT_TIME_OUT, kwargs['data'].pop('duration')) self.assertLess(0., kwargs['data'].pop('bot_overhead')) self.assertLess( 0., kwargs['data']['isolated_stats']['download'].pop('duration')) self.assertLess( 0., kwargs['data']['isolated_stats']['upload'].pop('duration')) # Makes the diffing easier. for k in ('download', 'upload'): for j in ('items_cold', 'items_hot'): kwargs['data']['isolated_stats'][k][j] = large.unpack( base64.b64decode(kwargs['data']['isolated_stats'][k][j])) # The command print the pid of this child and grand-child processes, each # on its line. output = base64.b64decode(kwargs['data'].pop('output', '')) for line in output.splitlines(): try: to_kill.append(int(line)) except ValueError: pass self.assertEqual( { 'data': { 'exit_code': exit_code, 'hard_timeout': False, 'id': u'localhost', 'io_timeout': True, 'isolated_stats': { u'download': { u'initial_number_items': 0, u'initial_size': 0, u'items_cold': [144, 150, 285, 307], u'items_hot': [], }, u'upload': { u'items_cold': [], u'items_hot': [], }, }, 'output_chunk_start': 0, 'task_id': 23, }, }, kwargs)
def check_final(kwargs): # Warning: this modifies input arguments. self.assertLess(0, kwargs['data'].pop('cost_usd')) self.assertLess(0, kwargs['data'].pop('bot_overhead')) self.assertLess(0, kwargs['data'].pop('duration')) self.assertLess( 0., kwargs['data']['isolated_stats']['download'].pop('duration')) # duration==0 can happen on Windows when the clock is in the default # resolution, 15.6ms. self.assertLessEqual( 0., kwargs['data']['isolated_stats']['upload'].pop('duration')) # Makes the diffing easier. kwargs['data']['output'] = base64.b64decode(kwargs['data']['output']) for k in ('download', 'upload'): for j in ('items_cold', 'items_hot'): kwargs['data']['isolated_stats'][k][j] = large.unpack( base64.b64decode(kwargs['data']['isolated_stats'][k][j])) self.assertEqual( { 'data': { 'exit_code': 0, 'hard_timeout': False, 'id': u'localhost', 'io_timeout': False, 'isolated_stats': { u'download': { u'initial_number_items': 0, u'initial_size': 0, u'items_cold': [10, 86, 94, 276], u'items_hot': [], }, u'upload': { u'items_cold': [], u'items_hot': [], }, }, 'output': 'hi\n', 'output_chunk_start': 0, 'task_id': 23, }, }, kwargs)
def test_1m_1000(self): array = [i * 1000 for i in xrange(1000000)] data = large.pack(array) self.assertGreater(2000, len(data)) self.assertEqual(array, large.unpack(data))
def test_1m_1(self): array = range(1000000) data = large.pack(array) self.assertGreater(1000, len(data)) self.assertEqual(array, large.unpack(data))
u'items_cold': [len(isolated_out_json)], u'items_hot': [15], }, }, }, u'version': 5, } actual = tools.read_json(out) # duration can be exactly 0 due to low timer resolution, especially but not # exclusively on Windows. self.assertLessEqual(0, actual.pop(u'duration')) actual_isolated_stats = actual[u'stats'][u'isolated'] self.assertLessEqual( 0, actual_isolated_stats[u'download'].pop(u'duration')) self.assertLessEqual(0, actual_isolated_stats[u'upload'].pop(u'duration')) for i in (u'download', u'upload'): for j in (u'items_cold', u'items_hot'): actual_isolated_stats[i][j] = large.unpack( base64.b64decode(actual_isolated_stats[i][j])) self.assertEqual(expected, actual) if __name__ == '__main__': fix_encoding.fix_encoding() if '-v' in sys.argv: unittest.TestCase.maxDiff = None logging.basicConfig( level=logging.DEBUG if '-v' in sys.argv else logging.ERROR) unittest.main()
def test_1m_1000(self): array = [i*1000 for i in xrange(1000000)] data = large.pack(array) self.assertGreater(2000, len(data)) self.assertEqual(array, large.unpack(data))
def test_empty(self): self.assertEqual('', large.pack([])) self.assertEqual([], large.unpack(''))
def test_empty(self): self.assertEqual('', large.pack([])) self.assertEqual([], large.unpack(''))
u'items_cold': [len(isolated_in_json)], u'items_hot': [], }, u'upload': { u'items_cold': [len(isolated_out_json)], u'items_hot': [15], }, }, u'version': 3, } actual = tools.read_json(out) # duration can be exactly 0 due to low timer resolution, especially but not # exclusively on Windows. self.assertLessEqual(0, actual.pop(u'duration')) self.assertLessEqual(0, actual[u'stats'][u'download'].pop(u'duration')) self.assertLessEqual(0, actual[u'stats'][u'upload'].pop(u'duration')) for i in (u'download', u'upload'): for j in (u'items_cold', u'items_hot'): actual[u'stats'][i][j] = large.unpack( base64.b64decode(actual[u'stats'][i][j])) self.assertEqual(expected, actual) if __name__ == '__main__': fix_encoding.fix_encoding() if '-v' in sys.argv: unittest.TestCase.maxDiff = None logging.basicConfig( level=logging.DEBUG if '-v' in sys.argv else logging.ERROR) unittest.main()
def test_1m_1(self): array = range(1000000) data = large.pack(array) self.assertGreater(1000, len(data)) self.assertEqual(array, large.unpack(data))