class TableStorageHandlerTest(_TestCase): def _divide_key(self, key): divided = [] hostname = gethostname() if key.find(hostname) >= 0: preceding, hostname, remaining = key.rpartition(hostname) preceding = preceding[:-1] if preceding.endswith('-') else preceding divided.extend(preceding.split('-')) divided.append(hostname) remaining = remaining[1:] if remaining.startswith('-') else remaining divided.extend(remaining.split('-')) else: divided.extend(key.split('-')) return iter(divided) def _get_formatter_name(self, handler_name, formatter_type): name = _get_handler_config_value(handler_name, formatter_type) if name: if name.startswith('cfg://formatters.'): name = name.split('.')[1] return name def _get_partition_key_formatter_name(self, handler_name): return self._get_formatter_name(handler_name, 'partition_key_formatter') def _get_row_key_formatter_name(self, handler_name): return self._get_formatter_name(handler_name, 'row_key_formatter') def setUp(self): self.service = TableService(account_name=ACCOUNT_NAME, account_key=ACCOUNT_KEY, is_emulated=_EMULATED) # ensure that there's no entity in the table before each test tables = set() for cfg in LOGGING['handlers'].values(): if 'table' in cfg: tables.add(cfg['table']) for table in self.service.list_tables(): if table.name in tables: for entity in self.service.query_entities(table.name): self.service.delete_entity(table.name, entity.PartitionKey, entity.RowKey) def test_logging(self): # get the logger for the test logger_name = 'table' logger = logging.getLogger(logger_name) handler_name = _get_handler_name(logger_name) # perform logging log_text = 'logging test' logging_started = datetime.now() logger.info(log_text) logging_finished = datetime.now() # confirm that the entity has correct log text table = _get_handler_config_value(handler_name, 'table') entities = iter(self.service.query_entities(table)) entity = next(entities) self.assertEqual(entity.message, 'INFO %s' % log_text) # confirm that the entity has the default partitiok key fmt = '%Y%m%d%H%M' try: self.assertEqual(entity.PartitionKey, logging_started.strftime(fmt)) except AssertionError: if logging_started == logging_finished: raise self.assertEqual(entity.PartitionKey, logging_finished.strftime(fmt)) # confirm that the entity has the default row key divided = self._divide_key(entity.RowKey) timestamp = next(divided) fmt = '%Y%m%d%H%M%S' self.assertGreaterEqual(timestamp[:-3], logging_started.strftime(fmt)) self.assertLessEqual(timestamp[:-3], logging_finished.strftime(fmt)) self.assertRegex(timestamp[-3:], '^[0-9]{3}$') self.assertEqual(next(divided), gethostname()) self.assertEqual(int(next(divided)), os.getpid()) self.assertEqual(next(divided), '00') with self.assertRaises(StopIteration): next(divided) # confirm that there's no more entity in the table with self.assertRaises(StopIteration): next(entities) @unittest.skipIf(_EMULATED, "Azure Storage Emulator doesn't support batch operation.") def test_batch(self): # get the logger for the test logger_name = 'batch' logger = logging.getLogger(logger_name) handler_name = _get_handler_name(logger_name) # perform logging and execute the first batch batch_size = _get_handler_config_value(handler_name, 'batch_size') log_text = 'batch logging test' for i in range(batch_size + int(batch_size/2)): logger.info('%s#%02d' % (log_text, i)) # confirm that only batch_size entities are committed at this point table = _get_handler_config_value(handler_name, 'table') entities = list(iter(self.service.query_entities(table))) self.assertEqual(len(entities), batch_size) rowno_found = set() seq_found = set() for entity in entities: # partition key self.assertEqual(entity.PartitionKey, 'batch-%s' % gethostname()) # row key rowno = entity.RowKey.split('-')[-1] self.assertLess(int(rowno), batch_size) self.assertNotIn(rowno, rowno_found) rowno_found.add(rowno) # message message, seq = entity.message.split('#') self.assertEqual(message, 'INFO %s' % log_text) self.assertLess(int(seq), batch_size) self.assertNotIn(seq, seq_found) seq_found.add(seq) # remove currently created entities before the next batch for entity in entities: self.service.delete_entity(table, entity.PartitionKey, entity.RowKey) # perform logging again and execute the next batch for j in range(i+1, int(batch_size/2)+i+1): logger.info('%s#%02d' % (log_text, j)) # confirm that the remaining entities are committed in the next batch entities = list(iter(self.service.query_entities(table))) self.assertEqual(len(entities), batch_size) rowno_found.clear() for entity in entities: # partition key self.assertEqual(entity.PartitionKey, 'batch-%s' % gethostname()) # row key rowno = entity.RowKey.split('-')[-1] self.assertLess(int(rowno), batch_size) self.assertNotIn(rowno, rowno_found) rowno_found.add(rowno) # message message, seq = entity.message.split('#') self.assertEqual(message, 'INFO %s' % log_text) self.assertGreaterEqual(int(seq), batch_size) self.assertLess(int(seq), batch_size*2) self.assertNotIn(seq, seq_found) seq_found.add(seq) def test_extra_properties(self): # get the logger for the test logger_name = 'extra_properties' logger = logging.getLogger(logger_name) handler_name = _get_handler_name(logger_name) # perform logging log_text = 'extra properties test' logger.info(log_text) # confirm that the entity has correct log text table = _get_handler_config_value(handler_name, 'table') entities = iter(self.service.query_entities(table)) entity = next(entities) self.assertEqual(entity.message, 'INFO %s' % log_text) # confirm that the extra properties have correct values entity = next(iter(self.service.query_entities(table))) self.assertEqual(entity.hostname, gethostname()) self.assertEqual(entity.levelname, 'INFO') self.assertEqual(int(entity.levelno), logging.INFO) self.assertEqual(entity.module, os.path.basename(__file__).rpartition('.')[0]) self.assertEqual(entity.name, logger_name) self.assertEqual(int(entity.process), os.getpid()) self.assertEqual(int(entity.thread), current_thread().ident) # confirm that there's no more entity in the table with self.assertRaises(StopIteration): next(entities) def test_custom_key_formatters(self): # get the logger for the test logger_name = 'custom_keys' logger = logging.getLogger(logger_name) handler_name = _get_handler_name(logger_name) # perform logging log_text = 'custom key formatters test' logging_started = datetime.now() logger.info(log_text) logging_finished = datetime.now() # confirm that the entity correct log text table = _get_handler_config_value(handler_name, 'table') entities = iter(self.service.query_entities(table)) entity = next(entities) self.assertEqual(entity.message, 'INFO %s' % log_text) # confirm that the entity has a custom partitiok key divided = self._divide_key(entity.PartitionKey) self.assertEqual(next(divided), 'mycustompartitionkey') self.assertEqual(next(divided), gethostname()) formatter_name = self._get_partition_key_formatter_name(handler_name) fmt = _get_formatter_config_value(formatter_name, 'datefmt') asctime = next(divided) try: self.assertEqual(asctime, logging_started.strftime(fmt)) except AssertionError: if logging_started == logging_finished: raise self.assertEqual(asctime, logging_finished.strftime(fmt)) with self.assertRaises(StopIteration): next(divided) # confirm that the entity has a custom row key divided = self._divide_key(entity.RowKey) self.assertEqual(next(divided), 'mycustomrowkey') self.assertEqual(next(divided), gethostname()) formatter_name = self._get_row_key_formatter_name(handler_name) fmt = _get_formatter_config_value(formatter_name, 'datefmt') asctime = next(divided) try: self.assertEqual(asctime, logging_started.strftime(fmt)) except AssertionError: if logging_started == logging_finished: raise self.assertEqual(asctime, logging_finished.strftime(fmt)) with self.assertRaises(StopIteration): next(divided) # confirm that there's no more entity in the table with self.assertRaises(StopIteration): next(entities)
class az(object): def __init__(self, default_table_name=DEFAULT_TABLE, partitionKey='default'): self.TABLE_STORAGE_KEY = os.getenv('AZURE_STORAGE_KEY') self.STORAGE_NAME = os.getenv('STORAGE_NAME') self.default_table_name = default_table_name self.default_partition = partitionKey if self.TABLE_STORAGE_KEY == None: from tokens import TABLE_STORAGE_ACCESS_KEY, STORAGE_ACCOUNT_NAME self.TABLE_STORAGE_KEY = TABLE_STORAGE_ACCESS_KEY self.STORAGE_NAME = STORAGE_ACCOUNT_NAME self.table_service = TableService(account_name=self.STORAGE_NAME, account_key=self.TABLE_STORAGE_KEY) #create_table_if_does_not_exists(self.default_table_name) def insert_or_replace_entity_to_azure(self, rowKey, entry, t_name=DEFAULT_TABLE): ''' takes table service Takes a list Uploads to azure table storage ''' segment = Entity() segment.PartitionKey = self.default_partition segment.RowKey = str(rowKey).zfill(8) segment.latA = str(entry['latA']) segment.longA = str(entry['longA']) segment.latB = str(entry['latB']) segment.longB = str(entry['longB']) segment.colorKey = str(entry['color']) #print segment.colorKey if os.name == 'nt': self.table_service.insert_or_replace_entity( t_name, self.default_partition, str(rowKey).zfill(8), segment) else: self.table_service.insert_or_replace_entity(t_name, segment) def create_table(self, name): return self.table_service.create_table(name) def delete_table(self, name): return self.table_service.delete_table(name) def delete_entity_by_rowKey(self, rowKey, table_name=DEFAULT_TABLE): return self.table_service.delete_entity(table_name, self.default_partition, rowKey) def does_table_exist(self, table_name): if os.name == 'nt': for i in self.table_service.query_tables(): if i.name == table_name: return True else: for i in self.table_service.list_tables(): if i.name == table_name: return True return False def list_tables(self): if os.name == 'nt': for j in self.table_service.query_tables(): print j.name else: for j in self.table_service.list_tables(): print j.name def create_table_if_does_not_exist(self, table_name=DEFAULT_TABLE): if self.does_table_exist(table_name): return 'already exists' else: self.table_service.create_table(table_name) def create_entry(self, latA, lonA, latB, lonB, bumpiness): x = { 'latA': latA, 'longA': lonA, 'latB': latB, 'longB': lonB, 'color': bumpiness } return x def create_random_entry(self): x = { 'latA': random.uniform(37, 38), 'longA': random.uniform(-122, -123), 'latB': random.uniform(37, 38), 'longB': random.uniform(-122, -123), 'color': random.randint(0, 7) } return x def create_and_insert_or_replace_entity_azure(self, latA, lonA, latB, lonB, bumpiness, rowKey, table_name=DEFAULT_TABLE): return self.insert_or_replace_entity_to_azure( rowKey, create_entry(latA, lonA, latB, lonB, bumpiness), table_name)
class TableStorageHandlerTest(_TestCase): def _divide_key(self, key): divided = [] hostname = gethostname() if key.find(hostname) >= 0: preceding, hostname, remaining = key.rpartition(hostname) preceding = preceding[:-1] if preceding.endswith( '-') else preceding divided.extend(preceding.split('-')) divided.append(hostname) remaining = remaining[1:] if remaining.startswith( '-') else remaining divided.extend(remaining.split('-')) else: divided.extend(key.split('-')) return iter(divided) def _get_formatter_name(self, handler_name, formatter_type): name = _get_handler_config_value(handler_name, formatter_type) if name: if name.startswith('cfg://formatters.'): name = name.split('.')[1] return name def _get_partition_key_formatter_name(self, handler_name): return self._get_formatter_name(handler_name, 'partition_key_formatter') def _get_row_key_formatter_name(self, handler_name): return self._get_formatter_name(handler_name, 'row_key_formatter') def setUp(self): self.service = TableService(account_name=ACCOUNT_NAME, account_key=ACCOUNT_KEY, is_emulated=_EMULATED) # ensure that there's no entity in the table before each test tables = set() for cfg in LOGGING['handlers'].values(): if 'table' in cfg: tables.add(cfg['table']) for table in self.service.list_tables(): if table.name in tables: for entity in self.service.query_entities(table.name): self.service.delete_entity(table.name, entity.PartitionKey, entity.RowKey) def test_logging(self): # get the logger for the test logger_name = 'table' logger = logging.getLogger(logger_name) handler_name = _get_handler_name(logger_name) # perform logging log_text = 'logging test' logging_started = datetime.now() logger.info(log_text) logging_finished = datetime.now() # confirm that the entity has correct log text table = _get_handler_config_value(handler_name, 'table') entities = iter(self.service.query_entities(table)) entity = next(entities) self.assertEqual(entity.message, 'INFO %s' % log_text) # confirm that the entity has the default partitiok key fmt = '%Y%m%d%H%M' try: self.assertEqual(entity.PartitionKey, logging_started.strftime(fmt)) except AssertionError: if logging_started == logging_finished: raise self.assertEqual(entity.PartitionKey, logging_finished.strftime(fmt)) # confirm that the entity has the default row key divided = self._divide_key(entity.RowKey) timestamp = next(divided) fmt = '%Y%m%d%H%M%S' self.assertGreaterEqual(timestamp[:-3], logging_started.strftime(fmt)) self.assertLessEqual(timestamp[:-3], logging_finished.strftime(fmt)) self.assertRegex(timestamp[-3:], '^[0-9]{3}$') self.assertEqual(next(divided), gethostname()) self.assertEqual(int(next(divided)), os.getpid()) self.assertEqual(next(divided), '00') with self.assertRaises(StopIteration): next(divided) # confirm that there's no more entity in the table with self.assertRaises(StopIteration): next(entities) @unittest.skipIf(_EMULATED, "Azure Storage Emulator doesn't support batch operation.") def test_batch(self): # get the logger for the test logger_name = 'batch' logger = logging.getLogger(logger_name) handler_name = _get_handler_name(logger_name) # perform logging and execute the first batch batch_size = _get_handler_config_value(handler_name, 'batch_size') log_text = 'batch logging test' for i in range(batch_size + int(batch_size / 2)): logger.info('%s#%02d' % (log_text, i)) # confirm that only batch_size entities are committed at this point table = _get_handler_config_value(handler_name, 'table') entities = list(iter(self.service.query_entities(table))) self.assertEqual(len(entities), batch_size) rowno_found = set() seq_found = set() for entity in entities: # partition key self.assertEqual(entity.PartitionKey, 'batch-%s' % gethostname()) # row key rowno = entity.RowKey.split('-')[-1] self.assertLess(int(rowno), batch_size) self.assertNotIn(rowno, rowno_found) rowno_found.add(rowno) # message message, seq = entity.message.split('#') self.assertEqual(message, 'INFO %s' % log_text) self.assertLess(int(seq), batch_size) self.assertNotIn(seq, seq_found) seq_found.add(seq) # remove currently created entities before the next batch for entity in entities: self.service.delete_entity(table, entity.PartitionKey, entity.RowKey) # perform logging again and execute the next batch for j in range(i + 1, int(batch_size / 2) + i + 1): logger.info('%s#%02d' % (log_text, j)) # confirm that the remaining entities are committed in the next batch entities = list(iter(self.service.query_entities(table))) self.assertEqual(len(entities), batch_size) rowno_found.clear() for entity in entities: # partition key self.assertEqual(entity.PartitionKey, 'batch-%s' % gethostname()) # row key rowno = entity.RowKey.split('-')[-1] self.assertLess(int(rowno), batch_size) self.assertNotIn(rowno, rowno_found) rowno_found.add(rowno) # message message, seq = entity.message.split('#') self.assertEqual(message, 'INFO %s' % log_text) self.assertGreaterEqual(int(seq), batch_size) self.assertLess(int(seq), batch_size * 2) self.assertNotIn(seq, seq_found) seq_found.add(seq) def test_extra_properties(self): # get the logger for the test logger_name = 'extra_properties' logger = logging.getLogger(logger_name) handler_name = _get_handler_name(logger_name) # perform logging log_text = 'extra properties test' logger.info(log_text) # confirm that the entity has correct log text table = _get_handler_config_value(handler_name, 'table') entities = iter(self.service.query_entities(table)) entity = next(entities) self.assertEqual(entity.message, 'INFO %s' % log_text) # confirm that the extra properties have correct values entity = next(iter(self.service.query_entities(table))) self.assertEqual(entity.hostname, gethostname()) self.assertEqual(entity.levelname, 'INFO') self.assertEqual(int(entity.levelno), logging.INFO) self.assertEqual(entity.module, os.path.basename(__file__).rpartition('.')[0]) self.assertEqual(entity.name, logger_name) self.assertEqual(int(entity.process), os.getpid()) self.assertEqual(int(entity.thread), current_thread().ident) # confirm that there's no more entity in the table with self.assertRaises(StopIteration): next(entities) def test_custom_key_formatters(self): # get the logger for the test logger_name = 'custom_keys' logger = logging.getLogger(logger_name) handler_name = _get_handler_name(logger_name) # perform logging log_text = 'custom key formatters test' logging_started = datetime.now() logger.info(log_text) logging_finished = datetime.now() # confirm that the entity correct log text table = _get_handler_config_value(handler_name, 'table') entities = iter(self.service.query_entities(table)) entity = next(entities) self.assertEqual(entity.message, 'INFO %s' % log_text) # confirm that the entity has a custom partitiok key divided = self._divide_key(entity.PartitionKey) self.assertEqual(next(divided), 'mycustompartitionkey') self.assertEqual(next(divided), gethostname()) formatter_name = self._get_partition_key_formatter_name(handler_name) fmt = _get_formatter_config_value(formatter_name, 'datefmt') asctime = next(divided) try: self.assertEqual(asctime, logging_started.strftime(fmt)) except AssertionError: if logging_started == logging_finished: raise self.assertEqual(asctime, logging_finished.strftime(fmt)) with self.assertRaises(StopIteration): next(divided) # confirm that the entity has a custom row key divided = self._divide_key(entity.RowKey) self.assertEqual(next(divided), 'mycustomrowkey') self.assertEqual(next(divided), gethostname()) formatter_name = self._get_row_key_formatter_name(handler_name) fmt = _get_formatter_config_value(formatter_name, 'datefmt') asctime = next(divided) try: self.assertEqual(asctime, logging_started.strftime(fmt)) except AssertionError: if logging_started == logging_finished: raise self.assertEqual(asctime, logging_finished.strftime(fmt)) with self.assertRaises(StopIteration): next(divided) # confirm that there's no more entity in the table with self.assertRaises(StopIteration): next(entities)
class az(object): def __init__(self, default_table_name=DEFAULT_TABLE, partitionKey='default'): self.TABLE_STORAGE_KEY = os.getenv('AZURE_STORAGE_KEY') self.STORAGE_NAME = os.getenv('STORAGE_NAME') self.default_table_name = default_table_name self.default_partition = partitionKey if self.TABLE_STORAGE_KEY == None: from tokens import TABLE_STORAGE_ACCESS_KEY, STORAGE_ACCOUNT_NAME self.TABLE_STORAGE_KEY = TABLE_STORAGE_ACCESS_KEY self.STORAGE_NAME = STORAGE_ACCOUNT_NAME self.table_service = TableService(account_name=self.STORAGE_NAME, account_key=self.TABLE_STORAGE_KEY) #create_table_if_does_not_exists(self.default_table_name) def insert_or_replace_entity_to_azure(self, rowKey, entry, t_name=DEFAULT_TABLE): ''' takes table service Takes a list Uploads to azure table storage ''' segment = Entity() segment.PartitionKey = self.default_partition segment.RowKey = str(rowKey).zfill(8) segment.latA = str(entry['latA']) segment.longA = str(entry['longA']) segment.latB = str(entry['latB']) segment.longB = str(entry['longB']) segment.colorKey = str(entry['color']) #print segment.colorKey if os.name == 'nt': self.table_service.insert_or_replace_entity(t_name, self.default_partition, str(rowKey).zfill(8), segment) else: self.table_service.insert_or_replace_entity(t_name, segment) def create_table(self, name): return self.table_service.create_table(name) def delete_table(self, name): return self.table_service.delete_table(name) def delete_entity_by_rowKey(self, rowKey, table_name=DEFAULT_TABLE): return self.table_service.delete_entity(table_name, self.default_partition, rowKey) def does_table_exist(self, table_name): if os.name == 'nt': for i in self.table_service.query_tables(): if i.name == table_name: return True else: for i in self.table_service.list_tables(): if i.name == table_name: return True return False def list_tables(self): if os.name == 'nt': for j in self.table_service.query_tables(): print j.name else: for j in self.table_service.list_tables(): print j.name def create_table_if_does_not_exist(self, table_name=DEFAULT_TABLE): if self.does_table_exist(table_name): return 'already exists' else: self.table_service.create_table(table_name) def create_entry(self, latA, lonA, latB, lonB, bumpiness): x = { 'latA':latA, 'longA':lonA, 'latB':latB, 'longB':lonB, 'color': bumpiness } return x def create_random_entry(self): x = { 'latA':random.uniform(37,38), 'longA':random.uniform(-122,-123), 'latB':random.uniform(37,38), 'longB':random.uniform(-122,-123), 'color': random.randint(0,7) } return x def create_and_insert_or_replace_entity_azure(self, latA, lonA, latB, lonB, bumpiness, rowKey, table_name=DEFAULT_TABLE ): return self.insert_or_replace_entity_to_azure(rowKey, create_entry(latA, lonA, latB, lonB, bumpiness), table_name)