class TableStorageHandlerTest(_TestCase):
def _divide_key(self, key):
divided = []
hostname = gethostname()
if key.find(hostname) >= 0:
preceding, hostname, remaining = key.rpartition(hostname)
preceding = preceding[:-1] if preceding.endswith('-') else preceding
divided.extend(preceding.split('-'))
divided.append(hostname)
remaining = remaining[1:] if remaining.startswith('-') else remaining
divided.extend(remaining.split('-'))
else:
divided.extend(key.split('-'))
return iter(divided)
def _get_formatter_name(self, handler_name, formatter_type):
name = _get_handler_config_value(handler_name, formatter_type)
if name:
if name.startswith('cfg://formatters.'):
name = name.split('.')[1]
return name
def _get_partition_key_formatter_name(self, handler_name):
return self._get_formatter_name(handler_name, 'partition_key_formatter')
def _get_row_key_formatter_name(self, handler_name):
return self._get_formatter_name(handler_name, 'row_key_formatter')
def setUp(self):
self.service = TableService(account_name=ACCOUNT_NAME,
account_key=ACCOUNT_KEY,
is_emulated=_EMULATED)
# ensure that there's no entity in the table before each test
tables = set()
for cfg in LOGGING['handlers'].values():
if 'table' in cfg:
tables.add(cfg['table'])
for table in self.service.list_tables():
if table.name in tables:
for entity in self.service.query_entities(table.name):
self.service.delete_entity(table.name,
entity.PartitionKey,
entity.RowKey)
def test_logging(self):
# get the logger for the test
logger_name = 'table'
logger = logging.getLogger(logger_name)
handler_name = _get_handler_name(logger_name)
# perform logging
log_text = 'logging test'
logging_started = datetime.now()
logger.info(log_text)
logging_finished = datetime.now()
# confirm that the entity has correct log text
table = _get_handler_config_value(handler_name, 'table')
entities = iter(self.service.query_entities(table))
entity = next(entities)
self.assertEqual(entity.message, 'INFO %s' % log_text)
# confirm that the entity has the default partitiok key
fmt = '%Y%m%d%H%M'
try:
self.assertEqual(entity.PartitionKey, logging_started.strftime(fmt))
except AssertionError:
if logging_started == logging_finished:
raise
self.assertEqual(entity.PartitionKey, logging_finished.strftime(fmt))
# confirm that the entity has the default row key
divided = self._divide_key(entity.RowKey)
timestamp = next(divided)
fmt = '%Y%m%d%H%M%S'
self.assertGreaterEqual(timestamp[:-3], logging_started.strftime(fmt))
self.assertLessEqual(timestamp[:-3], logging_finished.strftime(fmt))
self.assertRegex(timestamp[-3:], '^[0-9]{3}$')
self.assertEqual(next(divided), gethostname())
self.assertEqual(int(next(divided)), os.getpid())
self.assertEqual(next(divided), '00')
with self.assertRaises(StopIteration):
next(divided)
# confirm that there's no more entity in the table
with self.assertRaises(StopIteration):
next(entities)
@unittest.skipIf(_EMULATED, "Azure Storage Emulator doesn't support batch operation.")
def test_batch(self):
# get the logger for the test
logger_name = 'batch'
logger = logging.getLogger(logger_name)
handler_name = _get_handler_name(logger_name)
# perform logging and execute the first batch
batch_size = _get_handler_config_value(handler_name, 'batch_size')
log_text = 'batch logging test'
for i in range(batch_size + int(batch_size/2)):
logger.info('%s#%02d' % (log_text, i))
# confirm that only batch_size entities are committed at this point
table = _get_handler_config_value(handler_name, 'table')
entities = list(iter(self.service.query_entities(table)))
self.assertEqual(len(entities), batch_size)
rowno_found = set()
seq_found = set()
for entity in entities:
# partition key
self.assertEqual(entity.PartitionKey, 'batch-%s' % gethostname())
# row key
rowno = entity.RowKey.split('-')[-1]
self.assertLess(int(rowno), batch_size)
self.assertNotIn(rowno, rowno_found)
rowno_found.add(rowno)
# message
message, seq = entity.message.split('#')
self.assertEqual(message, 'INFO %s' % log_text)
self.assertLess(int(seq), batch_size)
self.assertNotIn(seq, seq_found)
seq_found.add(seq)
# remove currently created entities before the next batch
for entity in entities:
self.service.delete_entity(table,
entity.PartitionKey,
entity.RowKey)
# perform logging again and execute the next batch
for j in range(i+1, int(batch_size/2)+i+1):
logger.info('%s#%02d' % (log_text, j))
# confirm that the remaining entities are committed in the next batch
entities = list(iter(self.service.query_entities(table)))
self.assertEqual(len(entities), batch_size)
rowno_found.clear()
for entity in entities:
# partition key
self.assertEqual(entity.PartitionKey, 'batch-%s' % gethostname())
# row key
rowno = entity.RowKey.split('-')[-1]
self.assertLess(int(rowno), batch_size)
self.assertNotIn(rowno, rowno_found)
rowno_found.add(rowno)
# message
message, seq = entity.message.split('#')
self.assertEqual(message, 'INFO %s' % log_text)
self.assertGreaterEqual(int(seq), batch_size)
self.assertLess(int(seq), batch_size*2)
self.assertNotIn(seq, seq_found)
seq_found.add(seq)
def test_extra_properties(self):
# get the logger for the test
logger_name = 'extra_properties'
logger = logging.getLogger(logger_name)
handler_name = _get_handler_name(logger_name)
# perform logging
log_text = 'extra properties test'
logger.info(log_text)
# confirm that the entity has correct log text
table = _get_handler_config_value(handler_name, 'table')
entities = iter(self.service.query_entities(table))
entity = next(entities)
self.assertEqual(entity.message, 'INFO %s' % log_text)
# confirm that the extra properties have correct values
entity = next(iter(self.service.query_entities(table)))
self.assertEqual(entity.hostname, gethostname())
self.assertEqual(entity.levelname, 'INFO')
self.assertEqual(int(entity.levelno), logging.INFO)
self.assertEqual(entity.module, os.path.basename(__file__).rpartition('.')[0])
self.assertEqual(entity.name, logger_name)
self.assertEqual(int(entity.process), os.getpid())
self.assertEqual(int(entity.thread), current_thread().ident)
# confirm that there's no more entity in the table
with self.assertRaises(StopIteration):
next(entities)
def test_custom_key_formatters(self):
# get the logger for the test
logger_name = 'custom_keys'
logger = logging.getLogger(logger_name)
handler_name = _get_handler_name(logger_name)
# perform logging
log_text = 'custom key formatters test'
logging_started = datetime.now()
logger.info(log_text)
logging_finished = datetime.now()
# confirm that the entity correct log text
table = _get_handler_config_value(handler_name, 'table')
entities = iter(self.service.query_entities(table))
entity = next(entities)
self.assertEqual(entity.message, 'INFO %s' % log_text)
# confirm that the entity has a custom partitiok key
divided = self._divide_key(entity.PartitionKey)
self.assertEqual(next(divided), 'mycustompartitionkey')
self.assertEqual(next(divided), gethostname())
formatter_name = self._get_partition_key_formatter_name(handler_name)
fmt = _get_formatter_config_value(formatter_name, 'datefmt')
asctime = next(divided)
try:
self.assertEqual(asctime, logging_started.strftime(fmt))
except AssertionError:
if logging_started == logging_finished:
raise
self.assertEqual(asctime, logging_finished.strftime(fmt))
with self.assertRaises(StopIteration):
next(divided)
# confirm that the entity has a custom row key
divided = self._divide_key(entity.RowKey)
self.assertEqual(next(divided), 'mycustomrowkey')
self.assertEqual(next(divided), gethostname())
formatter_name = self._get_row_key_formatter_name(handler_name)
fmt = _get_formatter_config_value(formatter_name, 'datefmt')
asctime = next(divided)
try:
self.assertEqual(asctime, logging_started.strftime(fmt))
except AssertionError:
if logging_started == logging_finished:
raise
self.assertEqual(asctime, logging_finished.strftime(fmt))
with self.assertRaises(StopIteration):
next(divided)
# confirm that there's no more entity in the table
with self.assertRaises(StopIteration):
next(entities)
class az(object):
def __init__(self,
default_table_name=DEFAULT_TABLE,
partitionKey='default'):
self.TABLE_STORAGE_KEY = os.getenv('AZURE_STORAGE_KEY')
self.STORAGE_NAME = os.getenv('STORAGE_NAME')
self.default_table_name = default_table_name
self.default_partition = partitionKey
if self.TABLE_STORAGE_KEY == None:
from tokens import TABLE_STORAGE_ACCESS_KEY, STORAGE_ACCOUNT_NAME
self.TABLE_STORAGE_KEY = TABLE_STORAGE_ACCESS_KEY
self.STORAGE_NAME = STORAGE_ACCOUNT_NAME
self.table_service = TableService(account_name=self.STORAGE_NAME,
account_key=self.TABLE_STORAGE_KEY)
#create_table_if_does_not_exists(self.default_table_name)
def insert_or_replace_entity_to_azure(self,
rowKey,
entry,
t_name=DEFAULT_TABLE):
'''
takes table service
Takes a list
Uploads to azure table storage
'''
segment = Entity()
segment.PartitionKey = self.default_partition
segment.RowKey = str(rowKey).zfill(8)
segment.latA = str(entry['latA'])
segment.longA = str(entry['longA'])
segment.latB = str(entry['latB'])
segment.longB = str(entry['longB'])
segment.colorKey = str(entry['color'])
#print segment.colorKey
if os.name == 'nt':
self.table_service.insert_or_replace_entity(
t_name, self.default_partition,
str(rowKey).zfill(8), segment)
else:
self.table_service.insert_or_replace_entity(t_name, segment)
def create_table(self, name):
return self.table_service.create_table(name)
def delete_table(self, name):
return self.table_service.delete_table(name)
def delete_entity_by_rowKey(self, rowKey, table_name=DEFAULT_TABLE):
return self.table_service.delete_entity(table_name,
self.default_partition, rowKey)
def does_table_exist(self, table_name):
if os.name == 'nt':
for i in self.table_service.query_tables():
if i.name == table_name:
return True
else:
for i in self.table_service.list_tables():
if i.name == table_name:
return True
return False
def list_tables(self):
if os.name == 'nt':
for j in self.table_service.query_tables():
print j.name
else:
for j in self.table_service.list_tables():
print j.name
def create_table_if_does_not_exist(self, table_name=DEFAULT_TABLE):
if self.does_table_exist(table_name):
return 'already exists'
else:
self.table_service.create_table(table_name)
def create_entry(self, latA, lonA, latB, lonB, bumpiness):
x = {
'latA': latA,
'longA': lonA,
'latB': latB,
'longB': lonB,
'color': bumpiness
}
return x
def create_random_entry(self):
x = {
'latA': random.uniform(37, 38),
'longA': random.uniform(-122, -123),
'latB': random.uniform(37, 38),
'longB': random.uniform(-122, -123),
'color': random.randint(0, 7)
}
return x
def create_and_insert_or_replace_entity_azure(self,
latA,
lonA,
latB,
lonB,
bumpiness,
rowKey,
table_name=DEFAULT_TABLE):
return self.insert_or_replace_entity_to_azure(
rowKey, create_entry(latA, lonA, latB, lonB, bumpiness),
table_name)
class TableStorageHandlerTest(_TestCase):
def _divide_key(self, key):
divided = []
hostname = gethostname()
if key.find(hostname) >= 0:
preceding, hostname, remaining = key.rpartition(hostname)
preceding = preceding[:-1] if preceding.endswith(
'-') else preceding
divided.extend(preceding.split('-'))
divided.append(hostname)
remaining = remaining[1:] if remaining.startswith(
'-') else remaining
divided.extend(remaining.split('-'))
else:
divided.extend(key.split('-'))
return iter(divided)
def _get_formatter_name(self, handler_name, formatter_type):
name = _get_handler_config_value(handler_name, formatter_type)
if name:
if name.startswith('cfg://formatters.'):
name = name.split('.')[1]
return name
def _get_partition_key_formatter_name(self, handler_name):
return self._get_formatter_name(handler_name,
'partition_key_formatter')
def _get_row_key_formatter_name(self, handler_name):
return self._get_formatter_name(handler_name, 'row_key_formatter')
def setUp(self):
self.service = TableService(account_name=ACCOUNT_NAME,
account_key=ACCOUNT_KEY,
is_emulated=_EMULATED)
# ensure that there's no entity in the table before each test
tables = set()
for cfg in LOGGING['handlers'].values():
if 'table' in cfg:
tables.add(cfg['table'])
for table in self.service.list_tables():
if table.name in tables:
for entity in self.service.query_entities(table.name):
self.service.delete_entity(table.name, entity.PartitionKey,
entity.RowKey)
def test_logging(self):
# get the logger for the test
logger_name = 'table'
logger = logging.getLogger(logger_name)
handler_name = _get_handler_name(logger_name)
# perform logging
log_text = 'logging test'
logging_started = datetime.now()
logger.info(log_text)
logging_finished = datetime.now()
# confirm that the entity has correct log text
table = _get_handler_config_value(handler_name, 'table')
entities = iter(self.service.query_entities(table))
entity = next(entities)
self.assertEqual(entity.message, 'INFO %s' % log_text)
# confirm that the entity has the default partitiok key
fmt = '%Y%m%d%H%M'
try:
self.assertEqual(entity.PartitionKey,
logging_started.strftime(fmt))
except AssertionError:
if logging_started == logging_finished:
raise
self.assertEqual(entity.PartitionKey,
logging_finished.strftime(fmt))
# confirm that the entity has the default row key
divided = self._divide_key(entity.RowKey)
timestamp = next(divided)
fmt = '%Y%m%d%H%M%S'
self.assertGreaterEqual(timestamp[:-3], logging_started.strftime(fmt))
self.assertLessEqual(timestamp[:-3], logging_finished.strftime(fmt))
self.assertRegex(timestamp[-3:], '^[0-9]{3}$')
self.assertEqual(next(divided), gethostname())
self.assertEqual(int(next(divided)), os.getpid())
self.assertEqual(next(divided), '00')
with self.assertRaises(StopIteration):
next(divided)
# confirm that there's no more entity in the table
with self.assertRaises(StopIteration):
next(entities)
@unittest.skipIf(_EMULATED,
"Azure Storage Emulator doesn't support batch operation.")
def test_batch(self):
# get the logger for the test
logger_name = 'batch'
logger = logging.getLogger(logger_name)
handler_name = _get_handler_name(logger_name)
# perform logging and execute the first batch
batch_size = _get_handler_config_value(handler_name, 'batch_size')
log_text = 'batch logging test'
for i in range(batch_size + int(batch_size / 2)):
logger.info('%s#%02d' % (log_text, i))
# confirm that only batch_size entities are committed at this point
table = _get_handler_config_value(handler_name, 'table')
entities = list(iter(self.service.query_entities(table)))
self.assertEqual(len(entities), batch_size)
rowno_found = set()
seq_found = set()
for entity in entities:
# partition key
self.assertEqual(entity.PartitionKey, 'batch-%s' % gethostname())
# row key
rowno = entity.RowKey.split('-')[-1]
self.assertLess(int(rowno), batch_size)
self.assertNotIn(rowno, rowno_found)
rowno_found.add(rowno)
# message
message, seq = entity.message.split('#')
self.assertEqual(message, 'INFO %s' % log_text)
self.assertLess(int(seq), batch_size)
self.assertNotIn(seq, seq_found)
seq_found.add(seq)
# remove currently created entities before the next batch
for entity in entities:
self.service.delete_entity(table, entity.PartitionKey,
entity.RowKey)
# perform logging again and execute the next batch
for j in range(i + 1, int(batch_size / 2) + i + 1):
logger.info('%s#%02d' % (log_text, j))
# confirm that the remaining entities are committed in the next batch
entities = list(iter(self.service.query_entities(table)))
self.assertEqual(len(entities), batch_size)
rowno_found.clear()
for entity in entities:
# partition key
self.assertEqual(entity.PartitionKey, 'batch-%s' % gethostname())
# row key
rowno = entity.RowKey.split('-')[-1]
self.assertLess(int(rowno), batch_size)
self.assertNotIn(rowno, rowno_found)
rowno_found.add(rowno)
# message
message, seq = entity.message.split('#')
self.assertEqual(message, 'INFO %s' % log_text)
self.assertGreaterEqual(int(seq), batch_size)
self.assertLess(int(seq), batch_size * 2)
self.assertNotIn(seq, seq_found)
seq_found.add(seq)
def test_extra_properties(self):
# get the logger for the test
logger_name = 'extra_properties'
logger = logging.getLogger(logger_name)
handler_name = _get_handler_name(logger_name)
# perform logging
log_text = 'extra properties test'
logger.info(log_text)
# confirm that the entity has correct log text
table = _get_handler_config_value(handler_name, 'table')
entities = iter(self.service.query_entities(table))
entity = next(entities)
self.assertEqual(entity.message, 'INFO %s' % log_text)
# confirm that the extra properties have correct values
entity = next(iter(self.service.query_entities(table)))
self.assertEqual(entity.hostname, gethostname())
self.assertEqual(entity.levelname, 'INFO')
self.assertEqual(int(entity.levelno), logging.INFO)
self.assertEqual(entity.module,
os.path.basename(__file__).rpartition('.')[0])
self.assertEqual(entity.name, logger_name)
self.assertEqual(int(entity.process), os.getpid())
self.assertEqual(int(entity.thread), current_thread().ident)
# confirm that there's no more entity in the table
with self.assertRaises(StopIteration):
next(entities)
def test_custom_key_formatters(self):
# get the logger for the test
logger_name = 'custom_keys'
logger = logging.getLogger(logger_name)
handler_name = _get_handler_name(logger_name)
# perform logging
log_text = 'custom key formatters test'
logging_started = datetime.now()
logger.info(log_text)
logging_finished = datetime.now()
# confirm that the entity correct log text
table = _get_handler_config_value(handler_name, 'table')
entities = iter(self.service.query_entities(table))
entity = next(entities)
self.assertEqual(entity.message, 'INFO %s' % log_text)
# confirm that the entity has a custom partitiok key
divided = self._divide_key(entity.PartitionKey)
self.assertEqual(next(divided), 'mycustompartitionkey')
self.assertEqual(next(divided), gethostname())
formatter_name = self._get_partition_key_formatter_name(handler_name)
fmt = _get_formatter_config_value(formatter_name, 'datefmt')
asctime = next(divided)
try:
self.assertEqual(asctime, logging_started.strftime(fmt))
except AssertionError:
if logging_started == logging_finished:
raise
self.assertEqual(asctime, logging_finished.strftime(fmt))
with self.assertRaises(StopIteration):
next(divided)
# confirm that the entity has a custom row key
divided = self._divide_key(entity.RowKey)
self.assertEqual(next(divided), 'mycustomrowkey')
self.assertEqual(next(divided), gethostname())
formatter_name = self._get_row_key_formatter_name(handler_name)
fmt = _get_formatter_config_value(formatter_name, 'datefmt')
asctime = next(divided)
try:
self.assertEqual(asctime, logging_started.strftime(fmt))
except AssertionError:
if logging_started == logging_finished:
raise
self.assertEqual(asctime, logging_finished.strftime(fmt))
with self.assertRaises(StopIteration):
next(divided)
# confirm that there's no more entity in the table
with self.assertRaises(StopIteration):
next(entities)
class az(object):
def __init__(self, default_table_name=DEFAULT_TABLE, partitionKey='default'):
self.TABLE_STORAGE_KEY = os.getenv('AZURE_STORAGE_KEY')
self.STORAGE_NAME = os.getenv('STORAGE_NAME')
self.default_table_name = default_table_name
self.default_partition = partitionKey
if self.TABLE_STORAGE_KEY == None:
from tokens import TABLE_STORAGE_ACCESS_KEY, STORAGE_ACCOUNT_NAME
self.TABLE_STORAGE_KEY = TABLE_STORAGE_ACCESS_KEY
self.STORAGE_NAME = STORAGE_ACCOUNT_NAME
self.table_service = TableService(account_name=self.STORAGE_NAME, account_key=self.TABLE_STORAGE_KEY)
#create_table_if_does_not_exists(self.default_table_name)
def insert_or_replace_entity_to_azure(self, rowKey, entry, t_name=DEFAULT_TABLE):
'''
takes table service
Takes a list
Uploads to azure table storage
'''
segment = Entity()
segment.PartitionKey = self.default_partition
segment.RowKey = str(rowKey).zfill(8)
segment.latA = str(entry['latA'])
segment.longA = str(entry['longA'])
segment.latB = str(entry['latB'])
segment.longB = str(entry['longB'])
segment.colorKey = str(entry['color'])
#print segment.colorKey
if os.name == 'nt':
self.table_service.insert_or_replace_entity(t_name, self.default_partition, str(rowKey).zfill(8), segment)
else:
self.table_service.insert_or_replace_entity(t_name, segment)
def create_table(self, name):
return self.table_service.create_table(name)
def delete_table(self, name):
return self.table_service.delete_table(name)
def delete_entity_by_rowKey(self, rowKey, table_name=DEFAULT_TABLE):
return self.table_service.delete_entity(table_name, self.default_partition, rowKey)
def does_table_exist(self, table_name):
if os.name == 'nt':
for i in self.table_service.query_tables():
if i.name == table_name:
return True
else:
for i in self.table_service.list_tables():
if i.name == table_name:
return True
return False
def list_tables(self):
if os.name == 'nt':
for j in self.table_service.query_tables():
print j.name
else:
for j in self.table_service.list_tables():
print j.name
def create_table_if_does_not_exist(self, table_name=DEFAULT_TABLE):
if self.does_table_exist(table_name):
return 'already exists'
else:
self.table_service.create_table(table_name)
def create_entry(self, latA, lonA, latB, lonB, bumpiness):
x = {
'latA':latA,
'longA':lonA,
'latB':latB,
'longB':lonB,
'color': bumpiness
}
return x
def create_random_entry(self):
x = {
'latA':random.uniform(37,38),
'longA':random.uniform(-122,-123),
'latB':random.uniform(37,38),
'longB':random.uniform(-122,-123),
'color': random.randint(0,7)
}
return x
def create_and_insert_or_replace_entity_azure(self, latA, lonA, latB, lonB, bumpiness, rowKey, table_name=DEFAULT_TABLE ):
return self.insert_or_replace_entity_to_azure(rowKey, create_entry(latA, lonA, latB, lonB, bumpiness), table_name)
