class TableStorageHandler(logging.Handler): """ Handler class which writes log messages to a Azure Storage table. """ MAX_BATCH_SIZE = 100 def __init__(self, account_name=None, account_key=None, protocol='https', table='logs', batch_size=0, extra_properties=None, partition_key_formatter=None, row_key_formatter=None, ): """ Initialize the handler. """ logging.Handler.__init__(self) self.service = TableService(account_name=account_name, account_key=account_key, protocol=protocol) self.meta = {'hostname': gethostname(), 'process': os.getpid()} self.table = _formatName(table, self.meta) self.ready = False self.rowno = 0 if not partition_key_formatter: # default format for partition keys fmt = '%(asctime)s' datefmt = '%Y%m%d%H%M' partition_key_formatter = logging.Formatter(fmt, datefmt) self.partition_key_formatter = partition_key_formatter if not row_key_formatter: # default format for row keys fmt = '%(asctime)s%(msecs)03d-%(hostname)s-%(process)d-%(rowno)02d' datefmt = '%Y%m%d%H%M%S' row_key_formatter = logging.Formatter(fmt, datefmt) self.row_key_formatter = row_key_formatter # extra properties and formatters for them self.extra_properties = extra_properties if extra_properties: self.extra_property_formatters = {} self.extra_property_names = {} for extra in extra_properties: if _PY3: f = logging.Formatter(fmt=extra, style=extra[0]) else: f = logging.Formatter(fmt=extra) self.extra_property_formatters[extra] = f self.extra_property_names[extra] = self._getFormatName(extra) # the storage emulator doesn't support batch operations if batch_size <= 1 or self.service.use_local_storage: self.batch = False else: self.batch = True if batch_size > TableStorageHandler.MAX_BATCH_SIZE: self.batch_size = TableStorageHandler.MAX_BATCH_SIZE else: self.batch_size = batch_size if self.batch: self.current_partition_key = None def _copyLogRecord(self, record): copy = logging.makeLogRecord(record.__dict__) copy.exc_info = None copy.exc_text = None if _PY3: copy.stack_info = None return copy def _getFormatName(self, extra): name = extra style = extra[0] if style == '%': name = extra[2:extra.index(')')] elif _PY3: if style == '{': name = next(string.Formatter().parse(extra))[1] elif style == '$': name = extra[1:] if name.startswith('{'): name = name[1:-1] return name def emit(self, record): """ Emit a record. Format the record and send it to the specified table. """ try: if not self.ready: self.service.create_table(self.table) if self.batch: self.service.begin_batch() self.ready = True # generate partition key for the entity record.hostname = self.meta['hostname'] copy = self._copyLogRecord(record) partition_key = self.partition_key_formatter.format(copy) # ensure entities in the batch all have the same patition key if self.batch: if self.current_partition_key is not None: if partition_key != self.current_partition_key: self.flush() self.current_partition_key = partition_key # add log message and extra properties to the entity entity = {} if self.extra_properties: for extra in self.extra_properties: formatter = self.extra_property_formatters[extra] name = self.extra_property_names[extra] entity[name] = formatter.format(copy) entity['message'] = self.format(record) # generate row key for the entity copy.rowno = self.rowno row_key = self.row_key_formatter.format(copy) # add entitiy to the table self.service.insert_or_replace_entity(self.table, partition_key, row_key, entity) # commit the ongoing batch if it reaches the high mark if self.batch: self.rowno += 1 if self.rowno >= self.batch_size: self.flush() except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def flush(self): """ Ensure all logging output has been flushed. """ if self.batch and self.rowno > 0: try: self.service.commit_batch() finally: self.rowno = 0 self.service.begin_batch() def setFormatter(self, fmt): """ Set the message formatter. """ super(TableStorageHandler, self).setFormatter(fmt) if self.extra_properties: logging._acquireLock() try: for extra in self.extra_property_formatters.values(): extra.converter = fmt.converter extra.datefmt = fmt.datefmt if _PY3: extra.default_time_format = fmt.default_time_format extra.default_msec_format = fmt.default_msec_format finally: logging._releaseLock() def setPartitionKeyFormatter(self, fmt): """ Set the partition key formatter. """ self.partition_key_formatter = fmt def setRowKeyFormatter(self, fmt): """ Set the row key formatter. """ self.row_key_formatter = fmt
class TableStorageHandler(logging.Handler): """ Handler class which writes log messages to a Azure Storage table. """ MAX_BATCH_SIZE = 100 def __init__(self, account_name=None, account_key=None, protocol='https', table='logs', batch_size=0, extra_properties=None, partition_key_formatter=None, row_key_formatter=None, ): """ Initialize the handler. """ logging.Handler.__init__(self) self.service = TableService(account_name=account_name, account_key=account_key, protocol=protocol) self.meta = {'hostname': gethostname(), 'process': os.getpid()} self.table = _formatName(table, self.meta) self.ready = False self.rowno = 0 if not partition_key_formatter: # default format for partition keys fmt = '%(asctime)s' datefmt = '%Y%m%d%H%M' partition_key_formatter = logging.Formatter(fmt, datefmt) self.partition_key_formatter = partition_key_formatter if not row_key_formatter: # default format for row keys fmt = '%(asctime)s%(msecs)03d-%(hostname)s-%(process)d-%(rowno)02d' datefmt = '%Y%m%d%H%M%S' row_key_formatter = logging.Formatter(fmt, datefmt) self.row_key_formatter = row_key_formatter # extra properties and formatters for them self.extra_properties = extra_properties if extra_properties: self.extra_property_formatters = {} self.extra_property_names = {} for extra in extra_properties: if _PY3: f = logging.Formatter(fmt=extra, style=extra[0]) else: f = logging.Formatter(fmt=extra) self.extra_property_formatters[extra] = f self.extra_property_names[extra] = self._getFormatName(extra) # the storage emulator doesn't support batch operations if batch_size <= 1 or self.service.use_local_storage: self.batch = False else: self.batch = True if batch_size > TableStorageHandler.MAX_BATCH_SIZE: self.batch_size = TableStorageHandler.MAX_BATCH_SIZE else: self.batch_size = batch_size if self.batch: self.current_partition_key = None def _copyLogRecord(self, record): copy = logging.makeLogRecord(record.__dict__) copy.exc_info = None copy.exc_text = None if _PY3: copy.stack_info = None return copy def _getFormatName(self, extra): name = extra style = extra[0] if style == '%': name = extra[2:extra.index(')')] elif _PY3: if style == '{': name = next(string.Formatter().parse(extra))[1] elif style == '$': name = extra[1:] if name.startswith('{'): name = name[1:-1] return name def emit(self, record): """ Emit a record. Format the record and send it to the specified table. """ try: if not self.ready: self.service.create_table(self.table) if self.batch: self.service.begin_batch() self.ready = True # generate partition key for the entity record.hostname = self.meta['hostname'] copy = self._copyLogRecord(record) partition_key = self.partition_key_formatter.format(copy) # ensure entities in the batch all have the same patition key if self.batch: if self.current_partition_key is not None: if partition_key != self.current_partition_key: self.flush() self.current_partition_key = partition_key # add log message and extra properties to the entity entity = {} if self.extra_properties: for extra in self.extra_properties: formatter = self.extra_property_formatters[extra] name = self.extra_property_names[extra] entity[name] = formatter.format(copy) entity['message'] = self.format(record) # generate row key for the entity copy.rowno = self.rowno row_key = self.row_key_formatter.format(copy) # add entitiy to the table self.service.insert_or_replace_entity(self.table, partition_key, row_key, entity) # commit the ongoing batch if it reaches the high mark if self.batch: self.rowno += 1 if self.rowno >= self.batch_size: self.flush() except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def flush(self): """ Ensure all logging output has been flushed. """ if self.batch and self.rowno > 0: try: self.service.commit_batch() finally: self.rowno = 0 self.service.begin_batch() def setFormatter(self, fmt): """ Set the message formatter. """ super(TableStorageHandler, self).setFormatter(fmt) if self.extra_properties: logging._acquireLock() try: for extra in self.extra_property_formatters.values(): extra.converter = fmt.converter extra.datefmt = fmt.datefmt if _PY3: extra.default_time_format = fmt.default_time_format extra.default_msec_format = fmt.default_msec_format finally: logging._releaseLock() def setPartitionKeyFormatter(self, fmt): """ Set the partition key formatter. """ self.partition_key_formatter = fmt def setRowKeyFormatter(self, fmt): """ Set the row key formatter. """ self.row_key_formatter = fmt
task = {'description': 'Buy detergent', 'priority': 300} table_service.insert_or_replace_entity('tasktable', 'tasksSeattle', '3', task) task10 = { 'PartitionKey': 'tasksSeattle', 'RowKey': '10', 'description': 'Go grocery shopping', 'priority': 400 } task11 = { 'PartitionKey': 'tasksSeattle', 'RowKey': '11', 'description': 'Clean the bathroom', 'priority': 100 } table_service.begin_batch() table_service.insert_entity('tasktable', task10) table_service.insert_entity('tasktable', task11) table_service.commit_batch() task = table_service.get_entity('tasktable', 'tasksSeattle', '1') print(task.description) print(task.priority) tasks = table_service.query_entities('tasktable', "PartitionKey eq 'tasksSeattle'") for task in tasks: print(task.description) print(task.priority) tasks = table_service.query_entities('tasktable',
task.description = 'Wash the car' task.priority = 100 table_service.insert_entity('tasktable', task) # task = {'description' : 'Take out the garbage', 'priority' : 250} table_service.update_entity('tasktable', 'tasksSeattle', '1', task) # task = {'description' : 'Take out the garbage again', 'priority' : 250} table_service.insert_or_replace_entity('tasktable', 'tasksSeattle', '1', task) task = {'description' : 'Buy detergent', 'priority' : 300} table_service.insert_or_replace_entity('tasktable', 'tasksSeattle', '3', task) task10 = {'PartitionKey': 'tasksSeattle', 'RowKey': '10', 'description' : 'Go grocery shopping', 'priority' : 400} task11 = {'PartitionKey': 'tasksSeattle', 'RowKey': '11', 'description' : 'Clean the bathroom', 'priority' : 100} table_service.begin_batch() table_service.insert_entity('tasktable', task10) table_service.insert_entity('tasktable', task11) table_service.commit_batch() task = table_service.get_entity('tasktable', 'tasksSeattle', '1') print(task.description) print(task.priority) tasks = table_service.query_entities('tasktable', "PartitionKey eq 'tasksSeattle'") for task in tasks: print(task.description) print(task.priority) tasks = table_service.query_entities('tasktable', "PartitionKey eq 'tasksSeattle'", 'description') for task in tasks: