def test_batch_insert_merge(self): # Arrange # Act entity = Entity() entity.PartitionKey = '001' entity.RowKey = 'batch_insert_merge' entity.test = EntityProperty(EdmType.BOOLEAN, 'true') entity.test2 = 'value' entity.test3 = 3 entity.test4 = EntityProperty(EdmType.INT64, '1234567890') entity.test5 = datetime.utcnow() batch = TableBatch() batch.insert_or_merge_entity(entity) resp = self.ts.commit_batch(self.table_name, batch) # Assert self.assertIsNotNone(resp) entity = self.ts.get_entity(self.table_name, '001', 'batch_insert_merge') self.assertIsNotNone(entity) self.assertEqual('value', entity.test2) self.assertEqual(1234567890, entity.test4) self.assertEqual(resp[0], entity.etag)
def test_batch_insert_merge(self): # Arrange # Act entity = Entity() entity.PartitionKey = '001' entity.RowKey = 'batch_insert_merge' entity.test = EntityProperty(EdmType.BOOLEAN, 'true') entity.test2 = 'value' entity.test3 = 3 entity.test4 = EntityProperty(EdmType.INT64, '1234567890') entity.test5 = datetime.utcnow() batch = TableBatch() batch.insert_or_merge_entity(entity) resp = self.ts.commit_batch(self.table_name, batch) # Assert self.assertIsNotNone(resp) entity = self.ts.get_entity( self.table_name, '001', 'batch_insert_merge') self.assertIsNotNone(entity) self.assertEqual('value', entity.test2) self.assertEqual(1234567890, entity.test4) self.assertEqual(resp[0], entity.etag)
def batch(self): table_name = self._create_table() entity = Entity() entity.PartitionKey = 'batch' entity.test = True # All operations in the same batch must have the same partition key but different row keys # Batches can hold from 1 to 100 entities # Batches are atomic. All operations completed simulatenously. If one operation fails, they all fail. # Insert, update, merge, insert or merge, insert or replace, and delete entity operations are supported # Context manager style with self.service.batch(table_name) as batch: for i in range(0, 5): entity.RowKey = 'context_{}'.format(i) batch.insert_entity(entity) # Commit style batch = TableBatch() for i in range(0, 5): entity.RowKey = 'commit_{}'.format(i) batch.insert_entity(entity) self.service.commit_batch(table_name, batch) self.service.delete_table(table_name)
def test_query_entities_large(self): # Arrange table_name = self._create_query_table(0) total_entities_count = 1000 entities_per_batch = 50 for j in range(total_entities_count // entities_per_batch): batch = TableBatch() for i in range(entities_per_batch): entity = Entity() entity.PartitionKey = 'large' entity.RowKey = 'batch{0}-item{1}'.format(j, i) entity.test = EntityProperty(EdmType.BOOLEAN, 'true') entity.test2 = 'hello world;' * 100 entity.test3 = 3 entity.test4 = EntityProperty(EdmType.INT64, '1234567890') entity.test5 = datetime(2016, 12, 31, 11, 59, 59, 0) batch.insert_entity(entity) self.ts.commit_batch(table_name, batch) # Act start_time = datetime.now() entities = list(self.ts.query_entities(table_name)) elapsed_time = datetime.now() - start_time # Assert print('query_entities took {0} secs.'.format( elapsed_time.total_seconds())) # azure allocates 5 seconds to execute a query # if it runs slowly, it will return fewer results and make the test fail self.assertEqual(len(entities), total_entities_count)
def test_query_entities_large(self): # Arrange table_name = self._create_query_table(0) total_entities_count = 1000 entities_per_batch = 50 for j in range(total_entities_count // entities_per_batch): batch = TableBatch() for i in range(entities_per_batch): entity = Entity() entity.PartitionKey = 'large' entity.RowKey = 'batch{0}-item{1}'.format(j, i) entity.test = EntityProperty(EdmType.BOOLEAN, 'true') entity.test2 = 'hello world;' * 100 entity.test3 = 3 entity.test4 = EntityProperty(EdmType.INT64, '1234567890') entity.test5 = datetime(2016, 12, 31, 11, 59, 59, 0) batch.insert_entity(entity) self.ts.commit_batch(table_name, batch) # Act start_time = datetime.now() entities = list(self.ts.query_entities(table_name)) elapsed_time = datetime.now() - start_time # Assert print('query_entities took {0} secs.'.format(elapsed_time.total_seconds())) # azure allocates 5 seconds to execute a query # if it runs slowly, it will return fewer results and make the test fail self.assertEqual(len(entities), total_entities_count)
def test_batch_update_if_doesnt_match(self): # Arrange entity = self._create_default_entity_dict() self.ts.insert_entity(self.table_name, entity) # Act sent_entity1 = self._create_updated_entity_dict( entity['PartitionKey'], entity['RowKey']) batch = TableBatch() batch.update_entity( sent_entity1, if_match=u'W/"datetime\'2012-06-15T22%3A51%3A44.9662825Z\'"') try: self.ts.commit_batch(self.table_name, batch) except AzureBatchOperationError as error: self.assertEqual(error.code, 'UpdateConditionNotSatisfied') self.assertTrue( str(error).startswith( 'The update condition specified in the request was not satisfied.' )) else: self.fail('AzureBatchOperationError was expected') # Assert received_entity = self.ts.get_entity(self.table_name, entity['PartitionKey'], entity['RowKey']) self._assert_default_entity(received_entity)
def __init__( self, account_name=None, account_key=None, protocol='https', table='logs', batch_size=0, extra_properties=None, partition_key_formatter=None, row_key_formatter=None, is_emulated=False, ): """ Initialize the handler. """ logging.Handler.__init__(self) self.service = TableService(account_name=account_name, account_key=account_key, is_emulated=is_emulated, protocol=protocol) self.meta = {'hostname': gethostname(), 'process': os.getpid()} self.table = _formatName(table, self.meta) self.ready = False self.rowno = 0 if not partition_key_formatter: # default format for partition keys fmt = '%(asctime)s' datefmt = '%Y%m%d%H%M' partition_key_formatter = logging.Formatter(fmt, datefmt) self.partition_key_formatter = partition_key_formatter if not row_key_formatter: # default format for row keys fmt = '%(asctime)s%(msecs)03d-%(hostname)s-%(process)d-%(rowno)02d' datefmt = '%Y%m%d%H%M%S' row_key_formatter = logging.Formatter(fmt, datefmt) self.row_key_formatter = row_key_formatter # extra properties and formatters for them self.extra_properties = extra_properties if extra_properties: self.extra_property_formatters = {} self.extra_property_names = {} for extra in extra_properties: if _PY3: f = logging.Formatter(fmt=extra, style=extra[0]) else: f = logging.Formatter(fmt=extra) self.extra_property_formatters[extra] = f self.extra_property_names[extra] = self._getFormatName(extra) # the storage emulator doesn't support batch operations if batch_size <= 1 or is_emulated: self.batch = None else: self.batch = TableBatch() if batch_size > TableStorageHandler.MAX_BATCH_SIZE: self.batch_size = TableStorageHandler.MAX_BATCH_SIZE else: self.batch_size = batch_size if self.batch: self.current_partition_key = None
def flush(self): """ Ensure all logging output has been flushed. """ if self.batch and self.rowno > 0: try: self.service.commit_batch(self.table, self.batch) finally: self.rowno = 0 self.batch = TableBatch()
def test_batch_strict_mode(self): # Arrange self.ts.require_encryption = True entity = self._create_default_entity_for_encryption() # Act batch = TableBatch(require_encryption=True) # Assert with self.assertRaises(ValueError): batch.insert_entity(entity)
def test_batch_too_many_ops(self): # Arrange entity = self._create_default_entity_dict('001', 'batch_negative_1') self.ts.insert_entity(self.table_name, entity) # Act with self.assertRaises(AzureBatchValidationError): batch = TableBatch() for i in range(0, 101): entity = Entity() entity.PartitionKey = 'large' entity.RowKey = 'item{0}'.format(i) batch.insert_entity(entity) self.ts.commit_batch(self.table_name, batch)
def test_invalid_encryption_operations_fail_batch(self): # Arrange entity = self._create_default_entity_for_encryption() self.ts.key_encryption_key = KeyWrapper('key1') self.ts.insert_entity(self.table_name, entity) # Act batch = TableBatch(require_encryption=True, key_encryption_key=self.ts.key_encryption_key) # Assert with self.assertRaises(ValueError): batch.merge_entity(entity) with self.assertRaises(ValueError): batch.insert_or_merge_entity(entity)
def test_batch_update_if_match(self): # Arrange entity = self._create_default_entity_dict() etag = self.ts.insert_entity(self.table_name, entity) # Act sent_entity = self._create_updated_entity_dict(entity['PartitionKey'], entity['RowKey']) batch = TableBatch() batch.update_entity(sent_entity, etag) resp = self.ts.commit_batch(self.table_name, batch) # Assert self.assertIsNotNone(resp) received_entity = self.ts.get_entity(self.table_name, entity['PartitionKey'], entity['RowKey']) self._assert_updated_entity(received_entity) self.assertEqual(resp[0], received_entity.etag)
def test_batch_different_partition_operations_fail(self): # Arrange entity = self._create_default_entity_dict('001', 'batch_negative_1') self.ts.insert_entity(self.table_name, entity) # Act with self.assertRaises(AzureBatchValidationError): batch = TableBatch() entity = self._create_updated_entity_dict( '001', 'batch_negative_1') batch.update_entity(entity) entity = self._create_default_entity_dict( '002', 'batch_negative_1') batch.insert_entity(entity)
def __init__(self, account_name=None, account_key=None, protocol='https', table='logs', batch_size=0, extra_properties=None, partition_key_formatter=None, row_key_formatter=None, is_emulated=False, ): """ Initialize the handler. """ logging.Handler.__init__(self) self.service = TableService(account_name=account_name, account_key=account_key, is_emulated=is_emulated, protocol=protocol) self.meta = {'hostname': gethostname(), 'process': os.getpid()} self.table = _formatName(table, self.meta) self.ready = False self.rowno = 0 if not partition_key_formatter: # default format for partition keys fmt = '%(asctime)s' datefmt = '%Y%m%d%H%M' partition_key_formatter = logging.Formatter(fmt, datefmt) self.partition_key_formatter = partition_key_formatter if not row_key_formatter: # default format for row keys fmt = '%(asctime)s%(msecs)03d-%(hostname)s-%(process)d-%(rowno)02d' datefmt = '%Y%m%d%H%M%S' row_key_formatter = logging.Formatter(fmt, datefmt) self.row_key_formatter = row_key_formatter # extra properties and formatters for them self.extra_properties = extra_properties if extra_properties: self.extra_property_formatters = {} self.extra_property_names = {} for extra in extra_properties: if _PY3: f = logging.Formatter(fmt=extra, style=extra[0]) else: f = logging.Formatter(fmt=extra) self.extra_property_formatters[extra] = f self.extra_property_names[extra] = self._getFormatName(extra) # the storage emulator doesn't support batch operations if batch_size <= 1 or is_emulated: self.batch = None else: self.batch = TableBatch() if batch_size > TableStorageHandler.MAX_BATCH_SIZE: self.batch_size = TableStorageHandler.MAX_BATCH_SIZE else: self.batch_size = batch_size if self.batch: self.current_partition_key = None
def delete_persongroup(cls, persongroupid): try: # delete partition key from azure table entities = cls.table_service.query_entities( table_name=config.AZURESTORAGE_TABLENAME, filter="PartitionKey eq '{}'".format(persongroupid)) batch = TableBatch() for entity in entities.items: batch.delete_entity(partition_key=entity["PartitionKey"], row_key=entity["RowKey"]) cls.table_service.commit_batch( table_name=config.AZURESTORAGE_TABLENAME, batch=batch) CF.person_group.delete(persongroupid) except Exception as e: logger.error(traceback.format_exc()) raise e
def _setup_tables(account_name, account_key, table_name, batch_size=100, max_num=1000000): table_service = TableService(account_name, account_key) table_service.create_table(table_name) partitions = defaultdict(list) for num in range(1, max_num + 1): partitions[('%03d' % num)[:3]].append(str(num)) for partition, nums in partitions.items(): for batch_num, batch in enumerate(_grouper(nums, batch_size), start=1): table_batch = TableBatch() for num in filter(None, batch): table_batch.insert_entity({ 'PartitionKey': partition, 'RowKey': num, 'value': str(uuid4()), }) table_service.commit_batch(table_name, table_batch) print('Done with partition %s, batch %d' % (partition, batch_num))
def _batch_upload(self, table_name, post_entities): # Context manager style if (len(post_entities) == 0): return if (len(post_entities) > 100): raise ValueError("batch cannot be over 100 entries") # Context manager style # with self.service.batch(table_name) as batch: # for entity in post_entities: # batch.insert_entity(entity) # Commit style batch = TableBatch() for entity in post_entities: if (entity == None): print "entity none" break batch.insert_entity(entity) self.service.commit_batch(table_name, batch)
def batch_encrypted_entities(self): table_name = self._create_table() entity1 = self._create_entity_for_encryption() entity2 = self._create_entity_for_encryption() entity2['PartitionKey'] = entity1['PartitionKey'] # Batches will encrypt the entities at the time of inserting into the batch, not # committing the batch to the service, so the encryption policy must be # passed in at the time of batch creation. kek = KeyWrapper('key1') batch = TableBatch(require_encryption=True, key_encryption_key=kek) batch.insert_entity(entity1) batch.insert_entity(entity2) self.service.commit_batch(table_name, batch) # When using the batch as a context manager, the tableservice object will # automatically apply its encryption policy to the batch. entity3 = self._create_entity_for_encryption() entity4 = self._create_entity_for_encryption() entity4['PartitionKey'] = entity3['PartitionKey'] self.service.key_encryption_key = KeyWrapper('key1') with self.service.batch(table_name) as batch: batch.insert_entity(entity3) batch.insert_entity(entity4) # Note that batches follow all the same client-side-encryption behavior as # the corresponding individual table operations. self.service.delete_table(table_name)
def test_batch_inserts(self): # Arrange # Act entity = Entity() entity.PartitionKey = 'batch_inserts' entity.test = EntityProperty(EdmType.BOOLEAN, 'true') entity.test2 = 'value' entity.test3 = 3 entity.test4 = EntityProperty(EdmType.INT64, '1234567890') batch = TableBatch() for i in range(100): entity.RowKey = str(i) batch.insert_entity(entity) self.ts.commit_batch(self.table_name, batch) entities = list(self.ts.query_entities(self.table_name, "PartitionKey eq 'batch_inserts'", '')) # Assert self.assertIsNotNone(entities) self.assertEqual(100, len(entities))
def test_batch_update_if_doesnt_match(self): # Arrange entity = self._create_default_entity_dict() self.ts.insert_entity(self.table_name, entity) # Act sent_entity1 = self._create_updated_entity_dict(entity['PartitionKey'], entity['RowKey']) batch = TableBatch() batch.update_entity( sent_entity1, if_match=u'W/"datetime\'2012-06-15T22%3A51%3A44.9662825Z\'"') try: self.ts.commit_batch(self.table_name, batch) except AzureBatchOperationError as error: self.assertEqual(error.code, 'UpdateConditionNotSatisfied') self.assertTrue(str(error).startswith('The update condition specified in the request was not satisfied.')) else: self.fail('AzureBatchOperationError was expected') # Assert received_entity = self.ts.get_entity(self.table_name, entity['PartitionKey'], entity['RowKey']) self._assert_default_entity(received_entity)
def test_batch_delete(self): # Arrange # Act entity = Entity() entity.PartitionKey = '001' entity.RowKey = 'batch_delete' entity.test = EntityProperty(EdmType.BOOLEAN, 'true') entity.test2 = 'value' entity.test3 = 3 entity.test4 = EntityProperty(EdmType.INT64, '1234567890') entity.test5 = datetime.utcnow() self.ts.insert_entity(self.table_name, entity) entity = self.ts.get_entity(self.table_name, '001', 'batch_delete') self.assertEqual(3, entity.test3) batch = TableBatch() batch.delete_entity('001', 'batch_delete') resp = self.ts.commit_batch(self.table_name, batch) # Assert self.assertIsNotNone(resp) self.assertIsNone(resp[0])
def test_batch_inserts(self): # Arrange # Act entity = Entity() entity.PartitionKey = 'batch_inserts' entity.test = EntityProperty(EdmType.BOOLEAN, 'true') entity.test2 = 'value' entity.test3 = 3 entity.test4 = EntityProperty(EdmType.INT64, '1234567890') batch = TableBatch() for i in range(100): entity.RowKey = str(i) batch.insert_entity(entity) self.ts.commit_batch(self.table_name, batch) entities = list( self.ts.query_entities(self.table_name, "PartitionKey eq 'batch_inserts'", '')) # Assert self.assertIsNotNone(entities) self.assertEqual(100, len(entities))
def test_batch_different_partition_operations_fail(self): # Arrange entity = self._create_default_entity_dict('001', 'batch_negative_1') self.ts.insert_entity(self.table_name, entity) # Act with self.assertRaises(AzureBatchValidationError): batch = TableBatch() entity = self._create_updated_entity_dict('001', 'batch_negative_1') batch.update_entity(entity) entity = self._create_default_entity_dict('002', 'batch_negative_1') batch.insert_entity(entity)
def wait_for_tasks_to_complete(table_service, batch_client, entity_pk, entity_rk, job_id): """ Returns when all tasks in the specified job reach the Completed state. """ while True: entity = table_service.get_entity('SearchEntity', entity_pk, entity_rk) tasks = batch_client.task.list(job_id) incomplete_tasks = [ task for task in tasks if task.id != 'JobManager' and task.state != batchmodels.TaskState.completed ] complete_tasks = [ task for task in tasks if task.id != 'JobManager' and task.state == batchmodels.TaskState.completed ] failed_tasks = [ task for task in complete_tasks if task.execution_info.exit_code != 0 or task.execution_info.scheduling_error is not None ] queries = table_service.query_entities( 'SearchQueryEntity', filter="PartitionKey eq '{}'".format(entity.RowKey)) current_batch_count = 0 updateBatch = TableBatch() for task in tasks: matching_queries = [q for q in queries if q.RowKey == task.id] if not matching_queries: print('Could not find query {}'.format(task.id)) continue query = matching_queries[0] update = False state = get_query_state(task) if query._State != state: query._State = state update = True if task.state == batchmodels.TaskState.running: if not hasattr(query, 'StartTime'): query.StartTime = task.execution_info.start_time update = True if task.state == batchmodels.TaskState.completed: if not hasattr(query, 'EndTime'): query.EndTime = task.execution_info.end_time update = True if update: updateBatch.update_entity(query) current_batch_count += 1 if current_batch_count == 99: table_service.commit_batch('SearchQueryEntity', updateBatch) current_batch_count = 0 updateBatch = TableBatch() if current_batch_count > 0: table_service.commit_batch('SearchQueryEntity', updateBatch) all_tasks_complete = not incomplete_tasks any_failures = len(failed_tasks) > 0 entity.CompletedTasks = len(complete_tasks) entity._State = get_search_state(all_tasks_complete, any_failures) if not incomplete_tasks: entity.EndTime = datetime.datetime.utcnow() table_service.update_entity('SearchEntity', entity) return else: table_service.update_entity('SearchEntity', entity) time.sleep(5)
import vk, urllib.request, csv, json, pprint, time from pprint import pprint from azure.storage.table import TableService, Entity, TableBatch table_service = TableService( account_name='seva', account_key= 'SgbxLwWkBH4XuGebxECoXfNVG3mVM5YjOs+SWTDUSacc+3YgUmcafYXrXdz5k0HtlZQ3AuEJ1IcFtZYeGVR9Hw==' ) batch = TableBatch() #table_service.delete_table('MyVkApp') #table_service.create_table('MyVkApp') login = input("Введите имя пользователя: ") password = input("Введите пароль: ") session = vk.AuthSession(app_id='5889724', user_login=login, user_password=password) api = vk.API(session) followers = api.users.getFollowers(count='') #table_service.create_table('MyVkApp') k = 0 for d in followers: user = api.users.get(user_ids=d) k = k + 1 # writer.writerow({'first_name': user[0]['first_name'], 'last_name': user[0]['last_name']}) followers_info = { 'PartitionKey': 'my_followers', 'RowKey': str(k), 'first_name': user[0]['first_name'], 'last_name': user[0]['last_name'] }
def _insertOrReplaceEntityViaBatch(entity, batch: TableBatch): return batch.insert_or_replace_entity(entity)
def _insertEntityViaBatch(entity, batch: TableBatch): return batch.insert_entity(entity)
class TableStorageHandler(logging.Handler): """ Handler class which writes log messages to a Azure Storage table. """ MAX_BATCH_SIZE = 100 def __init__(self, account_name=None, account_key=None, protocol='https', table='logs', batch_size=0, extra_properties=None, partition_key_formatter=None, row_key_formatter=None, is_emulated=False, ): """ Initialize the handler. """ logging.Handler.__init__(self) self.service = TableService(account_name=account_name, account_key=account_key, is_emulated=is_emulated, protocol=protocol) self.meta = {'hostname': gethostname(), 'process': os.getpid()} self.table = _formatName(table, self.meta) self.ready = False self.rowno = 0 if not partition_key_formatter: # default format for partition keys fmt = '%(asctime)s' datefmt = '%Y%m%d%H%M' partition_key_formatter = logging.Formatter(fmt, datefmt) self.partition_key_formatter = partition_key_formatter if not row_key_formatter: # default format for row keys fmt = '%(asctime)s%(msecs)03d-%(hostname)s-%(process)d-%(rowno)02d' datefmt = '%Y%m%d%H%M%S' row_key_formatter = logging.Formatter(fmt, datefmt) self.row_key_formatter = row_key_formatter # extra properties and formatters for them self.extra_properties = extra_properties if extra_properties: self.extra_property_formatters = {} self.extra_property_names = {} for extra in extra_properties: if _PY3: f = logging.Formatter(fmt=extra, style=extra[0]) else: f = logging.Formatter(fmt=extra) self.extra_property_formatters[extra] = f self.extra_property_names[extra] = self._getFormatName(extra) # the storage emulator doesn't support batch operations if batch_size <= 1 or is_emulated: self.batch = None else: self.batch = TableBatch() if batch_size > TableStorageHandler.MAX_BATCH_SIZE: self.batch_size = TableStorageHandler.MAX_BATCH_SIZE else: self.batch_size = batch_size if self.batch: self.current_partition_key = None def _copyLogRecord(self, record): copy = logging.makeLogRecord(record.__dict__) copy.exc_info = None copy.exc_text = None if _PY3: copy.stack_info = None return copy def _getFormatName(self, extra): name = extra style = extra[0] if style == '%': name = extra[2:extra.index(')')] elif _PY3: if style == '{': name = next(string.Formatter().parse(extra))[1] elif style == '$': name = extra[1:] if name.startswith('{'): name = name[1:-1] return name def emit(self, record): """ Emit a record. Format the record and send it to the specified table. """ try: if not self.ready: self.service.create_table(self.table) self.ready = True # generate partition key for the entity record.hostname = self.meta['hostname'] copy = self._copyLogRecord(record) partition_key = self.partition_key_formatter.format(copy) # ensure entities in the batch all have the same patition key if self.batch: if self.current_partition_key is not None: if partition_key != self.current_partition_key: self.flush() self.current_partition_key = partition_key # add log message and extra properties to the entity entity = {} if self.extra_properties: for extra in self.extra_properties: formatter = self.extra_property_formatters[extra] name = self.extra_property_names[extra] entity[name] = formatter.format(copy) entity['message'] = self.format(record) # generate row key for the entity copy.rowno = self.rowno row_key = self.row_key_formatter.format(copy) # add entitiy to the table entity['PartitionKey'] = partition_key entity['RowKey'] = row_key if not self.batch: self.service.insert_or_replace_entity(self.table, entity) else: self.batch.insert_or_replace_entity(entity) # commit the ongoing batch if it reaches the high mark self.rowno += 1 if self.rowno >= self.batch_size: self.flush() except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def flush(self): """ Ensure all logging output has been flushed. """ if self.batch and self.rowno > 0: try: self.service.commit_batch(self.table, self.batch) finally: self.rowno = 0 self.batch = TableBatch() def setFormatter(self, fmt): """ Set the message formatter. """ super(TableStorageHandler, self).setFormatter(fmt) if self.extra_properties: logging._acquireLock() try: for extra in self.extra_property_formatters.values(): extra.converter = fmt.converter extra.datefmt = fmt.datefmt if _PY3: extra.default_time_format = fmt.default_time_format extra.default_msec_format = fmt.default_msec_format finally: logging._releaseLock() def setPartitionKeyFormatter(self, fmt): """ Set the partition key formatter. """ self.partition_key_formatter = fmt def setRowKeyFormatter(self, fmt): """ Set the row key formatter. """ self.row_key_formatter = fmt
import vk, urllib.request, csv, json, pprint, time from pprint import pprint from azure.storage.table import TableService, Entity, TableBatch table_service = TableService( account_name='seva', account_key= 'SgbxLwWkBH4XuGebxECoXfNVG3mVM5YjOs+SWTDUSacc+3YgUmcafYXrXdz5k0HtlZQ3AuEJ1IcFtZYeGVR9Hw==' ) batch = TableBatch() #table_service.delete_table('MyVkApp') #table_service.create_table('MyVkApp') login = input("Введите имя пользователя: ") password = input("Введите пароль: ") session = vk.AuthSession(app_id='5889724', user_login=login, user_password=password) api = vk.API(session) friends = api.friends.get(count='') #table_service.create_table('MyVkApp') k = 1 user = api.users.get(user_ids=75301884) friends_info = { 'PartitionKey': 'my_friends1', 'RowKey': str(k), 'first_name': user[0]['first_name'], 'last_name': user[0]['last_name'], 'user_id': user[0]['uid'] } batch.insert_entity(friends_info)
import vk, urllib.request, csv, json, pprint, time from pprint import pprint from azure.storage.table import TableService, Entity, TableBatch table_service = TableService( account_name='seva', account_key= 'SgbxLwWkBH4XuGebxECoXfNVG3mVM5YjOs+SWTDUSacc+3YgUmcafYXrXdz5k0HtlZQ3AuEJ1IcFtZYeGVR9Hw==' ) batch = TableBatch() #table_service.delete_table('MyVkApp') #table_service.create_table('MyVkApp') login = input("Введите имя пользователя: ") password = input("Введите пароль: ") session = vk.AuthSession(app_id='5889724', user_login=login, user_password=password) api = vk.API(session) groups = api.groups.get(count='') #table_service.create_table('MyVkApp') k = 0 for g in groups: group = api.groups.getById(group_ids=g) k = k + 1 # writer.writerow({'group_name': group[0]['name'], 'group_id': group[0]['gid']}) groups_info = { 'PartitionKey': 'my_groups', 'RowKey': str(k), 'group_name': group[0]['name'], 'group_id': group[0]['gid'] }
def test_batch_reuse(self): # Arrange table2 = self._get_table_reference('table2') self.ts.create_table(table2) # Act entity = Entity() entity.PartitionKey = '003' entity.RowKey = 'batch_all_operations_together-1' entity.test = EntityProperty(EdmType.BOOLEAN, 'true') entity.test2 = 'value' entity.test3 = 3 entity.test4 = EntityProperty(EdmType.INT64, '1234567890') entity.test5 = datetime.utcnow() batch = TableBatch() batch.insert_entity(entity) entity.RowKey = 'batch_all_operations_together-2' batch.insert_entity(entity) entity.RowKey = 'batch_all_operations_together-3' batch.insert_entity(entity) entity.RowKey = 'batch_all_operations_together-4' batch.insert_entity(entity) self.ts.commit_batch(self.table_name, batch) self.ts.commit_batch(table2, batch) batch = TableBatch() entity.RowKey = 'batch_all_operations_together' batch.insert_entity(entity) entity.RowKey = 'batch_all_operations_together-1' batch.delete_entity(entity.PartitionKey, entity.RowKey) entity.RowKey = 'batch_all_operations_together-2' entity.test3 = 10 batch.update_entity(entity) entity.RowKey = 'batch_all_operations_together-3' entity.test3 = 100 batch.merge_entity(entity) entity.RowKey = 'batch_all_operations_together-4' entity.test3 = 10 batch.insert_or_replace_entity(entity) entity.RowKey = 'batch_all_operations_together-5' batch.insert_or_merge_entity(entity) self.ts.commit_batch(self.table_name, batch) resp = self.ts.commit_batch(table2, batch) # Assert self.assertEqual(6, len(resp)) entities = list(self.ts.query_entities(self.table_name, "PartitionKey eq '003'", '')) self.assertEqual(5, len(entities))
class TableStorageHandler(logging.Handler): """ Handler class which writes log messages to a Azure Storage table. """ MAX_BATCH_SIZE = 100 def __init__( self, account_name=None, account_key=None, protocol='https', table='logs', batch_size=0, extra_properties=None, partition_key_formatter=None, row_key_formatter=None, is_emulated=False, ): """ Initialize the handler. """ logging.Handler.__init__(self) self.service = TableService(account_name=account_name, account_key=account_key, is_emulated=is_emulated, protocol=protocol) self.meta = {'hostname': gethostname(), 'process': os.getpid()} self.table = _formatName(table, self.meta) self.ready = False self.rowno = 0 if not partition_key_formatter: # default format for partition keys fmt = '%(asctime)s' datefmt = '%Y%m%d%H%M' partition_key_formatter = logging.Formatter(fmt, datefmt) self.partition_key_formatter = partition_key_formatter if not row_key_formatter: # default format for row keys fmt = '%(asctime)s%(msecs)03d-%(hostname)s-%(process)d-%(rowno)02d' datefmt = '%Y%m%d%H%M%S' row_key_formatter = logging.Formatter(fmt, datefmt) self.row_key_formatter = row_key_formatter # extra properties and formatters for them self.extra_properties = extra_properties if extra_properties: self.extra_property_formatters = {} self.extra_property_names = {} for extra in extra_properties: if _PY3: f = logging.Formatter(fmt=extra, style=extra[0]) else: f = logging.Formatter(fmt=extra) self.extra_property_formatters[extra] = f self.extra_property_names[extra] = self._getFormatName(extra) # the storage emulator doesn't support batch operations if batch_size <= 1 or is_emulated: self.batch = None else: self.batch = TableBatch() if batch_size > TableStorageHandler.MAX_BATCH_SIZE: self.batch_size = TableStorageHandler.MAX_BATCH_SIZE else: self.batch_size = batch_size if self.batch: self.current_partition_key = None def _copyLogRecord(self, record): copy = logging.makeLogRecord(record.__dict__) copy.exc_info = None copy.exc_text = None if _PY3: copy.stack_info = None return copy def _getFormatName(self, extra): name = extra style = extra[0] if style == '%': name = extra[2:extra.index(')')] elif _PY3: if style == '{': name = next(string.Formatter().parse(extra))[1] elif style == '$': name = extra[1:] if name.startswith('{'): name = name[1:-1] return name def emit(self, record): """ Emit a record. Format the record and send it to the specified table. """ try: if not self.ready: self.service.create_table(self.table) self.ready = True # generate partition key for the entity record.hostname = self.meta['hostname'] copy = self._copyLogRecord(record) partition_key = self.partition_key_formatter.format(copy) # ensure entities in the batch all have the same patition key if self.batch: if self.current_partition_key is not None: if partition_key != self.current_partition_key: self.flush() self.current_partition_key = partition_key # add log message and extra properties to the entity entity = {} if self.extra_properties: for extra in self.extra_properties: formatter = self.extra_property_formatters[extra] name = self.extra_property_names[extra] entity[name] = formatter.format(copy) entity['message'] = self.format(record) # generate row key for the entity copy.rowno = self.rowno row_key = self.row_key_formatter.format(copy) # add entitiy to the table entity['PartitionKey'] = partition_key entity['RowKey'] = row_key if not self.batch: self.service.insert_or_replace_entity(self.table, entity) else: self.batch.insert_or_replace_entity(entity) # commit the ongoing batch if it reaches the high mark self.rowno += 1 if self.rowno >= self.batch_size: self.flush() except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def flush(self): """ Ensure all logging output has been flushed. """ if self.batch and self.rowno > 0: try: self.service.commit_batch(self.table, self.batch) finally: self.rowno = 0 self.batch = TableBatch() def setFormatter(self, fmt): """ Set the message formatter. """ super(TableStorageHandler, self).setFormatter(fmt) if self.extra_properties: logging._acquireLock() try: for extra in self.extra_property_formatters.values(): extra.converter = fmt.converter extra.datefmt = fmt.datefmt if _PY3: extra.default_time_format = fmt.default_time_format extra.default_msec_format = fmt.default_msec_format finally: logging._releaseLock() def setPartitionKeyFormatter(self, fmt): """ Set the partition key formatter. """ self.partition_key_formatter = fmt def setRowKeyFormatter(self, fmt): """ Set the row key formatter. """ self.row_key_formatter = fmt
import vk, urllib.request, csv, json, pprint, time from pprint import pprint from azure.storage.table import TableService, Entity, TableBatch table_service = TableService(account_name='seva', account_key='SgbxLwWkBH4XuGebxECoXfNVG3mVM5YjOs+SWTDUSacc+3YgUmcafYXrXdz5k0HtlZQ3AuEJ1IcFtZYeGVR9Hw==') batch =TableBatch() #table_service.delete_table('MyVkApp') #table_service.create_table('MyVkApp') login = input("Введите имя пользователя: ") password = input("Введите пароль: ") session = vk.AuthSession(app_id='5889724', user_login=login, user_password=password) api = vk.API(session) posts =api.wall.get(count='') #table_service.create_table('MyVkApp') all_posts_info ={'PartitionKey': 'posts', 'RowKey': '001', 'all_posts': posts[0]} all_posts_info =table_service.insert_or_replace_entity('MyVkApp', all_posts_info) all_posts_info =table_service.get_entity('MyVkApp', 'posts', '001') print(all_posts_info.all_posts)
def insert_options(optionChain): # receives the optionChain object containing all options for all expiration dates # for the selected symbol. inserts rows into the database options table for # each option. Performs a db INSERT statement. If the row already exists, # the database will generate an invalid key error to prevent the row from # being duplicated in the table. In this case, the error is ignored. # # changes to use an entity instead of a parameter list so we can cast the date objects for dates in Azure ###with pypyodbc.connect('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx') as connection: ### cursor = connection.cursor() ### stockPrice = optionChain.stockPrice ### for o in optionChain.options: ### original code: ###param_list = [o.symbol, o.optionDate, o.expiration, o.callPut, o.strike, o.bid, o.ask, o.lastPrice, o.volume, o.openInterest, o.impliedVolatility, stockPrice] ###try: ### updateString = ('insert into Options (Symbol, OptionDate, Expiration, CallPut, Strike, Bid, Ask, LastPrice, Volume, OpenInterest, IV, StockPrice)' ### ' values (?,?,?,?,?,?,?,?,?,?,?,?)') ### cursor.execute(updateString, param_list) ###except Exception as err: ### lg.error('Insert failed for symbol ' + o.symbol + ' exp: ' + o.expiration) ###connection.commit() ### ### replacement code to use entity objects and Azure table storage: account = ut.get_azure_account() # CloudStorageAccount(is_emulated=True) table_service = None try: table_service = account.create_table_service() if not table_service.exists(ut.TABLE_NAME_OPTIONS): # create the table try: table_service.create_table(ut.TABLE_NAME_OPTIONS) except Exception as err: print('Error creating table, ' + ut.TABLE_NAME_OPTIONS + 'check if it already exists') lg.error( 'Tried and failed to create the table for the symbols. Program terminating...' ) exit() batch = TableBatch() batchCount = 0 rowCount = 0 print('Number entries to handle is ' + str(len(optionChain.options))) for o in optionChain.options: rowCount += 1 if rowCount > 100: # Azure restricts the batch size to a max of a hundred entries. Since we're at our # limit, we'll commit these and start a new batch table_service.commit_batch(ut.TABLE_NAME_OPTIONS, batch) batch = TableBatch() rowCount = 1 batchCount += 1 option = Entity() option.PartitionKey = o.symbol # rowkey comprises the concatination of symbols to ensure the key is unique for the symbol. # we'll use the callPut, optionDate, expirationDate, and strike price. Dates will be in format yyyymmdd option.RowKey = o.callPut + o.optionDate.strftime( '%Y%m%d') + o.expiration.strftime('%Y%m%d') + str(o.strike) option.OptionDate = ut.date_for_azure(o.optionDate) option.Expiration = ut.date_for_azure(o.expiration) option.CallPut = o.callPut option.Strike = o.strike option.Bid = o.bid option.Ask = o.ask option.LastPrice = o.lastPrice option.Volume = o.volume option.OpenInterest = o.openInterest option.IV = o.impliedVolatility option.StockPrice = o.stockPrice batch.insert_entity(option) table_service.commit_batch(ut.TABLE_NAME_OPTIONS, batch) except Exception as e: print( 'Error occurred in the sample. If you are using the emulator, please make sure the emulator is running.', e) lg.error('Error adding rows to the options table')
def insert_options_azure(optionChain): # receives the optionChain object containing all options for all expiration dates # for the selected symbol. inserts rows into the database options table for # each option. Performs a db INSERT statement. If the row already exists, # the database will generate an invalid key error to prevent the row from # being duplicated in the table. In this case, the error is ignored. # account = ut.get_azure_account() table_service = None table_name = ut.TABLE_NAME_OPTIONS try: if config.IS_EMULATED: table_service = TableService(is_emulated=True) else: table_service = TableService( account_name=config.STORAGE_ACCOUNT_NAME, account_key=config.STORAGE_ACCOUNT_KEY) if not table_service.exists(table_name): # create the table try: table_service.create_table(table_name) except Exception as err: print('Error creating table, ' + table_name + 'check if it already exists') lg.error( 'Tried and failed to create the table for the symbols. Program terminating...' ) exit() batch = TableBatch() batchCount = 0 rowCount = 0 for o in optionChain.options: rowCount += 1 if rowCount > 100: # Azure restricts the batch size to a max of a hundred entries. Since we're at our # limit, we'll commit these and start a new batch table_service.commit_batch(table_name, batch) batch = TableBatch() rowCount = 1 batchCount += 1 option = Entity() option.PartitionKey = o.PartitionKey # rowkey comprises the concatination of symbols to ensure the key is unique for the symbol. # we'll use the callPut, optionDate, expirationDate, and strike price. Dates will be in format yyyymmdd option.RowKey = o.RowKey option.OptionDate = o.optionDate # dates are already cast as Entity Property with an aware date value option.Expiration = o.expiration option.CallPut = o.callPut option.Strike = o.strike option.Bid = o.bid option.Ask = o.ask option.LastPrice = o.lastPrice option.Volume = o.volume option.OpenInterest = o.openInterest option.IV = o.impliedVolatility option.StockPrice = o.stockPrice batch.insert_entity(option) table_service.commit_batch(table_name, batch) except Exception as e: print('Error adding option ' + symbol + '. Error is: ', e) lg.error('Error adding rows to the options table')
def insert_historical_option_data(rows): account = ut.get_azure_account() table_service = None table_name = 'optionSandbox' # ut.TABLE_NAME_OPTIONS try: if config.IS_EMULATED: table_service = TableService(is_emulated=True) else: table_service = TableService( account_name=config.STORAGE_ACCOUNT_NAME, account_key=config.STORAGE_ACCOUNT_KEY) if not table_service.exists(table_name): # create the table try: table_service.create_table(table_name) except Exception as err: print('Error creating table, ' + table_name + 'check if it already exists') lg.error( 'Tried and failed to create the table for the symbols. Program terminating...' ) exit() batch = TableBatch() batchCount = 0 rowCount = 0 for row in rows: option = Entity() callPut = str(row[5]).strip() optionDate = row[7] expiration = row[6] strike = float(row[8]) option.PartitionKey = str(row[0]).strip() # rowkey comprises the concatination of symbols to ensure the key is unique for the symbol. #option.RowKey = callPut + optionDate.strftime('%Y%m%d') + expiration.strftime('%Y%m%d') + str(strike) optionDateYYYY = optionDate[-4:] optionDateMM = optionDate[:2] optionDateDD = optionDate[3:5] optionDateRowKey = optionDateYYYY + optionDateMM + optionDateDD expDateYYYY = expiration[-4:] expDateMM = expiration[:2] expDateDD = expiration[3:5] expDateRowKey = expDateYYYY + expDateMM + expDateDD option.RowKey = callPut + optionDateRowKey + expDateRowKey + str( strike) option.OptionDate = ut.historicalLoadDates[ optionDate] # ut.date_for_azure(optionDate) option.Expiration = ut.historicalLoadDates[ expiration] # ut.date_for_azure(expiration) option.CallPut = callPut option.Strike = strike option.Bid = float(row[10]) option.Ask = float(row[11]) option.LastPrice = float(row[9]) option.Volume = float(row[12]) option.OpenInterest = int(row[13]) option.StockPrice = float(row[1]) batch.insert_or_replace_entity(option) table_service.commit_batch(table_name, batch) except Exception as e: print('Error importing option ' + symbol + '. Error is: ', e) lg.error('Error importing rows to the options table')
for abc in myposts: #time.sleep(1) try: a=a+int(abc['likes']['count']) #print(str(j) + " - " + str(a)) if a > 0 : likedByOtherCountList.append(a) except: # print('Ошибка') a =a j=j+1 #print(a) return a, likedByOtherCountList table_service = TableService(account_name='seva', account_key='SgbxLwWkBH4XuGebxECoXfNVG3mVM5YjOs+SWTDUSacc+3YgUmcafYXrXdz5k0HtlZQ3AuEJ1IcFtZYeGVR9Hw==') batch =TableBatch() #table_service.delete_table('MyVkApp') #table_service.create_table('MyVkApp') login = input("Введите имя пользователя: ") password = input("Введите пароль: ") session = vk.AuthSession(app_id='5889724', user_login=login, user_password=password) api = vk.API(session) #Вычисление количества друзей текущего пользователя friends =api.friends.get(count='') friends_list =table_service.query_entities('MyVkApp', filter="PartitionKey eq 'my_friends1'") k =1
def test_batch_reuse(self): # Arrange table2 = self._get_table_reference('table2') self.ts.create_table(table2) # Act entity = Entity() entity.PartitionKey = '003' entity.RowKey = 'batch_all_operations_together-1' entity.test = EntityProperty(EdmType.BOOLEAN, 'true') entity.test2 = 'value' entity.test3 = 3 entity.test4 = EntityProperty(EdmType.INT64, '1234567890') entity.test5 = datetime.utcnow() batch = TableBatch() batch.insert_entity(entity) entity.RowKey = 'batch_all_operations_together-2' batch.insert_entity(entity) entity.RowKey = 'batch_all_operations_together-3' batch.insert_entity(entity) entity.RowKey = 'batch_all_operations_together-4' batch.insert_entity(entity) self.ts.commit_batch(self.table_name, batch) self.ts.commit_batch(table2, batch) batch = TableBatch() entity.RowKey = 'batch_all_operations_together' batch.insert_entity(entity) entity.RowKey = 'batch_all_operations_together-1' batch.delete_entity(entity.PartitionKey, entity.RowKey) entity.RowKey = 'batch_all_operations_together-2' entity.test3 = 10 batch.update_entity(entity) entity.RowKey = 'batch_all_operations_together-3' entity.test3 = 100 batch.merge_entity(entity) entity.RowKey = 'batch_all_operations_together-4' entity.test3 = 10 batch.insert_or_replace_entity(entity) entity.RowKey = 'batch_all_operations_together-5' batch.insert_or_merge_entity(entity) self.ts.commit_batch(self.table_name, batch) resp = self.ts.commit_batch(table2, batch) # Assert self.assertEqual(6, len(resp)) entities = list( self.ts.query_entities(self.table_name, "PartitionKey eq '003'", '')) self.assertEqual(5, len(entities))