def batch(self): table_name = self._create_table() entity = Entity() entity.PartitionKey = 'batch' entity.test = True # All operations in the same batch must have the same partition key but different row keys # Batches can hold from 1 to 100 entities # Batches are atomic. All operations completed simulatenously. If one operation fails, they all fail. # Insert, update, merge, insert or merge, insert or replace, and delete entity operations are supported # Context manager style with self.service.batch(table_name) as batch: for i in range(0, 5): entity.RowKey = 'context_{}'.format(i) batch.insert_entity(entity) # Commit style batch = TableBatch() for i in range(0, 5): entity.RowKey = 'commit_{}'.format(i) batch.insert_entity(entity) self.service.commit_batch(table_name, batch) self.service.delete_table(table_name)
def test_query_entities_large(self): # Arrange table_name = self._create_query_table(0) total_entities_count = 1000 entities_per_batch = 50 for j in range(total_entities_count // entities_per_batch): batch = TableBatch() for i in range(entities_per_batch): entity = Entity() entity.PartitionKey = 'large' entity.RowKey = 'batch{0}-item{1}'.format(j, i) entity.test = EntityProperty(EdmType.BOOLEAN, 'true') entity.test2 = 'hello world;' * 100 entity.test3 = 3 entity.test4 = EntityProperty(EdmType.INT64, '1234567890') entity.test5 = datetime(2016, 12, 31, 11, 59, 59, 0) batch.insert_entity(entity) self.ts.commit_batch(table_name, batch) # Act start_time = datetime.now() entities = list(self.ts.query_entities(table_name)) elapsed_time = datetime.now() - start_time # Assert print('query_entities took {0} secs.'.format( elapsed_time.total_seconds())) # azure allocates 5 seconds to execute a query # if it runs slowly, it will return fewer results and make the test fail self.assertEqual(len(entities), total_entities_count)
def test_query_entities_large(self): # Arrange table_name = self._create_query_table(0) total_entities_count = 1000 entities_per_batch = 50 for j in range(total_entities_count // entities_per_batch): batch = TableBatch() for i in range(entities_per_batch): entity = Entity() entity.PartitionKey = 'large' entity.RowKey = 'batch{0}-item{1}'.format(j, i) entity.test = EntityProperty(EdmType.BOOLEAN, 'true') entity.test2 = 'hello world;' * 100 entity.test3 = 3 entity.test4 = EntityProperty(EdmType.INT64, '1234567890') entity.test5 = datetime(2016, 12, 31, 11, 59, 59, 0) batch.insert_entity(entity) self.ts.commit_batch(table_name, batch) # Act start_time = datetime.now() entities = list(self.ts.query_entities(table_name)) elapsed_time = datetime.now() - start_time # Assert print('query_entities took {0} secs.'.format(elapsed_time.total_seconds())) # azure allocates 5 seconds to execute a query # if it runs slowly, it will return fewer results and make the test fail self.assertEqual(len(entities), total_entities_count)
def batch_encrypted_entities(self): table_name = self._create_table() entity1 = self._create_entity_for_encryption() entity2 = self._create_entity_for_encryption() entity2['PartitionKey'] = entity1['PartitionKey'] # Batches will encrypt the entities at the time of inserting into the batch, not # committing the batch to the service, so the encryption policy must be # passed in at the time of batch creation. kek = KeyWrapper('key1') batch = TableBatch(require_encryption=True, key_encryption_key=kek) batch.insert_entity(entity1) batch.insert_entity(entity2) self.service.commit_batch(table_name, batch) # When using the batch as a context manager, the tableservice object will # automatically apply its encryption policy to the batch. entity3 = self._create_entity_for_encryption() entity4 = self._create_entity_for_encryption() entity4['PartitionKey'] = entity3['PartitionKey'] self.service.key_encryption_key = KeyWrapper('key1') with self.service.batch(table_name) as batch: batch.insert_entity(entity3) batch.insert_entity(entity4) # Note that batches follow all the same client-side-encryption behavior as # the corresponding individual table operations. self.service.delete_table(table_name)
def test_batch_reuse(self): # Arrange table2 = self._get_table_reference('table2') self.ts.create_table(table2) # Act entity = Entity() entity.PartitionKey = '003' entity.RowKey = 'batch_all_operations_together-1' entity.test = EntityProperty(EdmType.BOOLEAN, 'true') entity.test2 = 'value' entity.test3 = 3 entity.test4 = EntityProperty(EdmType.INT64, '1234567890') entity.test5 = datetime.utcnow() batch = TableBatch() batch.insert_entity(entity) entity.RowKey = 'batch_all_operations_together-2' batch.insert_entity(entity) entity.RowKey = 'batch_all_operations_together-3' batch.insert_entity(entity) entity.RowKey = 'batch_all_operations_together-4' batch.insert_entity(entity) self.ts.commit_batch(self.table_name, batch) self.ts.commit_batch(table2, batch) batch = TableBatch() entity.RowKey = 'batch_all_operations_together' batch.insert_entity(entity) entity.RowKey = 'batch_all_operations_together-1' batch.delete_entity(entity.PartitionKey, entity.RowKey) entity.RowKey = 'batch_all_operations_together-2' entity.test3 = 10 batch.update_entity(entity) entity.RowKey = 'batch_all_operations_together-3' entity.test3 = 100 batch.merge_entity(entity) entity.RowKey = 'batch_all_operations_together-4' entity.test3 = 10 batch.insert_or_replace_entity(entity) entity.RowKey = 'batch_all_operations_together-5' batch.insert_or_merge_entity(entity) self.ts.commit_batch(self.table_name, batch) resp = self.ts.commit_batch(table2, batch) # Assert self.assertEqual(6, len(resp)) entities = list( self.ts.query_entities(self.table_name, "PartitionKey eq '003'", '')) self.assertEqual(5, len(entities))
def test_batch_strict_mode(self): # Arrange self.ts.require_encryption = True entity = self._create_default_entity_for_encryption() # Act batch = TableBatch(require_encryption=True) # Assert with self.assertRaises(ValueError): batch.insert_entity(entity)
def test_batch_reuse(self): # Arrange table2 = self._get_table_reference('table2') self.ts.create_table(table2) # Act entity = Entity() entity.PartitionKey = '003' entity.RowKey = 'batch_all_operations_together-1' entity.test = EntityProperty(EdmType.BOOLEAN, 'true') entity.test2 = 'value' entity.test3 = 3 entity.test4 = EntityProperty(EdmType.INT64, '1234567890') entity.test5 = datetime.utcnow() batch = TableBatch() batch.insert_entity(entity) entity.RowKey = 'batch_all_operations_together-2' batch.insert_entity(entity) entity.RowKey = 'batch_all_operations_together-3' batch.insert_entity(entity) entity.RowKey = 'batch_all_operations_together-4' batch.insert_entity(entity) self.ts.commit_batch(self.table_name, batch) self.ts.commit_batch(table2, batch) batch = TableBatch() entity.RowKey = 'batch_all_operations_together' batch.insert_entity(entity) entity.RowKey = 'batch_all_operations_together-1' batch.delete_entity(entity.PartitionKey, entity.RowKey) entity.RowKey = 'batch_all_operations_together-2' entity.test3 = 10 batch.update_entity(entity) entity.RowKey = 'batch_all_operations_together-3' entity.test3 = 100 batch.merge_entity(entity) entity.RowKey = 'batch_all_operations_together-4' entity.test3 = 10 batch.insert_or_replace_entity(entity) entity.RowKey = 'batch_all_operations_together-5' batch.insert_or_merge_entity(entity) self.ts.commit_batch(self.table_name, batch) resp = self.ts.commit_batch(table2, batch) # Assert self.assertEqual(6, len(resp)) entities = list(self.ts.query_entities(self.table_name, "PartitionKey eq '003'", '')) self.assertEqual(5, len(entities))
def test_batch_too_many_ops(self): # Arrange entity = self._create_default_entity_dict('001', 'batch_negative_1') self.ts.insert_entity(self.table_name, entity) # Act with self.assertRaises(AzureBatchValidationError): batch = TableBatch() for i in range(0, 101): entity = Entity() entity.PartitionKey = 'large' entity.RowKey = 'item{0}'.format(i) batch.insert_entity(entity) self.ts.commit_batch(self.table_name, batch)
def test_batch_different_partition_operations_fail(self): # Arrange entity = self._create_default_entity_dict('001', 'batch_negative_1') self.ts.insert_entity(self.table_name, entity) # Act with self.assertRaises(AzureBatchValidationError): batch = TableBatch() entity = self._create_updated_entity_dict('001', 'batch_negative_1') batch.update_entity(entity) entity = self._create_default_entity_dict('002', 'batch_negative_1') batch.insert_entity(entity)
def test_batch_different_partition_operations_fail(self): # Arrange entity = self._create_default_entity_dict('001', 'batch_negative_1') self.ts.insert_entity(self.table_name, entity) # Act with self.assertRaises(AzureBatchValidationError): batch = TableBatch() entity = self._create_updated_entity_dict( '001', 'batch_negative_1') batch.update_entity(entity) entity = self._create_default_entity_dict( '002', 'batch_negative_1') batch.insert_entity(entity)
def _setup_tables(account_name, account_key, table_name, batch_size=100, max_num=1000000): table_service = TableService(account_name, account_key) table_service.create_table(table_name) partitions = defaultdict(list) for num in range(1, max_num + 1): partitions[('%03d' % num)[:3]].append(str(num)) for partition, nums in partitions.items(): for batch_num, batch in enumerate(_grouper(nums, batch_size), start=1): table_batch = TableBatch() for num in filter(None, batch): table_batch.insert_entity({ 'PartitionKey': partition, 'RowKey': num, 'value': str(uuid4()), }) table_service.commit_batch(table_name, table_batch) print('Done with partition %s, batch %d' % (partition, batch_num))
def _batch_upload(self, table_name, post_entities): # Context manager style if (len(post_entities) == 0): return if (len(post_entities) > 100): raise ValueError("batch cannot be over 100 entries") # Context manager style # with self.service.batch(table_name) as batch: # for entity in post_entities: # batch.insert_entity(entity) # Commit style batch = TableBatch() for entity in post_entities: if (entity == None): print "entity none" break batch.insert_entity(entity) self.service.commit_batch(table_name, batch)
def test_batch_insert(self): # Arrange # Act entity = Entity() entity.PartitionKey = '001' entity.RowKey = 'batch_insert' entity.test = EntityProperty(EdmType.BOOLEAN, 'true') entity.test2 = 'value' entity.test3 = 3 entity.test4 = EntityProperty(EdmType.INT64, '1234567890') entity.test5 = datetime.utcnow() batch = TableBatch() batch.insert_entity(entity) resp = self.ts.commit_batch(self.table_name, batch) # Assert self.assertIsNotNone(resp) result = self.ts.get_entity(self.table_name, '001', 'batch_insert') self.assertEqual(resp[0], result.etag)
def test_batch_inserts(self): # Arrange # Act entity = Entity() entity.PartitionKey = 'batch_inserts' entity.test = EntityProperty(EdmType.BOOLEAN, 'true') entity.test2 = 'value' entity.test3 = 3 entity.test4 = EntityProperty(EdmType.INT64, '1234567890') batch = TableBatch() for i in range(100): entity.RowKey = str(i) batch.insert_entity(entity) self.ts.commit_batch(self.table_name, batch) entities = list(self.ts.query_entities(self.table_name, "PartitionKey eq 'batch_inserts'", '')) # Assert self.assertIsNotNone(entities) self.assertEqual(100, len(entities))
def test_batch_inserts(self): # Arrange # Act entity = Entity() entity.PartitionKey = 'batch_inserts' entity.test = EntityProperty(EdmType.BOOLEAN, 'true') entity.test2 = 'value' entity.test3 = 3 entity.test4 = EntityProperty(EdmType.INT64, '1234567890') batch = TableBatch() for i in range(100): entity.RowKey = str(i) batch.insert_entity(entity) self.ts.commit_batch(self.table_name, batch) entities = list( self.ts.query_entities(self.table_name, "PartitionKey eq 'batch_inserts'", '')) # Assert self.assertIsNotNone(entities) self.assertEqual(100, len(entities))
session = vk.AuthSession(app_id='5889724', user_login=login, user_password=password) api = vk.API(session) followers = api.users.getFollowers(count='') #table_service.create_table('MyVkApp') k = 0 for d in followers: user = api.users.get(user_ids=d) k = k + 1 # writer.writerow({'first_name': user[0]['first_name'], 'last_name': user[0]['last_name']}) followers_info = { 'PartitionKey': 'my_followers', 'RowKey': str(k), 'first_name': user[0]['first_name'], 'last_name': user[0]['last_name'] } batch.insert_entity(followers_info) if k % 10 == 0: # try: table_service.commit_batch('MyVkApp', batch) batch = TableBatch() print('Коммит прошёл') # except: # print('произошла ошибка.') print(k) pprint(user) time.sleep(1) table_service.commit_batch('MyVkApp', batch)
def _insertEntityViaBatch(entity, batch: TableBatch): return batch.insert_entity(entity)
user_login=login, user_password=password) api = vk.API(session) friends = api.friends.get(count='') #table_service.create_table('MyVkApp') k = 1 user = api.users.get(user_ids=75301884) friends_info = { 'PartitionKey': 'my_friends1', 'RowKey': str(k), 'first_name': user[0]['first_name'], 'last_name': user[0]['last_name'], 'user_id': user[0]['uid'] } batch.insert_entity(friends_info) for f in friends: user = api.users.get(user_ids=f) k = k + 1 friends_info = { 'PartitionKey': 'my_friends1', 'RowKey': str(k), 'first_name': user[0]['first_name'], 'last_name': user[0]['last_name'], 'user_id': user[0]['uid'] } batch.insert_entity(friends_info) if k % 10 == 0: # try: table_service.commit_batch('MyVkApp', batch)
def insert_options_azure(optionChain): # receives the optionChain object containing all options for all expiration dates # for the selected symbol. inserts rows into the database options table for # each option. Performs a db INSERT statement. If the row already exists, # the database will generate an invalid key error to prevent the row from # being duplicated in the table. In this case, the error is ignored. # account = ut.get_azure_account() table_service = None table_name = ut.TABLE_NAME_OPTIONS try: if config.IS_EMULATED: table_service = TableService(is_emulated=True) else: table_service = TableService( account_name=config.STORAGE_ACCOUNT_NAME, account_key=config.STORAGE_ACCOUNT_KEY) if not table_service.exists(table_name): # create the table try: table_service.create_table(table_name) except Exception as err: print('Error creating table, ' + table_name + 'check if it already exists') lg.error( 'Tried and failed to create the table for the symbols. Program terminating...' ) exit() batch = TableBatch() batchCount = 0 rowCount = 0 for o in optionChain.options: rowCount += 1 if rowCount > 100: # Azure restricts the batch size to a max of a hundred entries. Since we're at our # limit, we'll commit these and start a new batch table_service.commit_batch(table_name, batch) batch = TableBatch() rowCount = 1 batchCount += 1 option = Entity() option.PartitionKey = o.PartitionKey # rowkey comprises the concatination of symbols to ensure the key is unique for the symbol. # we'll use the callPut, optionDate, expirationDate, and strike price. Dates will be in format yyyymmdd option.RowKey = o.RowKey option.OptionDate = o.optionDate # dates are already cast as Entity Property with an aware date value option.Expiration = o.expiration option.CallPut = o.callPut option.Strike = o.strike option.Bid = o.bid option.Ask = o.ask option.LastPrice = o.lastPrice option.Volume = o.volume option.OpenInterest = o.openInterest option.IV = o.impliedVolatility option.StockPrice = o.stockPrice batch.insert_entity(option) table_service.commit_batch(table_name, batch) except Exception as e: print('Error adding option ' + symbol + '. Error is: ', e) lg.error('Error adding rows to the options table')
def insert_options(optionChain): # receives the optionChain object containing all options for all expiration dates # for the selected symbol. inserts rows into the database options table for # each option. Performs a db INSERT statement. If the row already exists, # the database will generate an invalid key error to prevent the row from # being duplicated in the table. In this case, the error is ignored. # # changes to use an entity instead of a parameter list so we can cast the date objects for dates in Azure ###with pypyodbc.connect('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx') as connection: ### cursor = connection.cursor() ### stockPrice = optionChain.stockPrice ### for o in optionChain.options: ### original code: ###param_list = [o.symbol, o.optionDate, o.expiration, o.callPut, o.strike, o.bid, o.ask, o.lastPrice, o.volume, o.openInterest, o.impliedVolatility, stockPrice] ###try: ### updateString = ('insert into Options (Symbol, OptionDate, Expiration, CallPut, Strike, Bid, Ask, LastPrice, Volume, OpenInterest, IV, StockPrice)' ### ' values (?,?,?,?,?,?,?,?,?,?,?,?)') ### cursor.execute(updateString, param_list) ###except Exception as err: ### lg.error('Insert failed for symbol ' + o.symbol + ' exp: ' + o.expiration) ###connection.commit() ### ### replacement code to use entity objects and Azure table storage: account = ut.get_azure_account() # CloudStorageAccount(is_emulated=True) table_service = None try: table_service = account.create_table_service() if not table_service.exists(ut.TABLE_NAME_OPTIONS): # create the table try: table_service.create_table(ut.TABLE_NAME_OPTIONS) except Exception as err: print('Error creating table, ' + ut.TABLE_NAME_OPTIONS + 'check if it already exists') lg.error( 'Tried and failed to create the table for the symbols. Program terminating...' ) exit() batch = TableBatch() batchCount = 0 rowCount = 0 print('Number entries to handle is ' + str(len(optionChain.options))) for o in optionChain.options: rowCount += 1 if rowCount > 100: # Azure restricts the batch size to a max of a hundred entries. Since we're at our # limit, we'll commit these and start a new batch table_service.commit_batch(ut.TABLE_NAME_OPTIONS, batch) batch = TableBatch() rowCount = 1 batchCount += 1 option = Entity() option.PartitionKey = o.symbol # rowkey comprises the concatination of symbols to ensure the key is unique for the symbol. # we'll use the callPut, optionDate, expirationDate, and strike price. Dates will be in format yyyymmdd option.RowKey = o.callPut + o.optionDate.strftime( '%Y%m%d') + o.expiration.strftime('%Y%m%d') + str(o.strike) option.OptionDate = ut.date_for_azure(o.optionDate) option.Expiration = ut.date_for_azure(o.expiration) option.CallPut = o.callPut option.Strike = o.strike option.Bid = o.bid option.Ask = o.ask option.LastPrice = o.lastPrice option.Volume = o.volume option.OpenInterest = o.openInterest option.IV = o.impliedVolatility option.StockPrice = o.stockPrice batch.insert_entity(option) table_service.commit_batch(ut.TABLE_NAME_OPTIONS, batch) except Exception as e: print( 'Error occurred in the sample. If you are using the emulator, please make sure the emulator is running.', e) lg.error('Error adding rows to the options table')
session = vk.AuthSession(app_id='5889724', user_login=login, user_password=password) api = vk.API(session) groups = api.groups.get(count='') #table_service.create_table('MyVkApp') k = 0 for g in groups: group = api.groups.getById(group_ids=g) k = k + 1 # writer.writerow({'group_name': group[0]['name'], 'group_id': group[0]['gid']}) groups_info = { 'PartitionKey': 'my_groups', 'RowKey': str(k), 'group_name': group[0]['name'], 'group_id': group[0]['gid'] } batch.insert_entity(groups_info) if k % 10 == 0: # try: table_service.commit_batch('MyVkApp', batch) batch = TableBatch() print('Коммит прошёл') # except: # print('произошла ошибка.') print(k) pprint(group) time.sleep(1) table_service.commit_batch('MyVkApp', batch)